diff --git a/.gitignore b/.gitignore index 60d96a6d7e..0f22ee4a51 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ postgres/pgsql-test/output/ .env.local graphql/server/logs/ graphql/server/*.heapsnapshot +graphile-density-artifacts/ +/research/graphile-density/artifacts/ # Ephemeral pgpm modules installed by `pnpm fixtures:install` (pgpm install) /extensions/ diff --git a/graphile/graphile-bucket-provisioner-plugin/__tests__/plugin.test.ts b/graphile/graphile-bucket-provisioner-plugin/__tests__/plugin.test.ts index c767326646..ab059933f9 100644 --- a/graphile/graphile-bucket-provisioner-plugin/__tests__/plugin.test.ts +++ b/graphile/graphile-bucket-provisioner-plugin/__tests__/plugin.test.ts @@ -75,10 +75,32 @@ jest.mock('graphile-utils', () => ({ })); import { createBucketProvisionerPlugin } from '../src/plugin'; -import type { BucketProvisionerPluginOptions } from '../src/types'; +import type { + BucketProvisionerPluginOptions, + BucketProvisionerStorageModule, +} from '../src/types'; // --- Test helpers --- +function storageModule( + overrides: Partial = {}, +): BucketProvisionerStorageModule { + return { + id: 'sm-uuid-456', + bucketsQualifiedName: 'app_public.buckets', + schemaName: 'app_public', + bucketsTableName: 'buckets', + scope: 'app', + entityTableId: null, + entityQualifiedName: null, + endpoint: null, + publicUrlPrefix: null, + provider: null, + allowedOrigins: null, + ...overrides, + }; +} + function createDefaultOptions( overrides: Partial = {}, ): BucketProvisionerPluginOptions { @@ -91,6 +113,7 @@ function createDefaultOptions( secretAccessKey: 'minioadmin', }, allowedOrigins: ['https://app.example.com'], + preloadedStorageModules: [storageModule()], ...overrides, }; } @@ -100,17 +123,6 @@ function createMockPgClient(overrides: Record = {}) { 'jwt_private.current_database_id': { rows: [{ id: 'db-uuid-123' }], }, - 'metaschema_modules_public.storage_module': { - rows: [{ - id: 'sm-uuid-456', - buckets_schema: 'app_public', - buckets_table: 'buckets', - endpoint: null, - public_url_prefix: null, - provider: null, - allowed_origins: null, - }], - }, app_public: { rows: [{ id: 'bucket-uuid-789', @@ -127,6 +139,9 @@ function createMockPgClient(overrides: Record = {}) { return { query: jest.fn((arg: any) => { const sql: string = typeof arg === 'string' ? arg : arg.text; + if (sql.includes('UPDATE') && sql.includes('SET physical_name')) { + return Promise.resolve({ rows: [{ physical_name: arg.values[0] }] }); + } for (const [key, value] of Object.entries({ ...defaultQueries, ...overrides })) { if (sql.includes(key)) { return Promise.resolve(value); @@ -369,11 +384,11 @@ describe('createBucketProvisionerPlugin', () => { }); it('throws STORAGE_MODULE_NOT_PROVISIONED when no storage module exists', async () => { - createBucketProvisionerPlugin(createDefaultOptions()); + createBucketProvisionerPlugin(createDefaultOptions({ + preloadedStorageModules: [], + })); - const pgClient = createMockPgClient({ - 'metaschema_modules_public.storage_module': { rows: [] }, - }); + const pgClient = createMockPgClient(); const mockWithPgClient = jest.fn((_settings: any, callback: any) => callback(pgClient), ); @@ -423,7 +438,7 @@ describe('createBucketProvisionerPlugin', () => { }); expect(result.success).toBe(false); - expect(result.error).toBe('S3 connection refused'); + expect(result.error).toBe('BUCKET_PROVISIONING_FAILED'); expect(result.bucketName).toBe('public'); }); @@ -451,6 +466,8 @@ describe('createBucketProvisionerPlugin', () => { expect(update![0].text).toContain('physical_name IS NULL'); // Records the exact name returned by the provisioner against the row id. expect(update![0].values).toEqual(['public', 'bucket-uuid-789']); + expect(mockWithPgClient).toHaveBeenCalledTimes(1); + expect(mockWithPgClient.mock.calls[0][0]).toEqual({ role: 'admin' }); }); it('provisions the stored physical_name verbatim when already recorded', async () => { @@ -521,20 +538,15 @@ describe('createBucketProvisionerPlugin', () => { }); it('applies per-database endpoint override from storage module', async () => { - createBucketProvisionerPlugin(createDefaultOptions()); + createBucketProvisionerPlugin(createDefaultOptions({ + preloadedStorageModules: [storageModule({ + endpoint: 'http://custom-minio:9000', + publicUrlPrefix: 'https://cdn.example.com', + provider: 'minio', + })], + })); - const pgClient = createMockPgClient({ - 'metaschema_modules_public.storage_module': { - rows: [{ - id: 'sm-uuid-456', - buckets_schema: 'app_public', - buckets_table: 'buckets', - endpoint: 'http://custom-minio:9000', - public_url_prefix: 'https://cdn.example.com', - provider: 'minio', - }], - }, - }); + const pgClient = createMockPgClient(); const mockWithPgClient = jest.fn((_settings: any, callback: any) => callback(pgClient), ); @@ -578,20 +590,13 @@ describe('createBucketProvisionerPlugin', () => { }); it('passes publicUrlPrefix from storage module to provision call', async () => { - createBucketProvisionerPlugin(createDefaultOptions()); + createBucketProvisionerPlugin(createDefaultOptions({ + preloadedStorageModules: [storageModule({ + publicUrlPrefix: 'https://cdn.example.com', + })], + })); - const pgClient = createMockPgClient({ - 'metaschema_modules_public.storage_module': { - rows: [{ - id: 'sm-uuid-456', - buckets_schema: 'app_public', - buckets_table: 'buckets', - endpoint: null, - public_url_prefix: 'https://cdn.example.com', - provider: null, - }], - }, - }); + const pgClient = createMockPgClient(); const mockWithPgClient = jest.fn((_settings: any, callback: any) => callback(pgClient), ); @@ -610,6 +615,108 @@ describe('createBucketProvisionerPlugin', () => { }); }); + describe('storage snapshot isolation', () => { + it('rejects missing request settings before acquiring a PostgreSQL client', async () => { + createBucketProvisionerPlugin(createDefaultOptions()); + const withPgClient = jest.fn(); + + await expect(capturedLambdaCallback!({ + input: { bucketKey: 'public' }, + withPgClient, + pgSettings: null, + })).rejects.toThrow('STORAGE_REQUEST_SETTINGS_UNAVAILABLE'); + expect(withPgClient).not.toHaveBeenCalled(); + expect(mockProvision).not.toHaveBeenCalled(); + }); + + it('fails closed without a snapshot and never queries metaschema or calls S3', async () => { + createBucketProvisionerPlugin(createDefaultOptions({ + preloadedStorageModules: undefined, + })); + const pgClient = createMockPgClient(); + const withPgClient = jest.fn((_settings: any, callback: any) => callback(pgClient)); + + await expect(capturedLambdaCallback!({ + input: { bucketKey: 'public' }, + withPgClient, + pgSettings: { role: 'tenant_member' }, + })).rejects.toThrow('STORAGE_MODULE_SNAPSHOT_REQUIRED'); + + expect(pgClient.query.mock.calls.map((call: any[]) => call[0].text).join('\n')) + .not.toContain('metaschema_'); + expect(mockProvision).not.toHaveBeenCalled(); + }); + + it('rejects duplicate app modules before reading a bucket or calling S3', async () => { + createBucketProvisionerPlugin(createDefaultOptions({ + preloadedStorageModules: [ + storageModule({ id: 'app-a' }), + storageModule({ id: 'app-b', bucketsTableName: 'other_buckets' }), + ], + })); + const pgClient = createMockPgClient(); + const withPgClient = jest.fn((_settings: any, callback: any) => callback(pgClient)); + + await expect(capturedLambdaCallback!({ + input: { bucketKey: 'public' }, + withPgClient, + pgSettings: { role: 'tenant_member' }, + })).rejects.toThrow('STORAGE_MODULE_AMBIGUOUS:app'); + expect(mockProvision).not.toHaveBeenCalled(); + }); + + it('probes only safely quoted preloaded entity tables and rejects ambiguous owners', async () => { + createBucketProvisionerPlugin(createDefaultOptions({ + preloadedStorageModules: [ + storageModule({ + id: 'team-a', + scope: 'team-a', + schemaName: 'team_a_public', + bucketsTableName: 'buckets', + entityTableId: 'entity-a', + entityQualifiedName: '"tenant-a"."teams"', + }), + storageModule({ + id: 'team-b', + scope: 'team-b', + schemaName: 'team_b_public', + bucketsTableName: 'buckets', + entityTableId: 'entity-b', + entityQualifiedName: '"tenant-b"."teams"', + }), + ], + })); + const pgClient = createMockPgClient({ + '"tenant-a".teams': { rows: [{ '?column?': 1 }] }, + '"tenant-b".teams': { rows: [{ '?column?': 1 }] }, + }); + const withPgClient = jest.fn((_settings: any, callback: any) => callback(pgClient)); + + await expect(capturedLambdaCallback!({ + input: { bucketKey: 'private', ownerId: 'owner-a' }, + withPgClient, + pgSettings: { role: 'tenant_member' }, + })).rejects.toThrow('STORAGE_MODULE_AMBIGUOUS:owner'); + + const sql = pgClient.query.mock.calls.map((call: any[]) => call[0].text).join('\n'); + expect(sql).toContain('FROM "tenant-a".teams'); + expect(sql).toContain('FROM "tenant-b".teams'); + expect(sql).not.toContain('metaschema_'); + expect(mockProvision).not.toHaveBeenCalled(); + }); + + it('rejects an expression masquerading as an entity table at build time', () => { + expect(() => createBucketProvisionerPlugin(createDefaultOptions({ + preloadedStorageModules: [storageModule({ + id: 'malicious', + scope: 'team', + entityTableId: 'entity-a', + entityQualifiedName: 'safe.teams; SELECT pg_sleep(10)', + })], + }))).toThrow('STORAGE_MODULE_METADATA_INVALID:entity:malicious'); + }); + }); + describe('connection config resolution', () => { it('resolves static connection config', () => { const options = createDefaultOptions(); @@ -1110,7 +1217,11 @@ describe('CORS resolution hierarchy', () => { lifecycleRules: [], }); - createBucketProvisionerPlugin(createDefaultOptions()); + createBucketProvisionerPlugin(createDefaultOptions({ + preloadedStorageModules: [storageModule({ + allowedOrigins: ['https://db-default.example.com'], + })], + })); const pgClient = createMockPgClient({ app_public: { @@ -1122,17 +1233,6 @@ describe('CORS resolution hierarchy', () => { allowed_origins: ['*'], }], }, - 'metaschema_modules_public.storage_module': { - rows: [{ - id: 'sm-uuid-456', - buckets_schema: 'app_public', - buckets_table: 'buckets', - endpoint: null, - public_url_prefix: null, - provider: null, - allowed_origins: ['https://db-default.example.com'], - }], - }, }); const mockWithPgClient = jest.fn((_settings: any, callback: any) => callback(pgClient), @@ -1167,7 +1267,11 @@ describe('CORS resolution hierarchy', () => { lifecycleRules: [], }); - createBucketProvisionerPlugin(createDefaultOptions()); + createBucketProvisionerPlugin(createDefaultOptions({ + preloadedStorageModules: [storageModule({ + allowedOrigins: ['https://db-default.example.com'], + })], + })); const pgClient = createMockPgClient({ app_public: { @@ -1179,17 +1283,6 @@ describe('CORS resolution hierarchy', () => { allowed_origins: null, // No bucket-level override }], }, - 'metaschema_modules_public.storage_module': { - rows: [{ - id: 'sm-uuid-456', - buckets_schema: 'app_public', - buckets_table: 'buckets', - endpoint: null, - public_url_prefix: null, - provider: null, - allowed_origins: ['https://db-default.example.com'], - }], - }, }); const mockWithPgClient = jest.fn((_settings: any, callback: any) => callback(pgClient), @@ -1238,17 +1331,6 @@ describe('CORS resolution hierarchy', () => { allowed_origins: null, }], }, - 'metaschema_modules_public.storage_module': { - rows: [{ - id: 'sm-uuid-456', - buckets_schema: 'app_public', - buckets_table: 'buckets', - endpoint: null, - public_url_prefix: null, - provider: null, - allowed_origins: null, - }], - }, }); const mockWithPgClient = jest.fn((_settings: any, callback: any) => callback(pgClient), @@ -1346,6 +1428,7 @@ describe('bucket name resolution', () => { secretAccessKey: 'test', }, allowedOrigins: ['https://app.example.com'], + preloadedStorageModules: [storageModule()], }); const pgClient = createMockPgClient({ @@ -1400,6 +1483,7 @@ describe('bucket name resolution', () => { secretAccessKey: 'test', }, allowedOrigins: ['https://app.example.com'], + preloadedStorageModules: [storageModule()], bucketNamePrefix: 'should-be-ignored', resolveBucketName: customResolver, }); diff --git a/graphile/graphile-bucket-provisioner-plugin/src/index.ts b/graphile/graphile-bucket-provisioner-plugin/src/index.ts index 77919ee720..f437e82c41 100644 --- a/graphile/graphile-bucket-provisioner-plugin/src/index.ts +++ b/graphile/graphile-bucket-provisioner-plugin/src/index.ts @@ -44,6 +44,7 @@ export type { BucketAccessType, BucketNameResolver, BucketProvisionerPluginOptions, + BucketProvisionerStorageModule, ConnectionConfigOrGetter, ProvisionBucketInput, ProvisionBucketPayload, diff --git a/graphile/graphile-bucket-provisioner-plugin/src/plugin.ts b/graphile/graphile-bucket-provisioner-plugin/src/plugin.ts index 847d9cd6e8..4f3f898d9d 100644 --- a/graphile/graphile-bucket-provisioner-plugin/src/plugin.ts +++ b/graphile/graphile-bucket-provisioner-plugin/src/plugin.ts @@ -43,70 +43,95 @@ import { extendSchema, gql } from 'graphile-utils'; import type { BucketProvisionerPluginOptions, + BucketProvisionerStorageModule, } from './types'; const log = new Logger('graphile-bucket-provisioner:plugin'); -// --- Storage module queries --- +const QUALIFIED_IDENTIFIER = /^("(?:[^"]|"")+"|[a-z_][a-z0-9_$]*)\.("(?:[^"]|"")+"|[a-z_][a-z0-9_$]*)$/; -/** - * Resolve the app-level storage module (scope = 'app'). - */ -const APP_STORAGE_MODULE_QUERY = ` - SELECT - sm.id, - sm.scope, - sm.entity_table_id, - bs.schema_name AS buckets_schema, - bt.name AS buckets_table, - sm.endpoint, - sm.public_url_prefix, - sm.provider, - sm.allowed_origins - FROM metaschema_modules_public.storage_module sm - JOIN metaschema_public.table bt ON bt.id = sm.buckets_table_id - JOIN metaschema_public.schema bs ON bs.id = bt.schema_id - WHERE sm.database_id = $1 - AND sm.scope = 'app' - LIMIT 1 -`; +function decodeIdentifier(identifier: string): string { + return identifier.startsWith('"') + ? identifier.slice(1, -1).replace(/""/g, '"') + : identifier; +} /** - * Resolve ALL storage modules for a database (for ownerId-based resolution). + * Parse exactly two SQL identifiers, then quote both components again. This + * accepts the canonical quoted names emitted by the control-plane loader but + * rejects expressions, search paths, comments, and extra qualification. */ -const ALL_STORAGE_MODULES_QUERY = ` - SELECT - sm.id, - sm.scope, - sm.entity_table_id, - bs.schema_name AS buckets_schema, - bt.name AS buckets_table, - sm.endpoint, - sm.public_url_prefix, - sm.provider, - sm.allowed_origins, - es.schema_name AS entity_schema, - et.name AS entity_table - FROM metaschema_modules_public.storage_module sm - JOIN metaschema_public.table bt ON bt.id = sm.buckets_table_id - JOIN metaschema_public.schema bs ON bs.id = bt.schema_id - LEFT JOIN metaschema_public.table et ON et.id = sm.entity_table_id - LEFT JOIN metaschema_public.schema es ON es.id = et.schema_id - WHERE sm.database_id = $1 -`; - -interface StorageModuleRow { - id: string; - scope: string; - entity_table_id: string | null; - buckets_schema: string; - buckets_table: string; - endpoint: string | null; - public_url_prefix: string | null; - provider: string | null; - allowed_origins: string[] | null; - entity_schema?: string | null; - entity_table?: string | null; +function quotePreloadedQualifiedIdentifier(value: string, label: string): string { + const match = QUALIFIED_IDENTIFIER.exec(value); + if (!match) { + throw new Error(`STORAGE_MODULE_METADATA_INVALID:${label}`); + } + const schema = decodeIdentifier(match[1]); + const objectName = decodeIdentifier(match[2]); + if ( + schema.length === 0 || + objectName.length === 0 || + schema.includes('\0') || + objectName.includes('\0') || + Buffer.byteLength(schema, 'utf8') > 63 || + Buffer.byteLength(objectName, 'utf8') > 63 + ) { + throw new Error(`STORAGE_MODULE_METADATA_INVALID:${label}`); + } + return QuoteUtils.quoteQualifiedIdentifier(schema, objectName); +} + +function snapshotStorageModules( + modules: readonly BucketProvisionerStorageModule[] | undefined, +): readonly BucketProvisionerStorageModule[] | undefined { + if (modules === undefined) return undefined; + + for (const module of modules) { + if ( + !module || + typeof module.id !== 'string' || + module.id.length === 0 || + typeof module.scope !== 'string' || + module.scope.length === 0 || + typeof module.schemaName !== 'string' || + module.schemaName.length === 0 || + module.schemaName.includes('\0') || + Buffer.byteLength(module.schemaName, 'utf8') > 63 || + typeof module.bucketsTableName !== 'string' || + module.bucketsTableName.length === 0 || + module.bucketsTableName.includes('\0') || + Buffer.byteLength(module.bucketsTableName, 'utf8') > 63 + ) { + throw new Error('STORAGE_MODULE_METADATA_INVALID'); + } + QuoteUtils.quoteQualifiedIdentifier(module.schemaName, module.bucketsTableName); + if (module.scope === 'app') { + if (module.entityTableId !== null || module.entityQualifiedName !== null) { + throw new Error(`STORAGE_MODULE_METADATA_INVALID:${module.id}`); + } + } else if (!module.entityTableId || !module.entityQualifiedName) { + throw new Error(`STORAGE_MODULE_METADATA_INVALID:${module.id}`); + } else { + quotePreloadedQualifiedIdentifier(module.entityQualifiedName, `entity:${module.id}`); + } + } + + if ( + Object.isFrozen(modules) && + modules.every((module) => + Object.isFrozen(module) && + (module.allowedOrigins === null || Object.isFrozen(module.allowedOrigins)), + ) + ) { + return modules; + } + + return Object.freeze(modules.map((module) => Object.freeze({ + ...module, + allowedOrigins: module.allowedOrigins === null + ? null + : Object.freeze([...module.allowedOrigins]), + }))); } /** @@ -125,6 +150,18 @@ function runQuery( return pgClient.query(values === undefined ? { text } : { text, values }); } +function assertStorageRequestContext(withPgClient: unknown, pgSettings: unknown): asserts withPgClient is ( + settings: Record, + callback: (pgClient: any) => Promise, +) => Promise { + if (typeof withPgClient !== 'function') { + throw new Error('STORAGE_CONTEXT_UNAVAILABLE'); + } + if (typeof pgSettings !== 'object' || pgSettings === null || Array.isArray(pgSettings)) { + throw new Error('STORAGE_REQUEST_SETTINGS_UNAVAILABLE'); + } +} + /** * Resolve the storage module for a given scope. * If ownerId is provided, probes entity tables to find the matching module. @@ -132,33 +169,43 @@ function runQuery( */ async function resolveStorageModule( pgClient: any, - databaseId: string, + modules: readonly BucketProvisionerStorageModule[] | undefined, ownerId?: string, -): Promise { +): Promise { + if (modules === undefined) { + throw new Error('STORAGE_MODULE_SNAPSHOT_REQUIRED'); + } + if (!ownerId) { - // App-level resolution - const result = await runQuery(pgClient, APP_STORAGE_MODULE_QUERY, [databaseId]); - return (result.rows[0] as StorageModuleRow) ?? null; + const appModules = modules.filter((module) => module.scope === 'app'); + if (appModules.length > 1) { + throw new Error('STORAGE_MODULE_AMBIGUOUS:app'); + } + return appModules[0] ?? null; } - // Entity-scoped: load all modules and probe entity tables - const result = await runQuery(pgClient, ALL_STORAGE_MODULES_QUERY, [databaseId]); - const modules = result.rows as StorageModuleRow[]; - const entityModules = modules.filter((m) => m.entity_schema && m.entity_table); + const entityModules = modules.filter((module) => module.scope !== 'app'); + const matches: BucketProvisionerStorageModule[] = []; for (const mod of entityModules) { - const entityTable = QuoteUtils.quoteQualifiedIdentifier(mod.entity_schema!, mod.entity_table!); + const entityTable = quotePreloadedQualifiedIdentifier( + mod.entityQualifiedName!, + `entity:${mod.id}`, + ); const probe = await runQuery( pgClient, `SELECT 1 FROM ${entityTable} WHERE id = $1 LIMIT 1`, [ownerId], ); if (probe.rows.length > 0) { - return mod; + matches.push(mod); } } - return null; + if (matches.length > 1) { + throw new Error('STORAGE_MODULE_AMBIGUOUS:owner'); + } + return matches[0] ?? null; } interface BucketRow { @@ -185,24 +232,47 @@ function storedPhysicalName(row: Pick): string | nul /** * Record the physical S3 bucket name on the source bucket row. * - * Runs in the system lane (`withPgClient(null, ...)`) — server bookkeeping, - * RLS-independent. Idempotent via the `physical_name IS NULL` guard so a - * re-provision never clobbers an already-recorded coordinate. + * Runs on the request's already-scoped client. The write must satisfy the same + * role, claims, and RLS policies as the bucket read that authorized + * provisioning; a policy denial fails closed. It is idempotent via the + * `physical_name IS NULL` guard so a re-provision never clobbers an + * already-recorded coordinate. */ async function recordPhysicalName( - withPgClient: (pgSettings: null, cb: (client: any) => Promise) => Promise, + pgClient: any, bucketsTable: string, bucketId: string, physicalName: string, -): Promise { - await withPgClient(null, (client: any) => - runQuery( - client, - `UPDATE ${bucketsTable} SET physical_name = $1 WHERE id = $2 AND physical_name IS NULL`, - [physicalName, bucketId], - ), +): Promise { + const updated = await runQuery( + pgClient, + `UPDATE ${bucketsTable} + SET physical_name = $1 + WHERE id = $2 AND physical_name IS NULL + RETURNING physical_name`, + [physicalName, bucketId], ); - log.info(`Recorded physical_name="${physicalName}" on bucket ${bucketId}`); + const written = updated.rows[0]?.physical_name; + if (updated.rows.length === 1 && typeof written === 'string') { + log.info(`Recorded physical_name="${written}" on bucket ${bucketId}`); + return written; + } + if (updated.rows.length > 1) { + throw new Error('BUCKET_COORDINATE_AMBIGUOUS'); + } + + // Another process may have won the first-provision race. Route to the + // durable value it recorded; never return a losing candidate. + const existing = await runQuery( + pgClient, + `SELECT physical_name FROM ${bucketsTable} WHERE id = $1 LIMIT 2`, + [bucketId], + ); + const authoritative = existing.rows[0]?.physical_name; + if (existing.rows.length !== 1 || typeof authoritative !== 'string') { + throw new Error('BUCKET_COORDINATE_WRITE_FAILED'); + } + return authoritative; } // --- Helpers --- @@ -259,16 +329,16 @@ async function resolveDatabaseId(pgClient: any): Promise { */ function resolveAllowedOrigins( bucketOrigins: string[] | null | undefined, - storageModuleOrigins: string[] | null | undefined, + storageModuleOrigins: readonly string[] | null | undefined, pluginOrigins: string[], ): string[] { if (bucketOrigins && bucketOrigins.length > 0) { - return bucketOrigins; + return [...bucketOrigins]; } if (storageModuleOrigins && storageModuleOrigins.length > 0) { - return storageModuleOrigins; + return [...storageModuleOrigins]; } - return pluginOrigins; + return [...pluginOrigins]; } /** @@ -276,14 +346,14 @@ function resolveAllowedOrigins( */ function buildProvisioner( options: BucketProvisionerPluginOptions, - storageModule: StorageModuleRow | null, + storageModule: BucketProvisionerStorageModule, effectiveOrigins: string[], ): BucketProvisioner { const connection = resolveConnection(options); const effectiveConnection: StorageConnectionConfig = { ...connection, - ...(storageModule?.endpoint ? { endpoint: storageModule.endpoint } : {}), - ...(storageModule?.provider + ...(storageModule.endpoint ? { endpoint: storageModule.endpoint } : {}), + ...(storageModule.provider ? { provider: storageModule.provider as StorageConnectionConfig['provider'] } : {}), }; @@ -299,23 +369,20 @@ function buildProvisioner( * auto-provisioning hook. */ async function provisionBucketForRow( - pgClient: any, databaseId: string, bucketKey: string, bucketType: string, bucketAllowedOrigins: string[] | null | undefined, options: BucketProvisionerPluginOptions, s3BucketName: string, + storageModule: BucketProvisionerStorageModule, ): Promise { const accessType = bucketType as 'public' | 'private' | 'temp'; - // Read storage module config to check for endpoint/provider/CORS overrides - const storageModule = await resolveStorageModule(pgClient, databaseId); - // Resolve CORS origins using the 3-tier hierarchy const effectiveOrigins = resolveAllowedOrigins( bucketAllowedOrigins, - storageModule?.allowed_origins, + storageModule.allowedOrigins, options.allowedOrigins, ); @@ -330,7 +397,7 @@ async function provisionBucketForRow( bucketName: s3BucketName, accessType, versioning: options.versioning ?? false, - publicUrlPrefix: storageModule?.public_url_prefix ?? undefined, + publicUrlPrefix: storageModule.publicUrlPrefix ?? undefined, allowedOrigins: effectiveOrigins, }); @@ -346,21 +413,19 @@ async function provisionBucketForRow( * Update CORS on an existing S3 bucket when allowed_origins changes. */ async function updateBucketCors( - pgClient: any, databaseId: string, bucketKey: string, bucketType: string, bucketAllowedOrigins: string[] | null | undefined, options: BucketProvisionerPluginOptions, s3BucketName: string, + storageModule: BucketProvisionerStorageModule, ): Promise { const accessType = bucketType as 'public' | 'private' | 'temp'; - const storageModule = await resolveStorageModule(pgClient, databaseId); - const effectiveOrigins = resolveAllowedOrigins( bucketAllowedOrigins, - storageModule?.allowed_origins, + storageModule.allowedOrigins, options.allowedOrigins, ); @@ -401,6 +466,9 @@ export function createBucketProvisionerPlugin( options: BucketProvisionerPluginOptions, ): GraphileConfig.Plugin { const autoProvision = options.autoProvision ?? true; + const preloadedStorageModules = snapshotStorageModules( + options.preloadedStorageModules, + ); // The extendSchema plugin adds the explicit provisionBucket mutation const mutationPlugin = extendSchema(() => ({ @@ -461,6 +529,8 @@ export function createBucketProvisionerPlugin( throw new Error('INVALID_BUCKET_KEY'); } + assertStorageRequestContext(withPgClient, pgSettings); + return withPgClient(pgSettings, async (pgClient: any) => { // Resolve database ID from JWT context const databaseId = await resolveDatabaseId(pgClient); @@ -469,7 +539,11 @@ export function createBucketProvisionerPlugin( } // Resolve storage module (app-level or entity-scoped via ownerId) - const storageModule = await resolveStorageModule(pgClient, databaseId, ownerId); + const storageModule = await resolveStorageModule( + pgClient, + preloadedStorageModules, + ownerId, + ); if (!storageModule) { throw new Error( ownerId @@ -480,24 +554,30 @@ export function createBucketProvisionerPlugin( // Look up the bucket row (RLS enforced via pgSettings) const hasOwner = ownerId && storageModule.scope !== 'app'; - const bucketsTable = QuoteUtils.quoteQualifiedIdentifier(storageModule.buckets_schema, storageModule.buckets_table); + const bucketsTable = QuoteUtils.quoteQualifiedIdentifier( + storageModule.schemaName, + storageModule.bucketsTableName, + ); const bucketResult = await runQuery( pgClient, hasOwner ? `SELECT id, key, type, is_public, allowed_origins, physical_name FROM ${bucketsTable} WHERE key = $1 AND owner_id = $2 - LIMIT 1` + LIMIT 2` : `SELECT id, key, type, is_public, allowed_origins, physical_name FROM ${bucketsTable} WHERE key = $1 - LIMIT 1`, + LIMIT 2`, hasOwner ? [bucketKey, ownerId] : [bucketKey], ); if (bucketResult.rows.length === 0) { throw new Error('BUCKET_NOT_FOUND'); } + if (bucketResult.rows.length > 1) { + throw new Error('BUCKET_AMBIGUOUS'); + } const bucket = bucketResult.rows[0] as BucketRow; @@ -510,21 +590,26 @@ export function createBucketProvisionerPlugin( try { const result = await provisionBucketForRow( - pgClient, databaseId, bucket.key, bucket.type, bucket.allowed_origins, options, s3BucketName, + storageModule, ); // Record the exact provisioned name on the source row. - await recordPhysicalName(withPgClient, bucketsTable, bucket.id, result.bucketName); + const authoritativeBucketName = await recordPhysicalName( + pgClient, + bucketsTable, + bucket.id, + result.bucketName, + ); return { success: true, - bucketName: result.bucketName, + bucketName: authoritativeBucketName, accessType: result.accessType, provider: result.provider, endpoint: result.endpoint, @@ -538,7 +623,7 @@ export function createBucketProvisionerPlugin( accessType: bucket.type, provider: resolveConnection(options).provider, endpoint: resolveConnection(options).endpoint ?? null, - error: err.message, + error: 'BUCKET_PROVISIONING_FAILED', }; } }); @@ -622,10 +707,7 @@ export function createBucketProvisionerPlugin( const withPgClient = graphqlContext.withPgClient; const pgSettings = graphqlContext.pgSettings; - if (!withPgClient) { - log.warn(`${isCreate ? 'Auto-provision' : 'CORS update'} skipped: withPgClient not available in context`); - return result; - } + assertStorageRequestContext(withPgClient, pgSettings); if (isCreate) { // --- CREATE: full provisioning --- @@ -645,27 +727,49 @@ export function createBucketProvisionerPlugin( } // Newly-created row has no stored coordinate yet — mint on first provision. - const result = await provisionBucketForRow( + const storageModule = await resolveStorageModule( pgClient, + preloadedStorageModules, + ); + if (!storageModule) { + throw new Error('STORAGE_MODULE_NOT_PROVISIONED'); + } + + const result = await provisionBucketForRow( databaseId, bucketInput.key, bucketInput.type, bucketInput.allowedOrigins ?? bucketInput.allowed_origins ?? null, options, resolveBucketName(bucketInput.key, databaseId, options), + storageModule, ); // Record the provisioned name on the just-created row. - const storageModule = await resolveStorageModule(pgClient, databaseId); - if (storageModule) { - const bucketsTable = QuoteUtils.quoteQualifiedIdentifier(storageModule.buckets_schema, storageModule.buckets_table); - const idResult = await runQuery( + const bucketsTable = QuoteUtils.quoteQualifiedIdentifier( + storageModule.schemaName, + storageModule.bucketsTableName, + ); + const idResult = await runQuery( + pgClient, + `SELECT id FROM ${bucketsTable} WHERE key = $1 LIMIT 2`, + [bucketInput.key], + ); + if (idResult.rows.length !== 1) { + throw new Error( + idResult.rows.length === 0 + ? 'BUCKET_NOT_FOUND' + : 'BUCKET_AMBIGUOUS', + ); + } + const bucketId = idResult.rows[0]?.id; + if (bucketId) { + await recordPhysicalName( pgClient, - `SELECT id FROM ${bucketsTable} WHERE key = $1 LIMIT 1`, - [bucketInput.key], + bucketsTable, + bucketId, + result.bucketName, ); - const bucketId = idResult.rows[0]?.id; - if (bucketId) await recordPhysicalName(withPgClient, bucketsTable, bucketId, result.bucketName); } }); } else { @@ -686,7 +790,10 @@ export function createBucketProvisionerPlugin( } // Read the storage module config (app-level; auto-hook doesn't have ownerId context) - const storageModule = await resolveStorageModule(pgClient, databaseId); + const storageModule = await resolveStorageModule( + pgClient, + preloadedStorageModules, + ); if (!storageModule) { log.warn('CORS update skipped: storage module not provisioned'); return; @@ -705,13 +812,16 @@ export function createBucketProvisionerPlugin( } // Read the full bucket row (post-update) to get type + origins - const bucketsTable = QuoteUtils.quoteQualifiedIdentifier(storageModule.buckets_schema, storageModule.buckets_table); + const bucketsTable = QuoteUtils.quoteQualifiedIdentifier( + storageModule.schemaName, + storageModule.bucketsTableName, + ); const bucketResult = await runQuery( pgClient, `SELECT id, key, type, is_public, allowed_origins, physical_name FROM ${bucketsTable} WHERE key = $1 - LIMIT 1`, + LIMIT 2`, [patchKey], ); @@ -719,6 +829,9 @@ export function createBucketProvisionerPlugin( log.warn(`CORS update skipped: bucket "${patchKey}" not found`); return; } + if (bucketResult.rows.length > 1) { + throw new Error('BUCKET_AMBIGUOUS'); + } const bucket = bucketResult.rows[0] as BucketRow; @@ -728,7 +841,6 @@ export function createBucketProvisionerPlugin( const recorded = storedPhysicalName(bucket); await updateBucketCors( - pgClient, databaseId, bucket.key, bucket.type, @@ -737,6 +849,7 @@ export function createBucketProvisionerPlugin( recorded === null ? resolveBucketName(bucket.key, databaseId, options) : recorded, + storageModule, ); }); } diff --git a/graphile/graphile-bucket-provisioner-plugin/src/types.ts b/graphile/graphile-bucket-provisioner-plugin/src/types.ts index 45a5c172ba..7003c308f6 100644 --- a/graphile/graphile-bucket-provisioner-plugin/src/types.ts +++ b/graphile/graphile-bucket-provisioner-plugin/src/types.ts @@ -36,6 +36,25 @@ export type ConnectionConfigOrGetter = */ export type BucketNameResolver = (bucketKey: string, databaseId: string) => string; +/** + * Immutable storage routing metadata resolved by the control plane for one + * exact Graphile build. This intentionally mirrors the subset consumed by the + * provisioner without making the provisioner depend on the presigned plugin. + */ +export interface BucketProvisionerStorageModule { + id: string; + bucketsQualifiedName: string; + schemaName: string; + bucketsTableName: string; + scope: string; + entityTableId: string | null; + entityQualifiedName: string | null; + endpoint: string | null; + publicUrlPrefix: string | null; + provider: string | null; + allowedOrigins: readonly string[] | null; +} + /** * Plugin options for the bucket provisioner plugin. */ @@ -53,6 +72,14 @@ export interface BucketProvisionerPluginOptions { */ allowedOrigins: string[]; + /** + * Exact-build storage metadata supplied by the control plane. An empty list + * is authoritative. If this is omitted, provisioning fails closed; the + * plugin never discovers tenant routing metadata from a request-time SQL + * query. + */ + preloadedStorageModules?: readonly BucketProvisionerStorageModule[]; + /** * Optional prefix for S3 bucket names. * When set, the S3 bucket name becomes `{prefix}-{bucketKey}`. diff --git a/graphile/graphile-bulk-mutations/__tests__/bulk-where-type.test.ts b/graphile/graphile-bulk-mutations/__tests__/bulk-where-type.test.ts new file mode 100644 index 0000000000..775692b809 --- /dev/null +++ b/graphile/graphile-bulk-mutations/__tests__/bulk-where-type.test.ts @@ -0,0 +1,38 @@ +import { GraphQLInputObjectType } from 'graphql'; + +import { resolveBulkWhereType } from '../src/plugins/BulkTypesPlugin'; + +describe('bulk mutation where type resolution', () => { + it('uses connection-filter without touching a disabled condition inflector', () => { + const filter = new GraphQLInputObjectType({ name: 'ItemFilter', fields: {} }); + const getTypeByName = jest.fn((name: string) => name === 'ItemFilter' ? filter : undefined); + const conditionType = jest.fn(() => { + throw new Error('disabled condition inflector must not be called'); + }); + + expect(resolveBulkWhereType({ getTypeByName } as any, { conditionType }, 'Item')).toBe(filter); + expect(conditionType).not.toHaveBeenCalled(); + expect(getTypeByName).toHaveBeenCalledTimes(1); + }); + + it('falls back to the built-in condition type when no filter exists', () => { + const condition = new GraphQLInputObjectType({ name: 'ItemCondition', fields: {} }); + const getTypeByName = jest.fn((name: string) => + name === 'ItemCondition' ? condition : undefined + ); + + expect(resolveBulkWhereType( + { getTypeByName } as any, + { conditionType: () => 'ItemCondition' }, + 'Item' + )).toBe(condition); + }); + + it('returns undefined when neither predicate plugin is enabled', () => { + expect(resolveBulkWhereType( + { getTypeByName: (): undefined => undefined } as any, + {}, + 'Item' + )).toBeUndefined(); + }); +}); diff --git a/graphile/graphile-bulk-mutations/__tests__/identifier-quoting.test.ts b/graphile/graphile-bulk-mutations/__tests__/identifier-quoting.test.ts new file mode 100644 index 0000000000..9c091f4da4 --- /dev/null +++ b/graphile/graphile-bulk-mutations/__tests__/identifier-quoting.test.ts @@ -0,0 +1,42 @@ +import { + buildBulkDeleteSQL, + buildBulkInsertSQL, + buildBulkUpdateSQL, +} from '../src/utils/sql-builder'; + +describe('bulk mutation catalog identifier quoting', () => { + const hostile = 'value" RETURNING secret --'; + const quoted = '"value"" RETURNING secret --"'; + + it('escapes insert, conflict, update, and returning identifiers', () => { + const [query] = buildBulkInsertSQL( + 'tenant_a.items', + [{ name: hostile, sqlType: 'text' }], + [{ [hostile]: 'safe-value' }], + [hostile], + { conflictColumns: [hostile], action: 'UPDATE', updateColumns: [hostile] } + ); + + expect(query.text).toContain(`(${quoted})`); + expect(query.text).toContain(`ON CONFLICT (${quoted})`); + expect(query.text).toContain(`${quoted} = EXCLUDED.${quoted}`); + expect(query.text).toContain(`RETURNING ${quoted}`); + expect(query.values).toEqual(['safe-value']); + }); + + it('escapes update and delete identifiers', () => { + const update = buildBulkUpdateSQL( + 'tenant_a.items', + { [hostile]: 'safe-value' }, + [{ name: hostile, sqlType: 'text' }], + [hostile], + 'TRUE', + [] + ); + const deletion = buildBulkDeleteSQL('tenant_a.items', [hostile], 'TRUE', []); + + expect(update.text).toContain(`${quoted} = $1::text`); + expect(update.text).toContain(`RETURNING ${quoted}`); + expect(deletion.text).toContain(`RETURNING ${quoted}`); + }); +}); diff --git a/graphile/graphile-bulk-mutations/__tests__/pg-client-query-contract.test.ts b/graphile/graphile-bulk-mutations/__tests__/pg-client-query-contract.test.ts new file mode 100644 index 0000000000..183b58c8e2 --- /dev/null +++ b/graphile/graphile-bulk-mutations/__tests__/pg-client-query-contract.test.ts @@ -0,0 +1,29 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const plugins = [ + 'BulkInsertPlugin', + 'BulkUpsertPlugin', + 'BulkUpdatePlugin', + 'BulkDeletePlugin' +] as const; + +describe.each(plugins)('%s PgClient query contract', (plugin) => { + const source = readFileSync( + join(__dirname, '..', 'src', 'plugins', `${plugin}.ts`), + 'utf8' + ); + + it('types the callback as the @dataplan/pg PgClient', () => { + expect(source).toContain('pgClient: PgClient'); + }); + + it('uses object-form query arguments instead of node-postgres positional arguments', () => { + const queryCall = String.raw`pgClient\.query(?:<[^)]+>)?\(`; + expect(source).toMatch(new RegExp(`${queryCall}\\s*\\{`)); + expect(source).not.toMatch(new RegExp(`${queryCall}\\s*(?:\`|'|")`)); + expect(source).not.toMatch( + new RegExp(`${queryCall}\\s*[A-Za-z_$][\\w$]*\\s*,`) + ); + }); +}); diff --git a/graphile/graphile-bulk-mutations/package.json b/graphile/graphile-bulk-mutations/package.json index d0257abee5..f3e641b890 100644 --- a/graphile/graphile-bulk-mutations/package.json +++ b/graphile/graphile-bulk-mutations/package.json @@ -41,6 +41,9 @@ "bugs": { "url": "https://github.com/constructive-io/constructive/issues" }, + "dependencies": { + "@pgsql/quotes": "^18.1.0" + }, "devDependencies": { "@types/node": "^22.19.11", "graphile-test": "workspace:^", diff --git a/graphile/graphile-bulk-mutations/src/plugins/BulkDeletePlugin.ts b/graphile/graphile-bulk-mutations/src/plugins/BulkDeletePlugin.ts index 4731702283..4bc8d6d384 100644 --- a/graphile/graphile-bulk-mutations/src/plugins/BulkDeletePlugin.ts +++ b/graphile/graphile-bulk-mutations/src/plugins/BulkDeletePlugin.ts @@ -1,10 +1,12 @@ import '../augmentations'; -import { sideEffectWithPgClient } from '@dataplan/pg'; +import { type PgClient,sideEffectWithPgClient } from '@dataplan/pg'; +import { QuoteUtils } from '@pgsql/quotes'; import type { GraphileConfig } from 'graphile-config'; import type { GraphQLInputType,GraphQLOutputType } from 'graphql'; const version = '0.1.0'; +const qi = (name: string): string => QuoteUtils.quoteIdentifier(name); /** * BulkDeletePlugin @@ -79,7 +81,7 @@ export const BulkDeletePlugin: GraphileConfig.Plugin = { // Extract primary key columns for RETURNING clause const primaryUnique = resource.uniques.find((u: any) => u.isPrimary) ?? resource.uniques[0]; const pkColumns: string[] = primaryUnique.attributes; - const pkReturning = pkColumns.map((c) => `"${c}"`).join(', '); + const pkReturning = pkColumns.map(qi).join(', '); const compiledFrom = sql.compile(resource.from).text; @@ -105,7 +107,7 @@ export const BulkDeletePlugin: GraphileConfig.Plugin = { const $result = sideEffectWithPgClient( executor, $input, - async (pgClient: any, input: any) => { + async (pgClient: PgClient, input: any) => { if (requireWhere && (!input.where || Object.keys(input.where).length === 0)) { throw new Error( 'Bulk delete requires a non-empty where condition. Set bulkRequireWhere: false to allow unrestricted deletes.' @@ -125,11 +127,11 @@ export const BulkDeletePlugin: GraphileConfig.Plugin = { const sqlType = attrToSqlType[attrName]; if (spec === null) { - whereClauses.push(`"${attrName}" IS NULL`); + whereClauses.push(`${qi(attrName)} IS NULL`); } else if (spec !== undefined && typeof spec !== 'object') { // Simple equality (Condition type) values.push(spec); - whereClauses.push(`"${attrName}" = $${values.length}::${sqlType}`); + whereClauses.push(`${qi(attrName)} = $${values.length}::${sqlType}`); } else if (spec && typeof spec === 'object') { // Operator-based (Filter type) for (const [op, val] of Object.entries(spec) as [string, any][]) { @@ -137,22 +139,22 @@ export const BulkDeletePlugin: GraphileConfig.Plugin = { const paramRef = `$${values.length}::${sqlType}`; switch (op) { case 'equalTo': - whereClauses.push(`"${attrName}" = ${paramRef}`); + whereClauses.push(`${qi(attrName)} = ${paramRef}`); break; case 'notEqualTo': - whereClauses.push(`"${attrName}" != ${paramRef}`); + whereClauses.push(`${qi(attrName)} != ${paramRef}`); break; case 'greaterThan': - whereClauses.push(`"${attrName}" > ${paramRef}`); + whereClauses.push(`${qi(attrName)} > ${paramRef}`); break; case 'greaterThanOrEqualTo': - whereClauses.push(`"${attrName}" >= ${paramRef}`); + whereClauses.push(`${qi(attrName)} >= ${paramRef}`); break; case 'lessThan': - whereClauses.push(`"${attrName}" < ${paramRef}`); + whereClauses.push(`${qi(attrName)} < ${paramRef}`); break; case 'lessThanOrEqualTo': - whereClauses.push(`"${attrName}" <= ${paramRef}`); + whereClauses.push(`${qi(attrName)} <= ${paramRef}`); break; case 'in': if (Array.isArray(val)) { @@ -161,15 +163,15 @@ export const BulkDeletePlugin: GraphileConfig.Plugin = { return `$${values.length}::${sqlType}`; }); values.pop(); - whereClauses.push(`"${attrName}" IN (${placeholders.join(', ')})`); + whereClauses.push(`${qi(attrName)} IN (${placeholders.join(', ')})`); } break; case 'isNull': values.pop(); if (val) { - whereClauses.push(`"${attrName}" IS NULL`); + whereClauses.push(`${qi(attrName)} IS NULL`); } else { - whereClauses.push(`"${attrName}" IS NOT NULL`); + whereClauses.push(`${qi(attrName)} IS NOT NULL`); } break; default: @@ -194,7 +196,7 @@ export const BulkDeletePlugin: GraphileConfig.Plugin = { // Use RETURNING instead of RETURNING * // For delete, we capture PKs before rows are gone const text = `DELETE FROM ${compiledFrom}\nWHERE ${whereStr}\nRETURNING ${pkReturning}`; - const mutationResult = await pgClient.query(text, values); + const mutationResult = await pgClient.query({ text, values }); const affectedCount = mutationResult.rowCount ?? 0; // For delete, rows no longer exist so we can't do a diff --git a/graphile/graphile-bulk-mutations/src/plugins/BulkInsertPlugin.ts b/graphile/graphile-bulk-mutations/src/plugins/BulkInsertPlugin.ts index 6b128024a1..ea2af5a74a 100644 --- a/graphile/graphile-bulk-mutations/src/plugins/BulkInsertPlugin.ts +++ b/graphile/graphile-bulk-mutations/src/plugins/BulkInsertPlugin.ts @@ -1,6 +1,7 @@ import '../augmentations'; -import { sideEffectWithPgClient } from '@dataplan/pg'; +import { type PgClient,sideEffectWithPgClient } from '@dataplan/pg'; +import { QuoteUtils } from '@pgsql/quotes'; import type { GraphileConfig } from 'graphile-config'; import type { GraphQLInputType, GraphQLOutputType } from 'graphql'; @@ -10,6 +11,7 @@ import type { ColumnSpec } from '../utils/sql-builder'; import { buildBulkInsertSQL } from '../utils/sql-builder'; const version = '0.1.0'; +const qi = (name: string): string => QuoteUtils.quoteIdentifier(name); /** * BulkInsertPlugin @@ -131,7 +133,7 @@ export const BulkInsertPlugin: GraphileConfig.Plugin = { const $result = sideEffectWithPgClient( executor, $input, - async (pgClient: any, input: any) => { + async (pgClient: PgClient, input: any) => { const values = input.values; if (!values || !Array.isArray(values) || values.length === 0) { return { affectedCount: 0, returning: [] }; @@ -200,10 +202,10 @@ export const BulkInsertPlugin: GraphileConfig.Plugin = { const allPkRows: Record[] = []; for (const batch of batches) { - const result = await pgClient.query( - batch.text, - batch.values - ); + const result = await pgClient.query>({ + text: batch.text, + values: batch.values + }); totalAffected += result.rowCount ?? 0; if (result.rows) { allPkRows.push(...result.rows); @@ -259,10 +261,10 @@ export const BulkInsertPlugin: GraphileConfig.Plugin = { ); for (const batch of childBatches) { - const result = await pgClient.query( - batch.text, - batch.values - ); + const result = await pgClient.query({ + text: batch.text, + values: batch.values + }); totalAffected += result.rowCount ?? 0; } } @@ -275,18 +277,18 @@ export const BulkInsertPlugin: GraphileConfig.Plugin = { const pkConditions = allPkRows.map((pkRow, rowIdx) => { return pkColumns.map((col, colIdx) => { const paramIdx = rowIdx * pkColumns.length + colIdx + 1; - return `"${col}" = $${paramIdx}`; + return `${qi(col)} = $${paramIdx}`; }).join(' AND '); }); const whereClause = pkConditions.map((c) => `(${c})`).join(' OR '); const selectParams = allPkRows.flatMap((pkRow) => pkColumns.map((col) => pkRow[col]) ); - const selectResult = await pgClient.query( - `SELECT * FROM ${compiledFrom} WHERE ${whereClause}`, - selectParams - ); - returning = selectResult.rows || []; + const selectResult = await pgClient.query>({ + text: `SELECT * FROM ${compiledFrom} WHERE ${whereClause}`, + values: selectParams + }); + returning = [...selectResult.rows]; } return { diff --git a/graphile/graphile-bulk-mutations/src/plugins/BulkTypesPlugin.ts b/graphile/graphile-bulk-mutations/src/plugins/BulkTypesPlugin.ts index 18717fe73e..ecc34af7ed 100644 --- a/graphile/graphile-bulk-mutations/src/plugins/BulkTypesPlugin.ts +++ b/graphile/graphile-bulk-mutations/src/plugins/BulkTypesPlugin.ts @@ -25,6 +25,27 @@ function isBulkMutationCandidate(resource: any): boolean { ); } +/** + * Resolve the strongest available predicate input without assuming that the + * built-in condition plugin is enabled. Constructive disables that plugin when + * graphile-connection-filter supplies the richer `${typeName}Filter` type. + */ +export function resolveBulkWhereType( + build: Pick, + inflection: { conditionType?: (typeName: string) => string }, + typeName: string +): GraphQLInputType | undefined { + const filterType = build.getTypeByName(`${typeName}Filter`) as + | GraphQLInputType + | undefined; + if (filterType) return filterType; + + const conditionTypeName = inflection.conditionType?.(typeName); + return conditionTypeName + ? build.getTypeByName(conditionTypeName) as GraphQLInputType | undefined + : undefined; +} + /** * BulkTypesPlugin * @@ -465,16 +486,7 @@ export const BulkTypesPlugin: GraphileConfig.Plugin = { where: fieldWithHooks( { fieldName: 'where' }, () => { - // Try to use connection-filter type if available - const filterTypeName = `${typeName}Filter`; - const filterType = build.getTypeByName(filterTypeName) as GraphQLInputType | undefined; - // Fall back to PostGraphile's built-in condition type - const conditionTypeName = inflection.conditionType( - typeName - ); - const conditionType = - build.getTypeByName(conditionTypeName) as GraphQLInputType | undefined; - const whereType = filterType || conditionType; + const whereType = resolveBulkWhereType(build, inflection, typeName); return { description: 'Condition to select which rows to update.', @@ -511,14 +523,7 @@ export const BulkTypesPlugin: GraphileConfig.Plugin = { where: fieldWithHooks( { fieldName: 'where' }, () => { - const filterTypeName = `${typeName}Filter`; - const filterType = build.getTypeByName(filterTypeName) as GraphQLInputType | undefined; - const conditionTypeName = inflection.conditionType( - typeName - ); - const conditionType = - build.getTypeByName(conditionTypeName) as GraphQLInputType | undefined; - const whereType = filterType || conditionType; + const whereType = resolveBulkWhereType(build, inflection, typeName); return { description: 'Condition to select which rows to delete.', diff --git a/graphile/graphile-bulk-mutations/src/plugins/BulkUpdatePlugin.ts b/graphile/graphile-bulk-mutations/src/plugins/BulkUpdatePlugin.ts index 8ef1170067..e1fcdf3894 100644 --- a/graphile/graphile-bulk-mutations/src/plugins/BulkUpdatePlugin.ts +++ b/graphile/graphile-bulk-mutations/src/plugins/BulkUpdatePlugin.ts @@ -1,10 +1,12 @@ import '../augmentations'; -import { sideEffectWithPgClient } from '@dataplan/pg'; +import { type PgClient,sideEffectWithPgClient } from '@dataplan/pg'; +import { QuoteUtils } from '@pgsql/quotes'; import type { GraphileConfig } from 'graphile-config'; import type { GraphQLInputType, GraphQLOutputType } from 'graphql'; const version = '0.1.0'; +const qi = (name: string): string => QuoteUtils.quoteIdentifier(name); /** * BulkUpdatePlugin @@ -80,7 +82,7 @@ export const BulkUpdatePlugin: GraphileConfig.Plugin = { // Extract primary key columns for RETURNING clause const primaryUnique = resource.uniques.find((u: any) => u.isPrimary) ?? resource.uniques[0]; const pkColumns: string[] = primaryUnique.attributes; - const pkReturning = pkColumns.map((c) => `"${c}"`).join(', '); + const pkReturning = pkColumns.map(qi).join(', '); const compiledFrom = sql.compile(resource.from).text; @@ -106,7 +108,7 @@ export const BulkUpdatePlugin: GraphileConfig.Plugin = { const $result = sideEffectWithPgClient( executor, $input, - async (pgClient: any, input: any) => { + async (pgClient: PgClient, input: any) => { if (requireWhere && (!input.where || Object.keys(input.where).length === 0)) { throw new Error( 'Bulk update requires a non-empty where condition. Set bulkRequireWhere: false to allow unrestricted updates.' @@ -126,7 +128,7 @@ export const BulkUpdatePlugin: GraphileConfig.Plugin = { if (!attrName) continue; const sqlType = attrToSqlType[attrName]; values.push(val); - setClauses.push(`"${attrName}" = $${values.length}::${sqlType}`); + setClauses.push(`${qi(attrName)} = $${values.length}::${sqlType}`); } if (setClauses.length === 0) { @@ -144,11 +146,11 @@ export const BulkUpdatePlugin: GraphileConfig.Plugin = { const sqlType = attrToSqlType[attrName]; if (spec === null) { - whereClauses.push(`"${attrName}" IS NULL`); + whereClauses.push(`${qi(attrName)} IS NULL`); } else if (spec !== undefined && typeof spec !== 'object') { // Simple equality (Condition type) values.push(spec); - whereClauses.push(`"${attrName}" = $${values.length}::${sqlType}`); + whereClauses.push(`${qi(attrName)} = $${values.length}::${sqlType}`); } else if (spec && typeof spec === 'object') { // Operator-based (Filter type) for (const [op, val] of Object.entries(spec) as [string, any][]) { @@ -156,22 +158,22 @@ export const BulkUpdatePlugin: GraphileConfig.Plugin = { const paramRef = `$${values.length}::${sqlType}`; switch (op) { case 'equalTo': - whereClauses.push(`"${attrName}" = ${paramRef}`); + whereClauses.push(`${qi(attrName)} = ${paramRef}`); break; case 'notEqualTo': - whereClauses.push(`"${attrName}" != ${paramRef}`); + whereClauses.push(`${qi(attrName)} != ${paramRef}`); break; case 'greaterThan': - whereClauses.push(`"${attrName}" > ${paramRef}`); + whereClauses.push(`${qi(attrName)} > ${paramRef}`); break; case 'greaterThanOrEqualTo': - whereClauses.push(`"${attrName}" >= ${paramRef}`); + whereClauses.push(`${qi(attrName)} >= ${paramRef}`); break; case 'lessThan': - whereClauses.push(`"${attrName}" < ${paramRef}`); + whereClauses.push(`${qi(attrName)} < ${paramRef}`); break; case 'lessThanOrEqualTo': - whereClauses.push(`"${attrName}" <= ${paramRef}`); + whereClauses.push(`${qi(attrName)} <= ${paramRef}`); break; case 'in': if (Array.isArray(val)) { @@ -180,15 +182,15 @@ export const BulkUpdatePlugin: GraphileConfig.Plugin = { return `$${values.length}::${sqlType}`; }); values.pop(); - whereClauses.push(`"${attrName}" IN (${placeholders.join(', ')})`); + whereClauses.push(`${qi(attrName)} IN (${placeholders.join(', ')})`); } break; case 'isNull': values.pop(); if (val) { - whereClauses.push(`"${attrName}" IS NULL`); + whereClauses.push(`${qi(attrName)} IS NULL`); } else { - whereClauses.push(`"${attrName}" IS NOT NULL`); + whereClauses.push(`${qi(attrName)} IS NOT NULL`); } break; default: @@ -212,28 +214,31 @@ export const BulkUpdatePlugin: GraphileConfig.Plugin = { // Use RETURNING instead of RETURNING * const text = `UPDATE ${compiledFrom}\nSET ${setClauses.join(', ')}\nWHERE ${whereStr}\nRETURNING ${pkReturning}`; - const mutationResult = await pgClient.query(text, values); + const mutationResult = await pgClient.query>({ + text, + values + }); const affectedCount = mutationResult.rowCount ?? 0; // Follow-up SELECT using PKs to respect column-level grants let returning: unknown[] = []; if (mutationResult.rows && mutationResult.rows.length > 0) { - const pkRows: Record[] = mutationResult.rows; + const pkRows = mutationResult.rows; const pkConditions = pkRows.map((pkRow, rowIdx) => { return pkColumns.map((col, colIdx) => { const paramIdx = rowIdx * pkColumns.length + colIdx + 1; - return `"${col}" = $${paramIdx}`; + return `${qi(col)} = $${paramIdx}`; }).join(' AND '); }); const selectWhere = pkConditions.map((c) => `(${c})`).join(' OR '); const selectParams = pkRows.flatMap((pkRow) => pkColumns.map((col) => pkRow[col]) ); - const selectResult = await pgClient.query( - `SELECT * FROM ${compiledFrom} WHERE ${selectWhere}`, - selectParams - ); - returning = selectResult.rows || []; + const selectResult = await pgClient.query>({ + text: `SELECT * FROM ${compiledFrom} WHERE ${selectWhere}`, + values: selectParams + }); + returning = [...selectResult.rows]; } return { diff --git a/graphile/graphile-bulk-mutations/src/plugins/BulkUpsertPlugin.ts b/graphile/graphile-bulk-mutations/src/plugins/BulkUpsertPlugin.ts index 4b261c7562..dcd894cb05 100644 --- a/graphile/graphile-bulk-mutations/src/plugins/BulkUpsertPlugin.ts +++ b/graphile/graphile-bulk-mutations/src/plugins/BulkUpsertPlugin.ts @@ -1,6 +1,7 @@ import '../augmentations'; -import { sideEffectWithPgClient } from '@dataplan/pg'; +import { type PgClient,sideEffectWithPgClient } from '@dataplan/pg'; +import { QuoteUtils } from '@pgsql/quotes'; import type { GraphileConfig } from 'graphile-config'; import type { GraphQLInputType, GraphQLOutputType } from 'graphql'; @@ -8,6 +9,7 @@ import type { ColumnSpec } from '../utils/sql-builder'; import { buildBulkInsertSQL } from '../utils/sql-builder'; const version = '0.1.0'; +const qi = (name: string): string => QuoteUtils.quoteIdentifier(name); /** * BulkUpsertPlugin @@ -118,7 +120,7 @@ export const BulkUpsertPlugin: GraphileConfig.Plugin = { const $result = sideEffectWithPgClient( executor, $input, - async (pgClient: any, input: any) => { + async (pgClient: PgClient, input: any) => { const values = input.values; if (!values || !Array.isArray(values) || values.length === 0) { return { affectedCount: 0, returning: [] }; @@ -177,10 +179,10 @@ export const BulkUpsertPlugin: GraphileConfig.Plugin = { const allPkRows: Record[] = []; for (const batch of batches) { - const result = await pgClient.query( - batch.text, - batch.values - ); + const result = await pgClient.query>({ + text: batch.text, + values: batch.values + }); totalAffected += result.rowCount ?? 0; if (result.rows) { allPkRows.push(...result.rows); @@ -193,18 +195,18 @@ export const BulkUpsertPlugin: GraphileConfig.Plugin = { const pkConditions = allPkRows.map((pkRow, rowIdx) => { return pkColumns.map((col, colIdx) => { const paramIdx = rowIdx * pkColumns.length + colIdx + 1; - return `"${col}" = $${paramIdx}`; + return `${qi(col)} = $${paramIdx}`; }).join(' AND '); }); const whereClause = pkConditions.map((c) => `(${c})`).join(' OR '); const selectParams = allPkRows.flatMap((pkRow) => pkColumns.map((col) => pkRow[col]) ); - const selectResult = await pgClient.query( - `SELECT * FROM ${compiledFrom} WHERE ${whereClause}`, - selectParams - ); - returning = selectResult.rows || []; + const selectResult = await pgClient.query>({ + text: `SELECT * FROM ${compiledFrom} WHERE ${whereClause}`, + values: selectParams + }); + returning = [...selectResult.rows]; } return { diff --git a/graphile/graphile-bulk-mutations/src/utils/sql-builder.ts b/graphile/graphile-bulk-mutations/src/utils/sql-builder.ts index b649aa3cf6..906bb3ba4e 100644 --- a/graphile/graphile-bulk-mutations/src/utils/sql-builder.ts +++ b/graphile/graphile-bulk-mutations/src/utils/sql-builder.ts @@ -12,8 +12,12 @@ * See: https://github.com/pyramation/graphile-column-privileges-mutations */ +import { QuoteUtils } from '@pgsql/quotes'; + import { PG_MAX_PARAMS } from '../types'; +const qi = (name: string): string => QuoteUtils.quoteIdentifier(name); + export interface ColumnSpec { name: string; sqlType: string; @@ -45,12 +49,12 @@ export function buildBulkInsertSQL( updateColumns?: string[]; } ): InsertBatch[] { - const colNames = columns.map((c) => `"${c.name}"`); + const colNames = columns.map((c) => qi(c.name)); const colsPerRow = columns.length; const maxRowsPerBatch = Math.floor(PG_MAX_PARAMS / colsPerRow); const returningClause = returningColumns.length > 0 - ? returningColumns.map((c) => `"${c}"`).join(', ') + ? returningColumns.map(qi).join(', ') : '*'; const batches: InsertBatch[] = []; @@ -79,7 +83,7 @@ export function buildBulkInsertSQL( if (onConflict) { if (onConflict.conflictColumns && onConflict.conflictColumns.length > 0) { - const colList = onConflict.conflictColumns.map((c) => `"${c}"`).join(', '); + const colList = onConflict.conflictColumns.map(qi).join(', '); text += `\nON CONFLICT (${colList})`; } else { text += '\nON CONFLICT'; @@ -93,7 +97,7 @@ export function buildBulkInsertSQL( ? onConflict.updateColumns : columns.map((c) => c.name); const setClause = setCols - .map((c) => `"${c}" = EXCLUDED."${c}"`) + .map((c) => `${qi(c)} = EXCLUDED.${qi(c)}`) .join(', '); text += ` DO UPDATE SET ${setClause}`; } @@ -130,7 +134,7 @@ export function buildBulkUpdateSQL( if (value === undefined) continue; values.push(value); - setClauses.push(`"${col.name}" = $${values.length}::${col.sqlType}`); + setClauses.push(`${qi(col.name)} = $${values.length}::${col.sqlType}`); } if (setClauses.length === 0) { @@ -146,7 +150,7 @@ export function buildBulkUpdateSQL( values.push(...whereParams); const returningClause = returningColumns.length > 0 - ? returningColumns.map((c) => `"${c}"`).join(', ') + ? returningColumns.map(qi).join(', ') : '*'; const text = `UPDATE ${tableName}\nSET ${setClauses.join(', ')}\nWHERE ${renumberedWhere}\nRETURNING ${returningClause}`; @@ -167,7 +171,7 @@ export function buildBulkDeleteSQL( whereParams: unknown[] ): { text: string; values: unknown[] } { const returningClause = returningColumns.length > 0 - ? returningColumns.map((c) => `"${c}"`).join(', ') + ? returningColumns.map(qi).join(', ') : '*'; const text = `DELETE FROM ${tableName}\nWHERE ${whereClause}\nRETURNING ${returningClause}`; diff --git a/graphile/graphile-cache/README.md b/graphile/graphile-cache/README.md index 5eb5c674ba..7582dd6db9 100644 --- a/graphile/graphile-cache/README.md +++ b/graphile/graphile-cache/README.md @@ -13,7 +13,8 @@

-PostGraphile instance LRU cache with automatic cleanup when PostgreSQL pools are disposed. +Heap-budgeted PostGraphile v5 instance cache with request draining, serialized +build admission, and explicit PostgreSQL pool ownership. ## Installation @@ -21,119 +22,134 @@ PostGraphile instance LRU cache with automatic cleanup when PostgreSQL pools are npm install graphile-cache pg-cache ``` -Note: This package depends on `pg-cache` for the PostgreSQL pool management. +`graphile-cache` uses `pg-cache` leases to keep each resident instance's exact +runtime pool alive until the instance has fully drained and shut down. ## Features -- LRU cache for PostGraphile instances -- Automatic cleanup when associated PostgreSQL pools are disposed -- Integrates seamlessly with `pg-cache` -- Service cache re-exported for convenience -- TypeScript support +- Heap-derived residency limits plus an optional process-RSS admission ceiling +- Request-aware eviction that never tears down an instance in use +- Awaited HTTP, realtime, PostGraphile, and pool-lease teardown +- Memory-pressure refusal and eviction counters +- Exact pool identities protected by reference-counted `pg-cache` leases ## How It Works -When you import this package, it automatically registers a cleanup callback with `pg-cache`. When a PostgreSQL pool is disposed, any PostGraphile instances using that pool are automatically removed from the cache. +Long-lived callers acquire a `PgPoolLease`, configure PostGraphile with the +lease's pool, and pass the same lease to `createGraphileInstance()`. Ownership +transfers to the returned entry only when that promise resolves. If creation +rejects, the caller still owns the lease and must release it. + +Eviction marks the entry as disposing, waits for its requests to drain, closes +the HTTP server, stops realtime delivery, attempts `pgl.release()`, and finally +releases the pool lease. Every teardown stage is attempted even when an earlier +stage fails, and duplicate disposal calls share one promise. ## Usage -### Basic Usage +### Creating a leased instance ```typescript -import { graphileCache, GraphileCache } from 'graphile-cache'; -import { getPgPool } from 'pg-cache'; -import { postgraphile } from 'postgraphile'; - -// Create a PostGraphile instance -const pgPool = getPgPool({ database: 'mydb' }); -const handler = postgraphile(pgPool, 'public', { - // PostGraphile options -}); - -// Cache it -const cacheEntry: GraphileCache = { - pgPool, - pgPoolKey: 'mydb', - handler -}; - -graphileCache.set('mydb.public', cacheEntry); +import { + createGraphileInstance, + disposeUncachedEntry, + graphileCache +} from 'graphile-cache'; +import { acquirePgPool } from 'pg-cache'; + +const cacheKey = 'tenant-id:api-id:build-contract-hash'; +const lease = acquirePgPool( + { database: 'tenant_database' }, + { purpose: 'runtime', sanitizeOnCheckout: true } +); + +// Application code builds this preset with makePgService({ pool: lease.pool, +// schemas: ['tenant_api'] }) and its exact plugin/settings contract. +const preset = makePreset(lease.pool); +let entry; +try { + entry = await createGraphileInstance({ + preset, + cacheKey, + poolLease: lease, + poolIdentity: lease.identity, + enableRealtime: true, + realtimeSchema: 'tenant_a_realtime', + realtimeSourceSchemas: ['tenant_api'] + }); +} catch (error) { + // Creation rejected before ownership transfer. + lease.release(); + throw error; +} -// Retrieve it later -const cached = graphileCache.get('mydb.public'); -if (cached) { - // Use cached.handler +// Creation resolved, so disposal must now release the entry-owned lease if +// admission or publication fails. +try { + graphileCache.set(cacheKey, entry); +} catch (error) { + await disposeUncachedEntry(entry, cacheKey); + throw error; } ``` -### Automatic Cleanup +`poolIdentity` is optional when `poolLease` is present because the lease identity +becomes the entry's authoritative identity. Supplying both with different values +fails before ownership transfers. -The cleanup happens automatically: +### Serving and eviction ```typescript -import { pgCache } from 'pg-cache'; -import { graphileCache } from 'graphile-cache'; - -// Add entries -graphileCache.set('mydb.public', { pgPoolKey: 'mydb', ... }); -graphileCache.set('mydb.private', { pgPoolKey: 'mydb', ... }); - -// When the pool is removed... -pgCache.delete('mydb'); - -// Both graphile entries are automatically cleaned up! -console.log(graphileCache.has('mydb.public')); // false -console.log(graphileCache.has('mydb.private')); // false -``` - -### Complete Example - -```typescript -import { graphileCache, GraphileCache } from 'graphile-cache'; -import { getPgPool } from 'pg-cache'; -import { postgraphile } from 'postgraphile'; - -function getGraphileInstance(database: string, schema: string): GraphileCache { - const key = `${database}.${schema}`; - - // Check cache first - const cached = graphileCache.get(key); - if (cached) { - return cached; - } - - // Create new instance - const pgPool = getPgPool({ database }); - const handler = postgraphile(pgPool, schema, { - graphqlRoute: '/graphql', - graphiqlRoute: '/graphiql', - // other options... - }); - - const entry: GraphileCache = { - pgPool, - pgPoolKey: database, - handler - }; - - // Cache it - graphileCache.set(key, entry); - return entry; +import { + deleteGraphileCacheEntry, + graphileCache, + invokeEntryHandler +} from 'graphile-cache'; + +const entry = graphileCache.get(cacheKey); +if (entry && invokeEntryHandler(entry, req, res, next)) { + return; } -// Use in Express -app.use((req, res, next) => { - const { handler } = getGraphileInstance('mydb', 'public'); - handler(req, res, next); -}); +// Resolves only after teardown and pool-lease release complete. +await deleteGraphileCacheEntry(cacheKey, 'manual'); ``` +Use `invokeEntryHandler()` for resident traffic so disposal can observe in-flight +requests. A false return means the entry has started draining; route the request +through normal cache-miss/build admission instead. + +### Shared exact-topic realtime + +`sharedRealtime` is an opt-in build-time seam. The caller installs one +`ActivatableGenerationScopedRealtimeSubscriber` in the PostGraphile service, +collects the exact physical `@realtime` topics during schema construction, and +supplies a dedicated least-privilege listener login. Instance creation audits +that login on the broker's pinned client, acquires only those topics, and +activates the subscriber before the entry can be published. Audit and LISTEN +therefore remain safe when the notification pool has `max: 1`. + +One canonical host/port/database target may have only one active opaque listener +identity and role. TLS remains part of that listener identity, so a TLS, +credential, or pool-contract change fails closed while the old generation is +resident instead of opening a second listener and silently reducing density; +rotate by invalidating and draining the old generations first. Resolver output +must use stable canonical connection target values, because two DNS aliases for +the same server cannot be proven to name one physical database in-process. + +Successful role audits have an explicit TTL. One unref'ed timer per exact +listener identity proactively re-audits idle subscriptions, while HTTP and +WebSocket operation boundaries use the same coalesced refresh as an immediate +gate. Broker termination and privilege drift latch every affected generation +unavailable. The timer is cancelled after the last generation releases. The +default realtime mode remains the dedicated PostGraphile subscriber. + ### Graceful Shutdown ```typescript import { closeAllCaches } from 'graphile-cache'; -// This closes all caches including pg pools +// Drains Graphile entries first, then closes the remaining pg-cache pools. process.on('SIGTERM', async () => { await closeAllCaches(); process.exit(0); @@ -142,39 +158,64 @@ process.on('SIGTERM', async () => { ## API Reference -### graphileCache - -The main PostGraphile instance cache. - -- `get(key: string): GraphileCache | undefined` - Get a cached instance -- `set(key: string, value: GraphileCache): void` - Cache an instance -- `has(key: string): boolean` - Check if an instance is cached -- `delete(key: string): void` - Remove an instance -- `clear(): void` - Remove all instances - -### GraphileCache Interface - -```typescript -interface GraphileCache { - pgPool: pg.Pool; - pgPoolKey: string; - handler: HttpRequestHandler; -} -``` - -### closeAllCaches() - -Closes all caches including the service cache, graphile cache, and all PostgreSQL pools. - -### svcCache - -Re-exported from `pg-cache` for convenience. - -## Integration Details - -The integration with `pg-cache` happens automatically when this module is imported. The cleanup callback is registered immediately, ensuring that PostGraphile instances are cleaned up whenever their associated PostgreSQL pools are disposed. - -This design ensures: -- No memory leaks from orphaned PostGraphile instances -- Automatic cleanup without manual intervention -- Loose coupling between packages +### Main lifecycle APIs + +- `createGraphileInstance(options)` creates a ready PostGraphile entry and + accepts an optional retained `PgPoolLease`. Realtime callers may provide the + exact cursor-function schema through `realtimeSchema`; omission preserves the + `realtime_public` compatibility default. Realtime also requires exact + `realtimeSourceSchemas`; a foreign cursor row stops delivery before any row + in that batch is emitted. Cursor node IDs combine a process-unique replica + identity with the exact cache contract so replicas cannot share cursor state. + A fatal delivery-integrity failure latches that exact generation unhealthy; + the next request receives `503 GRAPHILE_REALTIME_UNAVAILABLE`, never enters + its Graphile handler, and identity-checks the generation before retiring it + so a later request can rebuild without risking a healthy replacement. +- `invokeEntryHandler(entry, req, res, next)` tracks a request against an exact + resident entry. +- `deleteGraphileCacheEntry(key, reason)` evicts and awaits teardown. +- `clearGraphileCache()` evicts and awaits every resident entry. +- `closeAllCaches()` drains Graphile entries, then closes `pg-cache`. + +### Capacity and observability + +- `prepareCacheForBuild()` serializes admission with awaited eviction. +- `getCacheConfig()` reports the heap-derived capacity and calibration sources. +- `getCacheStats()` reports residency, realtime-unhealthy generations, + aggregate credential-free listener-role attestation health, unique active + broker identities, and monotonic catalog-audit attempts/failures. Generation + references are reported separately, so three API surfaces sharing one role + audit don't triple-count its database QPS. +- `getCacheCounters()` reports monotonic admitted/completed HTTP and WebSocket + lifecycles alongside evictions, disposal failures, and build refusals. The + lifecycle counters make short-lived work observable even when both ends fall + between two state snapshots. +- `startMemoryGovernor()` starts pressure-driven idle eviction and returns an + idempotent stop callback. + +`GRAPHILE_CACHE_MAX` caps Graphile build contracts by heap budget. +`GRAPHILE_CACHE_ADMISSION_MODE=preserve-resident` makes that ceiling a strict +admission boundary: a new contract receives `resident_capacity` without +evicting an existing resident. The default, `evict-idle`, retains the ordinary +LRU replacement behavior. +`GRAPHILE_CACHE_RSS_LIMIT_BYTES` adds a fail-closed process-RSS ceiling, and +admission reserves `GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES` (768 MiB by +default) above current RSS before starting a build. When the RSS ceiling is not +set, RSS remains present in cache pressure telemetry but does not constrain +admission. `PG_CACHE_MAX` +caps PostgreSQL connection identities, which may include runtime, control-plane, +listener, and diagnostic pools. They are independent limits: a resident entry's +lease prevents ordinary pool LRU or TTL eviction, and acquiring a new identity +fails closed when every registry slot is leased. + +The LRU's internal ceiling scales with the configured V8 heap (one sparse slot +per 256 KiB, bounded from 1,024 to 65,536). It is only a backing-structure +limit; measured instance cost, server/build reserves, and live pressure still +decide how many entries may become resident. + +## Pool disposal integration + +The package still registers a `pg-cache` cleanup callback as a fail-safe for +legacy unleased entries and explicit process-wide shutdown. Normal resident +lifetime is lease-driven: Graphile disposal releases the lease, after which +`pg-cache` may evict or expire the now-idle pool identity. diff --git a/graphile/graphile-cache/package.json b/graphile/graphile-cache/package.json index d8f4ac6080..ea625078d1 100644 --- a/graphile/graphile-cache/package.json +++ b/graphile/graphile-cache/package.json @@ -2,7 +2,7 @@ "name": "graphile-cache", "version": "4.10.1", "author": "Constructive ", - "description": "PostGraphile v5 LRU cache with automatic pool cleanup integration", + "description": "Heap-aware PostGraphile v5 cache with leased PostgreSQL pool lifecycle", "main": "index.js", "module": "esm/index.js", "types": "index.d.ts", diff --git a/graphile/graphile-cache/src/__tests__/build-readiness.test.ts b/graphile/graphile-cache/src/__tests__/build-readiness.test.ts new file mode 100644 index 0000000000..c054217bd3 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/build-readiness.test.ts @@ -0,0 +1,73 @@ +import { awaitGraphileBuildReadiness } from '../build-readiness'; + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(error: Error): void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +const flushPromises = (): Promise => new Promise((resolve) => setImmediate(resolve)); + +describe('awaitGraphileBuildReadiness', () => { + it('does not resolve before the schema build completes', async () => { + const schemaResult = deferred(); + const release = jest.fn().mockResolvedValue(undefined); + let resolved = false; + const buildPromise = awaitGraphileBuildReadiness({ + schemaResult: schemaResult.promise, + addTo: jest.fn().mockResolvedValue(undefined), + ready: jest.fn().mockResolvedValue(undefined), + release + }).then(() => { + resolved = true; + }); + + await flushPromises(); + expect(resolved).toBe(false); + + schemaResult.resolve({}); + await buildPromise; + expect(release).not.toHaveBeenCalled(); + }); + + it('releases the failed generation before rejecting', async () => { + const schemaResult = deferred(); + const release = jest.fn().mockResolvedValue(undefined); + const buildPromise = awaitGraphileBuildReadiness({ + schemaResult: schemaResult.promise, + addTo: jest.fn().mockResolvedValue(undefined), + ready: jest.fn().mockResolvedValue(undefined), + release + }); + const failure = new Error('schema build failed'); + schemaResult.reject(failure); + + await expect(buildPromise).rejects.toBe(failure); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('preserves the build failure if cleanup also fails', async () => { + const failure = new Error('schema build failed'); + const cleanupFailure = new Error('release failed'); + const onReleaseError = jest.fn(); + + await expect(awaitGraphileBuildReadiness({ + schemaResult: Promise.reject(failure), + addTo: jest.fn().mockResolvedValue(undefined), + ready: jest.fn().mockResolvedValue(undefined), + release: jest.fn().mockRejectedValue(cleanupFailure), + onReleaseError + })).rejects.toBe(failure); + expect(onReleaseError).toHaveBeenCalledWith(cleanupFailure); + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/governor.test.ts b/graphile/graphile-cache/src/__tests__/governor.test.ts new file mode 100644 index 0000000000..54d40b7842 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/governor.test.ts @@ -0,0 +1,634 @@ +import { EventEmitter } from 'node:events'; + +import type { NextFunction, Request, Response } from 'express'; +import type { PgPoolLease } from 'pg-cache'; + +import { + computeBackingCacheMax, + computeCapacityFromBudget, + disposeUncachedEntry, + evaluateBuildAdmission, + getCacheConfig, + getCacheCounters, + getCacheStats, + getInstanceHeapEstimate, + getMemoryPressure, + graphileCache, + type GraphileCacheEntry, + invokeEntryHandler, + prepareCacheForBuild, + raceWithClearedTimeout, + recordInstanceHeapSample, + resetInstanceHeapSamples, + waitForEntryDisposal +} from '../graphile-cache'; +import { GRAPHILE_REALTIME_UNAVAILABLE_CODE } from '../realtime-readiness'; + +const MB = 1024 * 1024; + +const makeEntry = (releaseDelayMs = 0): GraphileCacheEntry => ({ + pgl: { + release: jest.fn(() => new Promise((resolve) => setTimeout(resolve, releaseDelayMs))) + } as unknown as GraphileCacheEntry['pgl'], + serv: {} as GraphileCacheEntry['serv'], + handler: jest.fn() as unknown as GraphileCacheEntry['handler'], + httpServer: null, + cacheKey: 'test', + createdAt: Date.now() +}); + +const makePoolLease = (onRelease?: () => void): PgPoolLease => ({ + pool: {} as PgPoolLease['pool'], + identity: 'pg:v1:test-runtime', + release: jest.fn(() => onRelease?.()) +}); + +describe('heap budget capacity', () => { + const calibrationEnv = [ + 'GRAPHILE_CACHE_MAX', + 'GRAPHILE_CACHE_ADMISSION_MODE', + 'GRAPHILE_CACHE_INSTANCE_HEAP_BYTES', + 'GRAPHILE_CACHE_SERVER_RESERVE_BYTES', + 'GRAPHILE_CACHE_BUILD_RESERVE_BYTES', + 'GRAPHILE_CACHE_RSS_LIMIT_BYTES', + 'GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES', + 'GRAPHILE_CACHE_CALIBRATION_ID' + ] as const; + let previousEnv: Record; + + beforeEach(() => { + previousEnv = Object.fromEntries( + calibrationEnv.map((name) => [name, process.env[name]]) + ); + for (const name of calibrationEnv) delete process.env[name]; + resetInstanceHeapSamples(); + }); + + afterEach(() => { + resetInstanceHeapSamples(); + for (const name of calibrationEnv) { + const value = previousEnv[name]; + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + it('fits residency and one serialized build transient', () => { + expect(computeCapacityFromBudget(3584 * MB, 1450 * MB, 256 * MB, 768 * MB)).toBe(2); + expect(computeCapacityFromBudget(2048 * MB, 1450 * MB, 256 * MB, 768 * MB)).toBe(1); + }); + + it('returns zero when even the server and build reserves cannot fit', () => { + expect(computeCapacityFromBudget(512 * MB, 64 * MB, 256 * MB, 768 * MB)).toBe(0); + }); + + it('does not hide validated density behind a fixed backing-cache ceiling', () => { + expect(computeCapacityFromBudget(238 * MB, MB, MB, MB)).toBe(237); + expect(computeBackingCacheMax(1024 * MB)).toBe(4096); + expect(computeBackingCacheMax(4096 * MB)).toBe(16_384); + expect(graphileCache.max).toBeGreaterThanOrEqual(4096); + }); + + it('derives the backing ceiling from the modeled heap rather than this process', () => { + expect(computeCapacityFromBudget(1024 * MB, 1, 1, 1)).toBe(4096); + }); + + it('treats runtime samples as a safety floor rather than an unsafe downsize', () => { + recordInstanceHeapSample(30 * MB); + recordInstanceHeapSample(32 * MB); + recordInstanceHeapSample(34 * MB); + expect(getInstanceHeapEstimate()).toBe(512 * MB); + expect(getCacheConfig().calibration).toMatchObject({ + instanceHeapSource: 'default', + instanceHeapSampleCount: 3 + }); + }); + + it('lets runtime samples raise an explicit calibrated floor', () => { + process.env.GRAPHILE_CACHE_INSTANCE_HEAP_BYTES = String(32 * MB); + recordInstanceHeapSample(40 * MB); + recordInstanceHeapSample(50 * MB); + recordInstanceHeapSample(60 * MB); + expect(getInstanceHeapEstimate()).toBe(60 * MB); + expect(getCacheConfig().calibration.instanceHeapSource).toBe( + 'runtime-safety-floor' + ); + }); + + it('reports explicit calibration provenance and respects the operator ceiling', () => { + process.env.GRAPHILE_CACHE_INSTANCE_HEAP_BYTES = String(MB); + process.env.GRAPHILE_CACHE_SERVER_RESERVE_BYTES = String(MB); + process.env.GRAPHILE_CACHE_BUILD_RESERVE_BYTES = String(MB); + process.env.GRAPHILE_CACHE_MAX = '128'; + process.env.GRAPHILE_CACHE_CALIBRATION_ID = 'cperf:fixture:sha256'; + const config = getCacheConfig(); + expect(config.max).toBe(128); + expect(config.calibration).toEqual({ + id: 'cperf:fixture:sha256', + instanceHeapSource: 'environment', + instanceHeapSampleCount: 0, + serverReserveSource: 'environment', + buildReserveSource: 'environment' + }); + }); + + it('defaults to idle eviction and strictly validates preserve-resident admission', () => { + expect(getCacheConfig().admissionMode).toBe('evict-idle'); + process.env.GRAPHILE_CACHE_ADMISSION_MODE = 'preserve-resident'; + expect(getCacheConfig().admissionMode).toBe('preserve-resident'); + process.env.GRAPHILE_CACHE_ADMISSION_MODE = 'preserve'; + expect(() => getCacheConfig()).toThrow( + 'GRAPHILE_CACHE_ADMISSION_MODE must be evict-idle or preserve-resident' + ); + }); + + it('reports an explicit RSS ceiling and transient reservation', () => { + process.env.GRAPHILE_CACHE_RSS_LIMIT_BYTES = String(3 * 1024 * MB); + process.env.GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES = String(96 * MB); + + const config = getCacheConfig(); + const pressure = getMemoryPressure(); + const stats = getCacheStats(); + + expect(config).toMatchObject({ + rssLimitBytes: 3 * 1024 * MB, + rssBuildReserveBytes: 96 * MB + }); + expect(pressure).toMatchObject({ + rssLimitBytes: 3 * 1024 * MB, + rssBytes: expect.any(Number), + rssRatio: expect.any(Number) + }); + expect(stats).toMatchObject({ + rssLimitBytes: 3 * 1024 * MB, + rssBuildReserveBytes: 96 * MB + }); + }); + + it('keeps RSS observable but unbounded unless an operator sets a ceiling', () => { + expect(getMemoryPressure()).toMatchObject({ + rssLimitBytes: null, + rssRatio: null, + rssLevel: 'unbounded', + rssBytes: expect.any(Number) + }); + }); + + it('refuses a build whose live RSS plus transient reserve crosses the ceiling', () => { + const rssBytes = process.memoryUsage().rss; + process.env.GRAPHILE_CACHE_RSS_LIMIT_BYTES = String(rssBytes * 4); + process.env.GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES = String(rssBytes * 5); + + expect(evaluateBuildAdmission(0)).toMatchObject({ + admit: false, + reason: 'rss_budget_exceeded', + rssLimitBytes: rssBytes * 4 + }); + }); + + it.each([ + ['GRAPHILE_CACHE_INSTANCE_HEAP_BYTES', '0'], + ['GRAPHILE_CACHE_SERVER_RESERVE_BYTES', '-1'], + ['GRAPHILE_CACHE_BUILD_RESERVE_BYTES', '1.5'], + ['GRAPHILE_CACHE_RSS_LIMIT_BYTES', '0'], + ['GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES', '-10'], + ['GRAPHILE_CACHE_MAX', '12entries'], + ['GRAPHILE_CACHE_MAX', String(Number.MAX_SAFE_INTEGER + 1)] + ])('rejects invalid explicit calibration %s=%s', (name, value) => { + process.env[name] = value; + expect(() => getCacheConfig()).toThrow('must be a positive safe integer'); + }); + + it('rejects an operator ceiling above the heap-scaled backing cache', () => { + process.env.GRAPHILE_CACHE_MAX = String(graphileCache.max + 1); + expect(() => getCacheConfig()).toThrow('exceeds heap-scaled backing ceiling'); + }); +}); + +describe('entry-scoped awaited disposal', () => { + afterEach(async () => { + graphileCache.clear(); + await new Promise((resolve) => setTimeout(resolve, 5)); + }); + + it('disposes distinct rebuilt entries with the same key exactly once each', async () => { + const first = makeEntry(); + const second = makeEntry(); + await Promise.all([ + disposeUncachedEntry(first, 'same-key'), + disposeUncachedEntry(second, 'same-key') + ]); + expect(first.pgl.release).toHaveBeenCalledTimes(1); + expect(second.pgl.release).toHaveBeenCalledTimes(1); + }); + + it('waits for a resident request before releasing the instance', async () => { + const entry = makeEntry(); + const response = new EventEmitter() as unknown as Response; + invokeEntryHandler( + entry, + {} as Request, + response, + (() => undefined) as NextFunction + ); + graphileCache.set('drain', entry); + graphileCache.delete('drain'); + + await new Promise((resolve) => setTimeout(resolve, 5)); + expect(entry.pgl.release).not.toHaveBeenCalled(); + (response as unknown as EventEmitter).emit('finish'); + await expect(waitForEntryDisposal(entry, 100)).resolves.toBe(true); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + }); + + it('does not enter an instance after the request has already closed', () => { + const entry = makeEntry(); + const request = Object.assign(new EventEmitter(), { + aborted: true, + destroyed: true, + socket: { destroyed: true } + }) as unknown as Request; + const response = Object.assign(new EventEmitter(), { + destroyed: true, + writableEnded: true + }) as unknown as Response; + + expect(invokeEntryHandler( + entry, + request, + response, + (() => undefined) as NextFunction + )).toBe(false); + expect(entry.handler).not.toHaveBeenCalled(); + expect(entry.inflight ?? 0).toBe(0); + }); + + it('does enter after a JSON body parser consumed the request stream', () => { + const entry = makeEntry(); + const countersBefore = getCacheCounters(); + const request = Object.assign(new EventEmitter(), { + aborted: false, + // Express/raw-body may destroy the readable request stream after fully + // consuming it while the underlying keep-alive socket remains healthy. + destroyed: true, + socket: { destroyed: false } + }) as unknown as Request; + const response = Object.assign(new EventEmitter(), { + destroyed: false, + writableEnded: false + }) as unknown as Response; + + expect(invokeEntryHandler( + entry, + request, + response, + (() => undefined) as NextFunction + )).toBe(true); + expect(entry.handler).toHaveBeenCalledTimes(1); + expect(entry.inflight).toBe(1); + expect(getCacheCounters().httpRequestsStarted).toBe( + countersBefore.httpRequestsStarted + 1 + ); + expect(getCacheCounters().httpRequestsCompleted).toBe( + countersBefore.httpRequestsCompleted + ); + (response as unknown as EventEmitter).emit('finish'); + // Express can emit close after finish; the completion counter remains + // monotonic and records this request exactly once. + (response as unknown as EventEmitter).emit('close'); + expect(entry.inflight).toBe(0); + expect(getCacheCounters().httpRequestsCompleted).toBe( + countersBefore.httpRequestsCompleted + 1 + ); + }); + + it('returns a stable 503 and retires the exact realtime-unhealthy generation', async () => { + const entry = makeEntry(); + entry.cacheKey = 'realtime-unhealthy'; + entry.realtimeHealth = { + status: 'failed', + failureCode: 'REALTIME_SOURCE_SCHEMA_VIOLATION', + failedAt: 1_000 + }; + graphileCache.set('realtime-unhealthy', entry); + expect(getCacheStats().realtimeUnhealthy).toBe(1); + const response = Object.assign(new EventEmitter(), { + destroyed: false, + writableEnded: false, + headersSent: false, + setHeader: jest.fn(), + status: jest.fn(), + json: jest.fn() + }); + response.status.mockReturnValue(response); + + expect(invokeEntryHandler( + entry, + {} as Request, + response as unknown as Response, + (() => undefined) as NextFunction + )).toBe(true); + + expect(entry.handler).not.toHaveBeenCalled(); + expect(entry.inflight ?? 0).toBe(0); + expect(response.setHeader).toHaveBeenCalledWith('Retry-After', '15'); + expect(response.status).toHaveBeenCalledWith(503); + expect(response.json).toHaveBeenCalledWith({ + error: { + code: GRAPHILE_REALTIME_UNAVAILABLE_CODE, + message: 'Realtime delivery is unavailable for this GraphQL instance' + } + }); + expect(graphileCache.has('realtime-unhealthy')).toBe(false); + await expect(waitForEntryDisposal(entry, 100)).resolves.toBe(true); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + }); + + it('fails closed when a listener-role attestation expires before invocation', () => { + const entry = makeEntry(); + entry.realtimeRoleAttestation = { + snapshot: jest.fn(() => ({ + version: 1, + mode: 'shared-exact', + listenerIdentity: 'opaque-listener-identity', + auditVersion: 'pg-notification-role:v1', + role: 'listener', + database: 'tenant_a', + lastAttestedAt: 1, + validUntil: 2, + checks: 1, + status: 'healthy', + failureCode: null as string | null, + failedAt: null as number | null + })), + revalidateIfDue: jest.fn(async () => true), + release: jest.fn() + }; + const response = Object.assign(new EventEmitter(), { + destroyed: false, + writableEnded: false, + headersSent: false, + setHeader: jest.fn(), + status: jest.fn(), + json: jest.fn() + }); + response.status.mockReturnValue(response); + + expect(invokeEntryHandler( + entry, + {} as Request, + response as unknown as Response, + (() => undefined) as NextFunction + )).toBe(true); + + expect(entry.handler).not.toHaveBeenCalled(); + expect(response.status).toHaveBeenCalledWith(503); + expect(response.json).toHaveBeenCalledWith({ + error: { + code: GRAPHILE_REALTIME_UNAVAILABLE_CODE, + message: 'Realtime delivery is unavailable for this GraphQL instance' + } + }); + }); + + it('never lets a stale realtime generation evict a healthy replacement', () => { + const stale = makeEntry(); + stale.cacheKey = 'shared-contract'; + stale.realtimeHealth = { + status: 'failed', + failureCode: 'REALTIME_SOURCE_SCHEMA_VIOLATION', + failedAt: 1_000 + }; + const replacement = makeEntry(); + replacement.cacheKey = 'shared-contract'; + graphileCache.set('shared-contract', replacement); + const response = Object.assign(new EventEmitter(), { + destroyed: false, + writableEnded: false, + headersSent: false, + setHeader: jest.fn(), + status: jest.fn(), + json: jest.fn() + }); + response.status.mockReturnValue(response); + + expect(invokeEntryHandler( + stale, + {} as Request, + response as unknown as Response, + (() => undefined) as NextFunction + )).toBe(true); + + expect(graphileCache.peek('shared-contract')).toBe(replacement); + expect(replacement.disposing).not.toBe(true); + expect(stale.disposing).not.toBe(true); + expect(stale.handler).not.toHaveBeenCalled(); + expect(response.status).toHaveBeenCalledWith(503); + }); + + it('releases if the response closes while terminal listeners are attached', () => { + const entry = makeEntry(); + const request = new EventEmitter() as unknown as Request; + const response = new EventEmitter() as unknown as Response; + let terminalChecks = 0; + Object.defineProperty(response, 'writableEnded', { + get: () => ++terminalChecks >= 2 + }); + + expect(invokeEntryHandler( + entry, + request, + response, + (() => undefined) as NextFunction + )).toBe(false); + expect(entry.handler).not.toHaveBeenCalled(); + expect(entry.inflight).toBe(0); + expect((response as unknown as EventEmitter).listenerCount('finish')).toBe(0); + expect((response as unknown as EventEmitter).listenerCount('close')).toBe(0); + }); + + it('releases the pool lease after the complete teardown sequence', async () => { + const events: string[] = []; + const entry = makeEntry(); + entry.httpServer = { + close: (callback: () => void) => { + events.push('http-close'); + callback(); + } + } as unknown as GraphileCacheEntry['httpServer']; + entry.realtimeManager = { + stop: jest.fn(async () => { + events.push('realtime-stop'); + }) + }; + entry.pgl = { + release: jest.fn(async () => { + events.push('postgraphile-release'); + }) + } as unknown as GraphileCacheEntry['pgl']; + entry.releasePresetServices = jest.fn(async () => { + events.push('preset-services-release'); + }); + entry.poolLease = makePoolLease(() => events.push('pool-lease-release')); + + const response = new EventEmitter() as unknown as Response; + invokeEntryHandler(entry, {} as Request, response, (() => undefined) as NextFunction); + const disposal = disposeUncachedEntry(entry, 'ordered'); + + await new Promise((resolve) => setImmediate(resolve)); + expect(events).toEqual([]); + + (response as unknown as EventEmitter).emit('finish'); + await disposal; + expect(events).toEqual([ + 'http-close', + 'postgraphile-release', + 'preset-services-release', + 'realtime-stop', + 'pool-lease-release' + ]); + }); + + it('releases the pool lease exactly once under duplicate disposal', async () => { + const entry = makeEntry(); + entry.poolLease = makePoolLease(); + + await Promise.all([ + disposeUncachedEntry(entry, 'duplicate'), + disposeUncachedEntry(entry, 'duplicate'), + disposeUncachedEntry(entry, 'duplicate') + ]); + + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + expect(entry.poolLease.release).toHaveBeenCalledTimes(1); + }); + + it('awaits released memory before admitting the next build', async () => { + const previousMax = process.env.GRAPHILE_CACHE_MAX; + const previousMode = process.env.GRAPHILE_CACHE_ADMISSION_MODE; + process.env.GRAPHILE_CACHE_MAX = '1'; + process.env.GRAPHILE_CACHE_ADMISSION_MODE = 'evict-idle'; + const entry = makeEntry(20); + graphileCache.set('resident', entry); + const startedAt = Date.now(); + try { + const result = await prepareCacheForBuild(200); + expect(result.evicted).toBe(1); + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(15); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + } finally { + if (previousMax === undefined) delete process.env.GRAPHILE_CACHE_MAX; + else process.env.GRAPHILE_CACHE_MAX = previousMax; + if (previousMode === undefined) delete process.env.GRAPHILE_CACHE_ADMISSION_MODE; + else process.env.GRAPHILE_CACHE_ADMISSION_MODE = previousMode; + } + }); + + it('refuses at preserve-resident capacity before evicting an idle resident', async () => { + const previousMax = process.env.GRAPHILE_CACHE_MAX; + const previousMode = process.env.GRAPHILE_CACHE_ADMISSION_MODE; + process.env.GRAPHILE_CACHE_MAX = '1'; + process.env.GRAPHILE_CACHE_ADMISSION_MODE = 'preserve-resident'; + const entry = makeEntry(); + graphileCache.set('preserved', entry); + try { + expect(evaluateBuildAdmission()).toMatchObject({ + admit: false, + reason: 'resident_capacity' + }); + await expect(prepareCacheForBuild(100)).rejects.toMatchObject({ + reason: 'resident_capacity' + }); + expect(graphileCache.peek('preserved')).toBe(entry); + expect(entry.pgl.release).not.toHaveBeenCalled(); + } finally { + graphileCache.delete('preserved'); + await waitForEntryDisposal(entry, 100); + if (previousMax === undefined) delete process.env.GRAPHILE_CACHE_MAX; + else process.env.GRAPHILE_CACHE_MAX = previousMax; + if (previousMode === undefined) delete process.env.GRAPHILE_CACHE_ADMISSION_MODE; + else process.env.GRAPHILE_CACHE_ADMISSION_MODE = previousMode; + } + }); + + it('refuses admission without evicting the only busy resident', async () => { + const previousMax = process.env.GRAPHILE_CACHE_MAX; + const previousMode = process.env.GRAPHILE_CACHE_ADMISSION_MODE; + process.env.GRAPHILE_CACHE_MAX = '1'; + process.env.GRAPHILE_CACHE_ADMISSION_MODE = 'evict-idle'; + const entry = makeEntry(); + const response = new EventEmitter() as unknown as Response; + invokeEntryHandler(entry, {} as Request, response, (() => undefined) as NextFunction); + graphileCache.set('busy', entry); + try { + await expect(prepareCacheForBuild(10)).rejects.toMatchObject({ + reason: 'resident_busy' + }); + expect(graphileCache.has('busy')).toBe(true); + } finally { + (response as unknown as EventEmitter).emit('finish'); + await waitForEntryDisposal(entry, 100); + if (previousMax === undefined) delete process.env.GRAPHILE_CACHE_MAX; + else process.env.GRAPHILE_CACHE_MAX = previousMax; + if (previousMode === undefined) delete process.env.GRAPHILE_CACHE_ADMISSION_MODE; + else process.env.GRAPHILE_CACHE_ADMISSION_MODE = previousMode; + } + }); + + it('releases the pool lease when PostGraphile release fails', async () => { + const releaseFailure = new Error('PostGraphile release failed'); + const events: string[] = []; + const entry = makeEntry(); + entry.pgl = { + release: jest.fn(async () => { + events.push('postgraphile-release'); + throw releaseFailure; + }) + } as unknown as GraphileCacheEntry['pgl']; + entry.poolLease = makePoolLease(() => events.push('pool-lease-release')); + + await expect(disposeUncachedEntry(entry, 'release-failure')).rejects.toBe( + releaseFailure + ); + expect(events).toEqual(['postgraphile-release', 'pool-lease-release']); + expect(entry.poolLease.release).toHaveBeenCalledTimes(1); + }); + + it('continues realtime and pool cleanup after a PostGraphile release failure', async () => { + const releaseFailure = new Error('PostGraphile release failed'); + const realtimeFailure = new Error('Realtime stop failed'); + const entry = makeEntry(); + entry.pgl = { + release: jest.fn(async () => { + throw releaseFailure; + }) + } as unknown as GraphileCacheEntry['pgl']; + entry.realtimeManager = { + stop: jest.fn(async () => { + throw realtimeFailure; + }) + }; + entry.poolLease = makePoolLease(); + + await expect(Promise.all([ + disposeUncachedEntry(entry, 'aggregate-release-failure'), + disposeUncachedEntry(entry, 'aggregate-release-failure') + ])).rejects.toBe(releaseFailure); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + expect(entry.realtimeManager.stop).toHaveBeenCalledTimes(1); + expect(entry.poolLease.release).toHaveBeenCalledTimes(1); + }); +}); + +describe('timer cleanup', () => { + it('clears the timeout when work settles first', async () => { + jest.useFakeTimers(); + try { + const result = await raceWithClearedTimeout(Promise.resolve('done'), 60_000); + expect(result).toEqual({ timedOut: false, value: 'done' }); + expect(jest.getTimerCount()).toBe(0); + } finally { + jest.useRealTimers(); + } + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/http-adapter.test.ts b/graphile/graphile-cache/src/__tests__/http-adapter.test.ts new file mode 100644 index 0000000000..9212215a61 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/http-adapter.test.ts @@ -0,0 +1,123 @@ +import type { Server } from 'node:http'; + +import express, { type Express } from 'express'; + +import { + disposeUncachedEntry, + type GraphileCacheEntry +} from '../graphile-cache'; +import { + attachGraphileHttpHandler, + createGraphileHttpHandler +} from '../http-adapter'; + +const closeServer = (server: Server): Promise => + new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + }); + +describe('lean Graphile HTTP adapter', () => { + it('matches the pinned Grafserv HTTP-only runtime contract', async () => { + // Use require so ts-jest's legacy resolver does not reject Grafserv's + // conditional `./express/v4` export, which the package build resolves. + const { grafserv } = require('grafserv/express/v4'); + const serv = grafserv({ + preset: { grafserv: { graphqlPath: '/graphql' } }, + schema: null + }); + const handler = createGraphileHttpHandler(); + + try { + await attachGraphileHttpHandler(serv, handler, serv.getPreset()); + expect((handler as any).stack).toHaveLength(1); + expect((handler as any).listen).toBeUndefined(); + } finally { + await serv.release(); + } + }); + + it('mounts Grafserv on a router with websocket/server allocation disabled', async () => { + const handler = createGraphileHttpHandler(); + const serv = { + addTo: jest.fn(async (app: Express) => { + app.use('/graphql', (_req, res) => { + res.status(200).json({ data: { adapter: 'router' } }); + }); + }) + }; + + await attachGraphileHttpHandler(serv, handler, { grafserv: {} }); + expect(serv.addTo).toHaveBeenCalledWith(handler, null, false); + expect((handler as any).listen).toBeUndefined(); + + const outerApp = express(); + outerApp.use(handler); + const outerServer = await new Promise((resolve, reject) => { + const server = outerApp.listen(0, '127.0.0.1', () => resolve(server)); + server.once('error', reject); + }); + try { + const address = outerServer.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected an address for the test HTTP server'); + } + const response = await fetch(`http://127.0.0.1:${address.port}/graphql`); + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ + data: { adapter: 'router' } + }); + } finally { + await closeServer(outerServer); + } + }); + + it('fails closed instead of silently disabling configured WebSockets', () => { + const handler = createGraphileHttpHandler(); + const serv = { addTo: jest.fn() }; + + expect(() => attachGraphileHttpHandler(serv, handler, { + grafserv: { websockets: true } + })).toThrow(/tenant-aware upgrade handler on the shared server/); + expect(serv.addTo).not.toHaveBeenCalled(); + }); + + it('mounts HTTP without an exclusive listener when shared routing is explicit', async () => { + const handler = createGraphileHttpHandler(); + const serv = { addTo: jest.fn() }; + + await attachGraphileHttpHandler(serv, handler, { + grafserv: { websockets: true } + }, { + sharedWebsocketRouting: true + }); + + expect(serv.addTo).toHaveBeenCalledWith(handler, null, false); + }); + + it('disposes a serverless adapter and its realtime manager in order', async () => { + const events: string[] = []; + const entry: GraphileCacheEntry = { + pgl: { + release: jest.fn(async () => { + events.push('postgraphile-release'); + }) + } as unknown as GraphileCacheEntry['pgl'], + serv: {} as GraphileCacheEntry['serv'], + handler: createGraphileHttpHandler(), + httpServer: null, + cacheKey: 'lean-adapter', + createdAt: Date.now(), + realtimeManager: { + stop: jest.fn(async () => { + events.push('realtime-stop'); + }) + } + }; + + await disposeUncachedEntry(entry); + + expect(events).toEqual(['postgraphile-release', 'realtime-stop']); + expect(entry.realtimeManager?.stop).toHaveBeenCalledTimes(1); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/preset-services.test.ts b/graphile/graphile-cache/src/__tests__/preset-services.test.ts new file mode 100644 index 0000000000..5807f04f72 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/preset-services.test.ts @@ -0,0 +1,29 @@ +import { createPresetServicesReleaser } from '../preset-services'; + +describe('preset service ownership', () => { + it('releases services in reverse order exactly once under concurrent teardown', async () => { + const events: string[] = []; + const first = { release: jest.fn(async () => { events.push('first'); }) }; + const second = { release: jest.fn(async () => { events.push('second'); }) }; + const release = createPresetServicesReleaser({ + pgServices: [first, second, first] + }); + + await Promise.all([release(), release(), release()]); + + expect(events).toEqual(['second', 'first']); + expect(first.release).toHaveBeenCalledTimes(1); + expect(second.release).toHaveBeenCalledTimes(1); + }); + + it('continues releasing services and preserves the first cleanup error', async () => { + const firstFailure = new Error('second failed'); + const first = { release: jest.fn(async (): Promise => undefined) }; + const second = { release: jest.fn(async () => { throw firstFailure; }) }; + const release = createPresetServicesReleaser({ pgServices: [first, second] }); + + await expect(release()).rejects.toBe(firstFailure); + expect(first.release).toHaveBeenCalledTimes(1); + expect(second.release).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/realtime-readiness.test.ts b/graphile/graphile-cache/src/__tests__/realtime-readiness.test.ts new file mode 100644 index 0000000000..897289a189 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/realtime-readiness.test.ts @@ -0,0 +1,204 @@ +import { + createGraphileRealtimeHealth, + createGraphileRealtimeNodeId, + DEFAULT_GRAPHILE_REALTIME_SCHEMA, + GraphileRealtimeStartupError, + startConfiguredRealtime, + withGraphileRealtimeFailure +} from '../realtime-readiness'; + +const makeManager = () => { + const start = jest.fn().mockResolvedValue(undefined); + const stop = jest.fn().mockResolvedValue(undefined); + const constructor = jest.fn().mockImplementation(() => ({ start, stop })); + return { constructor, start, stop }; +}; + +describe('configured realtime instance readiness', () => { + it('fails closed and releases PostGraphile when the subscriber is missing', async () => { + const manager = makeManager(); + const releasePostGraphile = jest.fn().mockResolvedValue(undefined); + + await expect(startConfiguredRealtime({ + cacheKey: 'missing-subscriber', + resolvedPreset: { + pgServices: [{ adaptorSettings: { pool: {} } }] + }, + allowedSourceSchemas: ['tenant_a'], + releasePostGraphile, + loadManager: async () => manager.constructor + })).rejects.toBeInstanceOf(GraphileRealtimeStartupError); + + expect(manager.constructor).not.toHaveBeenCalled(); + expect(releasePostGraphile).toHaveBeenCalledTimes(1); + }); + + it('stops a partially started manager and releases PostGraphile', async () => { + const manager = makeManager(); + const startupFailure = new Error('realtime startup failed'); + manager.start.mockRejectedValue(startupFailure); + const releasePostGraphile = jest.fn().mockResolvedValue(undefined); + + await expect(startConfiguredRealtime({ + cacheKey: 'start-failure', + resolvedPreset: { + pgServices: [{ + pgSubscriber: {}, + adaptorSettings: { pool: {} } + }] + }, + allowedSourceSchemas: ['tenant_a'], + releasePostGraphile, + loadManager: async () => manager.constructor + })).rejects.toMatchObject({ + code: 'GRAPHILE_REALTIME_STARTUP_FAILED', + cause: startupFailure + }); + + expect(manager.stop).toHaveBeenCalledTimes(1); + expect(releasePostGraphile).toHaveBeenCalledTimes(1); + }); + + it('returns a started manager without releasing a healthy generation', async () => { + const manager = makeManager(); + const releasePostGraphile = jest.fn().mockResolvedValue(undefined); + + const result = await startConfiguredRealtime({ + cacheKey: 'ready', + resolvedPreset: { + pgServices: [{ + pgSubscriber: { eventEmitter: { emit: jest.fn() } }, + adaptorSettings: { pool: {} } + }] + }, + allowedSourceSchemas: ['tenant_a'], + releasePostGraphile, + loadManager: async () => manager.constructor, + replicaIdentity: 'replica-a' + }); + + expect(result).toEqual({ start: manager.start, stop: manager.stop }); + expect(manager.constructor).toHaveBeenCalledWith(expect.objectContaining({ + schema: DEFAULT_GRAPHILE_REALTIME_SCHEMA, + allowedSourceSchemas: ['tenant_a'], + nodeId: 'graphile-cache:replica-a:ready' + })); + expect(manager.start).toHaveBeenCalledTimes(1); + expect(releasePostGraphile).not.toHaveBeenCalled(); + }); + + it('passes an exact custom cursor schema to the manager', async () => { + const manager = makeManager(); + const releasePostGraphile = jest.fn().mockResolvedValue(undefined); + const onFatalError = jest.fn(); + + await startConfiguredRealtime({ + cacheKey: 'tenant-a', + resolvedPreset: { + pgServices: [{ + pgSubscriber: { eventEmitter: { emit: jest.fn() } }, + adaptorSettings: { pool: {} } + }] + }, + realtimeSchema: 'ctf_a_realtime', + allowedSourceSchemas: ['ctf_a'], + onFatalError, + releasePostGraphile, + loadManager: async () => manager.constructor, + replicaIdentity: 'replica-a' + }); + + expect(manager.constructor).toHaveBeenCalledWith({ + pgSubscriber: { eventEmitter: { emit: expect.any(Function) } }, + pool: {}, + nodeId: 'graphile-cache:replica-a:tenant-a', + schema: 'ctf_a_realtime', + allowedSourceSchemas: ['ctf_a'], + onFatalError + }); + expect(releasePostGraphile).not.toHaveBeenCalled(); + }); + + it('uses an explicit generation publisher and configured cursor intervals', async () => { + const manager = makeManager(); + const releasePostGraphile = jest.fn().mockResolvedValue(undefined); + const publisher = { + assertTopics: jest.fn(), + publish: jest.fn() + }; + + await startConfiguredRealtime({ + cacheKey: 'shared-exact', + resolvedPreset: { + pgServices: [{ adaptorSettings: { pool: {} } }] + }, + publisher, + pollIntervalMs: 30_000, + heartbeatIntervalMs: 90_000, + allowedSourceSchemas: ['tenant_a'], + releasePostGraphile, + loadManager: async () => manager.constructor + }); + + expect(manager.constructor).toHaveBeenCalledWith(expect.objectContaining({ + publisher, + pollIntervalMs: 30_000, + heartbeatIntervalMs: 90_000 + })); + expect(manager.constructor.mock.calls[0][0]).not.toHaveProperty('pgSubscriber'); + }); + + it('fails closed before loading a manager when no source schema is allowed', async () => { + const manager = makeManager(); + const releasePostGraphile = jest.fn().mockResolvedValue(undefined); + + await expect(startConfiguredRealtime({ + cacheKey: 'no-sources', + resolvedPreset: { + pgServices: [{ + pgSubscriber: { eventEmitter: { emit: jest.fn() } }, + adaptorSettings: { pool: {} } + }] + }, + allowedSourceSchemas: [], + releasePostGraphile, + loadManager: async () => manager.constructor + })).rejects.toMatchObject({ + code: 'GRAPHILE_REALTIME_STARTUP_FAILED' + }); + + expect(manager.constructor).not.toHaveBeenCalled(); + expect(releasePostGraphile).toHaveBeenCalledTimes(1); + }); + + it('separates replica cursor identities while retaining the exact contract key', () => { + const cacheKey = 'graphile:v1:contract-a'; + const first = createGraphileRealtimeNodeId(cacheKey, 'replica-a'); + const second = createGraphileRealtimeNodeId(cacheKey, 'replica-b'); + + expect(first).not.toBe(second); + expect(first).toBe(`graphile-cache:replica-a:${cacheKey}`); + expect(second).toBe(`graphile-cache:replica-b:${cacheKey}`); + }); + + it('latches the first fatal delivery failure for one exact generation', () => { + const health = createGraphileRealtimeHealth(); + const first = Object.assign(new Error('foreign source'), { + code: 'REALTIME_SOURCE_SCHEMA_VIOLATION' + }); + const second = Object.assign(new Error('emitter missing'), { + code: 'REALTIME_SUBSCRIBER_UNAVAILABLE' + }); + + const failed = withGraphileRealtimeFailure(health, first, 1_000); + const stillFailed = withGraphileRealtimeFailure(failed, second, 2_000); + + expect(health).toEqual({ status: 'healthy' }); + expect(failed).toEqual({ + status: 'failed', + failureCode: 'REALTIME_SOURCE_SCHEMA_VIOLATION', + failedAt: 1_000 + }); + expect(stillFailed).toBe(failed); + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/shared-realtime.test.ts b/graphile/graphile-cache/src/__tests__/shared-realtime.test.ts new file mode 100644 index 0000000000..1d360a3875 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/shared-realtime.test.ts @@ -0,0 +1,479 @@ +const acquirePgNotificationBroker = jest.fn(); +const getPgNotificationBrokerStats = jest.fn(); +const getPgNotificationBrokerIdentity = jest.fn((config: { password?: string }) => + config.password === 'rotated-secret' + ? 'broker:v1:rotated' + : 'broker:v1:expected'); +const getPgNotificationDatabaseIdentity = jest.fn(() => 'database-target:v1:tenant-a'); + +jest.mock('pg-cache', () => ({ + acquirePgNotificationBroker, + getPgNotificationBrokerStats, + getPgNotificationBrokerIdentity, + getPgNotificationDatabaseIdentity, + PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE: 'PG_NOTIFICATION_LEASE_RELEASED' +})); + +import { + ActivatableGenerationScopedRealtimeSubscriber, + RealtimeTopicCollector +} from 'graphile-realtime-subscriptions'; + +import { + activateGraphileSharedRealtime, + getGraphileRealtimeRoleAuditStats, + GraphileSharedRealtimeDatabaseConflictError, + GraphileSharedRealtimeIdentityError +} from '../shared-realtime'; + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(reason: unknown): void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T) => void; + let reject!: (reason: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +const listenerConfig = { + host: 'db.internal', + port: 5432, + database: 'tenant_a', + user: 'tenant_a_notify', + password: 'never-log-this' +}; + +const successfulAudit = { + version: 'pg-notification-role:v1' as const, + role: 'tenant_a_notify', + database: 'tenant_a', + safe: true, + violations: [] as const +}; + +let brokerAuditAttempts = 0; +let brokerAuditFailures = 0; + +const makeCollector = (): RealtimeTopicCollector => { + const collector = new RealtimeTopicCollector(); + collector.collect([{ + topic: 'realtime:tenant_a.contacts', + schema: 'tenant_a', + table: 'contacts' + }]); + return collector; +}; + +const makeBrokerLease = ( + revalidate = async () => successfulAudit +) => { + const termination = deferred(); + const release = jest.fn(async (): Promise => { + termination.resolve(null); + }); + const revalidateRole = jest.fn(async () => { + brokerAuditAttempts++; + try { + return await revalidate(); + } catch (error) { + brokerAuditFailures++; + throw error; + } + }); + return { + identity: 'broker:v1:expected', + topics: ['realtime:tenant_a.contacts'], + terminated: termination.promise, + roleAudit: successfulAudit, + revalidateRole, + subscribe: jest.fn(() => { + const iterator: AsyncIterableIterator = { + [Symbol.asyncIterator]: () => iterator, + next: () => new Promise(() => undefined), + return: async (): Promise> => ({ + done: true, + value: undefined + }) + }; + return iterator; + }), + release, + termination + }; +}; + +const useBrokerLeases = (...leases: ReturnType[]): void => { + const pending = [...leases]; + acquirePgNotificationBroker.mockImplementation(async () => { + brokerAuditAttempts++; + const lease = pending.shift(); + if (!lease) throw new Error('No mocked notification broker lease remains'); + return lease; + }); +}; + +describe('shared exact realtime activation', () => { + beforeEach(() => { + jest.clearAllMocks(); + brokerAuditAttempts = 0; + brokerAuditFailures = 0; + getPgNotificationBrokerStats.mockImplementation(() => ({ + roleAuditAttempts: brokerAuditAttempts, + roleAuditFailures: brokerAuditFailures + })); + acquirePgNotificationBroker.mockImplementation(async () => { + brokerAuditAttempts++; + return makeBrokerLease(); + }); + }); + + it('installs exact topics only after the broker returns its pinned-client audit', async () => { + const order: string[] = []; + const broker = makeBrokerLease(); + acquirePgNotificationBroker.mockImplementation(async () => { + brokerAuditAttempts++; + order.push('broker'); + return broker; + }); + const subscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const onFatalError = jest.fn(); + + const attestation = await activateGraphileSharedRealtime({ + subscriber, + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError + }); + + expect(order).toEqual(['broker']); + expect(acquirePgNotificationBroker).toHaveBeenCalledWith(listenerConfig, { + topics: ['realtime:tenant_a.contacts'] + }); + expect(attestation.snapshot()).toMatchObject({ + mode: 'shared-exact', + listenerIdentity: 'broker:v1:expected', + auditVersion: 'pg-notification-role:v1', + role: 'tenant_a_notify', + database: 'tenant_a', + status: 'healthy', + checks: 1 + }); + + attestation.release(); + await subscriber.release(); + expect(broker.release).toHaveBeenCalledTimes(1); + }); + + it('latches broker termination into the exact generation health callback', async () => { + const broker = makeBrokerLease(); + useBrokerLeases(broker); + const subscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const onFatalError = jest.fn(); + const attestation = await activateGraphileSharedRealtime({ + subscriber, + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError + }); + const failure = Object.assign(new Error('listener ended'), { + code: 'PG_NOTIFICATION_BROKER_FAILED' + }); + + broker.termination.resolve(failure); + await Promise.resolve(); + await Promise.resolve(); + expect(onFatalError).toHaveBeenCalledWith(failure); + + attestation.release(); + await subscriber.release(); + }); + + it('proactively revalidates once per identity and cancels its unref timer', async () => { + jest.useFakeTimers(); + try { + jest.setSystemTime(1_000); + const firstBroker = makeBrokerLease(); + const secondBroker = makeBrokerLease(); + useBrokerLeases(firstBroker, secondBroker); + const firstSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const secondSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const common = { + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 100, + onFatalError: jest.fn() + }; + const first = await activateGraphileSharedRealtime({ + ...common, + subscriber: firstSubscriber + }); + const second = await activateGraphileSharedRealtime({ + ...common, + subscriber: secondSubscriber + }); + + expect(jest.getTimerCount()).toBe(1); + await jest.advanceTimersByTimeAsync(99); + expect(firstBroker.revalidateRole).not.toHaveBeenCalled(); + expect(secondBroker.revalidateRole).not.toHaveBeenCalled(); + await jest.advanceTimersByTimeAsync(1); + expect(firstBroker.revalidateRole).toHaveBeenCalledTimes(1); + expect(secondBroker.revalidateRole).not.toHaveBeenCalled(); + expect(first.snapshot()).toMatchObject({ + lastAttestedAt: 1_100, + checks: 3, + status: 'healthy' + }); + expect(second.snapshot()).toMatchObject({ checks: 3, status: 'healthy' }); + expect(jest.getTimerCount()).toBe(1); + + first.release(); + expect(jest.getTimerCount()).toBe(1); + second.release(); + expect(jest.getTimerCount()).toBe(0); + await Promise.all([firstSubscriber.release(), secondSubscriber.release()]); + await jest.advanceTimersByTimeAsync(100); + expect(firstBroker.revalidateRole).toHaveBeenCalledTimes(1); + expect(secondBroker.revalidateRole).not.toHaveBeenCalled(); + } finally { + jest.useRealTimers(); + } + }); + + it('retries another generation when the selected revalidator is released', async () => { + const now = jest.spyOn(Date, 'now').mockReturnValue(1_000); + const selectedAudit = deferred(); + const firstBroker = makeBrokerLease(() => selectedAudit.promise); + const secondBroker = makeBrokerLease(); + const thirdBroker = makeBrokerLease(); + useBrokerLeases(firstBroker, secondBroker, thirdBroker); + const firstSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const secondSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const thirdSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const firstFailure = jest.fn(); + const secondFailure = jest.fn(); + const thirdFailure = jest.fn(); + const common = { + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000 + }; + const first = await activateGraphileSharedRealtime({ + ...common, + subscriber: firstSubscriber, + onFatalError: firstFailure + }); + const second = await activateGraphileSharedRealtime({ + ...common, + subscriber: secondSubscriber, + onFatalError: secondFailure + }); + const third = await activateGraphileSharedRealtime({ + ...common, + subscriber: thirdSubscriber, + onFatalError: thirdFailure + }); + + now.mockReturnValue(61_001); + const refreshing = second.revalidateIfDue(); + await Promise.resolve(); + expect(firstBroker.revalidateRole).toHaveBeenCalledTimes(1); + + first.release(); + await firstSubscriber.release(); + selectedAudit.reject(Object.assign(new Error('lease released'), { + code: 'PG_NOTIFICATION_LEASE_RELEASED' + })); + + await expect(refreshing).resolves.toBe(true); + expect(secondBroker.revalidateRole).toHaveBeenCalledTimes(1); + expect(thirdBroker.revalidateRole).not.toHaveBeenCalled(); + expect(second.snapshot()).toMatchObject({ status: 'healthy', checks: 4 }); + expect(third.snapshot()).toMatchObject({ status: 'healthy', checks: 4 }); + expect(firstFailure).not.toHaveBeenCalled(); + expect(secondFailure).not.toHaveBeenCalled(); + expect(thirdFailure).not.toHaveBeenCalled(); + + second.release(); + third.release(); + await Promise.all([secondSubscriber.release(), thirdSubscriber.release()]); + now.mockRestore(); + }); + + it('coalesces TTL refresh and fails every sharing generation closed on drift', async () => { + const statsBefore = getGraphileRealtimeRoleAuditStats(); + const now = jest.spyOn(Date, 'now').mockReturnValue(1_000); + const drift = Object.assign(new Error('role drift'), { + code: 'PG_NOTIFICATION_ROLE_UNSAFE' + }); + const firstBroker = makeBrokerLease(async () => { + throw drift; + }); + const secondBroker = makeBrokerLease(); + useBrokerLeases(firstBroker, secondBroker); + const firstSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const secondSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const firstFailure = jest.fn(); + const secondFailure = jest.fn(); + const common = { + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000 + }; + const first = await activateGraphileSharedRealtime({ + ...common, + subscriber: firstSubscriber, + onFatalError: firstFailure + }); + const second = await activateGraphileSharedRealtime({ + ...common, + subscriber: secondSubscriber, + onFatalError: secondFailure + }); + expect(acquirePgNotificationBroker).toHaveBeenCalledTimes(2); + + now.mockReturnValue(61_001); + await expect(Promise.all([ + first.revalidateIfDue(), + second.revalidateIfDue() + ])).resolves.toEqual([false, false]); + + expect(firstBroker.revalidateRole).toHaveBeenCalledTimes(1); + expect(secondBroker.revalidateRole).not.toHaveBeenCalled(); + expect(firstFailure).toHaveBeenCalledWith(drift); + expect(secondFailure).toHaveBeenCalledWith(drift); + expect(first.snapshot()).toMatchObject({ + status: 'failed', + failureCode: 'PG_NOTIFICATION_ROLE_UNSAFE', + failedAt: 61_001 + }); + expect(getGraphileRealtimeRoleAuditStats()).toMatchObject({ + identities: 1, + failed: 1, + activeIdentityAuditAttempts: 3, + catalogAuditAttempts: statsBefore.catalogAuditAttempts + 3, + catalogAuditFailures: statsBefore.catalogAuditFailures + 1, + activeDatabaseTargets: 1 + }); + + first.release(); + second.release(); + await Promise.all([firstSubscriber.release(), secondSubscriber.release()]); + now.mockRestore(); + }); + + it('rejects a second active listener identity for one physical database', async () => { + const firstBroker = makeBrokerLease(); + const rotatedBroker = makeBrokerLease(); + useBrokerLeases(firstBroker, rotatedBroker); + const firstSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const first = await activateGraphileSharedRealtime({ + subscriber: firstSubscriber, + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError: jest.fn() + }); + const rotatedConfig = { + ...listenerConfig, + password: 'rotated-secret' + }; + const rotatedSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + + await expect(activateGraphileSharedRealtime({ + subscriber: rotatedSubscriber, + topicCollector: makeCollector(), + listenerPgConfig: rotatedConfig, + listenerIdentity: 'broker:v1:rotated', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError: jest.fn() + })).rejects.toBeInstanceOf(GraphileSharedRealtimeDatabaseConflictError); + expect(acquirePgNotificationBroker).toHaveBeenCalledTimes(1); + + first.release(); + await firstSubscriber.release(); + const rotated = await activateGraphileSharedRealtime({ + subscriber: rotatedSubscriber, + topicCollector: makeCollector(), + listenerPgConfig: rotatedConfig, + listenerIdentity: 'broker:v1:rotated', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError: jest.fn() + }); + expect(acquirePgNotificationBroker).toHaveBeenCalledTimes(2); + + rotated.release(); + await rotatedSubscriber.release(); + }); + + it('releases the physical-database reservation when the initial audit fails', async () => { + const auditFailure = new Error('catalog unavailable'); + acquirePgNotificationBroker.mockRejectedValueOnce(auditFailure); + const failedSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + await expect(activateGraphileSharedRealtime({ + subscriber: failedSubscriber, + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError: jest.fn() + })).rejects.toBe(auditFailure); + await failedSubscriber.release(); + + const rotatedBroker = makeBrokerLease(); + useBrokerLeases(rotatedBroker); + const rotatedSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const rotated = await activateGraphileSharedRealtime({ + subscriber: rotatedSubscriber, + topicCollector: makeCollector(), + listenerPgConfig: { + ...listenerConfig, + password: 'rotated-secret' + }, + listenerIdentity: 'broker:v1:rotated', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError: jest.fn() + }); + + rotated.release(); + await rotatedSubscriber.release(); + }); + + it('rejects a caller-supplied listener identity mismatch before audit', async () => { + await expect(activateGraphileSharedRealtime({ + subscriber: new ActivatableGenerationScopedRealtimeSubscriber(), + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:wrong', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError: jest.fn() + })).rejects.toBeInstanceOf(GraphileSharedRealtimeIdentityError); + expect(acquirePgNotificationBroker).not.toHaveBeenCalled(); + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/websocket-lifecycle.test.ts b/graphile/graphile-cache/src/__tests__/websocket-lifecycle.test.ts new file mode 100644 index 0000000000..abdf2734df --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/websocket-lifecycle.test.ts @@ -0,0 +1,263 @@ +import { once } from 'node:events'; +import { PassThrough } from 'node:stream'; + +import type { IncomingMessage } from 'http'; + +import { + disposeUncachedEntry, + getCacheCounters, + GRAPHILE_WEBSOCKET_UNAVAILABLE_CODE, + graphileCache, + type GraphileCacheEntry, + invokeEntryUpgradeHandler, + retireGraphileCacheEntry, + waitForEntryDisposal +} from '../graphile-cache'; +import { createGraphileHttpHandler } from '../http-adapter'; +import { GRAPHILE_REALTIME_UNAVAILABLE_CODE } from '../realtime-readiness'; + +const makeEntry = ( + overrides: Partial = {} +): GraphileCacheEntry => ({ + pgl: { + release: jest.fn(async (): Promise => undefined) + } as unknown as GraphileCacheEntry['pgl'], + serv: {} as GraphileCacheEntry['serv'], + handler: createGraphileHttpHandler(), + httpServer: null, + cacheKey: 'websocket-lifecycle', + createdAt: Date.now(), + ...overrides +}); + +const request = (): IncomingMessage => ({ + aborted: false +}) as IncomingMessage; + +describe('cached Graphile WebSocket lifecycle', () => { + it('retains an exact entry until its accepted socket closes', async () => { + const socket = new PassThrough(); + const upgradeHandler = jest.fn(); + const entry = makeEntry({ upgradeHandler }); + const countersBefore = getCacheCounters(); + + expect(invokeEntryUpgradeHandler(entry, request(), socket, Buffer.alloc(0))).toBe(true); + expect(upgradeHandler).toHaveBeenCalledWith( + expect.anything(), + socket, + expect.any(Buffer) + ); + expect(entry.inflight).toBe(1); + expect(entry.websocketSockets?.has(socket)).toBe(true); + expect(getCacheCounters().websocketUpgradesStarted).toBe( + countersBefore.websocketUpgradesStarted + 1 + ); + expect(getCacheCounters().websocketUpgradesCompleted).toBe( + countersBefore.websocketUpgradesCompleted + ); + + socket.destroy(); + await once(socket, 'close'); + + expect(entry.inflight).toBe(0); + expect(entry.websocketSockets?.size).toBe(0); + expect(getCacheCounters().websocketUpgradesCompleted).toBe( + countersBefore.websocketUpgradesCompleted + 1 + ); + }); + + it('transfers the outer transport only after the exact generation is retained', () => { + const socket = new PassThrough(); + const events: string[] = []; + const entry = makeEntry({ + upgradeHandler: jest.fn(() => events.push('grafserv')) + }); + + expect(invokeEntryUpgradeHandler( + entry, + request(), + socket, + Buffer.from('head'), + { onAccepted: () => events.push('accepted') } + )).toBe(true); + + expect(events).toEqual(['accepted', 'grafserv']); + expect(entry.inflight).toBe(1); + socket.destroy(); + }); + + it('terminates long-lived sockets before disposing their generation', async () => { + const socket = new PassThrough(); + const entry = makeEntry({ upgradeHandler: jest.fn() }); + expect(invokeEntryUpgradeHandler(entry, request(), socket, Buffer.alloc(0))).toBe(true); + + await disposeUncachedEntry(entry); + + expect(socket.destroyed).toBe(true); + expect(entry.inflight).toBe(0); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + }); + + it('retires the exact resident generation and its sockets on a fatal audit', async () => { + const socket = new PassThrough(); + const entry = makeEntry({ + cacheKey: 'websocket-fatal-audit', + upgradeHandler: jest.fn() + }); + graphileCache.set(entry.cacheKey, entry); + expect(invokeEntryUpgradeHandler( + entry, + request(), + socket, + Buffer.alloc(0) + )).toBe(true); + + expect(retireGraphileCacheEntry( + entry, + Object.assign(new Error('listener role changed'), { + code: 'INSUFFICIENT_PRIVILEGE' + }) + )).toBe(true); + + await once(socket, 'close'); + await expect(waitForEntryDisposal(entry, 100)).resolves.toBe(true); + expect(graphileCache.peek(entry.cacheKey)).toBeUndefined(); + expect(entry.realtimeHealth).toMatchObject({ + status: 'failed', + failureCode: 'INSUFFICIENT_PRIVILEGE' + }); + expect(entry.inflight).toBe(0); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + }); + + it('releases PgSubscriber before cursor cleanup in a saturated max=2 pool', async () => { + const socket = new PassThrough(); + const events: string[] = []; + // Model the two production runtime slots while a subscription is live: + // one PgSubscriber LISTEN checkout and one cursor-tracker checkout. + let occupiedSlots = 2; + const entry = makeEntry({ + upgradeHandler: jest.fn(), + pgl: { + release: jest.fn(async () => { + events.push('postgraphile-release'); + }) + } as unknown as GraphileCacheEntry['pgl'], + releasePresetServices: jest.fn(async () => { + events.push('preset-services-release'); + occupiedSlots -= 1; + }), + realtimeManager: { + stop: jest.fn(async () => { + events.push('realtime-stop'); + if (occupiedSlots >= 2) { + throw new Error('timeout exceeded when trying to connect'); + } + occupiedSlots -= 1; + }) + } + }); + expect(invokeEntryUpgradeHandler(entry, request(), socket, Buffer.alloc(0))).toBe(true); + + await expect(disposeUncachedEntry(entry, 'max-2-live-subscription')).resolves.toBeUndefined(); + + expect(socket.destroyed).toBe(true); + expect(occupiedSlots).toBe(0); + expect(events).toEqual([ + 'postgraphile-release', + 'preset-services-release', + 'realtime-stop' + ]); + }); + + it('releases a caller-owned shared subscriber and attestation exactly once', async () => { + const realtimeSubscriber = { + release: jest.fn(async (): Promise => undefined) + }; + const realtimeRoleAttestation = { + snapshot: jest.fn(), + revalidateIfDue: jest.fn(async () => true), + release: jest.fn() + }; + const entry = makeEntry({ + realtimeSubscriber, + realtimeRoleAttestation + }); + + const first = disposeUncachedEntry(entry, 'shared-owner'); + const second = disposeUncachedEntry(entry, 'shared-owner'); + expect(first).toBe(second); + await first; + + expect(realtimeRoleAttestation.release).toHaveBeenCalledTimes(1); + expect(realtimeSubscriber.release).toHaveBeenCalledTimes(1); + }); + + it('fails closed with a stable response when no upgrade handler exists', async () => { + const socket = new PassThrough(); + let response = ''; + socket.on('data', (chunk) => { + response += chunk.toString(); + }); + const ended = once(socket, 'end'); + + const rejected = jest.fn(); + expect(invokeEntryUpgradeHandler( + makeEntry(), + request(), + socket, + Buffer.alloc(0), + { onRejected: rejected } + )).toBe(true); + await ended; + + expect(response).toContain('HTTP/1.1 503'); + expect(response).toContain(GRAPHILE_WEBSOCKET_UNAVAILABLE_CODE); + expect(rejected).toHaveBeenCalledTimes(1); + }); + + it('rejects a WebSocket upgrade when its listener-role attestation is stale', async () => { + const socket = new PassThrough(); + let response = ''; + socket.on('data', (chunk) => { + response += chunk.toString(); + }); + const ended = once(socket, 'end'); + const entry = makeEntry({ + upgradeHandler: jest.fn(), + realtimeRoleAttestation: { + snapshot: jest.fn(() => ({ + version: 1, + mode: 'shared-exact', + listenerIdentity: 'opaque-listener-identity', + auditVersion: 'pg-notification-role:v1', + role: 'listener', + database: 'tenant_a', + lastAttestedAt: 1, + validUntil: 2, + checks: 1, + status: 'healthy', + failureCode: null as string | null, + failedAt: null as number | null + })), + revalidateIfDue: jest.fn(async () => true), + release: jest.fn() + } + }); + + const rejected = jest.fn(); + expect(invokeEntryUpgradeHandler( + entry, + request(), + socket, + Buffer.alloc(0), + { onRejected: rejected } + )).toBe(true); + await ended; + + expect(entry.upgradeHandler).not.toHaveBeenCalled(); + expect(rejected).toHaveBeenCalledTimes(1); + expect(response).toContain('HTTP/1.1 503'); + expect(response).toContain(GRAPHILE_REALTIME_UNAVAILABLE_CODE); + }); +}); diff --git a/graphile/graphile-cache/src/build-readiness.ts b/graphile/graphile-cache/src/build-readiness.ts new file mode 100644 index 0000000000..4b3d1a54c5 --- /dev/null +++ b/graphile/graphile-cache/src/build-readiness.ts @@ -0,0 +1,27 @@ +export interface GraphileBuildReadiness { + schemaResult: PromiseLike | unknown; + addTo(): PromiseLike | unknown; + ready(): PromiseLike | unknown; + release(): PromiseLike | unknown; + onReleaseError?(error: unknown): void; +} + +/** + * Keep the build coordinator occupied until both schema gathering and the + * HTTP adapter are ready. Failed generations are released before returning. + */ +export const awaitGraphileBuildReadiness = async ( + build: GraphileBuildReadiness +): Promise => { + try { + await build.addTo(); + await Promise.all([build.schemaResult, build.ready()]); + } catch (error) { + try { + await build.release(); + } catch (releaseError) { + build.onReleaseError?.(releaseError); + } + throw error; + } +}; diff --git a/graphile/graphile-cache/src/create-instance.ts b/graphile/graphile-cache/src/create-instance.ts index 575b767589..df973a4d48 100644 --- a/graphile/graphile-cache/src/create-instance.ts +++ b/graphile/graphile-cache/src/create-instance.ts @@ -1,23 +1,80 @@ -import { createServer } from 'node:http'; - import { Logger } from '@pgpmjs/logger'; -import express from 'express'; import { grafserv } from 'grafserv/express/v4'; +import { + ActivatableGenerationScopedRealtimeSubscriber, + type RealtimeTopicCollector +} from 'graphile-realtime-subscriptions'; +import type { PgNotificationListenerConfig, PgPoolLease } from 'pg-cache'; import { postgraphile } from 'postgraphile'; -import type { GraphileCacheEntry } from './graphile-cache'; +import { awaitGraphileBuildReadiness } from './build-readiness'; +import type { + GraphileCacheEntry, + GraphileUpgradeHandler +} from './graphile-cache'; +import { retireGraphileCacheEntry } from './graphile-cache'; +import { + attachGraphileHttpHandler, + createGraphileHttpHandler +} from './http-adapter'; +import { createPresetServicesReleaser } from './preset-services'; +import { + createGraphileRealtimeHealth, + GraphileRealtimeStartupError, + startConfiguredRealtime +} from './realtime-readiness'; +import { + activateGraphileSharedRealtime, + type GraphileRealtimeRoleAttestation +} from './shared-realtime'; const log = new Logger('graphile-cache:create'); -interface GraphileInstanceOptions { +export interface GraphileInstanceOptions { preset: any; cacheKey: string; + poolIdentity?: string; + /** + * Lease protecting the runtime pool for the lifetime of this instance. + * + * The caller owns the lease until `createGraphileInstance()` resolves. Once + * it resolves, ownership transfers to the returned cache entry and its + * disposal lifecycle releases the lease after PostGraphile teardown. + */ + poolLease?: PgPoolLease; + serviceKey?: string; + databaseId?: string | null; /** * When true, a RealtimeManager is created and started alongside the * PostGraphile instance. The pool is extracted from the preset's * pgServices (managed by pg-cache) rather than passed separately. */ enableRealtime?: boolean; + /** + * Build a no-server Grafserv upgrade handler for an outer tenant-aware + * router. The preset must explicitly enable `grafserv.websockets`; the + * cached instance still never attaches its own upgrade listener. + */ + enableWebsockets?: boolean; + /** + * Physical schema containing this instance's realtime cursor functions. + * Omit to use the compatibility default `realtime_public`. + */ + realtimeSchema?: string; + /** Exact physical source schemas allowed to produce realtime events. */ + realtimeSourceSchemas?: readonly string[]; + /** Cursor recovery polling interval; defaults to RealtimeManager's 5s. */ + realtimeCursorPollIntervalMs?: number; + /** Cursor listener heartbeat interval; defaults to RealtimeManager's 30s. */ + realtimeCursorHeartbeatIntervalMs?: number; + /** Opt-in exact-topic shared notification transport. */ + sharedRealtime?: { + subscriber: ActivatableGenerationScopedRealtimeSubscriber; + topicCollector: RealtimeTopicCollector; + listenerPgConfig: PgNotificationListenerConfig; + listenerIdentity: string; + roleRevalidationMs: number; + }; } /** @@ -29,6 +86,8 @@ interface GraphileInstanceOptions { * * Callers are responsible for building the `GraphileConfig.Preset` (including * pgServices, grafserv options, grafast context, etc.) before passing it here. + * When `poolLease` is supplied, ownership transfers only when this promise + * resolves. If instance creation rejects, the caller must release the lease. * * When `enableRealtime` is true, a RealtimeManager is created that bridges * cursor-tracked events from `drain_changes()` into the PostGraphile @@ -39,56 +98,182 @@ interface GraphileInstanceOptions { export const createGraphileInstance = async ( opts: GraphileInstanceOptions ): Promise => { - const { preset, cacheKey, enableRealtime = false } = opts; + const { + preset, + cacheKey, + poolIdentity, + poolLease, + serviceKey, + databaseId, + enableRealtime = false, + enableWebsockets = false, + realtimeSchema, + realtimeSourceSchemas, + realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs, + sharedRealtime + } = opts; + + if (poolLease && poolIdentity && poolLease.identity !== poolIdentity) { + throw new Error( + `PostGraphile[${cacheKey}] pool identity does not match its retained lease` + ); + } const pgl = postgraphile(preset); + const resolvedPreset = pgl.getResolvedPreset(); + const releasePresetServices = createPresetServicesReleaser(resolvedPreset); const serv = pgl.createServ(grafserv); + const handler = createGraphileHttpHandler(); + let upgradeHandler: GraphileUpgradeHandler | null = null; + let startupAttestation: GraphileRealtimeRoleAttestation | undefined; + let startupReleasePromise: Promise | null = null; + const releaseFailedGeneration = (): Promise => { + if (startupReleasePromise) return startupReleasePromise; + startupReleasePromise = (async () => { + let firstError: unknown; + try { + await pgl.release(); + } catch (error) { + firstError = error; + } + try { + await releasePresetServices(); + } catch (error) { + firstError ??= error; + } + try { + startupAttestation?.release(); + } catch (error) { + firstError ??= error; + } + try { + await sharedRealtime?.subscriber.release(); + } catch (error) { + firstError ??= error; + } + if (firstError) throw firstError; + })(); + return startupReleasePromise; + }; - const handler = express(); - const httpServer = createServer(handler); - await serv.addTo(handler, httpServer); - await serv.ready(); + // Start the schema build before wiring grafserv, but do not let this + // factory resolve until both are ready. `serv.ready()` alone does not + // guarantee that PostGraphile's gather/build phase has completed. + await awaitGraphileBuildReadiness({ + schemaResult: pgl.getSchemaResult(), + addTo: async () => { + const presetWebsockets = resolvedPreset.grafserv?.websockets === true; + if (presetWebsockets !== enableWebsockets) { + throw new Error( + `PostGraphile[${cacheKey}] websocket preset and shared routing must agree` + ); + } + await attachGraphileHttpHandler(serv, handler, resolvedPreset, { + sharedWebsocketRouting: enableWebsockets + }); + if (enableWebsockets) { + upgradeHandler = await serv.getUpgradeHandler(); + if (!upgradeHandler) { + throw new Error( + `PostGraphile[${cacheKey}] websocket upgrade handler is unavailable` + ); + } + } + }, + ready: () => serv.ready(), + release: releaseFailedGeneration, + onReleaseError: (releaseError) => { + log.error(`Failed to release PostGraphile[${cacheKey}] after build failure:`, releaseError); + } + }); const entry: GraphileCacheEntry = { pgl, serv, handler, - httpServer, + upgradeHandler, + httpServer: null, cacheKey, + poolIdentity: poolLease?.identity ?? poolIdentity, + poolLease, + releasePresetServices, + serviceKey, + databaseId, createdAt: Date.now(), + ...(sharedRealtime ? { realtimeSubscriber: sharedRealtime.subscriber } : {}) }; if (enableRealtime) { - try { - const { RealtimeManager } = await import('graphile-realtime-subscriptions'); - - // Extract PgSubscriber and pool from the resolved preset's pgServices. - // The pool is the same instance managed by pg-cache (via getPgPool) - // and threaded into the preset by makePgService({ pool, schemas }). - const resolvedPreset = pgl.getResolvedPreset(); - const pgService = (resolvedPreset as any).pgServices?.[0]; - const pgSubscriber = pgService?.pgSubscriber ?? null; - const pool = pgService?.adaptorSettings?.pool ?? null; - - if (!pgSubscriber) { - log.warn(`PostGraphile[${cacheKey}] has no pgSubscriber — RealtimeManager will not be started`); - } else if (!pool) { - log.warn(`PostGraphile[${cacheKey}] has no pool in pgService — RealtimeManager will not be started`); - } else { - const manager = new RealtimeManager({ - pgSubscriber, - pool, - nodeId: `graphile-cache:${cacheKey}`, - schema: 'realtime_public', + const realtimeHealth = createGraphileRealtimeHealth(); + entry.realtimeHealth = realtimeHealth; + const onFatalError = (error: Error): void => { + const alreadyFailed = entry.realtimeHealth?.status === 'failed'; + retireGraphileCacheEntry(entry, error); + if (!alreadyFailed) { + log.error( + `PostGraphile[${cacheKey}] realtime delivery became unavailable:`, + error + ); + } + }; + if (sharedRealtime) { + const pgService = (resolvedPreset as any)?.pgServices?.[0]; + if (pgService?.pgSubscriber !== sharedRealtime.subscriber) { + await releaseFailedGeneration(); + throw new GraphileRealtimeStartupError( + cacheKey, + new Error('Resolved pgService did not retain the provided generation subscriber') + ); + } + try { + startupAttestation = await activateGraphileSharedRealtime({ + ...sharedRealtime, + allowedSourceSchemas: realtimeSourceSchemas ?? [], + onFatalError }); - - await manager.start(); - entry.realtimeManager = manager; - log.info(`RealtimeManager started for PostGraphile[${cacheKey}]`); + entry.realtimeRoleAttestation = startupAttestation; + } catch (error) { + try { + await releaseFailedGeneration(); + } catch (releaseError) { + log.error( + `Failed to release PostGraphile[${cacheKey}] after shared realtime activation failure:`, + releaseError + ); + } + throw error instanceof GraphileRealtimeStartupError + ? error + : new GraphileRealtimeStartupError(cacheKey, error); + } + } + entry.realtimeManager = await startConfiguredRealtime({ + cacheKey, + resolvedPreset, + realtimeSchema, + allowedSourceSchemas: realtimeSourceSchemas ?? [], + ...(sharedRealtime ? { publisher: sharedRealtime.subscriber } : {}), + ...(realtimeCursorPollIntervalMs === undefined + ? {} + : { pollIntervalMs: realtimeCursorPollIntervalMs }), + ...(realtimeCursorHeartbeatIntervalMs === undefined + ? {} + : { heartbeatIntervalMs: realtimeCursorHeartbeatIntervalMs }), + onFatalError, + releasePostGraphile: releaseFailedGeneration + }); + if (entry.realtimeHealth.status === 'failed') { + try { + await entry.realtimeManager.stop(); + } finally { + await releaseFailedGeneration(); } - } catch (err) { - log.error(`Failed to start RealtimeManager for PostGraphile[${cacheKey}]:`, err); + throw new GraphileRealtimeStartupError( + cacheKey, + new Error('Realtime delivery failed during generation activation') + ); } + log.info(`RealtimeManager started for PostGraphile[${cacheKey}]`); } return entry; diff --git a/graphile/graphile-cache/src/graphile-cache.ts b/graphile/graphile-cache/src/graphile-cache.ts index 83782c6a21..bcd7a2a009 100644 --- a/graphile/graphile-cache/src/graphile-cache.ts +++ b/graphile/graphile-cache/src/graphile-cache.ts @@ -1,23 +1,111 @@ +import type { Duplex } from 'node:stream'; +import { getHeapStatistics } from 'node:v8'; + import { Logger } from '@pgpmjs/logger'; -import { parseEnvNumber } from '12factor-env'; import { EventEmitter } from 'events'; -import type { Express } from 'express'; +import type { NextFunction, Request, Response, Router } from 'express'; import type { GrafservBase } from 'grafserv'; -import type { Server as HttpServer } from 'http'; +import type { IncomingMessage, Server as HttpServer } from 'http'; import { LRUCache } from 'lru-cache'; -import { pgCache } from 'pg-cache'; +import { pgCache, type PgPoolLease } from 'pg-cache'; import type { PostGraphileInstance } from 'postgraphile'; +import { + GRAPHILE_REALTIME_UNAVAILABLE_CODE, + type GraphileRealtimeHealth, + withGraphileRealtimeFailure +} from './realtime-readiness'; +import { + getGraphileRealtimeRoleAuditStats, + type GraphileRealtimeRoleAttestation +} from './shared-realtime'; + const log = new Logger('graphile-cache'); +export const GRAPHILE_WEBSOCKET_UNAVAILABLE_CODE = + 'GRAPHILE_WEBSOCKET_UNAVAILABLE'; + +export type GraphileUpgradeHandler = ( + request: IncomingMessage, + socket: Duplex, + head: Buffer +) => void; + // --- Time Constants --- export const ONE_HOUR_MS = 1000 * 60 * 60; export const FIVE_MINUTES_MS = 1000 * 60 * 5; const ONE_DAY = ONE_HOUR_MS * 24; -const ONE_YEAR = ONE_DAY * 366; +const SIX_HOURS_MS = ONE_DAY / 4; // --- Eviction Types --- -export type EvictionReason = 'lru' | 'ttl' | 'manual'; +export type EvictionReason = + | 'lru' + | 'ttl' + | 'manual' + | 'governor' + | 'admission' + | 'realtime'; + +export interface CacheCounters { + /** Transient HTTP requests admitted to an exact resident handler. */ + httpRequestsStarted: number; + /** Admitted HTTP requests that reached a terminal response state. */ + httpRequestsCompleted: number; + /** WebSocket upgrades admitted to an exact resident upgrade handler. */ + websocketUpgradesStarted: number; + /** Admitted WebSocket lifecycles that closed or errored. */ + websocketUpgradesCompleted: number; + evictions: Record; + disposalsStarted: number; + disposalsCompleted: number; + disposalFailures: number; + drainTimeouts: number; + disposalTimeouts: number; + buildRefusals: Record; +} + +const cacheCounters: CacheCounters = { + httpRequestsStarted: 0, + httpRequestsCompleted: 0, + websocketUpgradesStarted: 0, + websocketUpgradesCompleted: 0, + evictions: { + lru: 0, + ttl: 0, + manual: 0, + governor: 0, + admission: 0, + realtime: 0 + }, + disposalsStarted: 0, + disposalsCompleted: 0, + disposalFailures: 0, + drainTimeouts: 0, + disposalTimeouts: 0, + buildRefusals: { + critical_pressure: 0, + insufficient_budget: 0, + rss_budget_exceeded: 0, + disposal_timeout: 0, + resident_busy: 0, + resident_capacity: 0, + disposal_failed: 0 + } +}; + +export const getCacheCounters = (): CacheCounters => ({ + httpRequestsStarted: cacheCounters.httpRequestsStarted, + httpRequestsCompleted: cacheCounters.httpRequestsCompleted, + websocketUpgradesStarted: cacheCounters.websocketUpgradesStarted, + websocketUpgradesCompleted: cacheCounters.websocketUpgradesCompleted, + evictions: { ...cacheCounters.evictions }, + disposalsStarted: cacheCounters.disposalsStarted, + disposalsCompleted: cacheCounters.disposalsCompleted, + disposalFailures: cacheCounters.disposalFailures, + drainTimeouts: cacheCounters.drainTimeouts, + disposalTimeouts: cacheCounters.disposalTimeouts, + buildRefusals: { ...cacheCounters.buildRefusals } +}); // --- Cache Event Emitter --- export interface CacheEvictionEvent { @@ -42,30 +130,243 @@ export const cacheEvents = new CacheEventEmitter(); export interface CacheConfig { max: number; ttl: number; + admissionMode: CacheAdmissionMode; + heapLimitBytes: number; + /** Explicit process-RSS ceiling. Null leaves RSS observable but unbounded. */ + rssLimitBytes: number | null; + instanceHeapBytes: number; + serverReserveBytes: number; + buildReserveBytes: number; + /** Transient RSS reserved before admitting one serialized build. */ + rssBuildReserveBytes: number; + budgetCapacity: number; + calibration: CacheCalibrationProvenance; +} + +export type CacheAdmissionMode = 'evict-idle' | 'preserve-resident'; + +export type CacheCalibrationSource = + | 'default' + | 'environment' + | 'runtime-safety-floor'; + +export interface CacheCalibrationProvenance { + id: string | null; + instanceHeapSource: CacheCalibrationSource; + instanceHeapSampleCount: number; + serverReserveSource: Exclude; + buildReserveSource: Exclude; } +const DEFAULT_INSTANCE_HEAP_BYTES = 512 * 1024 * 1024; +const DEFAULT_SERVER_RESERVE_BYTES = 256 * 1024 * 1024; +const DEFAULT_BUILD_RESERVE_BYTES = 768 * 1024 * 1024; +const DEFAULT_RSS_BUILD_RESERVE_BYTES = DEFAULT_BUILD_RESERVE_BYTES; +const MIN_BACKING_CACHE_ENTRIES = 1024; +const MAX_BACKING_CACHE_ENTRIES = 65_536; +// This is only a sparse-LRU allocation budget, never an estimate of a real +// Graphile instance. Keep it comfortably below every measured instance cost so +// the backing data structure cannot become the density limit before heap +// admission does. +const BACKING_CACHE_BYTES_PER_ENTRY = 256 * 1024; + +export const computeBackingCacheMax = (heapLimitBytes: number): number => { + if (!Number.isFinite(heapLimitBytes) || heapLimitBytes <= 0) { + return MIN_BACKING_CACHE_ENTRIES; + } + return Math.max( + MIN_BACKING_CACHE_ENTRIES, + Math.min( + MAX_BACKING_CACHE_ENTRIES, + Math.floor(heapLimitBytes / BACKING_CACHE_BYTES_PER_ENTRY) + ) + ); +}; + +const BACKING_CACHE_MAX = computeBackingCacheMax( + getHeapStatistics().heap_size_limit +); + +const parsePositiveInt = (value: string | undefined, fallback: number): number => { + const parsed = value ? Number.parseInt(value, 10) : Number.NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +}; + +const parseExplicitPositiveInt = ( + name: string, + value: string | undefined +): number | undefined => { + if (value === undefined) return undefined; + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive safe integer`); + } + return parsed; +}; + +const parseAdmissionMode = (value: string | undefined): CacheAdmissionMode => { + if (value === undefined || value === 'evict-idle') return 'evict-idle'; + if (value === 'preserve-resident') return 'preserve-resident'; + throw new Error( + 'GRAPHILE_CACHE_ADMISSION_MODE must be evict-idle or preserve-resident' + ); +}; + +const resolveCalibrationValue = ( + name: string, + fallback: number +): { bytes: number; source: 'default' | 'environment' } => { + const configured = parseExplicitPositiveInt(name, process.env[name]); + return configured === undefined + ? { bytes: fallback, source: 'default' } + : { bytes: configured, source: 'environment' }; +}; + +const measuredInstanceSamples: number[] = []; + +/** Record a retained-heap sample from a validated warm instance. */ +export const recordInstanceHeapSample = (bytes: number): void => { + if (!Number.isFinite(bytes) || bytes <= 0) return; + measuredInstanceSamples.push(Math.round(bytes)); + if (measuredInstanceSamples.length > 31) measuredInstanceSamples.shift(); +}; + +export const resetInstanceHeapSamples = (): void => { + measuredInstanceSamples.length = 0; +}; + +const median = (values: number[]): number => { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]; +}; + +const resolveInstanceHeapEstimate = (): { + bytes: number; + source: CacheCalibrationSource; +} => { + const configured = resolveCalibrationValue( + 'GRAPHILE_CACHE_INSTANCE_HEAP_BYTES', + DEFAULT_INSTANCE_HEAP_BYTES + ); + if (measuredInstanceSamples.length === 0) return configured; + const measuredWithReserve = Math.ceil(median(measuredInstanceSamples) * 1.2); + if (measuredWithReserve <= configured.bytes) return configured; + return { bytes: measuredWithReserve, source: 'runtime-safety-floor' }; +}; + +export const getInstanceHeapEstimate = (): number => + resolveInstanceHeapEstimate().bytes; + +/** + * Return the number of resident instances for which both steady-state and + * one-build-transient budgets fit. Zero means a build cannot be admitted. + */ +export const computeCapacityFromBudget = ( + heapLimitBytes: number, + instanceHeapBytes: number, + serverReserveBytes = DEFAULT_SERVER_RESERVE_BYTES, + buildReserveBytes = DEFAULT_BUILD_RESERVE_BYTES +): number => { + if ( + heapLimitBytes <= 0 || + instanceHeapBytes <= 0 || + serverReserveBytes + buildReserveBytes > heapLimitBytes + ) { + return 0; + } + const byResidency = Math.floor( + (heapLimitBytes - serverReserveBytes) / instanceHeapBytes + ); + const byRebuild = Math.floor( + (heapLimitBytes - serverReserveBytes - buildReserveBytes) / instanceHeapBytes + ) + 1; + return Math.max( + 0, + Math.min(computeBackingCacheMax(heapLimitBytes), byResidency, byRebuild) + ); +}; + /** * Get cache configuration from environment variables * * Supports: - * - GRAPHILE_CACHE_MAX: Maximum number of entries (default: 50) + * - GRAPHILE_CACHE_MAX: Operator ceiling (default: heap-budget-derived) + * - GRAPHILE_CACHE_ADMISSION_MODE: evict-idle (default) or preserve-resident + * - GRAPHILE_CACHE_RSS_LIMIT_BYTES: Optional absolute process-RSS ceiling + * - GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES: RSS reserved for one build * - GRAPHILE_CACHE_TTL_MS: TTL in milliseconds * - Production default: ONE_YEAR * - Development default: FIVE_MINUTES_MS * - * NOTE: This value should be <= PG_CACHE_MAX (also default: 50) so that - * every cached PostGraphile instance has a live pool backing it. + * Resident instances protect their exact runtime pools with `PgPoolLease`, so + * pool capacity and Graphile heap capacity are independent limits. Pool + * exhaustion fails closed when every registry identity is leased. */ export function getCacheConfig(): CacheConfig { const isDevelopment = process.env.NODE_ENV === 'development'; + const heapLimitBytes = getHeapStatistics().heap_size_limit; + const instanceHeap = resolveInstanceHeapEstimate(); + const serverReserve = resolveCalibrationValue( + 'GRAPHILE_CACHE_SERVER_RESERVE_BYTES', + DEFAULT_SERVER_RESERVE_BYTES + ); + const buildReserve = resolveCalibrationValue( + 'GRAPHILE_CACHE_BUILD_RESERVE_BYTES', + DEFAULT_BUILD_RESERVE_BYTES + ); + const rssLimitBytes = parseExplicitPositiveInt( + 'GRAPHILE_CACHE_RSS_LIMIT_BYTES', + process.env.GRAPHILE_CACHE_RSS_LIMIT_BYTES + ) ?? null; + const rssBuildReserveBytes = parseExplicitPositiveInt( + 'GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES', + process.env.GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES + ) ?? DEFAULT_RSS_BUILD_RESERVE_BYTES; + const instanceHeapBytes = instanceHeap.bytes; + const serverReserveBytes = serverReserve.bytes; + const buildReserveBytes = buildReserve.bytes; + const budgetCapacity = computeCapacityFromBudget( + heapLimitBytes, + instanceHeapBytes, + serverReserveBytes, + buildReserveBytes + ); + const requestedMax = parseExplicitPositiveInt( + 'GRAPHILE_CACHE_MAX', + process.env.GRAPHILE_CACHE_MAX + ) ?? (budgetCapacity || 1); + if (requestedMax > BACKING_CACHE_MAX) { + throw new Error( + `GRAPHILE_CACHE_MAX exceeds heap-scaled backing ceiling ${BACKING_CACHE_MAX}` + ); + } + // The backing LRU requires at least one slot. Admission still fails closed + // when budgetCapacity is zero, so the synthetic slot is never built into. + const max = Math.max(1, Math.min(requestedMax, budgetCapacity || 1)); + const ttl = parsePositiveInt( + process.env.GRAPHILE_CACHE_TTL_MS, + isDevelopment ? FIVE_MINUTES_MS : SIX_HOURS_MS + ); - const max = parseEnvNumber(process.env.GRAPHILE_CACHE_MAX) ?? 50; - - const ttl = - parseEnvNumber(process.env.GRAPHILE_CACHE_TTL_MS) ?? - (isDevelopment ? FIVE_MINUTES_MS : ONE_YEAR); - - return { max, ttl }; + return { + max, + ttl, + admissionMode: parseAdmissionMode(process.env.GRAPHILE_CACHE_ADMISSION_MODE), + heapLimitBytes, + rssLimitBytes, + instanceHeapBytes, + serverReserveBytes, + buildReserveBytes, + rssBuildReserveBytes, + budgetCapacity, + calibration: { + id: process.env.GRAPHILE_CACHE_CALIBRATION_ID?.trim() || null, + instanceHeapSource: instanceHeap.source, + instanceHeapSampleCount: measuredInstanceSamples.length, + serverReserveSource: serverReserve.source, + buildReserveSource: buildReserve.source + } + }; } /** @@ -74,79 +375,230 @@ export function getCacheConfig(): CacheConfig { * Each entry contains: * - pgl: The PostGraphile instance (manages schema, plugins, etc.) * - serv: The Grafserv server instance (handles HTTP/WS) - * - handler: Express app for routing requests - * - httpServer: Node HTTP server (required by grafserv) + * - handler: Lean Express router for routing requests + * - httpServer: Optional legacy/custom server; cached instances use the shared + * outer server and leave this null * - cacheKey: Unique identifier for this entry * - createdAt: Timestamp when this entry was created */ export interface GraphileCacheEntry { pgl: PostGraphileInstance; serv: GrafservBase; - handler: Express; - httpServer: HttpServer; + handler: Router; + /** No-server Grafserv handler selected only after exact tenant routing. */ + upgradeHandler?: GraphileUpgradeHandler | null; + /** Raw sockets retained so disposal can terminate long-lived subscriptions. */ + websocketSockets?: Set; + httpServer: HttpServer | null; cacheKey: string; + /** Opaque pg-cache identity used by this instance. */ + poolIdentity?: string; + /** + * Runtime pool ownership transferred from `createGraphileInstance()`. + * Disposal releases it only after requests and long-lived resources drain. + */ + poolLease?: PgPoolLease; + /** Idempotent release for pgServices owned by this exact preset generation. */ + releasePresetServices?: () => Promise; + /** Routing label for diagnostics and targeted invalidation only. */ + serviceKey?: string; + /** Tenant database id for targeted invalidation. */ + databaseId?: string | null; createdAt: number; /** Optional RealtimeManager for cursor-tracked subscription delivery */ realtimeManager?: { stop(): Promise } | null; + /** Caller-provided shared subscriber; preset services do not own it. */ + realtimeSubscriber?: { release(): Promise } | null; + /** Credential-free role-audit provenance plus coalesced TTL refresh. */ + realtimeRoleAttestation?: GraphileRealtimeRoleAttestation; + /** Fatal delivery failures latch this generation unavailable until rebuilt. */ + realtimeHealth?: GraphileRealtimeHealth; + /** Requests currently executing through this exact instance. */ + inflight?: number; + /** Once true, no new request may enter this instance. */ + disposing?: boolean; + /** Optional retained-heap measurement supplied by the validation harness. */ + retainedHeapBytes?: number; } -// Track disposed entries to prevent double-disposal -const disposedKeys = new Set(); +const disposalPromises = new WeakMap>(); +const activeDisposals = new Set>(); +let failedDisposalCount = 0; +const drainWaiters = new WeakMap void>>(); +const pendingEvictionReasons = new Map(); -// Track keys that are being manually evicted for accurate eviction reason -const manualEvictionKeys = new Set(); +export const getDrainingCount = (): number => activeDisposals.size; + +const notifyDrained = (entry: GraphileCacheEntry): void => { + if ((entry.inflight ?? 0) > 0) return; + const waiters = drainWaiters.get(entry); + if (!waiters) return; + drainWaiters.delete(entry); + for (const resolve of waiters) resolve(); +}; + +const waitForEntryDrain = (entry: GraphileCacheEntry): Promise => { + if ((entry.inflight ?? 0) === 0) return Promise.resolve(); + return new Promise((resolve) => { + const waiters = drainWaiters.get(entry) ?? new Set<() => void>(); + waiters.add(resolve); + drainWaiters.set(entry, waiters); + }); +}; + +export const raceWithClearedTimeout = async ( + promise: Promise, + timeoutMs: number +): Promise<{ timedOut: false; value: T } | { timedOut: true }> => { + let timer: ReturnType | undefined; + const timeout = new Promise<{ timedOut: true }>((resolve) => { + timer = setTimeout(() => resolve({ timedOut: true }), timeoutMs); + timer.unref?.(); + }); + try { + return await Promise.race([ + promise.then((value) => ({ timedOut: false as const, value })), + timeout + ]); + } finally { + if (timer) clearTimeout(timer); + } +}; /** * Dispose a PostGraphile v5 cache entry * * Properly releases resources by: - * 1. Closing the HTTP server if listening - * 2. Releasing the PostGraphile instance (which internally releases grafserv) + * 1. Waiting for resident requests to drain + * 2. Closing the HTTP server and releasing PostGraphile/Grafserv + * 3. Releasing the generation's preset services and PgSubscriber checkout + * 4. Stopping cursor-tracked realtime delivery + * 4. Releasing the retained runtime-pool lease * - * Uses disposedKeys set to prevent double-disposal when closeAllCaches() - * explicitly disposes entries and then clear() triggers the dispose callback. + * The promise is keyed by entry identity, so two generations with the same + * cache key both release exactly once and duplicate teardown is coalesced. */ -const disposeEntry = async (entry: GraphileCacheEntry, key: string): Promise => { - // Prevent double-disposal - if (disposedKeys.has(key)) { - return; - } - disposedKeys.add(key); +const scheduleDisposal = (entry: GraphileCacheEntry, key: string): Promise => { + const existing = disposalPromises.get(entry); + if (existing) return existing; - log.debug(`Disposing PostGraphile[${key}]`); - try { - // Close HTTP server if it's listening - if (entry.httpServer?.listening) { - await new Promise((resolve) => { - entry.httpServer.close(() => resolve()); - }); + entry.disposing = true; + // WebSocket subscriptions are deliberately long-lived. Waiting for clients + // to leave voluntarily would make LRU eviction and shutdown unbounded, so a + // retiring generation terminates only its own exact sockets before draining. + for (const socket of entry.websocketSockets ?? []) socket.destroy(); + cacheCounters.disposalsStarted++; + const pending = (async () => { + const drainTimeoutMs = parsePositiveInt( + process.env.GRAPHILE_CACHE_DRAIN_TIMEOUT_MS, + 30_000 + ); + const initialDrain = await raceWithClearedTimeout(waitForEntryDrain(entry), drainTimeoutMs); + if (initialDrain.timedOut) { + cacheCounters.drainTimeouts++; + log.warn( + `PostGraphile[${key}] still has ${entry.inflight ?? 0} request(s) after ` + + `${drainTimeoutMs}ms; teardown remains deferred until they finish` + ); + // Correctness wins over reclaim speed: never release an instance while a + // resident request is still executing through it. + await waitForEntryDrain(entry); } - // Stop RealtimeManager if present (before releasing PostGraphile) - if (entry.realtimeManager) { - try { - await entry.realtimeManager.stop(); - } catch (err) { - log.error(`Error stopping RealtimeManager for PostGraphile[${key}]:`, err); + + log.debug(`Disposing PostGraphile[${key}]`); + let firstError: unknown; + try { + if (entry.httpServer) { + await new Promise((resolve) => entry.httpServer.close(() => resolve())); } + } catch (error) { + firstError = error; } - // Release PostGraphile instance (this also releases grafserv internally) - if (entry.pgl) { + try { await entry.pgl.release(); + } catch (error) { + firstError ??= error; } - } catch (err) { - log.error(`Error disposing PostGraphile[${key}]:`, err); - } finally { - disposedKeys.delete(key); - } + try { + await entry.releasePresetServices?.(); + } catch (error) { + firstError ??= error; + } + try { + // A live GraphQL subscription may hold the PgSubscriber checkout while + // cursor tracking uses the other slot in the minimum max=2 runtime pool. + // Release Grafserv and the preset services first so cursor cleanup cannot + // deadlock waiting for a checkout that only PgSubscriber teardown returns. + if (entry.realtimeManager) await entry.realtimeManager.stop(); + } catch (error) { + firstError ??= error; + } + try { + entry.realtimeRoleAttestation?.release(); + } catch (error) { + firstError ??= error; + } + try { + await entry.realtimeSubscriber?.release(); + } catch (error) { + firstError ??= error; + } + try { + entry.poolLease?.release(); + } catch (error) { + firstError ??= error; + } + if (firstError) throw firstError; + cacheCounters.disposalsCompleted++; + })(); + + disposalPromises.set(entry, pending); + activeDisposals.add(pending); + void pending + .catch((error) => { + failedDisposalCount++; + cacheCounters.disposalFailures++; + log.error(`Failed to dispose PostGraphile[${key}]:`, error); + }) + .finally(() => activeDisposals.delete(pending)); + return pending; +}; + +/** Dispose an instance that finished building after its contract was invalidated. */ +export const disposeUncachedEntry = ( + entry: GraphileCacheEntry, + key = entry.cacheKey +): Promise => scheduleDisposal(entry, key); + +export const waitForEntryDisposal = async ( + entry: GraphileCacheEntry, + timeoutMs = 20_000 +): Promise => { + const pending = disposalPromises.get(entry); + if (!pending) return true; + const result = await raceWithClearedTimeout(pending, timeoutMs); + if (result.timedOut) cacheCounters.disposalTimeouts++; + return !result.timedOut; +}; + +export const waitForActiveDisposals = async (timeoutMs = 20_000): Promise => { + if (activeDisposals.size === 0) return true; + const result = await raceWithClearedTimeout( + Promise.allSettled([...activeDisposals]), + timeoutMs + ); + if (result.timedOut) cacheCounters.disposalTimeouts++; + return !result.timedOut; }; /** * Determine the eviction reason for a cache entry */ const getEvictionReason = (key: string, entry: GraphileCacheEntry): EvictionReason => { - if (manualEvictionKeys.has(key)) { - manualEvictionKeys.delete(key); - return 'manual'; + const explicit = pendingEvictionReasons.get(key); + if (explicit) { + pendingEvictionReasons.delete(key); + return explicit; } // Check if TTL expired @@ -164,32 +616,597 @@ const initialConfig = getCacheConfig(); // --- Graphile Cache --- export const graphileCache = new LRUCache({ - max: initialConfig.max, + // Admission enforces the dynamic heap-derived maximum. Keep the backing LRU + // at the hard ceiling so validated lower per-instance measurements can raise + // density without reconstructing the cache object. + max: BACKING_CACHE_MAX, ttl: initialConfig.ttl, updateAgeOnGet: true, dispose: (entry, key) => { - // Determine eviction reason before disposal const reason = getEvictionReason(key, entry); + cacheCounters.evictions[reason]++; // Emit eviction event cacheEvents.emitEviction({ key, reason, entry }); log.debug(`Evicting PostGraphile[${key}] (reason: ${reason})`); - // LRU dispose is synchronous, but v5 disposal is async - // Fire and forget the async cleanup - disposeEntry(entry, key).catch((err) => { - log.error(`Failed to dispose PostGraphile[${key}]:`, err); - }); + scheduleDisposal(entry, key); } }); +/** + * The server normally refreshes an expired role attestation before invoking a + * resident entry. Keep the cache boundary fail-closed too: direct consumers + * and synchronous WebSocket upgrades must not serve through an expired or + * failed listener-role proof. + */ +export const isEntryRealtimeUnavailable = (entry: GraphileCacheEntry): boolean => { + if (entry.realtimeHealth?.status === 'failed') return true; + const attestation = entry.realtimeRoleAttestation?.snapshot(); + return Boolean( + attestation + && (attestation.status === 'failed' || Date.now() >= attestation.validUntil) + ); +}; + +/** + * Permanently retire one exact generation after a fail-closed safety check. + * Marking the entry unavailable happens before cache removal so no concurrent + * HTTP request or WebSocket operation can enter between the failure and the + * disposal callback. Only the same resident object may be evicted; a healthy + * replacement with the same deterministic contract key is never touched. + */ +export const retireGraphileCacheEntry = ( + entry: GraphileCacheEntry, + error: unknown, + reason: EvictionReason = 'realtime' +): boolean => { + entry.realtimeHealth = withGraphileRealtimeFailure( + entry.realtimeHealth ?? { status: 'healthy' }, + error + ); + const resident = graphileCache.peek(entry.cacheKey, { allowStale: true }); + if (resident === entry) { + entry.disposing = true; + pendingEvictionReasons.set(entry.cacheKey, reason); + graphileCache.delete(entry.cacheKey); + return true; + } + + // An unpublished failed candidate must be rejected by publication. A stale + // object racing a healthy replacement is already detached and must not alter + // that replacement's lifecycle or masquerade as its disposal. + if (!resident) entry.disposing = true; + + // Cache removal normally destroys these through scheduleDisposal(). Keep the + // boundary fail-closed for an entry racing publication/removal as well. + for (const socket of entry.websocketSockets ?? []) socket.destroy(); + return false; +}; + +/** Enter an instance only while it is resident and not being torn down. */ +export const invokeEntryHandler = ( + entry: GraphileCacheEntry, + req: Request, + res: Response, + next: NextFunction +): boolean => { + const requestEnded = (): boolean => + Boolean( + req.aborted + || req.socket?.destroyed + || res.destroyed + || res.writableEnded + ); + if (requestEnded()) return false; + if (isEntryRealtimeUnavailable(entry)) { + // Retire only this exact resident generation. A delayed fatal callback or + // stale in-flight waiter must never evict a healthy replacement that uses + // the same deterministic build-contract key. + retireGraphileCacheEntry( + entry, + new Error('Graphile realtime generation is unavailable') + ); + if (!res.headersSent) { + res.setHeader('Retry-After', '15'); + res.status(503).json({ + error: { + code: GRAPHILE_REALTIME_UNAVAILABLE_CODE, + message: 'Realtime delivery is unavailable for this GraphQL instance' + } + }); + } + return true; + } + if (entry.disposing) return false; + cacheCounters.httpRequestsStarted++; + entry.inflight = (entry.inflight ?? 0) + 1; + let released = false; + const release = (): void => { + if (released) return; + released = true; + cacheCounters.httpRequestsCompleted++; + entry.inflight = Math.max(0, (entry.inflight ?? 1) - 1); + notifyDrained(entry); + }; + res.once('finish', release); + res.once('close', release); + // The response can close between the initial check and listener attachment. + // Rechecking after attachment turns that race into an ordinary release. + if (requestEnded()) { + res.removeListener('finish', release); + res.removeListener('close', release); + release(); + return false; + } + try { + entry.handler(req, res, next); + } catch (error) { + release(); + throw error; + } + return true; +}; + +/** + * Refresh an expired shared-listener role audit before serving through a + * resident generation. A failed refresh latches realtimeHealth via the + * activation observer, so the normal invocation boundary returns 503. + */ +export const revalidateEntryRealtimeRole = async ( + entry: GraphileCacheEntry +): Promise => { + if (!entry.realtimeRoleAttestation) return true; + return entry.realtimeRoleAttestation.revalidateIfDue(); +}; + +export interface GraphileUpgradeInvocationOptions { + /** Transfer the outer transport after this exact generation is retained. */ + onAccepted?: () => void; + /** Retire outer admission state before a stable cache-level rejection. */ + onRejected?: () => void; +} + +const writeUpgradeError = ( + socket: Duplex, + status: number, + code: string, + retryAfter?: number +): void => { + if (socket.destroyed) return; + const body = JSON.stringify({ error: { code } }); + const headers = [ + `HTTP/1.1 ${status} Service Unavailable`, + 'Connection: close', + 'Content-Type: application/json; charset=utf-8', + `Content-Length: ${Buffer.byteLength(body)}`, + ...(retryAfter == null ? [] : [`Retry-After: ${retryAfter}`]), + '', + body + ].join('\r\n'); + try { + socket.end(headers); + } catch { + socket.destroy(); + } +}; + +/** + * Route one already-authorized WebSocket upgrade into an exact cache entry. + * The outer server owns host/path/API selection; this function owns generation + * health, drain accounting, and bounded teardown of the accepted socket. + */ +export const invokeEntryUpgradeHandler = ( + entry: GraphileCacheEntry, + request: IncomingMessage, + socket: Duplex, + head: Buffer, + options: GraphileUpgradeInvocationOptions = {} +): boolean => { + if (request.aborted || socket.destroyed) return false; + if (isEntryRealtimeUnavailable(entry)) { + retireGraphileCacheEntry( + entry, + new Error('Graphile realtime generation is unavailable') + ); + try { + options.onRejected?.(); + writeUpgradeError(socket, 503, GRAPHILE_REALTIME_UNAVAILABLE_CODE, 15); + } catch (error) { + socket.destroy(); + throw error; + } + return true; + } + if (entry.disposing) return false; + if (!entry.upgradeHandler) { + try { + options.onRejected?.(); + writeUpgradeError(socket, 503, GRAPHILE_WEBSOCKET_UNAVAILABLE_CODE, 15); + } catch (error) { + socket.destroy(); + throw error; + } + return true; + } + + cacheCounters.websocketUpgradesStarted++; + entry.inflight = (entry.inflight ?? 0) + 1; + const sockets = entry.websocketSockets ?? new Set(); + entry.websocketSockets = sockets; + sockets.add(socket); + let released = false; + const release = (): void => { + if (released) return; + released = true; + cacheCounters.websocketUpgradesCompleted++; + socket.removeListener('close', release); + socket.removeListener('error', release); + sockets.delete(socket); + entry.inflight = Math.max(0, (entry.inflight ?? 1) - 1); + notifyDrained(entry); + }; + socket.once('close', release); + socket.once('error', release); + if (request.aborted || socket.destroyed || entry.disposing) { + release(); + return false; + } + try { + // The outer router may own a synthetic HTTP response while it runs tenant + // routing, authentication, and build admission. Transfer that transport + // only after this exact generation has passed every fail-closed check and + // is already accounted as in-flight. + options.onAccepted?.(); + entry.upgradeHandler(request, socket, head); + } catch (error) { + release(); + socket.destroy(); + throw error; + } + return true; +}; + +export type MemoryPressureLevel = 'ok' | 'elevated' | 'critical'; + +export interface MemoryPressure { + level: MemoryPressureLevel; + heapLevel: MemoryPressureLevel; + rssLevel: MemoryPressureLevel | 'unbounded'; + heapUsed: number; + heapLimit: number; + available: number; + ratio: number; + rssBytes: number; + rssLimitBytes: number | null; + rssRatio: number | null; +} + +const parseFraction = (value: string | undefined, fallback: number): number => { + const parsed = value ? Number.parseFloat(value) : Number.NaN; + return Number.isFinite(parsed) && parsed > 0 && parsed < 1 ? parsed : fallback; +}; + +const pressureLevel = ( + ratio: number, + elevatedAt: number, + criticalAt: number +): MemoryPressureLevel => ratio >= criticalAt + ? 'critical' + : ratio >= elevatedAt + ? 'elevated' + : 'ok'; + +export const getMemoryPressure = (): MemoryPressure => { + const stats = getHeapStatistics(); + const memory = process.memoryUsage(); + const heapUsed = memory.heapUsed; + const available = stats.total_available_size ?? Math.max(0, stats.heap_size_limit - heapUsed); + const exhaustible = heapUsed + available; + const ratio = exhaustible > 0 ? heapUsed / exhaustible : 0; + const elevatedAt = parseFraction( + process.env.GRAPHILE_MEMORY_GOVERNOR_ELEVATED, + 0.85 + ); + const criticalAt = parseFraction( + process.env.GRAPHILE_MEMORY_GOVERNOR_CRITICAL, + 0.92 + ); + const heapLevel = pressureLevel(ratio, elevatedAt, criticalAt); + const rssLimitBytes = getCacheConfig().rssLimitBytes; + const rssRatio = rssLimitBytes == null ? null : memory.rss / rssLimitBytes; + const rssLevel = rssRatio == null + ? 'unbounded' as const + : pressureLevel(rssRatio, elevatedAt, criticalAt); + const level: MemoryPressureLevel = heapLevel === 'critical' || rssLevel === 'critical' + ? 'critical' + : heapLevel === 'elevated' || rssLevel === 'elevated' + ? 'elevated' + : 'ok'; + return { + level, + heapLevel, + rssLevel, + heapUsed, + heapLimit: stats.heap_size_limit, + available, + ratio, + rssBytes: memory.rss, + rssLimitBytes, + rssRatio + }; +}; + +export type BuildRefusalReason = + | 'critical_pressure' + | 'insufficient_budget' + | 'rss_budget_exceeded' + | 'disposal_timeout' + | 'resident_busy' + | 'resident_capacity' + | 'disposal_failed'; + +export interface BuildAdmissionDecision { + admit: boolean; + reason?: BuildRefusalReason; + pressure: MemoryPressure; + projectedBytes: number; + heapLimitBytes: number; + projectedRssBytes: number; + rssLimitBytes: number | null; +} + +export const evaluateBuildAdmission = ( + residentCount = graphileCache.size +): BuildAdmissionDecision => { + const config = getCacheConfig(); + const pressure = getMemoryPressure(); + const projectedBytes = + config.serverReserveBytes + + residentCount * config.instanceHeapBytes + + config.buildReserveBytes; + const projectedRssBytes = pressure.rssBytes + config.rssBuildReserveBytes; + if (pressure.level === 'critical') { + return { + admit: false, + reason: 'critical_pressure', + pressure, + projectedBytes, + heapLimitBytes: config.heapLimitBytes, + projectedRssBytes, + rssLimitBytes: config.rssLimitBytes + }; + } + if (failedDisposalCount > 0) { + return { + admit: false, + reason: 'disposal_failed', + pressure, + projectedBytes, + heapLimitBytes: config.heapLimitBytes, + projectedRssBytes, + rssLimitBytes: config.rssLimitBytes + }; + } + // The preserve-resident mode turns the calibrated ceiling into a hard + // admission boundary. Check it before the transient-build calculation: the + // default mode deliberately evaluates a full cache, evicts one idle entry, + // and then evaluates the transient budget again. + if (config.admissionMode === 'preserve-resident' && residentCount >= config.max) { + return { + admit: false, + reason: 'resident_capacity', + pressure, + projectedBytes, + heapLimitBytes: config.heapLimitBytes, + projectedRssBytes, + rssLimitBytes: config.rssLimitBytes + }; + } + if (config.budgetCapacity === 0 || projectedBytes > config.heapLimitBytes) { + return { + admit: false, + reason: 'insufficient_budget', + pressure, + projectedBytes, + heapLimitBytes: config.heapLimitBytes, + projectedRssBytes, + rssLimitBytes: config.rssLimitBytes + }; + } + if ( + config.rssLimitBytes != null + && projectedRssBytes > config.rssLimitBytes + ) { + return { + admit: false, + reason: 'rss_budget_exceeded', + pressure, + projectedBytes, + heapLimitBytes: config.heapLimitBytes, + projectedRssBytes, + rssLimitBytes: config.rssLimitBytes + }; + } + return { + admit: true, + pressure, + projectedBytes, + heapLimitBytes: config.heapLimitBytes, + projectedRssBytes, + rssLimitBytes: config.rssLimitBytes + }; +}; + +export const recordBuildRefusal = (reason: BuildRefusalReason): void => { + cacheCounters.buildRefusals[reason]++; +}; + +export class CacheBuildAdmissionError extends Error { + readonly retryAfterSeconds = 15; + + constructor(readonly reason: BuildRefusalReason) { + super(`Graphile build admission refused: ${reason}`); + this.name = 'CacheBuildAdmissionError'; + } +} + +const evictEntry = ( + key: string, + reason: EvictionReason +): GraphileCacheEntry | undefined => { + const entry = graphileCache.peek(key); + if (!entry) return undefined; + pendingEvictionReasons.set(key, reason); + graphileCache.delete(key); + return entry; +}; + +export const deleteGraphileCacheEntry = async ( + key: string, + reason: EvictionReason = 'manual' +): Promise => { + const entry = evictEntry(key, reason); + if (!entry) return false; + await (disposalPromises.get(entry) ?? Promise.resolve()); + return true; +}; + +/** + * Make one build slot and wait until every evicted instance has truly released. + * This runs inside the global build coordinator, so the size check and eviction + * cannot race another large build. + */ +export const prepareCacheForBuild = async ( + timeoutMs = 20_000 +): Promise<{ evicted: number; decision: BuildAdmissionDecision }> => { + const initial = evaluateBuildAdmission(); + if ( + !initial.admit && + (initial.reason === 'critical_pressure' + || initial.reason === 'disposal_failed' + || initial.reason === 'resident_capacity') + ) { + recordBuildRefusal(initial.reason); + throw new CacheBuildAdmissionError(initial.reason); + } + + const startedAt = Date.now(); + if (!await waitForActiveDisposals(timeoutMs)) { + recordBuildRefusal('disposal_timeout'); + throw new CacheBuildAdmissionError('disposal_timeout'); + } + const targetSize = Math.max(0, getCacheConfig().max - 1); + let evicted = 0; + while (graphileCache.size > targetSize) { + const keys = [...graphileCache.rkeys()]; + const idleKey = keys.find((key) => { + const entry = graphileCache.peek(key); + return entry && !entry.disposing && (entry.inflight ?? 0) === 0; + }); + if (!idleKey) { + recordBuildRefusal('resident_busy'); + throw new CacheBuildAdmissionError('resident_busy'); + } + const victimKey = idleKey; + const entry = evictEntry(victimKey, 'admission'); + if (!entry) continue; + evicted++; + + const remainingMs = Math.max(1, timeoutMs - (Date.now() - startedAt)); + let disposed = false; + try { + disposed = await waitForEntryDisposal(entry, remainingMs); + } catch (error) { + log.error(`PostGraphile[${victimKey}] disposal failed during build admission`, error); + } + if (!disposed) { + recordBuildRefusal('disposal_timeout'); + throw new CacheBuildAdmissionError('disposal_timeout'); + } + } + + const decision = evaluateBuildAdmission(graphileCache.size); + if (!decision.admit && decision.reason) { + recordBuildRefusal(decision.reason); + throw new CacheBuildAdmissionError(decision.reason); + } + return { evicted, decision }; +}; + +let governorTimer: ReturnType | null = null; +let governorUsers = 0; + +export const startMemoryGovernor = (intervalMs = 10_000): (() => void) => { + if (process.env.GRAPHILE_MEMORY_GOVERNOR === '0') return () => {}; + governorUsers++; + if (!governorTimer) { + governorTimer = setInterval(() => { + const pressure = getMemoryPressure(); + if (pressure.level === 'ok') return; + for (const key of graphileCache.rkeys()) { + const entry = graphileCache.peek(key); + // A pressure governor must not interrupt a resident request. + if (entry && !entry.disposing && (entry.inflight ?? 0) === 0) { + log.warn( + `Memory governor evicting PostGraphile[${key}] at ${pressure.level} pressure` + ); + evictEntry(key, 'governor'); + break; + } + } + }, intervalMs); + governorTimer.unref?.(); + } + let released = false; + return () => { + if (released) return; + released = true; + governorUsers = Math.max(0, governorUsers - 1); + if (governorUsers === 0 && governorTimer) { + clearInterval(governorTimer); + governorTimer = null; + } + }; +}; + +export const stopMemoryGovernor = (): void => { + governorUsers = 0; + if (!governorTimer) return; + clearInterval(governorTimer); + governorTimer = null; +}; + // --- Cache Stats --- export interface CacheStats { size: number; max: number; ttl: number; + admissionMode: CacheAdmissionMode; keys: string[]; + realtimeUnhealthy: number; + realtimeRoleAttestations: { + generations: number; + identities: number; + healthy: number; + failed: number; + stale: number; + activeIdentityAuditAttempts: number; + catalogAuditAttempts: number; + catalogAuditFailures: number; + activeDatabaseTargets: number; + databaseConfigurationConflicts: number; + oldestLastAttestedAt: number | null; + }; + draining: number; + budgetCapacity: number; + instanceHeapBytes: number; + heapLimitBytes: number; + rssLimitBytes: number | null; + rssBuildReserveBytes: number; + calibration: CacheCalibrationProvenance; + pressure: MemoryPressure; } /** @@ -197,11 +1214,30 @@ export interface CacheStats { */ export function getCacheStats(): CacheStats { const config = getCacheConfig(); + const realtimeRoleAttestationGenerations = [...graphileCache.values()] + .filter((entry) => Boolean(entry.realtimeRoleAttestation)).length; + const realtimeRoleAuditStats = getGraphileRealtimeRoleAuditStats(); return { size: graphileCache.size, max: config.max, ttl: config.ttl, - keys: [...graphileCache.keys()] + admissionMode: config.admissionMode, + keys: [...graphileCache.keys()], + realtimeUnhealthy: [...graphileCache.values()].filter( + (entry) => entry.realtimeHealth?.status === 'failed' + ).length, + realtimeRoleAttestations: { + generations: realtimeRoleAttestationGenerations, + ...realtimeRoleAuditStats + }, + draining: getDrainingCount(), + budgetCapacity: config.budgetCapacity, + instanceHeapBytes: config.instanceHeapBytes, + heapLimitBytes: config.heapLimitBytes, + rssLimitBytes: config.rssLimitBytes, + rssBuildReserveBytes: config.rssBuildReserveBytes, + calibration: config.calibration, + pressure: getMemoryPressure() }; } @@ -217,8 +1253,7 @@ export function clearMatchingEntries(pattern: RegExp): number { for (const key of graphileCache.keys()) { if (pattern.test(key)) { - // Mark as manual eviction before deleting - manualEvictionKeys.add(key); + pendingEvictionReasons.set(key, 'manual'); graphileCache.delete(key); cleared++; } @@ -227,16 +1262,17 @@ export function clearMatchingEntries(pattern: RegExp): number { return cleared; } -// Register cleanup callback with pgCache -// When a pg pool is disposed, clean up any graphile instances using it -const unregister = pgCache.registerCleanupCallback((pgPoolKey: string) => { +// A retained lease prevents ordinary pg-cache eviction while an entry is +// resident. This callback remains a fail-safe for legacy unleased entries and +// explicit process-wide pg-cache shutdown, which is allowed to override leases. +pgCache.registerCleanupCallback((pgPoolKey: string) => { log.debug(`pgPool[${pgPoolKey}] disposed - checking graphile entries`); // Remove graphile entries that reference this pool key graphileCache.forEach((entry, k) => { - if (entry.cacheKey.includes(pgPoolKey)) { + if (entry.poolIdentity === pgPoolKey) { log.debug(`Removing graphileCache[${k}] due to pgPool[${pgPoolKey}] disposal`); - manualEvictionKeys.add(k); + pendingEvictionReasons.set(k, 'manual'); graphileCache.delete(k); } }); @@ -245,6 +1281,18 @@ const unregister = pgCache.registerCleanupCallback((pgPoolKey: string) => { // Enhanced close function that handles all caches const closePromise: { promise: Promise | null } = { promise: null }; +export const clearGraphileCache = async (): Promise => { + const entries = [...graphileCache.entries()]; + for (const [key] of entries) pendingEvictionReasons.set(key, 'manual'); + graphileCache.clear(); + const disposePromises = entries.map(([, entry]) => disposalPromises.get(entry)); + await Promise.allSettled([ + ...disposePromises.filter((promise): promise is Promise => Boolean(promise)), + ...activeDisposals + ]); + pendingEvictionReasons.clear(); +}; + /** * Close all caches and release resources * @@ -262,28 +1310,9 @@ export const closeAllCaches = async (verbose = false): Promise => { closePromise.promise = (async () => { try { if (verbose) log.info('Closing all server caches...'); + stopMemoryGovernor(); - // Collect all entries and dispose them properly - const entries = [...graphileCache.entries()]; - - // Mark all as manual evictions - for (const [key] of entries) { - manualEvictionKeys.add(key); - } - - const disposePromises = entries.map(([key, entry]) => - disposeEntry(entry, key) - ); - - // Wait for all disposals to complete - await Promise.allSettled(disposePromises); - - // Clear the cache after disposal (dispose callback will no-op due to disposedKeys) - graphileCache.clear(); - - // Clear disposed keys tracking after full cleanup - disposedKeys.clear(); - manualEvictionKeys.clear(); + await clearGraphileCache(); // Close pg pools await pgCache.close(); diff --git a/graphile/graphile-cache/src/http-adapter.ts b/graphile/graphile-cache/src/http-adapter.ts new file mode 100644 index 0000000000..7b9e53ddc0 --- /dev/null +++ b/graphile/graphile-cache/src/http-adapter.ts @@ -0,0 +1,52 @@ +import type { Server as HttpServer } from 'node:http'; +import type { Server as HttpsServer } from 'node:https'; + +import express, { type Express, type Router } from 'express'; + +/** The narrow part of ExpressGrafserv used by a cached HTTP-only instance. */ +export interface GrafservExpressAttachment { + addTo( + app: Express, + server: HttpServer | HttpsServer | null, + addExclusiveWebsocketHandler?: boolean + ): PromiseLike | void; +} + +export interface GraphileHttpAttachmentOptions { + /** + * The caller will route upgrades to this exact cached instance from the + * shared outer HTTP server. Grafserv must never install an exclusive + * listener for a tenant instance because that listener would reject every + * other tenant's path. + */ + sharedWebsocketRouting?: boolean; +} + +/** Allocate only the middleware router that the shared outer server invokes. */ +export const createGraphileHttpHandler = (): Router => express.Router(); + +/** + * Attach Grafserv's HTTP middleware without a private Node server. + * + * With exclusive websocket handling disabled, Grafserv's Express adapter only + * calls `app.use(...)`; Router implements that exact runtime contract. A cached + * per-tenant server never listens, so websocket upgrades must be owned by the + * shared outer server rather than retained on an unreachable dummy server. + */ +export const attachGraphileHttpHandler = ( + serv: GrafservExpressAttachment, + handler: Router, + resolvedPreset: unknown, + options: GraphileHttpAttachmentOptions = {} +): PromiseLike | void => { + if ( + (resolvedPreset as any)?.grafserv?.websockets === true + && options.sharedWebsocketRouting !== true + ) { + throw new Error( + '[graphile-cache] Cached Grafserv instances cannot own WebSocket ' + + 'upgrades; configure a tenant-aware upgrade handler on the shared server' + ); + } + return serv.addTo(handler as unknown as Express, null, false); +}; diff --git a/graphile/graphile-cache/src/index.ts b/graphile/graphile-cache/src/index.ts index 9a845fafe3..2f38bf4abf 100644 --- a/graphile/graphile-cache/src/index.ts +++ b/graphile/graphile-cache/src/index.ts @@ -1,29 +1,96 @@ // Main exports from graphile-cache package export { + BuildAdmissionDecision, + BuildRefusalReason, + CacheAdmissionMode, + CacheBuildAdmissionError, + CacheCalibrationProvenance, + CacheCalibrationSource, // Cache configuration CacheConfig, + // Process counters + CacheCounters, // Event emitter for cache events CacheEventEmitter, cacheEvents, CacheEvictionEvent, // Cache stats CacheStats, + clearGraphileCache, // Clear matching entries clearMatchingEntries, closeAllCaches, + // Capacity model and measured instance cost + computeBackingCacheMax, + computeCapacityFromBudget, + deleteGraphileCacheEntry, + disposeUncachedEntry, + evaluateBuildAdmission, // Eviction tracking EvictionReason, FIVE_MINUTES_MS, getCacheConfig, + getCacheCounters, getCacheStats, + getDrainingCount, + getInstanceHeapEstimate, + // Memory pressure governor + getMemoryPressure, + GRAPHILE_WEBSOCKET_UNAVAILABLE_CODE, // Cache instance and entry type graphileCache, GraphileCacheEntry, + GraphileUpgradeHandler, + // Request draining and build admission + invokeEntryHandler, + invokeEntryUpgradeHandler, + isEntryRealtimeUnavailable, + MemoryPressure, + MemoryPressureLevel, // Time constants - ONE_HOUR_MS} from './graphile-cache'; + ONE_HOUR_MS, + prepareCacheForBuild, + raceWithClearedTimeout, + recordBuildRefusal, + recordInstanceHeapSample, + resetInstanceHeapSamples, + retireGraphileCacheEntry, + revalidateEntryRealtimeRole, + startMemoryGovernor, + stopMemoryGovernor, + waitForActiveDisposals, + waitForEntryDisposal +} from './graphile-cache'; // Factory for creating PostGraphile v5 instances +export type { GraphileInstanceOptions } from './create-instance'; export { createGraphileInstance } from './create-instance'; +export type { + GraphileRealtimeHealth, + GraphileRealtimeManager +} from './realtime-readiness'; +export { + createGraphileRealtimeHealth, + createGraphileRealtimeNodeId, + DEFAULT_GRAPHILE_REALTIME_SCHEMA, + GRAPHILE_REALTIME_UNAVAILABLE_CODE, + GraphileRealtimeStartupError, + startConfiguredRealtime, + withGraphileRealtimeFailure +} from './realtime-readiness'; +export type { + ActivateGraphileSharedRealtimeOptions, + GraphileRealtimeRoleAttestation, + GraphileRealtimeRoleAttestationSnapshot, + GraphileRealtimeRoleAuditStats} from './shared-realtime'; +export { + activateGraphileSharedRealtime, + getGraphileRealtimeRoleAuditStats, + GRAPHILE_SHARED_REALTIME_DATABASE_CONFLICT_ERROR_CODE, + GRAPHILE_SHARED_REALTIME_IDENTITY_ERROR_CODE, + GraphileSharedRealtimeDatabaseConflictError, + GraphileSharedRealtimeIdentityError +} from './shared-realtime'; // Generic module config cache for plugin lookups export { ModuleConfigCache, ModuleConfigCacheOptions } from './module-config-cache'; diff --git a/graphile/graphile-cache/src/preset-services.ts b/graphile/graphile-cache/src/preset-services.ts new file mode 100644 index 0000000000..c2210828b3 --- /dev/null +++ b/graphile/graphile-cache/src/preset-services.ts @@ -0,0 +1,33 @@ +interface ReleasablePresetService { + release?: () => void | Promise; +} + +/** + * Own the pgServices created for one resolved PostGraphile preset. + * + * PostGraphile 5.0.3 releases Grafserv but does not release pgServices. Cached + * generations therefore have to do this explicitly or an evicted + * PgSubscriber can retain its LISTEN checkout in the next generation's pool. + */ +export const createPresetServicesReleaser = ( + resolvedPreset: { pgServices?: readonly ReleasablePresetService[] } +): (() => Promise) => { + const services = [...new Set(resolvedPreset.pgServices ?? [])]; + let releasePromise: Promise | null = null; + + return (): Promise => { + if (releasePromise) return releasePromise; + releasePromise = (async () => { + let firstError: unknown; + for (const service of [...services].reverse()) { + try { + await service.release?.(); + } catch (error) { + firstError ??= error; + } + } + if (firstError) throw firstError; + })(); + return releasePromise; + }; +}; diff --git a/graphile/graphile-cache/src/realtime-readiness.ts b/graphile/graphile-cache/src/realtime-readiness.ts new file mode 100644 index 0000000000..81294d3716 --- /dev/null +++ b/graphile/graphile-cache/src/realtime-readiness.ts @@ -0,0 +1,186 @@ +import { randomUUID } from 'node:crypto'; + +import { Logger } from '@pgpmjs/logger'; +import type { RealtimePublisher } from 'graphile-realtime-subscriptions'; + +const log = new Logger('graphile-cache:realtime'); + +export const DEFAULT_GRAPHILE_REALTIME_SCHEMA = 'realtime_public'; +export const GRAPHILE_REALTIME_UNAVAILABLE_CODE = 'GRAPHILE_REALTIME_UNAVAILABLE'; + +// One module instance represents one Node.js process/worker runtime. A random +// component prevents two replicas serving the same exact build contract from +// sharing a database cursor identity and cleaning up each other's state. +const GRAPHILE_REALTIME_PROCESS_ID = `${process.pid}-${randomUUID()}`; + +export const createGraphileRealtimeNodeId = ( + cacheKey: string, + replicaIdentity = GRAPHILE_REALTIME_PROCESS_ID +): string => `graphile-cache:${replicaIdentity}:${cacheKey}`; + +export type GraphileRealtimeHealth = + | { readonly status: 'healthy' } + | { + readonly status: 'failed'; + readonly failureCode: string | null; + readonly failedAt: number; + }; + +export const createGraphileRealtimeHealth = (): GraphileRealtimeHealth => ({ + status: 'healthy' +}); + +const errorCode = (error: unknown): string | null => { + if (!error || typeof error !== 'object') return null; + const code = (error as { code?: unknown }).code; + return typeof code === 'string' && code.length > 0 ? code : null; +}; + +/** Return the first fatal delivery state; a failed generation stays failed. */ +export const withGraphileRealtimeFailure = ( + health: GraphileRealtimeHealth, + error: unknown, + failedAt = Date.now() +): GraphileRealtimeHealth => health.status === 'failed' + ? health + : { + status: 'failed', + failureCode: errorCode(error), + failedAt + }; + +export class GraphileRealtimeStartupError extends Error { + readonly code = 'GRAPHILE_REALTIME_STARTUP_FAILED'; + + constructor(cacheKey: string, readonly cause?: unknown) { + super(`PostGraphile[${cacheKey}] realtime was configured but could not start`); + this.name = 'GraphileRealtimeStartupError'; + } +} + +export interface GraphileRealtimeManager { + start(): Promise; + stop(): Promise; +} + +export interface GraphileRealtimeManagerConstructor { + new(options: { + pgSubscriber?: any; + publisher?: RealtimePublisher; + pool: any; + nodeId: string; + schema: string; + allowedSourceSchemas: readonly string[]; + pollIntervalMs?: number; + heartbeatIntervalMs?: number; + onFatalError?: (error: Error) => void; + }): GraphileRealtimeManager; +} + +export interface StartConfiguredRealtimeOptions { + cacheKey: string; + resolvedPreset: unknown; + /** + * Physical schema containing the cursor functions for this exact runtime + * identity. Omit to preserve the historical `realtime_public` behavior. + */ + realtimeSchema?: string; + /** Exact physical schemas exposed by this Graphile instance. */ + allowedSourceSchemas: readonly string[]; + /** Explicit generation-local publisher used by shared-exact mode. */ + publisher?: RealtimePublisher; + /** Cursor recovery polling interval. */ + pollIntervalMs?: number; + /** Cursor listener heartbeat interval. */ + heartbeatIntervalMs?: number; + /** Synchronous fatal-delivery callback used to remove the owner from service. */ + onFatalError?: (error: Error) => void; + releasePostGraphile(): PromiseLike | void; + loadManager?: () => Promise; + /** @internal Deterministic injection for replica-identity tests. */ + replicaIdentity?: string; +} + +const defaultLoadManager = async (): Promise => { + const { RealtimeManager } = await import('graphile-realtime-subscriptions'); + return RealtimeManager; +}; + +/** + * Realtime is part of readiness when configured. Any missing dependency or + * startup failure releases the PostGraphile generation before rejecting. + */ +export const startConfiguredRealtime = async ( + options: StartConfiguredRealtimeOptions +): Promise => { + const { + cacheKey, + resolvedPreset, + realtimeSchema = DEFAULT_GRAPHILE_REALTIME_SCHEMA, + allowedSourceSchemas, + publisher, + pollIntervalMs, + heartbeatIntervalMs, + onFatalError, + releasePostGraphile, + loadManager = defaultLoadManager, + replicaIdentity + } = options; + let manager: GraphileRealtimeManager | undefined; + try { + const pgService = (resolvedPreset as any)?.pgServices?.[0]; + const pgSubscriber = pgService?.pgSubscriber ?? null; + const pool = pgService?.adaptorSettings?.pool ?? null; + if (!publisher && !pgSubscriber) { + throw new Error(`PostGraphile[${cacheKey}] resolved without a pgSubscriber`); + } + if (!pool) { + throw new Error(`PostGraphile[${cacheKey}] resolved without a runtime pool`); + } + const exactSourceSchemas = [...new Set(allowedSourceSchemas ?? [])]; + if ( + exactSourceSchemas.length === 0 + || exactSourceSchemas.some( + (schema) => typeof schema !== 'string' || schema.length === 0 + ) + ) { + throw new Error( + `PostGraphile[${cacheKey}] realtime requires at least one allowed source schema` + ); + } + + const RealtimeManager = await loadManager(); + manager = new RealtimeManager({ + ...(publisher ? { publisher } : { pgSubscriber }), + pool, + nodeId: createGraphileRealtimeNodeId(cacheKey, replicaIdentity), + schema: realtimeSchema, + allowedSourceSchemas: exactSourceSchemas, + ...(pollIntervalMs === undefined ? {} : { pollIntervalMs }), + ...(heartbeatIntervalMs === undefined ? {} : { heartbeatIntervalMs }), + ...(onFatalError ? { onFatalError } : {}) + }); + await manager.start(); + return manager; + } catch (error) { + if (manager) { + try { + await manager.stop(); + } catch (stopError) { + log.error( + `Failed to stop partially started RealtimeManager for PostGraphile[${cacheKey}]:`, + stopError + ); + } + } + try { + await releasePostGraphile(); + } catch (releaseError) { + log.error( + `Failed to release PostGraphile[${cacheKey}] after realtime startup failure:`, + releaseError + ); + } + throw new GraphileRealtimeStartupError(cacheKey, error); + } +}; diff --git a/graphile/graphile-cache/src/shared-realtime.ts b/graphile/graphile-cache/src/shared-realtime.ts new file mode 100644 index 0000000000..51b3bdd788 --- /dev/null +++ b/graphile/graphile-cache/src/shared-realtime.ts @@ -0,0 +1,490 @@ +import { + ActivatableGenerationScopedRealtimeSubscriber, + type RealtimeTopicCollector +} from 'graphile-realtime-subscriptions'; +import { + acquirePgNotificationBroker, + getPgNotificationBrokerIdentity, + getPgNotificationBrokerStats, + getPgNotificationDatabaseIdentity, + PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE, + type PgAttestedNotificationBrokerLease, + type PgNotificationListenerConfig, + type PgNotificationRoleAudit +} from 'pg-cache'; + +export const GRAPHILE_SHARED_REALTIME_IDENTITY_ERROR_CODE = + 'GRAPHILE_SHARED_REALTIME_IDENTITY_MISMATCH'; +export const GRAPHILE_SHARED_REALTIME_DATABASE_CONFLICT_ERROR_CODE = + 'GRAPHILE_SHARED_REALTIME_DATABASE_CONFLICT'; + +export class GraphileSharedRealtimeIdentityError extends Error { + readonly code = GRAPHILE_SHARED_REALTIME_IDENTITY_ERROR_CODE; + + constructor() { + super('Shared realtime listener identity does not match its connection contract'); + this.name = 'GraphileSharedRealtimeIdentityError'; + } +} + +export class GraphileSharedRealtimeDatabaseConflictError extends Error { + readonly code = GRAPHILE_SHARED_REALTIME_DATABASE_CONFLICT_ERROR_CODE; + + constructor(database: string) { + super( + `Physical database ${JSON.stringify(database)} already has a different active ` + + 'shared realtime listener contract' + ); + this.name = 'GraphileSharedRealtimeDatabaseConflictError'; + } +} + +export interface GraphileRealtimeRoleAttestationSnapshot { + readonly version: 1; + readonly mode: 'shared-exact'; + readonly listenerIdentity: string; + readonly auditVersion: string; + readonly role: string; + readonly database: string; + readonly lastAttestedAt: number; + readonly validUntil: number; + readonly checks: number; + readonly status: 'healthy' | 'failed'; + readonly failureCode: string | null; + readonly failedAt: number | null; +} + +export interface GraphileRealtimeRoleAttestation { + snapshot(): Readonly; + /** Re-audit once this generation's explicit validity window has elapsed. */ + revalidateIfDue(): Promise; + release(): void; +} + +interface SharedAttestationRecord { + readonly identity: string; + readonly role: string; + readonly database: string; + audit: PgNotificationRoleAudit; + lastAttestedAt: number; + revalidationMs: number; + checks: number; + refreshPromise: Promise | null; + refreshTimer: ReturnType | null; + failure: { code: string | null; failedAt: number } | null; + bindings: Set; +} + +interface SharedAttestationBinding { + readonly revalidationMs: number; + readonly onFailure: (error: Error) => void; + readonly revalidateRole: () => Promise; +} + +interface ActiveDatabaseListenerContract { + readonly listenerIdentity: string; + readonly role: string; + references: number; +} + +const attestationRecords = new Map(); +const activeDatabaseListenerContracts = new Map< +string, +ActiveDatabaseListenerContract +>(); +let databaseConfigurationConflicts = 0; + +export interface GraphileRealtimeRoleAuditStats { + readonly identities: number; + readonly healthy: number; + readonly failed: number; + readonly stale: number; + readonly activeIdentityAuditAttempts: number; + readonly catalogAuditAttempts: number; + readonly catalogAuditFailures: number; + readonly activeDatabaseTargets: number; + readonly databaseConfigurationConflicts: number; + readonly oldestLastAttestedAt: number | null; +} + +/** Process-level unique identity counts plus monotonic catalog-audit counters. */ +export const getGraphileRealtimeRoleAuditStats = ( + now = Date.now() +): Readonly => { + const records = [...attestationRecords.values()]; + const brokerStats = getPgNotificationBrokerStats(); + return Object.freeze({ + identities: records.length, + healthy: records.filter(({ failure }) => !failure).length, + failed: records.filter(({ failure }) => Boolean(failure)).length, + stale: records.filter( + ({ lastAttestedAt, revalidationMs }) => now >= lastAttestedAt + revalidationMs + ).length, + activeIdentityAuditAttempts: records.reduce( + (sum, { checks }) => sum + checks, + 0 + ), + catalogAuditAttempts: brokerStats.roleAuditAttempts, + catalogAuditFailures: brokerStats.roleAuditFailures, + activeDatabaseTargets: activeDatabaseListenerContracts.size, + databaseConfigurationConflicts, + oldestLastAttestedAt: records.length === 0 + ? null + : Math.min(...records.map(({ lastAttestedAt }) => lastAttestedAt)) + }); +}; + +const errorCode = (error: unknown): string | null => { + if (!error || typeof error !== 'object') return null; + const code = (error as { code?: unknown }).code; + return typeof code === 'string' && code.length > 0 ? code : null; +}; + +const reserveDatabaseListenerContract = (options: { + databaseIdentity: string; + listenerIdentity: string; + role: string; + database: string; +}): (() => void) => { + const { databaseIdentity, listenerIdentity, role, database } = options; + let record = activeDatabaseListenerContracts.get(databaseIdentity); + if ( + record + && (record.listenerIdentity !== listenerIdentity || record.role !== role) + ) { + databaseConfigurationConflicts++; + throw new GraphileSharedRealtimeDatabaseConflictError(database); + } + if (record) { + record.references++; + } else { + record = { listenerIdentity, role, references: 1 }; + activeDatabaseListenerContracts.set(databaseIdentity, record); + } + let released = false; + return (): void => { + if (released) return; + released = true; + record!.references--; + if ( + record!.references === 0 + && activeDatabaseListenerContracts.get(databaseIdentity) === record + ) { + activeDatabaseListenerContracts.delete(databaseIdentity); + } + }; +}; + +const withDatabaseContractReservation = ( + source: PgAttestedNotificationBrokerLease, + releaseReservation: () => void +): PgAttestedNotificationBrokerLease => { + let releasePromise: Promise | null = null; + return Object.freeze({ + identity: source.identity, + topics: source.topics, + terminated: source.terminated, + get roleAudit(): PgNotificationRoleAudit { + return source.roleAudit; + }, + revalidateRole(): Promise { + return source.revalidateRole(); + }, + subscribe(topic: string): AsyncIterableIterator { + return source.subscribe(topic); + }, + release(): Promise { + if (releasePromise) return releasePromise; + releasePromise = (async () => { + try { + await source.release(); + } finally { + releaseReservation(); + } + })(); + return releasePromise; + } + }); +}; + +const MAX_TIMER_DELAY_MS = 2_147_483_647; + +const clearRefreshTimer = (record: SharedAttestationRecord): void => { + if (!record.refreshTimer) return; + clearTimeout(record.refreshTimer); + record.refreshTimer = null; +}; + +function scheduleRefresh(record: SharedAttestationRecord): void { + clearRefreshTimer(record); + if (record.failure || record.bindings.size === 0) return; + const dueAt = record.lastAttestedAt + record.revalidationMs; + const delay = Math.max( + 0, + Math.min(MAX_TIMER_DELAY_MS, dueAt - Date.now()) + ); + record.refreshTimer = setTimeout(() => { + record.refreshTimer = null; + if (record.failure || record.bindings.size === 0) return; + // Very large TTLs are scheduled in safe setTimeout-sized chunks. + if (Date.now() < record.lastAttestedAt + record.revalidationMs) { + scheduleRefresh(record); + return; + } + void refreshRecord(record); + }, delay); + record.refreshTimer.unref?.(); +} + +const revalidateWithActiveBinding = async ( + record: SharedAttestationRecord +): Promise => { + const attempted = new Set(); + for (;;) { + const binding = [...record.bindings].find((candidate) => !attempted.has(candidate)); + if (!binding) { + throw new Error('Shared realtime role attestation has no active broker lease'); + } + attempted.add(binding); + try { + return await binding.revalidateRole(); + } catch (error) { + if ( + errorCode(error) === PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE + && !record.bindings.has(binding) + ) { + continue; + } + throw error; + } + } +}; + +function refreshRecord(record: SharedAttestationRecord): Promise { + if (record.failure) return Promise.resolve(false); + if (record.refreshPromise) return record.refreshPromise; + record.checks++; + const pending = (async (): Promise => { + try { + const audit = await revalidateWithActiveBinding(record); + record.audit = audit; + record.lastAttestedAt = Date.now(); + return true; + } catch (reason) { + const error = reason instanceof Error ? reason : new Error(String(reason)); + record.failure = { + code: errorCode(error), + failedAt: Date.now() + }; + clearRefreshTimer(record); + for (const binding of [...record.bindings]) { + try { + binding.onFailure(error); + } catch { + // Every observer is advisory; the failed record remains latched. + } + } + return false; + } + })(); + record.refreshPromise = pending; + void pending.then(() => { + if (record.refreshPromise === pending) record.refreshPromise = null; + if (record.bindings.size === 0) { + clearRefreshTimer(record); + if (attestationRecords.get(record.identity) === record) { + attestationRecords.delete(record.identity); + } + } else if (!record.failure) { + scheduleRefresh(record); + } + }); + return pending; +} + +const registerAttestation = (options: { + identity: string; + audit: PgNotificationRoleAudit; + attestedAt: number; + revalidationMs: number; + onFailure(error: Error): void; + revalidateRole(): Promise; +}): GraphileRealtimeRoleAttestation => { + const { + identity, + audit, + attestedAt, + revalidationMs, + onFailure, + revalidateRole + } = options; + let record = attestationRecords.get(identity); + if (!record) { + record = { + identity, + role: audit.role, + database: audit.database, + audit, + lastAttestedAt: attestedAt, + revalidationMs, + checks: 1, + refreshPromise: null, + refreshTimer: null, + failure: null, + bindings: new Set() + }; + attestationRecords.set(identity, record); + } else { + // Broker identity covers credentials, database, pool, TLS, and driver. + // A freshly successful acquisition audit supersedes older provenance. + record.audit = audit; + record.lastAttestedAt = attestedAt; + record.checks++; + record.failure = null; + } + const binding: SharedAttestationBinding = { + revalidationMs, + onFailure, + revalidateRole + }; + record.bindings.add(binding); + record.revalidationMs = Math.min( + ...[...record.bindings].map((active) => active.revalidationMs) + ); + scheduleRefresh(record); + let released = false; + + return Object.freeze({ + snapshot(): Readonly { + const failure = record!.failure; + return Object.freeze({ + version: 1, + mode: 'shared-exact', + listenerIdentity: identity, + auditVersion: record!.audit.version, + role: record!.role, + database: record!.database, + lastAttestedAt: record!.lastAttestedAt, + validUntil: record!.lastAttestedAt + revalidationMs, + checks: record!.checks, + status: failure ? 'failed' : 'healthy', + failureCode: failure?.code ?? null, + failedAt: failure?.failedAt ?? null + }); + }, + async revalidateIfDue(): Promise { + if (released || record!.failure) return false; + if (Date.now() < record!.lastAttestedAt + revalidationMs) return true; + return refreshRecord(record!); + }, + release(): void { + if (released) return; + released = true; + record!.bindings.delete(binding); + if (record!.bindings.size === 0) { + clearRefreshTimer(record!); + if (!record!.refreshPromise) attestationRecords.delete(identity); + } else { + record!.revalidationMs = Math.min( + ...[...record!.bindings].map((active) => active.revalidationMs) + ); + scheduleRefresh(record!); + } + } + }); +}; + +export interface ActivateGraphileSharedRealtimeOptions { + subscriber: ActivatableGenerationScopedRealtimeSubscriber; + topicCollector: RealtimeTopicCollector; + listenerPgConfig: PgNotificationListenerConfig; + listenerIdentity: string; + allowedSourceSchemas: readonly string[]; + roleRevalidationMs: number; + onFatalError(error: Error): void; +} + +/** + * Cross the shared-listener publication boundary. Topic validation and a fresh + * role audit finish before the broker lease is installed into PostGraphile. + */ +export const activateGraphileSharedRealtime = async ( + options: ActivateGraphileSharedRealtimeOptions +): Promise => { + const { + subscriber, + topicCollector, + listenerPgConfig, + listenerIdentity, + allowedSourceSchemas, + roleRevalidationMs, + onFatalError + } = options; + const expectedIdentity = getPgNotificationBrokerIdentity(listenerPgConfig); + if (expectedIdentity !== listenerIdentity) { + throw new GraphileSharedRealtimeIdentityError(); + } + if (!Number.isSafeInteger(roleRevalidationMs) || roleRevalidationMs <= 0) { + throw new Error('Shared realtime role revalidation interval must be positive'); + } + const topics = topicCollector.exactTopics(allowedSourceSchemas); + const role = listenerPgConfig.user; + const database = listenerPgConfig.database; + const databaseIdentity = getPgNotificationDatabaseIdentity(listenerPgConfig); + const releaseDatabaseReservation = reserveDatabaseListenerContract({ + databaseIdentity, + listenerIdentity, + role, + database + }); + + // This audit is intentionally fresh for every generation acquisition. The + // role may have drifted since an older generation joined the same broker. + let brokerLease: Awaited>; + try { + // Broker admission serializes this generation's fresh role audit and LISTEN + // on the same pinned client, which remains safe with pool max=1. + brokerLease = await acquirePgNotificationBroker(listenerPgConfig, { topics }); + } catch (error) { + releaseDatabaseReservation(); + throw error; + } + + const reservedBrokerLease = withDatabaseContractReservation( + brokerLease, + releaseDatabaseReservation + ); + + const reportBrokerTermination = (failure: Error): void => { + try { + onFatalError(failure); + } catch { + // The subscriber still fails all streams even if an observer throws. + } + }; + void reservedBrokerLease.terminated.then((failure) => { + if (failure) reportBrokerTermination(failure); + }); + + try { + await subscriber.activate({ + source: reservedBrokerLease, + allowedTopics: topics + }); + } catch (error) { + try { + await reservedBrokerLease.release(); + } catch { + // Preserve the activation failure; reservation release runs in finally. + } + throw error; + } + return registerAttestation({ + identity: listenerIdentity, + audit: reservedBrokerLease.roleAudit, + attestedAt: Date.now(), + revalidationMs: roleRevalidationMs, + onFailure: reportBrokerTermination, + revalidateRole: () => reservedBrokerLease.revalidateRole() + }); +}; diff --git a/graphile/graphile-function-bindings/src/__tests__/preloaded-bindings.test.ts b/graphile/graphile-function-bindings/src/__tests__/preloaded-bindings.test.ts new file mode 100644 index 0000000000..b07ecdf452 --- /dev/null +++ b/graphile/graphile-function-bindings/src/__tests__/preloaded-bindings.test.ts @@ -0,0 +1,122 @@ +import { withPgClientFromPgService } from '@dataplan/pg'; +import type { GraphileConfig } from 'graphile-config'; + +import { createFunctionBindingsPlugin } from '../plugin'; +import type { + ComputeModuleNames, + PreloadedFunctionBinding +} from '../types'; + +jest.mock('@dataplan/pg', () => ({ + ...jest.requireActual('@dataplan/pg'), + withPgClientFromPgService: jest.fn() +})); + +const withPgClientMock = withPgClientFromPgService as unknown as jest.Mock; + +const moduleNames: ComputeModuleNames = { + computeSchema: 'compute_public', + bindingsTable: 'function_api_bindings', + definitionsTable: 'function_definitions', + invocationsSchema: 'compute_public', + invocationsTable: 'function_invocations', + invocationsEntityField: null +}; + +const binding = (): PreloadedFunctionBinding => ({ + bindingId: 'binding-1', + alias: 'send_email', + config: { + graphql: { enabled: true }, + schema: { + type: 'object', + properties: { to: { type: 'string' } } + } + }, + functionDefinitionId: 'definition-1', + taskIdentifier: 'app:send_email', + description: 'Send an email', + payloadArgs: [{ name: 'to', type: 'text' }], + module: { ...moduleNames } +}); + +async function runGather(plugin: GraphileConfig.Plugin, includePgService = false) { + const output: Record = {}; + const main = (plugin.gather as any).main as ( + output: Record, + info: Record + ) => Promise; + await main(output, { + resolvedPreset: { + pgServices: includePgService ? [{ name: 'main' }] : [] + } + }); + return (output.functionApiBindings as { + bindings: readonly PreloadedFunctionBinding[]; + }).bindings; +} + +describe('FunctionBindingsPlugin preloaded bindings', () => { + beforeEach(() => { + withPgClientMock.mockReset(); + }); + + it('treats an empty preloaded array as authoritative and performs zero SQL', async () => { + const plugin = createFunctionBindingsPlugin({ + apiId: 'api-1', + modules: [], + preloadedBindings: [] + }); + + await expect(runGather(plugin)).resolves.toEqual([]); + expect(withPgClientMock).not.toHaveBeenCalled(); + }); + + it('snapshots nonempty preloaded rows immutably and performs zero SQL', async () => { + const original = binding(); + const plugin = createFunctionBindingsPlugin({ + apiId: 'api-1', + modules: [moduleNames], + preloadedBindings: [original] + }); + + original.alias = 'mutated_after_plugin_creation'; + (original.config!.graphql as { enabled: boolean }).enabled = false; + original.payloadArgs![0].name = 'mutated'; + original.module.invocationsTable = 'mutated_invocations'; + + const loaded = await runGather(plugin); + + expect(withPgClientMock).not.toHaveBeenCalled(); + expect(loaded).toHaveLength(1); + expect(loaded[0]).toMatchObject({ + alias: 'send_email', + payloadArgs: [{ name: 'to', type: 'text' }], + module: { invocationsTable: 'function_invocations' } + }); + expect((loaded[0].config!.graphql as { enabled: boolean }).enabled).toBe(true); + expect(Object.isFrozen(loaded)).toBe(true); + expect(Object.isFrozen(loaded[0])).toBe(true); + expect(Object.isFrozen(loaded[0].config)).toBe(true); + expect(Object.isFrozen(loaded[0].payloadArgs)).toBe(true); + expect(Object.isFrozen(loaded[0].module)).toBe(true); + }); + + it('uses the generic SQL loader only when preloadedBindings is undefined', async () => { + const query = jest.fn().mockResolvedValue({ rows: [] }); + withPgClientMock.mockImplementation( + async (_pgService: unknown, settings: unknown, callback: (client: unknown) => unknown) => { + expect(settings).toBeNull(); + return callback({ query }); + } + ); + const plugin = createFunctionBindingsPlugin({ + apiId: 'api-1', + modules: [moduleNames] + }); + + await expect(runGather(plugin, true)).resolves.toEqual([]); + expect(withPgClientMock).toHaveBeenCalledTimes(1); + expect(query).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphile/graphile-function-bindings/src/index.ts b/graphile/graphile-function-bindings/src/index.ts index dc947b2c6c..663d9cc321 100644 --- a/graphile/graphile-function-bindings/src/index.ts +++ b/graphile/graphile-function-bindings/src/index.ts @@ -14,5 +14,6 @@ export type { FunctionBindingRow, FunctionBindingsPluginOptions, JsonSchemaNode, - PayloadArg + PayloadArg, + PreloadedFunctionBinding } from './types'; diff --git a/graphile/graphile-function-bindings/src/plugin.ts b/graphile/graphile-function-bindings/src/plugin.ts index 0881770327..750b167899 100644 --- a/graphile/graphile-function-bindings/src/plugin.ts +++ b/graphile/graphile-function-bindings/src/plugin.ts @@ -1,12 +1,12 @@ /** * PostGraphile v5 Function Bindings Plugin * - * Exposes API-bound compute functions as GraphQL mutations. At gather time - * the plugin queries the bindings table joined to the definitions table - * (schema/table names resolved from the constructive metaschema via the - * express-context compute module loader — never guessed or hard-coded) - * for the configured api_id and emits one mutation per graphql-enabled - * binding: + * Exposes API-bound compute functions as GraphQL mutations. A Constructive + * server can preload an authoritative control-plane snapshot, avoiding tenant + * runtime-pool metadata queries during gather. Generic callers may omit that + * snapshot and retain the bindings/definitions table query (schema/table names + * resolved from the constructive metaschema — never guessed or hard-coded). + * The plugin emits one mutation per graphql-enabled binding: * * (input: Input!): Payload * @@ -40,7 +40,12 @@ import { toCamelCase, toConstantCase, toPascalCase } from 'inflekt'; import type { DerivedField, DerivedInput } from './derive'; import { buildInvocationPayload, deriveInputFields, isGraphqlEnabled } from './derive'; -import type { ComputeModuleNames, FunctionBindingRow, FunctionBindingsPluginOptions } from './types'; +import type { + ComputeModuleNames, + FunctionBindingRow, + FunctionBindingsPluginOptions, + PreloadedFunctionBinding +} from './types'; const log = new Logger('graphile-function-bindings'); @@ -56,21 +61,63 @@ declare global { } } -/** A binding together with the module (scope) it was loaded from. */ -interface LoadedBinding extends FunctionBindingRow { - module: ComputeModuleNames; +interface FunctionBindingsBuildInput { + bindings: readonly PreloadedFunctionBinding[]; } -interface FunctionBindingsBuildInput { - bindings: LoadedBinding[]; +interface FunctionBindingsPluginOptionsSnapshot { + apiId: string; + modules: readonly ComputeModuleNames[]; + preloadedBindings: readonly PreloadedFunctionBinding[] | undefined; +} + +function deepFreeze(value: T): T { + if (value === null || typeof value !== 'object' || Object.isFrozen(value)) { + return value; + } + Object.freeze(value); + for (const child of Object.values(value as Record)) { + deepFreeze(child); + } + return value; +} + +function snapshotBinding(binding: PreloadedFunctionBinding): PreloadedFunctionBinding { + return deepFreeze({ + bindingId: binding.bindingId, + alias: binding.alias, + config: binding.config === null ? null : structuredClone(binding.config), + functionDefinitionId: binding.functionDefinitionId, + taskIdentifier: binding.taskIdentifier, + description: binding.description, + payloadArgs: binding.payloadArgs === null + ? null + : binding.payloadArgs.map((argument) => ({ ...argument })), + module: { ...binding.module } + }); +} + +function snapshotOptions( + options: FunctionBindingsPluginOptions +): FunctionBindingsPluginOptionsSnapshot { + const preloadedBindings = options.preloadedBindings === undefined + ? undefined + : options.preloadedBindings + .map(snapshotBinding) + .filter((binding) => isGraphqlEnabled(binding.config)); + return deepFreeze({ + apiId: options.apiId, + modules: options.modules.map((module) => ({ ...module })), + preloadedBindings + }); } async function loadBindings( pgService: GraphileConfig.PgServiceConfiguration, - options: FunctionBindingsPluginOptions + options: FunctionBindingsPluginOptionsSnapshot ): Promise { return withPgClientFromPgService(pgService, null, async (client) => { - const bindings: LoadedBinding[] = []; + const bindings: PreloadedFunctionBinding[] = []; for (const module of options.modules) { const { computeSchema, bindingsTable, definitionsTable } = module; const { text, values } = new QueryBuilder() @@ -124,6 +171,7 @@ async function loadBindings( export function createFunctionBindingsPlugin( options: FunctionBindingsPluginOptions ): GraphileConfig.Plugin { + const optionsSnapshot = snapshotOptions(options); return { name: 'FunctionBindingsPlugin', version: '0.1.0', @@ -136,20 +184,25 @@ export function createFunctionBindingsPlugin( namespace: 'functionBindings', helpers: {}, async main(output, info) { - const pgService = info.resolvedPreset.pgServices?.[0]; - if (!pgService) { - throw new Error('FunctionBindingsPlugin: no pgService configured'); - } - if (!options.apiId) { + if (!optionsSnapshot.apiId) { throw new Error('FunctionBindingsPlugin: apiId is required'); } - if (!options.modules?.length) { - throw new Error('FunctionBindingsPlugin: at least one compute module is required'); + let result: FunctionBindingsBuildInput; + if (optionsSnapshot.preloadedBindings !== undefined) { + result = { bindings: optionsSnapshot.preloadedBindings }; + } else { + const pgService = info.resolvedPreset.pgServices?.[0]; + if (!pgService) { + throw new Error('FunctionBindingsPlugin: no pgService configured'); + } + if (optionsSnapshot.modules.length === 0) { + throw new Error('FunctionBindingsPlugin: at least one compute module is required'); + } + result = await loadBindings(pgService, optionsSnapshot); } - const result = await loadBindings(pgService, options); (output as Record).functionApiBindings = result; log.debug( - `Loaded ${result.bindings.length} graphql-enabled function binding(s) for api ${options.apiId}` + `Loaded ${result.bindings.length} graphql-enabled function binding(s) for api ${optionsSnapshot.apiId}` ); } }, diff --git a/graphile/graphile-function-bindings/src/types.ts b/graphile/graphile-function-bindings/src/types.ts index 1fd3576eac..d73c09a216 100644 --- a/graphile/graphile-function-bindings/src/types.ts +++ b/graphile/graphile-function-bindings/src/types.ts @@ -23,7 +23,7 @@ export interface JsonSchemaNode { /** * A graphql-enabled function_api_bindings row joined to its - * function_definitions row, loaded at gather time. + * function_definitions row, either preloaded or loaded at gather time. */ export interface FunctionBindingRow { bindingId: string; @@ -60,6 +60,15 @@ export interface ComputeModuleNames { invocationsEntityField: string | null; } +/** + * A control-plane-resolved binding paired with the exact physical compute + * module used for invocation writes. Supplying these rows lets schema builds + * avoid querying tenant runtime pools for binding metadata. + */ +export interface PreloadedFunctionBinding extends FunctionBindingRow { + module: ComputeModuleNames; +} + export interface FunctionBindingsPluginOptions { /** Only bindings for this api are exposed as mutations. */ apiId: string; @@ -67,5 +76,12 @@ export interface FunctionBindingsPluginOptions { * One entry per provisioned function-module scope. Bindings from every * module are exposed; RLS on the underlying tables governs access. */ - modules: ComputeModuleNames[]; + modules: readonly ComputeModuleNames[]; + /** + * Authoritative control-plane-resolved bindings for this build. When this + * option is defined, including as an empty array, the plugin performs no + * gather-time binding metadata query. Omit it to retain the generic SQL + * loader for callers that do not have a control-plane snapshot. + */ + preloadedBindings?: readonly PreloadedFunctionBinding[]; } diff --git a/graphile/graphile-i18n/package.json b/graphile/graphile-i18n/package.json index 81447e3a9e..95c5cd72d9 100644 --- a/graphile/graphile-i18n/package.json +++ b/graphile/graphile-i18n/package.json @@ -29,6 +29,7 @@ "url": "https://github.com/constructive-io/constructive/issues" }, "dependencies": { + "@pgsql/quotes": "^18.1.0", "accept-language-parser": "^1.5.0" }, "peerDependencies": { diff --git a/graphile/graphile-i18n/src/__tests__/pg-query.test.ts b/graphile/graphile-i18n/src/__tests__/pg-query.test.ts new file mode 100644 index 0000000000..93c8c4f2aa --- /dev/null +++ b/graphile/graphile-i18n/src/__tests__/pg-query.test.ts @@ -0,0 +1,33 @@ +import type { PgClient } from '@dataplan/pg'; + +import { queryI18nRow } from '../pg-query'; + +describe('queryI18nRow', () => { + it('passes one query configuration object to the @dataplan/pg client', async () => { + const query = jest.fn().mockResolvedValue({ + rows: [{ lang_code: 'es', title: 'Hola' }], + rowCount: 1, + notices: [], + }); + const client = { query } as unknown as Pick; + const values = [1, ['es', 'en']]; + + await expect(queryI18nRow(client, 'SELECT $1, $2', values)).resolves.toEqual({ + lang_code: 'es', + title: 'Hola', + }); + expect(query).toHaveBeenCalledTimes(1); + expect(query.mock.calls[0]).toHaveLength(1); + expect(query).toHaveBeenCalledWith({ + text: 'SELECT $1, $2', + values, + }); + }); + + it('returns null when the translation query has no rows', async () => { + const query = jest.fn().mockResolvedValue({ rows: [], rowCount: 0, notices: [] }); + const client = { query } as unknown as Pick; + + await expect(queryI18nRow(client, 'SELECT 1', [])).resolves.toBeNull(); + }); +}); diff --git a/graphile/graphile-i18n/src/__tests__/plugin-isolation.test.ts b/graphile/graphile-i18n/src/__tests__/plugin-isolation.test.ts new file mode 100644 index 0000000000..4e577fdc8c --- /dev/null +++ b/graphile/graphile-i18n/src/__tests__/plugin-isolation.test.ts @@ -0,0 +1,81 @@ +import { + assertI18nRequestContext, + resolveI18nTableInfo, +} from '../plugin'; + +function fixture(duplicateSameSchema = false) { + const idCodec = { name: 'tenant_id', sqlType: { kind: 'tenant_id' } }; + const textCodec = { name: 'text', sqlType: { kind: 'text' } }; + const baseCodec = { + name: 'posts', + attributes: { + id: { codec: idCodec }, + title: { codec: textCodec }, + }, + extensions: { + pg: { serviceName: 'main', schemaName: 'tenant_a', name: 'posts' }, + tags: { i18n: 'posts_translations' }, + }, + }; + const translationCodec = (schemaName: string) => ({ + name: `${schemaName}PostsTranslations`, + attributes: { + posts_id: { codec: idCodec }, + lang_code: { codec: textCodec }, + title: { codec: textCodec, notNull: true }, + }, + extensions: { + pg: { serviceName: 'main', schemaName, name: 'posts_translations' }, + }, + }); + const tenantATranslation = translationCodec('tenant_a'); + const resources: Record = { + base: { + codec: baseCodec, + uniques: [{ isPrimary: true, attributes: ['id'] }], + }, + tenantATranslation: { codec: tenantATranslation }, + tenantBTranslation: { codec: translationCodec('tenant_b') }, + }; + if (duplicateSameSchema) { + resources.duplicateTenantATranslation = { codec: tenantATranslation }; + } + const build = { + input: { pgRegistry: { pgResources: resources } }, + inflection: { camelCase: (value: string) => value }, + sql: { + compile: (value: unknown) => ({ + text: value === idCodec.sqlType ? 'tenant_types.tenant_id' : 'text', + values: [] as unknown[], + }), + }, + }; + return { build, baseCodec }; +} + +describe('i18n exact-build isolation', () => { + it('resolves only the same-service, same-schema translation resource', () => { + const { build, baseCodec } = fixture(); + expect(resolveI18nTableInfo(build, baseCodec as any, 'lang_code', ['text'])) + .toMatchObject({ + schemaName: 'tenant_a', + baseTable: 'posts', + translationTable: 'posts_translations', + pkType: 'tenant_types.tenant_id', + }); + }); + + it('fails when the exact translation coordinate is ambiguous', () => { + const { build, baseCodec } = fixture(true); + expect(() => resolveI18nTableInfo(build, baseCodec as any, 'lang_code', ['text'])) + .toThrow(/matches=2/); + }); + + it.each([ + [undefined, {}, 1, 'I18N_PG_CLIENT_CONTEXT_UNAVAILABLE'], + [jest.fn(), null, 1, 'I18N_PG_SETTINGS_UNAVAILABLE'], + [jest.fn(), {}, undefined, 'I18N_PARENT_ID_UNAVAILABLE'], + ])('fails closed when request context is incomplete', (withPgClient, pgSettings, id, error) => { + expect(() => assertI18nRequestContext(withPgClient, pgSettings, id)).toThrow(error); + }); +}); diff --git a/graphile/graphile-i18n/src/pg-query.ts b/graphile/graphile-i18n/src/pg-query.ts new file mode 100644 index 0000000000..5eba921e2c --- /dev/null +++ b/graphile/graphile-i18n/src/pg-query.ts @@ -0,0 +1,10 @@ +import type { PgClient } from '@dataplan/pg'; + +export async function queryI18nRow( + client: Pick, + text: string, + values: any[] +): Promise | null> { + const { rows } = await client.query>({ text, values }); + return rows[0] ?? null; +} diff --git a/graphile/graphile-i18n/src/plugin.ts b/graphile/graphile-i18n/src/plugin.ts index 0fb1af80d6..1a75736f00 100644 --- a/graphile/graphile-i18n/src/plugin.ts +++ b/graphile/graphile-i18n/src/plugin.ts @@ -20,11 +20,13 @@ import 'graphile-build'; import 'graphile-build-pg'; -import type { PgCodecWithAttributes } from '@dataplan/pg'; +import type { PgClient, PgCodecWithAttributes } from '@dataplan/pg'; import { TYPES } from '@dataplan/pg'; +import { QuoteUtils } from '@pgsql/quotes'; import { context as grafastContext, lambda, object } from 'grafast'; import type { GraphileConfig } from 'graphile-config'; +import { queryI18nRow } from './pg-query'; import type { I18nPluginOptions, I18nTableInfo, TranslatableField } from './types'; // ─── Namespace Augmentations ───────────────────────────────────────────────── @@ -47,15 +49,6 @@ function hasI18nTag(codec: PgCodecWithAttributes): string | false { return false; } -function resolvePgTypeName(codec: any): string { - if (codec === TYPES.uuid) return 'uuid'; - if (codec === TYPES.int) return 'int4'; - if (codec === TYPES.bigint) return 'int8'; - if (codec === TYPES.text) return 'text'; - if (codec === TYPES.varchar) return 'text'; - return codec?.name ?? 'text'; -} - function resolveAttrPgType(codec: any): string { if (codec === TYPES.text) return 'text'; if (codec === TYPES.varchar) return 'text'; @@ -63,6 +56,172 @@ function resolveAttrPgType(codec: any): string { return codec?.name ?? 'text'; } +function resourceIdentity(resource: any, label: string): { + serviceName: string; + schemaName: string; + name: string; +} { + const pg = resource?.codec?.extensions?.pg ?? resource?.extensions?.pg; + if (!pg?.serviceName || !pg?.schemaName || !pg?.name) { + throw new Error(`[graphile-i18n] ${label} is missing exact service/schema/table metadata`); + } + return pg; +} + +function compilePgType(build: any, codec: any, label: string): string { + if (!codec?.sqlType || typeof build?.sql?.compile !== 'function') { + throw new Error(`[graphile-i18n] ${label} has no compilable PostgreSQL type`); + } + const compiled = build.sql.compile(codec.sqlType); + if (!compiled?.text || (compiled.values?.length ?? 0) !== 0) { + throw new Error(`[graphile-i18n] ${label} PostgreSQL type did not compile to a static identifier`); + } + return compiled.text; +} + +/** Resolve one @i18n tag exclusively against this exact build registry. */ +export function resolveI18nTableInfo( + build: any, + codec: PgCodecWithAttributes, + langCodeColumn: string, + allowedTypes: readonly string[] +): I18nTableInfo | null { + const translationTableName = hasI18nTag(codec); + if (!translationTableName) return null; + + const resources = Object.values(build.input?.pgRegistry?.pgResources ?? {}) as any[]; + const baseMatches = resources.filter( + (resource) => !resource?.parameters && resource?.codec === codec + ); + if (baseMatches.length !== 1) { + throw new Error( + `[graphile-i18n] @i18n codec '${codec.name}' must resolve exactly one base resource ` + + `(matches=${baseMatches.length})` + ); + } + const baseResource = baseMatches[0]; + const base = resourceIdentity(baseResource, 'base resource'); + + const primaryKeys = (baseResource.uniques as Array<{ + attributes: string[]; + isPrimary?: boolean; + }> | undefined)?.filter((unique) => unique.isPrimary) ?? []; + if (primaryKeys.length !== 1 || primaryKeys[0].attributes.length !== 1) { + throw new Error( + `[graphile-i18n] @i18n base '${base.schemaName}.${base.name}' requires one ` + + 'single-column primary key' + ); + } + const pkColumn = primaryKeys[0].attributes[0]; + const pkAttr = codec.attributes?.[pkColumn] as any; + if (!pkAttr) { + throw new Error( + `[graphile-i18n] Primary key '${pkColumn}' is missing from ` + + `'${base.schemaName}.${base.name}'` + ); + } + const pkType = compilePgType(build, pkAttr.codec, `${base.schemaName}.${base.name}.${pkColumn}`); + + const translationMatches = resources.filter((resource) => { + if (resource?.parameters || !resource?.codec?.attributes) return false; + const pg = resource.codec.extensions?.pg ?? resource.extensions?.pg; + return pg?.serviceName === base.serviceName && + pg?.schemaName === base.schemaName && + pg?.name === translationTableName; + }); + if (translationMatches.length !== 1) { + throw new Error( + `[graphile-i18n] @i18n on '${base.schemaName}.${base.name}' must resolve exactly ` + + `one same-service, same-schema '${translationTableName}' resource ` + + `(matches=${translationMatches.length})` + ); + } + + const translationResource = translationMatches[0]; + const translation = resourceIdentity(translationResource, 'translation resource'); + const translationCodec = translationResource.codec as PgCodecWithAttributes; + if (!translationCodec.attributes?.[langCodeColumn]) { + throw new Error( + `[graphile-i18n] Translation table '${translation.schemaName}.${translation.name}' ` + + `is missing language column '${langCodeColumn}'` + ); + } + + const conventionalFk = `${base.name}_id`; + const matchingFkColumns = Object.entries(translationCodec.attributes) + .filter(([attrName, attr]) => + attrName !== 'id' && + attrName !== langCodeColumn && + (attr as any).codec === pkAttr.codec + ) + .map(([attrName]) => attrName); + const fkColumn = matchingFkColumns.includes(conventionalFk) + ? conventionalFk + : matchingFkColumns.length === 1 + ? matchingFkColumns[0] + : null; + if (!fkColumn) { + throw new Error( + `[graphile-i18n] Translation table '${translation.schemaName}.${translation.name}' ` + + `has ambiguous or missing FK metadata for '${base.schemaName}.${base.name}'` + ); + } + + const fields: Record = {}; + for (const [attrName, attr] of Object.entries(translationCodec.attributes)) { + if (attrName === langCodeColumn || attrName === fkColumn) continue; + if (attrName === 'id' || attrName === 'created_at' || attrName === 'updated_at') continue; + + const pgType = resolveAttrPgType((attr as any).codec); + if (!allowedTypes.includes(pgType)) continue; + if (!codec.attributes?.[attrName]) { + throw new Error( + `[graphile-i18n] Translation field '${translation.schemaName}.${translation.name}.` + + `${attrName}' has no matching base field on '${base.schemaName}.${base.name}'` + ); + } + + const gqlName = build.inflection.camelCase(attrName); + fields[gqlName] = { + column: attrName, + type: pgType, + isNotNull: !!(attr as any).notNull, + }; + } + if (Object.keys(fields).length === 0) { + throw new Error( + `[graphile-i18n] Translation table '${translation.schemaName}.${translation.name}' ` + + 'has no eligible translatable fields' + ); + } + + return { + baseTable: base.name, + translationTable: translation.name, + schemaName: base.schemaName, + fkColumn, + pkColumn, + pkType, + fields, + }; +} + +export function assertI18nRequestContext( + withPgClient: unknown, + pgSettings: unknown, + id: unknown +): void { + if (typeof withPgClient !== 'function') { + throw new Error('I18N_PG_CLIENT_CONTEXT_UNAVAILABLE'); + } + if (typeof pgSettings !== 'object' || pgSettings === null || Array.isArray(pgSettings)) { + throw new Error('I18N_PG_SETTINGS_UNAVAILABLE'); + } + if (id === null || id === undefined) { + throw new Error('I18N_PARENT_ID_UNAVAILABLE'); + } +} + // ─── Plugin Factory ────────────────────────────────────────────────────────── export function createI18nPlugin(options: I18nPluginOptions = {}): GraphileConfig.Plugin { @@ -74,8 +233,8 @@ export function createI18nPlugin(options: I18nPluginOptions = {}): GraphileConfi } = options; // Closure-scoped state shared between init and field hooks - let i18nRegistry: Record = {}; - const localeTypeCache: Record = {}; + let i18nRegistry = new WeakMap(); + let localeTypeCache: Record = {}; return { name: 'I18nPlugin', @@ -85,119 +244,16 @@ export function createI18nPlugin(options: I18nPluginOptions = {}): GraphileConfi hooks: { init: { callback(_, build) { - i18nRegistry = {}; + i18nRegistry = new WeakMap(); + localeTypeCache = {}; for (const [, codec] of Object.entries(build.input.pgRegistry.pgCodecs)) { const c = codec as PgCodecWithAttributes; if (!c.attributes) continue; - const translationTableName = hasI18nTag(c); - if (!translationTableName) continue; - - // Get schema name from the codec's pg extensions - let schemaName = (c.extensions as any)?.pg?.schemaName ?? 'public'; - let pkColumn: string | null = null; - let pkType = 'text'; - for (const [, resource] of Object.entries(build.input.pgRegistry.pgResources)) { - const r = resource as any; - if (r.codec === c) { - // Try multiple sources for schema name - const rSchema = r.extensions?.pg?.schemaName ?? r.schemaName; - if (rSchema) schemaName = rSchema; - // Extract PK from the resource's uniques array - const uniques = r.uniques as Array<{ attributes: string[]; isPrimary?: boolean }> | undefined; - if (uniques) { - const pk = uniques.find((u: any) => u.isPrimary); - if (pk && pk.attributes.length === 1) { - pkColumn = pk.attributes[0]; - const pkAttr = c.attributes[pkColumn]; - if (pkAttr) { - pkType = resolvePgTypeName((pkAttr as any).codec); - } - } - } - break; - } - } - if (!pkColumn) continue; - - // Find the translation codec. The @i18n tag value is the SQL table name - // (e.g. 'posts_translations'), but PostGraphile inflects codec names - // to camelCase (e.g. 'postsTranslations'). Match via resource name. - let translationCodec: PgCodecWithAttributes | null = null; - for (const [, resource] of Object.entries(build.input.pgRegistry.pgResources)) { - const r = resource as any; - if (!r.codec?.attributes) continue; - // Match by the resource's SQL name (which preserves snake_case) - const sqlName = r.codec?.extensions?.pg?.name ?? r.name; - if (sqlName === translationTableName) { - translationCodec = r.codec as PgCodecWithAttributes; - break; - } - } - // Fallback: try matching the inflected codec name directly - if (!translationCodec) { - const inflectedName = build.inflection.camelCase(translationTableName); - for (const [, tCodec] of Object.entries(build.input.pgRegistry.pgCodecs)) { - const tc = tCodec as any; - if (!tc.attributes) continue; - if (tc.name === translationTableName || tc.name === inflectedName) { - translationCodec = tc; - break; - } - } - } - - if (!translationCodec) continue; - - // Find FK column on translation table — convention first, then type match - let fkColumn: string | null = null; - const conventionalFk = `${c.name}_id`; - if (translationCodec.attributes[conventionalFk]) { - fkColumn = conventionalFk; - } - if (!fkColumn) { - // Fallback: find a column with the same type as the PK, excluding - // common non-FK columns (id, lang_code) - for (const [attrName, attr] of Object.entries(translationCodec.attributes)) { - if (attrName === 'id' || attrName === langCodeColumn) continue; - const a = attr as any; - if (a.codec === (c.attributes[pkColumn] as any).codec) { - fkColumn = attrName; - break; - } - } - } - if (!fkColumn) continue; - - // Discover translatable fields - const fields: Record = {}; - for (const [attrName, attr] of Object.entries(translationCodec.attributes)) { - if (attrName === langCodeColumn || attrName === fkColumn) continue; - if (attrName === 'id' || attrName === 'created_at' || attrName === 'updated_at') continue; - - const pgType = resolveAttrPgType((attr as any).codec); - if (!allowedTypes.includes(pgType)) continue; - - const gqlName = build.inflection.camelCase(attrName); - fields[gqlName] = { - column: attrName, - type: pgType, - isNotNull: !!(attr as any).notNull, - }; - } - - if (Object.keys(fields).length === 0) continue; - - i18nRegistry[c.name] = { - baseTable: c.name, - translationTable: translationTableName, - schemaName, - fkColumn, - pkColumn, - pkType, - fields, - }; + if (!hasI18nTag(c)) continue; + const info = resolveI18nTableInfo(build, c, langCodeColumn, allowedTypes); + if (info) i18nRegistry.set(c, info); } return _; @@ -211,7 +267,7 @@ export function createI18nPlugin(options: I18nPluginOptions = {}): GraphileConfi if (!scope.pgCodec || !scope.isPgClassType) return fields; const codec = scope.pgCodec as PgCodecWithAttributes; - const info = i18nRegistry[codec.name]; + const info = i18nRegistry.get(codec); if (!info) return fields; const localeFieldsConfig: Record = { @@ -235,18 +291,22 @@ export function createI18nPlugin(options: I18nPluginOptions = {}): GraphileConfi const { schemaName, baseTable, translationTable, fkColumn, pkColumn, pkType, fields: i18nFields } = info; + const qi = (name: string): string => QuoteUtils.quoteIdentifier(name); const coalescedCols = Object.values(i18nFields) - .map(f => `coalesce(v."${f.column}", b."${f.column}") as "${f.column}"`) + .map(f => `coalesce(v.${qi(f.column)}, b.${qi(f.column)}) as ${qi(f.column)}`) .join(', '); + const baseTableRef = QuoteUtils.quoteQualifiedIdentifier(schemaName, baseTable); + const translationTableRef = QuoteUtils.quoteQualifiedIdentifier(schemaName, translationTable); + // Build the SQL query template - const sqlQuery = `SELECT v."${langCodeColumn}" AS "lang_code", ${coalescedCols} - FROM "${schemaName}"."${baseTable}" b - LEFT JOIN "${schemaName}"."${translationTable}" v - ON v."${fkColumn}" = b."${pkColumn}" - AND array_position($2::text[], v."${langCodeColumn}") IS NOT NULL - WHERE b."${pkColumn}" = $1::${pkType} - ORDER BY array_position($2::text[], v."${langCodeColumn}") ASC NULLS LAST + const sqlQuery = `SELECT v.${qi(langCodeColumn)} AS "lang_code", ${coalescedCols} + FROM ${baseTableRef} b + LEFT JOIN ${translationTableRef} v + ON v.${qi(fkColumn)} = b.${qi(pkColumn)} + AND array_position($2::text[], v.${qi(langCodeColumn)}) IS NOT NULL + WHERE b.${qi(pkColumn)} = $1::${pkType} + ORDER BY array_position($2::text[], v.${qi(langCodeColumn)}) ASC NULLS LAST LIMIT 1`; // Build column names list for mapping base values @@ -266,31 +326,26 @@ export function createI18nPlugin(options: I18nPluginOptions = {}): GraphileConfi $baseCols[column] = $parent.get(column); } const $withPgClient = (grafastContext() as any).get('withPgClient'); + const $pgSettings = (grafastContext() as any).get('pgSettings'); const $langCodes = (grafastContext() as any).get('langCodes'); // Combine all inputs into a single step const $input = object({ id: $id, withPgClient: $withPgClient, + pgSettings: $pgSettings, langCodes: $langCodes, ...$baseCols, }); return lambda($input, async (input: any) => { - const { id, withPgClient, langCodes: ctxLangCodes, ...baseCols } = input; + const { id, withPgClient, pgSettings, langCodes: ctxLangCodes, ...baseCols } = input; const langs: string[] = ctxLangCodes ?? defaultLanguages; - if (!withPgClient || !id) { - const result: Record = { [langCodeGqlField]: null }; - for (const { gqlName, column } of baseColNames) { - result[gqlName] = baseCols[column] ?? null; - } - return result; - } + assertI18nRequestContext(withPgClient, pgSettings, id); - const row = await withPgClient(null, async (client: any) => { - const { rows } = await client.query(sqlQuery, [id, langs]); - return rows[0] ?? null; + const row = await withPgClient(pgSettings, async (client: PgClient) => { + return queryI18nRow(client, sqlQuery, [id, langs]); }); if (!row) { diff --git a/graphile/graphile-llm/package.json b/graphile/graphile-llm/package.json index 9b754ff0bc..8b13c1c08e 100644 --- a/graphile/graphile-llm/package.json +++ b/graphile/graphile-llm/package.json @@ -32,6 +32,7 @@ "@agentic-kit/ollama": "workspace:*", "@constructive-io/express-context": "workspace:^", "@constructive-io/llm-env": "workspace:^", + "@pgsql/quotes": "^18.1.0", "graphile-cache": "workspace:^" }, "peerDependencies": { diff --git a/graphile/graphile-llm/src/__tests__/agent-discovery.test.ts b/graphile/graphile-llm/src/__tests__/agent-discovery.test.ts new file mode 100644 index 0000000000..85134b42e8 --- /dev/null +++ b/graphile/graphile-llm/src/__tests__/agent-discovery.test.ts @@ -0,0 +1,67 @@ +import { clearAgentDiscoveryCache, getAgentDiscovery } from '../plugins/agent-discovery-plugin'; + +const TENANT_A = '11111111-1111-1111-1111-111111111111'; +const TENANT_B = '22222222-2222-2222-2222-222222222222'; + +function makePool(schemaPrefix = 'tenant') { + const calls: Array<{ text: string; values?: unknown[] }> = []; + const rows: Record = { + [TENANT_A]: { + schema_name: `${schemaPrefix}_a_agent`, + thread_table_name: 'agent_thread', + message_table_name: 'agent_message', + task_table_name: 'agent_task' + }, + [TENANT_B]: { + schema_name: `${schemaPrefix}_b_agent`, + thread_table_name: 'agent_thread', + message_table_name: 'agent_message', + task_table_name: 'agent_task' + } + }; + return { + calls, + query: async (text: string, values?: unknown[]) => { + calls.push({ text, values }); + const row = rows[String(values?.[0])]; + return { rows: row ? [row] : [] }; + } + }; +} + +describe('agent discovery tenant isolation', () => { + beforeEach(() => clearAgentDiscoveryCache()); + + it('filters and caches discovery by database id', async () => { + const pool = makePool(); + const a = await getAgentDiscovery(pool as any, TENANT_A); + const cachedA = await getAgentDiscovery(pool as any, TENANT_A); + const b = await getAgentDiscovery(pool as any, TENANT_B); + + expect(pool.calls).toHaveLength(2); + expect(pool.calls[0].text).toContain('WHERE acm.database_id = $1'); + expect(pool.calls.map((call) => call.values)).toEqual([[TENANT_A], [TENANT_B]]); + expect(a).toEqual(cachedA); + expect(a?.thread?.schemaName).toBe('tenant_a_agent'); + expect(b?.thread?.schemaName).toBe('tenant_b_agent'); + }); + + it('fails closed when database id is absent', async () => { + const pool = makePool(); + await expect(getAgentDiscovery(pool as any, '')).rejects.toThrow(/databaseId is required/); + expect(pool.calls).toHaveLength(0); + }); + + it('does not share discovery across different physical pool identities', async () => { + const poolA = makePool('physical_a'); + const poolB = makePool('physical_b'); + + const a = await getAgentDiscovery(poolA as any, TENANT_A); + const b = await getAgentDiscovery(poolB as any, TENANT_A); + + expect(a?.thread?.schemaName).toBe('physical_a_a_agent'); + expect(b?.thread?.schemaName).toBe('physical_b_a_agent'); + expect(poolA.calls).toHaveLength(1); + expect(poolB.calls).toHaveLength(1); + }); +}); diff --git a/graphile/graphile-llm/src/__tests__/config-cache-isolation.test.ts b/graphile/graphile-llm/src/__tests__/config-cache-isolation.test.ts new file mode 100644 index 0000000000..7cf54f5875 --- /dev/null +++ b/graphile/graphile-llm/src/__tests__/config-cache-isolation.test.ts @@ -0,0 +1,66 @@ +import { + getLlmBillingConfig, + invalidateLlmBillingConfig, +} from '../config-cache'; + +function makeClient(privateSchema: string) { + const query = jest.fn(async (text: string) => { + if (text.includes('information_schema.schemata')) return { rows: [{ exists: 1 }] }; + if (text.includes('billing_module')) { + return { + rows: [{ + public_schema: `${privateSchema}_public`, + private_schema: privateSchema, + record_usage_function: 'record_usage', + }], + }; + } + if (text.includes('inference_log_module')) { + return { rows: [{ schema: privateSchema, table_name: 'usage_log_inference' }] }; + } + throw new Error('unexpected SQL'); + }); + return { query }; +} + +describe('LLM config cache isolation', () => { + beforeEach(() => invalidateLlmBillingConfig()); + + it('keys the same database UUID by opaque exact-build identity', async () => { + const databaseId = '11111111-1111-1111-1111-111111111111'; + const buildA = {}; + const buildB = {}; + const clientA = makeClient('tenant_a_private'); + const clientB = makeClient('tenant_b_private'); + + const firstA = await getLlmBillingConfig(clientA, databaseId, buildA); + const cachedA = await getLlmBillingConfig(clientA, databaseId, buildA); + const firstB = await getLlmBillingConfig(clientB, databaseId, buildB); + + expect(firstA).toBe(cachedA); + expect(firstA.billing?.privateSchema).toBe('tenant_a_private'); + expect(firstB.billing?.privateSchema).toBe('tenant_b_private'); + expect(clientA.query).toHaveBeenCalledTimes(4); + expect(clientB.query).toHaveBeenCalledTimes(4); + }); + + it('does not cache or downgrade incomplete configuration', async () => { + const client = { + query: jest.fn(async (text: string) => { + if (text.includes('information_schema.schemata')) return { rows: [{ exists: 1 }] }; + if (text.includes('billing_module')) { + return { rows: [{ private_schema: 'tenant_private' }] }; + } + return { rows: [] }; + }), + }; + + await expect(getLlmBillingConfig(client, 'database-a', {})) + .rejects.toThrow('LLM_BILLING_CONFIG_INCOMPLETE'); + }); + + it('requires an explicit exact-build cache scope', async () => { + await expect(getLlmBillingConfig(makeClient('tenant_private'), 'database-a', null as any)) + .rejects.toThrow('LLM_CONFIG_CACHE_SCOPE_UNAVAILABLE'); + }); +}); diff --git a/graphile/graphile-llm/src/__tests__/rag-sql.test.ts b/graphile/graphile-llm/src/__tests__/rag-sql.test.ts new file mode 100644 index 0000000000..79e99c0b70 --- /dev/null +++ b/graphile/graphile-llm/src/__tests__/rag-sql.test.ts @@ -0,0 +1,103 @@ +import { buildChunkSearchSql, discoverChunkTables } from '../plugins/rag-plugin'; +import type { ChunkTableInfo } from '../types'; + +const chunkTable = (overrides: Partial = {}): ChunkTableInfo => ({ + parentCodecName: 'articles', + chunksSchema: 'tenant-a-app-public', + vectorSchema: 'tenant-a-extensions', + chunksTableName: 'article_chunks', + parentFkField: 'article_id', + parentPkField: 'id', + embeddingField: 'embedding', + contentField: 'content', + ...overrides +}); + +describe('RAG SQL qualification', () => { + it('quotes tenant schemas and keeps parameter values separate', () => { + const query = buildChunkSearchSql(chunkTable(), '[1,0]', 7, 0.4); + expect(query.text).toContain('FROM "tenant-a-app-public".article_chunks'); + expect(query.text).toContain('$1::"tenant-a-extensions".vector'); + expect(query.text).toContain('OPERATOR("tenant-a-extensions".<=>)'); + expect(query.values).toEqual(['[1,0]', 0.4, 7]); + }); + + it('discovers the exact physical chunks resource and vector type schema', () => { + const vectorCodec = { + name: 'vector', + extensions: { + pg: { serviceName: 'main', schemaName: 'tenant-a-extensions', name: 'vector' } + } + }; + const chunkCodec = { + name: 'articleChunks', + attributes: { + article_id: {}, + content: {}, + embedding: { codec: vectorCodec } + }, + extensions: { + pg: { + serviceName: 'main', + schemaName: 'tenant-a-app-public', + name: 'article_chunks' + } + } + }; + const tables = discoverChunkTables({ + input: { + pgRegistry: { + pgCodecs: { + articles: { + name: 'articles', + attributes: { id: {} }, + extensions: { + pg: { + serviceName: 'main', + schemaName: 'tenant-a-app-public', + name: 'articles' + }, + tags: { + hasChunks: { + chunksTable: 'article_chunks', + parentFk: 'article_id' + } + } + } + } + }, + pgResources: { articleChunks: { codec: chunkCodec } } + } + }, + resolvedPreset: { + pgServices: [{ name: 'main', schemas: ['tenant-a-app-public'] }] + } + }); + expect(tables).toHaveLength(1); + expect(tables[0].chunksSchema).toBe('tenant-a-app-public'); + expect(tables[0].vectorSchema).toBe('tenant-a-extensions'); + }); + + it('rejects a chunks schema outside the exact service allowlist', () => { + expect(() => discoverChunkTables({ + input: { + pgRegistry: { + pgCodecs: { + articles: { + name: 'articles', + attributes: { id: {} }, + extensions: { + pg: { serviceName: 'main', schemaName: 'tenant_a', name: 'articles' }, + tags: { + hasChunks: { chunksSchema: 'tenant_b', chunksTable: 'article_chunks' } + } + } + } + }, + pgResources: {} + } + }, + resolvedPreset: { pgServices: [{ name: 'main', schemas: ['tenant_a'] }] } + })).toThrow(/outside service 'main'/); + }); +}); diff --git a/graphile/graphile-llm/src/config-cache.ts b/graphile/graphile-llm/src/config-cache.ts index c3a5ae82fb..a5d2b1db7d 100644 --- a/graphile/graphile-llm/src/config-cache.ts +++ b/graphile/graphile-llm/src/config-cache.ts @@ -1,7 +1,7 @@ /** * config-cache — Per-database LLM billing configuration cache * - * Caches resolved billing function names per database_id. + * Caches resolved billing function names per exact Graphile build and database_id. * Uses an LRU cache with TTL so config changes propagate within a bounded window * without requiring a server restart. * @@ -103,6 +103,21 @@ const billingCache = new ModuleConfigCache({ max: 50 }); +const billingScopeIds = new WeakMap(); +let nextBillingScopeId = 1; + +function scopedCacheKey(cacheScope: object, databaseId: string): string { + if ((typeof cacheScope !== 'object' && typeof cacheScope !== 'function') || cacheScope === null) { + throw new Error('LLM_CONFIG_CACHE_SCOPE_UNAVAILABLE'); + } + let scopeId = billingScopeIds.get(cacheScope); + if (scopeId === undefined) { + scopeId = nextBillingScopeId++; + billingScopeIds.set(cacheScope, scopeId); + } + return `${scopeId}\0${databaseId}`; +} + // ─── Resolution Functions ─────────────────────────────────────────────────── /** @@ -117,49 +132,46 @@ async function resolveInferenceLogConfig( pgClient: PgClient, databaseId: string ): Promise { - try { - const schemaCheck = await pgClient.query(SCHEMA_EXISTS_SQL, ['metaschema_modules_public']); - if (schemaCheck.rows.length === 0) return null; - - const result = await pgClient.query(INFERENCE_LOG_MODULE_SQL, [databaseId]); - const row = result.rows[0]; - if (!row?.schema || !row?.table_name) return null; - - return { - schema: row.schema as string, - tableName: row.table_name as string - }; - } catch { - return null; + const schemaCheck = await pgClient.query(SCHEMA_EXISTS_SQL, ['metaschema_modules_public']); + if (schemaCheck.rows.length === 0) return null; + + const result = await pgClient.query(INFERENCE_LOG_MODULE_SQL, [databaseId]); + const row = result.rows[0]; + if (!row) return null; + if (!row.schema || !row.table_name) { + throw new Error('LLM_INFERENCE_LOG_CONFIG_INCOMPLETE'); } + + return { + schema: row.schema as string, + tableName: row.table_name as string + }; } async function resolveBillingConfig( pgClient: PgClient, databaseId: string ): Promise { - try { - // Guard: check if the metaschema_modules_public schema exists. - // If the database doesn't have the billing module provisioned, - // this schema (or the billing_module table) won't exist. - const schemaCheck = await pgClient.query(SCHEMA_EXISTS_SQL, ['metaschema_modules_public']); - if (schemaCheck.rows.length === 0) return null; - - const result = await pgClient.query(BILLING_MODULE_SQL, [databaseId]); - const row = result.rows[0]; - if (!row?.record_usage_function) return null; - - return { - publicSchema: row.public_schema as string, - privateSchema: row.private_schema as string, - recordUsageFunction: row.record_usage_function as string, - // The check_billing_quota function name follows the inflection pattern - checkBillingQuotaFunction: 'check_billing_quota' - }; - } catch { - // Schema/table doesn't exist or query failed — billing not available - return null; + // Guard: check if the metaschema_modules_public schema exists. + // If the database doesn't have the billing module provisioned, + // this schema (or the billing_module table) won't exist. + const schemaCheck = await pgClient.query(SCHEMA_EXISTS_SQL, ['metaschema_modules_public']); + if (schemaCheck.rows.length === 0) return null; + + const result = await pgClient.query(BILLING_MODULE_SQL, [databaseId]); + const row = result.rows[0]; + if (!row) return null; + if (!row.public_schema || !row.private_schema || !row.record_usage_function) { + throw new Error('LLM_BILLING_CONFIG_INCOMPLETE'); } + + return { + publicSchema: row.public_schema as string, + privateSchema: row.private_schema as string, + recordUsageFunction: row.record_usage_function as string, + // The check_billing_quota function name follows the inflection pattern + checkBillingQuotaFunction: 'check_billing_quota' + }; } // ─── Public API ───────────────────────────────────────────────────────────── @@ -170,12 +182,16 @@ async function resolveBillingConfig( * * @param pgClient - A client connected to the tenant database (from withPgClient) * @param databaseId - The database UUID + * @param cacheScope - Opaque identity of the exact Graphile build/pool contract */ export async function getLlmBillingConfig( pgClient: PgClient, - databaseId: string + databaseId: string, + cacheScope: object ): Promise { - const cached = billingCache.get(databaseId); + if (!databaseId) throw new Error('LLM_DATABASE_ID_UNAVAILABLE'); + const cacheKey = scopedCacheKey(cacheScope, databaseId); + const cached = billingCache.get(cacheKey); if (cached) return cached; const [billing, inferenceLog] = await Promise.all([ @@ -184,19 +200,21 @@ export async function getLlmBillingConfig( ]); const entry: LlmBillingCacheEntry = { billing, inferenceLog }; - billingCache.set(databaseId, entry); + billingCache.set(cacheKey, entry); return entry; } /** * Invalidate the cached config for a specific database (or all). */ -export function invalidateLlmBillingConfig(databaseId?: string): void { - if (databaseId) { - billingCache.delete(databaseId); - } else { - billingCache.clear(); +export function invalidateLlmBillingConfig(databaseId?: string, cacheScope?: object): void { + if (databaseId && cacheScope) { + billingCache.delete(scopedCacheKey(cacheScope, databaseId)); + return; } + // A database UUID alone is not a safe cache identity. Conservatively clear + // every exact-build entry when the caller cannot provide the matching scope. + billingCache.clear(); } /** diff --git a/graphile/graphile-llm/src/plugins/agent-discovery-plugin.ts b/graphile/graphile-llm/src/plugins/agent-discovery-plugin.ts index 15347c94d7..345c256b0a 100644 --- a/graphile/graphile-llm/src/plugins/agent-discovery-plugin.ts +++ b/graphile/graphile-llm/src/plugins/agent-discovery-plugin.ts @@ -34,14 +34,23 @@ export interface AgentDiscovery { // ─── Cache ────────────────────────────────────────────────────────────────── -const agentDiscoveryCache = new ModuleConfigCache({ - name: 'agent-discovery', - ttlMs: 60_000 -}); +let agentDiscoveryCaches = new WeakMap>(); + +function cacheForPool(pool: object): ModuleConfigCache { + let cache = agentDiscoveryCaches.get(pool); + if (!cache) { + cache = new ModuleConfigCache({ + name: 'agent-discovery', + ttlMs: 60_000 + }); + agentDiscoveryCaches.set(pool, cache); + } + return cache; +} /** Clear all cached discovery results (for testing) */ export function clearAgentDiscoveryCache(): void { - agentDiscoveryCache.clear(); + agentDiscoveryCaches = new WeakMap>(); } // ─── Discovery Query ──────────────────────────────────────────────────────── @@ -53,7 +62,9 @@ const DISCOVERY_SQL = ` acm.message_table_name, acm.task_table_name FROM metaschema_modules_public.agent_chat_module acm - JOIN metaschema_public.schema s ON s.id = acm.schema_id + JOIN metaschema_public.schema s + ON s.id = acm.schema_id + AND s.database_id = acm.database_id WHERE acm.database_id = $1 LIMIT 1 `; @@ -83,6 +94,7 @@ export async function getAgentDiscovery( throw new Error('getAgentDiscovery: databaseId is required'); } + const agentDiscoveryCache = cacheForPool(pool); const cached = agentDiscoveryCache.get(databaseId); if (cached !== undefined) { return cached; diff --git a/graphile/graphile-llm/src/plugins/metering-plugin.ts b/graphile/graphile-llm/src/plugins/metering-plugin.ts index 754f4aabfd..40eb749138 100644 --- a/graphile/graphile-llm/src/plugins/metering-plugin.ts +++ b/graphile/graphile-llm/src/plugins/metering-plugin.ts @@ -26,7 +26,7 @@ * **Graceful behavior:** * - billing_module not provisioned → embedder passes through unmetered * - entity_id not available → embedder passes through unmetered - * - check_billing_quota throws → call is allowed (billing is opt-in) + * - check_billing_quota fails after billing is configured → call is denied * - record_usage throws → call succeeds, recording silently skipped * - quota exceeded → embedder returns null */ @@ -63,7 +63,8 @@ function defaultResolveEntityId(pgSettings: Record): string | nu async function buildMeteringContext( graphqlContext: any, - resolveEntityId: (pgSettings: Record) => string | null + resolveEntityId: (pgSettings: Record) => string | null, + cacheScope: object ): Promise { const pgSettings: Record = graphqlContext?.pgSettings ?? {}; const entityId = resolveEntityId(pgSettings); @@ -73,19 +74,15 @@ async function buildMeteringContext( if (!entityId || !databaseId) return null; const withPgClient: WithPgClient | undefined = graphqlContext?.withPgClient; - if (!withPgClient) return null; + if (!withPgClient) throw new Error('LLM_METERING_PG_CONTEXT_UNAVAILABLE'); let billingConfig = null; let inferenceLogConfig = null; - try { - await withPgClient(pgSettings, async (pgClient: PgClient) => { - const entry = await getLlmBillingConfig(pgClient, databaseId); - billingConfig = entry.billing; - inferenceLogConfig = entry.inferenceLog; - }); - } catch { - return null; - } + await withPgClient(pgSettings, async (pgClient: PgClient) => { + const entry = await getLlmBillingConfig(pgClient, databaseId, cacheScope); + billingConfig = entry.billing; + inferenceLogConfig = entry.inferenceLog; + }); if (!billingConfig) return null; @@ -210,12 +207,17 @@ export function createLlmMeteringPlugin( const defaultResolver = (obj: any) => obj[(context as any).scope.fieldName]; const { resolve: oldResolve = defaultResolver, ...rest } = field; + const exactBuildCacheScope = build as object; return { ...rest, async resolve(source: any, args: any, graphqlContext: any, info: any) { // Build the metering context for this request - const ctx = await buildMeteringContext(graphqlContext, resolveEntityId); + const ctx = await buildMeteringContext( + graphqlContext, + resolveEntityId, + exactBuildCacheScope + ); // Run the original resolver within the AsyncLocalStorage scope // so any embedder calls made by downstream plugins pick up the ctx diff --git a/graphile/graphile-llm/src/plugins/rag-plugin.ts b/graphile/graphile-llm/src/plugins/rag-plugin.ts index 3c1a3e15cf..e9870be67e 100644 --- a/graphile/graphile-llm/src/plugins/rag-plugin.ts +++ b/graphile/graphile-llm/src/plugins/rag-plugin.ts @@ -20,6 +20,7 @@ * 2. Falls back to error if not configured */ +import { QuoteUtils } from '@pgsql/quotes'; import { context as grafastContext, lambda, object } from 'grafast'; import type { GraphileConfig } from 'graphile-config'; import { extendSchema, gql } from 'graphile-utils'; @@ -76,6 +77,7 @@ function parseHasChunksTag(raw: any, codec: any): ChunkTableInfo | null { return { parentCodecName: codec.name || 'unknown', chunksSchema, + vectorSchema: '', chunksTableName: parsed.chunksTable, parentFkField: parsed.parentFk || 'parent_id', parentPkField: parsed.parentPk || 'id', @@ -84,10 +86,60 @@ function parseHasChunksTag(raw: any, codec: any): ChunkTableInfo | null { }; } +function requirePgIdentity(value: any, label: string): { + serviceName: string; + schemaName: string; + name: string; +} { + const pg = value?.extensions?.pg; + if (!pg?.serviceName || !pg?.schemaName || !pg?.name) { + throw new Error(`[graphile-llm] ${label} is missing exact service/schema/table metadata`); + } + return pg; +} + +function configuredSchemas(build: any, serviceName: string): ReadonlySet | null { + const services = build?.resolvedPreset?.pgServices; + if (!Array.isArray(services)) return null; + const matches = services.filter( + (service: any) => (service?.name ?? 'main') === serviceName + ); + if (matches.length !== 1) { + throw new Error( + `[graphile-llm] @hasChunks cannot resolve exact service '${serviceName}' ` + + `(matches=${matches.length})` + ); + } + const service = matches[0]; + const schemas = service?.schemas; + if (!Array.isArray(schemas) || schemas.length === 0) { + throw new Error( + `[graphile-llm] @hasChunks service '${serviceName}' has no configured schema allowlist` + ); + } + const dependencySchemas = service?.introspectionAllowedDependencySchemas; + if (dependencySchemas !== undefined && !Array.isArray(dependencySchemas)) { + throw new Error( + `[graphile-llm] @hasChunks service '${serviceName}' has an invalid dependency schema allowlist` + ); + } + return new Set([...schemas, ...(dependencySchemas ?? [])]); +} + +function requireField(codec: any, fieldName: string, label: string, table: string): any { + const field = codec?.attributes?.[fieldName]; + if (!field) { + throw new Error( + `[graphile-llm] @hasChunks ${label} '${fieldName}' does not exist on '${table}'` + ); + } + return field; +} + /** * Discover all chunk-aware tables from the pgRegistry. */ -function discoverChunkTables(build: any): ChunkTableInfo[] { +export function discoverChunkTables(build: any): ChunkTableInfo[] { const chunkTables: ChunkTableInfo[] = []; const pgRegistry = build.input?.pgRegistry ?? build.pgRegistry; if (!pgRegistry) return chunkTables; @@ -101,9 +153,67 @@ function discoverChunkTables(build: any): ChunkTableInfo[] { if (!tags?.hasChunks) continue; const info = parseHasChunksTag(tags.hasChunks, c); - if (info) { - chunkTables.push(info); + if (!info) { + throw new Error(`[graphile-llm] @hasChunks on '${c.name}' must be a valid JSON object`); + } + + const parent = requirePgIdentity(c, 'parent codec'); + if (!info.chunksSchema) { + throw new Error(`[graphile-llm] @hasChunks on '${parent.name}' has no chunks schema`); + } + const allowedSchemas = configuredSchemas(build, parent.serviceName); + if (allowedSchemas && !allowedSchemas.has(info.chunksSchema)) { + throw new Error( + `[graphile-llm] @hasChunks on '${parent.schemaName}.${parent.name}' references ` + + `schema '${info.chunksSchema}' outside service '${parent.serviceName}'` + ); + } + + const matches = Object.values(pgRegistry.pgResources ?? {}).filter((resource: any) => { + if (resource?.parameters || !resource?.codec?.attributes) return false; + const pg = resource.codec.extensions?.pg; + return pg?.serviceName === parent.serviceName && + pg?.schemaName === info.chunksSchema && + pg?.name === info.chunksTableName; + }) as any[]; + if (matches.length !== 1) { + throw new Error( + `[graphile-llm] @hasChunks on '${parent.schemaName}.${parent.name}' must resolve ` + + `exactly one '${info.chunksSchema}.${info.chunksTableName}' resource ` + + `(matches=${matches.length})` + ); } + + const chunksCodec = matches[0].codec; + const chunks = requirePgIdentity(chunksCodec, 'chunks codec'); + requireField(c, info.parentPkField, 'parentPk', `${parent.schemaName}.${parent.name}`); + requireField(chunksCodec, info.parentFkField, 'parentFk', `${chunks.schemaName}.${chunks.name}`); + requireField(chunksCodec, info.contentField, 'contentField', `${chunks.schemaName}.${chunks.name}`); + const embedding = requireField( + chunksCodec, + info.embeddingField, + 'embeddingField', + `${chunks.schemaName}.${chunks.name}` + ); + const vectorPg = embedding.codec?.extensions?.pg; + if ( + vectorPg?.name !== 'vector' || + vectorPg?.serviceName !== parent.serviceName || + !vectorPg?.schemaName + ) { + throw new Error( + `[graphile-llm] @hasChunks embedding '${chunks.schemaName}.${chunks.name}.` + + `${info.embeddingField}' is not bound to an exact vector type for service ` + + `'${parent.serviceName}'` + ); + } + + chunkTables.push({ + ...info, + chunksSchema: chunks.schemaName, + chunksTableName: chunks.name, + vectorSchema: vectorPg.schemaName, + }); } return chunkTables; @@ -112,37 +222,43 @@ function discoverChunkTables(build: any): ChunkTableInfo[] { /** * Build a SQL query string to search a chunks table for similar embeddings. */ -function buildChunkSearchSql( +export function buildChunkSearchSql( table: ChunkTableInfo, vectorString: string, limit: number, maxDistance: number | null ): { text: string; values: any[] } { - const schema = table.chunksSchema; - const qualifiedTable = schema - ? `"${schema}"."${table.chunksTableName}"` - : `"${table.chunksTableName}"`; - - const embeddingCol = `"${table.embeddingField}"`; - const contentCol = `"${table.contentField}"`; - const parentFkCol = `"${table.parentFkField}"`; + const qualifiedTable = QuoteUtils.quoteQualifiedIdentifier( + table.chunksSchema || null, + table.chunksTableName + ); + + const embeddingCol = QuoteUtils.quoteIdentifier(table.embeddingField); + const contentCol = QuoteUtils.quoteIdentifier(table.contentField); + const parentFkCol = QuoteUtils.quoteIdentifier(table.parentFkField); + if (!table.vectorSchema) { + throw new Error('[graphile-llm] RAG chunk table is missing an exact vector schema'); + } + const vectorType = QuoteUtils.quoteQualifiedIdentifier(table.vectorSchema, 'vector'); + const vectorDistanceOperator = `OPERATOR(${QuoteUtils.quoteIdentifier(table.vectorSchema)}.<=>)`; let text = ` SELECT ${contentCol} AS content, ${parentFkCol}::text AS parent_id, - (${embeddingCol} <=> $1::vector) AS distance + (${embeddingCol} ${vectorDistanceOperator} $1::${vectorType}) AS distance FROM ${qualifiedTable} `; const values: any[] = [vectorString]; if (maxDistance !== null) { - text += ` WHERE (${embeddingCol} <=> $1::vector) <= $2`; + text += ` WHERE (${embeddingCol} ${vectorDistanceOperator} $1::${vectorType}) <= $2`; values.push(maxDistance); } - text += ` ORDER BY ${embeddingCol} <=> $1::vector LIMIT $${values.length + 1}`; + text += ` ORDER BY ${embeddingCol} ${vectorDistanceOperator} $1::${vectorType} ` + + `LIMIT $${values.length + 1}`; values.push(limit); return { text, values }; @@ -174,7 +290,7 @@ export function createLlmRagPlugin( let embedder: EmbedderFunction | null = null; let chatCompleter: ChatFunction | null = null; - const schemaExtension = extendSchema((build) => { + const schemaExtension = extendSchema((_build) => { return { typeDefs: gql` """A source chunk retrieved during RAG context assembly.""" @@ -307,6 +423,12 @@ export function createLlmRagPlugin( }> = []; if (chunkTables.length > 0) { + if (typeof withPgClient !== 'function') { + throw new Error('RAG_PG_CLIENT_CONTEXT_UNAVAILABLE'); + } + if (typeof pgSettings !== 'object' || pgSettings === null) { + throw new Error('RAG_PG_SETTINGS_UNAVAILABLE'); + } await withPgClient(pgSettings, async (pgClient: any) => { for (const table of chunkTables) { const query = buildChunkSearchSql(table, vectorString, limit, maxDistance); diff --git a/graphile/graphile-llm/src/types.ts b/graphile/graphile-llm/src/types.ts index c60d8e6bfe..78441627fa 100644 --- a/graphile/graphile-llm/src/types.ts +++ b/graphile/graphile-llm/src/types.ts @@ -174,6 +174,8 @@ export interface ChunkTableInfo { parentCodecName: string; /** Schema of the chunks table (or null for public/default) */ chunksSchema: string | null; + /** Exact schema containing the pgvector `vector` type for this service */ + vectorSchema: string; /** Name of the chunks table */ chunksTableName: string; /** FK column on chunks table pointing to parent */ diff --git a/graphile/graphile-ltree/src/__tests__/schema-qualified-sql.test.ts b/graphile/graphile-ltree/src/__tests__/schema-qualified-sql.test.ts new file mode 100644 index 0000000000..e6677a52d7 --- /dev/null +++ b/graphile/graphile-ltree/src/__tests__/schema-qualified-sql.test.ts @@ -0,0 +1,176 @@ +import sql from 'pg-sql2'; + +import { createLtreeOperatorFactory } from '../plugins/connection-filter-operators'; +import { + resolveLtreeExtensionInfo, + type LtreeExtensionInfo, +} from '../plugins/detect-ltree'; +import { createFolderOperatorFactory } from '../plugins/folder-filter-operators'; +import { LtreeCodecPlugin } from '../plugins/ltree-codec'; + +const codec = (name: string, schemaName = 'extension_tools') => ({ + name, + extensions: { + pg: { + serviceName: 'tenant_service', + schemaName, + name, + }, + }, +}); + +const helperResource = ( + name: 'to_path' | 'to_query', + returnCodec: any, + schemaName = 'tenant_helpers' +) => ({ + name: `resource_${name}`, + parameters: [{ codec: codec('text', 'pg_catalog') }], + codec: returnCodec, + extensions: { + pg: { + serviceName: 'tenant_service', + schemaName, + name, + }, + }, +}); + +const registryBuild = (options: { + includeHelpers?: boolean; + ltreeCodec?: any; + lqueryCodec?: any; + resources?: Record; +} = {}) => { + const ltreeCodec = options.ltreeCodec ?? codec('ltree'); + const lqueryCodec = options.lqueryCodec ?? codec('lquery'); + const includeHelpers = options.includeHelpers ?? false; + return { + input: { + pgRegistry: { + pgCodecs: { ltree: ltreeCodec, lquery: lqueryCodec }, + pgResources: options.resources ?? (includeHelpers ? { + toPath: helperResource('to_path', ltreeCodec), + toQuery: helperResource('to_query', lqueryCodec), + } : {}), + }, + }, + }; +}; + +const resolveSql = ( + info: LtreeExtensionInfo, + factory: ReturnType, + operatorName: string, + input: string +) => { + const registration = factory({ pgLtreeExtensionInfo: info } as any) + .find((entry) => entry.operatorName === operatorName)!; + const fragment = registration.spec.resolve!( + sql.identifier('path'), + sql.null, + input, + null, + { fieldName: 'path', operatorName } + ); + return sql.compile(fragment!); +}; + +describe('ltree extension identity', () => { + it('qualifies and annotates a native codec from gather introspection', async () => { + const gatherHook = (LtreeCodecPlugin as any).gather.hooks.pgCodecs_findPgCodec; + const event: any = { + pgCodec: { + name: 'ltree', + sqlType: sql.fragment`ltree`, + extensions: undefined, + }, + pgType: { typname: 'ltree', typnamespace: '910', _id: '911' }, + serviceName: 'tenant_service', + }; + const originalCodec = event.pgCodec; + await gatherHook({ + helpers: { + pgIntrospection: { + getNamespace: jest.fn().mockResolvedValue({ nspname: 'extension_tools' }), + }, + }, + }, event); + + expect(event.pgCodec).toBe(originalCodec); + expect(event.pgCodec.extensions).toMatchObject({ + oid: '911', + pg: { + serviceName: 'tenant_service', + schemaName: 'extension_tools', + name: 'ltree', + }, + }); + expect(sql.compile(event.pgCodec.sqlType).text).toBe('"extension_tools"."ltree"'); + }); + + it('derives codec and actual helper schemas from one service/build', () => { + const info = resolveLtreeExtensionInfo(registryBuild({ includeHelpers: true })); + expect(info).toMatchObject({ + serviceName: 'tenant_service', + schemaName: 'extension_tools', + helperSchemaName: 'tenant_helpers', + }); + }); + + it('fails closed on missing codec identity and incomplete helpers', () => { + expect(() => resolveLtreeExtensionInfo(registryBuild({ + ltreeCodec: { name: 'ltree', extensions: { pg: { name: 'ltree' } } }, + }))).toThrow(/missing exact service\/schema metadata/); + + const ltreeCodec = codec('ltree'); + expect(() => resolveLtreeExtensionInfo(registryBuild({ + ltreeCodec, + resources: { + onlyPath: helperResource('to_path', ltreeCodec), + }, + }))).toThrow(/incomplete or ambiguous/); + }); + + it('fails closed when ltree and lquery identities disagree', () => { + expect(() => resolveLtreeExtensionInfo(registryBuild({ + lqueryCodec: codec('lquery', 'other_extension_schema'), + }))).toThrow(/does not match/); + }); +}); + +describe('ltree SQL qualification', () => { + it('qualifies helper functions and operators in the folder factory', () => { + const info = resolveLtreeExtensionInfo(registryBuild({ includeHelpers: true }))!; + const within = resolveSql(info, createFolderOperatorFactory(), 'within', '/a/b'); + const glob = resolveSql(info, createFolderOperatorFactory(), 'glob', '/a/*'); + + expect(within.text).toContain('OPERATOR("extension_tools".<@)'); + expect(within.text).toContain('"tenant_helpers"."to_path"($1)'); + expect(glob.text).toContain('OPERATOR("extension_tools".~)'); + expect(glob.text).toContain('"tenant_helpers"."to_query"($1)'); + }); + + it('qualifies inline casts when helper functions are absent', () => { + const info = resolveLtreeExtensionInfo(registryBuild())!; + const within = resolveSql(info, createFolderOperatorFactory(), 'within', '/a/b'); + const glob = resolveSql(info, createFolderOperatorFactory(), 'glob', '/a/*'); + + expect(within.text).toContain('::"extension_tools"."ltree"'); + expect(within.text).toContain('OPERATOR("extension_tools".<@)'); + expect(glob.text).toContain('::"extension_tools"."lquery"'); + expect(glob.text).toContain('OPERATOR("extension_tools".~)'); + }); + + it('qualifies the deprecated duplicate operator factory too', () => { + const info = resolveLtreeExtensionInfo(registryBuild())!; + const result = resolveSql( + info, + createLtreeOperatorFactory() as ReturnType, + 'isDescendantOf', + '/a/b' + ); + expect(result.text).toContain('OPERATOR("extension_tools".@>)'); + expect(result.text).toContain('::"extension_tools"."ltree"'); + }); +}); diff --git a/graphile/graphile-ltree/src/plugins/connection-filter-operators.ts b/graphile/graphile-ltree/src/plugins/connection-filter-operators.ts index aa2b6114dd..6fca79d787 100644 --- a/graphile/graphile-ltree/src/plugins/connection-filter-operators.ts +++ b/graphile/graphile-ltree/src/plugins/connection-filter-operators.ts @@ -10,35 +10,13 @@ import type { import type { SQL } from 'pg-sql2'; import sql from 'pg-sql2'; +import type { LtreeExtensionInfo } from './detect-ltree'; import { LTREE_SCALAR_NAME } from './ltree-codec'; - -function hasLtreeHelpers(build: any): boolean { - const pgRegistry = build.input?.pgRegistry; - if (!pgRegistry) return false; - for (const resource of Object.values(pgRegistry.pgResources)) { - const r = resource as any; - if (r?.extensions?.pg?.schemaName === 'ltree_helpers') return true; - } - return false; -} - -function toPathExpr(value: SQL, useHelpers: boolean): SQL { - if (useHelpers) { - return sql.fragment`ltree_helpers.to_path(${value})`; - } - return sql.fragment`replace(ltrim(${value}, '/'), '/', '.')::ltree`; -} - -function toQueryExpr(value: SQL, useHelpers: boolean): SQL { - if (useHelpers) { - return sql.fragment`ltree_helpers.to_query(${value})`; - } - // Glob → lquery conversion: - // ** → * (0+ labels in lquery) - // * → *{1} (exactly 1 label) - // We use a placeholder to avoid ** being affected by the * → *{1} step. - return sql.fragment`replace(replace(replace(replace(ltrim(${value}, '/'), '**', '__DSTAR__'), '*', '*{1}'), '__DSTAR__', '*'), '/', '.')::lquery`; -} +import { + ltreeOperatorExpression, + ltreePathExpression, + ltreeQueryExpression, +} from './qualified-sql'; /** * Creates the ltree connection filter operator factory. @@ -55,10 +33,10 @@ function toQueryExpr(value: SQL, useHelpers: boolean): SQL { */ export function createLtreeOperatorFactory(): ConnectionFilterOperatorFactory { return (build) => { - const ltreeInfo = (build as any).pgLtreeExtensionInfo; + const ltreeInfo: LtreeExtensionInfo | undefined = + (build as any).pgLtreeExtensionInfo; if (!ltreeInfo) return []; - const useHelpers = hasLtreeHelpers(build); const registrations: ConnectionFilterOperatorRegistration[] = []; registrations.push({ @@ -78,7 +56,12 @@ export function createLtreeOperatorFactory(): ConnectionFilterOperatorFactory { _details: { fieldName: string | null; operatorName: string } ) { const pathVal = sql.value(String(input)); - return sql.fragment`${sqlIdentifier} <@ ${toPathExpr(pathVal, useHelpers)}`; + return ltreeOperatorExpression( + '<@', + sqlIdentifier, + ltreePathExpression(pathVal, ltreeInfo), + ltreeInfo + ); } } satisfies ConnectionFilterOperatorSpec }); @@ -100,7 +83,12 @@ export function createLtreeOperatorFactory(): ConnectionFilterOperatorFactory { _details: { fieldName: string | null; operatorName: string } ) { const pathVal = sql.value(String(input)); - return sql.fragment`${sqlIdentifier} @> ${toPathExpr(pathVal, useHelpers)}`; + return ltreeOperatorExpression( + '@>', + sqlIdentifier, + ltreePathExpression(pathVal, ltreeInfo), + ltreeInfo + ); } } satisfies ConnectionFilterOperatorSpec }); @@ -122,7 +110,12 @@ export function createLtreeOperatorFactory(): ConnectionFilterOperatorFactory { _details: { fieldName: string | null; operatorName: string } ) { const globVal = sql.value(String(input)); - return sql.fragment`${sqlIdentifier} ~ ${toQueryExpr(globVal, useHelpers)}`; + return ltreeOperatorExpression( + '~', + sqlIdentifier, + ltreeQueryExpression(globVal, ltreeInfo), + ltreeInfo + ); } } satisfies ConnectionFilterOperatorSpec }); diff --git a/graphile/graphile-ltree/src/plugins/detect-ltree.ts b/graphile/graphile-ltree/src/plugins/detect-ltree.ts index 4dd689f473..dff817593f 100644 --- a/graphile/graphile-ltree/src/plugins/detect-ltree.ts +++ b/graphile/graphile-ltree/src/plugins/detect-ltree.ts @@ -5,8 +5,12 @@ import type { PgCodec } from '@dataplan/pg'; import type { GraphileConfig } from 'graphile-config'; export interface LtreeExtensionInfo { + serviceName: string; + schemaName: string; ltreeCodec: PgCodec; lqueryCodec: PgCodec | null; + /** Exact schema containing both validated helper functions, when present. */ + helperSchemaName: string | null; } function isLtreeCodec(codec: any): boolean { @@ -23,6 +27,117 @@ function isLqueryCodec(codec: any): boolean { ); } +function codecIdentity(codec: any, typeName: string): { + serviceName: string; + schemaName: string; +} { + const pg = codec?.extensions?.pg; + if (!pg?.serviceName || !pg?.schemaName) { + throw new Error( + `[graphile-ltree] ${typeName} codec is missing exact service/schema metadata` + ); + } + return { serviceName: pg.serviceName, schemaName: pg.schemaName }; +} + +function helperSchemaName( + pgRegistry: any, + serviceName: string +): string | null { + const matches: Record<'to_path' | 'to_query', any[]> = { + to_path: [], + to_query: [], + }; + + for (const resource of Object.values(pgRegistry.pgResources ?? {}) as any[]) { + if (!Array.isArray(resource?.parameters)) continue; + const pg = resource?.extensions?.pg; + const rawFunctionName = pg?.name ?? resource?.name; + if (rawFunctionName !== 'to_path' && rawFunctionName !== 'to_query') continue; + const functionName: 'to_path' | 'to_query' = rawFunctionName; + + const returnMatches = functionName === 'to_path' + ? isLtreeCodec(resource.codec) + : isLqueryCodec(resource.codec); + const parameter = resource.parameters[0]; + const parameterName = parameter?.codec?.extensions?.pg?.name ?? parameter?.codec?.name; + const signatureMatches = + returnMatches && + resource.parameters.length === 1 && + (parameterName === 'text' || parameterName === 'varchar' || parameterName === 'bpchar'); + if (!signatureMatches) continue; + + if (!pg?.serviceName || !pg?.schemaName) { + throw new Error( + `[graphile-ltree] ${functionName} helper is missing exact service/schema metadata` + ); + } + if (pg.serviceName !== serviceName) continue; + matches[functionName].push(resource); + } + + const pathMatches = matches.to_path; + const queryMatches = matches.to_query; + if (pathMatches.length === 0 && queryMatches.length === 0) return null; + if (pathMatches.length !== 1 || queryMatches.length !== 1) { + throw new Error( + `[graphile-ltree] Helper functions for service '${serviceName}' are incomplete ` + + `or ambiguous (to_path=${pathMatches.length}, to_query=${queryMatches.length})` + ); + } + + const pathSchema = pathMatches[0].extensions.pg.schemaName; + const querySchema = queryMatches[0].extensions.pg.schemaName; + if (pathSchema !== querySchema) { + throw new Error( + `[graphile-ltree] Helper functions for service '${serviceName}' resolve to ` + + `different schemas ('${pathSchema}', '${querySchema}')` + ); + } + return pathSchema; +} + +/** Resolve one unambiguous ltree identity from this exact build registry. */ +export function resolveLtreeExtensionInfo(build: any): LtreeExtensionInfo | undefined { + const pgRegistry = build.input?.pgRegistry; + if (!pgRegistry) return undefined; + + const ltreeCodecs = Object.values(pgRegistry.pgCodecs).filter(isLtreeCodec) as PgCodec[]; + const lqueryCodecs = Object.values(pgRegistry.pgCodecs).filter(isLqueryCodec) as PgCodec[]; + if (ltreeCodecs.length === 0) return undefined; + if (ltreeCodecs.length !== 1) { + throw new Error( + `[graphile-ltree] Expected one ltree codec per build, found ${ltreeCodecs.length}` + ); + } + + const ltreeCodec = ltreeCodecs[0]; + const identity = codecIdentity(ltreeCodec, 'ltree'); + const matchingLquery = lqueryCodecs.filter((codec) => { + const candidate = codecIdentity(codec, 'lquery'); + return candidate.serviceName === identity.serviceName && + candidate.schemaName === identity.schemaName; + }); + if (lqueryCodecs.length > 0 && matchingLquery.length !== lqueryCodecs.length) { + throw new Error( + '[graphile-ltree] lquery codec service/schema does not match the ltree codec' + ); + } + if (matchingLquery.length > 1) { + throw new Error( + `[graphile-ltree] Expected at most one matching lquery codec, found ` + + `${matchingLquery.length}` + ); + } + + return { + ...identity, + ltreeCodec, + lqueryCodec: matchingLquery[0] ?? null, + helperSchemaName: helperSchemaName(pgRegistry, identity.serviceName), + }; +} + /** * LtreeExtensionDetectionPlugin * @@ -40,30 +155,8 @@ export const LtreeExtensionDetectionPlugin: GraphileConfig.Plugin = { schema: { hooks: { build(build) { - const pgRegistry = build.input?.pgRegistry; - if (!pgRegistry) { - return build; - } - - let ltreeCodec: PgCodec | null = null; - let lqueryCodec: PgCodec | null = null; - - for (const codec of Object.values(pgRegistry.pgCodecs)) { - if (isLtreeCodec(codec)) { - ltreeCodec = codec; - } else if (isLqueryCodec(codec)) { - lqueryCodec = codec; - } - } - - if (!ltreeCodec) { - return build; - } - - const ltreeInfo: LtreeExtensionInfo = { - ltreeCodec, - lqueryCodec - }; + const ltreeInfo = resolveLtreeExtensionInfo(build); + if (!ltreeInfo) return build; return build.extend( build, diff --git a/graphile/graphile-ltree/src/plugins/folder-filter-operators.ts b/graphile/graphile-ltree/src/plugins/folder-filter-operators.ts index 88f39c3c83..e58bce645e 100644 --- a/graphile/graphile-ltree/src/plugins/folder-filter-operators.ts +++ b/graphile/graphile-ltree/src/plugins/folder-filter-operators.ts @@ -10,31 +10,13 @@ import type { import type { SQL } from 'pg-sql2'; import sql from 'pg-sql2'; +import type { LtreeExtensionInfo } from './detect-ltree'; import { LTREE_SCALAR_NAME } from './ltree-codec'; - -function hasLtreeHelpers(build: any): boolean { - const pgRegistry = build.input?.pgRegistry; - if (!pgRegistry) return false; - for (const resource of Object.values(pgRegistry.pgResources)) { - const r = resource as any; - if (r?.extensions?.pg?.schemaName === 'ltree_helpers') return true; - } - return false; -} - -function slashToLtree(value: SQL, useHelpers: boolean): SQL { - if (useHelpers) { - return sql.fragment`ltree_helpers.to_path(${value})`; - } - return sql.fragment`replace(ltrim(${value}, '/'), '/', '.')::ltree`; -} - -function slashGlobToLquery(value: SQL, useHelpers: boolean): SQL { - if (useHelpers) { - return sql.fragment`ltree_helpers.to_query(${value})`; - } - return sql.fragment`replace(replace(replace(replace(ltrim(${value}, '/'), '**', '__DSTAR__'), '*', '*{1}'), '__DSTAR__', '*'), '/', '.')::lquery`; -} +import { + ltreeOperatorExpression, + ltreePathExpression, + ltreeQueryExpression, +} from './qualified-sql'; /** * Creates folder-oriented connection filter operators for the LTree scalar. @@ -56,10 +38,10 @@ function slashGlobToLquery(value: SQL, useHelpers: boolean): SQL { */ export function createFolderOperatorFactory(): ConnectionFilterOperatorFactory { return (build) => { - const ltreeInfo = (build as any).pgLtreeExtensionInfo; + const ltreeInfo: LtreeExtensionInfo | undefined = + (build as any).pgLtreeExtensionInfo; if (!ltreeInfo) return []; - const useHelpers = hasLtreeHelpers(build); const registrations: ConnectionFilterOperatorRegistration[] = []; registrations.push({ @@ -79,7 +61,12 @@ export function createFolderOperatorFactory(): ConnectionFilterOperatorFactory { _details: { fieldName: string | null; operatorName: string } ) { const pathVal = sql.value(String(input)); - return sql.fragment`${sqlIdentifier} <@ ${slashToLtree(pathVal, useHelpers)}`; + return ltreeOperatorExpression( + '<@', + sqlIdentifier, + ltreePathExpression(pathVal, ltreeInfo), + ltreeInfo + ); } } satisfies ConnectionFilterOperatorSpec }); @@ -101,7 +88,12 @@ export function createFolderOperatorFactory(): ConnectionFilterOperatorFactory { _details: { fieldName: string | null; operatorName: string } ) { const pathVal = sql.value(String(input)); - return sql.fragment`${sqlIdentifier} @> ${slashToLtree(pathVal, useHelpers)}`; + return ltreeOperatorExpression( + '@>', + sqlIdentifier, + ltreePathExpression(pathVal, ltreeInfo), + ltreeInfo + ); } } satisfies ConnectionFilterOperatorSpec }); @@ -124,7 +116,12 @@ export function createFolderOperatorFactory(): ConnectionFilterOperatorFactory { _details: { fieldName: string | null; operatorName: string } ) { const globVal = sql.value(String(input)); - return sql.fragment`${sqlIdentifier} ~ ${slashGlobToLquery(globVal, useHelpers)}`; + return ltreeOperatorExpression( + '~', + sqlIdentifier, + ltreeQueryExpression(globVal, ltreeInfo), + ltreeInfo + ); } } satisfies ConnectionFilterOperatorSpec }); diff --git a/graphile/graphile-ltree/src/plugins/ltree-codec.ts b/graphile/graphile-ltree/src/plugins/ltree-codec.ts index db39dcd0a9..ce800bc3ac 100644 --- a/graphile/graphile-ltree/src/plugins/ltree-codec.ts +++ b/graphile/graphile-ltree/src/plugins/ltree-codec.ts @@ -56,8 +56,6 @@ export const LtreeCodecPlugin: GraphileConfig.Plugin = { gather: { hooks: { async pgCodecs_findPgCodec(info, event) { - if (event.pgCodec) return; - const { pgType: type, serviceName } = event; const isLtree = type.typname === 'ltree'; @@ -70,7 +68,39 @@ export const LtreeCodecPlugin: GraphileConfig.Plugin = { serviceName, type.typnamespace ); - const schemaName = ns?.nspname || 'pg_catalog'; + if (!ns?.nspname) { + throw new Error( + `[graphile-ltree] Cannot resolve namespace for ${type.typname} ` + + `codec in service '${serviceName}'` + ); + } + const schemaName = ns.nspname; + + if (event.pgCodec) { + const existingPg = event.pgCodec.extensions?.pg; + if ( + (existingPg?.serviceName && existingPg.serviceName !== serviceName) || + (existingPg?.schemaName && existingPg.schemaName !== schemaName) + ) { + throw new Error( + `[graphile-ltree] Existing ${type.typname} codec identity conflicts with ` + + `introspection for service '${serviceName}'` + ); + } + const existingCodec = event.pgCodec as any; + existingCodec.sqlType = sql.identifier(schemaName, type.typname); + existingCodec.extensions = { + ...existingCodec.extensions, + oid: type._id, + pg: { + ...existingPg, + serviceName, + schemaName, + name: type.typname, + }, + }; + return; + } event.pgCodec = { name: type.typname, diff --git a/graphile/graphile-ltree/src/plugins/qualified-sql.ts b/graphile/graphile-ltree/src/plugins/qualified-sql.ts new file mode 100644 index 0000000000..1c1b584d0d --- /dev/null +++ b/graphile/graphile-ltree/src/plugins/qualified-sql.ts @@ -0,0 +1,39 @@ +import type { SQL } from 'pg-sql2'; +import sql from 'pg-sql2'; + +import type { LtreeExtensionInfo } from './detect-ltree'; + +export function ltreePathExpression(value: SQL, info: LtreeExtensionInfo): SQL { + if (info.helperSchemaName) { + const toPath = sql.identifier(info.helperSchemaName, 'to_path'); + return sql.fragment`${toPath}(${value})`; + } + const ltreeType = sql.identifier(info.schemaName, 'ltree'); + return sql.fragment`replace(ltrim(${value}, '/'), '/', '.')::${ltreeType}`; +} + +export function ltreeQueryExpression(value: SQL, info: LtreeExtensionInfo): SQL { + if (info.helperSchemaName) { + const toQuery = sql.identifier(info.helperSchemaName, 'to_query'); + return sql.fragment`${toQuery}(${value})`; + } + const lqueryType = sql.identifier(info.schemaName, 'lquery'); + return sql.fragment`replace(replace(replace(replace(ltrim(${value}, '/'), '**', '__DSTAR__'), '*', '*{1}'), '__DSTAR__', '*'), '/', '.')::${lqueryType}`; +} + +export function ltreeOperatorExpression( + operator: '<@' | '@>' | '~', + left: SQL, + right: SQL, + info: LtreeExtensionInfo +): SQL { + const schema = sql.identifier(info.schemaName); + switch (operator) { + case '<@': + return sql.fragment`${left} OPERATOR(${schema}.<@) ${right}`; + case '@>': + return sql.fragment`${left} OPERATOR(${schema}.@>) ${right}`; + case '~': + return sql.fragment`${left} OPERATOR(${schema}.~) ${right}`; + } +} diff --git a/graphile/graphile-postgis/__tests__/codec.test.ts b/graphile/graphile-postgis/__tests__/codec.test.ts index 4f1e19e145..bd94c15c93 100644 --- a/graphile/graphile-postgis/__tests__/codec.test.ts +++ b/graphile/graphile-postgis/__tests__/codec.test.ts @@ -1,4 +1,5 @@ import type { PgCodec } from '@dataplan/pg'; +import sql from 'pg-sql2'; import { GisSubtype } from '../src/constants'; import { PostgisCodecPlugin } from '../src/plugins/codec'; @@ -27,16 +28,33 @@ describe('PostgisCodecPlugin', () => { const gatherHook = (PostgisCodecPlugin as { gather: { hooks: { pgCodecs_findPgCodec: Function } } }) .gather.hooks.pgCodecs_findPgCodec; - it('should skip if pgCodec is already set', async () => { - const info = { helpers: { pgIntrospection: { getNamespace: jest.fn() } } }; - const event = { pgCodec: { name: 'existing' }, pgType: { typname: 'geometry' }, serviceName: 'main' }; + it('should bind exact identity when a native pgCodec is already set', async () => { + const info = { + helpers: { + pgIntrospection: { + getNamespace: jest.fn().mockResolvedValue({ _id: '123', nspname: 'postgis_ext' }) + } + } + }; + const event = { + pgCodec: { name: 'geometry' } as PgCodec, + pgType: { typname: 'geometry', typnamespace: '123', _id: '456' }, + serviceName: 'main' + }; + const originalCodec = event.pgCodec; await gatherHook(info, event); - // Should not have called getNamespace since pgCodec was already set - expect(info.helpers.pgIntrospection.getNamespace).not.toHaveBeenCalled(); + expect(event.pgCodec).toBe(originalCodec); + expect(info.helpers.pgIntrospection.getNamespace).toHaveBeenCalledWith('main', '123'); + expect(event.pgCodec.extensions?.pg).toEqual({ + serviceName: 'main', + schemaName: 'postgis_ext', + name: 'geometry' + }); + expect(sql.compile(event.pgCodec.sqlType!).text).toBe('"postgis_ext"."geometry"'); }); - it('should skip if namespace is not found', async () => { + it('should fail closed if namespace is not found', async () => { const info = { helpers: { pgIntrospection: { getNamespace: jest.fn().mockResolvedValue(null) } } }; @@ -46,8 +64,9 @@ describe('PostgisCodecPlugin', () => { serviceName: 'main' }; - await gatherHook(info, event); - expect(event.pgCodec).toBeNull(); + await expect(gatherHook(info, event)).rejects.toThrow( + /Cannot resolve namespace for geometry codec/ + ); }); it('should create geometry codec when type is geometry', async () => { diff --git a/graphile/graphile-postgis/__tests__/connection-filter-operators.test.ts b/graphile/graphile-postgis/__tests__/connection-filter-operators.test.ts index 0fe7ed1388..ccb1aba3be 100644 --- a/graphile/graphile-postgis/__tests__/connection-filter-operators.test.ts +++ b/graphile/graphile-postgis/__tests__/connection-filter-operators.test.ts @@ -343,31 +343,31 @@ describe('PostGIS operator factory (createPostgisOperatorFactory)', () => { it('generates correct SQL for = operator', () => { expect(runOp('exactlyEquals').text).toBe( - '"col" = "public"."st_geomfromgeojson"($1::text)' + '"col" OPERATOR("public".=) "public"."st_geomfromgeojson"($1::text)' ); }); it('generates correct SQL for && operator', () => { expect(runOp('bboxIntersects2D').text).toBe( - '"col" && "public"."st_geomfromgeojson"($1::text)' + '"col" OPERATOR("public".&&) "public"."st_geomfromgeojson"($1::text)' ); }); it('generates correct SQL for ~ operator', () => { expect(runOp('bboxContains').text).toBe( - '"col" ~ "public"."st_geomfromgeojson"($1::text)' + '"col" OPERATOR("public".~) "public"."st_geomfromgeojson"($1::text)' ); }); it('generates correct SQL for ~= operator', () => { expect(runOp('bboxEquals').text).toBe( - '"col" ~= "public"."st_geomfromgeojson"($1::text)' + '"col" OPERATOR("public".~=) "public"."st_geomfromgeojson"($1::text)' ); }); it('generates correct SQL for &&& operator', () => { expect(runOp('bboxIntersectsND').text).toBe( - '"col" &&& "public"."st_geomfromgeojson"($1::text)' + '"col" OPERATOR("public".&&&) "public"."st_geomfromgeojson"($1::text)' ); }); }); diff --git a/graphile/graphile-postgis/__tests__/detect-extension.test.ts b/graphile/graphile-postgis/__tests__/detect-extension.test.ts index cd29d9b90c..9416012dbd 100644 --- a/graphile/graphile-postgis/__tests__/detect-extension.test.ts +++ b/graphile/graphile-postgis/__tests__/detect-extension.test.ts @@ -40,7 +40,7 @@ describe('PostgisExtensionDetectionPlugin', () => { it('should detect PostGIS with only geometry codec (no geography)', () => { const geometryCodec = { name: 'geometry', - extensions: { pg: { name: 'geometry', schemaName: 'public' } } + extensions: { pg: { name: 'geometry', schemaName: 'public', serviceName: 'main' } } }; const build = { input: { @@ -57,17 +57,18 @@ describe('PostgisExtensionDetectionPlugin', () => { expect(result.pgGISExtensionInfo).toBeDefined(); expect(result.pgGISExtensionInfo.geometryCodec).toBe(geometryCodec); expect(result.pgGISExtensionInfo.geographyCodec).toBeNull(); + expect(result.pgGISExtensionInfo.serviceName).toBe('main'); expect(result.pgGISExtensionInfo.schemaName).toBe('public'); }); it('should detect PostGIS when both geometry and geography codecs exist', () => { const geometryCodec = { name: 'geometry', - extensions: { pg: { name: 'geometry', schemaName: 'public' } } + extensions: { pg: { name: 'geometry', schemaName: 'public', serviceName: 'main' } } }; const geographyCodec = { name: 'geography', - extensions: { pg: { name: 'geography', schemaName: 'public' } } + extensions: { pg: { name: 'geography', schemaName: 'public', serviceName: 'main' } } }; const build = { @@ -90,11 +91,11 @@ describe('PostgisExtensionDetectionPlugin', () => { it('should detect custom schema for PostGIS installation', () => { const geometryCodec = { name: 'geometry', - extensions: { pg: { name: 'geometry', schemaName: 'postgis' } } + extensions: { pg: { name: 'geometry', schemaName: 'postgis', serviceName: 'main' } } }; const geographyCodec = { name: 'geography', - extensions: { pg: { name: 'geography', schemaName: 'postgis' } } + extensions: { pg: { name: 'geography', schemaName: 'postgis', serviceName: 'main' } } }; const build = { @@ -113,11 +114,11 @@ describe('PostgisExtensionDetectionPlugin', () => { it('should skip codecs without pg extensions', () => { const geometryCodec = { name: 'geometry', - extensions: { pg: { name: 'geometry', schemaName: 'public' } } + extensions: { pg: { name: 'geometry', schemaName: 'public', serviceName: 'main' } } }; const geographyCodec = { name: 'geography', - extensions: { pg: { name: 'geography', schemaName: 'public' } } + extensions: { pg: { name: 'geography', schemaName: 'public', serviceName: 'main' } } }; const otherCodec = { name: 'custom', @@ -140,5 +141,44 @@ describe('PostgisExtensionDetectionPlugin', () => { const result = buildHook(build); expect(result.pgGISExtensionInfo).toBeDefined(); }); + + it('fails closed when codec identity is missing or inconsistent', () => { + const extend = (base: any, ext: any) => ({ ...base, ...ext }); + expect(() => buildHook({ + input: { + pgRegistry: { + pgCodecs: { + geometry: { + name: 'geometry', + extensions: { pg: { name: 'geometry', schemaName: 'postgis' } } + } + } + } + }, + extend + })).toThrow(/missing exact service\/schema metadata/); + + expect(() => buildHook({ + input: { + pgRegistry: { + pgCodecs: { + geometry: { + name: 'geometry', + extensions: { + pg: { name: 'geometry', schemaName: 'postgis_a', serviceName: 'main' } + } + }, + geography: { + name: 'geography', + extensions: { + pg: { name: 'geography', schemaName: 'postgis_b', serviceName: 'main' } + } + } + } + } + }, + extend + })).toThrow(/different service\/schema identities/); + }); }); }); diff --git a/graphile/graphile-postgis/__tests__/spatial-relations.test.ts b/graphile/graphile-postgis/__tests__/spatial-relations.test.ts index 8e29ba7d7b..441337e28f 100644 --- a/graphile/graphile-postgis/__tests__/spatial-relations.test.ts +++ b/graphile/graphile-postgis/__tests__/spatial-relations.test.ts @@ -1,6 +1,7 @@ import sql from 'pg-sql2'; import { + buildSpatialJoinFragment, collectSpatialRelations, OPERATOR_REGISTRY, parseSpatialRelationTag, @@ -130,6 +131,23 @@ describe('OPERATOR_REGISTRY', () => { } } }); + + it('schema-qualifies the PostGIS infix operator in relation SQL', () => { + const fragment = buildSpatialJoinFragment( + { + ownerAttributeName: 'location', + targetAttributeName: 'geom', + operator: OPERATOR_REGISTRY.st_bbox_intersects, + } as any, + 'postgis_ext', + sql.identifier('owner'), + sql.identifier('target'), + null + ); + expect(sql.compile(fragment).text).toBe( + '"owner"."location" OPERATOR("postgis_ext".&&) "target"."geom"' + ); + }); }); // --------------------------------------------------------------------------- diff --git a/graphile/graphile-postgis/src/plugins/codec.ts b/graphile/graphile-postgis/src/plugins/codec.ts index 59dfde1679..cb9b566b87 100644 --- a/graphile/graphile-postgis/src/plugins/codec.ts +++ b/graphile/graphile-postgis/src/plugins/codec.ts @@ -199,12 +199,12 @@ export const PostgisCodecPlugin: GraphileConfig.Plugin = { gather: { hooks: { async pgCodecs_findPgCodec(info, event) { - if (event.pgCodec) { + const { pgType: type, serviceName } = event; + + if (type.typname !== 'geometry' && type.typname !== 'geography') { return; } - const { pgType: type, serviceName } = event; - // Find the namespace for this type by its OID const typeNamespace = await info.helpers.pgIntrospection.getNamespace( serviceName, @@ -212,6 +212,35 @@ export const PostgisCodecPlugin: GraphileConfig.Plugin = { ); if (!typeNamespace) { + throw new Error( + `[graphile-postgis] Cannot resolve namespace for ${type.typname} ` + + `codec in service '${serviceName}'` + ); + } + + if (event.pgCodec) { + const existingPg = event.pgCodec.extensions?.pg; + if ( + (existingPg?.serviceName && existingPg.serviceName !== serviceName) || + (existingPg?.schemaName && existingPg.schemaName !== typeNamespace.nspname) + ) { + throw new Error( + `[graphile-postgis] Existing ${type.typname} codec identity conflicts ` + + `with introspection for service '${serviceName}'` + ); + } + const existingCodec = event.pgCodec as any; + existingCodec.sqlType = sql.identifier(typeNamespace.nspname, type.typname); + existingCodec.extensions = { + ...existingCodec.extensions, + oid: type._id, + pg: { + ...existingPg, + serviceName, + schemaName: typeNamespace.nspname, + name: type.typname, + }, + }; return; } diff --git a/graphile/graphile-postgis/src/plugins/connection-filter-operators.ts b/graphile/graphile-postgis/src/plugins/connection-filter-operators.ts index 5431c81fc3..d94f18f680 100644 --- a/graphile/graphile-postgis/src/plugins/connection-filter-operators.ts +++ b/graphile/graphile-postgis/src/plugins/connection-filter-operators.ts @@ -18,21 +18,22 @@ import type { PostgisExtensionInfo } from './detect-extension'; * Builds an infix operator SQL fragment from a validated operator string. * Uses explicit template literals for each operator to avoid sql.raw. */ -function buildOperatorExpr(op: string, i: SQL, v: SQL): SQL { +function buildOperatorExpr(schemaName: string, op: string, i: SQL, v: SQL): SQL { + const schema = sql.identifier(schemaName); switch (op) { - case '=': return sql.fragment`${i} = ${v}`; - case '&&': return sql.fragment`${i} && ${v}`; - case '&&&': return sql.fragment`${i} &&& ${v}`; - case '&<': return sql.fragment`${i} &< ${v}`; - case '&<|': return sql.fragment`${i} &<| ${v}`; - case '&>': return sql.fragment`${i} &> ${v}`; - case '|&>': return sql.fragment`${i} |&> ${v}`; - case '<<': return sql.fragment`${i} << ${v}`; - case '<<|': return sql.fragment`${i} <<| ${v}`; - case '>>': return sql.fragment`${i} >> ${v}`; - case '|>>': return sql.fragment`${i} |>> ${v}`; - case '~': return sql.fragment`${i} ~ ${v}`; - case '~=': return sql.fragment`${i} ~= ${v}`; + case '=': return sql.fragment`${i} OPERATOR(${schema}.=) ${v}`; + case '&&': return sql.fragment`${i} OPERATOR(${schema}.&&) ${v}`; + case '&&&': return sql.fragment`${i} OPERATOR(${schema}.&&&) ${v}`; + case '&<': return sql.fragment`${i} OPERATOR(${schema}.&<) ${v}`; + case '&<|': return sql.fragment`${i} OPERATOR(${schema}.&<|) ${v}`; + case '&>': return sql.fragment`${i} OPERATOR(${schema}.&>) ${v}`; + case '|&>': return sql.fragment`${i} OPERATOR(${schema}.|&>) ${v}`; + case '<<': return sql.fragment`${i} OPERATOR(${schema}.<<) ${v}`; + case '<<|': return sql.fragment`${i} OPERATOR(${schema}.<<|) ${v}`; + case '>>': return sql.fragment`${i} OPERATOR(${schema}.>>) ${v}`; + case '|>>': return sql.fragment`${i} OPERATOR(${schema}.|>>) ${v}`; + case '~': return sql.fragment`${i} OPERATOR(${schema}.~) ${v}`; + case '~=': return sql.fragment`${i} OPERATOR(${schema}.~=) ${v}`; default: throw new Error(`Unexpected PostGIS SQL operator: ${op}`); } @@ -289,7 +290,8 @@ export function createPostgisOperatorFactory(): ConnectionFilterOperatorFactory operatorName, description, baseType: baseType as 'geometry' | 'geography', - resolve: (i: SQL, v: SQL) => buildOperatorExpr(capturedOp, i, v) + resolve: (i: SQL, v: SQL) => + buildOperatorExpr(schemaName, capturedOp, i, v) }); } } diff --git a/graphile/graphile-postgis/src/plugins/detect-extension.ts b/graphile/graphile-postgis/src/plugins/detect-extension.ts index b91f0ea182..85dba50051 100644 --- a/graphile/graphile-postgis/src/plugins/detect-extension.ts +++ b/graphile/graphile-postgis/src/plugins/detect-extension.ts @@ -8,6 +8,62 @@ import type { PostgisExtensionInfo } from '../types'; export type { PostgisExtensionInfo } from '../types'; +function codecIdentity(codec: any, typeName: string): { + serviceName: string; + schemaName: string; +} { + const pg = codec?.extensions?.pg; + if (!pg?.serviceName || !pg?.schemaName) { + throw new Error( + `[graphile-postgis] ${typeName} codec is missing exact service/schema metadata` + ); + } + return { serviceName: pg.serviceName, schemaName: pg.schemaName }; +} + +/** Resolve one unambiguous PostGIS installation identity for this build. */ +export function resolvePostgisExtensionInfo(build: any): PostgisExtensionInfo | undefined { + const pgRegistry = build.input?.pgRegistry; + if (!pgRegistry) return undefined; + + const geometryCodecs: PgCodec[] = []; + const geographyCodecs: PgCodec[] = []; + for (const codec of Object.values(pgRegistry.pgCodecs) as PgCodec[]) { + const name = codec?.extensions?.pg?.name; + if (name === 'geometry') geometryCodecs.push(codec); + if (name === 'geography') geographyCodecs.push(codec); + } + if (geometryCodecs.length === 0 && geographyCodecs.length === 0) return undefined; + if (geometryCodecs.length > 1 || geographyCodecs.length > 1) { + throw new Error( + `[graphile-postgis] Ambiguous codecs in one build ` + + `(geometry=${geometryCodecs.length}, geography=${geographyCodecs.length})` + ); + } + + const geometryCodec = geometryCodecs[0] ?? null; + const geographyCodec = geographyCodecs[0] ?? null; + const primary = geometryCodec ?? geographyCodec!; + const identity = codecIdentity(primary, geometryCodec ? 'geometry' : 'geography'); + if (geometryCodec && geographyCodec) { + const geographyIdentity = codecIdentity(geographyCodec, 'geography'); + if ( + geographyIdentity.serviceName !== identity.serviceName || + geographyIdentity.schemaName !== identity.schemaName + ) { + throw new Error( + '[graphile-postgis] geometry/geography codecs resolve to different service/schema identities' + ); + } + } + + return { + ...identity, + geometryCodec, + geographyCodec, + }; +} + /** * PostgisExtensionDetectionPlugin * @@ -25,44 +81,8 @@ export const PostgisExtensionDetectionPlugin: GraphileConfig.Plugin = { schema: { hooks: { build(build) { - const pgRegistry = build.input?.pgRegistry; - if (!pgRegistry) { - return build; - } - - let geometryCodec: PgCodec | null = null; - let geographyCodec: PgCodec | null = null; - let schemaName: string = 'public'; - - // Search through codecs for geometry and geography types - for (const codec of Object.values(pgRegistry.pgCodecs)) { - const pg = codec?.extensions?.pg; - if (!pg) continue; - - if (pg.name === 'geometry') { - geometryCodec = codec; - schemaName = pg.schemaName || 'public'; - } else if (pg.name === 'geography') { - geographyCodec = codec; - if (!geometryCodec) { - schemaName = pg.schemaName || 'public'; - } - } - } - - // PostGIS is detected when at least one of geometry or geography - // codecs is present. Some databases use only geography columns - // (e.g. use_geography: true in SearchSpatial), so PostGraphile may - // introspect geography but not geometry. - if (!geometryCodec && !geographyCodec) { - return build; - } - - const postgisInfo: PostgisExtensionInfo = { - schemaName, - geometryCodec, - geographyCodec - }; + const postgisInfo = resolvePostgisExtensionInfo(build); + if (!postgisInfo) return build; return build.extend(build, { pgGISExtensionInfo: postgisInfo, diff --git a/graphile/graphile-postgis/src/plugins/spatial-relations.ts b/graphile/graphile-postgis/src/plugins/spatial-relations.ts index 20adc5132c..68ecb1ba8e 100644 --- a/graphile/graphile-postgis/src/plugins/spatial-relations.ts +++ b/graphile/graphile-postgis/src/plugins/spatial-relations.ts @@ -451,7 +451,7 @@ function spatialFilterTypeName(build: any, rel: SpatialRelationInfo): string { * Build the SQL fragment that joins the inner (target) row to the outer * (owner) row using the resolved PostGIS predicate. */ -function buildSpatialJoinFragment( +export function buildSpatialJoinFragment( rel: SpatialRelationInfo, schemaName: string, outerAlias: SQL, @@ -467,8 +467,8 @@ function buildSpatialJoinFragment( const ownerExpr = sql`${outerAlias}.${sql.identifier(rel.ownerAttributeName)}`; const targetExpr = sql`${innerAlias}.${sql.identifier(rel.targetAttributeName)}`; if (rel.operator.kind === 'infix') { - // Only `&&` today — simple inline (symmetric). - return sql`${ownerExpr} && ${targetExpr}`; + // Only `&&` today. Bind it to this build's exact PostGIS namespace. + return sql`${ownerExpr} OPERATOR(${sql.identifier(schemaName)}.&&) ${targetExpr}`; } const fn = sql.identifier(schemaName, rel.operator.pgToken); if (rel.operator.parametric) { diff --git a/graphile/graphile-postgis/src/types.ts b/graphile/graphile-postgis/src/types.ts index 60defc8ea8..aa56511f48 100644 --- a/graphile/graphile-postgis/src/types.ts +++ b/graphile/graphile-postgis/src/types.ts @@ -22,6 +22,8 @@ export interface GisFieldValue { * PostGIS extension detection result stored on the build object. */ export interface PostgisExtensionInfo { + /** Exact Graphile PostgreSQL service that owns these codecs. */ + serviceName: string; /** The schema name where PostGIS is installed (e.g. 'public') */ schemaName: string; /** The geometry codec from the registry (null if only geography columns are used) */ diff --git a/graphile/graphile-presigned-url-plugin/__tests__/cache-isolation.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/cache-isolation.test.ts new file mode 100644 index 0000000000..46969196ca --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/__tests__/cache-isolation.test.ts @@ -0,0 +1,761 @@ +import { + getBucketConfig, + getStorageModuleCacheScope, + getStorageModuleConfig, + getStorageModuleConfigForOwner, + isS3BucketProvisioned, + loadAllStorageModules, + markS3BucketProvisioned, + resolveStorageConfigFromCodec, +} from '../src/storage-module-cache'; +import { resolveDownloadStorageTarget } from '../src/download-url-field'; +import { mintPhysicalBucketName, resolveS3Config } from '../src/s3-config'; +import { + loadStorageModulesForBuild, + snapshotPreloadedStorageModules, + type StorageWithPgClient, +} from '../src/storage-module-source'; +import type { WithPgClient } from '../src/request-pg-client'; +import type { + PresignedUrlPluginOptions, + StorageModuleConfig, +} from '../src/types'; + +const DATABASE_ID = '00000000-0000-0000-0000-000000000001'; +const MODULE_ID = '00000000-0000-0000-0000-000000000002'; +const BUCKET_ID = '00000000-0000-0000-0000-000000000003'; +type QueryOptions = { text: string; values?: unknown[] }; + +/** + * Model Graphile's real request-client contract: acquire without settings, + * open one explicit transaction, then apply every request GUC with set_config + * before tenant-bound SQL is allowed to run. + */ +function requestPgHarness( + pgSettings: Record, + requestQuery: (query: QueryOptions) => Promise<{ rows: any[] }>, +) { + let transactionActive = false; + let appliedSettings: Record | null = null; + const pgClient: any = {}; + + const query = jest.fn(async (queryOptions: QueryOptions) => { + if (queryOptions.text.includes('SELECT set_config')) { + expect(transactionActive).toBe(true); + const entries = JSON.parse(String(queryOptions.values?.[0])) as Array<[string, string]>; + appliedSettings = Object.fromEntries(entries); + expect(appliedSettings).toEqual(pgSettings); + return { rows: [] }; + } + + expect(transactionActive).toBe(true); + expect(appliedSettings).toEqual(pgSettings); + return requestQuery(queryOptions); + }); + const withTransaction = jest.fn(async (callback: (tx: any) => Promise) => { + expect(transactionActive).toBe(false); + transactionActive = true; + try { + return await callback(pgClient); + } finally { + appliedSettings = null; + transactionActive = false; + } + }); + pgClient.query = query; + pgClient.withTransaction = withTransaction; + + const withPgClientMock = jest.fn(async ( + settings: Record | null, + callback: (client: any) => Promise, + ) => { + expect(settings).toBeNull(); + return callback(pgClient); + }); + + return { + withPgClient: withPgClientMock as unknown as WithPgClient & StorageWithPgClient, + withPgClientMock, + withTransaction, + query, + }; +} + +function storageRow(publicUrlPrefix: string, maxFileSize = 1024): Record { + return { + id: MODULE_ID, + database_id: DATABASE_ID, + scope: 'app', + entity_table_id: null, + buckets_database_id: DATABASE_ID, + buckets_schema: 'storage_public', + buckets_schema_database_id: DATABASE_ID, + buckets_table: 'app_buckets', + files_database_id: DATABASE_ID, + files_schema: 'storage_public', + files_schema_database_id: DATABASE_ID, + files_table: 'app_files', + endpoint: null, + public_url_prefix: publicUrlPrefix, + provider: 's3', + allowed_origins: null, + upload_url_expiry_seconds: 900, + download_url_expiry_seconds: 3600, + default_max_file_size: maxFileSize, + max_filename_length: 1024, + cache_ttl_seconds: 3600, + max_bulk_files: 100, + max_bulk_total_size: 1073741824, + has_path_shares: false, + entity_database_id: null, + entity_schema_database_id: null, + entity_schema: null, + entity_table: null, + }; +} + +function bucketRow(maxFileSize: number): Record { + return { + id: BUCKET_ID, + key: 'private', + type: 'private', + is_public: false, + owner_id: null, + allowed_mime_types: ['application/pdf'], + max_file_size: maxFileSize, + allow_custom_keys: false, + physical_name: 'persisted-test-bucket', + }; +} + +function options(bucket: string): PresignedUrlPluginOptions { + return { + s3: { + client: {} as any, + bucket, + publicUrlPrefix: `https://${bucket}.example`, + }, + }; +} + +function preloadedConfig(publicUrlPrefix: string): StorageModuleConfig { + return { + id: MODULE_ID, + bucketsQualifiedName: 'storage_public.app_buckets', + filesQualifiedName: 'storage_public.app_files', + schemaName: 'storage_public', + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + scope: 'app', + entityTableId: null, + entityQualifiedName: null, + endpoint: null, + publicUrlPrefix, + provider: 's3', + allowedOrigins: ['https://app.example'], + uploadUrlExpirySeconds: 900, + downloadUrlExpirySeconds: 1234, + defaultMaxFileSize: 1024, + maxFilenameLength: 1024, + cacheTtlSeconds: 3600, + hasPathShares: false, + maxBulkFiles: 100, + maxBulkTotalSize: 1073741824, + }; +} + +describe('build-local storage metadata caches', () => { + it('uses fixed-expiry caches rather than extending metadata lifetime on reads', () => { + const scope = getStorageModuleCacheScope({}); + expect(scope.storageModuleCache.updateAgeOnGet).toBe(false); + expect(scope.bucketCache.updateAgeOnGet).toBe(false); + }); + + it('binds every legacy metadata join to the requested physical database', async () => { + const scope = getStorageModuleCacheScope({}); + const client = { + query: jest.fn(async (_query: QueryOptions) => ({ + rows: [storageRow('https://tenant.example')], + })), + }; + + await getStorageModuleConfig(client, DATABASE_ID, scope); + + const query = client.query.mock.calls[0][0] as QueryOptions; + expect(query.values).toEqual([DATABASE_ID]); + expect(query.text).toContain('bt.database_id = sm.database_id'); + expect(query.text).toContain('bs.database_id = sm.database_id'); + expect(query.text).toContain('ft.database_id = sm.database_id'); + expect(query.text).toContain('fs.database_id = sm.database_id'); + expect(query.text).not.toContain('LIMIT 1'); + }); + + it('rejects wrong-database metadata rows and never caches them', async () => { + const scope = getStorageModuleCacheScope({}); + const client = { + query: jest.fn(async () => ({ + rows: [{ + ...storageRow('https://tenant.example'), + buckets_schema_database_id: '00000000-0000-0000-0000-000000000099', + }], + })), + }; + + await expect(getStorageModuleConfig(client, DATABASE_ID, scope)) + .rejects.toThrow(`STORAGE_MODULE_CROSS_DATABASE_METADATA:${MODULE_ID}`); + await expect(getStorageModuleConfig(client, DATABASE_ID, scope)) + .rejects.toThrow(`STORAGE_MODULE_CROSS_DATABASE_METADATA:${MODULE_ID}`); + expect(client.query).toHaveBeenCalledTimes(2); + }); + + it('rejects duplicate app metadata instead of selecting the first row', async () => { + const scope = getStorageModuleCacheScope({}); + const client = { + query: jest.fn(async () => ({ + rows: [ + storageRow('https://tenant-a.example'), + { ...storageRow('https://tenant-b.example'), id: 'module-b' }, + ], + })), + }; + + await expect(getStorageModuleConfig(client, DATABASE_ID, scope)) + .rejects.toThrow('STORAGE_MODULE_METADATA_AMBIGUOUS:app'); + }); + + it('rejects a codec mapped by more than one preloaded module', () => { + const first = preloadedConfig('https://tenant-a.example'); + const second = { ...preloadedConfig('https://tenant-b.example'), id: 'module-b' }; + + expect(() => resolveStorageConfigFromCodec({ + name: 'app_files', + extensions: { pg: { schemaName: 'storage_public', name: 'app_files' } }, + }, [first, second])).toThrow('STORAGE_MODULE_AMBIGUOUS:codec'); + }); + + it('isolates identical logical IDs and object names by exact Graphile build', async () => { + const firstBuild = {}; + const secondBuild = {}; + const firstScope = getStorageModuleCacheScope(firstBuild); + const secondScope = getStorageModuleCacheScope(secondBuild); + + expect(getStorageModuleCacheScope(firstBuild)).toBe(firstScope); + expect(secondScope).not.toBe(firstScope); + + const firstClient = { + query: jest.fn(async ({ text }: { text: string }) => ({ + rows: text.includes('metaschema_modules_public.storage_module') + ? [storageRow('https://tenant-a.example', 111)] + : /SELECT id\s+FROM/.test(text) + ? [{ id: BUCKET_ID }] + : [bucketRow(111)], + })), + }; + const secondClient = { + query: jest.fn(async ({ text }: { text: string }) => ({ + rows: text.includes('metaschema_modules_public.storage_module') + ? [storageRow('https://tenant-b.example', 222)] + : /SELECT id\s+FROM/.test(text) + ? [{ id: BUCKET_ID }] + : [bucketRow(222)], + })), + }; + + const [firstModules, secondModules] = await Promise.all([ + loadAllStorageModules(firstClient, DATABASE_ID, firstScope), + loadAllStorageModules(secondClient, DATABASE_ID, secondScope), + ]); + + expect(firstModules[0]).toMatchObject({ + id: MODULE_ID, + bucketsQualifiedName: 'storage_public.app_buckets', + publicUrlPrefix: 'https://tenant-a.example', + defaultMaxFileSize: 111, + }); + expect(secondModules[0]).toMatchObject({ + id: MODULE_ID, + bucketsQualifiedName: 'storage_public.app_buckets', + publicUrlPrefix: 'https://tenant-b.example', + defaultMaxFileSize: 222, + }); + + const firstBucket = await getBucketConfig( + firstClient, + firstModules[0], + DATABASE_ID, + 'private', + undefined, + firstScope, + ); + const secondBucket = await getBucketConfig( + secondClient, + secondModules[0], + DATABASE_ID, + 'private', + undefined, + secondScope, + ); + + expect(firstBucket).toMatchObject({ id: BUCKET_ID, key: 'private', max_file_size: 111 }); + expect(secondBucket).toMatchObject({ id: BUCKET_ID, key: 'private', max_file_size: 222 }); + + markS3BucketProvisioned('same-physical-bucket-name', firstScope); + expect(isS3BucketProvisioned('same-physical-bucket-name', firstScope)).toBe(true); + expect(isS3BucketProvisioned('same-physical-bucket-name', secondScope)).toBe(false); + }); + + it('negative-caches metadata inside one build without poisoning another build', async () => { + const missingScope = getStorageModuleCacheScope({}); + const presentScope = getStorageModuleCacheScope({}); + const missingClient = { + query: jest.fn(async (_opts: QueryOptions): Promise<{ rows: unknown[] }> => ({ rows: [] })), + }; + const presentClient = { + query: jest.fn(async () => ({ rows: [storageRow('https://present.example')] })), + }; + + await expect(getStorageModuleConfig(missingClient, DATABASE_ID, missingScope)).resolves.toBeNull(); + await expect(getStorageModuleConfig(missingClient, DATABASE_ID, missingScope)).resolves.toBeNull(); + expect(missingClient.query).toHaveBeenCalledTimes(1); + + await expect(getStorageModuleConfig(presentClient, DATABASE_ID, presentScope)).resolves.toMatchObject({ + id: MODULE_ID, + publicUrlPrefix: 'https://present.example', + }); + expect(presentClient.query).toHaveBeenCalledTimes(1); + }); + + it('negative-caches bucket metadata but rechecks RLS on every cache hit', async () => { + const scope = getStorageModuleCacheScope({}); + const moduleClient = { + query: jest.fn(async () => ({ rows: [storageRow('https://tenant.example')] })), + }; + const [storageConfig] = await loadAllStorageModules(moduleClient, DATABASE_ID, scope); + const bucketClient = { + query: jest.fn(async (_opts: QueryOptions): Promise<{ rows: unknown[] }> => ({ rows: [] })), + }; + + await expect( + getBucketConfig(bucketClient, storageConfig, DATABASE_ID, 'private', undefined, scope), + ).resolves.toBeNull(); + await expect( + getBucketConfig(bucketClient, storageConfig, DATABASE_ID, 'private', undefined, scope), + ).resolves.toBeNull(); + + expect(bucketClient.query).toHaveBeenCalledTimes(2); + const calls = bucketClient.query.mock.calls as unknown as Array<[QueryOptions]>; + expect(calls[0][0].text).toContain('allowed_mime_types'); + expect(calls[1][0].text).toMatch(/SELECT id\s+FROM/); + }); + + it('does not return a positive bucket cache hit when the current RLS context cannot see it', async () => { + const scope = getStorageModuleCacheScope({}); + const moduleClient = { + query: jest.fn(async () => ({ rows: [storageRow('https://tenant.example')] })), + }; + const [storageConfig] = await loadAllStorageModules(moduleClient, DATABASE_ID, scope); + const authorizedClient = { + query: jest.fn(async () => ({ rows: [bucketRow(512)] })), + }; + const unauthorizedClient = { + query: jest.fn(async (_opts: QueryOptions): Promise<{ rows: unknown[] }> => ({ rows: [] })), + }; + + await expect( + getBucketConfig(authorizedClient, storageConfig, DATABASE_ID, 'private', undefined, scope), + ).resolves.toMatchObject({ id: BUCKET_ID, max_file_size: 512 }); + await expect( + getBucketConfig(unauthorizedClient, storageConfig, DATABASE_ID, 'private', undefined, scope), + ).resolves.toBeNull(); + + const calls = unauthorizedClient.query.mock.calls as unknown as Array<[QueryOptions]>; + expect(calls).toHaveLength(1); + expect(calls[0][0].text).toMatch(/SELECT id\s+FROM/); + }); + + it('memoizes lazy S3 configuration per build instead of on shared preset options', () => { + const firstScope = getStorageModuleCacheScope({}); + const secondScope = getStorageModuleCacheScope({}); + let activeBucket = 'tenant-a-bucket'; + const getter = jest.fn(() => ({ + client: {} as any, + bucket: activeBucket, + })); + const sharedOptions: PresignedUrlPluginOptions = { s3: getter }; + + const first = resolveS3Config(sharedOptions, firstScope); + activeBucket = 'tenant-b-bucket'; + const second = resolveS3Config(sharedOptions, secondScope); + activeBucket = 'unrelated-later-value'; + + expect(resolveS3Config(sharedOptions, firstScope)).toBe(first); + expect(resolveS3Config(sharedOptions, secondScope)).toBe(second); + expect(first.bucket).toBe('tenant-a-bucket'); + expect(second.bucket).toBe('tenant-b-bucket'); + expect(getter).toHaveBeenCalledTimes(2); + expect(typeof sharedOptions.s3).toBe('function'); + }); + + it('matches the bucket provisioner resolver argument order on first provision', () => { + const cacheScope = getStorageModuleCacheScope({}); + const resolver = jest.fn((bucketKey: string, databaseId: string) => + `physical-${bucketKey}-${databaseId}`, + ); + + expect(mintPhysicalBucketName( + { ...options('fallback'), resolveBucketName: resolver }, + 'private', + DATABASE_ID, + cacheScope, + )).toBe(`physical-private-${DATABASE_ID}`); + expect(resolver).toHaveBeenCalledWith('private', DATABASE_ID); + }); + + it('does not cache an owner lookup across RLS principals', async () => { + const scope = getStorageModuleCacheScope({}); + const entityRow = { + ...storageRow('https://tenant.example'), + scope: 'team', + entity_table_id: '00000000-0000-0000-0000-000000000004', + entity_database_id: DATABASE_ID, + entity_schema_database_id: DATABASE_ID, + entity_schema: 'app_public', + entity_table: 'teams', + }; + const authorizedClient = { + query: jest.fn(async ({ text }: QueryOptions) => ({ + rows: text.includes('metaschema_modules_public.storage_module') + ? [entityRow] + : [{ '?column?': 1 }], + })), + }; + const unauthorizedClient = { + query: jest.fn(async (_opts: QueryOptions): Promise<{ rows: unknown[] }> => ({ rows: [] })), + }; + + await expect( + getStorageModuleConfigForOwner(authorizedClient, DATABASE_ID, 'owner-id', scope), + ).resolves.toMatchObject({ id: MODULE_ID, scope: 'team' }); + await expect( + getStorageModuleConfigForOwner(unauthorizedClient, DATABASE_ID, 'owner-id', scope), + ).resolves.toBeNull(); + + expect(unauthorizedClient.query).toHaveBeenCalledTimes(1); + const calls = unauthorizedClient.query.mock.calls as unknown as Array<[QueryOptions]>; + expect(calls[0][0].text).toContain('WHERE id = $1'); + }); +}); + +describe('download target fail-closed resolution', () => { + const codec = { + name: 'app_files', + extensions: { pg: { schemaName: 'storage_public', name: 'app_files' } }, + }; + + it('keeps identical logical tenants on their own build configuration', async () => { + async function resolveForBuild( + build: object, + publicUrlPrefix: string, + s3Bucket: string, + ) { + const pgSettings = { + role: 'member', + 'jwt.claims.api_id': 'api-a', + 'jwt.claims.database_id': DATABASE_ID, + 'jwt.claims.user_id': '', + }; + const requestQuery = jest.fn(async ({ text }: QueryOptions) => { + if (text.includes('current_database_id')) return { rows: [{ id: DATABASE_ID }] }; + if (text.includes('metaschema_modules_public.storage_module')) { + return { rows: [storageRow(publicUrlPrefix)] }; + } + if (text.includes('SELECT key, physical_name FROM')) { + return { rows: [{ key: 'private', physical_name: s3Bucket }] }; + } + throw new Error(`Unexpected query: ${text}`); + }); + const { withPgClient } = requestPgHarness(pgSettings, requestQuery); + + return resolveDownloadStorageTarget({ + options: options(s3Bucket), + preloadedStorageModules: undefined, + cacheScope: getStorageModuleCacheScope(build), + codec, + withPgClient, + pgSettings, + bucketId: BUCKET_ID, + }); + } + + const [first, second] = await Promise.all([ + resolveForBuild({}, 'https://tenant-a.example', 'tenant-a-bucket'), + resolveForBuild({}, 'https://tenant-b.example', 'tenant-b-bucket'), + ]); + + expect(first.s3).toMatchObject({ + bucket: 'tenant-a-bucket', + publicUrlPrefix: 'https://tenant-a.example', + }); + expect(second.s3).toMatchObject({ + bucket: 'tenant-b-bucket', + publicUrlPrefix: 'https://tenant-b.example', + }); + }); + + it('propagates lookup failures and never resolves fallback signing config', async () => { + const pgSettings = { + role: 'member', + 'jwt.claims.api_id': 'api-a', + 'jwt.claims.database_id': DATABASE_ID, + 'jwt.claims.user_id': '', + }; + const s3Getter = jest.fn(() => ({ + client: {} as any, + bucket: 'unsafe-global-fallback', + })); + const requestQuery = jest.fn(async ({ text }: QueryOptions) => { + if (text.includes('current_database_id')) return { rows: [{ id: DATABASE_ID }] }; + throw new Error('tenant metadata lookup failed'); + }); + const { withPgClient } = requestPgHarness(pgSettings, requestQuery); + + await expect(resolveDownloadStorageTarget({ + options: { s3: s3Getter }, + preloadedStorageModules: undefined, + cacheScope: getStorageModuleCacheScope({}), + codec, + withPgClient, + pgSettings, + bucketId: BUCKET_ID, + })).rejects.toThrow('tenant metadata lookup failed'); + expect(s3Getter).not.toHaveBeenCalled(); + }); + + it('does not consult global signing config when tenant metadata is absent', async () => { + const pgSettings = { + role: 'member', + 'jwt.claims.api_id': 'api-a', + 'jwt.claims.database_id': DATABASE_ID, + 'jwt.claims.user_id': '', + }; + const s3Getter = jest.fn(() => ({ + client: {} as any, + bucket: 'unsafe-global-fallback', + })); + const requestQuery = jest.fn(async ({ text }: QueryOptions) => { + if (text.includes('current_database_id')) return { rows: [{ id: DATABASE_ID }] }; + if (text.includes('metaschema_modules_public.storage_module')) return { rows: [] }; + throw new Error(`Unexpected query: ${text}`); + }); + const { withPgClient } = requestPgHarness(pgSettings, requestQuery); + + await expect(resolveDownloadStorageTarget({ + options: { s3: s3Getter }, + preloadedStorageModules: undefined, + cacheScope: getStorageModuleCacheScope({}), + codec, + withPgClient, + pgSettings, + bucketId: BUCKET_ID, + })).rejects.toThrow('STORAGE_MODULE_NOT_FOUND'); + expect(s3Getter).not.toHaveBeenCalled(); + }); + + it('rejects missing request settings before acquiring a metadata client or signing', async () => { + const s3Getter = jest.fn(() => ({ + client: {} as any, + bucket: 'unsafe-global-fallback', + })); + const withPgClient = jest.fn(); + + await expect(resolveDownloadStorageTarget({ + options: { s3: s3Getter }, + preloadedStorageModules: undefined, + cacheScope: getStorageModuleCacheScope({}), + codec, + withPgClient, + pgSettings: null, + bucketId: BUCKET_ID, + })).rejects.toThrow('STORAGE_REQUEST_SETTINGS_UNAVAILABLE'); + expect(withPgClient).not.toHaveBeenCalled(); + expect(s3Getter).not.toHaveBeenCalled(); + }); + + it('uses persisted physical_name verbatim without consulting the naming resolver', async () => { + const resolver = jest.fn(() => 'recomputed-and-wrong'); + const pgSettings = { role: 'member' }; + const requestQuery = jest.fn(async ({ text }: QueryOptions) => { + if (text.includes('current_database_id')) return { rows: [{ id: DATABASE_ID }] }; + if (text.includes('SELECT key, physical_name FROM')) { + return { rows: [{ key: 'private', physical_name: 'persisted-physical-bucket' }] }; + } + throw new Error(`Unexpected query: ${text}`); + }); + const { withPgClient } = requestPgHarness(pgSettings, requestQuery); + + await expect(resolveDownloadStorageTarget({ + options: { + ...options('global-fallback'), + resolveBucketName: resolver, + }, + preloadedStorageModules: snapshotPreloadedStorageModules([ + preloadedConfig('https://tenant.example'), + ]), + cacheScope: getStorageModuleCacheScope({}), + codec, + withPgClient, + pgSettings, + bucketId: BUCKET_ID, + })).resolves.toMatchObject({ + s3: { bucket: 'persisted-physical-bucket' }, + }); + expect(resolver).not.toHaveBeenCalled(); + }); +}); + +describe('preloaded storage-module configuration', () => { + const codec = { + name: 'app_files', + extensions: { pg: { schemaName: 'storage_public', name: 'app_files' } }, + }; + + it('takes an immutable snapshot and performs zero metadata SQL', async () => { + const source = [preloadedConfig('https://tenant.example')]; + const snapshot = snapshotPreloadedStorageModules(source)!; + source[0].publicUrlPrefix = 'https://mutated.example'; + source[0].allowedOrigins!.push('https://mutated.example'); + const withPgClient = jest.fn(async () => { + throw new Error('preloaded configuration must not acquire a metadata client'); + }); + + await expect(loadStorageModulesForBuild( + snapshot, + withPgClient, + { role: 'tenant_member' }, + DATABASE_ID, + getStorageModuleCacheScope({}), + )).resolves.toBe(snapshot); + + expect(withPgClient).not.toHaveBeenCalled(); + expect(Object.isFrozen(snapshot)).toBe(true); + expect(Object.isFrozen(snapshot[0])).toBe(true); + expect(Object.isFrozen(snapshot[0].allowedOrigins)).toBe(true); + expect(snapshotPreloadedStorageModules(snapshot)).toBe(snapshot); + expect(snapshot[0].publicUrlPrefix).toBe('https://tenant.example'); + expect(snapshot[0].allowedOrigins).toEqual(['https://app.example']); + }); + + it('rejects executable SQL and inconsistent object names in a preload', () => { + expect(() => snapshotPreloadedStorageModules([{ + ...preloadedConfig('https://tenant.example'), + bucketsQualifiedName: 'storage_public.app_buckets; SELECT pg_sleep(10)', + }])).toThrow(`STORAGE_MODULE_METADATA_INVALID:buckets:${MODULE_ID}`); + + expect(() => snapshotPreloadedStorageModules([{ + ...preloadedConfig('https://tenant.example'), + bucketsQualifiedName: 'other_schema.app_buckets', + }])).toThrow(`STORAGE_MODULE_METADATA_INCONSISTENT:${MODULE_ID}`); + }); + + it('rejects duplicate scope metadata in a preload', () => { + expect(() => snapshotPreloadedStorageModules([ + preloadedConfig('https://tenant-a.example'), + { ...preloadedConfig('https://tenant-b.example'), id: 'module-b' }, + ])).toThrow('STORAGE_MODULE_METADATA_AMBIGUOUS'); + }); + + it('treats an empty preloaded list as authoritative instead of falling back', async () => { + const snapshot = snapshotPreloadedStorageModules([])!; + const withPgClient = jest.fn(async () => { + throw new Error('empty preload must not fall back'); + }); + + await expect(loadStorageModulesForBuild( + snapshot, + withPgClient, + { role: 'tenant_member' }, + DATABASE_ID, + getStorageModuleCacheScope({}), + )).resolves.toEqual([]); + expect(withPgClient).not.toHaveBeenCalled(); + }); + + it('keeps request identity and bucket authorization under pgSettings', async () => { + const pgSettings = { role: 'tenant_member' }; + const requestQuery = jest.fn(async ({ text }: QueryOptions) => { + if (text.includes('current_database_id')) { + return { rows: [{ id: DATABASE_ID }] }; + } + if (text.includes('SELECT key, physical_name FROM storage_public.app_buckets')) { + return { rows: [{ key: 'private', physical_name: 'tenant-bucket' }] }; + } + if (text.includes('metaschema_')) { + throw new Error('metadata SQL must not run'); + } + throw new Error(`Unexpected query: ${text}`); + }); + const { + withPgClient, + withPgClientMock, + withTransaction, + query, + } = requestPgHarness(pgSettings, requestQuery); + const preloaded = snapshotPreloadedStorageModules([ + preloadedConfig('https://tenant.example'), + ]); + + await expect(resolveDownloadStorageTarget({ + options: options('tenant-bucket'), + preloadedStorageModules: preloaded, + cacheScope: getStorageModuleCacheScope({}), + codec, + withPgClient, + pgSettings, + bucketId: BUCKET_ID, + })).resolves.toMatchObject({ + s3: { + bucket: 'tenant-bucket', + publicUrlPrefix: 'https://tenant.example', + }, + downloadUrlExpirySeconds: 1234, + }); + + expect(withPgClientMock).toHaveBeenCalledTimes(1); + expect(withTransaction).toHaveBeenCalledTimes(1); + expect(query).toHaveBeenCalledTimes(3); + expect(requestQuery).toHaveBeenCalledTimes(2); + expect(requestQuery.mock.calls.map(([arg]) => arg.text).join('\n')).not.toContain('metaschema_'); + }); + + it('retains the metadata SQL fallback only when preloaded modules are undefined and applies request settings', async () => { + const pgSettings = { + role: 'tenant_member', + 'jwt.claims.api_id': 'api-a', + 'jwt.claims.database_id': DATABASE_ID, + 'jwt.claims.user_id': '', + 'transaction_read_only': 'off', + 'row_security': 'on', + }; + const pgClient = { + query: jest.fn(async () => ({ rows: [storageRow('https://generic.example')] })), + }; + const withPgClient = jest.fn(async ( + settings: unknown, + callback: (client: typeof pgClient) => Promise | unknown, + ) => { + expect(settings).toBe(pgSettings); + return callback(pgClient); + }) as unknown as StorageWithPgClient; + + await expect(loadStorageModulesForBuild( + undefined, + withPgClient, + pgSettings, + DATABASE_ID, + getStorageModuleCacheScope({}), + )).resolves.toHaveLength(1); + + expect(withPgClient).toHaveBeenCalledTimes(1); + expect(pgClient.query).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphile/graphile-presigned-url-plugin/__tests__/upload-owner-scope.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/upload-owner-scope.test.ts new file mode 100644 index 0000000000..34bd240a8e --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/__tests__/upload-owner-scope.test.ts @@ -0,0 +1,82 @@ +import { resolveUploadOwnerScope } from '../src/plugin'; +import type { StorageModuleConfig } from '../src/types'; + +const moduleConfig = (scope: string): StorageModuleConfig => ({ + id: '00000000-0000-4000-8000-000000000001', + bucketsQualifiedName: 'storage_public.app_buckets', + filesQualifiedName: 'storage_public.app_files', + schemaName: 'storage_public', + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + scope, + entityTableId: scope === 'app' ? null : '00000000-0000-4000-8000-000000000002', + entityQualifiedName: scope === 'app' ? null : 'app_public.organizations', + endpoint: null, + publicUrlPrefix: null, + provider: 's3', + allowedOrigins: null, + uploadUrlExpirySeconds: 900, + downloadUrlExpirySeconds: 3600, + defaultMaxFileSize: 1024, + maxFilenameLength: 1024, + cacheTtlSeconds: 300, + hasPathShares: false, + maxBulkFiles: 100, + maxBulkTotalSize: 1024, +}); + +const filesCodec = { + name: 'app_files', + extensions: { + pg: { schemaName: 'storage_public', name: 'app_files' }, + }, +}; + +const bucketCodec = (withOwnerColumn = true) => ({ + name: 'app_buckets', + attributes: withOwnerColumn ? { owner_id: { codec: 'uuid-codec' } } : {}, + extensions: { + pg: { schemaName: 'storage_public', name: 'app_buckets' }, + }, +}); + +describe('upload owner scope', () => { + it('keeps ownerId optional for an authoritative app-scoped module', () => { + expect(resolveUploadOwnerScope( + filesCodec, + bucketCodec(), + [moduleConfig('app')], + )).toEqual({ hasOwnerId: false, ownerIdCodec: 'uuid-codec' }); + }); + + it('requires ownerId for an authoritative entity-scoped module', () => { + expect(resolveUploadOwnerScope( + filesCodec, + bucketCodec(), + [moduleConfig('organization')], + )).toEqual({ hasOwnerId: true, ownerIdCodec: 'uuid-codec' }); + }); + + it('fails closed when entity scope has no owner_id column', () => { + expect(() => resolveUploadOwnerScope( + filesCodec, + bucketCodec(false), + [moduleConfig('organization')], + )).toThrow('STORAGE_OWNER_COLUMN_REQUIRED:storage_public.app_buckets'); + }); + + it('omits upload fields when an authoritative snapshot has no matching module', () => { + expect(resolveUploadOwnerScope(filesCodec, bucketCodec(), [])).toBeNull(); + }); + + it('retains column inference only for generic consumers without a snapshot', () => { + expect(resolveUploadOwnerScope(filesCodec, bucketCodec(), undefined)).toEqual({ + hasOwnerId: true, + ownerIdCodec: 'uuid-codec', + }); + expect(resolveUploadOwnerScope(filesCodec, bucketCodec(false), undefined)).toEqual({ + hasOwnerId: false, + ownerIdCodec: null, + }); + }); +}); diff --git a/graphile/graphile-presigned-url-plugin/src/download-url-field.ts b/graphile/graphile-presigned-url-plugin/src/download-url-field.ts index b0d2bc9a73..7825825d2e 100644 --- a/graphile/graphile-presigned-url-plugin/src/download-url-field.ts +++ b/graphile/graphile-presigned-url-plugin/src/download-url-field.ts @@ -25,10 +25,27 @@ import { Logger } from '@pgpmjs/logger'; import { context as grafastContext, lambda, object } from 'grafast'; import type { GraphileConfig } from 'graphile-config'; -import { withRequestPgClient } from './request-pg-client'; +import { + type RequestPgClient, + type WithPgClient, + withRequestPgClient, +} from './request-pg-client'; import { generatePresignedGetUrl } from './s3-signer'; -import { loadAllStorageModules, resolveStorageConfigFromCodec, storedPhysicalName } from './storage-module-cache'; -import type { PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types'; +import { resolveS3ConfigForPhysicalBucket } from './s3-config'; +import { + getStorageModuleCacheScope, + resolveStorageConfigFromCodec, + storedPhysicalName, + type StorageModuleCacheScope, +} from './storage-module-cache'; +import { + assertStorageRequestContext, + loadStorageModulesForBuild, + snapshotPreloadedStorageModules, + type PreloadedStorageModules, + type StorageWithPgClient, +} from './storage-module-source'; +import type { PresignedUrlPluginOptions, S3Config } from './types'; const log = new Logger('graphile-presigned-url:download-url'); @@ -41,48 +58,101 @@ const log = new Logger('graphile-presigned-url:download-url'); * the storage module's files table, which we discover at schema-build time * via the `@storageFiles` smart tag. */ -/** - * Resolve the S3 config from the options. If the option is a lazy getter - * function, call it (and cache the result). - */ -function resolveS3(options: PresignedUrlPluginOptions): S3Config { - if (typeof options.s3 === 'function') { - const resolved = options.s3(); - options.s3 = resolved; - return resolved; - } - return options.s3; +interface DownloadStorageTargetOptions { + options: PresignedUrlPluginOptions; + preloadedStorageModules: PreloadedStorageModules; + cacheScope: StorageModuleCacheScope; + codec: { + name: string; + extensions?: { pg?: { schemaName?: string; name?: string } }; + sqlType?: string; + }; + withPgClient: (WithPgClient & StorageWithPgClient) | null | undefined; + pgSettings: unknown; + bucketId: string | null | undefined; +} + +function withCurrentRequestPgClient( + pgClient: RequestPgClient, +): StorageWithPgClient { + return async (_pgSettings, callback) => callback(pgClient); } /** - * Build a per-database S3Config for a *known* physical bucket. `physicalName` - * is required — the stored coordinate on the bucket row is the only source; - * no name is ever recomputed here. Same logic as plugin.ts resolveS3ForDatabase. + * Resolve every tenant-bound input before choosing credentials or signing. + * Any missing metadata or database error rejects the field instead of signing + * the same key with process-global fallback configuration. */ -function resolveS3ForDatabase( - options: PresignedUrlPluginOptions, - storageConfig: StorageModuleConfig, - physicalName: string, -): S3Config { - const globalS3 = resolveS3(options); - const publicUrlPrefix = storageConfig.publicUrlPrefix != null - ? storageConfig.publicUrlPrefix - : globalS3.publicUrlPrefix; - - if (physicalName === globalS3.bucket && publicUrlPrefix === globalS3.publicUrlPrefix) { - return globalS3; - } +export async function resolveDownloadStorageTarget({ + options, + preloadedStorageModules, + cacheScope, + codec, + withPgClient, + pgSettings, + bucketId, +}: DownloadStorageTargetOptions): Promise<{ + s3: S3Config; + downloadUrlExpirySeconds: number; +}> { + assertStorageRequestContext(withPgClient, pgSettings); + const requestSettings = pgSettings as Record; - return { - ...globalS3, - bucket: physicalName, - ...(publicUrlPrefix != null ? { publicUrlPrefix } : {}), - }; + return withRequestPgClient(withPgClient, requestSettings, async (pgClient) => { + const result = await pgClient.query({ + text: `SELECT jwt_private.current_database_id() AS id`, + }); + const databaseId = result.rows[0]?.id as string | null | undefined; + if (!databaseId) { + throw new Error('DATABASE_NOT_FOUND'); + } + + const allConfigs = await loadStorageModulesForBuild( + preloadedStorageModules, + withCurrentRequestPgClient(pgClient), + requestSettings, + databaseId, + cacheScope, + ); + const storageConfig = resolveStorageConfigFromCodec(codec, allConfigs); + if (!storageConfig) { + throw new Error('STORAGE_MODULE_NOT_FOUND'); + } + if (!bucketId) { + throw new Error('BUCKET_NOT_FOUND'); + } + + const bucketResult = await pgClient.query({ + text: `SELECT key, physical_name FROM ${storageConfig.bucketsQualifiedName} WHERE id = $1 LIMIT 1`, + values: [bucketId], + }); + const bucketRow = bucketResult.rows[0] as { key: string; physical_name?: string | null } | undefined; + if (!bucketRow) { + throw new Error('BUCKET_NOT_FOUND'); + } + const physicalName = storedPhysicalName(bucketRow); + if (physicalName === null) { + throw new Error('BUCKET_NOT_PROVISIONED'); + } + + return { + s3: resolveS3ConfigForPhysicalBucket( + options, + storageConfig, + physicalName, + cacheScope, + ), + downloadUrlExpirySeconds: storageConfig.downloadUrlExpirySeconds, + }; + }); } export function createDownloadUrlPlugin( options: PresignedUrlPluginOptions, ): GraphileConfig.Plugin { + const preloadedStorageModules = snapshotPreloadedStorageModules( + options.preloadedStorageModules, + ); return { name: 'PresignedUrlDownloadPlugin', @@ -108,6 +178,7 @@ export function createDownloadUrlPlugin( } log.debug(`Adding downloadUrl field to type: ${pgCodec.name} (has @storageFiles tag)`); + const cacheScope = getStorageModuleCacheScope(build); const { graphql: { GraphQLString }, @@ -146,56 +217,32 @@ export function createDownloadUrlPlugin( return lambda($combined, async ({ key, isPublic, filename, bucketId, withPgClient, pgSettings }: any) => { if (!key) return null; - let s3ForDb = resolveS3(options); - let downloadUrlExpirySeconds = 3600; + let target: Awaited>; try { - if (withPgClient && pgSettings) { - const databaseId = await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { - const dbResult = await pgClient.query({ - text: `SELECT jwt_private.current_database_id() AS id`, - }); - return (dbResult.rows[0]?.id as string | undefined) ?? null; - }); - // Module registration is server config, not user data: - // resolve it without the request role's pgSettings. - const config = databaseId - ? resolveStorageConfigFromCodec( - capturedCodec, - await withPgClient(null, (pgClient: any) => loadAllStorageModules(pgClient, databaseId)), - ) - : null; - const resolved = config && bucketId - ? await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { - // Look up the stored physical coordinate for scoped S3 resolution - const bucketResult = await pgClient.query({ - text: `SELECT key, physical_name FROM ${config.bucketsQualifiedName} WHERE id = $1 LIMIT 1`, - values: [bucketId], - }); - const row = bucketResult.rows[0] as { key: string; physical_name?: string | null } | undefined; - return row ? { config, physicalName: storedPhysicalName(row) } : null; - }) - : null; - if (resolved) { - if (resolved.physicalName === null) { - // No physical bucket was ever provisioned — no object can exist. - return null; - } - downloadUrlExpirySeconds = resolved.config.downloadUrlExpirySeconds; - s3ForDb = resolveS3ForDatabase(options, resolved.config, resolved.physicalName); - } + target = await resolveDownloadStorageTarget({ + options, + preloadedStorageModules, + cacheScope, + codec: capturedCodec, + withPgClient, + pgSettings, + bucketId, + }); + } catch (error) { + if (error instanceof Error && error.message === 'BUCKET_NOT_PROVISIONED') { + return null; } - } catch { - // Fall back to global config if lookup fails + throw error; } - if (isPublic && s3ForDb.publicUrlPrefix) { - return `${s3ForDb.publicUrlPrefix}/${s3ForDb.bucket}/${key}`; + if (isPublic && target.s3.publicUrlPrefix) { + return `${target.s3.publicUrlPrefix}/${target.s3.bucket}/${key}`; } return generatePresignedGetUrl( - s3ForDb, + target.s3, key, - downloadUrlExpirySeconds, + target.downloadUrlExpirySeconds, filename || undefined, ); }); diff --git a/graphile/graphile-presigned-url-plugin/src/index.ts b/graphile/graphile-presigned-url-plugin/src/index.ts index 3d8ee4fe44..6cc338aa4f 100644 --- a/graphile/graphile-presigned-url-plugin/src/index.ts +++ b/graphile/graphile-presigned-url-plugin/src/index.ts @@ -28,10 +28,25 @@ */ export { createDownloadUrlPlugin } from './download-url-field'; -export { createPresignedUrlPlugin,PresignedUrlPlugin } from './plugin'; +export { createPresignedUrlPlugin, PresignedUrlPlugin } from './plugin'; export { PresignedUrlPreset } from './preset'; +export { snapshotPreloadedStorageModules } from './storage-module-source'; export { deleteS3Object, generatePresignedGetUrl, generatePresignedPutUrl, headObject } from './s3-signer'; -export { clearBucketCache, clearStorageModuleCache, getBucketConfig, getStorageModuleConfig, getStorageModuleConfigForOwner, isS3BucketProvisioned, loadAllStorageModules, markS3BucketProvisioned,resolveStorageConfigFromCodec, resolveStorageModuleByFileId } from './storage-module-cache'; +export { + getStorageModuleConfig, + getStorageModuleConfigForOwner, + getBucketConfig, + resolveStorageModuleByFileId, + loadAllStorageModules, + resolveStorageConfigFromCodec, + clearStorageModuleCache, + clearBucketCache, + isS3BucketProvisioned, + markS3BucketProvisioned, + StorageModuleCacheScope, + getStorageModuleCacheScope, + storedPhysicalName, +} from './storage-module-cache'; export type { BucketConfig, BucketNameResolver, diff --git a/graphile/graphile-presigned-url-plugin/src/plugin.ts b/graphile/graphile-presigned-url-plugin/src/plugin.ts index 6d27b29b19..90d49840ff 100644 --- a/graphile/graphile-presigned-url-plugin/src/plugin.ts +++ b/graphile/graphile-presigned-url-plugin/src/plugin.ts @@ -23,10 +23,32 @@ import { Logger } from '@pgpmjs/logger'; import { access, context as grafastContext, lambda, object } from 'grafast'; import type { GraphileConfig } from 'graphile-config'; -import { type WithPgClient,withRequestPgClient } from './request-pg-client'; -import { deleteS3Object,generatePresignedPutUrl } from './s3-signer'; -import { getBucketConfig, isS3BucketProvisioned, loadAllStorageModules, markS3BucketProvisioned,resolveStorageConfigFromCodec, storedPhysicalName } from './storage-module-cache'; -import type { BucketConfig,PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types'; +import { + type RequestPgClient, + type WithPgClient, + withRequestPgClient, +} from './request-pg-client'; +import { + mintPhysicalBucketName, + resolveS3ConfigForPhysicalBucket, +} from './s3-config'; +import { deleteS3Object, generatePresignedPutUrl } from './s3-signer'; +import { + getBucketConfig, + getStorageModuleCacheScope, + isS3BucketProvisioned, + markS3BucketProvisioned, + resolveStorageConfigFromCodec, + type StorageModuleCacheScope, + storedPhysicalName, +} from './storage-module-cache'; +import { + assertStorageRequestContext, + loadStorageModulesForBuild, + snapshotPreloadedStorageModules, + type StorageWithPgClient, +} from './storage-module-source'; +import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types'; const log = new Logger('graphile-presigned-url:plugin'); @@ -38,6 +60,49 @@ const MAX_CUSTOM_KEY_LENGTH = 1024; const SHA256_HEX_REGEX = /^[a-f0-9]{64}$/; const CUSTOM_KEY_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9_.\-/]*$/; +type TaggedStorageCodec = { + name: string; + attributes?: Record; + extensions?: { pg?: { schemaName?: string; name?: string } }; +}; + +/** + * Resolve the upload input's owner scope at schema-build time. + * + * Exact-build preloaded metadata is authoritative: app-scoped modules must + * not expose a required ownerId merely because their canonical bucket table + * retains a nullable owner_id column. Generic consumers without a preloaded + * snapshot retain the legacy column-based inference because their module + * scope is available only at request time. + */ +export function resolveUploadOwnerScope( + filesCodec: TaggedStorageCodec, + bucketCodec: TaggedStorageCodec, + preloadedStorageModules: readonly StorageModuleConfig[] | undefined, +): { hasOwnerId: boolean; ownerIdCodec: unknown | null } | null { + const ownerIdCodec = bucketCodec.attributes?.owner_id?.codec ?? null; + if (preloadedStorageModules === undefined) { + return { + hasOwnerId: ownerIdCodec !== null, + ownerIdCodec, + }; + } + + const storageConfig = resolveStorageConfigFromCodec( + filesCodec, + preloadedStorageModules, + ); + if (!storageConfig) return null; + + const hasOwnerId = storageConfig.scope !== 'app'; + if (hasOwnerId && ownerIdCodec === null) { + throw new Error( + `STORAGE_OWNER_COLUMN_REQUIRED:${storageConfig.schemaName}.${storageConfig.bucketsTableName}`, + ); + } + return { hasOwnerId, ownerIdCodec }; +} + // --- Helpers --- function isValidSha256(hash: string): boolean { @@ -81,58 +146,39 @@ async function resolveDatabaseId(pgClient: any): Promise { return result.rows[0]?.id ?? null; } -function resolveS3(options: PresignedUrlPluginOptions): S3Config { - if (typeof options.s3 === 'function') { - const resolved = options.s3(); - options.s3 = resolved; - return resolved; - } - return options.s3; -} +type RequestStorageWithPgClient = WithPgClient & StorageWithPgClient; -/** - * Mint the physical S3 bucket name for a logical bucket's first provision. - * - * This is a naming *policy*, consulted exactly once per bucket — before the - * physical bucket exists. Once provisioned, the recorded `physical_name` on - * the row is authoritative and this function must not be consulted again. - */ -function mintPhysicalBucketName( - options: PresignedUrlPluginOptions, - databaseId: string, - bucketKey: string, -): string { - if (options.resolveBucketName) { - return options.resolveBucketName(databaseId, bucketKey); - } - // Single-bucket deployment: the globally configured bucket is the physical bucket. - return resolveS3(options).bucket; +function withCurrentRequestPgClient( + pgClient: RequestPgClient, +): StorageWithPgClient { + return async (_pgSettings, callback) => callback(pgClient); } -/** - * Build the S3 config for a *known* physical bucket. `physicalName` is - * required — callers must resolve the coordinate (stored row value, or a - * freshly provisioned name) before getting here. No name is ever recomputed. - */ -function resolveS3ForDatabase( - options: PresignedUrlPluginOptions, - storageConfig: StorageModuleConfig, - physicalName: string, -): S3Config { - const globalS3 = resolveS3(options); - const publicUrlPrefix = storageConfig.publicUrlPrefix != null - ? storageConfig.publicUrlPrefix - : globalS3.publicUrlPrefix; - - if (physicalName === globalS3.bucket && publicUrlPrefix === globalS3.publicUrlPrefix) { - return globalS3; - } - - return { - ...globalS3, - bucket: physicalName, - ...(publicUrlPrefix != null ? { publicUrlPrefix } : {}), - }; +async function resolveRequestStorageModules( + preloadedStorageModules: readonly StorageModuleConfig[] | undefined, + cacheScope: StorageModuleCacheScope, + withPgClient: RequestStorageWithPgClient | null | undefined, + pgSettings: unknown, +): Promise<{ + databaseId: string; + allConfigs: readonly StorageModuleConfig[]; +}> { + assertStorageRequestContext(withPgClient, pgSettings); + const requestSettings = pgSettings as Record; + + return withRequestPgClient(withPgClient, requestSettings, async (pgClient) => { + const databaseId = await resolveDatabaseId(pgClient); + if (!databaseId) throw new Error('DATABASE_NOT_FOUND'); + + const allConfigs = await loadStorageModulesForBuild( + preloadedStorageModules, + withCurrentRequestPgClient(pgClient), + requestSettings, + databaseId, + cacheScope, + ); + return { databaseId, allConfigs }; + }); } /** @@ -144,48 +190,71 @@ function resolveS3ForDatabase( * value is the durable coordinate: route resolution and every later read use * it verbatim; nothing is recomputed. * - * The record write runs in the system lane (privileged role, so it bypasses the - * RLS that stops request roles from UPDATE-ing bucket rows) — it is server - * bookkeeping, not request data. It still carries the tenant `database_id` - * claim, because the buckets table's catalog-sync trigger calls - * `jwt_private.current_database_id()` and would otherwise raise - * DATABASE_CLAIM_REQUIRED; `withRequestPgClient` applies that claim inside the - * write's transaction without switching off the privileged role. - * `bucket` (the cached config) is mutated in place so subsequent reads observe - * the recorded name without a DB round-trip. + * The record write reuses the complete request settings. It must satisfy the + * same role, claims, and RLS policies as the bucket read that authorized the + * upload; a policy denial fails closed. `bucket` (the cached config) is mutated + * in place so subsequent reads observe the recorded name without a DB round-trip. */ async function provisionAndRecordPhysicalBucket( options: PresignedUrlPluginOptions, withPgClient: WithPgClient, + pgSettings: Record, storageConfig: StorageModuleConfig, databaseId: string, bucket: BucketConfig, allowedOrigins: string[] | null, + cacheScope: StorageModuleCacheScope, ): Promise { - const s3BucketName = mintPhysicalBucketName(options, databaseId, bucket.key); + const s3BucketName = mintPhysicalBucketName( + options, + bucket.key, + databaseId, + cacheScope, + ); - if (options.ensureBucketProvisioned && !isS3BucketProvisioned(s3BucketName)) { + if (options.ensureBucketProvisioned && !isS3BucketProvisioned(s3BucketName, cacheScope)) { log.info(`Lazy-provisioning S3 bucket "${s3BucketName}" for database ${databaseId}`); await options.ensureBucketProvisioned(s3BucketName, bucket.type, databaseId, allowedOrigins); - markS3BucketProvisioned(s3BucketName); + markS3BucketProvisioned(s3BucketName, cacheScope); log.info(`Lazy-provisioned S3 bucket "${s3BucketName}" successfully`); } - // Record the physical coordinate on the source row. The `physical_name IS NULL` - // guard keeps this idempotent and race-safe across concurrent first uploads. - // The catalog-sync trigger on this UPDATE needs `jwt.claims.database_id`, so the - // write runs under the resolved database claim (privileged role preserved). - await withRequestPgClient(withPgClient, { 'jwt.claims.database_id': databaseId }, (client) => - client.query({ - text: `UPDATE ${storageConfig.bucketsQualifiedName} + // Record the physical coordinate on the source row. If another process won + // the race, read its value and route to that authoritative coordinate rather + // than signing against this process's losing candidate. + // Keep the complete request role and claims active for both race-resolution + // statements; a policy denial must fail closed rather than use a system lane. + const recordedPhysicalName = await withRequestPgClient( + withPgClient, + pgSettings, + async (client) => { + const updated = await client.query({ + text: `UPDATE ${storageConfig.bucketsQualifiedName} SET physical_name = $1 - WHERE id = $2 AND physical_name IS NULL`, - values: [s3BucketName, bucket.id], - }), + WHERE id = $2 AND physical_name IS NULL + RETURNING physical_name`, + values: [s3BucketName, bucket.id], + }); + const updatedName = storedPhysicalName(updated.rows[0] ?? {}); + if (updatedName !== null) return updatedName; + + const existing = await client.query({ + text: `SELECT physical_name + FROM ${storageConfig.bucketsQualifiedName} + WHERE id = $1 + LIMIT 1`, + values: [bucket.id], + }); + return storedPhysicalName(existing.rows[0] ?? {}); + }, ); - bucket.physical_name = s3BucketName; - log.info(`Recorded physical_name="${s3BucketName}" on bucket ${bucket.id}`); - return s3BucketName; + if (recordedPhysicalName === null) { + throw new Error('BUCKET_PHYSICAL_NAME_NOT_RECORDED'); + } + + bucket.physical_name = recordedPhysicalName; + log.info(`Using physical_name="${recordedPhysicalName}" for bucket ${bucket.id}`); + return recordedPhysicalName; } // --- Plugin factory --- @@ -193,6 +262,9 @@ async function provisionAndRecordPhysicalBucket( export function createPresignedUrlPlugin( options: PresignedUrlPluginOptions, ): GraphileConfig.Plugin { + const preloadedStorageModules = snapshotPreloadedStorageModules( + options.preloadedStorageModules, + ); return { name: 'PresignedUrlPlugin', @@ -259,11 +331,22 @@ export function createPresignedUrlPlugin( continue; } - const hasOwnerId = !!matchingBucketCodec.attributes.owner_id; + const ownerScope = resolveUploadOwnerScope( + filesCodec, + matchingBucketCodec, + preloadedStorageModules, + ); + if (!ownerScope) { + log.debug( + `Skipping upload mutation for ${filesCodec.name}: no authoritative preloaded storage module`, + ); + continue; + } + const { hasOwnerId, ownerIdCodec } = ownerScope; const mutationName = `upload${filesTypeName}`; const ownerIdGqlType = hasOwnerId - ? (build as any).getGraphQLTypeByPgCodec(matchingBucketCodec.attributes.owner_id.codec, 'input') + ? (build as any).getGraphQLTypeByPgCodec(ownerIdCodec, 'input') : null; const InputType = new GraphQLInputObjectType({ @@ -294,6 +377,7 @@ export function createPresignedUrlPlugin( }); const capturedFilesCodec = filesCodec; + const cacheScope = getStorageModuleCacheScope(build); log.debug(`Adding file upload mutation "${mutationName}" for ${filesTypeName} (entity-scoped=${hasOwnerId})`); @@ -330,34 +414,50 @@ export function createPresignedUrlPlugin( }); return lambda($combined, async (vals: any) => { - // Request-lane reads/writes run under the request role's pgSettings - // inside an explicit transaction so the jwt claims stay applied - // across every statement (see withRequestPgClient). - const databaseId = await withRequestPgClient(vals.withPgClient, vals.pgSettings, (pgClient) => - resolveDatabaseId(pgClient), - ); - if (!databaseId) throw new Error('DATABASE_NOT_FOUND'); - - // Module registration is server config, not user data: - // resolve it without the request role's pgSettings. - const allConfigs = await vals.withPgClient(null, (pgClient: any) => - loadAllStorageModules(pgClient, databaseId), + // Resolve the request tenant and exact build-scoped module + // metadata inside one transaction carrying every request GUC. + const { databaseId, allConfigs } = await resolveRequestStorageModules( + preloadedStorageModules, + cacheScope, + vals.withPgClient, + vals.pgSettings, ); const storageConfig = resolveStorageConfigFromCodec(capturedFilesCodec, allConfigs); if (!storageConfig) throw new Error('STORAGE_MODULE_NOT_FOUND'); // Bucket config read under the request role (RLS-gated visibility). const bucket = await withRequestPgClient(vals.withPgClient, vals.pgSettings, (pgClient) => - getBucketConfig(pgClient, storageConfig, databaseId, vals.bucketKey, vals.ownerId || undefined), + getBucketConfig( + pgClient, + storageConfig, + databaseId, + vals.bucketKey, + vals.ownerId || undefined, + cacheScope, + ), ); if (!bucket) throw new Error('BUCKET_NOT_FOUND'); // First provision mints + records the coordinate; afterwards the // stored physical_name is authoritative and nothing is recomputed. const physicalName = bucket.physical_name === null - ? await provisionAndRecordPhysicalBucket(options, vals.withPgClient, storageConfig, databaseId, bucket, storageConfig.allowedOrigins) + ? await provisionAndRecordPhysicalBucket( + options, + vals.withPgClient, + vals.pgSettings, + storageConfig, + databaseId, + bucket, + storageConfig.allowedOrigins, + cacheScope, + ) : bucket.physical_name; - const s3ForDb = resolveS3ForDatabase(options, storageConfig, physicalName); + const s3ForDb = resolveS3ConfigForPhysicalBucket( + options, + storageConfig, + physicalName, + cacheScope, + ); // File row INSERT under the request role (RLS enforced). return withRequestPgClient(vals.withPgClient, vals.pgSettings, (txClient) => @@ -444,25 +544,27 @@ export function createPresignedUrlPlugin( }); return lambda($combined, async (vals: any) => { - // Request-lane reads/writes run under the request role's pgSettings - // inside an explicit transaction so the jwt claims stay applied - // across every statement (see withRequestPgClient). - const databaseId = await withRequestPgClient(vals.withPgClient, vals.pgSettings, (pgClient) => - resolveDatabaseId(pgClient), - ); - if (!databaseId) throw new Error('DATABASE_NOT_FOUND'); - - // Module registration is server config, not user data: - // resolve it without the request role's pgSettings. - const allConfigs = await vals.withPgClient(null, (pgClient: any) => - loadAllStorageModules(pgClient, databaseId), + // Resolve the request tenant and exact build-scoped module + // metadata inside one transaction carrying every request GUC. + const { databaseId, allConfigs } = await resolveRequestStorageModules( + preloadedStorageModules, + cacheScope, + vals.withPgClient, + vals.pgSettings, ); const storageConfig = resolveStorageConfigFromCodec(capturedFilesCodec, allConfigs); if (!storageConfig) throw new Error('STORAGE_MODULE_NOT_FOUND'); // Bucket config read under the request role (RLS-gated visibility). const bucket = await withRequestPgClient(vals.withPgClient, vals.pgSettings, (pgClient) => - getBucketConfig(pgClient, storageConfig, databaseId, vals.bucketKey, vals.ownerId || undefined), + getBucketConfig( + pgClient, + storageConfig, + databaseId, + vals.bucketKey, + vals.ownerId || undefined, + cacheScope, + ), ); if (!bucket) throw new Error('BUCKET_NOT_FOUND'); @@ -483,9 +585,23 @@ export function createPresignedUrlPlugin( // First provision mints + records the coordinate; afterwards the // stored physical_name is authoritative and nothing is recomputed. const physicalName = bucket.physical_name === null - ? await provisionAndRecordPhysicalBucket(options, vals.withPgClient, storageConfig, databaseId, bucket, storageConfig.allowedOrigins) + ? await provisionAndRecordPhysicalBucket( + options, + vals.withPgClient, + vals.pgSettings, + storageConfig, + databaseId, + bucket, + storageConfig.allowedOrigins, + cacheScope, + ) : bucket.physical_name; - const s3ForDb = resolveS3ForDatabase(options, storageConfig, physicalName); + const s3ForDb = resolveS3ConfigForPhysicalBucket( + options, + storageConfig, + physicalName, + cacheScope, + ); // File row INSERTs under the request role (RLS enforced). return withRequestPgClient(vals.withPgClient, vals.pgSettings, async (txClient) => { @@ -548,6 +664,7 @@ export function createPresignedUrlPlugin( const defaultResolver = (obj: any) => obj[fieldName]; const { resolve: oldResolve = defaultResolver, ...rest } = field; const capturedCodec = pgCodec; + const cacheScope = getStorageModuleCacheScope(build); return { ...rest, @@ -567,12 +684,14 @@ export function createPresignedUrlPlugin( if (withPgClient) { try { - const databaseId = await withRequestPgClient(withPgClient, pgSettings, (pgClient) => resolveDatabaseId(pgClient)); - // Module registration is server config, not user data: - // resolve it without the request role's pgSettings. - const allConfigs = databaseId - ? await withPgClient(null, (pgClient: any) => loadAllStorageModules(pgClient, databaseId)) - : []; + // Prefer the exact-build control-plane snapshot; the SQL + // fallback exists only for generic package consumers. + const { allConfigs } = await resolveRequestStorageModules( + preloadedStorageModules, + cacheScope, + withPgClient, + pgSettings, + ); const storageConfig = resolveStorageConfigFromCodec(capturedCodec, allConfigs); if (storageConfig) { @@ -603,12 +722,14 @@ export function createPresignedUrlPlugin( if (withPgClient) { try { - const databaseId = await withRequestPgClient(withPgClient, pgSettings, (pgClient) => resolveDatabaseId(pgClient)); - // Module registration is server config, not user data: - // resolve it without the request role's pgSettings. - const allConfigs = databaseId - ? await withPgClient(null, (pgClient: any) => loadAllStorageModules(pgClient, databaseId)) - : []; + // Prefer the exact-build control-plane snapshot; the SQL + // fallback exists only for generic package consumers. + const { allConfigs } = await resolveRequestStorageModules( + preloadedStorageModules, + cacheScope, + withPgClient, + pgSettings, + ); const storageConfig = resolveStorageConfigFromCodec(capturedCodec, allConfigs); if (storageConfig) await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { @@ -642,7 +763,12 @@ export function createPresignedUrlPlugin( log.warn(`Bucket ${fileRow!.bucket_id} has no physical_name; skipping S3 delete`); return; } - const s3ForDb = resolveS3ForDatabase(options, storageConfig, physicalName); + const s3ForDb = resolveS3ConfigForPhysicalBucket( + options, + storageConfig, + physicalName, + cacheScope, + ); await deleteS3Object(s3ForDb, fileRow!.key); log.info(`Sync S3 delete succeeded for key=${fileRow!.key}`); }); diff --git a/graphile/graphile-presigned-url-plugin/src/preset.ts b/graphile/graphile-presigned-url-plugin/src/preset.ts index 3ab986721e..2cdbee629d 100644 --- a/graphile/graphile-presigned-url-plugin/src/preset.ts +++ b/graphile/graphile-presigned-url-plugin/src/preset.ts @@ -10,6 +10,7 @@ import type { GraphileConfig } from 'graphile-config'; import { createDownloadUrlPlugin } from './download-url-field'; import { createPresignedUrlPlugin } from './plugin'; +import { snapshotPreloadedStorageModules } from './storage-module-source'; import type { PresignedUrlPluginOptions } from './types'; /** @@ -38,10 +39,17 @@ import type { PresignedUrlPluginOptions } from './types'; export function PresignedUrlPreset( options: PresignedUrlPluginOptions, ): GraphileConfig.Preset { + const preloadedStorageModules = snapshotPreloadedStorageModules( + options.preloadedStorageModules, + ); + const buildOptions = preloadedStorageModules === undefined + ? options + : { ...options, preloadedStorageModules }; + return { plugins: [ - createPresignedUrlPlugin(options), - createDownloadUrlPlugin(options), + createPresignedUrlPlugin(buildOptions), + createDownloadUrlPlugin(buildOptions), ], }; } diff --git a/graphile/graphile-presigned-url-plugin/src/s3-config.ts b/graphile/graphile-presigned-url-plugin/src/s3-config.ts new file mode 100644 index 0000000000..87d9f7944f --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/src/s3-config.ts @@ -0,0 +1,85 @@ +import type { + PresignedUrlPluginOptions, + S3Config, + StorageModuleConfig, +} from './types'; +import type { StorageModuleCacheScope } from './storage-module-cache'; + +const s3ConfigsByBuild = new WeakMap< + StorageModuleCacheScope, + WeakMap +>(); + +/** + * Resolve the current runtime's S3 configuration without mutating the shared + * preset options. A preset may be reused by several Graphile builds, so + * memoizing the first getter result here would bind later builds to it. + */ +export function resolveS3Config( + options: PresignedUrlPluginOptions, + cacheScope: StorageModuleCacheScope, +): S3Config { + if (typeof options.s3 !== 'function') { + return options.s3; + } + + let configsForBuild = s3ConfigsByBuild.get(cacheScope); + if (!configsForBuild) { + configsForBuild = new WeakMap(); + s3ConfigsByBuild.set(cacheScope, configsForBuild); + } + + const cached = configsForBuild.get(options); + if (cached) { + return cached; + } + + const resolved = options.s3(); + configsForBuild.set(options, resolved); + return resolved; +} + +/** + * Mint a physical bucket name for a bucket that has never been provisioned. + * + * This deliberately matches graphile-bucket-provisioner-plugin's resolver + * contract: logical bucket key first, database ID second. Callers must persist + * the result and use the stored physical_name for every later operation. + */ +export function mintPhysicalBucketName( + options: PresignedUrlPluginOptions, + bucketKey: string, + databaseId: string, + cacheScope: StorageModuleCacheScope, +): string { + const base = resolveS3Config(options, cacheScope); + return options.resolveBucketName + ? options.resolveBucketName(bucketKey, databaseId) + : base.bucket; +} + +/** + * Build an S3 config for a persisted physical coordinate. + * + * No naming resolver is consulted here: physical_name is authoritative once + * recorded, even if naming policy or environment configuration later changes. + */ +export function resolveS3ConfigForPhysicalBucket( + options: PresignedUrlPluginOptions, + storageConfig: StorageModuleConfig, + physicalBucketName: string, + cacheScope: StorageModuleCacheScope, +): S3Config { + const base = resolveS3Config(options, cacheScope); + const publicUrlPrefix = storageConfig.publicUrlPrefix ?? base.publicUrlPrefix; + + if (physicalBucketName === base.bucket && publicUrlPrefix === base.publicUrlPrefix) { + return base; + } + + return { + ...base, + bucket: physicalBucketName, + ...(publicUrlPrefix != null ? { publicUrlPrefix } : {}), + }; +} diff --git a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts index 12e48b6781..76ebc94ce7 100644 --- a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts +++ b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts @@ -18,21 +18,62 @@ const DEFAULT_MAX_BULK_TOTAL_SIZE = 1073741824; // 1GB const FIVE_MINUTES_MS = 1000 * 60 * 5; const ONE_HOUR_MS = 1000 * 60 * 60; +type StorageCacheEntry = + | { kind: 'config'; value: StorageModuleConfig | null } + | { kind: 'list'; value: StorageModuleConfig[] }; + +const CACHE_TTL_MS = process.env.NODE_ENV === 'development' + ? FIVE_MINUTES_MS + : ONE_HOUR_MS; + /** - * LRU cache for per-database StorageModuleConfig. - * - * Each PostGraphile instance serves a single database, but the presigned URL - * plugin needs to know the generated table names (buckets, files) - * and their schemas. This cache avoids re-querying metaschema - * on every request. + * Metadata owned by one exact Graphile build. * - * Pattern: same as graphile-cache's LRU with TTL-based eviction. + * Logical database/module identifiers are deliberately only keys inside this + * scope. They never select the scope itself, because separate physical pools + * may legitimately expose identical identifiers. + */ +export class StorageModuleCacheScope { + readonly storageModuleCache = new LRUCache({ + max: 100, + ttl: CACHE_TTL_MS, + updateAgeOnGet: false, + }); + + readonly bucketCache = new LRUCache({ + max: 500, + ttl: CACHE_TTL_MS, + updateAgeOnGet: false, + }); + + readonly provisionedBuckets = new Set(); + + clear(): void { + this.storageModuleCache.clear(); + this.bucketCache.clear(); + this.provisionedBuckets.clear(); + } +} + +/** + * Weak ownership ties cached metadata to the exact Graphile build object. + * Reusing a preset/plugin object for another build therefore cannot reuse the + * first build's tenant metadata, and releasing the build releases its cache. */ -const storageModuleCache = new LRUCache({ - max: 50, - ttl: process.env.NODE_ENV === 'development' ? FIVE_MINUTES_MS : ONE_HOUR_MS, - updateAgeOnGet: true, -}); +const cacheScopesByBuild = new WeakMap(); + +export function getStorageModuleCacheScope(build: object): StorageModuleCacheScope { + if ((typeof build !== 'object' || build === null) && typeof build !== 'function') { + throw new TypeError('A Graphile build object is required for storage cache isolation'); + } + + let scope = cacheScopesByBuild.get(build); + if (!scope) { + scope = new StorageModuleCacheScope(); + cacheScopesByBuild.set(build, scope); + } + return scope; +} /** * SQL query to resolve the app-level storage module config for a database. @@ -45,11 +86,16 @@ const storageModuleCache = new LRUCache({ const APP_STORAGE_MODULE_QUERY = ` SELECT sm.id, + sm.database_id, sm.scope, sm.entity_table_id, + bt.database_id AS buckets_database_id, bs.schema_name AS buckets_schema, + bs.database_id AS buckets_schema_database_id, bt.name AS buckets_table, + ft.database_id AS files_database_id, fs.schema_name AS files_schema, + fs.database_id AS files_schema_database_id, ft.name AS files_table, sm.endpoint, sm.public_url_prefix, @@ -63,16 +109,26 @@ const APP_STORAGE_MODULE_QUERY = ` sm.max_bulk_files, sm.max_bulk_total_size, sm.has_path_shares, + NULL AS entity_database_id, + NULL AS entity_schema_database_id, NULL AS entity_schema, NULL AS entity_table FROM metaschema_modules_public.storage_module sm - JOIN metaschema_public.table bt ON bt.id = sm.buckets_table_id - JOIN metaschema_public.schema bs ON bs.id = bt.schema_id - JOIN metaschema_public.table ft ON ft.id = sm.files_table_id - JOIN metaschema_public.schema fs ON fs.id = ft.schema_id + JOIN metaschema_public.table bt + ON bt.id = sm.buckets_table_id + AND bt.database_id = sm.database_id + JOIN metaschema_public.schema bs + ON bs.id = bt.schema_id + AND bs.database_id = sm.database_id + JOIN metaschema_public.table ft + ON ft.id = sm.files_table_id + AND ft.database_id = sm.database_id + JOIN metaschema_public.schema fs + ON fs.id = ft.schema_id + AND fs.database_id = sm.database_id WHERE sm.database_id = $1 AND sm.scope = 'app' - LIMIT 1 + ORDER BY sm.id `; /** @@ -84,11 +140,16 @@ const APP_STORAGE_MODULE_QUERY = ` const ALL_STORAGE_MODULES_QUERY = ` SELECT sm.id, + sm.database_id, sm.scope, sm.entity_table_id, + bt.database_id AS buckets_database_id, bs.schema_name AS buckets_schema, + bs.database_id AS buckets_schema_database_id, bt.name AS buckets_table, + ft.database_id AS files_database_id, fs.schema_name AS files_schema, + fs.database_id AS files_schema_database_id, ft.name AS files_table, sm.endpoint, sm.public_url_prefix, @@ -102,25 +163,45 @@ const ALL_STORAGE_MODULES_QUERY = ` sm.max_bulk_files, sm.max_bulk_total_size, sm.has_path_shares, + et.database_id AS entity_database_id, + es.database_id AS entity_schema_database_id, es.schema_name AS entity_schema, et.name AS entity_table FROM metaschema_modules_public.storage_module sm - JOIN metaschema_public.table bt ON bt.id = sm.buckets_table_id - JOIN metaschema_public.schema bs ON bs.id = bt.schema_id - JOIN metaschema_public.table ft ON ft.id = sm.files_table_id - JOIN metaschema_public.schema fs ON fs.id = ft.schema_id - LEFT JOIN metaschema_public.table et ON et.id = sm.entity_table_id - LEFT JOIN metaschema_public.schema es ON es.id = et.schema_id + JOIN metaschema_public.table bt + ON bt.id = sm.buckets_table_id + AND bt.database_id = sm.database_id + JOIN metaschema_public.schema bs + ON bs.id = bt.schema_id + AND bs.database_id = sm.database_id + JOIN metaschema_public.table ft + ON ft.id = sm.files_table_id + AND ft.database_id = sm.database_id + JOIN metaschema_public.schema fs + ON fs.id = ft.schema_id + AND fs.database_id = sm.database_id + LEFT JOIN metaschema_public.table et + ON et.id = sm.entity_table_id + AND et.database_id = sm.database_id + LEFT JOIN metaschema_public.schema es + ON es.id = et.schema_id + AND es.database_id = sm.database_id WHERE sm.database_id = $1 + ORDER BY sm.scope, sm.id `; interface StorageModuleRow { id: string; + database_id: string; scope: string; entity_table_id: string | null; + buckets_database_id: string; buckets_schema: string; + buckets_schema_database_id: string; buckets_table: string; + files_database_id: string; files_schema: string; + files_schema_database_id: string; files_table: string; endpoint: string | null; public_url_prefix: string | null; @@ -134,6 +215,8 @@ interface StorageModuleRow { max_bulk_files: number | null; max_bulk_total_size: number | null; has_path_shares: boolean; + entity_database_id: string | null; + entity_schema_database_id: string | null; entity_schema: string | null; entity_table: string | null; } @@ -141,19 +224,82 @@ interface StorageModuleRow { /** * Build a StorageModuleConfig from a raw DB row. */ -function buildConfig(row: StorageModuleRow): StorageModuleConfig { +function quoteMetadataIdentifier(schema: string, objectName: string, label: string): string { + if ( + typeof schema !== 'string' || + schema.length === 0 || + schema.includes('\0') || + Buffer.byteLength(schema, 'utf8') > 63 || + typeof objectName !== 'string' || + objectName.length === 0 || + objectName.includes('\0') || + Buffer.byteLength(objectName, 'utf8') > 63 + ) { + throw new Error(`STORAGE_MODULE_METADATA_INVALID:${label}`); + } + return QuoteUtils.quoteQualifiedIdentifier(schema, objectName); +} + +function buildConfig(row: StorageModuleRow, databaseId: string): StorageModuleConfig { + const objectDatabaseIds = [ + row.database_id, + row.buckets_database_id, + row.buckets_schema_database_id, + row.files_database_id, + row.files_schema_database_id, + ]; + if (objectDatabaseIds.some((id) => id !== databaseId)) { + throw new Error(`STORAGE_MODULE_CROSS_DATABASE_METADATA:${row.id}`); + } + if ( + typeof row.id !== 'string' || + row.id.length === 0 || + typeof row.scope !== 'string' || + row.scope.length === 0 + ) { + throw new Error('STORAGE_MODULE_METADATA_INVALID'); + } + + if (row.entity_table_id === null) { + if ( + row.scope !== 'app' || + row.entity_database_id !== null || + row.entity_schema_database_id !== null || + row.entity_schema !== null || + row.entity_table !== null + ) { + throw new Error(`STORAGE_MODULE_METADATA_INVALID:${row.id}`); + } + } else if ( + row.scope === 'app' || + row.entity_database_id !== databaseId || + row.entity_schema_database_id !== databaseId || + !row.entity_schema || + !row.entity_table + ) { + throw new Error(`STORAGE_MODULE_CROSS_DATABASE_METADATA:${row.id}`); + } + const cacheTtlSeconds = row.cache_ttl_seconds ?? DEFAULT_CACHE_TTL_SECONDS; return { id: row.id, - bucketsQualifiedName: QuoteUtils.quoteQualifiedIdentifier(row.buckets_schema, row.buckets_table), - filesQualifiedName: QuoteUtils.quoteQualifiedIdentifier(row.files_schema, row.files_table), + bucketsQualifiedName: quoteMetadataIdentifier( + row.buckets_schema, + row.buckets_table, + `buckets:${row.id}`, + ), + filesQualifiedName: quoteMetadataIdentifier( + row.files_schema, + row.files_table, + `files:${row.id}`, + ), schemaName: row.buckets_schema, bucketsTableName: row.buckets_table, filesTableName: row.files_table, scope: row.scope, entityTableId: row.entity_table_id, entityQualifiedName: row.entity_schema && row.entity_table - ? QuoteUtils.quoteQualifiedIdentifier(row.entity_schema, row.entity_table) + ? quoteMetadataIdentifier(row.entity_schema, row.entity_table, `entity:${row.id}`) : null, endpoint: row.endpoint, publicUrlPrefix: row.public_url_prefix, @@ -170,6 +316,28 @@ function buildConfig(row: StorageModuleRow): StorageModuleConfig { }; } +function assertUnambiguousModules(configs: readonly StorageModuleConfig[]): void { + const ids = new Set(); + const scopes = new Set(); + const buckets = new Set(); + const files = new Set(); + + for (const config of configs) { + if ( + ids.has(config.id) || + scopes.has(config.scope) || + buckets.has(config.bucketsQualifiedName) || + files.has(config.filesQualifiedName) + ) { + throw new Error('STORAGE_MODULE_METADATA_AMBIGUOUS'); + } + ids.add(config.id); + scopes.add(config.scope); + buckets.add(config.bucketsQualifiedName); + files.add(config.filesQualifiedName); + } +} + /** * Resolve the app-level storage module config for a database, using the LRU cache. * @@ -183,11 +351,16 @@ function buildConfig(row: StorageModuleRow): StorageModuleConfig { export async function getStorageModuleConfig( pgClient: { query: (opts: { text: string; values?: unknown[] }) => Promise<{ rows: unknown[] }> }, databaseId: string, + cacheScope: StorageModuleCacheScope, ): Promise { + const { storageModuleCache } = cacheScope; const cacheKey = `storage:${databaseId}:app`; - const cached = storageModuleCache.get(cacheKey); - if (cached) { - return cached; + if (storageModuleCache.has(cacheKey)) { + const cached = storageModuleCache.get(cacheKey); + if (cached?.kind !== 'config') { + throw new Error('STORAGE_CACHE_INTEGRITY_ERROR'); + } + return cached.value; } log.debug(`Cache miss for app-level storage in database ${databaseId}, querying metaschema...`); @@ -196,11 +369,15 @@ export async function getStorageModuleConfig( if (result.rows.length === 0) { log.warn(`No app-level storage module found for database ${databaseId}`); + storageModuleCache.set(cacheKey, { kind: 'config', value: null }); return null; } + if (result.rows.length !== 1) { + throw new Error('STORAGE_MODULE_METADATA_AMBIGUOUS:app'); + } - const config = buildConfig(result.rows[0] as StorageModuleRow); - storageModuleCache.set(cacheKey, config); + const config = buildConfig(result.rows[0] as StorageModuleRow, databaseId); + storageModuleCache.set(cacheKey, { kind: 'config', value: config }); log.debug(`Cached app-level storage config for database ${databaseId}: ${config.bucketsQualifiedName}`); return config; @@ -225,57 +402,35 @@ export async function getStorageModuleConfigForOwner( pgClient: { query: (opts: { text: string; values?: unknown[] }) => Promise<{ rows: unknown[] }> }, databaseId: string, ownerId: string, + cacheScope: StorageModuleCacheScope, ): Promise { - // Check if we already have a cached mapping for this ownerId - const ownerCacheKey = `storage:${databaseId}:owner:${ownerId}`; - const cachedOwner = storageModuleCache.get(ownerCacheKey); - if (cachedOwner) { - return cachedOwner; - } + const allConfigs = await loadAllStorageModules(pgClient, databaseId, cacheScope); - // Load all storage modules for this database - const allModulesCacheKey = `storage:${databaseId}:all`; - let allConfigs: StorageModuleConfig[]; - const cachedAll = storageModuleCache.get(allModulesCacheKey); - if (cachedAll) { - // We stored a sentinel; re-derive from individual caches - // Actually, let's just query fresh — this is the cache-miss path - allConfigs = []; - } else { - allConfigs = []; - } - - if (allConfigs.length === 0) { - log.debug(`Loading all storage modules for database ${databaseId} to resolve ownerId ${ownerId}`); - const result = await pgClient.query({ text: ALL_STORAGE_MODULES_QUERY, values: [databaseId] }); - allConfigs = (result.rows as StorageModuleRow[]).map(buildConfig); - - // Cache each individual config by its scope - for (const config of allConfigs) { - const key = `storage:${databaseId}:scope:${config.scope}`; - storageModuleCache.set(key, config); - } - } - - // Find entity-scoped modules and probe their entity tables for the ownerId + // The module list is build-local configuration, but owner visibility is + // request/RLS-specific. Always probe it under the current pgClient instead + // of caching one principal's authorization decision for another principal. const entityModules = allConfigs.filter((c) => c.entityQualifiedName !== null); + const matches: StorageModuleConfig[] = []; for (const mod of entityModules) { const probeResult = await pgClient.query({ text: `SELECT 1 FROM ${mod.entityQualifiedName} WHERE id = $1 LIMIT 1`, values: [ownerId], }); if (probeResult.rows.length > 0) { - // Found the matching module — cache the ownerId→module mapping - storageModuleCache.set(ownerCacheKey, mod); log.debug( `Resolved ownerId ${ownerId} to storage module ${mod.id} ` + `(scope=${mod.scope}, table=${mod.bucketsQualifiedName})`, ); - return mod; + matches.push(mod); } } + if (matches.length > 1) { + throw new Error('STORAGE_MODULE_AMBIGUOUS:owner'); + } + if (matches.length === 1) return matches[0]; + log.warn(`No entity-scoped storage module found for ownerId ${ownerId} in database ${databaseId}`); return null; } @@ -299,10 +454,15 @@ export async function resolveStorageModuleByFileId( log.debug(`Resolving file ${fileId} across all storage modules for database ${databaseId}`); const allConfigs = (await pgClient.query({ text: ALL_STORAGE_MODULES_QUERY, values: [databaseId] })).rows.map( - (row: unknown) => buildConfig(row as StorageModuleRow), + (row: unknown) => buildConfig(row as StorageModuleRow, databaseId), ); + assertUnambiguousModules(allConfigs); // Probe each module's files table for the fileId + const matches: Array<{ + storageConfig: StorageModuleConfig; + file: { id: string; key: string; mime_type: string; bucket_id: string }; + }> = []; for (const config of allConfigs) { const fileResult = await pgClient.query({ text: `SELECT id, key, mime_type, bucket_id @@ -313,10 +473,15 @@ export async function resolveStorageModuleByFileId( }); if (fileResult.rows.length > 0) { const file = fileResult.rows[0] as { id: string; key: string; mime_type: string; bucket_id: string }; - return { storageConfig: config, file }; + matches.push({ storageConfig: config, file }); } } + if (matches.length > 1) { + throw new Error('STORAGE_MODULE_AMBIGUOUS:file'); + } + if (matches.length === 1) return matches[0]; + return null; } @@ -329,28 +494,27 @@ export async function resolveStorageModuleByFileId( export async function loadAllStorageModules( pgClient: { query: (opts: { text: string; values?: unknown[] }) => Promise<{ rows: unknown[] }> }, databaseId: string, + cacheScope: StorageModuleCacheScope, ): Promise { + const { storageModuleCache } = cacheScope; const cacheKey = `storage:${databaseId}:all-list`; - const cached = storageModuleCache.get(cacheKey); - if (cached) { - return (cached as any)._allConfigs as StorageModuleConfig[]; + if (storageModuleCache.has(cacheKey)) { + const cached = storageModuleCache.get(cacheKey); + if (cached?.kind !== 'list') { + throw new Error('STORAGE_CACHE_INTEGRITY_ERROR'); + } + return cached.value; } log.debug(`Loading all storage modules for database ${databaseId}`); const result = await pgClient.query({ text: ALL_STORAGE_MODULES_QUERY, values: [databaseId] }); - const configs = (result.rows as StorageModuleRow[]).map(buildConfig); - - // Cache each individual config by its scope - for (const config of configs) { - const key = `storage:${databaseId}:scope:${config.scope}`; - storageModuleCache.set(key, config); - } - - // Store the full list under a sentinel key (only if non-empty to avoid caching failed lookups) - if (configs.length > 0) { - const sentinel = { ...configs[0], _allConfigs: configs } as any; - storageModuleCache.set(cacheKey, sentinel); - } + const configs = (result.rows as StorageModuleRow[]).map((row) => + buildConfig(row, databaseId), + ); + assertUnambiguousModules(configs); + // Empty results are intentional negative cache entries. Query failures are + // never cached, so a transient control-plane error cannot become a miss. + storageModuleCache.set(cacheKey, { kind: 'list', value: configs }); return configs; } @@ -367,17 +531,21 @@ export async function loadAllStorageModules( */ export function resolveStorageConfigFromCodec( pgCodec: { name: string; extensions?: { pg?: { schemaName?: string; name?: string } }; sqlType?: string }, - allConfigs: StorageModuleConfig[], + allConfigs: readonly StorageModuleConfig[], ): StorageModuleConfig | null { const schemaName = pgCodec.extensions?.pg?.schemaName; const tableName = pgCodec.extensions?.pg?.name ?? pgCodec.name; if (!schemaName || !tableName) return null; - return allConfigs.find((c) => + const matches = allConfigs.filter((c) => (c.filesTableName === tableName && c.schemaName === schemaName) || (c.bucketsTableName === tableName && c.schemaName === schemaName), - ) || null; + ); + if (matches.length > 1) { + throw new Error('STORAGE_MODULE_AMBIGUOUS:codec'); + } + return matches[0] ?? null; } // --- Bucket metadata cache --- @@ -386,21 +554,13 @@ export function resolveStorageConfigFromCodec( * LRU cache for per-database bucket metadata. * * Buckets are essentially static config — created once and rarely changed. - * Caching avoids a DB query on every requestUploadUrl call. The bucket - * lookup in the plugin runs under RLS, but since AuthzEntityMembership - * grants all org members access to all org buckets, and the cached data - * is just config (mime types, size limits), bypassing RLS on cache hits - * is safe. The important RLS is on the files table (INSERT/UPDATE), - * which is never cached. + * Cache hits still execute an exact ID lookup through the request's RLS + * context. The cached metadata is returned only when that authorized ID + * matches, so cached data never substitutes for row authorization. * - * Keys: `bucket:${databaseId}:${storageModuleId}:${bucketKey}` - * TTL: same as storage module cache (5min dev / 1hr prod) + * Keys are local to the exact build scope; database/module identifiers never + * select cache entries belonging to another physical Graphile build. */ -const bucketCache = new LRUCache({ - max: 500, // many buckets across many databases - ttl: process.env.NODE_ENV === 'development' ? FIVE_MINUTES_MS : ONE_HOUR_MS, - updateAgeOnGet: true, -}); /** * Normalize the recorded physical coordinate at the DB boundary. @@ -432,36 +592,71 @@ export async function getBucketConfig( storageConfig: StorageModuleConfig, databaseId: string, bucketKey: string, - ownerId?: string, + ownerId: string | undefined, + cacheScope: StorageModuleCacheScope, ): Promise { + const { bucketCache } = cacheScope; const cacheKey = `bucket:${databaseId}:${storageConfig.id}:${bucketKey}${ownerId ? `:${ownerId}` : ''}`; - const cached = bucketCache.get(cacheKey); - if (cached) { - return cached; + // Entity-scoped buckets use (owner_id, key) composite lookup; + // app-level buckets just use key. + const isEntityScoped = storageConfig.scope !== 'app'; + if (isEntityScoped && !ownerId) { + throw new Error('STORAGE_OWNER_REQUIRED'); + } + const hasOwner = Boolean(ownerId && isEntityScoped); + const whereSql = hasOwner + ? 'key = $1 AND owner_id = $2' + : 'key = $1'; + const values = hasOwner ? [bucketKey, ownerId] : [bucketKey]; + + if (bucketCache.has(cacheKey)) { + // This query runs with the current request's pgSettings/RLS context. Do + // not return even immutable cached metadata without reauthorizing it. + const authorized = await pgClient.query({ + text: `SELECT id + FROM ${storageConfig.bucketsQualifiedName} + WHERE ${whereSql} + LIMIT 2`, + values, + }); + const authorizedId = (authorized.rows[0] as { id?: string } | undefined)?.id; + if (authorized.rows.length > 1) { + throw new Error('STORAGE_BUCKET_AMBIGUOUS'); + } + if (!authorizedId) { + return null; + } + + const cached = bucketCache.get(cacheKey); + if (cached?.id === authorizedId) { + return cached; + } + // A formerly missing bucket may now exist, or a bucket may have been + // replaced under the same key. Reload its immutable metadata below. } log.debug(`Bucket cache miss for ${databaseId}:${bucketKey}${ownerId ? ` (owner=${ownerId})` : ''}, querying DB...`); - // Entity-scoped buckets use (owner_id, key) composite lookup; - // app-level buckets just use key. - const isEntityScoped = storageConfig.scope !== 'app'; - const hasOwner = ownerId && isEntityScoped; const result = await pgClient.query({ text: hasOwner ? `SELECT id, key, type, is_public, owner_id, allowed_mime_types, max_file_size, allow_custom_keys, physical_name FROM ${storageConfig.bucketsQualifiedName} - WHERE key = $1 AND owner_id = $2 - LIMIT 1` + WHERE ${whereSql} + LIMIT 2` : `SELECT id, key, type, is_public, ${isEntityScoped ? 'owner_id,' : ''} allowed_mime_types, max_file_size, allow_custom_keys, physical_name FROM ${storageConfig.bucketsQualifiedName} - WHERE key = $1 - LIMIT 1`, - values: hasOwner ? [bucketKey, ownerId] : [bucketKey], + WHERE ${whereSql} + LIMIT 2`, + values, }); if (result.rows.length === 0) { + bucketCache.set(cacheKey, null); return null; } + if (result.rows.length > 1) { + throw new Error('STORAGE_BUCKET_AMBIGUOUS'); + } const row = result.rows[0] as { id: string; @@ -508,20 +703,24 @@ export async function getBucketConfig( * The set resets on server restart, which is fine because the * provisioner's createBucket is idempotent (handles "already exists"). */ -const provisionedBuckets = new Set(); - /** * Check whether an S3 bucket has already been provisioned (cached). */ -export function isS3BucketProvisioned(s3BucketName: string): boolean { - return provisionedBuckets.has(s3BucketName); +export function isS3BucketProvisioned( + s3BucketName: string, + cacheScope: StorageModuleCacheScope, +): boolean { + return cacheScope.provisionedBuckets.has(s3BucketName); } /** * Mark an S3 bucket as provisioned in the in-memory cache. */ -export function markS3BucketProvisioned(s3BucketName: string): void { - provisionedBuckets.add(s3BucketName); +export function markS3BucketProvisioned( + s3BucketName: string, + cacheScope: StorageModuleCacheScope, +): void { + cacheScope.provisionedBuckets.add(s3BucketName); log.debug(`Marked S3 bucket "${s3BucketName}" as provisioned`); } @@ -529,17 +728,19 @@ export function markS3BucketProvisioned(s3BucketName: string): void { * Clear the storage module cache AND bucket cache. * Useful for testing or schema changes. */ -export function clearStorageModuleCache(): void { - storageModuleCache.clear(); - bucketCache.clear(); - provisionedBuckets.clear(); +export function clearStorageModuleCache(cacheScope: StorageModuleCacheScope): void { + cacheScope.clear(); } /** * Clear cached bucket entries for a specific database. * Useful when bucket config changes are detected. */ -export function clearBucketCache(databaseId?: string): void { +export function clearBucketCache( + databaseId: string | undefined, + cacheScope: StorageModuleCacheScope, +): void { + const { bucketCache } = cacheScope; if (!databaseId) { bucketCache.clear(); return; diff --git a/graphile/graphile-presigned-url-plugin/src/storage-module-source.ts b/graphile/graphile-presigned-url-plugin/src/storage-module-source.ts new file mode 100644 index 0000000000..cd5ce25023 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/src/storage-module-source.ts @@ -0,0 +1,185 @@ +import { QuoteUtils } from '@pgsql/quotes'; + +import type { StorageModuleConfig } from './types'; +import { + loadAllStorageModules, + type StorageModuleCacheScope, +} from './storage-module-cache'; + +export type StoragePgClient = { + query: (opts: { + text: string; + values?: unknown[]; + }) => Promise<{ rows: any[] }>; +}; + +export type StorageWithPgClient = ( + pgSettings: unknown, + callback: (pgClient: StoragePgClient) => Promise | T, +) => Promise; + +export function assertStorageRequestContext( + withPgClient: StorageWithPgClient | null | undefined, + pgSettings: unknown, +): asserts withPgClient is StorageWithPgClient { + if (typeof withPgClient !== 'function') { + throw new Error('STORAGE_CONTEXT_UNAVAILABLE'); + } + if (typeof pgSettings !== 'object' || pgSettings === null || Array.isArray(pgSettings)) { + throw new Error('STORAGE_REQUEST_SETTINGS_UNAVAILABLE'); + } +} + +export type PreloadedStorageModules = readonly StorageModuleConfig[] | undefined; + +const QUALIFIED_IDENTIFIER = /^("(?:[^"]|"")+"|[a-z_][a-z0-9_$]*)\.("(?:[^"]|"")+"|[a-z_][a-z0-9_$]*)$/; + +function decodeIdentifier(identifier: string): string { + return identifier.startsWith('"') + ? identifier.slice(1, -1).replace(/""/g, '"') + : identifier; +} + +function normalizeQualifiedIdentifier( + value: string, + label: string, +): { schema: string; objectName: string; sql: string } { + const match = QUALIFIED_IDENTIFIER.exec(value); + if (!match) throw new Error(`STORAGE_MODULE_METADATA_INVALID:${label}`); + + const schema = decodeIdentifier(match[1]); + const objectName = decodeIdentifier(match[2]); + if ( + schema.length === 0 || + objectName.length === 0 || + schema.includes('\0') || + objectName.includes('\0') || + Buffer.byteLength(schema, 'utf8') > 63 || + Buffer.byteLength(objectName, 'utf8') > 63 + ) { + throw new Error(`STORAGE_MODULE_METADATA_INVALID:${label}`); + } + return { + schema, + objectName, + sql: QuoteUtils.quoteQualifiedIdentifier(schema, objectName), + }; +} + +/** + * Capture an immutable per-build snapshot. `undefined` deliberately remains + * distinct from an empty list: only the former enables the generic SQL path. + */ +export function snapshotPreloadedStorageModules( + modules: readonly StorageModuleConfig[] | undefined, +): PreloadedStorageModules { + if (modules === undefined) { + return undefined; + } + + const ids = new Set(); + const scopes = new Set(); + const buckets = new Set(); + const files = new Set(); + const normalized = modules.map((module) => { + if ( + !module || + typeof module.id !== 'string' || + module.id.length === 0 || + typeof module.scope !== 'string' || + module.scope.length === 0 || + typeof module.schemaName !== 'string' || + typeof module.bucketsTableName !== 'string' || + typeof module.filesTableName !== 'string' + ) { + throw new Error('STORAGE_MODULE_METADATA_INVALID'); + } + + const bucketName = normalizeQualifiedIdentifier( + module.bucketsQualifiedName, + `buckets:${module.id}`, + ); + const fileName = normalizeQualifiedIdentifier( + module.filesQualifiedName, + `files:${module.id}`, + ); + if ( + bucketName.schema !== module.schemaName || + bucketName.objectName !== module.bucketsTableName || + fileName.schema !== module.schemaName || + fileName.objectName !== module.filesTableName + ) { + throw new Error(`STORAGE_MODULE_METADATA_INCONSISTENT:${module.id}`); + } + + let entityQualifiedName: string | null = null; + if (module.scope === 'app') { + if (module.entityTableId !== null || module.entityQualifiedName !== null) { + throw new Error(`STORAGE_MODULE_METADATA_INVALID:${module.id}`); + } + } else { + if (!module.entityTableId || !module.entityQualifiedName) { + throw new Error(`STORAGE_MODULE_METADATA_INVALID:${module.id}`); + } + entityQualifiedName = normalizeQualifiedIdentifier( + module.entityQualifiedName, + `entity:${module.id}`, + ).sql; + } + + if ( + ids.has(module.id) || + scopes.has(module.scope) || + buckets.has(bucketName.sql) || + files.has(fileName.sql) + ) { + throw new Error('STORAGE_MODULE_METADATA_AMBIGUOUS'); + } + ids.add(module.id); + scopes.add(module.scope); + buckets.add(bucketName.sql); + files.add(fileName.sql); + + return Object.freeze({ + ...module, + bucketsQualifiedName: bucketName.sql, + filesQualifiedName: fileName.sql, + entityQualifiedName, + allowedOrigins: module.allowedOrigins + ? Object.freeze([...module.allowedOrigins]) + : null, + }) as StorageModuleConfig; + }); + + const unchanged = Object.isFrozen(modules) && modules.every((module, index) => + Object.isFrozen(module) && + (module.allowedOrigins === null || Object.isFrozen(module.allowedOrigins)) && + module.bucketsQualifiedName === normalized[index].bucketsQualifiedName && + module.filesQualifiedName === normalized[index].filesQualifiedName && + module.entityQualifiedName === normalized[index].entityQualifiedName, + ); + return unchanged ? modules : Object.freeze(normalized); +} + +/** + * Preloaded configuration is authoritative and never acquires a metadata + * client. The database branch exists only for generic package consumers that + * did not supply a control-plane snapshot; because callers reach this helper + * from request execution, that fallback must carry the exact request settings. + */ +export async function loadStorageModulesForBuild( + preloaded: PreloadedStorageModules, + withPgClient: StorageWithPgClient, + pgSettings: unknown, + databaseId: string, + cacheScope: StorageModuleCacheScope, +): Promise { + if (preloaded !== undefined) { + return preloaded; + } + + assertStorageRequestContext(withPgClient, pgSettings); + return withPgClient(pgSettings, (pgClient) => + loadAllStorageModules(pgClient, databaseId, cacheScope), + ); +} diff --git a/graphile/graphile-presigned-url-plugin/src/types.ts b/graphile/graphile-presigned-url-plugin/src/types.ts index 79fe686af1..2adc3182d8 100644 --- a/graphile/graphile-presigned-url-plugin/src/types.ts +++ b/graphile/graphile-presigned-url-plugin/src/types.ts @@ -149,24 +149,25 @@ export interface S3Config { /** * S3 configuration or a lazy getter that returns it on first use. - * When a function is provided, it will only be called when the first - * mutation or resolver actually needs the S3 client — avoiding eager - * env-var reads and S3Client creation at module import time. + * When a function is provided, it is called lazily once per exact Graphile + * build — avoiding eager env-var reads while keeping credentials isolated + * when a shared preset is used for multiple physical pools. */ export type S3ConfigOrGetter = S3Config | (() => S3Config); /** - * Function to derive the actual S3 bucket name for a given database and bucket key. + * Function to derive the actual S3 bucket name for a logical bucket on its + * first provision. The returned name is persisted as physical_name and is not + * recomputed for later operations. * - * When provided, the presigned URL plugin calls this on every request - * to determine which S3 bucket to use — enabling per-(database, bucketKey) - * isolation. If not provided, falls back to `s3Config.bucket` (global). + * When provided, the presigned URL plugin calls this only when physical_name + * is absent. If not provided, first provision uses `s3Config.bucket`. * - * @param databaseId - The metaschema database UUID * @param bucketKey - The logical bucket key (e.g., "public", "private") + * @param databaseId - The metaschema database UUID * @returns The S3 bucket name for this database + bucket key */ -export type BucketNameResolver = (databaseId: string, bucketKey: string) => string; +export type BucketNameResolver = (bucketKey: string, databaseId: string) => string; /** * Callback to lazily provision an S3 bucket on first use. @@ -196,6 +197,16 @@ export interface PresignedUrlPluginOptions { /** S3 configuration (concrete or lazy getter) */ s3: S3ConfigOrGetter; + /** + * Storage-module configuration resolved by the control plane for this exact + * Graphile build. Supplying this option, including an empty array, disables + * runtime metaschema SQL; the plugin treats an immutable snapshot of this + * list as authoritative for the build. + * + * Omit only for generic integrations that need the legacy database lookup. + */ + preloadedStorageModules?: readonly StorageModuleConfig[]; + /** * Optional function to resolve S3 bucket name per-database. * When set, each database gets its own S3 bucket instead of sharing diff --git a/graphile/graphile-realtime-subscriptions/README.md b/graphile/graphile-realtime-subscriptions/README.md index b546a5067c..3ef950d1e2 100644 --- a/graphile/graphile-realtime-subscriptions/README.md +++ b/graphile/graphile-realtime-subscriptions/README.md @@ -30,6 +30,29 @@ const preset = { 4. The subscription re-queries the source table with RLS enforced 5. The client receives `{ event, row }` where `row` reflects the current state +## Generation-Scoped Delivery + +`GenerationScopedRealtimeSubscriber` wraps a shared Grafast notification +source with an exact topic allowlist. Database notifications still fan out to +every generation that leased that topic, while `publish()` sends cursor +catch-up events only to subscriptions owned by that one Graphile generation. +The facade uses fixed bounded queues, fails a slow subscription on overflow, +and awaits its source iterators and source lease during `release()`. + +`RealtimeManager` accepts this explicit publisher capability. A transitional +`createPgSubscriberPublisher()` adapter retains compatibility with the current +`@dataplan/pg` subscriber, keeping its private emitter access out of the +manager. New shared-listener integrations should use the generation-scoped +facade so cursor events cannot cross generation boundaries. + +`RealtimeTopicCollector` receives the plugin's physical schema/table +descriptors during build and rejects missing, empty, changed, malformed, or +foreign topic sets. `ActivatableGenerationScopedRealtimeSubscriber` gives +PostGraphile a stable subscriber identity before schema construction, but +fails every subscribe/publish call until the validated exact-topic source is +installed. This two-phase boundary prevents an instance from serving while its +shared listener is incomplete. + ## Subscription Modes ### Phase 3a (current) diff --git a/graphile/graphile-realtime-subscriptions/__tests__/cursor-tracker.test.ts b/graphile/graphile-realtime-subscriptions/__tests__/cursor-tracker.test.ts index 05a3c9b50a..bd43487f7b 100644 --- a/graphile/graphile-realtime-subscriptions/__tests__/cursor-tracker.test.ts +++ b/graphile/graphile-realtime-subscriptions/__tests__/cursor-tracker.test.ts @@ -21,6 +21,7 @@ jest.mock('@pgpmjs/logger', () => ({ import { CursorTracker, + CursorTrackerStartAbortedError, DEFAULT_BATCH_LIMIT, DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_POLL_INTERVAL_MS, @@ -51,6 +52,16 @@ function createChangeLogEntry(overrides: Partial = {}): ChangeLo }; } +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + // --- Tests --- describe('CursorTracker defaults', () => { @@ -152,6 +163,38 @@ describe('CursorTracker.start()', () => { await tracker.stop(); }); + + it('fails readiness and rolls back when listener registration fails', async () => { + const error = new Error('touch denied'); + const pool: Queryable = { query: jest.fn().mockRejectedValue(error) }; + const onError = jest.fn(); + const tracker = new CursorTracker({ pool, onError }); + + await expect(tracker.start()).rejects.toBe(error); + + expect(tracker.isRunning).toBe(false); + expect(onError).toHaveBeenCalledWith(error); + expect((pool.query as jest.Mock).mock.calls).toHaveLength(1); + }); + + it('fails readiness and cleans up when the initial drain fails', async () => { + const error = new Error('drain denied'); + const pool: Queryable = { + query: jest.fn().mockImplementation(async (sql: string) => { + if (sql.includes('drain_changes')) throw error; + return { rows: [] }; + }) + }; + const tracker = new CursorTracker({ nodeId: 'strict-node', pool }); + + await expect(tracker.start()).rejects.toBe(error); + + expect(tracker.isRunning).toBe(false); + expect(pool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['strict-node'] + ); + }); }); describe('CursorTracker.stop()', () => { @@ -223,6 +266,112 @@ describe('CursorTracker.stop()', () => { expect(clearSpy).toHaveBeenCalledTimes(2); clearSpy.mockRestore(); }); + + it('waits for an active poll and suppresses its dispatch after stop begins', async () => { + const pool = createMockPool(); + const onChanges = jest.fn(); + const tracker = new CursorTracker({ + nodeId: 'poll-stop-node', + pool, + onChanges, + }); + await tracker.start(); + + const poll = deferred<{ rows: { drain_changes: ChangeLogEntry }[] }>(); + pool.query.mockImplementation((sql: string) => { + if (sql.includes('drain_changes')) return poll.promise; + return Promise.resolve({ rows: [] }); + }); + pool.query.mockClear(); + + const activeDrain = tracker.drain(); + const stopping = tracker.stop(); + let stopped = false; + void stopping.then(() => { + stopped = true; + }); + await Promise.resolve(); + + expect(stopped).toBe(false); + expect(pool.query.mock.calls.some(([sql]) => sql.includes('cleanup_ephemeral'))).toBe(false); + + const entry = createChangeLogEntry(); + poll.resolve({ rows: [{ drain_changes: entry }] }); + await expect(activeDrain).resolves.toEqual([entry]); + await stopping; + + expect(onChanges).not.toHaveBeenCalled(); + expect(pool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['poll-stop-node'] + ); + }); + + it('waits for an active heartbeat before cleaning up the listener', async () => { + const pool = createMockPool(); + const tracker = new CursorTracker({ + nodeId: 'heartbeat-stop-node', + pool, + }); + await tracker.start(); + + const heartbeat = deferred<{ rows: never[] }>(); + pool.query.mockImplementation((sql: string) => { + if (sql.includes('touch_listener')) return heartbeat.promise; + return Promise.resolve({ rows: [] }); + }); + pool.query.mockClear(); + + const activeHeartbeat = tracker.touchListener(); + const stopping = tracker.stop(); + let stopped = false; + void stopping.then(() => { + stopped = true; + }); + await Promise.resolve(); + + expect(stopped).toBe(false); + expect(pool.query.mock.calls.some(([sql]) => sql.includes('cleanup_ephemeral'))).toBe(false); + + heartbeat.resolve({ rows: [] }); + await activeHeartbeat; + await stopping; + + expect(pool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['heartbeat-stop-node'] + ); + }); + + it('aborts startup deterministically when stop wins the registration race', async () => { + const registration = deferred<{ rows: never[] }>(); + const pool: jest.Mocked = { + query: jest.fn().mockImplementation((sql: string) => { + if (sql.includes('touch_listener')) return registration.promise; + return Promise.resolve({ rows: [] }); + }), + }; + const tracker = new CursorTracker({ + nodeId: 'start-stop-node', + pool, + }); + + const starting = tracker.start(); + const startResult = expect(starting).rejects.toBeInstanceOf(CursorTrackerStartAbortedError); + await Promise.resolve(); + await Promise.resolve(); + expect(pool.query.mock.calls.some(([sql]) => sql.includes('touch_listener'))).toBe(true); + + const stopping = tracker.stop(); + registration.resolve({ rows: [] }); + + await startResult; + await stopping; + + expect(tracker.isRunning).toBe(false); + expect(pool.query.mock.calls.some(([sql]) => sql.includes('drain_changes'))).toBe(false); + expect(pool.query.mock.calls.filter(([sql]) => sql.includes('cleanup_ephemeral'))).toHaveLength(1); + }); }); describe('CursorTracker.drain()', () => { diff --git a/graphile/graphile-realtime-subscriptions/__tests__/generation-subscriber.test.ts b/graphile/graphile-realtime-subscriptions/__tests__/generation-subscriber.test.ts new file mode 100644 index 0000000000..8ee65a7e1c --- /dev/null +++ b/graphile/graphile-realtime-subscriptions/__tests__/generation-subscriber.test.ts @@ -0,0 +1,285 @@ +import type { GrafastSubscriber } from 'grafast'; + +import { + ActivatableGenerationScopedRealtimeSubscriber, + GENERATION_SUBSCRIBER_QUEUE_CAPACITY, + GenerationScopedRealtimeSubscriber, + RealtimeGenerationNotActiveError, + RealtimeGenerationOverflowError, + RealtimeGenerationSourceEndedError, + RealtimeGenerationTopicError +} from '../src/generation-subscriber'; + +interface Deferred { + promise: Promise; + resolve(value: T | PromiseLike): void; + reject(error: unknown): void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +class ManualIterator implements AsyncIterableIterator { + private readonly buffered: string[] = []; + private readonly waiting: Deferred>[] = []; + private failure: Error | null = null; + private done = false; + readonly returnMock = jest.fn(async (): Promise> => { + this.complete(); + return { done: true, value: undefined }; + }); + + [Symbol.asyncIterator](): AsyncIterableIterator { + return this; + } + + next(): Promise> { + const value = this.buffered.shift(); + if (value !== undefined) return Promise.resolve({ done: false, value }); + if (this.failure) return Promise.reject(this.failure); + if (this.done) return Promise.resolve({ done: true, value: undefined }); + const result = deferred>(); + this.waiting.push(result); + return result.promise; + } + + return(): Promise> { + return this.returnMock(); + } + + throw(error?: unknown): Promise> { + const failure = error instanceof Error ? error : new Error(String(error)); + this.fail(failure); + return Promise.reject(failure); + } + + push(value: string): void { + const waiter = this.waiting.shift(); + if (waiter) waiter.resolve({ done: false, value }); + else this.buffered.push(value); + } + + fail(error: Error): void { + this.failure = error; + for (const waiter of this.waiting.splice(0)) waiter.reject(error); + } + + complete(): void { + this.done = true; + for (const waiter of this.waiting.splice(0)) { + waiter.resolve({ done: true, value: undefined }); + } + } +} + +class ManualSource implements GrafastSubscriber> { + readonly streams = new Map>(); + readonly release = jest.fn(async (): Promise => {}); + + subscribe(topic: string): AsyncIterableIterator { + const stream = new ManualIterator(); + let streams = this.streams.get(topic); + if (!streams) { + streams = new Set(); + this.streams.set(topic, streams); + } + streams.add(stream); + return stream; + } + + publish(topic: string, payload: string): void { + for (const stream of this.streams.get(topic) ?? []) stream.push(payload); + } + + fail(topic: string, error: Error): void { + for (const stream of this.streams.get(topic) ?? []) stream.fail(error); + } + + complete(topic: string): void { + for (const stream of this.streams.get(topic) ?? []) stream.complete(); + } +} + +const flushMicrotasks = async (): Promise => { + for (let index = 0; index < 8; index++) await Promise.resolve(); +}; + +describe('GenerationScopedRealtimeSubscriber', () => { + it('merges database notifications with generation-local cursor publications', async () => { + const source = new ManualSource(); + const facade = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['realtime:tenant_a.contacts'] + }); + const stream = facade.subscribe('realtime:tenant_a.contacts'); + await flushMicrotasks(); + + source.publish('realtime:tenant_a.contacts', 'INSERT:db-row'); + await expect(stream.next()).resolves.toMatchObject({ value: 'INSERT:db-row' }); + + facade.publish('realtime:tenant_a.contacts', 'UPDATE:cursor-row'); + await expect(stream.next()).resolves.toMatchObject({ value: 'UPDATE:cursor-row' }); + await facade.release(); + expect(source.release).toHaveBeenCalledTimes(1); + }); + + it('enforces exact allowlists rather than prefixes', async () => { + const source = new ManualSource(); + const facade = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['realtime:tenant.contacts'] + }); + + expect(() => facade.subscribe('realtime:tenant.contacts.private')) + .toThrow(RealtimeGenerationTopicError); + expect(() => facade.publish('realtime:tenant', 'INSERT:wrong')) + .toThrow(RealtimeGenerationTopicError); + await facade.release(); + }); + + it('keeps cursor publications inside their Graphile generation', async () => { + const source = new ManualSource(); + const first = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['realtime:shared.contacts'], + releaseSourceOnRelease: false + }); + const second = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['realtime:shared.contacts'], + releaseSourceOnRelease: false + }); + const firstStream = first.subscribe('realtime:shared.contacts'); + const secondStream = second.subscribe('realtime:shared.contacts'); + await flushMicrotasks(); + + first.publish('realtime:shared.contacts', 'INSERT:first-cursor'); + await expect(firstStream.next()).resolves.toMatchObject({ + value: 'INSERT:first-cursor' + }); + + source.publish('realtime:shared.contacts', 'UPDATE:database'); + await expect(firstStream.next()).resolves.toMatchObject({ value: 'UPDATE:database' }); + await expect(secondStream.next()).resolves.toMatchObject({ value: 'UPDATE:database' }); + await Promise.all([first.release(), second.release()]); + }); + + it('fails an overflowing local subscriber without poisoning its peers', async () => { + const source = new ManualSource(); + const facade = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['realtime:events'] + }); + const slow = facade.subscribe('realtime:events'); + + for (let index = 0; index <= GENERATION_SUBSCRIBER_QUEUE_CAPACITY; index++) { + facade.publish('realtime:events', `INSERT:${index}`); + } + await expect(slow.next()).rejects.toBeInstanceOf(RealtimeGenerationOverflowError); + + const healthy = facade.subscribe('realtime:events'); + facade.publish('realtime:events', 'INSERT:healthy'); + await expect(healthy.next()).resolves.toMatchObject({ value: 'INSERT:healthy' }); + await facade.release(); + }); + + it('propagates source failure and unexpected completion', async () => { + const source = new ManualSource(); + const facade = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['a', 'b'] + }); + const failed = facade.subscribe('a'); + const ended = facade.subscribe('b'); + await flushMicrotasks(); + + source.fail('a', new Error('listener failed')); + source.complete('b'); + + await expect(failed.next()).rejects.toThrow('listener failed'); + await expect(ended.next()).rejects.toBeInstanceOf( + RealtimeGenerationSourceEndedError + ); + await facade.release(); + }); + + it('makes release idempotent and awaits stream and source teardown', async () => { + const source = new ManualSource(); + const streamReleased = deferred>(); + const sourceReleased = deferred(); + const facade = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['a'] + }); + facade.subscribe('a'); + await flushMicrotasks(); + const sourceStream = [...source.streams.get('a')!][0]; + sourceStream.returnMock.mockImplementation(async () => streamReleased.promise); + source.release.mockImplementation(async () => sourceReleased.promise); + + const first = facade.release(); + const second = facade.release(); + expect(first).toBe(second); + await flushMicrotasks(); + expect(source.release).not.toHaveBeenCalled(); + + streamReleased.resolve({ done: true, value: undefined }); + await flushMicrotasks(); + expect(source.release).toHaveBeenCalledTimes(1); + + let settled = false; + void first.then(() => { + settled = true; + }); + await flushMicrotasks(); + expect(settled).toBe(false); + sourceReleased.resolve(); + await first; + expect(settled).toBe(true); + }); +}); + +describe('ActivatableGenerationScopedRealtimeSubscriber', () => { + it('fails closed before activation and owns an activated source exactly once', async () => { + const source = new ManualSource(); + const facade = new ActivatableGenerationScopedRealtimeSubscriber(); + + expect(() => facade.subscribe('realtime:tenant_a.contacts')) + .toThrow(RealtimeGenerationNotActiveError); + await facade.activate({ + source, + allowedTopics: ['realtime:tenant_a.contacts'] + }); + + const stream = facade.subscribe('realtime:tenant_a.contacts'); + await flushMicrotasks(); + source.publish('realtime:tenant_a.contacts', 'INSERT:row-a'); + await expect(stream.next()).resolves.toMatchObject({ value: 'INSERT:row-a' }); + + const first = facade.release(); + const second = facade.release(); + expect(first).toBe(second); + await first; + expect(source.release).toHaveBeenCalledTimes(1); + }); + + it('releases a rejected second activation source', async () => { + const firstSource = new ManualSource(); + const secondSource = new ManualSource(); + const facade = new ActivatableGenerationScopedRealtimeSubscriber(); + await facade.activate({ source: firstSource, allowedTopics: ['a'] }); + + await expect(facade.activate({ source: secondSource, allowedTopics: ['a'] })) + .rejects.toMatchObject({ code: 'REALTIME_GENERATION_ALREADY_ACTIVE' }); + expect(secondSource.release).toHaveBeenCalledTimes(1); + await facade.release(); + expect(firstSource.release).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphile/graphile-realtime-subscriptions/__tests__/plugin.test.ts b/graphile/graphile-realtime-subscriptions/__tests__/plugin.test.ts index 723560853a..8e5992522f 100644 --- a/graphile/graphile-realtime-subscriptions/__tests__/plugin.test.ts +++ b/graphile/graphile-realtime-subscriptions/__tests__/plugin.test.ts @@ -12,7 +12,7 @@ * - NOTIFY payload parsing (TG_OP:id1,id2,... and INVALIDATE) * - Per-subscriber event throttling with configurable limit * - Sparse set subscriptions (ids: [UUID!]) with row ID intersection filtering - * - RLS-aware rowId masking in payload resolvers + * - RLS-aware event suppression and rowId masking */ jest.mock('@pgpmjs/logger', () => ({ @@ -26,17 +26,20 @@ jest.mock('@pgpmjs/logger', () => ({ const mockListen = jest.fn(); const mockConstant = jest.fn((val: any) => `constant(${val})`); -const mockObject = jest.fn((obj: any) => obj); const mockLambda = jest.fn((input: any, fn: Function) => fn(input)); +const mockGet = jest.fn((parent: any, key: string) => + typeof parent?.get === 'function' ? parent.get(key) : parent?.[key] +); +let mockPgSubscriber: any = 'mock-pgSubscriber'; const mockContext = jest.fn(() => ({ - get: jest.fn((key: string) => `mock-${key}`), + get: jest.fn((key: string) => key === 'pgSubscriber' ? mockPgSubscriber : `mock-${key}`), })); jest.mock('grafast', () => ({ context: mockContext, listen: mockListen, - object: mockObject, constant: mockConstant, + get: mockGet, lambda: mockLambda, })); @@ -59,6 +62,7 @@ import { EventThrottle, parseNotifyPayload, RealtimeSubscriptionsPlugin, + selectCandidateRowId, } from '../src/plugin'; // --- Test helpers --- @@ -82,8 +86,48 @@ function createMockCodec( }; } -function createMockResource(name: string, codec: any) { - return { codec, name }; +function createMockExecutorContext( + visibleIds: readonly string[] = [], + pgSettings: Record = { role: 'tenant_runtime' }, +) { + const visible = new Set(visibleIds); + const query = jest.fn(async ({ values }: { text: string; values?: unknown[] }) => { + const requestedIds = (values?.[0] ?? []) as string[]; + const rows = requestedIds + .filter((rowId) => visible.has(rowId)) + .map((id) => ({ id })); + return { rows }; + }); + const withPgClient = jest.fn(async (_settings: unknown, callback: Function) => + callback({ query }) + ); + + return { + executorContext: { pgSettings, withPgClient }, + query, + withPgClient, + }; +} + +function createMockResource(name: string, codec: any, executorContext?: any) { + const context = executorContext ?? createMockExecutorContext().executorContext; + return { + codec, + name, + executor: { + context: jest.fn(() => context), + }, + }; +} + +async function* notifications(payloads: readonly string[]) { + for (const payload of payloads) yield payload; +} + +async function collectNotifications(iterable: AsyncIterable) { + const result: unknown[] = []; + for await (const payload of iterable) result.push(payload); + return result; } function createMockBuild(resources: Record, inflectionOverrides: Record = {}) { @@ -227,6 +271,7 @@ describe('createRealtimeSubscriptionsPlugin', () => { beforeEach(() => { jest.clearAllMocks(); capturedFactory = null; + mockPgSubscriber = 'mock-pgSubscriber'; }); describe('plugin structure', () => { @@ -243,6 +288,38 @@ describe('createRealtimeSubscriptionsPlugin', () => { }); describe('table discovery', () => { + it('reports sorted credential-free physical topic descriptors during build', () => { + const onTopicsDiscovered = jest.fn(); + createRealtimeSubscriptionsPlugin({ onTopicsDiscovered }); + + const zeta = createMockCodec('zeta', { + realtime: true, + schemaName: 'tenant_a' + }); + const alpha = createMockCodec('alpha', { + realtime: true, + schemaName: 'tenant_a' + }); + capturedFactory!(createMockBuild({ + zeta: createMockResource('zeta', zeta), + alpha: createMockResource('alpha', alpha) + })); + + expect(onTopicsDiscovered).toHaveBeenCalledTimes(1); + expect(onTopicsDiscovered).toHaveBeenCalledWith([ + { topic: 'realtime:tenant_a.alpha', schema: 'tenant_a', table: 'alpha' }, + { topic: 'realtime:tenant_a.zeta', schema: 'tenant_a', table: 'zeta' } + ]); + }); + + it('reports an explicit empty topic set', () => { + const onTopicsDiscovered = jest.fn(); + createRealtimeSubscriptionsPlugin({ onTopicsDiscovered }); + capturedFactory!(createMockBuild({})); + + expect(onTopicsDiscovered).toHaveBeenCalledWith([]); + }); + it('discovers tables with @realtime tag', () => { createRealtimeSubscriptionsPlugin(); @@ -344,7 +421,7 @@ describe('createRealtimeSubscriptionsPlugin', () => { expect(result.typeDefs).toContain('documents: Documents'); expect(result.typeDefs).toContain('rowId: UUID'); expect(result.typeDefs).toContain('overflow: Boolean!'); - expect(result.typeDefs).toContain('masked when RLS denies access'); + expect(result.typeDefs).toContain('after RLS authorization'); }); it('extends Subscription type', () => { @@ -379,7 +456,7 @@ describe('createRealtimeSubscriptionsPlugin', () => { expect(result.plans['Subscription']).toBeDefined(); expect(result.plans['Subscription']['onProjectsChanged']).toBeDefined(); - const mockArgs = { getRaw: jest.fn(() => 'test-id') }; + const mockArgs = { getRaw: jest.fn(() => ['test-id']) }; result.plans['Subscription']['onProjectsChanged'].subscribePlan(null, mockArgs); expect(mockConstant).toHaveBeenCalledWith('realtime:app_public.projects'); @@ -398,7 +475,7 @@ describe('createRealtimeSubscriptionsPlugin', () => { const result = capturedFactory!(build); - const mockArgs = { getRaw: jest.fn(() => 'test-id') }; + const mockArgs = { getRaw: jest.fn(() => ['test-id']) }; result.plans['Subscription']['onItemsChanged'].subscribePlan(null, mockArgs); expect(mockConstant).toHaveBeenCalledWith('realtime:inventory_public.items'); @@ -430,7 +507,7 @@ describe('createRealtimeSubscriptionsPlugin', () => { }); const result = capturedFactory!(build); - const mockArgs = { getRaw: jest.fn(() => 'some-id') }; + const mockArgs = { getRaw: jest.fn(() => ['some-id']) }; result.plans['Subscription']['onTasksChanged'].subscribePlan(null, mockArgs); @@ -564,12 +641,19 @@ describe('createRealtimeSubscriptionsPlugin', () => { }); describe('sparse set filtering (ids argument)', () => { - it('subscribePlan passes ids through object step', () => { + it('threads ids into the pre-delivery authorization filter', async () => { createRealtimeSubscriptionsPlugin(); const codec = createMockCodec('tasks', { realtime: true }); + const { executorContext, query } = createMockExecutorContext(['id-a']); + mockPgSubscriber = { + subscribe: jest.fn(() => notifications([ + 'INSERT:id-other', + 'UPDATE:id-a', + ])), + }; const build = createMockBuild({ - tasks: createMockResource('tasks', codec), + tasks: createMockResource('tasks', codec, executorContext), }); const result = capturedFactory!(build); @@ -581,46 +665,28 @@ describe('createRealtimeSubscriptionsPlugin', () => { result.plans['Subscription']['onTasksChanged'].subscribePlan(null, mockArgs); expect(mockArgs.getRaw).toHaveBeenCalledWith('ids'); - - // The listen callback is captured but not invoked by the mock. - // Invoke it manually to verify ids are threaded through. expect(mockListen).toHaveBeenCalled(); - const listenCallback = mockListen.mock.calls[mockListen.mock.calls.length - 1][2]; - listenCallback('INSERT:id-a'); - - expect(mockObject).toHaveBeenCalled(); - const objectArg = mockObject.mock.calls[mockObject.mock.calls.length - 1][0]; - expect(objectArg).toHaveProperty('subscribedIds'); - }); - - it('drops events with no row ID intersection in sparse set mode', () => { - const parsed = parseNotifyPayload('INSERT:id-x,id-y'); - const subscribedIds = ['id-a', 'id-b']; - - const hasMatch = parsed.rowIds.some((rid: string) => subscribedIds.includes(rid)); - expect(hasMatch).toBe(false); - }); - - it('delivers events with row ID intersection in sparse set mode', () => { - const parsed = parseNotifyPayload('UPDATE:id-a,id-x'); - const subscribedIds = ['id-a', 'id-b']; - - const hasMatch = parsed.rowIds.some((rid: string) => subscribedIds.includes(rid)); - expect(hasMatch).toBe(true); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + const events = await collectNotifications( + authorizedSubscriber.subscribe('realtime:app_public.tasks'), + ); + + expect(events).toEqual([{ + parsed: { event: 'UPDATE', rowIds: ['id-a'], overflow: false }, + subscribedIds: ['id-a', 'id-b'], + }]); + expect(query).toHaveBeenCalledTimes(1); + expect(query.mock.calls[0][0].values).toEqual([['id-a']]); }); - it('delivers INVALIDATE events regardless of sparse set', () => { - const parsed = parseNotifyPayload('INVALIDATE'); - expect(parsed.overflow).toBe(true); - expect(parsed.rowIds).toEqual([]); - }); - - it('rowId resolver returns first matching ID from sparse set', () => { + it('rowId resolver returns a sparse-set ID only when RLS exposes the row', () => { createRealtimeSubscriptionsPlugin(); const codec = createMockCodec('tasks', { realtime: true }); + const getAuthorizedId = jest.fn(() => 'id-b'); + const get = jest.fn(() => ({ get: getAuthorizedId })); const build = createMockBuild({ - tasks: { ...createMockResource('tasks', codec), get: jest.fn() }, + tasks: { ...createMockResource('tasks', codec), get }, }); const result = capturedFactory!(build); @@ -632,38 +698,46 @@ describe('createRealtimeSubscriptionsPlugin', () => { return null; }) }; - payload.rowId(mockParent); + expect(payload.rowId(mockParent)).toBe('id-b'); expect(mockParent.get).toHaveBeenCalledWith('parsed'); expect(mockParent.get).toHaveBeenCalledWith('subscribedIds'); + expect(get).toHaveBeenCalledWith({ id: 'id-b' }); + expect(getAuthorizedId).toHaveBeenCalledWith('id'); }); - it('rowId resolver returns null when no sparse set match', () => { + it('rowId resolver returns null when RLS hides a sparse-set row', () => { createRealtimeSubscriptionsPlugin(); const codec = createMockCodec('tasks', { realtime: true }); + const getAuthorizedId = jest.fn((): null => null); + const get = jest.fn(() => ({ get: getAuthorizedId })); const build = createMockBuild({ - tasks: { ...createMockResource('tasks', codec), get: jest.fn() }, + tasks: { ...createMockResource('tasks', codec), get }, }); const result = capturedFactory!(build); const payload = result.plans['TasksSubscriptionPayload']; const mockParent = { get: jest.fn((key: string) => { - if (key === 'parsed') return { event: 'INSERT', rowIds: ['id-x'], overflow: false }; + if (key === 'parsed') return { event: 'INSERT', rowIds: ['id-a'], overflow: false }; if (key === 'subscribedIds') return ['id-a', 'id-b']; return null; }) }; - payload.rowId(mockParent); + expect(payload.rowId(mockParent)).toBeNull(); expect(mockParent.get).toHaveBeenCalledWith('subscribedIds'); + expect(get).toHaveBeenCalledWith({ id: 'id-a' }); + expect(getAuthorizedId).toHaveBeenCalledWith('id'); }); - it('rowId resolver falls back to first rowId when no sparse set provided', () => { + it('rowId resolver never exposes IDs in collection mode', () => { createRealtimeSubscriptionsPlugin(); const codec = createMockCodec('tasks', { realtime: true }); + const getAuthorizedId = jest.fn((): null => null); + const get = jest.fn(() => ({ get: getAuthorizedId })); const build = createMockBuild({ - tasks: { ...createMockResource('tasks', codec), get: jest.fn() }, + tasks: { ...createMockResource('tasks', codec), get }, }); const result = capturedFactory!(build); @@ -675,13 +749,226 @@ describe('createRealtimeSubscriptionsPlugin', () => { return null; }) }; - payload.rowId(mockParent); + expect(payload.rowId(mockParent)).toBeNull(); expect(mockParent.get).toHaveBeenCalledWith('subscribedIds'); + expect(get).toHaveBeenCalledWith({ id: null }); + expect(getAuthorizedId).toHaveBeenCalledWith('id'); + }); + + it.each(['DELETE', 'INVALIDATE'])('rowId resolver never exposes IDs for %s', (event) => { + createRealtimeSubscriptionsPlugin(); + + const codec = createMockCodec('tasks', { realtime: true }); + const getAuthorizedId = jest.fn((): null => null); + const get = jest.fn(() => ({ get: getAuthorizedId })); + const build = createMockBuild({ + tasks: { ...createMockResource('tasks', codec), get }, + }); + + const result = capturedFactory!(build); + const payload = result.plans['TasksSubscriptionPayload']; + const mockParent = { get: jest.fn((key: string) => { + if (key === 'parsed') { + return { + event, + rowIds: event === 'INVALIDATE' ? [] : ['id-a'], + overflow: event === 'INVALIDATE', + }; + } + if (key === 'subscribedIds') return ['id-a']; + return null; + }) }; + + expect(payload.rowId(mockParent)).toBeNull(); + expect(get).toHaveBeenCalledWith({ id: null }); + expect(getAuthorizedId).toHaveBeenCalledWith('id'); }); }); describe('RLS-aware event delivery', () => { - it('rowId doc comment mentions RLS masking', () => { + it('suppresses an unauthorized collection event before Grafast can emit its timing or type', async () => { + createRealtimeSubscriptionsPlugin(); + + const codec = createMockCodec('items', { realtime: true }); + const { executorContext, query, withPgClient } = createMockExecutorContext( + ['visible-id'], + { role: 'tenant_a', 'jwt.claims.tenant_id': 'tenant-a' }, + ); + mockPgSubscriber = { + subscribe: jest.fn(() => notifications([ + 'INSERT:hidden-id', + 'UPDATE:hidden-id,visible-id', + ])), + }; + const build = createMockBuild({ + items: createMockResource('items', codec, executorContext), + }); + + const result = capturedFactory!(build); + result.plans['Subscription']['onItemsChanged'].subscribePlan( + null, + { getRaw: jest.fn((): null => null) }, + ); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + const events = await collectNotifications( + authorizedSubscriber.subscribe('realtime:app_public.items'), + ); + + expect(events).toEqual([{ + parsed: { event: 'UPDATE', rowIds: ['visible-id'], overflow: false }, + subscribedIds: null, + }]); + expect(query.mock.calls.map(([request]) => request.values)).toEqual([ + [['hidden-id']], + [['hidden-id', 'visible-id']], + ]); + expect(withPgClient).toHaveBeenCalledWith( + { role: 'tenant_a', 'jwt.claims.tenant_id': 'tenant-a' }, + expect.any(Function), + ); + }); + + it('suppresses DELETE, database INVALIDATE, and malformed operations without querying', async () => { + createRealtimeSubscriptionsPlugin(); + + const codec = createMockCodec('items', { realtime: true }); + const { executorContext, query } = createMockExecutorContext(['visible-id']); + mockPgSubscriber = { + subscribe: jest.fn(() => notifications([ + 'DELETE:visible-id', + 'INVALIDATE', + 'UPDATE', + 'TRUNCATE:visible-id', + 'INSERT:visible-id', + ])), + }; + const build = createMockBuild({ + items: createMockResource('items', codec, executorContext), + }); + + const result = capturedFactory!(build); + result.plans['Subscription']['onItemsChanged'].subscribePlan( + null, + { getRaw: jest.fn((): null => null) }, + ); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + const events = await collectNotifications( + authorizedSubscriber.subscribe('realtime:app_public.items'), + ); + + expect(events).toEqual([{ + parsed: { event: 'INSERT', rowIds: ['visible-id'], overflow: false }, + subscribedIds: null, + }]); + expect(query).toHaveBeenCalledTimes(1); + }); + + it('fails closed when the RLS visibility query errors', async () => { + createRealtimeSubscriptionsPlugin(); + + const codec = createMockCodec('items', { realtime: true }); + const withPgClient = jest.fn(async () => { + throw new Error('database unavailable'); + }); + const executorContext = { + pgSettings: { role: 'tenant_runtime' }, + withPgClient, + }; + mockPgSubscriber = { + subscribe: jest.fn(() => notifications(['INSERT:possibly-visible-id'])), + }; + const build = createMockBuild({ + items: createMockResource('items', codec, executorContext), + }); + + const result = capturedFactory!(build); + result.plans['Subscription']['onItemsChanged'].subscribePlan( + null, + { getRaw: jest.fn((): null => null) }, + ); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + + await expect(collectNotifications( + authorizedSubscriber.subscribe('realtime:app_public.items'), + )).resolves.toEqual([]); + expect(withPgClient).toHaveBeenCalledTimes(1); + }); + + it('quotes physical identifiers and binds hostile row IDs as values', async () => { + createRealtimeSubscriptionsPlugin(); + + const hostileId = "00000000-0000-0000-0000-000000000000' OR true --"; + const codec = createMockCodec('tasks', { + realtime: true, + schemaName: 'tenant"; set role postgres; --', + }); + codec.extensions.pg.name = 'tasks"; drop table audit; --'; + const { executorContext, query } = createMockExecutorContext([hostileId]); + mockPgSubscriber = { + subscribe: jest.fn(() => notifications([`INSERT:${hostileId}`])), + }; + const build = createMockBuild({ + tasks: createMockResource('tasks', codec, executorContext), + }); + + const result = capturedFactory!(build); + result.plans['Subscription']['onTasksChanged'].subscribePlan( + null, + { getRaw: jest.fn((): null => null) }, + ); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + await collectNotifications( + authorizedSubscriber.subscribe('realtime:hostile'), + ); + + const request = query.mock.calls[0][0]; + expect(request.text).toContain( + '"tenant""; set role postgres; --"."tasks""; drop table audit; --"', + ); + expect(request.text).toContain('any($1::text[])'); + expect(request.text).not.toContain(hostileId); + expect(request.values).toEqual([[hostileId]]); + }); + + it('counts only authorized events toward the subscriber throttle', async () => { + createRealtimeSubscriptionsPlugin({ overflowThreshold: 1 }); + + const codec = createMockCodec('items', { realtime: true }); + const { executorContext } = createMockExecutorContext(['visible-a', 'visible-b']); + mockPgSubscriber = { + subscribe: jest.fn(() => notifications([ + 'INSERT:hidden-id', + 'INSERT:visible-a', + 'UPDATE:visible-b', + ])), + }; + const build = createMockBuild({ + items: createMockResource('items', codec, executorContext), + }); + + const result = capturedFactory!(build); + result.plans['Subscription']['onItemsChanged'].subscribePlan( + null, + { getRaw: jest.fn((): null => null) }, + ); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + const events = await collectNotifications( + authorizedSubscriber.subscribe('realtime:app_public.items'), + ); + + expect(events).toEqual([ + { + parsed: { event: 'INSERT', rowIds: ['visible-a'], overflow: false }, + subscribedIds: null, + }, + { + parsed: { event: 'INVALIDATE', rowIds: [], overflow: true }, + subscribedIds: null, + }, + ]); + }); + + it('rowId doc comment states the fail-closed visibility rules', () => { createRealtimeSubscriptionsPlugin(); const codec = createMockCodec('items', { realtime: true }); @@ -690,7 +977,8 @@ describe('createRealtimeSubscriptionsPlugin', () => { }); const result = capturedFactory!(build); - expect(result.typeDefs).toContain('masked when RLS denies access'); + expect(result.typeDefs).toContain('after RLS authorization'); + expect(result.typeDefs).toContain('Null for collection, INVALIDATE, or denied rows'); }); it('type defs include sparse set ids argument', () => { @@ -719,3 +1007,25 @@ describe('createRealtimeSubscriptionsPlugin', () => { }); }); }); + +describe('selectCandidateRowId', () => { + const insert = { event: 'INSERT', rowIds: ['id-a', 'id-b'], overflow: false }; + + it('allows collection row fetching without allowing collection rowId exposure', () => { + expect(selectCandidateRowId(insert, null, true)).toBe('id-a'); + expect(selectCandidateRowId(insert, null, false)).toBeNull(); + }); + + it('selects only a caller-supplied sparse ID', () => { + expect(selectCandidateRowId(insert, ['id-b'], false)).toBe('id-b'); + expect(selectCandidateRowId(insert, ['id-x'], false)).toBeNull(); + }); + + it.each(['DELETE', 'INVALIDATE', 'UNKNOWN'])('rejects %s before any row lookup', (event) => { + expect(selectCandidateRowId({ + event, + rowIds: ['id-a'], + overflow: event === 'INVALIDATE', + }, ['id-a'], false)).toBeNull(); + }); +}); diff --git a/graphile/graphile-realtime-subscriptions/__tests__/realtime-manager.test.ts b/graphile/graphile-realtime-subscriptions/__tests__/realtime-manager.test.ts index c0d10650e0..3d97d1209d 100644 --- a/graphile/graphile-realtime-subscriptions/__tests__/realtime-manager.test.ts +++ b/graphile/graphile-realtime-subscriptions/__tests__/realtime-manager.test.ts @@ -1,7 +1,14 @@ import { EventEmitter } from 'events'; -import { RealtimeManager } from '../src/realtime-manager'; -import { entryToChannel,entryToNotifyPayload, extractRowId } from '../src/realtime-manager'; +import { + entryToChannel, + entryToNotifyPayload, + extractRowId, + RealtimeManager, + RealtimeSourceSchemaConfigurationError, + RealtimeSourceSchemaViolationError, + RealtimeSubscriberUnavailableError +} from '../src/realtime-manager'; import type { ChangeLogEntry, Queryable } from '../src/types'; // --------------------------------------------------------------------------- @@ -34,6 +41,20 @@ function createMockPgSubscriber() { return { eventEmitter, subscribe: jest.fn() }; } +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +async function flushMicrotasks(): Promise { + for (let i = 0; i < 6; i++) await Promise.resolve(); +} + // --------------------------------------------------------------------------- // Unit tests: helper functions // --------------------------------------------------------------------------- @@ -129,6 +150,7 @@ describe('RealtimeManager', () => { return new RealtimeManager({ pgSubscriber: mockSubscriber, pool: mockPool, + allowedSourceSchemas: ['public', 'billing'], nodeId: 'test-manager-node', pollIntervalMs: 1000, heartbeatIntervalMs: 5000, @@ -175,6 +197,108 @@ describe('RealtimeManager', () => { ); }); + it('fails startup before registration when the subscriber emitter is unavailable', async () => { + const manager = createManager({ pgSubscriber: {} }); + + await expect(manager.start()).rejects.toBeInstanceOf( + RealtimeSubscriberUnavailableError + ); + + expect(manager.isRunning).toBe(false); + expect(mockPool.query).not.toHaveBeenCalled(); + }); + + it('uses an explicit publisher without inspecting PgSubscriber internals', async () => { + const publish = jest.fn(); + const opaqueSubscriber = Object.defineProperty({}, 'eventEmitter', { + get() { + throw new Error('private field accessed'); + } + }); + mockPool.query.mockImplementation(async (sql: string) => { + if (sql.includes('drain_changes')) { + return { + rows: [{ + drain_changes: makeEntry({ payload_after: { id: 'cursor-row' } }) + }] + }; + } + return { rows: [] }; + }); + const manager = createManager({ + publisher: { publish }, + pgSubscriber: opaqueSubscriber + }); + + await manager.start(); + expect(publish).toHaveBeenCalledWith( + 'realtime:public.contact', + 'INSERT:cursor-row' + ); + await manager.stop(); + }); + + it('fails the generation when the explicit publisher rejects delivery', async () => { + const failure = new Error('generation released'); + const fatalErrors: Error[] = []; + mockPool.query.mockImplementation(async (sql: string) => { + if (sql.includes('drain_changes')) { + return { rows: [{ drain_changes: makeEntry() }] }; + } + return { rows: [] }; + }); + const manager = createManager({ + publisher: { + publish() { + throw failure; + } + }, + onFatalError: (error: Error) => fatalErrors.push(error) + }); + + await expect(manager.start()).rejects.toBe(failure); + expect(fatalErrors).toEqual([failure]); + expect(manager.isRunning).toBe(false); + }); + + it('preflights every cursor topic before publishing any row in the batch', async () => { + const publish = jest.fn(); + const topicFailure = new Error('topic outside generation'); + mockPool.query.mockImplementation(async (sql: string) => { + if (sql.includes('drain_changes')) { + return { + rows: [ + { drain_changes: makeEntry({ source_table: 'contact' }) }, + { drain_changes: makeEntry({ source_table: 'private_table' }) } + ] + }; + } + return { rows: [] }; + }); + const manager = createManager({ + publisher: { + assertTopics(topics: readonly string[]) { + if (topics.includes('realtime:public.private_table')) throw topicFailure; + }, + publish + } + }); + + await expect(manager.start()).rejects.toBe(topicFailure); + expect(publish).not.toHaveBeenCalled(); + }); + + it('fails startup before registration when no source schema is allowed', async () => { + const manager = createManager({ allowedSourceSchemas: [] }); + + await expect(manager.start()).rejects.toBeInstanceOf( + RealtimeSourceSchemaConfigurationError + ); + + expect(manager.isRunning).toBe(false); + expect(mockPool.query).not.toHaveBeenCalled(); + }); + it('is idempotent for start', async () => { const manager = createManager(); await manager.start(); @@ -190,7 +314,184 @@ describe('RealtimeManager', () => { await manager.stop(); // should be no-op }); + it('fails a running generation when periodic cursor polling fails', async () => { + const failure = new Error('periodic drain failed'); + const errors: Error[] = []; + const fatalErrors: Error[] = []; + let rejectDrain = false; + mockPool.query.mockImplementation(async (sql: string) => { + if (rejectDrain && sql.includes('drain_changes')) throw failure; + return { rows: [] }; + }); + const manager = createManager({ + onError: (error: Error) => errors.push(error), + onFatalError: (error: Error) => fatalErrors.push(error) + }); + + await manager.start(); + rejectDrain = true; + await jest.advanceTimersByTimeAsync(1000); + await flushMicrotasks(); + await manager.stop(); + + expect(errors).toEqual([failure]); + expect(fatalErrors).toEqual([failure]); + expect(manager.isRunning).toBe(false); + expect(mockPool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['test-manager-node'] + ); + }); + + it('fails a running generation when its periodic heartbeat fails', async () => { + const failure = new Error('periodic heartbeat failed'); + const errors: Error[] = []; + const fatalErrors: Error[] = []; + let rejectHeartbeat = false; + mockPool.query.mockImplementation(async (sql: string) => { + if (rejectHeartbeat && sql.includes('touch_listener')) throw failure; + return { rows: [] }; + }); + const manager = createManager({ + onError: (error: Error) => errors.push(error), + onFatalError: (error: Error) => fatalErrors.push(error) + }); + + await manager.start(); + rejectHeartbeat = true; + await jest.advanceTimersByTimeAsync(5000); + await flushMicrotasks(); + await manager.stop(); + + expect(errors).toEqual([failure]); + expect(fatalErrors).toEqual([failure]); + expect(manager.isRunning).toBe(false); + expect(mockPool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['test-manager-node'] + ); + }); + + it('does not dispatch a deferred startup drain after stop begins', async () => { + const entry = makeEntry({ payload_after: { id: 'late-row' } }); + const drain = deferred<{ rows: { drain_changes: ChangeLogEntry }[] }>(); + const emitted: string[] = []; + mockSubscriber.eventEmitter.on('realtime:public.contact', (payload: string) => { + emitted.push(payload); + }); + mockPool.query.mockImplementation((sql: string) => { + if (sql.includes('drain_changes')) return drain.promise; + return Promise.resolve({ rows: [] }); + }); + + const manager = createManager(); + const starting = manager.start(); + const startResult = expect(starting).rejects.toMatchObject({ + code: 'CURSOR_TRACKER_START_ABORTED', + }); + await flushMicrotasks(); + expect(mockPool.query.mock.calls.some(([sql]) => sql.includes('drain_changes'))).toBe(true); + + const stopping = manager.stop(); + drain.resolve({ rows: [{ drain_changes: entry }] }); + + await startResult; + await stopping; + + expect(emitted).toEqual([]); + expect(manager.isRunning).toBe(false); + expect(mockPool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['test-manager-node'] + ); + }); + describe('event dispatching', () => { + it('rejects a mixed batch atomically when it contains a foreign source schema', async () => { + const emitted: string[] = []; + const errors: Error[] = []; + const fatalErrors: Error[] = []; + mockSubscriber.eventEmitter.on('realtime:public.contact', (payload: string) => { + emitted.push(payload); + }); + const entries = [ + makeEntry({ payload_after: { id: 'allowed-row' } }), + makeEntry({ + source_schema: 'tenant_b', + payload_after: { id: 'foreign-row' } + }) + ]; + mockPool.query.mockImplementation(async (sql: string) => { + if (sql.includes('drain_changes')) { + return { rows: entries.map((entry) => ({ drain_changes: entry })) }; + } + return { rows: [] }; + }); + + const manager = createManager({ + allowedSourceSchemas: ['public'], + onError: (error: Error) => errors.push(error), + onFatalError: (error: Error) => fatalErrors.push(error) + }); + + await expect(manager.start()).rejects.toBeInstanceOf( + RealtimeSourceSchemaViolationError + ); + await manager.stop(); + + expect(emitted).toEqual([]); + expect(errors).toHaveLength(1); + expect(errors[0]).toMatchObject({ + code: 'REALTIME_SOURCE_SCHEMA_VIOLATION', + sourceSchema: 'tenant_b', + allowedSourceSchemas: ['public'] + }); + expect(fatalErrors).toEqual([errors[0]]); + expect(manager.isRunning).toBe(false); + }); + + it('stops a running manager before a foreign periodic batch can emit', async () => { + const errors: Error[] = []; + const fatalErrors: Error[] = []; + const emitted: string[] = []; + mockSubscriber.eventEmitter.on('realtime:public.contact', (payload: string) => { + emitted.push(payload); + }); + const manager = createManager({ + allowedSourceSchemas: ['public'], + onError: (error: Error) => errors.push(error), + onFatalError: (error: Error) => fatalErrors.push(error) + }); + await manager.start(); + mockPool.query.mockImplementation(async (sql: string) => { + if (sql.includes('drain_changes')) { + return { + rows: [{ + drain_changes: makeEntry({ + source_schema: 'tenant_b', + payload_after: { id: 'foreign-periodic-row' } + }) + }] + }; + } + return { rows: [] }; + }); + + await jest.advanceTimersByTimeAsync(1000); + await flushMicrotasks(); + await manager.stop(); + + expect(emitted).toEqual([]); + expect(errors).toHaveLength(1); + expect(errors[0]).toBeInstanceOf(RealtimeSourceSchemaViolationError); + expect(fatalErrors).toEqual([errors[0]]); + expect(manager.isRunning).toBe(false); + expect(mockPool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['test-manager-node'] + ); + }); + it('emits cursor-tracked events on PgSubscriber eventEmitter', async () => { const emitted: { channel: string; payload: string }[] = []; mockSubscriber.eventEmitter.on('realtime:public.contact', (payload: string) => { @@ -290,7 +591,7 @@ describe('RealtimeManager', () => { }); describe('error handling', () => { - it('calls onError when drain fails', async () => { + it('fails startup and rolls back readiness when the initial drain fails', async () => { const errors: Error[] = []; mockPool.query.mockImplementation(async (sql: string) => { @@ -301,30 +602,12 @@ describe('RealtimeManager', () => { }); const manager = createManager({ onError: (err: Error) => errors.push(err) }); - await manager.start(); + await expect(manager.start()).rejects.toThrow('drain failed'); expect(errors).toHaveLength(1); expect(errors[0].message).toBe('drain failed'); - - await manager.stop(); + expect(manager.isRunning).toBe(false); }); - it('handles missing eventEmitter gracefully', async () => { - const entries: ChangeLogEntry[] = [ - makeEntry({ operation: 'INSERT', payload_after: { id: 'row-x' } }), - ]; - - mockPool.query.mockImplementation(async (sql: string) => { - if (typeof sql === 'string' && sql.includes('drain_changes')) { - return { rows: entries.map((e) => ({ drain_changes: e })) }; - } - return { rows: [] }; - }); - - // pgSubscriber without eventEmitter — should not crash - const manager = createManager({ pgSubscriber: {} }); - await manager.start(); - await manager.stop(); - }); }); }); diff --git a/graphile/graphile-realtime-subscriptions/__tests__/topic-collector.test.ts b/graphile/graphile-realtime-subscriptions/__tests__/topic-collector.test.ts new file mode 100644 index 0000000000..f38af65514 --- /dev/null +++ b/graphile/graphile-realtime-subscriptions/__tests__/topic-collector.test.ts @@ -0,0 +1,66 @@ +import { + RealtimeTopicCollector, + RealtimeTopicDiscoveryError +} from '../src/topic-collector'; + +describe('RealtimeTopicCollector', () => { + it('returns sorted exact physical topics for allowed schemas', () => { + const collector = new RealtimeTopicCollector(); + collector.collect([ + { topic: 'realtime:tenant_a.z', schema: 'tenant_a', table: 'z' }, + { topic: 'realtime:tenant_a.a', schema: 'tenant_a', table: 'a' } + ]); + + expect(collector.exactTopics(['tenant_a'])).toEqual([ + 'realtime:tenant_a.a', + 'realtime:tenant_a.z' + ]); + }); + + it.each([ + { + descriptors: [], + schemas: ['tenant_a'], + code: 'REALTIME_TOPIC_DISCOVERY_EMPTY' + }, + { + descriptors: [ + { topic: 'realtime:tenant_b.items', schema: 'tenant_b', table: 'items' } + ], + schemas: ['tenant_a'], + code: 'REALTIME_TOPIC_DISCOVERY_FOREIGN' + }, + { + descriptors: [ + { topic: 'realtime:tenant.a.items', schema: 'tenant.a', table: 'items' } + ], + schemas: ['tenant.a'], + code: 'REALTIME_TOPIC_DISCOVERY_INVALID' + } + ])('fails closed for $code', ({ descriptors, schemas, code }) => { + const collector = new RealtimeTopicCollector(); + expect(() => { + collector.collect(descriptors); + collector.exactTopics(schemas); + }).toThrow(expect.objectContaining({ + code + }) as RealtimeTopicDiscoveryError); + }); + + it('rejects missing discovery and post-discovery topic drift', () => { + const missing = new RealtimeTopicCollector(); + expect(() => missing.exactTopics(['tenant_a'])).toThrow(expect.objectContaining({ + code: 'REALTIME_TOPIC_DISCOVERY_MISSING' + }) as RealtimeTopicDiscoveryError); + + const changed = new RealtimeTopicCollector(); + changed.collect([ + { topic: 'realtime:tenant_a.items', schema: 'tenant_a', table: 'items' } + ]); + expect(() => changed.collect([ + { topic: 'realtime:tenant_a.users', schema: 'tenant_a', table: 'users' } + ])).toThrow(expect.objectContaining({ + code: 'REALTIME_TOPIC_DISCOVERY_CHANGED' + }) as RealtimeTopicDiscoveryError); + }); +}); diff --git a/graphile/graphile-realtime-subscriptions/src/cursor-tracker.ts b/graphile/graphile-realtime-subscriptions/src/cursor-tracker.ts index ab1f1204b9..b18849c1bc 100644 --- a/graphile/graphile-realtime-subscriptions/src/cursor-tracker.ts +++ b/graphile/graphile-realtime-subscriptions/src/cursor-tracker.ts @@ -30,6 +30,17 @@ const DEFAULT_HEARTBEAT_INTERVAL_MS = 30000; const DEFAULT_BATCH_LIMIT = 500; const DEFAULT_SCHEMA = 'realtime_public'; +type CursorTrackerState = 'stopped' | 'starting' | 'running' | 'stopping'; + +export class CursorTrackerStartAbortedError extends Error { + readonly code = 'CURSOR_TRACKER_START_ABORTED'; + + constructor() { + super('CursorTracker was stopped before startup completed'); + this.name = 'CursorTrackerStartAbortedError'; + } +} + export class CursorTracker { readonly nodeId: string; @@ -43,8 +54,13 @@ export class CursorTracker { private pollTimer: ReturnType | null = null; private heartbeatTimer: ReturnType | null = null; - private running = false; - private draining = false; + private state: CursorTrackerState = 'stopped'; + private generation = 0; + private registered = false; + private startPromise: Promise | null = null; + private stopPromise: Promise | null = null; + private activeDrain: Promise | null = null; + private activeHeartbeat: Promise | null = null; constructor(options: CursorTrackerOptions) { this.nodeId = options.nodeId ?? randomUUID(); @@ -59,32 +75,107 @@ export class CursorTracker { }); } - async start(): Promise { - if (this.running) return; - this.running = true; + start(): Promise { + if (this.state === 'running') return Promise.resolve(); + if (this.state === 'starting') return this.startPromise!; + if (this.state === 'stopping') { + return (this.stopPromise ?? Promise.resolve()).then(() => this.start()); + } + const generation = ++this.generation; + this.state = 'starting'; + const pending = this.startInternal(generation); + this.startPromise = pending; + void pending.then( + () => { + if (this.startPromise === pending) this.startPromise = null; + }, + () => { + if (this.startPromise === pending) this.startPromise = null; + } + ); + return pending; + } + + private async startInternal(generation: number): Promise { log.info(`Starting cursor tracker: node=${this.nodeId}, schema=${this.schema}`); + try { + // A manual operation may have started while the tracker was stopped. + // Readiness must execute its own strict registration and drain rather + // than coalescing onto a non-strict operation. + await this.waitForActiveWork(); + this.assertStartCurrent(generation); - await this.touchListener(); + // Startup is a readiness boundary: the instance must not become resident + // when the runtime role cannot register or drain the configured schema. + await this.touchListenerInternal(true); + this.registered = true; + this.assertStartCurrent(generation); - // Initial drain immediately after registration - await this.drain(); + // A caller can request a manual drain while registration is in flight. + // Let it settle, then run the strict readiness drain ourselves so a + // best-effort call can never satisfy the startup boundary. + await this.waitForActiveWork(); + this.assertStartCurrent(generation); + await this.drainInternal(true, generation); + this.assertStartCurrent(generation); - this.pollTimer = setInterval(() => { - void this.drain(); - }, this.pollIntervalMs); + this.state = 'running'; - this.heartbeatTimer = setInterval(() => { - void this.touchListener(); - }, this.heartbeatIntervalMs); + this.pollTimer = setInterval(() => { + void this.drain(); + }, this.pollIntervalMs); + this.pollTimer.unref?.(); + + this.heartbeatTimer = setInterval(() => { + void this.touchListener(); + }, this.heartbeatIntervalMs); + this.heartbeatTimer.unref?.(); + } catch (error) { + this.clearTimers(); + if (this.registered) { + await this.cleanupEphemeralInternal(); + this.registered = false; + } + if (this.state === 'starting') this.state = 'stopped'; + throw error; + } } - async stop(): Promise { - if (!this.running) return; - this.running = false; + stop(): Promise { + if (this.state === 'stopped') return Promise.resolve(); + if (this.state === 'stopping') return this.stopPromise!; + + const startInFlight = this.startPromise; + ++this.generation; + this.state = 'stopping'; + this.clearTimers(); log.info(`Stopping cursor tracker: node=${this.nodeId}`); + const pending = this.stopInternal(startInFlight); + this.stopPromise = pending; + void pending.then( + () => { + if (this.stopPromise === pending) this.stopPromise = null; + }, + () => { + if (this.stopPromise === pending) this.stopPromise = null; + } + ); + return pending; + } + private async stopInternal(startInFlight: Promise | null): Promise { + if (startInFlight) await Promise.allSettled([startInFlight]); + await this.waitForActiveWork(); + if (this.registered) { + await this.cleanupEphemeralInternal(); + this.registered = false; + } + this.state = 'stopped'; + } + + private clearTimers(): void { if (this.pollTimer) { clearInterval(this.pollTimer); this.pollTimer = null; @@ -94,14 +185,39 @@ export class CursorTracker { clearInterval(this.heartbeatTimer); this.heartbeatTimer = null; } + } - await this.cleanupEphemeral(); + drain(): Promise { + if (this.state === 'stopping') return Promise.resolve([]); + const dispatchGeneration = this.state === 'starting' || this.state === 'running' + ? this.generation + : undefined; + return this.drainInternal(false, dispatchGeneration); } - async drain(): Promise { - if (this.draining) return []; - this.draining = true; + private drainInternal( + throwOnError: boolean, + dispatchGeneration?: number + ): Promise { + if (this.activeDrain) return Promise.resolve([]); + const pending = this.executeDrain(throwOnError, dispatchGeneration); + this.activeDrain = pending; + void pending.then( + () => { + if (this.activeDrain === pending) this.activeDrain = null; + }, + () => { + if (this.activeDrain === pending) this.activeDrain = null; + } + ); + return pending; + } + + private async executeDrain( + throwOnError: boolean, + dispatchGeneration?: number + ): Promise { try { const sql = `SELECT * FROM ${this.quoteIdent(this.schema)}.drain_changes($1, $2)`; const result = await this.pool.query<{ drain_changes: ChangeLogEntry }>( @@ -110,30 +226,56 @@ export class CursorTracker { ); const entries = result.rows.map((row) => row.drain_changes); - if (entries.length > 0) { + if (entries.length > 0 && this.mayDispatch(dispatchGeneration)) { log.info(`Drained ${entries.length} change(s) for node=${this.nodeId}`); this.onChanges(entries); } return entries; } catch (err) { - this.onError(err instanceof Error ? err : new Error(String(err))); + const error = err instanceof Error ? err : new Error(String(err)); + this.onError(error); + if (throwOnError) throw error; return []; - } finally { - this.draining = false; } } - async touchListener(): Promise { + touchListener(): Promise { + if (this.state === 'stopping') return Promise.resolve(); + return this.touchListenerInternal(false); + } + + private touchListenerInternal(throwOnError: boolean): Promise { + if (this.activeHeartbeat) return this.activeHeartbeat; + const pending = this.executeTouchListener(throwOnError); + this.activeHeartbeat = pending; + void pending.then( + () => { + if (this.activeHeartbeat === pending) this.activeHeartbeat = null; + }, + () => { + if (this.activeHeartbeat === pending) this.activeHeartbeat = null; + } + ); + return pending; + } + + private async executeTouchListener(throwOnError: boolean): Promise { try { const sql = `SELECT ${this.quoteIdent(this.schema)}.touch_listener($1)`; await this.pool.query(sql, [this.nodeId]); } catch (err) { - this.onError(err instanceof Error ? err : new Error(String(err))); + const error = err instanceof Error ? err : new Error(String(err)); + this.onError(error); + if (throwOnError) throw error; } } async cleanupEphemeral(): Promise { + await this.cleanupEphemeralInternal(); + } + + private async cleanupEphemeralInternal(): Promise { try { const sql = `SELECT ${this.quoteIdent(this.schema)}.cleanup_ephemeral($1)`; await this.pool.query(sql, [this.nodeId]); @@ -144,7 +286,26 @@ export class CursorTracker { } get isRunning(): boolean { - return this.running; + return this.state === 'running'; + } + + private assertStartCurrent(generation: number): void { + if (this.state !== 'starting' || this.generation !== generation) { + throw new CursorTrackerStartAbortedError(); + } + } + + private mayDispatch(generation: number | undefined): boolean { + if (generation === undefined) return this.state !== 'stopping'; + return this.generation === generation + && (this.state === 'starting' || this.state === 'running'); + } + + private async waitForActiveWork(): Promise { + const active: Promise[] = []; + if (this.activeDrain) active.push(this.activeDrain); + if (this.activeHeartbeat) active.push(this.activeHeartbeat); + if (active.length > 0) await Promise.allSettled(active); } private quoteIdent(identifier: string): string { diff --git a/graphile/graphile-realtime-subscriptions/src/generation-subscriber.ts b/graphile/graphile-realtime-subscriptions/src/generation-subscriber.ts new file mode 100644 index 0000000000..4b94da1502 --- /dev/null +++ b/graphile/graphile-realtime-subscriptions/src/generation-subscriber.ts @@ -0,0 +1,423 @@ +import type { GrafastSubscriber } from 'grafast'; + +import type { RealtimePublisher } from './types'; + +export const GENERATION_SUBSCRIBER_QUEUE_CAPACITY = 256; +export const REALTIME_GENERATION_TOPIC_ERROR_CODE = 'REALTIME_GENERATION_TOPIC_INVALID'; +export const REALTIME_GENERATION_RELEASED_ERROR_CODE = 'REALTIME_GENERATION_RELEASED'; +export const REALTIME_GENERATION_OVERFLOW_ERROR_CODE = 'REALTIME_GENERATION_OVERFLOW'; +export const REALTIME_GENERATION_SOURCE_ENDED_ERROR_CODE = 'REALTIME_GENERATION_SOURCE_ENDED'; +export const REALTIME_GENERATION_NOT_ACTIVE_ERROR_CODE = 'REALTIME_GENERATION_NOT_ACTIVE'; +export const REALTIME_GENERATION_ALREADY_ACTIVE_ERROR_CODE = 'REALTIME_GENERATION_ALREADY_ACTIVE'; + +type RealtimeTopicMap = Record; + +export interface GenerationScopedRealtimeSubscriberOptions< + TTopics extends RealtimeTopicMap +> { + /** Shared database notification source owned by this generation facade. */ + source: GrafastSubscriber; + /** Exact topics compiled into this Graphile generation. */ + allowedTopics: readonly (keyof TTopics & string)[]; + /** Defaults to true; set false only when lifecycle ownership lives elsewhere. */ + releaseSourceOnRelease?: boolean; +} + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(error: unknown): void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +export class RealtimeGenerationTopicError extends Error { + readonly code = REALTIME_GENERATION_TOPIC_ERROR_CODE; + + constructor(readonly topic: unknown) { + super(`Realtime topic ${JSON.stringify(topic)} is outside this generation's allowlist`); + this.name = 'RealtimeGenerationTopicError'; + } +} + +export class RealtimeGenerationReleasedError extends Error { + readonly code = REALTIME_GENERATION_RELEASED_ERROR_CODE; + + constructor() { + super('Realtime generation subscriber has been released'); + this.name = 'RealtimeGenerationReleasedError'; + } +} + +export class RealtimeGenerationOverflowError extends Error { + readonly code = REALTIME_GENERATION_OVERFLOW_ERROR_CODE; + + constructor( + readonly topic: string, + readonly capacity: number + ) { + super( + `Realtime generation queue for ${JSON.stringify(topic)} exceeded its ` + + `fixed capacity of ${capacity}` + ); + this.name = 'RealtimeGenerationOverflowError'; + } +} + +export class RealtimeGenerationSourceEndedError extends Error { + readonly code = REALTIME_GENERATION_SOURCE_ENDED_ERROR_CODE; + + constructor(readonly topic: string) { + super(`Realtime source for ${JSON.stringify(topic)} ended unexpectedly`); + this.name = 'RealtimeGenerationSourceEndedError'; + } +} + +export class RealtimeGenerationNotActiveError extends Error { + readonly code = REALTIME_GENERATION_NOT_ACTIVE_ERROR_CODE; + + constructor() { + super('Realtime generation subscriber has not been activated'); + this.name = 'RealtimeGenerationNotActiveError'; + } +} + +export class RealtimeGenerationAlreadyActiveError extends Error { + readonly code = REALTIME_GENERATION_ALREADY_ACTIVE_ERROR_CODE; + + constructor() { + super('Realtime generation subscriber has already been activated'); + this.name = 'RealtimeGenerationAlreadyActiveError'; + } +} + +class LocalQueue { + private readonly buffered: T[] = []; + private readonly waiting: Deferred>[] = []; + private terminal: 'open' | 'complete' | 'failed' = 'open'; + private failure: Error | null = null; + + constructor( + private readonly topic: string, + private readonly capacity: number + ) {} + + next(): Promise> { + if (this.buffered.length > 0) { + return Promise.resolve({ done: false, value: this.buffered.shift()! }); + } + if (this.terminal === 'failed') return Promise.reject(this.failure); + if (this.terminal === 'complete') { + return Promise.resolve({ done: true, value: undefined }); + } + const result = deferred>(); + this.waiting.push(result); + return result.promise; + } + + push(value: T): RealtimeGenerationOverflowError | null { + if (this.terminal !== 'open') return null; + const waiter = this.waiting.shift(); + if (waiter) { + waiter.resolve({ done: false, value }); + return null; + } + if (this.buffered.length >= this.capacity) { + const error = new RealtimeGenerationOverflowError(this.topic, this.capacity); + this.fail(error); + return error; + } + this.buffered.push(value); + return null; + } + + complete(): void { + if (this.terminal !== 'open') return; + this.terminal = 'complete'; + this.buffered.length = 0; + for (const waiter of this.waiting.splice(0)) { + waiter.resolve({ done: true, value: undefined }); + } + } + + fail(error: Error): void { + if (this.terminal !== 'open') return; + this.terminal = 'failed'; + this.failure = error; + this.buffered.length = 0; + for (const waiter of this.waiting.splice(0)) waiter.reject(error); + } +} + +class GenerationSubscription implements AsyncIterableIterator { + private readonly queue: LocalQueue; + private readonly sourceIteratorPromise: Promise>; + private sourceReturnPromise: Promise | null = null; + private stopped = false; + private stopPromise: Promise | null = null; + + constructor( + readonly topic: string, + source: GrafastSubscriber>, + private readonly onStop: (subscription: GenerationSubscription) => void + ) { + this.queue = new LocalQueue(topic, GENERATION_SUBSCRIBER_QUEUE_CAPACITY); + this.sourceIteratorPromise = Promise.resolve().then(() => source.subscribe(topic)); + void this.pump(); + } + + [Symbol.asyncIterator](): AsyncIterableIterator { + return this; + } + + next(): Promise> { + return this.queue.next(); + } + + async return(value?: unknown): Promise> { + await this.stop(); + return { done: true, value: value as T }; + } + + async throw(error?: unknown): Promise> { + const failure = error instanceof Error ? error : new Error(String(error)); + await this.stop(failure); + throw failure; + } + + publish(value: T): void { + if (this.stopped) return; + const overflow = this.queue.push(value); + if (overflow) void this.stop(overflow).catch(() => {}); + } + + fail(error: Error): void { + if (this.stopped) return; + void this.stop(error).catch(() => {}); + } + + stop(error?: Error): Promise { + if (this.stopPromise) return this.stopPromise; + this.stopped = true; + if (error) this.queue.fail(error); + else this.queue.complete(); + this.stopPromise = this.returnSource().finally(() => this.onStop(this)); + return this.stopPromise; + } + + private async pump(): Promise { + try { + const iterator = await this.sourceIteratorPromise; + if (this.stopped) { + await this.returnSource(); + return; + } + for (;;) { + const result = await iterator.next(); + if (this.stopped) return; + if (result.done) { + this.fail(new RealtimeGenerationSourceEndedError(this.topic)); + return; + } + this.publish(result.value); + } + } catch (error) { + if (!this.stopped) { + this.fail(error instanceof Error ? error : new Error(String(error))); + } + } + } + + private returnSource(): Promise { + if (this.sourceReturnPromise) return this.sourceReturnPromise; + this.sourceReturnPromise = this.sourceIteratorPromise.then(async (iterator) => { + await iterator.return?.(); + }, () => { + // Source acquisition failure is already delivered to the output queue. + }); + return this.sourceReturnPromise; + } +} + +/** + * A Graphile-generation-local GrafastSubscriber. Database notifications are + * forwarded from the shared source, while cursor catch-up events published + * through publish() remain inside this exact generation. + */ +export class GenerationScopedRealtimeSubscriber< + TTopics extends RealtimeTopicMap = RealtimeTopicMap +> implements GrafastSubscriber, RealtimePublisher { + readonly allowedTopics: readonly (keyof TTopics & string)[]; + private readonly allowedTopicSet: ReadonlySet; + private readonly subscriptions = new Map< + string, + Set> + >(); + private readonly releaseSourceOnRelease: boolean; + private readonly source: GrafastSubscriber; + private released = false; + private releasePromise: Promise | null = null; + + constructor(options: GenerationScopedRealtimeSubscriberOptions) { + if (!Array.isArray(options.allowedTopics) || options.allowedTopics.length === 0) { + throw new RealtimeGenerationTopicError(options.allowedTopics); + } + if (options.allowedTopics.some((topic) => typeof topic !== 'string')) { + throw new RealtimeGenerationTopicError(options.allowedTopics); + } + this.allowedTopics = Object.freeze([...new Set(options.allowedTopics)]); + this.allowedTopicSet = new Set(this.allowedTopics); + this.source = options.source; + this.releaseSourceOnRelease = options.releaseSourceOnRelease ?? true; + } + + subscribe( + topic: TTopic + ): AsyncIterableIterator { + if (this.released) throw new RealtimeGenerationReleasedError(); + if (typeof topic !== 'string' || !this.allowedTopicSet.has(topic)) { + throw new RealtimeGenerationTopicError(topic); + } + + let topicSubscriptions = this.subscriptions.get(topic); + if (!topicSubscriptions) { + topicSubscriptions = new Set(); + this.subscriptions.set(topic, topicSubscriptions); + } + const subscription = new GenerationSubscription( + topic, + this.source as GrafastSubscriber>, + (stopped) => { + topicSubscriptions!.delete(stopped); + if (topicSubscriptions!.size === 0) this.subscriptions.delete(topic); + } + ); + topicSubscriptions.add(subscription); + return subscription as AsyncIterableIterator; + } + + assertTopics(topics: readonly string[]): void { + if (this.released) throw new RealtimeGenerationReleasedError(); + const invalid = topics.find((topic) => !this.allowedTopicSet.has(topic)); + if (invalid !== undefined) throw new RealtimeGenerationTopicError(invalid); + } + + publish(topic: string, payload: string): void { + this.assertTopics([topic]); + const subscriptions = this.subscriptions.get(topic); + if (!subscriptions) return; + for (const subscription of [...subscriptions]) subscription.publish(payload); + } + + release(): Promise { + if (this.releasePromise) return this.releasePromise; + this.released = true; + const active = [...this.subscriptions.values()].flatMap((entries) => [...entries]); + this.releasePromise = (async () => { + const results = await Promise.allSettled(active.map((subscription) => subscription.stop())); + if (this.releaseSourceOnRelease) await this.source.release?.(); + const rejected = results.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ); + if (rejected) throw rejected.reason; + })(); + return this.releasePromise; + } +} + +/** + * Stable subscriber identity installed into a PostGraphile pgService before + * schema construction. Activation installs the exact generation facade only + * after the build has reported all physical @realtime topics. + */ +export class ActivatableGenerationScopedRealtimeSubscriber< + TTopics extends RealtimeTopicMap = RealtimeTopicMap +> implements GrafastSubscriber, RealtimePublisher { + private delegate: GenerationScopedRealtimeSubscriber | null = null; + private released = false; + private releasePromise: Promise | null = null; + + async activate( + options: GenerationScopedRealtimeSubscriberOptions + ): Promise { + if (this.released) { + await options.source.release?.(); + throw new RealtimeGenerationReleasedError(); + } + if (this.delegate) { + await options.source.release?.(); + throw new RealtimeGenerationAlreadyActiveError(); + } + + try { + this.delegate = new GenerationScopedRealtimeSubscriber(options); + } catch (error) { + await options.source.release?.(); + throw error; + } + } + + subscribe( + topic: TTopic + ): AsyncIterableIterator { + if (this.released) throw new RealtimeGenerationReleasedError(); + if (!this.delegate) throw new RealtimeGenerationNotActiveError(); + return this.delegate.subscribe(topic); + } + + assertTopics(topics: readonly string[]): void { + if (this.released) throw new RealtimeGenerationReleasedError(); + if (!this.delegate) throw new RealtimeGenerationNotActiveError(); + this.delegate.assertTopics(topics); + } + + publish(topic: string, payload: string): void { + if (this.released) throw new RealtimeGenerationReleasedError(); + if (!this.delegate) throw new RealtimeGenerationNotActiveError(); + this.delegate.publish(topic, payload); + } + + release(): Promise { + if (this.releasePromise) return this.releasePromise; + this.released = true; + this.releasePromise = this.delegate?.release() ?? Promise.resolve(); + return this.releasePromise; + } +} + +type LegacyEventEmitter = { + emit(topic: string, payload: string): boolean; +}; + +/** + * Transitional adapter for @dataplan/pg's current PgSubscriber. Private-field + * access is quarantined here; RealtimeManager and new integrations depend only + * on the explicit publisher capability. + */ +export const createPgSubscriberPublisher = ( + pgSubscriber: unknown +): RealtimePublisher | null => { + const candidate = pgSubscriber as { eventEmitter?: LegacyEventEmitter } | null; + const emitter = candidate && typeof candidate === 'object' + ? candidate.eventEmitter + : null; + if (!emitter || typeof emitter.emit !== 'function') return null; + const emit = emitter.emit.bind(emitter); + return Object.freeze({ + assertTopics(): void { + // The legacy PgSubscriber owns topic validation. New integrations use + // GenerationScopedRealtimeSubscriber's exact preflight instead. + }, + publish(topic: string, payload: string): void { + emit(topic, payload); + } + }); +}; diff --git a/graphile/graphile-realtime-subscriptions/src/index.ts b/graphile/graphile-realtime-subscriptions/src/index.ts index d0fdf741ca..456531e860 100644 --- a/graphile/graphile-realtime-subscriptions/src/index.ts +++ b/graphile/graphile-realtime-subscriptions/src/index.ts @@ -17,14 +17,52 @@ * ``` */ -export { CursorTracker } from './cursor-tracker'; +export { CursorTracker, CursorTrackerStartAbortedError } from './cursor-tracker'; +export type { GenerationScopedRealtimeSubscriberOptions } from './generation-subscriber'; +export { + ActivatableGenerationScopedRealtimeSubscriber, + createPgSubscriberPublisher, + GENERATION_SUBSCRIBER_QUEUE_CAPACITY, + GenerationScopedRealtimeSubscriber, + REALTIME_GENERATION_ALREADY_ACTIVE_ERROR_CODE, + REALTIME_GENERATION_NOT_ACTIVE_ERROR_CODE, + REALTIME_GENERATION_OVERFLOW_ERROR_CODE, + REALTIME_GENERATION_RELEASED_ERROR_CODE, + REALTIME_GENERATION_SOURCE_ENDED_ERROR_CODE, + REALTIME_GENERATION_TOPIC_ERROR_CODE, + RealtimeGenerationAlreadyActiveError, + RealtimeGenerationNotActiveError, + RealtimeGenerationOverflowError, + RealtimeGenerationReleasedError, + RealtimeGenerationSourceEndedError, + RealtimeGenerationTopicError +} from './generation-subscriber'; export { createRealtimeSubscriptionsPlugin, RealtimeSubscriptionsPlugin } from './plugin'; export { RealtimeSubscriptionsPreset } from './preset'; -export { RealtimeManager } from './realtime-manager'; -export type { RealtimeSubscriptionsPluginOptions } from './types'; +export { + RealtimeManager, + RealtimeManagerStartAbortedError, + RealtimeSourceSchemaConfigurationError, + RealtimeSourceSchemaViolationError, + RealtimeSubscriberUnavailableError +} from './realtime-manager'; +export { + REALTIME_TOPIC_DISCOVERY_CHANGED_ERROR_CODE, + REALTIME_TOPIC_DISCOVERY_EMPTY_ERROR_CODE, + REALTIME_TOPIC_DISCOVERY_FOREIGN_ERROR_CODE, + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + REALTIME_TOPIC_DISCOVERY_MISSING_ERROR_CODE, + RealtimeTopicCollector, + RealtimeTopicDiscoveryError +} from './topic-collector'; +export type { + RealtimeSubscriptionsPluginOptions, + RealtimeTopicDescriptor +} from './types'; export type { ChangeLogEntry, CursorTrackerOptions, Queryable, RealtimeManagerOptions, + RealtimePublisher, } from './types'; diff --git a/graphile/graphile-realtime-subscriptions/src/plugin.ts b/graphile/graphile-realtime-subscriptions/src/plugin.ts index 85a38958d7..7a0b0a8d17 100644 --- a/graphile/graphile-realtime-subscriptions/src/plugin.ts +++ b/graphile/graphile-realtime-subscriptions/src/plugin.ts @@ -35,23 +35,32 @@ * drops individual events and sends a single INVALIDATE when exceeded * * Security / RLS enforcement: - * - Row data is always fetched via resource.get() which runs through the - * authenticated user's connection with their JWT role and pgSettings applied. - * - For INSERT/UPDATE events, if RLS denies access (resource.get returns null), - * the rowId is masked (set to null) to prevent metadata leaks. - * - For DELETE events, row is naturally null (the row no longer exists). - * - For INVALIDATE (overflow), the client should refetch via a normal query - * which is also RLS-gated. + * - INSERT/UPDATE notifications are filtered at the AsyncIterable boundary by + * a parameterized visibility query under the request role and pgSettings. + * Grafast never observes an event unless at least one changed row is visible. + * - Row data is fetched via resource.get() under the same request RLS context. + * - Collection subscriptions never expose row IDs because merely observing + * identifiers from rows hidden by RLS is a metadata leak. + * - Sparse INSERT/UPDATE subscriptions expose a requested row ID only after + * resource.get() confirms that the row remains visible under request RLS. + * - DELETE and database-originated INVALIDATE events are suppressed because + * neither carries a sound post-change audience proof. Plugin throttling may + * emit INVALIDATE only after an authorized INSERT/UPDATE event. * - When ids are provided, only events for those specific rows are delivered, * preventing cross-tenant event leaks. */ import { Logger } from '@pgpmjs/logger'; -import { constant, context as grafastContext, lambda,listen, object } from 'grafast'; +import { QuoteUtils } from '@pgsql/quotes'; +import type { Step } from 'grafast'; +import { constant, context as grafastContext, get, lambda, listen } from 'grafast'; import type { GraphileConfig } from 'graphile-config'; import { extendSchema } from 'graphile-utils'; -import type { RealtimeSubscriptionsPluginOptions } from './types'; +import type { + RealtimeSubscriptionsPluginOptions, + RealtimeTopicDescriptor, +} from './types'; const log = new Logger('graphile-realtime-subscriptions'); @@ -75,6 +84,158 @@ interface ParsedPayload { overflow: boolean; } +interface RealtimeEvent { + parsed: ParsedPayload; + subscribedIds: string[] | null | undefined; +} + +interface PgExecutorContextLike { + pgSettings: Record | null; + withPgClient( + pgSettings: Record | null, + callback: (client: { + query(query: { + text: string; + values?: unknown[]; + }): Promise<{ rows: readonly TData[] }>; + }) => Promise | T, + ): Promise; +} + +interface RealtimeSubscriberLike { + subscribe(topic: string | number): + | AsyncIterableIterator + | Promise>; +} + +/** + * Select the row that may be fetched under the request's RLS context. + * + * Collection subscriptions may fetch INSERT/UPDATE rows, but their public + * rowId field remains hidden. Sparse subscriptions only consider IDs the + * caller supplied. DELETE cannot be authorized after the row is gone. + */ +function selectCandidateRowId( + parsed: ParsedPayload | null, + subscribedIds: string[] | null | undefined, + allowCollection: boolean, +): string | null { + if ( + !parsed + || parsed.overflow + || (parsed.event !== 'INSERT' && parsed.event !== 'UPDATE') + || parsed.rowIds.length === 0 + ) { + return null; + } + + if (subscribedIds && subscribedIds.length > 0) { + return parsed.rowIds.find((rowId) => subscribedIds.includes(rowId)) ?? null; + } + + return allowCollection ? parsed.rowIds[0] : null; +} + +function selectCandidateRowIds( + parsed: ParsedPayload, + subscribedIds: string[] | null | undefined, +): string[] { + if ( + parsed.overflow + || (parsed.event !== 'INSERT' && parsed.event !== 'UPDATE') + ) { + return []; + } + + const candidates = subscribedIds && subscribedIds.length > 0 + ? parsed.rowIds.filter((rowId) => subscribedIds.includes(rowId)) + : parsed.rowIds; + + return [...new Set(candidates)]; +} + +/** + * Filter the notification stream before Grafast observes a subscription event. + * Returning a nullable payload from an item plan would still emit an observable + * GraphQL result, so authorization has to happen at the AsyncIterable boundary. + */ +async function* authorizeNotificationStream( + sourceOrPromise: + | AsyncIterableIterator + | Promise>, + executorContext: PgExecutorContextLike, + subscribedIds: string[] | null | undefined, + visibilitySql: string, + overflowThreshold: number, +): AsyncGenerator { + const source = await sourceOrPromise; + const throttle = new EventThrottle(overflowThreshold); + + for await (const raw of source) { + const parsed = parseNotifyPayload(String(raw)); + const candidateRowIds = selectCandidateRowIds(parsed, subscribedIds); + + // DELETE cannot be reauthorized after the row is gone. Database-originated + // INVALIDATE and malformed/unknown events carry no audience proof either. + if (candidateRowIds.length === 0) continue; + + let visibleRowIds: Set; + try { + visibleRowIds = await executorContext.withPgClient( + executorContext.pgSettings, + async (client) => { + const result = await client.query<{ id: string }>({ + text: visibilitySql, + values: [candidateRowIds], + }); + return new Set(result.rows.map((row) => String(row.id))); + }, + ); + } catch { + // Authorization errors must never turn into an event-existence oracle. + log.warn('Suppressing realtime event because RLS reauthorization failed'); + continue; + } + + const authorizedRowIds = candidateRowIds.filter((rowId) => visibleRowIds.has(rowId)); + if (authorizedRowIds.length === 0) continue; + + // Count only authorized events. Hidden-tenant traffic must not influence a + // subscriber's throttle state because that would be an observable side channel. + const action = throttle.check(); + if (action === 'drop') continue; + + const authorizedPayload = action === 'overflow' + ? { event: 'INVALIDATE', rowIds: [], overflow: true } + : { ...parsed, rowIds: authorizedRowIds }; + + yield { + parsed: authorizedPayload, + subscribedIds, + }; + } +} + +function createRlsAuthorizedSubscriber( + subscriber: RealtimeSubscriberLike, + executorContext: PgExecutorContextLike, + subscribedIds: string[] | null | undefined, + visibilitySql: string, + overflowThreshold: number, +): RealtimeSubscriberLike { + return { + subscribe(topic: string | number) { + return authorizeNotificationStream( + subscriber.subscribe(topic), + executorContext, + subscribedIds, + visibilitySql, + overflowThreshold, + ); + }, + }; +} + /** * Parse the NOTIFY payload from emit_change. * Format: "TG_OP:id1,id2,..." or "INVALIDATE" @@ -186,11 +347,11 @@ function buildTypeDefs(tables: RealtimeTableInfo[]): string { .map(({ payloadTypeName, typeName, rowFieldName }) => `"""Payload delivered when a ${typeName} row changes."""\n` + `type ${payloadTypeName} {\n` + - ` """The DML operation: INSERT, UPDATE, DELETE, or INVALIDATE."""\n` + + ` """The authorized operation: INSERT, UPDATE, or plugin-generated INVALIDATE."""\n` + ` event: String!\n` + - ` """The current state of the row (null for DELETE, INVALIDATE, or if RLS denies access)."""\n` + + ` """The current state of the row (null for INVALIDATE or an RLS visibility race)."""\n` + ` ${rowFieldName}: ${typeName}\n` + - ` """The ID of the changed row (null for INVALIDATE, or masked when RLS denies access)."""\n` + + ` """The requested row ID for a sparse INSERT/UPDATE subscription after RLS authorization. Null for collection, INVALIDATE, or denied rows."""\n` + ` rowId: UUID\n` + ` """True when too many changes occurred and the client should refetch."""\n` + ` overflow: Boolean!\n` + @@ -208,48 +369,50 @@ function buildPlans( const subscriptionPlans: Record = {}; const allPlans: Record = {}; - for (const { resource, fieldName, payloadTypeName, rowFieldName, notifyChannel } of tables) { - const throttle = new EventThrottle(overflowThreshold); + for (const { + resource, + fieldName, + payloadTypeName, + rowFieldName, + notifyChannel, + pgSchema, + pgTable, + } of tables) { + const qualifiedTable = QuoteUtils.quoteQualifiedIdentifier(pgSchema, pgTable); + const idColumn = QuoteUtils.quoteIdentifier('id'); + const visibilitySql = + `select ${idColumn}::text as id from ${qualifiedTable} ` + // Notification payloads are text, and @realtime tables may use UUID, + // integer, bigint, or text primary keys. Comparing their canonical text + // form keeps the query parameterized and avoids a UUID-only cast that + // silently suppresses otherwise authorized events. + + `where ${idColumn}::text = any($1::text[])`; subscriptionPlans[fieldName] = { subscribePlan(_$root: any, args: any) { const $pgSubscriber = (grafastContext() as any).get('pgSubscriber'); + const $executorContext = resource.executor.context(); const $topic = constant(notifyChannel); const $ids = args.getRaw('ids'); + const $authorizedSubscriber = lambda( + [$pgSubscriber, $executorContext, $ids], + (values: unknown) => { + const [subscriber, executorContext, subscribedIds] = values as readonly [ + RealtimeSubscriberLike, + PgExecutorContextLike, + string[] | null | undefined, + ]; + return createRlsAuthorizedSubscriber( + subscriber, + executorContext, + subscribedIds, + visibilitySql, + overflowThreshold, + ); + }, + ); - return listen($pgSubscriber, $topic, ($payload: any) => { - const $parsed = lambda([$payload, $ids], (pair: unknown) => { - const [raw, subscribedIds] = pair as readonly [unknown, string[] | null | undefined]; - const parsed = parseNotifyPayload(String(raw)); - - const action = parsed.overflow ? 'deliver' : throttle.check(); - - if (action === 'drop') { - return null; - } - - if (action === 'overflow') { - return { - event: 'INVALIDATE', - rowIds: [], - overflow: true, - }; - } - - // Sparse set filtering: only deliver events for subscribed row IDs - if (subscribedIds && subscribedIds.length > 0) { - const hasMatch = parsed.rowIds.some((rid: string) => subscribedIds.includes(rid)); - if (!hasMatch) return null; - } - - return parsed; - }); - - return object({ - parsed: $parsed, - subscribedIds: $ids, - }); - }); + return listen($authorizedSubscriber, $topic); }, plan($event: any) { return $event; @@ -257,47 +420,45 @@ function buildPlans( }; allPlans[payloadTypeName] = { - event($parent: any) { - const $parsed = $parent.get('parsed'); + event($parent: Step) { + const $parsed = get($parent, 'parsed'); return lambda($parsed, (p: unknown) => (p as ParsedPayload | null)?.event ?? 'UNKNOWN'); }, - rowId($parent: any) { - const $parsed = $parent.get('parsed'); - const $subscribedIds = $parent.get('subscribedIds'); - return lambda([$parsed, $subscribedIds], (pair: unknown) => { - const [p, subscribedIds] = pair as readonly [ParsedPayload | null, string[] | null | undefined]; - if (!p || p.overflow || p.rowIds.length === 0) return null; - - // When ids are provided, return the first matching row ID - if (subscribedIds && subscribedIds.length > 0) { - return p.rowIds.find((rid: string) => subscribedIds.includes(rid)) ?? null; - } - - return p.rowIds[0]; - }); + rowId($parent: Step) { + const $parsed = get($parent, 'parsed'); + const $subscribedIds = get($parent, 'subscribedIds'); + const $candidateRowId = lambda( + [$parsed, $subscribedIds], + (pair: unknown) => { + const [parsed, subscribedIds] = pair as readonly [ + ParsedPayload | null, + string[] | null | undefined, + ]; + // Collection mode deliberately cannot surface a row identifier. + return selectCandidateRowId(parsed, subscribedIds, false); + }, + ); + const $authorizedRow = resource.get({ id: $candidateRowId }); + // Selecting through the PgSelectSingleStep makes the ID null whenever + // request RLS hides the row; the raw notification ID is never returned. + return $authorizedRow.get('id'); }, - overflow($parent: any) { - const $parsed = $parent.get('parsed'); + overflow($parent: Step) { + const $parsed = get($parent, 'parsed'); return lambda($parsed, (p: unknown) => (p as ParsedPayload | null)?.overflow ?? false); }, - [rowFieldName]($parent: any) { - const $parsed = $parent.get('parsed'); - const $subscribedIds = $parent.get('subscribedIds'); + [rowFieldName]($parent: Step) { + const $parsed = get($parent, 'parsed'); + const $subscribedIds = get($parent, 'subscribedIds'); const $rowId = lambda( [$parsed, $subscribedIds], (tuple: unknown) => { - const [p, subscribedIds] = tuple as readonly [ + const [parsed, subscribedIds] = tuple as readonly [ ParsedPayload | null, string[] | null | undefined, ]; - if (!p || p.overflow || p.rowIds.length === 0) return null; - // When ids are provided, return first matching row ID - if (subscribedIds && subscribedIds.length > 0) { - return p.rowIds.find((rid: string) => subscribedIds.includes(rid)) ?? null; - } - // Full collection mode: return first row ID - return p.rowIds[0]; + return selectCandidateRowId(parsed, subscribedIds, true); }, ); @@ -318,6 +479,16 @@ export function createRealtimeSubscriptionsPlugin( return extendSchema( (build) => { const tables = discoverRealtimeTables(build); + const discoveredTopics: readonly RealtimeTopicDescriptor[] = Object.freeze( + tables + .map(({ notifyChannel, pgSchema, pgTable }) => Object.freeze({ + topic: notifyChannel, + schema: pgSchema, + table: pgTable, + })) + .sort((left, right) => left.topic.localeCompare(right.topic)), + ); + options.onTopicsDiscovered?.(discoveredTopics); if (tables.length === 0) { log.info('No tables with @realtime tag found — skipping subscription generation'); @@ -344,4 +515,9 @@ export { RealtimeManager } from './realtime-manager'; export type { ChangeLogEntry, CursorTrackerOptions, Queryable, RealtimeManagerOptions } from './types'; // Exported for testing -export { DEFAULT_OVERFLOW_THRESHOLD,EventThrottle, parseNotifyPayload }; +export { + DEFAULT_OVERFLOW_THRESHOLD, + EventThrottle, + parseNotifyPayload, + selectCandidateRowId, +}; diff --git a/graphile/graphile-realtime-subscriptions/src/realtime-manager.ts b/graphile/graphile-realtime-subscriptions/src/realtime-manager.ts index 58bcf11926..53c77362dd 100644 --- a/graphile/graphile-realtime-subscriptions/src/realtime-manager.ts +++ b/graphile/graphile-realtime-subscriptions/src/realtime-manager.ts @@ -1,17 +1,13 @@ /** * RealtimeManager — bridges CursorTracker (polling drain_changes) into - * PostGraphile's PgSubscriber so cursor-tracked events flow through the - * same subscription plans as NOTIFY events. + * a generation-local publisher so cursor-tracked events flow through the same + * subscription plans as NOTIFY events. * * Architecture: - * PgSubscriber uses an internal EventEmitter. NOTIFY payloads arrive via - * pg's `notification` event and are emitted as `eventEmitter.emit(channel, payload)`. - * The `listen()` step in grafast subscribes to the same EventEmitter. - * * RealtimeManager converts ChangeLogEntry objects from drain_changes() into - * the same NOTIFY payload format ("OP:rowId1,rowId2,...") and emits them on - * the PgSubscriber's EventEmitter, so existing subscription plans handle - * them identically to real NOTIFY events. + * the same NOTIFY payload format ("OP:rowId1,rowId2,...") and publishes them + * through an explicit capability. The generation-scoped subscriber keeps + * these cursor events local even when PostgreSQL LISTEN is shared. * * This provides at-least-once delivery: NOTIFY is instant but best-effort; * cursor polling catches up on anything missed (disconnects, restarts). @@ -26,13 +22,59 @@ import { Logger } from '@pgpmjs/logger'; import { CursorTracker } from './cursor-tracker'; +import { createPgSubscriberPublisher } from './generation-subscriber'; import type { ChangeLogEntry, RealtimeManagerOptions, + RealtimePublisher, } from './types'; const log = new Logger('realtime-manager'); +type RealtimeManagerState = 'stopped' | 'starting' | 'running' | 'stopping'; + +export class RealtimeManagerStartAbortedError extends Error { + readonly code = 'REALTIME_MANAGER_START_ABORTED'; + + constructor() { + super('RealtimeManager was stopped before startup completed'); + this.name = 'RealtimeManagerStartAbortedError'; + } +} + +export class RealtimeSubscriberUnavailableError extends Error { + readonly code = 'REALTIME_SUBSCRIBER_UNAVAILABLE'; + + constructor() { + super('RealtimeManager requires a usable local publisher'); + this.name = 'RealtimeSubscriberUnavailableError'; + } +} + +export class RealtimeSourceSchemaViolationError extends Error { + readonly code = 'REALTIME_SOURCE_SCHEMA_VIOLATION'; + + constructor( + readonly sourceSchema: unknown, + readonly allowedSourceSchemas: readonly string[] + ) { + super( + `Realtime cursor returned source schema ${JSON.stringify(sourceSchema)} ` + + `outside the allowed Graphile schemas: ${allowedSourceSchemas.join(', ')}` + ); + this.name = 'RealtimeSourceSchemaViolationError'; + } +} + +export class RealtimeSourceSchemaConfigurationError extends Error { + readonly code = 'REALTIME_SOURCE_SCHEMAS_REQUIRED'; + + constructor() { + super('RealtimeManager requires at least one exact allowed source schema'); + this.name = 'RealtimeSourceSchemaConfigurationError'; + } +} + /** * Extract row IDs from a ChangeLogEntry. * @@ -69,12 +111,37 @@ function entryToChannel(entry: ChangeLogEntry): string { export class RealtimeManager { private readonly cursorTracker: CursorTracker; - private readonly subscriber: unknown; - private started = false; + private readonly publisher: RealtimePublisher | null; + private readonly allowedSourceSchemas: ReadonlySet; + private readonly allowedSourceSchemaList: readonly string[]; + private readonly sourceSchemaConfigurationValid: boolean; + private readonly onFatalError?: (error: Error) => void; + private state: RealtimeManagerState = 'stopped'; + private generation = 0; + private dispatchEnabled = false; + private fatalError: Error | null = null; + private startPromise: Promise | null = null; + private stopPromise: Promise | null = null; constructor(options: RealtimeManagerOptions) { - const { pgSubscriber, pool, ...cursorOpts } = options; - this.subscriber = pgSubscriber; + const { + publisher, + pgSubscriber, + pool, + allowedSourceSchemas, + onFatalError, + ...cursorOpts + } = options; + this.publisher = publisher ?? createPgSubscriberPublisher(pgSubscriber); + this.onFatalError = onFatalError; + this.sourceSchemaConfigurationValid = Array.isArray(allowedSourceSchemas) + && allowedSourceSchemas.every( + (schema) => typeof schema === 'string' && schema.length > 0 + ); + this.allowedSourceSchemaList = Object.freeze([ + ...new Set(allowedSourceSchemas ?? []) + ]); + this.allowedSourceSchemas = new Set(this.allowedSourceSchemaList); this.cursorTracker = new CursorTracker({ nodeId: cursorOpts.nodeId, @@ -84,9 +151,26 @@ export class RealtimeManager { batchLimit: cursorOpts.batchLimit, pool, onChanges: (entries) => this.dispatchEntries(entries), - onError: cursorOpts.onError ?? ((err) => { - log.error(`RealtimeManager error: ${err.message}`); - }), + onError: (error) => { + // Once readiness has completed, losing either cursor polling or the + // listener heartbeat means at-least-once delivery can no longer be + // claimed. Disable dispatch and begin shutdown before invoking the + // observational callback so a callback cannot leave a stale + // generation serving traffic by throwing or stopping it itself. + if (this.state === 'running') this.failDelivery(error); + + try { + if (cursorOpts.onError) { + cursorOpts.onError(error); + } else { + log.error(`RealtimeManager error: ${error.message}`); + } + } catch (callbackError) { + log.error( + `RealtimeManager error callback failed: ${String(callbackError)}` + ); + } + }, }); } @@ -95,62 +179,160 @@ export class RealtimeManager { } get isRunning(): boolean { - return this.started && this.cursorTracker.isRunning; + return this.state === 'running' && this.cursorTracker.isRunning; } - async start(): Promise { - if (this.started) return; - this.started = true; + start(): Promise { + if (this.state === 'running') return Promise.resolve(); + if (this.state === 'starting') return this.startPromise!; + if (this.state === 'stopping') { + return (this.stopPromise ?? Promise.resolve()).then(() => this.start()); + } + const generation = ++this.generation; + this.state = 'starting'; + this.dispatchEnabled = true; log.info(`Starting RealtimeManager: node=${this.nodeId}`); - await this.cursorTracker.start(); + const pending = this.startInternal(generation); + this.startPromise = pending; + void pending.then( + () => { + if (this.startPromise === pending) this.startPromise = null; + }, + () => { + if (this.startPromise === pending) this.startPromise = null; + } + ); + return pending; + } + + private async startInternal(generation: number): Promise { + try { + if ( + !this.sourceSchemaConfigurationValid + || this.allowedSourceSchemas.size === 0 + ) { + throw new RealtimeSourceSchemaConfigurationError(); + } + if (!this.publisher || typeof this.publisher.publish !== 'function') { + throw new RealtimeSubscriberUnavailableError(); + } + await this.cursorTracker.start(); + if (this.state !== 'starting' || this.generation !== generation) { + throw new RealtimeManagerStartAbortedError(); + } + this.state = 'running'; + } catch (error) { + this.dispatchEnabled = false; + if (this.state === 'starting') this.state = 'stopped'; + throw error; + } } - async stop(): Promise { - if (!this.started) return; - this.started = false; + stop(): Promise { + if (this.state === 'stopped') return Promise.resolve(); + if (this.state === 'stopping') return this.stopPromise!; + const startInFlight = this.startPromise; + ++this.generation; + this.state = 'stopping'; + this.dispatchEnabled = false; log.info(`Stopping RealtimeManager: node=${this.nodeId}`); - await this.cursorTracker.stop(); + // Start the tracker shutdown synchronously so an in-flight drain is + // invalidated before it can dispatch after this method is called. + const trackerStop = this.cursorTracker.stop(); + const pending = this.stopInternal(startInFlight, trackerStop); + this.stopPromise = pending; + void pending.then( + () => { + if (this.stopPromise === pending) this.stopPromise = null; + }, + () => { + if (this.stopPromise === pending) this.stopPromise = null; + } + ); + return pending; + } + + private async stopInternal( + startInFlight: Promise | null, + trackerStop: Promise + ): Promise { + try { + if (startInFlight) await Promise.allSettled([startInFlight]); + await trackerStop; + } finally { + this.state = 'stopped'; + this.dispatchEnabled = false; + } } /** - * Convert ChangeLogEntry objects to NOTIFY-format payloads and emit - * them on the PgSubscriber's internal EventEmitter. + * Convert ChangeLogEntry objects to NOTIFY-format payloads and publish them + * through the exact generation's explicit local capability. */ private dispatchEntries(entries: ChangeLogEntry[]): void { - const emitter = this.getEventEmitter(); - if (!emitter) { - log.warn('PgSubscriber has no eventEmitter; cursor events cannot be dispatched'); - return; + if (!this.dispatchEnabled) return; + + const publisher = this.publisher; + if (!publisher) { + const error = new RealtimeSubscriberUnavailableError(); + this.failDelivery(error); + throw error; } - for (const entry of entries) { - const channel = entryToChannel(entry); - const payload = entryToNotifyPayload(entry); - emitter.emit(channel, payload); + // Validate the complete batch before emitting the first event. This keeps + // a mixed valid/foreign batch atomic from the tenant-isolation boundary's + // perspective: no event is delivered when routing is inconclusive. + const foreignEntry = entries.find( + (entry) => !this.allowedSourceSchemas.has(entry.source_schema) + ); + if (foreignEntry) { + const error = new RealtimeSourceSchemaViolationError( + foreignEntry.source_schema, + this.allowedSourceSchemaList + ); + this.failDelivery(error); + throw error; } - log.info(`Dispatched ${entries.length} cursor-tracked event(s) to PgSubscriber`); + const notifications = entries.map((entry) => ({ + channel: entryToChannel(entry), + payload: entryToNotifyPayload(entry) + })); + try { + publisher.assertTopics?.(notifications.map(({ channel }) => channel)); + for (const { channel, payload } of notifications) { + publisher.publish(channel, payload); + } + } catch (reason) { + const error = reason instanceof Error ? reason : new Error(String(reason)); + this.failDelivery(error); + throw error; + } + + log.info(`Dispatched ${entries.length} cursor-tracked event(s)`); } - /** - * Access PgSubscriber's internal EventEmitter. - * - * PgSubscriber from @dataplan/pg stores an EventEmitter3 instance as - * `this.eventEmitter`. It is private but stable across v1.x releases. - * This is the same emitter that NOTIFY events are dispatched through. - */ - private getEventEmitter(): { emit(event: string, payload: string): boolean } | null { - const sub = this.subscriber as Record; - if (sub && typeof sub === 'object' && 'eventEmitter' in sub) { - const ee = sub.eventEmitter as { emit(event: string, payload: string): boolean }; - if (typeof ee?.emit === 'function') { - return ee; + private failDelivery(error: Error): void { + this.dispatchEnabled = false; + const stopping = this.stop(); + if (!this.fatalError) { + this.fatalError = error; + try { + this.onFatalError?.(error); + } catch (callbackError) { + log.error( + `RealtimeManager fatal-error callback failed: ${String(callbackError)}` + ); } } - return null; + void stopping.catch((stopError) => { + log.error( + `RealtimeManager failed to stop after a delivery violation: ${String(stopError)}` + ); + }); } } -export { entryToChannel,entryToNotifyPayload, extractRowId }; +export { entryToChannel, entryToNotifyPayload, extractRowId }; diff --git a/graphile/graphile-realtime-subscriptions/src/topic-collector.ts b/graphile/graphile-realtime-subscriptions/src/topic-collector.ts new file mode 100644 index 0000000000..b65e88f0f0 --- /dev/null +++ b/graphile/graphile-realtime-subscriptions/src/topic-collector.ts @@ -0,0 +1,171 @@ +import type { RealtimeTopicDescriptor } from './types'; + +export const REALTIME_TOPIC_DISCOVERY_MISSING_ERROR_CODE = + 'REALTIME_TOPIC_DISCOVERY_MISSING'; +export const REALTIME_TOPIC_DISCOVERY_EMPTY_ERROR_CODE = + 'REALTIME_TOPIC_DISCOVERY_EMPTY'; +export const REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE = + 'REALTIME_TOPIC_DISCOVERY_INVALID'; +export const REALTIME_TOPIC_DISCOVERY_FOREIGN_ERROR_CODE = + 'REALTIME_TOPIC_DISCOVERY_FOREIGN'; +export const REALTIME_TOPIC_DISCOVERY_CHANGED_ERROR_CODE = + 'REALTIME_TOPIC_DISCOVERY_CHANGED'; + +type RealtimeTopicDiscoveryCode = + | typeof REALTIME_TOPIC_DISCOVERY_MISSING_ERROR_CODE + | typeof REALTIME_TOPIC_DISCOVERY_EMPTY_ERROR_CODE + | typeof REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE + | typeof REALTIME_TOPIC_DISCOVERY_FOREIGN_ERROR_CODE + | typeof REALTIME_TOPIC_DISCOVERY_CHANGED_ERROR_CODE; + +export class RealtimeTopicDiscoveryError extends Error { + constructor( + readonly code: RealtimeTopicDiscoveryCode, + message: string + ) { + super(message); + this.name = 'RealtimeTopicDiscoveryError'; + } +} + +const containsUnpairedSurrogate = (value: string): boolean => { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return true; + index++; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +}; + +const assertIdentifierPart = ( + part: 'schema' | 'table', + value: unknown +): string => { + if (typeof value !== 'string' || value.length === 0) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + `Realtime ${part} must be a non-empty string` + ); + } + if ( + value.includes('\0') + || value.includes('.') + || containsUnpairedSurrogate(value) + ) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + `Realtime ${part} cannot be represented unambiguously in a notification topic` + ); + } + return value; +}; + +const normalizeDescriptor = ( + descriptor: RealtimeTopicDescriptor +): Readonly => { + const schema = assertIdentifierPart('schema', descriptor?.schema); + const table = assertIdentifierPart('table', descriptor?.table); + const expectedTopic = `realtime:${schema}.${table}`; + if ( + descriptor?.topic !== expectedTopic + || expectedTopic.includes('\0') + || containsUnpairedSurrogate(expectedTopic) + || Buffer.byteLength(expectedTopic, 'utf8') > 63 + ) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + 'Realtime topic does not exactly match its physical schema/table or exceeds PostgreSQL limits' + ); + } + return Object.freeze({ topic: expectedTopic, schema, table }); +}; + +const descriptorKey = (descriptor: RealtimeTopicDescriptor): string => + `${descriptor.schema}\0${descriptor.table}\0${descriptor.topic}`; + +/** + * One schema-generation collector. It accepts repeated byte-equivalent build + * callbacks, but rejects topic drift so an already activated listener cannot + * silently become incomplete after a Graphile rebuild. + */ +export class RealtimeTopicCollector { + private descriptors: readonly Readonly[] | null = null; + + readonly collect = (input: readonly RealtimeTopicDescriptor[]): void => { + if (!Array.isArray(input)) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + 'Realtime topic discovery did not provide an array' + ); + } + const byTopic = new Map>(); + for (const candidate of input) { + const descriptor = normalizeDescriptor(candidate); + const previous = byTopic.get(descriptor.topic); + if (previous && descriptorKey(previous) !== descriptorKey(descriptor)) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + `Realtime notification topic ${JSON.stringify(descriptor.topic)} is ambiguous` + ); + } + byTopic.set(descriptor.topic, descriptor); + } + const next = Object.freeze( + [...byTopic.values()].sort((left, right) => left.topic.localeCompare(right.topic)) + ); + if (this.descriptors) { + const previousKeys = this.descriptors.map(descriptorKey); + const nextKeys = next.map(descriptorKey); + if ( + previousKeys.length !== nextKeys.length + || previousKeys.some((key, index) => key !== nextKeys[index]) + ) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_CHANGED_ERROR_CODE, + 'Realtime topics changed after the generation discovery boundary' + ); + } + return; + } + this.descriptors = next; + }; + + exactTopics(allowedSchemas: readonly string[]): readonly string[] { + if (!this.descriptors) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_MISSING_ERROR_CODE, + 'Realtime plugin did not report its compiled notification topics' + ); + } + if (this.descriptors.length === 0) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_EMPTY_ERROR_CODE, + 'Shared realtime requires at least one compiled @realtime topic' + ); + } + if ( + !Array.isArray(allowedSchemas) + || allowedSchemas.length === 0 + || allowedSchemas.some((schema) => typeof schema !== 'string' || schema.length === 0) + ) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + 'Shared realtime requires at least one exact allowed physical schema' + ); + } + const allowed = new Set(allowedSchemas); + const foreign = this.descriptors.find(({ schema }) => !allowed.has(schema)); + if (foreign) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_FOREIGN_ERROR_CODE, + `Realtime topic ${JSON.stringify(foreign.topic)} is outside this Graphile generation` + ); + } + return Object.freeze(this.descriptors.map(({ topic }) => topic)); + } +} diff --git a/graphile/graphile-realtime-subscriptions/src/types.ts b/graphile/graphile-realtime-subscriptions/src/types.ts index bbf220ba4d..37de46fb30 100644 --- a/graphile/graphile-realtime-subscriptions/src/types.ts +++ b/graphile/graphile-realtime-subscriptions/src/types.ts @@ -11,6 +11,25 @@ export interface RealtimeSubscriptionsPluginOptions { * Default: 50 */ overflowThreshold?: number; + + /** + * Receives the exact physical PostgreSQL notification topics compiled into + * this schema. The callback runs during schema construction, including with + * an empty list when no @realtime table was discovered. + * + * This is a build-time integration seam. It must not retain Graphile build + * objects or database resources; descriptors contain strings only. + */ + onTopicsDiscovered?: ( + topics: readonly RealtimeTopicDescriptor[] + ) => void; +} + +/** Credential-free description of one compiled @realtime channel. */ +export interface RealtimeTopicDescriptor { + readonly topic: string; + readonly schema: string; + readonly table: string; } /** @@ -28,6 +47,13 @@ export interface Queryable { ): Promise<{ rows: R[] }>; } +/** Explicit local delivery capability used by cursor catch-up. */ +export interface RealtimePublisher { + /** Optional batch preflight used to keep routing violations fail-closed. */ + assertTopics?(topics: readonly string[]): void; + publish(topic: string, payload: string): void; +} + /** * A single entry from drain_changes(), representing a change_log row * matched against subscriber tables. @@ -111,11 +137,31 @@ export interface CursorTrackerOptions { */ export interface RealtimeManagerOptions { /** - * The PgSubscriber instance from PostGraphile's context. - * RealtimeManager emits cursor-tracked events on its internal EventEmitter - * so they flow through existing subscription plans. + * Generation-local publisher used for cursor catch-up delivery. New callers + * should always provide this capability explicitly. + */ + publisher?: RealtimePublisher; + + /** + * Transitional compatibility input for the current @dataplan/pg + * PgSubscriber. Its private emitter is adapted outside RealtimeManager. + * @deprecated Provide publisher instead. + */ + pgSubscriber?: unknown; + + /** + * Exact physical schemas this Graphile instance exposes. Cursor rows naming + * any other source schema stop delivery and surface an error before any row + * in that batch is emitted. */ - pgSubscriber: unknown; + allowedSourceSchemas: readonly string[]; + + /** + * Called once when delivery can no longer be trusted, after new dispatch is + * disabled and manager shutdown has begun. Callers should synchronously + * remove the owning Graphile generation from service. + */ + onFatalError?: (error: Error) => void; /** * A query-capable object (typically a pg.Pool from pg-cache) used by @@ -160,8 +206,10 @@ export interface RealtimeManagerOptions { batchLimit?: number; /** - * Called when an error occurs during polling, heartbeat, or cleanup. - * If not provided, errors are logged via @pgpmjs/logger. + * Observes polling, heartbeat, or cleanup errors. A polling or heartbeat + * error after startup is independently treated as fatal and delivered to + * onFatalError because cursor recovery can no longer be guaranteed. + * If omitted, the error is logged via @pgpmjs/logger. */ onError?: (error: Error) => void; } diff --git a/graphile/graphile-schema/__tests__/scoped-introspection-equivalence.test.ts b/graphile/graphile-schema/__tests__/scoped-introspection-equivalence.test.ts new file mode 100644 index 0000000000..1e7c7f96dd --- /dev/null +++ b/graphile/graphile-schema/__tests__/scoped-introspection-equivalence.test.ts @@ -0,0 +1,155 @@ +import path from 'node:path'; + +import { makeSchema } from 'graphile-build'; +import { makePgService, MinimalPreset } from 'graphile-settings'; +import { + lexicographicSortSchema, + parse, + printSchema, + type ExecutionResult +} from 'graphql'; +import type { Pool } from 'pg'; +import { getConnections, PgTestClient } from 'pgsql-test'; + +const SCHEMA = 'scoped_equivalence'; + +// graphile-schema consumes these through graphile-settings in production; the +// test resolves that package's exact dependency instances without adding +// test-only runtime dependencies to graphile-schema. +const graphileSettingsDirectory = path.dirname(require.resolve('graphile-settings')); +const { execute } = require(require.resolve('grafast', { + paths: [graphileSettingsDirectory] +})); +const { withPgClientFromPgService } = require(require.resolve('graphile-build-pg', { + paths: [graphileSettingsDirectory] +})); + +let pg: PgTestClient; +let pool: Pool; +let teardown: () => Promise; + +beforeAll(async () => { + const connections = await getConnections({}, []); + ({ pg, teardown } = connections); + pool = connections.manager.getPool(pg.config); + + await pg.query(` + CREATE SCHEMA ${SCHEMA}; + CREATE TYPE ${SCHEMA}.item_state AS ENUM ('draft', 'published'); + + CREATE TABLE ${SCHEMA}.organizations ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + name text NOT NULL UNIQUE + ); + + CREATE TABLE ${SCHEMA}.items ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + organization_id bigint NOT NULL REFERENCES ${SCHEMA}.organizations(id), + state ${SCHEMA}.item_state NOT NULL DEFAULT 'draft', + title text NOT NULL + ); + + CREATE FUNCTION ${SCHEMA}.item_title(item ${SCHEMA}.items) + RETURNS text + LANGUAGE sql + IMMUTABLE + AS 'SELECT item.title'; + + CREATE FUNCTION ${SCHEMA}.tenant_token() + RETURNS text + LANGUAGE sql + STABLE + AS 'SELECT ''scoped-equivalence-token''::text'; + + COMMENT ON TABLE ${SCHEMA}.items IS 'Scoped-introspection equivalence canary'; + `); +}); + +beforeEach(async () => { + await pg.beforeEach(); +}); + +afterEach(async () => { + await pg.afterEach(); +}); + +afterAll(async () => { + await teardown(); +}); + +async function build( + mode: 'stock' | 'scoped-required', + schemas = [SCHEMA], + scopedCatalogTypes?: 'all' | 'dependency-closure' +) { + const pgService = makePgService({ + pool, + schemas, + introspectionMode: mode, + ...(scopedCatalogTypes === undefined + ? {} + : { introspectionScopedCatalogTypes: scopedCatalogTypes }) + }); + const built = await makeSchema({ + extends: [MinimalPreset], + pgServices: [pgService] + }); + return { ...built, pgService }; +} + +describe('scoped introspection schema equivalence', () => { + it('builds byte-equivalent SDL and executes the same token in every arm', async () => { + const stockBuild = await build('stock'); + const scopedAllBuild = await build('scoped-required'); + const scopedClosureBuild = await build( + 'scoped-required', + [SCHEMA], + 'dependency-closure' + ); + const stock = printSchema(lexicographicSortSchema(stockBuild.schema)); + const scopedAll = printSchema(lexicographicSortSchema(scopedAllBuild.schema)); + const scopedClosure = printSchema(lexicographicSortSchema( + scopedClosureBuild.schema + )); + + expect(scopedAll).toBe(stock); + expect(scopedClosure).toBe(stock); + expect(scopedClosure).toContain('type Item'); + expect(scopedClosure).toContain('enum ItemState'); + + for (const built of [stockBuild, scopedAllBuild, scopedClosureBuild]) { + const withPgClientKey = built.pgService.withPgClientKey ?? 'withPgClient'; + const result = await execute({ + schema: built.schema, + document: parse('{ tenantToken }'), + contextValue: { + pgSettings: {}, + [withPgClientKey]: withPgClientFromPgService.bind( + null, + built.pgService + ) + }, + resolvedPreset: built.resolvedPreset + }) as ExecutionResult<{ tenantToken?: unknown }>; + if (Symbol.asyncIterator in result) { + throw new Error('tenant token canary unexpectedly returned a stream'); + } + expect(result.errors).toBeUndefined(); + expect(result.data).toEqual({ tenantToken: 'scoped-equivalence-token' }); + } + }); + + it.each([undefined, 'dependency-closure'] as const)( + 'fails closed when a required schema is absent (catalog types: %s)', + async (scopedCatalogTypes) => { + await expect(build( + 'scoped-required', + ['missing_required_schema'], + scopedCatalogTypes + )) + .rejects.toThrow( + 'did not find required schema(s): missing_required_schema' + ); + } + ); +}); diff --git a/graphile/graphile-search/package.json b/graphile/graphile-search/package.json index 076e377cca..8e739f9404 100644 --- a/graphile/graphile-search/package.json +++ b/graphile/graphile-search/package.json @@ -29,6 +29,7 @@ "url": "https://github.com/constructive-io/constructive/issues" }, "dependencies": { + "@pgsql/quotes": "^18.1.0", "graphile-plugin-utils": "workspace:^" }, "devDependencies": { diff --git a/graphile/graphile-search/src/__tests__/bm25-codec.test.ts b/graphile/graphile-search/src/__tests__/bm25-codec.test.ts new file mode 100644 index 0000000000..ad49436a45 --- /dev/null +++ b/graphile/graphile-search/src/__tests__/bm25-codec.test.ts @@ -0,0 +1,353 @@ +import { createBm25Adapter } from '../adapters/bm25'; +import { collectBm25Indexes } from '../codecs/bm25-codec'; + +interface MockIndexOptions { + schema: string; + classId: string; + table?: string; + column?: string; + attributeNumber?: number; + index: string; + accessMethod?: string; + valid?: boolean | null; + ready?: boolean | null; + live?: boolean | null; +} + +const mockIndex = ({ + schema, + classId, + table = 'documents', + column = 'body', + attributeNumber = 2, + index, + accessMethod = 'bm25', + valid = true, + ready = true, + live = true +}: MockIndexOptions) => { + const namespace = { nspname: schema }; + const tableClass = { + _id: classId, + relname: table, + relnamespace: schema, + getNamespace: () => namespace + }; + const indexClass = { + relname: index, + getAccessMethod: () => ({ amname: accessMethod }) + }; + const attribute = { attnum: attributeNumber, attname: column }; + return { + indisvalid: valid, + indisready: ready, + indislive: live, + indnkeyatts: 1, + indkey: [attributeNumber], + getIndexClass: () => indexClass, + getClass: () => tableClass, + getKeys: () => [attribute] + }; +}; + +const introspectionWith = (...indexes: ReturnType[]) => ({ + indexes, + extensions: [{ extname: 'pg_textsearch', extnamespace: '900' }], + getNamespace: ({ id }: { id: string }) => id === '900' + ? { _id: '900', nspname: 'extension_tools' } + : { _id: id, nspname: id } +} as any); + +describe('BM25 introspection binding', () => { + it('keeps identical tenant table names isolated by the configured schema', () => { + const introspection = introspectionWith( + mockIndex({ + schema: 'tenant_a', + classId: '100', + index: 'tenant_a_documents_body_bm25_idx' + }), + mockIndex({ + schema: 'tenant_b', + classId: '200', + index: 'tenant_b_documents_body_bm25_idx' + }) + ); + + const tenantA = collectBm25Indexes(introspection, ['tenant_a'], 'service_a'); + const tenantB = collectBm25Indexes(introspection, ['tenant_b'], 'service_b'); + + expect([...tenantA.values()]).toEqual([ + { + serviceName: 'service_a', + extensionSchema: 'extension_tools', + schemaName: 'tenant_a', + tableName: 'documents', + columnName: 'body', + indexName: 'tenant_a_documents_body_bm25_idx' + } + ]); + expect([...tenantB.values()]).toEqual([ + { + serviceName: 'service_b', + extensionSchema: 'extension_tools', + schemaName: 'tenant_b', + tableName: 'documents', + columnName: 'body', + indexName: 'tenant_b_documents_body_bm25_idx' + } + ]); + }); + + it('does not retain indexes across rebuilds', () => { + const first = collectBm25Indexes( + introspectionWith( + mockIndex({ schema: 'tenant_a', classId: '100', index: 'first_idx' }) + ), + ['tenant_a'], + 'main' + ); + const rebuilt = collectBm25Indexes(introspectionWith(), ['tenant_a'], 'main'); + + expect(first.size).toBe(1); + expect(rebuilt.size).toBe(0); + }); + + it('skips invalid and non-BM25 indexes', () => { + const result = collectBm25Indexes( + introspectionWith( + mockIndex({ + schema: 'tenant_a', + classId: '100', + index: 'invalid_idx', + valid: false + }), + mockIndex({ + schema: 'tenant_a', + classId: '100', + index: 'unknown_state_idx', + live: null + }), + mockIndex({ + schema: 'tenant_a', + classId: '100', + index: 'btree_idx', + accessMethod: 'btree' + }) + ), + ['tenant_a'], + 'main' + ); + + expect(result.size).toBe(0); + }); + + it('fails deterministically when two BM25 indexes target one attribute', () => { + const introspection = introspectionWith( + mockIndex({ schema: 'tenant_a', classId: '100', index: 'z_idx' }), + mockIndex({ schema: 'tenant_a', classId: '100', index: 'a_idx' }) + ); + + expect(() => collectBm25Indexes(introspection, ['tenant_a'], 'main')).toThrow( + 'Multiple BM25 indexes target tenant_a.documents.body: a_idx, z_idx' + ); + }); + + it('lets the adapter consume metadata bound to the exact codec attribute', () => { + const adapter = createBm25Adapter(); + const codec = { + attributes: { + body: { + codec: { name: 'text' }, + extensions: { + bm25Index: { + serviceName: 'main', + extensionSchema: 'extension_tools', + schemaName: 'tenant_a', + tableName: 'documents', + columnName: 'body', + indexName: 'tenant_a_documents_body_bm25_idx' + } + } + } + } + }; + + expect(adapter.detectColumns(codec, {})).toEqual([ + { + attributeName: 'body', + adapterData: { + bm25Index: codec.attributes.body.extensions.bm25Index, + chunksInfo: undefined + } + } + ]); + }); + + it('binds chunk search to its introspected physical index name', () => { + const adapter = createBm25Adapter(); + const parentIndex = { + serviceName: 'service_a', + extensionSchema: 'extension_tools', + schemaName: 'tenant_a', + tableName: 'documents', + columnName: 'body', + indexName: 'parent_idx' + }; + const chunkIndex = { + serviceName: 'service_a', + extensionSchema: 'extension_tools', + schemaName: 'tenant_a', + tableName: 'document_chunks', + columnName: 'content', + indexName: 'nonconventional_exact_chunk_idx' + }; + const parentCodec = { + attributes: { + id: { codec: { name: 'uuid' } }, + body: { + codec: { name: 'text' }, + extensions: { bm25Index: parentIndex } + } + }, + extensions: { + pg: { serviceName: 'service_a', schemaName: 'tenant_a', name: 'documents' }, + tags: { + hasChunks: { + chunksTable: 'document_chunks', + contentField: 'content', + searchIndexes: ['bm25'] + } + } + } + }; + const chunksCodec = { + attributes: { + parent_id: {}, + embedding: {}, + content: { extensions: { bm25Index: chunkIndex } } + }, + extensions: { + pg: { + serviceName: 'service_a', + schemaName: 'tenant_a', + name: 'document_chunks' + } + } + }; + const build = { + input: { + pgRegistry: { + pgCodecs: { + wrongServiceChunks: { + attributes: { + content: { + extensions: { + bm25Index: { + ...chunkIndex, + serviceName: 'service_b', + indexName: 'wrong_service_idx' + } + } + } + } + }, + chunks: chunksCodec + }, + pgResources: { chunks: { codec: chunksCodec } } + } + }, + resolvedPreset: { + pgServices: [{ name: 'service_a', schemas: ['tenant_a'] }] + } + }; + + const [column] = adapter.detectColumns(parentCodec, build); + expect(column.adapterData).toEqual({ + bm25Index: parentIndex, + chunksInfo: { + chunksSchema: 'tenant_a', + chunksTableName: 'document_chunks', + parentFkField: 'parent_id', + parentPkField: 'id', + embeddingField: 'embedding', + contentField: 'content', + searchField: null, + searchIndexes: ['bm25'] + }, + chunkBm25Index: chunkIndex + }); + + const sql = Object.assign( + (strings: TemplateStringsArray, ...values: any[]) => + strings.reduce((text, part, index) => text + part + (values[index] ?? ''), ''), + { + identifier: (...names: string[]) => names.map((name) => `"${name}"`).join('.'), + value: (value: any) => `'${value}'` + } + ); + const result = adapter.buildFilterApply( + sql, + 'docs' as any, + column, + { query: 'tenant memory' }, + build + ); + expect(String(result?.scoreExpression)).toContain( + 'tenant_a.nonconventional_exact_chunk_idx' + ); + expect(String(result?.scoreExpression)).toContain( + 'OPERATOR("extension_tools".<@>)' + ); + expect(String(result?.scoreExpression)).not.toContain('wrong_service_idx'); + }); + + it('fails closed when chunk BM25 metadata cannot be bound', () => { + const adapter = createBm25Adapter(); + const codec = { + attributes: { + id: { codec: { name: 'uuid' } }, + body: { + codec: { name: 'text' }, + extensions: { + bm25Index: { + serviceName: 'main', + extensionSchema: 'extension_tools', + schemaName: 'tenant_a', + tableName: 'documents', + columnName: 'body', + indexName: 'parent_idx' + } + } + } + }, + extensions: { + pg: { serviceName: 'main', schemaName: 'tenant_a', name: 'documents' }, + tags: { + hasChunks: { + chunksTable: 'document_chunks', + searchIndexes: ['bm25'] + } + } + } + }; + const chunksCodec = { + attributes: { parent_id: {}, embedding: {}, content: {} }, + extensions: { + pg: { serviceName: 'main', schemaName: 'tenant_a', name: 'document_chunks' } + } + }; + + expect(() => adapter.detectColumns(codec, { + input: { + pgRegistry: { + pgCodecs: { chunks: chunksCodec }, + pgResources: { chunks: { codec: chunksCodec } } + } + }, + resolvedPreset: { pgServices: [{ name: 'main', schemas: ['tenant_a'] }] } + })).toThrow( + 'BM25 chunk search could not bind an introspected index for ' + + 'tenant_a.document_chunks.content' + ); + }); +}); diff --git a/graphile/graphile-search/src/__tests__/chunks-isolation.test.ts b/graphile/graphile-search/src/__tests__/chunks-isolation.test.ts new file mode 100644 index 0000000000..e4e38fab02 --- /dev/null +++ b/graphile/graphile-search/src/__tests__/chunks-isolation.test.ts @@ -0,0 +1,92 @@ +import { getChunksInfo } from '../adapters/chunks'; + +function fixture(overrides: { + chunksSchema?: string; + duplicate?: boolean; + dependencySchemas?: string[]; +} = {}) { + const parentCodec = { + name: 'documents', + attributes: { id: {} }, + extensions: { + pg: { serviceName: 'main', schemaName: 'tenant_a', name: 'documents' }, + tags: { + hasChunks: { + chunksSchema: overrides.chunksSchema ?? 'tenant_a', + chunksTable: 'documents_chunks', + parentFk: 'document_id', + }, + }, + }, + }; + const chunkCodec = { + name: 'documentsChunks', + attributes: { + document_id: {}, + embedding: {}, + content: {}, + }, + extensions: { + pg: { + serviceName: 'main', + schemaName: overrides.chunksSchema ?? 'tenant_a', + name: 'documents_chunks', + }, + }, + }; + const resource = { codec: chunkCodec }; + return { + parentCodec, + build: { + input: { + pgRegistry: { + pgResources: overrides.duplicate + ? { first: resource, second: { codec: chunkCodec } } + : { chunks: resource }, + }, + }, + resolvedPreset: { + pgServices: [{ + name: 'main', + schemas: ['tenant_a'], + introspectionAllowedDependencySchemas: overrides.dependencySchemas ?? [], + }], + }, + }, + }; +} + +describe('@hasChunks exact-build isolation', () => { + it('resolves one exact resource inside the service schema allowlist', () => { + const { parentCodec, build } = fixture(); + expect(getChunksInfo(parentCodec, build)).toMatchObject({ + chunksSchema: 'tenant_a', + chunksTableName: 'documents_chunks', + parentFkField: 'document_id', + }); + }); + + it('rejects a cross-schema resource even when it exists in the registry', () => { + const { parentCodec, build } = fixture({ chunksSchema: 'tenant_b' }); + expect(() => getChunksInfo(parentCodec, build)).toThrow(/outside service 'main'/); + }); + + it('accepts a resource only when its dependency schema is explicitly allowlisted', () => { + const { parentCodec, build } = fixture({ + chunksSchema: 'tenant_chunks', + dependencySchemas: ['tenant_chunks'], + }); + expect(getChunksInfo(parentCodec, build)?.chunksSchema).toBe('tenant_chunks'); + }); + + it('rejects ambiguous exact resource matches', () => { + const { parentCodec, build } = fixture({ duplicate: true }); + expect(() => getChunksInfo(parentCodec, build)).toThrow(/matches=2/); + }); + + it('rejects malformed tags instead of silently disabling chunk routing', () => { + const { parentCodec, build } = fixture(); + parentCodec.extensions.tags.hasChunks = 'not-json' as any; + expect(() => getChunksInfo(parentCodec, build)).toThrow(/valid JSON/); + }); +}); diff --git a/graphile/graphile-search/src/__tests__/extension-schema-qualification.test.ts b/graphile/graphile-search/src/__tests__/extension-schema-qualification.test.ts new file mode 100644 index 0000000000..95141bf70d --- /dev/null +++ b/graphile/graphile-search/src/__tests__/extension-schema-qualification.test.ts @@ -0,0 +1,322 @@ +import sql from 'pg-sql2'; + +import { createPgvectorAdapter } from '../adapters/pgvector'; +import { createTrgmAdapter } from '../adapters/trgm'; +import { createTrgmOperatorFactories } from '../codecs/operator-factories'; +import { VectorCodecPlugin } from '../codecs/vector-codec'; +import { + collectSearchExtensionSchemas, + extensionSchemasByService, + requireBuildExtensionSchema, + resolveBuildExtensionSchema, + type SearchExtensionSchemas, +} from '../extension-metadata'; + +const extensionBinding = ( + overrides: Partial = {} +): SearchExtensionSchemas => ({ + serviceName: 'tenant_service', + pgTrgmSchema: 'extension_tools', + pgvectorSchema: 'extension_tools', + ...overrides, +}); + +const introspection = (extensions: any[]) => ({ + extensions, + getNamespace: ({ id }: { id: string }) => + id === '910' ? { nspname: 'extension_tools' } : undefined, +} as any); + +describe('search extension schema binding', () => { + it('collects exact pg_trgm and pgvector schemas from one service introspection', () => { + expect(collectSearchExtensionSchemas( + introspection([ + { extname: 'pg_trgm', extnamespace: '910' }, + { extname: 'vector', extnamespace: '910' }, + ]), + 'tenant_service' + )).toEqual(extensionBinding()); + }); + + it('fails closed on ambiguous or unresolved extension metadata', () => { + expect(() => collectSearchExtensionSchemas( + introspection([ + { extname: 'pg_trgm', extnamespace: '910' }, + { extname: 'pg_trgm', extnamespace: '910' }, + ]), + 'tenant_service' + )).toThrow(/ambiguous pg_trgm/); + + expect(() => collectSearchExtensionSchemas( + introspection([{ extname: 'pg_trgm', extnamespace: '999' }]), + 'tenant_service' + )).toThrow(/cannot resolve the namespace/); + }); + + it('requires one complete build-wide schema for shared operator factories', () => { + expect(requireBuildExtensionSchema({ + pgSearchExtensionSchemasByService: new Map([ + ['tenant_service', extensionBinding()], + ]), + }, 'pg_trgm')).toBe('extension_tools'); + + expect(() => requireBuildExtensionSchema({ + pgSearchExtensionSchemasByService: new Map([ + ['a', extensionBinding({ serviceName: 'a', pgTrgmSchema: 'ext_a' })], + ['b', extensionBinding({ serviceName: 'b', pgTrgmSchema: 'ext_b' })], + ]), + }, 'pg_trgm')).toThrow(/ambiguous schemas/); + + const absentBuild = { + pgSearchExtensionSchemasByService: new Map([ + ['tenant_service', extensionBinding({ pgTrgmSchema: null })], + ]), + }; + expect(resolveBuildExtensionSchema(absentBuild, 'pg_trgm')).toBeNull(); + expect(() => requireBuildExtensionSchema(absentBuild, 'pg_trgm')).toThrow( + /required by this feature but is not installed/ + ); + }); + + it('disables optional operators for an empty service and still fails on missing record metadata', () => { + const emptyBuild = { + input: { pgRegistry: { pgCodecs: { text: { name: 'text' } } } }, + }; + expect(resolveBuildExtensionSchema(emptyBuild, 'pg_trgm')).toBeNull(); + expect(createTrgmOperatorFactories()(emptyBuild as any)).toEqual([]); + + const recordWithoutBinding = { + input: { + pgRegistry: { + pgCodecs: { + animals: { name: 'animals', attributes: {} }, + }, + }, + }, + }; + expect(() => resolveBuildExtensionSchema( + recordWithoutBinding, + 'pg_trgm' + )).toThrow(/requires service-bound extension metadata/); + }); + + it('retains service identity on a record codec even without attributes', () => { + const binding = extensionBinding(); + const build = { + input: { + pgRegistry: { + pgCodecs: { + emptyRecord: { + name: 'empty_record', + extensions: { searchExtensionSchemas: binding }, + }, + }, + }, + }, + }; + expect(extensionSchemasByService(build).get('tenant_service')).toBe(binding); + }); +}); + +describe('pg_trgm SQL qualification', () => { + const adapter = createTrgmAdapter({ requireIntentionalSearch: false }); + + it('binds metadata to the eligible attribute and qualifies similarity', () => { + const codec = { + name: 'documents', + attributes: { + title: { + codec: { name: 'text' }, + extensions: { searchExtensionSchemas: extensionBinding() }, + }, + }, + }; + const [column] = adapter.detectColumns(codec, {}); + const result = adapter.buildFilterApply( + sql, + sql.identifier('documents'), + column, + { value: 'memory density', threshold: 0.2 }, + {} + ); + const compiled = sql.compile(result!.whereClause!); + expect(compiled.text).toContain( + '"extension_tools"."similarity"("documents"."title", $1)' + ); + expect(compiled.text).not.toMatch(/(^|[^."])similarity\(/); + }); + + it('qualifies similarity and word_similarity in operator factories', () => { + const registrations = createTrgmOperatorFactories()({ + sql, + pgSearchExtensionSchemasByService: new Map([ + ['tenant_service', extensionBinding()], + ]), + getTypeByName: () => ({ name: 'TrgmSearchInput' }), + } as any); + const similar = registrations.find((entry) => entry.operatorName === 'similarTo')!; + const word = registrations.find((entry) => entry.operatorName === 'wordSimilarTo')!; + + const similarSql = similar.spec.resolve!( + sql.identifier('title'), + sql.null, + { value: 'memory', threshold: 0.3 }, + null, + { fieldName: 'title', operatorName: 'similarTo' } + ); + const wordSql = word.spec.resolve!( + sql.identifier('title'), + sql.null, + { value: 'memory', threshold: 0.3 }, + null, + { fieldName: 'title', operatorName: 'wordSimilarTo' } + ); + expect(sql.compile(similarSql!).text).toContain( + '"extension_tools"."similarity"' + ); + expect(sql.compile(wordSql!).text).toContain( + '"extension_tools"."word_similarity"' + ); + }); + + it('fails closed when an eligible attribute has no bound schema', () => { + expect(() => adapter.detectColumns({ + name: 'documents', + attributes: { title: { codec: { name: 'text' } } }, + }, {})).toThrow(/missing service-bound extension schema/); + }); + + it('disables the adapter and operator factory when pg_trgm is absent', () => { + const absent = extensionBinding({ pgTrgmSchema: null }); + const defaultAdapter = createTrgmAdapter(); + expect(defaultAdapter.detectColumns({ + name: 'documents', + attributes: { + title: { + codec: { name: 'text' }, + extensions: { searchExtensionSchemas: absent }, + }, + }, + }, {})).toEqual([]); + + expect(createTrgmOperatorFactories()({ + sql, + pgSearchExtensionSchemasByService: new Map([ + ['tenant_service', absent], + ]), + } as any)).toEqual([]); + }); + + it('fails closed when an explicit trgm feature requires an absent extension', () => { + expect(() => adapter.detectColumns({ + name: 'documents', + attributes: { + title: { + codec: { name: 'text' }, + extensions: { + searchExtensionSchemas: extensionBinding({ pgTrgmSchema: null }), + }, + }, + }, + }, {})).toThrow(/required .* but is not installed/); + }); +}); + +describe('pgvector SQL qualification', () => { + const vectorCodec = { + name: 'vector', + extensions: { + pg: { + serviceName: 'tenant_service', + schemaName: 'extension_tools', + name: 'vector', + }, + }, + }; + + it('qualifies and annotates a native vector codec during gather', async () => { + const gatherHook = (VectorCodecPlugin as any).gather.hooks.pgCodecs_findPgCodec; + const event: any = { + pgCodec: { + name: 'vector', + sqlType: sql.fragment`vector`, + extensions: undefined, + }, + pgType: { typname: 'vector', typnamespace: '910', _id: '912' }, + serviceName: 'tenant_service', + }; + const originalCodec = event.pgCodec; + await gatherHook({ + helpers: { + pgIntrospection: { + getNamespace: jest.fn().mockResolvedValue({ nspname: 'extension_tools' }), + }, + }, + }, event); + + expect(event.pgCodec).toBe(originalCodec); + expect(event.pgCodec.extensions).toMatchObject({ + oid: '912', + pg: { + serviceName: 'tenant_service', + schemaName: 'extension_tools', + name: 'vector', + }, + }); + expect(sql.compile(event.pgCodec.sqlType).text).toBe('"extension_tools"."vector"'); + }); + + it('fails closed when the vector type namespace cannot be resolved', async () => { + const gatherHook = (VectorCodecPlugin as any).gather.hooks.pgCodecs_findPgCodec; + await expect(gatherHook({ + helpers: { + pgIntrospection: { + getNamespace: jest.fn().mockResolvedValue(undefined), + }, + }, + }, { + pgType: { typname: 'vector', typnamespace: '999', _id: '912' }, + serviceName: 'tenant_service', + })).rejects.toThrow(/cannot resolve the vector type namespace/i); + }); + + it('qualifies the vector cast and distance operator', () => { + const adapter = createPgvectorAdapter(); + const [column] = adapter.detectColumns({ + name: 'documents', + attributes: { + embedding: { + codec: vectorCodec, + extensions: { searchExtensionSchemas: extensionBinding() }, + }, + }, + }, {}); + const result = adapter.buildFilterApply( + sql, + sql.identifier('documents'), + column, + { vector: [1, 0, 0], metric: 'COSINE' }, + {} + ); + const compiled = sql.compile(result!.scoreExpression); + expect(compiled.text).toContain('::"extension_tools"."vector"'); + expect(compiled.text).toContain('OPERATOR("extension_tools".<=>)'); + }); + + it('fails closed when codec and extension identities disagree', () => { + const adapter = createPgvectorAdapter(); + expect(() => adapter.detectColumns({ + name: 'documents', + attributes: { + embedding: { + codec: vectorCodec, + extensions: { + searchExtensionSchemas: extensionBinding({ + pgvectorSchema: 'other_extension_schema', + }), + }, + }, + }, + }, {})).toThrow(/does not match extension/); + }); +}); diff --git a/graphile/graphile-search/src/__tests__/plugin-cache-isolation.test.ts b/graphile/graphile-search/src/__tests__/plugin-cache-isolation.test.ts new file mode 100644 index 0000000000..07848e9977 --- /dev/null +++ b/graphile/graphile-search/src/__tests__/plugin-cache-isolation.test.ts @@ -0,0 +1,66 @@ +import type { SearchAdapter } from '../types'; +import { createUnifiedSearchPlugin } from '../plugin'; + +describe('UnifiedSearchPlugin build isolation', () => { + it('does not reuse discovery results across same-named codecs from different builds', () => { + const detectColumns = jest.fn((codec: any, build: any) => [ + { attributeName: build.tenantColumn ?? codec.extensions.tenantColumn }, + ]); + const adapter: SearchAdapter = { + name: 'tenant-test', + filterPrefix: 'tenantTest', + scoreSemantics: { metric: 'score', lowerIsBetter: false, range: null }, + detectColumns, + registerTypes: jest.fn(), + getFilterTypeName: jest.fn(() => 'TenantTestInput'), + buildFilterApply: jest.fn(), + }; + const plugin = createUnifiedSearchPlugin({ + adapters: [adapter], + enableSearchScore: false, + enableUnifiedSearch: false, + }); + const callback = (plugin.schema!.entityBehavior!.pgCodecAttribute as any) + .inferred.callback; + const buildA = { tenantColumn: 'tenant_a_search' }; + const buildB = { tenantColumn: 'tenant_b_search' }; + const codecA = { + name: 'documents', + attributes: { tenant_a_search: {} }, + extensions: { tenantColumn: 'tenant_a_search' }, + }; + const codecB = { + name: 'documents', + attributes: { tenant_b_search: {} }, + extensions: { tenantColumn: 'tenant_b_search' }, + }; + + expect(callback('default', [codecA, 'tenant_a_search'], buildA)).toContain( + 'unifiedSearch:select' + ); + expect(callback('default', [codecB, 'tenant_b_search'], buildB)).toContain( + 'unifiedSearch:select' + ); + expect(callback('default', [codecB, 'tenant_a_search'], buildB)).toBe('default'); + expect(detectColumns).toHaveBeenCalledTimes(2); + + // Reusing the same codec object within one build should still hit the cache. + callback('default', [codecB, 'tenant_b_search'], buildB); + expect(detectColumns).toHaveBeenCalledTimes(2); + + // Discovery is build-dependent, so even a reused codec object must not + // carry adapter metadata across builds. + const sharedCodec = { + name: 'shared_documents', + attributes: { tenant_a_search: {}, tenant_b_search: {} }, + extensions: { tenantColumn: 'unused' }, + }; + expect(callback('default', [sharedCodec, 'tenant_a_search'], buildA)).toContain( + 'unifiedSearch:select' + ); + expect(callback('default', [sharedCodec, 'tenant_b_search'], buildB)).toContain( + 'unifiedSearch:select' + ); + expect(detectColumns).toHaveBeenCalledTimes(4); + }); +}); diff --git a/graphile/graphile-search/src/__tests__/search-config.test.ts b/graphile/graphile-search/src/__tests__/search-config.test.ts index fd7a559161..7132cb0547 100644 --- a/graphile/graphile-search/src/__tests__/search-config.test.ts +++ b/graphile/graphile-search/src/__tests__/search-config.test.ts @@ -13,6 +13,31 @@ import { createPgvectorAdapter } from '../adapters/pgvector'; import { createTsvectorAdapter } from '../adapters/tsvector'; import { createUnifiedSearchPlugin } from '../plugin'; +const VECTOR_ADAPTER_IDENTITY = { + serviceName: 'main', + extensionSchema: 'extension_tools', +}; + +const vectorAttribute = () => ({ + codec: { + name: 'vector', + extensions: { + pg: { + serviceName: 'main', + schemaName: 'extension_tools', + name: 'vector', + }, + }, + }, + extensions: { + searchExtensionSchemas: { + serviceName: 'main', + pgTrgmSchema: 'extension_tools', + pgvectorSchema: 'extension_tools', + }, + }, +}); + // ─── pgvector adapter: chunk detection ──────────────────────────────────────── describe('pgvector adapter — chunk querying (Phase E)', () => { @@ -24,7 +49,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { name: 'documents', attributes: { id: { codec: { name: 'uuid' } }, - embedding: { codec: { name: 'vector' } }, + embedding: vectorAttribute(), }, extensions: { tags: {} }, }; @@ -32,7 +57,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { const columns = adapter.detectColumns(codec, {}); expect(columns).toHaveLength(1); expect(columns[0].attributeName).toBe('embedding'); - expect(columns[0].adapterData).toBeUndefined(); + expect(columns[0].adapterData).toEqual(VECTOR_ADAPTER_IDENTITY); }); it('includes chunksInfo when @hasChunks smart tag has metadata', () => { @@ -40,7 +65,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { name: 'documents', attributes: { id: { codec: { name: 'uuid' } }, - embedding: { codec: { name: 'vector' } }, + embedding: vectorAttribute(), }, extensions: { tags: { @@ -58,6 +83,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { expect(columns).toHaveLength(1); expect(columns[0].attributeName).toBe('embedding'); expect(columns[0].adapterData).toEqual({ + ...VECTOR_ADAPTER_IDENTITY, chunksInfo: { chunksSchema: 'app_public', chunksTableName: 'documents_chunks', @@ -75,7 +101,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { const codec = { name: 'documents', attributes: { - embedding: { codec: { name: 'vector' } }, + embedding: vectorAttribute(), }, extensions: { tags: { @@ -93,6 +119,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { const columns = adapter.detectColumns(codec, {}); expect(columns).toHaveLength(1); expect(columns[0].adapterData).toEqual({ + ...VECTOR_ADAPTER_IDENTITY, chunksInfo: { chunksSchema: 'private_schema', chunksTableName: 'doc_chunks', @@ -106,11 +133,11 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { }); }); - it('uses default parentFk, parentPk, and embeddingField when not specified', () => { + it('fails closed when neither the tag nor parent codec identifies a schema', () => { const codec = { name: 'documents', attributes: { - embedding: { codec: { name: 'vector' } }, + embedding: vectorAttribute(), }, extensions: { tags: { @@ -119,26 +146,14 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { }, }; - const columns = adapter.detectColumns(codec, {}); - expect(columns[0].adapterData).toEqual({ - chunksInfo: { - chunksSchema: null, - chunksTableName: 'my_chunks', - parentFkField: 'parent_id', - parentPkField: 'id', - embeddingField: 'embedding', - contentField: 'content', - searchField: null, - searchIndexes: [], - }, - }); + expect(() => adapter.detectColumns(codec, {})).toThrow(/chunksSchema/); }); it('inherits schema from parent codec when not explicitly set', () => { const codec = { name: 'documents', attributes: { - embedding: { codec: { name: 'vector' } }, + embedding: vectorAttribute(), }, extensions: { tags: { @@ -150,6 +165,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { const columns = adapter.detectColumns(codec, {}); expect(columns[0].adapterData).toEqual({ + ...VECTOR_ADAPTER_IDENTITY, chunksInfo: { chunksSchema: 'my_schema', chunksTableName: 'my_chunks', @@ -163,36 +179,32 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { }); }); - it('ignores boolean true @hasChunks (no metadata to resolve)', () => { + it('rejects boolean true @hasChunks (no metadata to resolve)', () => { const codec = { name: 'documents', attributes: { - embedding: { codec: { name: 'vector' } }, + embedding: vectorAttribute(), }, extensions: { tags: { hasChunks: true }, }, }; - const columns = adapter.detectColumns(codec, {}); - expect(columns).toHaveLength(1); - expect(columns[0].adapterData).toBeUndefined(); + expect(() => adapter.detectColumns(codec, {})).toThrow(/JSON object/); }); - it('ignores invalid JSON in @hasChunks string', () => { + it('rejects invalid JSON in @hasChunks string', () => { const codec = { name: 'documents', attributes: { - embedding: { codec: { name: 'vector' } }, + embedding: vectorAttribute(), }, extensions: { tags: { hasChunks: 'not-valid-json' }, }, }; - const columns = adapter.detectColumns(codec, {}); - expect(columns).toHaveLength(1); - expect(columns[0].adapterData).toBeUndefined(); + expect(() => adapter.detectColumns(codec, {})).toThrow(/valid JSON/); }); it('does not detect chunks when enableChunkQuerying is false', () => { @@ -200,7 +212,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { const codec = { name: 'documents', attributes: { - embedding: { codec: { name: 'vector' } }, + embedding: vectorAttribute(), }, extensions: { tags: { @@ -211,7 +223,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { const columns = noChunksAdapter.detectColumns(codec, {}); expect(columns).toHaveLength(1); - expect(columns[0].adapterData).toBeUndefined(); + expect(columns[0].adapterData).toEqual(VECTOR_ADAPTER_IDENTITY); }); }); @@ -220,7 +232,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { // Mock sql object that mimics pg-sql2 behavior const mockSql = { - identifier: (name: string) => `"${name}"`, + identifier: (...names: string[]) => names.map((name) => `"${name}"`).join('.'), value: (val: any) => `'${val}'`, raw: (s: string) => s, fragment: (strings: TemplateStringsArray, ...values: any[]) => { @@ -251,7 +263,10 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { const result = adapter.buildFilterApply( sql, 'tbl' as any, - { attributeName: 'embedding' }, + { + attributeName: 'embedding', + adapterData: VECTOR_ADAPTER_IDENTITY, + }, { vector: [1, 0, 0], metric: 'COSINE' }, {}, ); @@ -269,6 +284,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { { attributeName: 'embedding', adapterData: { + ...VECTOR_ADAPTER_IDENTITY, chunksInfo: { chunksSchema: null, chunksTableName: 'documents_chunks', @@ -296,6 +312,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { { attributeName: 'embedding', adapterData: { + ...VECTOR_ADAPTER_IDENTITY, chunksInfo: { chunksSchema: null, chunksTableName: 'documents_chunks', @@ -323,6 +340,7 @@ describe('pgvector adapter — chunk querying (Phase E)', () => { { attributeName: 'embedding', adapterData: { + ...VECTOR_ADAPTER_IDENTITY, chunksInfo: { chunksSchema: 'app_private', chunksTableName: 'doc_chunks', @@ -423,3 +441,44 @@ describe('VectorNearbyInput includeChunks field (Phase E)', () => { expect(fields.includeChunks.description).toContain('chunks'); }); }); + +describe('BM25 index qualification', () => { + const sql = Object.assign( + (strings: TemplateStringsArray, ...values: any[]) => + strings.reduce((text, part, index) => text + part + (values[index] ?? ''), ''), + { + identifier: (name: string) => `"${name}"`, + value: (value: any) => `'${value}'` + } + ); + + it('passes the physical schema-qualified index name as the regclass-like value', () => { + const adapter = createBm25Adapter(); + const result = adapter.buildFilterApply( + sql, + 'docs' as any, + { + attributeName: 'body', + adapterData: { + bm25Index: { + serviceName: 'main', + extensionSchema: 'extension_tools', + schemaName: 'tenant-a-app-public', + tableName: 'documents', + columnName: 'body', + indexName: 'documents_body_bm25_idx' + } + } + }, + { query: 'memory density' }, + {} + ); + + expect(String(result?.scoreExpression)).toContain( + '"tenant-a-app-public".documents_body_bm25_idx' + ); + expect(String(result?.scoreExpression)).toContain( + 'OPERATOR("extension_tools".<@>)' + ); + }); +}); diff --git a/graphile/graphile-search/src/adapters/bm25.ts b/graphile/graphile-search/src/adapters/bm25.ts index d5ebd22254..25aa6f338f 100644 --- a/graphile/graphile-search/src/adapters/bm25.ts +++ b/graphile/graphile-search/src/adapters/bm25.ts @@ -4,8 +4,8 @@ * Detects text columns with BM25 indexes (via pg_textsearch) and generates * BM25 relevance scoring. Wraps the same SQL logic as graphile-bm25. * - * Requires the Bm25CodecPlugin to be loaded first (for index discovery). - * The adapter reads from the bm25IndexStore populated during the gather phase. + * Requires the Bm25CodecPlugin to be loaded first. The adapter reads index + * metadata attached to the exact codec attribute during the same gather. * * Supports chunk-aware querying via @hasChunks smart tag: when the parent * table has chunks with a BM25 index, the adapter includes a lateral @@ -13,26 +13,20 @@ * LEAST(parent_score, chunk_score) (lower = better for BM25). */ +import { QuoteUtils } from '@pgsql/quotes'; import type { SQL } from 'pg-sql2'; -import { bm25IndexStore as moduleBm25IndexStore } from '../codecs/bm25-codec'; -import type { FilterApplyResult,SearchableColumn, SearchAdapter } from '../types'; +import type { Bm25IndexInfo } from '../codecs/bm25-codec'; +import type { FilterApplyResult, SearchableColumn, SearchAdapter } from '../types'; import { type ChunksInfo,getChunksInfo } from './chunks'; -/** - * BM25 index info discovered during gather phase. - */ -export interface Bm25IndexInfo { - schemaName: string; - tableName: string; - columnName: string; - indexName: string; -} +export type { Bm25IndexInfo } from '../codecs/bm25-codec'; /** Combined adapter data for a BM25-searchable column */ interface Bm25ColumnData { bm25Index: Bm25IndexInfo; chunksInfo?: ChunksInfo; + chunkBm25Index?: Bm25IndexInfo; } function isTextCodec(codec: any): boolean { @@ -46,42 +40,44 @@ export interface Bm25AdapterOptions { * @default 'bm25' */ filterPrefix?: string; - - /** - * External BM25 index store. If not provided, the adapter will attempt - * to read from the build object's `pgBm25IndexStore`. - */ - bm25IndexStore?: Map; } export function createBm25Adapter( options: Bm25AdapterOptions = {} ): SearchAdapter { - const { filterPrefix = 'bm25', bm25IndexStore } = options; - - function getIndexStore(build: any): Map | undefined { - if (bm25IndexStore) return bm25IndexStore; - // Try build.pgBm25IndexStore (set by standalone Bm25SearchPlugin's build hook) - const buildStore = build.pgBm25IndexStore as Map | undefined; - if (buildStore && buildStore.size > 0) return buildStore; - // Fall back to module-level store populated by Bm25CodecPlugin's gather phase - if (moduleBm25IndexStore && moduleBm25IndexStore.size > 0) return moduleBm25IndexStore; - return undefined; - } + const { filterPrefix = 'bm25' } = options; function getBm25IndexForAttribute( codec: any, - attributeName: string, - build: any, + attributeName: string ): Bm25IndexInfo | undefined { - const store = getIndexStore(build); - if (!store) return undefined; - - const pg = codec?.extensions?.pg; - if (!pg) return undefined; + const bound = codec.attributes?.[attributeName]?.extensions?.bm25Index; + return bound as Bm25IndexInfo | undefined; + } - const key = `${pg.schemaName}.${pg.name}.${attributeName}`; - return store.get(key); + function findBoundBm25Index( + build: any, + serviceName: string, + schemaName: string, + tableName: string, + columnName: string + ): Bm25IndexInfo | undefined { + for (const codec of Object.values( + build?.input?.pgRegistry?.pgCodecs ?? {} + ) as any[]) { + for (const attribute of Object.values(codec?.attributes ?? {}) as any[]) { + const index = attribute?.extensions?.bm25Index as Bm25IndexInfo | undefined; + if ( + index?.serviceName === serviceName + && index.schemaName === schemaName + && index.tableName === tableName + && index.columnName === columnName + ) { + return index; + } + } + } + return undefined; } return { @@ -110,17 +106,45 @@ export function createBm25Adapter( codec.attributes as Record )) { if (!isTextCodec(attribute.codec)) continue; - const bm25Index = getBm25IndexForAttribute(codec, attributeName, build); + const bm25Index = getBm25IndexForAttribute(codec, attributeName); if (!bm25Index) continue; // Check for chunk-aware BM25 - const chunksInfo = getChunksInfo(codec); - const hasChunkBm25 = chunksInfo?.searchIndexes.includes('bm25'); + const chunksInfo = getChunksInfo(codec, build); + const hasChunkBm25 = chunksInfo?.searchIndexes.includes('bm25') === true; + let chunkBm25Index: Bm25IndexInfo | undefined; + if (hasChunkBm25) { + if (!bm25Index.serviceName) { + throw new Error('BM25 chunk search requires a bound PostgreSQL service identity'); + } + if (!chunksInfo?.chunksSchema) { + throw new Error( + `BM25 chunk search for '${chunksInfo?.chunksTableName}' requires a physical schema` + ); + } + chunkBm25Index = findBoundBm25Index( + build, + bm25Index.serviceName, + chunksInfo.chunksSchema, + chunksInfo.chunksTableName, + chunksInfo.contentField + ); + if (!chunkBm25Index) { + throw new Error( + 'BM25 chunk search could not bind an introspected index for ' + + `${chunksInfo.chunksSchema}.${chunksInfo.chunksTableName}.` + + chunksInfo.contentField + ); + } + } const columnData: Bm25ColumnData = { - bm25Index, - chunksInfo: hasChunkBm25 ? chunksInfo : undefined, + bm25Index }; + if (hasChunkBm25) { + columnData.chunksInfo = chunksInfo; + columnData.chunkBm25Index = chunkBm25Index; + } columns.push({ attributeName, adapterData: columnData }); } return columns; @@ -181,14 +205,25 @@ export function createBm25Adapter( const bm25Index = columnData.bm25Index; const columnExpr = sql`${alias}.${sql.identifier(column.attributeName)}`; - // Use quoteQualifiedIdentifier to produce the qualified index name - const qualifiedIndexName = `"${bm25Index.schemaName}"."${bm25Index.indexName}"`; - const bm25queryExpr = sql`to_bm25query(${sql.value(query)}, ${sql.value(qualifiedIndexName)})`; - const scoreExpr = sql`(${columnExpr} <@> ${bm25queryExpr})`; + const qualifiedIndexName = QuoteUtils.quoteQualifiedIdentifier( + bm25Index.schemaName, + bm25Index.indexName + ); + const bm25queryExpr = sql`${sql.identifier( + bm25Index.extensionSchema, + 'to_bm25query' + )}(${sql.value(query)}, ${sql.value(qualifiedIndexName)})`; + const scoreExpr = sql`(${columnExpr} OPERATOR(${sql.identifier( + bm25Index.extensionSchema + )}.<@>) ${bm25queryExpr})`; // Check for chunk-aware querying const chunksInfo = columnData.chunksInfo; if (chunksInfo && chunksInfo.searchIndexes.includes('bm25') && (includeChunks !== false)) { + const chunkBm25Index = columnData.chunkBm25Index; + if (!chunkBm25Index) { + throw new Error('BM25 chunk search is missing its bound introspected index'); + } const chunksTableRef = chunksInfo.chunksSchema ? sql`${sql.identifier(chunksInfo.chunksSchema)}.${sql.identifier(chunksInfo.chunksTableName)}` : sql`${sql.identifier(chunksInfo.chunksTableName)}`; @@ -197,12 +232,17 @@ export function createBm25Adapter( const parentId = sql`${alias}.${sql.identifier(chunksInfo.parentPkField)}`; const chunksAlias = sql.identifier('__bm25_chunks'); - // BM25 on chunks requires an index name on the chunks table. - // We construct it from the chunks table schema + a conventional index name. - // The BM25 index on chunks is named: {chunks_table}_{content_field}_bm25_idx - const chunksIndexName = `"${chunksInfo.chunksSchema || bm25Index.schemaName}"."${chunksInfo.chunksTableName}_${chunksInfo.contentField}_bm25_idx"`; - const chunkBm25queryExpr = sql`to_bm25query(${sql.value(query)}, ${sql.value(chunksIndexName)})`; - const chunkScoreExpr = sql`(${chunksAlias}.${chunkContentField} <@> ${chunkBm25queryExpr})`; + const chunksIndexName = QuoteUtils.quoteQualifiedIdentifier( + chunkBm25Index.schemaName, + chunkBm25Index.indexName + ); + const chunkBm25queryExpr = sql`${sql.identifier( + chunkBm25Index.extensionSchema, + 'to_bm25query' + )}(${sql.value(query)}, ${sql.value(chunksIndexName)})`; + const chunkScoreExpr = sql`(${chunksAlias}.${chunkContentField} OPERATOR(${sql.identifier( + chunkBm25Index.extensionSchema + )}.<@>) ${chunkBm25queryExpr})`; // Subquery: MIN(bm25_score) across chunks (lower = better for BM25) const chunkScoreSubquery = sql`( diff --git a/graphile/graphile-search/src/adapters/chunks.ts b/graphile/graphile-search/src/adapters/chunks.ts index 7c9bb47968..4f1f60db71 100644 --- a/graphile/graphile-search/src/adapters/chunks.ts +++ b/graphile/graphile-search/src/adapters/chunks.ts @@ -23,6 +23,77 @@ export interface ChunksInfo { searchIndexes: string[]; } +interface PgIdentity { + serviceName: string; + schemaName: string; + name: string; +} + +function requireNonEmptyString(value: unknown, label: string): string { + if (typeof value !== 'string' || value.length === 0 || value.includes('\0')) { + throw new Error(`[graphile-search] @hasChunks ${label} must be a non-empty PostgreSQL identifier`); + } + return value; +} + +function pgIdentity(value: any, label: string): PgIdentity { + const pg = value?.extensions?.pg; + if (!pg?.serviceName || !pg?.schemaName || !pg?.name) { + throw new Error( + `[graphile-search] ${label} is missing exact service/schema/table metadata` + ); + } + return { + serviceName: pg.serviceName, + schemaName: pg.schemaName, + name: pg.name, + }; +} + +function configuredSchemas(build: any, serviceName: string): ReadonlySet | null { + const services = build?.resolvedPreset?.pgServices; + if (!Array.isArray(services)) return null; + const matches = services.filter( + (service: any) => (service?.name ?? 'main') === serviceName + ); + if (matches.length !== 1) { + throw new Error( + `[graphile-search] @hasChunks cannot resolve exact service '${serviceName}' ` + + `(matches=${matches.length})` + ); + } + const service = matches[0]; + const schemas = service?.schemas; + if (!Array.isArray(schemas) || schemas.length === 0) { + throw new Error( + `[graphile-search] @hasChunks service '${serviceName}' has no configured schema allowlist` + ); + } + const dependencySchemas = service?.introspectionAllowedDependencySchemas; + if (dependencySchemas !== undefined && !Array.isArray(dependencySchemas)) { + throw new Error( + `[graphile-search] @hasChunks service '${serviceName}' has an invalid dependency schema allowlist` + ); + } + return new Set([...schemas, ...(dependencySchemas ?? [])]); +} + +function parseSearchIndexes(value: unknown): string[] { + let parsed = value; + if (typeof parsed === 'string') { + try { + parsed = JSON.parse(parsed); + } catch { + throw new Error('[graphile-search] @hasChunks searchIndexes must be a JSON string array'); + } + } + if (parsed == null) return []; + if (!Array.isArray(parsed)) { + throw new Error('[graphile-search] @hasChunks searchIndexes must be an array'); + } + return parsed.map((entry) => requireNonEmptyString(entry, 'searchIndexes entry')); +} + /** * Read @hasChunks smart tag from codec extensions. * @@ -38,7 +109,7 @@ export interface ChunksInfo { * "searchIndexes": ["fulltext","bm25"] // optional, defaults to [] * } */ -export function getChunksInfo(codec: any): ChunksInfo | undefined { +export function getChunksInfo(codec: any, build?: any): ChunksInfo | undefined { const tags = codec?.extensions?.tags; if (!tags) return undefined; const raw = tags.hasChunks; @@ -49,43 +120,89 @@ export function getChunksInfo(codec: any): ChunksInfo | undefined { try { parsed = JSON.parse(raw); } catch { - return undefined; + throw new Error('[graphile-search] @hasChunks must contain valid JSON'); } - } else if (typeof raw === 'object') { + } else if (typeof raw === 'object' && raw !== null && !Array.isArray(raw)) { parsed = raw; - } else if (raw === true) { - return undefined; } else { - return undefined; + throw new Error('[graphile-search] @hasChunks must be a JSON object'); } - if (!parsed.chunksTable) return undefined; + const chunksTableName = requireNonEmptyString(parsed.chunksTable, 'chunksTable'); + const parentPg = codec?.extensions?.pg; + const chunksSchema = requireNonEmptyString( + parsed.chunksSchema || parentPg?.schemaName, + 'chunksSchema' + ); + const parentFkField = requireNonEmptyString(parsed.parentFk || 'parent_id', 'parentFk'); + const parentPkField = requireNonEmptyString(parsed.parentPk || 'id', 'parentPk'); + const embeddingField = requireNonEmptyString( + parsed.embeddingField || 'embedding', + 'embeddingField' + ); + const contentField = requireNonEmptyString(parsed.contentField || 'content', 'contentField'); + const searchField = parsed.searchField == null + ? null + : requireNonEmptyString(parsed.searchField, 'searchField'); + const searchIndexes = parseSearchIndexes(parsed.searchIndexes); - const chunksSchema = parsed.chunksSchema - || codec?.extensions?.pg?.schemaName - || null; + const pgRegistry = build?.input?.pgRegistry ?? build?.pgRegistry; + if (pgRegistry) { + const parentIdentity = pgIdentity(codec, 'parent codec'); + const allowedSchemas = configuredSchemas(build, parentIdentity.serviceName); + if (allowedSchemas && !allowedSchemas.has(chunksSchema)) { + throw new Error( + `[graphile-search] @hasChunks on '${parentIdentity.schemaName}.${parentIdentity.name}' ` + + `references schema '${chunksSchema}' outside service '${parentIdentity.serviceName}'` + ); + } - // Parse searchIndexes from tag (may be array or JSON string) - let searchIndexes: string[] = []; - if (Array.isArray(parsed.searchIndexes)) { - searchIndexes = parsed.searchIndexes; - } else if (typeof parsed.searchIndexes === 'string') { - try { - const arr = JSON.parse(parsed.searchIndexes); - if (Array.isArray(arr)) searchIndexes = arr; - } catch { - // ignore + const matches = Object.values(pgRegistry.pgResources ?? {}).filter((resource: any) => { + if (resource?.parameters || !resource?.codec?.attributes) return false; + const pg = resource.codec.extensions?.pg; + return pg?.serviceName === parentIdentity.serviceName && + pg?.schemaName === chunksSchema && + pg?.name === chunksTableName; + }) as any[]; + if (matches.length !== 1) { + throw new Error( + `[graphile-search] @hasChunks on '${parentIdentity.schemaName}.${parentIdentity.name}' ` + + `must resolve exactly one '${chunksSchema}.${chunksTableName}' resource ` + + `(matches=${matches.length})` + ); + } + + const chunkIdentity = pgIdentity(matches[0].codec, 'chunks codec'); + const chunkAttributes = matches[0].codec.attributes; + for (const [field, label] of [ + [parentFkField, 'parentFk'], + [embeddingField, 'embeddingField'], + [contentField, 'contentField'], + ...(searchField ? [[searchField, 'searchField']] : []), + ] as Array<[string, string]>) { + if (!chunkAttributes[field]) { + throw new Error( + `[graphile-search] @hasChunks ${label} '${field}' does not exist on ` + + `'${chunkIdentity.schemaName}.${chunkIdentity.name}'` + ); + } + } + if (!codec?.attributes?.[parentPkField]) { + throw new Error( + `[graphile-search] @hasChunks parentPk '${parentPkField}' does not exist on ` + + `'${parentIdentity.schemaName}.${parentIdentity.name}'` + ); } } return { chunksSchema, - chunksTableName: parsed.chunksTable, - parentFkField: parsed.parentFk || 'parent_id', - parentPkField: parsed.parentPk || 'id', - embeddingField: parsed.embeddingField || 'embedding', - contentField: parsed.contentField || 'content', - searchField: parsed.searchField || null, + chunksTableName, + parentFkField, + parentPkField, + embeddingField, + contentField, + searchField, searchIndexes, }; } diff --git a/graphile/graphile-search/src/adapters/pgvector.ts b/graphile/graphile-search/src/adapters/pgvector.ts index 038509bfab..7b0f97c0ed 100644 --- a/graphile/graphile-search/src/adapters/pgvector.ts +++ b/graphile/graphile-search/src/adapters/pgvector.ts @@ -8,9 +8,16 @@ import type { SQL } from 'pg-sql2'; +import type { SearchExtensionSchemas } from '../extension-metadata'; import type { FilterApplyResult,SearchableColumn, SearchAdapter } from '../types'; import { type ChunksInfo,getChunksInfo } from './chunks'; +interface PgvectorColumnData { + serviceName: string; + extensionSchema: string; + chunksInfo?: ChunksInfo; +} + /** * Build a distance expression for the given metric. * Uses explicit SQL template literals for each operator to avoid sql.raw. @@ -20,15 +27,16 @@ function buildDistanceExpr( columnExpr: SQL, vectorExpr: SQL, metric: string, + extensionSchema: string, ): SQL { switch (metric) { case 'L2': - return sql`(${columnExpr} <-> ${vectorExpr})`; + return sql`(${columnExpr} OPERATOR(${sql.identifier(extensionSchema)}.<->) ${vectorExpr})`; case 'IP': - return sql`(${columnExpr} <#> ${vectorExpr})`; + return sql`(${columnExpr} OPERATOR(${sql.identifier(extensionSchema)}.<#>) ${vectorExpr})`; case 'COSINE': default: - return sql`(${columnExpr} <=> ${vectorExpr})`; + return sql`(${columnExpr} OPERATOR(${sql.identifier(extensionSchema)}.<=>) ${vectorExpr})`; } } @@ -82,19 +90,43 @@ export function createPgvectorAdapter( supportsTextSearch: false, // pgvector requires a vector array, not plain text — no buildTextSearchInput - detectColumns(codec: any, _build: any): SearchableColumn[] { + detectColumns(codec: any, build: any): SearchableColumn[] { if (!codec?.attributes) return []; const columns: SearchableColumn[] = []; - const chunksInfo = enableChunkQuerying ? getChunksInfo(codec) : undefined; + const chunksInfo = enableChunkQuerying ? getChunksInfo(codec, build) : undefined; for (const [attributeName, attribute] of Object.entries( codec.attributes as Record )) { if (isVectorCodec(attribute.codec)) { + const binding: SearchExtensionSchemas | undefined = + attribute?.extensions?.searchExtensionSchemas; + const codecPg = attribute.codec?.extensions?.pg; + if (!binding?.pgvectorSchema || !codecPg?.schemaName || !codecPg?.serviceName) { + const tableName = codec?.extensions?.pg?.name ?? codec?.name ?? ''; + throw new Error( + `[graphile-search] pgvector column '${tableName}.${attributeName}' is ` + + 'missing exact codec/service extension metadata' + ); + } + if ( + codecPg.schemaName !== binding.pgvectorSchema || + codecPg.serviceName !== binding.serviceName + ) { + throw new Error( + `[graphile-search] pgvector column '${attributeName}' codec identity ` + + `'${codecPg.serviceName}/${codecPg.schemaName}' does not match extension ` + + `'${binding.serviceName}/${binding.pgvectorSchema}'` + ); + } columns.push({ attributeName, - adapterData: chunksInfo ? { chunksInfo } : undefined, + adapterData: { + serviceName: binding.serviceName, + extensionSchema: binding.pgvectorSchema, + ...(chunksInfo ? { chunksInfo } : {}), + } satisfies PgvectorColumnData, }); } } @@ -195,12 +227,21 @@ export function createPgvectorAdapter( const { vector, metric, distance, includeChunks } = filterValue; if (!vector || !Array.isArray(vector) || vector.length === 0) return null; + const adapterData = column.adapterData as PgvectorColumnData | undefined; + if (!adapterData?.extensionSchema || !adapterData.serviceName) { + throw new Error( + `[graphile-search] pgvector column '${column.attributeName}' has no bound ` + + 'extension schema' + ); + } const resolvedMetric = metric || defaultMetric; const vectorString = `[${vector.join(',')}]`; - const vectorExpr = sql`${sql.value(vectorString)}::vector`; + const vectorExpr = sql`${sql.value(vectorString)}::${sql.identifier( + adapterData.extensionSchema, + 'vector' + )}`; // Check if this column has chunks info and chunk querying is requested - const adapterData = column.adapterData as { chunksInfo?: ChunksInfo } | undefined; const chunksInfo = adapterData?.chunksInfo; if (chunksInfo && (includeChunks !== false)) { @@ -217,7 +258,13 @@ export function createPgvectorAdapter( const chunksAlias = sql.identifier('__chunks'); // Subquery: SELECT MIN(distance) FROM chunks WHERE chunks.parent_fk = parent.pk - const chunkDistanceExpr = buildDistanceExpr(sql, sql`${chunksAlias}.${chunkEmbedding}`, vectorExpr, resolvedMetric); + const chunkDistanceExpr = buildDistanceExpr( + sql, + sql`${chunksAlias}.${chunkEmbedding}`, + vectorExpr, + resolvedMetric, + adapterData.extensionSchema + ); const chunkDistanceSubquery = sql`( SELECT MIN(${chunkDistanceExpr}) FROM ${chunksTableRef} AS ${chunksAlias} @@ -226,7 +273,13 @@ export function createPgvectorAdapter( // Also compute direct parent distance if the parent has an embedding const parentColumnExpr = sql`${alias}.${sql.identifier(column.attributeName)}`; - const parentDistanceExpr = buildDistanceExpr(sql, parentColumnExpr, vectorExpr, resolvedMetric); + const parentDistanceExpr = buildDistanceExpr( + sql, + parentColumnExpr, + vectorExpr, + resolvedMetric, + adapterData.extensionSchema + ); // Use LEAST of parent distance and closest chunk distance // COALESCE handles cases where parent or chunks may not have embeddings @@ -248,7 +301,13 @@ export function createPgvectorAdapter( // Standard (non-chunk) query const columnExpr = sql`${alias}.${sql.identifier(column.attributeName)}`; - const distanceExpr = buildDistanceExpr(sql, columnExpr, vectorExpr, resolvedMetric); + const distanceExpr = buildDistanceExpr( + sql, + columnExpr, + vectorExpr, + resolvedMetric, + adapterData.extensionSchema + ); let whereClause: SQL | null = null; if (distance !== undefined && distance !== null) { diff --git a/graphile/graphile-search/src/adapters/trgm.ts b/graphile/graphile-search/src/adapters/trgm.ts index 103e6c7bd4..b7eff47744 100644 --- a/graphile/graphile-search/src/adapters/trgm.ts +++ b/graphile/graphile-search/src/adapters/trgm.ts @@ -12,6 +12,7 @@ import type { SQL } from 'pg-sql2'; +import type { SearchExtensionSchemas } from '../extension-metadata'; import type { FilterApplyResult,SearchableColumn, SearchAdapter } from '../types'; import { type ChunksInfo,getChunksInfo } from './chunks'; @@ -48,6 +49,12 @@ export interface TrgmAdapterOptions { requireIntentionalSearch?: boolean; } +interface TrgmColumnData { + serviceName: string; + extensionSchema: string; + chunksInfo?: ChunksInfo; +} + export function createTrgmAdapter( options: TrgmAdapterOptions = {} ): SearchAdapter { @@ -81,7 +88,7 @@ export function createTrgmAdapter( return { value: text }; }, - detectColumns(codec: any, _build: any): SearchableColumn[] { + detectColumns(codec: any, build: any): SearchableColumn[] { if (!codec?.attributes) return []; const columns: SearchableColumn[] = []; @@ -89,12 +96,41 @@ export function createTrgmAdapter( codec.attributes as Record )) { if (isTextCodec(attribute.codec)) { + const binding: SearchExtensionSchemas | undefined = + attribute?.extensions?.searchExtensionSchemas; + if (!binding) { + const tableName = codec?.extensions?.pg?.name ?? codec?.name ?? ''; + throw new Error( + `[graphile-search] pg_trgm column '${tableName}.${attributeName}' is ` + + 'missing service-bound extension schema metadata' + ); + } + if (!binding.pgTrgmSchema) { + const explicitlyRequired = + requireIntentionalSearch === false || + codec?.extensions?.tags?.trgmSearch === true || + attribute?.extensions?.tags?.trgmSearch === true; + if (explicitlyRequired) { + const tableName = codec?.extensions?.pg?.name ?? codec?.name ?? ''; + throw new Error( + `[graphile-search] pg_trgm is required for '${tableName}.${attributeName}' ` + + `but is not installed for service '${binding.serviceName}'` + ); + } + // The adapter is enabled in the preset, but this service does not + // install pg_trgm. Leave the attribute untouched. + continue; + } // Store chunks info if available and chunks have trigram search - const chunksInfo = getChunksInfo(codec); + const chunksInfo = getChunksInfo(codec, build); const hasChunkTrgm = chunksInfo?.searchIndexes.includes('trigram'); columns.push({ attributeName, - adapterData: hasChunkTrgm ? chunksInfo : undefined, + adapterData: { + serviceName: binding.serviceName, + extensionSchema: binding.pgTrgmSchema, + ...(hasChunkTrgm ? { chunksInfo } : {}), + } satisfies TrgmColumnData, }); } } @@ -152,12 +188,20 @@ export function createTrgmAdapter( const { value, threshold, includeChunks } = filterValue; if (!value || typeof value !== 'string' || value.trim().length === 0) return null; + const columnData = column.adapterData as TrgmColumnData | undefined; + if (!columnData?.extensionSchema || !columnData.serviceName) { + throw new Error( + `[graphile-search] pg_trgm column '${column.attributeName}' has no bound ` + + 'extension schema' + ); + } const th = threshold != null ? threshold : defaultThreshold; const columnExpr = sql`${alias}.${sql.identifier(column.attributeName)}`; - const similarityExpr = sql`similarity(${columnExpr}, ${sql.value(value)})`; + const similarity = sql.identifier(columnData.extensionSchema, 'similarity'); + const similarityExpr = sql`${similarity}(${columnExpr}, ${sql.value(value)})`; // Check for chunk-aware querying - const chunksInfo = column.adapterData as ChunksInfo | undefined; + const chunksInfo = columnData.chunksInfo; if (chunksInfo && chunksInfo.searchIndexes.includes('trigram') && (includeChunks !== false)) { const chunksTableRef = chunksInfo.chunksSchema ? sql`${sql.identifier(chunksInfo.chunksSchema)}.${sql.identifier(chunksInfo.chunksTableName)}` @@ -169,10 +213,10 @@ export function createTrgmAdapter( // Subquery: MAX(similarity) across chunks (higher = better for trgm) const chunkSimilaritySubquery = sql`( - SELECT MAX(similarity(${chunksAlias}.${chunkContentField}, ${sql.value(value)})) + SELECT MAX(${similarity}(${chunksAlias}.${chunkContentField}, ${sql.value(value)})) FROM ${chunksTableRef} AS ${chunksAlias} WHERE ${chunksAlias}.${parentFk} = ${parentId} - AND similarity(${chunksAlias}.${chunkContentField}, ${sql.value(value)}) > ${sql.value(th)} + AND ${similarity}(${chunksAlias}.${chunkContentField}, ${sql.value(value)}) > ${sql.value(th)} )`; // Combined: GREATEST of parent similarity and best chunk similarity diff --git a/graphile/graphile-search/src/adapters/tsvector.ts b/graphile/graphile-search/src/adapters/tsvector.ts index 308086bb68..d7eb51dbb8 100644 --- a/graphile/graphile-search/src/adapters/tsvector.ts +++ b/graphile/graphile-search/src/adapters/tsvector.ts @@ -63,7 +63,7 @@ export function createTsvectorAdapter( return text; }, - detectColumns(codec: any, _build: any): SearchableColumn[] { + detectColumns(codec: any, build: any): SearchableColumn[] { if (!codec?.attributes) return []; const columns: SearchableColumn[] = []; @@ -72,7 +72,7 @@ export function createTsvectorAdapter( )) { if (isTsvectorCodec(attribute.codec)) { // Store chunks info if available and chunks have fulltext search - const chunksInfo = getChunksInfo(codec); + const chunksInfo = getChunksInfo(codec, build); const hasChunkFulltext = chunksInfo?.searchField && chunksInfo.searchIndexes.includes('fulltext'); columns.push({ diff --git a/graphile/graphile-search/src/codecs/bm25-codec.ts b/graphile/graphile-search/src/codecs/bm25-codec.ts index b48beceeda..ba3d018e35 100644 --- a/graphile/graphile-search/src/codecs/bm25-codec.ts +++ b/graphile/graphile-search/src/codecs/bm25-codec.ts @@ -2,25 +2,37 @@ * Bm25CodecPlugin * * Teaches PostGraphile v5 how to handle the pg_textsearch `bm25query` type - * and discovers all BM25 indexes in the database. + * and discovers BM25 indexes from Graphile's scoped introspection payload. * * This plugin: * 1. Creates a codec for bm25query via gather.hooks.pgCodecs_findPgCodec - * 2. Discovers all BM25 indexes via gather.hooks.pgIntrospection_introspection - * by querying pg_index + pg_am + pg_class + pg_attribute - * 3. Stores discovered BM25 index info in a module-level Map for use by - * the BM25 adapter during the schema build phase + * 2. Discovers requested-schema BM25 indexes without issuing side-channel SQL + * 3. Attaches index metadata to the exact codec attribute for this build */ import 'graphile-build-pg'; import type { GraphileConfig } from 'graphile-config'; +import { gatherConfig } from 'graphile-build'; import sql from 'pg-sql2'; +type Introspection = Parameters< + GraphileConfig.GatherHooks['pgIntrospection_introspection'] +>[0]['introspection']; + +interface ScopedPgService { + name?: string; + schemas?: readonly string[]; +} + /** * Represents a discovered BM25 index in the database. */ export interface Bm25IndexInfo { + /** Graphile PostgreSQL service that owns this index. */ + serviceName: string; + /** Schema containing pg_textsearch's functions and operators. */ + extensionSchema: string; /** Schema name (e.g. 'public') */ schemaName: string; /** Table name (e.g. 'documents') */ @@ -31,46 +43,105 @@ export interface Bm25IndexInfo { indexName: string; } -/** - * Module-level store for discovered BM25 indexes. - * Populated during the gather phase, read during the schema build phase. - * - * Key: "schemaName.tableName.columnName" - * Value: Bm25IndexInfo - */ -export const bm25IndexStore = new Map(); - -/** - * Whether pg_textsearch extension was detected in the database. - */ -export let bm25ExtensionDetected = false; +declare global { + namespace GraphileConfig { + interface GatherHelpers { + bm25Codec: Record; + } + } + + namespace DataplanPg { + interface PgCodecAttributeExtensions { + /** Exact physical BM25 index bound during this gather generation. */ + bm25Index?: Bm25IndexInfo; + } + } +} -/** - * The SQL query that discovers BM25 indexes in the database. - * Joins pg_index -> pg_class -> pg_am to find all indexes using the 'bm25' - * access method, then resolves the schema, table, column, and index names. - */ -const BM25_DISCOVERY_SQL = ` - SELECT - n.nspname AS schema_name, - c.relname AS table_name, - a.attname AS column_name, - i.relname AS index_name - FROM pg_index ix - JOIN pg_class i ON i.oid = ix.indexrelid - JOIN pg_am am ON am.oid = i.relam - JOIN pg_class c ON c.oid = ix.indrelid - JOIN pg_namespace n ON n.oid = c.relnamespace - JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = ANY(ix.indkey) - WHERE am.amname = 'bm25' -`; +const attributeKey = (classId: string, attributeNumber: number): string => + `${classId}:${attributeNumber}`; + +/** Collect exact requested-schema indexes from this build's own introspection. */ +export const collectBm25Indexes = ( + introspection: Introspection, + schemas: readonly string[], + serviceName: string +): Map => { + const allowedSchemas = new Set(schemas); + const discovered = new Map(); + const extension = introspection.extensions.find( + (candidate) => candidate.extname === 'pg_textsearch' + ); + const extensionNamespace = extension?.extnamespace + ? introspection.getNamespace({ id: extension.extnamespace }) + : undefined; + const indexes = [...introspection.indexes].sort((left, right) => { + const leftName = left.getIndexClass()?.relname ?? ''; + const rightName = right.getIndexClass()?.relname ?? ''; + return leftName.localeCompare(rightName); + }); + + for (const index of indexes) { + if (index.indisvalid !== true || index.indisready !== true || index.indislive !== true) { + continue; + } + const indexClass = index.getIndexClass(); + const tableClass = index.getClass(); + const namespace = tableClass + ? introspection.getNamespace({ id: tableClass.relnamespace }) + : undefined; + if ( + !indexClass + || indexClass.getAccessMethod()?.amname !== 'bm25' + || !tableClass + || !namespace + || !allowedSchemas.has(namespace.nspname) + ) { + continue; + } + + const keyCount = index.indnkeyatts ?? index.indkey.length; + for (const attribute of index.getKeys().slice(0, keyCount)) { + if (!attribute) continue; + const key = attributeKey(tableClass._id, attribute.attnum); + if (!extensionNamespace) { + throw new Error( + `BM25 index ${namespace.nspname}.${indexClass.relname} has no ` + + 'introspected pg_textsearch extension schema' + ); + } + const indexInfo: Bm25IndexInfo = { + serviceName, + extensionSchema: extensionNamespace.nspname, + schemaName: namespace.nspname, + tableName: tableClass.relname, + columnName: attribute.attname, + indexName: indexClass.relname + }; + const existing = discovered.get(key); + if (existing && existing.indexName !== indexInfo.indexName) { + throw new Error( + `Multiple BM25 indexes target ${indexInfo.schemaName}.${indexInfo.tableName}.` + + `${indexInfo.columnName}: ${existing.indexName}, ${indexInfo.indexName}` + ); + } + discovered.set(key, indexInfo); + } + } + return discovered; +}; export const Bm25CodecPlugin: GraphileConfig.Plugin = { name: 'Bm25CodecPlugin', version: '1.0.0', description: 'Registers a codec for the pg_textsearch bm25query type and discovers BM25 indexes', - gather: { + gather: gatherConfig({ + namespace: 'bm25Codec', + initialState: () => ({ + indexesByService: new Map>() + }), + helpers: {}, hooks: { /** * Register the bm25query codec when detected during type introspection. @@ -112,65 +183,34 @@ export const Bm25CodecPlugin: GraphileConfig.Plugin = { }; }, - /** - * After introspection completes, query for all BM25 indexes. - * Uses the pgService's adaptorSettings to create a direct pg.Pool - * connection and runs the BM25 discovery query. - */ - async pgIntrospection_introspection(info, event) { - const { serviceName } = event; - - // Get the pgService from the resolved preset - const pgService = info.resolvedPreset?.pgServices?.find( - (s: { name?: string }) => (s.name ?? 'main') === serviceName + pgIntrospection_introspection(info, event) { + const { introspection, serviceName } = event; + const pgServices = info.resolvedPreset.pgServices as + | readonly ScopedPgService[] + | undefined; + const pgService = pgServices?.find( + (service) => (service.name ?? 'main') === serviceName ); - if (!pgService) return; - - // Clear previous entries for this introspection run - bm25IndexStore.clear(); - - try { - const adaptorSettings = (pgService as any).adaptorSettings; - if (!adaptorSettings?.connectionString && !adaptorSettings?.pool) { - return; - } - - // Import pg dynamically for the discovery query - const { Pool } = await import('pg'); - const existingPool = adaptorSettings.pool; - const pool = existingPool ?? new Pool({ - connectionString: adaptorSettings.connectionString, - max: 1, - }); - const isOwnPool = !existingPool; - - try { - const result = await pool.query(BM25_DISCOVERY_SQL); - - if (result.rows && result.rows.length > 0) { - bm25ExtensionDetected = true; - for (const row of result.rows) { - const key = `${row.schema_name}.${row.table_name}.${row.column_name}`; - bm25IndexStore.set(key, { - schemaName: row.schema_name, - tableName: row.table_name, - columnName: row.column_name, - indexName: row.index_name, - }); - } - } - } finally { - if (isOwnPool) { - await pool.end(); - } - } - } catch { - // pg_textsearch not installed or query failed — gracefully skip - bm25ExtensionDetected = false; + if (!pgService) throw new Error(`BM25 gather could not find service '${serviceName}'`); + if (!pgService.schemas?.length) { + throw new Error(`BM25 gather requires configured schemas for service '${serviceName}'`); } + info.state.indexesByService.set( + serviceName, + collectBm25Indexes(introspection, pgService.schemas, serviceName) + ); + }, + + pgCodecs_attribute(info, event) { + const indexInfo = info.state.indexesByService + .get(event.serviceName) + ?.get(attributeKey(event.pgClass._id, event.pgAttribute.attnum)); + if (!indexInfo) return; + event.attribute.extensions ??= Object.create(null); + event.attribute.extensions.bm25Index = indexInfo; }, }, - }, + }), schema: { hooks: { diff --git a/graphile/graphile-search/src/codecs/index.ts b/graphile/graphile-search/src/codecs/index.ts index 41a283597d..3cc733771a 100644 --- a/graphile/graphile-search/src/codecs/index.ts +++ b/graphile/graphile-search/src/codecs/index.ts @@ -10,8 +10,6 @@ export type { Bm25IndexInfo } from './bm25-codec'; export { Bm25CodecPlugin, Bm25CodecPreset, - bm25ExtensionDetected, - bm25IndexStore, } from './bm25-codec'; export type { TsvectorCodecPluginOptions } from './tsvector-codec'; export { diff --git a/graphile/graphile-search/src/codecs/operator-factories.ts b/graphile/graphile-search/src/codecs/operator-factories.ts index a8e8a80b0f..f884c3f579 100644 --- a/graphile/graphile-search/src/codecs/operator-factories.ts +++ b/graphile/graphile-search/src/codecs/operator-factories.ts @@ -11,6 +11,8 @@ import type { ConnectionFilterOperatorFactory } from 'graphile-connection-filter'; import type { SQL } from 'pg-sql2'; +import { resolveBuildExtensionSchema } from '../extension-metadata'; + /** * Creates the `matches` filter operator factory for full-text search. * Declared here so it's registered via the declarative @@ -59,6 +61,10 @@ export function createMatchesOperatorFactory( export function createTrgmOperatorFactories(): ConnectionFilterOperatorFactory { return (build) => { const { sql } = build; + const extensionSchema = resolveBuildExtensionSchema(build, 'pg_trgm'); + if (!extensionSchema) return []; + const similarity = sql.identifier(extensionSchema, 'similarity'); + const wordSimilarity = sql.identifier(extensionSchema, 'word_similarity'); return [ { @@ -82,7 +88,7 @@ export function createTrgmOperatorFactories(): ConnectionFilterOperatorFactory { return null; } const th = threshold != null ? threshold : 0.3; - return sql`similarity(${sqlIdentifier}, ${sql.value(value)}) > ${sql.value(th)}`; + return sql`${similarity}(${sqlIdentifier}, ${sql.value(value)}) > ${sql.value(th)}`; }, }, }, @@ -107,7 +113,7 @@ export function createTrgmOperatorFactories(): ConnectionFilterOperatorFactory { return null; } const th = threshold != null ? threshold : 0.3; - return sql`word_similarity(${sql.value(value)}, ${sqlIdentifier}) > ${sql.value(th)}`; + return sql`${wordSimilarity}(${sql.value(value)}, ${sqlIdentifier}) > ${sql.value(th)}`; }, }, }, diff --git a/graphile/graphile-search/src/codecs/vector-codec.ts b/graphile/graphile-search/src/codecs/vector-codec.ts index e764ba1238..539f22dc87 100644 --- a/graphile/graphile-search/src/codecs/vector-codec.ts +++ b/graphile/graphile-search/src/codecs/vector-codec.ts @@ -26,8 +26,6 @@ export const VectorCodecPlugin: GraphileConfig.Plugin = { gather: { hooks: { async pgCodecs_findPgCodec(info, event) { - if (event.pgCodec) return; - const { pgType: type, serviceName } = event; if (type.typname !== 'vector') return; @@ -35,10 +33,41 @@ export const VectorCodecPlugin: GraphileConfig.Plugin = { serviceName, type.typnamespace ); - if (!typeNamespace) return; + if (!typeNamespace?.nspname) { + throw new Error( + `[graphile-search] Cannot resolve the vector type namespace for ` + + `service '${serviceName}'` + ); + } const schemaName = typeNamespace.nspname; + if (event.pgCodec) { + const existingPg = event.pgCodec.extensions?.pg; + if ( + (existingPg?.serviceName && existingPg.serviceName !== serviceName) || + (existingPg?.schemaName && existingPg.schemaName !== schemaName) + ) { + throw new Error( + `[graphile-search] Existing vector codec identity conflicts with ` + + `introspection for service '${serviceName}'` + ); + } + const existingCodec = event.pgCodec as any; + existingCodec.sqlType = sql.identifier(schemaName, 'vector'); + existingCodec.extensions = { + ...existingCodec.extensions, + oid: type._id, + pg: { + ...existingPg, + serviceName, + schemaName, + name: 'vector', + }, + }; + return; + } + event.pgCodec = { name: 'vector', sqlType: sql.identifier(schemaName, 'vector'), diff --git a/graphile/graphile-search/src/extension-metadata.ts b/graphile/graphile-search/src/extension-metadata.ts new file mode 100644 index 0000000000..1317a54f46 --- /dev/null +++ b/graphile/graphile-search/src/extension-metadata.ts @@ -0,0 +1,245 @@ +import 'graphile-build'; +import 'graphile-build-pg'; + +import type { GraphileConfig } from 'graphile-config'; +import { gatherConfig } from 'graphile-build'; + +type Introspection = Parameters< + GraphileConfig.GatherHooks['pgIntrospection_introspection'] +>[0]['introspection']; + +/** Extension namespaces discovered for one exact Graphile PostgreSQL service. */ +export interface SearchExtensionSchemas { + serviceName: string; + pgTrgmSchema: string | null; + pgvectorSchema: string | null; +} + +declare global { + namespace GraphileConfig { + interface GatherHelpers { + unifiedSearchExtensionMetadata: Record; + } + } + + namespace DataplanPg { + interface PgCodecExtensions { + /** Exact extension schemas for the service that owns this record codec. */ + searchExtensionSchemas?: SearchExtensionSchemas; + } + + interface PgCodecAttributeExtensions { + /** Exact extension schemas bound from this service's introspection generation. */ + searchExtensionSchemas?: SearchExtensionSchemas; + } + } + + namespace GraphileBuild { + interface Build { + /** Per-service extension schemas for this build only. */ + pgSearchExtensionSchemasByService?: ReadonlyMap; + } + } +} + +function extensionSchema( + introspection: Introspection, + extensionName: string, + serviceName: string +): string | null { + const matches = introspection.extensions.filter( + (extension) => extension.extname === extensionName + ); + if (matches.length > 1) { + throw new Error( + `[graphile-search] Service '${serviceName}' has ambiguous ${extensionName} ` + + `extension metadata (${matches.length} entries)` + ); + } + if (matches.length === 0) return null; + + const extension = matches[0]; + if (extension.extnamespace == null) { + throw new Error( + `[graphile-search] Service '${serviceName}' has ${extensionName} without an ` + + 'introspected extension namespace' + ); + } + const namespace = introspection.getNamespace({ id: extension.extnamespace }); + if (!namespace?.nspname) { + throw new Error( + `[graphile-search] Service '${serviceName}' cannot resolve the namespace for ` + + `${extensionName}` + ); + } + return namespace.nspname; +} + +/** Resolve extension schemas exclusively from the current service introspection. */ +export function collectSearchExtensionSchemas( + introspection: Introspection, + serviceName: string +): SearchExtensionSchemas { + return Object.freeze({ + serviceName, + pgTrgmSchema: extensionSchema(introspection, 'pg_trgm', serviceName), + pgvectorSchema: extensionSchema(introspection, 'vector', serviceName), + }); +} + +/** + * Gather configuration used by UnifiedSearchPlugin. + * + * Metadata is attached to every attribute while its service identity is still + * explicit. Adapters later retain only the exact binding for eligible columns. + */ +export const SearchExtensionMetadataGather = gatherConfig({ + namespace: 'unifiedSearchExtensionMetadata', + initialState: () => ({ + schemasByService: new Map(), + }), + helpers: {}, + hooks: { + pgIntrospection_introspection(info, event) { + const { introspection, serviceName } = event; + info.state.schemasByService.set( + serviceName, + collectSearchExtensionSchemas(introspection, serviceName) + ); + }, + + pgCodecs_PgCodec(info, event) { + // Record codecs are service-local, unlike built-in scalar codecs that may + // be shared. Keeping one carrier per exposed class also covers builds + // whose attributes are later reduced from the registry. + if (!event.pgClass) return; + const binding = info.state.schemasByService.get(event.serviceName); + if (!binding) { + throw new Error( + `[graphile-search] No extension metadata was gathered for service ` + + `'${event.serviceName}'` + ); + } + event.pgCodec.extensions ??= Object.create(null); + event.pgCodec.extensions.searchExtensionSchemas = binding; + }, + + pgCodecs_attribute(info, event) { + const binding = info.state.schemasByService.get(event.serviceName); + if (!binding) { + throw new Error( + `[graphile-search] No extension metadata was gathered for service ` + + `'${event.serviceName}'` + ); + } + event.attribute.extensions ??= Object.create(null); + event.attribute.extensions.searchExtensionSchemas = binding; + }, + }, +}); + +/** Build an immutable, consistency-checked service map from bound attributes. */ +export function extensionSchemasByService(build: any): ReadonlyMap { + const schemasByService = new Map(); + const codecs = build.input?.pgRegistry?.pgCodecs; + if (!codecs) return schemasByService; + + const addBinding = (binding: SearchExtensionSchemas): void => { + const existing = schemasByService.get(binding.serviceName); + if ( + existing && + (existing.pgTrgmSchema !== binding.pgTrgmSchema || + existing.pgvectorSchema !== binding.pgvectorSchema) + ) { + throw new Error( + `[graphile-search] Conflicting extension metadata for service ` + + `'${binding.serviceName}' in one build` + ); + } + schemasByService.set(binding.serviceName, binding); + }; + + for (const codec of Object.values(codecs) as any[]) { + const codecBinding: SearchExtensionSchemas | undefined = + codec?.extensions?.searchExtensionSchemas; + if (codecBinding) { + addBinding(codecBinding); + } + if (!codec?.attributes) continue; + for (const attribute of Object.values(codec.attributes) as any[]) { + const binding: SearchExtensionSchemas | undefined = + attribute?.extensions?.searchExtensionSchemas; + if (!binding) continue; + addBinding(binding); + } + } + return schemasByService; +} + +/** + * Resolve one extension namespace for a build-wide operator factory. + * Shared GraphQL filter types cannot safely route to different schemas, so a + * multi-service build with partial or differing namespaces is rejected. A + * build where every service lacks the optional extension resolves to null. + */ +export function resolveBuildExtensionSchema( + build: any, + extension: 'pg_trgm' | 'vector' +): string | null { + const schemasByService: ReadonlyMap = + build.pgSearchExtensionSchemasByService ?? extensionSchemasByService(build); + if (schemasByService.size === 0) { + const codecs = build.input?.pgRegistry?.pgCodecs; + const hasServiceBoundCodec = codecs && Object.values(codecs).some( + (codec: any) => + codec?.attributes != null || codec?.extensions?.pg?.serviceName != null + ); + if (!hasServiceBoundCodec) { + // An empty exposed schema has no service-local codec on which gather can + // carry optional extension metadata. No operator is registered, so no + // unqualified SQL path is created. + return null; + } + throw new Error( + `[graphile-search] ${extension} requires service-bound extension metadata` + ); + } + + const field = extension === 'pg_trgm' ? 'pgTrgmSchema' : 'pgvectorSchema'; + const schemas = new Set(); + let missingCount = 0; + for (const binding of schemasByService.values()) { + const schemaName = binding[field]; + if (!schemaName) { + missingCount++; + continue; + } + schemas.add(schemaName); + } + if (schemas.size === 0) return null; + if (missingCount > 0) { + throw new Error( + `[graphile-search] ${extension} is present for only part of this multi-service build` + ); + } + if (schemas.size !== 1) { + throw new Error( + `[graphile-search] ${extension} has ambiguous schemas across this build: ` + + [...schemas].sort().join(', ') + ); + } + return schemas.values().next().value!; +} + +export function requireBuildExtensionSchema( + build: any, + extension: 'pg_trgm' | 'vector' +): string { + const schemaName = resolveBuildExtensionSchema(build, extension); + if (!schemaName) { + throw new Error( + `[graphile-search] ${extension} is required by this feature but is not installed` + ); + } + return schemaName; +} diff --git a/graphile/graphile-search/src/index.ts b/graphile/graphile-search/src/index.ts index b28afee219..de01a0e8f1 100644 --- a/graphile/graphile-search/src/index.ts +++ b/graphile/graphile-search/src/index.ts @@ -33,6 +33,14 @@ // Core plugin export { createUnifiedSearchPlugin } from './plugin'; +// Exact per-service extension namespace binding +export type { SearchExtensionSchemas } from './extension-metadata'; +export { + collectSearchExtensionSchemas, + requireBuildExtensionSchema, + resolveBuildExtensionSchema, +} from './extension-metadata'; + // Preset export type { UnifiedSearchPresetOptions } from './preset'; export { UnifiedSearchPreset } from './preset'; @@ -68,7 +76,6 @@ export type { export { Bm25CodecPlugin, Bm25CodecPreset, - bm25IndexStore, createTsvectorCodecPlugin, TsvectorCodecPlugin, TsvectorCodecPreset, diff --git a/graphile/graphile-search/src/plugin.ts b/graphile/graphile-search/src/plugin.ts index c67f796be3..6673a47927 100644 --- a/graphile/graphile-search/src/plugin.ts +++ b/graphile/graphile-search/src/plugin.ts @@ -26,6 +26,10 @@ import { TYPES } from '@dataplan/pg'; import type { GraphileConfig } from 'graphile-config'; import { getQueryBuilder } from 'graphile-plugin-utils'; +import { + extensionSchemasByService, + SearchExtensionMetadataGather, +} from './extension-metadata'; import type { SearchableColumn, SearchAdapter, UnifiedSearchOptions } from './types'; // ─── TypeScript Namespace Augmentations ────────────────────────────────────── @@ -171,8 +175,14 @@ export function createUnifiedSearchPlugin( ): GraphileConfig.Plugin { const { adapters, enableSearchScore = true, enableUnifiedSearch = true, rrfK = 60 } = options; - // Per-codec cache of discovered columns, keyed by codec name - const codecCache = new Map(); + // Adapter discovery may depend on both the codec and the surrounding build + // registry. Object-identity keys prevent a long-lived preset from reusing + // tenant A's result for tenant B, even if a caller reuses a codec object. + // Weak keys also allow disposed builds and codecs to be collected. + const buildCodecCache = new WeakMap< + object, + WeakMap + >(); // Bridge between orderBy enum apply and filter apply. // The orderBy enum runs on the PgSelectStep while the filter runs on @@ -195,9 +205,14 @@ export function createUnifiedSearchPlugin( * count as intentional search. */ function getAdapterColumns(codec: PgCodecWithAttributes, build: any): AdapterColumnCache[] { - const cacheKey = codec.name; - if (codecCache.has(cacheKey)) { - return codecCache.get(cacheKey)!; + let codecCache = buildCodecCache.get(build); + if (!codecCache) { + codecCache = new WeakMap(); + buildCodecCache.set(build, codecCache); + } + const cached = codecCache.get(codec); + if (cached) { + return cached; } const primaryAdapters = adapters.filter((a) => !a.isSupplementary); @@ -238,7 +253,7 @@ export function createUnifiedSearchPlugin( } } - codecCache.set(cacheKey, results); + codecCache.set(codec, results); return results; } @@ -260,6 +275,8 @@ export function createUnifiedSearchPlugin( 'VectorCodecPlugin', ], + gather: SearchExtensionMetadataGather, + // ─── Custom Inflection Methods ───────────────────────────────────── inflection: { add: { @@ -328,6 +345,17 @@ export function createUnifiedSearchPlugin( }, hooks: { + /** Publish only this build's consistency-checked extension bindings. */ + build(build) { + return build.extend( + build, + { + pgSearchExtensionSchemasByService: extensionSchemasByService(build), + }, + 'UnifiedSearchPlugin adding per-service extension schemas' + ); + }, + /** * Register all adapter-specific GraphQL types during init. */ @@ -375,7 +403,7 @@ export function createUnifiedSearchPlugin( inflection, sql, graphql: { GraphQLFloat }, - grafast: { lambda }, + grafast: { constant, lambda }, } = build; const { scope: { isPgClassType, pgCodec: rawPgCodec }, @@ -424,7 +452,7 @@ export function createUnifiedSearchPlugin( const $select = typeof $row.getClassStep === 'function' ? $row.getClassStep() : null; - if (!$select) return build.grafast.constant(null); + if (!$select) return constant(null); if (typeof $select.setInliningForbidden === 'function') { $select.setInliningForbidden(); @@ -522,7 +550,7 @@ export function createUnifiedSearchPlugin( const $select = typeof $row.getClassStep === 'function' ? $row.getClassStep() : null; - if (!$select) return build.grafast.constant(null); + if (!$select) return constant(null); if (typeof $select.setInliningForbidden === 'function') { $select.setInliningForbidden(); diff --git a/graphile/graphile-settings/README.md b/graphile/graphile-settings/README.md index 84a4156058..edd3aa6f8d 100644 --- a/graphile/graphile-settings/README.md +++ b/graphile/graphile-settings/README.md @@ -40,6 +40,10 @@ const preset = { makePgService({ connectionString: 'postgres://user:pass@localhost/mydb', schemas: ['app_public'], + // Optional density optimization: retire the exact connection that ran + // catalog introspection instead of returning its enlarged backend to + // the request pool. The default is 'reuse'. + introspectionClientReleaseMode: 'destroy', }), ], }; @@ -52,6 +56,12 @@ serv.addTo(app, httpServer); httpServer.listen(5000); ``` +`introspectionClientReleaseMode: 'destroy'` applies only to the connection +checked out by Graphile for the catalog gather query. It fails closed when the +configured adaptor cannot prove that it owns and can destroy that exact +connection; caller-owned clients are never destroyed. Runtime requests still +use a dedicated tenant/API service and its normal pool. + ## Features The `ConstructivePreset` combines multiple plugins and configurations to provide a clean, opinionated GraphQL API. Below is a detailed breakdown of each feature. diff --git a/graphile/graphile-settings/__tests__/PublicKeySignature.test.ts b/graphile/graphile-settings/__tests__/PublicKeySignature.test.ts index cc833a3333..2214bc92e7 100644 --- a/graphile/graphile-settings/__tests__/PublicKeySignature.test.ts +++ b/graphile/graphile-settings/__tests__/PublicKeySignature.test.ts @@ -1,8 +1,12 @@ import type { PublicKeyChallengeConfig } from '../src/plugins/PublicKeySignature'; -import { PublicKeySignature } from '../src/plugins/PublicKeySignature'; +import { + PublicKeySignature, + withAnonymousPublicKeyClient, +} from '../src/plugins/PublicKeySignature'; const defaultConfig: PublicKeyChallengeConfig = { schema: 'app_private', + anonymousRole: 'api_anonymous', crypto_network: 'btc', sign_up_with_key: 'sign_up_with_key', sign_in_request_challenge: 'sign_in_request_challenge', @@ -29,6 +33,7 @@ describe('PublicKeySignature plugin factory', () => { it('accepts custom config values', () => { const customConfig: PublicKeyChallengeConfig = { schema: 'custom_schema', + anonymousRole: 'custom_anonymous', crypto_network: 'eth', sign_up_with_key: 'custom_signup', sign_in_request_challenge: 'custom_challenge', @@ -59,6 +64,11 @@ describe('PublicKeySignature config validation', () => { expect(() => PublicKeySignature({ ...defaultConfig, schema: 'DROP TABLE' })).toThrow(/invalid schema/); }); + it('throws on an invalid anonymous role', () => { + expect(() => PublicKeySignature({ ...defaultConfig, anonymousRole: 'tenant-a; SET ROLE owner' })) + .toThrow(/invalid anonymousRole/); + }); + it('throws on invalid function name', () => { expect(() => PublicKeySignature({ ...defaultConfig, sign_up_with_key: 'evil"; DROP' })).toThrow( /invalid sign_up_with_key/, @@ -87,3 +97,52 @@ describe('PublicKeySignature config validation', () => { expect(() => PublicKeySignature(defaultConfig)).not.toThrow(); }); }); + +describe('PublicKeySignature request context', () => { + it('preserves the complete request GUC contract while forcing the anonymous role', async () => { + const pgSettings = { + role: 'authenticated', + 'jwt.claims.api_id': 'api-a', + 'jwt.claims.database_id': 'database-a', + 'jwt.claims.user_id': '', + 'jwt.claims.session_id': '', + 'request.id': 'request-a', + transaction_read_only: 'off', + search_path: 'pg_catalog, "tenant_a"', + row_security: 'on', + }; + const pgClient = { + query: jest.fn(async (): Promise<{ rows: Record[] }> => ({ rows: [] })), + }; + const callback = jest.fn(async (client) => client); + const withPgClient = jest.fn(async (settings, fn) => { + expect(settings).toEqual({ ...pgSettings, role: 'api_anonymous' }); + expect(settings).not.toBe(pgSettings); + return fn(pgClient); + }); + + await expect(withAnonymousPublicKeyClient( + withPgClient, + pgSettings, + 'api_anonymous', + callback, + )).resolves.toBe(pgClient); + + expect(withPgClient).toHaveBeenCalledTimes(1); + expect(callback).toHaveBeenCalledWith(pgClient); + expect(pgSettings.role).toBe('authenticated'); + }); + + it.each([ + ['missing withPgClient', undefined, { role: 'anonymous' }, 'PG_CLIENT_CONTEXT_UNAVAILABLE'], + ['missing pgSettings', jest.fn(), undefined, 'PG_SETTINGS_UNAVAILABLE'], + ['null pgSettings', jest.fn(), null, 'PG_SETTINGS_UNAVAILABLE'], + ])('fails closed for %s', async (_label, withPgClient, pgSettings, expected) => { + await expect(withAnonymousPublicKeyClient( + withPgClient as any, + pgSettings, + 'api_anonymous', + async (): Promise => null, + )).rejects.toThrow(expected); + }); +}); diff --git a/graphile/graphile-settings/__tests__/build-state-retirement.test.ts b/graphile/graphile-settings/__tests__/build-state-retirement.test.ts new file mode 100644 index 0000000000..5a2273107b --- /dev/null +++ b/graphile/graphile-settings/__tests__/build-state-retirement.test.ts @@ -0,0 +1,203 @@ +import { + BUILD_STATE_RELEASED_ERROR_CODE, + defaultPreset, + getBuilder, + isBuildStateReleased, + releaseBuildState +} from 'graphile-build'; +import { + GraphQLInterfaceType, + GraphQLObjectType, + GraphQLSchema, + GraphQLString, + graphqlSync, + printSchema +} from 'graphql'; + +type CapturedBuild = GraphileBuild.BuildBase & Partial; + +interface Capture { + builds: CapturedBuild[]; + owned: Map[]; + disposalOrder: string[]; +} + +const makeCapturePlugin = ( + capture: Capture, + configure?: (build: CapturedBuild) => void +): GraphileConfig.Plugin => ({ + name: 'BuildStateRetirementContractPlugin', + version: '0.0.0', + schema: { + hooks: { + build(build) { + const owned = new Map([['construction-only', 'retained']]); + capture.builds.push(build); + capture.owned.push(owned); + build.registerBuildStateDisposer(() => { + capture.disposalOrder.push('owned'); + owned.clear(); + }); + configure?.(build); + return build; + } + } + } +}); + +const makeCapture = (): Capture => ({ + builds: [], + owned: [], + disposalOrder: [] +}); + +const buildTestSchema = ( + capture: Capture, + releaseBuildStateAfterValidation: boolean, + extraPlugins: GraphileConfig.Plugin[] = [] +): ReturnType['buildSchema']> => getBuilder({ + extends: [defaultPreset], + plugins: [makeCapturePlugin(capture), ...extraPlugins], + schema: { releaseBuildStateAfterValidation } +}).buildSchema(Object.create(null)); + +const expectReleasedError = (callback: () => unknown): void => { + try { + callback(); + throw new Error('Expected released build state to reject late access'); + } catch (error) { + expect((error as Error & { code?: string }).code).toBe( + BUILD_STATE_RELEASED_ERROR_CODE + ); + } +}; + +describe('Graphile build-state retirement contract', () => { + it('is default-off and retains plugin-owned construction state', () => { + const capture = makeCapture(); + const schema = buildTestSchema(capture, false); + + expect(capture.disposalOrder).toEqual([]); + expect(capture.owned[0].size).toBe(1); + expect(isBuildStateReleased(capture.builds[0] as GraphileBuild.Build)).toBe(false); + expect(capture.builds[0].getAllTypes()).toBeDefined(); + expect(graphqlSync({ schema, source: '{ __typename }' })).toEqual({ + data: { __typename: 'Query' } + }); + }); + + it('preserves schema bytes and execution while failing closed on late access', () => { + const baselineCapture = makeCapture(); + const candidateCapture = makeCapture(); + const baseline = buildTestSchema(baselineCapture, false); + const candidate = buildTestSchema(candidateCapture, true); + const releasedBuild = candidateCapture.builds[0]; + + expect(printSchema(candidate)).toBe(printSchema(baseline)); + expect(candidateCapture.disposalOrder).toEqual(['owned']); + expect(candidateCapture.owned[0].size).toBe(0); + expect(isBuildStateReleased(releasedBuild as GraphileBuild.Build)).toBe(true); + expect(graphqlSync({ schema: candidate, source: '{ __typename }' })).toEqual({ + data: { __typename: 'Query' } + }); + expectReleasedError(() => releasedBuild.input); + expectReleasedError(() => releasedBuild.scopeByType); + expectReleasedError(() => releasedBuild.getAllTypes()); + expectReleasedError(() => releasedBuild.behavior!.getDefaultBehaviorFor('string')); + expect(releaseBuildState(releasedBuild as GraphileBuild.Build)).toBe(false); + }); + + it('runs every disposer in LIFO order and aggregates disposal failures', () => { + const capture = makeCapture(); + const plugin = makeCapturePlugin(capture, (build) => { + build.registerBuildStateDisposer(() => capture.disposalOrder.push('first')); + build.registerBuildStateDisposer(() => { + capture.disposalOrder.push('second-error'); + throw new Error('synthetic disposal failure'); + }); + build.registerBuildStateDisposer(() => capture.disposalOrder.push('third')); + }); + const builder = getBuilder({ + extends: [defaultPreset], + plugins: [plugin], + schema: { releaseBuildStateAfterValidation: true } + }); + + let thrown: unknown; + try { + builder.buildSchema(Object.create(null)); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(AggregateError); + expect((thrown as AggregateError).errors).toHaveLength(1); + expect((thrown as AggregateError).errors[0]).toEqual( + new Error('synthetic disposal failure') + ); + expect(capture.disposalOrder).toEqual([ + 'third', + 'second-error', + 'first', + 'owned' + ]); + expect(capture.owned[0].size).toBe(0); + expect(isBuildStateReleased(capture.builds[0] as GraphileBuild.Build)).toBe(true); + }); + + it('keeps diagnostic state when schema validation fails', () => { + const capture = makeCapture(); + const invalidSchemaPlugin: GraphileConfig.Plugin = { + name: 'InvalidSchemaForRetirementContractPlugin', + version: '0.0.0', + schema: { + hooks: { + finalize(schema) { + const requiredInterface = new GraphQLInterfaceType({ + name: 'RetirementRequiredInterface', + fields: { required: { type: GraphQLString } } + }); + const brokenObject = new GraphQLObjectType({ + name: 'RetirementBrokenObject', + interfaces: [requiredInterface], + fields: { other: { type: GraphQLString } } + }); + const config = schema.toConfig(); + return new GraphQLSchema({ + ...config, + types: [...config.types, requiredInterface, brokenObject] + }); + } + } + } + }; + + expect(() => buildTestSchema(capture, true, [invalidSchemaPlugin])) + .toThrow(/validation failure/i); + expect(capture.disposalOrder).toEqual([]); + expect(capture.owned[0].size).toBe(1); + expect(isBuildStateReleased(capture.builds[0] as GraphileBuild.Build)).toBe(false); + expect(capture.builds[0].getAllTypes()).toBeDefined(); + }); + + it('retires each rebuild without clearing the builder hook registry', () => { + const capture = makeCapture(); + const builder = getBuilder({ + extends: [defaultPreset], + plugins: [makeCapturePlugin(capture)], + schema: { releaseBuildStateAfterValidation: true } + }); + + const first = builder.buildSchema(Object.create(null)); + const second = builder.buildSchema(Object.create(null)); + + expect(capture.builds).toHaveLength(2); + expect(capture.owned).toHaveLength(2); + expect(capture.owned.every((owned) => owned.size === 0)).toBe(true); + expect(capture.builds.every((build) => + isBuildStateReleased(build as GraphileBuild.Build) + )).toBe(true); + expect(capture.disposalOrder).toEqual(['owned', 'owned']); + expect(printSchema(second)).toBe(printSchema(first)); + }); +}); diff --git a/graphile/graphile-settings/__tests__/constructive-preset-bucket-wiring.test.ts b/graphile/graphile-settings/__tests__/constructive-preset-bucket-wiring.test.ts index e045370cd4..16b2c23d41 100644 --- a/graphile/graphile-settings/__tests__/constructive-preset-bucket-wiring.test.ts +++ b/graphile/graphile-settings/__tests__/constructive-preset-bucket-wiring.test.ts @@ -8,7 +8,7 @@ * cross-tenant bucket-name collisions. */ -const captured: { bucketProvisionerOptions?: any } = {}; +const captured: { bucketProvisionerOptions?: any; presignedOptions?: any } = {}; // Capture the options handed to BucketProvisionerPreset without pulling in the // real plugin (and its S3 machinery). @@ -19,6 +19,17 @@ jest.mock('graphile-bucket-provisioner-plugin', () => ({ }), })); +jest.mock('graphile-presigned-url-plugin', () => ({ + PresignedUrlPreset: jest.fn((options: any) => { + captured.presignedOptions = options; + return { plugins: [] as any[] }; + }), + snapshotPreloadedStorageModules: jest.fn((modules: any[] | undefined) => { + if (modules === undefined) return undefined; + return Object.freeze(modules.map((module) => Object.freeze({ ...module }))); + }), +})); + // The preset reads CDN config eagerly when building the presigned/provisioner // plugin options; provide a prefix so name minting is deterministic. const PREFIX = 'test-bucket'; @@ -42,6 +53,7 @@ const DATABASE_ID = '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9'; describe('ConstructivePreset bucket-provisioner wiring', () => { beforeEach(() => { captured.bucketProvisionerOptions = undefined; + captured.presignedOptions = undefined; }); it('passes a resolveBucketName into BucketProvisionerPreset when presigned uploads are enabled', () => { @@ -50,6 +62,10 @@ describe('ConstructivePreset bucket-provisioner wiring', () => { const options = captured.bucketProvisionerOptions; expect(options).toBeDefined(); expect(typeof options.resolveBucketName).toBe('function'); + expect(options.preloadedStorageModules).toEqual([]); + expect(options.preloadedStorageModules).toBe( + captured.presignedOptions.preloadedStorageModules, + ); }); it('the wired resolver mints the tenant-aware {prefix}-{bucketKey}-{databaseId} name', () => { @@ -66,6 +82,17 @@ describe('ConstructivePreset bucket-provisioner wiring', () => { expect(captured.bucketProvisionerOptions.autoProvision).toBe(false); }); + it('passes the same immutable build snapshot to both storage plugins', () => { + const modules = [{ id: 'storage-module-a' }] as any[]; + createConstructivePreset({ preloadedStorageModules: modules }); + + const provisionerSnapshot = captured.bucketProvisionerOptions.preloadedStorageModules; + expect(provisionerSnapshot).toBe(captured.presignedOptions.preloadedStorageModules); + expect(provisionerSnapshot).not.toBe(modules); + expect(Object.isFrozen(provisionerSnapshot)).toBe(true); + expect(Object.isFrozen(provisionerSnapshot[0])).toBe(true); + }); + it('does not wire the provisioner preset when presigned uploads are disabled', () => { createConstructivePreset({ enablePresignedUploads: false }); expect(captured.bucketProvisionerOptions).toBeUndefined(); diff --git a/graphile/graphile-settings/__tests__/grafast-cache-limits.test.ts b/graphile/graphile-settings/__tests__/grafast-cache-limits.test.ts new file mode 100644 index 0000000000..c5d1197bfb --- /dev/null +++ b/graphile/graphile-settings/__tests__/grafast-cache-limits.test.ts @@ -0,0 +1,55 @@ +import type { GraphQLSchemaConfig } from 'graphql'; + +import { + applyGrafastCacheLimits, + createGrafastCacheLimitsPlugin, + createGrafastCacheLimitsPreset, + normalizeGrafastCacheLimits +} from '../src/grafast-cache-limits'; + +const schemaConfig = (): GraphQLSchemaConfig => ({ + extensions: { + existing: true, + grafast: { queryCacheMaxLength: 99 } + } +}); + +describe('Grafast schema-local cache limits', () => { + it('preserves unrelated schema and Grafast extensions', () => { + const result = applyGrafastCacheLimits(schemaConfig(), { + operationsCacheMaxLength: 16, + operationOperationPlansCacheMaxLength: 8 + }); + + expect(result.extensions).toMatchObject({ + existing: true, + grafast: { + queryCacheMaxLength: 99, + operationsCacheMaxLength: 16, + operationOperationPlansCacheMaxLength: 8 + } + }); + }); + + it.each([0, 1, -1, 1.5, Number.MAX_SAFE_INTEGER + 1])( + 'rejects an unsafe bound %s', + (value) => { + expect(() => normalizeGrafastCacheLimits({ + operationsCacheMaxLength: value + })).toThrow('must be a safe integer of at least 2'); + } + ); + + it('installs a reusable GraphQLSchema hook', () => { + const plugin = createGrafastCacheLimitsPlugin({ operationsCacheMaxLength: 8 }); + const hook = plugin.schema?.hooks?.GraphQLSchema; + expect(typeof hook).toBe('function'); + + const result = (hook as Function)(schemaConfig(), {}, {}); + expect(result.extensions?.grafast?.operationsCacheMaxLength).toBe(8); + }); + + it('is inert when no limits are configured', () => { + expect(createGrafastCacheLimitsPreset({})).toEqual({}); + }); +}); diff --git a/graphile/graphile-settings/__tests__/introspection-capabilities.test.ts b/graphile/graphile-settings/__tests__/introspection-capabilities.test.ts new file mode 100644 index 0000000000..7d2d195d5b --- /dev/null +++ b/graphile/graphile-settings/__tests__/introspection-capabilities.test.ts @@ -0,0 +1,26 @@ +import { resolveConstructiveIntrospectionCapabilityExtensions } from + '../src/presets/constructive-preset'; + +describe('Constructive scoped-introspection extension capabilities', () => { + it('derives only the exact extensions required by enabled plugins', () => { + expect(resolveConstructiveIntrospectionCapabilityExtensions()).toEqual([ + 'pg_trgm', + 'vector', + 'pg_textsearch', + 'postgis', + 'ltree' + ]); + expect(resolveConstructiveIntrospectionCapabilityExtensions({ + enableSearch: false, + enableLlm: false, + enablePostgis: false, + enableLtree: false + })).toEqual([]); + expect(resolveConstructiveIntrospectionCapabilityExtensions({ + enableSearch: false, + enableLlm: true, + enablePostgis: false, + enableLtree: false + })).toEqual(['vector']); + }); +}); diff --git a/graphile/graphile-settings/__tests__/introspection-client-release.test.ts b/graphile/graphile-settings/__tests__/introspection-client-release.test.ts new file mode 100644 index 0000000000..37bc725e81 --- /dev/null +++ b/graphile/graphile-settings/__tests__/introspection-client-release.test.ts @@ -0,0 +1,253 @@ +import { withPgClientFromPgService } from '@dataplan/pg'; +import { makeSchema } from 'graphile-build'; +import { Pool } from 'pg'; + +import { assertIntrospectionClientReleaseCapabilities } from '../src/introspection-client-release'; +import { MinimalPreset } from '../src/plugins'; + +type TestWithPgClient = { + ( + pgSettings: Record | null, + callback: (client: { rawClient: unknown }) => T | Promise, + options?: { clientReleaseMode?: 'reuse' | 'destroy' } + ): Promise; + supportedClientReleaseModes?: readonly ('reuse' | 'destroy')[]; +}; + +const { + makePgAdaptorWithPgClient, + makeWithPgClientViaPgClientAlreadyInTransaction +} = require('@dataplan/pg/adaptors/pg') as { + makePgAdaptorWithPgClient(pool: unknown): TestWithPgClient; + makeWithPgClientViaPgClientAlreadyInTransaction(client: unknown): TestWithPgClient; +}; + +const { makePgService: makePostGraphilePgService } = require( + 'postgraphile/adaptors/pg' +) as { + makePgService(options: Record): Record; +}; + +const makeRawClient = () => ({ + query: jest.fn().mockResolvedValue({ rows: [], rowCount: 0 }), + release: jest.fn(), + addListener: jest.fn(), + removeListener: jest.fn() +}); + +const makeNodePostgresPool = (rawClient: ReturnType) => { + const pool = Object.create(Pool.prototype) as Pool & { + connect: jest.Mock; + }; + pool.connect = jest.fn().mockResolvedValue(rawClient); + return pool; +}; + +describe('published dependency capability gate', () => { + const supported = { + dataplanPg: 'dataplan-pg-exact-client-destroy-v1', + graphileBuildPg: 'graphile-build-pg-exact-client-destroy-v1' + }; + + it('allows reuse without patched downstream dependencies', () => { + expect(() => assertIntrospectionClientReleaseCapabilities('reuse', { + dataplanPg: undefined, + graphileBuildPg: undefined + })).not.toThrow(); + }); + + it('requires both exact destroy protocol capabilities', () => { + expect(() => assertIntrospectionClientReleaseCapabilities( + 'destroy', + supported + )).not.toThrow(); + expect(() => assertIntrospectionClientReleaseCapabilities('destroy', { + ...supported, + dataplanPg: undefined + })).toThrow( + 'GRAPHILE_INTROSPECTION_CLIENT_DESTROY_UNSUPPORTED:@dataplan/pg' + ); + expect(() => assertIntrospectionClientReleaseCapabilities('destroy', { + ...supported, + graphileBuildPg: undefined + })).toThrow( + 'GRAPHILE_INTROSPECTION_CLIENT_DESTROY_UNSUPPORTED:graphile-build-pg' + ); + }); +}); + +describe('per-use PostgreSQL client release mode', () => { + it('destroys the exact successful checkout once', async () => { + const rawClient = makeRawClient(); + const pool = makeNodePostgresPool(rawClient); + const withPgClient = makePgAdaptorWithPgClient(pool as never); + + expect(withPgClient.supportedClientReleaseModes).toEqual(['reuse', 'destroy']); + await expect(withPgClient( + null, + async (client) => client.rawClient, + { clientReleaseMode: 'destroy' } + )).resolves.toBe(rawClient); + + expect(pool.connect).toHaveBeenCalledTimes(1); + expect(rawClient.release.mock.calls).toEqual([[true]]); + }); + + it('destroys the exact failed checkout once', async () => { + const marker = new Error('callback failed'); + const rawClient = makeRawClient(); + const pool = makeNodePostgresPool(rawClient); + const withPgClient = makePgAdaptorWithPgClient(pool as never); + + await expect(withPgClient( + null, + async () => { + throw marker; + }, + { clientReleaseMode: 'destroy' } + )).rejects.toBe(marker); + + expect(pool.connect).toHaveBeenCalledTimes(1); + expect(rawClient.release.mock.calls).toEqual([[true]]); + }); + + it('destroys the exact checkout when first-use setup throws synchronously', async () => { + const marker = new Error('client setup failed'); + const rawClient = makeRawClient(); + rawClient.query.mockImplementationOnce(() => { + throw marker; + }); + const pool = makeNodePostgresPool(rawClient); + const callback = jest.fn(); + const withPgClient = makePgAdaptorWithPgClient(pool as never); + + await expect(withPgClient( + null, + callback, + { clientReleaseMode: 'destroy' } + )).rejects.toBe(marker); + + expect(callback).not.toHaveBeenCalled(); + expect(pool.connect).toHaveBeenCalledTimes(1); + expect(rawClient.release.mock.calls).toEqual([[true]]); + }); + + it('rejects destroy mode for a structurally compatible custom pool', async () => { + const rawClient = makeRawClient(); + const pool = { connect: jest.fn().mockResolvedValue(rawClient) }; + const callback = jest.fn(); + const withPgClient = makePgAdaptorWithPgClient(pool as never); + + expect(withPgClient.supportedClientReleaseModes).toEqual(['reuse']); + await expect(withPgClient( + null, + callback, + { clientReleaseMode: 'destroy' } + )).rejects.toThrow( + 'Exact PostgreSQL client destruction requires a node-postgres Pool' + ); + + expect(pool.connect).not.toHaveBeenCalled(); + expect(callback).not.toHaveBeenCalled(); + expect(rawClient.release).not.toHaveBeenCalled(); + }); + + it('reuses the exact checkout once by default', async () => { + const rawClient = makeRawClient(); + const pool = makeNodePostgresPool(rawClient); + const withPgClient = makePgAdaptorWithPgClient(pool as never); + + await withPgClient(null, async (client) => client.rawClient); + + expect(pool.connect).toHaveBeenCalledTimes(1); + expect(rawClient.release.mock.calls).toEqual([[]]); + }); + + it('fails closed before invoking an adaptor that does not advertise destruction', async () => { + const callback = jest.fn(); + const originalWithPgClient = Object.assign( + jest.fn(async (): Promise => undefined), + { release: jest.fn() } + ); + const service = { + name: 'unsupported-adaptor', + adaptor: { + createWithPgClient: jest.fn().mockReturnValue(originalWithPgClient) + } + }; + + await expect(withPgClientFromPgService( + service as never, + null, + callback, + { clientReleaseMode: 'destroy' } + )).rejects.toThrow( + "PostgreSQL service 'unsupported-adaptor' does not support exact client destruction" + ); + + expect(callback).not.toHaveBeenCalled(); + expect(originalWithPgClient).not.toHaveBeenCalled(); + }); + + it('refuses to destroy a caller-owned client', async () => { + const callback = jest.fn(); + const rawClient = makeRawClient(); + const withPgClient = makeWithPgClientViaPgClientAlreadyInTransaction( + rawClient as never + ); + + expect(withPgClient.supportedClientReleaseModes).toEqual(['reuse']); + await expect(withPgClient( + null, + callback, + { clientReleaseMode: 'destroy' } + )).rejects.toThrow('Cannot destroy a caller-owned PostgreSQL client'); + + expect(callback).not.toHaveBeenCalled(); + expect(rawClient.release).not.toHaveBeenCalled(); + }); +}); + +describe('introspection client release forwarding', () => { + it.each([ + ['destroy', 'destroy', [[true]]], + ['default reuse', undefined, [[]]] + ] as const)('uses %s for the introspection checkout', async ( + _label, + clientReleaseMode, + expectedReleaseCalls + ) => { + const marker = new Error('captured introspection query'); + let sawIntrospection = false; + const rawClient = makeRawClient(); + rawClient.query.mockImplementation(async (query: string | { text: string }) => { + if ( + typeof query === 'object' + && query.text.includes('requested_schema_names') + ) { + sawIntrospection = true; + throw marker; + } + return { rows: [], rowCount: 0 }; + }); + const pool = makeNodePostgresPool(rawClient); + + await expect(makeSchema({ + extends: [MinimalPreset], + pgServices: [Object.assign(makePostGraphilePgService({ + pool: pool as never, + pubsub: false, + schemas: ['tenant_a'] + }), { + introspectionMode: 'scoped-required', + ...(clientReleaseMode === undefined + ? {} + : { introspectionClientReleaseMode: clientReleaseMode }) + }) as never] + })).rejects.toBe(marker); + + expect(sawIntrospection).toBe(true); + expect(pool.connect).toHaveBeenCalledTimes(1); + expect(rawClient.release.mock.calls).toEqual(expectedReleaseCalls); + }); +}); diff --git a/graphile/graphile-settings/__tests__/make-pg-service.test.ts b/graphile/graphile-settings/__tests__/make-pg-service.test.ts new file mode 100644 index 0000000000..f5890731b5 --- /dev/null +++ b/graphile/graphile-settings/__tests__/make-pg-service.test.ts @@ -0,0 +1,42 @@ +import { + normalizeIntrospectionDependencySchemas, + resolveIntrospectionSettings +} from '../src/introspection-settings'; + +describe('resolveIntrospectionSettings', () => { + it('disables JIT only for scoped introspection', () => { + const scoped = resolveIntrospectionSettings( + 'scoped-required', + { statement_timeout: '5000', jit: 'on', work_mem: '16MB' } + ); + const stock = resolveIntrospectionSettings('stock', { statement_timeout: '5000' }); + const defaultBound = resolveIntrospectionSettings('stock', undefined); + + expect(scoped).toEqual({ + statement_timeout: '5000', + jit: 'off', + work_mem: '512kB' + }); + expect(stock).toEqual({ statement_timeout: '5000' }); + expect(defaultBound).toEqual({ statement_timeout: '120s' }); + }); +}); + +describe('normalizeIntrospectionDependencySchemas', () => { + it('preserves lookup order while trimming and deduplicating', () => { + expect(normalizeIntrospectionDependencySchemas([ + ' extensions ', + 'shared_api', + 'extensions' + ])).toEqual(['extensions', 'shared_api']); + }); + + it.each(['pg_catalog', 'pg_toast', 'information_schema'])( + 'rejects system dependency schema %s', + (schema) => { + expect(() => normalizeIntrospectionDependencySchemas([schema])).toThrow( + 'must not be a system schema' + ); + } + ); +}); diff --git a/graphile/graphile-settings/__tests__/scoped-bm25-cross-database.integration.test.ts b/graphile/graphile-settings/__tests__/scoped-bm25-cross-database.integration.test.ts new file mode 100644 index 0000000000..fb57c9bb61 --- /dev/null +++ b/graphile/graphile-settings/__tests__/scoped-bm25-cross-database.integration.test.ts @@ -0,0 +1,362 @@ +import { randomUUID } from 'crypto'; + +import { QuoteUtils } from '@pgsql/quotes'; +import { execute } from 'grafast'; +import { makeSchema } from 'graphile-build'; +import { withPgClientFromPgService } from 'graphile-build-pg'; +import type { GraphileConfig } from 'graphile-config'; +import { parse, type ExecutionResult } from 'graphql'; +import { Pool } from 'pg'; + +import { resolveIntrospectionSettings } from '../src/introspection-settings'; +import { ConstructivePreset } from '../src/presets/constructive-preset'; + +const { makePgService: makePostGraphilePgService } = require('postgraphile/adaptors/pg') as { + makePgService(options: Record): any; +}; + +jest.setTimeout(120000); + +const API_SCHEMA = 'tenant_api'; + +function makePgService( + options: Record & { + introspectionMode: 'stock' | 'scoped-required'; + } +) { + const pgSettingsForIntrospection = resolveIntrospectionSettings( + options.introspectionMode, + options.pgSettingsForIntrospection as Record | undefined + ); + return Object.assign(makePostGraphilePgService({ + ...options, + pgSettingsForIntrospection + }), { + introspectionMode: options.introspectionMode, + introspectionAllowedDependencySchemas: options.introspectionAllowedDependencySchemas, + introspectionCapabilityExtensions: options.introspectionCapabilityExtensions + }); +} + +interface FixtureSpec { + database: string; + role: string; + unrelatedAclRole: string; + password: string; + index: string; + token: string; +} + +interface DocumentsResult { + documents: { + nodes: Array<{ + body: string; + bodyBm25Score: number | null; + }>; + }; +} + +interface BuiltApi { + pool: Pool; + query: (term: string) => Promise>; +} + +async function readPlannerSettings(pool: Pool) { + return (await pool.query<{ + pid: number; + jit: string; + work_mem: string; + }>(` + SELECT + pg_backend_pid() AS pid, + current_setting('jit') AS jit, + current_setting('work_mem') AS work_mem + `)).rows[0]; +} + +function fixtureSpec(label: 'a' | 'b', token: string): FixtureSpec { + const suffix = randomUUID().replace(/-/g, '').slice(0, 10); + return { + database: `gsi_bm25_${label}_${suffix}`, + role: `gsi_bm25_${label}_role_${suffix}`, + unrelatedAclRole: `gsi_bm25_${label}_auditor_${suffix}`, + password: `gsi_${suffix}_${label}_local_only`, + index: `${label}_opaque_lexicon_${suffix}`, + token + }; +} + +async function provisionFixture( + adminPool: Pool, + spec: FixtureSpec, + createdDatabases: string[], + createdRoles: string[] +): Promise { + const database = QuoteUtils.quoteIdentifier(spec.database); + const role = QuoteUtils.quoteIdentifier(spec.role); + const unrelatedAclRole = QuoteUtils.quoteIdentifier(spec.unrelatedAclRole); + + await adminPool.query( + `CREATE ROLE ${role} + LOGIN PASSWORD ${QuoteUtils.escape(spec.password)} + NOSUPERUSER NOBYPASSRLS NOCREATEROLE NOCREATEDB NOREPLICATION NOINHERIT` + ); + createdRoles.push(spec.role); + + await adminPool.query( + `CREATE ROLE ${unrelatedAclRole} + NOLOGIN NOSUPERUSER NOBYPASSRLS NOCREATEROLE NOCREATEDB NOREPLICATION NOINHERIT` + ); + createdRoles.push(spec.unrelatedAclRole); + + await adminPool.query(`CREATE DATABASE ${database}`); + createdDatabases.push(spec.database); + await adminPool.query(`REVOKE CONNECT ON DATABASE ${database} FROM PUBLIC`); + await adminPool.query(`GRANT CONNECT ON DATABASE ${database} TO ${role}`); + + const ownerPool = new Pool({ database: spec.database }); + try { + await ownerPool.query('CREATE EXTENSION pg_textsearch'); + await ownerPool.query(`CREATE SCHEMA ${API_SCHEMA}`); + await ownerPool.query(`REVOKE ALL ON SCHEMA ${API_SCHEMA} FROM PUBLIC`); + await ownerPool.query(` + CREATE TABLE ${API_SCHEMA}.documents ( + id integer PRIMARY KEY, + body text NOT NULL + ) + `); + await ownerPool.query(` + INSERT INTO ${API_SCHEMA}.documents (id, body) + VALUES (1, $1) + `, [spec.token]); + await ownerPool.query(` + CREATE INDEX ${QuoteUtils.quoteIdentifier(spec.index)} + ON ${API_SCHEMA}.documents USING bm25(body) + WITH (text_config = 'english') + `); + await ownerPool.query(`GRANT USAGE ON SCHEMA ${API_SCHEMA} TO ${role}`); + await ownerPool.query(`GRANT SELECT ON ${API_SCHEMA}.documents TO ${role}`); + // PgRBACPlugin resolves every retained ACL entry, even when the grantee is + // unrelated to the runtime login. Scoped introspection must retain this role. + await ownerPool.query( + `GRANT SELECT ON ${API_SCHEMA}.documents TO ${unrelatedAclRole}` + ); + } finally { + await ownerPool.end(); + } +} + +async function assertLeastPrivilege(pool: Pool, spec: FixtureSpec): Promise { + const result = await pool.query<{ + rolname: string; + rolsuper: boolean; + rolbypassrls: boolean; + rolcreaterole: boolean; + rolcreatedb: boolean; + rolreplication: boolean; + rolinherit: boolean; + owns_database: boolean; + owns_schema: boolean; + can_create_in_schema: boolean; + }>(` + SELECT + role.rolname, + role.rolsuper, + role.rolbypassrls, + role.rolcreaterole, + role.rolcreatedb, + role.rolreplication, + role.rolinherit, + database.datdba = role.oid AS owns_database, + namespace.nspowner = role.oid AS owns_schema, + pg_catalog.has_schema_privilege( + role.oid, + namespace.oid, + 'CREATE' + ) AS can_create_in_schema + FROM pg_catalog.pg_roles AS role + JOIN pg_catalog.pg_database AS database + ON database.datname = current_database() + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.nspname = $1 + WHERE role.rolname = current_user + `, [API_SCHEMA]); + + expect(result.rows).toEqual([{ + rolname: spec.role, + rolsuper: false, + rolbypassrls: false, + rolcreaterole: false, + rolcreatedb: false, + rolreplication: false, + rolinherit: false, + owns_database: false, + owns_schema: false, + can_create_in_schema: false + }]); +} + +async function expectUnapprovedExtensionSchemaRejected(spec: FixtureSpec): Promise { + const pool = new Pool({ + database: spec.database, + user: spec.role, + password: spec.password, + max: 1 + }); + try { + const settingsBefore = await readPlannerSettings(pool); + const pgService = makePgService({ + pool, + schemas: [API_SCHEMA], + introspectionMode: 'scoped-required', + // Naming a capability controls retained extension metadata; it must not + // silently approve the extension's physical schema. + introspectionCapabilityExtensions: ['pg_textsearch'] + }); + await expect(makeSchema({ + extends: [ConstructivePreset], + pgServices: [pgService] + })).rejects.toThrow('crossed into unapproved dependency schema(s): public'); + expect(await readPlannerSettings(pool)).toEqual(settingsBefore); + } finally { + await pool.end(); + } +} + +async function buildApi(spec: FixtureSpec): Promise { + const pool = new Pool({ + database: spec.database, + user: spec.role, + password: spec.password, + max: 1 + }); + + try { + await assertLeastPrivilege(pool, spec); + const settingsBefore = await readPlannerSettings(pool); + + const pgService = makePgService({ + pool, + schemas: [API_SCHEMA], + introspectionMode: 'scoped-required', + introspectionAllowedDependencySchemas: ['public'], + introspectionCapabilityExtensions: ['pg_textsearch'] + }); + const preset: GraphileConfig.Preset = { + // Deliberately reuse the module-level preset, including its plugin objects. + extends: [ConstructivePreset], + pgServices: [pgService] + }; + const { schema, resolvedPreset } = await makeSchema(preset); + expect(await readPlannerSettings(pool)).toEqual(settingsBefore); + const withPgClientKey = pgService.withPgClientKey ?? 'withPgClient'; + + return { + pool, + async query(term: string) { + const result = await execute({ + schema, + document: parse(` + query ScopedBm25Isolation($term: String!) { + documents(where: { bm25Body: { query: $term } }) { + nodes { + body + bodyBm25Score + } + } + } + `), + variableValues: { term }, + contextValue: { + pgSettings: {}, + [withPgClientKey]: withPgClientFromPgService.bind(null, pgService) + }, + resolvedPreset + }); + if (Symbol.asyncIterator in result) { + throw new Error('BM25 isolation query unexpectedly returned a stream'); + } + return result as unknown as ExecutionResult; + } + }; + } catch (error) { + await pool.end(); + throw error; + } +} + +async function expectDatabaseConnectionDenied( + source: FixtureSpec, + target: FixtureSpec +): Promise { + const crossTenantPool = new Pool({ + database: target.database, + user: source.role, + password: source.password, + max: 1 + }); + try { + await expect(crossTenantPool.query('SELECT 1')).rejects.toMatchObject({ + code: '42501' + }); + } finally { + await crossTenantPool.end(); + } +} + +describe('scoped BM25 cross-database isolation', () => { + it('keeps same-named tenant APIs bound to their own database and BM25 index', async () => { + const adminPool = new Pool({ database: 'postgres', max: 1 }); + const createdDatabases: string[] = []; + const createdRoles: string[] = []; + const builtApis: BuiltApi[] = []; + const tenantA = fixtureSpec('a', 'amber lunar archive tenant-a-only'); + const tenantB = fixtureSpec('b', 'violet orchard ledger tenant-b-only'); + + try { + await provisionFixture(adminPool, tenantA, createdDatabases, createdRoles); + await provisionFixture(adminPool, tenantB, createdDatabases, createdRoles); + + await expectDatabaseConnectionDenied(tenantA, tenantB); + await expectDatabaseConnectionDenied(tenantB, tenantA); + await expectUnapprovedExtensionSchemaRejected(tenantA); + + // Build sequentially so the second build exercises reuse of the exact same + // long-lived ConstructivePreset and its UnifiedSearchPlugin instance. + const apiA = await buildApi(tenantA); + builtApis.push(apiA); + const apiB = await buildApi(tenantB); + builtApis.push(apiB); + + const [resultA, resultB] = await Promise.all([ + apiA.query('lunar'), + apiB.query('orchard') + ]); + + expect(resultA.errors).toBeUndefined(); + expect(resultB.errors).toBeUndefined(); + expect(resultA.data?.documents.nodes).toEqual([{ + body: tenantA.token, + bodyBm25Score: expect.any(Number) + }]); + expect(resultB.data?.documents.nodes).toEqual([{ + body: tenantB.token, + bodyBm25Score: expect.any(Number) + }]); + } finally { + await Promise.allSettled(builtApis.map(({ pool }) => pool.end())); + + for (const databaseName of [...createdDatabases].reverse()) { + await adminPool.query( + `DROP DATABASE IF EXISTS ${QuoteUtils.quoteIdentifier(databaseName)} WITH (FORCE)` + ); + } + for (const roleName of [...createdRoles].reverse()) { + await adminPool.query( + `DROP ROLE IF EXISTS ${QuoteUtils.quoteIdentifier(roleName)}` + ); + } + await adminPool.end(); + } + }); +}); diff --git a/graphile/graphile-settings/__tests__/scoped-introspection-cache-lifecycle.test.ts b/graphile/graphile-settings/__tests__/scoped-introspection-cache-lifecycle.test.ts new file mode 100644 index 0000000000..23834db60a --- /dev/null +++ b/graphile/graphile-settings/__tests__/scoped-introspection-cache-lifecycle.test.ts @@ -0,0 +1,211 @@ +import { watchGather } from 'graphile-build'; +import { PgIntrospectionPlugin } from 'graphile-build-pg'; +import type { GraphileConfig } from 'graphile-config'; + +const SCHEMA = 'tenant_a'; + +const introspectionText = JSON.stringify({ + database: { datdba: '10', datacl: null }, + namespaces: [{ + _id: '2200', + oid: '2200', + nspname: SCHEMA, + nspowner: '10', + nspacl: null + }], + classes: [], + attributes: [], + constraints: [], + procs: [], + roles: [{ + _id: '10', + oid: '10', + rolname: 'postgres', + rolsuper: true, + rolinherit: true, + rolcreaterole: true, + rolcreatedb: true, + rolcanlogin: true, + rolreplication: true, + rolconnlimit: -1, + rolpassword: null, + rolvaliduntil: null, + rolbypassrls: true, + rolconfig: null + }], + auth_members: [], + types: [], + enums: [], + extensions: [], + indexes: [], + languages: [], + ranges: [], + depends: [], + descriptions: [], + inherits: [], + am: [], + catalog_by_oid: { + 2615: 'pg_namespace', + 1259: 'pg_class', + 1255: 'pg_proc', + 1247: 'pg_type', + 2606: 'pg_constraint', + 3079: 'pg_extension' + }, + current_user: 'postgres', + server_version_num: 180004 +}); +const missingSchemaIntrospectionText = JSON.stringify({ + ...JSON.parse(introspectionText), + namespaces: [] +}); + +interface GatherResult { + input: Record | null; + error?: Error; +} + +function makeResultQueue() { + const queued: GatherResult[] = []; + const waiters: Array<(result: GatherResult) => void> = []; + + return { + push(result: GatherResult) { + const waiter = waiters.shift(); + if (waiter) waiter(result); + else queued.push(result); + }, + next(): Promise { + const result = queued.shift(); + if (result) return Promise.resolve(result); + return new Promise((resolve) => waiters.push(resolve)); + } + }; +} + +describe('scoped introspection raw-text lifecycle', () => { + it('releases raw text, re-queries fresh data, and fails closed on regather errors', async () => { + let cache: { introspectionResultsPromise: Promise | null } | null = null; + let triggerRegather: (() => void) | null = null; + let queryError: Error | null = null; + let nextIntrospectionText = introspectionText; + const seenNamespaceNames: string[] = []; + const query = jest.fn(async () => { + if (queryError) { + const error = queryError; + queryError = null; + throw error; + } + return { rows: [{ introspection: nextIntrospectionText }] }; + }); + const withPgClient = Object.assign( + async ( + _settings: Record | null, + callback: (client: { query: typeof query }) => unknown + ) => callback({ query }), + { release: jest.fn() } + ); + const adaptor = { + createWithPgClient: jest.fn(async () => withPgClient) + }; + + const originalGather = PgIntrospectionPlugin.gather!; + const capturingIntrospectionPlugin = { + ...PgIntrospectionPlugin, + gather: { + ...originalGather, + initialCache(info: never) { + cache = originalGather.initialCache!(info) as typeof cache; + return cache; + }, + // A deterministic test trigger drives the same persistent gather cache + // without needing a live LISTEN/NOTIFY subscriber. + watch: undefined + } + } as unknown as GraphileConfig.Plugin; + const observerPlugin = { + name: 'ScopedIntrospectionCacheObserverPlugin', + gather: { + namespace: 'scopedIntrospectionCacheObserver', + async main(output: Record, info: any) { + const [result] = await info.helpers.pgIntrospection.getIntrospection(); + const namespace = result.introspection.namespaces[0]; + seenNamespaceNames.push(namespace.nspname); + output.namespaceName = namespace.nspname; + // Graphile plugins may mutate their gather-local parsed graph. A later + // gather must never observe this mutation. + namespace.nspname = 'mutated_by_plugin'; + }, + watch(_info: never, callback: () => void) { + triggerRegather = callback; + return (): void => undefined; + } + } + } as unknown as GraphileConfig.Plugin; + const pgService = { + name: 'main', + schemas: [SCHEMA], + introspectionMode: 'scoped-required', + introspectionAllowedDependencySchemas: [] as readonly string[], + adaptor, + adaptorSettings: {}, + withPgClientKey: 'withPgClient', + pgSettingsKey: 'pgSettings' + }; + const results = makeResultQueue(); + + const stopWatching = await watchGather({ + plugins: [capturingIntrospectionPlugin, observerPlugin], + pgServices: [pgService as never] + }, undefined, (input, error) => { + results.push({ + input: input as unknown as Record | null, + error: error as Error | undefined + }); + }); + + try { + const first = await results.next(); + expect(first.error).toBeUndefined(); + expect(first.input).toMatchObject({ namespaceName: SCHEMA }); + expect(query).toHaveBeenCalledTimes(1); + expect(cache!.introspectionResultsPromise).toBeNull(); + + triggerRegather!(); + const second = await results.next(); + expect(second.error).toBeUndefined(); + expect(second.input).toMatchObject({ namespaceName: SCHEMA }); + expect(query).toHaveBeenCalledTimes(2); + expect(seenNamespaceNames).toEqual([SCHEMA, SCHEMA]); + expect(cache!.introspectionResultsPromise).toBeNull(); + + nextIntrospectionText = missingSchemaIntrospectionText; + triggerRegather!(); + const invalid = await results.next(); + expect(invalid.input).toBeNull(); + expect(invalid.error?.message).toContain( + `did not find required schema(s): ${SCHEMA}` + ); + expect(query).toHaveBeenCalledTimes(3); + expect(cache!.introspectionResultsPromise).toBeNull(); + + nextIntrospectionText = introspectionText; + triggerRegather!(); + const recovered = await results.next(); + expect(recovered.error).toBeUndefined(); + expect(recovered.input).toMatchObject({ namespaceName: SCHEMA }); + expect(query).toHaveBeenCalledTimes(4); + + const marker = new Error('scoped introspection re-query failed'); + queryError = marker; + triggerRegather!(); + const failed = await results.next(); + expect(failed.input).toBeNull(); + expect(failed.error).toBe(marker); + expect(query).toHaveBeenCalledTimes(5); + expect(cache!.introspectionResultsPromise).toBeNull(); + } finally { + stopWatching(); + } + }); +}); diff --git a/graphile/graphile-settings/__tests__/scoped-introspection-capability-closure.integration.test.ts b/graphile/graphile-settings/__tests__/scoped-introspection-capability-closure.integration.test.ts new file mode 100644 index 0000000000..6d0789621a --- /dev/null +++ b/graphile/graphile-settings/__tests__/scoped-introspection-capability-closure.integration.test.ts @@ -0,0 +1,729 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; + +import { execute } from 'grafast'; +import { makeSchema } from 'graphile-build'; +import { withPgClientFromPgService } from 'graphile-build-pg'; +import type { GraphileConfig } from 'graphile-config'; +import { + type ExecutionResult, + type GraphQLSchema, + isObjectType, + lexicographicSortSchema, + parse, + printSchema +} from 'graphql'; +import type { Pool } from 'pg'; + +import { resolveIntrospectionSettings } from '../src/introspection-settings'; +import { createConstructivePreset } from '../src/presets/constructive-preset'; + +jest.setTimeout(180000); + +const API_SCHEMA = 'closure_api'; +const PLAIN_API_SCHEMA = 'closure_plain_api'; +const EXTENSION_SCHEMA = 'closure_ext'; +const FIXTURE = join( + __dirname, + '../sql/scoped-introspection-capability-closure.sql' +); +const REQUIRED_EXTENSIONS = [ + 'ltree', + 'pg_textsearch', + 'pg_trgm', + 'postgis', + 'vector' +] as const; +const REQUIRED_INDEXES = [ + 'closure_records_body_bm25_idx', + 'closure_records_embedding_idx', + 'closure_records_geom_idx', + 'closure_records_name_trgm_idx', + 'closure_records_path_idx', + 'closure_records_pkey', + 'closure_records_score_window_idx', + 'closure_records_search_document_idx' +] as const; +const REQUIRED_RECORD_FIELDS = [ + 'id', + 'name', + 'body', + 'status', + 'statuses', + 'label', + 'labels', + 'payload', + 'payloads', + 'scoreWindow', + 'scoreWindowArray', + 'scoreWindows', + 'searchDocument', + 'path', + 'paths', + 'embedding', + 'embeddings', + 'geom', + 'geoms' +] as const; +const MAX_SERVICE_CATALOG_ROWS = 128; + +const CORE_DOCUMENT = parse(` + query ClosureCoreTypes { + closureRecords { + nodes { + id + name + status + statuses + label + labels + payload { label weight } + payloads { label weight } + scoreWindow { + start { value inclusive } + end { value inclusive } + } + scoreWindowArray { + start { value inclusive } + end { value inclusive } + } + scoreWindows { + start { value inclusive } + end { value inclusive } + } + path + paths + embedding + embeddings + geom { geojson } + } + } + closureFunctionProbe { + nodes { id name } + } + closureMultirangeProbe( + expected: [ + { + start: { value: "40", inclusive: true } + end: { value: "50", inclusive: false } + } + { + start: { value: "60", inclusive: true } + end: { value: "70", inclusive: true } + } + ] + ) { + start { value inclusive } + end { value inclusive } + } + closureTimestampMultirangeProbe( + expected: [ + { + start: { value: "2026-08-01T00:00:00Z", inclusive: true } + end: { value: "2026-08-02T00:00:00Z", inclusive: false } + } + { + start: { value: "2026-08-03T00:00:00Z", inclusive: true } + end: { value: "2026-08-04T00:00:00Z", inclusive: true } + } + ] + ) { + start { value inclusive } + end { value inclusive } + } + } +`); + +const EXTENSION_DOCUMENT = parse(` + query ClosureExtensionCapabilities($bbox: GeoJSON!) { + tsvector: closureRecords( + where: { tsvSearchDocument: "tenant" } + ) { + nodes { id searchDocumentTsvRank } + } + trigram: closureRecords( + where: { trgmName: { value: "Acme", threshold: 0.1 } } + ) { + nodes { id nameTrgmSimilarity } + } + bm25: closureRecords( + where: { bm25Body: { query: "tenant" } } + ) { + nodes { id bodyBm25Score } + } + vector: closureRecords( + where: { + vectorEmbedding: { + nearby: { embedding: [1, 0, 0], distance: 0.1 } + } + } + ) { + nodes { id embeddingVectorDistance } + } + postgis: closureRecords( + where: { geom: { intersects: $bbox } } + ) { + nodes { id } + } + ltree: closureRecords( + where: { path: { within: "/customers" } } + ) { + nodes { id path } + } + } +`); + +const PLAIN_EXTENSION_DOCUMENT = parse(` + query InstalledExtensionWithoutObjectDependency { + plainRecords( + where: { trgmBody: { value: "tenant", threshold: 0.1 } } + ) { + nodes { id body bodyTrgmSimilarity } + } + } +`); + +const BBOX = { + type: 'Polygon', + coordinates: [[ + [-74.1, 40.6], + [-73.9, 40.6], + [-73.9, 40.9], + [-74.1, 40.9], + [-74.1, 40.6] + ]], + crs: { + type: 'name', + properties: { name: 'EPSG:4326' } + } +}; + +type PgTestClientLike = { + config: Record; + query: (text: string, values?: unknown[]) => Promise<{ + rows: T[]; + }>; + beforeEach: () => Promise; + afterEach: () => Promise; +}; + +type ConnectionResult = { + pg: PgTestClientLike; + manager: { getPool: (config: Record) => Pool }; + teardown: () => Promise; +}; + +type RawIntrospection = { + namespaces: Array<{ _id: string; nspname: string }>; + classes: Array<{ _id: string; relname: string }>; + types: Array<{ + _id: string; + typname: string; + typnamespace: string; + typtype: string; + typelem: string; + typarray: string; + }>; + extensions: Array<{ extname: string }>; + languages: Array<{ lanname: string }>; + am: Array<{ amname: string }>; + indexes: Array<{ indexrelid: string }>; +}; + +type BuiltSchema = Awaited> & { + pgService: ReturnType; + schemaName: string; +}; + +const { makePgService: makePostGraphilePgService } = require( + 'postgraphile/adaptors/pg' +) as { + makePgService: ( + options: Record + ) => GraphileConfig.PgServiceConfiguration; +}; + +function makePgService(options: { + pool: Pool; + schemas: readonly string[]; + introspectionMode: 'stock' | 'scoped-required'; + introspectionScopedCatalogTypes?: 'dependency-closure'; + introspectionAllowedDependencySchemas?: readonly string[]; + introspectionCapabilityExtensions?: readonly string[]; +}) { + const pgSettingsForIntrospection = resolveIntrospectionSettings( + options.introspectionMode, + undefined + ); + return Object.assign(makePostGraphilePgService({ + pool: options.pool, + schemas: options.schemas, + pgSettingsForIntrospection + }), { + introspectionMode: options.introspectionMode, + introspectionScopedCatalogTypes: options.introspectionScopedCatalogTypes, + introspectionAllowedDependencySchemas: + options.introspectionAllowedDependencySchemas ?? [], + ...(options.introspectionCapabilityExtensions === undefined + ? {} + : { + introspectionCapabilityExtensions: + options.introspectionCapabilityExtensions + }) + }); +} + +const graphileTestDirectory = dirname(require.resolve('graphile-test')); +const { getConnections } = require(require.resolve('pgsql-test', { + paths: [graphileTestDirectory] +})) as { + getConnections: ( + options: Record, + seeders: never[] + ) => Promise; +}; + +const graphileBuildPgDirectory = dirname(require.resolve('graphile-build-pg')); +const { + makeIntrospectionQuery, + makeSchemaScopedIntrospectionQuery, + parseIntrospectionResults +} = require(require.resolve('pg-introspection', { + paths: [graphileBuildPgDirectory] +})) as { + makeIntrospectionQuery: () => string; + makeSchemaScopedIntrospectionQuery: ( + schemas: readonly string[], + options: { + catalogTypes: 'dependency-closure'; + capabilityExtensions?: readonly string[]; + } + ) => { text: string; values: [string[], string[]] }; + parseIntrospectionResults: (value: string) => RawIntrospection; +}; + +function unsupportedExtensions(missing: readonly string[]): Error & { + code: string; +} { + const code = 'GRAPHILE_CLOSURE_UNSUPPORTED_EXTENSIONS'; + return Object.assign( + new Error(`${code}: ${sorted(missing).join(', ')}`), + { code } + ); +} + +function unsupportedExtensionSchema(extension: string): Error & { + code: string; + extension: string; +} { + const code = 'GRAPHILE_CLOSURE_UNSUPPORTED_EXTENSION_SCHEMA'; + return Object.assign(new Error(`${code}: ${extension}`), { + code, + extension + }); +} + +function extensionNamedBy(error: unknown): string | undefined { + const candidate = error as { message?: unknown; detail?: unknown }; + const diagnostic = [candidate?.message, candidate?.detail] + .filter((value): value is string => typeof value === 'string') + .join(' ') + .toLowerCase(); + return REQUIRED_EXTENSIONS.find((extension) => + diagnostic.includes(extension.toLowerCase()) + ); +} + +function sorted(values: Iterable): string[] { + return [...values].sort((left, right) => left.localeCompare(right)); +} + +function captureCatalog(introspection: RawIntrospection) { + const namespaces = new Map( + introspection.namespaces.map(({ _id, nspname }) => [String(_id), nspname]) + ); + const classes = new Map( + introspection.classes.map(({ _id, relname }) => [String(_id), relname]) + ); + const capturedTypeNames = new Set([ + `${API_SCHEMA}.closure_status`, + `${API_SCHEMA}._closure_status`, + `${API_SCHEMA}.closure_label`, + `${API_SCHEMA}._closure_label`, + `${API_SCHEMA}.closure_payload`, + `${API_SCHEMA}._closure_payload`, + `${API_SCHEMA}.score_window`, + `${API_SCHEMA}._score_window`, + `${API_SCHEMA}.score_window_multirange`, + `${API_SCHEMA}._score_window_multirange`, + `${EXTENSION_SCHEMA}.ltree`, + `${EXTENSION_SCHEMA}._ltree`, + `${EXTENSION_SCHEMA}.vector`, + `${EXTENSION_SCHEMA}._vector`, + `${EXTENSION_SCHEMA}.geometry`, + `${EXTENSION_SCHEMA}._geometry`, + 'pg_catalog.tsvector', + 'pg_catalog._tsvector' + ]); + + return { + namespaces: sorted( + introspection.namespaces + .map(({ nspname }) => nspname) + .filter((name) => + [API_SCHEMA, EXTENSION_SCHEMA, 'pg_catalog'].includes(name) + ) + ), + extensions: sorted( + introspection.extensions + .map(({ extname }) => extname) + .filter((name) => (REQUIRED_EXTENSIONS as readonly string[]).includes(name)) + ), + indexes: sorted( + introspection.indexes + .map(({ indexrelid }) => classes.get(String(indexrelid))) + .filter((name): name is string => + name !== undefined && (REQUIRED_INDEXES as readonly string[]).includes(name) + ) + ), + types: introspection.types + .map((type) => ({ + name: `${namespaces.get(String(type.typnamespace))}.${type.typname}`, + typtype: type.typtype, + typelem: String(type.typelem), + typarray: String(type.typarray) + })) + .filter(({ name }) => capturedTypeNames.has(name)) + .sort((left, right) => left.name.localeCompare(right.name)) + }; +} + +function recordFields(schema: GraphQLSchema): string[] { + const type = schema.getType('ClosureRecord'); + if (!type || !isObjectType(type)) return []; + return sorted(Object.keys(type.getFields())); +} + +function qualification(schema: GraphQLSchema): { + code?: string; + missingFields: string[]; + productionQualified: boolean; +} { + const fields = new Set(recordFields(schema)); + const queryFields = new Set(Object.keys(schema.getQueryType()?.getFields() ?? {})); + const missingFields = [ + ...REQUIRED_RECORD_FIELDS + .filter((field) => !fields.has(field)) + .map((field) => `ClosureRecord.${field}`), + ...[ + 'closureRecords', + 'closureFunctionProbe', + 'closureMultirangeProbe', + 'closureTimestampMultirangeProbe' + ] + .filter((field) => !queryFields.has(field)) + .map((field) => `Query.${field}`) + ]; + + return missingFields.length === 0 + ? { productionQualified: true, missingFields: [] } + : { + code: 'GRAPHILE_CLOSURE_CAPABILITY_MISMATCH', + missingFields, + productionQualified: false + }; +} + +async function build( + pool: Pool, + mode: 'stock' | 'scoped-required', + schemaName = API_SCHEMA +) { + const pgService = makePgService({ + pool, + schemas: [schemaName], + introspectionMode: mode, + ...(mode === 'scoped-required' + ? { + introspectionScopedCatalogTypes: 'dependency-closure' as const, + introspectionAllowedDependencySchemas: [EXTENSION_SCHEMA], + introspectionCapabilityExtensions: REQUIRED_EXTENSIONS + } + : {}) + }); + const built = await makeSchema({ + extends: [createConstructivePreset({ preloadedStorageModules: [] })], + pgServices: [pgService] + }); + return { ...built, pgService, schemaName }; +} + +async function runDocument( + built: BuiltSchema, + document: ReturnType, + variableValues: Record = {} +): Promise>> { + const withPgClientKey = built.pgService.withPgClientKey ?? 'withPgClient'; + const result = await execute({ + schema: built.schema, + document, + variableValues, + contextValue: { + // Shared extension schemas are deliberately absent. Plugin SQL must use + // the introspected physical schema instead of relying on search_path. + pgSettings: { search_path: `pg_catalog,${built.schemaName}` }, + [withPgClientKey]: withPgClientFromPgService.bind( + null, + built.pgService + ) + }, + resolvedPreset: built.resolvedPreset + }) as ExecutionResult>; + if (Symbol.asyncIterator in result) { + throw new Error('capability closure canary unexpectedly returned a stream'); + } + return result; +} + +describe('schema-scoped dependency-closure capability matrix', () => { + let pg: PgTestClientLike; + let pool: Pool; + let teardown: () => Promise; + let stock: BuiltSchema; + let scoped: BuiltSchema; + let plainStock: BuiltSchema; + let plainScoped: BuiltSchema; + let stockIntrospection: RawIntrospection; + let scopedIntrospection: RawIntrospection; + let plainScopedIntrospection: RawIntrospection; + let transactionStarted = false; + + beforeAll(async () => { + const connections = await getConnections({}, []); + ({ pg, teardown } = connections); + pool = connections.manager.getPool(pg.config); + + const available = await pg.query<{ name: string }>(` + SELECT name + FROM pg_catalog.pg_available_extensions + WHERE name = ANY($1::text[]) + `, [[...REQUIRED_EXTENSIONS]]); + const found = new Set(available.rows.map(({ name }) => name)); + const missing = REQUIRED_EXTENSIONS.filter((name) => !found.has(name)); + if (missing.length > 0) throw unsupportedExtensions(missing); + + try { + await pg.query(readFileSync(FIXTURE, 'utf8')); + } catch (error) { + const extension = extensionNamedBy(error); + if (extension) throw unsupportedExtensionSchema(extension); + throw error; + } + + const stockQuery = { text: makeIntrospectionQuery() }; + const scopedQuery = makeSchemaScopedIntrospectionQuery([API_SCHEMA], { + catalogTypes: 'dependency-closure', + capabilityExtensions: REQUIRED_EXTENSIONS + }); + const plainScopedQuery = makeSchemaScopedIntrospectionQuery( + [PLAIN_API_SCHEMA], + { + catalogTypes: 'dependency-closure', + capabilityExtensions: REQUIRED_EXTENSIONS + } + ); + const [stockResult, scopedResult, plainScopedResult] = await Promise.all([ + pool.query<{ introspection: string }>(stockQuery), + pool.query<{ introspection: string }>(scopedQuery), + pool.query<{ introspection: string }>(plainScopedQuery) + ]); + stockIntrospection = parseIntrospectionResults( + stockResult.rows[0].introspection + ); + scopedIntrospection = parseIntrospectionResults( + scopedResult.rows[0].introspection + ); + plainScopedIntrospection = parseIntrospectionResults( + plainScopedResult.rows[0].introspection + ); + + stock = await build(pool, 'stock'); + scoped = await build(pool, 'scoped-required'); + plainStock = await build(pool, 'stock', PLAIN_API_SCHEMA); + plainScoped = await build(pool, 'scoped-required', PLAIN_API_SCHEMA); + }); + + beforeEach(async () => { + await pg.beforeEach(); + transactionStarted = true; + }); + + afterEach(async () => { + if (transactionStarted) { + transactionStarted = false; + await pg.afterEach(); + } + }); + + afterAll(async () => { + if (teardown) await teardown(); + }); + + it('retains the required type, extension, namespace, and index closure', async () => { + const stockCatalog = captureCatalog(stockIntrospection); + const scopedCatalog = captureCatalog(scopedIntrospection); + + expect(scopedCatalog).toEqual(stockCatalog); + expect(scopedCatalog.namespaces).toEqual([ + API_SCHEMA, + EXTENSION_SCHEMA, + 'pg_catalog' + ]); + expect(scopedCatalog.extensions).toEqual([...REQUIRED_EXTENSIONS]); + expect(scopedCatalog.indexes).toEqual([...REQUIRED_INDEXES]); + expect(scopedCatalog.types.map(({ name, typtype }) => ({ name, typtype }))) + .toEqual(expect.arrayContaining([ + { name: `${API_SCHEMA}.closure_status`, typtype: 'e' }, + { name: `${API_SCHEMA}.closure_label`, typtype: 'd' }, + { name: `${API_SCHEMA}.closure_payload`, typtype: 'c' }, + { name: `${API_SCHEMA}.score_window`, typtype: 'r' }, + { name: `${API_SCHEMA}.score_window_multirange`, typtype: 'm' }, + { name: `${API_SCHEMA}._closure_status`, typtype: 'b' }, + { name: `${API_SCHEMA}._closure_label`, typtype: 'b' }, + { name: `${API_SCHEMA}._closure_payload`, typtype: 'b' }, + { name: `${API_SCHEMA}._score_window`, typtype: 'b' }, + { name: `${API_SCHEMA}._score_window_multirange`, typtype: 'b' } + ])); + + const installed = await pg.query<{ + extname: string; + nspname: string; + }>(` + SELECT extension.extname, namespace.nspname + FROM pg_catalog.pg_extension AS extension + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = extension.extnamespace + WHERE extension.extname = ANY($1::text[]) + ORDER BY extension.extname + `, [[...REQUIRED_EXTENSIONS]]); + expect(installed.rows).toEqual( + REQUIRED_EXTENSIONS.map((extname) => ({ + extname, + nspname: EXTENSION_SCHEMA + })) + ); + }); + + it('builds byte-equivalent SDL and qualifies every required field', () => { + const stockSdl = printSchema(lexicographicSortSchema(stock.schema)); + const scopedSdl = printSchema(lexicographicSortSchema(scoped.schema)); + + expect(scopedSdl).toBe(stockSdl); + expect(recordFields(stock.schema)).toEqual( + expect.arrayContaining(REQUIRED_RECORD_FIELDS) + ); + expect(recordFields(scoped.schema)).toEqual( + expect.arrayContaining(REQUIRED_RECORD_FIELDS) + ); + expect(qualification(stock.schema)).toEqual({ + missingFields: [], + productionQualified: true + }); + expect(qualification(scoped.schema)).toEqual({ + missingFields: [], + productionQualified: true + }); + }); + + it('retains installed extension capabilities without an object dependency', async () => { + const stockExtensions = sorted(stockIntrospection.extensions + .map(({ extname }) => extname) + .filter((name) => (REQUIRED_EXTENSIONS as readonly string[]).includes(name))); + const scopedExtensions = sorted( + plainScopedIntrospection.extensions.map(({ extname }) => extname) + ); + const stockLanguages = sorted( + stockIntrospection.languages.map(({ lanname }) => lanname) + ); + const scopedLanguages = sorted( + plainScopedIntrospection.languages.map(({ lanname }) => lanname) + ); + const stockAccessMethods = sorted( + stockIntrospection.am.map(({ amname }) => amname) + ); + const scopedAccessMethods = sorted( + plainScopedIntrospection.am.map(({ amname }) => amname) + ); + + expect(scopedExtensions).toEqual(stockExtensions); + expect(scopedLanguages).toEqual(stockLanguages); + expect(scopedAccessMethods).toEqual(stockAccessMethods); + expect(scopedExtensions).toEqual(expect.arrayContaining([ + ...REQUIRED_EXTENSIONS + ])); + expect(scopedExtensions).not.toContain('plpgsql'); + expect(scopedExtensions.length).toBeLessThan(MAX_SERVICE_CATALOG_ROWS); + expect(scopedLanguages.length).toBeLessThan(MAX_SERVICE_CATALOG_ROWS); + expect(scopedAccessMethods.length).toBeLessThan(MAX_SERVICE_CATALOG_ROWS); + expect(sorted( + plainScopedIntrospection.namespaces.map(({ nspname }) => nspname) + )).toEqual([EXTENSION_SCHEMA, PLAIN_API_SCHEMA, 'pg_catalog']); + + const stockSdl = printSchema(lexicographicSortSchema(plainStock.schema)); + const scopedSdl = printSchema(lexicographicSortSchema(plainScoped.schema)); + expect(scopedSdl).toBe(stockSdl); + + const plainRecord = plainStock.schema.getType('PlainRecord'); + expect(plainRecord && isObjectType(plainRecord) + ? Object.keys(plainRecord.getFields()) + : []).toEqual(expect.arrayContaining(['body', 'bodyTrgmSimilarity'])); + expect(plainStock.schema.getQueryType()?.getFields().plainRecords) + .toBeDefined(); + + const [stockResult, scopedResult] = await Promise.all([ + runDocument(plainStock, PLAIN_EXTENSION_DOCUMENT), + runDocument(plainScoped, PLAIN_EXTENSION_DOCUMENT) + ]); + expect(stockResult.errors).toBeUndefined(); + expect(scopedResult).toEqual(stockResult); + expect((stockResult.data?.plainRecords as { nodes: unknown[] }).nodes) + .toHaveLength(1); + }); + + it('executes matching core and extension documents without extension search_path', async () => { + const [stockCore, scopedCore, stockExtensions, scopedExtensions] = + await Promise.all([ + runDocument(stock, CORE_DOCUMENT), + runDocument(scoped, CORE_DOCUMENT), + runDocument(stock, EXTENSION_DOCUMENT, { bbox: BBOX }), + runDocument(scoped, EXTENSION_DOCUMENT, { bbox: BBOX }) + ]); + + expect(stockCore.errors).toBeUndefined(); + expect(scopedCore).toEqual(stockCore); + expect(scopedExtensions).toEqual(stockExtensions); + expect(stockExtensions.errors).toBeUndefined(); + + const records = (stockCore.data?.closureRecords as { + nodes: Array<{ scoreWindows: unknown[] }>; + }).nodes; + expect(records[0].scoreWindows).toHaveLength(2); + expect(stockCore.data?.closureMultirangeProbe).toHaveLength(2); + expect(stockCore.data?.closureTimestampMultirangeProbe).toHaveLength(2); + + const extensionData = stockExtensions.data as Record< + string, + { nodes: unknown[] } | null + >; + for (const alias of [ + 'tsvector', + 'trigram', + 'bm25', + 'vector', + 'postgis', + 'ltree' + ]) { + expect(extensionData[alias]?.nodes).toHaveLength(1); + } + }); +}); diff --git a/graphile/graphile-settings/__tests__/scoped-introspection-runtime.test.ts b/graphile/graphile-settings/__tests__/scoped-introspection-runtime.test.ts new file mode 100644 index 0000000000..471de308cf --- /dev/null +++ b/graphile/graphile-settings/__tests__/scoped-introspection-runtime.test.ts @@ -0,0 +1,125 @@ +import { makeSchema } from 'graphile-build'; +import { MinimalPreset } from '../src/plugins'; + +const { makePgService: makePostGraphilePgService } = require('postgraphile/adaptors/pg') as { + makePgService(options: Record): Record; +}; + +describe('schema-scoped introspection runtime integration', () => { + it.each([ + ['all catalog types by default', undefined, true], + ['dependency-closure catalog types', 'dependency-closure', false] + ] as const)('executes the parameterized scoped query with %s', async ( + _label, + scopedCatalogTypes, + retainsAllCatalogTypes + ) => { + const marker = new Error('captured introspection query'); + let captured: { text: string; values?: unknown[] } | null = null; + const client = { + query: jest.fn(async (query: string | { text: string; values?: unknown[] }) => { + if (typeof query === 'string') return { rows: [] as unknown[] }; + captured = query; + throw marker; + }), + release: jest.fn(), + addListener: jest.fn(), + removeListener: jest.fn() + }; + const pool = { + connect: jest.fn().mockResolvedValue(client) + }; + + await expect(makeSchema({ + extends: [MinimalPreset], + pgServices: [Object.assign(makePostGraphilePgService({ + pool: pool as never, + schemas: ['tenant_a'] + }), { + introspectionMode: 'scoped-required', + introspectionCapabilityExtensions: ['pg_trgm'], + ...(scopedCatalogTypes === undefined + ? {} + : { introspectionScopedCatalogTypes: scopedCatalogTypes }) + }) as never] + })).rejects.toBe(marker); + + expect(captured).not.toBeNull(); + expect(captured!.text).toContain('requested_schema_names'); + expect(captured!.text).not.toBe('select introspection'); + expect(captured!.values).toEqual([['tenant_a'], ['pg_trgm']]); + expect(captured!.text.includes( + "or pg_type.typnamespace = 'pg_catalog'::regnamespace" + )).toBe(retainsAllCatalogTypes); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it('fails closed when a retained entity references a missing type', async () => { + const introspection = JSON.stringify({ + database: {}, + namespaces: [{ + _id: '100', + nspname: 'tenant_a', + nspowner: '10', + nspacl: null + }], + classes: [{ + _id: '200', + relname: 'broken_items', + relnamespace: '100', + reltype: '999', + reloftype: null + }], + attributes: [], + constraints: [], + procs: [], + roles: [], + auth_members: [], + types: [], + enums: [], + extensions: [], + indexes: [], + inherits: [], + languages: [], + policies: [], + ranges: [], + depends: [], + descriptions: [], + am: [], + catalog_by_oid: { + 1255: 'pg_proc', + 1247: 'pg_type', + 1259: 'pg_class', + 2606: 'pg_constraint', + 2615: 'pg_namespace', + 3079: 'pg_extension' + }, + current_user: 'runtime_role', + pg_version: 'PostgreSQL test fixture', + introspection_version: 1 + }); + const client = { + query: jest.fn().mockResolvedValue({ rows: [{ introspection }] }), + release: jest.fn(), + addListener: jest.fn(), + removeListener: jest.fn() + }; + const pool = { + connect: jest.fn().mockResolvedValue(client) + }; + + await expect(makeSchema({ + extends: [MinimalPreset], + pgServices: [Object.assign(makePostGraphilePgService({ + pool: pool as never, + schemas: ['tenant_a'] + }), { + introspectionMode: 'scoped-required', + introspectionScopedCatalogTypes: 'dependency-closure' + }) as never] + })).rejects.toThrow( + /service '.+' retained pg_class 'broken_items \(200\)' field 'reltype' referencing missing pg_type OID '999'/ + ); + expect(client.release).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphile/graphile-settings/sql/scoped-introspection-capability-closure.sql b/graphile/graphile-settings/sql/scoped-introspection-capability-closure.sql new file mode 100644 index 0000000000..fd3812f524 --- /dev/null +++ b/graphile/graphile-settings/sql/scoped-introspection-capability-closure.sql @@ -0,0 +1,208 @@ +-- Capability fixture for schema-scoped dependency-closure introspection. +-- +-- The integration test performs an explicit pg_available_extensions preflight +-- before executing this file. Keep extension creation unconditional: a missing +-- or partially installed capability must fail the fixture instead of silently +-- producing a smaller GraphQL schema. + +BEGIN; + +CREATE SCHEMA closure_ext; + +CREATE EXTENSION ltree WITH SCHEMA closure_ext; +CREATE EXTENSION pg_textsearch WITH SCHEMA closure_ext; +CREATE EXTENSION pg_trgm WITH SCHEMA closure_ext; +CREATE EXTENSION postgis WITH SCHEMA closure_ext; +CREATE EXTENSION vector WITH SCHEMA closure_ext; + +CREATE SCHEMA closure_api; +SET LOCAL search_path = closure_api, pg_catalog, closure_ext; + +-- Exercise every user-defined pg_type.typtype that Graphile is expected to +-- retain, plus their generated array types. PostgreSQL 18 creates the +-- multirange type alongside the range type. +CREATE TYPE closure_api.closure_status AS ENUM ('ready', 'archived'); + +CREATE DOMAIN closure_api.closure_label AS text + CHECK (VALUE <> ''); + +CREATE TYPE closure_api.closure_payload AS ( + label closure_api.closure_label, + weight integer +); + +CREATE TYPE closure_api.score_window AS RANGE ( + subtype = numeric, + multirange_type_name = score_window_multirange +); + +-- Exercise multirange input serialization independently from table output. +-- The runtime search_path omits closure_ext, so the signature and body remain +-- bound to the tenant schema through explicit identifiers. +CREATE FUNCTION closure_api.closure_multirange_probe( + expected closure_api.score_window_multirange +) +RETURNS closure_api.score_window_multirange +LANGUAGE sql +IMMUTABLE +STRICT +SET search_path = pg_catalog, closure_api +AS $function$ + SELECT expected +$function$; + +-- Timestamp codecs use Graphile's SQL cast path, so this probe covers the +-- multirange representation that cannot be decoded from raw PostgreSQL text. +CREATE FUNCTION closure_api.closure_timestamp_multirange_probe( + expected pg_catalog.tstzmultirange +) +RETURNS pg_catalog.tstzmultirange +LANGUAGE sql +IMMUTABLE +STRICT +SET search_path = pg_catalog, closure_api +AS $function$ + SELECT expected +$function$; + +CREATE TABLE closure_api.closure_records ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + name text NOT NULL, + body text NOT NULL, + status closure_api.closure_status NOT NULL, + statuses closure_api.closure_status[] NOT NULL, + label closure_api.closure_label NOT NULL, + labels closure_api.closure_label[] NOT NULL, + payload closure_api.closure_payload NOT NULL, + payloads closure_api.closure_payload[] NOT NULL, + score_window closure_api.score_window NOT NULL, + score_window_array closure_api.score_window[] NOT NULL, + score_windows closure_api.score_window_multirange NOT NULL, + search_document tsvector NOT NULL, + path closure_ext.ltree NOT NULL, + paths closure_ext.ltree[] NOT NULL, + embedding closure_ext.vector(3) NOT NULL, + embeddings closure_ext.vector(3)[] NOT NULL, + geom closure_ext.geometry(Point, 4326) NOT NULL, + geoms closure_ext.geometry(Point, 4326)[] NOT NULL +); + +CREATE INDEX closure_records_search_document_idx + ON closure_api.closure_records USING gin(search_document); + +CREATE INDEX closure_records_name_trgm_idx + ON closure_api.closure_records + USING gin(name closure_ext.gin_trgm_ops); + +CREATE INDEX closure_records_body_bm25_idx + ON closure_api.closure_records USING bm25(body) + WITH (text_config = 'english'); + +CREATE INDEX closure_records_embedding_idx + ON closure_api.closure_records + USING hnsw (embedding closure_ext.vector_cosine_ops); + +CREATE INDEX closure_records_geom_idx + ON closure_api.closure_records USING gist(geom); + +CREATE INDEX closure_records_path_idx + ON closure_api.closure_records + USING gist(path closure_ext.gist_ltree_ops); + +CREATE INDEX closure_records_score_window_idx + ON closure_api.closure_records USING gist(score_window); + +INSERT INTO closure_api.closure_records ( + name, + body, + status, + statuses, + label, + labels, + payload, + payloads, + score_window, + score_window_array, + score_windows, + search_document, + path, + paths, + embedding, + embeddings, + geom, + geoms +) +VALUES ( + 'Acme tenant', + 'tenant memory density isolation canary', + 'ready', + ARRAY['ready', 'archived']::closure_api.closure_status[], + 'primary', + ARRAY['primary', 'secondary']::closure_api.closure_label[], + ROW('payload', 7)::closure_api.closure_payload, + ARRAY[ + ROW('payload', 7)::closure_api.closure_payload, + ROW('archive', 3)::closure_api.closure_payload + ], + closure_api.score_window(0, 10, '[]'), + ARRAY[ + closure_api.score_window(0, 10, '[]'), + closure_api.score_window(20, 30, '[)') + ], + closure_api.score_window_multirange( + closure_api.score_window(0, 10, '[]'), + closure_api.score_window(20, 30, '[)') + ), + to_tsvector('english', 'tenant memory density isolation canary'), + 'customers.acme', + ARRAY['customers.acme', 'customers.acme.documents']::closure_ext.ltree[], + '[1,0,0]', + ARRAY['[1,0,0]', '[0,1,0]']::closure_ext.vector(3)[], + closure_ext.st_setsrid(closure_ext.st_makepoint(-73.968, 40.785), 4326), + ARRAY[ + closure_ext.st_setsrid(closure_ext.st_makepoint(-73.968, 40.785), 4326), + closure_ext.st_setsrid(closure_ext.st_makepoint(-73.969, 40.786), 4326) + ]::closure_ext.geometry(Point, 4326)[] +); + +-- Defaults make this a root query field while its signature forces closure +-- traversal through enum, ltree, vector, and PostGIS types. The SQL body uses +-- fully qualified extension objects, so runtime search_path never needs the +-- shared extension schema. +CREATE FUNCTION closure_api.closure_function_probe( + expected_status closure_api.closure_status DEFAULT 'ready', + expected_path closure_ext.ltree DEFAULT 'customers.acme', + expected_embedding closure_ext.vector(3) DEFAULT '[1,0,0]', + expected_geom closure_ext.geometry DEFAULT + closure_ext.st_setsrid(closure_ext.st_makepoint(-73.968, 40.785), 4326) +) +RETURNS SETOF closure_api.closure_records +LANGUAGE sql +STABLE +SET search_path = pg_catalog, closure_api +AS $function$ + SELECT record.* + FROM closure_api.closure_records AS record + WHERE record.status = expected_status + AND record.path OPERATOR(closure_ext.<@) expected_path + AND record.embedding OPERATOR(closure_ext.<=>) expected_embedding < 0.01 + AND closure_ext.st_intersects(record.geom, expected_geom) +$function$; + +-- This schema deliberately has no type, function, operator-class, or index +-- dependency on pg_trgm/vector. Installed extensions are still a service-level +-- Graphile capability: @trgmSearch must behave identically under stock and +-- scoped introspection even when no retained object points at pg_trgm. +CREATE SCHEMA closure_plain_api; + +CREATE TABLE closure_plain_api.plain_records ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + body text NOT NULL +); + +COMMENT ON COLUMN closure_plain_api.plain_records.body IS E'@trgmSearch'; + +INSERT INTO closure_plain_api.plain_records (body) +VALUES ('tenant density capability without an extension-owned index'); + +COMMIT; diff --git a/graphile/graphile-settings/src/grafast-cache-limits.ts b/graphile/graphile-settings/src/grafast-cache-limits.ts new file mode 100644 index 0000000000..25855167c3 --- /dev/null +++ b/graphile/graphile-settings/src/grafast-cache-limits.ts @@ -0,0 +1,68 @@ +import type { GrafastCacheLimits } from '@constructive-io/graphql-types'; +import type { GraphileConfig } from 'graphile-config'; +import type { GraphQLSchemaConfig } from 'graphql'; + +const LIMIT_KEYS = [ + 'queryCacheMaxLength', + 'operationsCacheMaxLength', + 'operationOperationPlansCacheMaxLength' +] as const; + +/** Validate cache bounds before they reach Grafast's LRU constructors. */ +export const normalizeGrafastCacheLimits = ( + limits: GrafastCacheLimits +): Readonly => { + const normalized: GrafastCacheLimits = {}; + for (const key of LIMIT_KEYS) { + const value = limits[key]; + if (value === undefined) continue; + if (!Number.isSafeInteger(value) || value < 2) { + throw new Error(`grafastCache.${key} must be a safe integer of at least 2`); + } + normalized[key] = value; + } + return Object.freeze(normalized); +}; + +/** Apply authoritative per-schema cache limits without disturbing other extensions. */ +export const applyGrafastCacheLimits = ( + config: GraphQLSchemaConfig, + limits: Readonly +): GraphQLSchemaConfig => ({ + ...config, + extensions: { + ...(config.extensions ?? {}), + grafast: { + ...(config.extensions?.grafast ?? {}), + ...limits + } + } +}); + +/** Reusable plugin API for bounding Grafast's schema-local memory growth. */ +export const createGrafastCacheLimitsPlugin = ( + limits: GrafastCacheLimits +): GraphileConfig.Plugin => { + const normalized = normalizeGrafastCacheLimits(limits); + return { + name: 'GrafastCacheLimitsPlugin', + version: '1.0.0', + description: 'Bounds schema-local Grafast parse and operation-plan caches', + schema: { + hooks: { + GraphQLSchema(config) { + return applyGrafastCacheLimits(config, normalized); + } + } + } + }; +}; + +export const createGrafastCacheLimitsPreset = ( + limits: GrafastCacheLimits +): GraphileConfig.Preset => { + const normalized = normalizeGrafastCacheLimits(limits); + return Object.keys(normalized).length === 0 + ? {} + : { plugins: [createGrafastCacheLimitsPlugin(normalized)] }; +}; diff --git a/graphile/graphile-settings/src/index.ts b/graphile/graphile-settings/src/index.ts index afa9154a82..4b3b929260 100644 --- a/graphile/graphile-settings/src/index.ts +++ b/graphile/graphile-settings/src/index.ts @@ -35,7 +35,22 @@ import 'postgraphile/grafserv'; import 'graphile-build'; -import { makePgService } from 'postgraphile/adaptors/pg'; +import { makePgService as makePostGraphilePgService } from 'postgraphile/adaptors/pg'; + +import { + normalizeIntrospectionDependencySchemas, + resolveIntrospectionSettings +} from './introspection-settings'; +import { assertIntrospectionClientReleaseCapabilities } from './introspection-client-release'; + +export * from './introspection-client-release'; + +export { + applyGrafastCacheLimits, + createGrafastCacheLimitsPlugin, + createGrafastCacheLimitsPreset, + normalizeGrafastCacheLimits +} from './grafast-cache-limits'; // ============================================================================ // Re-export all plugins and presets @@ -43,7 +58,11 @@ import { makePgService } from 'postgraphile/adaptors/pg'; // Main preset + factory export type { ConstructivePresetOptions } from './presets/constructive-preset'; -export { ConstructivePreset, createConstructivePreset } from './presets/constructive-preset'; +export { + ConstructivePreset, + createConstructivePreset, + resolveConstructiveIntrospectionCapabilityExtensions +} from './presets/constructive-preset'; // Re-export all plugins for convenience export * from './plugins/index'; @@ -55,8 +74,106 @@ export * from './presets/index'; // Utilities // ============================================================================ -// Re-export makePgService for convenience -export { makePgService }; +export type ConstructivePgServiceOptions = Parameters[0] & { + introspectionMode?: 'stock' | 'scoped-required'; + introspectionScopedCatalogTypes?: 'all' | 'dependency-closure'; + introspectionAllowedDependencySchemas?: readonly string[]; + introspectionCapabilityExtensions?: readonly string[]; + introspectionClientReleaseMode?: 'reuse' | 'destroy'; +}; + +const normalizeIntrospectionCapabilityExtensions = ( + extensions: readonly string[] | undefined +): readonly string[] => { + if (extensions === undefined) return []; + if (!Array.isArray(extensions)) { + throw new Error('introspectionCapabilityExtensions must be an array'); + } + return [...new Set(extensions.map((extension) => { + if ( + typeof extension !== 'string' + || extension.length === 0 + || extension.trim() !== extension + || extension.includes('\0') + ) { + throw new Error( + 'introspectionCapabilityExtensions must contain exact non-empty extension names' + ); + } + return extension; + }))]; +}; + +/** + * Constructive's pgService factory adds the explicit catalog-introspection mode + * consumed by Graphile's gather phase. + */ +export const makePgService = (options: ConstructivePgServiceOptions) => { + const introspectionMode = options.introspectionMode ?? 'stock'; + const introspectionScopedCatalogTypes = options.introspectionScopedCatalogTypes; + const introspectionCapabilityExtensions = normalizeIntrospectionCapabilityExtensions( + options.introspectionCapabilityExtensions + ); + const introspectionClientReleaseMode = options.introspectionClientReleaseMode ?? 'reuse'; + if ( + introspectionScopedCatalogTypes !== undefined + && introspectionScopedCatalogTypes !== 'all' + && introspectionScopedCatalogTypes !== 'dependency-closure' + ) { + throw new Error( + `Unsupported scoped catalog type policy '${introspectionScopedCatalogTypes}'` + ); + } + if (introspectionMode === 'stock' && introspectionScopedCatalogTypes !== undefined) { + throw new Error( + 'introspectionScopedCatalogTypes requires scoped-required introspection' + ); + } + if ( + introspectionMode === 'stock' + && options.introspectionCapabilityExtensions !== undefined + ) { + throw new Error( + 'introspectionCapabilityExtensions requires scoped-required introspection' + ); + } + if ( + introspectionClientReleaseMode !== 'reuse' + && introspectionClientReleaseMode !== 'destroy' + ) { + throw new Error( + `Unsupported introspection client release mode '${introspectionClientReleaseMode}'` + ); + } + assertIntrospectionClientReleaseCapabilities(introspectionClientReleaseMode); + const introspectionAllowedDependencySchemas = normalizeIntrospectionDependencySchemas( + options.introspectionAllowedDependencySchemas + ); + const pgSettingsForIntrospection = resolveIntrospectionSettings( + introspectionMode, + options.pgSettingsForIntrospection + ); + const service = makePostGraphilePgService({ + ...options, + pgSettingsForIntrospection + }); + return Object.assign(service, { + introspectionMode, + ...(introspectionScopedCatalogTypes === undefined + ? {} + : { introspectionScopedCatalogTypes }), + introspectionAllowedDependencySchemas, + ...(introspectionMode === 'scoped-required' + ? { introspectionCapabilityExtensions } + : {}), + introspectionClientReleaseMode + }); +}; + +export { + normalizeIntrospectionDependencySchemas, + resolveIntrospectionSettings +} from './introspection-settings'; // Presigned URL utilities export { getPresignedUrlS3Config } from './presigned-url-resolver'; diff --git a/graphile/graphile-settings/src/introspection-client-release.ts b/graphile/graphile-settings/src/introspection-client-release.ts new file mode 100644 index 0000000000..1e72e4f0a3 --- /dev/null +++ b/graphile/graphile-settings/src/introspection-client-release.ts @@ -0,0 +1,49 @@ +import * as dataplanPg from '@dataplan/pg'; +import * as graphileBuildPg from 'graphile-build-pg'; + +export type IntrospectionClientReleaseMode = 'reuse' | 'destroy'; + +export interface IntrospectionClientReleaseCapabilities { + dataplanPg: unknown; + graphileBuildPg: unknown; +} + +const REQUIRED_DATAPLAN_PG_RELEASE_CAPABILITY = + 'dataplan-pg-exact-client-destroy-v1'; +const REQUIRED_GRAPHILE_BUILD_PG_RELEASE_CAPABILITY = + 'graphile-build-pg-exact-client-destroy-v1'; + +const runtimeIntrospectionClientReleaseCapabilities = Object.freeze({ + dataplanPg: (dataplanPg as Record).exactClientReleaseCapability, + graphileBuildPg: + (graphileBuildPg as Record).introspectionClientReleaseCapability +}); + +/** + * Dependency patches do not propagate through a published package. Destroy + * mode is accepted only when both upstream seams advertise the exact protocol + * this package was tested against. + */ +export function assertIntrospectionClientReleaseCapabilities( + mode: IntrospectionClientReleaseMode, + capabilities: IntrospectionClientReleaseCapabilities = + runtimeIntrospectionClientReleaseCapabilities +): void { + if (mode === 'reuse') return; + + const missing: string[] = []; + if (capabilities.dataplanPg !== REQUIRED_DATAPLAN_PG_RELEASE_CAPABILITY) { + missing.push('@dataplan/pg'); + } + if ( + capabilities.graphileBuildPg + !== REQUIRED_GRAPHILE_BUILD_PG_RELEASE_CAPABILITY + ) { + missing.push('graphile-build-pg'); + } + if (missing.length > 0) { + throw new Error( + `GRAPHILE_INTROSPECTION_CLIENT_DESTROY_UNSUPPORTED:${missing.join(',')}` + ); + } +} diff --git a/graphile/graphile-settings/src/introspection-settings.ts b/graphile/graphile-settings/src/introspection-settings.ts new file mode 100644 index 0000000000..a3099c652a --- /dev/null +++ b/graphile/graphile-settings/src/introspection-settings.ts @@ -0,0 +1,43 @@ +export type GraphileIntrospectionMode = 'stock' | 'scoped-required'; + +export const DEFAULT_INTROSPECTION_STATEMENT_TIMEOUT = '120s'; + +export const normalizeIntrospectionDependencySchemas = ( + schemas: readonly string[] | null | undefined +): string[] => [...new Set((schemas ?? []).map((schema) => { + if (typeof schema !== 'string' || schema.trim().length === 0) { + throw new Error('Introspection dependency schemas must be non-empty strings'); + } + const normalized = schema.trim(); + if (normalized === 'information_schema' || normalized.startsWith('pg_')) { + throw new Error(`Introspection dependency schema '${normalized}' must not be a system schema`); + } + if (normalized.includes('\0')) { + throw new Error('Introspection dependency schemas must not contain NUL bytes'); + } + return normalized; +}))]; + +export const resolveIntrospectionSettings = ( + mode: GraphileIntrospectionMode, + settings: Record | null | undefined +): Record => { + const boundedSettings = { ...settings }; + if (!boundedSettings.statement_timeout) { + // An admitted build owns the sole process-wide heap slot. Bound catalog + // SQL so a lock wait or pathological plan cannot block every cold tenant + // indefinitely; the setting is transaction-local in @dataplan/pg. + boundedSettings.statement_timeout = DEFAULT_INTROSPECTION_STATEMENT_TIMEOUT; + } + return mode === 'scoped-required' + ? { + ...boundedSettings, + // The scoped recursive query is deliberately short-lived. PostgreSQL's + // JIT compilation costs more than the catalog work, while wide catalog + // hashes multiply work_mem. Bound both only inside the introspection + // transaction; @dataplan/pg restores the runtime session afterwards. + jit: 'off', + work_mem: '512kB' + } + : boundedSettings; +}; diff --git a/graphile/graphile-settings/src/plugins/PublicKeySignature.ts b/graphile/graphile-settings/src/plugins/PublicKeySignature.ts index 2d01aa9712..861c3b5b67 100644 --- a/graphile/graphile-settings/src/plugins/PublicKeySignature.ts +++ b/graphile/graphile-settings/src/plugins/PublicKeySignature.ts @@ -4,10 +4,11 @@ import { QuoteUtils } from '@pgsql/quotes'; import { context as grafastContext, lambda, object } from 'grafast'; import type { GraphileConfig } from 'graphile-config'; import { extendSchema, gql } from 'graphile-utils'; -import pgQueryWithContext from 'pg-query-context'; export interface PublicKeyChallengeConfig { schema: string; + /** Exact anonymous role configured for this Graphile API surface. */ + anonymousRole: string; crypto_network: string; // crypto_network: keyof typeof Networks; sign_up_with_key: string; @@ -38,9 +39,54 @@ const MAX_MESSAGE_LENGTH = 4096; const MAX_SIGNATURE_LENGTH = 1024; const ENABLE_SIGNATURE_VERIFICATION = process.env.ENABLE_SIGNATURE_VERIFICATION === 'true'; +type PublicKeyPgSettings = Record; + +export type PublicKeyPgClient = { + query>(opts: { + text: string; + values?: unknown[]; + }): Promise<{ rows: TData[] }>; +}; + +export type PublicKeyWithPgClient = ( + pgSettings: PublicKeyPgSettings, + callback: (pgClient: PublicKeyPgClient) => Promise | T, +) => Promise; + +/** + * Run a public-key authentication operation in the request's complete GUC + * context while retaining the plugin's deliberately anonymous database role. + * + * The copy is important: Grafast's request context remains immutable, and the + * role override cannot leak back into other plans sharing that context. + */ +export async function withAnonymousPublicKeyClient( + withPgClient: PublicKeyWithPgClient | null | undefined, + pgSettings: unknown, + anonymousRole: string, + callback: (pgClient: PublicKeyPgClient) => Promise | T, +): Promise { + if (typeof withPgClient !== 'function') { + throw new Error('PG_CLIENT_CONTEXT_UNAVAILABLE'); + } + if (typeof pgSettings !== 'object' || pgSettings === null || Array.isArray(pgSettings)) { + throw new Error('PG_SETTINGS_UNAVAILABLE'); + } + validateIdentifier(anonymousRole, 'anonymousRole'); + + return withPgClient( + { + ...(pgSettings as PublicKeyPgSettings), + role: anonymousRole, + }, + callback, + ); +} + export const PublicKeySignature = (pubkey_challenge: PublicKeyChallengeConfig): GraphileConfig.Plugin => { const { schema, + anonymousRole, crypto_network, sign_up_with_key, sign_in_request_challenge, @@ -49,6 +95,7 @@ export const PublicKeySignature = (pubkey_challenge: PublicKeyChallengeConfig): } = pubkey_challenge; validateIdentifier(schema, 'schema'); + validateIdentifier(anonymousRole, 'anonymousRole'); validateIdentifier(sign_up_with_key, 'sign_up_with_key'); validateIdentifier(sign_in_request_challenge, 'sign_in_request_challenge'); validateIdentifier(sign_in_record_failure, 'sign_in_record_failure'); @@ -103,40 +150,32 @@ export const PublicKeySignature = (pubkey_challenge: PublicKeyChallengeConfig): createUserAccountWithPublicKey(_$mutation: any, fieldArgs: any) { const $input = fieldArgs.getRaw('input'); const $withPgClient = (grafastContext() as any).get('withPgClient'); - const $combined = object({ input: $input, withPgClient: $withPgClient }); + const $pgSettings = (grafastContext() as any).get('pgSettings'); + const $combined = object({ + input: $input, + withPgClient: $withPgClient, + pgSettings: $pgSettings, + }); - return lambda($combined, async ({ input, withPgClient }: any) => { + return lambda($combined, async ({ input, withPgClient, pgSettings }: any) => { if (!input.publicKey || typeof input.publicKey !== 'string' || input.publicKey.length > MAX_PUBLIC_KEY_LENGTH) { throw new Error('INVALID_PUBLIC_KEY'); } - return withPgClient(null, async (pgClient: any) => { - await pgClient.query('BEGIN'); - try { - await pgQueryWithContext({ - client: pgClient, - context: { role: 'anonymous' }, - query: `SELECT * FROM ${QuoteUtils.quoteQualifiedIdentifier(schema, sign_up_with_key)}($1)`, - variables: [input.publicKey], - skipTransaction: true - }); - - const { - rows: [{ [sign_in_request_challenge]: message }] - } = await pgQueryWithContext({ - client: pgClient, - context: { role: 'anonymous' }, - query: `SELECT * FROM ${QuoteUtils.quoteQualifiedIdentifier(schema, sign_in_request_challenge)}($1)`, - variables: [input.publicKey], - skipTransaction: true - }); - - await pgClient.query('COMMIT'); - return { message }; - } catch (err) { - await pgClient.query('ROLLBACK'); - throw err; - } + return withAnonymousPublicKeyClient(withPgClient, pgSettings, anonymousRole, async (pgClient) => { + await pgClient.query({ + text: `SELECT * FROM ${QuoteUtils.quoteQualifiedIdentifier(schema, sign_up_with_key)}($1)`, + values: [input.publicKey], + }); + + const { + rows: [{ [sign_in_request_challenge]: message }] + } = await pgClient.query>({ + text: `SELECT * FROM ${QuoteUtils.quoteQualifiedIdentifier(schema, sign_in_request_challenge)}($1)`, + values: [input.publicKey], + }); + + return { message }; }); }); }, @@ -144,21 +183,24 @@ export const PublicKeySignature = (pubkey_challenge: PublicKeyChallengeConfig): getMessageForSigning(_$mutation: any, fieldArgs: any) { const $input = fieldArgs.getRaw('input'); const $withPgClient = (grafastContext() as any).get('withPgClient'); - const $combined = object({ input: $input, withPgClient: $withPgClient }); + const $pgSettings = (grafastContext() as any).get('pgSettings'); + const $combined = object({ + input: $input, + withPgClient: $withPgClient, + pgSettings: $pgSettings, + }); - return lambda($combined, async ({ input, withPgClient }: any) => { + return lambda($combined, async ({ input, withPgClient, pgSettings }: any) => { if (!input.publicKey || typeof input.publicKey !== 'string' || input.publicKey.length > MAX_PUBLIC_KEY_LENGTH) { throw new Error('INVALID_PUBLIC_KEY'); } - return withPgClient(null, async (pgClient: any) => { + return withAnonymousPublicKeyClient(withPgClient, pgSettings, anonymousRole, async (pgClient) => { const { rows: [{ [sign_in_request_challenge]: message }] - } = await pgQueryWithContext({ - client: pgClient, - context: { role: 'anonymous' }, - query: `SELECT * FROM ${QuoteUtils.quoteQualifiedIdentifier(schema, sign_in_request_challenge)}($1)`, - variables: [input.publicKey] + } = await pgClient.query>({ + text: `SELECT * FROM ${QuoteUtils.quoteQualifiedIdentifier(schema, sign_in_request_challenge)}($1)`, + values: [input.publicKey], }); if (!message) throw new Error('NO_ACCOUNT_EXISTS'); @@ -173,9 +215,14 @@ export const PublicKeySignature = (pubkey_challenge: PublicKeyChallengeConfig): verifyMessageForSigning(_$mutation: any, fieldArgs: any) { const $input = fieldArgs.getRaw('input'); const $withPgClient = (grafastContext() as any).get('withPgClient'); - const $combined = object({ input: $input, withPgClient: $withPgClient }); + const $pgSettings = (grafastContext() as any).get('pgSettings'); + const $combined = object({ + input: $input, + withPgClient: $withPgClient, + pgSettings: $pgSettings, + }); - return lambda($combined, async ({ input, withPgClient }: any) => { + return lambda($combined, async ({ input, withPgClient, pgSettings }: any) => { const { publicKey, message, signature: _signature } = input; if (!publicKey || typeof publicKey !== 'string' || publicKey.length > MAX_PUBLIC_KEY_LENGTH) { @@ -194,31 +241,20 @@ export const PublicKeySignature = (pubkey_challenge: PublicKeyChallengeConfig): throw new Error('FEATURE_DISABLED'); } - return withPgClient(null, async (pgClient: any) => { - // Only the success path needs a transaction (multi-step) - await pgClient.query('BEGIN'); - try { - const { - rows: [token] - } = await pgQueryWithContext({ - client: pgClient, - context: { role: 'anonymous' }, - query: `SELECT * FROM ${QuoteUtils.quoteQualifiedIdentifier(schema, sign_in_with_challenge)}($1, $2)`, - variables: [publicKey, message], - skipTransaction: true - }); - - if (!token?.access_token) throw new Error('BAD_SIGNIN'); - - await pgClient.query('COMMIT'); - return { - access_token: token.access_token, - access_token_expires_at: token.access_token_expires_at - }; - } catch (err) { - await pgClient.query('ROLLBACK'); - throw err; - } + return withAnonymousPublicKeyClient(withPgClient, pgSettings, anonymousRole, async (pgClient) => { + const { + rows: [token] + } = await pgClient.query>({ + text: `SELECT * FROM ${QuoteUtils.quoteQualifiedIdentifier(schema, sign_in_with_challenge)}($1, $2)`, + values: [publicKey, message], + }); + + if (!token?.access_token) throw new Error('BAD_SIGNIN'); + + return { + access_token: token.access_token, + access_token_expires_at: token.access_token_expires_at + }; }); }); } diff --git a/graphile/graphile-settings/src/plugins/index.ts b/graphile/graphile-settings/src/plugins/index.ts index 829bba18c6..fe62a0b129 100644 --- a/graphile/graphile-settings/src/plugins/index.ts +++ b/graphile/graphile-settings/src/plugins/index.ts @@ -106,14 +106,12 @@ export type { export { Bm25CodecPlugin, Bm25CodecPreset, - bm25IndexStore, createBm25Adapter, // Operator factories for connection filter integration createMatchesOperatorFactory, createPgvectorAdapter, createTrgmAdapter, createTrgmOperatorFactories, - // Adapters createTsvectorAdapter, createTsvectorCodecPlugin, // Core plugin + preset diff --git a/graphile/graphile-settings/src/presets/constructive-preset.ts b/graphile/graphile-settings/src/presets/constructive-preset.ts index fdaf48fcd9..ddaf8593d4 100644 --- a/graphile/graphile-settings/src/presets/constructive-preset.ts +++ b/graphile/graphile-settings/src/presets/constructive-preset.ts @@ -8,7 +8,9 @@ import { GraphileLlmPreset } from 'graphile-llm'; import { createFolderOperatorFactory, GraphileLtreePreset } from 'graphile-ltree'; import { PgAggregatesPreset } from 'graphile-pg-aggregates'; import { createPostgisOperatorFactory,GraphilePostgisPreset } from 'graphile-postgis'; -import { PresignedUrlPreset } from 'graphile-presigned-url-plugin'; +import type { StorageModuleConfig } from 'graphile-presigned-url-plugin'; +import { PresignedUrlPreset, snapshotPreloadedStorageModules } from 'graphile-presigned-url-plugin'; +import type { RealtimeSubscriptionsPluginOptions } from 'graphile-realtime-subscriptions'; import { RealtimeSubscriptionsPreset } from 'graphile-realtime-subscriptions'; import { createMatchesOperatorFactory, createTrgmOperatorFactories,UnifiedSearchPreset } from 'graphile-search'; import { UploadPreset } from 'graphile-upload-plugin'; @@ -52,6 +54,14 @@ export interface ConstructivePresetOptions { enableBulk?: boolean; enableI18n?: boolean; enableHistory?: boolean; + /** Build-time and delivery settings forwarded to the realtime plugin. */ + realtimeSubscriptions?: RealtimeSubscriptionsPluginOptions; + /** + * Control-plane-resolved metadata. Constructive treats omission and an empty + * list as authoritative absence; it never falls back to runtime metaschema + * discovery. Standalone storage-plugin consumers retain the legacy mode. + */ + preloadedStorageModules?: readonly StorageModuleConfig[]; } /** @@ -72,7 +82,10 @@ function assertSupportedNodeVersion(): void { } } -const DEFAULTS: Required = { +const DEFAULTS: Required> = { enableAggregates: false, enablePostgis: true, enableSearch: true, @@ -88,6 +101,31 @@ const DEFAULTS: Required = { enableHistory: false }; +/** + * Extension metadata required by Constructive's enabled Graphile plugins. + * + * These are extension names, not schemas. Scoped introspection uses the exact + * names to retain optional capability metadata without treating every + * installed extension as part of the tenant's runtime surface. + */ +export function resolveConstructiveIntrospectionCapabilityExtensions( + options?: ConstructivePresetOptions +): readonly string[] { + const opts = { ...DEFAULTS, ...options }; + const extensions = new Set(); + + if (opts.enableSearch) { + extensions.add('pg_trgm'); + extensions.add('vector'); + extensions.add('pg_textsearch'); + } + if (opts.enableLlm) extensions.add('vector'); + if (opts.enablePostgis) extensions.add('postgis'); + if (opts.enableLtree) extensions.add('ltree'); + + return Object.freeze([...extensions]); +} + /** * Create a Constructive PostGraphile v5 Preset. * @@ -198,15 +236,24 @@ export function createConstructivePreset( } if (opts.enablePresignedUploads) { + // Freeze one authoritative control-plane snapshot and hand the exact same + // object to both storage plugins. Constructive always chooses the strict + // path: omitted metadata becomes an authoritative empty snapshot rather + // than enabling either plugin's legacy runtime lookup. + const storageModules = snapshotPreloadedStorageModules( + opts.preloadedStorageModules ?? [], + ); presets.push( PresignedUrlPreset({ s3: getPresignedUrlS3Config, resolveBucketName: createBucketNameResolver(), - ensureBucketProvisioned: createEnsureBucketProvisioned() + ensureBucketProvisioned: createEnsureBucketProvisioned(), + preloadedStorageModules: storageModules }), BucketProvisionerPreset({ connection: getBucketProvisionerConnection, allowedOrigins: getAllowedOrigins(), + preloadedStorageModules: storageModules, // Same tenant-aware naming policy as the presigned (lazy) path, so the // eager provisionBucket mutation mints the identical physical name // (`{prefix}-{bucketKey}-{databaseId}`) instead of falling back to the @@ -226,7 +273,7 @@ export function createConstructivePreset( } if (opts.enableRealtime) { - presets.push(RealtimeSubscriptionsPreset()); + presets.push(RealtimeSubscriptionsPreset(opts.realtimeSubscriptions)); } if (opts.enableBulk) { diff --git a/graphql/env/README.md b/graphql/env/README.md index e5084a59d8..c4c56978ce 100644 --- a/graphql/env/README.md +++ b/graphql/env/README.md @@ -43,6 +43,13 @@ In addition to all environment variables supported by `@pgpmjs/env`, this packag ### GraphQL Schema - `GRAPHILE_SCHEMA` - Comma-separated list of PostgreSQL schemas to expose +- `GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE` - `reuse` preserves the introspection backend; `destroy` retires that exact client after gather and reconnects lazily for runtime traffic; defaults to `reuse` +- `GRAPHILE_REALTIME_SCHEMA` - Exact physical schema containing realtime cursor functions; omission preserves `realtime_public` +- `GRAPHILE_REALTIME_NOTIFICATION_MODE` - `dedicated` keeps one PostGraphile subscriber per instance; `shared-exact` opts into the per-database exact-topic broker and requires an application `notificationPgResolver`; defaults to `dedicated` +- `GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS` - Maximum age of a successful shared-listener role audit; defaults to `60000` +- `GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS` - Realtime cursor recovery poll interval; defaults to `5000` +- `GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS` - Realtime cursor heartbeat interval; defaults to `30000` +- `GRAPHILE_RELEASE_BUILD_STATE_AFTER_VALIDATION` - Opt in to releasing schema-construction-only Graphile state after successful validation; defaults to `false` ### Feature Flags - `FEATURES_SIMPLE_INFLECTION` - Enable simple inflection plugin @@ -54,8 +61,18 @@ In addition to all environment variables supported by `@pgpmjs/env`, this packag - `API_IS_PUBLIC` - Whether API is public - `API_EXPOSED_SCHEMAS` - Comma-separated list of exposed schemas - `API_META_SCHEMAS` - Comma-separated list of meta schemas +- `API_ALLOW_META_SCHEMA_HEADER` - Explicitly enable the privileged `X-Meta-Schema` control-plane surface. Defaults to false and must only be used on a separate private admin ingress. - `API_ANON_ROLE` - Anonymous role name - `API_ROLE_NAME` - Default role name +- `GRAPHQL_INTERNAL_REQUEST_SECRET` - Minimum-32-byte token required before private routing/actor headers or the HTTP cache flush endpoint are trusted. `X-Schemata` remains prohibited; use an authoritative API name. + +### Routing Metadata Cache +- `GRAPHQL_ROUTING_CACHE_MAX_ENTRIES` - Capacity reserved for routing metadata diagnostics. Security-sensitive request routing is resolved authoritatively and never served from this cache. + +### Runtime PostgreSQL credentials + +- `GRAPHQL_RUNTIME_PGUSER` and `GRAPHQL_RUNTIME_PGPASSWORD` populate the legacy static `runtimePg` login. +- Production and `GRAPHILE_INTROSPECTION_MODE=scoped-required` do not accept those two values as a dynamic multi-tenant credential source. Use a programmatic `runtimePgResolver`; for a dedicated one-route server, pair an explicit static database with `runtimePgStaticIdentity` in trusted configuration. ## Defaults @@ -75,8 +92,10 @@ GraphQL defaults are provided by `@constructive-io/graphql-types`: roleName: 'administrator', isPublic: true, metaSchemas: ['routing_public', 'metaschema_public', 'metaschema_modules_public'], + allowMetaSchemaHeader: false, routingSchema: 'routing_public' - } + }, + routingCache: {} } ``` diff --git a/graphql/env/__tests__/__snapshots__/merge.test.ts.snap b/graphql/env/__tests__/__snapshots__/merge.test.ts.snap index 6383de2044..d16e94d4dc 100644 --- a/graphql/env/__tests__/__snapshots__/merge.test.ts.snap +++ b/graphql/env/__tests__/__snapshots__/merge.test.ts.snap @@ -3,6 +3,7 @@ exports[`getEnvOptions merges pgpm defaults, graphql defaults, config, env, and overrides 1`] = ` { "api": { + "allowMetaSchemaHeader": false, "anonRole": "env_anon", "exposedSchemas": [ "public", @@ -70,10 +71,19 @@ exports[`getEnvOptions merges pgpm defaults, graphql defaults, config, env, and }, "graphile": { "extends": [], + "introspectionClientReleaseMode": "reuse", + "introspectionDependencySchemas": [], + "introspectionMode": "stock", "preset": {}, + "realtimeCursorHeartbeatIntervalMs": 30000, + "realtimeCursorPollIntervalMs": 5000, + "realtimeNotificationMode": "dedicated", + "realtimeNotificationRoleRevalidationMs": 60000, + "releaseBuildStateAfterValidation": false, "schema": [ "override_schema", ], + "trustCallerPresetsInProduction": false, }, "migrations": { "codegen": { @@ -87,6 +97,7 @@ exports[`getEnvOptions merges pgpm defaults, graphql defaults, config, env, and "port": 5432, "user": "env-user", }, + "routingCache": {}, "server": { "host": "localhost", "port": 5000, diff --git a/graphql/env/__tests__/merge.test.ts b/graphql/env/__tests__/merge.test.ts index fa7dd645e8..d548504bb8 100644 --- a/graphql/env/__tests__/merge.test.ts +++ b/graphql/env/__tests__/merge.test.ts @@ -138,6 +138,46 @@ describe('getEnvOptions', () => { expect(result.api?.metaSchemas).toEqual(['env_meta', 'override_meta']); }); + it('parses the internal request secret without exposing a default', () => { + const secret = '0123456789abcdef0123456789abcdef'; + + expect(getGraphQLEnvVars({ GRAPHQL_INTERNAL_REQUEST_SECRET: secret }).api) + .toMatchObject({ internalRequestSecret: secret }); + expect(getGraphQLEnvVars({}).api?.internalRequestSecret).toBeUndefined(); + }); + + it('keeps the privileged metadata header disabled unless explicitly configured', () => { + expect(getGraphQLEnvVars({ API_ALLOW_META_SCHEMA_HEADER: 'true' }).api) + .toMatchObject({ allowMetaSchemaHeader: true }); + expect(getGraphQLEnvVars({ API_ALLOW_META_SCHEMA_HEADER: 'false' }).api) + .toMatchObject({ allowMetaSchemaHeader: false }); + expect(getGraphQLEnvVars({}).api?.allowMetaSchemaHeader).toBeUndefined(); + }); + + it('preserves the exact static runtime route contract from trusted config', () => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'graphql-env-runtime-pg-')); + const identity = { + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['tenant_a_public', 'tenant_a_auth'], + roles: ['tenant_a_anon', 'tenant_a_user'] + }; + writeConfig(tempDir, { + runtimePg: { + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + }, + runtimePgStaticIdentity: identity + }); + + const result = getEnvOptions({}, tempDir, {}); + + expect(result.runtimePgStaticIdentity).toEqual(identity); + expect(result.runtimePg?.database).toBe('tenant_a'); + }); + it('parses SMS environment variables into typed options', () => { const result = getGraphQLEnvVars({ SMS_PROVIDER: 'devsms', diff --git a/graphql/env/src/__tests__/runtime-pg.test.ts b/graphql/env/src/__tests__/runtime-pg.test.ts new file mode 100644 index 0000000000..5d00400674 --- /dev/null +++ b/graphql/env/src/__tests__/runtime-pg.test.ts @@ -0,0 +1,172 @@ +import { getGraphQLEnvVars } from '../env'; + +describe('GraphQL runtime PostgreSQL environment', () => { + it('maps the dedicated runtime credentials without changing control-plane pg', () => { + const result = getGraphQLEnvVars({ + GRAPHQL_RUNTIME_PGUSER: 'graphql_runtime', + GRAPHQL_RUNTIME_PGPASSWORD: 'runtime-secret' + }); + + expect(result.runtimePg).toEqual({ + user: 'graphql_runtime', + password: 'runtime-secret' + }); + expect(result.pg).toBeUndefined(); + }); + + it('does not create a runtime override when both variables are absent', () => { + expect(getGraphQLEnvVars({}).runtimePg).toBeUndefined(); + }); +}); + +describe('Graphile introspection environment', () => { + it.each(['stock', 'scoped-required'] as const)( + 'accepts the explicit %s mode', + (introspectionMode) => { + expect( + getGraphQLEnvVars({ GRAPHILE_INTROSPECTION_MODE: introspectionMode }).graphile + ).toEqual({ introspectionMode }); + } + ); + + it('rejects unknown modes instead of falling back to stock', () => { + expect(() => + getGraphQLEnvVars({ GRAPHILE_INTROSPECTION_MODE: 'scoped-if-possible' }) + ).toThrow("GRAPHILE_INTROSPECTION_MODE must be 'stock' or 'scoped-required'"); + }); + + it.each(['reuse', 'destroy'] as const)( + 'accepts the explicit %s introspection-client release mode', + (introspectionClientReleaseMode) => { + expect(getGraphQLEnvVars({ + GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE: introspectionClientReleaseMode + }).graphile).toEqual({ introspectionClientReleaseMode }); + } + ); + + it('rejects an unknown introspection-client release mode', () => { + expect(() => getGraphQLEnvVars({ + GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE: 'best-effort' + })).toThrow( + "GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE must be 'reuse' or 'destroy'" + ); + }); + + it('parses the ordered dependency-schema allowlist without duplicates', () => { + expect(getGraphQLEnvVars({ + GRAPHILE_INTROSPECTION_DEPENDENCY_SCHEMAS: 'extensions, shared_api,extensions' + }).graphile).toEqual({ + introspectionDependencySchemas: ['extensions', 'shared_api'] + }); + }); + + it('rejects an empty dependency-schema entry', () => { + expect(() => getGraphQLEnvVars({ + GRAPHILE_INTROSPECTION_DEPENDENCY_SCHEMAS: 'extensions, ,shared_api' + })).toThrow('must be a comma-separated list of non-empty schema names'); + }); +}); + +describe('Graphile realtime environment', () => { + it.each(['dedicated', 'shared-exact'] as const)( + 'maps the explicit %s notification mode', + (realtimeNotificationMode) => { + expect(getGraphQLEnvVars({ + GRAPHILE_REALTIME_NOTIFICATION_MODE: realtimeNotificationMode + }).graphile).toEqual({ realtimeNotificationMode }); + } + ); + + it('rejects unknown notification modes', () => { + expect(() => getGraphQLEnvVars({ + GRAPHILE_REALTIME_NOTIFICATION_MODE: 'shared-prefix' + })).toThrow("must be 'dedicated' or 'shared-exact'"); + }); + + it('maps role revalidation and cursor timing intervals', () => { + expect(getGraphQLEnvVars({ + GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS: '60000', + GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS: '30000', + GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS: '90000' + }).graphile).toEqual({ + realtimeNotificationRoleRevalidationMs: 60_000, + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 90_000 + }); + }); + + it('maps one exact cursor-function schema', () => { + expect(getGraphQLEnvVars({ + GRAPHILE_REALTIME_SCHEMA: ' tenant_a_realtime ' + }).graphile).toEqual({ + realtimeSchema: 'tenant_a_realtime' + }); + }); + + it('rejects a whitespace-only cursor schema', () => { + expect(() => getGraphQLEnvVars({ + GRAPHILE_REALTIME_SCHEMA: ' ' + })).toThrow('GRAPHILE_REALTIME_SCHEMA must be one non-empty exact schema name'); + }); + + it('preserves the compatibility default by omitting absent configuration', () => { + expect(getGraphQLEnvVars({}).graphile?.realtimeSchema).toBeUndefined(); + }); +}); + +describe('Grafast cache-limit environment', () => { + it('maps all three schema-local cache bounds', () => { + expect(getGraphQLEnvVars({ + GRAPHILE_QUERY_CACHE_MAX_LENGTH: '64', + GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH: '32', + GRAPHILE_OPERATION_PLANS_CACHE_MAX_LENGTH: '8' + }).graphile?.grafastCache).toEqual({ + queryCacheMaxLength: 64, + operationsCacheMaxLength: 32, + operationOperationPlansCacheMaxLength: 8 + }); + }); + + it.each(['0', '-1', '1.5', '12entries'])( + 'rejects an invalid cache bound %s', + (value) => { + expect(() => getGraphQLEnvVars({ + GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH: value + })).toThrow('must be a positive safe integer'); + } + ); +}); + +describe('Graphile build-state retirement environment', () => { + it.each([ + ['true', true], + ['false', false] + ])('maps the explicit %s value', (value, expected) => { + expect(getGraphQLEnvVars({ + GRAPHILE_RELEASE_BUILD_STATE_AFTER_VALIDATION: value + }).graphile?.releaseBuildStateAfterValidation).toBe(expected); + }); + + it('keeps retirement absent unless explicitly configured', () => { + expect( + getGraphQLEnvVars({}).graphile?.releaseBuildStateAfterValidation + ).toBeUndefined(); + }); +}); + +describe('Routing metadata cache environment', () => { + it('maps the explicit process capacity', () => { + expect(getGraphQLEnvVars({ + GRAPHQL_ROUTING_CACHE_MAX_ENTRIES: '4096' + }).routingCache).toEqual({ maxEntries: 4096 }); + }); + + it.each(['0', '-1', '1.5', '12entries'])( + 'rejects an invalid routing cache capacity %s', + (value) => { + expect(() => getGraphQLEnvVars({ + GRAPHQL_ROUTING_CACHE_MAX_ENTRIES: value + })).toThrow('must be a positive safe integer'); + } + ); +}); diff --git a/graphql/env/src/env.ts b/graphql/env/src/env.ts index 014924ef24..3345d7cfcb 100644 --- a/graphql/env/src/env.ts +++ b/graphql/env/src/env.ts @@ -7,6 +7,20 @@ import { parseEnvBoolean, parseEnvNumber } from '12factor-env'; export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial => { const { GRAPHILE_SCHEMA, + GRAPHILE_INTROSPECTION_MODE, + GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE, + GRAPHILE_INTROSPECTION_DEPENDENCY_SCHEMAS, + GRAPHILE_QUERY_CACHE_MAX_LENGTH, + GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH, + GRAPHILE_OPERATION_PLANS_CACHE_MAX_LENGTH, + GRAPHILE_REALTIME_SCHEMA, + GRAPHILE_REALTIME_NOTIFICATION_MODE, + GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS, + GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS, + GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS, + GRAPHILE_RELEASE_BUILD_STATE_AFTER_VALIDATION, + + GRAPHQL_ROUTING_CACHE_MAX_ENTRIES, FEATURES_SIMPLE_INFLECTION, FEATURES_OPPOSITE_BASE_NAMES, @@ -16,9 +30,15 @@ export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial API_IS_PUBLIC, API_EXPOSED_SCHEMAS, API_META_SCHEMAS, + API_ALLOW_META_SCHEMA_HEADER, API_ANON_ROLE, API_ROLE_NAME, + GRAPHQL_INTERNAL_REQUEST_SECRET, + + GRAPHQL_RUNTIME_PGUSER, + GRAPHQL_RUNTIME_PGPASSWORD, + EMBEDDER_PROVIDER, EMBEDDER_MODEL, EMBEDDER_BASE_URL, @@ -47,7 +67,93 @@ export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial ); return { + ...((GRAPHQL_RUNTIME_PGUSER || GRAPHQL_RUNTIME_PGPASSWORD) && { + runtimePg: { + ...(GRAPHQL_RUNTIME_PGUSER && { user: GRAPHQL_RUNTIME_PGUSER }), + ...(GRAPHQL_RUNTIME_PGPASSWORD && { password: GRAPHQL_RUNTIME_PGPASSWORD }) + } + }), + ...(GRAPHQL_ROUTING_CACHE_MAX_ENTRIES && { + routingCache: { + maxEntries: parsePositiveSafeInteger( + GRAPHQL_ROUTING_CACHE_MAX_ENTRIES, + 'GRAPHQL_ROUTING_CACHE_MAX_ENTRIES' + ) + } + }), graphile: { + ...(GRAPHILE_INTROSPECTION_MODE && { + introspectionMode: parseGraphileIntrospectionMode(GRAPHILE_INTROSPECTION_MODE) + }), + ...(GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE && { + introspectionClientReleaseMode: parseGraphileIntrospectionClientReleaseMode( + GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE + ) + }), + ...(GRAPHILE_INTROSPECTION_DEPENDENCY_SCHEMAS && { + introspectionDependencySchemas: parseSchemaList( + GRAPHILE_INTROSPECTION_DEPENDENCY_SCHEMAS, + 'GRAPHILE_INTROSPECTION_DEPENDENCY_SCHEMAS' + ) + }), + ...((GRAPHILE_QUERY_CACHE_MAX_LENGTH + || GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH + || GRAPHILE_OPERATION_PLANS_CACHE_MAX_LENGTH) && { + grafastCache: { + ...(GRAPHILE_QUERY_CACHE_MAX_LENGTH && { + queryCacheMaxLength: parsePositiveSafeInteger( + GRAPHILE_QUERY_CACHE_MAX_LENGTH, + 'GRAPHILE_QUERY_CACHE_MAX_LENGTH' + ) + }), + ...(GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH && { + operationsCacheMaxLength: parsePositiveSafeInteger( + GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH, + 'GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH' + ) + }), + ...(GRAPHILE_OPERATION_PLANS_CACHE_MAX_LENGTH && { + operationOperationPlansCacheMaxLength: parsePositiveSafeInteger( + GRAPHILE_OPERATION_PLANS_CACHE_MAX_LENGTH, + 'GRAPHILE_OPERATION_PLANS_CACHE_MAX_LENGTH' + ) + }) + } + }), + ...(GRAPHILE_REALTIME_SCHEMA && { + realtimeSchema: parseExactSchemaName( + GRAPHILE_REALTIME_SCHEMA, + 'GRAPHILE_REALTIME_SCHEMA' + ) + }), + ...(GRAPHILE_REALTIME_NOTIFICATION_MODE && { + realtimeNotificationMode: parseGraphileRealtimeNotificationMode( + GRAPHILE_REALTIME_NOTIFICATION_MODE + ) + }), + ...(GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS && { + realtimeNotificationRoleRevalidationMs: parsePositiveSafeInteger( + GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS, + 'GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS' + ) + }), + ...(GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS && { + realtimeCursorPollIntervalMs: parsePositiveSafeInteger( + GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS, + 'GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS' + ) + }), + ...(GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS && { + realtimeCursorHeartbeatIntervalMs: parsePositiveSafeInteger( + GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS, + 'GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS' + ) + }), + ...(GRAPHILE_RELEASE_BUILD_STATE_AFTER_VALIDATION && { + releaseBuildStateAfterValidation: parseEnvBoolean( + GRAPHILE_RELEASE_BUILD_STATE_AFTER_VALIDATION + ) + }), ...(GRAPHILE_SCHEMA && { schema: GRAPHILE_SCHEMA.includes(',') ? GRAPHILE_SCHEMA.split(',').map(s => s.trim()) @@ -64,8 +170,14 @@ export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial ...(API_IS_PUBLIC && { isPublic: parseEnvBoolean(API_IS_PUBLIC) }), ...(API_EXPOSED_SCHEMAS && { exposedSchemas: API_EXPOSED_SCHEMAS.split(',').map(s => s.trim()) }), ...(API_META_SCHEMAS && { metaSchemas: API_META_SCHEMAS.split(',').map(s => s.trim()) }), + ...(API_ALLOW_META_SCHEMA_HEADER && { + allowMetaSchemaHeader: parseEnvBoolean(API_ALLOW_META_SCHEMA_HEADER) + }), ...(API_ANON_ROLE && { anonRole: API_ANON_ROLE }), - ...(API_ROLE_NAME && { roleName: API_ROLE_NAME }) + ...(API_ROLE_NAME && { roleName: API_ROLE_NAME }), + ...(GRAPHQL_INTERNAL_REQUEST_SECRET && { + internalRequestSecret: GRAPHQL_INTERNAL_REQUEST_SECRET + }) }, ...((EMBEDDER_PROVIDER || CHAT_PROVIDER) && { llm: { @@ -102,3 +214,56 @@ export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial }) }; }; + +const parseGraphileIntrospectionMode = ( + value: string +): 'stock' | 'scoped-required' => { + if (value === 'stock' || value === 'scoped-required') return value; + throw new Error( + `GRAPHILE_INTROSPECTION_MODE must be 'stock' or 'scoped-required'; received '${value}'` + ); +}; + +const parseGraphileIntrospectionClientReleaseMode = ( + value: string +): 'reuse' | 'destroy' => { + if (value === 'reuse' || value === 'destroy') return value; + throw new Error( + "GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE must be 'reuse' or 'destroy'; " + + `received '${value}'` + ); +}; + +const parseGraphileRealtimeNotificationMode = ( + value: string +): 'dedicated' | 'shared-exact' => { + if (value === 'dedicated' || value === 'shared-exact') return value; + throw new Error( + "GRAPHILE_REALTIME_NOTIFICATION_MODE must be 'dedicated' or 'shared-exact'; " + + `received '${value}'` + ); +}; + +const parseSchemaList = (value: string, variable: string): string[] => { + const schemas = value.split(',').map((schema) => schema.trim()); + if (schemas.some((schema) => schema.length === 0)) { + throw new Error(`${variable} must be a comma-separated list of non-empty schema names`); + } + return [...new Set(schemas)]; +}; + +const parseExactSchemaName = (value: string, variable: string): string => { + const schema = value.trim(); + if (schema.length === 0) { + throw new Error(`${variable} must be one non-empty exact schema name`); + } + return schema; +}; + +const parsePositiveSafeInteger = (value: string, variable: string): number => { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${variable} must be a positive safe integer; received '${value}'`); + } + return parsed; +}; diff --git a/graphql/env/src/merge.ts b/graphql/env/src/merge.ts index 15f1402c53..58ca0146fb 100644 --- a/graphql/env/src/merge.ts +++ b/graphql/env/src/merge.ts @@ -44,6 +44,11 @@ export const getEnvOptions = ( ...(configOptions.graphile && { graphile: configOptions.graphile }), ...(configOptions.features && { features: configOptions.features }), ...(configOptions.api && { api: configOptions.api }), + ...(configOptions.routingCache && { routingCache: configOptions.routingCache }), + ...(configOptions.runtimePg && { runtimePg: configOptions.runtimePg }), + ...(configOptions.runtimePgStaticIdentity && { + runtimePgStaticIdentity: configOptions.runtimePgStaticIdentity + }), ...(configOptions.sms && { sms: configOptions.sms }), }, graphqlEnvOptions, diff --git a/graphql/query/tsconfig.esm.json b/graphql/query/tsconfig.esm.json index 800d7506d3..d767c87fcd 100644 --- a/graphql/query/tsconfig.esm.json +++ b/graphql/query/tsconfig.esm.json @@ -3,6 +3,7 @@ "compilerOptions": { "outDir": "dist/esm", "module": "es2022", + "moduleResolution": "bundler", "rootDir": "src/", "declaration": false } diff --git a/graphql/query/tsconfig.json b/graphql/query/tsconfig.json index 1a9d5696cb..1141f4f971 100644 --- a/graphql/query/tsconfig.json +++ b/graphql/query/tsconfig.json @@ -2,7 +2,10 @@ "extends": "../../tsconfig.json", "compilerOptions": { "outDir": "dist", - "rootDir": "src/" + "rootDir": "src/", + "module": "nodenext", + "moduleResolution": "nodenext", + "isolatedModules": true }, "include": ["src/**/*.ts"], "exclude": ["dist", "node_modules", "**/*.spec.*", "**/*.test.*"] diff --git a/graphql/server-test/__tests__/server.integration.test.ts b/graphql/server-test/__tests__/server.integration.test.ts index cec379129a..23dfef27f3 100644 --- a/graphql/server-test/__tests__/server.integration.test.ts +++ b/graphql/server-test/__tests__/server.integration.test.ts @@ -14,7 +14,11 @@ import path from 'path'; import type supertest from 'supertest'; -import { getConnections, seed } from '../src'; +import { + getConnections, + seed, + TEST_INTERNAL_REQUEST_SECRET +} from '../src'; import type { ServerInfo } from '../src/types'; jest.setTimeout(60000); @@ -51,6 +55,7 @@ type Scenario = { isPublic: boolean; metaSchemas?: string[]; routingSchema?: string; + allowMetaSchemaHeader?: boolean; }; headers?: Record; }; @@ -87,17 +92,8 @@ const scenarios: Scenario[] = [ api: { isPublic: false, metaSchemas: scopedMetaSchemas }, headers: { 'X-Database-Id': scopedDatabaseId, - 'X-Api-Name': 'private' - } - }, - { - name: 'scoped private via X-Schemata', - seedDir: 'simple-seed-scoped', - useRouting: true, - api: { isPublic: false, metaSchemas: scopedMetaSchemas }, - headers: { - 'X-Database-Id': scopedDatabaseId, - 'X-Schemata': schemas.join(',') + 'X-Api-Name': 'private', + 'X-Constructive-Internal-Token': TEST_INTERNAL_REQUEST_SECRET } } ]; @@ -267,6 +263,7 @@ describe('scoped private via X-Meta-Schema', () => { const headers: Record = { 'X-Database-Id': scopedDatabaseId, 'X-Meta-Schema': 'true', + 'X-Constructive-Internal-Token': TEST_INTERNAL_REQUEST_SECRET, ...extraHeaders }; for (const [header, value] of Object.entries(headers)) { @@ -284,7 +281,8 @@ describe('scoped private via X-Meta-Schema', () => { useRouting: true, api: { isPublic: false, - metaSchemas: metaApiSchemas + metaSchemas: metaApiSchemas, + allowMetaSchemaHeader: true } } }, @@ -342,7 +340,7 @@ describe('scoped private via X-Meta-Schema', () => { * Error path tests * * Exercise the api middleware error conditions under scoped routing: - * - Invalid X-Schemata (ApiError with errorHtml → 404) + * - Raw X-Schemata is rejected before database routing (→ 403) * - Host that resolves to no route (→ 404, no legacy fallback) * - NO_VALID_SCHEMAS (configured metaSchemas absent → 404) */ @@ -368,16 +366,40 @@ describe('Error paths', () => { teardowns.push(teardown); }); - describe('Invalid X-Schemata (returns 404)', () => { - it('should return 404 when X-Schemata contains schemas not in the DB', async () => { + describe('Raw X-Schemata (returns 403)', () => { + it('rejects physical schema selection even from an authenticated internal caller', async () => { const res = await request .post('/graphql') .set('X-Database-Id', scopedDatabaseId) .set('X-Schemata', 'nonexistent_schema_abc,another_fake_schema') + .set('X-Constructive-Internal-Token', TEST_INTERNAL_REQUEST_SECRET) .send({ query: '{ __typename }' }); - expect(res.status).toBe(404); - expect(res.text).toContain('No valid schemas found for the supplied X-Schemata header'); + expect(res.status).toBe(403); + expect(res.text).toBe('Forbidden'); + }); + }); + + describe('Unauthenticated internal selectors (returns 403)', () => { + it('rejects API/database selectors before any routing query', async () => { + const res = await request + .post('/graphql') + .set('X-Database-Id', scopedDatabaseId) + .set('X-Api-Name', 'private') + .send({ query: '{ __typename }' }); + + expect(res.status).toBe(403); + expect(res.text).toBe('Forbidden'); + }); + + it('rejects actor claims before any routing query', async () => { + const res = await request + .post('/graphql') + .set('X-Actor-Id', 'attacker-controlled-actor') + .send({ query: '{ __typename }' }); + + expect(res.status).toBe(403); + expect(res.text).toBe('Forbidden'); }); }); @@ -408,7 +430,8 @@ describe('Error paths', () => { useRouting: true, api: { isPublic: false, - metaSchemas: scopedMetaSchemas + metaSchemas: scopedMetaSchemas, + allowMetaSchemaHeader: true } } }, @@ -422,6 +445,7 @@ describe('Error paths', () => { .post('/graphql') .set('X-Database-Id', 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee') .set('X-Meta-Schema', 'true') + .set('X-Constructive-Internal-Token', TEST_INTERNAL_REQUEST_SECRET) .send({ query: '{ __typename }' }); expect(res.status).toBe(404); diff --git a/graphql/server-test/__tests__/upload.integration.test.ts b/graphql/server-test/__tests__/upload.integration.test.ts index a95abbd053..b6b3137bdb 100644 --- a/graphql/server-test/__tests__/upload.integration.test.ts +++ b/graphql/server-test/__tests__/upload.integration.test.ts @@ -33,7 +33,11 @@ import path from 'path'; import type { PgTestClient } from 'pgsql-test'; import type supertest from 'supertest'; -import { getConnections, seed } from '../src'; +import { + getConnections, + seed, + TEST_INTERNAL_REQUEST_SECRET +} from '../src'; jest.setTimeout(120000); @@ -288,7 +292,8 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { return request .post('/graphql') .set('X-Database-Id', aliceDatabaseId) - .set('X-Schemata', aliceSchemas.join(',')) + .set('X-Api-Name', 'app') + .set('X-Constructive-Internal-Token', TEST_INTERNAL_REQUEST_SECRET) .send(payload); }; @@ -304,6 +309,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { .post('/graphql') .set('X-Database-Id', databaseId) .set('X-Api-Name', apiName) + .set('X-Constructive-Internal-Token', TEST_INTERNAL_REQUEST_SECRET) .send(payload); }; @@ -319,6 +325,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { .post('/graphql') .set('X-Database-Id', databaseId) .set('X-Schemata', schemas.join(',')) + .set('X-Constructive-Internal-Token', TEST_INTERNAL_REQUEST_SECRET) .send(payload); }; @@ -1050,27 +1057,16 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { expect(res.status).toBe(404); }); - it('X-Schemata with Bob schema + Alice database_id does NOT leak Alice data', async () => { + it('rejects Bob physical schemas paired with Alice database_id', async () => { const res = await postGraphQLViaSchemata(aliceDatabaseId, bobSchemas, { query: APP_FILES }); - if (res.status === 200 && res.body.data) { - const names = (res.body.data.appFiles?.nodes ?? []).map( - (f: { filename: string }) => f.filename - ); - expect(names).not.toContain('hello-public.txt'); - expect(names).not.toContain('hello-private.txt'); - } + expect(res.status).toBe(403); + expect(res.text).toBe('Forbidden'); }); - it('X-Schemata with Mallory schema + Bob database_id does NOT leak Bob data', async () => { + it('rejects Mallory physical schemas paired with Bob database_id', async () => { const res = await postGraphQLViaSchemata(bobDatabaseId, mallorySchemas, { query: APP_FILES }); - if (res.status === 200 && res.body.data) { - const names = (res.body.data.appFiles?.nodes ?? []).map( - (f: { filename: string }) => f.filename - ); - expect(names).not.toContain('bob-file.txt'); - expect(names).not.toContain('bob-seeded-public.txt'); - expect(names).not.toContain('bob-seeded-private.txt'); - } + expect(res.status).toBe(403); + expect(res.text).toBe('Forbidden'); }); }); @@ -1106,4 +1102,3 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { }); }); }); - diff --git a/graphql/server-test/src/get-connections.ts b/graphql/server-test/src/get-connections.ts index 875c30b9cd..0e20088b01 100644 --- a/graphql/server-test/src/get-connections.ts +++ b/graphql/server-test/src/get-connections.ts @@ -3,7 +3,11 @@ import type { GetConnectionOpts, GetConnectionResult } from 'pgsql-test'; import { getConnections as getPgConnections } from 'pgsql-test'; import type { SeedAdapter } from 'pgsql-test/seed/types'; -import { createDevTestServer, createTestServer } from './server'; +import { + createDevTestServer, + createTestServer, + TEST_INTERNAL_REQUEST_SECRET +} from './server'; import { createQueryFn,createSuperTestAgent } from './supertest'; import type { GetConnectionsInput, GetConnectionsResult } from './types'; @@ -51,6 +55,12 @@ export const getConnections = async ( api: { // Start with user-provided api options from server.api ...input.server?.api, + // Production routing/identity headers fail closed unless the ingress is + // authenticated. Tests use one fixture-only credential and send it only + // on cases that intentionally exercise the reserved header boundary. + internalRequestSecret: + input.server?.api?.internalRequestSecret + ?? TEST_INTERNAL_REQUEST_SECRET, // Apply convenience properties (these take precedence) exposedSchemas: input.schemas, ...(input.authRole && { anonRole: input.authRole, roleName: input.authRole }) diff --git a/graphql/server-test/src/index.ts b/graphql/server-test/src/index.ts index 5b4b7e7f82..bf9bc942a0 100644 --- a/graphql/server-test/src/index.ts +++ b/graphql/server-test/src/index.ts @@ -2,7 +2,7 @@ export * from './types'; // Export server utilities -export { createTestServer } from './server'; +export { createTestServer, TEST_INTERNAL_REQUEST_SECRET } from './server'; // Export SuperTest utilities export { createSuperTestAgent } from './supertest'; diff --git a/graphql/server-test/src/server.ts b/graphql/server-test/src/server.ts index c8fbfc1e2d..87c0ef2ae5 100644 --- a/graphql/server-test/src/server.ts +++ b/graphql/server-test/src/server.ts @@ -5,6 +5,10 @@ import { Server as HttpServer } from 'http'; import type { ServerInfo, ServerOptions } from './types'; +/** Credential used only by in-process integration fixtures. */ +export const TEST_INTERNAL_REQUEST_SECRET = + 'graphql-server-test-internal-secret-32-bytes'; + /** * Create a single-tenant dev test server (no scoped routing, no database id). * diff --git a/graphql/server/README.md b/graphql/server/README.md index f874d18e0f..80d9f6a84e 100644 --- a/graphql/server/README.md +++ b/graphql/server/README.md @@ -65,7 +65,7 @@ Runs an Express server that wires CORS, uploads, domain parsing, auth, and PostG - Meta-schema routing by domain + subdomain - File uploads via `graphql-upload` - GraphiQL and health check endpoints -- Schema cache flush via `/flush` or database notifications +- Schema cache flush via authenticated `/flush` or database notifications - Opt-in observability for memory, DB activity, and Graphile build debugging ## Observability @@ -95,7 +95,7 @@ For the operational workflow, sampler output, and heap snapshot usage, see [docs - `GET /graphiql` -> GraphiQL UI - `GET /graphql` / `POST /graphql` -> GraphQL endpoint - `POST /graphql` (multipart) -> file uploads -- `POST /flush` -> clears cached Graphile schema for the current API +- `POST /flush` -> clears the current API's cached Graphile schema; requires `X-Constructive-Internal-Token` - `GET /debug/memory` -> memory/process/Graphile debug snapshot when observability is enabled - `GET /debug/db` -> PostgreSQL activity/locks/pool debug snapshot when observability is enabled @@ -103,14 +103,61 @@ For the operational workflow, sampler output, and heap snapshot usage, see [docs This is a production-only server: every request is resolved through the scoped-routing plane. There is no static single-tenant mode and no flag to disable routing. For single-database local development without route resolution or a database id, use [`@constructive-io/graphql-dev-server`](../dev-server/README.md). -- The server resolves the request host with a single `resolve_route()` call against the compiled route bindings in the scoped routing schema (`API_ROUTING_SCHEMA`, default `routing_public`), mapping host → tenant/api/database/role. +- The server resolves every request host with a fresh `resolve_route()` call against the compiled route bindings in the scoped routing schema (`API_ROUTING_SCHEMA`, default `routing_public`), mapping host → tenant/api/database/role. Routing metadata is not served from the process cache because a missed notification must never retain an old hostname-to-tenant assignment. - Only APIs where `api.is_public` matches `API_IS_PUBLIC` are served. -- In private mode (`API_IS_PUBLIC=false`), you can override with headers: +- In private mode (`API_IS_PUBLIC=false`), an internal caller can select an authoritative surface with these headers only when it also supplies the exact `X-Constructive-Internal-Token` configured by `GRAPHQL_INTERNAL_REQUEST_SECRET`: - `X-Api-Name` + `X-Database-Id` - - `X-Schemata` + `X-Database-Id` - - `X-Meta-Schema` + `X-Database-Id` +- `X-Meta-Schema` is a privileged, potentially cross-tenant control-plane API. It is rejected by default and can only be enabled with `API_ALLOW_META_SCHEMA_HEADER=true` on a separate private admin ingress; it is never a tenant-routing mechanism. +- `X-Schemata` is rejected even from an authenticated internal caller because an unchecked physical schema list is not a tenant-safe routing contract. Provision an API record and select it by name instead. +- The ingress must remove any caller-supplied reserved headers before injecting its own token and selectors, and the hop to this server must use an authenticated encrypted channel. - A resolved database id is always required. There is no default database, so a request that resolves without a database id is rejected (`NO_DATABASE_ID` → HTTP 500). +Production multi-tenant execution requires `runtimePgResolver`. The server calls +it once per request with the credential-free exact route contract: database id, +physical database name, API id, ordered physical schemas, and roles in +`[anonymous, authenticated]` order. The result must contain an explicit user, +password, and matching database; `connectionString` and control-plane credential +fallbacks are rejected. The secret-bearing result remains in a server-owned +`WeakMap`, while Express context and Graphile consume the same frozen resolution +and independently verify its opaque pool identity. + +```typescript +GraphQLServer({ + pg: controlPlanePg, + graphile: { introspectionMode: 'scoped-required' }, + runtimePgResolver: async ({ databaseId, databaseName, apiId, schemas, roles }) => { + const login = await credentialStore.get({ + databaseId, + databaseName, + apiId, + schemas, + roles + }); + return { + database: databaseName, + user: login.user, + password: login.password + }; + } +}); +``` + +`runtimePg` remains a compatibility path for one statically configured route. +In production or `scoped-required` mode it must include an explicit database and +be paired with an exact credential-free `runtimePgStaticIdentity`; any request +whose database/API/schema/role contract differs fails closed. A dynamic server +must use the resolver even when several databases happen to share a login. + +The resolver is part of the trusted routing boundary and must key its lookup by +immutable `databaseId`. The server requires its normalized host, port, database, +and TLS policy to match the control-plane tenant connection exactly, then binds +the complete target/login/pool contract into an opaque identity and rechecks the +route before every consumer reads it. A deployment where tenant databases live +on different network endpoints needs one future per-route resolver shared by +both control and runtime lanes; this implementation rejects that topology +rather than authenticating/configuring against one server and executing against +another. + ## Configuration Configuration is merged from defaults, config files, and env vars via `@constructive-io/graphql-env`. See `graphql/env/README.md` for the full list and examples. @@ -123,6 +170,13 @@ Configuration is merged from defaults, config files, and env vars via `@construc | `PGPASSWORD` | Postgres password | `password` | | `PGDATABASE` | Postgres database | `postgres` | | `GRAPHILE_SCHEMA` | Comma-separated schemas to expose | empty | +| `GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE` | Reuse or destroy the exact catalog-introspection client after gather | `reuse` | +| `GRAPHILE_REALTIME_SCHEMA` | Exact schema containing realtime cursor functions | `realtime_public` | +| `GRAPHILE_REALTIME_NOTIFICATION_MODE` | Dedicated subscriber or opt-in exact-topic broker | `dedicated` | +| `GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS` | Maximum age of shared-listener role audit | `60000` | +| `GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS` | Cursor recovery poll interval | `5000` | +| `GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS` | Cursor listener heartbeat interval | `30000` | +| `GRAPHILE_RELEASE_BUILD_STATE_AFTER_VALIDATION` | Release schema-construction-only state after successful validation | `false` | | `FEATURES_SIMPLE_INFLECTION` | Enable simple inflection | `true` | | `FEATURES_OPPOSITE_BASE_NAMES` | Enable opposite base names | `true` | | `FEATURES_POSTGIS` | Enable PostGIS support | `true` | @@ -130,12 +184,64 @@ Configuration is merged from defaults, config files, and env vars via `@construc | `API_IS_PUBLIC` | Serve public APIs only | `true` | | `API_EXPOSED_SCHEMAS` | Additional schemas to expose | empty | | `API_META_SCHEMAS` | Meta schemas to query | `routing_public,metaschema_public,metaschema_modules_public` | +| `API_ALLOW_META_SCHEMA_HEADER` | Enable the privileged metadata admin surface on an isolated private ingress | `false` | | `API_ANON_ROLE` | Anonymous role name | `administrator` | | `API_ROLE_NAME` | Authenticated role name | `administrator` | +| `GRAPHQL_INTERNAL_REQUEST_SECRET` | Minimum-32-byte token for reserved routing, actor-identity, and cache-administration headers | empty; reserved headers fail closed | +| `GRAPHQL_ROUTING_CACHE_MAX_ENTRIES` | Resolved routing/service labels retained per process; must be at least the effective Graphile resident capacity | `max(1024, effective Graphile capacity)` | | `GRAPHQL_OBSERVABILITY_ENABLED` | Master switch for debug routes and sampler | `false` | +| `GRAPHQL_OBSERVABILITY_TOKEN` | Bearer token (minimum 32 bytes) required for loopback-only production observability | empty | | `GRAPHQL_DEBUG_SAMPLER_ENABLED` | Enables periodic NDJSON sampling when observability is on | `true` | | `GRAPHQL_DEBUG_SAMPLER_INTERVAL_MS` | Sampler interval in milliseconds | `10000` | | `GRAPHQL_DEBUG_SAMPLER_DIR` | Override output directory for sampler logs | `graphql/server/logs` | +| `GRAPHILE_BUILD_WATCHDOG_MS` | Latch schema-build admission unhealthy after one admitted build exceeds this duration; recovery requires a process restart | `300000` | + +The build watchdog never cancels or releases an overdue build, because JavaScript +and plugin work cannot be canceled safely. It rejects queued and subsequent +builds with `GRAPHILE_BUILD_STUCK_RESTART_REQUIRED`, prevents late publication, +and leaves resident handlers available while the process is restarted. + +Programmatic `graphile.extends` and `graphile.preset` values are applied after +Constructive's feature preset, so trusted caller plugins and ordinary Graphile +schema/runtime settings take effect. They cannot replace the exact tenant +`pgServices`, security-GUC context, GraphQL/WebSocket transport policy, error +masking, or server-owned auth/admission plugins; explicit attempts fail startup +with `GRAPHILE_PROTECTED_PRESET_OVERRIDE`. Graphile plugins execute trusted +server-side code, so this boundary prevents structural misconfiguration rather +than sandboxing a hostile plugin implementation. + +The routing cache stores host/header labels and their resolved API metadata. Its +capacity is independent from Graphile build identity: evicting a routing label +causes the next request to resolve that label again, but it never disposes a +valid resident Graphile instance. `/debug/memory` reports its size, capacity, +hits, misses, and capacity/TTL evictions. + +`GRAPHILE_REALTIME_SCHEMA` changes only the exact cursor-function schema for an +API whose database settings enable realtime. Cursor events are accepted only +from that API's exposed physical schemas. A foreign cursor row or lost +subscriber emitter latches that exact generation unavailable, and the next HTTP +request receives `503 GRAPHILE_REALTIME_UNAVAILABLE` instead of entering its +Graphile handler. The failed generation is identity-checked and retired so the +following request can build a fresh one without a stale callback evicting a +healthy replacement. Realtime-enabled cached instances expose a no-server +Grafserv upgrade handler. The shared server routes `/graphql` upgrades through +the same API resolution, origin, authentication, request-context, build +contract, runtime-role, listener-attestation, and cache-admission path as HTTP; +other paths and failed admission close with stable metadata-free errors. +Accepted sockets retain their exact cache generation until close and are +destroyed before that generation is disposed. + +`GRAPHILE_REALTIME_NOTIFICATION_MODE=shared-exact` is an experimental, +default-off transport seam and additionally requires a +`notificationPgResolver` in `ConstructiveOptions`. It must return explicit +credentials for a dedicated listener login and the exact routed physical +database; runtime or control-plane credentials are never a fallback. The +listener identity in a Graphile build contract is an opaque digest, and raw +connection configuration is neither serialized into the contract nor exposed +through cache statistics. The transport remains experimental until the hostile +cross-tenant subscription suite and loaded churn qualification pass on the +production-shaped fixture; the upgrade router itself is now production-wired +and fail-closed. ## Testing diff --git a/graphql/server/package.json b/graphql/server/package.json index f653266ba3..ad86b1db0e 100644 --- a/graphql/server/package.json +++ b/graphql/server/package.json @@ -67,6 +67,7 @@ "graphile-cache": "workspace:^", "graphile-config": "1.0.1", "graphile-function-bindings": "workspace:^", + "graphile-realtime-subscriptions": "workspace:^", "graphile-settings": "workspace:^", "graphile-utils": "5.0.1", "graphql": "16.13.0", @@ -94,6 +95,7 @@ "makage": "^0.3.0", "nodemon": "^3.1.14", "supertest": "^7.2.2", + "pg-introspection": "1.0.1", "ts-node": "^10.9.2" } } diff --git a/graphql/server/src/__tests__/server-cache-lifecycle.test.ts b/graphql/server/src/__tests__/server-cache-lifecycle.test.ts new file mode 100644 index 0000000000..c95982e383 --- /dev/null +++ b/graphql/server/src/__tests__/server-cache-lifecycle.test.ts @@ -0,0 +1,98 @@ +import { + getGraphileGovernorCounters, + reopenGraphileBuildCoordinator, + runGraphileBuild +} from '../middleware/graphile-build-governor'; +import { + GRAPHILE_CACHE_SHUTDOWN_DRAIN_TIMEOUT_CODE, + GraphileCacheShutdownError, + Server +} from '../server'; + +const deferred = () => { + let resolve!: (value: T) => void; + const promise = new Promise((accept) => { + resolve = accept; + }); + return { promise, resolve }; +}; + +const settle = async (): Promise => { + await Promise.resolve(); + await Promise.resolve(); +}; + +describe('process-wide Graphile cache lifecycle', () => { + const previousShutdownTimeout = process.env.GRAPHILE_BUILD_SHUTDOWN_TIMEOUT_MS; + + afterEach(() => { + if (previousShutdownTimeout === undefined) { + delete process.env.GRAPHILE_BUILD_SHUTDOWN_TIMEOUT_MS; + } else { + process.env.GRAPHILE_BUILD_SHUTDOWN_TIMEOUT_MS = previousShutdownTimeout; + } + jest.useRealTimers(); + }); + + it('drains an admitted build before a direct process-wide cache clear', async () => { + const gate = deferred(); + const build = runGraphileBuild(() => gate.promise); + await settle(); + + let closeSettled = false; + const close = Server.closeCaches().then(() => { + closeSettled = true; + }); + await settle(); + + expect(closeSettled).toBe(false); + expect(getGraphileGovernorCounters().activeBuilds).toBe(1); + + gate.resolve('built'); + await expect(build).resolves.toBe('built'); + await expect(close).resolves.toBeUndefined(); + expect(closeSettled).toBe(true); + await expect(runGraphileBuild(async () => 'reopened')).resolves.toBe('reopened'); + }); + + it('also drains when caches are requested after an ordinary Server close', async () => { + const gate = deferred(); + const build = runGraphileBuild(() => gate.promise); + await settle(); + const server = Object.create(Server.prototype) as Server; + Object.assign(server, { closed: true }); + + let closeSettled = false; + const close = server.close({ closeCaches: true }).then(() => { + closeSettled = true; + }); + await settle(); + expect(closeSettled).toBe(false); + + gate.resolve('built'); + await build; + await close; + expect(closeSettled).toBe(true); + await expect(runGraphileBuild(async () => 'reopened')).resolves.toBe('reopened'); + }); + + it('leaves caches intact and admission closed when a build cannot drain', async () => { + jest.useFakeTimers(); + process.env.GRAPHILE_BUILD_SHUTDOWN_TIMEOUT_MS = '10'; + const gate = deferred(); + const build = runGraphileBuild(() => gate.promise); + await settle(); + + const closeFailure = expect(Server.closeCaches()).rejects.toMatchObject({ + code: GRAPHILE_CACHE_SHUTDOWN_DRAIN_TIMEOUT_CODE + } satisfies Partial); + await settle(); + await jest.advanceTimersByTimeAsync(10); + await closeFailure; + + gate.resolve('late completion'); + await expect(build).resolves.toBe('late completion'); + expect(reopenGraphileBuildCoordinator()).toBe(true); + await expect(runGraphileBuild(async () => 'recovered')).resolves.toBe('recovered'); + }); +}); diff --git a/graphql/server/src/__tests__/server-pool-listener-lease.test.ts b/graphql/server/src/__tests__/server-pool-listener-lease.test.ts new file mode 100644 index 0000000000..a661fba7e1 --- /dev/null +++ b/graphql/server/src/__tests__/server-pool-listener-lease.test.ts @@ -0,0 +1,230 @@ +import { EventEmitter } from 'node:events'; + +jest.mock('pg-cache', () => { + class MockPgPoolCapacityError extends Error { + readonly code = 'PG_POOL_CAPACITY'; + readonly retryAfterSeconds = 15; + } + return { + acquirePgPool: jest.fn(), + getPgPool: jest.fn(), + pgCache: { + registerCleanupCallback: jest.fn(() => jest.fn()) + }, + PgPoolCapacityError: MockPgPoolCapacityError + }; +}); + +import type { PoolClient } from 'pg'; +import { acquirePgPool, PgPoolCapacityError } from 'pg-cache'; + +import { Server } from '../server'; + +const mockAcquirePgPool = acquirePgPool as jest.MockedFunction; + +class FakeClient extends EventEmitter { + query = jest.fn().mockResolvedValue({}); +} + +const settle = async (): Promise => { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +}; + +const serverWithoutConstructor = (): Server => { + const server = Object.create(Server.prototype) as Server & Record; + Object.assign(server, { + opts: { pg: { database: 'routing' } }, + listenAttempt: null, + listenRetryTimer: null, + listenCleanupTasks: new Set>(), + shuttingDown: false, + closed: false, + moduleRegistry: { invalidate: jest.fn() } + }); + server.error = jest.fn(); + server.log = jest.fn(); + server.flush = jest.fn().mockResolvedValue(undefined); + return server; +}; + +const poolLease = (connect: jest.Mock) => { + const release = jest.fn(); + return { + value: { + pool: { connect } as never, + identity: 'pg:control', + release + }, + release + }; +}; + +describe('server LISTEN PostgreSQL pool-lease lifecycle', () => { + beforeEach(() => { + jest.useFakeTimers(); + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('uses a dedicated identity so a one-client routing pool is not starved', async () => { + const retained = poolLease(jest.fn()); + mockAcquirePgPool.mockReturnValue(retained.value); + const server = serverWithoutConstructor(); + + server.addEventListener(); + + expect(mockAcquirePgPool).toHaveBeenCalledWith( + { database: 'routing' }, + { purpose: 'notifications' } + ); + (server as unknown as { shuttingDown: boolean }).shuttingDown = true; + await server.removeEventListener(); + }); + + it('releases the pool lease and schedules one retry when checkout fails', async () => { + const retained = poolLease(jest.fn((callback) => { + callback(new Error('connect failed')); + })); + mockAcquirePgPool.mockReturnValue(retained.value); + const server = serverWithoutConstructor(); + + server.addEventListener(); + await settle(); + + expect(retained.release).toHaveBeenCalledTimes(1); + expect(jest.getTimerCount()).toBe(1); + await server.removeEventListener(); + expect(jest.getTimerCount()).toBe(0); + }); + + it('releases both ownership layers and cancels retry when LISTEN rejects', async () => { + const client = new FakeClient(); + const listenFailure = new Error('LISTEN failed'); + client.query.mockRejectedValueOnce(listenFailure); + const releaseClient = jest.fn(); + const retained = poolLease(jest.fn((callback) => { + callback(null, client as unknown as PoolClient, releaseClient); + })); + mockAcquirePgPool.mockReturnValue(retained.value); + const server = serverWithoutConstructor(); + + server.addEventListener(); + await settle(); + + expect(releaseClient).toHaveBeenCalledTimes(1); + expect(releaseClient).toHaveBeenCalledWith(listenFailure); + expect(retained.release).toHaveBeenCalledTimes(1); + expect(jest.getTimerCount()).toBe(1); + await server.removeEventListener(); + expect(jest.getTimerCount()).toBe(0); + }); + + it('coalesces repeated connection errors into one cleanup and one reconnect', async () => { + const firstClient = new FakeClient(); + const firstClientRelease = jest.fn(); + const first = poolLease(jest.fn((callback) => { + callback(null, firstClient as unknown as PoolClient, firstClientRelease); + })); + const secondConnect = jest.fn(); + const second = poolLease(secondConnect); + mockAcquirePgPool + .mockReturnValueOnce(first.value) + .mockReturnValueOnce(second.value); + const server = serverWithoutConstructor(); + + server.addEventListener(); + await settle(); + const registry = (server as unknown as { + moduleRegistry: { invalidate: jest.Mock }; + }).moduleRegistry; + expect(registry.invalidate).toHaveBeenCalledTimes(1); + const errorHandler = firstClient.listeners('error')[0] as (error: Error) => void; + const socketFailure = new Error('socket failed'); + errorHandler(socketFailure); + errorHandler(new Error('duplicate socket failure')); + await settle(); + + expect(firstClientRelease).toHaveBeenCalledTimes(1); + expect(firstClientRelease).toHaveBeenCalledWith(socketFailure); + expect(first.release).toHaveBeenCalledTimes(1); + expect(registry.invalidate).toHaveBeenCalledTimes(2); + expect(jest.getTimerCount()).toBe(1); + + await jest.advanceTimersByTimeAsync(5000); + expect(mockAcquirePgPool).toHaveBeenCalledTimes(2); + expect(secondConnect).toHaveBeenCalledTimes(1); + + (server as unknown as { shuttingDown: boolean }).shuttingDown = true; + await server.removeEventListener(); + expect(second.release).toHaveBeenCalledTimes(1); + expect(jest.getTimerCount()).toBe(0); + }); + + it('releases a late successful checkout after shutdown without double-releasing its pool', async () => { + let connectCallback!: ( + error: Error | null, + client?: PoolClient, + release?: () => void + ) => void; + const retained = poolLease(jest.fn((callback) => { + connectCallback = callback; + })); + mockAcquirePgPool.mockReturnValue(retained.value); + const server = serverWithoutConstructor(); + const client = new FakeClient(); + const releaseClient = jest.fn(); + + server.addEventListener(); + (server as unknown as { shuttingDown: boolean }).shuttingDown = true; + await server.removeEventListener(); + connectCallback(null, client as unknown as PoolClient, releaseClient); + await settle(); + + expect(client.query).not.toHaveBeenCalled(); + expect(releaseClient).toHaveBeenCalledTimes(1); + expect(retained.release).toHaveBeenCalledTimes(1); + expect(jest.getTimerCount()).toBe(0); + }); + + it('UNLISTENs and releases exactly once during active shutdown', async () => { + const client = new FakeClient(); + const releaseClient = jest.fn(); + const retained = poolLease(jest.fn((callback) => { + callback(null, client as unknown as PoolClient, releaseClient); + })); + mockAcquirePgPool.mockReturnValue(retained.value); + const server = serverWithoutConstructor(); + + server.addEventListener(); + await settle(); + (server as unknown as { shuttingDown: boolean }).shuttingDown = true; + await server.removeEventListener(); + + expect(client.query).toHaveBeenNthCalledWith(1, 'LISTEN "schema:update"'); + expect(client.query).toHaveBeenNthCalledWith(2, 'UNLISTEN "schema:update"'); + expect(releaseClient).toHaveBeenCalledTimes(1); + expect(retained.release).toHaveBeenCalledTimes(1); + expect(jest.getTimerCount()).toBe(0); + }); + + it('cancels a capacity retry timer during shutdown', async () => { + mockAcquirePgPool.mockImplementation(() => { + throw new PgPoolCapacityError(1, 1, 1); + }); + const server = serverWithoutConstructor(); + + server.addEventListener(); + expect(jest.getTimerCount()).toBe(1); + (server as unknown as { shuttingDown: boolean }).shuttingDown = true; + await server.removeEventListener(); + + expect(jest.getTimerCount()).toBe(0); + await jest.advanceTimersByTimeAsync(15_000); + expect(mockAcquirePgPool).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphql/server/src/__tests__/server-process-shutdown.test.ts b/graphql/server/src/__tests__/server-process-shutdown.test.ts new file mode 100644 index 0000000000..ac1397742c --- /dev/null +++ b/graphql/server/src/__tests__/server-process-shutdown.test.ts @@ -0,0 +1,71 @@ +import { EventEmitter } from 'node:events'; + +import { + installProcessShutdownHandlers, + type ProcessShutdownTarget +} from '../server'; + +class FakeProcess extends EventEmitter implements ProcessShutdownTarget { + readonly exit = jest.fn((_code?: number): void => undefined); +} + +const flushPromises = async (): Promise => { + await Promise.resolve(); + await Promise.resolve(); +}; + +describe('GraphQL server process shutdown', () => { + afterEach(() => { + jest.useRealTimers(); + }); + + it('drains once and exits cleanly on the first signal', async () => { + const processTarget = new FakeProcess(); + const shutdown = jest.fn(async (): Promise => undefined); + + installProcessShutdownHandlers(shutdown, { processTarget, timeoutMs: 1000 }); + processTarget.emit('SIGTERM'); + await flushPromises(); + + expect(shutdown).toHaveBeenCalledTimes(1); + expect(processTarget.exit).toHaveBeenCalledTimes(1); + expect(processTarget.exit).toHaveBeenCalledWith(0); + expect(processTarget.listenerCount('SIGINT')).toBe(0); + expect(processTarget.listenerCount('SIGTERM')).toBe(0); + }); + + it('forces exit on a repeated signal without starting a second drain', async () => { + let resolveShutdown!: () => void; + const processTarget = new FakeProcess(); + const shutdown = jest.fn(() => new Promise((resolve) => { + resolveShutdown = resolve; + })); + + installProcessShutdownHandlers(shutdown, { processTarget, timeoutMs: 1000 }); + processTarget.emit('SIGTERM'); + processTarget.emit('SIGINT'); + + expect(shutdown).toHaveBeenCalledTimes(1); + expect(processTarget.exit).toHaveBeenCalledTimes(1); + expect(processTarget.exit).toHaveBeenCalledWith(1); + + resolveShutdown(); + await flushPromises(); + expect(processTarget.exit).toHaveBeenCalledTimes(1); + }); + + it('forces exit when graceful shutdown exceeds its deadline', () => { + jest.useFakeTimers(); + const processTarget = new FakeProcess(); + const shutdown = jest.fn(() => new Promise(() => undefined)); + + installProcessShutdownHandlers(shutdown, { processTarget, timeoutMs: 1000 }); + processTarget.emit('SIGTERM'); + jest.advanceTimersByTime(1000); + + expect(shutdown).toHaveBeenCalledTimes(1); + expect(processTarget.exit).toHaveBeenCalledTimes(1); + expect(processTarget.exit).toHaveBeenCalledWith(1); + expect(jest.getTimerCount()).toBe(0); + }); +}); diff --git a/graphql/server/src/__tests__/websocket-upgrade.test.ts b/graphql/server/src/__tests__/websocket-upgrade.test.ts new file mode 100644 index 0000000000..739fa5d81e --- /dev/null +++ b/graphql/server/src/__tests__/websocket-upgrade.test.ts @@ -0,0 +1,324 @@ +import { EventEmitter } from 'node:events'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { PassThrough } from 'node:stream'; + +import express, { + type Express, + type NextFunction, + type Request, + type Response +} from 'express'; + +import { + createGraphileWebSocketOriginGuard, + createGraphileWebSocketUpgradeGateway, + getGraphileWebSocketUpgradeTransport, + GRAPHILE_WEBSOCKET_ADMISSION_FAILED_CODE, + GRAPHILE_WEBSOCKET_ADMISSION_TIMEOUT_CODE, + GRAPHILE_WEBSOCKET_BAD_UPGRADE_CODE, + GRAPHILE_WEBSOCKET_ROUTE_NOT_FOUND_CODE, + handoffGraphileWebSocketUpgrade +} from '../websocket-upgrade'; + +const makeRequest = ( + overrides: Partial = {} +): IncomingMessage => Object.assign(new EventEmitter(), { + method: 'GET', + url: '/graphql', + headers: { + connection: 'keep-alive, Upgrade', + upgrade: 'websocket', + host: 'a.example.test' + }, + aborted: false, + httpVersion: '1.1', + httpVersionMajor: 1, + httpVersionMinor: 1, + socket: { destroyed: false } +}, overrides) as unknown as IncomingMessage; + +const makeSocket = (): PassThrough => new PassThrough(); + +const outputFrom = (socket: PassThrough): { read(): string } => { + let value = ''; + socket.on('data', (chunk) => { + value += chunk.toString(); + }); + return { read: () => value }; +}; + +const settle = async (): Promise => { + await new Promise((resolve) => setImmediate(resolve)); +}; + +describe('production Graphile WebSocket upgrade gateway', () => { + afterEach(() => { + jest.useRealTimers(); + }); + + it('rejects the wrong path before tenant routing runs', async () => { + const app = jest.fn() as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app); + const socket = makeSocket(); + const output = outputFrom(socket); + + gateway.handle(makeRequest({ url: '/graphiql' }), socket, Buffer.alloc(0)); + await settle(); + + expect(app).not.toHaveBeenCalled(); + expect(output.read()).toContain('HTTP/1.1 404'); + expect(output.read()).toContain(GRAPHILE_WEBSOCKET_ROUTE_NOT_FOUND_CODE); + }); + + it('rejects malformed upgrades before tenant routing runs', async () => { + const app = jest.fn() as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app); + const socket = makeSocket(); + const output = outputFrom(socket); + + gateway.handle(makeRequest({ + headers: { host: 'a.example.test', upgrade: 'h2c' } + }), socket, Buffer.alloc(0)); + await settle(); + + expect(app).not.toHaveBeenCalled(); + expect(output.read()).toContain('HTTP/1.1 400'); + expect(output.read()).toContain(GRAPHILE_WEBSOCKET_BAD_UPGRADE_CODE); + }); + + it('preserves A/B routing and auth state on the exact handed-off request', () => { + const observed: Array> = []; + const app = express(); + app.use((request, _response, next) => { + const host = request.headers.host; + request.api = { + dbname: host === 'a.example.test' ? 'tenant_a' : 'tenant_b', + databaseId: host === 'a.example.test' ? 'database-a' : 'database-b', + apiId: host === 'a.example.test' ? 'api-a' : 'api-b', + schema: [host === 'a.example.test' ? 'a_public' : 'b_public'], + anonRole: 'tenant_anon', + roleName: 'tenant_user', + domains: [], + isPublic: true + }; + next(); + }); + app.use((request, _response, next) => { + request.token = { + user_id: request.headers.authorization?.slice('Bearer '.length) + }; + next(); + }); + app.use((request, response) => { + const transport = getGraphileWebSocketUpgradeTransport(request); + const accepted = handoffGraphileWebSocketUpgrade(request, response); + observed.push({ + request, + databaseId: request.api.databaseId, + userId: request.token.user_id, + socket: accepted.socket, + head: accepted.head, + transport + }); + }); + const gateway = createGraphileWebSocketUpgradeGateway(app); + const firstSocket = makeSocket(); + const secondSocket = makeSocket(); + const firstHead = Buffer.from('first-head'); + const secondHead = Buffer.from('second-head'); + const first = makeRequest({ + headers: { + connection: 'upgrade', + upgrade: 'websocket', + host: 'a.example.test', + authorization: 'Bearer actor-a' + } + }); + const second = makeRequest({ + headers: { + connection: 'upgrade', + upgrade: 'websocket', + host: 'b.example.test', + authorization: 'Bearer actor-b' + } + }); + + gateway.handle(first, firstSocket, firstHead); + gateway.handle(second, secondSocket, secondHead); + + expect(observed).toHaveLength(2); + expect(observed[0]).toMatchObject({ + request: first, + databaseId: 'database-a', + userId: 'actor-a', + socket: firstSocket, + head: firstHead, + transport: { socket: firstSocket, head: firstHead } + }); + expect(observed[1]).toMatchObject({ + request: second, + databaseId: 'database-b', + userId: 'actor-b', + socket: secondSocket, + head: secondHead, + transport: { socket: secondSocket, head: secondHead } + }); + expect(gateway.pendingCount).toBe(0); + }); + + it('retires and detaches the synthetic response before handoff', () => { + const socket = makeSocket(); + const head = Buffer.from('preserved-head'); + let responseAtHandoff: ServerResponse | undefined; + let releaseCount = 0; + const app = ((request: Request, response: Response): void => { + responseAtHandoff = response as unknown as ServerResponse; + response.once('close', () => { + releaseCount++; + }); + handoffGraphileWebSocketUpgrade(request, response); + }) as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app); + + gateway.handle(makeRequest(), socket, head); + + expect(responseAtHandoff?.socket).toBeNull(); + expect(releaseCount).toBe(1); + expect(socket.destroyed).toBe(false); + expect(gateway.pendingCount).toBe(0); + }); + + it('does not leak middleware response bodies or tenant metadata', async () => { + const app = ((_request: Request, response: Response): void => { + response.statusCode = 500; + response.end('tenant_a secret-password cache-key'); + }) as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app); + const socket = makeSocket(); + const output = outputFrom(socket); + + gateway.handle(makeRequest(), socket, Buffer.alloc(0)); + await settle(); + + expect(output.read()).toContain('HTTP/1.1 500'); + expect(output.read()).toContain(GRAPHILE_WEBSOCKET_ADMISSION_FAILED_CODE); + expect(output.read()).not.toContain('tenant_a'); + expect(output.read()).not.toContain('secret-password'); + expect(output.read()).not.toContain('cache-key'); + }); + + it('rejects an untrusted browser origin before authentication work', async () => { + const authenticate = jest.fn(( + _request: Request, + _response: Response, + next: NextFunction + ) => next()); + const app = express(); + app.use((request, _response, next) => { + request.api = { + databaseId: 'database-a', + dbname: 'tenant_a', + schema: ['a_public'], + anonRole: 'tenant_anon', + roleName: 'tenant_user', + domains: [], + corsOrigins: ['https://console.example.test'] + }; + next(); + }); + app.use(createGraphileWebSocketOriginGuard()); + app.use(authenticate); + const gateway = createGraphileWebSocketUpgradeGateway(app); + const socket = makeSocket(); + const output = outputFrom(socket); + + gateway.handle(makeRequest({ + headers: { + connection: 'upgrade', + upgrade: 'websocket', + host: 'a.example.test', + origin: 'https://attacker.example.test', + cookie: 'constructive_session=session-a' + } + }), socket, Buffer.alloc(0)); + await settle(); + + expect(authenticate).not.toHaveBeenCalled(); + expect(output.read()).toContain('HTTP/1.1 403'); + expect(output.read()).toContain('GRAPHILE_WEBSOCKET_AUTH_REJECTED'); + }); + + it('aborts bounded pre-upgrade work when the peer disconnects', async () => { + let aborted = 0; + const app = ((request: Request): void => { + request.once('aborted', () => { + aborted++; + }); + }) as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app); + const socket = makeSocket(); + + gateway.handle(makeRequest(), socket, Buffer.alloc(0)); + socket.destroy(); + await settle(); + + expect(aborted).toBe(1); + expect(gateway.pendingCount).toBe(0); + }); + + it('times out pre-upgrade work with a stable fail-closed response', async () => { + jest.useFakeTimers(); + let aborted = 0; + let responseClosed = 0; + const app = ((request: Request, response: Response): void => { + request.once('aborted', () => { + aborted++; + }); + response.once('close', () => { + responseClosed++; + }); + }) as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app, { + admissionTimeoutMs: 25 + }); + const socket = makeSocket(); + const output = outputFrom(socket); + + gateway.handle(makeRequest(), socket, Buffer.alloc(0)); + await jest.advanceTimersByTimeAsync(25); + + expect(aborted).toBe(1); + expect(responseClosed).toBe(1); + expect(output.read()).toContain('HTTP/1.1 503'); + expect(output.read()).toContain(GRAPHILE_WEBSOCKET_ADMISSION_TIMEOUT_CODE); + expect(output.read()).toContain('Retry-After: 1'); + expect(gateway.pendingCount).toBe(0); + }); + + it('aborts waiters and closes synthetic responses before shutdown rejection', async () => { + let aborted = 0; + let responseClosed = 0; + const app = ((request: Request, response: Response): void => { + request.once('aborted', () => { + aborted++; + }); + response.once('close', () => { + responseClosed++; + }); + }) as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app); + const socket = makeSocket(); + const output = outputFrom(socket); + + gateway.handle(makeRequest(), socket, Buffer.alloc(0)); + expect(gateway.pendingCount).toBe(1); + gateway.close(); + await settle(); + + expect(aborted).toBe(1); + expect(responseClosed).toBe(1); + expect(gateway.pendingCount).toBe(0); + expect(output.read()).toContain('HTTP/1.1 503'); + expect(output.read()).toContain('GRAPHILE_WEBSOCKET_SERVER_CLOSING'); + }); +}); diff --git a/graphql/server/src/agentic/router.ts b/graphql/server/src/agentic/router.ts index d1faf65320..151836ce27 100644 --- a/graphql/server/src/agentic/router.ts +++ b/graphql/server/src/agentic/router.ts @@ -19,7 +19,11 @@ */ import { OllamaAdapter } from '@agentic-kit/ollama'; -import type { BillingClient, LlmConfig } from '@constructive-io/express-context'; +import { + quoteQualifiedSqlIdentifier, + type BillingClient, + type LlmConfig +} from '@constructive-io/express-context'; import { getEnvOptions as getLlmEnvOptions } from '@constructive-io/llm-env'; import { Logger } from '@pgpmjs/logger'; import express, { Request, Response,Router } from 'express'; @@ -129,10 +133,15 @@ async function handleCreateThread( const body: CreateThreadBody = req.body || {}; const { schemaName, threadTableName } = agentChat; + const threadTableSql = quoteQualifiedSqlIdentifier( + schemaName, + threadTableName, + 'agent thread table' + ); const result = await ctx.withPgClient(async (client) => { const { rows } = await client.query( - `INSERT INTO "${schemaName}"."${threadTableName}" + `INSERT INTO ${threadTableSql} (entity_id, owner_id, mode, model, system_prompt, title) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, mode, model, system_prompt, status, created_at`, @@ -182,6 +191,16 @@ async function handleSendMessage( } const { schemaName, threadTableName, messageTableName } = agentChat; + const threadTableSql = quoteQualifiedSqlIdentifier( + schemaName, + threadTableName, + 'agent thread table' + ); + const messageTableSql = quoteQualifiedSqlIdentifier( + schemaName, + messageTableName, + 'agent message table' + ); const threadId = req.params.thread_id; const userId = ctx.userId; @@ -189,7 +208,7 @@ async function handleSendMessage( const threadRow = await ctx.withPgClient(async (client) => { const { rows } = await client.query( `SELECT id, mode, model, system_prompt, status - FROM "${schemaName}"."${threadTableName}" + FROM ${threadTableSql} WHERE id = $1`, [threadId] ); @@ -231,9 +250,9 @@ async function handleSendMessage( for (const msg of body.messages) { if (msg.role === 'user') { await client.query( - `INSERT INTO "${schemaName}"."${messageTableName}" + `INSERT INTO ${messageTableSql} (thread_id, owner_id, entity_id, author_role, parts) - VALUES ($1, $2, (SELECT entity_id FROM "${schemaName}"."${threadTableName}" WHERE id = $1), $3, $4)`, + VALUES ($1, $2, (SELECT entity_id FROM ${threadTableSql} WHERE id = $1), $3, $4)`, [threadId, userId, 'user', JSON.stringify([{ type: 'text', text: msg.content }])] ); } @@ -244,7 +263,7 @@ async function handleSendMessage( const history = await ctx.withPgClient(async (client) => { const { rows } = await client.query( `SELECT author_role, parts, created_at - FROM "${schemaName}"."${messageTableName}" + FROM ${messageTableSql} WHERE thread_id = $1 ORDER BY created_at ASC`, [threadId] @@ -277,14 +296,14 @@ async function handleSendMessage( await handleStreamingResponse(req, res, { ctx, chatAdapter, model, llmMessages, body, entityId, userId, threadId, - schemaName, threadTableName, messageTableName, + threadTableSql, messageTableSql, billing, startTime, meterSlug }); } else { await handleBatchResponse(req, res, { ctx, chatAdapter, model, llmMessages, body, entityId, userId, threadId, - schemaName, threadTableName, messageTableName, + threadTableSql, messageTableSql, billing, startTime, meterSlug }); } @@ -299,9 +318,8 @@ interface MessageContext { entityId: string; userId: string; threadId: string; - schemaName: string; - threadTableName: string; - messageTableName: string; + threadTableSql: string; + messageTableSql: string; billing: BillingClient | null; startTime: number; meterSlug: string; @@ -312,7 +330,7 @@ async function handleStreamingResponse( res: Response, mc: MessageContext ): Promise { - const { ctx, chatAdapter, model, llmMessages, body, entityId, userId, threadId, schemaName, threadTableName, messageTableName, billing, startTime, meterSlug } = mc; + const { ctx, chatAdapter, model, llmMessages, body, entityId, userId, threadId, threadTableSql, messageTableSql, billing, startTime, meterSlug } = mc; res.writeHead(200, { 'Content-Type': 'text/event-stream', @@ -375,9 +393,9 @@ async function handleStreamingResponse( if (content) { ctx.withPgClient(async (client) => { await client.query( - `INSERT INTO "${schemaName}"."${messageTableName}" + `INSERT INTO ${messageTableSql} (thread_id, owner_id, entity_id, author_role, parts, model) - VALUES ($1, $2, (SELECT entity_id FROM "${schemaName}"."${threadTableName}" WHERE id = $1), $3, $4, $5)`, + VALUES ($1, $2, (SELECT entity_id FROM ${threadTableSql} WHERE id = $1), $3, $4, $5)`, [threadId, userId, 'assistant', JSON.stringify([{ type: 'text', text: content }]), model] ); }).catch((err) => log.error('Failed to persist assistant message:', err)); @@ -416,7 +434,7 @@ async function handleBatchResponse( res: Response, mc: MessageContext ): Promise { - const { ctx, chatAdapter, model, llmMessages, body, entityId, userId, threadId, schemaName, threadTableName, messageTableName, billing, startTime, meterSlug } = mc; + const { ctx, chatAdapter, model, llmMessages, body, entityId, userId, threadId, threadTableSql, messageTableSql, billing, startTime, meterSlug } = mc; const systemMsg = llmMessages.find(m => m.role === 'system'); const nonSystem = llmMessages.filter(m => m.role !== 'system'); @@ -451,9 +469,9 @@ async function handleBatchResponse( // Persist assistant message await ctx.withPgClient(async (client) => { await client.query( - `INSERT INTO "${schemaName}"."${messageTableName}" + `INSERT INTO ${messageTableSql} (thread_id, owner_id, entity_id, author_role, parts, model) - VALUES ($1, $2, (SELECT entity_id FROM "${schemaName}"."${threadTableName}" WHERE id = $1), $3, $4, $5)`, + VALUES ($1, $2, (SELECT entity_id FROM ${threadTableSql} WHERE id = $1), $3, $4, $5)`, [threadId, userId, 'assistant', JSON.stringify([{ type: 'text', text: content }]), model] ); }); diff --git a/graphql/server/src/diagnostics/__tests__/observability.test.ts b/graphql/server/src/diagnostics/__tests__/observability.test.ts index 507584a5ce..799dde74cd 100644 --- a/graphql/server/src/diagnostics/__tests__/observability.test.ts +++ b/graphql/server/src/diagnostics/__tests__/observability.test.ts @@ -2,6 +2,7 @@ import { isDevelopmentObservabilityMode, isGraphqlDebugSamplerEnabled, isGraphqlObservabilityEnabled, + isGraphqlObservabilityTokenValid, isLoopbackAddress, isLoopbackHost, } from '../observability'; @@ -36,12 +37,27 @@ describe('observability helpers', () => { expect(isGraphqlDebugSamplerEnabled('0.0.0.0')).toBe(false); }); - it('disables observability outside development even when requested', () => { + it('disables production observability without a strong token', () => { process.env.NODE_ENV = 'production'; process.env.GRAPHQL_OBSERVABILITY_ENABLED = 'true'; expect(isDevelopmentObservabilityMode()).toBe(false); expect(isGraphqlObservabilityEnabled('localhost')).toBe(false); expect(isGraphqlDebugSamplerEnabled('localhost')).toBe(false); + + process.env.GRAPHQL_OBSERVABILITY_TOKEN = 'too-short'; + expect(isGraphqlObservabilityEnabled('localhost')).toBe(false); + }); + + it('allows token-authenticated production observability only on loopback', () => { + process.env.NODE_ENV = 'production'; + process.env.GRAPHQL_OBSERVABILITY_ENABLED = 'true'; + process.env.GRAPHQL_OBSERVABILITY_TOKEN = 'a'.repeat(64); + + expect(isGraphqlObservabilityEnabled('localhost')).toBe(true); + expect(isGraphqlDebugSamplerEnabled('127.0.0.1')).toBe(true); + expect(isGraphqlObservabilityEnabled('0.0.0.0')).toBe(false); + expect(isGraphqlObservabilityTokenValid('a'.repeat(64))).toBe(true); + expect(isGraphqlObservabilityTokenValid('b'.repeat(64))).toBe(false); }); }); diff --git a/graphql/server/src/diagnostics/debug-db-snapshot.ts b/graphql/server/src/diagnostics/debug-db-snapshot.ts index 666ad9619b..75932986d8 100644 --- a/graphql/server/src/diagnostics/debug-db-snapshot.ts +++ b/graphql/server/src/diagnostics/debug-db-snapshot.ts @@ -203,7 +203,7 @@ export interface DebugDatabaseSnapshot { export const getDebugDatabaseSnapshot = async ( opts: ConstructiveOptions, ): Promise => { - const appPool = getPgPool(opts.pg); + const appPool = getPgPool(opts.pg, { purpose: 'diagnostics' }); const { activity, blocked, diff --git a/graphql/server/src/diagnostics/debug-memory-snapshot.ts b/graphql/server/src/diagnostics/debug-memory-snapshot.ts index b35f5e8779..f85e42d765 100644 --- a/graphql/server/src/diagnostics/debug-memory-snapshot.ts +++ b/graphql/server/src/diagnostics/debug-memory-snapshot.ts @@ -1,11 +1,14 @@ import os from 'node:os'; import v8 from 'node:v8'; -import { SVC_CACHE_TTL_MS,svcCache } from '@pgpmjs/server-utils'; -import { getCacheStats } from 'graphile-cache'; +import { getSvcCacheStats } from '@pgpmjs/server-utils'; +import { getCacheCounters, getCacheStats } from 'graphile-cache'; +import { getPgCacheStats, getPgCheckoutSanitizerStats } from 'pg-cache'; import { getInFlightCount, getInFlightKeys } from '../middleware/graphile'; +import { getGraphileGovernorCounters } from '../middleware/graphile-build-governor'; import { getGraphileBuildStats } from '../middleware/observability/graphile-build-stats'; +import { getRuntimeRoleSafetyStats } from '../middleware/runtime-role-safety'; const toMB = (bytes: number): string => `${(bytes / 1024 / 1024).toFixed(1)} MB`; @@ -43,13 +46,12 @@ export interface DebugMemorySnapshot { }>; }; graphileCache: ReturnType; - svcCache: { - size: number; - max: number; - ttlMs: number; - oldestKeyAgeMs: number | null; - keys: string[]; - }; + graphileCacheCounters: ReturnType; + graphileGovernor: ReturnType; + pgCache: ReturnType; + pgCheckoutSanitizer: ReturnType; + runtimeRoleSafety: ReturnType; + svcCache: ReturnType; inFlight: { count: number; keys: string[]; @@ -97,23 +99,12 @@ export const getDebugMemorySnapshot = (): DebugMemorySnapshot => { heapSpaces, }, graphileCache: getCacheStats(), - svcCache: { - size: svcCache.size, - max: svcCache.max, - ttlMs: SVC_CACHE_TTL_MS, - // Note: with updateAgeOnGet: true, this is "time since last access" not "time since creation" - oldestKeyAgeMs: (() => { - let minRemaining = Infinity; - for (const key of svcCache.keys()) { - const remaining = svcCache.getRemainingTTL(key); - if (remaining < minRemaining) { - minRemaining = remaining; - } - } - return Number.isFinite(minRemaining) ? SVC_CACHE_TTL_MS - minRemaining : null; - })(), - keys: [...svcCache.keys()].slice(0, 200), - }, + graphileCacheCounters: getCacheCounters(), + graphileGovernor: getGraphileGovernorCounters(), + pgCache: getPgCacheStats(), + pgCheckoutSanitizer: getPgCheckoutSanitizerStats(), + runtimeRoleSafety: getRuntimeRoleSafetyStats(), + svcCache: getSvcCacheStats(), inFlight: { count: getInFlightCount(), keys: getInFlightKeys(), diff --git a/graphql/server/src/diagnostics/observability.ts b/graphql/server/src/diagnostics/observability.ts index bf0e8d466a..09d1e9c642 100644 --- a/graphql/server/src/diagnostics/observability.ts +++ b/graphql/server/src/diagnostics/observability.ts @@ -1,5 +1,8 @@ +import { timingSafeEqual } from 'node:crypto'; + const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1', '[::1]']); const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1']); +const MIN_OBSERVABILITY_TOKEN_BYTES = 32; const parseBooleanEnv = (value: string | undefined, fallback: boolean): boolean => { if (value == null) { @@ -46,6 +49,29 @@ const normalizeAddress = (value: string | null | undefined): string | null => { export const isDevelopmentObservabilityMode = (): boolean => process.env.NODE_ENV === 'development'; +/** + * Production observability is reserved for an explicitly authenticated local + * process such as cperf. Reject short secrets so an accidental boolean-like + * value cannot turn a production debug route on. + */ +export const getGraphqlObservabilityToken = (): string | null => { + const token = process.env.GRAPHQL_OBSERVABILITY_TOKEN?.trim(); + if (!token || Buffer.byteLength(token) < MIN_OBSERVABILITY_TOKEN_BYTES) { + return null; + } + return token; +}; + +export const isGraphqlObservabilityTokenValid = ( + candidate: string | null | undefined +): boolean => { + const token = getGraphqlObservabilityToken(); + if (!token || !candidate) return false; + const expected = Buffer.from(token); + const actual = Buffer.from(candidate); + return expected.length === actual.length && timingSafeEqual(expected, actual); +}; + export const isLoopbackHost = (value: string | null | undefined): boolean => { const normalized = normalizeHost(value); return normalized != null && LOOPBACK_HOSTS.has(normalized); @@ -60,9 +86,9 @@ export const isGraphqlObservabilityRequested = (): boolean => parseBooleanEnv(process.env.GRAPHQL_OBSERVABILITY_ENABLED, false); export const isGraphqlObservabilityEnabled = (serverHost?: string | null): boolean => - isDevelopmentObservabilityMode() && isGraphqlObservabilityRequested() && - isLoopbackHost(serverHost); + isLoopbackHost(serverHost) && + (isDevelopmentObservabilityMode() || getGraphqlObservabilityToken() !== null); export const isGraphqlDebugSamplerEnabled = (serverHost?: string | null): boolean => isGraphqlObservabilityEnabled(serverHost) && diff --git a/graphql/server/src/index.ts b/graphql/server/src/index.ts index edd35483ad..75cb2de241 100644 --- a/graphql/server/src/index.ts +++ b/graphql/server/src/index.ts @@ -6,3 +6,7 @@ export { createAuthenticateMiddleware } from './middleware/auth'; export { cors } from './middleware/cors'; export { flush, flushService } from './middleware/flush'; export { graphile } from './middleware/graphile'; +export { + GRAPHILE_PROTECTED_PRESET_OVERRIDE_CODE, + GraphileProtectedPresetOverrideError +} from './middleware/graphile-preset-composition'; diff --git a/graphql/server/src/middleware/__tests__/api.test.ts b/graphql/server/src/middleware/__tests__/api.test.ts index 965d01094c..4f4c02b0d4 100644 --- a/graphql/server/src/middleware/__tests__/api.test.ts +++ b/graphql/server/src/middleware/__tests__/api.test.ts @@ -1,22 +1,68 @@ jest.mock('pg-cache', () => ({ - getPgPool: jest.fn() + acquirePgPool: jest.fn(), + getPgPoolIdentity: jest.fn(), + PG_POOL_CAPACITY_ERROR_CODE: 'PG_POOL_CAPACITY' })); jest.mock('@constructive-io/express-context', () => ({ createDefaultRegistry: jest.fn(() => ({ - resolve: jest.fn().mockResolvedValue(undefined) + resolve: jest.fn(async (name: string) => name === 'databaseSettings' ? { + enableAggregates: false, + enablePostgis: false, + enableSearch: false, + enableDirectUploads: false, + enablePresignedUploads: false, + enableManyToMany: false, + enableConnectionFilter: false, + enableLtree: false, + enableLlm: false, + enableRealtime: false, + enableBulk: false, + enableI18n: false + } : undefined) })) })); +import { createDefaultRegistry } from '@constructive-io/express-context'; import { svcCache } from '@pgpmjs/server-utils'; -import type { Request } from 'express'; +import type { NextFunction, Request, Response } from 'express'; import type { Pool } from 'pg'; -import { getPgPool } from 'pg-cache'; +import { acquirePgPool, getPgPoolIdentity } from 'pg-cache'; import type { ApiOptions } from '../../types'; -import { getApiConfig, getSvcKey } from '../api'; +import { + createApiMiddleware, + getApiConfig, + getSvcCacheKey, + getSvcKey +} from '../api'; +import { + authorizeInternalRequest, + INTERNAL_REQUEST_TOKEN_HEADER +} from '../internal-request'; -const mockGetPgPool = getPgPool as jest.MockedFunction; +const INTERNAL_SECRET = 'test-internal-secret-with-at-least-32-bytes'; + +const withInternalAuth = ( + headers: Record +): Record => ({ + ...headers, + [INTERNAL_REQUEST_TOKEN_HEADER]: INTERNAL_SECRET +}); + +const mockAcquirePgPool = acquirePgPool as jest.MockedFunction; +const mockGetPgPoolIdentity = getPgPoolIdentity as jest.MockedFunction< + typeof getPgPoolIdentity +>; +const mockRegistryResolve = ( + createDefaultRegistry as jest.MockedFunction +).mock.results[0].value.resolve as jest.Mock; + +const leasePool = (pool: Pool, release = jest.fn()) => ({ + pool, + identity: 'pg:test', + release +}); const createRequest = (headers: Record): Request => { const normalized = new Map( @@ -36,7 +82,8 @@ const createPrivateOptions = (): ApiOptions => ({ }, api: { isPublic: false, - metaSchemas: ['metaschema_public'] + metaSchemas: ['metaschema_public'], + internalRequestSecret: INTERNAL_SECRET } } as unknown as ApiOptions); @@ -44,24 +91,28 @@ describe('api middleware routing priority', () => { beforeEach(() => { svcCache.clear(); jest.clearAllMocks(); + mockGetPgPoolIdentity.mockImplementation((config) => + `pg:${config.host ?? 'test'}` + ); }); afterEach(() => { svcCache.clear(); }); - it('uses X-Api-Name before X-Schemata when building private service keys', () => { - const req = createRequest({ + it('uses an authenticated X-Api-Name when building private service keys', () => { + const opts = createPrivateOptions(); + const req = createRequest(withInternalAuth({ host: 'admin.localhost', 'X-Database-Id': 'db-123', - 'X-Api-Name': 'customer-api', - 'X-Schemata': 'app_public' - }); + 'X-Api-Name': 'customer-api' + })); + authorizeInternalRequest(opts, req); - expect(getSvcKey(createPrivateOptions(), req)).toBe('api:db-123:customer-api'); + expect(getSvcKey(opts, req)).toBe('api:db-123:customer-api'); }); - it('uses the same X-Api-Name priority when resolving and caching API config', async () => { + it('resolves an authenticated X-Api-Name without caching routing authority', async () => { const query = jest.fn(async (_sql: string, params: unknown[]) => { if (Array.isArray(params[0])) { return { @@ -88,18 +139,27 @@ describe('api middleware routing priority', () => { return { rows: [] }; }); - mockGetPgPool.mockReturnValue({ query } as unknown as Pool); + const pool = { query } as unknown as Pool; + const releases: jest.Mock[] = []; + mockAcquirePgPool.mockImplementation(() => { + const release = jest.fn(); + releases.push(release); + return leasePool(pool, release); + }); - const req = createRequest({ + const req = createRequest(withInternalAuth({ host: 'admin.localhost', 'X-Database-Id': 'db-123', - 'X-Api-Name': 'customer-api', - 'X-Schemata': 'app_public' - }); + 'X-Api-Name': 'customer-api' + })); const result = await getApiConfig(createPrivateOptions(), req); expect(req.svc_key).toBe('api:db-123:customer-api'); + expect(req.svc_cache_key).toBe(getSvcCacheKey( + createPrivateOptions(), + 'api:db-123:customer-api' + )); expect(result).toMatchObject({ apiId: 'api-123', dbname: 'tenant_db', @@ -109,9 +169,284 @@ describe('api middleware routing priority', () => { databaseId: 'db-123', isPublic: false }); - expect(svcCache.get('api:db-123:customer-api')).toBe(result); + expect(svcCache.has(getSvcCacheKey( + createPrivateOptions(), + 'api:db-123:customer-api' + ))).toBe(false); expect(query.mock.calls).toEqual(expect.arrayContaining([ [expect.stringContaining('FROM "routing_public".apis'), ['db-123', 'customer-api']] ])); + expect(query.mock.calls.some(([sql]) => + String(sql).includes('aps.database_id = a.database_id') + )).toBe(true); + expect(mockAcquirePgPool).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ database: 'constructive' }), + { purpose: 'routing-request-control', sanitizeOnCheckout: true } + ); + expect(mockAcquirePgPool).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ database: 'tenant_db' }), + { purpose: 'tenant-request-control', sanitizeOnCheckout: true } + ); + expect(releases).toHaveLength(2); + expect(releases.every((release) => release.mock.calls.length === 1)).toBe(true); + }); + + it('fails closed when an authenticated API selector resolves an incomplete contract', async () => { + const query = jest.fn(async (_sql: string, params: unknown[]) => { + if (Array.isArray(params[0])) { + return { + rows: (params[0] as string[]).map((schemaName) => ({ schema_name: schemaName })) + }; + } + return { + rows: [{ + api_id: 'api-123', + database_id: 'db-123', + dbname: 'tenant_db', + role_name: '', + anon_role: 'api_anon', + is_public: false, + schemas: ['api_public'] + }] + }; + }); + mockAcquirePgPool.mockReturnValue(leasePool({ query } as unknown as Pool)); + + const result = await getApiConfig(createPrivateOptions(), createRequest(withInternalAuth({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Api-Name': 'customer-api' + }))); + + expect(result).toBeNull(); + }); + + it('fails closed when an exact tenant API has no feature contract', async () => { + const query = jest.fn(async (_sql: string, params: unknown[]) => { + if (Array.isArray(params[0])) { + return { + rows: (params[0] as string[]).map((schemaName) => ({ schema_name: schemaName })) + }; + } + return { rows: [{ + api_id: 'api-123', + database_id: 'db-123', + dbname: 'tenant_db', + role_name: 'api_user', + anon_role: 'api_anon', + is_public: false, + schemas: ['api_public'] + }] }; + }); + mockAcquirePgPool.mockReturnValue(leasePool({ query } as unknown as Pool)); + mockRegistryResolve.mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined); + const req = createRequest(withInternalAuth({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Api-Name': 'customer-api' + })); + + await expect(getApiConfig(createPrivateOptions(), req)).rejects.toMatchObject({ + code: 'GRAPHILE_DATABASE_FEATURE_CONTRACT_MISSING' + }); + }); + + it('ignores a stale routing-cache entry and resolves the authoritative API', async () => { + const opts = createPrivateOptions(); + const req = createRequest(withInternalAuth({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Api-Name': 'customer-api' + })); + svcCache.set(getSvcCacheKey( + opts, + 'api:db-123:customer-api' + ), { databaseId: 'db-old', dbname: 'wrong_tenant' }); + + const query = jest.fn(async (_sql: string, params: unknown[]) => { + if (Array.isArray(params[0])) { + return { + rows: (params[0] as string[]).map((schemaName) => ({ + schema_name: schemaName + })) + }; + } + return { + rows: [{ + api_id: 'api-123', + database_id: 'db-123', + dbname: 'tenant_fresh', + role_name: 'api_role', + anon_role: 'api_anon', + is_public: false, + schemas: ['api_public'] + }] + }; + }); + mockAcquirePgPool.mockImplementation(() => + leasePool({ query } as unknown as Pool) + ); + + await expect(getApiConfig(opts, req)).resolves.toMatchObject({ + databaseId: 'db-123', + dbname: 'tenant_fresh' + }); + expect(mockAcquirePgPool).toHaveBeenCalled(); + }); + + it('isolates one routing label across exact control-pool contracts', () => { + const optsA = createPrivateOptions(); + const optsB = createPrivateOptions(); + optsA.pg = { ...optsA.pg, host: 'routing-a.internal' }; + optsB.pg = { ...optsB.pg, host: 'routing-b.internal' }; + const label = 'api:db-123:customer-api'; + + expect(getSvcCacheKey(optsA, label)).not.toBe(getSvcCacheKey(optsB, label)); + }); + + it('does not publish authoritative meta-schema routing results', async () => { + const query = jest.fn(async (_sql: string, params: unknown[]) => ({ + rows: (params[0] as string[]).map((schemaName) => ({ schema_name: schemaName })) + })); + mockAcquirePgPool.mockReturnValue(leasePool({ query } as unknown as Pool)); + const opts = createPrivateOptions(); + opts.api!.allowMetaSchemaHeader = true; + const req = createRequest(withInternalAuth({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Meta-Schema': 'metaschema_public' + })); + + await expect(getApiConfig(opts, req)).resolves.toMatchObject({ databaseId: 'db-123' }); + expect(svcCache.has(req.svc_cache_key!)).toBe(false); + }); + + it('rejects raw physical schema routing even with a valid internal token', async () => { + const opts = createPrivateOptions(); + const req = createRequest(withInternalAuth({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Schemata': 'app_public' + })); + + await expect(getApiConfig(opts, req)).rejects.toMatchObject({ + code: 'INTERNAL_REQUEST_FORBIDDEN' + }); + expect(mockAcquirePgPool).not.toHaveBeenCalled(); + }); + + it('rejects unauthenticated private routing headers before touching PostgreSQL', async () => { + const req = createRequest({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Api-Name': 'customer-api' + }); + + await expect(getApiConfig(createPrivateOptions(), req)).rejects.toMatchObject({ + code: 'INTERNAL_REQUEST_FORBIDDEN' + }); + expect(mockAcquirePgPool).not.toHaveBeenCalled(); + }); + + it('returns 403 for an invalid internal token without leaking configuration', async () => { + const req = createRequest({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Api-Name': 'customer-api', + [INTERNAL_REQUEST_TOKEN_HEADER]: 'wrong-secret-with-at-least-32-bytes' + }); + const res = { + status: jest.fn().mockReturnThis(), + send: jest.fn() + } as unknown as Response; + const next = jest.fn() as NextFunction; + + await createApiMiddleware(createPrivateOptions())(req, res, next); + + expect(res.status).toHaveBeenCalledWith(403); + expect(res.send).toHaveBeenCalledWith('Forbidden'); + expect(next).not.toHaveBeenCalled(); + expect(mockAcquirePgPool).not.toHaveBeenCalled(); + }); + + it('releases the routing lease when schema validation fails', async () => { + const release = jest.fn(); + const query = jest.fn().mockRejectedValue(new Error('validation failed')); + mockAcquirePgPool.mockReturnValue( + leasePool({ query } as unknown as Pool, release) + ); + + const req = createRequest({ host: 'admin.localhost' }); + await expect(getApiConfig(createPrivateOptions(), req)).rejects.toThrow('validation failed'); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('releases tenant and routing leases when module resolution fails', async () => { + const query = jest.fn(async (_sql: string, params: unknown[]) => { + if (Array.isArray(params[0])) { + return { + rows: (params[0] as string[]).map((schemaName) => ({ + schema_name: schemaName + })) + }; + } + return { + rows: [{ + api_id: 'api-123', + database_id: 'db-123', + dbname: 'tenant_db', + role_name: 'api_role', + anon_role: 'api_anon', + is_public: false, + schemas: ['api_public'] + }] + }; + }); + const releaseOrder: string[] = []; + mockAcquirePgPool + .mockReturnValueOnce(leasePool( + { query } as unknown as Pool, + jest.fn(() => releaseOrder.push('routing')) + )) + .mockReturnValueOnce(leasePool( + { query } as unknown as Pool, + jest.fn(() => releaseOrder.push('tenant')) + )); + mockRegistryResolve.mockRejectedValueOnce(new Error('loader failed')); + const req = createRequest(withInternalAuth({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Api-Name': 'customer-api' + })); + + await expect(getApiConfig(createPrivateOptions(), req)).rejects.toThrow('loader failed'); + expect(releaseOrder).toEqual(['tenant', 'routing']); + }); + + it('forwards pool-capacity refusal to the shared HTTP error handler', async () => { + const capacityError = Object.assign(new Error('sensitive capacity details'), { + code: 'PG_POOL_CAPACITY' + }); + mockAcquirePgPool.mockImplementation(() => { + throw capacityError; + }); + const req = createRequest({ host: 'admin.localhost' }); + const res = { + status: jest.fn().mockReturnThis(), + send: jest.fn() + } as unknown as Response; + const next = jest.fn() as NextFunction; + + await createApiMiddleware(createPrivateOptions())(req, res, next); + + expect(next).toHaveBeenCalledWith(capacityError); + expect((res.status as jest.Mock)).not.toHaveBeenCalled(); }); }); diff --git a/graphql/server/src/middleware/__tests__/auth-pool-lease.test.ts b/graphql/server/src/middleware/__tests__/auth-pool-lease.test.ts new file mode 100644 index 0000000000..2ae5daa1c9 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/auth-pool-lease.test.ts @@ -0,0 +1,268 @@ +jest.mock('pg-cache', () => ({ + acquirePgPool: jest.fn(), + PG_POOL_CAPACITY_ERROR_CODE: 'PG_POOL_CAPACITY' +})); + +jest.mock('pg-query-context', () => ({ + __esModule: true, + default: jest.fn() +})); + +import type { PgpmOptions } from '@pgpmjs/types'; +import type { NextFunction, Request, Response } from 'express'; +import { acquirePgPool } from 'pg-cache'; +import pgQueryContext from 'pg-query-context'; + +import type { ApiStructure, RlsModule } from '../../types'; +import { createAuthenticateMiddleware } from '../auth'; + +const mockAcquirePgPool = acquirePgPool as jest.MockedFunction; +const mockPgQueryContext = pgQueryContext as jest.MockedFunction; + +const rlsModule: RlsModule = { + authenticate: 'authenticate', + authenticateStrict: 'authenticate_strict', + privateSchema: { schemaName: 'auth_private' }, + publicSchema: { schemaName: 'auth_public' }, + currentRole: 'current_role', + currentRoleId: 'current_role_id', + currentIpAddress: 'current_ip_address', + currentUserAgent: 'current_user_agent' +}; + +const api = (overrides: Partial = {}): ApiStructure => ({ + dbname: 'tenant_db', + anonRole: 'anonymous', + roleName: 'authenticated', + schema: ['app_public'], + databaseId: 'db-1', + rlsModule, + ...overrides +}); + +const request = ( + apiConfig: ApiStructure, + headers: Record = {} +): Request => { + const normalized = Object.fromEntries( + Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]) + ); + return { + api: apiConfig, + clientIp: '127.0.0.1', + headers: normalized, + get: jest.fn((name: string) => normalized[name.toLowerCase()]) + } as unknown as Request; +}; + +const response = (): Response => { + const res = { + status: jest.fn(), + json: jest.fn(), + send: jest.fn() + }; + res.status.mockReturnValue(res); + return res as unknown as Response; +}; + +const opts = { + pg: { + database: 'routing_db', + user: 'control_user' + }, + server: { + strictAuth: false + } +} as unknown as PgpmOptions; + +describe('authenticate middleware PostgreSQL pool leases', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it.each([ + ['an API without RLS', api({ rlsModule: undefined }), {}], + ['an anonymous request', api(), {}] + ])('does not allocate a tenant control pool for %s', async (_label, apiConfig, headers) => { + const req = request(apiConfig as ApiStructure, headers as Record); + const res = response(); + const next = jest.fn() as NextFunction; + + await createAuthenticateMiddleware(opts)(req, res, next); + + expect(mockAcquirePgPool).not.toHaveBeenCalled(); + expect(mockPgQueryContext).not.toHaveBeenCalled(); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('fails closed when the selected authentication function is absent', async () => { + const req = request(api({ + rlsModule: { ...rlsModule, authenticate: '' } + }), { authorization: 'Bearer credential' }); + const res = response(); + const next = jest.fn() as NextFunction; + + await createAuthenticateMiddleware(opts)(req, res, next); + + expect(mockAcquirePgPool).not.toHaveBeenCalled(); + expect(mockPgQueryContext).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(200); + expect(next).not.toHaveBeenCalled(); + }); + + it('fails closed when strict authentication cannot resolve an RLS module', async () => { + const strictOpts = { + ...opts, + server: { ...opts.server, strictAuth: true } + } as unknown as PgpmOptions; + const res = response(); + const next = jest.fn() as NextFunction; + + await createAuthenticateMiddleware(strictOpts)( + request(api({ rlsModule: undefined })), + res, + next + ); + + expect(mockAcquirePgPool).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(200); + expect(next).not.toHaveBeenCalled(); + }); + + it('quotes metadata-derived authentication identifiers', async () => { + const release = jest.fn(); + mockAcquirePgPool.mockReturnValue({ + pool: {} as never, + identity: 'pg:tenant-control', + release + }); + mockPgQueryContext.mockResolvedValue({ + rowCount: 1, + rows: [{ role: 'authenticated', user_id: 'user-1' }] + } as never); + const req = request(api({ + rlsModule: { + ...rlsModule, + privateSchema: { schemaName: 'auth";select pg_sleep(9);--' }, + authenticate: 'authenticate";drop schema public;--' + } + }), { authorization: 'Bearer credential' }); + + await createAuthenticateMiddleware(opts)( + req, + response(), + jest.fn() as NextFunction + ); + + expect(mockPgQueryContext).toHaveBeenCalledWith(expect.objectContaining({ + query: 'SELECT * FROM "auth"";select pg_sleep(9);--"."authenticate"";drop schema public;--"($1)', + variables: ['credential'] + })); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('leases the exact tenant control pool only for the credential query', async () => { + const release = jest.fn(); + const pool = { query: jest.fn() }; + mockAcquirePgPool.mockReturnValue({ + pool: pool as never, + identity: 'pg:tenant-control', + release + }); + mockPgQueryContext.mockResolvedValue({ + rowCount: 1, + rows: [{ role: 'authenticated', user_id: 'user-1' }] + } as never); + const req = request(api(), { authorization: 'Bearer credential' }); + const res = response(); + const next = jest.fn() as NextFunction; + + await createAuthenticateMiddleware(opts)(req, res, next); + + expect(mockAcquirePgPool).toHaveBeenCalledTimes(1); + expect(mockAcquirePgPool).toHaveBeenCalledWith( + expect.objectContaining({ + database: 'tenant_db', + user: 'control_user' + }), + { purpose: 'tenant-request-control', sanitizeOnCheckout: true } + ); + expect(mockPgQueryContext).toHaveBeenCalledWith(expect.objectContaining({ + client: pool, + query: 'SELECT * FROM "auth_private"."authenticate"($1)', + variables: ['credential'] + })); + expect(release).toHaveBeenCalledTimes(1); + expect(req.token).toEqual(expect.objectContaining({ user_id: 'user-1' })); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('omits an unavailable client IP from the credential-query context', async () => { + const release = jest.fn(); + mockAcquirePgPool.mockReturnValue({ + pool: {} as never, + identity: 'pg:tenant-control', + release + }); + mockPgQueryContext.mockResolvedValue({ + rowCount: 1, + rows: [{ role: 'authenticated', user_id: 'user-1' }] + } as never); + const req = request(api(), { authorization: 'Bearer credential' }); + req.clientIp = undefined; + + await createAuthenticateMiddleware( + opts + )(req, response(), jest.fn() as NextFunction); + + expect(mockPgQueryContext).toHaveBeenCalledWith(expect.objectContaining({ + context: expect.objectContaining({ + 'jwt.claims.ip_address': '', + 'jwt.claims.origin': '', + 'jwt.claims.user_agent': '', + 'jwt.claims.database_id': 'db-1', + 'row_security': 'on', + 'search_path': 'pg_catalog', + 'transaction_read_only': 'on' + }) + })); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('releases the tenant control pool when the credential query fails', async () => { + const release = jest.fn(); + mockAcquirePgPool.mockReturnValue({ + pool: {} as never, + identity: 'pg:tenant-control', + release + }); + mockPgQueryContext.mockRejectedValue(new Error('query failed')); + const req = request(api(), { authorization: 'Bearer credential' }); + const res = response(); + const next = jest.fn() as NextFunction; + + await createAuthenticateMiddleware(opts)(req, res, next); + + expect(release).toHaveBeenCalledTimes(1); + expect(res.status).toHaveBeenCalledWith(200); + expect(next).not.toHaveBeenCalled(); + }); + + it('forwards pool-capacity refusal to the shared HTTP error handler', async () => { + const capacityError = Object.assign(new Error('sensitive capacity details'), { + code: 'PG_POOL_CAPACITY' + }); + mockAcquirePgPool.mockImplementation(() => { + throw capacityError; + }); + const req = request(api(), { authorization: 'Bearer credential' }); + const res = response(); + const next = jest.fn() as NextFunction; + + await createAuthenticateMiddleware(opts)(req, res, next); + + expect(next).toHaveBeenCalledWith(capacityError); + expect(mockPgQueryContext).not.toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/captcha.test.ts b/graphql/server/src/middleware/__tests__/captcha.test.ts new file mode 100644 index 0000000000..f25ab7ddc8 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/captcha.test.ts @@ -0,0 +1,294 @@ +import express, { type NextFunction, type Request, type Response } from 'express'; +import supertest from 'supertest'; + +import type { ApiStructure } from '../../types'; +import { + createCaptchaGraphqlBodyParsers, + createCaptchaMiddleware, + inspectCaptchaOperation +} from '../captcha'; + +const api = (enableCaptcha: boolean): ApiStructure => ({ + dbname: 'tenant_db', + anonRole: 'anonymous', + roleName: 'authenticated', + schema: ['app_public'], + databaseId: 'database-a', + apiId: 'api-a', + authSettings: { + cookieSecure: true, + cookieSamesite: 'lax', + cookieDomain: null, + cookieHttponly: true, + cookieMaxAge: null, + cookiePath: '/', + rememberMeDuration: null, + enableCaptcha, + captchaSiteKey: null + } +}); + +interface RequestOptions { + operationName?: unknown; + enableCaptcha?: boolean; + headers?: Record; + method?: string; + path?: string; + body?: unknown; +} + +const request = ( + query: string, + options: RequestOptions = {} +): Request => { + const normalized = Object.fromEntries( + Object.entries(options.headers ?? {}) + .map(([name, value]) => [name.toLowerCase(), value]) + ); + return { + api: api(options.enableCaptcha ?? true), + body: options.body ?? { query, operationName: options.operationName }, + method: options.method ?? 'POST', + path: options.path ?? '/graphql', + query: {}, + get: jest.fn((name: string) => normalized[name.toLowerCase()]) + } as unknown as Request; +}; + +const response = (): Response => { + const res = { + status: jest.fn(), + json: jest.fn() + }; + res.status.mockReturnValue(res); + return res as unknown as Response; +}; + +describe('CAPTCHA GraphQL operation inspection', () => { + it.each([ + ['an arbitrary operation label', 'mutation Harmless { signUp }'], + ['a root field alias', 'mutation Harmless { allowed: resetPassword }'], + [ + 'a fragment spread', + 'mutation Harmless { ...Protected } fragment Protected on Mutation { signUpWithSms }' + ], + [ + 'an inline fragment', + 'mutation Harmless { ... on Mutation { requestPasswordReset } }' + ] + ])('finds a protected root mutation through %s', (_label, query) => { + expect(inspectCaptchaOperation(query, 'Harmless')).toEqual({ + kind: 'protected', + fields: expect.any(Array) + }); + }); + + it('uses operationName only to select one operation from a multi-operation document', () => { + const query = ` + query Safe { viewer { id } } + mutation Protected { signUp } + `; + + expect(inspectCaptchaOperation(query, 'Safe')).toEqual({ + kind: 'not-protected' + }); + expect(inspectCaptchaOperation(query, 'Protected')).toEqual({ + kind: 'protected', + fields: ['signUp'] + }); + expect(inspectCaptchaOperation(query, undefined)).toEqual({ + kind: 'invalid', + reason: 'ambiguous or missing GraphQL operation' + }); + }); + + it.each([ + ['malformed syntax', 'mutation {', undefined], + [ + 'a missing fragment', + 'mutation Protected { ...Missing }', + 'Protected' + ], + [ + 'a cyclic fragment', + `mutation Protected { ...A } + fragment A on Mutation { ...B } + fragment B on Mutation { ...A }`, + 'Protected' + ], + ['a missing selected operation', 'query Safe { viewer { id } }', 'Other'] + ])('fails closed for %s', (_label, query, operationName) => { + expect(inspectCaptchaOperation(query, operationName)).toEqual( + expect.objectContaining({ kind: 'invalid' }) + ); + }); +}); + +describe('captcha middleware admission', () => { + const originalSecret = process.env.RECAPTCHA_SECRET_KEY; + + beforeEach(() => { + delete process.env.RECAPTCHA_SECRET_KEY; + }); + + afterAll(() => { + if (originalSecret === undefined) { + delete process.env.RECAPTCHA_SECRET_KEY; + } else { + process.env.RECAPTCHA_SECRET_KEY = originalSecret; + } + }); + + it('fails closed in production when tenant policy enables CAPTCHA', async () => { + const req = request('mutation AnyName { signUp }'); + const res = response(); + const next = jest.fn() as NextFunction; + + await createCaptchaMiddleware({ nodeEnv: 'production' })(req, res, next); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith({ + errors: [{ + message: 'Something went wrong: authentication failed', + extensions: expect.objectContaining({ + code: 'INTERNAL_FAILURE', + http: 500 + }) + }] + }); + expect(next).not.toHaveBeenCalled(); + }); + + it('fails closed under strict authentication outside production', async () => { + const res = response(); + const next = jest.fn() as NextFunction; + + await createCaptchaMiddleware({ + strictAuth: true, + nodeEnv: 'development' + })(request('mutation Reset { requestPasswordReset }'), res, next); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ + errors: [expect.objectContaining({ + extensions: expect.objectContaining({ code: 'INTERNAL_FAILURE' }) + })] + })); + expect(next).not.toHaveBeenCalled(); + }); + + it('preserves the local non-strict compatibility behavior', async () => { + const res = response(); + const next = jest.fn() as NextFunction; + + await createCaptchaMiddleware({ + strictAuth: false, + nodeEnv: 'development' + })(request('mutation Register { signUp }'), res, next); + + expect(next).toHaveBeenCalledTimes(1); + expect(res.status).not.toHaveBeenCalled(); + }); + + it.each([ + [ + 'disabled tenant policy', + request('mutation Register { signUp }', { enableCaptcha: false }) + ], + ['an unprotected mutation', request('mutation Login { signIn }')], + ['a query', request('query Viewer { viewer { id } }')], + [ + 'a non-GraphQL route', + request('mutation Register { signUp }', { path: '/fn/register' }) + ], + [ + 'a WebSocket handshake', + request('', { + method: 'GET', + headers: { upgrade: 'websocket' } + }) + ] + ])('does not require a secret for %s', async (_label, req) => { + const res = response(); + const next = jest.fn() as NextFunction; + + await createCaptchaMiddleware({ + strictAuth: true, + nodeEnv: 'production' + })(req, res, next); + + expect(next).toHaveBeenCalledTimes(1); + expect(res.status).not.toHaveBeenCalled(); + }); + + it.each([ + ['a malformed document', request('mutation {')], + [ + 'an ambiguous document', + request('query A { viewer { id } } query B { viewer { id } }') + ], + ['a missing body', request('', { body: null })], + ['a batched body', request('', { body: [] })] + ])('fails closed before GraphQL for %s', async (_label, req) => { + const res = response(); + const next = jest.fn() as NextFunction; + + await createCaptchaMiddleware({ + strictAuth: false, + nodeEnv: 'development' + })(req, res, next); + + expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ + errors: [expect.objectContaining({ + extensions: expect.objectContaining({ code: 'INTERNAL_FAILURE' }) + })] + })); + expect(next).not.toHaveBeenCalled(); + }); + + it('still requires a CAPTCHA token when the secret is configured', async () => { + process.env.RECAPTCHA_SECRET_KEY = 'server-side-secret'; + const res = response(); + const next = jest.fn() as NextFunction; + + await createCaptchaMiddleware({ nodeEnv: 'production' })( + request('mutation Reset { resetPassword }'), + res, + next + ); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ + errors: [expect.objectContaining({ + extensions: expect.objectContaining({ code: 'CAPTCHA_REQUIRED' }) + })] + })); + expect(next).not.toHaveBeenCalled(); + }); + + it.each([ + ['application/json', { query: 'mutation Register { signUp }' }], + ['application/graphql', 'mutation Register { signUp }'], + [ + 'application/x-www-form-urlencoded', + 'query=mutation%20Register%20%7B%20signUp%20%7D' + ] + ])('parses and gates a real %s request before Grafserv', async (contentType, body) => { + const app = express(); + app.use((req, _res, next) => { + req.api = api(true); + next(); + }); + app.use('/graphql', ...createCaptchaGraphqlBodyParsers()); + app.use(createCaptchaMiddleware({ nodeEnv: 'production' })); + app.use((_req, res) => res.status(204).end()); + + const result = await supertest(app) + .post('/graphql') + .set('content-type', contentType) + .send(body); + + expect(result.status).toBe(200); + expect(result.body.errors[0].extensions.code).toBe('INTERNAL_FAILURE'); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/error-handler.test.ts b/graphql/server/src/middleware/__tests__/error-handler.test.ts new file mode 100644 index 0000000000..1b54a1421e --- /dev/null +++ b/graphql/server/src/middleware/__tests__/error-handler.test.ts @@ -0,0 +1,43 @@ +import type { NextFunction, Request, Response } from 'express'; + +import { errorHandler } from '../error-handler'; + +describe('shared error handler pool-capacity response', () => { + it('returns a stable retryable 503 without exposing capacity details', () => { + const req = { + requestId: 'request-1', + path: '/graphql', + method: 'POST', + get: jest.fn((name: string) => { + if (name === 'Accept') return 'text/html'; + if (name === 'host') return 'api.example.com'; + return undefined; + }) + } as unknown as Request; + const res = { + headersSent: false, + set: jest.fn(), + status: jest.fn(), + json: jest.fn(), + send: jest.fn() + }; + res.status.mockReturnValue(res); + const error = Object.assign( + new Error('PostgreSQL pool capacity exhausted: 2050/2064 and 2050 leased'), + { code: 'PG_POOL_CAPACITY' } + ); + + errorHandler(error, req, res as unknown as Response, jest.fn() as NextFunction); + + expect(res.set).toHaveBeenCalledWith('Retry-After', '15'); + expect(res.status).toHaveBeenCalledWith(503); + expect(res.json).toHaveBeenCalledWith({ + error: { + code: 'PG_POOL_CAPACITY', + message: 'Service temporarily unavailable', + requestId: 'request-1' + } + }); + expect(JSON.stringify(res.json.mock.calls)).not.toContain('2050'); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/flush-auth.test.ts b/graphql/server/src/middleware/__tests__/flush-auth.test.ts new file mode 100644 index 0000000000..cdb9d69cc6 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/flush-auth.test.ts @@ -0,0 +1,61 @@ +jest.mock('../graphile', () => ({ + invalidateInFlightBuilds: jest.fn() +})); + +import type { LoaderRegistry } from '@constructive-io/express-context'; +import type { NextFunction, Request, Response } from 'express'; + +import { createFlushMiddleware, flush } from '../flush'; + +const response = (): Response => ({ + status: jest.fn().mockReturnThis(), + send: jest.fn() +} as unknown as Response); + +describe('HTTP cache flush authorization', () => { + it('rejects an unauthenticated cache flush', async () => { + const req = { url: '/flush', internalTrusted: false } as Request; + const res = response(); + const next = jest.fn() as NextFunction; + + await flush(req, res, next); + + expect(res.status).toHaveBeenCalledWith(403); + expect(res.send).toHaveBeenCalledWith('Forbidden'); + expect(next).not.toHaveBeenCalled(); + }); + + it('allows a request already authenticated at the internal ingress boundary', async () => { + const req = { + url: '/flush', + internalTrusted: true, + svc_key: 'api.example.test' + } as Request; + const res = response(); + const next = jest.fn() as NextFunction; + + await flush(req, res, next); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.send).toHaveBeenCalledWith('OK'); + expect(next).not.toHaveBeenCalled(); + }); + + it('invalidates module metadata before acknowledging an authenticated flush', async () => { + const invalidate = jest.fn(); + const registry = { invalidate } as unknown as LoaderRegistry; + const req = { + url: '/flush', + internalTrusted: true, + svc_key: 'api.example.test', + databaseId: 'database-123' + } as Request; + const res = response(); + const next = jest.fn() as NextFunction; + + await createFlushMiddleware(registry)(req, res, next); + + expect(invalidate).toHaveBeenCalledWith('database-123'); + expect(res.status).toHaveBeenCalledWith(200); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/flush-pool-lease.test.ts b/graphql/server/src/middleware/__tests__/flush-pool-lease.test.ts new file mode 100644 index 0000000000..49731183b8 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/flush-pool-lease.test.ts @@ -0,0 +1,160 @@ +jest.mock('pg-cache', () => ({ + acquirePgPool: jest.fn(), + getPgPoolIdentity: jest.fn((config: { host?: string }) => + `pg:${config.host ?? 'control'}` + ) +})); + +jest.mock('graphile-cache', () => ({ + deleteGraphileCacheEntry: jest.fn().mockResolvedValue(true), + graphileCache: new Map() +})); + +jest.mock('../graphile', () => ({ + invalidateInFlightBuilds: jest.fn() +})); + +import type { ConstructiveOptions } from '@constructive-io/graphql-types'; +import { svcCache } from '@pgpmjs/server-utils'; +import { deleteGraphileCacheEntry, graphileCache } from 'graphile-cache'; +import { acquirePgPool } from 'pg-cache'; + +import { getSvcCacheKey } from '../api'; +import { flushService } from '../flush'; + +const mockAcquirePgPool = acquirePgPool as jest.MockedFunction; +const mockDeleteGraphileCacheEntry = deleteGraphileCacheEntry as jest.MockedFunction< + typeof deleteGraphileCacheEntry +>; + +describe('flushService PostgreSQL pool ownership', () => { + beforeEach(() => { + jest.clearAllMocks(); + graphileCache.clear(); + svcCache.clear(); + mockDeleteGraphileCacheEntry.mockImplementation(async (key) => graphileCache.delete(key)); + }); + + it.each([true, false])( + 'evicts database entries before a routing failure for isPublic=%s', + async (isPublic) => { + const queryFailure = new Error('routing query failed'); + const release = jest.fn(); + const order: string[] = []; + graphileCache.set('database-a-public', { + databaseId: 'database-a', + serviceKey: 'api:database-a:public' + } as never); + graphileCache.set('database-a-private', { + databaseId: 'database-a', + serviceKey: 'api:database-a:private' + } as never); + graphileCache.set('database-b', { + databaseId: 'database-b', + serviceKey: 'api:database-b:public' + } as never); + mockDeleteGraphileCacheEntry.mockImplementation(async (key) => { + order.push(`delete:${key}`); + return graphileCache.delete(key); + }); + mockAcquirePgPool.mockReturnValue({ + identity: 'pg:control', + pool: { + query: jest.fn().mockImplementation(async () => { + order.push('query'); + throw queryFailure; + }) + } as never, + release + }); + const options = { + pg: { database: 'routing' }, + api: { isPublic } + } as ConstructiveOptions; + + await expect(flushService(options, 'database-a')).rejects.toBe(queryFailure); + + expect(mockAcquirePgPool).toHaveBeenCalledWith( + { database: 'routing' }, + { purpose: 'routing-request-control', sanitizeOnCheckout: true } + ); + expect(mockDeleteGraphileCacheEntry).toHaveBeenCalledTimes(2); + expect(mockDeleteGraphileCacheEntry).toHaveBeenCalledWith('database-a-public'); + expect(mockDeleteGraphileCacheEntry).toHaveBeenCalledWith('database-a-private'); + expect(graphileCache.has('database-b')).toBe(true); + expect(order.indexOf('delete:database-a-public')).toBeLessThan(order.indexOf('query')); + expect(order.indexOf('delete:database-a-private')).toBeLessThan(order.indexOf('query')); + expect(release).toHaveBeenCalledTimes(1); + } + ); + + it.each([true, false])( + 'evicts database entries when routing has no domains for isPublic=%s', + async (isPublic) => { + const release = jest.fn(); + const order: string[] = []; + graphileCache.set('database-a', { + databaseId: 'database-a', + serviceKey: 'api:database-a:public' + } as never); + graphileCache.set('database-b', { + databaseId: 'database-b', + serviceKey: 'api:database-b:public' + } as never); + mockDeleteGraphileCacheEntry.mockImplementation(async (key) => { + order.push(`delete:${key}`); + return graphileCache.delete(key); + }); + mockAcquirePgPool.mockReturnValue({ + identity: 'pg:control', + pool: { + query: jest.fn().mockImplementation(async () => { + order.push('query'); + return { rows: [], rowCount: 0 }; + }) + } as never, + release + }); + const options = { + pg: { database: 'routing' }, + api: { isPublic } + } as ConstructiveOptions; + + await flushService(options, 'database-a'); + + expect(mockDeleteGraphileCacheEntry).toHaveBeenCalledTimes(1); + expect(mockDeleteGraphileCacheEntry).toHaveBeenCalledWith('database-a'); + expect(graphileCache.has('database-b')).toBe(true); + expect(order).toEqual(['delete:database-a', 'query']); + expect(release).toHaveBeenCalledTimes(1); + } + ); + + it('invalidates routing metadata only inside the exact control-pool contract', async () => { + const optsA = { + pg: { database: 'routing', host: 'routing-a.internal' }, + api: { isPublic: true } + } as ConstructiveOptions; + const optsB = { + pg: { database: 'routing', host: 'routing-b.internal' }, + api: { isPublic: true } + } as ConstructiveOptions; + const serviceKey = 'api.example.com'; + const keyA = getSvcCacheKey(optsA, serviceKey); + const keyB = getSvcCacheKey(optsB, serviceKey); + svcCache.set(keyA, { databaseId: 'database-a' }); + svcCache.set(keyB, { databaseId: 'database-a' }); + mockAcquirePgPool.mockReturnValue({ + identity: 'pg:routing-a.internal', + pool: { + query: jest.fn().mockResolvedValue({ rows: [], rowCount: 0 }) + } as never, + release: jest.fn() + }); + + await flushService(optsA, 'database-a'); + + expect(svcCache.has(keyA)).toBe(false); + expect(svcCache.peek(keyB)).toEqual({ databaseId: 'database-a' }); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-build-admission-response.test.ts b/graphql/server/src/middleware/__tests__/graphile-build-admission-response.test.ts new file mode 100644 index 0000000000..d1b34d6233 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-build-admission-response.test.ts @@ -0,0 +1,74 @@ +import type { Response } from 'express'; +import { + CacheBuildAdmissionError, + GraphileRealtimeStartupError +} from 'graphile-cache'; + +import { + GRAPHILE_BUILD_RESIDENT_CAPACITY_CODE, + handleBuildAvailabilityError +} from '../graphile'; +import { GraphileRealtimeNotificationConfigError } from '../realtime-notification-config'; + +describe('Graphile build admission responses', () => { + it('maps preserved resident capacity to a stable retryable 503', () => { + const response = { + destroyed: false, + writableEnded: false, + setHeader: jest.fn(), + status: jest.fn(), + json: jest.fn() + }; + response.status.mockReturnValue(response); + + expect(handleBuildAvailabilityError( + response as unknown as Response, + new CacheBuildAdmissionError('resident_capacity') + )).toBe(true); + expect(response.setHeader).toHaveBeenCalledWith('Retry-After', '15'); + expect(response.status).toHaveBeenCalledWith(503); + expect(response.json).toHaveBeenCalledWith({ + error: { + code: GRAPHILE_BUILD_RESIDENT_CAPACITY_CODE, + message: 'GraphQL schema capacity is temporarily unavailable' + } + }); + }); + + it.each([ + [ + new GraphileRealtimeNotificationConfigError('secret resolver detail'), + 'GRAPHILE_REALTIME_NOTIFICATION_CONFIG_INVALID', + 'Shared realtime notification configuration is unavailable' + ], + [ + new GraphileRealtimeStartupError('opaque-cache-key', new Error('secret startup detail')), + 'GRAPHILE_REALTIME_STARTUP_FAILED', + 'Realtime delivery could not be activated for this GraphQL instance' + ] + ])('maps realtime activation failures to credential-free stable 503s', ( + error, + code, + message + ) => { + const response = { + destroyed: false, + writableEnded: false, + setHeader: jest.fn(), + status: jest.fn(), + json: jest.fn() + }; + response.status.mockReturnValue(response); + + expect(handleBuildAvailabilityError( + response as unknown as Response, + error + )).toBe(true); + expect(response.setHeader).toHaveBeenCalledWith('Retry-After', '15'); + expect(response.status).toHaveBeenCalledWith(503); + expect(response.json).toHaveBeenCalledWith({ + error: { code, message } + }); + expect(JSON.stringify(response.json.mock.calls)).not.toContain('secret'); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-build-contract.test.ts b/graphql/server/src/middleware/__tests__/graphile-build-contract.test.ts new file mode 100644 index 0000000000..ab7d4b0925 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-build-contract.test.ts @@ -0,0 +1,296 @@ +import { + createGraphileBuildContract, + type CreateGraphileBuildContractInput, + hashGraphileBuildContract +} from '../graphile-build-contract'; + +const makeContract = (overrides: Partial = {}) => + createGraphileBuildContract({ + configurationIdentity: 'graphile-configuration:v1:test', + poolIdentity: 'pg:v1:abc', + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['tenant_a_public', 'tenant_a_private'], + authenticatedRole: 'tenant_user', + anonymousRole: 'tenant_anon', + graphiql: true, + graphiqlOnGraphQLGET: false, + pluginSettings: { + enableAggregates: false, + enablePostgis: true, + enableSearch: true, + enableDirectUploads: false, + enablePresignedUploads: true, + enableManyToMany: true, + enableConnectionFilter: true, + enableLtree: true, + enableLlm: true, + enableRealtime: false, + enableBulk: true, + enableI18n: true + }, + ...overrides + }); + +describe('GraphileBuildContractV1', () => { + it('is deterministic across object property ordering', () => { + const first = makeContract(); + const second = { ...makeContract(), roles: { ...makeContract().roles } }; + expect(hashGraphileBuildContract(first)).toBe(hashGraphileBuildContract(second)); + }); + + it('isolates every build-affecting tenant boundary', () => { + const base = makeContract(); + const variants = [ + { ...base, configurationIdentity: 'graphile-configuration:v1:other' }, + { ...base, poolIdentity: 'pg:v1:different' }, + { ...base, databaseId: 'database-b' }, + { ...base, apiId: 'api-b' }, + { ...base, schemas: [...base.schemas].reverse() }, + { ...base, roles: { ...base.roles, anonymous: 'different_anon' } }, + { + ...base, + storageModules: [{ + id: 'storage-a', + bucketsQualifiedName: '"tenant"."buckets"', + filesQualifiedName: '"tenant"."files"', + schemaName: 'tenant', + bucketsTableName: 'buckets', + filesTableName: 'files', + scope: 'app', + entityTableId: null, + entityQualifiedName: null, + endpoint: null, + publicUrlPrefix: null, + provider: null, + allowedOrigins: null, + uploadUrlExpirySeconds: 900, + downloadUrlExpirySeconds: 3600, + defaultMaxFileSize: 1024, + maxFilenameLength: 255, + cacheTtlSeconds: 300, + hasPathShares: false, + maxBulkFiles: 100, + maxBulkTotalSize: 1024 + }] + }, + { ...base, introspectionMode: 'scoped-required' as const }, + { ...base, introspectionClientReleaseMode: 'destroy' as const }, + { + ...base, + graphileSettings: { releaseBuildStateAfterValidation: true } + }, + { + ...base, + surface: { + ...base.surface, + enableRealtime: true, + realtimeSchema: 'tenant_a_realtime' + } + }, + { ...base, surface: { ...base.surface, graphiql: false } }, + { ...base, surface: { ...base.surface, graphiqlOnGraphQLGET: true } } + ]; + + for (const variant of variants) { + expect(hashGraphileBuildContract(variant)).not.toBe(hashGraphileBuildContract(base)); + } + }); + + it('accepts exact GraphiQL surface flags and preserves legacy defaults', () => { + const explicit = makeContract({ + graphiql: false, + graphiqlOnGraphQLGET: true + }); + expect(explicit.surface.graphiql).toBe(false); + expect(explicit.surface.graphiqlOnGraphQLGET).toBe(true); + + const legacy = createGraphileBuildContract({ + configurationIdentity: 'graphile-configuration:v1:legacy', + poolIdentity: 'pg:v1:legacy', + databaseId: 'database-legacy', + databaseName: 'tenant_legacy', + apiId: 'api-legacy', + schemas: ['tenant_legacy_public'], + authenticatedRole: 'tenant_user', + anonymousRole: 'tenant_anon' + }); + expect(legacy.surface.graphiql).toBe(true); + expect(legacy.surface.graphiqlOnGraphQLGET).toBe(false); + expect(legacy.surface.realtimeSchema).toBeNull(); + expect(legacy.introspectionClientReleaseMode).toBe('reuse'); + }); + + it('separates same-source plugin closures with different captured values', () => { + const pluginFactory = (tenantPolicy: string) => () => tenantPolicy; + const policyA = pluginFactory('policy-a'); + const policyB = pluginFactory('policy-b'); + + expect(policyA.toString()).toBe(policyB.toString()); + const first = makeContract({ + graphileSettings: { + preset: { schema: { policy: policyA } } as any + } + }); + const repeated = makeContract({ + graphileSettings: { + preset: { schema: { policy: policyA } } as any + } + }); + const distinctClosure = makeContract({ + graphileSettings: { + preset: { schema: { policy: policyB } } as any + } + }); + + expect(hashGraphileBuildContract(first)).toBe( + hashGraphileBuildContract(repeated) + ); + expect(hashGraphileBuildContract(first)).not.toBe( + hashGraphileBuildContract(distinctClosure) + ); + }); + + it('binds ordered caller plugin code and settings but ignores admission policy', () => { + const firstHook = () => 'first'; + const secondHook = () => 'second'; + const firstPlugin = { + name: 'FirstCallerPlugin', + version: '1.0.0', + schema: { hooks: { build: firstHook } } + }; + const secondPlugin = { + name: 'SecondCallerPlugin', + version: '2.0.0', + schema: { hooks: { build: secondHook } } + }; + const first = makeContract({ + graphileSettings: { + extends: [{ plugins: [firstPlugin, secondPlugin] } as any], + trustCallerPresetsInProduction: false + } + }); + const reordered = makeContract({ + graphileSettings: { + extends: [{ plugins: [secondPlugin, firstPlugin] } as any], + trustCallerPresetsInProduction: false + } + }); + const admissionOnly = makeContract({ + graphileSettings: { + extends: [{ plugins: [firstPlugin, secondPlugin] } as any], + trustCallerPresetsInProduction: true + } + }); + + expect(hashGraphileBuildContract(first)).not.toBe( + hashGraphileBuildContract(reordered) + ); + expect(hashGraphileBuildContract(first)).toBe( + hashGraphileBuildContract(admissionOnly) + ); + }); + + it('binds the effective realtime cursor schema only when realtime is enabled', () => { + const compatibilityDefault = makeContract({ enableRealtime: true }); + const explicitDefault = makeContract({ + enableRealtime: true, + realtimeSchema: 'realtime_public' + }); + const tenantScoped = makeContract({ + enableRealtime: true, + realtimeSchema: 'ctf_a_realtime' + }); + const disabled = makeContract({ + enableRealtime: false, + realtimeSchema: 'ignored_when_disabled' + }); + + expect(compatibilityDefault.surface.realtimeSchema).toBe('realtime_public'); + expect(hashGraphileBuildContract(compatibilityDefault)).toBe( + hashGraphileBuildContract(explicitDefault) + ); + expect(hashGraphileBuildContract(tenantScoped)).not.toBe( + hashGraphileBuildContract(compatibilityDefault) + ); + expect(disabled.surface.realtimeSchema).toBeNull(); + expect(disabled.surface.realtimeNotificationMode).toBeNull(); + expect(disabled.surface.realtimeCursorPollIntervalMs).toBeNull(); + }); + + it('binds shared transport identity, role TTL, and cursor timings', () => { + const first = makeContract({ + enableRealtime: true, + realtimeNotificationMode: 'shared-exact', + realtimeListenerPoolIdentity: 'pg-notification-broker:v1:first', + realtimeNotificationRoleRevalidationMs: 60_000, + realtimeCursorPollIntervalMs: 5_000, + realtimeCursorHeartbeatIntervalMs: 30_000 + }); + const changedIdentity = makeContract({ + enableRealtime: true, + realtimeNotificationMode: 'shared-exact', + realtimeListenerPoolIdentity: 'pg-notification-broker:v1:second', + realtimeNotificationRoleRevalidationMs: 60_000, + realtimeCursorPollIntervalMs: 5_000, + realtimeCursorHeartbeatIntervalMs: 30_000 + }); + const changedPolling = makeContract({ + enableRealtime: true, + realtimeNotificationMode: 'shared-exact', + realtimeListenerPoolIdentity: 'pg-notification-broker:v1:first', + realtimeNotificationRoleRevalidationMs: 60_000, + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 30_000 + }); + + expect(first.surface).toMatchObject({ + realtimeNotificationMode: 'shared-exact', + realtimeListenerPoolIdentity: 'pg-notification-broker:v1:first', + realtimeNotificationRoleRevalidationMs: 60_000, + realtimeCursorPollIntervalMs: 5_000, + realtimeCursorHeartbeatIntervalMs: 30_000 + }); + expect(hashGraphileBuildContract(first)).not.toBe( + hashGraphileBuildContract(changedIdentity) + ); + expect(hashGraphileBuildContract(first)).not.toBe( + hashGraphileBuildContract(changedPolling) + ); + }); + + it('rejects shared transport without an opaque listener pool identity', () => { + expect(() => makeContract({ + enableRealtime: true, + realtimeNotificationMode: 'shared-exact' + })).toThrow('requires an opaque listener pool identity'); + }); + + it('does not split realtime-disabled identities on an irrelevant cursor schema', () => { + const first = makeContract({ + enableRealtime: false, + graphileSettings: { + realtimeSchema: 'tenant_a_realtime', + realtimeNotificationMode: 'shared-exact', + realtimeNotificationRoleRevalidationMs: 1, + realtimeCursorPollIntervalMs: 1, + realtimeCursorHeartbeatIntervalMs: 1 + } + }); + const second = makeContract({ + enableRealtime: false, + graphileSettings: { + realtimeSchema: 'tenant_b_realtime', + realtimeNotificationMode: 'dedicated', + realtimeNotificationRoleRevalidationMs: 120_000, + realtimeCursorPollIntervalMs: 60_000, + realtimeCursorHeartbeatIntervalMs: 180_000 + } + }); + + expect(first.graphileSettings).toEqual({}); + expect(second.graphileSettings).toEqual({}); + expect(hashGraphileBuildContract(first)).toBe(hashGraphileBuildContract(second)); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-build-governor.test.ts b/graphql/server/src/middleware/__tests__/graphile-build-governor.test.ts new file mode 100644 index 0000000000..31267f25d0 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-build-governor.test.ts @@ -0,0 +1,215 @@ +import { + BuildCoordinator, + captureGraphileBuildGeneration, + closeGraphileBuildCoordinator, + getGraphileGovernorCounters, + GRAPHILE_BUILD_QUEUE_FULL_CODE, + GRAPHILE_BUILD_SHUTTING_DOWN_CODE, + GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE, + GraphileBuildCoordinatorError, + GraphileBuildWaitAbortedError, + isGraphileBuildGenerationCurrent, + reopenGraphileBuildCoordinator, + runGraphileBuild, + waitForGraphileBuild +} from '../graphile-build-governor'; + +const deferred = () => { + let resolve!: (value: T) => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +}; + +describe('Graphile build coordinator', () => { + it('serializes large builds process-wide', async () => { + const firstGate = deferred(); + const secondStarted = jest.fn(); + const first = runGraphileBuild(async () => { + await firstGate.promise; + return 'first'; + }); + const second = runGraphileBuild(async () => { + secondStarted(); + return 'second'; + }); + + await Promise.resolve(); + expect(secondStarted).not.toHaveBeenCalled(); + expect(getGraphileGovernorCounters().queueDepth).toBe(1); + firstGate.resolve(); + await expect(first).resolves.toBe('first'); + await expect(second).resolves.toBe('second'); + expect(secondStarted).toHaveBeenCalledTimes(1); + }); + + it('clears its wait timer when a build settles', async () => { + jest.useFakeTimers(); + try { + await expect(waitForGraphileBuild(Promise.resolve('ready'), 60_000)).resolves.toBe('ready'); + expect(jest.getTimerCount()).toBe(0); + } finally { + jest.useRealTimers(); + } + }); + + it('returns null with a stable timeout counter while the build continues', async () => { + jest.useFakeTimers(); + try { + const before = getGraphileGovernorCounters().buildWaitTimeouts; + const pending = deferred(); + const waiting = waitForGraphileBuild(pending.promise, 25); + await jest.advanceTimersByTimeAsync(25); + await expect(waiting).resolves.toBeNull(); + expect(getGraphileGovernorCounters().buildWaitTimeouts).toBe(before + 1); + pending.resolve('eventually cached'); + } finally { + jest.useRealTimers(); + } + }); + + it('bounds queued builds with a stable refusal code', async () => { + const coordinator = new BuildCoordinator(1, 1); + const releaseFirst = await coordinator.acquire(); + const queued = coordinator.acquire(); + + await expect(coordinator.acquire()).rejects.toMatchObject({ + code: GRAPHILE_BUILD_QUEUE_FULL_CODE, + retryAfterSeconds: 1 + } satisfies Partial); + + releaseFirst(); + const releaseQueued = await queued; + releaseQueued(); + }); + + it('removes an aborted waiter before it can start', async () => { + const coordinator = new BuildCoordinator(1, 1); + const releaseFirst = await coordinator.acquire(); + const abortController = new AbortController(); + const admitted = jest.fn(); + const queued = coordinator.acquire({ + signal: abortController.signal, + onAdmitted: admitted + }); + expect(coordinator.queueDepth).toBe(1); + + abortController.abort(); + await expect(queued).rejects.toBeInstanceOf(GraphileBuildWaitAbortedError); + expect(coordinator.queueDepth).toBe(0); + releaseFirst(); + expect(admitted).not.toHaveBeenCalled(); + }); + + it('aborts request waiting without canceling an already active build', async () => { + const pending = deferred(); + const abortController = new AbortController(); + const waiting = waitForGraphileBuild( + pending.promise, + 60_000, + abortController.signal + ); + + abortController.abort(); + await expect(waiting).rejects.toBeInstanceOf(GraphileBuildWaitAbortedError); + pending.resolve('still allowed to finish'); + await expect(pending.promise).resolves.toBe('still allowed to finish'); + }); + + it('rejects unsafe concurrent-build configuration', () => { + expect(() => new BuildCoordinator(2, 1)).toThrow( + 'GRAPHILE_BUILD_CONCURRENCY must be exactly 1' + ); + }); + + it('latches an unhealthy restart-required state without releasing the active slot', async () => { + jest.useFakeTimers(); + try { + const coordinator = new BuildCoordinator(1, 1, 25); + const queuedAdmitted = jest.fn(); + const releaseActive = await coordinator.acquire(); + const queued = coordinator.acquire({ onAdmitted: queuedAdmitted }); + const queuedRefusal = expect(queued).rejects.toMatchObject({ + code: GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE, + retryAfterSeconds: 30 + } satisfies Partial); + + await jest.advanceTimersByTimeAsync(25); + await queuedRefusal; + + expect(coordinator.isUnhealthy).toBe(true); + expect(coordinator.stuckSinceMs).not.toBeNull(); + expect(coordinator.activeCount).toBe(1); + expect(coordinator.queueDepth).toBe(0); + expect(queuedAdmitted).not.toHaveBeenCalled(); + await expect(coordinator.acquire()).rejects.toMatchObject({ + code: GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE + } satisfies Partial); + + // Only completion of the real build may release its slot. The unhealthy + // latch remains, so the same process still cannot admit replacement work. + releaseActive(); + expect(coordinator.activeCount).toBe(0); + expect(coordinator.isUnhealthy).toBe(true); + await expect(coordinator.acquire()).rejects.toMatchObject({ + code: GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE + } satisfies Partial); + } finally { + jest.useRealTimers(); + } + }); + + it('notifies active request waiters when the build watchdog trips', async () => { + jest.useFakeTimers(); + try { + const coordinator = new BuildCoordinator(1, 0, 10); + const release = await coordinator.acquire(); + const stuck = new Promise((resolve) => { + coordinator.onStuck(resolve); + }); + + await jest.advanceTimersByTimeAsync(10); + await expect(stuck).resolves.toMatchObject({ + code: GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE + }); + expect(coordinator.activeCount).toBe(1); + release(); + } finally { + jest.useRealTimers(); + } + }); + + it('rejects an invalid build watchdog configuration', () => { + expect(() => new BuildCoordinator(1, 1, 0)).toThrow( + 'GRAPHILE_BUILD_WATCHDOG_MS must be a positive safe integer' + ); + }); + + it('closes admission, rejects queued work, drains active work, and advances generation', async () => { + const firstGate = deferred(); + const queuedStarted = jest.fn(); + const generation = captureGraphileBuildGeneration(); + const first = runGraphileBuild(async () => { + await firstGate.promise; + return 'first'; + }); + const queued = runGraphileBuild(async () => { + queuedStarted(); + return 'queued'; + }); + await Promise.resolve(); + + const draining = closeGraphileBuildCoordinator(100); + await expect(queued).rejects.toMatchObject({ + code: GRAPHILE_BUILD_SHUTTING_DOWN_CODE + } satisfies Partial); + expect(isGraphileBuildGenerationCurrent(generation)).toBe(false); + firstGate.resolve(); + await expect(first).resolves.toBe('first'); + await expect(draining).resolves.toBe(true); + expect(queuedStarted).not.toHaveBeenCalled(); + expect(reopenGraphileBuildCoordinator()).toBe(true); + await expect(runGraphileBuild(async () => 'restarted')).resolves.toBe('restarted'); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-internal-claims.test.ts b/graphql/server/src/middleware/__tests__/graphile-internal-claims.test.ts new file mode 100644 index 0000000000..3110724de9 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-internal-claims.test.ts @@ -0,0 +1,77 @@ +import type { Request } from 'express'; + +import { getTrustedInternalClaims } from '../internal-request'; + +const request = ({ + isPublic, + internalTrusted, + userId, + headers = {} +}: { + isPublic: boolean; + internalTrusted: boolean; + userId?: string; + headers?: Record; +}): Request => { + const normalized = new Map( + Object.entries(headers).map(([name, value]) => [name.toLowerCase(), value]) + ); + return { + api: { + dbname: 'tenant', + schema: ['app_public'], + anonRole: 'api_anon', + roleName: 'api_role', + isPublic + }, + internalTrusted, + token: userId ? { user_id: userId } : undefined, + get: jest.fn((name: string) => normalized.get(name.toLowerCase())) + } as unknown as Request; +}; + +describe('private ingress actor claims', () => { + const actorHeaders = { + 'X-Actor-Id': 'actor-a', + 'X-Entity-Id': 'entity-a', + 'X-Organization-Id': 'organization-a' + }; + + it('does not trust actor headers from the public ingress', () => { + expect(getTrustedInternalClaims(request({ + isPublic: true, + internalTrusted: true, + headers: actorHeaders + }))).toEqual({}); + }); + + it('does not trust actor headers without internal request authentication', () => { + expect(getTrustedInternalClaims(request({ + isPublic: false, + internalTrusted: false, + headers: actorHeaders + }))).toEqual({}); + }); + + it('lets an authenticated user token outrank internal actor headers', () => { + expect(getTrustedInternalClaims(request({ + isPublic: false, + internalTrusted: true, + userId: 'token-user', + headers: actorHeaders + }))).toEqual({}); + }); + + it('maps authenticated private actor headers onto the exact claim allowlist', () => { + expect(getTrustedInternalClaims(request({ + isPublic: false, + internalTrusted: true, + headers: actorHeaders + }))).toEqual({ + 'jwt.claims.user_id': 'actor-a', + 'jwt.claims.principal_id': 'actor-a', + 'jwt.claims.entity_id': 'entity-a', + 'jwt.claims.organization_id': 'organization-a' + }); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-pool-lease-publication.test.ts b/graphql/server/src/middleware/__tests__/graphile-pool-lease-publication.test.ts new file mode 100644 index 0000000000..a5a3fec66d --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-pool-lease-publication.test.ts @@ -0,0 +1,222 @@ +import type { GraphileCacheEntry } from 'graphile-cache'; +import type { PgPoolLease } from 'pg-cache'; + +import { + GraphileBuildPoolLeaseOwner, + GraphileBuildPublicationError, + publishGraphileBuild +} from '../graphile'; + +const lease = (identity = 'pg:runtime') => { + const release = jest.fn(); + return { + value: { + identity, + pool: {} as PgPoolLease['pool'], + release + } as PgPoolLease, + release + }; +}; + +const entry = ( + cacheKey: string, + overrides: Partial = {} +): GraphileCacheEntry => ({ + cacheKey, + poolIdentity: overrides.poolLease?.identity, + createdAt: Date.now(), + ...overrides +} as GraphileCacheEntry); + +describe('Graphile build PostgreSQL pool-lease publication', () => { + it('keeps the lease with the build until a matching entry accepts ownership', () => { + const retained = lease(); + const owner = new GraphileBuildPoolLeaseOwner(retained.value); + const candidate = entry('build-a', { + poolLease: retained.value, + poolIdentity: retained.value.identity + }); + + owner.transferTo(candidate); + owner.release(); + owner.release(); + + expect(retained.release).not.toHaveBeenCalled(); + candidate.poolLease?.release(); + expect(retained.release).toHaveBeenCalledTimes(1); + }); + + it('releases an untransferred build lease exactly once', () => { + const retained = lease(); + const owner = new GraphileBuildPoolLeaseOwner(retained.value); + + expect(() => owner.transferTo(entry('build-a'))).toThrow( + 'did not retain the build pool lease' + ); + owner.release(); + owner.release(); + + expect(retained.release).toHaveBeenCalledTimes(1); + }); + + it('leaves identity-mismatch cleanup to the entry that received the lease', () => { + const retained = lease(); + const owner = new GraphileBuildPoolLeaseOwner(retained.value); + const candidate = entry('build-a', { + poolLease: retained.value, + poolIdentity: 'pg:wrong' + }); + + expect(() => owner.transferTo(candidate)).toThrow('unexpected pool identity'); + owner.release(); + expect(retained.release).not.toHaveBeenCalled(); + candidate.poolLease?.release(); + expect(retained.release).toHaveBeenCalledTimes(1); + }); + + it('publishes one candidate without disposing its retained lease', async () => { + const values = new Map(); + const cache = { + get: jest.fn((key: string) => values.get(key)), + set: jest.fn((key: string, value: GraphileCacheEntry) => values.set(key, value)), + delete: jest.fn((key: string) => values.delete(key)) + }; + const dispose = jest.fn(async (): Promise => undefined); + const candidate = entry('build-a'); + + await expect(publishGraphileBuild('build-a', candidate, false, { + cache, + dispose + })).resolves.toBe(candidate); + expect(cache.set).toHaveBeenCalledTimes(1); + expect(dispose).not.toHaveBeenCalled(); + }); + + it('disposes a candidate when cache publication throws', async () => { + const candidate = entry('build-a'); + const dispose = jest.fn(async (): Promise => undefined); + const cache = { + get: jest.fn((): GraphileCacheEntry | undefined => undefined), + set: jest.fn(() => { + throw new Error('set failed'); + }), + delete: jest.fn(() => false) + }; + + await expect(publishGraphileBuild('build-a', candidate, false, { + cache, + dispose + })).rejects.toMatchObject({ + code: 'GRAPHILE_BUILD_PUBLICATION_FAILED' + } satisfies Partial); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it('disposes an unexpected duplicate and returns the authoritative resident', async () => { + const candidate = entry('build-a'); + const resident = entry('build-a'); + const dispose = jest.fn(async (): Promise => undefined); + const cache = { + get: jest.fn(() => resident), + set: jest.fn(), + delete: jest.fn(() => false) + }; + + await expect(publishGraphileBuild('build-a', candidate, false, { + cache, + dispose + })).resolves.toBe(resident); + expect(cache.set).not.toHaveBeenCalled(); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it('disposes a candidate replaced during publication and returns the stable replacement', async () => { + const candidate = entry('build-a'); + const resident = entry('build-a'); + const dispose = jest.fn(async (): Promise => undefined); + let reads = 0; + const cache = { + get: jest.fn((): GraphileCacheEntry | undefined => { + reads++; + return reads === 1 ? undefined : resident; + }), + set: jest.fn(), + delete: jest.fn(() => false) + }; + + await expect(publishGraphileBuild('build-a', candidate, false, { + cache, + dispose + })).resolves.toBe(resident); + expect(cache.set).toHaveBeenCalledTimes(1); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it('disposes an invalidated candidate before rejecting the build', async () => { + const candidate = entry('build-a'); + const dispose = jest.fn(async (): Promise => undefined); + + await expect(publishGraphileBuild('build-a', candidate, true, { + cache: { get: jest.fn(), set: jest.fn(), delete: jest.fn() }, + dispose + })).rejects.toMatchObject({ code: 'GRAPHILE_BUILD_INVALIDATED' }); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it('rejects a candidate that failed before publication', async () => { + const candidate = entry('build-a', { + disposing: true, + realtimeHealth: { + status: 'failed', + failureCode: 'INSUFFICIENT_PRIVILEGE', + failedAt: Date.now() + } + }); + const dispose = jest.fn(async (): Promise => undefined); + const cache = { + get: jest.fn((): GraphileCacheEntry | undefined => undefined), + set: jest.fn(), + delete: jest.fn(() => false) + }; + + await expect(publishGraphileBuild('build-a', candidate, false, { + cache, + dispose + })).rejects.toMatchObject({ + code: 'GRAPHILE_BUILD_PUBLICATION_FAILED' + }); + expect(cache.set).not.toHaveBeenCalled(); + expect(cache.delete).not.toHaveBeenCalled(); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it('removes and disposes a candidate that fails during publication', async () => { + const candidate = entry('build-a'); + const values = new Map(); + const dispose = jest.fn(async (): Promise => undefined); + const cache = { + get: jest.fn((key: string) => values.get(key)), + set: jest.fn((key: string, value: GraphileCacheEntry) => { + values.set(key, value); + value.realtimeHealth = { + status: 'failed', + failureCode: 'INSUFFICIENT_PRIVILEGE', + failedAt: Date.now() + }; + }), + delete: jest.fn((key: string) => values.delete(key)) + }; + + await expect(publishGraphileBuild('build-a', candidate, false, { + cache, + dispose + })).rejects.toMatchObject({ + code: 'GRAPHILE_BUILD_PUBLICATION_FAILED' + }); + expect(cache.set).toHaveBeenCalledTimes(1); + expect(cache.delete).toHaveBeenCalledWith('build-a'); + expect(values.has('build-a')).toBe(false); + expect(dispose).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-preset-composition.test.ts b/graphql/server/src/middleware/__tests__/graphile-preset-composition.test.ts new file mode 100644 index 0000000000..b45f15d7cd --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-preset-composition.test.ts @@ -0,0 +1,188 @@ +import type { GraphileConfig } from 'graphile-config'; +import { resolvePreset } from 'graphile-config'; + +import { + composeGraphilePreset, + GRAPHILE_CALLER_PRESET_NOT_TRUSTED_CODE, + GRAPHILE_PROTECTED_PRESET_OVERRIDE_CODE, + GraphileCallerPresetNotTrustedError, + GraphileProtectedPresetOverrideError +} from '../graphile-preset-composition'; + +const callerPlugin: GraphileConfig.Plugin = { + name: 'CallerSchemaPlugin', + version: '1.0.0' +}; + +const authPlugin: GraphileConfig.Plugin = { + name: 'AuthCookiePlugin', + version: '1.0.0' +}; + +const websocketAdmissionPlugin: GraphileConfig.Plugin = { + name: 'ConstructiveWebSocketOperationAdmissionPlugin', + version: '1.0.0' +}; + +const exactService = { + name: 'main', + adaptor: 'constructive-test-adaptor' +} as unknown as NonNullable[number]; + +const protectedContext = jest.fn(() => ({ + pgSettings: { role: 'tenant_runtime' } +})); +const protectedMaskError = jest.fn((error) => error); + +const compose = ( + overrides: Partial[0]> = {} +): GraphileConfig.Preset => composeGraphilePreset({ + basePresets: [], + callerPresetsTrusted: true, + protectedPlugins: [authPlugin, websocketAdmissionPlugin], + pgServices: [exactService], + schema: { releaseBuildStateAfterValidation: true }, + grafserv: { + graphqlPath: '/graphql', + graphiqlPath: '/graphiql', + graphiql: true, + graphiqlOnGraphQLGET: false, + websockets: true, + maskError: protectedMaskError + }, + grafast: { + context: protectedContext, + explain: false + }, + ...overrides +}); + +describe('Graphile caller preset composition', () => { + it('rejects every non-empty caller preset unless startup admitted it as trusted code', () => { + for (const input of [ + { callerExtends: [{ plugins: [callerPlugin] }] }, + { callerPreset: { schema: { defaultBehavior: '-delete' } } } + ]) { + expect(() => compose({ + ...input, + callerPresetsTrusted: false + })).toThrow(expect.objectContaining({ + code: GRAPHILE_CALLER_PRESET_NOT_TRUSTED_CODE + })); + } + }); + + it('allows empty defaults without widening the production trust boundary', () => { + expect(() => compose({ + callerExtends: [], + callerPreset: {}, + callerPresetsTrusted: false + })).not.toThrow(); + }); + + it('reports caller trust admission as a startup configuration error', () => { + expect(() => compose({ + callerExtends: [{ plugins: [callerPlugin] }], + callerPresetsTrusted: false + })).toThrow(GraphileCallerPresetNotTrustedError); + }); + + it('applies caller plugins and safe schema/runtime configuration', () => { + const resolved = resolvePreset(compose({ + callerExtends: [{ plugins: [callerPlugin] }], + callerPreset: { + schema: { defaultBehavior: '-delete' }, + grafserv: { maxRequestLength: 123_456 } + } + })); + + expect(resolved.plugins).toEqual(expect.arrayContaining([ + callerPlugin, + authPlugin, + websocketAdmissionPlugin + ])); + expect(resolved.schema).toMatchObject({ + defaultBehavior: '-delete', + releaseBuildStateAfterValidation: true + }); + expect(resolved.grafserv).toMatchObject({ + maxRequestLength: 123_456, + graphqlPath: '/graphql', + websockets: true, + maskError: protectedMaskError + }); + expect(resolved.grafast).toMatchObject({ + context: protectedContext, + explain: false + }); + expect(resolved.pgServices).toEqual([exactService]); + }); + + it.each([ + [ + 'runtime service', + { pgServices: [{ name: 'attacker' }] }, + 'pgServices' + ], + [ + 'tenant context', + { grafast: { context: () => ({ pgSettings: {} }) } }, + 'grafast.context' + ], + [ + 'error masking', + { grafserv: { maskError: (error: unknown) => error } }, + 'grafserv.maskError' + ], + [ + 'WebSocket transport', + { grafserv: { websockets: false } }, + 'grafserv.websockets' + ], + [ + 'server build-state policy', + { schema: { releaseBuildStateAfterValidation: false } }, + 'schema.releaseBuildStateAfterValidation' + ], + [ + 'protected plugin replacement', + { plugins: [{ name: 'AuthCookiePlugin' }] }, + 'plugins.AuthCookiePlugin' + ], + [ + 'protected plugin disable', + { disablePlugins: ['ConstructiveWebSocketOperationAdmissionPlugin'] }, + 'disablePlugins.ConstructiveWebSocketOperationAdmissionPlugin' + ] + ])('rejects caller %s overrides', (_label, callerPreset, protectedSetting) => { + let thrown: unknown; + try { + compose({ + callerPreset: callerPreset as unknown as GraphileConfig.Preset + }); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(GraphileProtectedPresetOverrideError); + expect(thrown).toMatchObject({ + code: GRAPHILE_PROTECTED_PRESET_OVERRIDE_CODE, + presetPath: 'graphile.preset', + protectedSetting + }); + }); + + it('rejects protected overrides hidden in nested caller extends', () => { + expect(() => compose({ + callerExtends: [{ + extends: [{ + pgServices: [{ name: 'attacker' }] + } as unknown as GraphileConfig.Preset] + }] + })).toThrow(expect.objectContaining({ + code: GRAPHILE_PROTECTED_PRESET_OVERRIDE_CODE, + presetPath: 'graphile.extends[0].extends[0]', + protectedSetting: 'pgServices' + })); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-request-terminal.test.ts b/graphql/server/src/middleware/__tests__/graphile-request-terminal.test.ts new file mode 100644 index 0000000000..e7d998d347 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-request-terminal.test.ts @@ -0,0 +1,142 @@ +import { EventEmitter } from 'node:events'; + +import type { Request, Response } from 'express'; + +import { isGraphileWebSocketOriginAllowed } from '../../websocket-upgrade'; +import { + getGraphileTransportRequest, + isGraphileRequestTerminal +} from '../graphile'; + +const makeRequest = (overrides: Record = {}): Request => + Object.assign(new EventEmitter(), { + aborted: false, + destroyed: false, + readableEnded: false, + complete: false, + socket: { destroyed: false }, + ...overrides + }) as unknown as Request; + +const makeResponse = (overrides: Record = {}): Response => + Object.assign(new EventEmitter(), { + destroyed: false, + writableEnded: false, + ...overrides + }) as unknown as Response; + +describe('Graphile request terminal detection', () => { + it('keeps serving a parsed POST whose consumed request stream was auto-destroyed', () => { + const request = makeRequest({ + destroyed: true, + readableEnded: true, + complete: true + }); + + expect(isGraphileRequestTerminal(request, makeResponse())).toBe(false); + }); + + it.each([ + ['request aborted', { aborted: true }, {}], + ['socket destroyed', { socket: { destroyed: true } }, {}], + ['response destroyed', {}, { destroyed: true }], + ['response ended', {}, { writableEnded: true }] + ])('detects a terminal %s', (_label, request, response) => { + expect(isGraphileRequestTerminal( + makeRequest(request), + makeResponse(response) + )).toBe(true); + }); +}); + +describe('Graphile transport request identity', () => { + it('uses the already-routed and authenticated request for WebSocket execution', () => { + const request = makeRequest({ + api: { databaseId: 'database-a', schema: ['a_public'] }, + token: { user_id: 'actor-a' } + }); + + expect(getGraphileTransportRequest({ + ws: { request } + } as unknown as Partial)).toBe(request); + }); + + it('keeps the existing Express request path for HTTP execution', () => { + const request = makeRequest({ + api: { databaseId: 'database-b', schema: ['b_public'] }, + token: { user_id: 'actor-b' } + }); + + expect(getGraphileTransportRequest({ + expressv4: { req: request } + } as unknown as Partial)).toBe(request); + }); +}); + +describe('Graphile WebSocket origin policy', () => { + const originRequest = ( + headers: Record, + corsOrigins: string[] = [] + ): Request => makeRequest({ + headers, + api: { + databaseId: 'database-a', + dbname: 'tenant_a', + schema: ['a_public'], + anonRole: 'tenant_anon', + roleName: 'tenant_user', + domains: [], + corsOrigins + }, + get(name: string) { + return headers[name.toLowerCase()]; + } + }); + + it('allows same-host and configured browser origins', () => { + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test', + origin: 'https://a.example.test', + cookie: 'constructive_session=session-a' + }))).toBe(true); + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test', + origin: 'https://console.example.test', + cookie: 'constructive_session=session-a' + }, ['https://console.example.test']))).toBe(true); + }); + + it('does not authorize cookie WebSockets through wildcard or localhost shortcuts', () => { + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test', + origin: 'https://attacker.example.test', + cookie: 'constructive_session=session-a' + }), '*')).toBe(false); + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'api.localhost:3000', + origin: 'http://attacker.localhost:3001', + cookie: 'constructive_session=session-a' + }))).toBe(false); + }); + + it('rejects a cross-origin browser and originless cookie authentication', () => { + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test', + origin: 'https://attacker.example.test' + }))).toBe(false); + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test', + cookie: 'constructive_session=session-a' + }))).toBe(false); + }); + + it('allows originless bearer and anonymous non-browser clients', () => { + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test', + authorization: 'Bearer token-a' + }))).toBe(true); + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test' + }))).toBe(true); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/internal-request.test.ts b/graphql/server/src/middleware/__tests__/internal-request.test.ts new file mode 100644 index 0000000000..8665a07a6c --- /dev/null +++ b/graphql/server/src/middleware/__tests__/internal-request.test.ts @@ -0,0 +1,167 @@ +import type { Request } from 'express'; + +import type { ApiOptions } from '../../types'; +import { + assertInternalRequestSecret, + authorizeInternalRequest, + INTERNAL_REQUEST_TOKEN_HEADER +} from '../internal-request'; + +const SECRET = '0123456789abcdef0123456789abcdef'; + +const request = (headers: Record): Request => { + const normalized = new Map( + Object.entries(headers).map(([name, value]) => [name.toLowerCase(), value]) + ); + return { + get: jest.fn((name: string) => normalized.get(name.toLowerCase())) + } as unknown as Request; +}; + +const options = (isPublic: boolean, secret: string | undefined = SECRET): ApiOptions => ({ + api: { + isPublic, + ...(secret === undefined ? {} : { internalRequestSecret: secret }) + } +} as ApiOptions); + +describe('internal request boundary', () => { + it('allows ordinary requests without granting internal trust', () => { + const req = request({ host: 'api.example.com' }); + + authorizeInternalRequest(options(true), req); + + expect(req.internalTrusted).toBe(false); + }); + + it('authenticates a token-only administrative request in constant-time path', () => { + const req = request({ [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET }); + + authorizeInternalRequest(options(true), req); + + expect(req.internalTrusted).toBe(true); + }); + + it('rejects private actor claims without the internal token', () => { + const req = request({ 'X-Actor-Id': 'actor-a' }); + + expect(() => authorizeInternalRequest(options(false), req)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + expect(req.internalTrusted).toBe(false); + }); + + it('accepts private API and actor selectors only with the exact token', () => { + const req = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Api-Name': 'api-a', + 'X-Actor-Id': 'actor-a' + }); + + authorizeInternalRequest(options(false), req); + + expect(req.internalTrusted).toBe(true); + }); + + it('rejects private selectors on a public ingress even with the exact token', () => { + const req = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Api-Name': 'api-a' + }); + + expect(() => authorizeInternalRequest(options(true), req)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + expect(req.internalTrusted).toBe(false); + }); + + it('always rejects caller-supplied physical schemas', () => { + const req = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Schemata': 'tenant_b_public' + }); + + expect(() => authorizeInternalRequest(options(false), req)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + expect(req.internalTrusted).toBe(false); + }); + + it('rejects the privileged metadata surface unless explicitly enabled', () => { + const req = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Meta-Schema': 'true' + }); + + expect(() => authorizeInternalRequest(options(false), req)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + expect(req.internalTrusted).toBe(false); + }); + + it('allows the privileged metadata surface only under an explicit private-ingress gate', () => { + const opts = options(false); + opts.api!.allowMetaSchemaHeader = true; + const req = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Meta-Schema': 'true' + }); + + authorizeInternalRequest(opts, req); + + expect(req.internalTrusted).toBe(true); + }); + + it('rejects missing and conflicting private-selector identities', () => { + const opts = options(false); + opts.api!.allowMetaSchemaHeader = true; + const missingDatabase = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Api-Name': 'api-a' + }); + const conflicting = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Api-Name': 'api-a', + 'X-Meta-Schema': 'true' + }); + + expect(() => authorizeInternalRequest(opts, missingDatabase)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + expect(() => authorizeInternalRequest(opts, conflicting)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + }); + + it('rejects empty reserved routing and identity values', () => { + const blankApi = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Api-Name': ' ' + }); + const blankActor = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Actor-Id': '' + }); + + expect(() => authorizeInternalRequest(options(false), blankApi)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + expect(() => authorizeInternalRequest(options(false), blankActor)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + }); + + it('rejects short configured secrets at startup', () => { + expect(() => assertInternalRequestSecret(options(false, 'too-short'))).toThrow( + 'at least 32 bytes' + ); + expect(() => assertInternalRequestSecret(options(false, undefined))).not.toThrow(); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/pg-introspection-memo-contract.test.ts b/graphql/server/src/middleware/__tests__/pg-introspection-memo-contract.test.ts new file mode 100644 index 0000000000..0ccf64ee5d --- /dev/null +++ b/graphql/server/src/middleware/__tests__/pg-introspection-memo-contract.test.ts @@ -0,0 +1,140 @@ +import { parseIntrospectionResults } from 'pg-introspection'; + +function makeIntrospectionText(): string { + return JSON.stringify({ + database: { + _id: '1', + oid: '1', + datname: 'memo_contract', + datdba: '10', + datacl: null + }, + namespaces: [{ + _id: '2200', + oid: '2200', + nspname: 'tenant_api', + nspowner: '10', + nspacl: null + }], + classes: [], + attributes: [], + constraints: [], + procs: [], + roles: [{ + _id: '10', + oid: '10', + rolname: 'runtime_role', + rolsuper: false, + rolinherit: false, + rolcreaterole: false, + rolcreatedb: false, + rolcanlogin: true, + rolreplication: false, + rolconnlimit: -1, + rolpassword: null, + rolvaliduntil: null, + rolbypassrls: false, + rolconfig: null + }], + auth_members: [], + types: [], + enums: [], + extensions: [], + indexes: [], + inherits: [], + languages: [], + policies: [], + ranges: [], + depends: [], + descriptions: [], + am: [], + catalog_by_oid: { + 1247: 'pg_type', + 1255: 'pg_proc', + 1259: 'pg_class', + 2606: 'pg_constraint', + 2615: 'pg_namespace', + 3079: 'pg_extension' + }, + current_user: 'runtime_role', + pg_version: 'PostgreSQL test', + introspection_version: 1 + }); +} + +const parseFixture = (): any => parseIntrospectionResults(makeIntrospectionText()); + +describe('pg-introspection memo helper contract', () => { + it('installs memoized helpers as ordinary own data-function properties', () => { + const introspection = parseFixture(); + const namespace = introspection.namespaces[0]; + const getOwner = namespace.getOwner; + const descriptor = Object.getOwnPropertyDescriptor(namespace, 'getOwner'); + + expect(descriptor).toMatchObject({ + value: getOwner, + enumerable: true, + writable: true, + configurable: true + }); + expect(descriptor).not.toHaveProperty('get'); + expect(Object.keys(namespace)).toContain('getOwner'); + expect(getOwner.length).toBe(0); + expect(getOwner()).toBe(introspection.roles[0]); + expect(getOwner()).toBe(introspection.roles[0]); + expect(getOwner.call({ unrelated: true })).toBe(introspection.roles[0]); + expect(namespace.getOwner).toBe(getOwner); + expect(() => Reflect.construct(getOwner, [])).toThrow(TypeError); + + const spread = { ...namespace }; + expect(spread.getOwner).toBe(getOwner); + expect(JSON.parse(JSON.stringify(namespace))).toMatchObject({ + _id: '2200', + nspname: 'tenant_api' + }); + expect(JSON.stringify(namespace)).not.toContain('getOwner'); + }); + + it('preserves assignment, deletion, and post-freeze memoization', () => { + const mutable = parseFixture().namespaces[0]; + const replacement = jest.fn(() => 'replacement'); + + mutable.getOwner = replacement; + expect(mutable.getOwner()).toBe('replacement'); + expect(Reflect.deleteProperty(mutable, 'getOwner')).toBe(true); + expect(Object.prototype.hasOwnProperty.call(mutable, 'getOwner')).toBe(false); + + const frozen = parseFixture().namespaces[0]; + const getTags = frozen.getTags; + Object.freeze(frozen); + const first = getTags(); + + expect(getTags()).toBe(first); + expect(frozen.getTags).toBe(getTags); + expect(Object.isFrozen(frozen)).toBe(true); + }); + + it('preserves lazy helper identity and memoization on non-extensible entities', () => { + for (const lock of [Object.preventExtensions, Object.seal, Object.freeze]) { + const introspection = parseFixture(); + const namespace = introspection.namespaces[0]; + lock(namespace); + + const getOwner = namespace.getOwner; + expect(namespace.getOwner).toBe(getOwner); + expect(getOwner()).toBe(introspection.roles[0]); + expect(getOwner()).toBe(introspection.roles[0]); + expect(getOwner.call({ unrelated: true })).toBe(introspection.roles[0]); + } + }); + + it('keeps memo state isolated between parsed builds', () => { + const first = parseFixture(); + const second = parseFixture(); + + expect(first.namespaces[0].getOwner).not.toBe(second.namespaces[0].getOwner); + expect(first.namespaces[0].getOwner()).toBe(first.roles[0]); + expect(second.namespaces[0].getOwner()).toBe(second.roles[0]); + expect(first.roles[0]).not.toBe(second.roles[0]); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/realtime-config.test.ts b/graphql/server/src/middleware/__tests__/realtime-config.test.ts new file mode 100644 index 0000000000..26201a16cd --- /dev/null +++ b/graphql/server/src/middleware/__tests__/realtime-config.test.ts @@ -0,0 +1,43 @@ +import type { ConstructiveOptions } from '@constructive-io/graphql-types'; + +import { + addRealtimeRuntimeDependencySchema, + resolveGraphileRealtimeSchema +} from '../realtime-config'; + +describe('Graphile realtime configuration', () => { + it('preserves the compatibility default only for enabled realtime surfaces', () => { + expect(resolveGraphileRealtimeSchema({} as ConstructiveOptions, true)).toBe( + 'realtime_public' + ); + expect(resolveGraphileRealtimeSchema({} as ConstructiveOptions, false)).toBeNull(); + }); + + it('preserves one exact configured cursor schema', () => { + const options = { + graphile: { realtimeSchema: 'tenant_a_realtime' } + } as ConstructiveOptions; + + expect(resolveGraphileRealtimeSchema(options, true)).toBe('tenant_a_realtime'); + }); + + it('rejects an empty configured schema when realtime is enabled', () => { + const options = { + graphile: { realtimeSchema: '' } + } as ConstructiveOptions; + + expect(() => resolveGraphileRealtimeSchema(options, true)).toThrow( + 'graphile.realtimeSchema must be one non-empty exact schema name' + ); + }); + + it('adds and deduplicates the cursor schema only in the runtime allowlist', () => { + expect(addRealtimeRuntimeDependencySchema( + ['extensions', 'tenant_a_realtime'], + 'tenant_a_realtime' + )).toEqual(['extensions', 'tenant_a_realtime']); + expect(addRealtimeRuntimeDependencySchema(['extensions'], null)).toEqual([ + 'extensions' + ]); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/realtime-notification-config.test.ts b/graphql/server/src/middleware/__tests__/realtime-notification-config.test.ts new file mode 100644 index 0000000000..c9bebec256 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/realtime-notification-config.test.ts @@ -0,0 +1,129 @@ +import type { + ConstructiveOptions, + NotificationPgResolverInput +} from '@constructive-io/graphql-types'; + +import { + GraphileRealtimeNotificationConfigError, + resolveRealtimeCursorIntervals, + resolveRealtimeNotificationMode, + resolveRealtimeNotificationPgConfig, + resolveRealtimeNotificationRoleRevalidationMs +} from '../realtime-notification-config'; + +const route = { + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['tenant_a_public'] +}; + +describe('shared realtime notification configuration', () => { + it('defaults to the current dedicated subscriber and current cursor timings', () => { + const options = {} as ConstructiveOptions; + expect(resolveRealtimeNotificationMode(options)).toBe('dedicated'); + expect(resolveRealtimeNotificationRoleRevalidationMs(options)).toBe(60_000); + expect(resolveRealtimeCursorIntervals(options)).toEqual({ + pollIntervalMs: 5_000, + heartbeatIntervalMs: 30_000 + }); + }); + + it('accepts explicit shared mode and cursor timing contracts', () => { + const options = { + graphile: { + realtimeNotificationMode: 'shared-exact', + realtimeNotificationRoleRevalidationMs: 30_000, + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 90_000 + } + } as ConstructiveOptions; + + expect(resolveRealtimeNotificationMode(options)).toBe('shared-exact'); + expect(resolveRealtimeNotificationRoleRevalidationMs(options)).toBe(30_000); + expect(resolveRealtimeCursorIntervals(options)).toEqual({ + pollIntervalMs: 30_000, + heartbeatIntervalMs: 90_000 + }); + }); + + it('requires explicit listener credentials and never falls back to control credentials', async () => { + const options = { + pg: { + host: 'db.internal', + port: 5432, + database: 'control', + user: 'control_owner', + password: 'control-secret' + }, + notificationPgResolver: () => ({ + database: 'tenant_a', + user: 'tenant_a_notify' + }) + } as ConstructiveOptions; + + await expect(resolveRealtimeNotificationPgConfig(options, route)) + .rejects.toThrow('must return an explicit password'); + }); + + it('combines network defaults with one exact per-database listener identity', async () => { + const resolver = jest.fn((_input: Readonly) => ({ + database: 'tenant_a', + user: 'tenant_a_notify', + password: 'notification-secret', + pool: { max: 2 } + })); + const options = { + pg: { + host: 'db.internal', + port: 6432, + database: 'control', + user: 'control_owner', + password: 'control-secret', + ssl: true + }, + notificationPgResolver: resolver + } as ConstructiveOptions; + + await expect(resolveRealtimeNotificationPgConfig(options, route)).resolves.toEqual({ + host: 'db.internal', + port: 6432, + database: 'tenant_a', + user: 'tenant_a_notify', + password: 'notification-secret', + ssl: true, + pool: { max: 2 } + }); + const input = resolver.mock.calls[0][0]; + expect(input).toEqual(route); + expect(Object.isFrozen(input)).toBe(true); + expect(Object.isFrozen(input.schemas)).toBe(true); + }); + + it('rejects a resolver that routes to a different physical database', async () => { + const options = { + notificationPgResolver: () => ({ + database: 'tenant_b', + user: 'tenant_b_notify', + password: 'notification-secret' + }) + } as ConstructiveOptions; + + await expect(resolveRealtimeNotificationPgConfig(options, route)).rejects + .toBeInstanceOf(GraphileRealtimeNotificationConfigError); + }); + + it('rejects an ambiguous connection string even when explicit fields are present', async () => { + const options = { + notificationPgResolver: () => ({ + database: 'tenant_a', + user: 'tenant_a_notify', + password: 'explicit-secret', + connectionString: 'postgres://other:override@foreign/tenant_b' + }) + } as ConstructiveOptions; + + await expect(resolveRealtimeNotificationPgConfig(options, route)).rejects + .toThrow('must not return a connectionString'); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/routing.test.ts b/graphql/server/src/middleware/__tests__/routing.test.ts index bcd013a888..9176e2bd5f 100644 --- a/graphql/server/src/middleware/__tests__/routing.test.ts +++ b/graphql/server/src/middleware/__tests__/routing.test.ts @@ -1,23 +1,38 @@ jest.mock('pg-cache', () => ({ - getPgPool: jest.fn() + acquirePgPool: jest.fn(), + getPgPoolIdentity: jest.fn().mockReturnValue('pg:test'), + PG_POOL_CAPACITY_ERROR_CODE: 'PG_POOL_CAPACITY' })); jest.mock('@constructive-io/express-context', () => ({ createDefaultRegistry: jest.fn(() => ({ - resolve: jest.fn().mockResolvedValue(undefined) + resolve: jest.fn(async (name: string) => name === 'databaseSettings' ? { + enableAggregates: false, + enablePostgis: false, + enableSearch: false, + enableDirectUploads: false, + enablePresignedUploads: false, + enableManyToMany: false, + enableConnectionFilter: false, + enableLtree: false, + enableLlm: false, + enableRealtime: false, + enableBulk: false, + enableI18n: false + } : undefined) })) })); import { svcCache } from '@pgpmjs/server-utils'; import type { Request } from 'express'; import type { Pool } from 'pg'; -import { getPgPool } from 'pg-cache'; +import { acquirePgPool } from 'pg-cache'; import type { ApiOptions } from '../../types'; import { getApiConfig } from '../api'; import { ResolvedRoute, resolveRoute, routeToApiStructure } from '../routing'; -const mockGetPgPool = getPgPool as jest.MockedFunction; +const mockAcquirePgPool = acquirePgPool as jest.MockedFunction; const matchedRoute = (overrides: Partial = {}): ResolvedRoute => ({ route_binding_id: 'rb-1', @@ -29,7 +44,7 @@ const matchedRoute = (overrides: Partial = {}): ResolvedRoute => domain_id: 'dom-1', target_catalog_id: 'cat-1', target_module: 'apis', - target_source_id: 'api-src-1', + target_source_id: 'api-1', target_owner_scope: 'database', target_owner_key: 'db-1', resolved_config: { @@ -51,6 +66,11 @@ const noMatchRoute = (): ResolvedRoute => matchedRoute({ route_binding_id: null, target_module: null, resolved_config: null }); const createPool = (query: jest.Mock): Pool => ({ query } as unknown as Pool); +const leasePool = (pool: Pool) => ({ + pool, + identity: 'pg:test', + release: jest.fn() +}); describe('resolveRoute', () => { it('returns the row when a route matches', async () => { @@ -69,6 +89,22 @@ describe('resolveRoute', () => { expect(row).toBeNull(); }); + it('fails closed when the resolver violates its exactly-one-row contract', async () => { + const zeroRows = await resolveRoute( + createPool(jest.fn().mockResolvedValue({ rows: [] })), + 'constructive_routing_public', + 'api.example.com' + ); + const duplicateRows = await resolveRoute( + createPool(jest.fn().mockResolvedValue({ rows: [matchedRoute(), matchedRoute()] })), + 'constructive_routing_public', + 'api.example.com' + ); + + expect(zeroRows).toBeNull(); + expect(duplicateRows).toBeNull(); + }); + it('returns null when the resolver function is not installed', async () => { const query = jest.fn().mockRejectedValue(Object.assign(new Error('undefined function'), { code: '42883' })); const row = await resolveRoute(createPool(query), 'constructive_routing_public', 'api.example.com'); @@ -121,6 +157,55 @@ describe('routeToApiStructure', () => { it('returns null when resolved_config lacks api essentials', () => { expect(routeToApiStructure(matchedRoute({ resolved_config: {} }), opts)).toBeNull(); }); + + it('fails closed when route visibility does not match the server ingress', () => { + const privateRoute = matchedRoute({ + resolved_config: { + ...(matchedRoute().resolved_config as Record), + is_public: false + } + }); + + expect(routeToApiStructure(privateRoute, opts)).toBeNull(); + }); + + it('fails closed when exact roles or physical schemas are absent', () => { + expect(routeToApiStructure(matchedRoute({ + resolved_config: { + ...(matchedRoute().resolved_config as Record), + anon_role: undefined + } + }), opts)).toBeNull(); + expect(routeToApiStructure(matchedRoute({ + resolved_config: { + ...(matchedRoute().resolved_config as Record), + schemas: ['app_public', 'app_public'] + } + }), opts)).toBeNull(); + }); + + it('accepts Constructive dash-prefixed physical schemas', () => { + expect(routeToApiStructure(matchedRoute({ + resolved_config: { + ...(matchedRoute().resolved_config as Record), + schemas: ['customer-db-a1b2c3d4-app-public'] + } + }), opts)).toMatchObject({ + schema: ['customer-db-a1b2c3d4-app-public'] + }); + }); + + it('fails closed when route and resolved-config identities disagree', () => { + expect(routeToApiStructure(matchedRoute({ + target_source_id: 'another-api' + }), opts)).toBeNull(); + expect(routeToApiStructure(matchedRoute({ + target_owner_key: 'another-database' + }), opts)).toBeNull(); + expect(routeToApiStructure(matchedRoute({ + target_owner_scope: 'organization' + }), opts)).toBeNull(); + }); }); describe('getApiConfig with scoped routing enabled', () => { @@ -164,7 +249,7 @@ describe('getApiConfig with scoped routing enabled', () => { if (sql.includes('resolve_route')) return { rows: [matchedRoute()] }; throw new Error(`unexpected query: ${sql}`); }); - mockGetPgPool.mockReturnValue(createPool(query) as never); + mockAcquirePgPool.mockImplementation(() => leasePool(createPool(query))); const result = await getApiConfig(createOptions(), createRequest({ host: 'api.example.com' })); @@ -181,7 +266,7 @@ describe('getApiConfig with scoped routing enabled', () => { if (sql.includes('resolve_route')) return { rows: [noMatchRoute()] }; throw new Error(`unexpected query (no legacy fallback): ${sql}`); }); - mockGetPgPool.mockReturnValue(createPool(query) as never); + mockAcquirePgPool.mockImplementation(() => leasePool(createPool(query))); const result = await getApiConfig(createOptions(), createRequest({ host: 'nomatch.example.com' })); @@ -189,7 +274,43 @@ describe('getApiConfig with scoped routing enabled', () => { expect(query.mock.calls.some(([sql]) => String(sql).includes('services_public'))).toBe(false); }); - it('throws NO_DATABASE_ID when a route resolves without a database id (no default database)', async () => { + it('re-resolves a hot hostname so a reassignment cannot use stale tenant metadata', async () => { + let route = matchedRoute(); + const query = jest.fn(async (sql: string, params: unknown[]) => { + if (sql.includes('information_schema.schemata')) return schemaValidationRows(params); + if (sql.includes('resolve_route')) return { rows: [route] }; + throw new Error(`unexpected query: ${sql}`); + }); + mockAcquirePgPool.mockImplementation(() => leasePool(createPool(query))); + + const first = await getApiConfig( + createOptions(), + createRequest({ host: 'api.example.com' }) + ); + route = matchedRoute({ + target_source_id: 'api-2', + target_owner_key: 'db-2', + resolved_config: { + api_id: 'api-2', + database_id: 'db-2', + dbname: 'tenant_db_2', + role_name: 'api_role_2', + anon_role: 'api_anon_2', + is_public: true, + schemas: ['app_two_public'] + } + }); + const second = await getApiConfig( + createOptions(), + createRequest({ host: 'api.example.com' }) + ); + + expect(first).toMatchObject({ databaseId: 'db-1' }); + expect(second).toMatchObject({ databaseId: 'db-2', dbname: 'tenant_db_2' }); + expect(query.mock.calls.filter(([sql]) => String(sql).includes('resolve_route'))).toHaveLength(2); + }); + + it('fails closed when a route resolves without a database id', async () => { const routeWithoutDbId = matchedRoute({ resolved_config: { api_id: 'api-1', @@ -205,10 +326,10 @@ describe('getApiConfig with scoped routing enabled', () => { if (sql.includes('resolve_route')) return { rows: [routeWithoutDbId] }; throw new Error(`unexpected query: ${sql}`); }); - mockGetPgPool.mockReturnValue(createPool(query) as never); + mockAcquirePgPool.mockImplementation(() => leasePool(createPool(query))); await expect( getApiConfig(createOptions(), createRequest({ host: 'api.example.com' })) - ).rejects.toMatchObject({ code: 'NO_DATABASE_ID' }); + ).resolves.toBeNull(); }); }); diff --git a/graphql/server/src/middleware/__tests__/runtime-pg-config.test.ts b/graphql/server/src/middleware/__tests__/runtime-pg-config.test.ts new file mode 100644 index 0000000000..d396ad985d --- /dev/null +++ b/graphql/server/src/middleware/__tests__/runtime-pg-config.test.ts @@ -0,0 +1,203 @@ +import { EventEmitter } from 'node:events'; + +import type { + ConstructiveOptions, + RuntimePgResolverInput +} from '@constructive-io/graphql-types'; +import type { NextFunction, Request, Response } from 'express'; + +import { + createRuntimePgResolutionStore, + resolveRuntimePgConfig +} from '../runtime-pg-config'; +import { InvalidRuntimePgConfigurationError } from '../runtime-pg-requirements'; + +const route: RuntimePgResolverInput = { + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['tenant_a_public', 'tenant_a_auth'], + roles: ['tenant_a_anonymous', 'tenant_a_authenticated'] +}; + +const resolverOptions = ( + resolver: ConstructiveOptions['runtimePgResolver'] +): ConstructiveOptions => ({ + pg: { + host: 'db.internal', + port: 6432, + database: 'control', + user: 'control_owner', + password: 'control-secret', + ssl: true + }, + graphile: { introspectionMode: 'scoped-required' }, + runtimePgResolver: resolver +}); + +describe('exact runtime PostgreSQL resolution', () => { + it('resolves one frozen credential-free route and normalizes an opaque pool identity', async () => { + const resolver = jest.fn((_input: Readonly) => ({ + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret', + pool: { max: 2, maxUses: 1 } + })); + + const resolution = await resolveRuntimePgConfig( + resolverOptions(resolver), + route, + 'production' + ); + + expect(resolution.pgConfig).toEqual({ + host: 'db.internal', + port: 6432, + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret', + ssl: true, + pool: { max: 2, maxUses: 1 } + }); + expect(resolution.poolIdentity).toMatch(/^pg:v1:/); + expect(Object.isFrozen(resolution)).toBe(true); + expect(Object.isFrozen(resolution.pgConfig)).toBe(true); + expect(resolver).toHaveBeenCalledTimes(1); + const input = resolver.mock.calls[0][0]; + expect(input).toEqual(route); + expect(Object.isFrozen(input)).toBe(true); + expect(Object.isFrozen(input.schemas)).toBe(true); + expect(Object.isFrozen(input.roles)).toBe(true); + }); + + it('rejects ambiguous connection strings and physical database mismatches', async () => { + await expect(resolveRuntimePgConfig(resolverOptions(() => ({ + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret', + connectionString: 'postgres://other:secret@foreign/tenant_b' + } as never)), route, 'production')).rejects.toThrow( + 'must not return a connectionString' + ); + + await expect(resolveRuntimePgConfig(resolverOptions(() => ({ + database: 'tenant_b', + user: 'tenant_b_runtime', + password: 'runtime-secret' + })), route, 'production')).rejects.toThrow( + 'does not match the routed physical database' + ); + }); + + it('binds login and pool policy into the opaque identity on one attested target', async () => { + const base = await resolveRuntimePgConfig(resolverOptions(() => ({ + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret', + pool: { max: 2 } + })), route, 'production'); + const otherTarget = await resolveRuntimePgConfig(resolverOptions(() => ({ + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'rotated-runtime-secret', + pool: { max: 3 } + })), route, 'production'); + + expect(otherTarget.poolIdentity).not.toBe(base.poolIdentity); + }); + + it.each([ + { host: 'other.internal' }, + { port: 5433 }, + { ssl: false } + ])('rejects runtime/control endpoint divergence: %p', async (networkOverride) => { + await expect(resolveRuntimePgConfig(resolverOptions(() => ({ + ...networkOverride, + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + })), route, 'production')).rejects.toThrow( + 'network/TLS endpoint does not match the routed control-plane database' + ); + }); + + it('authorizes static credentials for one exact ordered route only', async () => { + const options: ConstructiveOptions = { + pg: { host: 'db.internal', port: 5432, ssl: true }, + graphile: { introspectionMode: 'scoped-required' }, + runtimePg: { + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + }, + runtimePgStaticIdentity: route + }; + + await expect(resolveRuntimePgConfig(options, route, 'production')) + .resolves.toMatchObject({ + pgConfig: { + database: 'tenant_a', + user: 'tenant_a_runtime' + } + }); + await expect(resolveRuntimePgConfig(options, { + ...route, + schemas: [...route.schemas].reverse() + }, 'production')).rejects.toThrow( + 'not authorized for the requested exact route' + ); + await expect(resolveRuntimePgConfig(options, { + ...route, + roles: [route.roles[1], route.roles[0]] + }, 'production')).rejects.toThrow( + 'not authorized for the requested exact route' + ); + }); + + it('keeps the secret-bearing resolution outside req and resolves only once', async () => { + const resolver = jest.fn(() => ({ + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + })); + const store = createRuntimePgResolutionStore(resolverOptions(resolver)); + const req = Object.assign(new EventEmitter(), { + api: { + apiId: route.apiId, + databaseId: route.databaseId, + dbname: route.databaseName, + schema: [...route.schemas], + anonRole: route.roles[0], + roleName: route.roles[1] + } + }) as unknown as Request; + const res = new EventEmitter() as unknown as Response; + const next = jest.fn() as unknown as NextFunction; + + await store.middleware(req, res, next); + + expect(next).toHaveBeenCalledWith(); + expect(resolver).toHaveBeenCalledTimes(1); + const first = store.getRuntimePgResolution(req); + const second = store.getRuntimePgResolution(req); + expect(second).toBe(first); + expect(Reflect.ownKeys(req)).not.toContain('runtimePg'); + expect(JSON.stringify(req)).not.toContain('runtime-secret'); + + req.api = { + ...req.api!, + databaseId: 'database-b' + }; + expect(() => store.getRuntimePgResolution(req)).toThrow( + 'Authoritative API route changed after runtime PostgreSQL resolution' + ); + req.api = { + ...req.api, + databaseId: route.databaseId + }; + + (res as unknown as EventEmitter).emit('finish'); + expect(() => store.getRuntimePgResolution(req)) + .toThrow(InvalidRuntimePgConfigurationError); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/runtime-pg-requirements.test.ts b/graphql/server/src/middleware/__tests__/runtime-pg-requirements.test.ts new file mode 100644 index 0000000000..7a4c7576e6 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/runtime-pg-requirements.test.ts @@ -0,0 +1,111 @@ +import { + assertRuntimePgCredentials, + InvalidRuntimePgConfigurationError, + MissingRuntimePgCredentialsError, + shouldValidateRuntimeRoleSafety, + usesUnsafeDevelopmentRuntimePgFallback +} from '../runtime-pg-requirements'; + +describe('GraphQL runtime PostgreSQL requirements', () => { + it('preserves an explicitly named stock-mode fallback only outside production', () => { + const options = { + graphile: { introspectionMode: 'stock' } + } as const; + expect(usesUnsafeDevelopmentRuntimePgFallback(options, 'development')).toBe(true); + expect(usesUnsafeDevelopmentRuntimePgFallback(options, 'test')).toBe(true); + expect(usesUnsafeDevelopmentRuntimePgFallback(options, 'production')).toBe(false); + expect(() => assertRuntimePgCredentials(options, 'development')).not.toThrow(); + }); + + it('rejects production stock mode without an explicit runtime login', () => { + expect(() => assertRuntimePgCredentials({ + graphile: { introspectionMode: 'stock' } + }, 'production')).toThrow(MissingRuntimePgCredentialsError); + }); + + it.each([ + undefined, + {}, + { user: 'runtime' }, + { password: 'secret' }, + { user: ' ', password: 'secret' }, + { user: 'runtime', password: '' }, + { user: 42, password: 'secret' }, + { user: 'runtime', password: async () => 'secret' } + ])('rejects scoped mode without a complete explicit runtime login: %p', (runtimePg) => { + expect(() => assertRuntimePgCredentials({ + graphile: { introspectionMode: 'scoped-required' }, + runtimePg: runtimePg as never + }, 'test')).toThrow(MissingRuntimePgCredentialsError); + }); + + it('rejects an incomplete explicitly supplied login even in stock development', () => { + expect(() => assertRuntimePgCredentials({ + graphile: { introspectionMode: 'stock' }, + runtimePg: { user: 'runtime' } + }, 'development')).toThrow(MissingRuntimePgCredentialsError); + }); + + it('accepts a static login in stock development compatibility mode', () => { + const stock = { + graphile: { introspectionMode: 'stock' as const }, + runtimePg: { user: 'runtime', password: 'secret' } + }; + expect(() => assertRuntimePgCredentials(stock, 'development')).not.toThrow(); + expect(shouldValidateRuntimeRoleSafety(stock, 'development')).toBe(true); + }); + + it('requires a resolver or one exact static route in scoped/production modes', () => { + expect(() => assertRuntimePgCredentials({ + graphile: { introspectionMode: 'scoped-required' }, + runtimePg: { user: 'runtime', password: 'secret' } + }, 'test')).toThrow(InvalidRuntimePgConfigurationError); + + expect(() => assertRuntimePgCredentials({ + graphile: { introspectionMode: 'scoped-required' }, + runtimePg: { + database: 'tenant_a', + user: 'runtime', + password: 'secret' + }, + runtimePgStaticIdentity: { + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['tenant_a_public'], + roles: ['anonymous', 'authenticated'] + } + }, 'test')).not.toThrow(); + + expect(() => assertRuntimePgCredentials({ + graphile: { introspectionMode: 'scoped-required' }, + runtimePgResolver: async () => ({ + database: 'tenant_a', + user: 'runtime', + password: 'secret' + }) + }, 'test')).not.toThrow(); + }); + + it('always enables role safety in production, even for stock mode', () => { + expect(shouldValidateRuntimeRoleSafety({ + graphile: { introspectionMode: 'stock' }, + runtimePgResolver: () => ({ + database: 'tenant_a', + user: 'runtime', + password: 'secret' + }) + }, 'production')).toBe(true); + }); + + it('rejects ambiguous static and resolver credentials', () => { + expect(() => assertRuntimePgCredentials({ + runtimePg: { user: 'runtime', password: 'secret' }, + runtimePgResolver: () => ({ + database: 'tenant_a', + user: 'other', + password: 'other-secret' + }) + }, 'development')).toThrow(InvalidRuntimePgConfigurationError); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/runtime-role-safety.integration.test.ts b/graphql/server/src/middleware/__tests__/runtime-role-safety.integration.test.ts new file mode 100644 index 0000000000..4e55e34679 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/runtime-role-safety.integration.test.ts @@ -0,0 +1,199 @@ +import { randomUUID } from 'node:crypto'; + +import pg from 'pg'; +import { getPgEnvOptions } from 'pg-env'; + +import { + assertRuntimeRoleSafety, + UnsafeRuntimeRoleError +} from '../runtime-role-safety'; + +const describeWithPostgres = + process.env.GRAPHQL_SERVER_RUN_ROLE_SAFETY_INTEGRATION === '1' + ? describe + : describe.skip; + +describeWithPostgres('runtime role safety PostgreSQL integration', () => { + jest.setTimeout(30_000); + + it('rejects ownership inherited only after SET ROLE to a configured request role', async () => { + const suffix = randomUUID().replace(/-/g, '').slice(0, 12); + const parentRole = `rrs_parent_${suffix}`; + const requestRole = `rrs_request_${suffix}`; + const runtimeRole = `rrs_runtime_${suffix}`; + const schema = `rrs_schema_${suffix}`; + const password = `rrs-${randomUUID()}`; + const quoteIdentifier = pg.escapeIdentifier; + const adminConfig = getPgEnvOptions({}); + const adminPool = new pg.Pool({ ...adminConfig, max: 1 }); + let runtimePool: pg.Pool | null = null; + + try { + await adminPool.query(` + CREATE ROLE ${quoteIdentifier(parentRole)} NOLOGIN; + CREATE ROLE ${quoteIdentifier(requestRole)} NOLOGIN INHERIT; + CREATE ROLE ${quoteIdentifier(runtimeRole)} LOGIN NOINHERIT + PASSWORD ${pg.escapeLiteral(password)}; + CREATE SCHEMA ${quoteIdentifier(schema)} AUTHORIZATION CURRENT_USER; + CREATE TABLE ${quoteIdentifier(schema)}.owned_table (id integer); + ALTER TABLE ${quoteIdentifier(schema)}.owned_table + OWNER TO ${quoteIdentifier(parentRole)}; + GRANT USAGE ON SCHEMA ${quoteIdentifier(schema)} + TO ${quoteIdentifier(requestRole)}; + GRANT ${quoteIdentifier(parentRole)} TO ${quoteIdentifier(requestRole)} + WITH INHERIT TRUE, SET FALSE; + GRANT ${quoteIdentifier(requestRole)} TO ${quoteIdentifier(runtimeRole)} + WITH INHERIT FALSE, SET TRUE; + `); + + runtimePool = new pg.Pool({ + ...adminConfig, + user: runtimeRole, + password, + max: 1 + }); + const client = await runtimePool.connect(); + try { + const before = await client.query<{ + parent_usage: boolean; + parent_set: boolean; + request_set: boolean; + }>(` + SELECT pg_catalog.pg_has_role(current_user, $1, 'USAGE') AS parent_usage, + pg_catalog.pg_has_role(current_user, $1, 'SET') AS parent_set, + pg_catalog.pg_has_role(current_user, $2, 'SET') AS request_set + `, [parentRole, requestRole]); + expect(before.rows[0]).toEqual({ + parent_usage: false, + parent_set: false, + request_set: true + }); + + await client.query('BEGIN'); + await client.query(`SET ROLE ${quoteIdentifier(requestRole)}`); + const after = await client.query<{ parent_usage: boolean }>( + 'SELECT pg_catalog.pg_has_role(current_user, $1, \'USAGE\') AS parent_usage', + [parentRole] + ); + expect(after.rows[0]?.parent_usage).toBe(true); + await expect(client.query( + `ALTER TABLE ${quoteIdentifier(schema)}.owned_table ADD COLUMN escaped integer` + )).resolves.toBeDefined(); + await client.query('ROLLBACK'); + } finally { + client.release(true); + } + + let rejected: unknown; + try { + await assertRuntimeRoleSafety( + runtimePool, + [requestRole], + [schema] + ); + } catch (error) { + rejected = error; + } + expect(rejected).toBeInstanceOf(UnsafeRuntimeRoleError); + expect((rejected as UnsafeRuntimeRoleError).violations).toContain( + `${requestRole} can reach role ${parentRole}` + + ' after SET ROLE (USAGE=true,SET=false)' + ); + } finally { + await runtimePool?.end(); + await adminPool.query(` + DROP SCHEMA IF EXISTS ${quoteIdentifier(schema)} CASCADE; + DROP ROLE IF EXISTS ${quoteIdentifier(runtimeRole)}; + DROP ROLE IF EXISTS ${quoteIdentifier(requestRole)}; + DROP ROLE IF EXISTS ${quoteIdentifier(parentRole)}; + `); + await adminPool.end(); + } + }); + + it('rejects BYPASSRLS, object ownership, SECURITY DEFINER, and cross-schema privileges', async () => { + const suffix = randomUUID().replace(/-/g, '').slice(0, 12); + const requestRole = `rrs_request_${suffix}`; + const runtimeRole = `rrs_runtime_${suffix}`; + const approvedSchema = `rrs_approved_${suffix}`; + const externalSchema = `rrs_external_${suffix}`; + const password = `rrs-${randomUUID()}`; + const quoteIdentifier = pg.escapeIdentifier; + const adminConfig = getPgEnvOptions({}); + const adminPool = new pg.Pool({ ...adminConfig, max: 1 }); + let runtimePool: pg.Pool | null = null; + + try { + await adminPool.query(` + CREATE ROLE ${quoteIdentifier(requestRole)} NOLOGIN NOINHERIT BYPASSRLS; + CREATE ROLE ${quoteIdentifier(runtimeRole)} LOGIN NOINHERIT + PASSWORD ${pg.escapeLiteral(password)}; + CREATE SCHEMA ${quoteIdentifier(approvedSchema)} AUTHORIZATION CURRENT_USER; + CREATE SCHEMA ${quoteIdentifier(externalSchema)} AUTHORIZATION CURRENT_USER; + CREATE TABLE ${quoteIdentifier(approvedSchema)}.owned_table (id integer); + ALTER TABLE ${quoteIdentifier(approvedSchema)}.owned_table + OWNER TO ${quoteIdentifier(requestRole)}; + CREATE FUNCTION ${quoteIdentifier(approvedSchema)}.privileged_function() + RETURNS integer LANGUAGE sql SECURITY DEFINER AS 'SELECT 1'; + CREATE TABLE ${quoteIdentifier(externalSchema)}.external_table (id integer); + CREATE SEQUENCE ${quoteIdentifier(externalSchema)}.external_sequence; + CREATE FUNCTION ${quoteIdentifier(externalSchema)}.external_function() + RETURNS integer LANGUAGE sql AS 'SELECT 1'; + CREATE TYPE ${quoteIdentifier(externalSchema)}.external_type AS ENUM ('one'); + GRANT USAGE ON SCHEMA ${quoteIdentifier(approvedSchema)}, + ${quoteIdentifier(externalSchema)} TO ${quoteIdentifier(requestRole)}; + GRANT SELECT ON ${quoteIdentifier(externalSchema)}.external_table + TO ${quoteIdentifier(requestRole)}; + GRANT USAGE ON SEQUENCE ${quoteIdentifier(externalSchema)}.external_sequence + TO ${quoteIdentifier(requestRole)}; + GRANT EXECUTE ON FUNCTION ${quoteIdentifier(externalSchema)}.external_function() + TO ${quoteIdentifier(requestRole)}; + GRANT USAGE ON TYPE ${quoteIdentifier(externalSchema)}.external_type + TO ${quoteIdentifier(requestRole)}; + GRANT ${quoteIdentifier(requestRole)} TO ${quoteIdentifier(runtimeRole)} + WITH INHERIT FALSE, SET TRUE; + `); + + runtimePool = new pg.Pool({ + ...adminConfig, + user: runtimeRole, + password, + max: 1 + }); + + let rejected: unknown; + try { + await assertRuntimeRoleSafety( + runtimePool, + [requestRole], + [approvedSchema] + ); + } catch (error) { + rejected = error; + } + + expect(rejected).toBeInstanceOf(UnsafeRuntimeRoleError); + const violations = (rejected as UnsafeRuntimeRoleError).violations; + expect(violations).toContain(`${requestRole} has BYPASSRLS`); + expect(violations).toContain( + `${requestRole} owns RELATION ${approvedSchema}.owned_table` + ); + expect(violations).toContain( + `SECURITY DEFINER FUNCTION ${approvedSchema}.privileged_function ` + + 'is not allowed in the approved GraphQL schema scope' + ); + expect(violations).toContain( + `${requestRole} has RELATION,SEQUENCE,FUNCTION,TYPE on unapproved schema ${externalSchema}` + ); + } finally { + await runtimePool?.end(); + await adminPool.query(` + DROP SCHEMA IF EXISTS ${quoteIdentifier(approvedSchema)} CASCADE; + DROP SCHEMA IF EXISTS ${quoteIdentifier(externalSchema)} CASCADE; + DROP ROLE IF EXISTS ${quoteIdentifier(runtimeRole)}; + DROP ROLE IF EXISTS ${quoteIdentifier(requestRole)}; + `); + await adminPool.end(); + } + }); +}); diff --git a/graphql/server/src/middleware/__tests__/runtime-role-safety.test.ts b/graphql/server/src/middleware/__tests__/runtime-role-safety.test.ts new file mode 100644 index 0000000000..a09dafa84d --- /dev/null +++ b/graphql/server/src/middleware/__tests__/runtime-role-safety.test.ts @@ -0,0 +1,514 @@ +import type { Pool } from 'pg'; + +import { + assertRuntimeRoleSafety, + DEFAULT_RUNTIME_ROLE_SAFETY_MAX_AGE_MS, + ensureRuntimeRoleSafety, + getRuntimeRoleSafetyStats, + invalidateRuntimeRoleSafety, + MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS, + refreshRuntimeRoleSafety, + RUNTIME_ROLE_SAFETY_SQL, + UnsafeRuntimeRoleError +} from '../runtime-role-safety'; + +const poolWithRow = (row: Record) => { + const client = { + query: jest.fn(async (query: string) => query === RUNTIME_ROLE_SAFETY_SQL + ? { rows: [row] } + : { rows: [] }), + release: jest.fn() + }; + return { + pool: { connect: jest.fn(async () => client) } as unknown as Pool, + client + }; +}; + +const safeRow = { + login_role: 'graphql_runtime', + login_role_violations: [] as Array<{ capabilities: string[] }>, + inherited_role_violations: [] as Array<{ rolname: string }>, + unexpected_set_role_violations: [] as Array<{ rolname: string }>, + request_role_reachability_violations: [] as Array<{ + request_role: string; + reachable_role: string; + via_usage: boolean; + via_set: boolean; + }>, + role_violations: [] as Array<{ rolname: string; capabilities: string[] }>, + database_violations: [] as Array<{ + rolname: string; + datname: string; + capability: string; + }>, + cross_database_violations: [] as Array<{ + rolname: string; + datname: string; + }>, + schema_violations: [] as Array<{ rolname: string; nspname: string; capability: string }>, + cross_schema_violations: [] as Array<{ + rolname: string; + nspname: string; + capabilities: string[]; + }>, + object_owner_violations: [] as Array<{ + rolname: string; + nspname: string; + object_name: string; + object_kind: string; + }>, + privileged_object_violations: [] as Array<{ + nspname: string; + object_name: string; + reason: string; + }>, + stored_dependency_violations: [] as Array<{ + nspname: string; + object_name: string; + reason: string; + dependency: string; + }>, + missing_roles: [] as string[], + inaccessible_roles: [] as string[], + missing_schemas: [] as string[] +}; + +describe('runtime role safety', () => { + it('accepts a least-privilege login and parameterizes roles and schemas', async () => { + const { pool, client } = poolWithRow(safeRow); + await expect(assertRuntimeRoleSafety( + pool, + ['tenant_anon', 'tenant_user'], + ['tenant_public'], + ['extensions'] + )).resolves.toBeUndefined(); + + expect(client.query).toHaveBeenNthCalledWith(2, RUNTIME_ROLE_SAFETY_SQL, [ + ['tenant_anon', 'tenant_user'], + ['tenant_public'], + ['extensions'] + ]); + expect(client.query).toHaveBeenNthCalledWith( + 1, + 'BEGIN READ ONLY; SET LOCAL jit TO off' + ); + expect(client.query).toHaveBeenNthCalledWith(3, 'COMMIT'); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it.each(['SUPERUSER', 'BYPASSRLS', 'CREATEROLE', 'CREATEDB', 'REPLICATION'])( + 'rejects %s', + async (capability) => { + const { pool } = poolWithRow({ + ...safeRow, + role_violations: [{ rolname: 'graphql_runtime', capabilities: [capability] }] + }); + await expect(assertRuntimeRoleSafety(pool, [], [])).rejects.toBeInstanceOf( + UnsafeRuntimeRoleError + ); + } + ); + + it('requires the runtime login to be NOINHERIT', async () => { + const { pool } = poolWithRow({ + ...safeRow, + login_role_violations: [{ capabilities: ['INHERIT'] }] + }); + await expect(assertRuntimeRoleSafety(pool, [], [])).rejects.toThrow( + 'graphql_runtime has INHERIT' + ); + }); + + it('rejects privileges inherited through a membership-level INHERIT grant', async () => { + const { pool } = poolWithRow({ + ...safeRow, + inherited_role_violations: [{ rolname: 'cross_tenant_reader' }] + }); + await expect(assertRuntimeRoleSafety(pool, [], [])).rejects.toThrow( + 'graphql_runtime inherits privileges from role cross_tenant_reader' + ); + }); + + it('rejects SET-able roles outside the exact configured request-role set', async () => { + const { pool } = poolWithRow({ + ...safeRow, + unexpected_set_role_violations: [{ rolname: 'tenant_admin' }] + }); + await expect(assertRuntimeRoleSafety( + pool, + ['tenant_anon', 'tenant_user'], + ['tenant_public'] + )).rejects.toThrow( + 'graphql_runtime can SET ROLE to unconfigured role tenant_admin' + ); + }); + + it('rejects roles reachable only after SET ROLE to a configured request role', async () => { + const { pool } = poolWithRow({ + ...safeRow, + request_role_reachability_violations: [{ + request_role: 'tenant_user', + reachable_role: 'tenant_owner', + via_usage: true, + via_set: false + }] + }); + await expect(assertRuntimeRoleSafety( + pool, + ['tenant_user'], + ['tenant_public'] + )).rejects.toThrow( + 'tenant_user can reach role tenant_owner after SET ROLE (USAGE=true,SET=false)' + ); + }); + + it.each(['OWNER', 'CREATE'])('rejects schema %s capability', async (capability) => { + const { pool } = poolWithRow({ + ...safeRow, + schema_violations: [{ + rolname: 'graphql_runtime', + nspname: 'tenant_public', + capability + }] + }); + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + `has ${capability} on schema tenant_public` + ); + }); + + it.each(['OWNER', 'CREATE', 'TEMP'])('rejects database %s capability', async (capability) => { + const { pool } = poolWithRow({ + ...safeRow, + database_violations: [{ + rolname: 'graphql_runtime', + datname: 'tenant_database', + capability + }] + }); + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + `graphql_runtime has ${capability} on database tenant_database` + ); + }); + + it('rejects CONNECT to a non-target database', async () => { + const { pool } = poolWithRow({ + ...safeRow, + cross_database_violations: [{ + rolname: 'tenant_user', + datname: 'tenant_b' + }] + }); + await expect(assertRuntimeRoleSafety( + pool, + ['tenant_user'], + ['tenant_public'] + )).rejects.toThrow( + 'tenant_user has CONNECT on non-target database tenant_b' + ); + }); + + it.each([ + 'login_role_violations', + 'inherited_role_violations', + 'database_violations', + 'cross_database_violations', + 'unexpected_set_role_violations', + 'request_role_reachability_violations', + 'role_violations', + 'schema_violations', + 'cross_schema_violations', + 'object_owner_violations', + 'privileged_object_violations', + 'stored_dependency_violations' + ])( + 'fails closed when the safety query omits %s', + async (column) => { + const row = { ...safeRow } as Record; + delete row[column]; + const { pool } = poolWithRow(row); + + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + `safety query did not return ${column} as a JSON array` + ); + } + ); + + it.each(['missing_roles', 'inaccessible_roles', 'missing_schemas'])( + 'fails closed when the safety query omits %s', + async (column) => { + const row = { ...safeRow } as Record; + delete row[column]; + const { pool } = poolWithRow(row); + + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + `safety query did not return ${column} as a text array` + ); + } + ); + + it('fails closed when the safety query omits the login role', async () => { + const { pool } = poolWithRow({ ...safeRow, login_role: null }); + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + 'safety query did not return a non-empty login_role' + ); + }); + + it('rejects effective object access to an unapproved tenant schema', async () => { + const { pool } = poolWithRow({ + ...safeRow, + cross_schema_violations: [{ + rolname: 'graphql_runtime', + nspname: 'tenant_b', + capabilities: ['RELATION', 'FUNCTION'] + }] + }); + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_a'])).rejects.toThrow( + 'graphql_runtime has RELATION,FUNCTION on unapproved schema tenant_b' + ); + }); + + it.each([ + 'SECURITY DEFINER FUNCTION', + 'OWNER-RIGHTS VIEW', + 'FOREIGN TABLE', + 'MATERIALIZED VIEW' + ])('rejects approved-scope %s paths that can escape invoker privileges', async (reason) => { + const { pool } = poolWithRow({ + ...safeRow, + privileged_object_violations: [{ + nspname: 'tenant_public', + object_name: 'unsafe_path', + reason + }] + }); + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + `${reason} tenant_public.unsafe_path is not allowed in the approved GraphQL schema scope` + ); + }); + + it('walks tracked dependencies transitively from every stored-expression class', () => { + expect(RUNTIME_ROLE_SAFETY_SQL).toContain('WITH RECURSIVE'); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain( + "pg_catalog.pg_has_role(current_user, r.oid, 'SET')" + ); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain( + "pg_catalog.pg_has_role(current_user, r.oid, 'USAGE')" + ); + expect(RUNTIME_ROLE_SAFETY_SQL).not.toContain( + "pg_catalog.pg_has_role(current_user, r.oid, 'MEMBER')" + ); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain('pg_catalog.current_database()'); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain( + "pg_catalog.has_database_privilege(r.rolname, d.oid, 'CREATE')" + ); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain( + "pg_catalog.has_database_privilege(r.rolname, d.oid, 'TEMP')" + ); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain( + 'd.oid <> current_database.oid' + ); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain( + "pg_catalog.has_database_privilege(r.rolname, d.oid, 'CONNECT')" + ); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain("'pg_catalog.pg_constraint'::regclass::oid"); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain("index_class.relkind IN ('i', 'I')"); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain('FROM stored_dependency_closure closure'); + }); + + it.each(['RELATION', 'SEQUENCE', 'FUNCTION', 'TYPE'])('rejects %s ownership', async (objectKind) => { + const { pool } = poolWithRow({ + ...safeRow, + object_owner_violations: [{ + rolname: 'tenant_user', + nspname: 'tenant_public', + object_name: 'owned_object', + object_kind: objectKind + }] + }); + await expect(assertRuntimeRoleSafety(pool, ['tenant_user'], ['tenant_public'])).rejects.toThrow( + `tenant_user owns ${objectKind} tenant_public.owned_object` + ); + }); + + it('rejects stored expressions that reach a privileged helper', async () => { + const { pool } = poolWithRow({ + ...safeRow, + stored_dependency_violations: [{ + nspname: 'tenant_public', + object_name: 'documents:stamp_owner', + reason: 'STORED EXPRESSION CALLS SECURITY DEFINER', + dependency: 'hidden_private.lookup_owner' + }] + }); + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + 'STORED EXPRESSION CALLS SECURITY DEFINER from tenant_public.documents:stamp_owner to hidden_private.lookup_owner' + ); + }); + + it('rejects missing roles and schemas instead of silently weakening the check', async () => { + const { pool } = poolWithRow({ + ...safeRow, + missing_roles: ['tenant_user'], + missing_schemas: ['tenant_public'] + }); + await expect(assertRuntimeRoleSafety(pool, ['tenant_user'], ['tenant_public'])).rejects.toThrow( + 'request role tenant_user does not exist' + ); + }); + + it('coalesces concurrent checks and bounds successful-result reuse from completion', async () => { + const now = jest.spyOn(Date, 'now').mockReturnValue(1_000); + const reuseOptions = { maxSuccessAgeMs: MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS }; + let releaseFirst!: () => void; + const first = new Promise((resolve) => { + releaseFirst = resolve; + }); + const query = jest.fn() + .mockImplementationOnce(async () => { + await first; + return { rows: [safeRow] }; + }) + .mockResolvedValue({ rows: [safeRow] }); + const client = { + query: jest.fn(async (sql: string) => sql === RUNTIME_ROLE_SAFETY_SQL + ? query() + : { rows: [] }), + release: jest.fn() + }; + const pool = { connect: jest.fn(async () => client) } as unknown as Pool; + + const one = ensureRuntimeRoleSafety(pool, [], ['tenant_public'], [], reuseOptions); + const concurrent = ensureRuntimeRoleSafety( + pool, + [], + ['tenant_public'], + [], + reuseOptions + ); + expect(pool.connect).toHaveBeenCalledTimes(1); + now.mockReturnValue(1_200); + releaseFirst(); + await Promise.all([one, concurrent]); + expect(query).toHaveBeenCalledTimes(1); + + now.mockReturnValue(1_200 + MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS - 1); + await ensureRuntimeRoleSafety(pool, [], ['tenant_public'], [], reuseOptions); + expect(query).toHaveBeenCalledTimes(1); + + now.mockReturnValue(1_200 + MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS); + await ensureRuntimeRoleSafety(pool, [], ['tenant_public'], [], reuseOptions); + expect(query).toHaveBeenCalledTimes(2); + now.mockRestore(); + }); + + it('defaults to a zero-age policy and supports explicit invalidation', async () => { + const client = { + query: jest.fn(async (sql: string) => sql === RUNTIME_ROLE_SAFETY_SQL + ? { rows: [safeRow] } + : { rows: [] }), + release: jest.fn() + }; + const pool = { connect: jest.fn(async () => client) } as unknown as Pool; + + expect(DEFAULT_RUNTIME_ROLE_SAFETY_MAX_AGE_MS).toBe(0); + await ensureRuntimeRoleSafety(pool, [], ['tenant_public']); + await ensureRuntimeRoleSafety(pool, [], ['tenant_public']); + expect(pool.connect).toHaveBeenCalledTimes(2); + + await ensureRuntimeRoleSafety( + pool, + [], + ['tenant_public'], + [], + { maxSuccessAgeMs: MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS } + ); + expect(pool.connect).toHaveBeenCalledTimes(2); + invalidateRuntimeRoleSafety(pool); + await ensureRuntimeRoleSafety(pool, [], ['tenant_public']); + expect(pool.connect).toHaveBeenCalledTimes(3); + + await refreshRuntimeRoleSafety(pool, [], ['tenant_public']); + expect(pool.connect).toHaveBeenCalledTimes(4); + }); + + it('never caches a failed catalog audit', async () => { + let auditAttempts = 0; + const client = { + query: jest.fn(async (sql: string) => { + if (sql !== RUNTIME_ROLE_SAFETY_SQL) return { rows: [] }; + auditAttempts += 1; + if (auditAttempts === 1) throw new Error('catalog unavailable'); + return { rows: [safeRow] }; + }), + release: jest.fn() + }; + const pool = { connect: jest.fn(async () => client) } as unknown as Pool; + + await expect( + ensureRuntimeRoleSafety(pool, [], ['tenant_public']) + ).rejects.toThrow('catalog unavailable'); + await expect( + ensureRuntimeRoleSafety(pool, [], ['tenant_public']) + ).resolves.toBeUndefined(); + + expect(auditAttempts).toBe(2); + expect(pool.connect).toHaveBeenCalledTimes(2); + expect(client.release).toHaveBeenNthCalledWith(1, true); + }); + + it('reports actual checks separately from coalesced and reused callers', async () => { + const before = getRuntimeRoleSafetyStats(); + const reuseOptions = { maxSuccessAgeMs: MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS }; + let resolveAudit!: () => void; + const auditBlocked = new Promise((resolve) => { + resolveAudit = resolve; + }); + const client = { + query: jest.fn(async (sql: string) => { + if (sql !== RUNTIME_ROLE_SAFETY_SQL) return { rows: [] }; + await auditBlocked; + return { rows: [safeRow] }; + }), + release: jest.fn() + }; + const pool = { connect: jest.fn(async () => client) } as unknown as Pool; + + const first = ensureRuntimeRoleSafety( + pool, + [], + ['tenant_public'], + [], + reuseOptions + ); + const coalesced = ensureRuntimeRoleSafety( + pool, + [], + ['tenant_public'], + [], + reuseOptions + ); + resolveAudit(); + await Promise.all([first, coalesced]); + await ensureRuntimeRoleSafety(pool, [], ['tenant_public'], [], reuseOptions); + + const after = getRuntimeRoleSafetyStats(); + expect(after.checksStarted - before.checksStarted).toBe(1); + expect(after.checksSucceeded - before.checksSucceeded).toBe(1); + expect(after.checksFailed - before.checksFailed).toBe(0); + expect(after.inFlightCoalesces - before.inFlightCoalesces).toBe(1); + expect(after.successfulResultReuses - before.successfulResultReuses).toBe(1); + expect(after.durationMsTotal).toBeGreaterThanOrEqual(before.durationMsTotal); + }); + + it('rejects attempts to extend the successful-audit freshness bound', () => { + const { pool } = poolWithRow(safeRow); + expect(() => ensureRuntimeRoleSafety( + pool, + [], + ['tenant_public'], + [], + { maxSuccessAgeMs: MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS + 1 } + )).toThrow('maxSuccessAgeMs must be an integer between 0 and'); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/scoped-introspection.test.ts b/graphql/server/src/middleware/__tests__/scoped-introspection.test.ts new file mode 100644 index 0000000000..a86789e8df --- /dev/null +++ b/graphql/server/src/middleware/__tests__/scoped-introspection.test.ts @@ -0,0 +1,112 @@ +import { createHash } from 'node:crypto'; + +import { + makeIntrospectionQuery, + makeSchemaScopedIntrospectionQuery +} from 'pg-introspection'; + +const makeScopedQueryWithCatalogTypes = makeSchemaScopedIntrospectionQuery as unknown as ( + schemas: readonly string[], + options?: { + catalogTypes?: 'all' | 'dependency-closure'; + capabilityExtensions?: readonly string[]; + } +) => ReturnType; + +describe('schema-scoped PostgreSQL introspection', () => { + it('preserves the stock query byte for byte', () => { + const stock = makeIntrospectionQuery(); + expect(stock).toHaveLength(7332); + expect(createHash('sha256').update(stock).digest('hex')).toBe( + 'c0ed817b912f78e1ea68c70d89ff4b7f9cb4c02d88112a69ac4109d5b996e4c5' + ); + }); + + it('keeps schema names in bind values and includes dependency closure', () => { + const schemas = ['tenant_a', "tenant_'quoted", 'tenant_a']; + const query = makeSchemaScopedIntrospectionQuery(schemas); + + expect(query.values).toEqual([['tenant_a', "tenant_'quoted"], []]); + expect(query.text).toContain('$1::text[]'); + expect(query.text).toContain('$2::text[]'); + expect(query.text).not.toContain('tenant_a'); + expect(query.text).not.toContain("tenant_'quoted"); + expect(query.text).toContain('object_closure(object_class, object_id)'); + expect(query.text).toContain('installed_extensions(_id, extnamespace)'); + expect(query.text).toContain('pg_depend.refclassid'); + expect(query.text).toContain("where deptype IN ('a', 'e')"); + expect(query.text).toContain('pg_constraint.conrelid = object_closure.object_id'); + expect(query.text).toContain('pg_proc.proallargtypes'); + expect(query.text).toContain('pg_type.typbasetype'); + expect(query.text).toContain('pg_inherits.inhparent'); + expect(query.text).toContain('pg_extension.extnamespace'); + expect(query.text).toContain( + 'pg_extension.oid = any (array(select installed_extensions._id' + ); + // ACL evaluation walks every grant entry, including roles unrelated to the + // runtime login. Keep the stock role and membership result sets complete so + // PgRBACPlugin cannot fail on a retained object's unrelated ACL grantee. + expect(query.text).toContain('from pg_catalog.pg_roles\n ),'); + expect(query.text).toContain('where roleid in (select roles._id from roles)'); + expect(query.text).not.toContain('pg_roles.rolname = current_user'); + expect(query.text).not.toContain('and member in (select roles._id from roles)'); + expect(query.text).toMatch(/from pg_catalog\.pg_language\s+where true/); + expect(query.text).toMatch(/from pg_catalog\.pg_am\s+where true/); + expect(query.text).not.toContain('namespace_closure'); + expect(query.text).not.toContain('pg_inherits.inhparent = object_closure.object_id'); + expect(query.text).toContain( + "or pg_type.typnamespace = 'pg_catalog'::regnamespace" + ); + }); + + it('can retain only catalog types reached by the dependency closure', () => { + const query = makeScopedQueryWithCatalogTypes(['tenant_a'], { + catalogTypes: 'dependency-closure', + capabilityExtensions: ['pg_trgm', 'vector', 'pg_trgm'] + }); + + expect(query.values).toEqual([['tenant_a'], ['pg_trgm', 'vector']]); + expect(query.text).not.toContain('pg_trgm'); + expect(query.text).not.toContain('vector'); + expect(query.text).toContain( + 'pg_extension.extname in (\n select capability_extension_names.extension_name' + ); + expect(query.text).toContain( + "pg_type.oid = any (array(select object_id from object_closure where object_class = 'pg_catalog.pg_type'::regclass))" + ); + expect(query.text).not.toContain( + "or pg_type.typnamespace = 'pg_catalog'::regnamespace" + ); + expect(query.text).toContain( + 'retained_index_support_objects(object_class, object_id)' + ); + expect(query.text).toContain('retained_index_metadata.indclass::oid[]'); + expect(query.text).toContain("'pg_catalog.pg_opclass'::regclass::oid"); + expect(query.text).toContain("'pg_catalog.pg_opfamily'::regclass::oid"); + expect(query.text).toContain("'pg_catalog.pg_operator'::regclass::oid"); + expect(query.text).toContain('pg_catalog.pg_amop'); + expect(query.text).toContain('pg_catalog.pg_amproc'); + expect(query.text).toContain('retained_index_metadata.indcollation::oid[]'); + }); + + it('rejects empty and system-schema scopes', () => { + expect(() => makeSchemaScopedIntrospectionQuery([])).toThrow( + 'requires at least one schema' + ); + expect(() => makeSchemaScopedIntrospectionQuery(['pg_catalog'])).toThrow( + 'cannot expose system schema' + ); + expect(() => makeSchemaScopedIntrospectionQuery(['information_schema'])).toThrow( + 'cannot expose system schema' + ); + expect(() => makeScopedQueryWithCatalogTypes(['tenant_a'], { + catalogTypes: 'unknown' + } as never)).toThrow('Unsupported schema-scoped catalog type policy'); + expect(() => makeScopedQueryWithCatalogTypes(['tenant_a'], { + catalogType: 'dependency-closure' + } as never)).toThrow('Unsupported schema-scoped introspection option'); + expect(() => makeScopedQueryWithCatalogTypes(['tenant_a'], { + capabilityExtensions: [' pg_trgm'] + })).toThrow('must contain exact non-empty extension names'); + }); +}); diff --git a/graphql/server/src/middleware/api.ts b/graphql/server/src/middleware/api.ts index 24f28e9228..51a2a4f08e 100644 --- a/graphql/server/src/middleware/api.ts +++ b/graphql/server/src/middleware/api.ts @@ -10,12 +10,24 @@ import { Logger } from '@pgpmjs/logger'; import { svcCache } from '@pgpmjs/server-utils'; import { NextFunction, Request, Response } from 'express'; import { Pool } from 'pg'; -import { getPgPool } from 'pg-cache'; +import { + acquirePgPool, + getPgPoolIdentity, + PG_POOL_CAPACITY_ERROR_CODE, + type PgPoolLease +} from 'pg-cache'; import errorPage50x from '../errors/50x'; import errorPage404Message from '../errors/404-message'; import { ApiConfigResult, ApiError, ApiOptions, ApiStructure, AuthSettings, DatabaseSettings, PubkeyChallengeSettings, RlsModule, WebauthnSettings } from '../types'; -import { getRoutingSchema, isValidSchemaName, resolveRoute, routeToApiStructure } from './routing'; +import { authorizeInternalRequest } from './internal-request'; +import { + getRoutingSchema, + isValidPhysicalSchemaName, + isValidSchemaName, + resolveRoute, + routeToApiStructure +} from './routing'; const log = new Logger('api'); @@ -25,6 +37,118 @@ const log = new Logger('api'); const defaultRegistry: LoaderRegistry = createDefaultRegistry(); +const SVC_CACHE_CONTRACT_VERSION = 'constructive-routing-cache:v1'; + +interface SvcCacheContract { + routingPoolIdentity: string; + routingSchema: string; + serviceKey: string; +} + +const getSvcCacheContract = ( + opts: ApiOptions, + serviceKey: string +): SvcCacheContract => ({ + routingPoolIdentity: getPgPoolIdentity(opts.pg, { + purpose: 'routing-request-control', + sanitizeOnCheckout: true + }), + routingSchema: getRoutingSchema(opts), + serviceKey +}); + +export const getSvcCacheKey = ( + opts: ApiOptions, + serviceKey: string +): string => { + const contract = getSvcCacheContract(opts, serviceKey); + return JSON.stringify([ + SVC_CACHE_CONTRACT_VERSION, + contract.routingPoolIdentity, + contract.routingSchema, + contract.serviceKey + ]); +}; + +const parseSvcCacheKey = (key: string): SvcCacheContract | null => { + try { + const parsed = JSON.parse(key); + if ( + !Array.isArray(parsed) + || parsed.length !== 4 + || parsed[0] !== SVC_CACHE_CONTRACT_VERSION + || parsed.slice(1).some((value) => typeof value !== 'string') + ) { + return null; + } + return { + routingPoolIdentity: parsed[1], + routingSchema: parsed[2], + serviceKey: parsed[3] + }; + } catch { + return null; + } +}; + +/** Invalidate one exact physical routing entry left by an older caller. */ +export const invalidateSvcCacheKey = (cacheKey: string): boolean => { + return svcCache.delete(cacheKey); +}; + +const invalidateSvcCacheWhere = ( + predicate: (contract: SvcCacheContract, value: unknown) => boolean +): number => { + const keys: string[] = []; + for (const [key, value] of svcCache.entries()) { + const contract = parseSvcCacheKey(key); + if (contract && predicate(contract, value)) keys.push(key); + } + for (const key of keys) svcCache.delete(key); + return keys.length; +}; + +const sameSvcCacheScope = ( + left: SvcCacheContract, + right: SvcCacheContract +): boolean => + left.routingPoolIdentity === right.routingPoolIdentity + && left.routingSchema === right.routingSchema; + +export const invalidateSvcCacheForService = ( + opts: ApiOptions, + serviceKey: string +): number => { + const expected = getSvcCacheContract(opts, serviceKey); + return invalidateSvcCacheWhere((contract) => + sameSvcCacheScope(contract, expected) + && contract.serviceKey === serviceKey + ); +}; + +export const invalidateSvcCacheForDatabase = ( + opts: ApiOptions, + databaseId: string +): number => { + const expected = getSvcCacheContract(opts, ''); + const apiPrefix = `api:${databaseId}:`; + const schemataPrefix = `schemata:${databaseId}:`; + const metaKey = `metaschema:api:${databaseId}`; + return invalidateSvcCacheWhere((contract, value) => { + if (!sameSvcCacheScope(contract, expected)) return false; + const cachedDatabaseId = (value as { databaseId?: unknown })?.databaseId; + return cachedDatabaseId === databaseId + || contract.serviceKey.startsWith(apiPrefix) + || contract.serviceKey.startsWith(schemataPrefix) + || contract.serviceKey === metaKey; + }); +}; + +/** Clear process-wide routing metadata and retire every in-flight publication. */ +export const clearSvcCache = (): void => { + svcCache.clear(); +}; + // ============================================================================= // SQL Queries (API resolution only — module queries now live in loaders) // ============================================================================= @@ -44,12 +168,15 @@ const scopedApiNameLookupSql = (routingSchema: string): string => ` a.is_published as is_public, COALESCE(array_agg(s.schema_name) FILTER (WHERE s.schema_name IS NOT NULL), '{}') as schemas FROM "${routingSchema}".apis a - LEFT JOIN "${routingSchema}".api_schemas aps ON a.id = aps.api_id - LEFT JOIN metaschema_public.schema s ON aps.schema_id = s.id + LEFT JOIN "${routingSchema}".api_schemas aps + ON a.id = aps.api_id + AND aps.database_id = a.database_id + LEFT JOIN metaschema_public.schema s + ON aps.schema_id = s.id + AND s.database_id = a.database_id WHERE a.database_id = $1 AND a.name = $2 GROUP BY a.id, a.database_id, a.dbname, a.role_name, a.anon_role, a.is_published - LIMIT 1 `; // ============================================================================= @@ -68,7 +195,10 @@ interface ApiRow { interface ResolveContext { opts: ApiOptions; + registry: LoaderRegistry; pool: Pool; + routingPoolIdentity: string; + leases: PgPoolLease[]; domain: string; subdomain: string | null; cacheKey: string; @@ -77,7 +207,6 @@ interface ResolveContext { } type ResolutionMode = - | 'schemata-header' | 'api-name-header' | 'meta-schema-header' | 'scoped-route'; @@ -85,7 +214,6 @@ type ResolutionMode = type PrivateHeaderMode = Exclude; interface RoutingHeaders { - schemata?: string; apiName?: string; metaSchema?: string; databaseId?: string; @@ -104,22 +232,42 @@ interface ResolvedModuleSettings { webauthnSettings?: WebauthnSettings; } +export class MissingDatabaseFeatureContractError extends Error { + readonly code = 'GRAPHILE_DATABASE_FEATURE_CONTRACT_MISSING'; + + constructor(databaseId: string, apiId: string) { + super( + `No exact database feature contract resolved for database ${databaseId} and API ${apiId}` + ); + this.name = 'MissingDatabaseFeatureContractError'; + } +} + /** * Build a LoaderContext from the API row and options. * This is used to resolve per-database module settings via the loader registry. */ const buildLoaderContext = ( routingPool: Pool, + routingPoolIdentity: string, opts: ApiOptions, - row: ApiRow + row: ApiRow, + leases: PgPoolLease[] ): LoaderContext => { // Scoped APIs leave dbname NULL when their schemas live in the serving // database (pooled tenants); fall back to the server's own database. const dbname = row.dbname || opts.pg?.database || ''; + const tenantLease = acquirePgPool( + { ...opts.pg, database: dbname }, + { purpose: 'tenant-request-control', sanitizeOnCheckout: true } + ); + leases.push(tenantLease); return { routingPool, + routingPoolIdentity, routingSchema: getRoutingSchema(opts), - tenantPool: getPgPool({ ...opts.pg, database: dbname }), + tenantPool: tenantLease.pool, + tenantPoolIdentity: tenantLease.identity, databaseId: row.database_id, apiId: row.api_id, dbname @@ -150,6 +298,13 @@ const resolveModuleSettings = async ( registry.resolve('webauthnSettings', ctx) ]); + // These flags select executable plugins, realtime, uploads, and search. A + // missing metadata row must not silently expand the surface through preset + // defaults on an otherwise authoritative tenant route. + if (!databaseSettings) { + throw new MissingDatabaseFeatureContractError(ctx.databaseId, ctx.apiId ?? ''); + } + return { rlsModule, authSettings, @@ -182,18 +337,13 @@ const assertDatabaseId = (result: ApiStructure): void => { } }; -const parseCommaSeparatedHeader = (value: string): string[] => - value.split(',').map((s) => s.trim()).filter(Boolean); - const getPrivateHeaderMode = (headers: RoutingHeaders): PrivateHeaderMode | null => { if (headers.apiName) return 'api-name-header'; - if (headers.schemata) return 'schemata-header'; if (headers.metaSchema) return 'meta-schema-header'; return null; }; const getRoutingHeaders = (req: Request): RoutingHeaders => ({ - schemata: req.get('X-Schemata'), apiName: req.get('X-Api-Name'), metaSchema: req.get('X-Meta-Schema'), databaseId: req.get('X-Database-Id') @@ -217,15 +367,12 @@ export const getSvcKey = (opts: ApiOptions, req: Request): string => { const { domain, subdomains } = getUrlDomains(req); const baseKey = subdomains.filter((n) => n !== 'www').concat(domain).join('.'); - if (opts.api?.isPublic === false) { + if (opts.api?.isPublic === false && req.internalTrusted === true) { const headers = getRoutingHeaders(req); const mode = getPrivateHeaderMode(headers); if (mode === 'api-name-header') { return `api:${headers.databaseId}:${headers.apiName}`; } - if (mode === 'schemata-header') { - return `schemata:${headers.databaseId}:${headers.schemata}`; - } if (mode === 'meta-schema-header') { return `metaschema:api:${headers.databaseId}`; } @@ -236,9 +383,9 @@ export const getSvcKey = (opts: ApiOptions, req: Request): string => { const toApiStructure = (row: ApiRow, opts: ApiOptions, settings: ResolvedModuleSettings = {}): ApiStructure => ({ apiId: row.api_id, dbname: row.dbname || opts.pg?.database || '', - anonRole: row.anon_role || 'anon', - roleName: row.role_name || 'authenticated', - schema: row.schemas || [], + anonRole: row.anon_role, + roleName: row.role_name, + schema: row.schemas, rlsModule: settings.rlsModule, domains: [], databaseId: row.database_id, @@ -250,14 +397,31 @@ const toApiStructure = (row: ApiRow, opts: ApiOptions, settings: ResolvedModuleS webauthnSettings: settings.webauthnSettings }); +const isExactApiRow = (row: ApiRow, requestedDatabaseId: string): boolean => + typeof row.api_id === 'string' + && row.api_id.length > 0 + && row.database_id === requestedDatabaseId + && typeof row.role_name === 'string' + && row.role_name.length > 0 + && typeof row.anon_role === 'string' + && row.anon_role.length > 0 + && typeof row.is_public === 'boolean' + && Array.isArray(row.schemas) + && row.schemas.length > 0 + && row.schemas.every(isValidPhysicalSchemaName) + && new Set(row.schemas).size === row.schemas.length; + const createAdminStructure = ( opts: ApiOptions, schemas: string[], databaseId?: string ): ApiStructure => ({ dbname: opts.pg?.database ?? '', - anonRole: 'administrator', - roleName: 'administrator', + // Private header/meta-schema surfaces must be able to use a dedicated + // non-BYPASSRLS execution role. Keep the legacy default for compatibility; + // production admission will reject it unless operators configure safe roles. + anonRole: opts.api?.anonRole ?? 'administrator', + roleName: opts.api?.roleName ?? 'administrator', schema: schemas, domains: [], databaseId, @@ -288,7 +452,14 @@ const queryByApiName = async ( return null; } const result = await pool.query(scopedApiNameLookupSql(routingSchema), [databaseId, name]); - return result.rows[0] ?? null; + if (result.rows.length !== 1) { + log.warn( + `[api-name-lookup] expected one exact API row for databaseId=${databaseId}; ` + + `received ${result.rows.length}` + ); + return null; + } + return result.rows[0]; }; // ============================================================================= @@ -304,35 +475,25 @@ const determineMode = (ctx: ResolveContext): ResolutionMode => { return 'scoped-route'; }; -const resolveSchemataHeader = async ( - ctx: ResolveContext, - validatedSchemas: string[] -): Promise => { - const { opts, headers } = ctx; - const headerSchemas = parseCommaSeparatedHeader(headers.schemata!); - const validSet = new Set(validatedSchemas); - const validHeaderSchemas = headerSchemas.filter((s) => validSet.has(s)); - - if (validHeaderSchemas.length === 0) { - return { errorHtml: 'No valid schemas found for the supplied X-Schemata header.' }; - } - - return createAdminStructure(opts, validHeaderSchemas, headers.databaseId); -}; - const resolveApiNameHeader = async (ctx: ResolveContext): Promise => { const { opts, pool, headers } = ctx; if (!headers.databaseId) return null; const row = await queryByApiName(pool, opts, headers.databaseId, headers.apiName!); - if (!row) { + if (!row || !isExactApiRow(row, headers.databaseId)) { log.debug(`[api-name-lookup] No API found for databaseId=${headers.databaseId} name=${headers.apiName}`); return null; } - const loaderCtx = buildLoaderContext(pool, opts, row); - const settings = await resolveModuleSettings(defaultRegistry, loaderCtx); + const loaderCtx = buildLoaderContext( + pool, + ctx.routingPoolIdentity, + opts, + row, + ctx.leases + ); + const settings = await resolveModuleSettings(ctx.registry, loaderCtx); log.debug(`[api-name-lookup] resolved schemas: [${row.schemas?.join(', ')}], rlsModule: ${settings.rlsModule ? 'found' : 'none'}, authSettings: ${settings.authSettings ? 'found' : 'none'}`); return toApiStructure(row, opts, settings); }; @@ -366,7 +527,7 @@ const resolveScopedRoute = async (ctx: ResolveContext): Promise => { - const pool = getPgPool(opts.pg); + authorizeInternalRequest(opts, req); const { domain, subdomains } = getUrlDomains(req); const subdomain = getSubdomain(subdomains); - const cacheKey = getSvcKey(opts, req); - - req.svc_key = cacheKey; - - // Check cache first - if (svcCache.has(cacheKey)) { - log.debug(`Cache HIT for key=${cacheKey}`); - return svcCache.get(cacheKey) as ApiStructure; - } - - log.debug(`Cache MISS for key=${cacheKey}, resolving API`); - - const ctx: ResolveContext = { - opts, - pool, - domain, - subdomain, - cacheKey, - headers: getRoutingHeaders(req), - host: req.get('host') || '' - }; - - // Validate schemas upfront for modes that need them - const apiOpts = opts.api || {}; - const headerSchemas = ctx.headers.schemata ? parseCommaSeparatedHeader(ctx.headers.schemata) : []; - const candidateSchemas = - apiOpts.isPublic === false && headerSchemas.length - ? [...new Set([...(apiOpts.metaSchemas || []), ...headerSchemas])] - : apiOpts.metaSchemas || []; - - const validatedSchemas = await validateSchemata(pool, candidateSchemas); - - if (validatedSchemas.length === 0) { - const source = headerSchemas.length ? headerSchemas : apiOpts.metaSchemas || []; - const label = headerSchemas.length ? 'X-Schemata' : 'metaSchemas'; - const error = new Error(`No valid schemas found. Configured ${label}: [${source.join(', ')}]`) as Error & { code?: string }; - error.code = 'NO_VALID_SCHEMAS'; - throw error; - } + const serviceKey = getSvcKey(opts, req); + const cacheKey = getSvcCacheKey(opts, serviceKey); + + req.svc_key = serviceKey; + req.svc_cache_key = cacheKey; + + // Hostname and private-selector routing is an authorization boundary. LISTEN + // notifications are lossy across disconnects, so cached metadata cannot be + // authoritative after a domain or API is reassigned. Resolve every request; + // the independently keyed PostGraphile build cache still provides the large + // memory and build-latency win once this exact contract is known. + log.debug(`Authoritatively resolving API for key=${cacheKey}`); + const leases: PgPoolLease[] = []; + + try { + const routingLease = acquirePgPool(opts.pg, { + purpose: 'routing-request-control', + sanitizeOnCheckout: true + }); + leases.push(routingLease); + const pool = routingLease.pool; + const ctx: ResolveContext = { + opts, + pool, + routingPoolIdentity: routingLease.identity, + leases, + domain, + subdomain, + cacheKey, + headers: getRoutingHeaders(req), + host: req.get('host') || '', + registry + }; + + // Validate schemas upfront for modes that need them + const apiOpts = opts.api || {}; + const candidateSchemas = apiOpts.metaSchemas || []; + + const validatedSchemas = await validateSchemata(pool, candidateSchemas); + + if (validatedSchemas.length === 0) { + const source = apiOpts.metaSchemas || []; + const error = new Error(`No valid schemas found. Configured metaSchemas: [${source.join(', ')}]`) as Error & { code?: string }; + error.code = 'NO_VALID_SCHEMAS'; + throw error; + } - // Route to appropriate resolver based on mode - const mode = determineMode(ctx); - let result: ApiConfigResult; + // Route to appropriate resolver based on mode + const mode = determineMode(ctx); + let result: ApiConfigResult; - switch (mode) { - case 'schemata-header': - result = await resolveSchemataHeader(ctx, validatedSchemas); - break; + switch (mode) { + case 'api-name-header': + result = await resolveApiNameHeader(ctx); + break; - case 'api-name-header': - result = await resolveApiNameHeader(ctx); - break; + case 'meta-schema-header': + result = resolveMetaSchemaHeader(ctx, validatedSchemas); + break; - case 'meta-schema-header': - result = resolveMetaSchemaHeader(ctx, validatedSchemas); - break; + case 'scoped-route': + result = await resolveScopedRoute(ctx); + break; + } - case 'scoped-route': - result = await resolveScopedRoute(ctx); - break; - } + // Assert the complete routing identity before any downstream middleware. + // Deliberately do not publish this result to svcCache; see above. + if (result && !isApiError(result)) { + assertDatabaseId(result); + } - // Cache successful results - if (result && !isApiError(result)) { - assertDatabaseId(result); - svcCache.set(cacheKey, result); + return result; + } finally { + for (let i = leases.length - 1; i >= 0; i--) { + leases[i].release(); + } } - - return result; }; // ============================================================================= // Express Middleware // ============================================================================= -export const createApiMiddleware = (opts: ApiOptions) => { +export const createApiMiddleware = ( + opts: ApiOptions, + registry: LoaderRegistry = defaultRegistry +) => { return async (req: Request, res: Response, next: NextFunction): Promise => { log.debug(`[api-middleware] ${req.method} ${req.path}`); try { - const apiConfig = await getApiConfig(opts, req); + const apiConfig = await getApiConfig(opts, req, registry); if (isApiError(apiConfig)) { res.status(404).send(errorPage404Message('API not found', apiConfig.errorHtml)); @@ -497,6 +670,11 @@ export const createApiMiddleware = (opts: ApiOptions) => { } catch (error: unknown) { const err = error as Error & { code?: string }; + if (err.code === 'INTERNAL_REQUEST_FORBIDDEN') { + res.status(403).send('Forbidden'); + return; + } + if (err.code === 'NO_VALID_SCHEMAS') { res.status(404).send(errorPage404Message(err.message)); return; @@ -508,6 +686,11 @@ export const createApiMiddleware = (opts: ApiOptions) => { return; } + if (err.code === PG_POOL_CAPACITY_ERROR_CODE) { + next(err); + return; + } + if (err.message?.includes('does not exist')) { res.status(404).send(errorPage404Message("The resource you're looking for does not exist.")); return; diff --git a/graphql/server/src/middleware/auth.ts b/graphql/server/src/middleware/auth.ts index ef6da3f3a2..71c8e8d9ea 100644 --- a/graphql/server/src/middleware/auth.ts +++ b/graphql/server/src/middleware/auth.ts @@ -1,11 +1,19 @@ import './types'; // for Request type import { errors } from '@constructive-io/errors'; +import { + quoteQualifiedSqlIdentifier, + SECURITY_GUC_KEYS +} from '@constructive-io/express-context'; import { getNodeEnv } from '@pgpmjs/env'; import { Logger } from '@pgpmjs/logger'; import { PgpmOptions } from '@pgpmjs/types'; import { NextFunction, Request, RequestHandler, Response } from 'express'; -import { getPgPool } from 'pg-cache'; +import { + acquirePgPool, + PG_POOL_CAPACITY_ERROR_CODE, + type PgPoolLease +} from 'pg-cache'; import pgQueryContext from 'pg-query-context'; import { respondWithGraphQLError } from '../errors/graphql-response'; @@ -19,6 +27,23 @@ const SESSION_COOKIE_NAME = 'constructive_session'; /** Cookie name for trusted device tracking. */ const DEVICE_TOKEN_COOKIE_NAME = 'constructive_device_token'; +/** Complete transaction-local context for the sanitized authentication lane. */ +export const buildAuthenticationContext = ( + req: Request, + api: NonNullable +): Record => ({ + ...Object.fromEntries(SECURITY_GUC_KEYS.map((key) => [key, ''])), + 'jwt.claims.api_id': api.apiId ?? '', + 'jwt.claims.database_id': api.databaseId ?? '', + 'jwt.claims.ip_address': req.clientIp ?? '', + 'jwt.claims.origin': req.get('origin') ?? '', + 'jwt.claims.user_agent': req.get('User-Agent') ?? '', + 'request.id': req.requestId ?? '', + 'row_security': 'on', + 'search_path': 'pg_catalog', + 'transaction_read_only': 'on' +}); + /** * Extract a named cookie value from the raw Cookie header. * Avoids pulling in cookie-parser as a dependency. @@ -45,10 +70,6 @@ export const createAuthenticateMiddleware = ( return; } - const pool = getPgPool({ - ...opts.pg, - database: api.dbname, - }); const rlsModule = api.rlsModule; log.info( @@ -59,6 +80,18 @@ export const createAuthenticateMiddleware = ( ); if (!rlsModule) { + if (opts.server?.strictAuth) { + log.error('[auth] Strict authentication requires an RLS module'); + respondWithGraphQLError( + res, + errors.INTERNAL_FAILURE({ + details: isDev() + ? 'Strict authentication requires an RLS module' + : 'authentication failed' + }) + ); + return; + } log.info('[auth] No RLS module configured, skipping auth'); return next(); } @@ -71,6 +104,19 @@ export const createAuthenticateMiddleware = ( `[auth] strictAuth=${opts.server?.strictAuth ?? false}, authFn=${authFn ?? 'none'}` ); + if (!authFn || !rlsModule.privateSchema.schemaName) { + log.error('[auth] RLS authentication configuration is incomplete'); + respondWithGraphQLError( + res, + errors.INTERNAL_FAILURE({ + details: isDev() + ? 'RLS authentication configuration is incomplete' + : 'authentication failed' + }) + ); + return; + } + if (authFn && rlsModule.privateSchema.schemaName) { const { authorization = '' } = req.headers; const [authType, authToken] = authorization.split(' '); @@ -90,23 +136,38 @@ export const createAuthenticateMiddleware = ( if (effectiveToken) { log.info(`[auth] Processing ${tokenSource} authentication`); - const context: Record = { - 'jwt.claims.ip_address': req.clientIp, - }; + const context = buildAuthenticationContext(req, api); - if (req.get('origin')) { - context['jwt.claims.origin'] = req.get('origin'); - } - if (req.get('User-Agent')) { - context['jwt.claims.user_agent'] = req.get('User-Agent'); + let authQuery: string; + try { + authQuery = `SELECT * FROM ${quoteQualifiedSqlIdentifier( + rlsModule.privateSchema.schemaName, + authFn, + 'authentication function' + )}($1)`; + } catch (e: unknown) { + const message = e instanceof Error + ? e.message + : 'invalid authentication function'; + log.error('[auth] Invalid authentication function metadata:', message); + respondWithGraphQLError( + res, + errors.INTERNAL_FAILURE({ + details: isDev() ? message : 'authentication failed' + }) + ); + return; } - - const authQuery = `SELECT * FROM "${rlsModule.privateSchema.schemaName}"."${authFn}"($1)`; log.info(`[auth] Executing auth query: ${authQuery}`); + let poolLease: PgPoolLease | undefined; try { + poolLease = acquirePgPool({ + ...opts.pg, + database: api.dbname, + }, { purpose: 'tenant-request-control', sanitizeOnCheckout: true }); const result = await pgQueryContext({ - client: pool, + client: poolLease.pool, context, query: authQuery, variables: [effectiveToken], @@ -123,6 +184,10 @@ export const createAuthenticateMiddleware = ( token = result.rows[0]; log.info(`[auth] Auth success: role=${token.role}, user_id=${token.user_id}`); } catch (e: any) { + if (e?.code === PG_POOL_CAPACITY_ERROR_CODE) { + next(e); + return; + } log.error('[auth] Auth error:', e.message); respondWithGraphQLError( res, @@ -131,17 +196,14 @@ export const createAuthenticateMiddleware = ( }) ); return; + } finally { + poolLease?.release(); } } else { log.info('[auth] No credential provided (no bearer token or session cookie), using anonymous auth'); } req.token = token; - } else { - log.info( - `[auth] Skipping auth: authFn=${authFn ?? 'none'}, ` + - `privateSchema=${rlsModule.privateSchema?.schemaName ?? 'none'}` - ); } // Read device token cookie for trusted device tracking diff --git a/graphql/server/src/middleware/captcha.ts b/graphql/server/src/middleware/captcha.ts index 7a4da18955..8ca26d0ddd 100644 --- a/graphql/server/src/middleware/captcha.ts +++ b/graphql/server/src/middleware/captcha.ts @@ -1,8 +1,22 @@ import './types'; // for Request type import { errors } from '@constructive-io/errors'; +import { getNodeEnv } from '@pgpmjs/env'; import { Logger } from '@pgpmjs/logger'; -import type { NextFunction, Request, RequestHandler, Response } from 'express'; +import express, { + type NextFunction, + type Request, + type RequestHandler, + type Response +} from 'express'; +import { + Kind, + parse, + type DocumentNode, + type FragmentDefinitionNode, + type OperationDefinitionNode, + type SelectionSetNode +} from 'graphql'; import { respondWithGraphQLError } from '../errors/graphql-response'; @@ -17,6 +31,9 @@ const RECAPTCHA_VERIFY_URL = 'https://www.google.com/recaptcha/api/siteverify'; */ const CAPTCHA_HEADER = 'x-captcha-token'; +/** Match Grafserv's default maximum GraphQL request length. */ +export const CAPTCHA_GRAPHQL_BODY_LIMIT_BYTES = 100_000; + /** * GraphQL mutation names that require CAPTCHA verification when enabled. * Only sign-up and password-reset are gated; normal sign-in is not. @@ -29,25 +46,181 @@ const CAPTCHA_PROTECTED_OPERATIONS = new Set([ 'requestPasswordReset', ]); +export type CaptchaOperationInspection = + | { kind: 'protected'; fields: readonly string[] } + | { kind: 'not-protected' } + | { kind: 'invalid'; reason: string }; + interface RecaptchaResponse { success: boolean; 'error-codes'?: string[]; } +export interface CaptchaMiddlewareOptions { + /** Authentication-required deployments must never disable CAPTCHA implicitly. */ + strictAuth?: boolean; + /** @internal Deterministic environment seam for focused tests. */ + nodeEnv?: ReturnType; +} + /** - * Attempt to extract the GraphQL operation name from the request body. - * Works for both JSON and already-parsed bodies. + * Parse the GraphQL request formats Grafserv accepts before CAPTCHA admission. + * Multipart requests are deliberately left to graphql-upload, which supplies + * the same object-shaped body before the CAPTCHA middleware runs. */ -const getOperationName = (req: Request): string | undefined => { - const body = (req as any).body; - if (!body) return undefined; - // Already parsed (express.json ran first) - if (typeof body === 'object' && body.operationName) { - return body.operationName; +export const createCaptchaGraphqlBodyParsers = (): RequestHandler[] => [ + express.json({ limit: CAPTCHA_GRAPHQL_BODY_LIMIT_BYTES }), + express.text({ + type: 'application/graphql', + limit: CAPTCHA_GRAPHQL_BODY_LIMIT_BYTES + }), + express.urlencoded({ + extended: false, + limit: CAPTCHA_GRAPHQL_BODY_LIMIT_BYTES + }) +]; + +const selectOperation = ( + document: DocumentNode, + operationName: string | undefined +): OperationDefinitionNode | undefined => { + const operations = document.definitions.filter( + (definition): definition is OperationDefinitionNode => + definition.kind === Kind.OPERATION_DEFINITION + ); + if (operationName === undefined) { + return operations.length === 1 ? operations[0] : undefined; + } + const matches = operations.filter( + (operation) => operation.name?.value === operationName + ); + return matches.length === 1 ? matches[0] : undefined; +}; + +const collectRootFields = ( + selectionSet: SelectionSetNode, + fragments: ReadonlyMap, + activeFragments: Set, + fields: Set +): string | undefined => { + for (const selection of selectionSet.selections) { + if (selection.kind === Kind.FIELD) { + fields.add(selection.name.value); + continue; + } + if (selection.kind === Kind.INLINE_FRAGMENT) { + const invalid = collectRootFields( + selection.selectionSet, + fragments, + activeFragments, + fields + ); + if (invalid) return invalid; + continue; + } + + const fragmentName = selection.name.value; + const fragment = fragments.get(fragmentName); + if (!fragment) return `missing fragment ${fragmentName}`; + if (activeFragments.has(fragmentName)) { + return `cyclic fragment ${fragmentName}`; + } + activeFragments.add(fragmentName); + const invalid = collectRootFields( + fragment.selectionSet, + fragments, + activeFragments, + fields + ); + activeFragments.delete(fragmentName); + if (invalid) return invalid; } return undefined; }; +/** + * Classify the selected operation from the GraphQL document itself. Operation + * labels are client-controlled and therefore never stand in for root fields. + */ +export const inspectCaptchaOperation = ( + query: unknown, + operationName: unknown +): CaptchaOperationInspection => { + if (typeof query !== 'string' || query.trim().length === 0) { + return { kind: 'invalid', reason: 'missing GraphQL query' }; + } + if ( + operationName !== undefined + && operationName !== null + && (typeof operationName !== 'string' || operationName.length === 0) + ) { + return { kind: 'invalid', reason: 'invalid GraphQL operation name' }; + } + + let document: DocumentNode; + try { + document = parse(query); + } catch { + return { kind: 'invalid', reason: 'malformed GraphQL document' }; + } + + const selected = selectOperation( + document, + typeof operationName === 'string' ? operationName : undefined + ); + if (!selected) { + return { kind: 'invalid', reason: 'ambiguous or missing GraphQL operation' }; + } + if (selected.operation !== 'mutation') return { kind: 'not-protected' }; + + const fragments = new Map(); + for (const definition of document.definitions) { + if (definition.kind !== Kind.FRAGMENT_DEFINITION) continue; + if (fragments.has(definition.name.value)) { + return { kind: 'invalid', reason: `duplicate fragment ${definition.name.value}` }; + } + fragments.set(definition.name.value, definition); + } + + const fields = new Set(); + const invalid = collectRootFields( + selected.selectionSet, + fragments, + new Set(), + fields + ); + if (invalid) return { kind: 'invalid', reason: invalid }; + + const protectedFields = [...fields] + .filter((field) => CAPTCHA_PROTECTED_OPERATIONS.has(field)) + .sort(); + return protectedFields.length > 0 + ? { kind: 'protected', fields: protectedFields } + : { kind: 'not-protected' }; +}; + +const isGraphqlPath = (req: Request): boolean => req.path === '/graphql'; + +const isWebSocketUpgrade = (req: Request): boolean => + req.method === 'GET' + && req.get('upgrade')?.trim().toLowerCase() === 'websocket'; + +const inspectHttpRequest = (req: Request): CaptchaOperationInspection => { + if (req.method === 'GET' || req.method === 'HEAD') { + return inspectCaptchaOperation(req.query?.query, req.query?.operationName); + } + + const body = (req as Request & { body?: unknown }).body; + if (typeof body === 'string') { + return inspectCaptchaOperation(body, undefined); + } + if (!body || Array.isArray(body) || typeof body !== 'object') { + return { kind: 'invalid', reason: 'invalid GraphQL request body' }; + } + const graphqlBody = body as Record; + return inspectCaptchaOperation(graphqlBody.query, graphqlBody.operationName); +}; + /** * Verify a reCAPTCHA token with Google's API. */ @@ -80,9 +253,17 @@ const verifyToken = async (token: string, secretKey: string): Promise = * Skips verification when: * - CAPTCHA is not enabled in auth settings * - The request is not a protected mutation - * - No secret key is configured server-side + * - No secret key is configured in a non-production, non-strict local server + * + * Production and strict-auth servers fail closed when tenant policy enables + * CAPTCHA but the server-side secret is missing. */ -export const createCaptchaMiddleware = (): RequestHandler => { +export const createCaptchaMiddleware = ( + options: CaptchaMiddlewareOptions = {} +): RequestHandler => { + const failClosedWithoutSecret = options.strictAuth === true + || (options.nodeEnv ?? getNodeEnv()) === 'production'; + return async (req: Request, res: Response, next: NextFunction): Promise => { const authSettings = req.api?.authSettings; @@ -91,16 +272,40 @@ export const createCaptchaMiddleware = (): RequestHandler => { return next(); } - // Only gate protected operations - const opName = getOperationName(req); - if (!opName || !CAPTCHA_PROTECTED_OPERATIONS.has(opName)) { + // WebSocket handshakes have no operation document. The generation-scoped + // onSubscribe admission hook rejects protected mutations per operation. + if (!isGraphqlPath(req) || isWebSocketUpgrade(req) || req.method === 'OPTIONS') { return next(); } + const inspection = inspectHttpRequest(req); + if (inspection.kind === 'not-protected') return next(); + if (inspection.kind === 'invalid') { + log.warn(`[captcha] Rejecting GraphQL request: ${inspection.reason}`); + respondWithGraphQLError( + res, + errors.INTERNAL_FAILURE({ details: 'authentication failed' }) + ); + return; + } + // Secret key must be set server-side (env var, not stored in DB for security) const secretKey = process.env.RECAPTCHA_SECRET_KEY; - if (!secretKey) { - log.warn('[captcha] enable_captcha is true but RECAPTCHA_SECRET_KEY env var is not set; skipping verification'); + if (!secretKey?.trim()) { + if (failClosedWithoutSecret) { + log.error( + '[captcha] enable_captcha is true but RECAPTCHA_SECRET_KEY is not configured; rejecting protected operation' + ); + respondWithGraphQLError( + res, + errors.INTERNAL_FAILURE({ details: 'authentication failed' }) + ); + return; + } + log.warn( + '[captcha] enable_captcha is true but RECAPTCHA_SECRET_KEY is not configured; ' + + 'skipping verification only for non-production, non-strict local mode' + ); return next(); } @@ -116,7 +321,7 @@ export const createCaptchaMiddleware = (): RequestHandler => { return; } - log.info(`[captcha] Verified for operation=${opName}`); + log.info(`[captcha] Verified for fields=${inspection.fields.join(',')}`); next(); }; }; diff --git a/graphql/server/src/middleware/cors.ts b/graphql/server/src/middleware/cors.ts index 8bb7cefebb..b77e1d6d30 100644 --- a/graphql/server/src/middleware/cors.ts +++ b/graphql/server/src/middleware/cors.ts @@ -6,6 +6,41 @@ import type { Request, RequestHandler } from 'express'; import type { ApiStructure } from '../types'; +export interface CorsOriginInput { + origin?: string; + fallbackOrigin?: string; + api?: ApiStructure; + requestHost?: string; +} + +/** Shared HTTP/WebSocket origin policy. Missing origins are handled by the caller. */ +export const isCorsOriginAllowed = ({ + origin, + fallbackOrigin, + api, + requestHost +}: CorsOriginInput): boolean => { + if (!origin) return false; + const fallback = fallbackOrigin?.trim(); + if (fallback === '*') return true; + if (fallback && origin.trim() === fallback) return true; + + if ([...(api?.corsOrigins ?? []), ...(api?.domains ?? [])].includes(origin)) { + return true; + } + + try { + const parsedOrigin = new URL(origin); + if (requestHost && parsedOrigin.host.toLowerCase() === requestHost.toLowerCase()) { + return true; + } + const parsed = parseUrl(parsedOrigin); + return parsed.domain === 'localhost'; + } catch { + return false; + } +}; + /** * Unified CORS middleware for Constructive API * @@ -20,47 +55,13 @@ import type { ApiStructure } from '../types'; export const cors = (fallbackOrigin?: string): RequestHandler => { // Use the cors library's dynamic origin function to decide per request const dynamicOrigin = (origin: string | undefined, callback: (err: Error | null, allow?: boolean | string) => void, req: Request) => { - // 1) Global fallback (fast path) - if (fallbackOrigin && fallbackOrigin.trim().length) { - if (fallbackOrigin.trim() === '*') { - // Reflect whatever Origin the caller sent - return callback(null, true); - } - if (origin && origin.trim() === fallbackOrigin.trim()) { - return callback(null, true); - } - // If a strict fallback origin is provided and does not match, - // continue to per-API checks below (do not immediately deny). - } - - // 2) Per-API allowlist sourced from req.api (if available) - // createApiMiddleware runs before this in server.ts, so req.api should be set const api = (req as any).api as ApiStructure | undefined; - if (api) { - // Typed cors_settings origins - const typedOrigins = api.corsOrigins || []; - const siteUrls = api.domains || []; - const listOfDomains = [...typedOrigins, ...siteUrls]; - - if (origin && listOfDomains.includes(origin)) { - return callback(null, true); - } - } - - // 3) Localhost is always allowed - if (origin) { - try { - const parsed = parseUrl(new URL(origin)); - if (parsed.domain === 'localhost') { - return callback(null, true); - } - } catch { - // ignore invalid origin - } - } - - // Default: not allowed - return callback(null, false); + return callback(null, isCorsOriginAllowed({ + origin, + fallbackOrigin, + api, + requestHost: req.get('host') + })); }; // Wrap in the cors plugin with our dynamic origin resolver diff --git a/graphql/server/src/middleware/error-handler.ts b/graphql/server/src/middleware/error-handler.ts index bbf63de194..a4a17f4853 100644 --- a/graphql/server/src/middleware/error-handler.ts +++ b/graphql/server/src/middleware/error-handler.ts @@ -3,6 +3,7 @@ import './types'; import { getNodeEnv } from '@pgpmjs/env'; import { Logger } from '@pgpmjs/logger'; import type { ErrorRequestHandler, NextFunction, Request, Response } from 'express'; +import { PG_POOL_CAPACITY_ERROR_CODE } from 'pg-cache'; import errorPage50x from '../errors/50x'; import errorPage404Message from '../errors/404-message'; @@ -38,7 +39,18 @@ const isCsrfError = (err: Error): boolean => { return typeof code === 'string' && code.startsWith('CSRF_'); }; +const isPgPoolCapacityError = (err: Error): boolean => + (err as Error & { code?: string }).code === PG_POOL_CAPACITY_ERROR_CODE; + const categorizeError = (err: Error): ErrorResponse => { + if (isPgPoolCapacityError(err)) { + return { + statusCode: 503, + code: PG_POOL_CAPACITY_ERROR_CODE, + message: 'Service temporarily unavailable', + logLevel: 'warn' + }; + } if (isApiError(err)) { return { statusCode: err.statusCode, @@ -61,6 +73,11 @@ const categorizeError = (err: Error): ErrorResponse => { }; const sendResponse = (req: Request, res: Response, { statusCode, code, message }: ErrorResponse): void => { + if (code === PG_POOL_CAPACITY_ERROR_CODE) { + res.set('Retry-After', '15'); + res.status(statusCode).json({ error: { code, message, requestId: req.requestId } }); + return; + } if (wantsJson(req)) { res.status(statusCode).json({ error: { code, message, requestId: req.requestId } }); } else { @@ -79,7 +96,9 @@ const logError = (err: Error, req: Request, level: 'warn' | 'error'): void => { clientIp: req.clientIp, }; - if (isApiError(err)) { + if (isPgPoolCapacityError(err)) { + log.warn({ event: 'pg_pool_capacity', code: PG_POOL_CAPACITY_ERROR_CODE, ...context }); + } else if (isApiError(err)) { log[level]({ event: 'api_error', code: err.code, statusCode: err.statusCode, message: err.message, ...context }); } else { log[level]({ event: 'unexpected_error', name: err.name, message: err.message, stack: isDevelopment() ? err.stack : undefined, ...context }); diff --git a/graphql/server/src/middleware/flush.ts b/graphql/server/src/middleware/flush.ts index 1ff9ee6d3f..20a0530bd3 100644 --- a/graphql/server/src/middleware/flush.ts +++ b/graphql/server/src/middleware/flush.ts @@ -1,70 +1,127 @@ import './types'; // for Request type +import type { LoaderRegistry } from '@constructive-io/express-context'; import { ConstructiveOptions } from '@constructive-io/graphql-types'; import { Logger } from '@pgpmjs/logger'; -import { svcCache } from '@pgpmjs/server-utils'; import { NextFunction, Request, Response } from 'express'; -import { graphileCache } from 'graphile-cache'; -import { getPgPool } from 'pg-cache'; +import { deleteGraphileCacheEntry, graphileCache } from 'graphile-cache'; +import { acquirePgPool } from 'pg-cache'; +import { + invalidateSvcCacheForDatabase, + invalidateSvcCacheForService, + invalidateSvcCacheKey +} from './api'; +import { invalidateInFlightBuilds } from './graphile'; import { getRoutingSchema, isValidSchemaName } from './routing'; const log = new Logger('flush'); -export const flush = async ( +const flushRequest = async ( req: Request, res: Response, - next: NextFunction + next: NextFunction, + registry?: LoaderRegistry ): Promise => { if (req.url === '/flush') { - // TODO: check bearer for a flush / special key - graphileCache.delete((req as any).svc_key); - svcCache.delete((req as any).svc_key); + if (req.internalTrusted !== true) { + res.status(403).send('Forbidden'); + return; + } + const serviceKey = req.svc_key; + // Module metadata and Graphile residents are one publication boundary. + // Retire both before acknowledging the flush; otherwise a revoked module + // configuration can outlive the schema instance it helped configure. + registry?.invalidate(req.databaseId); + if (serviceKey) invalidateInFlightBuilds({ serviceKey }); + if (req.svc_cache_key) invalidateSvcCacheKey(req.svc_cache_key); + const cacheKeys = [...graphileCache.entries()] + .filter(([, entry]) => entry.serviceKey === serviceKey) + .map(([key]) => key); + await Promise.all(cacheKeys.map((key) => deleteGraphileCacheEntry(key))); res.status(200).send('OK'); return; } return next(); }; +export const flush = ( + req: Request, + res: Response, + next: NextFunction +): Promise => flushRequest(req, res, next); + +export const createFlushMiddleware = (registry: LoaderRegistry) => ( + req: Request, + res: Response, + next: NextFunction +): Promise => flushRequest(req, res, next, registry); + export const flushService = async ( opts: ConstructiveOptions, - databaseId: string + databaseId: string, + registry?: LoaderRegistry ): Promise => { - const pgPool = getPgPool(opts.pg); log.info('flushing db ' + databaseId); + registry?.invalidate(databaseId); + invalidateInFlightBuilds({ databaseId }); + invalidateSvcCacheForDatabase(opts, databaseId); const api = new RegExp(`^api:${databaseId}:.*`); const schemata = new RegExp(`^schemata:${databaseId}:.*`); const meta = new RegExp(`^metaschema:api:${databaseId}`); - if (!opts.api.isPublic) { - graphileCache.forEach((_, k: string) => { - if (api.test(k) || schemata.test(k) || meta.test(k)) { - graphileCache.delete(k); - svcCache.delete(k); + // Evict by the authoritative database identity before consulting routing. + // Routing is fallible and may legitimately return no domains; neither case + // may leave a resident instance for the database being flushed. + const databaseCacheKeys = new Set(); + graphileCache.forEach((entry, key: string) => { + if (entry.databaseId === databaseId) { + databaseCacheKeys.add(key); + } + + if (!opts.api.isPublic) { + const serviceKey = entry.serviceKey; + if (serviceKey && (api.test(serviceKey) || schemata.test(serviceKey) || meta.test(serviceKey))) { + invalidateSvcCacheForService(opts, serviceKey); } - }); - } + } + }); + await Promise.all([...databaseCacheKeys].map((key) => deleteGraphileCacheEntry(key))); const routingSchema = getRoutingSchema(opts); if (!isValidSchemaName(routingSchema)) { log.warn(`[flush] invalid routing schema name: ${routingSchema}`); return; } - const svc = await pgPool.query( - `SELECT hostname - FROM "${routingSchema}".domains - WHERE database_id = $1`, - [databaseId] - ); + const poolLease = acquirePgPool(opts.pg, { + purpose: 'routing-request-control', + sanitizeOnCheckout: true + }); + try { + const svc = await poolLease.pool.query( + `SELECT hostname + FROM "${routingSchema}".domains + WHERE database_id = $1`, + [databaseId] + ); - if (svc.rowCount === 0) return; + if (svc.rowCount === 0) return; - for (const row of svc.rows) { - const key: string | undefined = row.hostname || undefined; - if (key) { - graphileCache.delete(key); - svcCache.delete(key); + for (const row of svc.rows) { + const key: string | undefined = row.hostname || undefined; + if (key) { + const graphileKeys = new Set(); + graphileCache.forEach((entry, cacheKey) => { + if (entry.serviceKey === key || entry.databaseId === databaseId) { + graphileKeys.add(cacheKey); + } + }); + await Promise.all([...graphileKeys].map((cacheKey) => deleteGraphileCacheEntry(cacheKey))); + invalidateSvcCacheForService(opts, key); + } } + } finally { + poolLease.release(); } }; diff --git a/graphql/server/src/middleware/graphile-build-contract.ts b/graphql/server/src/middleware/graphile-build-contract.ts new file mode 100644 index 0000000000..becc854928 --- /dev/null +++ b/graphql/server/src/middleware/graphile-build-contract.ts @@ -0,0 +1,289 @@ +import { createHash, createHmac, randomBytes } from 'node:crypto'; + +import type { ComputeConfig, StorageConfig } from '@constructive-io/express-context'; +import type { ConstructiveOptions } from '@constructive-io/graphql-types'; + +import type { DatabaseSettings } from '../types'; + +type GraphileBuildSettings = Omit< + NonNullable, + | 'realtimeSchema' + | 'realtimeNotificationMode' + | 'realtimeNotificationRoleRevalidationMs' + | 'realtimeCursorPollIntervalMs' + | 'realtimeCursorHeartbeatIntervalMs' + | 'trustCallerPresetsInProduction' +>; + +// The resident cache is process-local, and its keys are emitted in diagnostics. +// A plain digest of plugin/settings configuration could act as an offline +// verifier for a low-entropy secret captured by a caller preset. Key the digest +// per process so equality remains stable for this cache lifetime without making +// the serialized contract portable or reversible evidence. +const graphileBuildContractHmacKey = randomBytes(32); + +export interface GraphileBuildContractV1 { + version: 1; + /** Process-local identity for the exact graphile(opts) configuration owner. */ + configurationIdentity: string; + poolIdentity: string; + databaseId: string; + databaseName: string; + apiId: string; + schemas: string[]; + roles: { + authenticated: string; + anonymous: string; + }; + pluginSettings: DatabaseSettings | null; + graphileSettings: GraphileBuildSettings | null; + computeModules: ComputeConfig['modules']; + computeBindings: ComputeConfig['bindings']; + storageModules: StorageConfig['modules']; + surface: { + isPublic: boolean; + enableRealtime: boolean; + realtimeSchema: string | null; + realtimeNotificationMode: 'dedicated' | 'shared-exact' | null; + realtimeListenerPoolIdentity: string | null; + realtimeNotificationRoleRevalidationMs: number | null; + realtimeCursorPollIntervalMs: number | null; + realtimeCursorHeartbeatIntervalMs: number | null; + graphiql: boolean; + graphiqlOnGraphQLGET: boolean; + explain: boolean; + }; + introspectionMode: 'stock' | 'scoped-required'; + introspectionClientReleaseMode: 'reuse' | 'destroy'; +} + +export interface CreateGraphileBuildContractInput { + configurationIdentity: string; + poolIdentity: string; + databaseId: string; + databaseName: string; + apiId: string; + schemas: string[]; + authenticatedRole: string; + anonymousRole: string; + pluginSettings?: DatabaseSettings; + graphileSettings?: ConstructiveOptions['graphile']; + compute?: ComputeConfig; + storage?: StorageConfig; + isPublic?: boolean; + enableRealtime?: boolean; + /** + * Physical schema containing realtime cursor functions. It is part of the + * exact instance identity only when realtime is enabled. + */ + realtimeSchema?: string; + realtimeNotificationMode?: 'dedicated' | 'shared-exact'; + /** Opaque digest only; raw listener connection configuration is forbidden. */ + realtimeListenerPoolIdentity?: string; + realtimeNotificationRoleRevalidationMs?: number; + realtimeCursorPollIntervalMs?: number; + realtimeCursorHeartbeatIntervalMs?: number; + /** Whether this exact build serves the GraphiQL UI. Defaults to the legacy true value. */ + graphiql?: boolean; + /** Whether GraphQL GET requests serve GraphiQL. Defaults to the legacy false value. */ + graphiqlOnGraphQLGET?: boolean; + explain?: boolean; + introspectionMode?: 'stock' | 'scoped-required'; + introspectionClientReleaseMode?: 'reuse' | 'destroy'; +} + +const graphileBuildSettings = ( + settings: ConstructiveOptions['graphile'] +): GraphileBuildSettings | null => { + if (!settings) return null; + const normalized = { ...settings }; + // Realtime cursor routing is represented by `surface.realtimeSchema` below. + // Keeping it here as well would split disabled surfaces on an irrelevant + // process-wide option and needlessly reduce resident tenant density. + delete normalized.realtimeSchema; + delete normalized.realtimeNotificationMode; + delete normalized.realtimeNotificationRoleRevalidationMs; + delete normalized.realtimeCursorPollIntervalMs; + delete normalized.realtimeCursorHeartbeatIntervalMs; + // Admission policy controls whether startup configuration may enter the + // process trust boundary; it does not change the admitted Graphile build. + delete normalized.trustCallerPresetsInProduction; + return normalized; +}; + +export const createGraphileBuildContract = ( + input: CreateGraphileBuildContractInput +): GraphileBuildContractV1 => { + const realtimeNotificationMode = input.enableRealtime + ? input.realtimeNotificationMode ?? 'dedicated' + : null; + if ( + realtimeNotificationMode === 'shared-exact' + && !input.realtimeListenerPoolIdentity + ) { + throw new Error('Shared realtime requires an opaque listener pool identity'); + } + return { + version: 1, + configurationIdentity: input.configurationIdentity, + poolIdentity: input.poolIdentity, + databaseId: input.databaseId, + databaseName: input.databaseName, + apiId: input.apiId, + schemas: [...input.schemas], + roles: { + authenticated: input.authenticatedRole, + anonymous: input.anonymousRole + }, + pluginSettings: input.pluginSettings ?? null, + graphileSettings: graphileBuildSettings(input.graphileSettings), + computeModules: input.compute?.modules.map((module) => ({ ...module })) ?? [], + computeBindings: input.compute?.bindings.map((binding) => ({ + ...binding, + module: { ...binding.module } + })) ?? [], + storageModules: input.storage?.modules.map((module) => ({ ...module })) ?? [], + surface: { + isPublic: input.isPublic ?? true, + enableRealtime: input.enableRealtime ?? false, + realtimeSchema: input.enableRealtime + ? input.realtimeSchema ?? 'realtime_public' + : null, + realtimeNotificationMode, + realtimeListenerPoolIdentity: + realtimeNotificationMode === 'shared-exact' + ? input.realtimeListenerPoolIdentity ?? null + : null, + realtimeNotificationRoleRevalidationMs: + realtimeNotificationMode === 'shared-exact' + ? input.realtimeNotificationRoleRevalidationMs ?? 60_000 + : null, + realtimeCursorPollIntervalMs: input.enableRealtime + ? input.realtimeCursorPollIntervalMs ?? 5_000 + : null, + realtimeCursorHeartbeatIntervalMs: input.enableRealtime + ? input.realtimeCursorHeartbeatIntervalMs ?? 30_000 + : null, + graphiql: input.graphiql ?? true, + graphiqlOnGraphQLGET: input.graphiqlOnGraphQLGET ?? false, + explain: input.explain ?? false + }, + introspectionMode: input.introspectionMode ?? 'stock', + introspectionClientReleaseMode: input.introspectionClientReleaseMode ?? 'reuse' + }; +}; + +let nextReferenceIdentity = 0; +const referenceIdentities = new WeakMap(); +const symbolIdentities = new Map(); + +const referenceIdentity = (value: object): number => { + let identity = referenceIdentities.get(value); + if (identity === undefined) { + identity = ++nextReferenceIdentity; + referenceIdentities.set(value, identity); + } + return identity; +}; + +const symbolIdentity = (value: symbol): number => { + let identity = symbolIdentities.get(value); + if (identity === undefined) { + identity = ++nextReferenceIdentity; + symbolIdentities.set(value, identity); + } + return identity; +}; + +const canonicalize = (value: unknown, ancestors = new Set()): unknown => { + if ( + value === null || + typeof value === 'string' || + typeof value === 'boolean' + ) { + return value; + } + if (typeof value === 'number') { + if (Number.isNaN(value)) return { $number: 'NaN' }; + if (value === Number.POSITIVE_INFINITY) return { $number: 'Infinity' }; + if (value === Number.NEGATIVE_INFINITY) return { $number: '-Infinity' }; + if (Object.is(value, -0)) return { $number: '-0' }; + return value; + } + if (value === undefined) return { $undefined: true }; + if (typeof value === 'function') { + return { + $function: value.name, + source: createHash('sha256').update(Function.prototype.toString.call(value)).digest('hex'), + // Function source cannot distinguish closures that captured different + // tenant/plugin configuration. Cache reuse is process-local, so bind + // the contract to the exact configured function object as well. + reference: referenceIdentity(value) + }; + } + if (typeof value === 'bigint') return { $bigint: value.toString() }; + if (typeof value === 'symbol') { + return { + $symbol: value.description ?? null, + reference: symbolIdentity(value) + }; + } + if (typeof value !== 'object') { + return { $type: typeof value, value: String(value) }; + } + + if (ancestors.has(value)) { + throw new Error('Graphile build contract contains a circular value'); + } + const nextAncestors = new Set(ancestors).add(value); + if (Array.isArray(value)) { + return value.map((item) => canonicalize(item, nextAncestors)); + } + if (value instanceof Date) return { $date: value.toISOString() }; + if (Buffer.isBuffer(value)) { + return { + $buffer: createHash('sha256').update(value).digest('hex'), + bytes: value.byteLength + }; + } + if (value instanceof RegExp) { + return { $regexp: value.source, flags: value.flags }; + } + + const record = value as Record; + const symbolRecord = value as Record; + const stringProperties = Object.fromEntries( + Object.keys(record) + .sort() + .map((key) => [key, canonicalize(record[key], nextAncestors)]) + ); + const symbolProperties = Object.getOwnPropertySymbols(record) + .map((key) => ({ + key: symbolIdentity(key), + description: key.description ?? null, + value: canonicalize(symbolRecord[key], nextAncestors) + })) + .sort((left, right) => left.key - right.key); + const prototype = Object.getPrototypeOf(record); + if (prototype === Object.prototype || prototype === null) { + return symbolProperties.length === 0 + ? stringProperties + : { $properties: stringProperties, $symbols: symbolProperties }; + } + // Unknown class instances can hide effective configuration in private + // fields or accessors. Their exact reference is safer than treating two + // empty-looking instances as equivalent. + return { + $instance: record.constructor?.name ?? null, + reference: referenceIdentity(record), + properties: stringProperties, + symbols: symbolProperties + }; +}; + +export const hashGraphileBuildContract = (contract: GraphileBuildContractV1): string => { + const canonical = JSON.stringify(canonicalize(contract)); + return `graphile:v1:${createHmac('sha256', graphileBuildContractHmacKey) + .update(canonical) + .digest('hex')}`; +}; diff --git a/graphql/server/src/middleware/graphile-build-governor.ts b/graphql/server/src/middleware/graphile-build-governor.ts new file mode 100644 index 0000000000..df903a2cf2 --- /dev/null +++ b/graphql/server/src/middleware/graphile-build-governor.ts @@ -0,0 +1,442 @@ +import { raceWithClearedTimeout } from 'graphile-cache'; + +export interface GraphileGovernorCounters { + buildsStarted: number; + coalescedRequests: number; + buildWaitTimeouts: number; + buildWaitAborts: number; + queueRefusals: number; + shutdownRefusals: number; + buildWatchdogTrips: number; + stuckRefusals: number; + queueDepth: number; + activeBuilds: number; + unhealthy: boolean; + restartRequired: boolean; + stuckSinceMs: number | null; + activeBuildAgeMs: number | null; + watchdogMs: number; +} + +export const GRAPHILE_BUILD_QUEUE_FULL_CODE = 'GRAPHILE_BUILD_QUEUE_FULL'; +export const GRAPHILE_BUILD_SHUTTING_DOWN_CODE = 'GRAPHILE_BUILD_SHUTTING_DOWN'; +export const GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE = + 'GRAPHILE_BUILD_STUCK_RESTART_REQUIRED'; + +type GraphileBuildCoordinatorErrorCode = + | typeof GRAPHILE_BUILD_QUEUE_FULL_CODE + | typeof GRAPHILE_BUILD_SHUTTING_DOWN_CODE + | typeof GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE; + +export class GraphileBuildCoordinatorError extends Error { + readonly retryAfterSeconds: number; + + constructor( + readonly code: GraphileBuildCoordinatorErrorCode, + message: string, + retryAfterSeconds = 1 + ) { + super(message); + this.name = 'GraphileBuildCoordinatorError'; + this.retryAfterSeconds = retryAfterSeconds; + } +} + +export class GraphileBuildWaitAbortedError extends Error { + readonly code = 'GRAPHILE_BUILD_WAIT_ABORTED'; + + constructor() { + super('The request ended while waiting for a GraphQL schema build'); + this.name = 'GraphileBuildWaitAbortedError'; + } +} + +interface BuildWaiter { + resolve(release: () => void): void; + reject(error: Error): void; + signal?: AbortSignal; + onAbort?: () => void; + onAdmitted?: () => void; +} + +export interface BuildAcquireOptions { + signal?: AbortSignal; + onAdmitted?: () => void; +} + +export class BuildCoordinator { + private active = 0; + private readonly waiters: BuildWaiter[] = []; + private readonly drainWaiters = new Set<() => void>(); + private readonly unhealthyListeners = new Set<( + error: GraphileBuildCoordinatorError + ) => void>(); + private closed = false; + private unhealthyError: GraphileBuildCoordinatorError | null = null; + private stuckAtMs: number | null = null; + private activeStartedAtMs: number | null = null; + private watchdogTimer: ReturnType | null = null; + + constructor( + private readonly capacity: number, + private readonly maxQueueDepth: number, + private readonly buildWatchdogMs = 300_000, + private readonly onUnhealthy?: () => void + ) { + if (capacity !== 1) { + throw new Error( + 'GRAPHILE_BUILD_CONCURRENCY must be exactly 1 until concurrent builds reserve independent heap budgets' + ); + } + if (!Number.isSafeInteger(maxQueueDepth) || maxQueueDepth < 0) { + throw new Error('GRAPHILE_BUILD_QUEUE_MAX must be a non-negative safe integer'); + } + if (!Number.isSafeInteger(buildWatchdogMs) || buildWatchdogMs <= 0) { + throw new Error('GRAPHILE_BUILD_WATCHDOG_MS must be a positive safe integer'); + } + } + + acquire(options: BuildAcquireOptions = {}): Promise<() => void> { + const { signal, onAdmitted } = options; + if (this.unhealthyError) { + counters.stuckRefusals++; + return Promise.reject(this.unhealthyError); + } + if (this.closed) { + counters.shutdownRefusals++; + return Promise.reject(new GraphileBuildCoordinatorError( + GRAPHILE_BUILD_SHUTTING_DOWN_CODE, + 'GraphQL schema build admission is closed for shutdown' + )); + } + if (signal?.aborted) { + return Promise.reject(new GraphileBuildWaitAbortedError()); + } + if (this.active < this.capacity) { + this.active++; + onAdmitted?.(); + return Promise.resolve(this.createRelease()); + } + if (this.waiters.length >= this.maxQueueDepth) { + counters.queueRefusals++; + return Promise.reject(new GraphileBuildCoordinatorError( + GRAPHILE_BUILD_QUEUE_FULL_CODE, + 'GraphQL schema build queue is full' + )); + } + return new Promise((resolve, reject) => { + const waiter: BuildWaiter = { resolve, reject, signal, onAdmitted }; + this.waiters.push(waiter); + if (signal) { + waiter.onAbort = () => { + const index = this.waiters.indexOf(waiter); + if (index < 0) return; + this.waiters.splice(index, 1); + reject(new GraphileBuildWaitAbortedError()); + }; + signal.addEventListener('abort', waiter.onAbort, { once: true }); + if (signal.aborted) waiter.onAbort(); + } + }); + } + + private createRelease(): () => void { + this.activeStartedAtMs = Date.now(); + this.watchdogTimer = setTimeout(() => this.markUnhealthy(), this.buildWatchdogMs); + this.watchdogTimer.unref?.(); + let released = false; + return () => { + if (released) return; + released = true; + if (this.watchdogTimer) { + clearTimeout(this.watchdogTimer); + this.watchdogTimer = null; + } + this.activeStartedAtMs = null; + if (this.unhealthyError) { + // The watchdog never releases this slot. Only completion of the actual + // build reaches this callback, and the latched unhealthy state still + // prevents this process from admitting another build. + this.active = Math.max(0, this.active - 1); + if (this.active === 0) { + for (const resolve of this.drainWaiters) resolve(); + this.drainWaiters.clear(); + } + return; + } + const next = this.waiters.shift(); + if (next) { + if (next.signal && next.onAbort) { + next.signal.removeEventListener('abort', next.onAbort); + } + next.onAdmitted?.(); + next.resolve(this.createRelease()); + } else { + this.active = Math.max(0, this.active - 1); + if (this.active === 0) { + for (const resolve of this.drainWaiters) resolve(); + this.drainWaiters.clear(); + } + } + }; + } + + private markUnhealthy(): void { + if (this.unhealthyError || this.active === 0) return; + this.stuckAtMs = Date.now(); + this.unhealthyError = new GraphileBuildCoordinatorError( + GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE, + 'GraphQL schema build exceeded its watchdog; process restart is required', + 30 + ); + counters.buildWatchdogTrips++; + this.onUnhealthy?.(); + + for (const waiter of this.waiters.splice(0)) { + if (waiter.signal && waiter.onAbort) { + waiter.signal.removeEventListener('abort', waiter.onAbort); + } + counters.stuckRefusals++; + waiter.reject(this.unhealthyError); + } + for (const listener of this.unhealthyListeners) { + listener(this.unhealthyError); + } + } + + onStuck( + listener: (error: GraphileBuildCoordinatorError) => void + ): () => void { + if (this.unhealthyError) { + listener(this.unhealthyError); + return () => undefined; + } + this.unhealthyListeners.add(listener); + return () => this.unhealthyListeners.delete(listener); + } + + close(): boolean { + if (this.closed) return false; + this.closed = true; + const error = new GraphileBuildCoordinatorError( + GRAPHILE_BUILD_SHUTTING_DOWN_CODE, + 'GraphQL schema build admission closed during shutdown' + ); + for (const waiter of this.waiters.splice(0)) { + if (waiter.signal && waiter.onAbort) { + waiter.signal.removeEventListener('abort', waiter.onAbort); + } + counters.shutdownRefusals++; + waiter.reject(error); + } + return true; + } + + async closeAndDrain(timeoutMs: number): Promise { + this.close(); + if (this.active === 0) return true; + const drained = new Promise((resolve) => this.drainWaiters.add(resolve)); + const result = await raceWithClearedTimeout(drained, timeoutMs); + if (result.timedOut) this.drainWaiters.clear(); + return !result.timedOut; + } + + get activeCount(): number { + return this.active; + } + + get queueDepth(): number { + return this.waiters.length; + } + + get isClosed(): boolean { + return this.closed; + } + + get isUnhealthy(): boolean { + return this.unhealthyError !== null; + } + + get stuckSinceMs(): number | null { + return this.stuckAtMs; + } + + get activeBuildAgeMs(): number | null { + return this.activeStartedAtMs == null + ? null + : Math.max(0, Date.now() - this.activeStartedAtMs); + } + + get watchdogMs(): number { + return this.buildWatchdogMs; + } +} + +const parsePositiveInt = (value: string | undefined, fallback: number): number => { + const parsed = value ? Number.parseInt(value, 10) : Number.NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +}; + +const parseNonNegativeInt = (value: string | undefined, fallback: number): number => { + if (value === undefined) return fallback; + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 0) { + throw new Error('GRAPHILE_BUILD_QUEUE_MAX must be a non-negative safe integer'); + } + return parsed; +}; + +const parseSerializedBuildConcurrency = (value: string | undefined): number => { + if (value === undefined) return 1; + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed !== 1) { + throw new Error( + 'GRAPHILE_BUILD_CONCURRENCY must be exactly 1 until concurrent builds reserve independent heap budgets' + ); + } + return parsed; +}; + +const parseBuildWatchdogMs = (value: string | undefined): number => { + if (value === undefined) return 300_000; + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error('GRAPHILE_BUILD_WATCHDOG_MS must be a positive safe integer'); + } + return parsed; +}; + +let coordinatorGeneration = 0; + +const createBuildCoordinator = (): BuildCoordinator => new BuildCoordinator( + parseSerializedBuildConcurrency(process.env.GRAPHILE_BUILD_CONCURRENCY), + parseNonNegativeInt(process.env.GRAPHILE_BUILD_QUEUE_MAX, 16), + parseBuildWatchdogMs(process.env.GRAPHILE_BUILD_WATCHDOG_MS), + () => { + // Late completion from the stuck generation must never publish. + coordinatorGeneration++; + } +); + +let coordinator = createBuildCoordinator(); + +const counters = { + buildsStarted: 0, + coalescedRequests: 0, + buildWaitTimeouts: 0, + buildWaitAborts: 0, + queueRefusals: 0, + shutdownRefusals: 0, + buildWatchdogTrips: 0, + stuckRefusals: 0 +}; + +export interface RunGraphileBuildOptions extends BuildAcquireOptions {} + +export const runGraphileBuild = async ( + build: () => Promise, + options: RunGraphileBuildOptions = {} +): Promise => { + const release = await coordinator.acquire(options); + counters.buildsStarted++; + try { + return await build(); + } finally { + release(); + } +}; + +export const recordCoalescedRequest = (): void => { + counters.coalescedRequests++; +}; + +export const waitForGraphileBuild = async ( + build: Promise, + timeoutMs = parsePositiveInt(process.env.GRAPHILE_BUILD_TIMEOUT_MS, 180_000), + signal?: AbortSignal +): Promise => { + if (signal?.aborted) { + counters.buildWaitAborts++; + throw new GraphileBuildWaitAbortedError(); + } + let timer: ReturnType | undefined; + let abortListener: (() => void) | undefined; + let removeStuckListener: (() => void) | undefined; + const timeout = new Promise<{ type: 'timeout' }>((resolve) => { + timer = setTimeout(() => resolve({ type: 'timeout' }), timeoutMs); + timer.unref?.(); + }); + const aborted = signal + ? new Promise<{ type: 'aborted' }>((resolve) => { + abortListener = () => resolve({ type: 'aborted' }); + signal.addEventListener('abort', abortListener, { once: true }); + if (signal.aborted) abortListener(); + }) + : new Promise(() => undefined); + const stuck = new Promise<{ + type: 'stuck'; + error: GraphileBuildCoordinatorError; + }>((resolve) => { + removeStuckListener = coordinator.onStuck((error) => { + resolve({ type: 'stuck', error }); + }); + }); + try { + const result = await Promise.race([ + build.then((value) => ({ type: 'ready' as const, value })), + timeout, + aborted, + stuck + ]); + if (result.type === 'timeout') { + counters.buildWaitTimeouts++; + return null; + } + if (result.type === 'aborted') { + counters.buildWaitAborts++; + throw new GraphileBuildWaitAbortedError(); + } + if (result.type === 'stuck') throw result.error; + return result.value; + } finally { + if (timer) clearTimeout(timer); + if (signal && abortListener) signal.removeEventListener('abort', abortListener); + removeStuckListener?.(); + } +}; + +export const captureGraphileBuildGeneration = (): number => coordinatorGeneration; + +export const isGraphileBuildGenerationCurrent = (generation: number): boolean => + generation === coordinatorGeneration; + +export const closeGraphileBuildCoordinator = async ( + timeoutMs = parsePositiveInt(process.env.GRAPHILE_BUILD_SHUTDOWN_TIMEOUT_MS, 30_000) +): Promise => { + if (coordinator.close()) coordinatorGeneration++; + return coordinator.closeAndDrain(timeoutMs); +}; + +/** + * Reopen admission only after the previous coordinator fully drained. This + * supports a clean in-process Server restart without ever overlapping a late + * build from the previous generation. + */ +export const reopenGraphileBuildCoordinator = (): boolean => { + if (coordinator.isUnhealthy) return false; + if (!coordinator.isClosed) return true; + if (coordinator.activeCount !== 0 || coordinator.queueDepth !== 0) return false; + coordinator = createBuildCoordinator(); + return true; +}; + +export const getGraphileGovernorCounters = (): GraphileGovernorCounters => ({ + ...counters, + queueDepth: coordinator.queueDepth, + activeBuilds: coordinator.activeCount, + unhealthy: coordinator.isUnhealthy, + restartRequired: coordinator.isUnhealthy, + stuckSinceMs: coordinator.stuckSinceMs, + activeBuildAgeMs: coordinator.activeBuildAgeMs, + watchdogMs: coordinator.watchdogMs +}); diff --git a/graphql/server/src/middleware/graphile-preset-composition.ts b/graphql/server/src/middleware/graphile-preset-composition.ts new file mode 100644 index 0000000000..2dc719c02a --- /dev/null +++ b/graphql/server/src/middleware/graphile-preset-composition.ts @@ -0,0 +1,221 @@ +import type { GraphileConfig } from 'graphile-config'; + +const hasOwn = (value: object, key: PropertyKey): boolean => + Object.prototype.hasOwnProperty.call(value, key); + +/** Names owned by the server even when a particular surface is disabled. */ +export const CONSTRUCTIVE_PROTECTED_GRAPHILE_PLUGINS = Object.freeze([ + 'AuthCookiePlugin', + 'ConstructiveWebSocketOperationAdmissionPlugin', + 'FunctionBindingsPlugin', + 'GrafastCacheLimitsPlugin' +] as const); + +const protectedPluginNames = new Set( + CONSTRUCTIVE_PROTECTED_GRAPHILE_PLUGINS +); + +const PROTECTED_SCOPE_FIELDS = Object.freeze({ + grafast: Object.freeze(['context', 'explain']), + grafserv: Object.freeze([ + 'graphqlPath', + 'graphiqlPath', + 'graphiql', + 'graphiqlOnGraphQLGET', + 'websockets', + 'maskError' + ]), + schema: Object.freeze(['releaseBuildStateAfterValidation']) +} as const); + +export const GRAPHILE_PROTECTED_PRESET_OVERRIDE_CODE = + 'GRAPHILE_PROTECTED_PRESET_OVERRIDE'; +export const GRAPHILE_CALLER_PRESET_NOT_TRUSTED_CODE = + 'GRAPHILE_CALLER_PRESET_NOT_TRUSTED'; + +/** + * Caller presets are executable server code, not tenant-scoped configuration. + * This error keeps production deny-by-default unless the deployment explicitly + * admits that code into the same trust boundary as Constructive itself. + */ +export class GraphileCallerPresetNotTrustedError extends Error { + readonly code = GRAPHILE_CALLER_PRESET_NOT_TRUSTED_CODE; + + constructor() { + super( + 'Graphile caller presets are disabled in production unless trustCallerPresetsInProduction is explicitly enabled' + ); + this.name = 'GraphileCallerPresetNotTrustedError'; + } +} + +/** + * A startup configuration error, never a request error. Values are omitted so + * connection credentials and plugin configuration cannot leak into logs. + */ +export class GraphileProtectedPresetOverrideError extends Error { + readonly code = GRAPHILE_PROTECTED_PRESET_OVERRIDE_CODE; + + constructor( + readonly presetPath: string, + readonly protectedSetting: string + ) { + super( + `Graphile caller preset '${presetPath}' may not configure protected setting '${protectedSetting}'` + ); + this.name = 'GraphileProtectedPresetOverrideError'; + } +} + +const reject = (path: string, setting: string): never => { + throw new GraphileProtectedPresetOverrideError(path, setting); +}; + +const assertScopeDoesNotOverride = ( + preset: Record, + path: string, + scope: keyof typeof PROTECTED_SCOPE_FIELDS +): void => { + const value = preset[scope]; + if (typeof value !== 'object' || value === null || Array.isArray(value)) return; + for (const field of PROTECTED_SCOPE_FIELDS[scope]) { + if (hasOwn(value, field)) reject(path, `${scope}.${field}`); + } +}; + +const assertPluginNamesAreNotProtected = ( + value: unknown, + path: string, + field: 'plugins' | 'disablePlugins' +): void => { + if (!Array.isArray(value)) return; + for (const plugin of value) { + const name = field === 'plugins' + ? (plugin as { name?: unknown } | null)?.name + : plugin; + if (typeof name === 'string' && protectedPluginNames.has(name)) { + reject(path, `${field}.${name}`); + } + } +}; + +const assertPresetDoesNotOverrideProtectedSettings = ( + preset: unknown, + path: string, + visiting: Set, + validated: Set +): void => { + if (typeof preset !== 'object' || preset === null || Array.isArray(preset)) return; + if (validated.has(preset)) return; + if (visiting.has(preset)) { + reject(path, 'extends.circular'); + } + + visiting.add(preset); + const record = preset as Record; + if (hasOwn(record, 'pgServices')) reject(path, 'pgServices'); + assertScopeDoesNotOverride(record, path, 'grafast'); + assertScopeDoesNotOverride(record, path, 'grafserv'); + assertScopeDoesNotOverride(record, path, 'schema'); + assertPluginNamesAreNotProtected(record.plugins, path, 'plugins'); + assertPluginNamesAreNotProtected(record.disablePlugins, path, 'disablePlugins'); + + if (Array.isArray(record.extends)) { + record.extends.forEach((nested, index) => { + assertPresetDoesNotOverrideProtectedSettings( + nested, + `${path}.extends[${index}]`, + visiting, + validated + ); + }); + } + visiting.delete(preset); + validated.add(preset); +}; + +export interface ComposeGraphilePresetInput { + /** Constructive feature presets applied before trusted caller customization. */ + basePresets: readonly GraphileConfig.Preset[]; + callerExtends?: readonly GraphileConfig.Preset[]; + callerPreset?: Partial; + /** Whether caller preset code has been explicitly admitted into the TCB. */ + callerPresetsTrusted: boolean; + /** Server-owned presets applied after callers, for example cache bounds. */ + protectedPresets?: readonly GraphileConfig.Preset[]; + protectedPlugins: GraphileConfig.Plugin[]; + pgServices: NonNullable; + schema: NonNullable; + grafserv: NonNullable; + grafast: NonNullable; +} + +export interface GraphileCallerPresetInput { + callerExtends?: readonly GraphileConfig.Preset[]; + callerPreset?: Partial; + /** Whether all caller preset code has been admitted into the process TCB. */ + callerPresetsTrusted: boolean; +} + +const hasCallerPresetConfiguration = ( + input: GraphileCallerPresetInput +): boolean => { + if ((input.callerExtends?.length ?? 0) > 0) return true; + const preset = input.callerPreset; + if (!preset) return false; + return Reflect.ownKeys(preset).length > 0; +}; + +/** Validate eagerly at server construction and again at lazy tenant build. */ +export const assertGraphileCallerPresetsSafe = ( + input: GraphileCallerPresetInput +): void => { + if (!input.callerPresetsTrusted && hasCallerPresetConfiguration(input)) { + throw new GraphileCallerPresetNotTrustedError(); + } + const callerExtends = input.callerExtends ?? []; + const validated = new Set(); + callerExtends.forEach((preset, index) => { + assertPresetDoesNotOverrideProtectedSettings( + preset, + `graphile.extends[${index}]`, + new Set(), + validated + ); + }); + if (input.callerPreset) { + assertPresetDoesNotOverrideProtectedSettings( + input.callerPreset, + 'graphile.preset', + new Set(), + validated + ); + } +}; + +/** + * Compose trusted caller Graphile configuration inside the server-owned tenant + * boundary. The root fields are deliberately written by Constructive after all + * caller presets, so the exact pool, request context, transports, and security + * plugins cannot be replaced through Graphile's shallow preset merging. + */ +export const composeGraphilePreset = ( + input: ComposeGraphilePresetInput +): GraphileConfig.Preset => { + const callerExtends = input.callerExtends ?? []; + assertGraphileCallerPresetsSafe(input); + + return { + extends: [ + ...input.basePresets, + ...callerExtends, + ...(input.callerPreset ? [input.callerPreset] : []), + ...(input.protectedPresets ?? []) + ], + plugins: input.protectedPlugins, + pgServices: input.pgServices, + schema: input.schema, + grafserv: input.grafserv, + grafast: input.grafast + }; +}; diff --git a/graphql/server/src/middleware/graphile.ts b/graphql/server/src/middleware/graphile.ts index e6de98f7ad..c530b41208 100644 --- a/graphql/server/src/middleware/graphile.ts +++ b/graphql/server/src/middleware/graphile.ts @@ -3,29 +3,134 @@ import './types'; // for Request type import crypto from 'node:crypto'; import { classify, type ErrorContext, errors, parse } from '@constructive-io/errors'; -import type { ComputeConfig } from '@constructive-io/express-context'; +import { + buildPgSettings, + type ComputeConfig, + type RuntimePgPoolResolution, + type StorageConfig +} from '@constructive-io/express-context'; import type { ConstructiveOptions } from '@constructive-io/graphql-types'; import { getNodeEnv } from '@pgpmjs/env'; import { Logger } from '@pgpmjs/logger'; import type { NextFunction, Request, RequestHandler, Response } from 'express'; -import type { GraphQLError, GraphQLFormattedError } from 'grafast/graphql'; -import { createGraphileInstance, graphileCache,type GraphileCacheEntry } from 'graphile-cache'; +import { + type BuildRefusalReason, + CacheBuildAdmissionError, + createGraphileInstance, + disposeUncachedEntry, + evaluateBuildAdmission, + graphileCache, + type GraphileCacheEntry, + GraphileRealtimeStartupError, + invokeEntryHandler, + invokeEntryUpgradeHandler, + isEntryRealtimeUnavailable, + prepareCacheForBuild, + recordBuildRefusal, + revalidateEntryRealtimeRole +} from 'graphile-cache'; import type { GraphileConfig } from 'graphile-config'; import { createFunctionBindingsPlugin } from 'graphile-function-bindings'; -import { createConstructivePreset, makePgService } from 'graphile-settings'; -import { getPgPool } from 'pg-cache'; -import { getPgEnvOptions } from 'pg-env'; +import { + ActivatableGenerationScopedRealtimeSubscriber, + RealtimeTopicCollector +} from 'graphile-realtime-subscriptions'; +import { + createConstructivePreset, + createGrafastCacheLimitsPreset, + makePgService, + normalizeIntrospectionDependencySchemas, + resolveConstructiveIntrospectionCapabilityExtensions +} from 'graphile-settings'; +import type { GraphQLError, GraphQLFormattedError } from 'graphql'; +import { + acquirePgPool, + getPgNotificationBrokerIdentity, + getPgPoolIdentity, + PgPoolCapacityError, + type PgPoolLease +} from 'pg-cache'; import { isGraphqlObservabilityEnabled } from '../diagnostics/observability'; import { HandlerCreationError } from '../errors/api-errors'; import { respondWithGraphQLError } from '../errors/graphql-response'; import { AuthCookiePlugin } from '../plugins/auth-cookie-plugin'; +import { + createGraphileWebSocketOperationAdmission, + type GraphileWebSocketOperationAdmission +} from '../plugins/websocket-operation-admission-plugin'; import type { DatabaseSettings } from '../types'; +import { + getGraphileWebSocketUpgradeTransport, + GRAPHILE_WEBSOCKET_AUTH_REJECTED_CODE, + handoffGraphileWebSocketUpgrade, + isGraphileWebSocketOriginAllowed +} from '../websocket-upgrade'; +import { + createGraphileBuildContract, + hashGraphileBuildContract +} from './graphile-build-contract'; +import { + captureGraphileBuildGeneration, + GRAPHILE_BUILD_QUEUE_FULL_CODE, + GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE, + GraphileBuildCoordinatorError, + GraphileBuildWaitAbortedError, + isGraphileBuildGenerationCurrent, + recordCoalescedRequest, + runGraphileBuild, + waitForGraphileBuild +} from './graphile-build-governor'; +import { + assertGraphileCallerPresetsSafe, + composeGraphilePreset +} from './graphile-preset-composition'; +import { getTrustedInternalClaims } from './internal-request'; import { observeGraphileBuild } from './observability/graphile-build-stats'; +import { + addRealtimeRuntimeDependencySchema, + resolveGraphileRealtimeSchema +} from './realtime-config'; +import { + GraphileRealtimeNotificationConfigError, + resolveRealtimeCursorIntervals, + resolveRealtimeNotificationMode, + resolveRealtimeNotificationPgConfig, + resolveRealtimeNotificationRoleRevalidationMs +} from './realtime-notification-config'; +import { + createRuntimePgResolverInput, + resolveRuntimePgConfig +} from './runtime-pg-config'; +import { + assertRuntimePgCredentials, + shouldValidateRuntimeRoleSafety +} from './runtime-pg-requirements'; +import { + ensureRuntimeRoleSafety, + refreshRuntimeRoleSafety +} from './runtime-role-safety'; const maskErrorLog = new Logger('graphile:maskError'); const isDev = (): boolean => getNodeEnv() === 'development'; +const GRAPHILE_SURFACE_FLAGS = Object.freeze({ + graphiql: true, + graphiqlOnGraphQLGET: false +}); + +let nextGraphileConfigurationIdentity = 0; + +/** @internal Resolve the same routed/authenticated request for HTTP and WS. */ +export const getGraphileTransportRequest = ( + requestContext: Partial +): Request | undefined => { + const typedContext = requestContext as { + expressv4?: { req?: Request }; + ws?: { request?: Request }; + }; + return typedContext.expressv4?.req ?? typedContext.ws?.request; +}; /** * GraphQL framework protocol codes. These originate in the GraphQL/grafast @@ -124,7 +229,17 @@ const maskError = (error: GraphQLError): GraphQLError | GraphQLFormattedError => * When multiple concurrent requests arrive for the same cache key, only the * first request creates the handler while others wait on the same promise. */ -const creating = new Map>(); +interface InFlightGraphileBuild { + promise: Promise; + serviceKey: string; + databaseId: string | null; + invalidated: boolean; + admitted: boolean; + waiterCount: number; + abortController: AbortController; +} + +const creating = new Map(); /** * Returns the number of currently in-flight handler creation operations. @@ -146,12 +261,80 @@ export function getInFlightKeys(): string[] { * Clears the in-flight map. Used for testing purposes. */ export function clearInFlightMap(): void { + for (const build of creating.values()) { + if (!build.admitted) build.abortController.abort(); + } creating.clear(); } +export const invalidateInFlightBuilds = (selector: { + serviceKey?: string; + databaseId?: string; +}): number => { + let invalidated = 0; + for (const build of creating.values()) { + if ( + (selector.serviceKey && build.serviceKey === selector.serviceKey) || + (selector.databaseId && build.databaseId === selector.databaseId) + ) { + build.invalidated = true; + invalidated++; + } + } + return invalidated; +}; + const log = new Logger('graphile'); const reqLabel = (req: Request): string => (req.requestId ? `[${req.requestId}]` : '[req]'); +/** + * A consumed IncomingMessage may be auto-destroyed while its keep-alive socket + * remains healthy. Only the transport socket, an explicit abort, or the + * response state tells us that the request can no longer receive a response. + */ +export const isGraphileRequestTerminal = (req: Request, res: Response): boolean => + Boolean( + req.aborted + || req.socket?.destroyed + || res.destroyed + || res.writableEnded + ); + +const createRequestAbortHandle = ( + req: Request, + res: Response +): { signal: AbortSignal; cleanup(): void } => { + const controller = new AbortController(); + const abort = (): void => controller.abort(); + req.once('aborted', abort); + res.once('close', abort); + if (isGraphileRequestTerminal(req, res)) abort(); + return { + signal: controller.signal, + cleanup: () => { + req.removeListener('aborted', abort); + res.removeListener('close', abort); + } + }; +}; + +const waitForInFlightGraphileBuild = async ( + build: InFlightGraphileBuild, + signal: AbortSignal +): Promise => { + build.waiterCount++; + try { + return await waitForGraphileBuild(build.promise, undefined, signal); + } finally { + build.waiterCount = Math.max(0, build.waiterCount - 1); + // Active builds are allowed to finish and become useful residents. Queued + // builds retain no pool lease and are canceled once nobody can consume them. + if (build.waiterCount === 0 && !build.admitted) { + build.abortController.abort(); + } + } +}; + /** * Build a PostGraphile v5 preset for a tenant. * @@ -167,12 +350,53 @@ const buildPreset = ( roleName: string, databaseSettings?: DatabaseSettings, apiId?: string, - compute?: ComputeConfig + compute?: ComputeConfig, + storage?: StorageConfig, + introspectionMode: 'stock' | 'scoped-required' = 'stock', + introspectionClientReleaseMode: 'reuse' | 'destroy' = 'reuse', + introspectionDependencySchemas: readonly string[] = [], + grafastCache: NonNullable['grafastCache'] = {}, + releaseBuildStateAfterValidation = false, + enableWebsockets = false, + sharedRealtime?: { + subscriber: ActivatableGenerationScopedRealtimeSubscriber; + topicCollector: RealtimeTopicCollector; + }, + websocketOperationAdmissionPlugin?: GraphileConfig.Plugin, + callerExtends: readonly GraphileConfig.Preset[] = [], + callerPreset?: Partial, + callerPresetsTrusted = false ): GraphileConfig.Preset => { - return { - extends: [createConstructivePreset(databaseSettings)], - plugins: [ + if (enableWebsockets && !websocketOperationAdmissionPlugin) { + throw new Error( + 'Graphile WebSockets require exact per-operation safety admission' + ); + } + return composeGraphilePreset({ + basePresets: [ + createConstructivePreset({ + ...databaseSettings, + // The server always supplies an authoritative control-plane snapshot. + // Undefined means "module not provisioned", not "query as runtime". + preloadedStorageModules: storage?.modules ?? [], + ...(sharedRealtime ? { + realtimeSubscriptions: { + onTopicsDiscovered: sharedRealtime.topicCollector.collect + } + } : {}) + }) + ], + callerExtends, + callerPreset, + callerPresetsTrusted, + protectedPresets: [ + createGrafastCacheLimitsPreset(grafastCache) + ], + protectedPlugins: [ AuthCookiePlugin, + ...(websocketOperationAdmissionPlugin + ? [websocketOperationAdmissionPlugin] + : []), // Only registered when the compute module is provisioned for this // database — all schema/table names come from the constructive // metaschema (express-context compute module loader); the plugin has @@ -188,6 +412,17 @@ const buildPreset = ( invocationsSchema: m.invocationsSchemaName, invocationsTable: m.invocationsTableName, invocationsEntityField: m.invocationsEntityField + })), + preloadedBindings: compute.bindings.map((binding) => ({ + ...binding, + module: { + computeSchema: binding.module.schemaName, + bindingsTable: binding.module.bindingsTableName, + definitionsTable: binding.module.definitionsTableName, + invocationsSchema: binding.module.invocationsSchemaName, + invocationsTable: binding.module.invocationsTableName, + invocationsEntityField: binding.module.invocationsEntityField + } })) }) ] @@ -196,134 +431,394 @@ const buildPreset = ( pgServices: [ makePgService({ pool, - schemas + schemas, + introspectionMode, + introspectionClientReleaseMode, + introspectionAllowedDependencySchemas: introspectionDependencySchemas, + ...(introspectionMode === 'scoped-required' ? { + introspectionCapabilityExtensions: + resolveConstructiveIntrospectionCapabilityExtensions(databaseSettings) + } : {}), + ...(sharedRealtime ? { + pubsub: false, + pgSubscriber: sharedRealtime.subscriber + } : {}) }) ], + schema: { + releaseBuildStateAfterValidation + }, grafserv: { graphqlPath: '/graphql', graphiqlPath: '/graphiql', - graphiql: true, - graphiqlOnGraphQLGET: false, + ...GRAPHILE_SURFACE_FLAGS, + websockets: enableWebsockets, maskError }, grafast: { - explain: process.env.NODE_ENV === 'development', + explain: isDev(), context: (requestContext: Partial) => { - // In grafserv/express/v4, the request is available at requestContext.expressv4.req - const req = (requestContext as { expressv4?: { req?: Request } })?.expressv4?.req; - const context: Record = {}; + // HTTP carries the Express request directly. WebSocket execution keeps + // the same already-routed/authenticated IncomingMessage under `ws`, so + // both transports derive identical roles, claims, and security GUCs. + const req = getGraphileTransportRequest(requestContext); + const api = req?.api ?? { + dbname: '', + schema: schemas, + anonRole, + roleName + }; + const trustedClaims = getTrustedInternalClaims(req); - if (req) { - if (req.databaseId) { - context['jwt.claims.database_id'] = req.databaseId; - } - // API provenance — which API surface this request arrived through. - // Derived server-side by resolving the hostname through the scoped - // routing plane (resolve_route -> api_id); never taken from - // client-supplied headers, body, or token payload. - if (req.api?.apiId) { - context['jwt.claims.api_id'] = req.api.apiId; - } - if (req.clientIp) { - context['jwt.claims.ip_address'] = req.clientIp; - } - if (req.get('origin')) { - context['jwt.claims.origin'] = req.get('origin') as string; - } - if (req.get('User-Agent')) { - context['jwt.claims.user_agent'] = req.get('User-Agent') as string; - } - if (req.deviceToken) { - context['jwt.claims.device_token'] = req.deviceToken; - } + return { + pgSettings: buildPgSettings({ + api, + token: req?.token ?? null, + requestId: req?.requestId ?? '', + clientIp: req?.clientIp, + origin: req?.get('origin'), + userAgent: req?.get('User-Agent'), + deviceToken: req?.deviceToken, + trustedClaims, + dependencySchemas: introspectionDependencySchemas + }) + }; + } + } + }); +}; - if (req.token?.user_id) { - const pgSettings: Record = { - role: roleName, - 'jwt.claims.token_id': req.token.id, - 'jwt.claims.user_id': req.token.user_id, - ...context - }; +export class GraphileBuildInvalidatedError extends Error { + readonly code = 'GRAPHILE_BUILD_INVALIDATED'; - if (req.token.session_id) { - pgSettings['jwt.claims.session_id'] = req.token.session_id; - } + constructor() { + super('Graphile build was invalidated before it could become resident'); + this.name = 'GraphileBuildInvalidatedError'; + } +} - // Propagate credential metadata as JWT claims so PG functions - // can read them via current_setting('jwt.claims.access_level') etc. - if (req.token.access_level) { - pgSettings['jwt.claims.access_level'] = req.token.access_level; - } - if (req.token.kind) { - pgSettings['jwt.claims.kind'] = req.token.kind; - } +export class GraphileBuildPublicationError extends Error { + readonly code = 'GRAPHILE_BUILD_PUBLICATION_FAILED'; - // Principal identity — always set; equals user_id for human sessions - pgSettings['jwt.claims.principal_id'] = req.token.principal_id || req.token.user_id; + constructor(message: string, readonly cause?: unknown) { + super(message); + this.name = 'GraphileBuildPublicationError'; + } +} - // Enforce read-only transactions for read_only credentials - if (req.token.access_level === 'read_only') { - pgSettings['default_transaction_read_only'] = 'on'; - } +/** @internal Explicit ownership state for the build-to-entry pool lease handoff. */ +export class GraphileBuildPoolLeaseOwner { + private pending: PgPoolLease | undefined; - if (req.requestId) { - pgSettings['request.id'] = req.requestId; - } + constructor(lease: PgPoolLease | undefined) { + this.pending = lease; + } - return { pgSettings }; - } + get lease(): PgPoolLease | undefined { + return this.pending; + } - // Private (in-cluster) surface: there is no token — identity - // arrives on the trusted internal X-* headers stamped by the - // dispatching worker/sync gateway (the same vocabulary as - // X-Database-Id above). Map it into per-request claims so writes - // made through this surface carry actor attribution. Never applied - // on the public surface, where client-supplied identity headers - // must not assert identity. - const headerActorId = req.get('X-Actor-Id'); - if (req.api?.isPublic === false && headerActorId) { - const pgSettings: Record = { - role: roleName, - 'jwt.claims.user_id': headerActorId, - 'jwt.claims.principal_id': headerActorId, - ...context - }; - const headerEntityId = req.get('X-Entity-Id'); - if (headerEntityId) { - pgSettings['jwt.claims.entity_id'] = headerEntityId; - } - const headerOrganizationId = req.get('X-Organization-Id'); - if (headerOrganizationId) { - pgSettings['jwt.claims.organization_id'] = headerOrganizationId; - } - if (req.requestId) { - pgSettings['request.id'] = req.requestId; - } - return { pgSettings }; - } - } + transferTo(entry: GraphileCacheEntry): void { + const expected = this.pending; + if (!expected) { + throw new GraphileBuildPublicationError( + `PostGraphile[${entry.cacheKey}] has no pending pool lease to transfer` + ); + } + if (entry.poolLease !== expected) { + throw new GraphileBuildPublicationError( + `PostGraphile[${entry.cacheKey}] did not retain the build pool lease` + ); + } - const anonSettings: Record = { - role: anonRole, - ...context - }; - if (req?.requestId) { - anonSettings['request.id'] = req.requestId; - } + // The entry now owns the lease even when a later identity assertion fails; + // its disposal path, rather than the build finally block, must release it. + this.pending = undefined; + if (entry.poolIdentity !== expected.identity) { + throw new GraphileBuildPublicationError( + `PostGraphile[${entry.cacheKey}] retained an unexpected pool identity` + ); + } + } - return { - pgSettings: anonSettings - }; - } + release(): void { + const pending = this.pending; + this.pending = undefined; + pending?.release(); + } +} + +interface GraphileBuildPublicationCache { + get(key: string): GraphileCacheEntry | undefined; + set(key: string, entry: GraphileCacheEntry): unknown; + delete(key: string): boolean; +} + +interface GraphileBuildPublicationDependencies { + cache?: GraphileBuildPublicationCache; + dispose?: (entry: GraphileCacheEntry, key: string) => Promise; +} + +/** @internal Publish exactly one authoritative entry or dispose the candidate. */ +export const publishGraphileBuild = async ( + key: string, + candidate: GraphileCacheEntry, + invalidated: boolean, + dependencies: GraphileBuildPublicationDependencies = {} +): Promise => { + const cache = dependencies.cache ?? graphileCache; + const dispose = dependencies.dispose ?? disposeUncachedEntry; + const cleanupCandidate = async (message: string): Promise => { + try { + await dispose(candidate, key); + } catch (cleanupError) { + throw new GraphileBuildPublicationError( + `${message}; candidate disposal also failed`, + cleanupError + ); } }; + const disposeCandidate = async (message: string, cause?: unknown): Promise => { + await cleanupCandidate(message); + throw new GraphileBuildPublicationError(message, cause); + }; + const candidateUnavailable = (): boolean => + candidate.disposing === true || isEntryRealtimeUnavailable(candidate); + const rejectPublishedCandidate = async (message: string): Promise => { + if (cache.get(key) === candidate) cache.delete(key); + return disposeCandidate(message); + }; + + if (invalidated) { + await cleanupCandidate(`PostGraphile[${key}] invalidation disposal failed`); + throw new GraphileBuildInvalidatedError(); + } + if (candidateUnavailable()) { + return disposeCandidate( + `PostGraphile[${key}] became unavailable before publication` + ); + } + + const resident = cache.get(key); + if (resident && resident !== candidate) { + if (resident.disposing) { + return disposeCandidate( + `PostGraphile[${key}] collided with a disposing resident entry` + ); + } + await cleanupCandidate(`PostGraphile[${key}] duplicate disposal failed`); + const authoritative = cache.get(key); + if (authoritative !== resident || resident.disposing) { + throw new GraphileBuildPublicationError( + `PostGraphile[${key}] resident changed while discarding a duplicate build` + ); + } + log.warn(`Discarded duplicate PostGraphile[${key}] build; using the resident entry`); + return resident; + } + if (resident === candidate) { + if (candidateUnavailable()) { + return rejectPublishedCandidate( + `PostGraphile[${key}] resident candidate became unavailable` + ); + } + return candidate; + } + + try { + cache.set(key, candidate); + } catch (error) { + return disposeCandidate(`Failed to publish PostGraphile[${key}]`, error); + } + + const published = cache.get(key); + if (published === candidate) { + if (candidateUnavailable()) { + return rejectPublishedCandidate( + `PostGraphile[${key}] became unavailable during publication` + ); + } + return candidate; + } + if (published && !published.disposing) { + await cleanupCandidate(`PostGraphile[${key}] replaced-candidate disposal failed`); + const authoritative = cache.get(key); + if (authoritative !== published || published.disposing) { + throw new GraphileBuildPublicationError( + `PostGraphile[${key}] resident changed after publication replacement` + ); + } + log.warn(`PostGraphile[${key}] publication was replaced; using the resident entry`); + return published; + } + return disposeCandidate(`PostGraphile[${key}] was not resident after publication`); }; -export const graphile = (opts: ConstructiveOptions): RequestHandler => { +export const GRAPHILE_BUILD_RESIDENT_CAPACITY_CODE = + 'GRAPHILE_BUILD_RESIDENT_CAPACITY'; + +export const BUILD_REFUSAL_CODES = { + critical_pressure: 'GRAPHILE_BUILD_MEMORY_PRESSURE', + insufficient_budget: 'GRAPHILE_BUILD_BUDGET_EXCEEDED', + rss_budget_exceeded: 'GRAPHILE_BUILD_RSS_BUDGET_EXCEEDED', + disposal_timeout: 'GRAPHILE_BUILD_DISPOSAL_TIMEOUT', + resident_busy: 'GRAPHILE_BUILD_CAPACITY_BUSY', + resident_capacity: GRAPHILE_BUILD_RESIDENT_CAPACITY_CODE, + disposal_failed: 'GRAPHILE_BUILD_DISPOSAL_FAILED' +} as const satisfies Record; + +const respondBuildUnavailable = ( + res: Response, + code: string, + message: string, + retryAfterSeconds = 15 +): void => { + if (res.destroyed || res.writableEnded) return; + res.setHeader('Retry-After', String(retryAfterSeconds)); + res.status(503).json({ error: { code, message } }); +}; + +export const handleBuildAvailabilityError = ( + res: Response, + error: unknown +): boolean => { + if (error instanceof PgPoolCapacityError) { + respondBuildUnavailable( + res, + error.code, + 'PostgreSQL connection capacity is temporarily unavailable', + error.retryAfterSeconds + ); + return true; + } + if (error instanceof CacheBuildAdmissionError) { + respondBuildUnavailable( + res, + BUILD_REFUSAL_CODES[error.reason], + 'GraphQL schema capacity is temporarily unavailable', + error.retryAfterSeconds + ); + return true; + } + if (error instanceof GraphileBuildCoordinatorError) { + respondBuildUnavailable( + res, + error.code, + error.code === GRAPHILE_BUILD_QUEUE_FULL_CODE + ? 'GraphQL schema build queue is full; retry shortly' + : error.code === GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE + ? 'GraphQL schema build admission is unhealthy; process restart is required' + : 'GraphQL schema build admission is closed for shutdown', + error.retryAfterSeconds + ); + return true; + } + if (error instanceof GraphileBuildInvalidatedError) { + respondBuildUnavailable( + res, + error.code, + 'GraphQL schema changed while it was building; retry shortly', + 1 + ); + return true; + } + if (error instanceof GraphileBuildPublicationError) { + respondBuildUnavailable( + res, + error.code, + 'GraphQL schema publication failed; retry shortly', + 1 + ); + return true; + } + if (error instanceof GraphileRealtimeNotificationConfigError) { + respondBuildUnavailable( + res, + error.code, + 'Shared realtime notification configuration is unavailable' + ); + return true; + } + if (error instanceof GraphileRealtimeStartupError) { + respondBuildUnavailable( + res, + error.code, + 'Realtime delivery could not be activated for this GraphQL instance' + ); + return true; + } + return false; +}; + +const handleBuildWaitAbort = ( + res: Response, + error: unknown, + requestSignal: AbortSignal +): boolean => { + if (!(error instanceof GraphileBuildWaitAbortedError)) return false; + if (!requestSignal.aborted) { + respondBuildUnavailable( + res, + 'GRAPHILE_BUILD_CANCELED', + 'GraphQL schema build was canceled before admission; retry shortly', + 1 + ); + } + return true; +}; + +export const graphile = ( + opts: ConstructiveOptions, + getRuntimePgResolution?: ( + req: Request + ) => Readonly +): RequestHandler => { + // The resident cache is process-wide, but caller presets may contain hooks + // whose captured state cannot be serialized. Never share a generation + // across two independently constructed server configurations, even when all + // visible data fields happen to compare equal. + const configurationIdentity = + `graphile-configuration:v1:${++nextGraphileConfigurationIdentity}`; + const callerPresetsTrusted = + getNodeEnv() !== 'production' + || opts.graphile?.trustCallerPresetsInProduction === true; + assertRuntimePgCredentials(opts, getNodeEnv()); + assertGraphileCallerPresetsSafe({ + callerExtends: opts.graphile?.extends, + callerPreset: opts.graphile?.preset, + callerPresetsTrusted + }); + const introspectionDependencySchemas = normalizeIntrospectionDependencySchemas( + opts.graphile?.introspectionDependencySchemas + ); const observabilityEnabled = isGraphqlObservabilityEnabled(opts.server?.host); + const runtimeSafetyRequired = shouldValidateRuntimeRoleSafety(opts, getNodeEnv()); + const realtimeNotificationMode = resolveRealtimeNotificationMode(opts); + const realtimeNotificationRoleRevalidationMs = + resolveRealtimeNotificationRoleRevalidationMs(opts); + const realtimeCursorIntervals = resolveRealtimeCursorIntervals(opts); return async (req: Request, res: Response, next: NextFunction) => { const label = reqLabel(req); + const requestAbort = createRequestAbortHandle(req, res); + const websocketUpgrade = getGraphileWebSocketUpgradeTransport(req); + const invokeEntry = (entry: GraphileCacheEntry): boolean => { + if (!websocketUpgrade) return invokeEntryHandler(entry, req, res, next); + return invokeEntryUpgradeHandler( + entry, + req, + websocketUpgrade.socket, + websocketUpgrade.head, + { + onAccepted: () => { + handoffGraphileWebSocketUpgrade(req, res); + }, + onRejected: () => { + handoffGraphileWebSocketUpgrade(req, res); + } + } + ); + }; try { const api = req.api; if (!api) { @@ -331,8 +826,20 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { respondWithGraphQLError(res, errors.INTERNAL_FAILURE({ details: 'Missing API info' })); return; } - const key = req.svc_key; - if (!key) { + if ( + websocketUpgrade + && !isGraphileWebSocketOriginAllowed(req, opts.server?.origin) + ) { + res.status(403).json({ + error: { + code: GRAPHILE_WEBSOCKET_AUTH_REJECTED_CODE, + message: 'WebSocket origin is not allowed' + } + }); + return; + } + const serviceKey = req.svc_key; + if (!serviceKey) { log.error(`${label} Missing service cache key`); respondWithGraphQLError( res, @@ -342,89 +849,361 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { } const { dbname, anonRole, roleName, schema } = api; const schemaLabel = schema?.join(',') || 'unknown'; + const poolOptions = { purpose: 'runtime', sanitizeOnCheckout: true } as const; + const runtimePgResolution = getRuntimePgResolution + ? getRuntimePgResolution(req) + : await resolveRuntimePgConfig( + opts, + createRuntimePgResolverInput(api) + ); + const pgConfig = runtimePgResolution.pgConfig; + const poolIdentity = runtimePgResolution.poolIdentity; + if (getPgPoolIdentity(pgConfig, poolOptions) !== poolIdentity) { + throw new Error( + 'Resolved runtime PostgreSQL pool identity changed before Graphile acquisition' + ); + } + if ( + req.constructive + && req.constructive.runtimePoolIdentity !== poolIdentity + ) { + throw new Error( + 'Request context and Graphile resolved different runtime PostgreSQL pools' + ); + } + const [compute, storage] = await Promise.all([ + api.apiId ? req.constructive?.useModule('compute') : undefined, + (api.databaseSettings?.enablePresignedUploads ?? true) + ? req.constructive?.useModule('storage') + : undefined + ]); + const introspectionMode = opts.graphile?.introspectionMode ?? 'stock'; + const introspectionClientReleaseMode = + opts.graphile?.introspectionClientReleaseMode ?? 'reuse'; + const realtimeEnabled = api.databaseSettings?.enableRealtime ?? false; + const realtimeSchema = resolveGraphileRealtimeSchema(opts, realtimeEnabled); + const notificationPgConfig = realtimeEnabled + && realtimeNotificationMode === 'shared-exact' + ? await resolveRealtimeNotificationPgConfig(opts, { + databaseId: api.databaseId ?? '', + databaseName: dbname, + apiId: api.apiId ?? '', + schemas: schema ?? [] + }) + : null; + const realtimeListenerPoolIdentity = notificationPgConfig + ? getPgNotificationBrokerIdentity(notificationPgConfig) + : null; + const runtimeDependencySchemas = addRealtimeRuntimeDependencySchema( + introspectionDependencySchemas, + realtimeSchema + ); + const buildContract = createGraphileBuildContract({ + configurationIdentity, + poolIdentity, + databaseId: api.databaseId ?? '', + databaseName: dbname, + apiId: api.apiId ?? '', + schemas: schema ?? [], + authenticatedRole: roleName, + anonymousRole: anonRole, + pluginSettings: api.databaseSettings, + graphileSettings: opts.graphile, + compute, + storage, + isPublic: api.isPublic, + enableRealtime: realtimeEnabled, + realtimeSchema: realtimeSchema ?? undefined, + realtimeNotificationMode, + realtimeListenerPoolIdentity: realtimeListenerPoolIdentity ?? undefined, + realtimeNotificationRoleRevalidationMs, + realtimeCursorPollIntervalMs: realtimeCursorIntervals.pollIntervalMs, + realtimeCursorHeartbeatIntervalMs: realtimeCursorIntervals.heartbeatIntervalMs, + ...GRAPHILE_SURFACE_FLAGS, + explain: isDev(), + introspectionMode, + introspectionClientReleaseMode + }); + const key = hashGraphileBuildContract(buildContract); + const ensureRuntimeSafety = async (): Promise => { + // Hold an operation-scoped lease even when a resident entry already + // owns this pool. The entry may be evicted while the async audit runs; + // this lease prevents pool teardown until the audit has settled. + const auditLease = acquirePgPool(pgConfig, poolOptions); + try { + await ensureRuntimeRoleSafety( + auditLease.pool, + [anonRole, roleName], + schema ?? [], + runtimeDependencySchemas + ); + } finally { + auditLease.release(); + } + }; - // ========================================================================= - // Phase A: Cache Check (fast path) - // ========================================================================= const cached = graphileCache.get(key); if (cached) { - log.debug(`${label} PostGraphile cache hit key=${key} db=${dbname} schemas=${schemaLabel}`); - return cached.handler(req, res, next); + // A role or schema can drift after the instance was built. Re-enter + // the fail-closed audit on every resident path; the audit itself + // coalesces requests and reuses only recent successful results. + if (runtimeSafetyRequired) await ensureRuntimeSafety(); + await revalidateEntryRealtimeRole(cached); + if (invokeEntry(cached)) { + log.debug(`${label} PostGraphile cache hit key=${key} route=${serviceKey} db=${dbname} schemas=${schemaLabel}`); + return; + } + if (isGraphileRequestTerminal(req, res)) return; } - log.debug(`${label} PostGraphile cache miss key=${key} db=${dbname} schemas=${schemaLabel}`); + log.debug(`${label} PostGraphile cache miss key=${key} route=${serviceKey} db=${dbname} schemas=${schemaLabel}`); + if (isGraphileRequestTerminal(req, res)) return; - // ========================================================================= - // Phase B: In-Flight Check (single-flight coalescing) - // ========================================================================= const inFlight = creating.get(key); if (inFlight) { + recordCoalescedRequest(); log.debug(`${label} Coalescing request for PostGraphile[${key}] - waiting for in-flight creation`); try { - const instance = await inFlight; - return instance.handler(req, res, next); + const instance = await waitForInFlightGraphileBuild(inFlight, requestAbort.signal); + if (!instance) { + respondBuildUnavailable( + res, + 'GRAPHILE_BUILD_WAIT_TIMEOUT', + 'GraphQL schema build is still in progress' + ); + return; + } + if (runtimeSafetyRequired) await ensureRuntimeSafety(); + await revalidateEntryRealtimeRole(instance); + if (invokeEntry(instance)) return; + respondBuildUnavailable( + res, + 'GRAPHILE_INSTANCE_ROTATING', + 'GraphQL schema instance is rotating', + 1 + ); + return; } catch (error) { - log.warn(`${label} Coalesced request failed for PostGraphile[${key}], retrying`); - // Fall through to Phase C to retry creation + if (handleBuildWaitAbort(res, error, requestAbort.signal)) return; + if (handleBuildAvailabilityError(res, error)) return; + throw error; } } - // ========================================================================= - // Phase C: Create New Handler (first request for this key) - // ========================================================================= - - // Re-check cache after coalesced request failure (another retry may have succeeded) - const recheckedCache = graphileCache.get(key); - if (recheckedCache) { - log.debug(`${label} PostGraphile cache hit on re-check key=${key}`); - return recheckedCache.handler(req, res, next); + const earlyDecision = evaluateBuildAdmission(); + if (!earlyDecision.admit && earlyDecision.reason === 'critical_pressure') { + recordBuildRefusal(earlyDecision.reason); + respondBuildUnavailable( + res, + 'GRAPHILE_BUILD_MEMORY_PRESSURE', + 'Server memory pressure is too high to start a new GraphQL schema build' + ); + return; } - - // Re-check in-flight map (another retry may have started creation) - const retryInFlight = creating.get(key); - if (retryInFlight) { - log.debug(`${label} Re-coalescing request for PostGraphile[${key}]`); - const retryInstance = await retryInFlight; - return retryInstance.handler(req, res, next); + if (!earlyDecision.admit && earlyDecision.reason === 'resident_capacity') { + recordBuildRefusal(earlyDecision.reason); + handleBuildAvailabilityError(res, new CacheBuildAdmissionError( + earlyDecision.reason + )); + return; } log.info( - `${label} Building PostGraphile v5 handler key=${key} db=${dbname} schemas=${schemaLabel} role=${roleName} anon=${anonRole}` + `${label} Building PostGraphile v5 handler key=${key} route=${serviceKey} db=${dbname} schemas=${schemaLabel} role=${roleName} anon=${anonRole}` ); - const pgConfig = getPgEnvOptions({ - ...opts.pg, - database: dbname - }); + const buildGeneration = captureGraphileBuildGeneration(); + const buildState: InFlightGraphileBuild = { + promise: null as unknown as Promise, + serviceKey, + databaseId: api.databaseId ?? null, + invalidated: false, + admitted: false, + waiterCount: 0, + abortController: new AbortController() + }; + const creationPromise = runGraphileBuild(async () => { + let poolLeaseOwner: GraphileBuildPoolLeaseOwner | undefined; + let sharedRealtimeBuild: { + subscriber: ActivatableGenerationScopedRealtimeSubscriber; + topicCollector: RealtimeTopicCollector; + } | undefined; + let sharedRealtimeOwnershipTransferred = false; + let websocketOperationAdmission: + GraphileWebSocketOperationAdmission | undefined; + try { + const builtWhileQueued = graphileCache.get(key); + if (builtWhileQueued && !builtWhileQueued.disposing) return builtWhileQueued; - // Route through pg-cache so the pool is tracked and can be cleaned up - // properly, preventing leaked connections during database teardown. - const pool = getPgPool(pgConfig); + await prepareCacheForBuild(); + if ( + buildState.invalidated + || !isGraphileBuildGenerationCurrent(buildGeneration) + ) { + throw new GraphileBuildInvalidatedError(); + } - // Create promise and store in in-flight map BEFORE try block - const compute = api.apiId ? await req.constructive?.useModule('compute') : undefined; - const preset = buildPreset(pool, schema || [], anonRole, roleName, api.databaseSettings, api.apiId, compute); - const creationPromise = observeGraphileBuild( - { - cacheKey: key, - serviceKey: key, - databaseId: api.databaseId ?? null - }, - () => createGraphileInstance({ - preset, - cacheKey: key, - enableRealtime: api.databaseSettings?.enableRealtime - }), - { enabled: observabilityEnabled } - ); - creating.set(key, creationPromise); + // A queued build retains only immutable contract inputs. The large + // preset and runtime-pool lease are acquired after the serialized + // heap slot is granted and are owned until publication or failure. + const buildPoolLease = acquirePgPool(pgConfig, poolOptions); + poolLeaseOwner = new GraphileBuildPoolLeaseOwner(buildPoolLease); + const pool = buildPoolLease.pool; + if (notificationPgConfig) { + sharedRealtimeBuild = { + subscriber: new ActivatableGenerationScopedRealtimeSubscriber(), + topicCollector: new RealtimeTopicCollector() + }; + } + if (realtimeEnabled) { + websocketOperationAdmission = createGraphileWebSocketOperationAdmission({ + cacheKey: key, + databaseId: api.databaseId ?? '', + databaseName: dbname, + apiId: api.apiId ?? '', + schemas: schema ?? [], + authenticatedRole: roleName, + anonymousRole: anonRole, + dependencySchemas: runtimeDependencySchemas, + runtimeSafetyRequired + }); + } + const preset = buildPreset( + pool, + schema || [], + anonRole, + roleName, + api.databaseSettings, + api.apiId, + compute, + storage, + introspectionMode, + introspectionClientReleaseMode, + introspectionDependencySchemas, + opts.graphile?.grafastCache, + opts.graphile?.releaseBuildStateAfterValidation ?? false, + realtimeEnabled, + sharedRealtimeBuild, + websocketOperationAdmission?.plugin, + opts.graphile?.extends, + opts.graphile?.preset, + callerPresetsTrusted + ); + + const instance = await observeGraphileBuild( + { + cacheKey: key, + serviceKey, + databaseId: api.databaseId ?? null + }, + async () => { + if (runtimeSafetyRequired) { + await refreshRuntimeRoleSafety( + pool, + [anonRole, roleName], + schema ?? [], + runtimeDependencySchemas + ); + } + const built = await createGraphileInstance({ + preset, + cacheKey: key, + poolIdentity, + poolLease: poolLeaseOwner!.lease, + serviceKey, + databaseId: api.databaseId ?? null, + enableRealtime: realtimeEnabled, + enableWebsockets: realtimeEnabled, + realtimeSchema: realtimeSchema ?? undefined, + realtimeSourceSchemas: schema ?? [], + realtimeCursorPollIntervalMs: realtimeCursorIntervals.pollIntervalMs, + realtimeCursorHeartbeatIntervalMs: + realtimeCursorIntervals.heartbeatIntervalMs, + ...(notificationPgConfig && sharedRealtimeBuild + && realtimeListenerPoolIdentity ? { + sharedRealtime: { + ...sharedRealtimeBuild, + listenerPgConfig: notificationPgConfig, + listenerIdentity: realtimeListenerPoolIdentity, + roleRevalidationMs: + realtimeNotificationRoleRevalidationMs + } + } : {}) + }); + sharedRealtimeOwnershipTransferred = Boolean(sharedRealtimeBuild); + try { + websocketOperationAdmission?.bind(built); + poolLeaseOwner!.transferTo(built); + } catch (transferError) { + try { + await disposeUncachedEntry(built, key); + } catch (cleanupError) { + throw new GraphileBuildPublicationError( + `PostGraphile[${key}] lease-transfer cleanup failed`, + cleanupError + ); + } + throw transferError; + } + return built; + }, + { enabled: observabilityEnabled } + ); + return publishGraphileBuild( + key, + instance, + buildState.invalidated || !isGraphileBuildGenerationCurrent(buildGeneration) + ); + } finally { + // Covers queued-cache hits, admission/safety failures, and rejected + // instance creation. Entry-owned leases were cleared above. + poolLeaseOwner?.release(); + if (sharedRealtimeBuild && !sharedRealtimeOwnershipTransferred) { + await sharedRealtimeBuild.subscriber.release(); + } + } + }, { + signal: buildState.abortController.signal, + onAdmitted: () => { + buildState.admitted = true; + } + }); + buildState.promise = creationPromise; + creating.set(key, buildState); + + void creationPromise + .then(() => log.info(`${label} PostGraphile v5 handler ready key=${key} db=${dbname}`)) + .catch(() => { + // The request path records the concrete failure. Detached builds may + // finish after a waiter timed out; their rejection is intentionally consumed. + }) + .finally(() => { + if (creating.get(key) === buildState) creating.delete(key); + }); try { - const instance = await creationPromise; - graphileCache.set(key, instance); - log.info(`${label} Cached PostGraphile v5 handler key=${key} db=${dbname}`); - return instance.handler(req, res, next); + const instance = await waitForInFlightGraphileBuild(buildState, requestAbort.signal); + if (!instance) { + respondBuildUnavailable( + res, + 'GRAPHILE_BUILD_WAIT_TIMEOUT', + 'GraphQL schema build is still in progress' + ); + return; + } + if (runtimeSafetyRequired) await ensureRuntimeSafety(); + await revalidateEntryRealtimeRole(instance); + if (invokeEntry(instance)) return; + respondBuildUnavailable( + res, + 'GRAPHILE_INSTANCE_ROTATING', + 'GraphQL schema instance is rotating', + 1 + ); + return; } catch (error) { + if (handleBuildWaitAbort(res, error, requestAbort.signal)) return; + if (handleBuildAvailabilityError(res, error)) return; log.error(`${label} Failed to create PostGraphile[${key}]:`, error); throw new HandlerCreationError( `Failed to create handler for ${key}: ${error instanceof Error ? error.message : String(error)}`, @@ -433,11 +1212,11 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { cause: error instanceof Error ? error.message : String(error) } ); - } finally { - // Always clean up in-flight tracker - creating.delete(key); } } catch (e: any) { + if (isGraphileRequestTerminal(req, res)) return; + if (handleBuildWaitAbort(res, e, requestAbort.signal)) return; + if (!res.headersSent && handleBuildAvailabilityError(res, e)) return; log.error(`${label} PostGraphile middleware error`, e); if (!res.headersSent) { respondWithGraphQLError( @@ -449,6 +1228,8 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { return; } next(e); + } finally { + requestAbort.cleanup(); } }; }; diff --git a/graphql/server/src/middleware/internal-request.ts b/graphql/server/src/middleware/internal-request.ts new file mode 100644 index 0000000000..56ba932882 --- /dev/null +++ b/graphql/server/src/middleware/internal-request.ts @@ -0,0 +1,156 @@ +import { timingSafeEqual } from 'node:crypto'; + +import type { SecurityGucKey } from '@constructive-io/express-context'; +import type { Request } from 'express'; + +import type { ApiOptions } from '../types'; + +export const INTERNAL_REQUEST_TOKEN_HEADER = 'X-Constructive-Internal-Token'; +export const MIN_INTERNAL_REQUEST_SECRET_BYTES = 32; + +const PRIVATE_ROUTING_HEADERS = [ + 'X-Api-Name', + 'X-Schemata', + 'X-Meta-Schema', + 'X-Database-Id' +] as const; + +const PRIVATE_IDENTITY_HEADERS = [ + 'X-Actor-Id', + 'X-Entity-Id', + 'X-Organization-Id' +] as const; + +const hasHeader = (req: Request, name: string): boolean => + req.get(name) !== undefined; + +const hasAnyHeader = (req: Request, names: readonly string[]): boolean => + names.some((name) => hasHeader(req, name)); + +const hasBlankHeader = (req: Request, names: readonly string[]): boolean => + names.some((name) => { + const value = req.get(name); + return value !== undefined && value.trim().length === 0; + }); + +const secretIsWellFormed = (secret: string | undefined): secret is string => + typeof secret === 'string' + && Buffer.byteLength(secret) >= MIN_INTERNAL_REQUEST_SECRET_BYTES; + +const secretsEqual = (expected: string, actual: string): boolean => { + const expectedBytes = Buffer.from(expected); + const actualBytes = Buffer.from(actual); + return expectedBytes.length === actualBytes.length + && timingSafeEqual(expectedBytes, actualBytes); +}; + +const forbidden = (message: string): Error & { code: string } => + Object.assign(new Error(message), { code: 'INTERNAL_REQUEST_FORBIDDEN' }); + +/** + * Reject a configured internal secret at startup when it cannot provide a + * meaningful bearer-token boundary. Omitting the secret is allowed, but then + * every reserved internal header and the HTTP cache flush endpoint fail closed. + */ +export const assertInternalRequestSecret = (opts: ApiOptions): void => { + const secret = opts.api?.internalRequestSecret; + if (secret !== undefined && !secretIsWellFormed(secret)) { + throw new Error( + `api.internalRequestSecret must contain at least ${MIN_INTERNAL_REQUEST_SECRET_BYTES} bytes` + ); + } +}; + +/** + * Authenticate reserved ingress headers before they can influence routing or + * database claims. The raw X-Schemata selector is deliberately prohibited: an + * authenticated proxy must select an authoritative API record by name instead + * of supplying an unchecked physical schema list. + */ +export const authorizeInternalRequest = ( + opts: ApiOptions, + req: Request +): void => { + req.internalTrusted = false; + + const hasRoutingHeaders = hasAnyHeader(req, PRIVATE_ROUTING_HEADERS); + const hasIdentityHeaders = hasAnyHeader(req, PRIVATE_IDENTITY_HEADERS); + const presentedSecret = req.get(INTERNAL_REQUEST_TOKEN_HEADER); + const hasInternalCredential = presentedSecret !== undefined; + + if (!hasRoutingHeaders && !hasIdentityHeaders && !hasInternalCredential) { + return; + } + + const configuredSecret = opts.api?.internalRequestSecret; + if ( + !secretIsWellFormed(configuredSecret) + || !presentedSecret + || !secretsEqual(configuredSecret, presentedSecret) + ) { + throw forbidden('Reserved internal request headers require authentication.'); + } + + // Internal route and actor selectors have no meaning on the public ingress. + // The token by itself remains valid there so operators can authenticate the + // cache-administration endpoint for an already-authoritatively-routed host. + if ((hasRoutingHeaders || hasIdentityHeaders) && opts.api?.isPublic !== false) { + throw forbidden('Private routing and identity headers are disabled on the public ingress.'); + } + + if (hasHeader(req, 'X-Schemata')) { + throw forbidden( + 'X-Schemata is not a production-safe routing contract; use X-Api-Name with X-Database-Id.' + ); + } + + if (hasBlankHeader(req, [...PRIVATE_ROUTING_HEADERS, ...PRIVATE_IDENTITY_HEADERS])) { + throw forbidden('Reserved internal request headers must not be empty.'); + } + + const hasApiName = hasHeader(req, 'X-Api-Name'); + const hasMetaSchema = hasHeader(req, 'X-Meta-Schema'); + const hasDatabaseId = hasHeader(req, 'X-Database-Id'); + if (hasApiName && hasMetaSchema) { + throw forbidden('Private requests must select exactly one API surface.'); + } + if (hasDatabaseId !== (hasApiName || hasMetaSchema)) { + throw forbidden( + 'X-Database-Id must be paired with exactly one private API selector.' + ); + } + if (hasMetaSchema && opts.api?.allowMetaSchemaHeader !== true) { + throw forbidden( + 'The privileged metadata API is disabled on this ingress.' + ); + } + + req.internalTrusted = true; +}; + +/** Translate authenticated private-ingress identity headers only. */ +export const getTrustedInternalClaims = ( + req: Request | undefined +): Partial> => { + if ( + !req + || req.api?.isPublic !== false + || req.internalTrusted !== true + || req.token?.user_id + ) { + return {}; + } + + const actorId = req.get('X-Actor-Id'); + if (!actorId) return {}; + + const claims: Partial> = { + 'jwt.claims.user_id': actorId, + 'jwt.claims.principal_id': actorId + }; + const entityId = req.get('X-Entity-Id'); + const organizationId = req.get('X-Organization-Id'); + if (entityId) claims['jwt.claims.entity_id'] = entityId; + if (organizationId) claims['jwt.claims.organization_id'] = organizationId; + return claims; +}; diff --git a/graphql/server/src/middleware/observability/__tests__/guard.test.ts b/graphql/server/src/middleware/observability/__tests__/guard.test.ts index 6473519968..e09d887bf7 100644 --- a/graphql/server/src/middleware/observability/__tests__/guard.test.ts +++ b/graphql/server/src/middleware/observability/__tests__/guard.test.ts @@ -2,12 +2,19 @@ import type { NextFunction, Request, Response } from 'express'; import { localObservabilityOnly } from '../guard'; -function makeReq(input: { remoteAddress?: string | null; host?: string } = {}): Request { +function makeReq(input: { + remoteAddress?: string | null; + host?: string; + authorization?: string; +} = {}): Request { return { socket: { remoteAddress: input.remoteAddress ?? '::1', }, - headers: input.host ? { host: input.host } : {}, + headers: { + ...(input.host ? { host: input.host } : {}), + ...(input.authorization ? { authorization: input.authorization } : {}) + }, } as unknown as Request; } @@ -23,6 +30,17 @@ function makeNext(): NextFunction { } describe('localObservabilityOnly', () => { + const originalEnv = { ...process.env }; + + beforeEach(() => { + process.env.NODE_ENV = 'development'; + delete process.env.GRAPHQL_OBSERVABILITY_TOKEN; + }); + + afterEach(() => { + process.env = { ...originalEnv }; + }); + it('allows loopback requests', () => { const req = makeReq({ remoteAddress: '::ffff:127.0.0.1', host: 'localhost:3000' }); const res = makeRes(); @@ -45,4 +63,35 @@ describe('localObservabilityOnly', () => { expect(res.status).toHaveBeenCalledWith(404); expect(res.send).toHaveBeenCalledWith('Not found'); }); + + it('requires the configured bearer token for production loopback requests', () => { + process.env.NODE_ENV = 'production'; + process.env.GRAPHQL_OBSERVABILITY_TOKEN = 'c'.repeat(64); + const missing = makeReq({ remoteAddress: '127.0.0.1' }); + const wrong = makeReq({ + remoteAddress: '127.0.0.1', + authorization: `Bearer ${'d'.repeat(64)}` + }); + const valid = makeReq({ + remoteAddress: '127.0.0.1', + authorization: `Bearer ${'c'.repeat(64)}` + }); + const missingRes = makeRes(); + const wrongRes = makeRes(); + const validRes = makeRes(); + const missingNext = makeNext(); + const wrongNext = makeNext(); + const validNext = makeNext(); + + localObservabilityOnly(missing, missingRes, missingNext); + localObservabilityOnly(wrong, wrongRes, wrongNext); + localObservabilityOnly(valid, validRes, validNext); + + expect(missingNext).not.toHaveBeenCalled(); + expect(wrongNext).not.toHaveBeenCalled(); + expect(missingRes.status).toHaveBeenCalledWith(404); + expect(wrongRes.status).toHaveBeenCalledWith(404); + expect(validNext).toHaveBeenCalledTimes(1); + expect(validRes.status).not.toHaveBeenCalled(); + }); }); diff --git a/graphql/server/src/middleware/observability/guard.ts b/graphql/server/src/middleware/observability/guard.ts index 6790e15e15..f68c6c5c0a 100644 --- a/graphql/server/src/middleware/observability/guard.ts +++ b/graphql/server/src/middleware/observability/guard.ts @@ -1,16 +1,26 @@ import type { RequestHandler } from 'express'; -import { isLoopbackAddress, isLoopbackHost } from '../../diagnostics/observability'; +import { + isDevelopmentObservabilityMode, + isGraphqlObservabilityTokenValid, + isLoopbackAddress, + isLoopbackHost +} from '../../diagnostics/observability'; + +const bearerToken = (authorization: string | undefined): string | null => { + const match = /^Bearer\s+(.+)$/i.exec(authorization?.trim() ?? ''); + return match?.[1] ?? null; +}; export const localObservabilityOnly: RequestHandler = (req, res, next) => { const remoteAddress = req.socket.remoteAddress; - if (isLoopbackAddress(remoteAddress)) { - next(); - return; - } - const hostHeader = req.headers.host; - if (!remoteAddress && isLoopbackHost(hostHeader)) { + const isLocal = isLoopbackAddress(remoteAddress) + || (!remoteAddress && isLoopbackHost(hostHeader)); + const isAuthorized = isDevelopmentObservabilityMode() + || isGraphqlObservabilityTokenValid(bearerToken(req.headers.authorization)); + + if (isLocal && isAuthorized) { next(); return; } diff --git a/graphql/server/src/middleware/realtime-config.ts b/graphql/server/src/middleware/realtime-config.ts new file mode 100644 index 0000000000..eb5a24914f --- /dev/null +++ b/graphql/server/src/middleware/realtime-config.ts @@ -0,0 +1,27 @@ +import type { ConstructiveOptions } from '@constructive-io/graphql-types'; +import { DEFAULT_GRAPHILE_REALTIME_SCHEMA } from 'graphile-cache'; + +/** Resolve the exact cursor-function schema for one enabled Graphile surface. */ +export const resolveGraphileRealtimeSchema = ( + opts: ConstructiveOptions, + enableRealtime: boolean +): string | null => { + if (!enableRealtime) return null; + const configured = opts.graphile?.realtimeSchema; + if (configured === undefined) return DEFAULT_GRAPHILE_REALTIME_SCHEMA; + if (typeof configured !== 'string' || configured.length === 0) { + throw new Error('graphile.realtimeSchema must be one non-empty exact schema name'); + } + return configured; +}; + +/** Approve the cursor schema for runtime-role safety without exposing it. */ +export const addRealtimeRuntimeDependencySchema = ( + dependencySchemas: readonly string[], + realtimeSchema: string | null +): string[] => [ + ...new Set([ + ...dependencySchemas, + ...(realtimeSchema ? [realtimeSchema] : []) + ]) +]; diff --git a/graphql/server/src/middleware/realtime-notification-config.ts b/graphql/server/src/middleware/realtime-notification-config.ts new file mode 100644 index 0000000000..531fcf345c --- /dev/null +++ b/graphql/server/src/middleware/realtime-notification-config.ts @@ -0,0 +1,144 @@ +import type { + ConstructiveOptions, + GraphileRealtimeNotificationMode, + NotificationPgResolverInput +} from '@constructive-io/graphql-types'; +import type { PgNotificationListenerConfig } from 'pg-cache'; +import { getPgEnvOptions } from 'pg-env'; + +export const GRAPHILE_REALTIME_NOTIFICATION_CONFIG_ERROR_CODE = + 'GRAPHILE_REALTIME_NOTIFICATION_CONFIG_INVALID'; + +export class GraphileRealtimeNotificationConfigError extends Error { + readonly code = GRAPHILE_REALTIME_NOTIFICATION_CONFIG_ERROR_CODE; + + constructor(message: string) { + super(message); + this.name = 'GraphileRealtimeNotificationConfigError'; + } +} + +export const resolveRealtimeNotificationMode = ( + options: ConstructiveOptions +): GraphileRealtimeNotificationMode => { + const mode = options.graphile?.realtimeNotificationMode ?? 'dedicated'; + if (mode !== 'dedicated' && mode !== 'shared-exact') { + throw new GraphileRealtimeNotificationConfigError( + 'graphile.realtimeNotificationMode must be dedicated or shared-exact' + ); + } + return mode; +}; + +export const resolveRealtimeNotificationRoleRevalidationMs = ( + options: ConstructiveOptions +): number => { + const value = options.graphile?.realtimeNotificationRoleRevalidationMs ?? 60_000; + if (!Number.isSafeInteger(value) || value <= 0) { + throw new GraphileRealtimeNotificationConfigError( + 'graphile.realtimeNotificationRoleRevalidationMs must be a positive safe integer' + ); + } + return value; +}; + +const positiveInterval = (value: number, setting: string): number => { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new GraphileRealtimeNotificationConfigError( + `${setting} must be a positive safe integer` + ); + } + return value; +}; + +export const resolveRealtimeCursorIntervals = ( + options: ConstructiveOptions +): { pollIntervalMs: number; heartbeatIntervalMs: number } => ({ + pollIntervalMs: positiveInterval( + options.graphile?.realtimeCursorPollIntervalMs ?? 5_000, + 'graphile.realtimeCursorPollIntervalMs' + ), + heartbeatIntervalMs: positiveInterval( + options.graphile?.realtimeCursorHeartbeatIntervalMs ?? 30_000, + 'graphile.realtimeCursorHeartbeatIntervalMs' + ) +}); + +/** + * Resolve one dedicated listener login without inheriting control-plane or + * runtime credentials. Network/TLS defaults may be shared, but user, password, + * and physical database must be explicit in every resolver result. + */ +export const resolveRealtimeNotificationPgConfig = async ( + options: ConstructiveOptions, + input: NotificationPgResolverInput +): Promise => { + const resolver = options.notificationPgResolver; + if (typeof resolver !== 'function') { + throw new GraphileRealtimeNotificationConfigError( + 'shared-exact realtime requires notificationPgResolver' + ); + } + + let resolved: Awaited>; + try { + resolved = await resolver(Object.freeze({ + databaseId: input.databaseId, + databaseName: input.databaseName, + apiId: input.apiId, + schemas: Object.freeze([...input.schemas]) + })); + } catch { + throw new GraphileRealtimeNotificationConfigError( + 'notificationPgResolver failed for the requested physical database' + ); + } + if (!resolved || typeof resolved !== 'object' || Array.isArray(resolved)) { + throw new GraphileRealtimeNotificationConfigError( + 'notificationPgResolver must return a PostgreSQL configuration object' + ); + } + if (Object.prototype.hasOwnProperty.call(resolved, 'connectionString')) { + throw new GraphileRealtimeNotificationConfigError( + 'notificationPgResolver must not return a connectionString; use explicit fields' + ); + } + if (typeof resolved.user !== 'string' || resolved.user.trim().length === 0) { + throw new GraphileRealtimeNotificationConfigError( + 'notificationPgResolver must return an explicit user' + ); + } + if (typeof resolved.password !== 'string' || resolved.password.length === 0) { + throw new GraphileRealtimeNotificationConfigError( + 'notificationPgResolver must return an explicit password' + ); + } + if (resolved.database !== input.databaseName) { + throw new GraphileRealtimeNotificationConfigError( + 'notificationPgResolver database does not match the routed physical database' + ); + } + + const networkDefaults = { + ...(options.pg?.host === undefined ? {} : { host: options.pg.host }), + ...(options.pg?.port === undefined ? {} : { port: options.pg.port }), + ...(options.pg?.ssl === undefined ? {} : { ssl: options.pg.ssl }) + }; + const normalized = getPgEnvOptions({ + ...networkDefaults, + ...resolved + }); + if ( + normalized.user !== resolved.user + || normalized.password !== resolved.password + || normalized.database !== input.databaseName + ) { + throw new GraphileRealtimeNotificationConfigError( + 'notification PostgreSQL identity changed during normalization' + ); + } + return { + ...normalized, + ...(resolved.pool ? { pool: { ...resolved.pool } } : {}) + }; +}; diff --git a/graphql/server/src/middleware/routing.ts b/graphql/server/src/middleware/routing.ts index 752b32bdff..6dc2db7474 100644 --- a/graphql/server/src/middleware/routing.ts +++ b/graphql/server/src/middleware/routing.ts @@ -53,6 +53,17 @@ export const getRoutingSchema = (opts: { export const isValidSchemaName = (name: string): boolean => /^[a-z_][a-z0-9_]*$/.test(name); +/** + * Constructive physical schemas may contain the generated dash separators used + * by tenant prefixes. They are always passed as data or quoted identifiers, but + * keep the accepted alphabet deliberately narrow and reject system namespaces. + */ +export const isValidPhysicalSchemaName = (name: string): boolean => + /^[a-z_][a-z0-9_-]*$/.test(name) + && name.length <= 63 + && name !== 'information_schema' + && !name.startsWith('pg_'); + /** * Resolve a hostname through the compiled scoped-routing plane (host-only: * path/method routing belongs to Traefik/Ingress, not the server). @@ -75,6 +86,13 @@ export const resolveRoute = async ( `SELECT * FROM "${schema}".${RESOLVER_FUNCTION}($1, '/', NULL)`, [host] ); + if (result.rows.length !== 1) { + log.warn( + `[resolve-route] expected exactly one resolver row for host=${host}; ` + + `received ${result.rows.length}` + ); + return null; + } const row = result.rows[0]; if (!row || row.route_binding_id === null) { log.debug(`[resolve-route] no match for host=${host}`); @@ -121,21 +139,47 @@ export const routeToApiStructure = ( } const config = (route.resolved_config ?? {}) as ApiSurfaceConfig; - if (!config.schemas?.length) { - log.debug('[resolve-route] api target missing schemas in resolved_config; no match'); + const expectedPublic = opts.api?.isPublic ?? false; + if (typeof config.is_public !== 'boolean' || config.is_public !== expectedPublic) { + log.warn('[resolve-route] api visibility does not match this server ingress; no match'); + return null; + } + + if ( + !config.api_id + || !config.database_id + || route.target_source_id !== config.api_id + || route.target_owner_scope !== 'database' + || route.target_owner_key !== config.database_id + ) { + log.warn('[resolve-route] api target missing exact api/database identity; no match'); + return null; + } + + if ( + !config.schemas?.length + || config.schemas.some((schema) => !isValidPhysicalSchemaName(schema)) + || new Set(config.schemas).size !== config.schemas.length + ) { + log.warn('[resolve-route] api target has an invalid physical schema contract; no match'); + return null; + } + + if (!config.role_name || !config.anon_role) { + log.warn('[resolve-route] api target missing exact request roles; no match'); return null; } return { - apiId: config.api_id ?? route.target_source_id ?? undefined, + apiId: config.api_id, // Scoped APIs leave dbname NULL when their schemas live in the serving // database; fall back to the server's own database in that case. dbname: config.dbname || opts.pg?.database || '', - anonRole: config.anon_role || 'anon', - roleName: config.role_name || 'authenticated', + anonRole: config.anon_role, + roleName: config.role_name, schema: config.schemas, domains: [], databaseId: config.database_id, - isPublic: config.is_public ?? (opts.api?.isPublic ?? false) + isPublic: config.is_public }; }; diff --git a/graphql/server/src/middleware/runtime-pg-config.ts b/graphql/server/src/middleware/runtime-pg-config.ts new file mode 100644 index 0000000000..a2da4c251e --- /dev/null +++ b/graphql/server/src/middleware/runtime-pg-config.ts @@ -0,0 +1,548 @@ +import type { + RuntimePgPoolResolution +} from '@constructive-io/express-context'; +import type { + ConstructiveOptions, + RuntimePgConfig, + RuntimePgResolverInput +} from '@constructive-io/graphql-types'; +import { getNodeEnv } from '@pgpmjs/env'; +import type { NextFunction, Request, RequestHandler, Response } from 'express'; +import { getPgPoolIdentity } from 'pg-cache'; +import { getPgEnvOptions } from 'pg-env'; + +import type { ApiStructure } from '../types'; +import { + assertRuntimePgCredentials, + InvalidRuntimePgConfigurationError, + requiresExactRuntimePgResolution +} from './runtime-pg-requirements'; + +const RUNTIME_POOL_OPTIONS = { + purpose: 'runtime', + sanitizeOnCheckout: true +} as const; + +const TARGET_ATTESTATION_POOL = Object.freeze({ + max: 1, + maxUses: 1, + idleTimeoutMillis: 0, + connectionTimeoutMillis: 0, + allowExitOnIdle: true +}); + +const TARGET_ATTESTATION_OPTIONS = { + purpose: 'runtime-target-attestation', + sanitizeOnCheckout: true +} as const; + +const CONFIG_KEYS = new Set([ + 'host', + 'port', + 'user', + 'password', + 'database', + 'ssl', + 'pool' +]); + +const POOL_KEYS = new Set([ + 'max', + 'maxUses', + 'idleTimeoutMillis', + 'connectionTimeoutMillis', + 'allowExitOnIdle' +]); + +const STATIC_IDENTITY_KEYS = new Set([ + 'databaseId', + 'databaseName', + 'apiId', + 'schemas', + 'roles' +]); + +const ownDataRecord = ( + value: unknown, + label: string, + allowedKeys: ReadonlySet +): Record => { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new InvalidRuntimePgConfigurationError( + `${label} must be a PostgreSQL configuration object` + ); + } + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new InvalidRuntimePgConfigurationError( + `${label} must contain only plain data` + ); + } + const keys = Reflect.ownKeys(value); + if (keys.some((key) => typeof key !== 'string')) { + throw new InvalidRuntimePgConfigurationError( + `${label} must not contain symbol properties` + ); + } + for (const key of keys as string[]) { + if (key === 'connectionString') { + throw new InvalidRuntimePgConfigurationError( + `${label} must not return a connectionString; use explicit fields` + ); + } + if (!allowedKeys.has(key)) { + throw new InvalidRuntimePgConfigurationError( + `${label} contains unsupported field '${key}'` + ); + } + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (!descriptor || !('value' in descriptor) || descriptor.value === undefined) { + throw new InvalidRuntimePgConfigurationError( + `${label}.${key} must be an explicit data value` + ); + } + } + return value as Record; +}; + +const exactArray = ( + value: unknown, + label: string, + length?: number +): readonly unknown[] => { + if (!Array.isArray(value) || (length !== undefined && value.length !== length)) { + throw new InvalidRuntimePgConfigurationError( + `${label} must be ${length === undefined ? 'an array' : `an array of length ${length}`}` + ); + } + const keys = Reflect.ownKeys(value); + if ( + keys.some((key) => typeof key !== 'string') + || keys.some((key) => key !== 'length' && !/^(?:0|[1-9]\d*)$/.test(key as string)) + || Object.keys(value).length !== value.length + ) { + throw new InvalidRuntimePgConfigurationError( + `${label} must be a dense array without custom properties` + ); + } + return value; +}; + +const exactString = ( + value: unknown, + label: string, + allowEmpty = false +): string => { + if ( + typeof value !== 'string' + || (!allowEmpty && value.trim().length === 0) + ) { + throw new InvalidRuntimePgConfigurationError( + `${label} must be ${allowEmpty ? 'a string' : 'a non-empty string'}` + ); + } + return value; +}; + +const normalizeResolverInput = ( + value: RuntimePgResolverInput, + label = 'runtime PostgreSQL route identity' +): Readonly => { + const record = ownDataRecord(value, label, STATIC_IDENTITY_KEYS); + if (Reflect.ownKeys(record).length !== STATIC_IDENTITY_KEYS.size) { + throw new InvalidRuntimePgConfigurationError( + `${label} must contain databaseId, databaseName, apiId, schemas, and roles` + ); + } + const schemas = exactArray(record.schemas, `${label}.schemas`).map( + (schema, index) => exactString(schema, `${label}.schemas[${index}]`) + ); + if (schemas.length === 0 || new Set(schemas).size !== schemas.length) { + throw new InvalidRuntimePgConfigurationError( + `${label}.schemas must contain at least one unique physical schema` + ); + } + const roles = exactArray(record.roles, `${label}.roles`, 2).map( + (role, index) => exactString(role, `${label}.roles[${index}]`) + ) as [string, string]; + return Object.freeze({ + databaseId: exactString(record.databaseId, `${label}.databaseId`), + databaseName: exactString(record.databaseName, `${label}.databaseName`), + apiId: exactString(record.apiId, `${label}.apiId`, true), + schemas: Object.freeze(schemas), + roles: Object.freeze(roles) as readonly [string, string] + }); +}; + +const sameResolverInput = ( + left: Readonly, + right: Readonly +): boolean => + left.databaseId === right.databaseId + && left.databaseName === right.databaseName + && left.apiId === right.apiId + && left.schemas.length === right.schemas.length + && left.schemas.every((schema, index) => schema === right.schemas[index]) + && left.roles[0] === right.roles[0] + && left.roles[1] === right.roles[1]; + +const cloneIdentityData = ( + value: unknown, + path: string, + ancestors = new Set() +): unknown => { + if ( + value === null + || typeof value === 'string' + || typeof value === 'number' + || typeof value === 'boolean' + ) { + return value; + } + if (Buffer.isBuffer(value)) return Buffer.from(value); + if (Array.isArray(value)) { + exactArray(value, path); + if (ancestors.has(value)) { + throw new InvalidRuntimePgConfigurationError(`${path} must not be cyclic`); + } + ancestors.add(value); + const cloned = value.map((entry, index) => + cloneIdentityData(entry, `${path}[${index}]`, ancestors) + ); + ancestors.delete(value); + return Object.freeze(cloned); + } + if (typeof value === 'object') { + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new InvalidRuntimePgConfigurationError( + `${path} must contain only plain data` + ); + } + if (ancestors.has(value)) { + throw new InvalidRuntimePgConfigurationError(`${path} must not be cyclic`); + } + ancestors.add(value); + const cloned: Record = {}; + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== 'string') { + throw new InvalidRuntimePgConfigurationError( + `${path} must not contain symbol properties` + ); + } + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (!descriptor || !('value' in descriptor) || descriptor.value === undefined) { + throw new InvalidRuntimePgConfigurationError( + `${path}.${key} must be an explicit data value` + ); + } + cloned[key] = cloneIdentityData( + descriptor.value, + `${path}.${key}`, + ancestors + ); + } + ancestors.delete(value); + return Object.freeze(cloned); + } + throw new InvalidRuntimePgConfigurationError( + `${path} must contain only deterministic data values` + ); +}; + +const networkDefaults = (options: ConstructiveOptions): RuntimePgConfig => ({ + ...(options.pg?.host === undefined ? {} : { host: options.pg.host }), + ...(options.pg?.port === undefined ? {} : { port: options.pg.port }), + ...(options.pg?.ssl === undefined + ? {} + : { ssl: cloneIdentityData(options.pg.ssl, 'pg.ssl') as RuntimePgConfig['ssl'] }) +}); + +const networkTargetIdentity = (config: RuntimePgConfig): string => + getPgPoolIdentity({ + host: config.host, + port: config.port, + database: config.database, + // Fixed non-connection sentinels make the existing exact/HMAC pool + // identity machinery attest only this physical network/TLS target. + user: 'constructive_target_attestation', + password: 'constructive_target_attestation', + ...(config.ssl === undefined ? {} : { ssl: config.ssl }), + pool: TARGET_ATTESTATION_POOL + }, TARGET_ATTESTATION_OPTIONS); + +const normalizeRuntimePgConfig = ( + options: ConstructiveOptions, + input: Readonly, + rawValue: unknown, + label: string +): Readonly => { + const raw = ownDataRecord(rawValue, label, CONFIG_KEYS); + const user = exactString(raw.user, `${label}.user`); + const password = exactString(raw.password, `${label}.password`); + const database = exactString(raw.database, `${label}.database`); + if (database !== input.databaseName) { + throw new InvalidRuntimePgConfigurationError( + `${label} database does not match the routed physical database` + ); + } + + let pool: RuntimePgConfig['pool']; + if (raw.pool !== undefined) { + const poolRecord = ownDataRecord(raw.pool, `${label}.pool`, POOL_KEYS); + pool = Object.freeze({ ...poolRecord }) as RuntimePgConfig['pool']; + } + const normalized = getPgEnvOptions({ + ...networkDefaults(options), + ...raw, + user, + password, + database, + ...(raw.ssl === undefined + ? {} + : { ssl: cloneIdentityData(raw.ssl, `${label}.ssl`) as RuntimePgConfig['ssl'] }) + }); + if ( + normalized.user !== user + || normalized.password !== password + || normalized.database !== input.databaseName + ) { + throw new InvalidRuntimePgConfigurationError( + `${label} identity changed during normalization` + ); + } + + try { + const controlTarget = getPgEnvOptions({ + ...networkDefaults(options), + database: input.databaseName + }); + if ( + networkTargetIdentity(normalized) + !== networkTargetIdentity(controlTarget) + ) { + throw new InvalidRuntimePgConfigurationError( + `${label} network/TLS endpoint does not match the routed control-plane database` + ); + } + } catch (error) { + if (error instanceof InvalidRuntimePgConfigurationError) throw error; + throw new InvalidRuntimePgConfigurationError( + `${label} could not attest the routed control-plane network/TLS endpoint` + ); + } + + const pgConfig = Object.freeze({ + host: normalized.host, + port: normalized.port, + user: normalized.user, + password: normalized.password, + database: normalized.database, + ...(normalized.ssl === undefined + ? {} + : { ssl: cloneIdentityData(normalized.ssl, `${label}.ssl`) as RuntimePgConfig['ssl'] }), + ...(pool ? { pool } : {}) + }) as RuntimePgConfig; + let poolIdentity: string; + try { + poolIdentity = getPgPoolIdentity(pgConfig, RUNTIME_POOL_OPTIONS); + } catch { + throw new InvalidRuntimePgConfigurationError( + `${label} could not form an exact normalized pool identity` + ); + } + return Object.freeze({ pgConfig, poolIdentity }); +}; + +/** Build the credential-free exact resolver key from authoritative routing. */ +export const createRuntimePgResolverInput = ( + api: ApiStructure +): Readonly => normalizeResolverInput({ + databaseId: api.databaseId ?? '', + databaseName: api.dbname, + apiId: api.apiId ?? '', + schemas: api.schema, + roles: [api.anonRole, api.roleName] +}); + +/** Resolve and normalize one request's tenant execution identity exactly once. */ +export const resolveRuntimePgConfig = async ( + options: ConstructiveOptions, + inputValue: RuntimePgResolverInput, + nodeEnv = getNodeEnv() +): Promise> => { + assertRuntimePgCredentials(options, nodeEnv); + const input = normalizeResolverInput(inputValue); + const resolver = options.runtimePgResolver; + if (resolver) { + let resolved: Awaited>; + try { + resolved = await resolver(input); + } catch { + throw new InvalidRuntimePgConfigurationError( + 'runtimePgResolver failed for the requested exact route' + ); + } + return normalizeRuntimePgConfig( + options, + input, + resolved, + 'runtimePgResolver result' + ); + } + + if (options.runtimePg) { + if (options.runtimePgStaticIdentity) { + const staticIdentity = normalizeResolverInput( + options.runtimePgStaticIdentity, + 'runtimePgStaticIdentity' + ); + if (!sameResolverInput(staticIdentity, input)) { + throw new InvalidRuntimePgConfigurationError( + 'Static runtimePg is not authorized for the requested exact route' + ); + } + } else if (requiresExactRuntimePgResolution(options, nodeEnv)) { + throw new InvalidRuntimePgConfigurationError( + 'Static runtimePg requires one exact runtimePgStaticIdentity' + ); + } + const configuredDatabase = options.runtimePg.database; + if ( + configuredDatabase !== undefined + && configuredDatabase !== input.databaseName + ) { + throw new InvalidRuntimePgConfigurationError( + 'runtimePg database does not match the routed physical database' + ); + } + return normalizeRuntimePgConfig( + options, + input, + { + ...options.runtimePg, + database: configuredDatabase ?? input.databaseName + }, + 'runtimePg' + ); + } + + // Explicitly unsafe compatibility path for stock local development/tests. + // The startup assertion above prevents this path in production or scoped mode. + const fallback = getPgEnvOptions({ + ...options.pg, + database: input.databaseName + }); + return normalizeRuntimePgConfig( + options, + input, + fallback, + 'development control-plane runtime fallback' + ); +}; + +export interface RuntimePgResolutionStore { + middleware: RequestHandler; + getRuntimePgResolution: ( + req: Request, + api?: ApiStructure + ) => Readonly; +} + +/** + * Keep raw credentials in a server-owned WeakMap, never on `req`. Context and + * Graphile receive the same frozen resolution and verify its opaque identity. + */ +export const createRuntimePgResolutionStore = ( + options: ConstructiveOptions +): RuntimePgResolutionStore => { + assertRuntimePgCredentials(options); + let staticResolution: StoredResolution | null = null; + if (options.runtimePg) { + ownDataRecord(options.runtimePg, 'runtimePg', CONFIG_KEYS); + if (options.runtimePgStaticIdentity) { + const input = normalizeResolverInput( + options.runtimePgStaticIdentity, + 'runtimePgStaticIdentity' + ); + const resolution = normalizeRuntimePgConfig( + options, + input, + options.runtimePg, + 'runtimePg' + ); + staticResolution = { input, resolution }; + } + } + interface StoredResolution { + input: Readonly; + resolution: Readonly; + } + const resolutions = new WeakMap(); + const getRuntimePgResolution = ( + req: Request, + api = req.api + ): Readonly => { + const stored = resolutions.get(req); + if (!stored || !api) { + throw new InvalidRuntimePgConfigurationError( + 'Runtime PostgreSQL resolution is unavailable for this request' + ); + } + const currentInput = createRuntimePgResolverInput(api); + if (!sameResolverInput(stored.input, currentInput)) { + throw new InvalidRuntimePgConfigurationError( + 'Authoritative API route changed after runtime PostgreSQL resolution' + ); + } + return stored.resolution; + }; + const middleware: RequestHandler = async ( + req: Request, + res: Response, + next: NextFunction + ): Promise => { + const requestEnded = (): boolean => Boolean( + req.aborted + || req.socket?.destroyed + || res.destroyed + || res.writableEnded + ); + const cleanup = (): void => { + resolutions.delete(req); + req.removeListener('aborted', cleanup); + res.removeListener('finish', cleanup); + res.removeListener('close', cleanup); + }; + try { + if (requestEnded()) return; + if (!req.api) { + throw new InvalidRuntimePgConfigurationError( + 'Runtime PostgreSQL resolution requires an authoritative API route' + ); + } + const input = createRuntimePgResolverInput(req.api); + let resolution: Readonly; + if (staticResolution) { + if (!sameResolverInput(staticResolution.input, input)) { + throw new InvalidRuntimePgConfigurationError( + 'Static runtimePg is not authorized for the requested exact route' + ); + } + resolution = staticResolution.resolution; + } else { + resolution = await resolveRuntimePgConfig(options, input); + } + if (requestEnded()) return; + resolutions.set(req, { input, resolution }); + req.once('aborted', cleanup); + res.once('finish', cleanup); + res.once('close', cleanup); + next(); + } catch (error) { + cleanup(); + next(error); + } + }; + return { middleware, getRuntimePgResolution }; +}; diff --git a/graphql/server/src/middleware/runtime-pg-requirements.ts b/graphql/server/src/middleware/runtime-pg-requirements.ts new file mode 100644 index 0000000000..b4e43aa05d --- /dev/null +++ b/graphql/server/src/middleware/runtime-pg-requirements.ts @@ -0,0 +1,113 @@ +import type { ConstructiveOptions } from '@constructive-io/graphql-types'; +import { getNodeEnv } from '@pgpmjs/env'; + +export class MissingRuntimePgCredentialsError extends Error { + readonly code = 'GRAPHILE_RUNTIME_PG_REQUIRED'; + + constructor() { + super( + 'GraphQL runtime execution requires an explicit PostgreSQL user and password' + ); + this.name = 'MissingRuntimePgCredentialsError'; + } +} + +export class InvalidRuntimePgConfigurationError extends Error { + readonly code = 'GRAPHILE_RUNTIME_PG_CONFIG_INVALID'; + + constructor(message: string) { + super(message); + this.name = 'InvalidRuntimePgConfigurationError'; + } +} + +export const requiresExactRuntimePgResolution = ( + options: ConstructiveOptions, + nodeEnv = getNodeEnv() +): boolean => + nodeEnv === 'production' + || options.graphile?.introspectionMode === 'scoped-required'; + +/** + * The control-plane login fallback exists only for backwards-compatible local + * stock-mode development and tests. Production and scoped introspection always + * use an explicit, independently audited runtime login. + */ +export const usesUnsafeDevelopmentRuntimePgFallback = ( + options: ConstructiveOptions, + nodeEnv = getNodeEnv() +): boolean => + !requiresExactRuntimePgResolution(options, nodeEnv) + && options.runtimePg === undefined + && options.runtimePgResolver === undefined; + +export const shouldValidateRuntimeRoleSafety = ( + options: ConstructiveOptions, + nodeEnv = getNodeEnv() +): boolean => + nodeEnv === 'production' + || options.runtimePg !== undefined + || options.runtimePgResolver !== undefined + || options.graphile?.introspectionMode === 'scoped-required'; + +export const assertRuntimePgCredentials = ( + options: ConstructiveOptions, + nodeEnv = getNodeEnv() +): void => { + const hasResolver = options.runtimePgResolver !== undefined; + const hasStatic = options.runtimePg !== undefined; + const hasStaticIdentity = options.runtimePgStaticIdentity !== undefined; + if (hasResolver && (hasStatic || hasStaticIdentity)) { + throw new InvalidRuntimePgConfigurationError( + 'runtimePgResolver is mutually exclusive with runtimePg and runtimePgStaticIdentity' + ); + } + if (hasResolver) { + if (typeof options.runtimePgResolver !== 'function') { + throw new InvalidRuntimePgConfigurationError( + 'runtimePgResolver must be a function' + ); + } + return; + } + if (hasStaticIdentity && !hasStatic) { + throw new InvalidRuntimePgConfigurationError( + 'runtimePgStaticIdentity requires runtimePg' + ); + } + if (usesUnsafeDevelopmentRuntimePgFallback(options, nodeEnv)) return; + if (!hasStatic) throw new MissingRuntimePgCredentialsError(); + if (Object.prototype.hasOwnProperty.call(options.runtimePg, 'connectionString')) { + throw new InvalidRuntimePgConfigurationError( + 'runtimePg must not contain a connectionString; use explicit fields' + ); + } + const user = options.runtimePg?.user; + const password = options.runtimePg?.password; + if ( + typeof user !== 'string' + || user.trim().length === 0 + || typeof password !== 'string' + || password.length === 0 + ) { + throw new MissingRuntimePgCredentialsError(); + } + if (requiresExactRuntimePgResolution(options, nodeEnv)) { + if (!hasStaticIdentity) { + throw new InvalidRuntimePgConfigurationError( + 'Production and scoped introspection require runtimePgResolver, or runtimePgStaticIdentity for one exact route' + ); + } + if ( + typeof options.runtimePg?.database !== 'string' + || options.runtimePg.database.length === 0 + ) { + throw new InvalidRuntimePgConfigurationError( + 'Static production runtimePg requires an explicit database' + ); + } + } +}; + +/** @deprecated Use `assertRuntimePgCredentials`; retained for package compatibility. */ +export const assertScopedRuntimePgCredentials = assertRuntimePgCredentials; diff --git a/graphql/server/src/middleware/runtime-role-safety.ts b/graphql/server/src/middleware/runtime-role-safety.ts new file mode 100644 index 0000000000..1cbff507d1 --- /dev/null +++ b/graphql/server/src/middleware/runtime-role-safety.ts @@ -0,0 +1,883 @@ +import { performance } from 'node:perf_hooks'; + +import type { Pool, PoolClient, QueryResult } from 'pg'; + +export const RUNTIME_ROLE_SAFETY_SQL = ` +WITH RECURSIVE execution_roles AS ( + SELECT r.oid, r.rolname + FROM pg_catalog.pg_roles r + WHERE r.rolname = current_user + OR r.rolname = ANY($1::text[]) +), execution_role_reachability AS MATERIALIZED ( + SELECT execution_role.oid AS execution_role_oid, + execution_role.rolname AS execution_role_name, + candidate.oid AS reachable_role_oid, + candidate.rolname AS reachable_role_name, + pg_catalog.pg_has_role(execution_role.oid, candidate.oid, 'USAGE') + AS via_usage, + pg_catalog.pg_has_role(execution_role.oid, candidate.oid, 'SET') + AS via_set + FROM execution_roles execution_role + INNER JOIN pg_catalog.pg_roles candidate + ON candidate.oid = execution_role.oid + OR pg_catalog.pg_has_role(execution_role.oid, candidate.oid, 'USAGE') + OR pg_catalog.pg_has_role(execution_role.oid, candidate.oid, 'SET') +), accessible_roles AS ( + SELECT r.oid, r.rolname, r.rolsuper, r.rolbypassrls, r.rolcreaterole, + r.rolcreatedb, r.rolreplication, r.rolinherit + FROM pg_catalog.pg_roles r + WHERE r.rolname = current_user + OR r.rolname = ANY($1::text[]) + OR pg_catalog.pg_has_role(current_user, r.oid, 'USAGE') + OR pg_catalog.pg_has_role(current_user, r.oid, 'SET') + OR EXISTS ( + SELECT 1 + FROM execution_role_reachability reachable + WHERE reachable.reachable_role_oid = r.oid + ) +), exposed_schemas AS ( + SELECT n.oid, n.nspname, n.nspowner + FROM pg_catalog.pg_namespace n + WHERE n.nspname = ANY($2::text[]) +), approved_schemas AS ( + SELECT n.oid, n.nspname, n.nspowner + FROM pg_catalog.pg_namespace n + WHERE n.nspname = ANY($2::text[] || $3::text[]) +), current_database_record AS ( + SELECT d.oid, d.datname, d.datdba + FROM pg_catalog.pg_database d + WHERE d.datname = pg_catalog.current_database() +), unapproved_schema_access AS MATERIALIZED ( + SELECT r.oid AS role_oid, r.rolname, n.oid AS namespace_oid, n.nspname, + n.nspowner = r.oid AS is_owner, + pg_catalog.has_schema_privilege(r.rolname, n.oid, 'CREATE') AS can_create, + pg_catalog.has_schema_privilege(r.rolname, n.oid, 'USAGE') AS can_use + FROM accessible_roles r + INNER JOIN pg_catalog.pg_namespace n ON true + WHERE n.nspname <> 'information_schema' + AND n.nspname NOT LIKE 'pg\\_%' + AND NOT EXISTS (SELECT 1 FROM approved_schemas a WHERE a.oid = n.oid) + AND ( + n.nspowner = r.oid + OR pg_catalog.has_schema_privilege(r.rolname, n.oid, 'CREATE') + OR pg_catalog.has_schema_privilege(r.rolname, n.oid, 'USAGE') + ) +), login_role_violations AS ( + SELECT array_remove(ARRAY[ + CASE WHEN rolinherit THEN 'INHERIT' END + ], NULL) AS capabilities + FROM accessible_roles + WHERE rolname = current_user AND rolinherit +), inherited_role_violations AS ( + SELECT r.rolname + FROM pg_catalog.pg_roles r + WHERE r.rolname <> current_user + AND pg_catalog.pg_has_role(current_user, r.oid, 'USAGE') +), unexpected_set_role_violations AS ( + SELECT r.rolname + FROM pg_catalog.pg_roles r + WHERE r.rolname <> current_user + AND NOT (r.rolname = ANY($1::text[])) + AND pg_catalog.pg_has_role(current_user, r.oid, 'SET') +), request_role_reachability_violations AS ( + SELECT reachable.execution_role_name AS request_role, + reachable.reachable_role_name AS reachable_role, + reachable.via_usage, + reachable.via_set + FROM execution_role_reachability reachable + WHERE reachable.execution_role_name = ANY($1::text[]) + AND reachable.execution_role_oid <> reachable.reachable_role_oid +), role_violations AS ( + SELECT rolname, + array_remove(ARRAY[ + CASE WHEN rolsuper THEN 'SUPERUSER' END, + CASE WHEN rolbypassrls THEN 'BYPASSRLS' END, + CASE WHEN rolcreaterole THEN 'CREATEROLE' END, + CASE WHEN rolcreatedb THEN 'CREATEDB' END, + CASE WHEN rolreplication THEN 'REPLICATION' END + ], NULL) AS capabilities + FROM accessible_roles + WHERE rolsuper OR rolbypassrls OR rolcreaterole OR rolcreatedb OR rolreplication +), database_violations AS ( + SELECT r.rolname, d.datname, violation.capability + FROM accessible_roles r + INNER JOIN current_database_record d ON true + CROSS JOIN LATERAL ( + VALUES + ('OWNER'::text, d.datdba = r.oid), + ('CREATE'::text, pg_catalog.has_database_privilege(r.rolname, d.oid, 'CREATE')), + ('TEMP'::text, pg_catalog.has_database_privilege(r.rolname, d.oid, 'TEMP')) + ) AS violation(capability, present) + WHERE violation.present +), cross_database_violations AS ( + SELECT r.rolname, d.datname + FROM accessible_roles r + INNER JOIN current_database_record current_database ON true + INNER JOIN pg_catalog.pg_database d ON d.oid <> current_database.oid + WHERE pg_catalog.has_database_privilege(r.rolname, d.oid, 'CONNECT') +), schema_violations AS ( + SELECT r.rolname, n.nspname, + CASE WHEN n.nspowner = r.oid THEN 'OWNER' ELSE 'CREATE' END AS capability + FROM accessible_roles r + INNER JOIN approved_schemas n ON true + WHERE n.nspowner = r.oid + OR pg_catalog.has_schema_privilege(r.rolname, n.nspname, 'CREATE') +), cross_schema_violations AS ( + SELECT access.rolname, access.nspname, + array_remove(ARRAY[ + CASE WHEN access.can_create OR access.is_owner THEN 'CREATE/OWNER' END, + CASE WHEN EXISTS ( + SELECT 1 + FROM pg_catalog.pg_class c + WHERE c.relnamespace = access.namespace_oid + AND c.relkind IN ('r', 'p', 'v', 'm', 'f') + AND pg_catalog.has_table_privilege( + access.rolname, + c.oid, + 'SELECT,INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER' + ) + ) AND access.can_use + THEN 'RELATION' END, + CASE WHEN EXISTS ( + SELECT 1 + FROM pg_catalog.pg_class c + WHERE c.relnamespace = access.namespace_oid + AND CASE WHEN c.relkind = 'S' + THEN pg_catalog.has_sequence_privilege( + access.rolname, + c.oid, + 'USAGE,SELECT,UPDATE' + ) + ELSE false + END + ) AND access.can_use + THEN 'SEQUENCE' END, + CASE WHEN EXISTS ( + SELECT 1 + FROM pg_catalog.pg_proc p + WHERE p.pronamespace = access.namespace_oid + AND pg_catalog.has_function_privilege(access.rolname, p.oid, 'EXECUTE') + ) AND access.can_use + THEN 'FUNCTION' END, + CASE WHEN EXISTS ( + SELECT 1 + FROM pg_catalog.pg_type t + WHERE t.typnamespace = access.namespace_oid + AND pg_catalog.has_type_privilege(access.rolname, t.oid, 'USAGE') + ) AND access.can_use + THEN 'TYPE' END + ], NULL) AS capabilities + FROM unapproved_schema_access access + WHERE access.can_create + OR access.is_owner + OR ( + access.can_use + AND ( + EXISTS ( + SELECT 1 + FROM pg_catalog.pg_class c + WHERE c.relnamespace = access.namespace_oid + AND ( + (c.relkind IN ('r', 'p', 'v', 'm', 'f') AND pg_catalog.has_table_privilege( + access.rolname, + c.oid, + 'SELECT,INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER' + )) + OR CASE WHEN c.relkind = 'S' + THEN pg_catalog.has_sequence_privilege( + access.rolname, + c.oid, + 'USAGE,SELECT,UPDATE' + ) + ELSE false + END + ) + ) + OR EXISTS ( + SELECT 1 + FROM pg_catalog.pg_proc p + WHERE p.pronamespace = access.namespace_oid + AND pg_catalog.has_function_privilege(access.rolname, p.oid, 'EXECUTE') + ) + OR EXISTS ( + SELECT 1 + FROM pg_catalog.pg_type t + WHERE t.typnamespace = access.namespace_oid + AND pg_catalog.has_type_privilege(access.rolname, t.oid, 'USAGE') + ) + ) + ) +), object_owner_violations AS ( + SELECT r.rolname, n.nspname, c.relname AS object_name, + CASE c.relkind + WHEN 'S' THEN 'SEQUENCE' + WHEN 'v' THEN 'VIEW' + WHEN 'm' THEN 'MATERIALIZED VIEW' + WHEN 'f' THEN 'FOREIGN TABLE' + ELSE 'RELATION' + END AS object_kind + FROM accessible_roles r + INNER JOIN approved_schemas n ON true + INNER JOIN pg_catalog.pg_class c + ON c.relnamespace = n.oid AND c.relowner = r.oid + + UNION ALL + + SELECT r.rolname, n.nspname, p.proname, 'FUNCTION' + FROM accessible_roles r + INNER JOIN approved_schemas n ON true + INNER JOIN pg_catalog.pg_proc p + ON p.pronamespace = n.oid AND p.proowner = r.oid + + UNION ALL + + SELECT r.rolname, n.nspname, t.typname, 'TYPE' + FROM accessible_roles r + INNER JOIN approved_schemas n ON true + INNER JOIN pg_catalog.pg_type t + ON t.typnamespace = n.oid AND t.typowner = r.oid +), stored_expression_roots AS ( + SELECT 'pg_catalog.pg_trigger'::regclass::oid AS root_class, + trigger.oid AS root_id, + namespace.nspname, + class.relname || ':' || trigger.tgname AS object_name + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_class class ON class.relnamespace = namespace.oid + INNER JOIN pg_catalog.pg_trigger trigger ON trigger.tgrelid = class.oid + WHERE NOT trigger.tgisinternal + + UNION ALL + + SELECT 'pg_catalog.pg_attrdef'::regclass::oid, + attribute_default.oid, + namespace.nspname, + class.relname || '.' || attribute.attname + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_class class ON class.relnamespace = namespace.oid + INNER JOIN pg_catalog.pg_attrdef attribute_default ON attribute_default.adrelid = class.oid + INNER JOIN pg_catalog.pg_attribute attribute + ON attribute.attrelid = class.oid AND attribute.attnum = attribute_default.adnum + + UNION ALL + + SELECT 'pg_catalog.pg_policy'::regclass::oid, + policy.oid, + namespace.nspname, + class.relname || ':' || policy.polname + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_class class ON class.relnamespace = namespace.oid + INNER JOIN pg_catalog.pg_policy policy ON policy.polrelid = class.oid + + UNION ALL + + SELECT 'pg_catalog.pg_rewrite'::regclass::oid, + rewrite.oid, + namespace.nspname, + class.relname || ':' || rewrite.rulename + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_class class ON class.relnamespace = namespace.oid + INNER JOIN pg_catalog.pg_rewrite rewrite ON rewrite.ev_class = class.oid + + UNION ALL + + SELECT 'pg_catalog.pg_constraint'::regclass::oid, + constraint_record.oid, + namespace.nspname, + COALESCE(class.relname || ':', '') || constraint_record.conname + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_constraint constraint_record + ON constraint_record.connamespace = namespace.oid + LEFT JOIN pg_catalog.pg_class class ON class.oid = constraint_record.conrelid + + UNION ALL + + SELECT 'pg_catalog.pg_class'::regclass::oid, + index_class.oid, + namespace.nspname, + index_class.relname + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_class index_class + ON index_class.relnamespace = namespace.oid + AND index_class.relkind IN ('i', 'I') + + UNION ALL + + SELECT 'pg_catalog.pg_proc'::regclass::oid, + procedure.oid, + namespace.nspname, + procedure.proname + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_proc procedure ON procedure.pronamespace = namespace.oid +), stored_dependency_closure( + root_class, + root_id, + nspname, + object_name, + dependency_class, + dependency_id +) AS ( + SELECT root.root_class, + root.root_id, + root.nspname, + root.object_name, + dependency.refclassid, + dependency.refobjid + FROM stored_expression_roots root + INNER JOIN pg_catalog.pg_depend dependency + ON dependency.classid = root.root_class + AND dependency.objid = root.root_id + WHERE dependency.refobjid <> 0 + + UNION + + SELECT closure.root_class, + closure.root_id, + closure.nspname, + closure.object_name, + dependency.refclassid, + dependency.refobjid + FROM stored_dependency_closure closure + INNER JOIN pg_catalog.pg_depend dependency + ON dependency.classid = closure.dependency_class + AND dependency.objid = closure.dependency_id + WHERE dependency.refobjid <> 0 +), stored_dependency_violations AS ( + SELECT DISTINCT root.nspname, root.object_name, + CASE + WHEN dependency_proc.prosecdef + THEN 'STORED EXPRESSION CALLS SECURITY DEFINER' + ELSE 'STORED EXPRESSION CROSSES SCHEMA' + END AS reason, + dependency_namespace.nspname || '.' || dependency_proc.proname AS dependency + FROM stored_dependency_closure root + INNER JOIN pg_catalog.pg_proc dependency_proc + ON root.dependency_class = 'pg_catalog.pg_proc'::regclass + AND dependency_proc.oid = root.dependency_id + INNER JOIN pg_catalog.pg_namespace dependency_namespace + ON dependency_namespace.oid = dependency_proc.pronamespace + WHERE dependency_proc.prosecdef + OR ( + dependency_namespace.nspname <> 'pg_catalog' + AND NOT EXISTS ( + SELECT 1 FROM approved_schemas approved + WHERE approved.oid = dependency_namespace.oid + ) + ) + + UNION + + SELECT DISTINCT root.nspname, root.object_name, + 'STORED EXPRESSION CROSSES SCHEMA', + dependency_namespace.nspname || '.' || dependency_class.relname + FROM stored_dependency_closure root + INNER JOIN pg_catalog.pg_class dependency_class + ON root.dependency_class = 'pg_catalog.pg_class'::regclass + AND dependency_class.oid = root.dependency_id + INNER JOIN pg_catalog.pg_namespace dependency_namespace + ON dependency_namespace.oid = dependency_class.relnamespace + WHERE dependency_namespace.nspname <> 'pg_catalog' + AND NOT EXISTS ( + SELECT 1 FROM approved_schemas approved + WHERE approved.oid = dependency_namespace.oid + ) +), privileged_object_violations AS ( + SELECT n.nspname, p.proname AS object_name, 'SECURITY DEFINER FUNCTION' AS reason + FROM approved_schemas n + INNER JOIN pg_catalog.pg_proc p ON p.pronamespace = n.oid + WHERE p.prosecdef + + UNION ALL + + SELECT n.nspname, c.relname, 'OWNER-RIGHTS VIEW' + FROM approved_schemas n + INNER JOIN pg_catalog.pg_class c ON c.relnamespace = n.oid + WHERE c.relkind = 'v' + AND NOT COALESCE(c.reloptions @> ARRAY['security_invoker=true'], false) + + UNION ALL + + SELECT n.nspname, c.relname, 'FOREIGN TABLE' + FROM approved_schemas n + INNER JOIN pg_catalog.pg_class c ON c.relnamespace = n.oid + WHERE c.relkind = 'f' + + UNION ALL + + SELECT n.nspname, c.relname, 'MATERIALIZED VIEW' + FROM approved_schemas n + INNER JOIN pg_catalog.pg_class c ON c.relnamespace = n.oid + WHERE c.relkind = 'm' +) +SELECT current_user AS login_role, + COALESCE((SELECT json_agg(login_role_violations) FROM login_role_violations), '[]'::json) AS login_role_violations, + COALESCE((SELECT json_agg(inherited_role_violations) FROM inherited_role_violations), '[]'::json) AS inherited_role_violations, + COALESCE((SELECT json_agg(unexpected_set_role_violations) FROM unexpected_set_role_violations), '[]'::json) AS unexpected_set_role_violations, + COALESCE((SELECT json_agg(request_role_reachability_violations) FROM request_role_reachability_violations), '[]'::json) AS request_role_reachability_violations, + COALESCE((SELECT json_agg(role_violations) FROM role_violations), '[]'::json) AS role_violations, + COALESCE((SELECT json_agg(database_violations) FROM database_violations), '[]'::json) AS database_violations, + COALESCE((SELECT json_agg(cross_database_violations) FROM cross_database_violations), '[]'::json) AS cross_database_violations, + COALESCE((SELECT json_agg(schema_violations) FROM schema_violations), '[]'::json) AS schema_violations, + COALESCE((SELECT json_agg(cross_schema_violations) FROM cross_schema_violations), '[]'::json) AS cross_schema_violations, + COALESCE((SELECT json_agg(object_owner_violations) FROM object_owner_violations), '[]'::json) AS object_owner_violations, + COALESCE((SELECT json_agg(privileged_object_violations) FROM privileged_object_violations), '[]'::json) AS privileged_object_violations, + COALESCE((SELECT json_agg(stored_dependency_violations) FROM stored_dependency_violations), '[]'::json) AS stored_dependency_violations, + ARRAY( + SELECT requested + FROM unnest($1::text[]) requested + WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.rolname = requested) + ) AS missing_roles, + ARRAY( + SELECT r.rolname::text + FROM pg_catalog.pg_roles r + WHERE r.rolname = ANY($1::text[]) + AND r.rolname <> current_user + AND NOT pg_catalog.pg_has_role(current_user, r.oid, 'SET') + ) AS inaccessible_roles, + ARRAY( + SELECT requested + FROM unnest($2::text[] || $3::text[]) requested + WHERE NOT EXISTS (SELECT 1 FROM approved_schemas n WHERE n.nspname = requested) + ) AS missing_schemas +`; + +interface RuntimeRoleSafetyRow { + login_role: string; + login_role_violations: Array<{ capabilities: string[] }> | string; + inherited_role_violations: Array<{ rolname: string }> | string; + unexpected_set_role_violations: Array<{ rolname: string }> | string; + request_role_reachability_violations: Array<{ + request_role: string; + reachable_role: string; + via_usage: boolean; + via_set: boolean; + }> | string; + role_violations: Array<{ rolname: string; capabilities: string[] }> | string; + database_violations: Array<{ + rolname: string; + datname: string; + capability: string; + }> | string; + cross_database_violations: Array<{ + rolname: string; + datname: string; + }> | string; + schema_violations: Array<{ rolname: string; nspname: string; capability: string }> | string; + cross_schema_violations: Array<{ + rolname: string; + nspname: string; + capabilities: string[]; + }> | string; + object_owner_violations: Array<{ + rolname: string; + nspname: string; + object_name: string; + object_kind: string; + }> | string; + privileged_object_violations: Array<{ + nspname: string; + object_name: string; + reason: string; + }> | string; + stored_dependency_violations: Array<{ + nspname: string; + object_name: string; + reason: string; + dependency: string; + }> | string; + missing_roles: string[]; + inaccessible_roles: string[]; + missing_schemas: string[]; +} + +const parseRequiredJsonColumn = ( + value: T[] | string | null | undefined, + column: string +): T[] => { + let parsed: unknown = value; + if (typeof value === 'string') { + try { + parsed = JSON.parse(value); + } catch { + throw new UnsafeRuntimeRoleError([ + `safety query returned invalid JSON for ${column}` + ]); + } + } + if (!Array.isArray(parsed)) { + throw new UnsafeRuntimeRoleError([ + `safety query did not return ${column} as a JSON array` + ]); + } + return parsed as T[]; +}; + +const parseRequiredTextArrayColumn = ( + value: string[] | null | undefined, + column: string +): string[] => { + if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) { + throw new UnsafeRuntimeRoleError([ + `safety query did not return ${column} as a text array` + ]); + } + return value; +}; + +export class UnsafeRuntimeRoleError extends Error { + readonly code = 'GRAPHILE_UNSAFE_RUNTIME_ROLE'; + + constructor(readonly violations: string[]) { + super(`GraphQL runtime role safety check failed: ${violations.join('; ')}`); + this.name = 'UnsafeRuntimeRoleError'; + } +} + +export const assertRuntimeRoleSafety = async ( + pool: Pool, + requestRoles: string[], + exposedSchemas: string[], + dependencySchemas: string[] = [] +): Promise => { + const uniqueRoles = [...new Set(requestRoles.filter(Boolean))]; + const uniqueExposedSchemas = [...new Set(exposedSchemas.filter(Boolean))]; + const uniqueDependencySchemas = [...new Set( + dependencySchemas.filter((schema) => schema && !uniqueExposedSchemas.includes(schema)) + )]; + const client: PoolClient = await pool.connect(); + let inTransaction = false; + let destroyClient = false; + let result: QueryResult; + try { + // Both commands use the simple protocol, so send them together and avoid a + // second network round trip without changing the read-only transaction or + // the per-audit JIT policy. + await client.query('BEGIN READ ONLY; SET LOCAL jit TO off'); + inTransaction = true; + // The catalog ACL audit has a deliberately broad static plan. On large + // catalogs PostgreSQL can spend over a second compiling hundreds of JIT + // functions for a query that executes in milliseconds once compiled. + result = await client.query(RUNTIME_ROLE_SAFETY_SQL, [ + uniqueRoles, + uniqueExposedSchemas, + uniqueDependencySchemas + ]); + await client.query('COMMIT'); + inTransaction = false; + } catch (error) { + destroyClient = true; + if (inTransaction) { + try { + await client.query('ROLLBACK'); + } catch { + // Preserve the safety-check failure; pg-pool will discard a broken + // connection through its normal error path. + } + } + throw error; + } finally { + client.release(destroyClient); + } + const row = result.rows[0]; + if (!row) { + throw new UnsafeRuntimeRoleError(['safety query returned no result']); + } + + if (typeof row.login_role !== 'string' || row.login_role.length === 0) { + throw new UnsafeRuntimeRoleError([ + 'safety query did not return a non-empty login_role' + ]); + } + + // Every result column participates in the tenant boundary. Treat query/result + // drift as an unsafe audit instead of interpreting an absent check as an + // empty violation set. + const loginRoleViolations = parseRequiredJsonColumn<{ + capabilities: string[]; + }>(row.login_role_violations, 'login_role_violations'); + const inheritedRoleViolations = parseRequiredJsonColumn<{ + rolname: string; + }>(row.inherited_role_violations, 'inherited_role_violations'); + const databaseViolations = parseRequiredJsonColumn<{ + rolname: string; + datname: string; + capability: string; + }>(row.database_violations, 'database_violations'); + const crossDatabaseViolations = parseRequiredJsonColumn<{ + rolname: string; + datname: string; + }>(row.cross_database_violations, 'cross_database_violations'); + const unexpectedSetRoleViolations = parseRequiredJsonColumn<{ + rolname: string; + }>(row.unexpected_set_role_violations, 'unexpected_set_role_violations'); + const requestRoleReachabilityViolations = parseRequiredJsonColumn<{ + request_role: string; + reachable_role: string; + via_usage: boolean; + via_set: boolean; + }>( + row.request_role_reachability_violations, + 'request_role_reachability_violations' + ); + const roleViolations = parseRequiredJsonColumn<{ + rolname: string; + capabilities: string[]; + }>(row.role_violations, 'role_violations'); + const schemaViolations = parseRequiredJsonColumn<{ + rolname: string; + nspname: string; + capability: string; + }>(row.schema_violations, 'schema_violations'); + const crossSchemaViolations = parseRequiredJsonColumn<{ + rolname: string; + nspname: string; + capabilities: string[]; + }>(row.cross_schema_violations, 'cross_schema_violations'); + const objectOwnerViolations = parseRequiredJsonColumn<{ + rolname: string; + nspname: string; + object_name: string; + object_kind: string; + }>(row.object_owner_violations, 'object_owner_violations'); + const privilegedObjectViolations = parseRequiredJsonColumn<{ + nspname: string; + object_name: string; + reason: string; + }>(row.privileged_object_violations, 'privileged_object_violations'); + const storedDependencyViolations = parseRequiredJsonColumn<{ + nspname: string; + object_name: string; + reason: string; + dependency: string; + }>(row.stored_dependency_violations, 'stored_dependency_violations'); + const missingRoles = parseRequiredTextArrayColumn( + row.missing_roles, + 'missing_roles' + ); + const inaccessibleRoles = parseRequiredTextArrayColumn( + row.inaccessible_roles, + 'inaccessible_roles' + ); + const missingSchemas = parseRequiredTextArrayColumn( + row.missing_schemas, + 'missing_schemas' + ); + + const violations = [ + ...loginRoleViolations.map( + (role) => `${row.login_role} has ${role.capabilities.join(',')}` + ), + ...inheritedRoleViolations.map( + (role) => `${row.login_role} inherits privileges from role ${role.rolname}` + ), + ...unexpectedSetRoleViolations.map( + (role) => `${row.login_role} can SET ROLE to unconfigured role ${role.rolname}` + ), + ...requestRoleReachabilityViolations.map( + (role) => `${role.request_role} can reach role ${role.reachable_role}` + + ` after SET ROLE (USAGE=${role.via_usage},SET=${role.via_set})` + ), + ...roleViolations.map( + (role) => `${role.rolname} has ${role.capabilities.join(',')}` + ), + ...databaseViolations.map( + (database) => `${database.rolname} has ${database.capability} on database ${database.datname}` + ), + ...crossDatabaseViolations.map( + (database) => `${database.rolname} has CONNECT on non-target database ${database.datname}` + ), + ...schemaViolations.map( + (schema) => `${schema.rolname} has ${schema.capability} on schema ${schema.nspname}` + ), + ...crossSchemaViolations.map( + (schema) => `${schema.rolname} has ${schema.capabilities.join(',')} on unapproved schema ${schema.nspname}` + ), + ...objectOwnerViolations.map( + (object) => `${object.rolname} owns ${object.object_kind} ${object.nspname}.${object.object_name}` + ), + ...privilegedObjectViolations.map( + (object) => `${object.reason} ${object.nspname}.${object.object_name} is not allowed in the approved GraphQL schema scope` + ), + ...storedDependencyViolations.map( + (object) => `${object.reason} from ${object.nspname}.${object.object_name} to ${object.dependency}` + ), + ...missingRoles.map((role) => `request role ${role} does not exist`), + ...inaccessibleRoles.map( + (role) => `runtime login ${row.login_role} cannot SET ROLE ${role}` + ), + ...missingSchemas.map((schema) => `exposed schema ${schema} does not exist`) + ]; + + if (violations.length > 0) throw new UnsafeRuntimeRoleError(violations); +}; + +interface CachedSafetyCheck { + promise: Promise; + /** Wall-clock time when the successful catalog audit completed. */ + validatedAt: number | null; +} + +export interface RuntimeRoleSafetyStats { + checksStarted: number; + checksSucceeded: number; + checksFailed: number; + inFlightCoalesces: number; + successfulResultReuses: number; + durationMsTotal: number; + durationMsMax: number; +} + +const runtimeRoleSafetyStats: RuntimeRoleSafetyStats = { + checksStarted: 0, + checksSucceeded: 0, + checksFailed: 0, + inFlightCoalesces: 0, + successfulResultReuses: 0, + durationMsTotal: 0, + durationMsMax: 0 +}; + +/** Process-level audit timing and coalescing telemetry for local diagnostics. */ +export const getRuntimeRoleSafetyStats = (): Readonly => ({ + ...runtimeRoleSafetyStats +}); + +const recordRuntimeRoleSafetyDuration = (startedAt: number): void => { + const durationMs = performance.now() - startedAt; + runtimeRoleSafetyStats.durationMsTotal += durationMs; + runtimeRoleSafetyStats.durationMsMax = Math.max( + runtimeRoleSafetyStats.durationMsMax, + durationMs + ); +}; + +/** + * Successful catalog audits may only be reused for a narrowly bounded window. + * Callers may choose a fresher policy, including zero for in-flight coalescing + * without any completed-result reuse, but may not extend this safety bound. + */ +// Without an authoritative control-plane epoch, a completed catalog result is +// stale immediately. Keep the opt-in cap small for deployments that wire the +// invalidation seam to every DDL/ACL commit, but make fresh checks the default. +export const DEFAULT_RUNTIME_ROLE_SAFETY_MAX_AGE_MS = 0; +export const MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS = 1_000; + +export interface RuntimeRoleSafetyCacheOptions { + maxSuccessAgeMs?: number; +} + +const safetyChecks = new WeakMap>(); + +const normalizeMaxSuccessAgeMs = (value: number | undefined): number => { + const maxSuccessAgeMs = value ?? DEFAULT_RUNTIME_ROLE_SAFETY_MAX_AGE_MS; + if ( + !Number.isSafeInteger(maxSuccessAgeMs) + || maxSuccessAgeMs < 0 + || maxSuccessAgeMs > MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS + ) { + throw new RangeError( + `runtime role safety maxSuccessAgeMs must be an integer between 0 and ${MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS}` + ); + } + return maxSuccessAgeMs; +}; + +/** Coalesce identical safety checks for every consumer of a runtime pool. */ +const safetyCheckKey = ( + requestRoles: string[], + exposedSchemas: string[], + dependencySchemas: string[] +): string => JSON.stringify([ + [...new Set(requestRoles.filter(Boolean))].sort(), + [...new Set(exposedSchemas.filter(Boolean))].sort(), + [...new Set(dependencySchemas.filter(Boolean))].sort() +]); + +/** Reuse a recent successful audit while coalescing concurrent callers. */ +export const ensureRuntimeRoleSafety = ( + pool: Pool, + requestRoles: string[], + exposedSchemas: string[], + dependencySchemas: string[] = [], + options: RuntimeRoleSafetyCacheOptions = {} +): Promise => { + const maxSuccessAgeMs = normalizeMaxSuccessAgeMs(options.maxSuccessAgeMs); + const key = safetyCheckKey(requestRoles, exposedSchemas, dependencySchemas); + let checksForPool = safetyChecks.get(pool); + if (!checksForPool) { + checksForPool = new Map(); + safetyChecks.set(pool, checksForPool); + } + const existing = checksForPool.get(key); + if (existing) { + if (existing.validatedAt == null) { + runtimeRoleSafetyStats.inFlightCoalesces++; + return existing.promise; + } + const now = Date.now(); + const successAgeMs = now - existing.validatedAt; + if ( + maxSuccessAgeMs > 0 + && successAgeMs >= 0 + && successAgeMs < maxSuccessAgeMs + ) { + runtimeRoleSafetyStats.successfulResultReuses++; + return existing.promise; + } + } + + let check!: CachedSafetyCheck; + runtimeRoleSafetyStats.checksStarted++; + const startedAt = performance.now(); + const pending = assertRuntimeRoleSafety( + pool, + requestRoles, + exposedSchemas, + dependencySchemas + ).then(() => { + runtimeRoleSafetyStats.checksSucceeded++; + recordRuntimeRoleSafetyDuration(startedAt); + check.validatedAt = Date.now(); + }).catch((error) => { + runtimeRoleSafetyStats.checksFailed++; + recordRuntimeRoleSafetyDuration(startedAt); + if (checksForPool?.get(key) === check) checksForPool.delete(key); + throw error; + }); + check = { promise: pending, validatedAt: null }; + checksForPool.set(key, check); + return pending; +}; + +/** + * Invalidate one audit contract, or every cached audit for the pool when the + * contract arguments are omitted. Control-plane DDL/GRANT/REVOKE paths should + * call this immediately after committing catalog changes. + */ +export const invalidateRuntimeRoleSafety = ( + pool: Pool, + requestRoles?: string[], + exposedSchemas?: string[], + dependencySchemas: string[] = [] +): void => { + const checksForPool = safetyChecks.get(pool); + if (!checksForPool) return; + if (requestRoles == null || exposedSchemas == null) { + safetyChecks.delete(pool); + return; + } + checksForPool.delete( + safetyCheckKey(requestRoles, exposedSchemas, dependencySchemas) + ); + if (checksForPool.size === 0) safetyChecks.delete(pool); +}; + +/** Force a new audit for schema build admission, even after a cached success. */ +export const refreshRuntimeRoleSafety = ( + pool: Pool, + requestRoles: string[], + exposedSchemas: string[], + dependencySchemas: string[] = [] +): Promise => { + invalidateRuntimeRoleSafety(pool, requestRoles, exposedSchemas, dependencySchemas); + return ensureRuntimeRoleSafety(pool, requestRoles, exposedSchemas, dependencySchemas); +}; diff --git a/graphql/server/src/middleware/types.ts b/graphql/server/src/middleware/types.ts index 5b0868f764..911939baf6 100644 --- a/graphql/server/src/middleware/types.ts +++ b/graphql/server/src/middleware/types.ts @@ -15,6 +15,10 @@ declare global { interface Request { api?: ApiStructure; svc_key?: string; + /** Opaque physical routing-cache identity; never used as a service label. */ + svc_cache_key?: string; + /** True only after constant-time authentication of the internal request token. */ + internalTrusted?: boolean; clientIp?: string; databaseId?: string; requestId?: string; diff --git a/graphql/server/src/plugins/__tests__/websocket-operation-admission-plugin.test.ts b/graphql/server/src/plugins/__tests__/websocket-operation-admission-plugin.test.ts new file mode 100644 index 0000000000..eee6885278 --- /dev/null +++ b/graphql/server/src/plugins/__tests__/websocket-operation-admission-plugin.test.ts @@ -0,0 +1,234 @@ +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; + +import type { Request } from 'express'; +import type { GraphileCacheEntry } from 'graphile-cache'; + +import { + createGraphileWebSocketOperationAdmission, + GRAPHILE_WEBSOCKET_CAPTCHA_REQUIRED_CODE, + GRAPHILE_WEBSOCKET_OPERATION_SAFETY_CODE +} from '../websocket-operation-admission-plugin'; + +const contract = { + cacheKey: 'contract-a', + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['a_public'], + authenticatedRole: 'a_user', + anonymousRole: 'a_anon', + dependencySchemas: ['realtime_a'], + runtimeSafetyRequired: true +} as const; + +const makeFixture = (options: { + enableCaptcha?: boolean; + query?: string; + operationName?: string; +} = {}) => { + const socket = new PassThrough(); + const request = Object.assign(new EventEmitter(), { + aborted: false, + socket, + api: { + databaseId: contract.databaseId, + dbname: contract.databaseName, + apiId: contract.apiId, + schema: [...contract.schemas], + roleName: contract.authenticatedRole, + anonRole: contract.anonymousRole, + authSettings: options.enableCaptcha + ? { enableCaptcha: true } + : undefined + } + }) as unknown as Request; + const entry = { + cacheKey: contract.cacheKey, + websocketSockets: new Set([socket]), + disposing: false + } as unknown as GraphileCacheEntry; + const ensureRuntimeSafety = jest.fn(async (): Promise => undefined); + const revalidateRealtimeRole = jest.fn(async (): Promise => true); + const retire = jest.fn((): boolean => true); + const admission = createGraphileWebSocketOperationAdmission(contract, { + ensureRuntimeSafety, + revalidateRealtimeRole, + retire + }); + admission.bind(entry); + const callback = ( + admission.plugin.grafserv?.middleware?.onSubscribe as { + callback: (next: () => unknown, event: unknown) => Promise; + } + ).callback; + const event = { + ctx: { extra: { request } }, + message: { + payload: { + query: options.query ?? 'subscription Events { events { id } }', + operationName: options.operationName + } + } + }; + return { + admission, + callback, + ensureRuntimeSafety, + entry, + event, + request, + retire, + revalidateRealtimeRole, + socket + }; +}; + +describe('Graphile WebSocket per-operation safety admission', () => { + it('revalidates both exact safety boundaries before every operation', async () => { + const fixture = makeFixture(); + const next = jest.fn(() => ({ accepted: true })); + + await expect(fixture.callback(next, fixture.event)).resolves.toEqual({ + accepted: true + }); + await expect(fixture.callback(next, fixture.event)).resolves.toEqual({ + accepted: true + }); + + expect(fixture.ensureRuntimeSafety).toHaveBeenCalledTimes(2); + expect(fixture.revalidateRealtimeRole).toHaveBeenCalledTimes(2); + expect(next).toHaveBeenCalledTimes(2); + expect(fixture.retire).not.toHaveBeenCalled(); + }); + + it('retires the exact generation when runtime-role safety cannot be proved', async () => { + const fixture = makeFixture(); + const failure = new Error('unsafe runtime role'); + fixture.ensureRuntimeSafety.mockRejectedValueOnce(failure); + const next = jest.fn(); + + const result = await fixture.callback(next, fixture.event) as Array<{ + extensions?: Record; + }>; + + expect(result[0]?.extensions?.code).toBe( + GRAPHILE_WEBSOCKET_OPERATION_SAFETY_CODE + ); + expect(fixture.retire).toHaveBeenCalledWith(fixture.entry, failure); + expect(fixture.revalidateRealtimeRole).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + }); + + it('retires the generation when listener-role refresh fails', async () => { + const fixture = makeFixture(); + fixture.revalidateRealtimeRole.mockResolvedValueOnce(false); + const next = jest.fn(); + + const result = await fixture.callback(next, fixture.event) as unknown[]; + + expect(result).toHaveLength(1); + expect(fixture.retire).toHaveBeenCalledTimes(1); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects a request whose routed tenant differs from the bound generation', async () => { + const fixture = makeFixture(); + fixture.request.api.databaseId = 'database-b'; + const next = jest.fn(); + + const result = await fixture.callback(next, fixture.event) as unknown[]; + + expect(result).toHaveLength(1); + expect(fixture.retire).toHaveBeenCalledTimes(1); + expect(fixture.ensureRuntimeSafety).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + }); + + it('does not enter GraphQL when invalidation wins an asynchronous audit race', async () => { + const fixture = makeFixture(); + fixture.ensureRuntimeSafety.mockImplementationOnce(async () => { + fixture.entry.disposing = true; + }); + const next = jest.fn(); + + const result = await fixture.callback(next, fixture.event) as unknown[]; + + expect(result).toHaveLength(1); + expect(next).not.toHaveBeenCalled(); + }); + + it.each([ + [ + 'an aliased root field', + 'mutation Harmless { allowed: signUp }', + 'Harmless' + ], + [ + 'a fragment root field', + `mutation Harmless { ...Protected } + fragment Protected on Mutation { requestPasswordReset }`, + 'Harmless' + ], + [ + 'the selected operation in a multi-operation document', + 'query Safe { viewer { id } } mutation Protected { resetPassword }', + 'Protected' + ] + ])('rejects CAPTCHA-protected WebSocket mutations through %s', async ( + _label, + query, + operationName + ) => { + const fixture = makeFixture({ enableCaptcha: true, query, operationName }); + const next = jest.fn(); + + const result = await fixture.callback(next, fixture.event) as Array<{ + extensions?: Record; + }>; + + expect(result[0]?.extensions?.code).toBe( + GRAPHILE_WEBSOCKET_CAPTCHA_REQUIRED_CODE + ); + expect(fixture.ensureRuntimeSafety).not.toHaveBeenCalled(); + expect(fixture.revalidateRealtimeRole).not.toHaveBeenCalled(); + expect(fixture.retire).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + }); + + it.each([ + ['an ambiguous document', 'query A { viewer { id } } query B { viewer { id } }'], + ['a malformed document', 'mutation {'] + ])('fails closed when a CAPTCHA-enabled WebSocket sends %s', async ( + _label, + query + ) => { + const fixture = makeFixture({ enableCaptcha: true, query }); + const next = jest.fn(); + + const result = await fixture.callback(next, fixture.event) as Array<{ + extensions?: Record; + }>; + + expect(result[0]?.extensions?.code).toBe( + GRAPHILE_WEBSOCKET_CAPTCHA_REQUIRED_CODE + ); + expect(next).not.toHaveBeenCalled(); + }); + + it('keeps ordinary subscriptions available when CAPTCHA is enabled', async () => { + const fixture = makeFixture({ + enableCaptcha: true, + query: 'subscription Events { events { id } }', + operationName: 'Events' + }); + const next = jest.fn(() => ({ accepted: true })); + + await expect(fixture.callback(next, fixture.event)).resolves.toEqual({ + accepted: true + }); + expect(fixture.ensureRuntimeSafety).toHaveBeenCalledTimes(1); + expect(fixture.revalidateRealtimeRole).toHaveBeenCalledTimes(1); + expect(next).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphql/server/src/plugins/websocket-operation-admission-plugin.ts b/graphql/server/src/plugins/websocket-operation-admission-plugin.ts new file mode 100644 index 0000000000..6241151193 --- /dev/null +++ b/graphql/server/src/plugins/websocket-operation-admission-plugin.ts @@ -0,0 +1,212 @@ +import type { Request } from 'express'; +import { + type GraphileCacheEntry, + isEntryRealtimeUnavailable, + retireGraphileCacheEntry, + revalidateEntryRealtimeRole +} from 'graphile-cache'; +import type { GraphileConfig } from 'graphile-config'; +import { GraphQLError } from 'graphql'; + +import { inspectCaptchaOperation } from '../middleware/captcha'; +import { ensureRuntimeRoleSafety } from '../middleware/runtime-role-safety'; + +export const GRAPHILE_WEBSOCKET_OPERATION_SAFETY_CODE = + 'GRAPHILE_WEBSOCKET_OPERATION_SAFETY_FAILED'; +export const GRAPHILE_WEBSOCKET_CAPTCHA_REQUIRED_CODE = 'CAPTCHA_REQUIRED'; + +export interface GraphileWebSocketOperationContract { + cacheKey: string; + databaseId: string; + databaseName: string; + apiId: string; + schemas: readonly string[]; + authenticatedRole: string; + anonymousRole: string; + dependencySchemas: readonly string[]; + runtimeSafetyRequired: boolean; +} + +interface OperationAdmissionDependencies { + ensureRuntimeSafety(entry: GraphileCacheEntry): Promise; + revalidateRealtimeRole(entry: GraphileCacheEntry): Promise; + retire(entry: GraphileCacheEntry, error: unknown): boolean; +} + +export interface GraphileWebSocketOperationAdmission { + readonly plugin: GraphileConfig.Plugin; + bind(entry: GraphileCacheEntry): void; +} + +const sameStrings = ( + left: readonly string[] | undefined, + right: readonly string[] +): boolean => Boolean( + left + && left.length === right.length + && left.every((value, index) => value === right[index]) +); + +const operationRequest = (event: { + ctx?: { extra?: unknown }; +}): Request | undefined => { + const extra = event.ctx?.extra as { request?: Request } | undefined; + return extra?.request; +}; + +const requestMatchesContract = ( + request: Request, + entry: GraphileCacheEntry, + contract: Readonly +): boolean => { + const api = request.api; + return Boolean( + api + && (api.databaseId ?? '') === contract.databaseId + && api.dbname === contract.databaseName + && (api.apiId ?? '') === contract.apiId + && api.roleName === contract.authenticatedRole + && api.anonRole === contract.anonymousRole + && sameStrings(api.schema, contract.schemas) + && entry.cacheKey === contract.cacheKey + && entry.websocketSockets?.has(request.socket) + ); +}; + +const unavailable = (): readonly GraphQLError[] => [ + new GraphQLError('WebSocket operation safety could not be verified', { + extensions: { code: GRAPHILE_WEBSOCKET_OPERATION_SAFETY_CODE } + }) +]; + +const captchaRequired = (): readonly GraphQLError[] => [ + new GraphQLError('CAPTCHA-protected mutations must use the HTTP endpoint', { + extensions: { code: GRAPHILE_WEBSOCKET_CAPTCHA_REQUIRED_CODE } + }) +]; + +const defaultDependencies = ( + contract: Readonly +): OperationAdmissionDependencies => ({ + ensureRuntimeSafety: async (entry) => { + if (!contract.runtimeSafetyRequired) return; + const pool = entry.poolLease?.pool; + if (!pool) { + throw new Error('Resident Graphile generation has no retained runtime pool'); + } + await ensureRuntimeRoleSafety( + pool, + [contract.anonymousRole, contract.authenticatedRole], + [...contract.schemas], + [...contract.dependencySchemas] + ); + }, + revalidateRealtimeRole: revalidateEntryRealtimeRole, + retire: retireGraphileCacheEntry +}); + +/** + * Bind Grafserv's per-operation WebSocket hook to one exact cache generation. + * The initial HTTP upgrade admission remains authoritative for routing and + * authentication; this hook prevents a long-lived socket from bypassing later + * role or listener-attestation checks when it starts another operation. + */ +export const createGraphileWebSocketOperationAdmission = ( + contract: Readonly, + dependencies: OperationAdmissionDependencies = defaultDependencies(contract) +): GraphileWebSocketOperationAdmission => { + const expected = Object.freeze({ + ...contract, + schemas: Object.freeze([...contract.schemas]), + dependencySchemas: Object.freeze([...contract.dependencySchemas]) + }); + let entry: GraphileCacheEntry | null = null; + + const reject = (error: unknown): readonly GraphQLError[] => { + if (entry) dependencies.retire(entry, error); + return unavailable(); + }; + + const plugin: GraphileConfig.Plugin = { + name: 'ConstructiveWebSocketOperationAdmissionPlugin', + version: '1.0.0', + grafserv: { + middleware: { + onSubscribe: { + callback: async (next, event) => { + const current = entry; + const request = operationRequest(event); + if (!current || !request) { + return reject(new Error('WebSocket operation has no bound generation')); + } + if ( + request.aborted + || request.socket.destroyed + || current.disposing + ) { + return unavailable(); + } + if (!requestMatchesContract(request, current, expected)) { + return reject(new Error( + 'WebSocket operation request does not match its bound generation' + )); + } + + if (request.api?.authSettings?.enableCaptcha) { + const message = (event as { + message?: { + payload?: { query?: unknown; operationName?: unknown }; + }; + }).message; + const inspection = inspectCaptchaOperation( + message?.payload?.query, + message?.payload?.operationName + ); + // CAPTCHA tokens are verified by the HTTP middleware. Protected + // mutations and documents we cannot classify never reach GraphQL + // over WebSocket, so a transport switch cannot bypass the gate. + if (inspection.kind !== 'not-protected') return captchaRequired(); + } + + try { + await dependencies.ensureRuntimeSafety(current); + const attested = await dependencies.revalidateRealtimeRole(current); + if (!attested || isEntryRealtimeUnavailable(current)) { + throw new Error( + 'WebSocket operation listener-role attestation is unavailable' + ); + } + } catch (error) { + return reject(error); + } + + // Schema invalidation or broker failure may retire the generation + // while either asynchronous audit is running. + if ( + request.aborted + || request.socket.destroyed + || current.disposing + || isEntryRealtimeUnavailable(current) + ) { + return unavailable(); + } + return next(); + } + } + } + } + }; + + return Object.freeze({ + plugin, + bind(candidate: GraphileCacheEntry): void { + if (candidate.cacheKey !== expected.cacheKey) { + throw new Error('WebSocket operation admission cache key mismatch'); + } + if (entry && entry !== candidate) { + throw new Error('WebSocket operation admission is already bound'); + } + entry = candidate; + } + }); +}; diff --git a/graphql/server/src/server.ts b/graphql/server/src/server.ts index 8ddd11c483..4e3b3fcfc2 100644 --- a/graphql/server/src/server.ts +++ b/graphql/server/src/server.ts @@ -1,18 +1,39 @@ import { createCsrfMiddleware } from '@constructive-io/csrf'; -import { createContextMiddleware, createDefaultRegistry, requestIdMiddleware } from '@constructive-io/express-context'; +import { + createContextMiddleware, + createDefaultRegistry, + type LoaderRegistry, + requestIdMiddleware +} from '@constructive-io/express-context'; import { getEnvOptions } from '@constructive-io/graphql-env'; import type { ConstructiveOptions } from '@constructive-io/graphql-types'; import { middleware as parseDomains } from '@constructive-io/url-domains'; +import { getNodeEnv } from '@pgpmjs/env'; import { Logger } from '@pgpmjs/logger'; -import { healthz, poweredBy, svcCache, trustProxy } from '@pgpmjs/server-utils'; +import { + configureSvcCache, + healthz, + poweredBy, + trustProxy +} from '@pgpmjs/server-utils'; import { PgpmOptions } from '@pgpmjs/types'; import cookieParser from 'cookie-parser'; import express, { Express, NextFunction, Request, RequestHandler, Response } from 'express'; -import { closeAllCaches,graphileCache } from 'graphile-cache'; +import { + clearGraphileCache, + closeAllCaches, + getCacheConfig, + startMemoryGovernor +} from 'graphile-cache'; import graphqlUpload from 'graphql-upload'; import type { Server as HttpServer } from 'http'; -import { Pool, PoolClient } from 'pg'; -import { getPgPool } from 'pg-cache'; +import { type Notification, Pool, type PoolClient } from 'pg'; +import { + acquirePgPool, + getPgPool, + PgPoolCapacityError, + type PgPoolLease +} from 'pg-cache'; import requestIp from 'request-ip'; import { createAgenticRouter } from './agentic'; @@ -20,31 +41,184 @@ import { closeDebugDatabasePools } from './diagnostics/debug-db-snapshot'; import type { DebugSamplerHandle } from './diagnostics/debug-sampler'; import { startDebugSampler } from './diagnostics/debug-sampler'; import { + getGraphqlObservabilityToken, isDevelopmentObservabilityMode, isGraphqlObservabilityEnabled, isGraphqlObservabilityRequested, isLoopbackHost } from './diagnostics/observability'; -import { createApiMiddleware } from './middleware/api'; +import { clearSvcCache, createApiMiddleware } from './middleware/api'; import { createAuthenticateMiddleware } from './middleware/auth'; // Auth cookie handling is done via AuthCookiePlugin in grafserv -import { createCaptchaMiddleware } from './middleware/captcha'; +import { + createCaptchaGraphqlBodyParsers, + createCaptchaMiddleware +} from './middleware/captcha'; import { parseCookieValue, SESSION_COOKIE_NAME } from './middleware/cookie'; import { cors } from './middleware/cors'; import { errorHandler, notFoundHandler } from './middleware/error-handler'; import { favicon } from './middleware/favicon'; -import { flush, flushService } from './middleware/flush'; +import { createFlushMiddleware, flushService } from './middleware/flush'; import { createFnRouter } from './middleware/fn'; import { graphile } from './middleware/graphile'; +import { + closeGraphileBuildCoordinator, + getGraphileGovernorCounters, + GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE, + reopenGraphileBuildCoordinator +} from './middleware/graphile-build-governor'; +import { assertInternalRequestSecret } from './middleware/internal-request'; import { multipartBridge } from './middleware/multipart-bridge'; import { createDebugDatabaseMiddleware } from './middleware/observability/debug-db'; import { debugMemory } from './middleware/observability/debug-memory'; import { localObservabilityOnly } from './middleware/observability/guard'; import { createRequestLogger } from './middleware/observability/request-logger'; +import { + addRealtimeRuntimeDependencySchema, + resolveGraphileRealtimeSchema +} from './middleware/realtime-config'; import { getRoutingSchema } from './middleware/routing'; +import { createRuntimePgResolutionStore } from './middleware/runtime-pg-config'; +import { + assertRuntimePgCredentials, + shouldValidateRuntimeRoleSafety +} from './middleware/runtime-pg-requirements'; +import { ensureRuntimeRoleSafety } from './middleware/runtime-role-safety'; +import { + createGraphileWebSocketOriginGuard, + createGraphileWebSocketUpgradeGateway, + type GraphileWebSocketUpgradeGateway +} from './websocket-upgrade'; const log = new Logger('server'); +export const GRAPHILE_CACHE_SHUTDOWN_DRAIN_TIMEOUT_CODE = + 'GRAPHILE_CACHE_SHUTDOWN_DRAIN_TIMEOUT'; +export const GRAPHILE_CACHE_SHUTDOWN_RESTART_REQUIRED_CODE = + 'GRAPHILE_CACHE_SHUTDOWN_RESTART_REQUIRED'; + +export class GraphileCacheShutdownError extends Error { + constructor( + readonly code: + | typeof GRAPHILE_CACHE_SHUTDOWN_DRAIN_TIMEOUT_CODE + | typeof GRAPHILE_CACHE_SHUTDOWN_RESTART_REQUIRED_CODE, + message: string + ) { + super(message); + this.name = 'GraphileCacheShutdownError'; + } +} + +// A process-wide cache clear owns the process-wide build coordinator. Coalesce +// concurrent callers so no invocation can reopen admission while another is +// still disposing residents or closing their pools. +let processCacheClose: Promise | null = null; +let processCacheClosePoolsRequested = false; + +const once = ( + callback: (...args: Args) => void +): ((...args: Args) => void) => { + let called = false; + return (...args: Args) => { + if (called) return; + called = true; + callback(...args); + }; +}; + +interface ListenAttempt { + releasePoolLease: () => void; + client: PoolClient | null; + releaseClient: ((error?: Error | boolean) => void) | null; + notificationHandler: ((message: Notification) => void) | null; + errorHandler: ((error: Error) => void) | null; + closed: boolean; + cleanupPromise: Promise | null; +} + +const PROCESS_SHUTDOWN_SIGNALS = ['SIGINT', 'SIGTERM'] as const; + +/** @internal Process seam used by the executable shutdown boundary and its tests. */ +export interface ProcessShutdownTarget { + on(signal: NodeJS.Signals, listener: () => void): unknown; + removeListener(signal: NodeJS.Signals, listener: () => void): unknown; + exit(code?: number): void; +} + +export interface ProcessShutdownOptions { + timeoutMs?: number; + processTarget?: ProcessShutdownTarget; +} + +/** + * Install process-level shutdown ownership at the executable boundary. + * A second signal forces exit, while the first gets a bounded graceful drain. + */ +export const installProcessShutdownHandlers = ( + shutdown: () => Promise, + options: ProcessShutdownOptions = {} +): (() => void) => { + const { timeoutMs = 30_000, processTarget = process } = options; + let started = false; + let finished = false; + let timeout: ReturnType | null = null; + const listeners = new Map void>(); + + const uninstall = (): void => { + for (const [signal, listener] of listeners) { + processTarget.removeListener(signal, listener); + } + listeners.clear(); + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + }; + + const finish = (exitCode: number): void => { + if (finished) return; + finished = true; + uninstall(); + processTarget.exit(exitCode); + }; + + const beginShutdown = (signal: NodeJS.Signals): void => { + if (started) { + log.warn(`Received ${signal} while shutdown is in progress; forcing exit`); + finish(1); + return; + } + started = true; + log.info(`Received ${signal}; draining GraphQL server resources`); + timeout = setTimeout(() => { + log.error(`GraphQL server shutdown exceeded ${timeoutMs}ms; forcing exit`); + finish(1); + }, Math.max(1, timeoutMs)); + timeout.unref?.(); + + let shutdownPromise: Promise; + try { + shutdownPromise = shutdown(); + } catch (error) { + shutdownPromise = Promise.reject(error); + } + void shutdownPromise.then( + () => finish(0), + (error) => { + log.error('GraphQL server shutdown failed', error); + finish(1); + } + ); + }; + + for (const signal of PROCESS_SHUTDOWN_SIGNALS) { + const listener = (): void => beginShutdown(signal); + listeners.set(signal, listener); + processTarget.on(signal, listener); + } + return uninstall; +}; + /** * Creates and starts a GraphQL server instance * @@ -71,26 +245,50 @@ export const GraphQLServer = (rawOpts: ConstructiveOptions | PgpmOptions = {}) = const app = new Server(opts); app.addEventListener(); app.listen(); + installProcessShutdownHandlers(() => app.close({ closeCaches: true })); }; class Server { private app: Express; private opts: ConstructiveOptions; - private listenClient: PoolClient | null = null; - private listenRelease: (() => void) | null = null; + private listenAttempt: ListenAttempt | null = null; + private listenRetryTimer: ReturnType | null = null; + private readonly listenCleanupTasks = new Set>(); private shuttingDown = false; private closed = false; private httpServer: HttpServer | null = null; private debugSampler: DebugSamplerHandle | null = null; + private stopMemoryGovernor: (() => void) | null = null; + private websocketUpgradeGateway: GraphileWebSocketUpgradeGateway | null = null; + private readonly moduleRegistry: LoaderRegistry; constructor(opts: ConstructiveOptions) { + if (!reopenGraphileBuildCoordinator()) { + log.warn( + 'GraphQL schema build admission remains closed because a previous generation is still draining' + ); + } this.opts = getEnvOptions(opts); + this.moduleRegistry = createDefaultRegistry(); const effectiveOpts = this.opts; + assertInternalRequestSecret(effectiveOpts); + const residentGraphileCapacity = getCacheConfig().max; + const routingCache = configureSvcCache({ + maxEntries: effectiveOpts.routingCache?.maxEntries, + minimumEntries: residentGraphileCapacity + }); + assertRuntimePgCredentials(effectiveOpts, getNodeEnv()); + const validateRuntimeRole = shouldValidateRuntimeRoleSafety( + effectiveOpts, + getNodeEnv() + ); const observabilityRequested = isGraphqlObservabilityRequested(); const observabilityEnabled = isGraphqlObservabilityEnabled(effectiveOpts.server?.host); + const runtimePgResolutions = createRuntimePgResolutionStore(effectiveOpts); const app = express(); - const api = createApiMiddleware(effectiveOpts); + this.stopMemoryGovernor = startMemoryGovernor(); + const api = createApiMiddleware(effectiveOpts, this.moduleRegistry); const authenticate = createAuthenticateMiddleware(effectiveOpts); const requestLogger = createRequestLogger({ observabilityEnabled }); @@ -105,13 +303,20 @@ class Server { apiIsPublic: apiOpts.isPublic, routingSchema: apiOpts.routingSchema, metaSchemas: apiOpts.metaSchemas?.join(',') || 'default', + routingCacheMaxEntries: routingCache.max, + residentGraphileCapacity, observabilityEnabled }); if (observabilityRequested && !observabilityEnabled) { const reasons = []; - if (!isDevelopmentObservabilityMode()) { - reasons.push('NODE_ENV must be development'); + if ( + !isDevelopmentObservabilityMode() + && !getGraphqlObservabilityToken() + ) { + reasons.push( + 'NODE_ENV must be development or GRAPHQL_OBSERVABILITY_TOKEN must contain at least 32 bytes' + ); } if (!isLoopbackHost(effectiveOpts.server?.host)) { reasons.push('server host must be localhost, 127.0.0.1, or ::1'); @@ -124,6 +329,21 @@ class Server { ); } + // Keep the generic health endpoint reusable, but fail this server's probe + // once the build watchdog has latched. Orchestrators can then replace the + // process; admitting a second build in-process would overlap an unknown + // amount of retained work from the stuck generation. + app.get('/healthz', (_req, res, next) => { + const governor = getGraphileGovernorCounters(); + if (!governor.restartRequired) { + next(); + return; + } + res.status(503).json({ + status: 'unhealthy', + code: GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE + }); + }); healthz(app); if (observabilityEnabled) { app.get('/debug/memory', localObservabilityOnly, debugMemory); @@ -150,6 +370,7 @@ class Server { app.use(poweredBy('constructive')); app.use(cookieParser()); app.use(cors(fallbackOrigin)); + app.use('/graphql', ...createCaptchaGraphqlBodyParsers()); app.use('/graphql', graphqlUpload.graphqlUploadExpress({ maxFileSize: 10 * 1024 * 1024, // 10 MB maxFiles: 10 @@ -162,13 +383,38 @@ class Server { app.use(requestIdMiddleware()); app.use(requestLogger); app.use(api); + // Browser WebSockets do not enforce CORS. Reject an untrusted Origin after + // exact tenant routing but before auth or any tenant-specific module I/O. + app.use(createGraphileWebSocketOriginGuard(fallbackOrigin)); app.use(authenticate); + app.use(runtimePgResolutions.middleware); app.use(createContextMiddleware({ pg: effectiveOpts.pg, - loaders: createDefaultRegistry(), + getRuntimePgResolution: runtimePgResolutions.getRuntimePgResolution, + dependencySchemas: effectiveOpts.graphile?.introspectionDependencySchemas, + validateRuntimePool: validateRuntimeRole + ? (pool, resolvedApi) => { + const realtimeSchema = resolveGraphileRealtimeSchema( + effectiveOpts, + resolvedApi.databaseSettings?.enableRealtime ?? false + ); + return ensureRuntimeRoleSafety( + pool, + [resolvedApi.anonRole, resolvedApi.roleName], + resolvedApi.schema, + addRealtimeRuntimeDependencySchema( + effectiveOpts.graphile?.introspectionDependencySchemas ?? [], + realtimeSchema + ) + ); + } + : undefined, + loaders: this.moduleRegistry, routingSchema: getRoutingSchema(effectiveOpts) })); - app.use(createCaptchaMiddleware()); + app.use(createCaptchaMiddleware({ + strictAuth: effectiveOpts.server?.strictAuth + })); // CSRF protection for cookie-authenticated requests // Skip CSRF for Bearer token auth (not vulnerable to CSRF) and anonymous requests @@ -206,14 +452,18 @@ class Server { // REST function invocation routes (POST /fn/:alias, GET /fn/invocations/:id) app.use(createFnRouter()); - app.use(graphile(effectiveOpts)); - app.use(flush); + app.use(graphile( + effectiveOpts, + runtimePgResolutions.getRuntimePgResolution + )); + app.use(createFlushMiddleware(this.moduleRegistry)); // Error handling - MUST be LAST app.use(notFoundHandler); // Catches unmatched routes (404) app.use(errorHandler); // Catches all thrown errors this.app = app; + this.websocketUpgradeGateway = createGraphileWebSocketUpgradeGateway(app); this.debugSampler = observabilityEnabled ? startDebugSampler(effectiveOpts) : null; } @@ -231,84 +481,227 @@ class Server { } throw err; }); + if (!this.websocketUpgradeGateway) { + throw new Error('Graphile WebSocket upgrade gateway is unavailable'); + } + httpServer.on('upgrade', this.websocketUpgradeGateway.handle); this.httpServer = httpServer; return httpServer; } async flush(databaseId: string): Promise { - await flushService(this.opts, databaseId); + await flushService(this.opts, databaseId, this.moduleRegistry); + } + + /** + * LISTEN delivery has no replay. Clear every local metadata publication when + * the listener is lost and again after LISTEN succeeds, so a missed change + * cannot extend a cached module value past reconnection. Security-sensitive + * auth/RLS loaders are additionally uncached and do not depend on this path. + */ + private invalidateConfigurationCaches(reason: string): void { + clearSvcCache(); + this.moduleRegistry.invalidate(); + log.info(`Invalidated configuration caches after notification ${reason}`); } getPool(): Pool { - return getPgPool(this.opts.pg); + return getPgPool(this.opts.pg, { purpose: 'control' }); } - addEventListener(): void { - if (this.shuttingDown) return; - const pgPool = this.getPool(); - pgPool.connect(this.listenForChanges.bind(this)); + private clearListenRetry(): void { + if (!this.listenRetryTimer) return; + clearTimeout(this.listenRetryTimer); + this.listenRetryTimer = null; } - listenForChanges(err: Error | null, client: PoolClient, release: () => void): void { - if (err) { - this.error('Error connecting with notify listener', err); + private scheduleListenRetry(delayMs: number): void { + if (this.shuttingDown || this.listenRetryTimer || this.listenAttempt) return; + this.listenRetryTimer = setTimeout(() => { + this.listenRetryTimer = null; + this.addEventListener(); + }, delayMs); + this.listenRetryTimer.unref?.(); + } + + private cleanupListenAttempt( + attempt: ListenAttempt, + unlisten: boolean, + connectionError?: Error + ): Promise { + if (attempt.cleanupPromise) return attempt.cleanupPromise; + attempt.closed = true; + if (this.listenAttempt === attempt) this.listenAttempt = null; + + const pending = (async () => { + const client = attempt.client; + if (client && attempt.notificationHandler) { + client.removeListener('notification', attempt.notificationHandler); + } + if (client && attempt.errorHandler) { + client.removeListener('error', attempt.errorHandler); + } + let clientReleaseError = connectionError; + if (client && unlisten) { + try { + // node-postgres serializes queries on one client. This also safely + // queues behind an in-progress LISTEN during a shutdown race. + await client.query('UNLISTEN "schema:update"'); + } catch (error) { + // The connection may already be unusable; release still must run. + clientReleaseError ??= error instanceof Error + ? error + : new Error(String(error)); + } + } + let releaseError: unknown; + try { + attempt.releaseClient?.(clientReleaseError); + } catch (error) { + releaseError = error; + } + attempt.releaseClient = null; + try { + attempt.releasePoolLease(); + } catch (error) { + releaseError ??= error; + } + if (releaseError) this.error('Error releasing database notify listener', releaseError); + })(); + attempt.cleanupPromise = pending; + this.listenCleanupTasks.add(pending); + void pending.then( + () => this.listenCleanupTasks.delete(pending), + () => this.listenCleanupTasks.delete(pending) + ); + return pending; + } + + addEventListener(): void { + if (this.shuttingDown || this.listenAttempt) return; + this.clearListenRetry(); + let lease: PgPoolLease; + try { + // LISTEN owns a client for the process lifetime. Give it a distinct + // identity so a one-client routing pool remains available to ordinary + // control-plane requests instead of being permanently starved. + lease = acquirePgPool(this.opts.pg, { purpose: 'notifications' }); + } catch (error) { + this.error('Error acquiring pool for notify listener', error); if (!this.shuttingDown) { - setTimeout(() => this.addEventListener(), 5000); + const retryMs = error instanceof PgPoolCapacityError + ? error.retryAfterSeconds * 1000 + : 5000; + this.scheduleListenRetry(retryMs); } return; } + const attempt: ListenAttempt = { + releasePoolLease: once(() => lease.release()), + client: null, + releaseClient: null, + notificationHandler: null, + errorHandler: null, + closed: false, + cleanupPromise: null + }; + this.listenAttempt = attempt; + lease.pool.connect((err, client, release) => { + void this.listenForChanges( + err ?? null, + client as PoolClient | undefined, + release as ((error?: Error | boolean) => void) | undefined, + attempt + ).catch(async (error) => { + this.error('Unexpected notify listener setup failure', error); + await this.cleanupListenAttempt( + attempt, + false, + error instanceof Error ? error : new Error(String(error)) + ); + this.scheduleListenRetry(5000); + }); + }); + } - if (this.shuttingDown) { - release(); + private async listenForChanges( + err: Error | null, + client: PoolClient | undefined, + release: ((error?: Error | boolean) => void) | undefined, + attempt: ListenAttempt + ): Promise { + if (attempt.closed || this.listenAttempt !== attempt || this.shuttingDown) { + release?.(); + attempt.releasePoolLease(); return; } - this.listenClient = client; - this.listenRelease = release; + if (err) { + this.error('Error connecting with notify listener', err); + this.invalidateConfigurationCaches('connection failure'); + await this.cleanupListenAttempt(attempt, false); + this.scheduleListenRetry(5000); + return; + } - client.on('notification', ({ channel, payload }) => { + if (!client || !release) { + this.error('Notify listener connected without a client release handle'); + this.invalidateConfigurationCaches('invalid checkout'); + await this.cleanupListenAttempt(attempt, false); + this.scheduleListenRetry(5000); + return; + } + + attempt.client = client; + attempt.releaseClient = once(release); + attempt.notificationHandler = ({ channel, payload }) => { if (channel === 'schema:update' && payload) { log.info('schema:update', payload); - this.flush(payload); + void this.flush(payload).catch((error) => { + this.error('Error flushing schema:update notification', error); + }); } - }); - - client.query('LISTEN "schema:update"'); - - client.on('error', (e) => { - if (this.shuttingDown) { - release(); - return; - } - this.error('Error with database notify listener', e); - release(); - this.addEventListener(); - }); + }; + attempt.errorHandler = (error) => { + if (attempt.closed) return; + if (!this.shuttingDown) this.error('Error with database notify listener', error); + this.invalidateConfigurationCaches('connection loss'); + void this.cleanupListenAttempt(attempt, false, error).then(() => { + this.scheduleListenRetry(5000); + }); + }; + client.on('notification', attempt.notificationHandler); + client.on('error', attempt.errorHandler); + try { + await client.query('LISTEN "schema:update"'); + } catch (error) { + this.error('Error starting database notify listener', error); + this.invalidateConfigurationCaches('LISTEN failure'); + await this.cleanupListenAttempt( + attempt, + false, + error instanceof Error ? error : new Error(String(error)) + ); + this.scheduleListenRetry(5000); + return; + } + if (attempt.closed || this.listenAttempt !== attempt || this.shuttingDown) { + await this.cleanupListenAttempt(attempt, true); + return; + } + this.invalidateConfigurationCaches('reconnect'); this.log('connected and listening for changes...'); } async removeEventListener(): Promise { - if (!this.listenClient || !this.listenRelease) { - return; + this.clearListenRetry(); + const attempt = this.listenAttempt; + if (attempt) await this.cleanupListenAttempt(attempt, true); + if (this.listenCleanupTasks.size > 0) { + await Promise.allSettled([...this.listenCleanupTasks]); } - - const client = this.listenClient; - const release = this.listenRelease; - this.listenClient = null; - this.listenRelease = null; - - client.removeAllListeners('notification'); - client.removeAllListeners('error'); - - try { - await client.query('UNLISTEN "schema:update"'); - } catch { - // Ignore listener cleanup errors during shutdown. - } - - release(); } async close(opts: { closeCaches?: boolean } = {}): Promise { @@ -321,30 +714,85 @@ class Server { } this.closed = true; this.shuttingDown = true; + // Only process-wide cache shutdown owns the process-global build + // coordinator. Closing one exported Server must not disable cold builds in + // another Server instance in the same process. + const buildDrain = closeCaches + ? closeGraphileBuildCoordinator() + : Promise.resolve(true); await this.removeEventListener(); + this.moduleRegistry.invalidate(); if (this.debugSampler) { await this.debugSampler.stop(); this.debugSampler = null; } + if (this.stopMemoryGovernor) { + this.stopMemoryGovernor(); + this.stopMemoryGovernor = null; + } + if (this.httpServer && this.websocketUpgradeGateway) { + this.httpServer.off('upgrade', this.websocketUpgradeGateway.handle); + } + this.websocketUpgradeGateway?.close(); if (this.httpServer?.listening) { await new Promise((resolve) => this.httpServer!.close(() => resolve())); } + const buildsDrained = await buildDrain; + if (!buildsDrained) { + log.warn( + 'GraphQL schema builds exceeded the shutdown drain deadline; late publication is disabled' + ); + } await closeDebugDatabasePools(); if (closeCaches) { await Server.closeCaches({ closePools: true }); + if (buildsDrained) reopenGraphileBuildCoordinator(); } } static async closeCaches(opts: { closePools?: boolean } = {}): Promise { - const { closePools = false } = opts; - svcCache.clear(); - // Use closeAllCaches to properly await async disposal of PostGraphile instances - // before closing pg pools - this ensures all connections are released - if (closePools) { - await closeAllCaches(); - } else { - graphileCache.clear(); + processCacheClosePoolsRequested ||= opts.closePools === true; + if (!processCacheClose) { + const closeTask = (async (): Promise => { + const buildsDrained = await closeGraphileBuildCoordinator(); + if (!buildsDrained) { + throw new GraphileCacheShutdownError( + GRAPHILE_CACHE_SHUTDOWN_DRAIN_TIMEOUT_CODE, + 'GraphQL schema builds did not drain; caches and pools were left intact' + ); + } + + clearSvcCache(); + let poolsClosed = false; + if (processCacheClosePoolsRequested) { + await closeAllCaches(); + poolsClosed = true; + } else { + await clearGraphileCache(); + } + // A concurrent closeCaches({ closePools: true }) may have joined while + // the resident-only clear was awaiting disposal. Honor that escalation + // before build admission can reopen. + if (processCacheClosePoolsRequested && !poolsClosed) { + await closeAllCaches(); + } + + if (!reopenGraphileBuildCoordinator()) { + throw new GraphileCacheShutdownError( + GRAPHILE_CACHE_SHUTDOWN_RESTART_REQUIRED_CODE, + 'GraphQL build admission cannot reopen safely; process restart is required' + ); + } + })(); + const tracked = closeTask.finally(() => { + if (processCacheClose === tracked) { + processCacheClose = null; + processCacheClosePoolsRequested = false; + } + }); + processCacheClose = tracked; } + return processCacheClose!; } log(text: string): void { diff --git a/graphql/server/src/websocket-upgrade.ts b/graphql/server/src/websocket-upgrade.ts new file mode 100644 index 0000000000..6bd3f5adcb --- /dev/null +++ b/graphql/server/src/websocket-upgrade.ts @@ -0,0 +1,447 @@ +import { type IncomingMessage,ServerResponse, STATUS_CODES } from 'node:http'; +import type { Socket } from 'node:net'; +import { type Duplex,PassThrough } from 'node:stream'; + +import type { + Express, + NextFunction, + Request, + RequestHandler, + Response +} from 'express'; + +import { parseCookieValue, SESSION_COOKIE_NAME } from './middleware/cookie'; +import { isCorsOriginAllowed } from './middleware/cors'; + +export const GRAPHILE_WEBSOCKET_PATH = '/graphql'; +export const GRAPHILE_WEBSOCKET_ROUTE_NOT_FOUND_CODE = + 'GRAPHILE_WEBSOCKET_ROUTE_NOT_FOUND'; +export const GRAPHILE_WEBSOCKET_BAD_UPGRADE_CODE = + 'GRAPHILE_WEBSOCKET_BAD_UPGRADE'; +export const GRAPHILE_WEBSOCKET_ADMISSION_TIMEOUT_CODE = + 'GRAPHILE_WEBSOCKET_ADMISSION_TIMEOUT'; +export const GRAPHILE_WEBSOCKET_ADMISSION_FAILED_CODE = + 'GRAPHILE_WEBSOCKET_ADMISSION_FAILED'; +export const GRAPHILE_WEBSOCKET_AUTH_REJECTED_CODE = + 'GRAPHILE_WEBSOCKET_AUTH_REJECTED'; +export const GRAPHILE_WEBSOCKET_SERVER_CLOSING_CODE = + 'GRAPHILE_WEBSOCKET_SERVER_CLOSING'; + +const DEFAULT_ADMISSION_TIMEOUT_MS = 180_000; + +interface UpgradeResponse { + status: number; + code: string; + retryAfterSeconds?: number; +} + +interface PendingUpgrade { + readonly request: IncomingMessage; + readonly socket: Duplex; + readonly head: Buffer; + readonly response: ServerResponse; + readonly responseSocket: Socket; + readonly timer: ReturnType; + readonly onSocketClose: () => void; + readonly onSocketError: () => void; + readonly onResponseFinish: () => void; + readonly onResponseClose: () => void; + readonly removePending: () => void; + handedOff: boolean; + terminal: boolean; +} + +const pendingByRequest = new WeakMap(); + +const safeStatus = (status: number): number => + Number.isSafeInteger(status) && status >= 400 && status <= 599 ? status : 500; + +const reasonPhrase = (status: number): string => + STATUS_CODES[status] ?? 'Error'; + +const writeUpgradeResponse = ( + socket: Duplex, + response: UpgradeResponse +): void => { + if (socket.destroyed || !socket.writable) return; + const status = safeStatus(response.status); + const body = JSON.stringify({ error: { code: response.code } }); + const headers = [ + `HTTP/1.1 ${status} ${reasonPhrase(status)}`, + 'Connection: close', + 'Content-Type: application/json; charset=utf-8', + `Content-Length: ${Buffer.byteLength(body)}`, + ...(response.retryAfterSeconds == null + ? [] + : [`Retry-After: ${response.retryAfterSeconds}`]), + '', + body + ].join('\r\n'); + try { + socket.end(headers); + } catch { + socket.destroy(); + } +}; + +const websocketPath = (request: IncomingMessage): string => { + const raw = request.url ?? ''; + const queryStart = raw.indexOf('?'); + return queryStart < 0 ? raw : raw.slice(0, queryStart); +}; + +const headerContainsToken = ( + value: string | string[] | undefined, + expected: string +): boolean => { + const values = Array.isArray(value) ? value : value == null ? [] : [value]; + return values.some((item) => + item.split(',').some((token) => token.trim().toLowerCase() === expected) + ); +}; + +const isGraphileWebSocketRequest = (request: IncomingMessage): boolean => + request.method === 'GET' + && websocketPath(request) === GRAPHILE_WEBSOCKET_PATH + && headerContainsToken(request.headers.connection, 'upgrade') + && headerContainsToken(request.headers.upgrade, 'websocket'); + +const responseFailure = (response: ServerResponse): UpgradeResponse => { + const status = safeStatus(response.statusCode); + const retryAfterValue = response.getHeader('Retry-After'); + const parsedRetryAfter = typeof retryAfterValue === 'string' + ? Number.parseInt(retryAfterValue, 10) + : typeof retryAfterValue === 'number' + ? retryAfterValue + : undefined; + const retryAfterSeconds = Number.isSafeInteger(parsedRetryAfter) + && (parsedRetryAfter as number) >= 0 + ? parsedRetryAfter + : undefined; + return { + status, + code: status === 401 || status === 403 + ? GRAPHILE_WEBSOCKET_AUTH_REJECTED_CODE + : status === 404 + ? GRAPHILE_WEBSOCKET_ROUTE_NOT_FOUND_CODE + : GRAPHILE_WEBSOCKET_ADMISSION_FAILED_CODE, + retryAfterSeconds + }; +}; + +export interface GraphileWebSocketUpgradeGatewayOptions { + /** Total time allowed for routing, auth, safety checks, and a cold build. */ + admissionTimeoutMs?: number; +} + +export interface GraphileWebSocketUpgradeGateway { + handle(request: IncomingMessage, socket: Duplex, head: Buffer): void; + close(): void; + readonly pendingCount: number; +} + +/** + * Feed upgrade requests through the same Express application as HTTP without + * exposing middleware-generated bodies on the wire. Express writes to a + * private sink; only stable, metadata-free admission errors reach the client. + */ +export const createGraphileWebSocketUpgradeGateway = ( + app: Express, + options: GraphileWebSocketUpgradeGatewayOptions = {} +): GraphileWebSocketUpgradeGateway => { + const admissionTimeoutMs = options.admissionTimeoutMs + ?? DEFAULT_ADMISSION_TIMEOUT_MS; + if (!Number.isSafeInteger(admissionTimeoutMs) || admissionTimeoutMs <= 0) { + throw new Error('WebSocket admission timeout must be a positive safe integer'); + } + + const pending = new Set(); + let closed = false; + + const cleanup = (context: PendingUpgrade): void => { + if (context.terminal) return; + context.terminal = true; + clearTimeout(context.timer); + pending.delete(context); + pendingByRequest.delete(context.request); + context.socket.removeListener('close', context.onSocketClose); + context.socket.removeListener('error', context.onSocketError); + context.response.removeListener('finish', context.onResponseFinish); + context.response.removeListener('close', context.onResponseClose); + if (context.response.socket === context.responseSocket) { + context.response.detachSocket(context.responseSocket); + } + context.responseSocket.destroy(); + }; + + const signalAdmissionAbort = (context: PendingUpgrade): void => { + // Remove only the gateway's terminal listeners before emitting the ordinary + // Express lifecycle signals. Request-scoped middleware must still observe + // them, but the gateway must retain the caller-selected stable response. + context.response.removeListener('finish', context.onResponseFinish); + context.response.removeListener('close', context.onResponseClose); + try { + context.request.emit('aborted'); + } catch { + // Cleanup and transport rejection remain mandatory even if an observer + // violates EventEmitter's no-throw expectation. + } + if (!context.response.destroyed && !context.response.writableEnded) { + try { + context.response.emit('close'); + } catch { + // See above: lifecycle observers are advisory to gateway cleanup. + } + } + }; + + const abortRequest = (context: PendingUpgrade): void => { + if (context.handedOff || context.terminal) return; + // Upgrade IncomingMessage instances are no longer owned by Node's HTTP + // parser, so a peer disconnect does not reliably emit `aborted`. Re-emit + // the ordinary request signal so queued Graphile builds release the waiter. + signalAdmissionAbort(context); + cleanup(context); + }; + + const rejectPending = ( + context: PendingUpgrade, + response: UpgradeResponse + ): void => { + if (context.handedOff || context.terminal) return; + cleanup(context); + writeUpgradeResponse(context.socket, response); + }; + + const abortAndRejectPending = ( + context: PendingUpgrade, + response: UpgradeResponse + ): void => { + if (context.handedOff || context.terminal) return; + signalAdmissionAbort(context); + rejectPending(context, response); + }; + + const handle = ( + request: IncomingMessage, + socket: Duplex, + head: Buffer + ): void => { + if (closed) { + writeUpgradeResponse(socket, { + status: 503, + code: GRAPHILE_WEBSOCKET_SERVER_CLOSING_CODE, + retryAfterSeconds: 1 + }); + return; + } + if (websocketPath(request) !== GRAPHILE_WEBSOCKET_PATH) { + writeUpgradeResponse(socket, { + status: 404, + code: GRAPHILE_WEBSOCKET_ROUTE_NOT_FOUND_CODE + }); + return; + } + if (!isGraphileWebSocketRequest(request)) { + writeUpgradeResponse(socket, { + status: 400, + code: GRAPHILE_WEBSOCKET_BAD_UPGRADE_CODE + }); + return; + } + + // Never attach the real upgrade socket to ServerResponse: an API/auth/build + // error may contain development detail. The response sink lets the normal + // Express lifecycle run while the gateway emits only stable error codes. + const response = new ServerResponse(request); + const responseSocket = new PassThrough() as unknown as Socket; + response.assignSocket(responseSocket); + + let context!: PendingUpgrade; + const onSocketClose = (): void => abortRequest(context); + const onSocketError = (): void => abortRequest(context); + const onResponseFinish = (): void => { + if (context.handedOff || context.terminal) return; + rejectPending(context, responseFailure(response)); + }; + const onResponseClose = (): void => { + if (context.handedOff || context.terminal) return; + rejectPending(context, responseFailure(response)); + }; + const timer = setTimeout(() => { + if (context.handedOff || context.terminal) return; + // Abort the build waiter before closing the transport, then surface a + // stable response that does not disclose the routed tenant or cache key. + abortAndRejectPending(context, { + status: 503, + code: GRAPHILE_WEBSOCKET_ADMISSION_TIMEOUT_CODE, + retryAfterSeconds: 1 + }); + }, admissionTimeoutMs); + timer.unref?.(); + + context = { + request, + socket, + head, + response, + responseSocket, + timer, + onSocketClose, + onSocketError, + onResponseFinish, + onResponseClose, + removePending: () => pending.delete(context), + handedOff: false, + terminal: false + }; + pending.add(context); + pendingByRequest.set(request, context); + socket.once('close', onSocketClose); + socket.once('error', onSocketError); + response.once('finish', onResponseFinish); + response.once('close', onResponseClose); + + try { + app(request, response); + } catch { + abortAndRejectPending(context, { + status: 500, + code: GRAPHILE_WEBSOCKET_ADMISSION_FAILED_CODE + }); + } + }; + + return { + handle, + close: () => { + if (closed) return; + closed = true; + for (const context of [...pending]) { + abortAndRejectPending(context, { + status: 503, + code: GRAPHILE_WEBSOCKET_SERVER_CLOSING_CODE, + retryAfterSeconds: 1 + }); + } + }, + get pendingCount(): number { + return pending.size; + } + }; +}; + +export const isGraphileWebSocketUpgrade = (request: Request): boolean => + pendingByRequest.has(request as unknown as IncomingMessage); + +/** Cookie-authenticated WebSockets require an origin a browser can prove. */ +export const isGraphileWebSocketOriginAllowed = ( + request: Request, + fallbackOrigin?: string +): boolean => { + const origin = request.get('origin'); + const bearer = request.headers.authorization + ?.toLowerCase().startsWith('bearer ') === true; + const sessionCookie = parseCookieValue(request, SESSION_COOKIE_NAME); + if (!origin) return bearer || !sessionCookie; + if (!sessionCookie) { + return isCorsOriginAllowed({ + origin, + fallbackOrigin, + api: request.api, + requestHost: request.get('host') + }); + } + + // A wildcard HTTP CORS policy and the localhost development convenience are + // not sufficient for a credentialed WebSocket: browsers attach cookies to + // the handshake but do not enforce CORS on the upgraded connection. Require + // an exact configured origin or exact same-host origin for session auth. + const normalizedOrigin = origin.trim(); + const fallback = fallbackOrigin?.trim(); + if (fallback && fallback !== '*' && normalizedOrigin === fallback) return true; + if ( + [...(request.api?.corsOrigins ?? []), ...(request.api?.domains ?? [])] + .includes(normalizedOrigin) + ) { + return true; + } + try { + return new URL(normalizedOrigin).host.toLowerCase() + === request.get('host')?.toLowerCase(); + } catch { + return false; + } +}; + +/** Mount immediately after API routing and before authentication/database I/O. */ +export const createGraphileWebSocketOriginGuard = ( + fallbackOrigin?: string +): RequestHandler => ( + request: Request, + response: Response, + next: NextFunction +): void => { + if ( + !isGraphileWebSocketUpgrade(request) + || isGraphileWebSocketOriginAllowed(request, fallbackOrigin) + ) { + next(); + return; + } + response.status(403).json({ + error: { + code: GRAPHILE_WEBSOCKET_AUTH_REJECTED_CODE, + message: 'WebSocket origin is not allowed' + } + }); +}; + +export interface AcceptedGraphileWebSocketUpgrade { + readonly socket: Duplex; + readonly head: Buffer; +} + +export const getGraphileWebSocketUpgradeTransport = ( + request: Request +): AcceptedGraphileWebSocketUpgrade | undefined => { + const context = pendingByRequest.get(request as unknown as IncomingMessage); + return context ? { socket: context.socket, head: context.head } : undefined; +}; + +/** + * Complete the synthetic response lifecycle before Grafserv owns the socket. + * This releases request-scoped pool leases while retaining the routed API and + * authenticated token on the IncomingMessage used by GraphQL over WebSocket. + */ +export const handoffGraphileWebSocketUpgrade = ( + request: Request, + response: Response +): AcceptedGraphileWebSocketUpgrade => { + const context = pendingByRequest.get(request as unknown as IncomingMessage); + if (!context || context.response !== (response as unknown as ServerResponse)) { + throw new Error('WebSocket upgrade context is unavailable'); + } + if (context.terminal || context.handedOff || context.socket.destroyed) { + throw new Error('WebSocket upgrade request is no longer active'); + } + if (context.response.socket !== context.responseSocket) { + throw new Error('Synthetic WebSocket admission response lost socket ownership'); + } + + context.handedOff = true; + clearTimeout(context.timer); + context.removePending(); + pendingByRequest.delete(context.request); + context.socket.removeListener('close', context.onSocketClose); + context.socket.removeListener('error', context.onSocketError); + context.response.removeListener('finish', context.onResponseFinish); + context.response.removeListener('close', context.onResponseClose); + context.response.detachSocket(context.responseSocket); + context.responseSocket.destroy(); + context.terminal = true; + // `finish` would mean an HTTP body was completed. `close` accurately tells + // request-scoped middleware that the synthetic response has been retired. + context.response.emit('close'); + return { socket: context.socket, head: context.head }; +}; diff --git a/graphql/types/README.md b/graphql/types/README.md index e071cedb40..01a6ec9f16 100644 --- a/graphql/types/README.md +++ b/graphql/types/README.md @@ -43,6 +43,14 @@ const config: ConstructiveOptions = { routingSchema: 'routing_public', exposedSchemas: ['public'], }, + routingCache: { + maxEntries: 4096, + }, + runtimePgResolver: async (route) => ({ + database: route.databaseName, + user: await runtimeUsers.forRoute(route), + password: await runtimePasswords.forRoute(route), + }), features: { simpleInflection: true, postgis: true, @@ -64,6 +72,27 @@ PostGraphile/Graphile configuration including schema, plugins, and build options Configuration for the Constructive API including meta API settings, exposed schemas, and role configuration. +### RoutingCacheOptions + +Configuration for the process-wide routing/service-label metadata cache. This +cache is independent from Graphile build identity and its `maxEntries` value +must be at least the effective resident Graphile capacity. + +### RuntimePgResolver + +Production multi-tenant servers resolve a least-privilege login from the exact +credential-free `RuntimePgResolverInput`: database id/name, API id, ordered +schemas, and `[anonymous, authenticated]` roles. The resolver must return an +explicit user, password, and matching database. A static `runtimePg` is accepted +in production or scoped introspection only with `runtimePgStaticIdentity`, which +binds it to one byte-exact route contract. + +`runtimePgResolver` is trusted infrastructure and should look up the login by +immutable `databaseId`. Its normalized host, port, database, and TLS policy must +match the control-plane tenant connection. Multi-cluster routing requires a +future per-route resolver shared by both lanes; runtime-only endpoint divergence +fails closed. + ### GraphileFeatureOptions Feature flags for GraphQL/Graphile including inflection settings and PostGIS support. diff --git a/graphql/types/src/constructive.ts b/graphql/types/src/constructive.ts index 485a4f4a59..35243cd45c 100644 --- a/graphql/types/src/constructive.ts +++ b/graphql/types/src/constructive.ts @@ -7,7 +7,7 @@ import { PgTestConnectionOptions, ServerOptions} from '@pgpmjs/types'; import deepmerge from 'deepmerge'; -import { PgConfig } from 'pg-env'; +import type { PgConfig, PgPoolConfig } from 'pg-env'; import { apiDefaults, @@ -19,6 +19,53 @@ import { import { LlmOptions } from './llm'; import { SmsOptions } from './sms'; +/** Process-wide routing-label metadata cache configuration. */ +export interface RoutingCacheOptions { + /** Maximum resolved service labels retained by one GraphQL server process. */ + maxEntries?: number; +} + +/** Credential-free routing input for resolving one physical listener login. */ +export interface NotificationPgResolverInput { + databaseId: string; + databaseName: string; + apiId: string; + schemas: readonly string[]; +} + +export type NotificationPgConfig = Partial & { pool?: PgPoolConfig }; + +/** + * Resolve a dedicated notification login for one physical database. The + * result must explicitly contain its user and password; server code never + * falls back to runtime or control-plane credentials. + */ +export type NotificationPgResolver = ( + input: Readonly +) => NotificationPgConfig | Promise; + +/** Credential-free exact route contract for one tenant execution identity. */ +export interface RuntimePgResolverInput { + databaseId: string; + databaseName: string; + apiId: string; + /** Physical schemas in Graphile exposure order. */ + schemas: readonly string[]; + /** Request roles in `[anonymous, authenticated]` order. */ + roles: readonly [anonymous: string, authenticated: string]; +} + +export type RuntimePgConfig = Partial & { pool?: PgPoolConfig }; + +/** + * Resolve one least-privilege tenant execution login from the exact routed + * contract. Results must contain explicit user, password, and database fields; + * control-plane credentials are never inherited. + */ +export type RuntimePgResolver = ( + input: Readonly +) => RuntimePgConfig | Promise; + /** * GraphQL-specific options for Constructive */ @@ -29,6 +76,8 @@ export interface ConstructiveGraphQLOptions { features?: GraphileFeatureOptions; /** API configuration options */ api?: ApiOptions; + /** Routing-label metadata cache configuration */ + routingCache?: RoutingCacheOptions; } /** @@ -40,6 +89,19 @@ export interface ConstructiveOptions extends PgpmOptions, ConstructiveGraphQLOpt db?: Partial; /** PostgreSQL connection configuration */ pg?: Partial; + /** + * Static least-privilege PostgreSQL login used for tenant GraphQL execution. + * Production and scoped introspection require `runtimePgStaticIdentity` and + * accept this login for that one exact route only. Multi-tenant servers must + * use `runtimePgResolver` instead. + */ + runtimePg?: RuntimePgConfig; + /** Exact credential-free route authorized to use the static `runtimePg`. */ + runtimePgStaticIdentity?: RuntimePgResolverInput; + /** Per-route least-privilege tenant execution login resolver. */ + runtimePgResolver?: RuntimePgResolver; + /** Per-physical-database login resolver used only by shared realtime LISTEN. */ + notificationPgResolver?: NotificationPgResolver; /** PostGraphile/Graphile configuration */ graphile?: GraphileOptions; /** HTTP server configuration */ @@ -48,6 +110,8 @@ export interface ConstructiveOptions extends PgpmOptions, ConstructiveGraphQLOpt features?: GraphileFeatureOptions; /** API configuration options */ api?: ApiOptions; + /** Routing-label metadata cache configuration */ + routingCache?: RoutingCacheOptions; /** CDN and file storage configuration */ cdn?: CDNOptions; /** Module deployment configuration */ @@ -66,7 +130,8 @@ export interface ConstructiveOptions extends PgpmOptions, ConstructiveGraphQLOpt export const constructiveGraphqlDefaults: ConstructiveGraphQLOptions = { graphile: graphileDefaults, features: graphileFeatureDefaults, - api: apiDefaults + api: apiDefaults, + routingCache: {} }; /** diff --git a/graphql/types/src/graphile.ts b/graphql/types/src/graphile.ts index 72fff4c739..f0edc3b61b 100644 --- a/graphql/types/src/graphile.ts +++ b/graphql/types/src/graphile.ts @@ -1,15 +1,88 @@ import type { GraphileConfig } from 'graphile-config'; +export type GraphileIntrospectionMode = 'stock' | 'scoped-required'; +export type GraphileIntrospectionClientReleaseMode = 'reuse' | 'destroy'; +export type GraphileRealtimeNotificationMode = 'dedicated' | 'shared-exact'; + +/** Per-schema Grafast parse, operation, and plan cache bounds. */ +export interface GrafastCacheLimits { + /** Maximum parsed and validated GraphQL documents retained by one schema. */ + queryCacheMaxLength?: number; + /** Maximum GraphQL operations with retained plan lookup state per schema. */ + operationsCacheMaxLength?: number; + /** Maximum context/variable-specific plans retained for one operation. */ + operationOperationPlansCacheMaxLength?: number; +} + /** * PostGraphile/Graphile v5 configuration */ export interface GraphileOptions { /** Database schema(s) to expose through GraphQL */ schema?: string | string[]; - /** Additional presets to extend */ + /** + * Additional trusted startup presets, applied after Constructive's feature + * preset. The server rejects nested attempts to replace its pgServices, + * tenant request context, transport/error policy, or fixed runtime plugins. + */ extends?: GraphileConfig.Preset[]; - /** Preset overrides */ + /** + * Trusted startup preset overrides. Safe schema and runtime settings plus + * caller plugins are applied; Constructive-owned tenant boundaries remain + * authoritative and fail closed on explicit override attempts. + */ preset?: Partial; + /** + * Admit `extends` and `preset` as fully trusted in-process code in production. + * + * Graphile plugins are not sandboxed: an admitted plugin can execute raw SQL + * through the configured PostgreSQL service and can access the Node.js + * process. Production therefore rejects every non-empty caller preset unless + * the deployment explicitly opts it into the server trust boundary. + */ + trustCallerPresetsInProduction?: boolean; + /** PostgreSQL catalog introspection strategy; scoped mode fails if any requested schema is absent */ + introspectionMode?: GraphileIntrospectionMode; + /** + * Whether the exact PostgreSQL client used for catalog introspection is + * returned to the runtime pool or destroyed after the gather query. Destroy + * avoids carrying catalog-query backend memory into request traffic and + * costs one lazy reconnect after each schema build. + */ + introspectionClientReleaseMode?: GraphileIntrospectionClientReleaseMode; + /** + * Ordered, non-writable schemas that exposed objects may depend on (for + * example the schema containing PostGIS or pgvector). Scoped mode fails if + * catalog closure reaches any other non-system schema. + */ + introspectionDependencySchemas?: string[]; + /** Explicit per-schema Grafast cache bounds used for tenant-density control. */ + grafastCache?: GrafastCacheLimits; + /** + * Release schema-construction-only Graphile state after successful schema + * validation. This is an opt-in density optimization; materialized schemas + * and runtime execution state remain tenant-dedicated. + */ + releaseBuildStateAfterValidation?: boolean; + /** + * Exact physical schema containing realtime cursor functions. Omit for the + * compatibility default `realtime_public`. + */ + realtimeSchema?: string; + /** + * PostgreSQL notification transport. `dedicated` preserves the current + * per-Graphile PgSubscriber; `shared-exact` is an experimental, default-off, + * role-attested broker whose leases are restricted to compiled physical + * topics. The GraphQL server routes WebSocket upgrades independently through + * the exact tenant build contract and admission boundary. + */ + realtimeNotificationMode?: GraphileRealtimeNotificationMode; + /** Maximum age of a successful shared-listener role attestation. */ + realtimeNotificationRoleRevalidationMs?: number; + /** Cursor recovery poll interval; lower values trade database QPS for latency. */ + realtimeCursorPollIntervalMs?: number; + /** Cursor listener heartbeat interval. */ + realtimeCursorHeartbeatIntervalMs?: number; } /** @@ -38,12 +111,24 @@ export interface ApiOptions { isPublic?: boolean; /** Schemas containing metadata tables */ metaSchemas?: string[]; + /** + * Allow the authenticated X-Meta-Schema private-header surface. This is a + * privileged, potentially cross-tenant control-plane API and is disabled by + * default; it must never share a tenant-facing ingress. + */ + allowMetaSchemaHeader?: boolean; /** * Schema containing the compiled resolve_route() resolver. Requests are * always resolved through the scoped-routing plane via * .resolve_route() (host → tenant/api/db/role). */ routingSchema?: string; + /** + * Process secret that authenticates reserved internal routing, identity, and + * cache-administration headers. It must contain at least 32 bytes. When it + * is absent, those headers are rejected rather than trusted from the network. + */ + internalRequestSecret?: string; } /** @@ -52,7 +137,16 @@ export interface ApiOptions { export const graphileDefaults: GraphileOptions = { schema: [], extends: [], - preset: {} + preset: {}, + trustCallerPresetsInProduction: false, + introspectionMode: 'stock', + introspectionClientReleaseMode: 'reuse', + introspectionDependencySchemas: [], + releaseBuildStateAfterValidation: false, + realtimeNotificationMode: 'dedicated', + realtimeNotificationRoleRevalidationMs: 60_000, + realtimeCursorPollIntervalMs: 5_000, + realtimeCursorHeartbeatIntervalMs: 30_000 }; /** @@ -77,5 +171,6 @@ export const apiDefaults: ApiOptions = { 'metaschema_public', 'metaschema_modules_public' ], + allowMetaSchemaHeader: false, routingSchema: 'routing_public' }; diff --git a/graphql/types/src/index.ts b/graphql/types/src/index.ts index 895604e137..aea05ed4ab 100644 --- a/graphql/types/src/index.ts +++ b/graphql/types/src/index.ts @@ -2,17 +2,30 @@ export { apiDefaults, ApiOptions, + GrafastCacheLimits, graphileDefaults, graphileFeatureDefaults, GraphileFeatureOptions, - GraphileOptions} from './graphile'; + GraphileIntrospectionClientReleaseMode, + GraphileIntrospectionMode, + GraphileOptions, + GraphileRealtimeNotificationMode, +} from './graphile'; // Export Constructive combined types export { constructiveDefaults, constructiveGraphqlDefaults, ConstructiveGraphQLOptions, - ConstructiveOptions} from './constructive'; + ConstructiveOptions, + NotificationPgConfig, + NotificationPgResolver, + NotificationPgResolverInput, + RoutingCacheOptions, + RuntimePgConfig, + RuntimePgResolver, + RuntimePgResolverInput +} from './constructive'; // Export GraphQL adapter types export { diff --git a/package.json b/package.json index 197e3b3514..fb4c9c2e8c 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,13 @@ "onlyBuiltDependencies": [ "@launchql/protobufjs", "core-js-pure" - ] + ], + "patchedDependencies": { + "@dataplan/pg@1.0.3": "patches/@dataplan__pg@1.0.3.patch", + "pg-introspection@1.0.1": "patches/pg-introspection@1.0.1.patch", + "graphile-build-pg@5.0.2": "patches/graphile-build-pg.patch", + "graphile-build@5.0.2": "patches/graphile-build@5.0.2.patch", + "@graphile-contrib/pg-many-to-many@2.0.0-rc.2": "patches/@graphile-contrib__pg-many-to-many@2.0.0-rc.2.patch" + } } } diff --git a/packages/cli/src/commands/explorer.ts b/packages/cli/src/commands/explorer.ts index 5d24a50f8b..e622204c51 100644 --- a/packages/cli/src/commands/explorer.ts +++ b/packages/cli/src/commands/explorer.ts @@ -104,9 +104,10 @@ export default async ( }); log.success('✅ Selected Configuration:'); - for (const [key, value] of Object.entries(options)) { - log.debug(`${key}: ${JSON.stringify(value)}`); - } + // The merged options object contains database and provider credentials. + // Keep startup diagnostics explicitly credential-free. + log.debug(`database: ${options.pg?.database ?? 'default'}`); + log.debug(`server: ${options.server?.host ?? 'localhost'}:${options.server?.port ?? port}`); log.success('🚀 Launching Explorer...\n'); explorer(options); diff --git a/packages/cli/src/commands/server.ts b/packages/cli/src/commands/server.ts index 0004362a17..8217be05a2 100644 --- a/packages/cli/src/commands/server.ts +++ b/packages/cli/src/commands/server.ts @@ -145,15 +145,19 @@ export default async ( } as ConstructiveOptions); log.success('✅ Selected Configuration:'); - for (const [key, value] of Object.entries(options)) { - log.debug(`${key}: ${JSON.stringify(value)}`); - } + // Never serialize the merged options object: it contains PostgreSQL + // passwords, provider credentials, and the internal-request secret. + log.debug(`database: ${options.pg?.database ?? selectedDb}`); + log.debug(`server: ${options.server?.host ?? 'localhost'}:${options.server?.port ?? port}`); // Debug: Log API routing configuration const apiOpts = (options as any).api || {}; log.debug(`📡 API Routing: isPublic=${apiOpts.isPublic}, routingSchema=${apiOpts.routingSchema}`); if (apiOpts.isPublic === false) { - log.debug(` Header-based routing enabled (X-Api-Name, X-Database-Id, X-Meta-Schema)`); + log.debug(` Authenticated header routing available (X-Api-Name, X-Database-Id)`); + if (apiOpts.allowMetaSchemaHeader === true) { + log.warn(' Privileged X-Meta-Schema admin routing is enabled; isolate this listener from tenant ingress'); + } } if (apiOpts.metaSchemas?.length) { log.debug(` Meta schemas: ${apiOpts.metaSchemas.join(', ')}`); diff --git a/packages/express-context/__tests__/context-pool-leases.test.ts b/packages/express-context/__tests__/context-pool-leases.test.ts new file mode 100644 index 0000000000..1d7b0b85f9 --- /dev/null +++ b/packages/express-context/__tests__/context-pool-leases.test.ts @@ -0,0 +1,279 @@ +import { EventEmitter } from 'node:events'; + +import type { NextFunction, Request, Response } from 'express'; +import type { Pool } from 'pg'; +import { acquirePgPool, getPgPool, getPgPoolIdentity } from 'pg-cache'; + +import { buildContext, createContextMiddleware } from '../src/context'; +import type { ApiStructure } from '../src/types'; + +jest.mock('pg-cache', () => ({ + acquirePgPool: jest.fn(), + getPgPool: jest.fn(), + getPgPoolIdentity: jest.fn(() => 'pg:v1:test') +})); + +const mockedAcquire = acquirePgPool as jest.MockedFunction; +const mockedGet = getPgPool as jest.MockedFunction; +const mockedIdentity = getPgPoolIdentity as jest.MockedFunction; + +const api: ApiStructure = { + apiId: 'api-a', + databaseId: 'database-a', + dbname: 'tenant_a', + anonRole: 'anonymous', + roleName: 'authenticated', + schema: ['tenant_a_public'], + domains: [], + isPublic: false +}; + +const makeRequest = (): Request => Object.assign(new EventEmitter(), { + api, + requestId: 'request-a', + get: jest.fn((): undefined => undefined), + aborted: false, + destroyed: false, + socket: { destroyed: false } +}) as unknown as Request; + +const makePool = (): Pool => ({ query: jest.fn() } as unknown as Pool); + +const makeResponse = (): Response => { + const response = new EventEmitter() as EventEmitter & { + destroyed: boolean; + writableEnded: boolean; + }; + response.destroyed = false; + response.writableEnded = false; + return response as unknown as Response; +}; + +const leaseFor = (pool: Pool) => ({ + pool, + identity: `pool-${Math.random()}`, + release: jest.fn() +}); + +describe('context PostgreSQL pool lifetimes', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('pins runtime and loader pools until the response finishes', () => { + const leases = [leaseFor(makePool()), leaseFor(makePool()), leaseFor(makePool())]; + mockedAcquire + .mockReturnValueOnce(leases[0]) + .mockReturnValueOnce(leases[1]) + .mockReturnValueOnce(leases[2]); + const response = makeResponse(); + const next = jest.fn() as unknown as NextFunction; + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + runtimePg: { user: 'runtime', password: 'secret' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware(makeRequest(), response, next); + + expect(next).toHaveBeenCalledWith(); + expect(mockedAcquire).toHaveBeenCalledTimes(3); + for (const lease of leases) expect(lease.release).not.toHaveBeenCalled(); + + response.emit('finish'); + response.emit('close'); + for (const lease of leases) expect(lease.release).toHaveBeenCalledTimes(1); + }); + + it('uses the exact server-owned runtime resolution and exposes only its opaque identity', () => { + const runtime = leaseFor(makePool()); + runtime.identity = 'pg:v1:exact-runtime'; + const otherLeases = [leaseFor(makePool()), leaseFor(makePool())]; + mockedAcquire + .mockReturnValueOnce(runtime) + .mockReturnValueOnce(otherLeases[0]) + .mockReturnValueOnce(otherLeases[1]); + const request = makeRequest(); + const getRuntimePgResolution = jest.fn(() => ({ + pgConfig: { + host: 'db.internal', + port: 5432, + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + }, + poolIdentity: runtime.identity + })); + + const context = buildContext(request, { + pg: { database: 'routing' }, + getRuntimePgResolution, + loaders: { resolve: jest.fn() } as any + }, []); + + expect(getRuntimePgResolution).toHaveBeenCalledWith(request, api); + expect(mockedAcquire.mock.calls[0][0]).toEqual({ + host: 'db.internal', + port: 5432, + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + }); + expect(context?.runtimePoolIdentity).toBe('pg:v1:exact-runtime'); + }); + + it('fails closed when the supplied runtime identity changes before acquisition', () => { + const runtime = leaseFor(makePool()); + runtime.identity = 'pg:v1:different-runtime'; + mockedAcquire.mockReturnValueOnce(runtime); + + expect(() => buildContext(makeRequest(), { + getRuntimePgResolution: () => ({ + pgConfig: { + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + }, + poolIdentity: 'pg:v1:expected-runtime' + }) + }, [])).toThrow('pool identity changed before context acquisition'); + }); + + it('does not acquire leases for a request that already ended', () => { + const request = makeRequest(); + Object.assign(request, { aborted: true }); + const next = jest.fn() as unknown as NextFunction; + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware(request, makeResponse(), next); + + expect(mockedAcquire).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + }); + + it('does not acquire leases after the request transport socket is destroyed', () => { + const request = makeRequest(); + Object.assign(request.socket, { destroyed: true }); + const next = jest.fn() as unknown as NextFunction; + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware(request, makeResponse(), next); + + expect(mockedAcquire).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + }); + + it('continues after a parser consumed and auto-destroyed the request stream', () => { + const request = makeRequest(); + Object.assign(request, { + destroyed: true, + readableEnded: true, + complete: true + }); + const leases = [leaseFor(makePool()), leaseFor(makePool()), leaseFor(makePool())]; + mockedAcquire + .mockReturnValueOnce(leases[0]) + .mockReturnValueOnce(leases[1]) + .mockReturnValueOnce(leases[2]); + const response = makeResponse(); + const next = jest.fn() as unknown as NextFunction; + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + runtimePg: { user: 'runtime', password: 'secret' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware(request, response, next); + + expect(next).toHaveBeenCalledWith(); + expect(mockedAcquire).toHaveBeenCalledTimes(3); + response.emit('finish'); + for (const lease of leases) expect(lease.release).toHaveBeenCalledTimes(1); + }); + + it('releases leases when the response ends during context construction', () => { + const response = makeResponse(); + const leases = [leaseFor(makePool()), leaseFor(makePool()), leaseFor(makePool())]; + mockedAcquire + .mockReturnValueOnce(leases[0]) + .mockImplementationOnce(() => { + Object.assign(response, { destroyed: true }); + return leases[1]; + }) + .mockReturnValueOnce(leases[2]); + const next = jest.fn() as unknown as NextFunction; + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + runtimePg: { user: 'runtime', password: 'secret' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware(makeRequest(), response, next); + + expect(mockedAcquire).toHaveBeenCalledTimes(3); + for (const lease of leases) expect(lease.release).toHaveBeenCalledTimes(1); + expect(next).not.toHaveBeenCalled(); + }); + + it('releases leases when the request aborts', () => { + const request = makeRequest(); + const response = makeResponse(); + const leases = [leaseFor(makePool()), leaseFor(makePool()), leaseFor(makePool())]; + mockedAcquire + .mockReturnValueOnce(leases[0]) + .mockReturnValueOnce(leases[1]) + .mockReturnValueOnce(leases[2]); + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + runtimePg: { user: 'runtime', password: 'secret' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware(request, response, jest.fn()); + request.emit('aborted'); + + for (const lease of leases) expect(lease.release).toHaveBeenCalledTimes(1); + }); + + it('releases earlier leases when a later acquisition fails', () => { + const first = leaseFor(makePool()); + const error = new Error('capacity'); + mockedAcquire.mockReturnValueOnce(first).mockImplementationOnce(() => { + throw error; + }); + const next = jest.fn() as unknown as NextFunction; + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware( + makeRequest(), + new EventEmitter() as unknown as Response, + next + ); + + expect(first.release).toHaveBeenCalledTimes(1); + expect(next).toHaveBeenCalledWith(error); + }); + + it('preserves unleased getPgPool behavior for direct buildContext callers', () => { + mockedGet.mockReturnValue(makePool()); + + const context = buildContext(makeRequest(), { + pg: { database: 'routing' }, + loaders: { resolve: jest.fn() } as any + }); + + expect(context).not.toBeNull(); + expect(mockedGet).toHaveBeenCalledTimes(3); + expect(mockedIdentity).toHaveBeenCalledTimes(3); + expect(mockedAcquire).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/express-context/__tests__/pg-settings.test.ts b/packages/express-context/__tests__/pg-settings.test.ts index 6b87b83b15..fafc90264f 100644 --- a/packages/express-context/__tests__/pg-settings.test.ts +++ b/packages/express-context/__tests__/pg-settings.test.ts @@ -29,14 +29,14 @@ describe('buildPgSettings — jwt.claims.api_id provenance', () => { expect(settings['jwt.claims.user_id']).toBe('u1'); }); - it('omits jwt.claims.api_id when the api has no apiId (non-API surface)', () => { + it('clears jwt.claims.api_id when the api has no apiId (non-API surface)', () => { const settings = buildPgSettings({ api: { ...api, apiId: undefined }, token: null, requestId: 'r1' }); - expect(settings['jwt.claims.api_id']).toBeUndefined(); + expect(settings['jwt.claims.api_id']).toBe(''); }); it('is derived only from the resolved api, never from the token', () => { diff --git a/packages/express-context/package.json b/packages/express-context/package.json index f3ab1cefd1..a508091a0c 100644 --- a/packages/express-context/package.json +++ b/packages/express-context/package.json @@ -34,6 +34,7 @@ "@pgpmjs/logger": "workspace:^", "@pgpmjs/server-utils": "workspace:^", "@pgpmjs/types": "workspace:^", + "@pgsql/quotes": "^18.2.0", "lru-cache": "^11.2.7", "pg": "^8.21.0", "pg-cache": "workspace:^", diff --git a/packages/express-context/src/__tests__/compute-loader.test.ts b/packages/express-context/src/__tests__/compute-loader.test.ts new file mode 100644 index 0000000000..5b7118648f --- /dev/null +++ b/packages/express-context/src/__tests__/compute-loader.test.ts @@ -0,0 +1,55 @@ +import type { Pool } from 'pg'; + +import { computeLoader } from '../loaders/compute'; + +describe('compute control-plane loader', () => { + afterEach(() => computeLoader.invalidate()); + + it('loads API bindings through the control-plane tenant pool', async () => { + const query = jest.fn() + .mockResolvedValueOnce({ + rows: [{ + functions_schema_name: 'compute"schema', + definitions_table_name: 'definitions', + bindings_table_name: 'bindings', + invocations_schema_name: 'invocations', + invocations_table_name: 'jobs', + invocations_entity_field: 'database_id' + }] + }) + .mockResolvedValueOnce({ + rows: [{ + id: 'binding-a', + alias: 'summarize', + config: { graphql: true }, + function_definition_id: 'definition-a', + task_identifier: 'summarize-task', + description: 'Summarize content', + payload_args: [{ name: 'body', type: 'string' }] + }] + }); + const ctx = { + routingPool: {} as Pool, + tenantPool: { query } as unknown as Pool, + databaseId: 'database-compute-loader-test', + apiId: 'api-a', + dbname: 'tenant_db' + }; + + const result = await computeLoader.resolve(ctx); + + expect(query).toHaveBeenCalledTimes(2); + expect(query.mock.calls[1][0]).toContain('FROM "compute""schema"."bindings" b'); + expect(query.mock.calls[1][1]).toEqual(['api-a']); + expect(result?.bindings).toEqual([{ + bindingId: 'binding-a', + alias: 'summarize', + config: { graphql: true }, + functionDefinitionId: 'definition-a', + taskIdentifier: 'summarize-task', + description: 'Summarize content', + payloadArgs: [{ name: 'body', type: 'string' }], + module: result?.modules[0] + }]); + }); +}); diff --git a/packages/express-context/src/__tests__/loader-cache-isolation.test.ts b/packages/express-context/src/__tests__/loader-cache-isolation.test.ts new file mode 100644 index 0000000000..c94c9b64fa --- /dev/null +++ b/packages/express-context/src/__tests__/loader-cache-isolation.test.ts @@ -0,0 +1,175 @@ +import type { Pool } from 'pg'; + +import { createModuleLoader } from '../loaders/create-loader'; +import type { LoaderContext } from '../loaders/types'; + +const context = ( + routingPool: Pool, + tenantPool: Pool, + suffix: string +): LoaderContext => ({ + routingPool, + routingPoolIdentity: `routing:${suffix}`, + tenantPool, + tenantPoolIdentity: `tenant:${suffix}`, + databaseId: 'cloned-database-id', + apiId: 'cloned-api-id', + dbname: 'cloned_database' +}); + +describe('module loader physical cache isolation', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('does not share one logical database/API entry across physical pool contracts', async () => { + const routingA = {} as Pool; + const routingB = {} as Pool; + const tenantA = {} as Pool; + const tenantB = {} as Pool; + const ctxA = context(routingA, tenantA, 'a'); + const ctxB = context(routingB, tenantB, 'b'); + const resolve = jest.fn(async (ctx: LoaderContext) => + ctx.tenantPoolIdentity === 'tenant:a' ? 'config-a' : 'config-b' + ); + const loader = createModuleLoader({ name: 'physical-isolation', resolve }); + + await expect(loader.resolve(ctxA)).resolves.toBe('config-a'); + await expect(loader.resolve(ctxB)).resolves.toBe('config-b'); + await expect(loader.resolve(ctxA)).resolves.toBe('config-a'); + await expect(loader.resolve(ctxB)).resolves.toBe('config-b'); + + expect(resolve).toHaveBeenCalledTimes(2); + }); + + it('can invalidate one physical contract without evicting its logical twin', async () => { + const ctxA = context({} as Pool, {} as Pool, 'a'); + const ctxB = context({} as Pool, {} as Pool, 'b'); + let generation = 0; + const resolve = jest.fn(async (ctx: LoaderContext) => + `${ctx.tenantPoolIdentity}:${++generation}` + ); + const loader = createModuleLoader({ name: 'physical-invalidation', resolve }); + + const firstA = await loader.resolve(ctxA); + const firstB = await loader.resolve(ctxB); + loader.invalidate(ctxA.databaseId, ctxA); + + await expect(loader.resolve(ctxB)).resolves.toBe(firstB); + await expect(loader.resolve(ctxA)).resolves.not.toBe(firstA); + expect(resolve).toHaveBeenCalledTimes(3); + }); + + it('falls back to pool object identity for generic callers without opaque identities', async () => { + const routingPool = {} as Pool; + const tenantA = {} as Pool; + const tenantB = {} as Pool; + const base = { + routingPool, + databaseId: 'cloned-database-id', + apiId: 'cloned-api-id', + dbname: 'cloned_database' + }; + const resolve = jest.fn(async (ctx: LoaderContext) => + ctx.tenantPool === tenantA ? 'config-a' : 'config-b' + ); + const loader = createModuleLoader({ name: 'object-isolation', resolve }); + + await expect(loader.resolve({ ...base, tenantPool: tenantA })).resolves.toBe('config-a'); + await expect(loader.resolve({ ...base, tenantPool: tenantB })).resolves.toBe('config-b'); + await expect(loader.resolve({ ...base, tenantPool: tenantA })).resolves.toBe('config-a'); + + expect(resolve).toHaveBeenCalledTimes(2); + }); + + it('isolates routing schemas even when the physical pools and logical IDs match', async () => { + const base = context({} as Pool, {} as Pool, 'shared'); + const resolve = jest.fn(async (ctx: LoaderContext) => ctx.routingSchema); + const loader = createModuleLoader({ name: 'routing-schema-isolation', resolve }); + + await expect(loader.resolve({ ...base, routingSchema: 'routing_a' })) + .resolves.toBe('routing_a'); + await expect(loader.resolve({ ...base, routingSchema: 'routing_b' })) + .resolves.toBe('routing_b'); + await expect(loader.resolve({ ...base, routingSchema: 'routing_a' })) + .resolves.toBe('routing_a'); + + expect(resolve).toHaveBeenCalledTimes(2); + }); + + it('coalesces concurrent misses for one exact build contract', async () => { + const ctx = context({} as Pool, {} as Pool, 'shared'); + const resolve = jest.fn(async () => 'shared-config'); + const loader = createModuleLoader({ name: 'concurrent-coalescing', resolve }); + + await expect(Promise.all([ + loader.resolve(ctx), + loader.resolve(ctx), + loader.resolve(ctx) + ])).resolves.toEqual(['shared-config', 'shared-config', 'shared-config']); + expect(resolve).toHaveBeenCalledTimes(1); + }); + + it('does not publish a resolution invalidated while its query is in flight', async () => { + const ctx = context({} as Pool, {} as Pool, 'shared'); + let complete!: (value: string) => void; + const first = new Promise((resolve) => { + complete = resolve; + }); + const resolve = jest.fn() + .mockImplementationOnce(() => first) + .mockResolvedValueOnce('fresh-config'); + const loader = createModuleLoader({ + name: 'inflight-invalidation', + resolve + }); + + const stale = loader.resolve(ctx); + loader.invalidate(ctx.databaseId, ctx); + const fresh = loader.resolve(ctx); + await expect(fresh).resolves.toBe('fresh-config'); + complete('stale-config'); + await expect(stale).resolves.toBe('stale-config'); + await expect(loader.resolve(ctx)).resolves.toBe('fresh-config'); + expect(resolve).toHaveBeenCalledTimes(2); + }); + + it('uses a hard TTL that cache hits cannot extend indefinitely', async () => { + let now = 1; + jest.spyOn(performance, 'now').mockImplementation(() => now); + const ctx = context({} as Pool, {} as Pool, 'shared'); + let generation = 0; + const resolve = jest.fn(async () => `config-${++generation}`); + const loader = createModuleLoader({ + name: 'hard-expiry', + ttlMs: 100, + resolve + }); + + await expect(loader.resolve(ctx)).resolves.toBe('config-1'); + now = 76; + await expect(loader.resolve(ctx)).resolves.toBe('config-1'); + now = 106; + await expect(loader.resolve(ctx)).resolves.toBe('config-2'); + expect(resolve).toHaveBeenCalledTimes(2); + }); + + it('does not cache or coalesce authoritative loader reads', async () => { + const ctx = context({} as Pool, {} as Pool, 'shared'); + let generation = 0; + const resolve = jest.fn(async () => `config-${++generation}`); + const loader = createModuleLoader({ + name: 'authoritative', + cache: false, + resolve + }); + + await expect(Promise.all([ + loader.resolve(ctx), + loader.resolve(ctx) + ])).resolves.toEqual(['config-1', 'config-2']); + await expect(loader.resolve(ctx)).resolves.toBe('config-3'); + expect(resolve).toHaveBeenCalledTimes(3); + expect(loader.cacheSize).toBe(0); + }); +}); diff --git a/packages/express-context/src/__tests__/pg-settings.test.ts b/packages/express-context/src/__tests__/pg-settings.test.ts new file mode 100644 index 0000000000..0bfaf36ec3 --- /dev/null +++ b/packages/express-context/src/__tests__/pg-settings.test.ts @@ -0,0 +1,104 @@ +import { buildPgSettings, SECURITY_GUC_KEYS } from '../pg-settings'; +import type { ApiStructure } from '../types'; + +const api: ApiStructure = { + apiId: 'api-a', + databaseId: 'database-a', + dbname: 'tenant_a', + schema: ['tenant_a_public'], + roleName: 'tenant_user', + anonRole: 'tenant_anon' +}; + +describe('buildPgSettings', () => { + it('initializes every security GUC and explicit transaction state for anonymous requests', () => { + const settings = buildPgSettings({ api, token: null, requestId: 'request-a' }); + + expect(settings.role).toBe('tenant_anon'); + expect(settings['request.id']).toBe('request-a'); + expect(settings['transaction_read_only']).toBe('off'); + expect(settings['search_path']).toBe('pg_catalog, "tenant_a_public"'); + expect(settings['row_security']).toBe('on'); + for (const key of SECURITY_GUC_KEYS) { + expect(Object.prototype.hasOwnProperty.call(settings, key)).toBe(true); + } + expect(settings['jwt.claims.user_id']).toBe(''); + }); + + it('quotes every physical schema in the pinned search path', () => { + const settings = buildPgSettings({ + api: { ...api, schema: ['tenant-a', 'quoted"schema'] }, + token: null, + requestId: 'request-search-path', + dependencySchemas: ['postgis-ext', 'shared"api'] + }); + + expect(settings['search_path']).toBe( + 'pg_catalog, "postgis-ext", "shared""api", "tenant-a", "quoted""schema"' + ); + }); + + it('sets known claims while keeping every absent claim empty', () => { + const settings = buildPgSettings({ + api, + token: { + id: 'token-a', + user_id: 'user-a', + access_level: 'read_only', + kind: 'api_key' + }, + requestId: 'request-b', + clientIp: '127.0.0.1', + origin: 'https://example.test', + userAgent: 'test-agent', + deviceToken: 'device-a' + }); + + expect(settings).toMatchObject({ + role: 'tenant_user', + 'jwt.claims.token_id': 'token-a', + 'jwt.claims.user_id': 'user-a', + 'jwt.claims.principal_id': 'user-a', + 'jwt.claims.session_id': '', + 'jwt.claims.access_level': 'read_only', + 'jwt.claims.device_token': 'device-a', + 'transaction_read_only': 'on' + }); + }); + + it('does not retain claims or read-only state across requests', () => { + buildPgSettings({ + api, + token: { user_id: 'user-a', access_level: 'read_only' }, + requestId: 'request-a' + }); + const next = buildPgSettings({ api, token: null, requestId: 'request-b' }); + + expect(next['jwt.claims.user_id']).toBe(''); + expect(next['jwt.claims.access_level']).toBe(''); + expect(next['transaction_read_only']).toBe('off'); + }); + + it('rejects runtime-shaped trusted claims that could override session state', () => { + expect(() => buildPgSettings({ + api, + token: null, + requestId: 'request-extra-claim', + trustedClaims: { + role: 'cross_tenant_owner' + } as unknown as Record<'jwt.claims.user_id', string> + })).toThrow("trustedClaims contains unsupported security GUC 'role'"); + + const claims = Object.create(null) as Record; + Object.defineProperty(claims, 'jwt.claims.user_id', { + enumerable: true, + get: () => 'getter-value' + }); + expect(() => buildPgSettings({ + api, + token: null, + requestId: 'request-accessor-claim', + trustedClaims: claims as Record<'jwt.claims.user_id', string> + })).toThrow('trustedClaims.jwt.claims.user_id must be a string data property'); + }); +}); diff --git a/packages/express-context/src/__tests__/security-loader-freshness.test.ts b/packages/express-context/src/__tests__/security-loader-freshness.test.ts new file mode 100644 index 0000000000..a8e9cc7f8d --- /dev/null +++ b/packages/express-context/src/__tests__/security-loader-freshness.test.ts @@ -0,0 +1,269 @@ +import type { Pool } from 'pg'; + +import { authSettingsLoader } from '../loaders/auth-settings'; +import { corsLoader } from '../loaders/cors'; +import { databaseSettingsLoader } from '../loaders/database-settings'; +import { pubkeyLoader } from '../loaders/pubkey'; +import { rlsLoader } from '../loaders/rls'; +import type { LoaderContext } from '../loaders/types'; +import { webauthnLoader } from '../loaders/webauthn'; + +const loaderContext = ( + routingPool: Pool, + tenantPool: Pool +): LoaderContext => ({ + routingPool, + routingPoolIdentity: 'routing:security-test', + routingSchema: 'routing_public', + tenantPool, + tenantPoolIdentity: 'tenant:security-test', + databaseId: 'database-123', + apiId: 'api-123', + dbname: 'tenant_database' +}); + +describe('security-sensitive module freshness', () => { + afterEach(() => { + rlsLoader.invalidate(); + authSettingsLoader.invalidate(); + corsLoader.invalidate(); + databaseSettingsLoader.invalidate(); + pubkeyLoader.invalidate(); + webauthnLoader.invalidate(); + }); + + it('reads RLS authentication routing authoritatively on every request', async () => { + const query = jest.fn() + .mockResolvedValueOnce({ rows: [{ + authenticate: 'authenticate_v1', + authenticate_strict: 'authenticate_strict_v1', + authenticate_schema: 'auth_private', + role_schema: 'auth_public', + current_role: 'current_role', + current_role_id: 'current_role_id', + current_ip_address: 'current_ip_address', + current_user_agent: 'current_user_agent' + }] }) + .mockResolvedValueOnce({ rows: [{ + authenticate: 'authenticate_v2', + authenticate_strict: 'authenticate_strict_v2', + authenticate_schema: 'auth_private', + role_schema: 'auth_public', + current_role: 'current_role', + current_role_id: 'current_role_id', + current_ip_address: 'current_ip_address', + current_user_agent: 'current_user_agent' + }] }); + const routingPool = { query } as unknown as Pool; + const ctx = loaderContext(routingPool, {} as Pool); + + await expect(rlsLoader.resolve(ctx)).resolves.toMatchObject({ + authenticate: 'authenticate_v1' + }); + await expect(rlsLoader.resolve(ctx)).resolves.toMatchObject({ + authenticate: 'authenticate_v2' + }); + + expect(query).toHaveBeenCalledTimes(2); + expect(rlsLoader.cacheSize).toBe(0); + }); + + it('reads cookie and CAPTCHA policy authoritatively on every request', async () => { + const query = jest.fn() + .mockResolvedValueOnce({ + rows: [{ schema_name: 'sessions_private', table_name: 'auth_settings' }] + }) + .mockResolvedValueOnce({ rows: [{ + cookie_secure: true, + cookie_samesite: 'lax', + cookie_domain: null, + cookie_httponly: true, + cookie_max_age: '3600', + cookie_path: '/', + remember_me_duration: '86400', + enable_captcha: false, + captcha_site_key: null + }] }) + .mockResolvedValueOnce({ + rows: [{ schema_name: 'sessions_private', table_name: 'auth_settings' }] + }) + .mockResolvedValueOnce({ rows: [{ + cookie_secure: true, + cookie_samesite: 'strict', + cookie_domain: null, + cookie_httponly: true, + cookie_max_age: '1800', + cookie_path: '/', + remember_me_duration: '43200', + enable_captcha: true, + captcha_site_key: 'site-key-v2' + }] }); + const tenantPool = { query } as unknown as Pool; + const ctx = loaderContext({} as Pool, tenantPool); + + await expect(authSettingsLoader.resolve(ctx)).resolves.toMatchObject({ + cookieSamesite: 'lax', + enableCaptcha: false + }); + await expect(authSettingsLoader.resolve(ctx)).resolves.toMatchObject({ + cookieSamesite: 'strict', + enableCaptcha: true + }); + + expect(query).toHaveBeenCalledTimes(4); + expect(authSettingsLoader.cacheSize).toBe(0); + }); + + it('does not retain revoked CORS policy', async () => { + const query = jest.fn() + .mockResolvedValueOnce({ rows: [{ allowed_origins: ['https://old.example'] }] }) + .mockResolvedValueOnce({ rows: [{ allowed_origins: [] }] }); + const ctx = loaderContext({ query } as unknown as Pool, {} as Pool); + + await expect(corsLoader.resolve(ctx)).resolves.toEqual(['https://old.example']); + await expect(corsLoader.resolve(ctx)).resolves.toEqual([]); + + expect(query).toHaveBeenCalledTimes(2); + expect(corsLoader.cacheSize).toBe(0); + }); + + it('does not retain a revoked Graphile/realtime feature surface', async () => { + const settings = (enabled: boolean) => ({ + resolved_enable_aggregates: enabled, + resolved_enable_postgis: enabled, + resolved_enable_search: enabled, + resolved_enable_direct_uploads: enabled, + resolved_enable_presigned_uploads: enabled, + resolved_enable_many_to_many: enabled, + resolved_enable_connection_filter: enabled, + resolved_enable_ltree: enabled, + resolved_enable_llm: enabled, + resolved_enable_realtime: enabled, + resolved_enable_bulk: enabled, + resolved_enable_i18n: enabled + }); + const query = jest.fn() + .mockResolvedValueOnce({ rows: [settings(true)] }) + .mockResolvedValueOnce({ rows: [settings(false)] }); + const ctx = loaderContext({ query } as unknown as Pool, {} as Pool); + + await expect(databaseSettingsLoader.resolve(ctx)).resolves.toMatchObject({ + enableRealtime: true, + enableSearch: true + }); + await expect(databaseSettingsLoader.resolve(ctx)).resolves.toMatchObject({ + enableRealtime: false, + enableSearch: false + }); + + expect(query).toHaveBeenCalledTimes(2); + expect(databaseSettingsLoader.cacheSize).toBe(0); + }); + + it('rejects ambiguous or incomplete Graphile feature contracts', async () => { + const settings = { + resolved_enable_aggregates: false, + resolved_enable_postgis: false, + resolved_enable_search: false, + resolved_enable_direct_uploads: false, + resolved_enable_presigned_uploads: false, + resolved_enable_many_to_many: false, + resolved_enable_connection_filter: false, + resolved_enable_ltree: false, + resolved_enable_llm: false, + resolved_enable_realtime: false, + resolved_enable_bulk: false, + resolved_enable_i18n: false + }; + const ambiguous = loaderContext({ + query: jest.fn().mockResolvedValue({ rows: [settings, settings] }) + } as unknown as Pool, {} as Pool); + const incomplete = loaderContext({ + query: jest.fn().mockResolvedValue({ + rows: [{ ...settings, resolved_enable_search: null }] + }) + } as unknown as Pool, {} as Pool); + + await expect(databaseSettingsLoader.resolve(ambiguous)) + .rejects.toThrow('Ambiguous database feature configuration'); + await expect(databaseSettingsLoader.resolve(incomplete)) + .rejects.toThrow('Incomplete database feature configuration'); + }); + + it('does not retain changed public-key or WebAuthn policy', async () => { + const pubkeyQuery = jest.fn() + .mockResolvedValueOnce({ rows: [{ + schema: 'auth_public', + crypto_network: 'mainnet', + sign_up_with_key: 'sign_up_v1', + sign_in_request_challenge: 'request_v1', + sign_in_record_failure: 'failure_v1', + sign_in_with_challenge: 'sign_in_v1' + }] }) + .mockResolvedValueOnce({ rows: [{ + schema: 'auth_public', + crypto_network: 'mainnet', + sign_up_with_key: 'sign_up_v2', + sign_in_request_challenge: 'request_v2', + sign_in_record_failure: 'failure_v2', + sign_in_with_challenge: 'sign_in_v2' + }] }); + const pubkeyContext = loaderContext( + { query: pubkeyQuery } as unknown as Pool, + {} as Pool + ); + + await expect(pubkeyLoader.resolve(pubkeyContext)).resolves.toMatchObject({ + signUpWithKey: 'sign_up_v1' + }); + await expect(pubkeyLoader.resolve(pubkeyContext)).resolves.toMatchObject({ + signUpWithKey: 'sign_up_v2' + }); + + const webauthnQuery = jest.fn() + .mockResolvedValueOnce({ rows: [{ + schema: 'auth_public', + credentials_schema: 'auth_private', + sessions_schema: 'sessions_private', + session_secrets_schema: 'sessions_private', + rp_id: 'old.example', + rp_name: 'Old', + origin_allowlist: ['https://old.example'], + attestation_type: 'none', + require_user_verification: false, + resident_key: 'preferred', + challenge_expiry_seconds: 300 + }] }) + .mockResolvedValueOnce({ rows: [{ + schema: 'auth_public', + credentials_schema: 'auth_private', + sessions_schema: 'sessions_private', + session_secrets_schema: 'sessions_private', + rp_id: 'new.example', + rp_name: 'New', + origin_allowlist: ['https://new.example'], + attestation_type: 'direct', + require_user_verification: true, + resident_key: 'required', + challenge_expiry_seconds: 60 + }] }); + const webauthnContext = loaderContext( + { query: webauthnQuery } as unknown as Pool, + {} as Pool + ); + + await expect(webauthnLoader.resolve(webauthnContext)).resolves.toMatchObject({ + rpId: 'old.example', + requireUserVerification: false + }); + await expect(webauthnLoader.resolve(webauthnContext)).resolves.toMatchObject({ + rpId: 'new.example', + requireUserVerification: true + }); + + expect(pubkeyQuery).toHaveBeenCalledTimes(2); + expect(webauthnQuery).toHaveBeenCalledTimes(2); + expect(pubkeyLoader.cacheSize).toBe(0); + expect(webauthnLoader.cacheSize).toBe(0); + }); +}); diff --git a/packages/express-context/src/__tests__/security-metadata.test.ts b/packages/express-context/src/__tests__/security-metadata.test.ts new file mode 100644 index 0000000000..935d6f4f5d --- /dev/null +++ b/packages/express-context/src/__tests__/security-metadata.test.ts @@ -0,0 +1,134 @@ +import type { Pool } from 'pg'; + +import { createBillingClient } from '../billing-client'; +import { quoteQualifiedSqlIdentifier, quoteSqlIdentifier } from '../sql-identifiers'; +import { agentChatLoader } from '../loaders/agent-chat'; +import { authSettingsLoader } from '../loaders/auth-settings'; +import { rlsLoader } from '../loaders/rls'; +import type { LoaderContext } from '../loaders/types'; + +const context = ( + routingQuery: jest.Mock, + tenantQuery: jest.Mock +): LoaderContext => ({ + routingPool: { query: routingQuery } as unknown as Pool, + routingPoolIdentity: 'routing-a', + tenantPool: { query: tenantQuery } as unknown as Pool, + tenantPoolIdentity: 'tenant-a', + databaseId: '11111111-1111-4111-8111-111111111111', + apiId: '22222222-2222-4222-8222-222222222222', + dbname: 'tenant_a' +}); + +describe('security-sensitive metadata SQL', () => { + afterEach(() => { + agentChatLoader.invalidate(); + authSettingsLoader.invalidate(); + rlsLoader.invalidate(); + }); + + it('quotes arbitrary PostgreSQL identifiers and rejects truncation/NUL cases', () => { + expect(quoteQualifiedSqlIdentifier('tenant-a', 'table"name')) + .toBe('"tenant-a"."table""name"'); + expect(() => quoteSqlIdentifier('')).toThrow('Invalid SQL identifier'); + expect(() => quoteSqlIdentifier('bad\0name')).toThrow('Invalid SQL identifier'); + expect(() => quoteSqlIdentifier('a'.repeat(64))).toThrow('Invalid SQL identifier'); + }); + + it('constrains RLS schemas and functions to the requested database and schema', async () => { + const routingQuery = jest.fn().mockResolvedValue({ rows: [{ + authenticate_schema: 'auth_private', + role_schema: 'auth_public', + authenticate: 'authenticate', + authenticate_strict: 'authenticate_strict', + current_role: 'current_role', + current_role_id: 'current_role_id', + current_user_agent: 'current_user_agent', + current_ip_address: 'current_ip_address' + }] }); + await rlsLoader.resolve(context(routingQuery, jest.fn())); + + const [sql, values] = routingQuery.mock.calls[0]; + expect(values).toEqual(['11111111-1111-4111-8111-111111111111']); + expect(sql).toContain('auth_fn.database_id = rs.database_id'); + expect(sql).toContain('auth_fn.schema_id = rs.authenticate_schema_id'); + expect(sql).toContain('role_fn.schema_id = rs.role_schema_id'); + }); + + it('rejects an RLS row whose referenced metadata did not resolve exactly', async () => { + const routingQuery = jest.fn().mockResolvedValue({ rows: [{ + authenticate_schema: null, + role_schema: 'auth_public', + authenticate: null, + authenticate_strict: null, + current_role: 'current_role', + current_role_id: 'current_role_id', + current_user_agent: 'current_user_agent', + current_ip_address: 'current_ip_address' + }] }); + + await expect(rlsLoader.resolve(context(routingQuery, jest.fn()))) + .rejects.toThrow('Incomplete or cross-database RLS module configuration'); + }); + + it('scopes tenant module discovery and safely quotes discovered identifiers', async () => { + const tenantQuery = jest.fn() + .mockResolvedValueOnce({ + rows: [{ schema_name: 'session-private', table_name: 'auth"settings' }] + }) + .mockResolvedValueOnce({ rows: [{ + cookie_secure: true, + cookie_samesite: 'lax', + cookie_domain: null, + cookie_httponly: true, + cookie_max_age: null, + cookie_path: '/', + remember_me_duration: null, + enable_captcha: false, + captcha_site_key: null + }] }); + await authSettingsLoader.resolve(context(jest.fn(), tenantQuery)); + + expect(tenantQuery.mock.calls[0][0]).toContain('WHERE sm.database_id = $1'); + expect(tenantQuery.mock.calls[0][1]).toEqual([ + '11111111-1111-4111-8111-111111111111' + ]); + expect(tenantQuery.mock.calls[1][0]) + .toContain('FROM "session-private"."auth""settings"'); + }); + + it('scopes agent chat discovery to the exact logical database', async () => { + const tenantQuery = jest.fn().mockResolvedValue({ rows: [{ + schema_name: 'agent_public', + thread_table_name: 'threads', + message_table_name: 'messages', + task_table_name: 'tasks' + }] }); + await agentChatLoader.resolve(context(jest.fn(), tenantQuery)); + + expect(tenantQuery.mock.calls[0][0]).toContain('WHERE acm.database_id = $1'); + expect(tenantQuery.mock.calls[0][1]).toEqual([ + '11111111-1111-4111-8111-111111111111' + ]); + }); + + it('fails a configured billing quota check closed and quotes its function', async () => { + const query = jest.fn().mockRejectedValue(new Error('billing unavailable')); + const withPgClient = jest.fn(async (callback) => callback({ query })); + const billing = createBillingClient( + withPgClient as never, + '33333333-3333-4333-8333-333333333333', + { + publicSchema: 'billing-public', + privateSchema: 'billing"private', + recordUsageFunction: 'record_usage', + checkBillingQuotaFunction: 'check"quota' + }, + null + ); + + await expect(billing.checkQuota('tokens')).resolves.toBe(false); + expect(query.mock.calls[0][0]) + .toContain('SELECT "billing""private"."check""quota"('); + }); +}); diff --git a/packages/express-context/src/__tests__/storage-loader.test.ts b/packages/express-context/src/__tests__/storage-loader.test.ts new file mode 100644 index 0000000000..416d27791c --- /dev/null +++ b/packages/express-context/src/__tests__/storage-loader.test.ts @@ -0,0 +1,55 @@ +import type { Pool } from 'pg'; + +import { storageLoader, STORAGE_MODULE_SQL } from '../loaders/storage'; + +describe('storage control-plane loader', () => { + afterEach(() => storageLoader.invalidate()); + + it('normalizes immutable module metadata and caches it by database contract', async () => { + const query = jest.fn().mockResolvedValue({ + rows: [{ + id: 'storage-a', + scope: 'app', + entity_table_id: null, + buckets_schema: 'tenant-a', + buckets_table: 'buckets"table', + files_schema: 'tenant-a', + files_table: 'files', + endpoint: null, + public_url_prefix: null, + provider: null, + allowed_origins: null, + upload_url_expiry_seconds: null, + download_url_expiry_seconds: null, + default_max_file_size: null, + max_filename_length: null, + cache_ttl_seconds: null, + max_bulk_files: null, + max_bulk_total_size: '1073741824', + has_path_shares: null, + entity_schema: null, + entity_table: null + }] + }); + const ctx = { + routingPool: {} as Pool, + tenantPool: { query } as unknown as Pool, + databaseId: 'database-storage-loader-test', + dbname: 'tenant_db' + }; + + const first = await storageLoader.resolve(ctx); + const cached = await storageLoader.resolve(ctx); + + expect(query).toHaveBeenCalledTimes(1); + expect(query).toHaveBeenCalledWith(STORAGE_MODULE_SQL, [ctx.databaseId]); + expect(cached).toBe(first); + expect(first?.modules[0]).toMatchObject({ + bucketsQualifiedName: '"tenant-a"."buckets""table"', + filesQualifiedName: '"tenant-a"."files"', + uploadUrlExpirySeconds: 900, + maxBulkTotalSize: 1073741824, + hasPathShares: false + }); + }); +}); diff --git a/packages/express-context/src/billing-client.ts b/packages/express-context/src/billing-client.ts index ae13066707..27e00d5313 100644 --- a/packages/express-context/src/billing-client.ts +++ b/packages/express-context/src/billing-client.ts @@ -15,6 +15,7 @@ import { Logger } from '@pgpmjs/logger'; +import { quoteQualifiedSqlIdentifier } from './sql-identifiers'; import type { BillingConfig, InferenceLogConfig, WithPgClient } from './types'; const log = new Logger('billing-client'); @@ -48,7 +49,8 @@ export interface BillingClient { * Check if the entity has sufficient quota for the requested amount. * Returns true if allowed, false if quota is exceeded. * - * Gracefully returns true if billing is not provisioned or errors. + * Returns true when billing is not provisioned. Once billing is configured, + * lookup failures deny the request so quota enforcement cannot fail open. */ checkQuota(meterSlug: string, amount?: number): Promise; @@ -79,14 +81,19 @@ export function createBillingClient( try { return await withPgClient(async (client) => { - const sql = `SELECT "${billing.privateSchema}"."${billing.checkBillingQuotaFunction}"($1, $2::uuid, $3) AS allowed`; + const fn = quoteQualifiedSqlIdentifier( + billing.privateSchema, + billing.checkBillingQuotaFunction, + 'billing quota function' + ); + const sql = `SELECT ${fn}($1, $2::uuid, $3) AS allowed`; const result = await client.query(sql, [meterSlug, entityId, amount]); return result.rows[0]?.allowed !== false; }); } catch (e: unknown) { const message = e instanceof Error ? e.message : String(e); - log.warn(`check_billing_quota failed (allowing): ${message}`); - return true; + log.warn(`check_billing_quota failed (denying): ${message}`); + return false; } }, @@ -95,7 +102,12 @@ export function createBillingClient( try { await withPgClient(async (client) => { - const sql = `SELECT "${billing.privateSchema}"."${billing.recordUsageFunction}"($1, $2::uuid, $3, $4::jsonb)`; + const fn = quoteQualifiedSqlIdentifier( + billing.privateSchema, + billing.recordUsageFunction, + 'billing usage function' + ); + const sql = `SELECT ${fn}($1, $2::uuid, $3, $4::jsonb)`; await client.query(sql, [meterSlug, entityId, amount, JSON.stringify(metadata ?? {})]); }); } catch (e: unknown) { @@ -109,8 +121,13 @@ export function createBillingClient( try { await withPgClient(async (client) => { + const table = quoteQualifiedSqlIdentifier( + inferenceLog.schema, + inferenceLog.tableName, + 'inference log table' + ); await client.query( - `INSERT INTO "${inferenceLog.schema}"."${inferenceLog.tableName}" + `INSERT INTO ${table} (entity_id, actor_id, model, provider, service, operation, input_tokens, output_tokens, total_tokens, latency_ms, status, cache_read_tokens, cache_write_tokens, diff --git a/packages/express-context/src/context.ts b/packages/express-context/src/context.ts index 82d87de9d3..31ad836510 100644 --- a/packages/express-context/src/context.ts +++ b/packages/express-context/src/context.ts @@ -17,7 +17,13 @@ import type { PgpmOptions } from '@pgpmjs/types'; import type { NextFunction, Request, RequestHandler, Response } from 'express'; import type { Pool } from 'pg'; -import { getPgPool } from 'pg-cache'; +import { + acquirePgPool, + getPgPool, + getPgPoolIdentity, + type GetPgPoolOptions, + type PgPoolLease +} from 'pg-cache'; import type { BillingClient } from './billing-client'; import { createBillingClient } from './billing-client'; @@ -25,23 +31,69 @@ import type { LoaderRegistry } from './loaders/registry'; import type { LoaderContext } from './loaders/types'; import { withPgClient as withPgClientFn } from './pg-client'; import { buildPgSettings } from './pg-settings'; -import type { BillingConfig, BuiltinModuleMap, ConstructiveContext, InferenceLogConfig, LlmConfig } from './types'; +import type { ApiStructure, BillingConfig, BuiltinModuleMap, ConstructiveContext, InferenceLogConfig, LlmConfig } from './types'; + +type PoolConfig = Parameters[0]; + +/** + * Secret-bearing connection config resolved by the owning server, paired with + * the opaque identity that both request context and Graphile must consume. + */ +export interface RuntimePgPoolResolution { + pgConfig: PoolConfig; + poolIdentity: string; +} export interface ContextMiddlewareOptions { /** Base PG options for pool creation (host, port, user, password) */ pg?: PgpmOptions['pg']; + /** Least-privilege tenant execution login; inherits unspecified pg fields. */ + runtimePg?: PgpmOptions['pg']; + /** + * Read the server-owned request resolution. Implementations should keep raw + * credentials outside the Express request object (for example in a WeakMap). + */ + getRuntimePgResolution?: ( + req: Request, + api: ApiStructure + ) => Readonly; + /** Optional fail-closed admission check for the tenant execution pool. */ + validateRuntimePool?: (pool: Pool, api: ApiStructure) => Promise; + /** Ordered, audited extension/shared schemas used by request SQL. */ + dependencySchemas?: readonly string[]; /** Module loader registry for per-database cached lookups */ loaders?: LoaderRegistry; /** Routing-plane schema loaders query (defaults to routing_public) */ routingSchema?: string; } +interface ResolvedPool { + pool: Pool; + identity: string; +} + +const resolvePool = ( + config: PoolConfig, + options: GetPgPoolOptions, + leases?: PgPoolLease[] +): ResolvedPool => { + if (!leases) { + return { + pool: getPgPool(config, options), + identity: getPgPoolIdentity(config, options) + }; + } + const lease = acquirePgPool(config, options); + leases.push(lease); + return { pool: lease.pool, identity: lease.identity }; +}; + /** * Create a `useModule` function bound to the given loader context. * - * Calling `useModule('rlsModule')` lazily resolves the RLS loader, - * hitting the DB only on cache miss. The function is a no-op (returns - * undefined) when no registry is configured. + * Calling `useModule('rlsModule')` lazily resolves the RLS loader according to + * that loader's freshness policy. The function is a no-op (returns undefined) + * when no registry is configured. */ function createUseModule( registry: LoaderRegistry | undefined, @@ -65,7 +117,9 @@ function createUseModule( */ export function buildContext( req: Request, - opts: ContextMiddlewareOptions = {} + opts: ContextMiddlewareOptions = {}, + /** Internal request lifetime. Omit for backwards-compatible direct use. */ + poolLeases?: PgPoolLease[] ): ConstructiveContext | null { const api = req.api; if (!api) return null; @@ -77,30 +131,75 @@ export function buildContext( api, token, requestId, - clientIp: req.clientIp + clientIp: req.clientIp, + origin: req.get('origin'), + userAgent: req.get('User-Agent'), + deviceToken: req.deviceToken, + dependencySchemas: opts.dependencySchemas }); - const tenantPool: Pool = getPgPool({ + const suppliedRuntimeResolution = opts.getRuntimePgResolution + ? opts.getRuntimePgResolution(req, api) + : undefined; + if (opts.getRuntimePgResolution && !suppliedRuntimeResolution) { + throw new Error( + 'Runtime PostgreSQL resolution provider returned no exact identity' + ); + } + const runtimeConfig = suppliedRuntimeResolution?.pgConfig ?? { ...opts.pg, + ...opts.runtimePg, database: api.dbname - }); + }; + const runtimePool = resolvePool( + runtimeConfig, + { purpose: 'runtime', sanitizeOnCheckout: true }, + poolLeases + ); + if ( + suppliedRuntimeResolution + && runtimePool.identity !== suppliedRuntimeResolution.poolIdentity + ) { + throw new Error( + 'Resolved runtime PostgreSQL pool identity changed before context acquisition' + ); + } + const tenantPool = runtimePool.pool; // Build loader context (if registry provided and databaseId known) let loaderCtx: LoaderContext | null = null; if (opts.loaders && api.databaseId) { - const routingPool: Pool = getPgPool(opts.pg); + const routingPool = resolvePool(opts.pg ?? {}, { + purpose: 'routing-request-control', + sanitizeOnCheckout: true + }, poolLeases); + const controlTenantPool = resolvePool({ + ...opts.pg, + database: api.dbname + }, { + purpose: 'tenant-request-control', + sanitizeOnCheckout: true + }, poolLeases); loaderCtx = { - routingPool, + routingPool: routingPool.pool, + routingPoolIdentity: routingPool.identity, routingSchema: opts.routingSchema, - tenantPool, + tenantPool: controlTenantPool.pool, + tenantPoolIdentity: controlTenantPool.identity, databaseId: api.databaseId, apiId: api.apiId, dbname: api.dbname }; } + let runtimeSafetyPromise: Promise | null = null; + const ensureRuntimePoolIsSafe = (): Promise => { + if (!opts.validateRuntimePool) return Promise.resolve(); + runtimeSafetyPromise ??= opts.validateRuntimePool(tenantPool, api); + return runtimeSafetyPromise; + }; const withPgClient = (fn: (client: any) => Promise) => - withPgClientFn(tenantPool, pgSettings, fn); + ensureRuntimePoolIsSafe().then(() => withPgClientFn(tenantPool, pgSettings, fn)); const useModule = createUseModule(opts.loaders, loaderCtx); // Lazy-initialized billing client (cached per request) @@ -116,6 +215,7 @@ export function buildContext( userId: token?.user_id ?? null, requestId, pool: tenantPool, + runtimePoolIdentity: runtimePool.identity, withPgClient, useModule, async useBilling() { @@ -172,8 +272,8 @@ export function buildContext( * // Downstream middleware/routes call useModule on demand: * app.post('/v1/chat', async (req, res) => { * const ctx = req.constructive; - * const rls = await ctx.useModule('rlsModule'); // only fires if not cached - * const auth = await ctx.useModule('authSettings'); // only fires if not cached + * const rls = await ctx.useModule('rlsModule'); // authoritative read + * const auth = await ctx.useModule('authSettings'); // authoritative read * // webauthnSettings loader never fires if nobody asks for it * }); * ``` @@ -181,11 +281,48 @@ export function buildContext( export function createContextMiddleware( opts: ContextMiddlewareOptions = {} ): RequestHandler { - return (req: Request, _res: Response, next: NextFunction): void => { - const ctx = buildContext(req, opts); - if (ctx) { + return (req: Request, res: Response, next: NextFunction): void => { + const requestEnded = (): boolean => + Boolean( + req.aborted + || req.socket?.destroyed + || res.destroyed + || res.writableEnded + ); + if (requestEnded()) return; + + const leases: PgPoolLease[] = []; + let released = false; + const releaseLeases = (): void => { + if (released) return; + released = true; + req.removeListener('aborted', releaseLeases); + res.removeListener('finish', releaseLeases); + res.removeListener('close', releaseLeases); + for (const lease of leases.reverse()) lease.release(); + }; + + try { + const ctx = buildContext(req, opts, leases); + if (!ctx) { + releaseLeases(); + next(); + return; + } req.constructive = ctx; + req.once('aborted', releaseLeases); + res.once('finish', releaseLeases); + res.once('close', releaseLeases); + // The response may have ended while the synchronous context builder was + // acquiring its pool leases, before these listeners could be attached. + if (requestEnded()) { + releaseLeases(); + return; + } + next(); + } catch (error) { + releaseLeases(); + next(error); } - next(); }; } diff --git a/packages/express-context/src/index.ts b/packages/express-context/src/index.ts index 013e195f5d..baa4a5a5fb 100644 --- a/packages/express-context/src/index.ts +++ b/packages/express-context/src/index.ts @@ -9,7 +9,7 @@ * - withPgClient (tenant-scoped RLS transaction helper) * - requestId middleware (UUID correlation ID) * - Context middleware (composes all of the above into req.constructive) - * - Module loaders (pluggable per-database cached lookups) + * - Module loaders (pluggable authoritative or hard-TTL lookups) * * @example * ```typescript @@ -28,8 +28,8 @@ * * app.post('/v1/chat', async (req, res) => { * const ctx = req.constructive; - * const rls = await ctx.useModule('rlsModule'); // only fires if not cached - * const auth = await ctx.useModule('authSettings'); // only fires if not cached + * const rls = await ctx.useModule('rlsModule'); // authoritative read + * const auth = await ctx.useModule('authSettings'); // authoritative read * // webauthnSettings loader never fires if nobody asks for it * }); * ``` @@ -45,6 +45,7 @@ export type { AuthSurface, BillingConfig, BuiltinModuleMap, + ComputeBindingConfig, ComputeConfig, ComputeModuleConfig, ConstructiveAPIToken, @@ -56,6 +57,8 @@ export type { LlmConfig, PubkeyChallengeSettings, RlsModule, + StorageConfig, + StorageModuleConfig, WebauthnSettings, WithPgClient, } from './types'; @@ -65,8 +68,15 @@ export type { BillingClient, InferenceLogEntry } from './billing-client'; export { createBillingClient } from './billing-client'; // pgSettings builder -export type { PgSettingsInput } from './pg-settings'; -export { buildPgSettings } from './pg-settings'; +export type { PgSettingsInput, SecurityGucKey } from './pg-settings'; +export { buildPgSettings, SECURITY_GUC_KEYS } from './pg-settings'; + +// Safe interpolation for trusted metadata identifiers. Request values still +// belong in query parameters. +export { + quoteQualifiedSqlIdentifier, + quoteSqlIdentifier +} from './sql-identifiers'; // withPgClient helper export { withPgClient } from './pg-client'; @@ -75,7 +85,10 @@ export { withPgClient } from './pg-client'; export { requestIdMiddleware } from './request-id'; // Context middleware -export type { ContextMiddlewareOptions } from './context'; +export type { + ContextMiddlewareOptions, + RuntimePgPoolResolution +} from './context'; export { buildContext, createContextMiddleware } from './context'; // Module loaders @@ -103,6 +116,7 @@ export { requireDatabaseId, requireIdentityProvider, rlsLoader, + storageLoader, webauthnLoader, } from './loaders'; diff --git a/packages/express-context/src/loaders/agent-chat.ts b/packages/express-context/src/loaders/agent-chat.ts index 7138dee58f..b41618b869 100644 --- a/packages/express-context/src/loaders/agent-chat.ts +++ b/packages/express-context/src/loaders/agent-chat.ts @@ -23,9 +23,10 @@ const AGENT_CHAT_MODULE_SQL = ` acm.message_table_name, acm.task_table_name FROM metaschema_modules_public.agent_chat_module acm - JOIN metaschema_public.schema s ON s.id = acm.schema_id + JOIN metaschema_public.schema s + ON s.id = acm.schema_id + AND s.database_id = acm.database_id WHERE acm.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -50,6 +51,9 @@ export const agentChatLoader: ModuleLoader = createModuleLoader AGENT_CHAT_MODULE_SQL, [databaseId], ); + if (result.rows.length > 1) { + throw new Error('Ambiguous agent chat module configuration'); + } const row = result.rows[0]; if (!row) return undefined; diff --git a/packages/express-context/src/loaders/auth-settings.ts b/packages/express-context/src/loaders/auth-settings.ts index e5734cc603..f03f7b8424 100644 --- a/packages/express-context/src/loaders/auth-settings.ts +++ b/packages/express-context/src/loaders/auth-settings.ts @@ -16,6 +16,7 @@ * makes that resolution sticky for its TTL. */ +import { quoteQualifiedSqlIdentifier } from '../sql-identifiers'; import type { AuthSettings } from '../types'; import { createModuleLoader } from './create-loader'; import type { LoaderContext, ModuleLoader } from './types'; @@ -26,9 +27,10 @@ import { requireDatabaseId } from './types'; const AUTH_SETTINGS_DISCOVERY_SQL = ` SELECT s.schema_name, sm.auth_settings_table_name AS table_name FROM metaschema_modules_public.sessions_module sm - JOIN metaschema_public.schema s ON s.id = sm.schema_id + JOIN metaschema_public.schema s + ON s.id = sm.schema_id + AND s.database_id = sm.database_id WHERE sm.database_id = $1 - LIMIT 1 `; const buildAuthSettingsQuery = (schemaName: string, tableName: string) => ` @@ -42,7 +44,7 @@ const buildAuthSettingsQuery = (schemaName: string, tableName: string) => ` remember_me_duration, enable_captcha, captcha_site_key - FROM "${schemaName}"."${tableName}" + FROM ${quoteQualifiedSqlIdentifier(schemaName, tableName, 'auth settings table')} LIMIT 1 `; @@ -64,7 +66,9 @@ interface AuthSettingsRow { export const authSettingsLoader: ModuleLoader = createModuleLoader({ name: 'authSettings', - ttlMs: 5 * 60_000, + // Cookie and CAPTCHA policy changes must take effect on the next request, + // independently of lossy LISTEN delivery. + cache: false, async resolve(ctx: LoaderContext) { const { tenantPool, databaseId } = ctx; requireDatabaseId(databaseId, 'authSettings'); @@ -74,6 +78,9 @@ export const authSettingsLoader: ModuleLoader = createModuleLoader AUTH_SETTINGS_DISCOVERY_SQL, [databaseId] ); + if (discovery.rows.length > 1) { + throw new Error('Ambiguous sessions module configuration'); + } const resolved = discovery.rows[0]; if (!resolved) return undefined; diff --git a/packages/express-context/src/loaders/billing.ts b/packages/express-context/src/loaders/billing.ts index 3e743499fb..02054bc092 100644 --- a/packages/express-context/src/loaders/billing.ts +++ b/packages/express-context/src/loaders/billing.ts @@ -17,10 +17,13 @@ const BILLING_MODULE_SQL = ` ps.schema_name AS private_schema, bm.record_usage_function FROM metaschema_modules_public.billing_module bm - JOIN metaschema_public.schema s ON bm.schema_id = s.id - JOIN metaschema_public.schema ps ON bm.private_schema_id = ps.id + JOIN metaschema_public.schema s + ON bm.schema_id = s.id + AND s.database_id = bm.database_id + JOIN metaschema_public.schema ps + ON bm.private_schema_id = ps.id + AND ps.database_id = bm.database_id WHERE bm.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -43,8 +46,14 @@ export const billingLoader: ModuleLoader = createModuleLoader 1) { + throw new Error('Ambiguous billing module configuration'); + } const row = result.rows[0]; - if (!row?.record_usage_function) return undefined; + if (!row) return undefined; + if (!row.public_schema || !row.private_schema || !row.record_usage_function) { + throw new Error('Incomplete or cross-database billing module configuration'); + } return { publicSchema: row.public_schema, diff --git a/packages/express-context/src/loaders/compute.ts b/packages/express-context/src/loaders/compute.ts index b9b52ca013..4c65dc6ace 100644 --- a/packages/express-context/src/loaders/compute.ts +++ b/packages/express-context/src/loaders/compute.ts @@ -12,9 +12,10 @@ * the underlying tables governs access. */ -import type { ComputeConfig } from '../types'; -import { createModuleLoader } from './create-loader'; +import type { ComputeBindingConfig, ComputeConfig, ComputeModuleConfig } from '../types'; +import { quoteQualifiedSqlIdentifier } from '../sql-identifiers'; import type { LoaderContext, ModuleLoader } from './types'; +import { createModuleLoader } from './create-loader'; // ─── SQL ──────────────────────────────────────────────────────────────────── @@ -27,10 +28,14 @@ const COMPUTE_MODULE_SQL = ` ivm.invocations_table_name, ivm.entity_field AS invocations_entity_field FROM metaschema_modules_public.function_module fm - JOIN metaschema_public.schema fs ON fs.id = fm.schema_id + JOIN metaschema_public.schema fs + ON fs.id = fm.schema_id + AND fs.database_id = fm.database_id JOIN metaschema_modules_public.function_invocation_module ivm ON ivm.database_id = fm.database_id AND ivm.scope = fm.scope - JOIN metaschema_public.schema ivs ON ivs.id = ivm.schema_id + JOIN metaschema_public.schema ivs + ON ivs.id = ivm.schema_id + AND ivs.database_id = ivm.database_id WHERE fm.database_id = $1 ORDER BY fs.schema_name `; @@ -46,6 +51,32 @@ interface ComputeModuleRow { invocations_entity_field: string | null; } +interface ComputeBindingRow { + id: string; + alias: string; + config: Record | null; + function_definition_id: string; + task_identifier: string; + description: string | null; + payload_args: ComputeBindingConfig['payloadArgs']; +} + +const bindingSql = (module: ComputeModuleConfig): string => ` + SELECT + b.id, + b.alias, + b.config, + b.function_definition_id, + d.task_identifier, + d.description, + d.payload_args + FROM ${quoteQualifiedSqlIdentifier(module.schemaName, module.bindingsTableName, 'compute bindings table')} b + JOIN ${quoteQualifiedSqlIdentifier(module.schemaName, module.definitionsTableName, 'compute definitions table')} d + ON d.id = b.function_definition_id + WHERE b.api_id = $1 + ORDER BY b.alias +`; + // ─── Loader ───────────────────────────────────────────────────────────────── export const computeLoader: ModuleLoader = createModuleLoader({ @@ -60,8 +91,7 @@ export const computeLoader: ModuleLoader = createModuleLoader ({ + const modules: ComputeModuleConfig[] = result.rows.map((row) => ({ schemaName: row.functions_schema_name, definitionsTableName: row.definitions_table_name, // Physical bindings table name, recorded by the metaschema generator @@ -72,8 +102,27 @@ export const computeLoader: ModuleLoader = createModuleLoader { + const bindingResult = await tenantPool.query( + bindingSql(module), + [ctx.apiId] + ); + return bindingResult.rows.map((row): ComputeBindingConfig => ({ + bindingId: row.id, + alias: row.alias, + config: row.config, + functionDefinitionId: row.function_definition_id, + taskIdentifier: row.task_identifier, + description: row.description, + payloadArgs: row.payload_args, + module + })); + }))).flat() + : []; + + return { modules, bindings }; }, }); diff --git a/packages/express-context/src/loaders/cors.ts b/packages/express-context/src/loaders/cors.ts index 2caa417bc4..219fb163ca 100644 --- a/packages/express-context/src/loaders/cors.ts +++ b/packages/express-context/src/loaders/cors.ts @@ -36,7 +36,8 @@ interface CorsSettingsRow { export const corsLoader: ModuleLoader = createModuleLoader({ name: 'corsOrigins', - ttlMs: 5 * 60_000, + // Revoking an allowed browser/WebSocket origin is a security policy change. + cache: false, async resolve(ctx: LoaderContext) { const { routingPool, databaseId, apiId } = ctx; const schema = routingSchemaOf(ctx); diff --git a/packages/express-context/src/loaders/create-loader.ts b/packages/express-context/src/loaders/create-loader.ts index 25aabd333e..2f0e191ea7 100644 --- a/packages/express-context/src/loaders/create-loader.ts +++ b/packages/express-context/src/loaders/create-loader.ts @@ -1,85 +1,205 @@ /** - * create-loader — Factory for building cached ModuleLoader instances. + * create-loader — Factory for building ModuleLoader instances. * - * Wraps a raw resolve function with an LRU cache keyed by databaseId:apiId. - * Each loader gets its own independent cache with configurable TTL and - * max entries. + * Optionally wraps a raw resolve function with an LRU cache keyed by the exact + * routing and tenant pool identities plus databaseId:apiId. Each cached loader + * gets its own independent hard TTL and max entries. */ import { Logger } from '@pgpmjs/logger'; import { LRUCache } from 'lru-cache'; -import type { LoaderContext, ModuleLoader } from './types'; +import { + type LoaderContext, + type ModuleLoader, + routingSchemaOf +} from './types'; export interface CreateLoaderOptions { /** Unique loader name (used in log prefix and modules map key) */ name: string; + /** + * Whether successful/absent results may be shared across requests. + * Security-boundary configuration should set this to false so every + * request observes an authoritative database read. + */ + cache?: boolean; /** TTL in milliseconds (default: 60_000 — 1 minute) */ ttlMs?: number; - /** Max cache entries before LRU eviction (default: 100) */ + /** Max cache entries before LRU eviction (default: 1024) */ max?: number; /** The actual resolution function. Called on cache miss. */ resolve: (ctx: LoaderContext) => Promise; } const DEFAULT_TTL_MS = 60_000; -const DEFAULT_MAX = 100; +// Match the Graphile instance governor's hard ceiling. A smaller hidden +// metadata cache would thrash control-plane queries long before heap pressure +// requires evicting the corresponding resident tenant handlers. +const DEFAULT_MAX = 1024; + +let nextPoolObjectIdentity = 0; +const poolObjectIdentities = new WeakMap(); + +const poolIdentity = (pool: object, explicitIdentity?: string): string => { + if (explicitIdentity?.trim()) return explicitIdentity; + let identity = poolObjectIdentities.get(pool); + if (!identity) { + identity = `pool-object:${++nextPoolObjectIdentity}`; + poolObjectIdentities.set(pool, identity); + } + return identity; +}; + +interface LoaderCacheContract { + databaseId: string; + routingSchema: string; + routingPoolIdentity: string; + tenantPoolIdentity: string; +} + +const cacheContract = (ctx: LoaderContext): LoaderCacheContract => ({ + databaseId: ctx.databaseId, + routingSchema: routingSchemaOf(ctx), + routingPoolIdentity: poolIdentity(ctx.routingPool, ctx.routingPoolIdentity), + tenantPoolIdentity: poolIdentity(ctx.tenantPool, ctx.tenantPoolIdentity) +}); + +const cacheKey = (ctx: LoaderContext, contract: LoaderCacheContract): string => + JSON.stringify([ + contract.routingPoolIdentity, + contract.tenantPoolIdentity, + contract.routingSchema, + contract.databaseId, + ctx.apiId ?? null + ]); + +interface LoaderCacheEntry { + contract: LoaderCacheContract; + value: T | undefined; +} + +interface PendingResolution { + contract: LoaderCacheContract; + invalidated: boolean; + promise: Promise; +} + +const samePhysicalContract = ( + left: LoaderCacheContract, + right: LoaderCacheContract +): boolean => + left.routingPoolIdentity === right.routingPoolIdentity + && left.tenantPoolIdentity === right.tenantPoolIdentity + && left.routingSchema === right.routingSchema; export function createModuleLoader(opts: CreateLoaderOptions): ModuleLoader { const log = new Logger(`loader:${opts.name}`); - const cache = new LRUCache({ + const cacheEnabled = opts.cache !== false; + const cache = new LRUCache>({ max: opts.max ?? DEFAULT_MAX, ttl: opts.ttlMs ?? DEFAULT_TTL_MS, - updateAgeOnGet: true, - allowStale: false, + ttlResolution: 0, + // A hit must never extend configuration lifetime indefinitely. This is a + // hard maximum staleness bound for non-security-sensitive module data. + updateAgeOnGet: false, + allowStale: false }); + const pending = new Map>(); return { name: opts.name, async resolve(ctx: LoaderContext): Promise { - const key = ctx.apiId ? `${ctx.databaseId}:${ctx.apiId}` : ctx.databaseId; + const logicalKey = ctx.apiId + ? `${ctx.databaseId}:${ctx.apiId}` + : ctx.databaseId; + + if (!cacheEnabled) { + log.debug(`Authoritative resolve databaseId=${logicalKey}`); + try { + return await opts.resolve(ctx); + } catch (e: any) { + if (e.code === '42P01') { + log.debug(`Module tables absent for databaseId=${logicalKey}: ${e.message}`); + return undefined; + } + log.warn(`Failed to resolve databaseId=${logicalKey}: ${e.message}`); + throw e; + } + } - if (cache.has(key)) { - log.debug(`Cache HIT databaseId=${key}`); - return cache.get(key); + const contract = cacheContract(ctx); + const key = cacheKey(ctx, contract); + const cached = cache.get(key); + if (cached !== undefined) { + log.debug(`Cache HIT databaseId=${logicalKey}`); + return cached.value; } - log.debug(`Cache MISS databaseId=${key}, resolving`); + const existing = pending.get(key); + if (existing && !existing.invalidated) { + log.debug(`Cache COALESCE databaseId=${logicalKey}`); + return existing.promise; + } + + log.debug(`Cache MISS databaseId=${logicalKey}, resolving`); // "Not provisioned" is expressed by the loader returning undefined, or // by the module's tables not existing at all (42P01 undefined_table). // Any other resolution error (bad query, ambiguous config) propagates — // never silently coerced into "module absent". - try { - const value = await opts.resolve(ctx); - cache.set(key, value); - return value; - } catch (e: any) { - if (e.code === '42P01') { - log.debug(`Module tables absent for databaseId=${key}: ${e.message}`); - cache.set(key, undefined); - return undefined; + const resolution: PendingResolution = { + contract, + invalidated: false, + promise: undefined as unknown as Promise + }; + resolution.promise = Promise.resolve().then(async () => { + try { + const value = await opts.resolve(ctx); + if (!resolution.invalidated) cache.set(key, { contract, value }); + return value; + } catch (e: any) { + if (e.code === '42P01') { + log.debug(`Module tables absent for databaseId=${logicalKey}: ${e.message}`); + if (!resolution.invalidated) { + cache.set(key, { contract, value: undefined }); + } + return undefined; + } + log.warn(`Failed to resolve databaseId=${logicalKey}: ${e.message}`); + throw e; + } finally { + if (pending.get(key) === resolution) pending.delete(key); } - log.warn(`Failed to resolve databaseId=${key}: ${e.message}`); - throw e; - } + }); + pending.set(key, resolution); + return resolution.promise; }, - invalidate(databaseId?: string): void { - if (databaseId) { - // Clear the plain databaseId key and any composite databaseId:apiId keys - let cleared = 0; - for (const k of cache.keys()) { - if (k === databaseId || k.startsWith(`${databaseId}:`)) { - cache.delete(k); - cleared++; - } - } - log.debug(`Invalidated ${cleared} entries for databaseId=${databaseId}`); - } else { + invalidate(databaseId?: string, context?: LoaderContext): void { + if (!databaseId && !context) { + const previousSize = cache.size; cache.clear(); - log.debug(`Invalidated all entries (was size=${cache.size})`); + for (const resolution of pending.values()) resolution.invalidated = true; + log.debug(`Invalidated all entries (was size=${previousSize})`); + return; + } + + const exact = context ? cacheContract(context) : null; + const matches = (contract: LoaderCacheContract): boolean => + (!databaseId || contract.databaseId === databaseId) + && (!exact || samePhysicalContract(contract, exact)); + let cleared = 0; + for (const [key, entry] of cache.entries()) { + if (!matches(entry.contract)) continue; + if (cache.delete(key)) cleared++; + } + for (const resolution of pending.values()) { + if (matches(resolution.contract)) resolution.invalidated = true; } + log.debug( + `Invalidated ${cleared} entries${databaseId ? ` for databaseId=${databaseId}` : ''}` + ); }, get cacheSize(): number { diff --git a/packages/express-context/src/loaders/database-settings.ts b/packages/express-context/src/loaders/database-settings.ts index aa7232f4fb..b10e318342 100644 --- a/packages/express-context/src/loaders/database-settings.ts +++ b/packages/express-context/src/loaders/database-settings.ts @@ -30,7 +30,6 @@ const databaseSettingsSql = (schema: string): string => ` FROM "${schema}".database_settings ds LEFT JOIN "${schema}".api_settings aps ON ds.database_id = aps.database_id AND aps.api_id = $2 WHERE ds.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -50,11 +49,28 @@ interface DatabaseSettingsRow { resolved_enable_i18n: boolean; } +const BOOLEAN_COLUMNS: readonly (keyof DatabaseSettingsRow)[] = [ + 'resolved_enable_aggregates', + 'resolved_enable_postgis', + 'resolved_enable_search', + 'resolved_enable_direct_uploads', + 'resolved_enable_presigned_uploads', + 'resolved_enable_many_to_many', + 'resolved_enable_connection_filter', + 'resolved_enable_ltree', + 'resolved_enable_llm', + 'resolved_enable_realtime', + 'resolved_enable_bulk', + 'resolved_enable_i18n' +]; + // ─── Loader ───────────────────────────────────────────────────────────────── export const databaseSettingsLoader: ModuleLoader = createModuleLoader({ name: 'databaseSettings', - ttlMs: 5 * 60_000, + // These flags select the executable Graphile/plugin surface and realtime + // admission, so a disable/revocation must alter the next build contract. + cache: false, async resolve(ctx: LoaderContext) { const { routingPool, databaseId, apiId } = ctx; @@ -62,8 +78,18 @@ export const databaseSettingsLoader: ModuleLoader = createModu databaseSettingsSql(routingSchemaOf(ctx)), [databaseId, apiId ?? null] ); + if (result.rows.length > 1) { + throw new Error( + `Ambiguous database feature configuration for database ${databaseId}` + ); + } const row = result.rows[0]; if (!row) return undefined; + if (BOOLEAN_COLUMNS.some((column) => typeof row[column] !== 'boolean')) { + throw new Error( + `Incomplete database feature configuration for database ${databaseId}` + ); + } return { enableAggregates: row.resolved_enable_aggregates, diff --git a/packages/express-context/src/loaders/index.ts b/packages/express-context/src/loaders/index.ts index a8a3203428..dfe045501d 100644 --- a/packages/express-context/src/loaders/index.ts +++ b/packages/express-context/src/loaders/index.ts @@ -55,6 +55,7 @@ export { inferenceLogLoader } from './inference-log'; export { llmLoader } from './llm'; export { pubkeyLoader } from './pubkey'; export { rlsLoader } from './rls'; +export { storageLoader } from './storage'; export { webauthnLoader } from './webauthn'; /** @@ -72,6 +73,7 @@ import { llmLoader } from './llm'; import { pubkeyLoader } from './pubkey'; import { createLoaderRegistry } from './registry'; import { rlsLoader } from './rls'; +import { storageLoader } from './storage'; import { webauthnLoader } from './webauthn'; export function createDefaultRegistry() { @@ -88,5 +90,6 @@ export function createDefaultRegistry() { registry.register(agentChatLoader); registry.register(llmLoader); registry.register(computeLoader); + registry.register(storageLoader); return registry; } diff --git a/packages/express-context/src/loaders/inference-log.ts b/packages/express-context/src/loaders/inference-log.ts index 2488792460..8d419c5754 100644 --- a/packages/express-context/src/loaders/inference-log.ts +++ b/packages/express-context/src/loaders/inference-log.ts @@ -16,9 +16,10 @@ const INFERENCE_LOG_MODULE_SQL = ` s.schema_name AS schema, ilm.inference_log_table_name AS table_name FROM metaschema_modules_public.inference_log_module ilm - JOIN metaschema_public.schema s ON ilm.schema_id = s.id + JOIN metaschema_public.schema s + ON ilm.schema_id = s.id + AND s.database_id = ilm.database_id WHERE ilm.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -40,8 +41,14 @@ export const inferenceLogLoader: ModuleLoader = createModule INFERENCE_LOG_MODULE_SQL, [databaseId], ); + if (result.rows.length > 1) { + throw new Error('Ambiguous inference-log module configuration'); + } const row = result.rows[0]; - if (!row?.schema || !row?.table_name) return undefined; + if (!row) return undefined; + if (!row.schema || !row.table_name) { + throw new Error('Incomplete or cross-database inference-log configuration'); + } return { schema: row.schema, diff --git a/packages/express-context/src/loaders/llm.ts b/packages/express-context/src/loaders/llm.ts index 33cb28f51d..8ded558f4c 100644 --- a/packages/express-context/src/loaders/llm.ts +++ b/packages/express-context/src/loaders/llm.ts @@ -28,7 +28,6 @@ const LLM_MODULE_SQL = ` lm.rag_context_limit FROM metaschema_modules_public.llm_module lm WHERE lm.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -58,6 +57,9 @@ export const llmLoader: ModuleLoader = createModuleLoader( LLM_MODULE_SQL, [databaseId], ); + if (result.rows.length > 1) { + throw new Error('Ambiguous LLM module configuration'); + } const row = result.rows[0]; if (!row) return undefined; diff --git a/packages/express-context/src/loaders/pubkey.ts b/packages/express-context/src/loaders/pubkey.ts index 9ff560d103..de1db8661c 100644 --- a/packages/express-context/src/loaders/pubkey.ts +++ b/packages/express-context/src/loaders/pubkey.ts @@ -22,13 +22,26 @@ const pubkeySettingsSql = (schema: string): string => ` sign_in_fail_fn.name AS sign_in_record_failure, sign_in_fn.name AS sign_in_with_challenge FROM "${schema}".pubkey_settings ps - LEFT JOIN metaschema_public.schema s ON ps.schema_id = s.id - LEFT JOIN metaschema_public.function sign_up_fn ON ps.sign_up_with_key_function_id = sign_up_fn.id - LEFT JOIN metaschema_public.function sign_in_req_fn ON ps.sign_in_request_challenge_function_id = sign_in_req_fn.id - LEFT JOIN metaschema_public.function sign_in_fail_fn ON ps.sign_in_record_failure_function_id = sign_in_fail_fn.id - LEFT JOIN metaschema_public.function sign_in_fn ON ps.sign_in_with_challenge_function_id = sign_in_fn.id + LEFT JOIN metaschema_public.schema s + ON ps.schema_id = s.id + AND s.database_id = ps.database_id + LEFT JOIN metaschema_public.function sign_up_fn + ON ps.sign_up_with_key_function_id = sign_up_fn.id + AND sign_up_fn.database_id = ps.database_id + AND sign_up_fn.schema_id = ps.schema_id + LEFT JOIN metaschema_public.function sign_in_req_fn + ON ps.sign_in_request_challenge_function_id = sign_in_req_fn.id + AND sign_in_req_fn.database_id = ps.database_id + AND sign_in_req_fn.schema_id = ps.schema_id + LEFT JOIN metaschema_public.function sign_in_fail_fn + ON ps.sign_in_record_failure_function_id = sign_in_fail_fn.id + AND sign_in_fail_fn.database_id = ps.database_id + AND sign_in_fail_fn.schema_id = ps.schema_id + LEFT JOIN metaschema_public.function sign_in_fn + ON ps.sign_in_with_challenge_function_id = sign_in_fn.id + AND sign_in_fn.database_id = ps.database_id + AND sign_in_fn.schema_id = ps.schema_id WHERE ps.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -45,7 +58,18 @@ interface PubkeySettingsRow { // ─── Transforms ───────────────────────────────────────────────────────────── function fromRow(row: PubkeySettingsRow | null): PubkeyChallengeSettings | undefined { - if (!row?.schema || !row?.sign_up_with_key) return undefined; + if (!row) return undefined; + const required = [ + row.schema, + row.crypto_network, + row.sign_up_with_key, + row.sign_in_request_challenge, + row.sign_in_record_failure, + row.sign_in_with_challenge + ]; + if (required.some((value) => typeof value !== 'string' || value.length === 0)) { + throw new Error('Incomplete or cross-database public-key authentication configuration'); + } return { schema: row.schema, cryptoNetwork: row.crypto_network, @@ -60,10 +84,14 @@ function fromRow(row: PubkeySettingsRow | null): PubkeyChallengeSettings | undef export const pubkeyLoader: ModuleLoader = createModuleLoader({ name: 'pubkeyChallengeSettings', - ttlMs: 5 * 60_000, + // Public-key authentication policy must be authoritative per request. + cache: false, async resolve(ctx: LoaderContext) { const { routingPool, databaseId } = ctx; const result = await routingPool.query(pubkeySettingsSql(routingSchemaOf(ctx)), [databaseId]); + if (result.rows.length > 1) { + throw new Error('Ambiguous public-key authentication configuration'); + } return fromRow(result.rows[0] ?? null); } }); diff --git a/packages/express-context/src/loaders/registry.ts b/packages/express-context/src/loaders/registry.ts index d4d8701eff..c632cc6c81 100644 --- a/packages/express-context/src/loaders/registry.ts +++ b/packages/express-context/src/loaders/registry.ts @@ -40,8 +40,8 @@ export interface LoaderRegistry { /** Check whether a loader is registered. */ has(name: string): boolean; - /** Invalidate caches for one database (or all databases if omitted). */ - invalidate(databaseId?: string): void; + /** Invalidate caches for one database, optionally limited to an exact pool pair. */ + invalidate(databaseId?: string, context?: LoaderContext): void; /** List all registered loader names. */ readonly names: string[]; @@ -96,9 +96,9 @@ export function createLoaderRegistry(): LoaderRegistry { return loaders.has(name); }, - invalidate(databaseId?: string): void { + invalidate(databaseId?: string, context?: LoaderContext): void { for (const loader of loaders.values()) { - loader.invalidate(databaseId); + loader.invalidate(databaseId, context); } log.debug( databaseId diff --git a/packages/express-context/src/loaders/rls.ts b/packages/express-context/src/loaders/rls.ts index 6fa7aa4425..c4a1e08c2a 100644 --- a/packages/express-context/src/loaders/rls.ts +++ b/packages/express-context/src/loaders/rls.ts @@ -24,16 +24,37 @@ const rlsSettingsSql = (schema: string): string => ` ua_fn.name AS current_user_agent, ip_fn.name AS current_ip_address FROM "${schema}".rls_settings rs - LEFT JOIN metaschema_public.schema auth_schema ON rs.authenticate_schema_id = auth_schema.id - LEFT JOIN metaschema_public.schema role_schema ON rs.role_schema_id = role_schema.id - LEFT JOIN metaschema_public.function auth_fn ON rs.authenticate_function_id = auth_fn.id - LEFT JOIN metaschema_public.function auth_strict_fn ON rs.authenticate_strict_function_id = auth_strict_fn.id - LEFT JOIN metaschema_public.function role_fn ON rs.current_role_function_id = role_fn.id - LEFT JOIN metaschema_public.function role_id_fn ON rs.current_role_id_function_id = role_id_fn.id - LEFT JOIN metaschema_public.function ua_fn ON rs.current_user_agent_function_id = ua_fn.id - LEFT JOIN metaschema_public.function ip_fn ON rs.current_ip_address_function_id = ip_fn.id + LEFT JOIN metaschema_public.schema auth_schema + ON rs.authenticate_schema_id = auth_schema.id + AND auth_schema.database_id = rs.database_id + LEFT JOIN metaschema_public.schema role_schema + ON rs.role_schema_id = role_schema.id + AND role_schema.database_id = rs.database_id + LEFT JOIN metaschema_public.function auth_fn + ON rs.authenticate_function_id = auth_fn.id + AND auth_fn.database_id = rs.database_id + AND auth_fn.schema_id = rs.authenticate_schema_id + LEFT JOIN metaschema_public.function auth_strict_fn + ON rs.authenticate_strict_function_id = auth_strict_fn.id + AND auth_strict_fn.database_id = rs.database_id + AND auth_strict_fn.schema_id = rs.authenticate_schema_id + LEFT JOIN metaschema_public.function role_fn + ON rs.current_role_function_id = role_fn.id + AND role_fn.database_id = rs.database_id + AND role_fn.schema_id = rs.role_schema_id + LEFT JOIN metaschema_public.function role_id_fn + ON rs.current_role_id_function_id = role_id_fn.id + AND role_id_fn.database_id = rs.database_id + AND role_id_fn.schema_id = rs.role_schema_id + LEFT JOIN metaschema_public.function ua_fn + ON rs.current_user_agent_function_id = ua_fn.id + AND ua_fn.database_id = rs.database_id + AND ua_fn.schema_id = rs.role_schema_id + LEFT JOIN metaschema_public.function ip_fn + ON rs.current_ip_address_function_id = ip_fn.id + AND ip_fn.database_id = rs.database_id + AND ip_fn.schema_id = rs.role_schema_id WHERE rs.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -53,7 +74,18 @@ interface RlsSettingsRow { function fromSettings(row: RlsSettingsRow | null): RlsModule | undefined { if (!row) return undefined; - if (!row.authenticate || !row.authenticate_schema) return undefined; + const required = [ + row.authenticate, + row.authenticate_schema, + row.role_schema, + row.current_role, + row.current_role_id, + row.current_ip_address, + row.current_user_agent + ]; + if (required.some((value) => typeof value !== 'string' || value.length === 0)) { + throw new Error('Incomplete or cross-database RLS module configuration'); + } return { authenticate: row.authenticate, authenticateStrict: row.authenticate_strict, @@ -70,10 +102,16 @@ function fromSettings(row: RlsSettingsRow | null): RlsModule | undefined { export const rlsLoader: ModuleLoader = createModuleLoader({ name: 'rlsModule', - ttlMs: 5 * 60_000, + // Authentication routing is an authorization boundary. Resolve it from the + // routing plane on every request; LISTEN notifications and TTLs are not an + // acceptable revocation mechanism because notifications can be missed. + cache: false, async resolve(ctx: LoaderContext) { const { routingPool, databaseId } = ctx; const result = await routingPool.query(rlsSettingsSql(routingSchemaOf(ctx)), [databaseId]); + if (result.rows.length > 1) { + throw new Error('Ambiguous RLS module configuration'); + } return fromSettings(result.rows[0] ?? null); } }); diff --git a/packages/express-context/src/loaders/storage.ts b/packages/express-context/src/loaders/storage.ts new file mode 100644 index 0000000000..f20859ab8f --- /dev/null +++ b/packages/express-context/src/loaders/storage.ts @@ -0,0 +1,157 @@ +/** + * Storage Module Loader + * + * Resolves immutable storage-module routing metadata through the privileged + * control-plane tenant pool. Graphile receives the normalized descriptors at + * build time, so its least-privilege runtime pool never reads metaschema + * configuration with `withPgClient(null)`. + */ + +import { quoteQualifiedSqlIdentifier } from '../sql-identifiers'; +import type { StorageConfig, StorageModuleConfig } from '../types'; +import { createModuleLoader } from './create-loader'; +import type { LoaderContext, ModuleLoader } from './types'; + +const DEFAULT_UPLOAD_URL_EXPIRY_SECONDS = 900; +const DEFAULT_DOWNLOAD_URL_EXPIRY_SECONDS = 3600; +const DEFAULT_MAX_FILE_SIZE = 200 * 1024 * 1024; +const DEFAULT_MAX_FILENAME_LENGTH = 1024; +const DEFAULT_CACHE_TTL_SECONDS = process.env.NODE_ENV === 'development' ? 300 : 3600; +const DEFAULT_MAX_BULK_FILES = 100; +const DEFAULT_MAX_BULK_TOTAL_SIZE = 1024 * 1024 * 1024; + +export const STORAGE_MODULE_SQL = ` + SELECT + sm.id, + sm.scope, + sm.entity_table_id, + bs.schema_name AS buckets_schema, + bt.name AS buckets_table, + fs.schema_name AS files_schema, + ft.name AS files_table, + sm.endpoint, + sm.public_url_prefix, + sm.provider, + sm.allowed_origins, + sm.upload_url_expiry_seconds, + sm.download_url_expiry_seconds, + sm.default_max_file_size, + sm.max_filename_length, + sm.cache_ttl_seconds, + sm.max_bulk_files, + sm.max_bulk_total_size, + sm.has_path_shares, + es.schema_name AS entity_schema, + et.name AS entity_table + FROM metaschema_modules_public.storage_module sm + JOIN metaschema_public.table bt + ON bt.id = sm.buckets_table_id + AND bt.database_id = sm.database_id + JOIN metaschema_public.schema bs + ON bs.id = bt.schema_id + AND bs.database_id = sm.database_id + JOIN metaschema_public.table ft + ON ft.id = sm.files_table_id + AND ft.database_id = sm.database_id + JOIN metaschema_public.schema fs + ON fs.id = ft.schema_id + AND fs.database_id = sm.database_id + LEFT JOIN metaschema_public.table et + ON et.id = sm.entity_table_id + AND et.database_id = sm.database_id + LEFT JOIN metaschema_public.schema es + ON es.id = et.schema_id + AND es.database_id = sm.database_id + WHERE sm.database_id = $1 + ORDER BY sm.scope, sm.id +`; + +interface StorageModuleRow { + id: string; + scope: string; + entity_table_id: string | null; + buckets_schema: string; + buckets_table: string; + files_schema: string; + files_table: string; + endpoint: string | null; + public_url_prefix: string | null; + provider: string | null; + allowed_origins: string[] | null; + upload_url_expiry_seconds: number | string | null; + download_url_expiry_seconds: number | string | null; + default_max_file_size: number | string | null; + max_filename_length: number | string | null; + cache_ttl_seconds: number | string | null; + max_bulk_files: number | string | null; + max_bulk_total_size: number | string | null; + has_path_shares: boolean | null; + entity_schema: string | null; + entity_table: string | null; +} + +const numberOr = (value: number | string | null, fallback: number): number => { + if (value == null) return fallback; + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 0) { + throw new Error(`Invalid storage module numeric setting '${value}'`); + } + return parsed; +}; + +export const normalizeStorageModule = (row: StorageModuleRow): StorageModuleConfig => ({ + id: row.id, + bucketsQualifiedName: quoteQualifiedSqlIdentifier( + row.buckets_schema, + row.buckets_table, + 'storage buckets table' + ), + filesQualifiedName: quoteQualifiedSqlIdentifier( + row.files_schema, + row.files_table, + 'storage files table' + ), + schemaName: row.buckets_schema, + bucketsTableName: row.buckets_table, + filesTableName: row.files_table, + scope: row.scope, + entityTableId: row.entity_table_id, + entityQualifiedName: row.entity_schema && row.entity_table + ? quoteQualifiedSqlIdentifier( + row.entity_schema, + row.entity_table, + 'storage entity table' + ) + : null, + endpoint: row.endpoint, + publicUrlPrefix: row.public_url_prefix, + provider: row.provider, + allowedOrigins: row.allowed_origins, + uploadUrlExpirySeconds: numberOr( + row.upload_url_expiry_seconds, + DEFAULT_UPLOAD_URL_EXPIRY_SECONDS + ), + downloadUrlExpirySeconds: numberOr( + row.download_url_expiry_seconds, + DEFAULT_DOWNLOAD_URL_EXPIRY_SECONDS + ), + defaultMaxFileSize: numberOr(row.default_max_file_size, DEFAULT_MAX_FILE_SIZE), + maxFilenameLength: numberOr(row.max_filename_length, DEFAULT_MAX_FILENAME_LENGTH), + cacheTtlSeconds: numberOr(row.cache_ttl_seconds, DEFAULT_CACHE_TTL_SECONDS), + hasPathShares: row.has_path_shares ?? false, + maxBulkFiles: numberOr(row.max_bulk_files, DEFAULT_MAX_BULK_FILES), + maxBulkTotalSize: numberOr(row.max_bulk_total_size, DEFAULT_MAX_BULK_TOTAL_SIZE) +}); + +export const storageLoader: ModuleLoader = createModuleLoader({ + name: 'storage', + ttlMs: 60_000, + async resolve(ctx: LoaderContext) { + const result = await ctx.tenantPool.query( + STORAGE_MODULE_SQL, + [ctx.databaseId] + ); + if (result.rows.length === 0) return undefined; + return { modules: result.rows.map(normalizeStorageModule) }; + } +}); diff --git a/packages/express-context/src/loaders/types.ts b/packages/express-context/src/loaders/types.ts index cec903d7b3..b41d9d39aa 100644 --- a/packages/express-context/src/loaders/types.ts +++ b/packages/express-context/src/loaders/types.ts @@ -1,9 +1,8 @@ /** * Module Loader Types * - * A ModuleLoader is a per-database cached lookup that resolves config - * from the routing DB or tenant DB. Each loader owns its own LRU cache - * keyed by databaseId, with independent TTL and eviction. + * A ModuleLoader resolves per-database config from the routing DB or tenant + * DB. Each loader chooses authoritative reads or an independent hard-TTL LRU. * * Loaders are registered in a LoaderRegistry and resolved in parallel * during context building. The result is a typed modules map on @@ -56,10 +55,14 @@ export function requireDatabaseId( export interface LoaderContext { /** Routing/configuration database pool (for routing-plane lookups) */ routingPool: Pool; + /** Opaque identity of the exact routing-pool connection contract. */ + routingPoolIdentity?: string; /** Routing-plane schema to query (defaults to the published routing_public) */ routingSchema?: string; /** Tenant database pool (for metaschema_modules_public.* lookups) */ tenantPool: Pool; + /** Opaque identity of the exact tenant control-pool connection contract. */ + tenantPoolIdentity?: string; /** UUID of the database being resolved */ databaseId: string; /** UUID of the API (if resolved from domain/api-name lookup) */ @@ -69,16 +72,19 @@ export interface LoaderContext { } /** - * A single module loader. Encapsulates the SQL query, type transform, - * and per-databaseId LRU cache for one piece of per-database config. + * A single module loader. Encapsulates the SQL query, type transform, and + * freshness policy for one piece of per-database config. */ export interface ModuleLoader { /** Unique name (used in log prefix and as the key in the modules map) */ readonly name: string; /** Resolve the module config for a given database. Returns undefined if not provisioned. */ resolve(ctx: LoaderContext): Promise; - /** Invalidate the cache for one database (or all databases if omitted) */ - invalidate(databaseId?: string): void; + /** + * Invalidate one logical database across all physical pools, or only the + * exact pool pair represented by `context`. Omitting both clears everything. + */ + invalidate(databaseId?: string, context?: LoaderContext): void; /** Current number of cached entries */ readonly cacheSize: number; } diff --git a/packages/express-context/src/loaders/webauthn.ts b/packages/express-context/src/loaders/webauthn.ts index 2d1642756f..0921235674 100644 --- a/packages/express-context/src/loaders/webauthn.ts +++ b/packages/express-context/src/loaders/webauthn.ts @@ -26,12 +26,19 @@ const webauthnSettingsSql = (schema: string): string => ` ws.resident_key, ws.challenge_expiry_seconds FROM "${schema}".webauthn_settings ws - LEFT JOIN metaschema_public.schema s ON ws.schema_id = s.id - LEFT JOIN metaschema_public.schema cred_s ON ws.credentials_schema_id = cred_s.id - LEFT JOIN metaschema_public.schema sess_s ON ws.sessions_schema_id = sess_s.id - LEFT JOIN metaschema_public.schema sec_s ON ws.session_secrets_schema_id = sec_s.id + LEFT JOIN metaschema_public.schema s + ON ws.schema_id = s.id + AND s.database_id = ws.database_id + LEFT JOIN metaschema_public.schema cred_s + ON ws.credentials_schema_id = cred_s.id + AND cred_s.database_id = ws.database_id + LEFT JOIN metaschema_public.schema sess_s + ON ws.sessions_schema_id = sess_s.id + AND sess_s.database_id = ws.database_id + LEFT JOIN metaschema_public.schema sec_s + ON ws.session_secrets_schema_id = sec_s.id + AND sec_s.database_id = ws.database_id WHERE ws.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -54,13 +61,36 @@ interface WebauthnSettingsRow { export const webauthnLoader: ModuleLoader = createModuleLoader({ name: 'webauthnSettings', - ttlMs: 5 * 60_000, + // RP/origin/verification policy revocation must take effect immediately. + cache: false, async resolve(ctx: LoaderContext) { const { routingPool, databaseId } = ctx; const result = await routingPool.query(webauthnSettingsSql(routingSchemaOf(ctx)), [databaseId]); + if (result.rows.length > 1) { + throw new Error('Ambiguous WebAuthn configuration'); + } const row = result.rows[0]; - if (!row?.schema) return undefined; + if (!row) return undefined; + const required = [ + row.schema, + row.credentials_schema, + row.sessions_schema, + row.session_secrets_schema, + row.rp_id, + row.rp_name, + row.attestation_type, + row.resident_key + ]; + if ( + required.some((value) => typeof value !== 'string' || value.length === 0) + || !Array.isArray(row.origin_allowlist) + || typeof row.require_user_verification !== 'boolean' + || !Number.isSafeInteger(row.challenge_expiry_seconds) + || row.challenge_expiry_seconds <= 0 + ) { + throw new Error('Incomplete or cross-database WebAuthn configuration'); + } return { schema: row.schema, diff --git a/packages/express-context/src/pg-settings.ts b/packages/express-context/src/pg-settings.ts index cb86336456..82daddc8f5 100644 --- a/packages/express-context/src/pg-settings.ts +++ b/packages/express-context/src/pg-settings.ts @@ -12,6 +12,56 @@ import type { ApiStructure, ConstructiveAPIToken } from './types'; +export const SECURITY_GUC_KEYS = [ + 'jwt.claims.access_level', + 'jwt.claims.api_id', + 'jwt.claims.database_id', + 'jwt.claims.device_token', + 'jwt.claims.email', + 'jwt.claims.entity_id', + 'jwt.claims.ip_address', + 'jwt.claims.kind', + 'jwt.claims.organization_id', + 'jwt.claims.origin', + 'jwt.claims.principal_id', + 'jwt.claims.role_type', + 'jwt.claims.session_id', + 'jwt.claims.tenant_id', + 'jwt.claims.token_id', + 'jwt.claims.user_agent', + 'jwt.claims.user_email', + 'jwt.claims.user_id' +] as const; + +export type SecurityGucKey = typeof SECURITY_GUC_KEYS[number]; + +const SECURITY_GUC_KEY_SET: ReadonlySet = new Set(SECURITY_GUC_KEYS); + +const applyTrustedClaims = ( + settings: Record, + trustedClaims: PgSettingsInput['trustedClaims'] +): void => { + if (trustedClaims === undefined) return; + if ( + typeof trustedClaims !== 'object' + || trustedClaims === null + || Array.isArray(trustedClaims) + ) { + throw new TypeError('trustedClaims must be an object of security GUC strings'); + } + + for (const key of Reflect.ownKeys(trustedClaims)) { + if (typeof key !== 'string' || !SECURITY_GUC_KEY_SET.has(key)) { + throw new TypeError(`trustedClaims contains unsupported security GUC '${String(key)}'`); + } + const descriptor = Object.getOwnPropertyDescriptor(trustedClaims, key); + if (!descriptor || !('value' in descriptor) || typeof descriptor.value !== 'string') { + throw new TypeError(`trustedClaims.${key} must be a string data property`); + } + settings[key] = descriptor.value; + } +}; + export interface PgSettingsInput { /** Resolved API config (provides role names, database_id) */ api: ApiStructure; @@ -21,8 +71,21 @@ export interface PgSettingsInput { requestId: string; /** Client IP address (from request-ip middleware) */ clientIp?: string; + /** Origin header captured by the server */ + origin?: string; + /** User-Agent header captured by the server */ + userAgent?: string; + /** Trusted device cookie resolved by authentication middleware */ + deviceToken?: string; + /** Server-derived claims for trusted private surfaces */ + trustedClaims?: Partial>; + /** Ordered, audited extension/shared schemas needed for runtime operators and functions. */ + dependencySchemas?: readonly string[]; } +const quoteIdentifier = (identifier: string): string => + `"${identifier.replace(/"/g, '""')}"`; + /** * Build pgSettings from the resolved API + auth token. * @@ -30,8 +93,10 @@ export interface PgSettingsInput { * making them available to RLS policies and SQL functions. */ export function buildPgSettings(input: PgSettingsInput): Record { - const { api, token, requestId, clientIp } = input; - const settings: Record = {}; + const { api, token, requestId, clientIp, origin, userAgent, deviceToken } = input; + const settings: Record = Object.fromEntries( + SECURITY_GUC_KEYS.map((key) => [key, '']) + ); // Role: from token (authenticated) or api (anonymous fallback) if (token?.user_id) { @@ -41,14 +106,24 @@ export function buildPgSettings(input: PgSettingsInput): Record settings['role'] = api.anonRole || 'anonymous'; } + if (token?.id) settings['jwt.claims.token_id'] = token.id; + if (token?.access_level) settings['jwt.claims.access_level'] = token.access_level; + if (token?.kind) settings['jwt.claims.kind'] = token.kind; + if (typeof token?.email === 'string') settings['jwt.claims.email'] = token.email; + if (typeof token?.user_email === 'string') settings['jwt.claims.user_email'] = token.user_email; + if (typeof token?.entity_id === 'string') settings['jwt.claims.entity_id'] = token.entity_id; + if (typeof token?.organization_id === 'string') settings['jwt.claims.organization_id'] = token.organization_id; + if (typeof token?.tenant_id === 'string') settings['jwt.claims.tenant_id'] = token.tenant_id; + if (typeof token?.role_type === 'string') settings['jwt.claims.role_type'] = token.role_type; + // Session claims if (token?.session_id) { settings['jwt.claims.session_id'] = token.session_id; } // Principal identity (service accounts / bots) - if (token?.principal_id) { - settings['jwt.claims.principal_id'] = token.principal_id; + if (token?.principal_id || token?.user_id) { + settings['jwt.claims.principal_id'] = token.principal_id || token.user_id || ''; } // Database context @@ -72,5 +147,27 @@ export function buildPgSettings(input: PgSettingsInput): Record settings['jwt.claims.ip_address'] = clientIp; } + if (origin) settings['jwt.claims.origin'] = origin; + if (userAgent) settings['jwt.claims.user_agent'] = userAgent; + if (deviceToken) settings['jwt.claims.device_token'] = deviceToken; + // This is an exported boundary and TypeScript types do not constrain runtime + // objects. Reject extra keys/accessors so a future caller cannot smuggle + // role, search_path, or other session state through this trusted seam. + applyTrustedClaims(settings, input.trustedClaims); + + // Explicitly undo read-only state inherited from a previous request. + settings['transaction_read_only'] = token?.access_level === 'read_only' ? 'on' : 'off'; + // Pin name resolution after SET ROLE. DISCARD ALL resets to role/database + // defaults, which are mutable control-plane state and must not route a + // request into an unapproved schema. + settings['search_path'] = [ + 'pg_catalog', + ...[...new Set(input.dependencySchemas ?? [])].map(quoteIdentifier), + ...api.schema.map(quoteIdentifier) + ].join(', '); + // Owners and BYPASSRLS logins are rejected separately, but this makes the + // intended RLS state explicit for every transaction and clears prior state. + settings['row_security'] = 'on'; + return settings; } diff --git a/packages/express-context/src/sql-identifiers.ts b/packages/express-context/src/sql-identifiers.ts new file mode 100644 index 0000000000..1850cdcee3 --- /dev/null +++ b/packages/express-context/src/sql-identifiers.ts @@ -0,0 +1,31 @@ +import { QuoteUtils } from '@pgsql/quotes'; + +const POSTGRES_IDENTIFIER_MAX_BYTES = 63; + +/** + * Quote a metadata-derived PostgreSQL identifier without accepting values that + * PostgreSQL would truncate or that cannot be identifiers at all. Request data + * must still be passed as query parameters. + */ +export const quoteSqlIdentifier = ( + identifier: string, + label = 'SQL identifier' +): string => { + if ( + typeof identifier !== 'string' + || identifier.length === 0 + || identifier.includes('\0') + || Buffer.byteLength(identifier, 'utf8') > POSTGRES_IDENTIFIER_MAX_BYTES + ) { + throw new Error(`Invalid ${label}`); + } + const quoted = QuoteUtils.quoteIdentifier(identifier); + return quoted.startsWith('"') ? quoted : `"${quoted}"`; +}; + +export const quoteQualifiedSqlIdentifier = ( + schema: string, + object: string, + label = 'qualified SQL identifier' +): string => + `${quoteSqlIdentifier(schema, `${label} schema`)}.${quoteSqlIdentifier(object, `${label} object`)}`; diff --git a/packages/express-context/src/types.ts b/packages/express-context/src/types.ts index 4316018209..9f479626b4 100644 --- a/packages/express-context/src/types.ts +++ b/packages/express-context/src/types.ts @@ -204,12 +204,54 @@ export interface ComputeModuleConfig { invocationsEntityField: string | null; } +export interface ComputeBindingConfig { + bindingId: string; + alias: string; + config: Record | null; + functionDefinitionId: string; + taskIdentifier: string; + description: string | null; + payloadArgs: Array<{ name: string; type: string }> | null; + module: ComputeModuleConfig; +} + /** * All function modules provisioned on the database. A database may have one * per scope; every module is exposed and RLS governs access to each. */ export interface ComputeConfig { modules: ComputeModuleConfig[]; + /** API-scoped binding metadata loaded with the control-plane tenant pool. */ + bindings: ComputeBindingConfig[]; +} + +/** Immutable storage-module routing metadata loaded through the control plane. */ +export interface StorageModuleConfig { + id: string; + bucketsQualifiedName: string; + filesQualifiedName: string; + schemaName: string; + bucketsTableName: string; + filesTableName: string; + scope: string; + entityTableId: string | null; + entityQualifiedName: string | null; + endpoint: string | null; + publicUrlPrefix: string | null; + provider: string | null; + allowedOrigins: string[] | null; + uploadUrlExpirySeconds: number; + downloadUrlExpirySeconds: number; + defaultMaxFileSize: number; + maxFilenameLength: number; + cacheTtlSeconds: number; + hasPathShares: boolean; + maxBulkFiles: number; + maxBulkTotalSize: number; +} + +export interface StorageConfig { + modules: StorageModuleConfig[]; } export interface LlmConfig { @@ -249,6 +291,7 @@ export interface BuiltinModuleMap { agentChat: AgentChatConfig; llm: LlmConfig; compute: ComputeConfig; + storage: StorageConfig; } // ─── Constructive Context ─────────────────────────────────────────────────── @@ -280,14 +323,16 @@ export interface ConstructiveContext { requestId: string; /** Tenant database connection pool */ pool: Pool; + /** Opaque exact identity of the tenant execution pool. */ + runtimePoolIdentity: string; /** Execute a function within a tenant-scoped RLS transaction */ withPgClient: WithPgClient; /** - * Resolve a per-database module on demand (lazy, cached). + * Resolve a per-database module on demand. * - * Only fires the SQL query on the first call per databaseId per TTL window. - * Subsequent calls return the cached result instantly. + * Each loader owns its freshness policy. Security-sensitive built-ins read + * authoritatively on every call; nonsecurity loaders may use a hard TTL. * * Built-in modules are typed: * const rls = await ctx.useModule('rlsModule'); // RlsModule | undefined @@ -340,6 +385,9 @@ declare global { clientIp?: string; requestId?: string; token?: ConstructiveAPIToken; + deviceToken?: string; + /** Set by the GraphQL ingress after authenticating reserved internal headers. */ + internalTrusted?: boolean; constructive?: ConstructiveContext; } } diff --git a/packages/perf-harness/README.md b/packages/perf-harness/README.md new file mode 100644 index 0000000000..c5fa156545 --- /dev/null +++ b/packages/perf-harness/README.md @@ -0,0 +1,210 @@ +# cperf Graphile density harness + +`cperf` is a local-only runner for the Graphile customer-density spike. It launches a fresh production-mode server process for each arm/heap/customer-count/repetition, warms every configured GraphQL surface with bounded concurrency and a fleet-size-scaled deadline, drives an open-loop workload, runs hostile isolation canaries, samples `/debug/memory` and the dedicated PostgreSQL container, and writes one timestamped JSON result per run plus an NDJSON ledger. + +The score is deliberately strict. A customer counts only when every declared surface warms, receives the configured minimum number of workload-phase requests, has an error rate of at most 0.5% and customer-workload p99 of at most 150 ms, runs all required capability operations, passes every required isolation canary conclusively with zero bleed, and sees no post-warmup Graphile or PostgreSQL pool eviction/refusal/build/disposal activity. Coverage probes prove operation support but do not contribute traffic, latency, or error samples. Results report customer workload RPS, periodic validation RPS, realtime validation RPS, and their combined HTTP RPS separately, so security probes cannot inflate customer throughput or pollute its latency percentiles. Runs shorter than 15 minutes always fail qualification, including `--smoke` runs. The JSON retains legacy `tenant*` aliases while the research interfaces migrate to customer terminology. + +## Capacity methodology + +The target is qualified complete customers per actual service memory unit, not fitting the fleet into a 1 GiB process. `tenantCountsByHeapMiB` retains its legacy name and supplies a different increasing customer ramp for each configured old-space size, while `tenantCounts` applies one ramp to every heap. A capacity result is complete only when all repetitions pass at one customer count and a greater count fails; an unbracketed last successful checkpoint is reported as observed capacity, not maximum capacity. + +Use exactly one load mode. `rps` holds total offered load fixed as tenants are added, which isolates memory capacity but reduces per-tenant traffic; `rpsPerTenant` holds per-tenant load fixed, so total offered load grows with the fleet. Every result records the resolved total and per-tenant load. `minWorkloadRequestsPerSurface` prevents a tenant from qualifying without representative traffic, and `warmupTimeoutPerSurfaceMs` scales the warmup allowance by the number of bounded-concurrency waves in addition to `warmupTimeoutMs`. + +Runs are deterministically interleaved across arms using `runOrderSeed`, so repeated experiments reproduce the order without always favoring the same arm. Each invocation also creates a random campaign ID and an immutable manifest for that exact ordered schedule. Per-campaign results form a forward SHA-256 chain, and report validation requires the manifest order, non-overlapping chronology, common runtime platform, and every chain pointer to agree; separately collected records cannot be spliced into qualification evidence. Results record the plan and fleet hashes, runtime versions, order, and resolved memory-governor policy. The runner also persists request, canary, memory, PostgreSQL, and workload-progress artifacts when a run fails partway through, so a failed capacity point remains diagnosable. + +Spawned Node arms may select only `v8Profile: stock`, `optimize-for-size`, `baseline-optimize-for-size`, or `jitless-optimize-for-size`. The baseline-size profile uses `--max-opt=1 --optimize-for-size`, retaining Sparkplug while excluding the higher optimization tiers. The runner injects the exact allowlisted flags, strips inherited copies, rejects managed flags hidden in the command or plan `NODE_OPTIONS`, and records the profile, sanitized `NODE_OPTIONS`, direct Node arguments, and their effective ordered combination in provenance. Every non-stock profile is an explicit candidate and must pass the full loaded density curve, p99, throughput, and isolation gates. + +`periodicCanarySchedule` defaults to the legacy `full-sweep` behavior. The `rotating-one` mode executes one deterministically staggered canary per tenant/surface in every timed round while retaining full initial and final sweeps. Timed rounds occupy only interval slots strictly before the workload deadline, so a 900-second run at a 60-second interval has exactly 14 rounds. Rounds are serialized and never dropped when one overlaps the next slot; each overlap, incomplete round, and deadline-late completion is recorded in `canary-schedule.json`. `canaryConcurrency` bounds parallelism across surfaces while every surface's probes stay sequential. A qualifying rotating plan should enable `requireCompletePeriodicCanaryCoverage`, which requires exact boundary sweeps, one exact result per selected target/round, complete configured-canary coverage, and every periodic round to finish by the workload deadline. + +Realtime GraphQL routes use the same strict timed-workload boundary. After one initial correlated mutation/subscription delivery, the driver schedules a fresh delivery in every 60-second slot strictly before the workload deadline, serializes rounds, and persists credential-free correlation receipts for every exact tenant/surface route. The report derives counts, globally unique ordered digests, prime-request volume, prime-response p99, and delivery p99 from those raw receipts instead of trusting the summary fields; append-only histories and the single timed-coverage completion transition are also verified. A late, missed, reused, or unverified recurring round sets qualified customers to zero even when the final post-workload probe succeeds, so a healthy connection at the two bookends cannot conceal a subscription that stopped delivering during the workload. + +Every persisted v6 result carries a SHA-256 binding over its complete result payload and the exact memory, PostgreSQL, request, canary, canary-schedule, retained-memory, workload-progress, realtime, and score-context evidence files. The credential-free score context binds the plan/fleet hashes and the few run facts that cannot be reconstructed from those raw files; workload load and warmup limits are re-derived from the plan and fleet. The report loader verifies every bound file, reconstructs the complete `scoreRun` input, reruns the scorer, and requires byte-equivalent result semantics, so recomputing public hashes cannot bless a hand-edited result. Soak records render in a separate section and never enter matrix medians, capacity boundaries, or candidate comparisons. + +An arm may declare `envByHeapMiB` to override its base environment for each configured heap. When present, it must contain exactly every plan heap and only string values. This is the intended path for measured Graphile governor calibration: each checkpoint can pin its cache ceiling, instance estimate, server/build reserves, RSS build reserve, and `GRAPHILE_CACHE_CALIBRATION_ID`, and qualifying physical-database runs verify that the live cache reports that identity and enough configured/budget capacity for every requested surface. Physical fixture cache keys are process-random keyed HMACs, so cross-process scoring compares the fleet against the fixture's credential-free Graphile contract fingerprints; the live keys remain in same-process guard state to prove that no resident entry changed during a run. + +The primary report metric is qualified customers divided by the maximum post-warmup time-aligned sum of current Node RSS and the dedicated PostgreSQL container's raw cgroup-v2 `memory.current` charge. On Linux, exact-process current RSS comes from `/proc` at 100 ms; on non-Linux diagnostic runs it comes from a bearer-authenticated loopback endpoint at 250 ms. Publication-quality qualification still requires Linux and cgroup v2. The runner pairs timestamped Node and PostgreSQL samples within one second and fails qualification when aligned service telemetry is unavailable. It also reports a conservative non-simultaneous upper bound—Node RSS high-water plus PostgreSQL peak—and retains Docker working set, configured old-space, and Node-only peak-RSS density as diagnostics. Candidate acceptance requires a complete paired matrix, the configured additional customers at every heap, the configured median improvement in both actual service-memory measures, and no per-heap regression in either of those measures; the heap sizes are measurement points rather than capacity targets. + +Smoke results also include `configuredCustomersPerAlignedServiceGiB` and `configuredCustomersPerServiceMemoryUpperBoundGiB`. These diagnostic fields make a short fully warmed mechanics run numerically useful, but they use configured rather than qualified customers and never participate in acceptance; `customersPerAlignedServiceGiB` remains zero until the full duration, traffic, correctness, isolation, and residency gates pass. + +Physical qualification arms must configure a prepare command that creates one fresh PostgreSQL fixture under the current run artifact directory before the measured Node process starts. The following audit binds the exact matrix coordinate, plan/fleet hashes, Docker image and resource/command configuration, cgroup-v2 identity, PostgreSQL system identifier and start time, exact database inventory, unique clone/nonce set, and recomputed live DDL/ACL/role/extension contracts. The sampler resolves the mutable Docker name once, pins the attested 64-character container ID for every `stats` and cgroup read, and revalidates the start time and cgroup identity after the final sample. The server then receives the attested manifest path, manifest hash, and clone ID as resolved command templates. Reusing or replacing any container, cluster, clone, attestation set, or nonce-set identity fails both the in-process schedule and cross-file report aggregation. + +Measured GraphQL canaries remain request-path correctness evidence; they are not a substitute for an induced hostile campaign. A qualifying plan must bind one immutable `exact-runtime-hostile-validation-v1` artifact per arm, including the exact runtime-artifact and configuration fingerprints. If those artifacts are absent or mismatched, the runner labels the entire campaign diagnostic and the report refuses to promote it, even when every passive canary passed. + +That full structural audit intentionally reads PostgreSQL catalogs before Graphile starts, so reported build latency is a post-attestation warm-catalog measurement. It is comparable across equally audited arms, but it is not evidence for a pristine-catalog cold start. + +## Commands + +```bash +pnpm --filter @constructive-io/perf-harness build + +node packages/perf-harness/dist/index.js validate --plan path/to/completed-plan.json + +node packages/perf-harness/dist/index.js run \ + --plan research/graphile-density/four-arm-plan.example.json \ + --smoke --arm scoped-introspection + +node packages/perf-harness/dist/index.js report \ + --plan research/graphile-density/four-arm-plan.example.json \ + --results graphile-density-artifacts/results.ndjson \ + --out graphile-density-artifacts/report.md +``` + +### Catalog cache-warmth benchmark + +Scoped runs accept `--scoped-catalog-types all|dependency-closure`. The default +is `all`, which preserves the current scoped-required query; the experimental +`dependency-closure` arm retains only catalog types reached by the requested +schemas' object closure. The flag is rejected for stock mode, and its value is +recorded in worker configs, results, summaries, provenance, and cache build +identities so the two scoped arms cannot share a Graphile instance. + +`--release-build-state-after-validation` enables the opt-in lifecycle candidate +for `catalog-bench`. Its boolean value is written to the worker config, progress, +result, summary, provenance, Graphile preset, and cache identity; omitting the +flag always measures the default retained-build-state behavior. + +`--introspection-client-release-mode reuse|destroy` selects how the PostgreSQL +checkout used for catalog introspection is released and defaults to `reuse`. +In `destroy` mode the worker proves the full PID plus SQL `backend_start` +identity has disappeared through a separate control connection before it +acquires a replacement; token canaries and cache-warm operations must then +leave that replacement identity unchanged. Snapshot RSS is the steady +replacement backend's RSS and its delta is relative to replacement acquisition. + +`--postgres-backend-sampler off|diagnostic-lower-bound` defaults to +`diagnostic-lower-bound` and gives paired sampler-on/off runs for quantifying +observer cost. Before each destroy-mode build, the fixed external sampler binds +the SQL `backend_start` to `/proc//stat` within an explicit 1.5-second +boot-time tolerance, then revalidates the immutable proc start token, PostgreSQL +process name, and PID namespace identity on every 10 ms sample. A Linux Docker +host prefers the container's procfs through host procfs; the fallback pins one +`docker exec` to the inspected 64-character container ID and revalidates the +name, ID, start time, and init PID after sampling. The fallback starts +`/usr/bin/env -i` with a path-only shell environment, but the initial Docker +exec process may briefly inherit the container's configured environment before +`env -i` clears it. Artifacts record only the allowlisted host variable names, +never their values. + +The worker traps shell exits, stops the sampler process group with bounded +graceful, TERM, and KILL phases, and awaits tree closure before backend +retirement. Even a cadence-complete `VmRSS`/`VmHWM` trace is a diagnostic lower +bound because Graphile has no pre-destroy acknowledgement guaranteeing a final +sample; artifacts never promote it to an exact peak or density authority. +Sampler launch and shutdown time are recorded without subtracting a correction. +Service-density authority remains the separately validated Linux cgroup-v2 +`memory.current` measurement, while Docker Desktop backend traces carry an +additional VM-boundary limitation. + +`--v8-profile stock|optimize-for-size|baseline-optimize-for-size|jitless-optimize-for-size` selects the +worker's named V8 configuration and defaults to `stock`. The parent sanitizes +inherited managed flags, launches the worker with the profile's exact direct +Node arguments, and pins `--heap-mib` through `NODE_OPTIONS`. Worker config, +progress, result, summary, and provenance artifacts record the selected profile, +the sanitized `NODE_OPTIONS`, its tokenization, `process.execArgv`, and the +effective ordered combination; a mismatch fails the run. Starting the parent +Node process with an optimization flag is not evidence that workers inherited +it, so benchmark comparisons must select the profile explicitly through this +flag. The baseline and jitless profiles remain opt-in candidates and must pass the same +loaded latency, throughput, and isolation gates as stock. + +`catalog-bench` can populate every resident schema's Grafast parse/query and +operation-plan caches with reproducible, distinct named operations. A nonzero +`--warm-operations-per-instance` requires one exact `--expected-tokens` value +per schema; every operation executes through `grafast({ source })`, and the +artifact records p50/p99 population latency plus conclusive token correctness. +`--warm-operation-replay-passes N` then replays that exact ordered source set +through Grafast `N` times for each instance. Replay execution counts, p50/p99 +latency, errors, exact-token correctness, mismatches, and cross-tenant results +are recorded separately from population, so cache-limit comparisons do not +mix cold source admission with cache-hit or cache-churn behavior. Replay is +disabled by default, and a positive pass count requires a nonempty population +set from `--warm-operations-per-instance`. +The three cache-limit flags are independently optional. Omitting all three uses +Grafast's defaults, while providing them installs the shared +`createGrafastCacheLimitsPreset` before schema construction. + +`--tenant-proxy-surfaces N` adds an explicitly synthetic density projection to +the parent `summary.json`; it does not change the worker or turn these fixtures +into measured complete tenants. The projection divides resident surface +instances into full groups of `N`, records any remainder, and reports group +density against both the configured `--max-old-space-size` GiB and the absolute +lifetime process peak-RSS GiB. It never uses baseline-relative RSS or the +per-instance slope as the peak-RSS denominator. Because the final checkpoint is +a scheduled stop rather than a discovered memory boundary, the summary records +`capacityBoundaryReached: false` and must be read as an observed synthetic +checkpoint, not maximum customer capacity. + +These four commands reproduce the default-versus-all8 comparison at 100 and +500 operations for the disposable density fixture, assuming the `PG*` +environment variables already select its least-privilege runtime login: + +```bash +node packages/perf-harness/dist/index.js catalog-bench --database graphile_density_20260801_a --mode scoped-required --schemas gd_t001_api --instances 1 --expected-tokens tenant-001-token --warm-operations-per-instance 100 --warm-operation-replay-passes 3 --heap-mib 2048 --repetitions 3 --postgres-container postgres --out /tmp/cperf-cache-default-100 + +node packages/perf-harness/dist/index.js catalog-bench --database graphile_density_20260801_a --mode scoped-required --schemas gd_t001_api --instances 1 --expected-tokens tenant-001-token --warm-operations-per-instance 500 --warm-operation-replay-passes 3 --heap-mib 2048 --repetitions 3 --postgres-container postgres --out /tmp/cperf-cache-default-500 + +node packages/perf-harness/dist/index.js catalog-bench --database graphile_density_20260801_a --mode scoped-required --schemas gd_t001_api --instances 1 --expected-tokens tenant-001-token --warm-operations-per-instance 100 --warm-operation-replay-passes 3 --grafast-query-cache-max 8 --grafast-operations-cache-max 8 --grafast-operation-plans-cache-max 8 --heap-mib 2048 --repetitions 3 --postgres-container postgres --out /tmp/cperf-cache-all8-100 + +node packages/perf-harness/dist/index.js catalog-bench --database graphile_density_20260801_a --mode scoped-required --schemas gd_t001_api --instances 1 --expected-tokens tenant-001-token --warm-operations-per-instance 500 --warm-operation-replay-passes 3 --grafast-query-cache-max 8 --grafast-operations-cache-max 8 --grafast-operation-plans-cache-max 8 --heap-mib 2048 --repetitions 3 --postgres-container postgres --out /tmp/cperf-cache-all8-500 +``` + +With no warmth or cache-limit flags, the command retains its prior behavior and +does not install a cache-limit preset. Each build now also records an +approximate transient heap/RSS peak sampled every 5 ms from an immediately +preceding forced-GC resident baseline. The process RSS high-water is recorded +as a backstop, but synchronous event-loop work can still hide a short heap peak, +so this number is a measured reserve input rather than an exact maximum. +`--heap-mib` configures Node's old-space flag; the worker records V8's effective +total heap limit separately because the two values are not interchangeable. + +The legacy `--schemas a,b --instances 1,2` form still means one schema per +resident instance. To measure one production-shaped surface that exposes an +ordered schema set, use `--surface-schemas` with exactly one instance and an +explicit `--allowed-dependency-schemas` list: + +```bash +node packages/perf-harness/dist/index.js catalog-bench \ + --database production_shape \ + --mode scoped-required \ + --scoped-catalog-types dependency-closure \ + --surface-schemas app_public,app_auth,app_users \ + --allowed-dependency-schemas app_extensions,jwt_private \ + --instances 1 \ + --heap-mib 2048 \ + --repetitions 3 \ + --out /tmp/cperf-production-shape-scoped +``` + +Both ordered lists are validated, included in the worker config and provenance, +and hashed into the Graphile build and fixture identities. The exposed list must +be nonempty; an explicitly supplied empty dependency value is retained as `[]` +and remains distinct from an omitted flag. Names must be unique within each +list and the lists must be disjoint, and a manually edited worker config fails +closed under the same checks. + +The checked-in example plan and fleet are deliberately incomplete placeholders, so `validate` rejects them until they are copied and filled with real tenant routes, queries, tokens, credentials, and separate worktree paths. Spawned arms must pin a commit. Each run verifies and records the actual Git HEAD and dirty state, command, working directory, entry and lockfile hashes, server PID, and effective V8 heap limit; optional `entrySha256` and `lockfileSha256` plan pins make mismatches fail before traffic starts. + +`postgresContainer` is sampled from raw cgroup-v2 `memory.current`, `memory.peak`, `memory.stat`, and `memory.events` at 250 ms when available; Docker working set is sampled separately at a lower frequency as a diagnostic. The recorded cold-build spike is the greatest sampled raw charge before the post-warmup boundary minus the first raw sample, and it is meaningful only when the container is dedicated to the arm. Backend process RSS is never summed because PostgreSQL processes share pages; backend and concrete pool-client counts are reported separately instead. The aligned service metric is meaningful only when that PostgreSQL container is dedicated to the measured Node process and no unrelated workload runs in either boundary. + +## Fleet contract + +A fleet contains customers (the legacy JSON key remains `tenants`), each with one or more named surfaces. A qualifying fleet also declares the exact customer → logical database → API topology: stable database/API IDs, a credential-free physical database label, ordered physical schemas, opaque credential-sensitive runtime-pool identities, and the surface names served by each API. Validation requires every surface to appear exactly once and rejects build-contract or runtime-pool identities reused across customers, so host labels and instance counts cannot be mistaken for customer isolation or density. + +Every surface defines a warmup query, weighted workload operations tagged with the capability they actually exercise, and isolation canaries. Operations may use typed `requiredMatches` and `forbiddenMatches` response oracles; a matching forbidden value produces `GRAPHQL_OPERATION_ORACLE_FORBIDDEN`, while a successful response missing required evidence produces `GRAPHQL_OPERATION_ORACLE_MISSING`. Wildcard-capable `invariants` add an exhaustive `everyEquals` assertion with positive `min` and optional `max`, so an empty collection or a later foreign row cannot pass after checking node zero. Transport and GraphQL failures remain inside the configured 0.5% error budget and are marked oracle-unavailable, while missing or unexpected evidence in a successful response still fails immediately and every operation must have exactly one conclusive coverage result. + +Mutations may declare an untimed `postCoverageVerification` query with the same oracle contract. `variablesFromResponse` binds each verification variable to exactly one JSON pointer from the primary response; a missing or ambiguous extraction fails closed before the verification query runs. This lets side-effect checks correlate by the ID returned by the current mutation instead of accepting a stale row selected by a reusable content hash. `requireConclusiveOperationOracles` rejects the plan unless every warmup and operation has direct or post-coverage evidence, and rejects the run on missing or foreign evidence without changing the production GraphQL API. + +Canaries use the same RFC 6901 JSON `path` plus exact JSON `value` model; point them at customer-specific result fields so unrelated strings elsewhere in a response cannot trigger or satisfy an isolation check. A realtime surface also declares its exact subscription and prime mutation with permanent required/forbidden identity invariants plus correlation paths. The driver replaces the declared prime variable with a fresh opaque nonce for every delivery round and requires that exact nonce in both the mutation response and subscription event, so a stale cursor replay cannot satisfy recurring coverage. Artifacts contain only ordered SHA-256 bindings of issued and verified nonces. Those clients remain in the driver rather than the measured server process, and sensitive HTTP/websocket headers are resolved from declared environment-variable names without entering the fleet or artifacts. + +If arms produce different cache identities, set `buildContracts` on each surface +with one exact hash per arm name. Validation rejects a partial mapping, and the +runner selects only the current arm's hash; a stock identity therefore cannot +silently satisfy a scoped run (or vice versa). + +Capabilities and canaries are plan-level allowlists. A run fails unless every tenant serves every configured operation and capability on its configured surface, every tenant covers every required capability, and every surface runs every required canary. The production plan should require generated Graphile plans plus i18n, LLM/RAG, BM25, tsvector, trigram, vector, PostGIS, ltree, uploads/storage, bulk mutations, realtime, and function bindings. It should also require cross-schema identifiers, metadata, functions, sequences, prepared-statement reuse, poisoned GUCs, rollback/savepoints, plugin raw SQL, owner/BYPASS-role probes, schema drift, cache invalidation, concurrent builds, and connection reuse. + +## Safety + +Ports 3000–3002, 5432, and 9000 are rejected unless `--allow-reserved-ports` is explicit. For each spawned process, cperf generates a fresh strong observability token and sends it only as an `Authorization` header to the loopback memory endpoint; the token is never put in a URL, log, provenance record, or artifact. Server credentials stay in the inherited environment and are never serialized into result files. An arm without a launch command is treated as an external reused server and can produce diagnostics, but it cannot qualify because the cache and process boundary are not fresh. Missing endpoint fields remain `null` and disqualify the run instead of becoming zero-valued measurements. + +This harness never provisions or modifies `constructive-db`. Fixture creation belongs in a disposable PostgreSQL database or an independently managed validation environment. diff --git a/packages/perf-harness/jest.config.js b/packages/perf-harness/jest.config.js new file mode 100644 index 0000000000..363ad24f79 --- /dev/null +++ b/packages/perf-harness/jest.config.js @@ -0,0 +1,11 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }] + }, + testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + modulePathIgnorePatterns: ['dist/*'] +}; diff --git a/packages/perf-harness/package.json b/packages/perf-harness/package.json new file mode 100644 index 0000000000..9e8e4d7e7d --- /dev/null +++ b/packages/perf-harness/package.json @@ -0,0 +1,41 @@ +{ + "name": "@constructive-io/perf-harness", + "version": "0.2.0", + "private": true, + "description": "Local Graphile tenant-density and isolation validation harness", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "bin": { + "cperf": "index.js" + }, + "scripts": { + "clean": "makage clean", + "build": "makage build", + "build:dev": "makage build --dev", + "lint": "eslint . --fix", + "test": "jest --passWithNoTests" + }, + "dependencies": { + "grafast": "1.0.2", + "graphile-build-pg": "5.0.2", + "graphile-cache": "workspace:^", + "graphile-settings": "workspace:^", + "graphql": "16.13.0", + "graphql-ws": "^6.0.8", + "pg": "^8.21.0", + "pg-env": "workspace:^", + "ws": "^8.20.0" + }, + "devDependencies": { + "@types/node": "^22.19.11", + "@types/pg": "^8.20.0", + "@types/ws": "^8.18.1", + "makage": "^0.3.0", + "ts-node": "^10.9.2" + }, + "engines": { + "node": ">=22" + }, + "license": "MIT" +} diff --git a/packages/perf-harness/src/__tests__/catalog-bench.test.ts b/packages/perf-harness/src/__tests__/catalog-bench.test.ts new file mode 100644 index 0000000000..6bf2c51839 --- /dev/null +++ b/packages/perf-harness/src/__tests__/catalog-bench.test.ts @@ -0,0 +1,1088 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { parse } from 'graphql'; + +import { + assertCatalogDockerContainerIdentity, + catalogBackendSamplerEnvironment, + catalogIntrospectionBuildIdentity, + type CatalogMemorySnapshot, + catalogPercentile, + catalogProgressPath, + catalogSchemaContractIdentity, + makeCatalogBackendSamplerLaunchSpec, + makeCatalogWarmOperationSource, + measureCatalogBuildWithBackendSampler, + parseCatalogBackendProcStatus, + parseCatalogBackendSamplerMode, + parseCatalogBuildStateRetirement, + parseCatalogDockerContainerIdentity, + parseCatalogIntrospectionClientReleaseMode, + parseCatalogSchemaLayout, + parseCatalogScopedCatalogTypes, + parseCatalogTenantProxySurfaces, + parseCatalogV8Profile, + parseCatalogWarmthCliOptions, + projectCatalogTenantDensity, + resolveCatalogBackendPidAfterBuild, + resolveCatalogSchemaLayout, + stopCatalogBackendSamplerProcessTree, + summarizeBuildTransientSamples, + summarizeCatalogBackendMemorySamples, + validateCatalogPostgresContainer, + validateCatalogRuntimeFlags, + validateCatalogWarmthConfig, + writeCatalogProgress +} from '../catalog-bench'; + +describe('catalog benchmark schema layout', () => { + it('preserves the legacy one-schema-per-instance CLI shape', () => { + expect(parseCatalogSchemaLayout([ + '--schemas', 'gd_t001_api,gd_t002_api' + ], 2)).toEqual({ + schemas: ['gd_t001_api', 'gd_t002_api'], + schemaSets: null, + allowedDependencySchemas: null + }); + }); + + it('parses one ordered multi-schema surface and explicit dependency closure', () => { + const layout = parseCatalogSchemaLayout([ + '--surface-schemas', 'app_public,app_auth,app_users', + '--allowed-dependency-schemas', 'app_extensions,jwt_private' + ], 1); + expect(layout).toEqual({ + schemas: ['app_public'], + schemaSets: [['app_public', 'app_auth', 'app_users']], + allowedDependencySchemas: ['app_extensions', 'jwt_private'] + }); + expect(resolveCatalogSchemaLayout({ + schemas: layout.schemas, + schemaSets: layout.schemaSets!, + allowedDependencySchemas: layout.allowedDependencySchemas!, + checkpoints: [1] + })).toEqual(layout); + expect(parseCatalogSchemaLayout([ + '--surface-schemas', 'app_public,app_auth', + '--allowed-dependency-schemas', '' + ], 1)).toEqual({ + schemas: ['app_public'], + schemaSets: [['app_public', 'app_auth']], + allowedDependencySchemas: [] + }); + }); + + it.each([ + [ + ['--surface-schemas', 'app_public,app_auth'], + 1, + 'requires --allowed-dependency-schemas' + ], + [ + [ + '--surface-schemas', 'app_public,,app_auth', + '--allowed-dependency-schemas', 'app_extensions' + ], + 1, + 'must not contain empty schema names' + ], + [ + [ + '--surface-schemas', 'app_public,app_public', + '--allowed-dependency-schemas', 'app_extensions' + ], + 1, + 'must contain unique schema names' + ], + [ + [ + '--surface-schemas', 'app_public,app_auth', + '--allowed-dependency-schemas', 'app_auth' + ], + 1, + 'must not overlap' + ], + [ + [ + '--surface-schemas', 'app_public,app_auth', + '--allowed-dependency-schemas', 'app_extensions' + ], + 2, + 'requires exactly one resident instance' + ], + [ + ['--schemas', 'app_public', '--surface-schemas', 'app_auth'], + 1, + 'mutually exclusive' + ], + [ + ['--schemas', 'app_public', '--allowed-dependency-schemas', 'app_extensions'], + 1, + 'requires --surface-schemas' + ] + ])('rejects ambiguous schema layout %#', (args, instances, message) => { + expect(() => parseCatalogSchemaLayout( + args as string[], + instances as number + )).toThrow(message as string); + }); + + it('rejects manually edited worker layouts and hashes both ordered lists', () => { + expect(() => resolveCatalogSchemaLayout({ + schemas: ['app_auth'], + schemaSets: [['app_public', 'app_auth']], + allowedDependencySchemas: ['app_extensions'], + checkpoints: [1] + })).toThrow('schemas[0] must equal the first ordered'); + expect(() => resolveCatalogSchemaLayout({ + schemas: ['app_public'], + schemaSets: [['app_public', 'app_auth']], + allowedDependencySchemas: ['app_extensions', 'app_extensions'], + checkpoints: [1] + })).toThrow('must contain unique schema names'); + + const identity = catalogSchemaContractIdentity( + ['app_public', 'app_auth'], + ['app_extensions', 'jwt_private'] + ); + expect(identity).not.toBe(catalogSchemaContractIdentity( + ['app_auth', 'app_public'], + ['app_extensions', 'jwt_private'] + )); + expect(identity).not.toBe(catalogSchemaContractIdentity( + ['app_public', 'app_auth'], + ['jwt_private', 'app_extensions'] + )); + expect(catalogSchemaContractIdentity(['app_public'], [])).not.toBe( + catalogSchemaContractIdentity(['app_public'], ['app_extensions']) + ); + }); +}); + +describe('catalog benchmark scoped catalog type policy', () => { + it('preserves all catalog types as the scoped default', () => { + expect(parseCatalogScopedCatalogTypes([], 'scoped-required')).toBe('all'); + expect(parseCatalogScopedCatalogTypes([], 'stock')).toBeNull(); + }); + + it('parses the dependency-closure experiment strictly', () => { + expect(parseCatalogScopedCatalogTypes([ + '--scoped-catalog-types', 'dependency-closure' + ], 'scoped-required')).toBe('dependency-closure'); + }); + + it.each([ + [['--scoped-catalog-types'], 'scoped-required', 'requires a value'], + [[ + '--scoped-catalog-types', 'all', + '--scoped-catalog-types', 'dependency-closure' + ], 'scoped-required', 'may only be specified once'], + [['--scoped-catalog-types', 'closure'], 'scoped-required', "must be 'all' or 'dependency-closure'"], + [['--scoped-catalog-types', 'all'], 'stock', 'requires --mode scoped-required'] + ])('rejects malformed catalog policy arguments %j', (args, mode, message) => { + expect(() => parseCatalogScopedCatalogTypes( + args as string[], + mode as 'stock' | 'scoped-required' + )).toThrow(message as string); + }); + + it('separates all-types and dependency-closure build identities', () => { + const all = catalogIntrospectionBuildIdentity('scoped-required', 'all'); + const closure = catalogIntrospectionBuildIdentity( + 'scoped-required', + 'dependency-closure' + ); + + expect(all).not.toBe(closure); + expect(catalogIntrospectionBuildIdentity('stock', null)).not.toBe(all); + expect(catalogIntrospectionBuildIdentity( + 'scoped-required', + 'dependency-closure', + true + )).not.toBe(closure); + expect(catalogIntrospectionBuildIdentity( + 'scoped-required', + 'dependency-closure', + false, + 'destroy' + )).not.toBe(closure); + }); + + it('keeps exact introspection client destruction explicitly opt-in', () => { + expect(parseCatalogIntrospectionClientReleaseMode([])).toBe('reuse'); + expect(parseCatalogIntrospectionClientReleaseMode([ + '--introspection-client-release-mode', 'destroy' + ])).toBe('destroy'); + }); + + it.each([ + [['--introspection-client-release-mode'], 'requires a value'], + [[ + '--introspection-client-release-mode', 'reuse', + '--introspection-client-release-mode', 'destroy' + ], 'may only be specified once'], + [[ + '--introspection-client-release-mode', 'discard' + ], "must be 'reuse' or 'destroy'"] + ])('rejects malformed introspection release arguments %j', (args, message) => { + expect(() => parseCatalogIntrospectionClientReleaseMode(args)).toThrow(message); + }); + + it('keeps build-state retirement explicitly opt-in', () => { + expect(parseCatalogBuildStateRetirement([])).toBe(false); + expect(parseCatalogBuildStateRetirement([ + '--release-build-state-after-validation' + ])).toBe(true); + expect(() => parseCatalogBuildStateRetirement([ + '--release-build-state-after-validation', + '--release-build-state-after-validation' + ])).toThrow('may only be specified once'); + }); +}); + +describe('catalog benchmark V8 runtime provenance', () => { + it('defaults to stock and parses only the named profiles', () => { + expect(parseCatalogV8Profile([])).toBe('stock'); + expect(parseCatalogV8Profile([ + '--v8-profile', 'optimize-for-size' + ])).toBe('optimize-for-size'); + expect(parseCatalogV8Profile([ + '--v8-profile', 'baseline-optimize-for-size' + ])).toBe('baseline-optimize-for-size'); + expect(parseCatalogV8Profile([ + '--v8-profile', 'jitless-optimize-for-size' + ])).toBe('jitless-optimize-for-size'); + }); + + it.each([ + [['--v8-profile'], 'requires a value'], + [[ + '--v8-profile', 'stock', + '--v8-profile', 'optimize-for-size' + ], 'may only be specified once'], + [['--v8-profile', 'jitless'], "v8Profile must be 'stock'"] + ])('rejects malformed V8 profile arguments %j', (args, message) => { + expect(() => parseCatalogV8Profile(args as string[])).toThrow(message as string); + }); + + it('proves the exact configured and observed worker flags', () => { + const runtime = { + heapMiB: 1024, + v8Profile: 'jitless-optimize-for-size' as const, + nodeOptions: '--max-old-space-size=1024', + nodeOptionsArgv: ['--max-old-space-size=1024'], + nodeExecArgv: ['--jitless', '--optimize-for-size', '--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--max-old-space-size=1024', + '--jitless', + '--optimize-for-size', + '--expose-gc' + ] + }; + + expect(() => validateCatalogRuntimeFlags(runtime, { + nodeOptions: runtime.nodeOptions, + nodeOptionsArgv: [...runtime.nodeOptionsArgv], + nodeExecArgv: [...runtime.nodeExecArgv], + effectiveNodeRuntimeFlags: [...runtime.effectiveNodeRuntimeFlags] + })).not.toThrow(); + expect(() => validateCatalogRuntimeFlags(runtime, { + nodeOptions: runtime.nodeOptions, + nodeOptionsArgv: [...runtime.nodeOptionsArgv], + nodeExecArgv: ['--optimize-for-size', '--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--max-old-space-size=1024', + '--optimize-for-size', + '--expose-gc' + ] + })).toThrow('process.execArgv does not match'); + }); + + it('proves the baseline-size worker flags exactly', () => { + const runtime = { + heapMiB: 1024, + v8Profile: 'baseline-optimize-for-size' as const, + nodeOptions: '--max-old-space-size=1024', + nodeOptionsArgv: ['--max-old-space-size=1024'], + nodeExecArgv: ['--max-opt=1', '--optimize-for-size', '--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--max-old-space-size=1024', + '--max-opt=1', + '--optimize-for-size', + '--expose-gc' + ] + }; + expect(() => validateCatalogRuntimeFlags(runtime, runtime)).not.toThrow(); + }); + + it('rejects a managed profile flag hidden in NODE_OPTIONS', () => { + expect(() => validateCatalogRuntimeFlags({ + heapMiB: 1024, + v8Profile: 'stock', + nodeOptions: '--jitless --max-old-space-size=1024', + nodeOptionsArgv: ['--jitless', '--max-old-space-size=1024'], + nodeExecArgv: ['--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--jitless', + '--max-old-space-size=1024', + '--expose-gc' + ] + }, { + nodeOptions: '--jitless --max-old-space-size=1024', + nodeOptionsArgv: ['--jitless', '--max-old-space-size=1024'], + nodeExecArgv: ['--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--jitless', + '--max-old-space-size=1024', + '--expose-gc' + ] + })).toThrow('configured Node runtime flags are inconsistent'); + }); +}); + +describe('catalog benchmark PostgreSQL backend lifecycle', () => { + const backendIdentity = { + pid: 101, + backendStartEpochMs: 1_700_000_000_500 + }; + const replacementIdentity = { + pid: 202, + backendStartEpochMs: 1_700_000_010_500 + }; + const backendStatus = (input: { + name?: string; + namespacePid?: number; + rssKiB?: number; + highWaterKiB?: number; + } = {}): string => [ + `Name:\t${input.name ?? 'postgres'}`, + `NSpid:\t70000\t${input.namespacePid ?? 101}`, + `VmHWM:\t${input.highWaterKiB ?? 2048} kB`, + `VmRSS:\t${input.rssKiB ?? 1024} kB` + ].join('\n'); + + const backendMeasurement = (input: { + source?: 'docker-container-procfs-diagnostic' | 'local-linux-procfs'; + } = {}) => summarizeCatalogBackendMemorySamples({ + backendIdentity, + samplerPid: 501, + source: input.source ?? 'docker-container-procfs-diagnostic', + postgresContainer: 'postgres-density', + samples: [ + { + monotonicMs: 1_000, + rssBytes: 100, + highWaterBytes: 150, + procStartTicks: 50_000, + procStartEpochMs: 1_700_000_000_000, + bootTimeEpochSeconds: 1_699_999_500, + clockTicksPerSecond: 100 + }, + { + monotonicMs: 1_010, + rssBytes: 180, + highWaterBytes: 220, + procStartTicks: 50_000, + procStartEpochMs: 1_700_000_000_000, + bootTimeEpochSeconds: 1_699_999_500, + clockTicksPerSecond: 100 + }, + { + monotonicMs: 1_020, + rssBytes: 140, + highWaterBytes: 240, + procStartTicks: 50_000, + procStartEpochMs: 1_700_000_000_000, + bootTimeEpochSeconds: 1_699_999_500, + clockTicksPerSecond: 100 + } + ], + targetExitedBeforeStop: true, + targetExitedAtMonotonicMs: 1_030, + samplerStartedAt: '2026-08-02T00:00:00.000Z', + samplerReadyAt: '2026-08-02T00:00:00.005Z', + buildStartedAt: '2026-08-02T00:00:00.010Z', + buildCompletedAt: '2026-08-02T00:00:00.040Z', + samplerStopRequestedAt: '2026-08-02T00:00:00.045Z', + samplerStoppedAt: '2026-08-02T00:00:00.050Z', + buildDurationMs: 30, + clientPlatform: 'linux', + clientArchitecture: 'x64' + }); + + it('binds proc status to the exact PostgreSQL namespace PID', () => { + expect(parseCatalogBackendProcStatus(backendStatus(), 101)).toEqual({ + rssBytes: 1024 * 1024, + highWaterBytes: 2048 * 1024 + }); + expect(() => parseCatalogBackendProcStatus( + backendStatus({ namespacePid: 202 }), + 101 + )).toThrow('identity did not match exact PID 101'); + expect(() => parseCatalogBackendProcStatus( + backendStatus({ name: 'node' }), + 101 + )).toThrow('identity did not match exact PID 101'); + expect(() => parseCatalogBackendProcStatus( + backendStatus().replace(/^VmHWM:.*$/m, ''), + 101 + )).toThrow('valid VmHWM'); + }); + + it('rejects container arguments that Docker could parse as options', () => { + expect(() => validateCatalogPostgresContainer('postgres-density.1')).not.toThrow(); + expect(() => validateCatalogPostgresContainer('--privileged')).toThrow( + "invalid PostgreSQL container name '--privileged'" + ); + expect(() => validateCatalogPostgresContainer('postgres/density')).toThrow( + 'invalid PostgreSQL container name' + ); + }); + + it('pins diagnostic Docker launches to an immutable ID and clears shell env', () => { + const secretVariableNames = [ + 'PGPASSWORD', + 'DATABASE_URL', + 'GRAPHQL_OBSERVABILITY_TOKEN', + 'AWS_SECRET_ACCESS_KEY' + ]; + const environment = { + PATH: '/usr/bin', + HOME: '/tmp/test-home', + DOCKER_HOST: 'unix:///tmp/docker.sock', + ...Object.fromEntries(secretVariableNames.map((name) => [name, `value-${name}`])) + }; + const containerIdentity = parseCatalogDockerContainerIdentity( + `${'a'.repeat(64)}\t2026-08-02T00:00:00.000Z\t70000`, + 'postgres-density' + ); + const launch = makeCatalogBackendSamplerLaunchSpec({ + backendIdentity, + containerIdentity, + clientPlatform: 'darwin', + environment + })!; + + expect(catalogBackendSamplerEnvironment(environment)).toEqual({ + PATH: '/usr/bin', + HOME: '/tmp/test-home', + DOCKER_HOST: 'unix:///tmp/docker.sock' + }); + expect(launch.command).toBe('docker'); + expect(launch.args.slice(0, 7)).toEqual([ + 'exec', + '-i', + 'a'.repeat(64), + '/usr/bin/env', + '-i', + 'PATH=/usr/bin:/bin', + '/bin/sh' + ]); + expect(launch.hostEnvironmentVariableNames).toEqual([ + 'DOCKER_HOST', + 'HOME', + 'PATH' + ]); + const serializedLaunch = JSON.stringify(launch); + for (const name of secretVariableNames) { + expect(launch.hostEnvironmentVariableNames).not.toContain(name); + expect(serializedLaunch).not.toContain(`value-${name}`); + } + const script = launch.args[launch.args.indexOf('-c') + 1]; + expect(script).toContain('trap cleanup_sampler EXIT'); + expect(script).toContain('wait "$sampler_pid"'); + }); + + it('fails immutable container revalidation even when a backend PID matches', () => { + const expected = parseCatalogDockerContainerIdentity( + `${'a'.repeat(64)}\t2026-08-02T00:00:00.000Z\t70000`, + 'postgres-density' + ); + const wrongContainer = parseCatalogDockerContainerIdentity( + `${'b'.repeat(64)}\t2026-08-02T00:00:00.000Z\t70001`, + 'postgres-density' + ); + expect(() => assertCatalogDockerContainerIdentity( + expected, + wrongContainer + )).toThrow('changed immutable identity'); + }); + + it('records identity-bound sampled peaks only as diagnostic lower bounds', () => { + expect(backendMeasurement()).toEqual(expect.objectContaining({ + backendPid: 101, + backendStartEpochMs: 1_700_000_000_500, + baselineRssBytes: 100, + baselineHighWaterBytes: 150, + sampledPeakRssLowerBoundBytes: 180, + sampledHighWaterLowerBoundBytes: 240, + sampledPeakRssDeltaLowerBoundBytes: 80, + sampledHighWaterDeltaLowerBoundBytes: 90, + sampleCount: 3, + targetExitedBeforeStop: true, + timing: expect.objectContaining({ + configuredIntervalMs: 10, + maximumConclusiveGapMs: 50, + maximumObservedGapMs: 10, + coveredBuildWindow: true, + cadenceConclusive: true, + samplerLaunchToReadyMs: 5, + samplerStopRequestToCloseMs: 5 + }), + observerEffect: expect.objectContaining({ + correctionApplied: false, + pairedComparisonSupported: true, + measuredLaunchToReadyMs: 5, + measuredStopRequestToCloseMs: 5 + }), + provenance: expect.objectContaining({ + samplerProcess: 'dedicated-external-procfs-loop', + samplerPid: 501, + source: 'docker-container-procfs-diagnostic', + backendSamplerAuthority: 'diagnostic-only', + serviceDensityMemoryAuthority: + 'separately-validated-linux-cgroup-v2-memory.current', + semantics: 'diagnostic-lower-bound-without-pre-destroy-acknowledgement', + dockerInitialExecEnvironment: + 'may-inherit-container-config-before-env-i', + samplerShellEnvironment: 'env-i-path-only', + backendIdentity: expect.objectContaining({ + sqlBackendStartEpochMs: 1_700_000_000_500, + procStartTicks: 50_000, + toleranceMs: 1_500 + }) + }) + })); + expect(backendMeasurement().provenance.limitation).toContain( + 'diagnostic lower bound' + ); + }); + + it('labels Docker transport separately without changing lower-bound semantics', () => { + const measurement = backendMeasurement(); + expect(measurement.timing.cadenceConclusive).toBe(true); + expect(measurement.provenance.backendSamplerAuthority).toBe('diagnostic-only'); + expect(measurement.provenance.limitation).toContain('Docker Desktop'); + expect(measurement.provenance.limitation).toContain( + 'separately validated Linux cgroup-v2 memory.current' + ); + }); + + it('rejects a changed proc start token and an out-of-tolerance SQL identity', () => { + const changedToken = backendMeasurement({ source: 'local-linux-procfs' }); + const mismatchedSamples = [ + { + monotonicMs: 1_000, + rssBytes: 100, + highWaterBytes: 150, + procStartTicks: 60_000, + procStartEpochMs: 1_700_000_010_000, + bootTimeEpochSeconds: 1_699_999_500, + clockTicksPerSecond: 100 + } + ]; + expect(changedToken.provenance.backendIdentity.procStartTicks).toBe(50_000); + expect(() => summarizeCatalogBackendMemorySamples({ + backendIdentity, + samplerPid: 501, + source: 'local-linux-procfs', + postgresContainer: null, + samples: mismatchedSamples, + targetExitedBeforeStop: false, + samplerStartedAt: '2026-08-02T00:00:00.000Z', + samplerReadyAt: '2026-08-02T00:00:00.005Z', + buildStartedAt: '2026-08-02T00:00:00.010Z', + buildCompletedAt: '2026-08-02T00:00:00.040Z', + samplerStopRequestedAt: '2026-08-02T00:00:00.045Z', + samplerStoppedAt: '2026-08-02T00:00:00.050Z', + buildDurationMs: 30 + })).toThrow('mismatched process start identity'); + }); + + it('marks sparse cadence inconclusive without promoting its lower bound', () => { + const measurement = summarizeCatalogBackendMemorySamples({ + backendIdentity, + samplerPid: 501, + source: 'local-linux-procfs', + postgresContainer: null, + samples: [ + { + monotonicMs: 1_000, + rssBytes: 100, + highWaterBytes: 150, + procStartTicks: 50_000, + procStartEpochMs: 1_700_000_000_000, + bootTimeEpochSeconds: 1_699_999_500, + clockTicksPerSecond: 100 + }, + { + monotonicMs: 1_075, + rssBytes: 180, + highWaterBytes: 220, + procStartTicks: 50_000, + procStartEpochMs: 1_700_000_000_000, + bootTimeEpochSeconds: 1_699_999_500, + clockTicksPerSecond: 100 + } + ], + targetExitedBeforeStop: false, + samplerStartedAt: '2026-08-02T00:00:00.000Z', + samplerReadyAt: '2026-08-02T00:00:00.005Z', + buildStartedAt: '2026-08-02T00:00:00.010Z', + buildCompletedAt: '2026-08-02T00:00:00.070Z', + samplerStopRequestedAt: '2026-08-02T00:00:00.075Z', + samplerStoppedAt: '2026-08-02T00:00:00.080Z', + buildDurationMs: 60 + }); + + expect(measurement.timing.maximumObservedGapMs).toBe(75); + expect(measurement.timing.cadenceConclusive).toBe(false); + expect(measurement.sampledHighWaterLowerBoundBytes).toBe(220); + expect(measurement.sampledHighWaterDeltaLowerBoundBytes).toBe(70); + expect(measurement.provenance.backendSamplerAuthority).toBe('diagnostic-only'); + expect(measurement.provenance.limitation).toContain('maximum observed gap'); + }); + + it('supports explicit sampler-on and sampler-off observer comparisons', () => { + expect(parseCatalogBackendSamplerMode([])).toBe('diagnostic-lower-bound'); + expect(parseCatalogBackendSamplerMode([ + '--postgres-backend-sampler', 'off' + ])).toBe('off'); + expect(() => parseCatalogBackendSamplerMode([ + '--postgres-backend-sampler', 'exact' + ])).toThrow("must be 'off' or 'diagnostic-lower-bound'"); + }); + + it('stops an already-exited worker without signaling a reused process group', async () => { + const requestGracefulStop = jest.fn(); + const signalProcessGroup = jest.fn(); + await expect(stopCatalogBackendSamplerProcessTree({ + requestGracefulStop, + waitForTreeExit: async () => true, + signalProcessGroup + })).resolves.toBe('already-exited'); + expect(requestGracefulStop).not.toHaveBeenCalled(); + expect(signalProcessGroup).not.toHaveBeenCalled(); + }); + + it('escalates bounded cleanup through TERM and KILL until no tree remains', async () => { + const waits = [false, false, false, true]; + const signalProcessGroup = jest.fn(); + await expect(stopCatalogBackendSamplerProcessTree({ + requestGracefulStop: jest.fn(), + waitForTreeExit: async () => waits.shift()!, + signalProcessGroup, + gracefulTimeoutMs: 1, + termTimeoutMs: 1, + killTimeoutMs: 1 + })).resolves.toBe('sigkill'); + expect(waits).toHaveLength(0); + expect(signalProcessGroup.mock.calls).toEqual([ + ['SIGTERM'], + ['SIGKILL'] + ]); + }); + + it('fails cleanup when the process tree survives KILL', async () => { + await expect(stopCatalogBackendSamplerProcessTree({ + requestGracefulStop: jest.fn(), + waitForTreeExit: async () => false, + signalProcessGroup: jest.fn(), + gracefulTimeoutMs: 1, + termTimeoutMs: 1, + killTimeoutMs: 1 + })).rejects.toThrow('survived SIGKILL'); + }); + + it('still reaps the tree when the graceful stop write fails', async () => { + const waits = [false, false, true]; + const signalProcessGroup = jest.fn(); + await expect(stopCatalogBackendSamplerProcessTree({ + requestGracefulStop: () => { + throw new Error('stdin failed'); + }, + waitForTreeExit: async () => waits.shift()!, + signalProcessGroup, + gracefulTimeoutMs: 1, + termTimeoutMs: 1, + killTimeoutMs: 1 + })).rejects.toThrow('stdin failed'); + expect(waits).toHaveLength(0); + expect(signalProcessGroup).toHaveBeenCalledWith('SIGTERM'); + }); + + it('starts before the build and stops before PID retirement and replacement', async () => { + const order: string[] = []; + const measurement = backendMeasurement(); + const sampled = await measureCatalogBuildWithBackendSampler({ + startSampler: async () => { + order.push('sampler:start'); + return { + stop: async () => { + order.push('sampler:stop'); + return measurement; + } + }; + }, + build: async () => { + order.push('build'); + return 'built'; + } + }); + const transition = await resolveCatalogBackendPidAfterBuild( + 'destroy', + backendIdentity, + { + waitForRetirement: async () => { + order.push('backend:retired'); + }, + acquireBackendIdentity: async () => { + order.push('replacement:acquired'); + return replacementIdentity; + } + } + ); + + expect(sampled.value).toBe('built'); + expect(sampled.backendMemoryLowerBound).toBe(measurement); + expect(transition.steadyBackendPid).toBe(202); + expect(order).toEqual([ + 'sampler:start', + 'build', + 'sampler:stop', + 'backend:retired', + 'replacement:acquired' + ]); + }); + + it('still stops the sampler when the Graphile build fails', async () => { + const stop = jest.fn(async () => backendMeasurement()); + await expect(measureCatalogBuildWithBackendSampler({ + startSampler: async () => ({ stop }), + build: async () => { + throw new Error('build failed'); + } + })).rejects.toThrow('build failed'); + expect(stop).toHaveBeenCalledTimes(1); + }); + + it('fails the build result when the configured sampler fails', async () => { + await expect(measureCatalogBuildWithBackendSampler({ + startSampler: async () => ({ + stop: async () => { + throw new Error('sampler failed'); + } + }), + build: async () => 'built' + })).rejects.toThrow('sampler failed'); + }); + + it('keeps the same backend in reuse mode without a retirement probe', async () => { + const waitForRetirement = jest.fn(async (): Promise => undefined); + const acquireBackendIdentity = jest.fn(async () => backendIdentity); + + await expect(resolveCatalogBackendPidAfterBuild('reuse', backendIdentity, { + waitForRetirement, + acquireBackendIdentity + })).resolves.toEqual({ + introspectionBackendPid: 101, + introspectionBackendStartEpochMs: 1_700_000_000_500, + steadyBackendPid: 101, + steadyBackendStartEpochMs: 1_700_000_000_500, + introspectionBackendRetired: false + }); + expect(waitForRetirement).not.toHaveBeenCalled(); + expect(acquireBackendIdentity).toHaveBeenCalledTimes(1); + }); + + it('proves retirement before acquiring and recording the replacement', async () => { + const order: string[] = []; + const waitForRetirement = jest.fn(async ( + identity: typeof backendIdentity + ): Promise => { + order.push(`retired:${identity.pid}`); + }); + const acquireBackendIdentity = jest.fn(async () => { + order.push('acquired:202'); + return replacementIdentity; + }); + + await expect(resolveCatalogBackendPidAfterBuild('destroy', backendIdentity, { + waitForRetirement, + acquireBackendIdentity + })).resolves.toEqual({ + introspectionBackendPid: 101, + introspectionBackendStartEpochMs: 1_700_000_000_500, + steadyBackendPid: 202, + steadyBackendStartEpochMs: 1_700_000_010_500, + introspectionBackendRetired: true + }); + expect(order).toEqual(['retired:101', 'acquired:202']); + }); + + it('fails closed on unexpected PID reuse or rotation', async () => { + await expect(resolveCatalogBackendPidAfterBuild('destroy', backendIdentity, { + waitForRetirement: async () => undefined, + acquireBackendIdentity: async () => ({ + pid: 101, + backendStartEpochMs: 1_700_000_020_500 + }) + })).rejects.toThrow('destroyed PostgreSQL introspection backend 101 was reused'); + await expect(resolveCatalogBackendPidAfterBuild('reuse', backendIdentity, { + waitForRetirement: async () => undefined, + acquireBackendIdentity: async () => replacementIdentity + })).rejects.toThrow('PostgreSQL benchmark backend identity changed'); + }); +}); + +describe('catalog benchmark cache warmth', () => { + it('preserves the disabled defaults', () => { + expect(parseCatalogWarmthCliOptions([])).toEqual({ + warmOperationsPerInstance: 0, + warmOperationReplayPasses: 0, + grafastCacheLimits: { + queryCacheMaxLength: null, + operationsCacheMaxLength: null, + operationOperationPlansCacheMaxLength: null + } + }); + }); + + it('parses independently configurable positive cache limits', () => { + expect(parseCatalogWarmthCliOptions([ + '--warm-operations-per-instance', '500', + '--warm-operation-replay-passes', '3', + '--grafast-query-cache-max', '8', + '--grafast-operations-cache-max', '16', + '--grafast-operation-plans-cache-max', '32' + ])).toEqual({ + warmOperationsPerInstance: 500, + warmOperationReplayPasses: 3, + grafastCacheLimits: { + queryCacheMaxLength: 8, + operationsCacheMaxLength: 16, + operationOperationPlansCacheMaxLength: 32 + } + }); + }); + + it.each([ + [['--warm-operations-per-instance'], 'requires a value'], + [['--warm-operations-per-instance', '-1'], 'non-negative integer'], + [['--warm-operations-per-instance', '1.5'], 'non-negative integer'], + [['--warm-operations-per-instance', '01'], 'non-negative integer'], + [['--warm-operation-replay-passes'], 'requires a value'], + [['--warm-operation-replay-passes', '-1'], 'non-negative integer'], + [['--warm-operation-replay-passes', '1.5'], 'non-negative integer'], + [['--warm-operation-replay-passes', '01'], 'non-negative integer'], + [['--grafast-query-cache-max', '0'], 'positive safe integer'], + [['--grafast-query-cache-max', '1'], 'safe integer of at least 2'], + [['--grafast-operations-cache-max', '-1'], 'positive integer'], + [['--grafast-operation-plans-cache-max', '1e2'], 'positive integer'], + [[ + '--grafast-query-cache-max', '8', + '--grafast-query-cache-max', '16' + ], 'may only be specified once'] + ])('rejects malformed warmth arguments %j', (args, message) => { + expect(() => parseCatalogWarmthCliOptions(args as string[])).toThrow(message as string); + }); + + it('rejects incomplete worker cache-limit configuration', () => { + expect(() => validateCatalogWarmthConfig({ + warmOperationsPerInstance: 1, + warmOperationReplayPasses: 0, + grafastCacheLimits: { + queryCacheMaxLength: null, + operationsCacheMaxLength: null + } as never + })).toThrow('must define all three cache limit fields'); + }); + + it('requires a populated source set when replay is enabled', () => { + expect(() => parseCatalogWarmthCliOptions([ + '--warm-operation-replay-passes', '1' + ])).toThrow( + 'warmOperationReplayPasses requires warmOperationsPerInstance' + ); + }); + + it('generates stable, distinct, valid source operations', () => { + const sources = Array.from( + { length: 500 }, + (_, index) => makeCatalogWarmOperationSource(index + 1) + ); + expect(new Set(sources).size).toBe(500); + expect(sources[0]).toBe( + 'query CatalogWarm1 { warmTenantToken: tenantToken }' + ); + expect(() => sources.forEach((source) => parse(source))).not.toThrow(); + expect(() => makeCatalogWarmOperationSource(0)).toThrow('positive safe integer'); + }); + + it('uses the nearest-rank percentile deterministically', () => { + expect(catalogPercentile([], 0.5)).toBeNull(); + expect(catalogPercentile([9, 1, 5, 3], 0.5)).toBe(3); + expect(catalogPercentile([9, 1, 5, 3], 0.99)).toBe(9); + expect(() => catalogPercentile([1], 0)).toThrow('percentile probability'); + }); + + it('summarizes sampled and process-high-water build transients', () => { + expect(summarizeBuildTransientSamples( + { heapUsedBytes: 100, rssBytes: 200, processPeakRssBytes: 250 }, + [ + { heapUsedBytes: 100, rssBytes: 200, processPeakRssBytes: 250 }, + { heapUsedBytes: 170, rssBytes: 260, processPeakRssBytes: 300 }, + { heapUsedBytes: 140, rssBytes: 240, processPeakRssBytes: 320 } + ] + )).toEqual({ + baselineHeapUsedBytes: 100, + baselineRssBytes: 200, + sampledPeakHeapUsedBytes: 170, + sampledPeakHeapDeltaBytes: 70, + sampledPeakRssBytes: 260, + sampledPeakRssDeltaBytes: 60, + processPeakRssBytes: 320, + processPeakRssDeltaBytes: 70, + sampleCount: 3 + }); + }); +}); + +describe('catalog benchmark tenant-density projection', () => { + it('is opt-in and parses one positive surface count', () => { + expect(parseCatalogTenantProxySurfaces([])).toBeNull(); + expect(parseCatalogTenantProxySurfaces([ + '--tenant-proxy-surfaces', '5' + ])).toBe(5); + }); + + it.each([ + [['--tenant-proxy-surfaces'], 'requires a value'], + [['--tenant-proxy-surfaces', '0'], 'positive safe integer'], + [['--tenant-proxy-surfaces', '1.5'], 'positive integer'], + [['--tenant-proxy-surfaces', '05'], 'positive integer'], + [[ + '--tenant-proxy-surfaces', '5', + '--tenant-proxy-surfaces', '6' + ], 'may only be specified once'] + ])('rejects malformed tenant proxy arguments %j', (args, message) => { + expect(() => parseCatalogTenantProxySurfaces(args as string[])).toThrow( + message as string + ); + }); + + it('projects the 350-instance result across configured old space and peak RSS', () => { + const density = projectCatalogTenantDensity({ + tenantProxySurfaces: 5, + configuredOldSpaceMiB: 1024, + snapshot: { + instances: 350, + processPeakRssBytes: 1_100_939_264, + processPeakRssDeltaBytes: 956_203_008 + } + }); + + expect(density).toMatchObject({ + residentSurfaceInstances: 350, + fullTenantProxyGroups: 70, + remainderSurfaceInstances: 0, + configuredOldSpaceMiB: 1024, + absolutePeakProcessRssBytes: 1_100_939_264, + groupsPerConfiguredOldSpaceGiB: 70 + }); + expect(density.groupsPerAbsolutePeakProcessRssGiB).toBeCloseTo( + 68.27073040061909, + 10 + ); + expect(density.groupsPerAbsolutePeakProcessRssGiB).not.toBeCloseTo( + 78.60457146773585, + 10 + ); + }); + + it('counts only full proxy groups and records leftover surface instances', () => { + expect(projectCatalogTenantDensity({ + tenantProxySurfaces: 5, + configuredOldSpaceMiB: 2048, + snapshot: { + instances: 24, + processPeakRssBytes: 2 ** 30, + processPeakRssDeltaBytes: 2 ** 29 + } + })).toEqual({ + residentSurfaceInstances: 24, + fullTenantProxyGroups: 4, + remainderSurfaceInstances: 4, + configuredOldSpaceMiB: 2048, + absolutePeakProcessRssBytes: 2 ** 30, + groupsPerConfiguredOldSpaceGiB: 2, + groupsPerAbsolutePeakProcessRssGiB: 4 + }); + }); +}); + +describe('catalog benchmark crash progress', () => { + it('atomically replaces one credential-free progress artifact', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-progress-')); + const resultFile = path.join(directory, 'result.json'); + const snapshot: CatalogMemorySnapshot = { + instances: 25, + heapUsedBytes: 100, + heapDeltaBytes: 50, + rssBytes: 200, + rssDeltaBytes: 75, + externalBytes: 10, + externalDeltaBytes: 1, + processPeakRssBytes: 220, + processPeakRssDeltaBytes: 80, + postgresBackendRssBytes: null, + postgresBackendRssDeltaBytes: null, + postgresBackendHighWaterBytes: null, + postgresBackendHighWaterDeltaBytes: null + }; + try { + writeCatalogProgress(resultFile, { + version: 1, + status: 'in-progress', + mode: 'scoped-required', + scopedCatalogTypes: 'dependency-closure', + introspectionClientReleaseMode: 'destroy', + postgresBackendSamplerMode: 'diagnostic-lower-bound', + releaseBuildStateAfterValidation: true, + repetition: 1, + heapMiB: 1024, + v8Profile: 'jitless-optimize-for-size', + nodeOptions: '--max-old-space-size=1024', + nodeOptionsArgv: ['--max-old-space-size=1024'], + nodeExecArgv: ['--jitless', '--optimize-for-size', '--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--max-old-space-size=1024', + '--jitless', + '--optimize-for-size', + '--expose-gc' + ], + targetInstances: 500, + completedInstances: 25, + configuredCheckpoints: [25, 500], + completedCheckpoints: [25], + buildsCompleted: 25, + canariesCompleted: 50, + mismatchViolations: 0, + crossTenantViolations: 0, + lastSnapshot: snapshot, + updatedAt: '2026-08-01T00:00:00.000Z' + }); + const progressFile = catalogProgressPath(resultFile); + expect(JSON.parse(fs.readFileSync(progressFile, 'utf8'))).toMatchObject({ + status: 'in-progress', + completedInstances: 25, + lastSnapshot: { instances: 25 } + }); + expect(fs.readdirSync(directory)).toEqual(['progress.json']); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/perf-harness/src/__tests__/config.test.ts b/packages/perf-harness/src/__tests__/config.test.ts new file mode 100644 index 0000000000..d8e07e6414 --- /dev/null +++ b/packages/perf-harness/src/__tests__/config.test.ts @@ -0,0 +1,875 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + armEnvironmentForHeap, + assertIsolatedPort, + assertLoopbackObservabilityUrl, + assertLoopbackRetainedHeapCheckpointUrl, + loadFleet, + loadPlan, + resolveTemplate, + tenantCountsForHeap, + validateAcceptanceGates, + validateCoverage, + validateWorkloadPlan} from '../config'; +import type { AcceptanceGates, DensityPlanV1, FleetV1 } from '../types'; + +const validGates: AcceptanceGates = { + maxErrorRate: 0.005, + maxP99Ms: 150, + maxPostWarmupHeapGrowthMiBPerHour: 5, + minMedianDensityImprovement: 0.15, + minAdditionalTenantsEveryRun: 1, + requireZeroBleed: true, + requireNoPostWarmupEvictions: true, + requireNoPostWarmupBuildRefusals: true, + requireNoPostWarmupBuilds: true, + requirePostgresMemoryTelemetry: false, + requireFreshPostgresRunAttestation: false, + requireRetainedMemoryCheckpoints: false, + requirePhysicalDatabaseTelemetry: false, + requireConclusiveCanaries: true, + requireCompletePeriodicCanaryCoverage: false, + requireConclusiveOperationOracles: false, + requireExplicitCustomerTopology: false, + requiredCacheAdmissionMode: null +}; + +describe('density harness configuration', () => { + it.each(Object.keys(validGates) as Array)( + 'fails closed when acceptance gate %s is omitted', + (key) => { + const malformed: Partial = { ...validGates }; + delete malformed[key]; + expect(() => validateAcceptanceGates(malformed as AcceptanceGates)).toThrow( + `plan.gates.${key}` + ); + } + ); + + it('validates optional aligned-memory cadence and workload-coverage gates', () => { + expect(() => validateAcceptanceGates({ + ...validGates, + maxAlignedMemorySampleGapMs: 1_000, + minAlignedMemoryCoverageRatio: 0.99 + })).not.toThrow(); + expect(() => validateAcceptanceGates({ + ...validGates, + maxAlignedMemorySampleGapMs: 0 + })).toThrow('plan.gates.maxAlignedMemorySampleGapMs must be positive'); + expect(() => validateAcceptanceGates({ + ...validGates, + minAlignedMemoryCoverageRatio: 1.01 + })).toThrow('plan.gates.minAlignedMemoryCoverageRatio must be at most 1'); + }); + + it('refuses shared workspace ports by default', () => { + expect(() => assertIsolatedPort(3000)).toThrow('reserved shared-workspace port'); + expect(() => assertIsolatedPort(5432)).toThrow('reserved shared-workspace port'); + expect(() => assertIsolatedPort(3345)).not.toThrow(); + expect(() => assertIsolatedPort(3000, true)).not.toThrow(); + }); + + it('resolves only known template variables', () => { + expect(resolveTemplate('http://127.0.0.1:{port}/{mode}', { + port: 3345, + mode: 'stock' + })).toBe('http://127.0.0.1:3345/stock'); + expect(() => resolveTemplate('{missing}', {})).toThrow("unknown template variable 'missing'"); + }); + + it('accepts exactly one offered-load mode and validates workload traffic budgets', () => { + const workload = { + durationSec: 900, + rps: 50, + minWorkloadRequestsPerSurface: 10, + requestTimeoutMs: 30_000, + maxInFlight: 128, + canaryIntervalSec: 60, + warmupTimeoutMs: 180_000, + warmupTimeoutPerSurfaceMs: 2_000 + }; + expect(() => validateWorkloadPlan(workload)).not.toThrow(); + expect(() => validateWorkloadPlan({ + ...workload, + rps: undefined, + rpsPerTenant: 0.2 + })).not.toThrow(); + expect(() => validateWorkloadPlan({ ...workload, rpsPerTenant: 1 })) + .toThrow('exactly one'); + expect(() => validateWorkloadPlan({ + ...workload, + rps: undefined, + rpsPerTenant: undefined + })).toThrow('exactly one'); + expect(() => validateWorkloadPlan({ + ...workload, + minWorkloadRequestsPerSurface: 0 + })).toThrow('minWorkloadRequestsPerSurface'); + expect(() => validateWorkloadPlan({ + ...workload, + periodicCanarySchedule: 'rotating-one', + canaryConcurrency: 16 + })).not.toThrow(); + expect(() => validateWorkloadPlan({ + ...workload, + periodicCanarySchedule: 'drop-overlap' as any + })).toThrow('periodicCanarySchedule'); + expect(() => validateWorkloadPlan({ + ...workload, + canaryConcurrency: 0 + })).toThrow('canaryConcurrency'); + }); + + it('resolves heap-specific ramps with a legacy fallback', () => { + const plan = { + tenantCounts: [1, 2], + tenantCountsByHeapMiB: { 2048: [4, 8] } + } as unknown as DensityPlanV1; + expect(tenantCountsForHeap(plan, 1024)).toEqual([1, 2]); + expect(tenantCountsForHeap(plan, 2048)).toEqual([4, 8]); + expect(() => tenantCountsForHeap({} as DensityPlanV1, 4096)) + .toThrow('no tenant-count ramp'); + }); + + it('overrides only the selected heap-specific environment', () => { + const arm = { + env: { SHARED: 'base', OVERRIDE: 'base' }, + envByHeapMiB: { + 1024: { OVERRIDE: 'one', CALIBRATION: 'cal-1' }, + 2048: { OVERRIDE: 'two', CALIBRATION: 'cal-2' } + } + }; + expect(armEnvironmentForHeap(arm, 1024)).toEqual({ + SHARED: 'base', + OVERRIDE: 'one', + CALIBRATION: 'cal-1' + }); + expect(armEnvironmentForHeap(arm, 2048).CALIBRATION).toBe('cal-2'); + }); + + it('requires a complete and exact heap-specific environment map', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-plan-env-')); + const file = path.join(directory, 'plan.json'); + const plan: any = { + version: 1, + fleetFile: 'fleet.json', + artifactDir: 'artifacts', + arms: [{ + name: 'calibrated', + port: 3345, + readinessUrl: 'http://127.0.0.1:3345/healthz', + memoryUrl: 'http://127.0.0.1:3345/debug/memory', + introspectionMode: 'stock', + envByHeapMiB: { 1024: { GRAPHILE_CACHE_CALIBRATION_ID: 'cal-1' } } + }], + heapMiB: [1024, 2048], + tenantCounts: [1], + repetitions: 1, + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + workload: { + durationSec: 900, + rpsPerTenant: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 30_000, + maxInFlight: 1, + canaryIntervalSec: 60, + warmupTimeoutMs: 30_000, + warmupTimeoutPerSurfaceMs: 30_000 + }, + gates: { ...validGates, requireExplicitCustomerTopology: false } + }; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow("envByHeapMiB is missing heap '2048'"); + plan.arms[0].envByHeapMiB['2048'] = { GRAPHILE_CACHE_CALIBRATION_ID: 'cal-2' }; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).not.toThrow(); + plan.arms[0].envByHeapMiB['4096'] = { GRAPHILE_CACHE_CALIBRATION_ID: 'cal-3' }; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow("contains unconfigured heap '4096'"); + }); + + it('validates an enabled soak against an exact configured arm and heap', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-plan-soak-')); + const file = path.join(directory, 'plan.json'); + const plan: any = { + version: 1, + fleetFile: 'fleet.json', + artifactDir: 'artifacts', + arms: [{ + name: 'candidate', + port: 3345, + readinessUrl: 'http://127.0.0.1:3345/healthz', + memoryUrl: 'http://127.0.0.1:3345/debug/memory', + introspectionMode: 'scoped-required' + }], + heapMiB: [1024], + tenantCounts: [1], + repetitions: 1, + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + workload: { + durationSec: 900, + rpsPerTenant: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 30_000, + maxInFlight: 1, + canaryIntervalSec: 60, + warmupTimeoutMs: 30_000, + warmupTimeoutPerSurfaceMs: 30_000 + }, + gates: validGates, + soak: { + enabled: true, + arm: 'candidate', + durationSec: 7_200, + tenantCount: 1, + heapMiB: 1024 + } + }; + const write = (): void => fs.writeFileSync(file, JSON.stringify(plan)); + + write(); + expect(() => loadPlan(file)).not.toThrow(); + plan.soak.heapMiB = 2048; + write(); + expect(() => loadPlan(file)).toThrow('plan.soak.heapMiB=2048 is not configured'); + plan.soak.heapMiB = 1024; + plan.soak.arm = 'missing'; + write(); + expect(() => loadPlan(file)).toThrow("plan.soak.arm 'missing' is not configured"); + plan.soak.arm = 'candidate'; + plan.soak.durationSec = 1.5; + write(); + expect(() => loadPlan(file)).toThrow('plan.soak.durationSec must be a safe integer'); + + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it('sends observability credentials only to the exact loopback memory route', () => { + expect(() => assertLoopbackObservabilityUrl( + 'http://127.0.0.1:3345/debug/memory', + 3345 + )).not.toThrow(); + expect(() => assertLoopbackObservabilityUrl( + 'http://[::1]:3345/debug/memory', + 3345 + )).not.toThrow(); + expect(() => assertLoopbackObservabilityUrl( + 'https://example.com:3345/debug/memory', + 3345 + )).toThrow('memoryUrl must be the credential-free URL'); + expect(() => assertLoopbackObservabilityUrl( + 'http://127.0.0.1:3345/debug/memory?token=secret', + 3345 + )).toThrow('memoryUrl must be the credential-free URL'); + }); + + it('accepts only the exact credential-free retained-memory checkpoint route', () => { + expect(() => assertLoopbackRetainedHeapCheckpointUrl( + 'http://127.0.0.1:3345/__cperf/retained-memory-checkpoint', + 3345 + )).not.toThrow(); + expect(() => assertLoopbackRetainedHeapCheckpointUrl( + 'http://127.0.0.1:3345/__cperf/retained-memory-checkpoint?token=secret', + 3345 + )).toThrow('retainedHeapCheckpointUrl must be the credential-free URL'); + expect(() => assertLoopbackRetainedHeapCheckpointUrl( + 'https://example.com:3345/__cperf/retained-memory-checkpoint', + 3345 + )).toThrow('retainedHeapCheckpointUrl must be the credential-free URL'); + }); + + it('requires spawned arms to expose GC and explicitly enable the checkpoint', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-plan-gc-')); + const file = path.join(directory, 'plan.json'); + const plan: any = { + version: 1, + fleetFile: 'fleet.json', + artifactDir: 'artifacts', + arms: [{ + name: 'candidate', + commit: 'a'.repeat(40), + command: [process.execPath, '/tmp/server.cjs'], + port: 3345, + readinessUrl: 'http://127.0.0.1:3345/healthz', + memoryUrl: 'http://127.0.0.1:3345/debug/memory', + retainedHeapCheckpointUrl: + 'http://127.0.0.1:{port}/__cperf/retained-memory-checkpoint', + introspectionMode: 'stock', + env: {} + }], + heapMiB: [1024], + tenantCounts: [1], + repetitions: 1, + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + workload: { + durationSec: 900, + rpsPerTenant: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 30_000, + maxInFlight: 1, + canaryIntervalSec: 60, + warmupTimeoutMs: 30_000, + warmupTimeoutPerSurfaceMs: 30_000 + }, + gates: { + ...validGates, + requireExplicitCustomerTopology: false, + requireRetainedMemoryCheckpoints: true + } + }; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('--expose-gc'); + plan.arms[0].command.splice(1, 0, '--expose-gc'); + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('GRAPHQL_CPERF_RETAINED_HEAP_ENABLED=true'); + plan.arms[0].env.GRAPHQL_CPERF_RETAINED_HEAP_ENABLED = 'true'; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).not.toThrow(); + plan.arms[0].v8Profile = 'jitless-optimize-for-size'; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).not.toThrow(); + plan.arms[0].v8Profile = 'baseline-optimize-for-size'; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).not.toThrow(); + plan.arms[0].v8Profile = 'jitless-optimize-for-size'; + plan.arms[0].command.splice(1, 0, '--jitless'); + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('managed V8 flags through v8Profile'); + plan.arms[0].command.splice(1, 1); + plan.arms[0].v8Profile = 'arbitrary-flags'; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('unknown v8Profile'); + plan.arms[0].v8Profile = 'stock'; + plan.gates.requiredCacheAdmissionMode = 'drop-resident'; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('requiredCacheAdmissionMode'); + }); + + it('requires a concrete fresh PostgreSQL prepare and server binding', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-plan-pg-run-')); + const file = path.join(directory, 'plan.json'); + const plan: any = { + version: 1, + fleetFile: 'fleet.json', + artifactDir: 'artifacts', + arms: [{ + name: 'candidate', + commit: 'a'.repeat(40), + command: [process.execPath, '/tmp/server.cjs'], + port: 3345, + readinessUrl: 'http://127.0.0.1:3345/healthz', + memoryUrl: 'http://127.0.0.1:3345/debug/memory', + introspectionMode: 'stock' + }], + heapMiB: [1024], + tenantCounts: [1], + repetitions: 1, + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + workload: { + durationSec: 900, + rpsPerTenant: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 30_000, + maxInFlight: 1, + canaryIntervalSec: 60, + warmupTimeoutMs: 30_000, + warmupTimeoutPerSurfaceMs: 30_000 + }, + gates: { + ...validGates, + requireExplicitCustomerTopology: false, + requireFreshPostgresRunAttestation: true + } + }; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('postgresRunAttestation.command'); + plan.arms[0].postgresRunAttestation = { + command: [process.execPath, '/tmp/audit.cjs'], + prepareCommand: [process.execPath, '/tmp/prepare.cjs'] + }; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('does not bind the fresh PostgreSQL fixture'); + plan.arms[0].command.push( + '{postgresManifestFile}', + '{postgresSecretsFile}', + '{postgresManifestSha256}', + '{postgresCloneId}' + ); + plan.arms[0].postgresRunAttestation.prepareCommand.push( + '{postgresFixtureDir}', + '{arm}', + '{heapMiB}', + '{tenantCount}', + '{repetition}', + '{runOrderIndex}' + ); + plan.arms[0].postgresRunAttestation.command.push( + '{postgresManifestFile}', + '{postgresSecretsFile}', + '{attestationFile}', + '{planSha256}', + '{fleetSha256}', + '{notBeforeEpochMs}' + ); + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).not.toThrow(); + }); + + it('rejects a surface without isolation canaries', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-config-')); + const file = path.join(directory, 'fleet.json'); + fs.writeFileSync(file, JSON.stringify({ + version: 1, + tenants: [{ + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: 'tenant-a-api', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [] + }] + }] + })); + expect(() => loadFleet(file)).toThrow('has no isolation canaries'); + }); + + it('rejects a canary that can pass on an empty result', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-config-')); + const file = path.join(directory, 'fleet.json'); + fs.writeFileSync(file, JSON.stringify({ + version: 1, + tenants: [{ + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: 'tenant-a-api', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ __typename }', + forbiddenMatches: [{ path: '/data/token', value: 'tenant-b' }] + }] + }] + }] + })); + expect(() => loadFleet(file)).toThrow('requiredMatches'); + }); + + it('validates paired operation oracles and post-coverage verification queries', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-operation-oracle-')); + const file = path.join(directory, 'fleet.json'); + const operation: any = { + name: 'upload', + capability: 'uploads', + query: 'mutation { upload { id } }', + postCoverageVerification: { + query: 'query ($fileId: UUID!, $contentHash: String!) { uploadedFiles(where: { id: { equalTo: $fileId }, contentHash: { equalTo: $contentHash } }) { nodes { physicalDatabaseIdentity } } }', + variables: { contentHash: 'fixture-hash' }, + variablesFromResponse: { fileId: '/data/upload/id' }, + requiredMatches: [{ + path: '/data/uploadedFiles/nodes/0/physicalDatabaseIdentity', + value: 'physical-db-a' + }], + forbiddenMatches: [{ + path: '/data/uploadedFiles/nodes/0/physicalDatabaseIdentity', + value: 'physical-db-b' + }], + invariants: [{ + path: '/data/uploadedFiles/nodes/*/physicalDatabaseIdentity', + everyEquals: 'physical-db-a', + min: 1, + max: 1 + }] + } + }; + const fleet = { + version: 1, + tenants: [{ + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: 'tenant-a-api', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [operation], + canaries: [{ + name: 'cross-schema', + query: '{ __typename }', + requiredMatches: [{ path: '/data/token', value: 'tenant-a' }], + forbiddenMatches: [{ path: '/data/token', value: 'tenant-b' }] + }] + }] + }] + }; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).not.toThrow(); + + operation.postCoverageVerification.variablesFromResponse.contentHash = + '/data/upload/contentHash'; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow('collides with a static variable'); + delete operation.postCoverageVerification.variablesFromResponse.contentHash; + + operation.postCoverageVerification.invariants[0].min = 0; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow('min must be a positive safe integer'); + operation.postCoverageVerification.invariants[0].min = 1; + + delete operation.postCoverageVerification.forbiddenMatches; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow( + 'postCoverageVerification.forbiddenMatches' + ); + + delete operation.postCoverageVerification; + operation.requiredMatches = [{ path: '/data/token', value: 'tenant-a' }]; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow( + 'must configure requiredMatches and forbiddenMatches together' + ); + }); + + it('validates exact realtime probes and keeps sensitive headers environment-backed', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-realtime-fleet-')); + const file = path.join(directory, 'fleet.json'); + const surface: any = { + name: 'api', + buildContract: 'customer-a-api', + url: 'http://127.0.0.1:3345/customer/customer-a/tenant/a/graphql', + headers: { 'accept-language': 'es' }, + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ token }', + requiredMatches: [{ path: '/data/token', value: 'tenant-a' }], + forbiddenMatches: [{ path: '/data/token', value: 'tenant-b' }] + }], + realtime: { + headersFromEnvironment: { authorization: 'CPERF_RUNTIME_TOKEN' }, + subscription: { + query: 'subscription { event { token } }', + requiredMatches: [{ path: '/data/event/token', value: 'tenant-a' }], + forbiddenMatches: [{ path: '/data/event/token', value: 'tenant-b' }] + }, + prime: { + query: 'mutation Prime($payload: String!) { prime(payload: $payload) { token payload } }', + variables: { payload: 'configured-placeholder' }, + requiredMatches: [{ path: '/data/prime/token', value: 'tenant-a' }], + forbiddenMatches: [{ path: '/data/prime/token', value: 'tenant-b' }] + }, + correlation: { + primeVariable: 'payload', + primeResponsePath: '/data/prime/payload', + subscriptionEventPath: '/data/event/payload' + } + } + }; + const fleet: any = { + version: 1, + tenants: [{ + id: 'customer-a', + databases: [{ + id: 'database-a', + physicalDatabase: 'customer_a', + apis: [{ + id: 'api-a', + runtimePoolIdentity: 'pg:v1:customer-a', + physicalSchemas: ['tenant_a'], + routingLabels: ['customer-a'], + realtime: true, + surfaces: ['api'] + }] + }], + surfaces: [surface] + }] + }; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).not.toThrow(); + + surface.realtime.subscription.requiredMatches.push({ + path: '/data/event/payload', + value: 'configured-placeholder' + }); + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow( + 'correlation paths must not carry a static required match' + ); + surface.realtime.subscription.requiredMatches.pop(); + + surface.realtime.correlation.subscriptionEventPath = '/data/event/*/payload'; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow('realtime.correlation is invalid'); + surface.realtime.correlation.subscriptionEventPath = '/data/event/payload'; + + surface.realtime.correlation.subscriptionEventPath = '/data/event/~2payload'; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow('realtime.correlation is invalid'); + surface.realtime.correlation.subscriptionEventPath = '/data/event/payload'; + + surface.headers.authorization = 'persisted-secret'; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow( + 'authorization must use realtime.headersFromEnvironment' + ); + }); + + it('fails coverage when a required canary is absent from any surface', () => { + const fleet = { + version: 1, + tenants: [{ + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: 'tenant-a-api', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ canary }', + forbiddenMatches: [{ path: '/data/canary', value: 'tenant-b' }], + requiredMatches: [{ path: '/data/canary', value: 'tenant-a' }] + }] + }] + }] + } as FleetV1; + const plan = { + tenantCounts: [1], + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema', 'prepared-reuse'] + } as DensityPlanV1; + expect(() => validateCoverage(plan, fleet)).toThrow('lacks canaries: prepared-reuse'); + }); + + it('requires every tenant to configure every required capability', () => { + const surface = (tenant: string, capability: string) => ({ + name: 'api', + buildContract: `${tenant}-api`, + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability, query: '{ __typename }' }, + operations: [{ name: 'read', capability, query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ token }', + forbiddenMatches: [{ path: '/data/token', value: 'other' }], + requiredMatches: [{ path: '/data/token', value: tenant }] + }] + }); + const fleet = { + version: 1, + tenants: [ + { id: 'tenant-a', surfaces: [surface('tenant-a', 'graphile')] }, + { id: 'tenant-b', surfaces: [surface('tenant-b', 'bm25')] } + ] + } as FleetV1; + const plan = { + tenantCounts: [2], + requiredCapabilities: ['graphile', 'bm25'], + requiredCanaries: ['cross-schema'] + } as DensityPlanV1; + + expect(() => validateCoverage(plan, fleet)).toThrow( + 'tenant-a has no operations for capabilities: bm25' + ); + }); + + it('requires an exact contract for every arm when arm-specific identities are used', () => { + const fleet = { + version: 1, + tenants: [{ + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: '', + buildContracts: { stock: 'stock-hash' }, + url: 'http://127.0.0.1:{port}/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ token }', + forbiddenMatches: [{ path: '/data/token', value: 'tenant-b' }], + requiredMatches: [{ path: '/data/token', value: 'tenant-a' }] + }] + }] + }] + } as FleetV1; + const plan = { + arms: [ + { name: 'stock' }, + { name: 'scoped' } + ], + tenantCounts: [1], + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'] + } as DensityPlanV1; + + expect(() => validateCoverage(plan, fleet)).toThrow( + 'lacks exact build contracts for arms: scoped' + ); + }); + + it('rejects one build contract reused across different tenants', () => { + const makeTenant = (id: string) => ({ + id, + surfaces: [{ + name: 'api', + buildContract: 'shared-contract', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ token }', + forbiddenMatches: [{ path: '/data/token', value: 'other' }], + requiredMatches: [{ path: '/data/token', value: id }] + }] + }] + }); + const fleet = { + version: 1, + tenants: [makeTenant('tenant-a'), makeTenant('tenant-b')] + } as FleetV1; + const plan = { + tenantCounts: [1, 2], + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'] + } as DensityPlanV1; + expect(() => validateCoverage(plan, fleet)).toThrow( + "build contract 'shared-contract' for arm 'default' is reused across tenants" + ); + }); + + it('requires an explicit customer/database/API map for qualifying fleets', () => { + const surface = { + name: 'api', + buildContract: 'tenant-a-build', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ token }', + forbiddenMatches: [{ path: '/data/token', value: 'tenant-b' }], + requiredMatches: [{ path: '/data/token', value: 'tenant-a' }] + }] + }; + const fleet = { + version: 1, + tenants: [{ id: 'customer-a', surfaces: [surface] }] + } as FleetV1; + const plan = { + tenantCounts: [1], + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + gates: { requireExplicitCustomerTopology: true } + } as DensityPlanV1; + expect(() => validateCoverage(plan, fleet)).toThrow( + 'customer-a has no explicit customer -> database -> API topology' + ); + }); + + it('rejects one runtime pool identity reused across customers', () => { + const customer = (id: string) => ({ + id, + databases: [{ + id: `${id}-database`, + physicalDatabase: 'fixture', + apis: [{ + id: `${id}-api`, + runtimePoolIdentity: 'pg:v1:shared', + physicalSchemas: [`${id}_api`], + routingLabels: [`${id}.api.localhost`], + realtime: false, + surfaces: ['api'] + }] + }], + surfaces: [{ + name: 'api', + buildContract: `${id}-build`, + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ token }', + forbiddenMatches: [{ path: '/data/token', value: 'other' }], + requiredMatches: [{ path: '/data/token', value: id }] + }] + }] + }); + const fleet = { + version: 1, + tenants: [customer('customer-a'), customer('customer-b')] + } as FleetV1; + const plan = { + tenantCounts: [1, 2], + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + gates: { requireExplicitCustomerTopology: true } + } as DensityPlanV1; + expect(() => validateCoverage(plan, fleet)).toThrow( + "runtime pool identity 'pg:v1:shared' for arm 'default' is reused across customers" + ); + }); + + it('rejects a strict rotating qualification with fewer rounds than canaries', () => { + const canaries = Array.from({ length: 4 }, (_, index) => ({ + name: `canary-${index}`, + query: '{ token }', + forbiddenMatches: [{ path: '/data/token', value: 'tenant-b' }], + requiredMatches: [{ path: '/data/token', value: 'tenant-a' }] + })); + const fleet = { + version: 1, + tenants: [{ + id: 'customer-a', + surfaces: [{ + name: 'api', + buildContract: 'customer-a-api', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries + }] + }] + } as FleetV1; + const plan = { + tenantCounts: [1], + requiredCapabilities: ['graphile'], + requiredCanaries: canaries.map((canary) => canary.name), + workload: { + durationSec: 120, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 60, + periodicCanarySchedule: 'rotating-one', + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100 + }, + gates: { + requireExplicitCustomerTopology: false, + requireCompletePeriodicCanaryCoverage: true + } + } as DensityPlanV1; + expect(() => validateCoverage(plan, fleet)).toThrow( + 'rotating periodic canary schedule has 1 timed rounds but a qualifying surface configures 4 canaries' + ); + plan.workload.durationSec = 300; + expect(() => validateCoverage(plan, fleet)).not.toThrow(); + }); +}); diff --git a/packages/perf-harness/src/__tests__/evidence.test.ts b/packages/perf-harness/src/__tests__/evidence.test.ts new file mode 100644 index 0000000000..ea73dbca94 --- /dev/null +++ b/packages/perf-harness/src/__tests__/evidence.test.ts @@ -0,0 +1,244 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + readRealtimeCoverageEvidence, + readScoreContextEvidence, + scoreContextFromInput, + writeScoreContext +} from '../evidence'; +import { summarizeRealtimeReceiptEvidence } from '../realtime-evidence'; +import type { ScoreInput } from '../score'; +import type { RealtimeCorrelationReceipt } from '../types'; + +const hash = (value: string): string => createHash('sha256') + .update(value) + .digest('hex'); + +const artifactDir = (): string => fs.mkdtempSync( + path.join(os.tmpdir(), 'cperf-evidence-test-') +); + +const contextInput = ( + command: string[] = ['node', 'server.cjs', '--secrets', '/tmp/runtime-secrets.json'], + executionErrors: string[] = [] +): ScoreInput => ({ + arm: 'stock', + evidenceMode: 'diagnostic', + runKind: 'matrix', + heapMiB: 2048, + tenants: [{ + id: 'customer-secret-id', + surfaces: [{ + name: 'api', + buildContract: 'customer-secret-contract', + url: 'http://127.0.0.1:3000/graphql', + headers: { authorization: 'Bearer customer-secret-token' } + }] + }], + repetition: 1, + runOrderIndex: 1, + startedAt: '2026-08-02T00:00:00.000Z', + endedAt: '2026-08-02T00:00:05.000Z', + configuredDurationSec: 5, + serverExit: null, + externalServer: false, + executionErrors, + provenance: { command }, + provenanceErrors: [], + postgresRunAttestation: null +} as unknown as ScoreInput); + +const metadata = (knownRuntimeSecretValues: readonly string[] = []) => ({ + planSha256: 'a'.repeat(64), + fleetSha256: 'b'.repeat(64), + campaignId: 'c'.repeat(64), + scheduleSha256: 'd'.repeat(64), + previousResultPayloadSha256: null as string | null, + notBeforeEpochMs: Date.parse('2026-08-02T00:00:00.000Z'), + knownRuntimeSecretValues +}); + +const receipt = ( + sequence: number, + nonce: string, + timed = true +): RealtimeCorrelationReceipt => { + const sha256 = hash(nonce); + return { + sequence, + timed, + deadlineAt: '2026-08-02T00:02:00.000Z', + issuedAt: '2026-08-02T00:01:00.000Z', + issuedSha256: sha256, + primeResponseAt: '2026-08-02T00:01:00.010Z', + primeResponseSha256: sha256, + eventAt: '2026-08-02T00:01:00.020Z', + eventSha256: sha256 + }; +}; + +const realtimeSurface = ( + tenantId: string, + nonce: string +) => ({ + tenantId, + surface: 'api', + route: `/customer/${tenantId}/graphql`, + active: true, + verified: true, + deliveryEvents: 1, + deliveryRoundsStarted: 1, + deliveryRoundsVerified: 1, + deliveryRoundPending: false, + timedRoundsExpected: 1, + timedRoundsStarted: 1, + timedRoundsVerified: 1, + timedRoundsDeadlineLate: 0, + correlationReceipts: [receipt(1, nonce)] +}); + +const realtimeSnapshot = (surfaces: ReturnType[]) => { + const coverage = summarizeRealtimeReceiptEvidence({ + deliveryIntervalMs: 60_000, + workloadStartedAt: '2026-08-02T00:00:00.000Z', + workloadDeadlineAt: '2026-08-02T00:02:00.000Z', + workloadEndedAt: '2026-08-02T00:02:00.000Z', + surfaces: surfaces.map((surface) => ({ + tenantId: surface.tenantId, + surface: surface.surface, + route: surface.route, + expectedRecurringRounds: surface.timedRoundsExpected, + startedRecurringRounds: surface.timedRoundsStarted, + verifiedRecurringRounds: surface.timedRoundsVerified, + deadlineLateRecurringRounds: surface.timedRoundsDeadlineLate, + receipts: surface.correlationReceipts + })) + }).coverage; + return { + expected: surfaces.length, + active: surfaces.length, + verified: surfaces.length, + deliveryIntervalMs: 60_000, + deliveryEvents: surfaces.length, + deliveryRoundsStarted: 1, + deliveryRoundsVerified: 1, + deliveryRoundsPending: 0, + timedCoverage: coverage, + errors: [] as string[], + surfaces + }; +}; + +describe('density score evidence', () => { + it('persists only credential-free scoring context', () => { + const dir = artifactDir(); + const knownSecret = 'known-runtime-secret-marker'; + writeScoreContext(dir, contextInput(), metadata([knownSecret])); + + const serialized = fs.readFileSync(path.join(dir, 'score-context.json'), 'utf8'); + expect(serialized).not.toContain(knownSecret); + expect(serialized).not.toContain('customer-secret-id'); + expect(serialized).not.toContain('customer-secret-contract'); + expect(serialized).not.toContain('customer-secret-token'); + expect(serialized).not.toContain('authorization'); + expect(serialized).toContain('/tmp/runtime-secrets.json'); + }); + + it.each([ + ['known runtime value', ['node', '--label=known-runtime-secret-marker'], ['known-runtime-secret-marker']], + ['separate password', ['node', '--password', 'literal-password'], []], + ['URL userinfo', ['node', 'postgres://runtime:literal-password@localhost/db'], []], + ['URL token parameter', ['node', 'https://localhost/start?token=literal-token'], []], + ['authorization header', ['node', 'Authorization: Bearer literal-token'], []] + ])('rejects credential-bearing provenance: %s', (_label, command, knownSecrets) => { + expect(() => scoreContextFromInput( + contextInput(command), + metadata(knownSecrets) + )).toThrow('provenance command contains credential material'); + }); + + it('requires execution failures to contain only a stable code and digest', () => { + expect(() => scoreContextFromInput( + contextInput(undefined, ['CAPACITY']), + metadata() + )).toThrow('code-and-SHA-256 evidence'); + + const safe = `CAPACITY:sha256:${'c'.repeat(64)}`; + expect(scoreContextFromInput( + contextInput(undefined, [safe]), + metadata() + ).executionErrors).toEqual([safe]); + }); + + it('rejects credential material nested anywhere in provenance', () => { + const input = contextInput(['node', 'server.cjs']); + input.provenance = { + ...input.provenance!, + memoryPolicy: { + nested: { + password: 'nested-secret-value' + } + } + } as unknown as ScoreInput['provenance']; + expect(() => scoreContextFromInput(input, metadata())).toThrow( + 'provenance contains credential material at provenance.memoryPolicy.nested.password' + ); + }); + + it('rejects unversioned additions to the persisted context shape', () => { + const dir = artifactDir(); + writeScoreContext(dir, contextInput(), metadata()); + const file = path.join(dir, 'score-context.json'); + const context = JSON.parse(fs.readFileSync(file, 'utf8')); + context.tenants = ['customer-secret-id']; + fs.writeFileSync(file, `${JSON.stringify(context)}\n`, 'utf8'); + + expect(() => readScoreContextEvidence(dir)).toThrow( + 'unexpected=tenants' + ); + }); + + it('rejects a correlation digest reused across tenant routes', () => { + const dir = artifactDir(); + const snapshot = realtimeSnapshot([ + realtimeSurface('customer-a', 'shared-nonce'), + realtimeSurface('customer-b', 'shared-nonce') + ]); + fs.writeFileSync(path.join(dir, 'realtime-driver.json'), `${JSON.stringify([{ + phase: 'timed-coverage-complete', + timestamp: '2026-08-02T00:02:00.000Z', + snapshot + }])}\n`, 'utf8'); + + expect(() => readRealtimeCoverageEvidence(dir)).toThrow( + 'reused realtime receipt digest: customer-b/api' + ); + }); + + it('requires receipt and error histories to be append-only', () => { + const dir = artifactDir(); + const surface = realtimeSurface('customer-a', 'nonce-a'); + const first = realtimeSnapshot([surface]); + first.errors = ['delivery failed']; + const second = realtimeSnapshot([surface]); + fs.writeFileSync(path.join(dir, 'realtime-driver.json'), `${JSON.stringify([ + { + phase: 'failed', + timestamp: '2026-08-02T00:02:00.000Z', + snapshot: first + }, + { + phase: 'disposed-after-failure', + timestamp: '2026-08-02T00:02:01.000Z', + snapshot: second + } + ])}\n`, 'utf8'); + + expect(() => readRealtimeCoverageEvidence(dir)).toThrow( + 'realtime error history is not append-only' + ); + }); +}); diff --git a/packages/perf-harness/src/__tests__/http.test.ts b/packages/perf-harness/src/__tests__/http.test.ts new file mode 100644 index 0000000000..905351f598 --- /dev/null +++ b/packages/perf-harness/src/__tests__/http.test.ts @@ -0,0 +1,792 @@ +import http from 'node:http'; + +import { + createWorkloadCapture, + deterministicCanaryOffset, + deterministicOperationOffset, + jsonPointerValues, + resolveOfferedLoad, + resolveWarmupTimeoutMs, + rotatingCanaryIndex, + runWorkload +} from '../http'; +import type { GraphqlSurface, TenantTarget } from '../types'; + +describe('open-loop workload', () => { + let server: http.Server; + let url: string; + let activeSlowRequests = 0; + let peakSlowRequests = 0; + let slowWarmRequests = 0; + const verificationVariables: Array> = []; + + beforeAll(async () => { + server = http.createServer(async (request, response) => { + let raw = ''; + for await (const chunk of request) raw += String(chunk); + const payload = JSON.parse(raw || '{}') as { + query?: string; + variables?: Record; + }; + const query = payload.query ?? ''; + if (query.includes('Slow')) { + if (query.includes('SlowWarm')) slowWarmRequests++; + activeSlowRequests++; + peakSlowRequests = Math.max(peakSlowRequests, activeSlowRequests); + await new Promise((resolve) => setTimeout(resolve, 40)); + activeSlowRequests--; + } + const physicalDatabaseIdentity = query.includes('ForeignPhysicalOracle') + ? 'physical-db-b' + : query.includes('MissingPhysicalOracle') + ? undefined + : 'physical-db-a'; + response.setHeader('content-type', 'application/json'); + if (query.includes('PartialForeignCanary')) { + response.end(JSON.stringify({ + data: { tenantToken: 'tenant-b-token' }, + errors: [{ message: 'partial resolver failure', extensions: { code: 'PARTIAL' } }] + })); + return; + } + if (query.includes('UniversalRows')) { + const nodes = query.includes('Empty') + ? [] + : query.includes('Foreign') + ? [ + { physicalDatabaseIdentity: 'physical-db-a' }, + { physicalDatabaseIdentity: 'physical-db-b' } + ] + : [{ physicalDatabaseIdentity: 'physical-db-a' }]; + response.end(JSON.stringify({ data: { documents: { nodes } } })); + return; + } + if (query.includes('CorrelatedUploadSubject')) { + response.end(JSON.stringify({ + data: { uploadAppFile: { fileId: 'file-current' } } + })); + return; + } + if (query.includes('AmbiguousCorrelationSubject')) { + response.end(JSON.stringify({ + data: { + uploads: [{ fileId: 'file-one' }, { fileId: 'file-two' }] + } + })); + return; + } + if (query.includes('MissingCorrelationSubject')) { + response.end(JSON.stringify({ data: { uploadAppFile: {} } })); + return; + } + if (query.includes('VerifyCorrelatedUpload')) { + verificationVariables.push(payload.variables ?? {}); + response.end(JSON.stringify({ + data: { + physicalDatabaseIdentity: payload.variables?.fileId === 'file-current' + ? 'physical-db-a' + : 'physical-db-b' + } + })); + return; + } + response.end(JSON.stringify({ + data: { + tenantToken: 'tenant-a-token', + ...(physicalDatabaseIdentity === undefined + ? {} + : { physicalDatabaseIdentity }) + }, + extensions: { note: 'tenant-b-token appears outside the asserted path' } + })); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('test server has no TCP address'); + url = `http://127.0.0.1:${address.port}/graphql`; + }); + + afterAll(async () => { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + }); + + const surface = ( + name: string, + warmupQuery = '{ tenantToken }', + operationQuery = '{ tenantToken }' + ): GraphqlSurface => ({ + name, + buildContract: `tenant-a-${name}`, + url, + warmup: { name: 'warm', capability: 'generated', query: warmupQuery }, + operations: [ + { name: 'generated', capability: 'generated', weight: 0.1, query: operationQuery }, + { name: 'search', capability: 'bm25', weight: 0.1, query: operationQuery } + ], + canaries: [{ + name: 'cross-schema', + query: '{ tenantToken }', + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b-token' }], + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a-token' }] + }] + }); + + it('stably staggers weighted operation cursors across tenant surfaces', () => { + const offsets = [ + deterministicOperationOffset('physical-customer-0001', 'a', 100), + deterministicOperationOffset('physical-customer-0001', 'b', 100), + deterministicOperationOffset('physical-customer-0002', 'a', 100) + ]; + expect(new Set(offsets).size).toBe(3); + expect(deterministicOperationOffset('physical-customer-0001', 'a', 100)) + .toBe(offsets[0]); + expect(offsets.every((offset) => offset >= 0 && offset < 100)).toBe(true); + expect(deterministicOperationOffset('tenant', 'api', 0)).toBe(0); + }); + + it('fails operation samples closed with stable missing and forbidden oracle codes', async () => { + const run = async (query: string) => { + const api = surface('api', '{ tenantToken }', query); + api.operations = [{ + name: 'physical-read', + capability: 'generated', + query, + requiredMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-a' + }], + forbiddenMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-b' + }] + }]; + return runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + }; + + const missing = await run('query MissingPhysicalOracle { physicalDatabaseIdentity }'); + expect(missing.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: false, + oracleConfigured: true, + oracleConclusive: false, + oracleViolation: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_MISSING' + }); + + const forbidden = await run('query ForeignPhysicalOracle { physicalDatabaseIdentity }'); + expect(forbidden.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: false, + oracleConfigured: true, + oracleViolation: true, + errorCode: 'GRAPHQL_OPERATION_ORACLE_FORBIDDEN' + }); + }); + + it('enforces nonempty cardinality and every-value invariants across collections', async () => { + const run = async (query: string, requiredMatches: any[]) => { + const api = surface('api', '{ tenantToken }', query); + api.operations = [{ + name: 'universal-read', + capability: 'generated', + query, + requiredMatches, + forbiddenMatches: [{ + path: '/data/documents/nodes/*/physicalDatabaseIdentity', + value: 'physical-db-c' + }], + invariants: [{ + path: '/data/documents/nodes/*/physicalDatabaseIdentity', + everyEquals: 'physical-db-a', + min: 1, + max: 1 + }] + }]; + return runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + }; + + const foreign = await run('query UniversalRowsForeign { documents { nodes { physicalDatabaseIdentity } } }', [{ + path: '/data/documents/nodes/0/physicalDatabaseIdentity', + value: 'physical-db-a' + }]); + expect(foreign.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: false, + oracleConclusive: true, + oracleViolation: true, + oracleUnavailable: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_INVARIANT_UNEXPECTED' + }); + + const empty = await run('query UniversalRowsEmpty { documents { nodes { physicalDatabaseIdentity } } }', [{ + path: '/data/documents/nodes', + value: [] + }]); + expect(empty.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: false, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_INVARIANT_MISSING' + }); + }); + + it('keeps forbidden canary evidence conclusive when GraphQL also returns errors', async () => { + const api = surface('api'); + api.canaries = [{ + name: 'partial-foreign', + query: 'query PartialForeignCanary { tenantToken }', + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a-token' }], + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b-token' }] + }]; + const result = await runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + expect(result.canaries).toHaveLength(2); + expect(result.canaries.every((canary) => + canary.conclusive && canary.violation + )).toBe(true); + expect(result.canaries[0].detail).toBe('GRAPHQL_OPERATION_ORACLE_FORBIDDEN'); + }); + + it('uses an untimed post-coverage query as mutation side-effect evidence', async () => { + const api = surface('api', '{ tenantToken }', 'mutation UploadSubject { __typename }'); + api.operations = [{ + name: 'upload-subject', + capability: 'uploads', + query: 'mutation UploadSubject { __typename }', + postCoverageVerification: { + query: 'query VerifySideEffect { physicalDatabaseIdentity }', + requiredMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-a' + }], + forbiddenMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-b' + }] + } + }]; + const result = await runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + expect(result.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + operation: 'upload-subject', + ok: true, + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: false, + postCoverageVerification: true + }); + }); + + it('extracts post-verification variables exactly from the primary response', async () => { + verificationVariables.length = 0; + const api = surface('api', '{ tenantToken }', 'mutation CorrelatedUploadSubject { uploadAppFile { fileId } }'); + api.operations = [{ + name: 'correlated-upload', + capability: 'uploads', + query: 'mutation CorrelatedUploadSubject { uploadAppFile { fileId } }', + postCoverageVerification: { + query: 'query VerifyCorrelatedUpload($fileId: ID!, $contentHash: String!) { physicalDatabaseIdentity }', + variables: { contentHash: 'current-hash' }, + variablesFromResponse: { fileId: '/data/uploadAppFile/fileId' }, + requiredMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-a' + }], + forbiddenMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-b' + }] + } + }]; + const result = await runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + expect(result.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: true, + oracleConclusive: true, + postCoverageVerification: true + }); + expect(verificationVariables).toEqual([{ + contentHash: 'current-hash', + fileId: 'file-current' + }]); + }); + + it('fails post-verification before I/O on missing or ambiguous correlation evidence', async () => { + const run = async (query: string, pointer: string) => { + const api = surface('api', '{ tenantToken }', query); + api.operations = [{ + name: 'correlation-failure', + capability: 'uploads', + query, + postCoverageVerification: { + query: 'query VerifyCorrelatedUpload($fileId: ID!) { physicalDatabaseIdentity }', + variablesFromResponse: { fileId: pointer }, + requiredMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-a' + }], + forbiddenMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-b' + }] + } + }]; + return runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + }; + + verificationVariables.length = 0; + const missing = await run( + 'mutation MissingCorrelationSubject { uploadAppFile { fileId } }', + '/data/uploadAppFile/fileId' + ); + expect(missing.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: false, + oracleConclusive: false, + oracleUnavailable: false, + errorCode: 'GRAPHQL_POST_COVERAGE_VARIABLE_MISSING' + }); + + const ambiguous = await run( + 'mutation AmbiguousCorrelationSubject { uploads { fileId } }', + '/data/uploads/*/fileId' + ); + expect(ambiguous.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: false, + oracleConclusive: false, + oracleUnavailable: false, + errorCode: 'GRAPHQL_POST_COVERAGE_VARIABLE_AMBIGUOUS' + }); + expect(verificationVariables).toEqual([]); + }); + + it('rotates through a stable, staggered 14-round canary permutation', () => { + const permutation = (tenantId: string, surfaceName: string) => + Array.from({ length: 14 }, (_unused, index) => + rotatingCanaryIndex(tenantId, surfaceName, 14, index + 1) + ); + const first = permutation('physical-customer-0001', 'api'); + expect(new Set(first)).toEqual(new Set(Array.from({ length: 14 }, (_, index) => index))); + expect(permutation('physical-customer-0001', 'api')).toEqual(first); + const offsets = [ + deterministicCanaryOffset('physical-customer-0001', 'api', 14), + deterministicCanaryOffset('physical-customer-0001', 'admin', 14), + deterministicCanaryOffset('physical-customer-0002', 'api', 14) + ]; + expect(new Set(offsets).size).toBeGreaterThan(1); + expect(permutation('physical-customer-0001', 'admin')).not.toEqual(first); + }); + + it('runs four rotating periodic canaries in four strict timed rounds', async () => { + const api = surface('api'); + api.canaries = Array.from({ length: 4 }, (_, index) => ({ + name: `canary-${index}`, + query: '{ tenantToken }', + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b-token' }], + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a-token' }] + })); + const result = await runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.25, + rps: 4, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 4, + canaryIntervalSec: 0.05, + periodicCanarySchedule: 'rotating-one', + canaryConcurrency: 2, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + + const periodic = result.canaries.filter((canary) => canary.phase === 'periodic'); + expect(periodic).toHaveLength(4); + expect(periodic.map((canary) => canary.periodicRound)).toEqual([1, 2, 3, 4]); + expect(new Set(periodic.map((canary) => canary.canary))).toEqual( + new Set(api.canaries.map((canary) => canary.name)) + ); + expect(result.canaries.filter((canary) => canary.phase === 'initial')).toHaveLength(4); + expect(result.canaries.filter((canary) => canary.phase === 'final')).toHaveLength(4); + expect(result.canarySchedule).toMatchObject({ + schedule: 'rotating-one', + planned: 4, + started: 4, + completed: 4, + missed: 0, + checksPlanned: 4, + checksStarted: 4, + checksCompleted: 4 + }); + expect(periodic.every((canary) => + Date.parse(canary.completedAt) >= Date.parse(canary.startedAt) + && canary.latencyMs >= 0 + )).toBe(true); + }); + + it('serializes overlapping rounds without dropping them and records deadline-late drain', async () => { + const api = surface('api'); + api.canaries = [{ + name: 'slow-canary', + query: '{ SlowCanary: tenantToken }', + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b-token' }], + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a-token' }] + }]; + const startedAt = performance.now(); + const result = await runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.1, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 2, + canaryIntervalSec: 0.02, + periodicCanarySchedule: 'rotating-one', + canaryConcurrency: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + + expect(performance.now() - startedAt).toBeLessThan(1_500); + expect(result.canaries.filter((canary) => canary.phase === 'periodic')).toHaveLength(4); + expect(result.canarySchedule).toMatchObject({ + planned: 4, + started: 4, + completed: 4, + missed: 0 + }); + expect(result.canarySchedule.overlapped).toBeGreaterThan(0); + expect(result.canarySchedule.deadlineLate).toBeGreaterThan(0); + expect(result.canarySchedule.rounds.every((round) => + round.targetsStarted === 1 + && round.targetsCompleted === 1 + && round.checksCompleted === 1 + )).toBe(true); + }); + + it('warms every surface and proves every configured operation received traffic', async () => { + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: [surface('api')] + }]; + const result = await runWorkload(tenants, { + durationSec: 0.2, + rps: 10, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 4, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + + expect(result.warmedSurfaces.get('tenant-a')).toEqual(new Set(['api'])); + expect(result.capabilities).toEqual(new Set(['generated', 'bm25'])); + expect(result.capabilitiesByTenantSurface.get('tenant-a/api')) + .toEqual(new Set(['generated', 'bm25'])); + expect(new Set(result.samples.filter((sample) => sample.ok).map((sample) => sample.operation))) + .toEqual(new Set(['generated', 'search'])); + expect(result.canaries.length).toBeGreaterThanOrEqual(2); + expect(result.canaries.every((canary) => canary.conclusive && !canary.violation)).toBe(true); + expect(result.missedArrivals).toBe(0); + }); + + it('signals the warm boundary after coverage and initial canaries but before timed load', async () => { + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: [surface('api'), surface('admin')] + }]; + const capture = createWorkloadCapture(); + let boundaryCalls = 0; + + const result = await runWorkload(tenants, { + durationSec: 0.1, + rps: 20, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 4, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 2 + }, async () => { + boundaryCalls++; + expect(capture.warmedSurfaces.get('tenant-a')).toEqual(new Set(['api', 'admin'])); + expect(capture.samples.filter((sample) => sample.phase === 'coverage')).toHaveLength(4); + expect(capture.samples.filter((sample) => sample.phase === 'workload')).toHaveLength(0); + expect(capture.capabilitiesByTenantSurface.get('tenant-a/api')) + .toEqual(new Set(['generated', 'bm25'])); + expect(capture.capabilitiesByTenantSurface.get('tenant-a/admin')) + .toEqual(new Set(['generated', 'bm25'])); + expect(capture.canaries).toHaveLength(2); + expect(capture.canaries.every((canary) => canary.conclusive && !canary.violation)) + .toBe(true); + }, capture); + + expect(boundaryCalls).toBe(1); + expect(result.samples.some((sample) => sample.phase === 'workload')).toBe(true); + expect(result.canaries.length).toBeGreaterThan(2); + }); + + it('fails closed before timed load when the warm-boundary callback rejects', async () => { + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: [surface('api')] + }]; + const capture = createWorkloadCapture(); + + await expect(runWorkload(tenants, { + durationSec: 0.1, + rps: 20, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 4, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }, async () => { + expect(capture.samples.filter((sample) => sample.phase === 'coverage')).toHaveLength(2); + expect(capture.canaries).toHaveLength(1); + throw new Error('warm-boundary setup failed'); + }, capture)).rejects.toThrow('warm-boundary setup failed'); + + expect(capture.samples.filter((sample) => sample.phase === 'workload')).toHaveLength(0); + }); + + it('submits each surface canary sequentially so validation cannot monopolize its pool', async () => { + peakSlowRequests = 0; + const api = surface('api'); + api.canaries = Array.from({ length: 4 }, (_, index) => ({ + name: `slow-canary-${index}`, + query: `{ SlowCanary${index}: tenantToken }`, + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b-token' }], + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a-token' }] + })); + + const result = await runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 8, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + + expect(result.canaries).toHaveLength(8); + expect(result.canaries.every((canary) => canary.conclusive && !canary.violation)) + .toBe(true); + expect(peakSlowRequests).toBe(1); + }); + + it('uses typed JSON pointers, including wildcards, instead of raw response substrings', () => { + const body = { + data: { rows: [{ token: 'tenant-a' }, { token: 'tenant-b' }] }, + extensions: { note: 'tenant-c' } + }; + expect(jsonPointerValues(body, '/data/rows/*/token')).toEqual(['tenant-a', 'tenant-b']); + expect(jsonPointerValues(body, '/data/missing')).toEqual([]); + expect(jsonPointerValues(body, '/extensions/note')).toEqual(['tenant-c']); + }); + + it('bounds concurrent warmups with the configured limit', async () => { + peakSlowRequests = 0; + slowWarmRequests = 0; + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: Array.from({ length: 5 }, (_, index) => + surface(`api-${index}`, `{ SlowWarm${index}: tenantToken }`) + ) + }]; + + const result = await runWorkload(tenants, { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 8, + canaryIntervalSec: 1, + warmupTimeoutMs: 2_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 2 + }); + + expect(result.warmedSurfaces.get('tenant-a')?.size).toBe(5); + expect(peakSlowRequests).toBeLessThanOrEqual(2); + }); + + it('uses one global warmup deadline instead of starting queued work after expiry', async () => { + slowWarmRequests = 0; + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: Array.from({ length: 5 }, (_, index) => + surface(`api-${index}`, `{ SlowWarm${index}: tenantToken }`) + ) + }]; + + const result = await runWorkload(tenants, { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 10, + warmupTimeoutPerSurfaceMs: 1, + warmupConcurrency: 1 + }); + + expect(slowWarmRequests).toBeLessThanOrEqual(1); + expect(result.warmedSurfaces.get('tenant-a')).toBeUndefined(); + }); + + it('records saturated arrivals as failures without dispatching a catch-up burst', async () => { + peakSlowRequests = 0; + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: [surface('api', '{ tenantToken }', '{ SlowOperation: tenantToken }')] + }]; + + const result = await runWorkload(tenants, { + durationSec: 0.12, + rps: 100, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + + expect(result.missedArrivals).toBeGreaterThan(0); + expect(result.samples.filter((sample) => + sample.errorCode === 'LOAD_GENERATOR_MISSED_ARRIVAL' + )).toHaveLength(result.missedArrivals); + expect(peakSlowRequests).toBe(1); + }); + + it('measures workload latency from the scheduled open-loop arrival', async () => { + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: [surface('api', '{ tenantToken }', '{ SlowOperation: tenantToken }')] + }]; + + const result = await runWorkload(tenants, { + durationSec: 0.08, + rps: 20, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 2, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + + const dispatched = result.samples.filter((sample) => + sample.phase === 'workload' + && sample.errorCode !== 'LOAD_GENERATOR_MISSED_ARRIVAL' + ); + expect(dispatched.length).toBeGreaterThan(0); + expect(dispatched.every((sample) => sample.scheduledAtMs != null)).toBe(true); + expect(dispatched.every((sample) => sample.latencyMs >= 35)).toBe(true); + }); + + it('resolves fixed-total and per-tenant offered load explicitly', () => { + expect(resolveOfferedLoad({ rps: 50 }, 10)).toEqual({ + mode: 'fixed-total', + configuredRps: 50, + tenantCount: 10, + totalRps: 50, + rpsPerTenant: 5 + }); + expect(resolveOfferedLoad({ rpsPerTenant: 2 }, 10)).toEqual({ + mode: 'per-tenant', + configuredRps: 2, + tenantCount: 10, + totalRps: 20, + rpsPerTenant: 2 + }); + expect(() => resolveOfferedLoad({ rps: 1, rpsPerTenant: 1 }, 1)) + .toThrow('exactly one'); + }); + + it('scales the global warmup deadline by concurrency waves', () => { + expect(resolveWarmupTimeoutMs({ + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 500, + warmupConcurrency: 2 + }, 10)).toBe(2_500); + expect(resolveWarmupTimeoutMs({ + warmupTimeoutMs: 10_000, + warmupTimeoutPerSurfaceMs: 500, + warmupConcurrency: 2 + }, 10)).toBe(10_000); + }); +}); diff --git a/packages/perf-harness/src/__tests__/memory.test.ts b/packages/perf-harness/src/__tests__/memory.test.ts new file mode 100644 index 0000000000..3378ac4c69 --- /dev/null +++ b/packages/perf-harness/src/__tests__/memory.test.ts @@ -0,0 +1,454 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + normalizeMemorySnapshot, + normalizeRetainedMemoryCheckpoint, + readLinuxProcessMemory, + startMemorySampler +} from '../memory'; + +describe('memory snapshot normalization', () => { + it('normalizes complete retained-memory checkpoints and rejects truncated samples', () => { + const guard = { + pid: 55, + graphileInFlight: 0, + residentBuildContracts: ['contract'], + stateSha256: `sha256:${'a'.repeat(64)}`, + state: { pid: 55, graphileInFlight: 0 } + }; + const raw = { + version: 1, + fixture: 'physical-database-density-v1', + pid: 55, + gcRounds: 8, + stableSampleCount: 3, + stable: true, + samples: Array.from({ length: 8 }, (_, index) => ({ + timestamp: `2026-08-01T00:00:0${index}.000Z`, + monotonicNs: String(index + 1), + heapUsedBytes: 100, + externalBytes: 20, + arrayBuffersBytes: 10, + rssBytes: 200 + })), + guardBefore: guard, + guardAfter: guard, + errors: [] as string[] + }; + expect(normalizeRetainedMemoryCheckpoint(raw)?.samples).toHaveLength(8); + expect(normalizeRetainedMemoryCheckpoint({ ...raw, samples: raw.samples.slice(1) })) + .toBeNull(); + }); + + it('sums stable cache/governor counters', () => { + const snapshot = normalizeMemorySnapshot({ + timestamp: '2026-07-31T00:00:00.000Z', + pid: 123, + nodeEnv: 'production', + memory: { heapUsedBytes: 10, rssBytes: 20 }, + resourceUsage: { maxRSS: 40 }, + v8: { heapStatistics: { heap_size_limit: 1024 } }, + graphileCache: { + size: 3, + max: 9, + admissionMode: 'preserve-resident', + budgetCapacity: 8, + instanceHeapBytes: 16 * 1024 ** 2, + calibration: { id: 'measured-cache-v1' }, + keys: ['build-a', 'build-b'] + }, + graphileCacheCounters: { + evictions: { lru: 1, ttl: 2 }, + buildRefusals: { critical_pressure: 4, resident_busy: 5 } + }, + graphileGovernor: { buildsStarted: 7 }, + graphileBuilds: { succeeded: 6, maxMs: 42 }, + pgCache: { + size: 12, + leasedPools: 4, + activeLeases: 6, + capacityEvictions: 1, + capacityRefusals: 2, + disposalFailures: 3 + }, + physicalDatabaseFixture: { + physicalDatabases: 3, + containerScope: { dedicated: true, unexpectedDatabases: 0 }, + pools: { + scope: 'runtime-only-exact-identities', + available: true, + requestedMaxUses: 1, + effectiveMaxUses: 1, + effectiveMaxUsesKnown: true, + maxUsesExact: true, + expectedPools: 9, + observedPools: 9, + totalClients: 8, + idleClients: 2, + waitingClients: 1 + }, + backends: { total: 9, active: 2, idle: 6, idleInTransaction: 1 }, + realtime: { + managersExpected: 6, + managersActive: 6, + transportsExpected: 6, + transportsActive: 6, + notificationMode: 'shared-exact', + notificationBrokers: { + brokers: 3, + listenerConnections: 3, + leases: 6, + topics: 6, + subscribers: 6, + queueOverflows: 0, + fatalFailures: 0 + }, + notificationRoleAudits: { + identities: 3, + healthy: 3, + failed: 0, + stale: 0, + catalogAuditAttempts: 6, + catalogAuditFailures: 0, + activeDatabaseTargets: 3, + databaseConfigurationConflicts: 0 + } + } + } + }); + expect(snapshot).toMatchObject({ + heapUsedBytes: 10, + rssBytes: 20, + pid: 123, + nodeEnv: 'production', + heapLimitBytes: 1024, + processPeakRssBytes: 40 * 1024, + cacheSize: 3, + cacheConfiguredMax: 9, + cacheBudgetCapacity: 8, + cacheInstanceHeapBytes: 16 * 1024 ** 2, + cacheCalibrationId: 'measured-cache-v1', + cacheAdmissionMode: 'preserve-resident', + residentBuildContracts: ['build-a', 'build-b'], + evictions: 3, + buildRefusals: 9, + buildsStarted: 7, + buildsSucceeded: 6, + buildMaxMs: 42, + pgPoolCacheSize: 12, + pgPoolLeasedPools: 4, + pgPoolActiveLeases: 6, + pgPoolCapacityEvictions: 1, + pgPoolCapacityRefusals: 2, + pgPoolDisposalFailures: 3, + pgPoolTotalClients: 8, + pgPoolIdleClients: 2, + pgPoolWaitingClients: 1, + runtimePoolTelemetryScope: 'runtime-only-exact-identities', + runtimePoolTelemetryAvailable: true, + runtimePoolRequestedMaxUses: 1, + runtimePoolEffectiveMaxUses: 1, + runtimePoolEffectiveMaxUsesKnown: true, + runtimePoolMaxUsesExact: true, + runtimePoolExpectedPools: 9, + runtimePoolObservedPools: 9, + runtimePoolTotalClients: 8, + runtimePoolIdleClients: 2, + runtimePoolWaitingClients: 1, + postgresBackendTotal: 9, + postgresBackendActive: 2, + postgresBackendIdle: 6, + postgresBackendIdleInTransaction: 1, + physicalDatabases: 3, + postgresContainerDedicated: true, + unexpectedPostgresDatabases: 0, + realtimeManagersExpected: 6, + realtimeManagersActive: 6, + realtimeTransportsExpected: 6, + realtimeTransportsActive: 6, + realtimeNotificationMode: 'shared-exact', + notificationBrokers: 3, + notificationListenerConnections: 3, + notificationBrokerLeases: 6, + notificationBrokerTopics: 6, + notificationBrokerSubscribers: 6, + notificationBrokerQueueOverflows: 0, + notificationBrokerFatalFailures: 0, + notificationAuditIdentities: 3, + notificationAuditsHealthy: 3, + notificationAuditsFailed: 0, + notificationAuditsStale: 0, + notificationAuditAttempts: 6, + notificationAuditFailures: 0, + notificationAuditActiveDatabaseTargets: 3, + notificationAuditDatabaseConflicts: 0, + cacheCountersAvailable: true, + buildCountersAvailable: true + }); + }); + + it('keeps missing measurements and counters null for an older endpoint', () => { + const snapshot = normalizeMemorySnapshot({ + graphileCache: {} + }); + expect(snapshot).toMatchObject({ + pid: null, + nodeEnv: null, + heapLimitBytes: null, + heapUsedBytes: null, + rssBytes: null, + processPeakRssBytes: null, + cacheSize: null, + cacheConfiguredMax: null, + cacheBudgetCapacity: null, + cacheInstanceHeapBytes: null, + cacheCalibrationId: null, + residentBuildContracts: null, + evictions: null, + buildRefusals: null, + buildsStarted: null, + buildsSucceeded: null, + buildMaxMs: null, + pgPoolCacheSize: null, + pgPoolLeasedPools: null, + pgPoolActiveLeases: null, + pgPoolCapacityEvictions: null, + pgPoolCapacityRefusals: null, + pgPoolDisposalFailures: null, + cacheCountersAvailable: false, + buildCountersAvailable: false + }); + }); + + it('reads Linux current and high-water RSS from the exact pid status file', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-proc-')); + const pid = 4321; + const pidDir = path.join(root, String(pid)); + fs.mkdirSync(pidDir); + fs.writeFileSync( + path.join(pidDir, 'status'), + 'Name:\tnode\nVmHWM:\t2048 kB\nVmRSS:\t1024 kB\n', + 'utf8' + ); + expect(readLinuxProcessMemory(pid, root)).toEqual({ + rssBytes: 1024 * 1024, + peakRssBytes: 2048 * 1024 + }); + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('reports proc read and malformed-status failures instead of silently dropping samples', () => { + const errors: string[] = []; + expect(readLinuxProcessMemory(9876, '/definitely-not-proc', (error) => { + errors.push(error); + })).toBeNull(); + expect(errors[0]).toContain('OS RSS proc read failed for pid 9876'); + + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-proc-invalid-')); + const pid = 9877; + const pidDir = path.join(root, String(pid)); + fs.mkdirSync(pidDir); + fs.writeFileSync(path.join(pidDir, 'status'), 'Name:\tnode\n', 'utf8'); + expect(readLinuxProcessMemory(pid, root, (error) => errors.push(error))).toBeNull(); + expect(errors.slice(-2)).toEqual([ + 'OS RSS proc status for pid 9877 omitted VmRSS', + 'OS RSS proc status for pid 9877 omitted VmHWM' + ]); + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('binds endpoint identity and heap limit while retaining resourceUsage peak RSS', async () => { + const originalFetch = global.fetch; + const authorization = `Bearer ${'test-observability-token-'.repeat(2)}`; + const fetchMock = jest.fn(async ( + _input: string | URL | Request, + _init?: RequestInit + ) => ({ + ok: true, + json: async () => ({ + timestamp: '2026-08-01T00:00:00.000Z', + pid: 55, + nodeEnv: 'production', + memory: { heapUsedBytes: 100, rssBytes: 200 }, + resourceUsage: { maxRSS: 300 }, + v8: { heapStatistics: { heap_size_limit: 400 } }, + graphileCache: { size: 1, keys: ['contract'] }, + graphileCacheCounters: { evictions: {}, buildRefusals: {} }, + graphileGovernor: { buildsStarted: 0 }, + graphileBuilds: { succeeded: 0, maxMs: 0 }, + pgCache: { + size: 2, + leasedPools: 1, + activeLeases: 1, + capacityEvictions: 0, + capacityRefusals: 0, + disposalFailures: 0 + } + }) + })); + global.fetch = fetchMock as unknown as typeof fetch; + try { + const sampler = startMemorySampler('http://127.0.0.1/debug/memory', { + intervalMs: 60_000, + osSampleIntervalMs: 60_000, + expectedPid: 55, + expectedHeapLimitBytes: 400, + procRoot: '/definitely-not-proc', + headers: { Authorization: authorization } + }); + await sampler.ready; + await sampler.markWarmupComplete(); + expect(sampler.warmupIndex).toBe(1); + expect(sampler.snapshots).toHaveLength(2); + await sampler.stop(); + expect(sampler.errors).toEqual([ + expect.stringContaining('OS RSS proc read failed for pid 55') + ]); + expect(sampler.snapshots[0]).toMatchObject({ + pid: 55, + heapLimitBytes: 400, + processPeakRssBytes: 300 * 1024 + }); + expect(fetchMock.mock.calls.every(([, init]) => + ((init as RequestInit).headers as Record)?.Authorization === authorization + )).toBe(true); + expect(JSON.stringify(sampler).includes(authorization)).toBe(false); + } finally { + global.fetch = originalFetch; + } + }); + + it('samples exact-pid current RSS through the authenticated endpoint', async () => { + const originalFetch = global.fetch; + const authorization = `Bearer ${'darwin-observability-token-'.repeat(2)}`; + const fetchMock = jest.fn(async ( + _input: string | URL | Request, + _init?: RequestInit + ) => ({ + ok: true, + json: async () => ({ + timestamp: '2000-01-01T00:00:00.000Z', + pid: 55, + nodeEnv: 'production', + memory: { heapUsedBytes: 100, rssBytes: 200 }, + resourceUsage: { maxRSS: 300 }, + v8: { heapStatistics: { heap_size_limit: 400 } }, + graphileCache: { size: 1, keys: ['contract'] }, + graphileCacheCounters: { evictions: {}, buildRefusals: {} }, + graphileGovernor: { buildsStarted: 0 }, + graphileBuilds: { succeeded: 0, maxMs: 0 }, + pgCache: { + size: 2, + leasedPools: 1, + activeLeases: 1, + capacityEvictions: 0, + capacityRefusals: 0, + disposalFailures: 0 + } + }) + })); + global.fetch = fetchMock as unknown as typeof fetch; + const startedAtMs = Date.now(); + try { + const sampler = startMemorySampler('http://127.0.0.1/debug/memory', { + intervalMs: 60_000, + osSampleIntervalMs: 60_000, + expectedPid: 55, + expectedHeapLimitBytes: 400, + currentRssSource: 'authenticated-endpoint', + headers: { Authorization: authorization } + }); + await sampler.ready; + await sampler.markWarmupComplete(); + await sampler.stop(); + + expect(sampler.errors).toEqual([]); + expect(sampler.osSnapshots.length).toBeGreaterThanOrEqual(3); + expect(sampler.osSnapshots.every(({ rssBytes, timestamp }) => + rssBytes === 200 + && Date.parse(timestamp) >= startedAtMs + && timestamp !== '2000-01-01T00:00:00.000Z' + )).toBe(true); + expect(sampler.osPeakRssBytes).toBe(300 * 1024); + expect(fetchMock.mock.calls.every(([, init]) => + ((init as RequestInit).headers as Record)?.Authorization + === authorization + )).toBe(true); + } finally { + global.fetch = originalFetch; + } + }); + + it('fails closed when endpoint RSS sampling lacks bearer authorization', async () => { + const originalFetch = global.fetch; + global.fetch = jest.fn(async () => ({ + ok: true, + json: async () => ({ + pid: 55, + nodeEnv: 'production', + memory: { heapUsedBytes: 100, rssBytes: 200 }, + resourceUsage: { maxRSS: 300 }, + v8: { heapStatistics: { heap_size_limit: 400 } }, + pgCache: { + size: 2, + leasedPools: 1, + activeLeases: 1, + capacityEvictions: 0, + capacityRefusals: 0, + disposalFailures: 0 + } + }) + })) as unknown as typeof fetch; + try { + const sampler = startMemorySampler('http://127.0.0.1/debug/memory', { + intervalMs: 60_000, + osSampleIntervalMs: 60_000, + expectedPid: 55, + expectedHeapLimitBytes: 400, + currentRssSource: 'authenticated-endpoint' + }); + await sampler.ready; + await sampler.stop(); + expect(sampler.osSnapshots).toHaveLength(0); + expect(sampler.errors).toContain( + 'authenticated memory-endpoint RSS sampling requires bearer authorization' + ); + } finally { + global.fetch = originalFetch; + } + }); + + it('records identity and heap mismatches instead of accepting the sample silently', async () => { + const originalFetch = global.fetch; + global.fetch = jest.fn(async () => ({ + ok: true, + json: async () => ({ + pid: 56, + nodeEnv: 'development', + memory: { heapUsedBytes: 100, rssBytes: 200 }, + resourceUsage: { maxRSS: 300 }, + v8: { heapStatistics: { heap_size_limit: 401 } } + }) + })) as unknown as typeof fetch; + try { + const sampler = startMemorySampler('http://127.0.0.1/debug/memory', { + intervalMs: 60_000, + osSampleIntervalMs: 60_000, + expectedPid: 55, + expectedHeapLimitBytes: 400, + procRoot: '/definitely-not-proc' + }); + await sampler.ready; + await sampler.stop(); + expect(sampler.errors).toEqual(expect.arrayContaining([ + 'memory endpoint pid mismatch: expected 55, observed 56', + 'memory endpoint NODE_ENV must be production, observed development', + 'V8 heap limit mismatch: expected 400, observed 401' + ])); + } finally { + global.fetch = originalFetch; + } + }); +}); diff --git a/packages/perf-harness/src/__tests__/postgres.test.ts b/packages/perf-harness/src/__tests__/postgres.test.ts new file mode 100644 index 0000000000..6aea1228b5 --- /dev/null +++ b/packages/perf-harness/src/__tests__/postgres.test.ts @@ -0,0 +1,63 @@ +import { + parseCgroupKeyValues, + parseCgroupV2Memory, + parseDockerBytes +} from '../postgres'; + +describe('PostgreSQL container telemetry', () => { + it('parses Docker memory units without decimal loss', () => { + expect(parseDockerBytes('1.5GiB')).toBe(1.5 * 1024 ** 3); + expect(parseDockerBytes('256MiB')).toBe(256 * 1024 ** 2); + expect(parseDockerBytes('unknown')).toBeNull(); + }); + + it('parses raw cgroup-v2 charge, peak, limits, stats, and events', () => { + const raw = [ + '__CPERF_CGROUP_FILE__ memory.current', + '104857600', + '__CPERF_CGROUP_FILE__ memory.peak', + '157286400', + '__CPERF_CGROUP_FILE__ memory.max', + '2147483648', + '__CPERF_CGROUP_FILE__ memory.stat', + 'anon 73400320', + 'file 20971520', + 'shmem 1048576', + '__CPERF_CGROUP_FILE__ memory.events', + 'low 0', + 'high 2', + 'oom 0', + 'oom_kill 0' + ].join('\n'); + expect(parseCgroupV2Memory(raw)).toEqual({ + currentBytes: 104857600, + peakBytes: 157286400, + maxBytes: 2147483648, + stat: { + anon: 73400320, + file: 20971520, + shmem: 1048576 + }, + events: { + low: 0, + high: 2, + oom: 0, + oom_kill: 0 + } + }); + }); + + it('treats an unlimited cgroup max as null and rejects missing current charge', () => { + expect(parseCgroupV2Memory([ + '__CPERF_CGROUP_FILE__ memory.current', + '4096', + '__CPERF_CGROUP_FILE__ memory.max', + 'max' + ].join('\n'))).toMatchObject({ currentBytes: 4096, maxBytes: null }); + expect(parseCgroupV2Memory('__CPERF_CGROUP_FILE__ memory.max\nmax')).toBeNull(); + expect(parseCgroupKeyValues('anon 10\ninvalid\nfile nope\nshmem 20')).toEqual({ + anon: 10, + shmem: 20 + }); + }); +}); diff --git a/packages/perf-harness/src/__tests__/process.test.ts b/packages/perf-harness/src/__tests__/process.test.ts new file mode 100644 index 0000000000..1bb781c5f1 --- /dev/null +++ b/packages/perf-harness/src/__tests__/process.test.ts @@ -0,0 +1,117 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { + collectArmProvenance, + createObservabilityHeaders, + expectedHeapLimitForNodeOptions, + nodeFlagsForV8Profile, + replaceMaxOldSpaceSize +} from '../process'; + +describe('arm process isolation and provenance', () => { + it('creates a fresh, strong bearer header without a separately exposed token', () => { + const first = createObservabilityHeaders(); + const second = createObservabilityHeaders(); + expect(Object.keys(first)).toEqual(['Authorization']); + expect(/^Bearer [A-Za-z0-9_-]+$/.test(first.Authorization)).toBe(true); + expect(Buffer.byteLength(first.Authorization.slice('Bearer '.length))).toBeGreaterThanOrEqual(32); + expect(first.Authorization === second.Authorization).toBe(false); + }); + + it('replaces all inherited max-old-space aliases without dropping other options', () => { + expect(replaceMaxOldSpaceSize( + '--trace-warnings --max-old-space-size=256 --max_old_space_size 512', + 1024 + )).toBe('--trace-warnings --max-old-space-size=1024'); + }); + + it('uses only the closed, named V8 profile flag combinations', () => { + expect(nodeFlagsForV8Profile('stock')).toEqual([]); + expect(nodeFlagsForV8Profile('optimize-for-size')).toEqual([ + '--optimize-for-size' + ]); + expect(nodeFlagsForV8Profile('baseline-optimize-for-size')).toEqual([ + '--max-opt=1', + '--optimize-for-size' + ]); + expect(nodeFlagsForV8Profile('jitless-optimize-for-size')).toEqual([ + '--jitless', + '--optimize-for-size' + ]); + expect(replaceMaxOldSpaceSize( + '--jitless --max-opt=1 --optimize-for-size --trace-warnings', + 1024 + )).toBe('--trace-warnings --max-old-space-size=1024'); + }); + + it('preserves quoted NODE_OPTIONS values while replacing the heap flag', () => { + expect(replaceMaxOldSpaceSize( + '--require "/tmp/a b.js" --max-old-space-size 128', + 2048 + )).toBe('--require "/tmp/a b.js" --max-old-space-size=2048'); + }); + + it('derives the actual V8 heap limit produced by the sanitized options', () => { + const nodeOptions = replaceMaxOldSpaceSize(undefined, 128); + expect(expectedHeapLimitForNodeOptions(nodeOptions)).toBeGreaterThanOrEqual(128 * 1024 ** 2); + }); + + it('records git, lockfile, entry, command, cwd, and server pid provenance', () => { + const cwd = path.resolve(__dirname, '../../../..'); + const entryPath = path.join(cwd, 'packages/perf-harness/src/index.ts'); + const result = collectArmProvenance(cwd, [process.execPath, entryPath], 9876); + expect(result.errors).toEqual([]); + expect(result.provenance).toMatchObject({ + cwd, + command: [process.execPath, entryPath], + serverPid: 9876, + worktreeDirty: expect.any(Boolean), + gitHead: expect.stringMatching(/^[0-9a-f]{40}$/), + gitStatusSha256: expect.stringMatching(/^[0-9a-f]{64}$/), + lockfilePath: path.join(cwd, 'pnpm-lock.yaml'), + entryPath, + v8Profile: 'stock', + nodeOptions: null, + nodeOptionsArgv: [], + nodeExecArgv: [], + effectiveNodeRuntimeFlags: [] + }); + expect(result.provenance.entrySha256).toBe( + createHash('sha256').update(fs.readFileSync(entryPath)).digest('hex') + ); + expect(result.provenance.lockfileSha256).toBe( + createHash('sha256').update(fs.readFileSync(path.join(cwd, 'pnpm-lock.yaml'))).digest('hex') + ); + }); + + it('binds the exact direct and NODE_OPTIONS runtime flags to provenance', () => { + const cwd = path.resolve(__dirname, '../../../..'); + const entryPath = path.join(cwd, 'packages/perf-harness/src/index.ts'); + const command = [ + process.execPath, + '--jitless', + '--optimize-for-size', + '--expose-gc', + entryPath + ]; + const result = collectArmProvenance(cwd, command, 9876, { + v8Profile: 'jitless-optimize-for-size', + nodeOptions: '--trace-warnings --max-old-space-size=1024', + nodeOptionsArgv: ['--trace-warnings', '--max-old-space-size=1024'], + nodeExecArgv: ['--jitless', '--optimize-for-size', '--expose-gc'] + }); + expect(result.provenance).toMatchObject({ + v8Profile: 'jitless-optimize-for-size', + nodeExecArgv: ['--jitless', '--optimize-for-size', '--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--trace-warnings', + '--max-old-space-size=1024', + '--jitless', + '--optimize-for-size', + '--expose-gc' + ] + }); + }); +}); diff --git a/packages/perf-harness/src/__tests__/realtime-evidence.test.ts b/packages/perf-harness/src/__tests__/realtime-evidence.test.ts new file mode 100644 index 0000000000..9cfad48d4c --- /dev/null +++ b/packages/perf-harness/src/__tests__/realtime-evidence.test.ts @@ -0,0 +1,137 @@ +import { createHash } from 'node:crypto'; + +import { summarizeRealtimeReceiptEvidence } from '../realtime-evidence'; +import type { RealtimeCorrelationReceipt } from '../types'; + +const digest = (value: string): string => createHash('sha256') + .update(value) + .digest('hex'); + +const receipt = ( + sequence: number, + issuedAt: string, + primeResponseAt: string, + eventAt: string, + value = `nonce-${sequence}` +): RealtimeCorrelationReceipt => { + const sha256 = digest(value); + const deadlineAt = new Date(Date.parse(issuedAt) + 30_000).toISOString(); + return { + sequence, + timed: true, + deadlineAt, + issuedAt, + issuedSha256: sha256, + primeResponseAt, + primeResponseSha256: sha256, + eventAt, + eventSha256: sha256 + }; +}; + +const evidence = (receipts: RealtimeCorrelationReceipt[]) => ({ + deliveryIntervalMs: 60_000, + workloadStartedAt: '2026-08-02T00:00:00.000Z', + workloadDeadlineAt: '2026-08-02T00:03:00.000Z', + workloadEndedAt: '2026-08-02T00:03:00.000Z', + surfaces: [{ + tenantId: 'customer-a', + surface: 'api-a', + route: '/customer/customer-a/tenant/a/graphql', + expectedRecurringRounds: receipts.length, + startedRecurringRounds: receipts.length, + verifiedRecurringRounds: receipts.length, + deadlineLateRecurringRounds: 0, + receipts + }] +}); + +describe('realtime receipt evidence', () => { + it('derives exact counts, digests, and latency from ordered receipts', () => { + const summary = summarizeRealtimeReceiptEvidence(evidence([ + receipt( + 1, + '2026-08-02T00:01:00.000Z', + '2026-08-02T00:01:00.020Z', + '2026-08-02T00:01:00.040Z' + ), + receipt( + 2, + '2026-08-02T00:02:00.000Z', + '2026-08-02T00:02:00.030Z', + '2026-08-02T00:02:00.050Z' + ) + ])); + + expect(summary.failures).toEqual([]); + expect(summary.coverage).toMatchObject({ + version: 2, + expectedRecurringRounds: 2, + startedRecurringRounds: 2, + verifiedRecurringRounds: 2, + primeRequests: 2, + primeResponseP99Ms: 30, + deliveryP99Ms: 50, + complete: true + }); + expect(summary.coverage.surfaces[0].issuedCorrelationSha256).toBe( + summary.coverage.surfaces[0].verifiedCorrelationSha256 + ); + }); + + it('rejects one digest reused across exact routes', () => { + const shared = receipt( + 1, + '2026-08-02T00:01:00.000Z', + '2026-08-02T00:01:00.020Z', + '2026-08-02T00:01:00.040Z', + 'shared-nonce' + ); + const input = evidence([shared]); + input.surfaces.push({ + ...input.surfaces[0], + tenantId: 'customer-b', + surface: 'api-b', + route: '/customer/customer-b/tenant/b/graphql', + receipts: [{ ...shared }] + }); + + const summary = summarizeRealtimeReceiptEvidence(input); + expect(summary.coverage.complete).toBe(false); + expect(summary.failures).toContain( + 'reused realtime receipt digest: customer-b/api-b' + ); + }); + + it('rejects a response or event timestamp preceding nonce issue', () => { + const summary = summarizeRealtimeReceiptEvidence(evidence([ + receipt( + 1, + '2026-08-02T00:01:00.100Z', + '2026-08-02T00:01:00.000Z', + '2026-08-02T00:01:00.050Z' + ) + ])); + + expect(summary.coverage.complete).toBe(false); + expect(summary.failures).toContain( + 'realtime verified receipt count mismatch: customer-a/api-a' + ); + }); + + it('rejects a self-issued deadline after the externally scheduled slot', () => { + const summary = summarizeRealtimeReceiptEvidence(evidence([ + receipt( + 1, + '2026-08-02T00:02:00.000Z', + '2026-08-02T00:02:00.020Z', + '2026-08-02T00:02:00.040Z' + ) + ])); + + expect(summary.coverage.complete).toBe(false); + expect(summary.failures).toContain( + 'invalid realtime receipt schedule deadline: customer-a/api-a' + ); + }); +}); diff --git a/packages/perf-harness/src/__tests__/realtime.test.ts b/packages/perf-harness/src/__tests__/realtime.test.ts new file mode 100644 index 0000000000..23a9dc1838 --- /dev/null +++ b/packages/perf-harness/src/__tests__/realtime.test.ts @@ -0,0 +1,575 @@ +import { + createRealtimeDriver, + type RealtimeClientFactoryInput, + realtimeHeaders, + realtimeWebSocketUrl +} from '../realtime'; +import type { GraphqlSurface, TenantTarget } from '../types'; + +const surface = ( + customer: string, + tenant: string, + foreignCustomer: string +): GraphqlSurface => { + const payload = `${customer}:${tenant}:resident`; + const physicalDatabaseIdentity = `database-${customer}`; + return { + name: `api-${tenant}`, + buildContract: `${customer}-${tenant}`, + url: `http://127.0.0.1:3410/customer/${customer}/tenant/${tenant}/graphql`, + headers: { 'accept-language': 'es' }, + warmup: { name: 'warm', capability: 'generated', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'generated', query: '{ __typename }' }], + canaries: [{ + name: 'isolation', + query: '{ tenantToken }', + requiredMatches: [{ path: '/data/tenantToken', value: tenant }], + forbiddenMatches: [{ path: '/data/tenantToken', value: 'foreign' }] + }], + realtime: { + headersFromEnvironment: { authorization: 'CPERF_TEST_TOKEN' }, + subscription: { + query: 'subscription Resident { changed { tenantId physicalDatabaseIdentity payload } }', + requiredMatches: [ + { path: '/data/changed/tenantId', value: tenant }, + { + path: '/data/changed/physicalDatabaseIdentity', + value: physicalDatabaseIdentity + } + ], + forbiddenMatches: [ + { path: '/data/changed/tenantId', value: 'foreign' }, + { + path: '/data/changed/physicalDatabaseIdentity', + value: `database-${foreignCustomer}` + } + ] + }, + prime: { + query: 'mutation Prime($payload: String!) { prime(payload: $payload) { tenantId physicalDatabaseIdentity payload } }', + variables: { payload }, + requiredMatches: [ + { path: '/data/prime/tenantId', value: tenant }, + { + path: '/data/prime/physicalDatabaseIdentity', + value: physicalDatabaseIdentity + } + ], + forbiddenMatches: [ + { path: '/data/prime/tenantId', value: 'foreign' }, + { + path: '/data/prime/physicalDatabaseIdentity', + value: `database-${foreignCustomer}` + } + ] + }, + correlation: { + primeVariable: 'payload', + primeResponsePath: '/data/prime/payload', + subscriptionEventPath: '/data/changed/payload' + } + } + }; +}; + +const fleet = (): TenantTarget[] => [ + { id: 'customer-1', surfaces: [surface('customer-1', 'a', 'customer-2')] }, + { id: 'customer-2', surfaces: [surface('customer-2', 'b', 'customer-1')] } +]; + +const waitFor = async (predicate: () => boolean, timeoutMs = 1_000): Promise => { + const deadline = Date.now() + timeoutMs; + while (!predicate() && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 2)); + } + if (!predicate()) throw new Error('TEST_WAIT_TIMEOUT'); +}; + +describe('external realtime driver', () => { + it('uses each exact route, verifies its event, and keeps credentials out of evidence', async () => { + const clients = new Map(); + const created: string[] = []; + const requested: string[] = []; + const previousCorrelationByRoute = new Map(); + const clientFactory = (input: RealtimeClientFactoryInput) => { + created.push(input.url); + const state = { input, unsubscribed: 0, disposed: 0 } as { + input: RealtimeClientFactoryInput; + sink?: { next(value: unknown): void }; + unsubscribed: number; + disposed: number; + }; + clients.set(input.url, state); + return { + subscribe: (_payload: unknown, sink: { next(value: unknown): void }) => { + state.sink = sink; + queueMicrotask(input.onConnected); + return () => { state.unsubscribed++; }; + }, + dispose: async () => { + state.disposed++; + input.onClosed(); + } + }; + }; + const fetchImpl = async (url: string | URL | Request, init?: RequestInit) => { + const href = String(url); + requested.push(href); + expect((init?.headers as Record).authorization).toBe('driver-secret'); + const parsedBody = JSON.parse(String(init?.body)); + const payload = parsedBody.variables.payload as string; + const tenant = href.includes('/tenant/a/') ? 'a' : 'b'; + const customer = href.includes('/customer/customer-1/') ? 'customer-1' : 'customer-2'; + const wsUrl = href.replace(/^http:/, 'ws:'); + // A cursor replay from the same exact tenant/database is legitimate, but + // even a nonce that proved the prior round must not prove this one. + clients.get(wsUrl)!.sink!.next({ + data: { + changed: { + tenantId: tenant, + physicalDatabaseIdentity: `database-${customer}`, + payload: previousCorrelationByRoute.get(href) + ?? 'earlier-valid-event' + } + } + }); + clients.get(wsUrl)!.sink!.next({ + data: { + changed: { + tenantId: tenant, + physicalDatabaseIdentity: `database-${customer}`, + payload + } + } + }); + previousCorrelationByRoute.set(href, payload); + return new Response(JSON.stringify({ + data: { + prime: { + tenantId: tenant, + physicalDatabaseIdentity: `database-${customer}`, + payload + } + } + }), { status: 200, headers: { 'content-type': 'application/json' } }); + }; + const driver = createRealtimeDriver(fleet(), { + concurrency: 1, + timeoutMs: 1_000 + }, { + clientFactory, + fetch: fetchImpl as typeof fetch, + environment: { CPERF_TEST_TOKEN: 'driver-secret' }, + correlationFactory: (surfaceKey, sequence) => + `test-correlation:${surfaceKey}:${sequence}:fresh-round`, + sleep: async () => undefined + }); + + await driver.startAndVerify(); + expect(created).toEqual([ + 'ws://127.0.0.1:3410/customer/customer-1/tenant/a/graphql', + 'ws://127.0.0.1:3410/customer/customer-2/tenant/b/graphql' + ]); + expect(requested).toEqual([ + 'http://127.0.0.1:3410/customer/customer-1/tenant/a/graphql', + 'http://127.0.0.1:3410/customer/customer-2/tenant/b/graphql' + ]); + expect(driver.snapshot()).toMatchObject({ + expected: 2, + active: 2, + verified: 2, + deliveryIntervalMs: 60_000, + deliveryEvents: 2, + deliveryRoundsStarted: 2, + deliveryRoundsVerified: 2, + deliveryRoundsPending: 0, + errors: [] + }); + expect(JSON.stringify(driver.snapshot())).not.toContain('driver-secret'); + driver.assertHealthy(); + + // Later legitimate workload writes change the payload but must preserve + // the permanent tenant/database invariants. + clients.get(created[0])!.sink!.next({ + data: { + changed: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload: 'tenant-a-workload-update' + } + } + }); + driver.assertHealthy(); + + await driver.verifyDeliveryNow(); + expect(driver.snapshot()).toMatchObject({ + deliveryEvents: 4, + deliveryRoundsStarted: 4, + deliveryRoundsVerified: 4, + deliveryRoundsPending: 0 + }); + for (const configured of driver.snapshot().surfaces) { + expect(configured.correlationReceipts).toHaveLength(2); + expect(configured.correlationReceipts.every((receipt) => + receipt.issuedSha256 === receipt.primeResponseSha256 + && receipt.issuedSha256 === receipt.eventSha256 + )).toBe(true); + } + + await driver.dispose(); + expect([...clients.values()].every((client) => + client.unsubscribed === 1 && client.disposed === 1 + )).toBe(true); + expect(driver.snapshot().active).toBe(0); + await driver.dispose(); + expect([...clients.values()].every((client) => client.disposed === 1)).toBe(true); + }); + + it('periodically requires a fresh matching event and never overlaps rounds', async () => { + let sink: { next(value: unknown): void } | null = null; + let primeCalls = 0; + let activePrimeCalls = 0; + let maximumActivePrimeCalls = 0; + let releasePeriodicPrime: (() => void) | null = null; + const periodicPrimeStarted = new Promise((resolve) => { + releasePeriodicPrime = resolve; + }); + let allowPeriodicPrimeToFinish: (() => void) | null = null; + const periodicPrimeCanFinish = new Promise((resolve) => { + allowPeriodicPrimeToFinish = resolve; + }); + const driver = createRealtimeDriver([fleet()[0]], { + concurrency: 1, + timeoutMs: 1_000, + deliveryIntervalMs: 20 + }, { + environment: { CPERF_TEST_TOKEN: 'driver-secret' }, + clientFactory: (input) => ({ + subscribe: (_payload, nextSink) => { + sink = nextSink; + queueMicrotask(input.onConnected); + return () => undefined; + }, + dispose: async () => undefined + }), + fetch: (async (_url, init) => { + primeCalls++; + activePrimeCalls++; + maximumActivePrimeCalls = Math.max(maximumActivePrimeCalls, activePrimeCalls); + const payload = JSON.parse(String(init?.body)).variables.payload; + if (primeCalls === 2) { + releasePeriodicPrime!(); + await periodicPrimeCanFinish; + } + sink!.next({ + data: { + changed: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }); + activePrimeCalls--; + return new Response(JSON.stringify({ + data: { + prime: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }), { status: 200 }); + }) as typeof fetch + }); + + try { + await driver.startAndVerify(); + driver.beginTimedCoverage(80); + await periodicPrimeStarted; + expect(driver.snapshot()).toMatchObject({ + deliveryEvents: 1, + deliveryRoundsStarted: 2, + deliveryRoundsVerified: 1, + deliveryRoundsPending: 1 + }); + + expect(primeCalls).toBe(2); + allowPeriodicPrimeToFinish!(); + await waitFor(() => + driver.snapshot().timedCoverage?.verifiedRecurringRounds === 3 + ); + await new Promise((resolve) => setTimeout(resolve, 25)); + const coverage = await driver.finishTimedCoverage(); + + expect(maximumActivePrimeCalls).toBe(1); + expect(coverage).toMatchObject({ + version: 2, + expectedRecurringRounds: 3, + startedRecurringRounds: 3, + verifiedRecurringRounds: 3, + deadlineLateRecurringRounds: 0, + complete: true, + primeRequests: 3, + surfaces: [{ + tenantId: 'customer-1', + surface: 'api-a', + expectedRecurringRounds: 3, + startedRecurringRounds: 3, + verifiedRecurringRounds: 3 + }] + }); + expect(coverage.surfaces[0].issuedCorrelationSha256).toMatch(/^[a-f0-9]{64}$/); + expect(coverage.surfaces[0].verifiedCorrelationSha256).toBe( + coverage.surfaces[0].issuedCorrelationSha256 + ); + expect(coverage.deliveryP99Ms).toBeGreaterThanOrEqual(0); + expect(driver.snapshot()).toMatchObject({ + deliveryEvents: 4, + deliveryRoundsStarted: 4, + deliveryRoundsVerified: 4, + deliveryRoundsPending: 0, + errors: [] + }); + } finally { + await driver.dispose(); + } + }); + + it('fails when a later round receives no matching event', async () => { + let sink: { next(value: unknown): void } | null = null; + let primeCalls = 0; + const driver = createRealtimeDriver([fleet()[0]], { + concurrency: 1, + timeoutMs: 35, + // Leave enough scheduling headroom that this exercises the event timeout, + // rather than the separate missed-deadline path on a busy test runner. + deliveryIntervalMs: 50 + }, { + environment: { CPERF_TEST_TOKEN: 'driver-secret' }, + clientFactory: (input) => ({ + subscribe: (_payload, nextSink) => { + sink = nextSink; + queueMicrotask(input.onConnected); + return () => undefined; + }, + dispose: async () => undefined + }), + fetch: (async (_url, init) => { + primeCalls++; + const payload = JSON.parse(String(init?.body)).variables.payload; + if (primeCalls === 1) { + sink!.next({ + data: { + changed: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }); + } + return new Response(JSON.stringify({ + data: { + prime: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }), { status: 200 }); + }) as typeof fetch + }); + + try { + await driver.startAndVerify(); + driver.beginTimedCoverage(200); + await waitFor(() => driver.snapshot().errors.length > 0); + expect(primeCalls).toBe(2); + expect(driver.snapshot()).toMatchObject({ + verified: 1, + deliveryEvents: 1, + deliveryRoundsStarted: 2, + deliveryRoundsVerified: 1, + deliveryRoundsPending: 0 + }); + await expect(driver.verifyDeliveryNow()).rejects.toThrow( + 'CPERF_REALTIME_EVENT_TIMEOUT:customer-1/api-a' + ); + } finally { + await driver.dispose(); + } + }); + + it('fails conclusively when the event came from another physical database', async () => { + let sink: { next(value: unknown): void } | null = null; + const driver = createRealtimeDriver([fleet()[0]], { + concurrency: 1, + timeoutMs: 1_000 + }, { + environment: { CPERF_TEST_TOKEN: 'driver-secret' }, + clientFactory: (input) => ({ + subscribe: (_payload, nextSink) => { + sink = nextSink; + queueMicrotask(input.onConnected); + return () => undefined; + }, + dispose: async () => undefined + }), + fetch: (async (_url, init) => { + const payload = JSON.parse(String(init?.body)).variables.payload; + sink!.next({ + data: { + changed: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-2', + payload + } + } + }); + return new Response(JSON.stringify({ + data: { + prime: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }), { status: 200 }); + }) as typeof fetch, + sleep: async () => undefined + }); + + await expect(driver.startAndVerify()).rejects.toThrow( + 'CPERF_REALTIME_FOREIGN_PAYLOAD:customer-1/api-a' + ); + expect(driver.snapshot().verified).toBe(0); + await driver.dispose(); + }); + + it('rejects a correlation digest reused by another exact route', async () => { + const sinks = new Map(); + const reused = 'same-correlation-across-all-routes'; + const driver = createRealtimeDriver(fleet(), { + concurrency: 1, + timeoutMs: 1_000 + }, { + environment: { CPERF_TEST_TOKEN: 'driver-secret' }, + correlationFactory: () => reused, + clientFactory: (input) => ({ + subscribe: (_payload, sink) => { + sinks.set(input.url, sink); + queueMicrotask(input.onConnected); + return () => undefined; + }, + dispose: async () => undefined + }), + fetch: (async (url, init) => { + const href = String(url); + const payload = JSON.parse(String(init?.body)).variables.payload; + const tenant = href.includes('/tenant/a/') ? 'a' : 'b'; + const customer = href.includes('/customer/customer-1/') + ? 'customer-1' + : 'customer-2'; + sinks.get(href.replace(/^http:/, 'ws:'))!.next({ + data: { + changed: { + tenantId: tenant, + physicalDatabaseIdentity: `database-${customer}`, + payload + } + } + }); + return new Response(JSON.stringify({ + data: { + prime: { + tenantId: tenant, + physicalDatabaseIdentity: `database-${customer}`, + payload + } + } + }), { status: 200 }); + }) as typeof fetch + }); + + await expect(driver.startAndVerify()).rejects.toThrow( + 'CPERF_REALTIME_CORRELATION_REUSED:customer-2/api-b' + ); + await driver.dispose(); + }); + + it('records a post-verification drop and fails the health check', async () => { + let callbacks: RealtimeClientFactoryInput | null = null; + let sink: { next(value: unknown): void } | null = null; + const target = fleet()[0]; + const driver = createRealtimeDriver([target], { + concurrency: 1, + timeoutMs: 1_000 + }, { + environment: { CPERF_TEST_TOKEN: 'driver-secret' }, + clientFactory: (input) => { + callbacks = input; + return { + subscribe: (_payload, nextSink) => { + sink = nextSink; + queueMicrotask(input.onConnected); + return () => undefined; + }, + dispose: async () => undefined + }; + }, + fetch: (async (_url, init) => { + const payload = JSON.parse(String(init?.body)).variables.payload; + sink!.next({ + data: { + changed: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }); + return new Response(JSON.stringify({ + data: { + prime: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }), { status: 200 }); + }) as typeof fetch, + sleep: async () => undefined + }); + + await driver.startAndVerify(); + callbacks!.onClosed(); + expect(() => driver.assertHealthy()).toThrow( + 'CPERF_REALTIME_TRANSPORT_DROPPED:customer-1/api-a' + ); + await driver.dispose(); + }); + + it('requires secret headers from the runtime environment', () => { + const configured = fleet()[0].surfaces[0]; + expect(() => realtimeHeaders(configured, {})).toThrow( + 'CPERF_REALTIME_HEADER_ENV_MISSING:api-a:CPERF_TEST_TOKEN' + ); + expect(realtimeWebSocketUrl(configured.url)).toBe( + 'ws://127.0.0.1:3410/customer/customer-1/tenant/a/graphql' + ); + expect(() => realtimeWebSocketUrl(`${configured.url}?token=secret`)).toThrow( + 'CPERF_REALTIME_SURFACE_URL_INVALID' + ); + expect(() => createRealtimeDriver([], { + concurrency: 1, + timeoutMs: 1_000, + deliveryIntervalMs: 0 + })).toThrow('CPERF_REALTIME_DELIVERY_INTERVAL_INVALID'); + }); +}); diff --git a/packages/perf-harness/src/__tests__/report.test.ts b/packages/perf-harness/src/__tests__/report.test.ts new file mode 100644 index 0000000000..27ad69f4ed --- /dev/null +++ b/packages/perf-harness/src/__tests__/report.test.ts @@ -0,0 +1,681 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { resolveTenants } from '../config'; +import { + bindResultEvidence, + RESULT_RAW_EVIDENCE_FILES, + writeScoreContext +} from '../evidence'; +import { + rejectDuplicatePostgresRunEpochs, + renderReport +} from '../report'; +import { + buildRunSchedule, + scheduleJobsForPlan, + scheduleManifestSha256, + type CampaignScheduleManifestV1 +} from '../schedule'; +import { scoreRun, type ScoreInput } from '../score'; +import type { + AcceptanceGates, + ArmPlan, + DensityPlanV1, + DensityRunResult, + FleetV1, + MemorySnapshot, + PostgresMemorySnapshot, + RealtimeDeliveryCoverage +} from '../types'; + +const artifactRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-report-test-')); +const planSha256 = 'a'.repeat(64); +const fleetSha256 = 'b'.repeat(64); +const CAPACITY_ERROR = `CAPACITY:sha256:${'9'.repeat(64)}`; +const campaignId = '7'.repeat(64); +const cohortSha256 = createHash('sha256') + .update(`${planSha256}\0${fleetSha256}`) + .digest('hex'); + +const gates: AcceptanceGates = { + maxErrorRate: 0.005, + maxP99Ms: 150, + maxPostWarmupHeapGrowthMiBPerHour: 5, + minMedianDensityImprovement: 0.15, + minAdditionalTenantsEveryRun: 1, + maxAlignedMemorySampleGapMs: 900_000, + minAlignedMemoryCoverageRatio: 0.99, + requireZeroBleed: true, + requireNoPostWarmupEvictions: true, + requireNoPostWarmupBuildRefusals: true, + requireNoPostWarmupBuilds: true, + requirePostgresMemoryTelemetry: false, + requireFreshPostgresRunAttestation: false, + requireRetainedMemoryCheckpoints: false, + requirePhysicalDatabaseTelemetry: false, + requireConclusiveCanaries: true, + requireCompletePeriodicCanaryCoverage: false, + requireConclusiveOperationOracles: false, + requireExplicitCustomerTopology: false, + requiredCacheAdmissionMode: null +}; + +const arms: ArmPlan[] = ['cache-governor-stock', 'scoped-introspection'].map( + (name, index) => ({ + name, + command: ['node', 'server.cjs'], + port: 3345 + index, + readinessUrl: `http://127.0.0.1:${3345 + index}/healthz`, + memoryUrl: `http://127.0.0.1:${3345 + index}/debug/memory`, + introspectionMode: 'scoped-required' + }) +); + +const fleet: FleetV1 = { + version: 1, + sourceSha256: fleetSha256, + tenants: [1, 2, 3].map((index) => ({ + id: `tenant-${index}`, + surfaces: [{ + name: 'api', + buildContract: `tenant-${index}-api`, + url: 'http://127.0.0.1:{port}/graphql', + warmup: { + name: 'warm', + capability: 'graphile', + query: '{ __typename }' + }, + operations: [{ + name: 'read', + capability: 'graphile', + query: '{ __typename }' + }], + canaries: [{ + name: 'cross-schema', + query: '{ __typename }', + requiredMatches: [{ path: '/data/tenant', value: `tenant-${index}` }], + forbiddenMatches: [{ path: '/data/tenant', value: 'foreign' }] + }] + }] + })) +}; + +const plan: DensityPlanV1 = { + version: 1, + sourceSha256: planSha256, + fleetFile: 'fleet.json', + artifactDir: artifactRoot, + arms, + heapMiB: [1024], + tenantCounts: [1, 2, 3], + repetitions: 1, + runOrderSeed: 'test-seed', + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + workload: { + durationSec: 900, + rpsPerTenant: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 4, + canaryIntervalSec: 60, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 1_000, + warmupConcurrency: 4 + }, + gates, + qualification: { + baselineArm: 'cache-governor-stock', + requiredHeapMiB: [1024], + minimumRepetitions: 1 + } +}; + +for (const [index, arm] of arms.entries()) { + const runtimeArtifactFingerprint = `sha256:${String(index + 1).repeat(64)}`; + const configurationFingerprint = `sha256:${String(index + 3).repeat(64)}`; + const artifactFile = path.join(artifactRoot, `hostile-${arm.name}.json`); + const artifact = { + version: 1, + kind: 'exact-runtime-hostile-validation-v1', + passed: true, + arm: arm.name, + runtimeArtifactFingerprint, + configurationFingerprint + }; + const bytes = `${JSON.stringify(artifact, null, 2)}\n`; + fs.writeFileSync(artifactFile, bytes, 'utf8'); + plan.qualification!.hostileValidationEvidence ??= {}; + plan.qualification!.hostileValidationEvidence[arm.name] = { + version: 1, + kind: 'exact-runtime-hostile-validation-v1', + artifactFile, + artifactSha256: createHash('sha256').update(bytes).digest('hex'), + runtimeArtifactFingerprint, + configurationFingerprint + }; +} + +let activePlan = plan; +let activeSchedule: ReturnType = []; +let activeScheduleSha256 = ''; +let previousResultPayloadSha256: string | null = null; + +const beginCampaign = (targetPlan: DensityPlanV1): void => { + activePlan = targetPlan; + activeSchedule = buildRunSchedule( + targetPlan, + targetPlan.arms, + targetPlan.heapMiB, + targetPlan.repetitions + ); + const manifest: CampaignScheduleManifestV1 = { + version: 1, + campaignId, + campaignStartedAt: '2026-08-01T23:59:00.000Z', + runOrderSeed: targetPlan.runOrderSeed!, + planSha256, + fleetSha256, + node: process.version, + v8: process.versions.v8, + platform: 'linux', + architecture: 'x64', + jobs: scheduleJobsForPlan(targetPlan, activeSchedule, true) + }; + activeScheduleSha256 = scheduleManifestSha256(manifest); + fs.writeFileSync( + path.join(artifactRoot, `campaign-${campaignId}.json`), + `${JSON.stringify({ + ...manifest, + scheduleSha256: activeScheduleSha256, + evidenceMode: 'qualification', + qualificationBlockers: [] + }, null, 2)}\n`, + 'utf8' + ); + previousResultPayloadSha256 = null; +}; + +const realtimeCoverage = ( + startedAt: string, + endedAt: string, + durationSec: number +): RealtimeDeliveryCoverage => ({ + version: 2, + deliveryIntervalMs: 60_000, + workloadStartedAt: startedAt, + workloadDeadlineAt: new Date(Date.parse(startedAt) + durationSec * 1000).toISOString(), + workloadEndedAt: endedAt, + expectedRecurringRounds: 0, + startedRecurringRounds: 0, + verifiedRecurringRounds: 0, + deadlineLateRecurringRounds: 0, + primeRequests: 0, + primeResponseP99Ms: 0, + deliveryP99Ms: 0, + complete: true, + surfaces: [] +}); + +const memorySnapshot = ( + timestamp: string, + contracts: string[], + nodeRssBytes: number +): MemorySnapshot => ({ + timestamp, + pid: 123, + nodeEnv: 'production', + heapLimitBytes: 1024 * 1024 ** 2, + heapUsedBytes: 100 * 1024 ** 2, + rssBytes: nodeRssBytes, + processPeakRssBytes: nodeRssBytes, + cacheSize: contracts.length, + residentBuildContractFingerprints: contracts, + residentBuildContracts: contracts, + evictions: 0, + buildRefusals: 0, + buildsStarted: contracts.length, + buildsSucceeded: contracts.length, + buildMaxMs: 80, + pgPoolCacheSize: contracts.length, + pgPoolLeasedPools: 0, + pgPoolActiveLeases: 0, + pgPoolCapacityEvictions: 0, + pgPoolCapacityRefusals: 0, + pgPoolDisposalFailures: 0, + cacheCountersAvailable: true, + buildCountersAvailable: true +}); + +const postgresSnapshot = ( + timestamp: string, + postgresBytes: number +): PostgresMemorySnapshot => ({ + timestamp, + usedBytes: postgresBytes, + limitBytes: 8 * 1024 ** 3, + source: 'cgroup-v2', + cgroupV2: { + currentBytes: postgresBytes, + peakBytes: postgresBytes, + maxBytes: 8 * 1024 ** 3, + stat: {}, + events: { oom: 0, oom_kill: 0 } + }, + raw: `${postgresBytes}B / 8GiB` +}); + +let artifactSequence = 0; + +const scoreInput = ( + armName: string, + configuredCustomers: number, + desiredServiceDensity: number, + executionErrors: string[] = [], + options: { runKind?: 'matrix' | 'soak'; durationSec?: number } = {} +): ScoreInput => { + const arm = arms.find((candidate) => candidate.name === armName)!; + const tenants = resolveTenants(fleet.tenants.slice(0, configuredCustomers), arm); + const durationSec = options.durationSec ?? 900; + const runKind = options.runKind ?? 'matrix'; + const repetition = runKind === 'soak' ? activePlan.repetitions + 1 : 1; + const scheduled = scheduleJobsForPlan(activePlan, activeSchedule, true).find((job) => + job.runKind === runKind + && job.arm === armName + && job.heapMiB === 1024 + && job.tenantCount === configuredCustomers + && job.repetition === repetition + ); + if (!scheduled) throw new Error('test coordinate is absent from the active campaign'); + const startedAt = new Date( + Date.parse('2026-08-02T00:00:00.000Z') + + (scheduled.orderIndex - 1) * 10_000_000 + ).toISOString(); + const endedAt = new Date(Date.parse(startedAt) + durationSec * 1000).toISOString(); + const targetServiceBytes = Math.round( + configuredCustomers / desiredServiceDensity * 1024 ** 3 + ); + const nodeRssBytes = Math.max(256 * 1024 ** 2, Math.floor(targetServiceBytes * 0.6)); + const postgresBytes = Math.max(1, targetServiceBytes - nodeRssBytes); + const contracts = tenants.flatMap((tenant) => + tenant.surfaces.map((surface) => surface.buildContract)); + const memorySnapshots = [ + memorySnapshot(startedAt, contracts, nodeRssBytes), + memorySnapshot(endedAt, contracts, nodeRssBytes) + ]; + const postgresSnapshots = [ + postgresSnapshot(startedAt, postgresBytes), + postgresSnapshot(endedAt, postgresBytes) + ]; + const runOrderIndex = scheduled.orderIndex; + const artifactDir = path.join(artifactRoot, String(++artifactSequence)); + const provenance: ScoreInput['provenance'] = { + cwd: '/tmp/repo', + command: ['node', 'server.cjs'], + gitHead: 'c'.repeat(40), + worktreeDirty: false, + gitStatusSha256: 'd'.repeat(64), + lockfilePath: '/tmp/repo/pnpm-lock.yaml', + lockfileSha256: 'e'.repeat(64), + entryPath: '/tmp/repo/server.cjs', + entrySha256: 'f'.repeat(64), + serverPid: 123, + v8Profile: 'stock', + nodeOptions: '--max-old-space-size=1024', + nodeOptionsArgv: ['--max-old-space-size=1024'], + nodeExecArgv: [], + effectiveNodeRuntimeFlags: ['--max-old-space-size=1024'], + planSha256, + fleetSha256, + node: process.version, + v8: process.versions.v8, + platform: 'linux', + architecture: 'x64', + runOrderSeed: 'test-seed', + runOrderIndex, + memoryPolicy: { + configuredMaxOldSpaceMiB: 1024, + expectedV8HeapLimitBytes: 1024 * 1024 ** 2, + graphileCacheMax: null, + graphileCacheInstanceHeapBytes: null, + graphileCacheServerReserveBytes: null, + graphileCacheBuildReserveBytes: null, + graphileCacheRssLimitBytes: null, + graphileCacheRssBuildReserveBytes: null, + graphileCacheCalibrationId: null, + graphileCacheAdmissionMode: null, + graphileBuildMaxConcurrency: null + } + }; + return { + arm: armName, + evidenceMode: 'qualification', + campaignId, + scheduleSha256: activeScheduleSha256, + previousResultPayloadSha256, + qualificationCohortSha256: cohortSha256, + introspectionMode: arm.introspectionMode, + heapMiB: 1024, + repetition, + expectedMatrixRepetitions: 1, + runKind, + runOrderSeed: 'test-seed', + runOrderIndex, + startedAt, + endedAt, + configuredDurationSec: durationSec, + workloadDurationMs: durationSec * 1000, + artifactDir, + tenants, + warmedSurfaces: new Map(tenants.map((tenant) => [ + tenant.id, + new Set(tenant.surfaces.map((surface) => surface.name)) + ])), + warmupLatencies: tenants.map(() => 80), + resolvedWarmupTimeoutMs: 1_000, + offeredLoad: { + mode: 'per-tenant', + configuredRps: 1, + tenantCount: configuredCustomers, + totalRps: configuredCustomers, + rpsPerTenant: 1 + }, + canaryIntervalSec: 60, + periodicCanarySchedule: 'full-sweep', + canarySchedule: null, + minWorkloadRequestsPerSurface: 1, + samples: tenants.flatMap((tenant) => tenant.surfaces.map((surface) => ({ + tenantId: tenant.id, + surface: surface.name, + operation: 'read', + capability: 'graphile', + latencyMs: 25, + status: 200, + ok: true, + phase: 'workload' as const + }))), + canaries: tenants.flatMap((tenant) => tenant.surfaces.map((surface) => ({ + tenantId: tenant.id, + surface: surface.name, + canary: 'cross-schema', + phase: 'initial' as const, + scheduledAt: startedAt, + startedAt, + completedAt: new Date(Date.parse(startedAt) + 20).toISOString(), + latencyMs: 20, + conclusive: true, + violation: false + }))), + memorySnapshots, + postWarmupSnapshots: memorySnapshots, + postWarmupNodeRssSnapshots: memorySnapshots.map((snapshot) => ({ + timestamp: snapshot.timestamp, + pid: 123, + source: 'proc' as const, + rssBytes: snapshot.rssBytes! + })), + retainedMemory: { baseline: null, final: null, errors: [] }, + memorySampleErrors: [], + postgresSnapshots, + postgresSampleErrors: [], + missedArrivals: 0, + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + gates, + serverExit: null, + provenance: { + ...provenance, + runOrderIndex + }, + provenanceErrors: [], + postgresRunAttestation: null, + realtimeDeliveryCoverage: realtimeCoverage(startedAt, endedAt, durationSec), + externalServer: false, + executionErrors + }; +}; + +const writeJson = (file: string, value: unknown): void => { + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, 'utf8'); +}; + +const persistResult = (input: ScoreInput): DensityRunResult => { + fs.mkdirSync(input.artifactDir, { recursive: true }); + writeJson(path.join(input.artifactDir, 'memory.json'), { + snapshots: input.memorySnapshots, + osSnapshots: input.postWarmupNodeRssSnapshots, + errors: input.memorySampleErrors, + warmupIndex: 0, + osWarmupIndex: 0, + osPeakRssBytes: Math.max(...input.postWarmupNodeRssSnapshots.map( + (snapshot) => snapshot.rssBytes + )) + }); + writeJson(path.join(input.artifactDir, 'postgres-memory.json'), { + snapshots: input.postgresSnapshots, + errors: input.postgresSampleErrors + }); + writeJson(path.join(input.artifactDir, 'canaries.json'), input.canaries); + writeJson(path.join(input.artifactDir, 'canary-schedule.json'), input.canarySchedule); + fs.writeFileSync( + path.join(input.artifactDir, 'requests.ndjson'), + `${input.samples.map((sample) => JSON.stringify(sample)).join('\n')}\n`, + 'utf8' + ); + writeJson(path.join(input.artifactDir, 'workload-progress.json'), { + warmedSurfaces: [...input.warmedSurfaces].map(([tenantId, surfaces]) => ({ + tenantId, + surfaces: [...surfaces].sort() + })), + warmupLatencies: input.warmupLatencies, + samples: input.samples.length, + canaries: input.canaries.length, + canarySchedule: input.canarySchedule, + offeredLoad: input.offeredLoad, + resolvedWarmupTimeoutMs: input.resolvedWarmupTimeoutMs, + workloadDurationMs: input.workloadDurationMs + }); + writeJson(path.join(input.artifactDir, 'retained-memory.json'), input.retainedMemory); + writeJson(path.join(input.artifactDir, 'realtime-driver.json'), [{ + phase: 'timed-coverage-complete', + timestamp: input.endedAt, + snapshot: { + expected: 0, + active: 0, + verified: 0, + deliveryIntervalMs: input.realtimeDeliveryCoverage!.deliveryIntervalMs, + deliveryEvents: 0, + deliveryRoundsStarted: 0, + deliveryRoundsVerified: 0, + deliveryRoundsPending: 0, + timedCoverage: input.realtimeDeliveryCoverage, + errors: [], + surfaces: [] + } + }]); + const result = scoreRun(input); + writeScoreContext(input.artifactDir, input, { + planSha256, + fleetSha256, + campaignId: input.campaignId, + scheduleSha256: input.scheduleSha256, + previousResultPayloadSha256: input.previousResultPayloadSha256, + notBeforeEpochMs: Date.parse(input.startedAt) + }); + bindResultEvidence(result); + previousResultPayloadSha256 = result.evidenceBinding!.resultPayloadSha256; + return result; +}; + +const persistConfiguredMatrix = (): DensityRunResult[] => activeSchedule.map((job) => { + const baseline = job.arm.name === 'cache-governor-stock'; + const acceptedBoundary = baseline ? 1 : 2; + return persistResult(scoreInput( + job.arm.name, + job.tenantCount, + baseline ? 1 : 2, + job.tenantCount <= acceptedBoundary ? [] : [CAPACITY_ERROR] + )); +}); + +describe('density report', () => { + beforeEach(() => beginCampaign(plan)); + + it('rejects a semantically edited result even after its public hashes are rebound', () => { + const first = activeSchedule[0]; + const value = persistResult(scoreInput(first.arm.name, first.tenantCount, 1)); + value.p99Ms += 1; + bindResultEvidence(value); + expect(() => renderReport([value], plan, fleet)).toThrow( + 'does not match semantic replay of raw evidence' + ); + + beginCampaign(plan); + const divergent = persistResult(scoreInput(first.arm.name, first.tenantCount, 1)); + fs.appendFileSync(path.join(divergent.artifactDir, 'requests.ndjson'), '{}\n'); + expect(() => renderReport([divergent], plan, fleet)).toThrow( + 'raw evidence file does not match: requests.ndjson' + ); + }); + + it('renders an exactly paired capacity decision from replayed evidence', () => { + const results = persistConfiguredMatrix(); + const report = renderReport(results, plan, fleet); + expect(report).toContain('Customers/aligned service GiB'); + expect(report).toContain('Customer workload RPS'); + expect(report).toContain('Periodic validation RPS'); + expect(report).toContain('Realtime validation RPS'); + expect(report).toContain('matrices are exactly paired: yes'); + expect(report).toContain('Materially better: **yes**'); + }); + + it('rejects reordered, cross-campaign, and overlapping result ledgers', () => { + const results = persistConfiguredMatrix(); + expect(() => renderReport([ + results[1], + results[0], + ...results.slice(2) + ], plan, fleet)).toThrow('campaign schedule or result chain'); + + const spliced = { ...results[1], campaignId: '8'.repeat(64) }; + bindResultEvidence(spliced); + expect(() => renderReport([ + results[0], + spliced, + ...results.slice(2) + ], plan, fleet)).toThrow('campaign schedule or result chain'); + + const overlapping = { + ...results[1], + startedAt: results[0].startedAt + }; + bindResultEvidence(overlapping); + expect(() => renderReport([ + results[0], + overlapping, + ...results.slice(2) + ], plan, fleet)).toThrow('campaign chronology is invalid or overlapping'); + }); + + it('rejects qualification without exact-runtime hostile validation artifacts', () => { + const unboundPlan: DensityPlanV1 = { + ...plan, + qualification: { + baselineArm: plan.qualification!.baselineArm, + requiredHeapMiB: [...plan.qualification!.requiredHeapMiB], + minimumRepetitions: plan.qualification!.minimumRepetitions + } + }; + beginCampaign(unboundPlan); + const first = activeSchedule[0]; + const result = persistResult(scoreInput(first.arm.name, first.tenantCount, 1)); + expect(() => renderReport([result], unboundPlan, fleet)).toThrow( + 'lacks exact-runtime hostile validation evidence' + ); + }); + + it('rejects malformed nested request evidence after rebinding its artifact hash', () => { + const first = activeSchedule[0]; + const result = persistResult(scoreInput(first.arm.name, first.tenantCount, 1)); + const requestsFile = path.join(result.artifactDir, 'requests.ndjson'); + const request = JSON.parse(fs.readFileSync(requestsFile, 'utf8').trim()); + request.latencyMs = null; + fs.writeFileSync(requestsFile, `${JSON.stringify(request)}\n`, 'utf8'); + bindResultEvidence(result); + expect(() => renderReport([result], plan, fleet)).toThrow( + 'latencyMs must be finite' + ); + }); + + it('requires one configured qualifying soak without mixing it into the matrix', () => { + const soakPlan: DensityPlanV1 = { + ...plan, + soak: { + enabled: true, + arm: 'scoped-introspection', + durationSec: 7_200, + tenantCount: 2, + heapMiB: 1024 + } + }; + beginCampaign(soakPlan); + const matrix = persistConfiguredMatrix(); + expect(renderReport(matrix, soakPlan, fleet)).toContain( + 'configured soak=0/1, accepted=no' + ); + const soak = persistResult(scoreInput( + 'scoped-introspection', + 2, + 2, + [], + { runKind: 'soak', durationSec: 7_200 } + )); + const report = renderReport([...matrix, soak], soakPlan, fleet); + expect(report).toContain('configured soak=1/1, accepted=yes'); + expect(report).toContain('Materially better: **yes**'); + }); + + it('rejects reuse of any PostgreSQL container, cluster, clone, or nonce identity', () => { + const first = persistResult(scoreInput( + activeSchedule[0].arm.name, + activeSchedule[0].tenantCount, + 2 + )); + const second = persistResult(scoreInput( + activeSchedule[1].arm.name, + activeSchedule[1].tenantCount, + 2 + )); + const evidence = { + epochId: `sha256:${'1'.repeat(64)}`, + containerId: '2'.repeat(64), + cgroupIdentitySha256: `sha256:${'3'.repeat(64)}`, + postgresSystemIdentifier: '7421234567890123456', + cloneId: 'measurement-clone-1', + cloneAttestationSetSha256: `sha256:${'4'.repeat(64)}`, + cloneNonceSetSha256: `sha256:${'5'.repeat(64)}` + } as DensityRunResult['postgresRunAttestation']; + first.postgresRunAttestation = evidence; + second.postgresRunAttestation = { + ...evidence!, + epochId: `sha256:${'6'.repeat(64)}`, + containerId: '7'.repeat(64), + cgroupIdentitySha256: `sha256:${'8'.repeat(64)}`, + postgresSystemIdentifier: '8421234567890123456', + cloneId: 'measurement-clone-2', + cloneAttestationSetSha256: `sha256:${'9'.repeat(64)}` + }; + const rejected = rejectDuplicatePostgresRunEpochs([first, second]); + expect(rejected.every((run) => !run.accepted)).toBe(true); + expect(rejected.every((run) => run.failures.some((failure) => + failure.includes('clone-nonce-set:') + ))).toBe(true); + }); + + it('binds every fixed raw-evidence file including the score context', () => { + expect(RESULT_RAW_EVIDENCE_FILES).toContain('score-context.json'); + }); +}); diff --git a/packages/perf-harness/src/__tests__/run-attestation.test.ts b/packages/perf-harness/src/__tests__/run-attestation.test.ts new file mode 100644 index 0000000000..f7724086c9 --- /dev/null +++ b/packages/perf-harness/src/__tests__/run-attestation.test.ts @@ -0,0 +1,164 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + collectPostgresRunAttestation, + normalizePostgresRunAttestation, + postgresRunIdentityClaims, + type RunAttestationContext +} from '../run-attestation'; +import type { ArmPlan } from '../types'; + +const canonicalize = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + const record = value as Record; + return Object.fromEntries(Object.keys(record).sort().map((key) => [ + key, + canonicalize(record[key]) + ])); +}; + +const sha256 = (value: unknown): string => `sha256:${createHash('sha256') + .update(JSON.stringify(canonicalize(value))) + .digest('hex')}`; + +const contextFor = (artifactDir: string): RunAttestationContext => ({ + arm: 'candidate', + heapMiB: 2048, + tenantCount: 1, + repetition: 2, + runOrderIndex: 7, + planSha256: 'a'.repeat(64), + fleetSha256: 'b'.repeat(64), + notBeforeEpochMs: Date.parse('2026-08-02T00:00:00.000Z'), + artifactDir +}); + +const envelopeFor = (context: RunAttestationContext) => { + const customerAudits = [{ + customerId: 'customer-1', + databaseContractFingerprint: `sha256:${'c'.repeat(64)}`, + structuralFingerprints: { combined: { sha256: `sha256:${'d'.repeat(64)}` } }, + cloneAttestationSha256: `sha256:${'e'.repeat(64)}`, + cloneNonceSha256: `sha256:${'f'.repeat(64)}` + }]; + const provisionClone = { + version: 1, + id: 'measurement-unique-clone', + purpose: 'measurement', + attestationSetSha256: `sha256:${'1'.repeat(64)}` + }; + const container = { + id: '2'.repeat(64), + startedAt: '2026-08-02T00:00:00.010Z' + }; + const cgroup = { + version: 1, + source: 'container-cgroup-v2', + identitySha256: `sha256:${'3'.repeat(64)}` + }; + const postgres = { + systemIdentifier: '7421234567890123456', + postmasterStartedAt: '2026-08-02T00:00:00.020Z' + }; + const immutableEpoch = { + dockerContainerId: container.id, + dockerStartedAt: container.startedAt, + containerConfigurationSha256: `sha256:${'4'.repeat(64)}`, + cgroupIdentitySha256: cgroup.identitySha256, + postgresSystemIdentifier: postgres.systemIdentifier, + postgresStartedAt: postgres.postmasterStartedAt, + cloneId: provisionClone.id, + cloneAttestationSetSha256: provisionClone.attestationSetSha256, + cloneNonceSetSha256: sha256(customerAudits.map((audit) => ({ + customerId: audit.customerId, + cloneNonceSha256: audit.cloneNonceSha256 + }))), + liveContractSetSha256: sha256(customerAudits.map((audit) => ({ + customerId: audit.customerId, + databaseContractFingerprint: audit.databaseContractFingerprint, + structuralFingerprint: audit.structuralFingerprints.combined.sha256 + }))) + }; + const payload = { + observedAt: '2026-08-02T00:00:01.000Z', + run: { + arm: context.arm, + heapMiB: context.heapMiB, + customerCount: context.tenantCount, + repetition: context.repetition, + runOrderIndex: context.runOrderIndex, + planSha256: `sha256:${context.planSha256}`, + fleetSha256: `sha256:${context.fleetSha256}` + }, + manifestSha256: `sha256:${'5'.repeat(64)}`, + containerTemplateSha256: `sha256:${'6'.repeat(64)}`, + canonicalDatabaseContractFingerprint: `sha256:${'7'.repeat(64)}`, + provisionClone, + container, + cgroup, + postgres, + customerAudits, + immutableEpoch, + epochId: sha256(immutableEpoch), + freshness: { + freshContainerForRun: true, + cgroupV2Verified: true, + notBeforeEpochMs: context.notBeforeEpochMs, + startToleranceMs: 0 + }, + catalogCacheState: 'warmed-by-live-contract-audit' + }; + return { + version: 1, + kind: 'physical-density-measurement-attestation-v1', + payload, + payloadSha256: sha256(payload) + }; +}; + +describe('PostgreSQL run attestation', () => { + it('binds every immutable container, cluster, clone, and live-contract identity', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-attestation-')); + const context = contextFor(directory); + const envelope = envelopeFor(context); + const artifact = path.join(directory, 'attestation.json'); + fs.writeFileSync(artifact, JSON.stringify(envelope)); + const evidence = normalizePostgresRunAttestation(envelope, context, artifact); + expect(evidence.cloneId).toBe('measurement-unique-clone'); + expect(evidence.cloneAttestationSetSha256).toBe(`sha256:${'1'.repeat(64)}`); + expect(postgresRunIdentityClaims(evidence)).toHaveLength(7); + + const tampered = structuredClone(envelope) as any; + tampered.payload.immutableEpoch.cloneId = 'different-clone'; + tampered.payload.epochId = sha256(tampered.payload.immutableEpoch); + tampered.payloadSha256 = sha256(tampered.payload); + expect(() => normalizePostgresRunAttestation(tampered, context, artifact)) + .toThrow('failed exact validation'); + }); + + it('refuses to overwrite a prior per-run attestation artifact', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-attestation-')); + const context = contextFor(directory); + fs.writeFileSync( + path.join(directory, 'postgres-run-attestation.json'), + '{}' + ); + const arm = { + name: 'candidate', + port: 3345, + readinessUrl: 'http://127.0.0.1:3345/healthz', + memoryUrl: 'http://127.0.0.1:3345/debug/memory', + introspectionMode: 'stock', + postgresRunAttestation: { + command: [process.execPath, '-e', 'process.exit(0)'], + prepareCommand: [process.execPath, '-e', 'process.exit(0)'] + } + } satisfies ArmPlan; + await expect(collectPostgresRunAttestation(arm, context)) + .rejects.toThrow('artifact already exists'); + }); +}); diff --git a/packages/perf-harness/src/__tests__/run.test.ts b/packages/perf-harness/src/__tests__/run.test.ts new file mode 100644 index 0000000000..8630a6650a --- /dev/null +++ b/packages/perf-harness/src/__tests__/run.test.ts @@ -0,0 +1,67 @@ +import { resolveTenants } from '../config'; +import { buildRunSchedule } from '../run'; +import type { ArmPlan, DensityPlanV1, TenantTarget } from '../types'; + +describe('arm-specific fleet resolution', () => { + it('selects the exact build identity for the running arm', () => { + const tenants = [{ + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: 'default-hash', + buildContracts: { + stock: 'stock-hash', + scoped: 'scoped-hash' + }, + url: 'http://127.0.0.1:{port}/{mode}', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [] + }] + }] as TenantTarget[]; + const arm = { + name: 'scoped', + port: 3345, + introspectionMode: 'scoped-required' + } as ArmPlan; + + const resolved = resolveTenants(tenants, arm); + expect(resolved[0].surfaces[0]).toMatchObject({ + buildContract: 'scoped-hash', + url: 'http://127.0.0.1:3345/scoped-required' + }); + }); + + it('uses heap-specific ramps and reproducibly interleaves arms within each cell', () => { + const arms = [ + { name: 'stock' }, + { name: 'scoped' } + ] as ArmPlan[]; + const plan = { + runOrderSeed: 'seed-a', + tenantCounts: [1], + tenantCountsByHeapMiB: { 2048: [2, 4] } + } as unknown as DensityPlanV1; + const first = buildRunSchedule(plan, arms, [1024, 2048], 2); + const second = buildRunSchedule(plan, arms, [1024, 2048], 2); + expect(first.map((job) => ({ + arm: job.arm.name, + heap: job.heapMiB, + tenants: job.tenantCount, + repetition: job.repetition, + order: job.orderIndex + }))).toEqual(second.map((job) => ({ + arm: job.arm.name, + heap: job.heapMiB, + tenants: job.tenantCount, + repetition: job.repetition, + order: job.orderIndex + }))); + expect(first).toHaveLength(12); + expect(first.slice(0, 2).map((job) => job.heapMiB)).toEqual([1024, 1024]); + expect(first.slice(2, 6).map((job) => job.tenantCount)).toEqual([2, 2, 4, 4]); + expect(first.map((job) => job.orderIndex)).toEqual( + Array.from({ length: 12 }, (_, index) => index + 1) + ); + }); +}); diff --git a/packages/perf-harness/src/__tests__/score.test.ts b/packages/perf-harness/src/__tests__/score.test.ts new file mode 100644 index 0000000000..3732047886 --- /dev/null +++ b/packages/perf-harness/src/__tests__/score.test.ts @@ -0,0 +1,1754 @@ +import { createHash } from 'node:crypto'; + +import { rotatingCanaryIndex } from '../http'; +import { + alignedServiceMemoryCoverage, + alignedServiceMemoryPeak, + compareDensity, + heapGrowthMiBPerHour, + percentile, + retainedMemoryGrowth, + type ScoreInput, + scoreRun, + summarizeCapacityBoundaries} from '../score'; +import type { + AcceptanceGates, + ArmProvenance, + DensityRunResult, + MemorySnapshot, + PostgresRunAttestationEvidence, + RetainedMemoryCheckpoint, + TenantTarget +} from '../types'; + +const gates: AcceptanceGates = { + maxErrorRate: 0.005, + maxP99Ms: 150, + maxPostWarmupHeapGrowthMiBPerHour: 5, + minMedianDensityImprovement: 0.15, + minAdditionalTenantsEveryRun: 1, + maxAlignedMemorySampleGapMs: 900_000, + minAlignedMemoryCoverageRatio: 0.99, + requireZeroBleed: true, + requireNoPostWarmupEvictions: true, + requireNoPostWarmupBuildRefusals: true, + requireNoPostWarmupBuilds: true, + requirePostgresMemoryTelemetry: true, + requireFreshPostgresRunAttestation: false, + requireRetainedMemoryCheckpoints: true, + requirePhysicalDatabaseTelemetry: false, + requireConclusiveCanaries: true, + requireCompletePeriodicCanaryCoverage: false, + requireConclusiveOperationOracles: false, + requireExplicitCustomerTopology: false, + requiredCacheAdmissionMode: null +}; + +const tenant: TenantTarget = { + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: 'tenant-a-api', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ __typename }', + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b' }], + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a' }] + }] + }] +}; + +const memory = (minute: number, heapMiB: number, evictions = 0): MemorySnapshot => ({ + timestamp: new Date(Date.UTC(2026, 6, 31, 0, minute)).toISOString(), + pid: 42, + nodeEnv: 'production', + heapLimitBytes: 1024 * 1024 ** 2, + heapUsedBytes: heapMiB * 1024 ** 2, + rssBytes: 200 * 1024 ** 2, + processPeakRssBytes: 220 * 1024 ** 2, + cacheSize: 1, + residentBuildContractFingerprints: ['tenant-a-api'], + residentBuildContracts: ['tenant-a-api'], + evictions, + buildRefusals: 0, + buildsStarted: 1, + buildsSucceeded: 1, + buildMaxMs: 80, + pgPoolCacheSize: 2, + pgPoolLeasedPools: 1, + pgPoolActiveLeases: 1, + pgPoolCapacityEvictions: 0, + pgPoolCapacityRefusals: 0, + pgPoolDisposalFailures: 0, + cacheCountersAvailable: true, + buildCountersAvailable: true +}); + +const canonicalJson = (value: unknown): string => { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value && typeof value === 'object') { + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => + `${JSON.stringify(key)}:${canonicalJson(record[key])}` + ).join(',')}}`; + } + return JSON.stringify(value) ?? 'null'; +}; + +const retainedCheckpoint = ( + minute: number, + heapMiB: number, + externalMiB = 10 +): RetainedMemoryCheckpoint => { + const state = { + pid: 42, + graphileInFlight: 0, + residentBuildContracts: ['tenant-a-api'], + residentBuildContractFingerprints: ['tenant-a-api'], + counters: { buildsStarted: 1, evictions: 0 } + }; + const stateSha256 = `sha256:${createHash('sha256') + .update(canonicalJson(state)) + .digest('hex')}`; + const guard = { + pid: 42, + graphileInFlight: 0, + residentBuildContracts: ['tenant-a-api'], + stateSha256, + state + }; + return { + version: 1, + fixture: 'physical-database-density-v1', + pid: 42, + gcRounds: 8, + stableSampleCount: 3, + stable: true, + samples: Array.from({ length: 8 }, (_, index) => ({ + timestamp: new Date(Date.UTC(2026, 6, 31, 0, minute, 0, index)).toISOString(), + monotonicNs: String(BigInt(minute) * 60_000_000_000n + BigInt(index + 1)), + heapUsedBytes: heapMiB * 1024 ** 2, + externalBytes: externalMiB * 1024 ** 2, + arrayBuffersBytes: 2 * 1024 ** 2, + rssBytes: 200 * 1024 ** 2 + })), + guardBefore: guard, + guardAfter: guard, + errors: [] + }; +}; + +const retainedCheckpointWithState = ( + minute: number, + state: Record +): RetainedMemoryCheckpoint => { + const checkpoint = retainedCheckpoint(minute, minute === 0 ? 100 : 101); + const stateHash = `sha256:${createHash('sha256') + .update(canonicalJson(state)) + .digest('hex')}`; + const residentBuildContracts = Array.isArray(state.residentBuildContracts) + ? state.residentBuildContracts as string[] + : ['tenant-a-api']; + const guard = { + pid: 42, + graphileInFlight: 0, + residentBuildContracts, + stateSha256: stateHash, + state + }; + return { ...checkpoint, guardBefore: guard, guardAfter: guard }; +}; + +const retainedPhysicalState = ( + httpRequestsStarted: number, + httpRequestsCompleted: number, + realtimeConnectionsActive = 1 +): Record => ({ + pid: 42, + graphileInFlight: 0, + residentBuildContracts: ['tenant-a-api'], + residentBuildContractFingerprints: ['tenant-a-api'], + cacheCounters: { + httpRequestsStarted, + httpRequestsCompleted, + websocketUpgradesStarted: 1, + websocketUpgradesCompleted: 0, + evictions: { lru: 0 }, + buildRefusals: { resident_capacity: 0 } + }, + realtime: { connectionsExpected: 1, connectionsActive: realtimeConnectionsActive } +}); + +const provenance: ArmProvenance = { + cwd: '/workspace/constructive', + command: ['/usr/bin/node', '/workspace/constructive/dist/server.js'], + gitHead: 'a'.repeat(40), + worktreeDirty: false, + gitStatusSha256: 'b'.repeat(64), + lockfilePath: '/workspace/constructive/pnpm-lock.yaml', + lockfileSha256: 'c'.repeat(64), + entryPath: '/workspace/constructive/dist/server.js', + entrySha256: 'd'.repeat(64), + serverPid: 42, + v8Profile: 'stock', + nodeOptions: '--max-old-space-size=1024', + nodeOptionsArgv: ['--max-old-space-size=1024'], + nodeExecArgv: [], + effectiveNodeRuntimeFlags: ['--max-old-space-size=1024'], + planSha256: 'e'.repeat(64), + fleetSha256: 'f'.repeat(64), + node: process.version, + v8: process.versions.v8, + platform: 'linux', + architecture: 'x64', + runOrderSeed: 'test-seed', + runOrderIndex: 1, + memoryPolicy: { + configuredMaxOldSpaceMiB: 1024, + expectedV8HeapLimitBytes: 1024 * 1024 ** 2, + graphileCacheMax: null, + graphileCacheInstanceHeapBytes: null, + graphileCacheServerReserveBytes: null, + graphileCacheBuildReserveBytes: null, + graphileCacheRssLimitBytes: null, + graphileCacheRssBuildReserveBytes: null, + graphileCacheCalibrationId: null, + graphileCacheAdmissionMode: null, + graphileBuildMaxConcurrency: null + } +}; + +const qualifyingInput = (): ScoreInput => ({ + arm: 'scoped-introspection', + evidenceMode: 'qualification', + campaignId: '8'.repeat(64), + scheduleSha256: '9'.repeat(64), + previousResultPayloadSha256: null, + qualificationCohortSha256: 'a'.repeat(64), + introspectionMode: 'scoped-required', + heapMiB: 1024, + repetition: 1, + expectedMatrixRepetitions: 1, + runKind: 'matrix', + runOrderSeed: 'test-seed', + runOrderIndex: 1, + startedAt: '2026-07-31T00:00:00.000Z', + endedAt: '2026-07-31T00:15:00.000Z', + configuredDurationSec: 900, + workloadDurationMs: 900_000, + artifactDir: '/tmp/result', + tenants: [tenant], + warmedSurfaces: new Map([['tenant-a', new Set(['api'])]]), + warmupLatencies: [80], + resolvedWarmupTimeoutMs: 180_000, + offeredLoad: { + mode: 'per-tenant', + configuredRps: 1, + tenantCount: 1, + totalRps: 1, + rpsPerTenant: 1 + }, + canaryIntervalSec: 60, + periodicCanarySchedule: 'full-sweep', + canarySchedule: null, + minWorkloadRequestsPerSurface: 1, + samples: [{ + tenantId: 'tenant-a', + surface: 'api', + operation: 'read', + capability: 'graphile', + latencyMs: 20, + status: 200, + ok: true, + phase: 'workload' + }], + canaries: [{ + tenantId: 'tenant-a', + surface: 'api', + canary: 'cross-schema', + phase: 'initial', + scheduledAt: '2026-07-31T00:00:00.000Z', + startedAt: '2026-07-31T00:00:00.000Z', + completedAt: '2026-07-31T00:00:00.020Z', + latencyMs: 20, + conclusive: true, + violation: false + }], + memorySnapshots: [memory(0, 100), memory(15, 101)], + postWarmupSnapshots: [memory(0, 100), memory(15, 101)], + postWarmupNodeRssSnapshots: [memory(0, 100), memory(15, 101)].map( + ({ timestamp, rssBytes }) => ({ + timestamp, + pid: 42, + source: 'proc', + rssBytes: rssBytes! + }) + ), + retainedMemory: { + baseline: retainedCheckpoint(0, 100), + final: retainedCheckpoint(15, 101), + errors: [] + }, + memorySampleErrors: [], + postgresSnapshots: [ + { + timestamp: '2026-07-31T00:00:00.100Z', + containerId: '4'.repeat(64), + cgroupIdentitySha256: `sha256:${'5'.repeat(64)}`, + usedBytes: 100, + limitBytes: 1_000, + source: 'cgroup-v2', + cgroupV2: { + currentBytes: 100, + peakBytes: 150, + maxBytes: 1_000, + stat: {}, + events: { oom: 0, oom_kill: 0 } + }, + raw: '100B / 1000B' + }, + { + timestamp: '2026-07-31T00:15:00.100Z', + containerId: '4'.repeat(64), + cgroupIdentitySha256: `sha256:${'5'.repeat(64)}`, + usedBytes: 120, + limitBytes: 1_000, + source: 'cgroup-v2', + cgroupV2: { + currentBytes: 120, + peakBytes: 180, + maxBytes: 1_000, + stat: {}, + events: { oom: 0, oom_kill: 0 } + }, + raw: '120B / 1000B' + } + ], + postgresSampleErrors: [], + missedArrivals: 0, + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + gates, + serverExit: null, + provenance, + provenanceErrors: [], + realtimeDeliveryCoverage: { + version: 2, + deliveryIntervalMs: 60_000, + workloadStartedAt: '2026-07-31T00:00:00.000Z', + workloadDeadlineAt: '2026-07-31T00:15:00.000Z', + workloadEndedAt: '2026-07-31T00:15:00.000Z', + expectedRecurringRounds: 0, + startedRecurringRounds: 0, + verifiedRecurringRounds: 0, + deadlineLateRecurringRounds: 0, + primeRequests: 0, + primeResponseP99Ms: 0, + deliveryP99Ms: 0, + complete: true, + surfaces: [] + }, + externalServer: false, + executionErrors: [] +}); + +const postgresRunAttestation = (): PostgresRunAttestationEvidence => ({ + version: 1, + kind: 'physical-density-measurement-attestation-v1', + artifactPath: '/tmp/result/postgres-run-attestation.json', + artifactSha256: `sha256:${'1'.repeat(64)}`, + payloadSha256: `sha256:${'2'.repeat(64)}`, + epochId: `sha256:${'3'.repeat(64)}`, + arm: 'scoped-introspection', + heapMiB: 1024, + tenantCount: 1, + repetition: 1, + runOrderIndex: 1, + planSha256: `sha256:${'e'.repeat(64)}`, + fleetSha256: `sha256:${'f'.repeat(64)}`, + containerId: '4'.repeat(64), + containerStartedAt: '2026-07-31T00:00:00.000Z', + cgroupIdentitySha256: `sha256:${'5'.repeat(64)}`, + containerConfigurationSha256: `sha256:${'6'.repeat(64)}`, + postgresSystemIdentifier: '7421234567890123456', + postgresStartedAt: '2026-07-31T00:00:00.010Z', + cloneId: 'measurement-unique-clone', + cloneAttestationSetSha256: `sha256:${'7'.repeat(64)}`, + cloneNonceSetSha256: `sha256:${'8'.repeat(64)}`, + liveContractSetSha256: `sha256:${'9'.repeat(64)}`, + manifestSha256: `sha256:${'a'.repeat(64)}`, + containerTemplateSha256: `sha256:${'b'.repeat(64)}`, + canonicalDatabaseContractFingerprint: `sha256:${'c'.repeat(64)}`, + freshContainerForRun: true, + cgroupV2Verified: true, + liveCustomerContractsAudited: 1, + catalogCacheState: 'warmed-by-live-contract-audit' +}); + +const strictCanaryInput = (): ScoreInput => { + const input = qualifyingInput(); + input.tenants = input.tenants.map((configuredTenant) => ({ + ...configuredTenant, + surfaces: configuredTenant.surfaces.map((configuredSurface) => ({ + ...configuredSurface, + canaries: [...configuredSurface.canaries] + })) + })); + const surface = input.tenants[0].surfaces[0]; + surface.canaries = [ + ...surface.canaries, + { + name: 'prepared-reuse', + query: '{ __typename }', + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b' }], + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a' }] + } + ]; + input.requiredCanaries = ['cross-schema', 'prepared-reuse']; + input.canaryIntervalSec = 300; + input.periodicCanarySchedule = 'rotating-one'; + input.gates = { ...input.gates, requireCompletePeriodicCanaryCoverage: true }; + const startedMs = Date.parse('2026-07-31T00:00:00.000Z'); + const evidence = ( + canary: string, + phase: 'initial' | 'periodic' | 'final', + scheduledMs: number, + periodicRound?: number + ) => ({ + tenantId: 'tenant-a', + surface: 'api', + canary, + phase, + ...(periodicRound != null ? { periodicRound } : {}), + scheduledAt: new Date(scheduledMs).toISOString(), + startedAt: new Date(scheduledMs + 10).toISOString(), + completedAt: new Date(scheduledMs + 20).toISOString(), + latencyMs: phase === 'periodic' ? 5_000 : 10, + conclusive: true, + violation: false + }); + const initial = surface.canaries.map((canary) => + evidence(canary.name, 'initial', startedMs - 1_000) + ); + const periodic = [1, 2].map((periodicRound) => { + const canary = surface.canaries[rotatingCanaryIndex( + 'tenant-a', + 'api', + surface.canaries.length, + periodicRound + )]; + return evidence( + canary.name, + 'periodic', + startedMs + periodicRound * 300_000, + periodicRound + ); + }); + const final = surface.canaries.map((canary) => + evidence(canary.name, 'final', startedMs + 900_000) + ); + input.canaries = [...initial, ...periodic, ...final]; + input.canarySchedule = { + schedule: 'rotating-one', + intervalMs: 300_000, + durationMs: 900_000, + canaryConcurrency: 1, + startedAt: new Date(startedMs).toISOString(), + deadlineAt: new Date(startedMs + 900_000).toISOString(), + planned: 2, + started: 2, + completed: 2, + missed: 0, + overlapped: 0, + deadlineLate: 0, + checksPlanned: 2, + checksStarted: 2, + checksCompleted: 2, + rounds: [1, 2].map((periodicRound) => ({ + periodicRound, + plannedAt: new Date(startedMs + periodicRound * 300_000).toISOString(), + startedAt: new Date(startedMs + periodicRound * 300_000 + 10).toISOString(), + completedAt: new Date(startedMs + periodicRound * 300_000 + 20).toISOString(), + targetsPlanned: 1, + targetsStarted: 1, + targetsCompleted: 1, + checksPlanned: 1, + checksStarted: 1, + checksCompleted: 1, + overlapped: false, + deadlineLate: false, + startDelayMs: 10, + durationMs: 10 + })) + }; + return input; +}; + +describe('density scoring', () => { + it('uses nearest-rank percentiles', () => { + expect(percentile([40, 10, 30, 20], 0.5)).toBe(20); + expect(percentile([40, 10, 30, 20], 0.99)).toBe(40); + }); + + it('qualifies only an exact fresh PostgreSQL run/server binding', () => { + const input = qualifyingInput(); + const attestation = postgresRunAttestation(); + input.gates = { ...input.gates, requireFreshPostgresRunAttestation: true }; + input.postgresRunAttestation = attestation; + input.provenance = { + ...input.provenance!, + command: [ + ...input.provenance!.command, + '--expected-manifest-sha256', attestation.manifestSha256, + '--clone-id', attestation.cloneId + ] + }; + expect(scoreRun(input).accepted).toBe(true); + + input.postgresRunAttestation = { + ...attestation, + freshContainerForRun: false + }; + expect(scoreRun(input).failures).toContain( + 'fresh PostgreSQL run attestation is incomplete or mismatched' + ); + input.postgresRunAttestation = null; + expect(scoreRun(input).failures).toContain( + 'fresh PostgreSQL run attestation unavailable' + ); + }); + + it('requires Linux /proc samples for the exact server PID to qualify', () => { + const nonLinux = qualifyingInput(); + nonLinux.provenance = { ...nonLinux.provenance!, platform: 'darwin' }; + expect(scoreRun(nonLinux).failures).toContain( + 'qualification requires exact-PID Linux /proc RSS evidence' + ); + + const wrongPid = qualifyingInput(); + wrongPid.postWarmupNodeRssSnapshots = wrongPid.postWarmupNodeRssSnapshots.map( + (snapshot) => ({ ...snapshot, pid: 999 }) + ); + expect(scoreRun(wrongPid).failures).toContain( + 'qualification requires exact-PID Linux /proc RSS evidence' + ); + + const endpointRss = qualifyingInput(); + endpointRss.postWarmupNodeRssSnapshots = endpointRss.postWarmupNodeRssSnapshots.map( + (snapshot) => ({ ...snapshot, source: 'authenticated-endpoint' }) + ); + expect(scoreRun(endpointRss).failures).toContain( + 'qualification requires exact-PID Linux /proc RSS evidence' + ); + }); + + it('measures a linear heap slope in MiB per hour', () => { + expect(heapGrowthMiBPerHour([memory(0, 100), memory(30, 102), memory(60, 104)])) + .toBeCloseTo(4, 5); + }); + + it('uses conservative converged bookends for retained heap and external growth', () => { + const summary = retainedMemoryGrowth({ + baseline: retainedCheckpoint(0, 100, 10), + final: retainedCheckpoint(15, 101, 10.5), + errors: [] + }, 42, new Set(['tenant-a-api'])); + expect(summary.errors).toEqual([]); + expect(summary.heapMiBPerHour).toBeCloseTo(4, 4); + expect(summary.externalMiBPerHour).toBeCloseTo(2, 4); + expect(summary.durationSec).toBeCloseTo(900, 4); + }); + + it('reports zero and negative retained growth without clamping', () => { + const zero = retainedMemoryGrowth({ + baseline: retainedCheckpoint(0, 100), + final: retainedCheckpoint(15, 100), + errors: [] + }); + const negative = retainedMemoryGrowth({ + baseline: retainedCheckpoint(0, 100), + final: retainedCheckpoint(15, 99), + errors: [] + }); + expect(zero.heapMiBPerHour).toBe(0); + expect(zero.externalMiBPerHour).toBe(0); + expect(negative.heapMiBPerHour).toBeCloseTo(-4, 4); + }); + + it('independently enforces the one-MiB convergence envelope', () => { + const within = retainedCheckpoint(15, 100); + const outside = retainedCheckpoint(15, 100); + within.samples.slice(-3).forEach((sample, index) => { + sample.heapUsedBytes += [0, 0.4, 0.9][index] * 1024 ** 2; + }); + outside.samples.slice(-3).forEach((sample, index) => { + sample.heapUsedBytes += [0, 2, 0][index] * 1024 ** 2; + }); + expect(retainedMemoryGrowth({ + baseline: retainedCheckpoint(0, 100), + final: within, + errors: [] + }).errors).toEqual([]); + expect(retainedMemoryGrowth({ + baseline: retainedCheckpoint(0, 100), + final: outside, + errors: [] + }).errors).toContain('final retained heapUsedBytes samples did not converge'); + }); + + it('keeps raw heap OLS diagnostic and gates on retained bookends', () => { + const input = qualifyingInput(); + input.postWarmupSnapshots = [memory(0, 100), memory(15, 200)]; + input.memorySnapshots = input.postWarmupSnapshots; + const result = scoreRun(input); + expect(result.rawPostWarmupHeapGrowthMiBPerHour).toBeCloseTo(400, 4); + expect(result.retainedHeapGrowthMiBPerHour).toBeCloseTo(4, 4); + expect(result.accepted).toBe(true); + }); + + it('allows only balanced HTTP lifecycle progress between retained bookends', () => { + const baseline = retainedCheckpointWithState(0, retainedPhysicalState(10, 10)); + const balanced = retainedCheckpointWithState(15, retainedPhysicalState(110, 110)); + expect(retainedMemoryGrowth({ baseline, final: balanced, errors: [] }).errors) + .toEqual([]); + + const unbalanced = retainedCheckpointWithState(15, retainedPhysicalState(110, 109)); + expect(retainedMemoryGrowth({ baseline, final: unbalanced, errors: [] }).errors) + .toContain( + 'retained-memory HTTP handler delta is unbalanced: started=100, completed=99' + ); + + const changedTopology = retainedCheckpointWithState( + 15, + retainedPhysicalState(110, 110, 2) + ); + expect(retainedMemoryGrowth({ baseline, final: changedTopology, errors: [] }).errors) + .toContain('retained-memory residency or non-HTTP counters changed across the workload'); + }); + + it('compares physical residency through stable fingerprints, not process-local HMAC keys', () => { + const state = { + pid: 42, + graphileInFlight: 0, + residentBuildContracts: ['graphile:v1:process-local-hmac'], + residentBuildContractFingerprints: ['tenant-a-api'], + counters: { buildsStarted: 1, evictions: 0 } + }; + const summary = retainedMemoryGrowth({ + baseline: retainedCheckpointWithState(0, state), + final: retainedCheckpointWithState(15, state), + errors: [] + }, 42, new Set(['tenant-a-api']), true); + expect(summary.errors).toEqual([]); + + const missingStable = { ...state }; + delete (missingStable as Partial).residentBuildContractFingerprints; + expect(retainedMemoryGrowth({ + baseline: retainedCheckpointWithState(0, missingStable), + final: retainedCheckpointWithState(15, missingStable), + errors: [] + }, 42, new Set(['tenant-a-api']), true).errors).toContain( + 'baseline retained-memory residency set mismatch' + ); + }); + + it('rejects retained external growth even when V8 retained heap passes', () => { + const input = qualifyingInput(); + input.retainedMemory.final = retainedCheckpoint(15, 101, 12); + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures).toContain( + 'retained external-memory growth 8.00MiB/hour exceeds 5' + ); + }); + + it('fails closed when a retained checkpoint is unstable', () => { + const input = qualifyingInput(); + input.retainedMemory.final = { + ...input.retainedMemory.final!, + stable: false, + errors: ['PDCF_RETAINED_HEAP_NOT_CONVERGED'] + }; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures.some((failure) => + failure.includes('retained-memory checkpoint errors') + )).toBe(true); + }); + + it('falls back to the raw heap-growth gate when retained checkpoints are optional', () => { + const input = qualifyingInput(); + input.gates = { ...input.gates, requireRetainedMemoryCheckpoints: false }; + input.retainedMemory = { baseline: null, final: null, errors: [] }; + const accepted = scoreRun(input); + expect(accepted.accepted).toBe(true); + expect(accepted.retainedHeapGrowthMiBPerHour).toBeNull(); + expect(accepted.retainedMemoryCheckpointErrors).toEqual([ + 'baseline retained-memory checkpoint is unavailable', + 'final retained-memory checkpoint is unavailable' + ]); + + input.postWarmupSnapshots = [memory(0, 100), memory(15, 110)]; + input.memorySnapshots = input.postWarmupSnapshots; + const rejected = scoreRun(input); + expect(rejected.accepted).toBe(false); + expect(rejected.failures).toContain('heap growth 40.00MiB/hour exceeds 5'); + }); + + it('scores the service footprint from near-simultaneous current RSS and PostgreSQL samples', () => { + const node = [ + { ...memory(0, 100), rssBytes: 200 }, + { ...memory(1, 100), rssBytes: 250 } + ]; + const postgres = [ + { timestamp: '2026-07-31T00:00:00.100Z', usedBytes: 50, limitBytes: 1_000, raw: '' }, + { timestamp: '2026-07-31T00:01:00.100Z', usedBytes: 80, limitBytes: 1_000, raw: '' } + ]; + expect(alignedServiceMemoryPeak(node, postgres)).toEqual({ + bytes: 330, + nodeRssBytes: 250, + postgresBytes: 80, + timestamp: '2026-07-31T00:01:00.000Z', + samples: 2, + maxSkewMs: 100 + }); + expect(alignedServiceMemoryPeak(node, [{ + ...postgres[0], + timestamp: '2026-07-31T00:10:00.000Z' + }])).toBeNull(); + expect(alignedServiceMemoryCoverage( + node, + postgres, + Date.parse('2026-07-31T00:00:00.000Z'), + 60_000 + )).toMatchObject({ + expectedDurationMs: 60_000, + coveredDurationMs: 60_000, + coverageRatio: 1, + maxGapMs: 60_000 + }); + }); + + it('uses cgroup memory.peak for the conservative denominator and limits current fallback to diagnostics', () => { + const qualifying = scoreRun(qualifyingInput()); + expect(qualifying.serviceMemoryUpperBoundPostgresSource) + .toBe('cgroup-v2-memory.peak'); + expect(qualifying.serviceMemoryUpperBoundBytes) + .toBe(220 * 1024 ** 2 + 180); + + const missingPeak = qualifyingInput(); + missingPeak.postgresSnapshots = missingPeak.postgresSnapshots.map((snapshot) => ({ + ...snapshot, + cgroupV2: { ...snapshot.cgroupV2!, peakBytes: null as number | null } + })); + const rejected = scoreRun(missingPeak); + expect(rejected.serviceMemoryUpperBoundBytes).toBeNull(); + expect(rejected.failures).toEqual(expect.arrayContaining([ + 'PostgreSQL cgroup-v2 memory.peak telemetry unavailable for conservative denominator', + 'conservative service-memory upper bound unavailable' + ])); + + const mismatchedCurrent = qualifyingInput(); + mismatchedCurrent.postgresSnapshots[0] = { + ...mismatchedCurrent.postgresSnapshots[0], + usedBytes: mismatchedCurrent.postgresSnapshots[0].usedBytes + 1 + }; + expect(scoreRun(mismatchedCurrent).failures).toContain( + 'PostgreSQL cgroup-v2 telemetry was incomplete' + ); + + missingPeak.evidenceMode = 'diagnostic'; + const diagnostic = scoreRun(missingPeak); + expect(diagnostic.serviceMemoryUpperBoundPostgresSource) + .toBe('sampled-current-diagnostic'); + expect(diagnostic.serviceMemoryUpperBoundBytes) + .toBe(220 * 1024 ** 2 + 120); + }); + + it('requires aligned cgroup telemetry to cover the entire post-warm workload at bounded cadence', () => { + const input = qualifyingInput(); + input.gates = { + ...input.gates, + maxAlignedMemorySampleGapMs: 1_000, + minAlignedMemoryCoverageRatio: 0.99 + }; + const startedAtMs = Date.parse('2026-07-31T00:00:00.000Z'); + input.postWarmupNodeRssSnapshots = Array.from({ length: 901 }, (_unused, index) => ({ + timestamp: new Date(startedAtMs + index * 1_000).toISOString(), + pid: 42, + source: 'proc' as const, + rssBytes: 200 * 1024 ** 2 + })); + input.postgresSnapshots = Array.from({ length: 901 }, (_unused, index) => ({ + timestamp: new Date(startedAtMs + index * 1_000 + 100).toISOString(), + usedBytes: 100 + index, + limitBytes: 1_000_000, + source: 'cgroup-v2' as const, + cgroupV2: { + currentBytes: 100 + index, + peakBytes: 1_000 + index, + maxBytes: 1_000_000, + stat: {}, + events: { oom: 0, oom_kill: 0 } + }, + raw: '' + })); + const complete = scoreRun(input); + expect(complete.accepted).toBe(true); + expect(complete.alignedServiceMemoryCoverageRatio).toBe(1); + expect(complete.alignedServiceMemoryMaxGapMs).toBe(1_000); + + const densePostgresSnapshots = input.postgresSnapshots; + input.postgresSnapshots = densePostgresSnapshots.filter((_snapshot, index) => + index % 2 === 0 + ); + const sparsePostgres = scoreRun(input); + expect(sparsePostgres.accepted).toBe(false); + expect(sparsePostgres.alignedServiceMemoryMaxGapMs).toBe(2_000); + expect(sparsePostgres.failures).toContain( + 'aligned service-memory maximum sample gap 2000ms exceeds 1000ms' + ); + input.postgresSnapshots = densePostgresSnapshots; + + input.postWarmupNodeRssSnapshots = input.postWarmupNodeRssSnapshots.slice(0, -10); + input.postgresSnapshots = input.postgresSnapshots.slice(0, -10); + const truncated = scoreRun(input); + expect(truncated.accepted).toBe(false); + expect(truncated.failures).toEqual(expect.arrayContaining([ + expect.stringContaining('maximum sample gap 10000ms exceeds 1000ms'), + expect.stringContaining('workload coverage 98.89% is below 99.00%') + ])); + }); + + it('qualifies only a complete, conclusive, resident tenant', () => { + const result = scoreRun(qualifyingInput()); + expect(result.accepted).toBe(true); + expect(result.qualifiedCustomers).toBe(1); + expect(result.qualifiedTenants).toBe(1); + expect(result.tenantsPerConfiguredOldSpaceGiB).toBe(1); + expect(result.configuredCustomersPerAlignedServiceGiB).toBeGreaterThan(0); + expect(result.observedHeapLimitBytes).toBe(1024 * 1024 ** 2); + expect(result).toMatchObject({ + pgPoolCacheSize: 2, + pgPoolLeasedPools: 1, + pgPoolActiveLeases: 1, + postWarmupPgPoolCapacityEvictions: 0, + postWarmupPgPoolCapacityRefusals: 0, + postWarmupPgPoolDisposalFailures: 0 + }); + }); + + it('requires conclusive per-operation coverage evidence when the gate is enabled', () => { + const input = qualifyingInput(); + input.gates = { ...input.gates, requireConclusiveOperationOracles: true }; + input.tenants[0].surfaces[0].operations[0] = { + ...input.tenants[0].surfaces[0].operations[0], + requiredMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-a' + }], + forbiddenMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-b' + }] + }; + input.samples.unshift({ + tenantId: 'tenant-a', + surface: 'api', + operation: 'read', + capability: 'graphile', + latencyMs: 10, + status: 200, + ok: true, + phase: 'coverage', + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: false + }); + expect(scoreRun(input).accepted).toBe(true); + + input.samples[0] = { + ...input.samples[0], + ok: false, + oracleConclusive: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_MISSING' + }; + const missing = scoreRun(input); + expect(missing.accepted).toBe(false); + expect(missing.operationOracleInconclusive).toBe(1); + expect(missing.missingOperationOracles).toEqual(['tenant-a/api/read']); + expect(missing.failures).toEqual(expect.arrayContaining([ + 'GraphQL operation response oracles inconclusive=1', + 'missing GraphQL operation response oracles: tenant-a/api/read' + ])); + + input.samples[0] = { + ...input.samples[0], + oracleConclusive: true, + oracleViolation: true, + errorCode: 'GRAPHQL_OPERATION_ORACLE_FORBIDDEN' + }; + const forbidden = scoreRun(input); + expect(forbidden.operationOracleViolations).toBe(1); + expect(forbidden.failures).toContain( + 'GraphQL operation response oracle violations=1' + ); + }); + + it('keeps the 0.5% request-error budget without treating unavailable oracles as bleed', () => { + const input = qualifyingInput(); + input.gates = { ...input.gates, requireConclusiveOperationOracles: true }; + input.tenants[0].surfaces[0].operations[0] = { + ...input.tenants[0].surfaces[0].operations[0], + requiredMatches: [{ path: '/data/physicalDatabaseIdentity', value: 'physical-db-a' }], + forbiddenMatches: [{ path: '/data/physicalDatabaseIdentity', value: 'physical-db-b' }] + }; + const baseSample = { + tenantId: 'tenant-a', + surface: 'api', + operation: 'read', + capability: 'graphile', + latencyMs: 20, + status: 200, + ok: true, + phase: 'workload' as const, + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: false, + oracleUnavailable: false + }; + input.samples = [ + { + ...baseSample, + phase: 'coverage', + }, + ...Array.from({ length: 199 }, () => ({ ...baseSample })), + { + ...baseSample, + status: 0, + ok: false, + errorCode: 'TIMEOUT', + oracleConclusive: false, + oracleUnavailable: true + } + ]; + const result = scoreRun(input); + expect(result.accepted).toBe(true); + expect(result.errorRate).toBe(0.005); + expect(result.operationOracleInconclusive).toBe(0); + expect(result.operationOracleViolations).toBe(0); + }); + + it('requires exactly one conclusive coverage result per operation', () => { + const input = qualifyingInput(); + input.gates = { ...input.gates, requireConclusiveOperationOracles: true }; + input.tenants[0].surfaces[0].operations[0] = { + ...input.tenants[0].surfaces[0].operations[0], + requiredMatches: [{ path: '/data/physicalDatabaseIdentity', value: 'physical-db-a' }], + forbiddenMatches: [{ path: '/data/physicalDatabaseIdentity', value: 'physical-db-b' }] + }; + const coverage = { + tenantId: 'tenant-a', + surface: 'api', + operation: 'read', + capability: 'graphile', + latencyMs: 10, + status: 200, + ok: true, + phase: 'coverage' as const, + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: false + }; + input.samples.unshift(coverage, { ...coverage }); + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.missingOperationOracles).toEqual(['tenant-a/api/read']); + }); + + it('requires exact initial, rotating periodic, and final canary evidence', () => { + const accepted = scoreRun(strictCanaryInput()); + expect(accepted.accepted).toBe(true); + expect(accepted).toMatchObject({ + customerWorkloadRps: 1 / 900, + periodicValidationRps: 2 / 900, + achievedRps: 1 / 900, + p99Ms: 20 + }); + expect(accepted.combinedHttpRps).toBeCloseTo(3 / 900, 12); + + for (const phase of ['initial', 'periodic', 'final'] as const) { + const input = strictCanaryInput(); + const removed = input.canaries.findIndex((canary) => canary.phase === phase); + input.canaries.splice(removed, 1); + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures.some((failure) => + failure.includes('missing exact canary evidence') + )).toBe(true); + if (phase === 'periodic') { + expect(result.failures.some((failure) => + failure.includes('periodic canary coverage is incomplete') + )).toBe(true); + expect(result.failures.some((failure) => + failure.includes('periodic target/round evidence mismatch') + )).toBe(true); + } + } + }); + + it('rejects missing, duplicate, and deadline-late periodic rounds', () => { + const missingRound = strictCanaryInput(); + missingRound.canarySchedule!.completed = 1; + missingRound.canarySchedule!.missed = 1; + missingRound.canarySchedule!.rounds[1].completedAt = null; + let result = scoreRun(missingRound); + expect(result.accepted).toBe(false); + expect(result.failures.some((failure) => + failure.includes('periodic canary rounds planned=2 started=2 completed=1 missed=1') + )).toBe(true); + + const duplicate = strictCanaryInput(); + duplicate.canaries.push({ ...duplicate.canaries.find((canary) => + canary.phase === 'periodic' + )! }); + result = scoreRun(duplicate); + expect(result.accepted).toBe(false); + expect(result.failures.some((failure) => + failure.includes('duplicate exact canary evidence') + )).toBe(true); + + const late = strictCanaryInput(); + const deadlineMs = Date.parse(late.canarySchedule!.deadlineAt); + late.canarySchedule!.deadlineLate = 1; + late.canarySchedule!.rounds[1].deadlineLate = true; + late.canarySchedule!.rounds[1].completedAt = new Date(deadlineMs + 1).toISOString(); + const lateResult = late.canaries.find((canary) => + canary.phase === 'periodic' && canary.periodicRound === 2 + )!; + lateResult.completedAt = new Date(deadlineMs + 1).toISOString(); + result = scoreRun(late); + expect(result.accepted).toBe(false); + expect(result.failures.some((failure) => + failure.includes('periodic canary rounds completed after deadline') + )).toBe(true); + }); + + it('fails closed when raw cgroup samples omit OOM event counters', () => { + const input = qualifyingInput(); + input.postgresSnapshots = input.postgresSnapshots.map((snapshot) => ({ + ...snapshot, + source: 'cgroup-v2' as const, + cgroupV2: { + currentBytes: snapshot.usedBytes, + peakBytes: snapshot.usedBytes, + maxBytes: snapshot.limitBytes, + stat: {}, + events: {} + } + })); + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.postgresOomEvents).toBeNull(); + expect(result.failures).toContain('PostgreSQL cgroup OOM event telemetry unavailable'); + }); + + it('never qualifies a smoke-length run or an eviction', () => { + const input = qualifyingInput(); + input.endedAt = '2026-07-31T00:00:05.000Z'; + input.configuredDurationSec = 5; + input.workloadDurationMs = 5_000; + input.memorySnapshots = [memory(0, 100), memory(1, 100, 1)]; + input.postWarmupSnapshots = input.memorySnapshots; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.qualifiedTenants).toBe(0); + expect(result.failures).toEqual(expect.arrayContaining([ + expect.stringContaining('15-minute'), + expect.stringContaining('evictions=1') + ])); + }); + + it('rejects matching cache counts with the wrong resident build identity', () => { + const input = qualifyingInput(); + input.memorySnapshots = input.memorySnapshots.map((snapshot): MemorySnapshot => ({ + ...snapshot, + residentBuildContracts: ['tenant-b-api'] + })); + input.postWarmupSnapshots = input.memorySnapshots; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures).toContain('resident Graphile build contracts missing: tenant-a-api'); + }); + + it('requires successful capability traffic for each configured tenant surface', () => { + const input = qualifyingInput(); + input.tenants = [{ + ...tenant, + surfaces: tenant.surfaces.map((surface) => ({ + ...surface, + operations: [ + ...surface.operations, + { name: 'search', capability: 'bm25', query: '{ search }' } + ] + })) + }]; + input.requiredCapabilities = ['graphile', 'bm25']; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.tenants[0]).toMatchObject({ + missingOperations: ['api/search'], + missingCapabilities: ['api/bm25', 'required/bm25'] + }); + expect(result.missingCapabilities).toEqual([ + 'tenant-a/api/bm25', + 'tenant-a/required/bm25' + ]); + }); + + it('does not let a healthy aggregate hide a surface that exceeds its SLA', () => { + const input = qualifyingInput(); + const apiSurface = input.tenants[0].surfaces[0]; + input.tenants = [{ + ...input.tenants[0], + surfaces: [ + apiSurface, + { ...apiSurface, name: 'admin' } + ] + }]; + input.warmedSurfaces = new Map([['tenant-a', new Set(['api', 'admin'])]]); + input.samples = ['api', 'admin'].flatMap((surface) => + Array.from({ length: 125 }, (_unused, index) => ({ + ...input.samples[0], + surface, + ok: !(surface === 'admin' && index === 0), + status: surface === 'admin' && index === 0 ? 500 : 200 + })) + ); + input.canaries = ['api', 'admin'].map((surface) => ({ + ...input.canaries[0], + surface + })); + const result = scoreRun(input); + expect(result.errorRate).toBe(0.004); + expect(result.errorRate).toBeLessThanOrEqual(input.gates.maxErrorRate); + expect(result.tenants[0].surfaces.find(({ surface }) => surface === 'admin')) + .toMatchObject({ errorRate: 0.008, qualified: false }); + expect(result.accepted).toBe(false); + }); + + it('does not let coverage-only traffic qualify a resident surface', () => { + const input = qualifyingInput(); + input.samples[0].phase = 'coverage'; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.tenants[0]).toMatchObject({ + surfacesWithTraffic: 0, + missingSurfaces: ['api'] + }); + }); + + it('keeps coverage samples out of workload latency and error metrics', () => { + const input = qualifyingInput(); + input.samples.unshift({ + tenantId: 'tenant-a', + surface: 'api', + operation: 'read', + capability: 'graphile', + latencyMs: 5_000, + status: 500, + ok: false, + phase: 'coverage', + errorCode: 'GRAPHQL_ERROR' + }); + const result = scoreRun(input); + expect(result.accepted).toBe(true); + expect(result).toMatchObject({ + coverageRequests: 1, + workloadRequests: 1, + errors: 0, + p99Ms: 20 + }); + }); + + it('rejects missing process RSS, missed arrivals, and incomplete provenance', () => { + const input = qualifyingInput(); + input.memorySnapshots = input.memorySnapshots.map((snapshot): MemorySnapshot => ({ + ...snapshot, + processPeakRssBytes: null, + pgPoolCacheSize: null + })); + input.postWarmupSnapshots = input.memorySnapshots; + input.missedArrivals = 2; + input.provenance = { ...provenance, entrySha256: null }; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures).toEqual(expect.arrayContaining([ + 'load generator missed scheduled arrivals=2', + 'OS process peak RSS telemetry unavailable', + 'PostgreSQL pool-cache telemetry unavailable', + expect.stringContaining('server provenance incomplete: entrySha256') + ])); + }); + + it('rejects post-warmup PostgreSQL pool churn and disposal failures', () => { + const input = qualifyingInput(); + input.postWarmupSnapshots = [ + memory(0, 100), + { + ...memory(15, 101), + pgPoolCapacityEvictions: 1, + pgPoolCapacityRefusals: 1, + pgPoolDisposalFailures: 1 + } + ]; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures).toEqual(expect.arrayContaining([ + 'post-warmup PostgreSQL pool capacity evictions=1', + 'post-warmup PostgreSQL pool capacity refusals=1', + 'post-warmup PostgreSQL pool disposal failures=1' + ])); + }); + + it('rejects cache, build, and pool counters that reset during the workload', () => { + const input = qualifyingInput(); + input.postWarmupSnapshots = [ + { + ...memory(0, 100, 2), + buildRefusals: 2, + buildsStarted: 2, + pgPoolCapacityEvictions: 2, + pgPoolCapacityRefusals: 2, + pgPoolDisposalFailures: 2 + }, + { + ...memory(7, 100, 3), + buildRefusals: 3, + buildsStarted: 3, + pgPoolCapacityEvictions: 3, + pgPoolCapacityRefusals: 3, + pgPoolDisposalFailures: 3 + }, + { + ...memory(15, 101, 2), + buildRefusals: 2, + buildsStarted: 2, + pgPoolCapacityEvictions: 2, + pgPoolCapacityRefusals: 2, + pgPoolDisposalFailures: 2 + } + ]; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures).toEqual(expect.arrayContaining([ + 'post-warmup evictions=unknown', + 'post-warmup build refusals=unknown', + 'post-warmup builds=unknown', + 'post-warmup PostgreSQL pool capacity evictions=unknown', + 'post-warmup PostgreSQL pool capacity refusals=unknown', + 'post-warmup PostgreSQL pool disposal failures=unknown' + ])); + }); + + it('requires physical database, backend, pool-client, and realtime residency when enabled', () => { + const input = qualifyingInput(); + input.gates = { ...gates, requirePhysicalDatabaseTelemetry: true }; + input.provenance = { + ...provenance, + memoryPolicy: { + ...provenance.memoryPolicy!, + graphileCacheCalibrationId: 'measured-cache-v1' + } + }; + input.tenants = [{ + ...tenant, + databases: [{ + id: 'logical:tenant-a', + physicalDatabase: 'physical_tenant_a', + apis: [{ + id: 'api:tenant-a', + runtimePoolIdentity: 'pg:v1:tenant-a', + physicalSchemas: ['tenant_a'], + routingLabels: ['tenant-a.localhost'], + realtime: true, + surfaces: ['api'] + }] + }] + }]; + input.memorySnapshots = input.memorySnapshots.map((snapshot): MemorySnapshot => ({ + ...snapshot, + cacheConfiguredMax: 3, + cacheBudgetCapacity: 3, + cacheInstanceHeapBytes: 16 * 1024 ** 2, + cacheCalibrationId: 'measured-cache-v1', + physicalDatabases: 1, + postgresContainerDedicated: true, + unexpectedPostgresDatabases: 0, + postgresBackendTotal: 1, + pgPoolTotalClients: 1, + pgPoolIdleClients: 0, + pgPoolWaitingClients: 0, + runtimePoolTelemetryScope: 'runtime-only-exact-identities', + runtimePoolTelemetryAvailable: true, + runtimePoolRequestedMaxUses: null, + runtimePoolEffectiveMaxUses: null, + runtimePoolEffectiveMaxUsesKnown: true, + runtimePoolMaxUsesExact: true, + runtimePoolExpectedPools: 1, + runtimePoolObservedPools: 1, + runtimePoolTotalClients: 1, + runtimePoolIdleClients: 0, + runtimePoolWaitingClients: 0, + realtimeManagersExpected: 1, + realtimeManagersActive: 1, + realtimeTransportsExpected: 1, + realtimeTransportsActive: 1, + realtimeNotificationMode: 'dedicated' + })); + input.postWarmupSnapshots = input.memorySnapshots; + const accepted = scoreRun(input); + expect(accepted.accepted).toBe(true); + expect(accepted).toMatchObject({ + residentPhysicalDatabases: 1, + cacheConfiguredMax: 3, + cacheBudgetCapacity: 3, + cacheCalibrationId: 'measured-cache-v1', + postgresContainerDedicated: true, + unexpectedPostgresDatabases: 0, + postgresBackendPeak: 1, + pgPoolTotalClients: 1, + runtimePoolExpectedPools: 1, + runtimePoolObservedPools: 1, + residentRealtimeManagers: 1, + residentRealtimeTransports: 1 + }); + + input.postWarmupSnapshots = input.memorySnapshots.map((snapshot): MemorySnapshot => ({ + ...snapshot, + runtimePoolRequestedMaxUses: 1, + runtimePoolEffectiveMaxUses: 1 + })); + const singleCheckout = scoreRun(input); + expect(singleCheckout.accepted).toBe(true); + expect(singleCheckout).toMatchObject({ + runtimePoolRequestedMaxUses: 1, + runtimePoolEffectiveMaxUses: 1 + }); + + const exactRuntimePoolSnapshots = input.postWarmupSnapshots; + input.postWarmupSnapshots = exactRuntimePoolSnapshots.map((snapshot, index) => ({ + ...snapshot, + runtimePoolExpectedPools: index === 0 ? 1 : 2, + runtimePoolObservedPools: index === 0 ? 1 : 2 + })); + const inexactCardinality = scoreRun(input); + expect(inexactCardinality.accepted).toBe(false); + expect(inexactCardinality.failures).toContain( + 'exact runtime PostgreSQL pool telemetry unavailable or inconsistent; observed=unknown, expected=1' + ); + + input.postWarmupSnapshots = exactRuntimePoolSnapshots.map((snapshot, index) => ({ + ...snapshot, + runtimePoolIdleClients: index === 0 ? 1 : 0 + })); + expect(scoreRun(input).failures).toContain( + 'runtime PostgreSQL maxUses=1 retained idle clients after warmup' + ); + + input.postWarmupSnapshots = input.memorySnapshots.map((snapshot): MemorySnapshot => ({ + ...snapshot, + postgresBackendTotal: null + })); + const rejected = scoreRun(input); + expect(rejected.accepted).toBe(false); + expect(rejected.failures).toContain('physical PostgreSQL backend telemetry unavailable'); + }); + + it('qualifies shared realtime from exact broker evidence without requiring one backend per API', () => { + const input = qualifyingInput(); + const residentContracts = [ + 'tenant-a-api', + 'tenant-a-admin', + 'tenant-a-private' + ]; + input.gates = { ...gates, requirePhysicalDatabaseTelemetry: true }; + input.provenance = { + ...provenance, + memoryPolicy: { + ...provenance.memoryPolicy!, + graphileCacheCalibrationId: 'measured-cache-v1' + } + }; + input.tenants = [{ + ...tenant, + surfaces: [ + tenant.surfaces[0], + { ...tenant.surfaces[0], name: 'admin', buildContract: 'tenant-a-admin' }, + { ...tenant.surfaces[0], name: 'private', buildContract: 'tenant-a-private' } + ], + databases: [{ + id: 'logical:tenant-a', + physicalDatabase: 'physical_tenant_a', + apis: ['api', 'admin', 'private'].map((name) => ({ + id: `${name}:tenant-a`, + runtimePoolIdentity: `pg:v1:tenant-a:${name}`, + physicalSchemas: [`tenant_a_${name}`], + routingLabels: [`${name}.tenant-a.localhost`], + realtime: true, + surfaces: [name] + })) + }] + }]; + input.warmedSurfaces = new Map([[ + 'tenant-a', + new Set(['api', 'admin', 'private']) + ]]); + input.samples = ['api', 'admin', 'private'].map((surface) => ({ + ...input.samples[0], + surface + })); + input.canaries = ['api', 'admin', 'private'].map((surface) => ({ + ...input.canaries[0], + surface + })); + input.memorySnapshots = input.memorySnapshots.map((snapshot): MemorySnapshot => ({ + ...snapshot, + cacheSize: 3, + residentBuildContractFingerprints: residentContracts, + residentBuildContracts: residentContracts, + cacheConfiguredMax: 3, + cacheBudgetCapacity: 3, + cacheInstanceHeapBytes: 16 * 1024 ** 2, + cacheCalibrationId: 'measured-cache-v1', + physicalDatabases: 1, + postgresContainerDedicated: true, + unexpectedPostgresDatabases: 0, + postgresBackendTotal: 1, + pgPoolTotalClients: 1, + pgPoolIdleClients: 0, + pgPoolWaitingClients: 0, + runtimePoolTelemetryScope: 'runtime-only-exact-identities', + runtimePoolTelemetryAvailable: true, + runtimePoolRequestedMaxUses: null, + runtimePoolEffectiveMaxUses: null, + runtimePoolEffectiveMaxUsesKnown: true, + runtimePoolMaxUsesExact: true, + runtimePoolExpectedPools: 3, + runtimePoolObservedPools: 3, + runtimePoolTotalClients: 1, + runtimePoolIdleClients: 0, + runtimePoolWaitingClients: 0, + realtimeManagersExpected: 3, + realtimeManagersActive: 3, + realtimeTransportsExpected: 3, + realtimeTransportsActive: 3, + realtimeNotificationMode: 'shared-exact' as const, + notificationBrokers: 1, + notificationListenerConnections: 1, + notificationBrokerLeases: 3, + notificationBrokerTopics: 3, + notificationBrokerSubscribers: 3, + notificationBrokerQueueOverflows: 0, + notificationBrokerFatalFailures: 0, + notificationAuditIdentities: 1, + notificationAuditsHealthy: 1, + notificationAuditsFailed: 0, + notificationAuditsStale: 0, + notificationAuditAttempts: 3, + notificationAuditFailures: 0, + notificationAuditActiveDatabaseTargets: 1, + notificationAuditDatabaseConflicts: 0 + })); + input.postWarmupSnapshots = input.memorySnapshots; + const bindRetainedResidency = ( + checkpoint: RetainedMemoryCheckpoint + ): RetainedMemoryCheckpoint => { + const state = { + ...checkpoint.guardAfter.state, + residentBuildContracts: residentContracts, + residentBuildContractFingerprints: residentContracts + }; + const guard = { + ...checkpoint.guardAfter, + residentBuildContracts: residentContracts, + state, + stateSha256: `sha256:${createHash('sha256') + .update(canonicalJson(state)) + .digest('hex')}` + }; + return { ...checkpoint, guardBefore: guard, guardAfter: guard }; + }; + input.retainedMemory = { + baseline: bindRetainedResidency(input.retainedMemory!.baseline!), + final: bindRetainedResidency(input.retainedMemory!.final!), + errors: [] + }; + + const accepted = scoreRun(input); + expect(accepted.failures).toEqual([]); + expect(accepted.accepted).toBe(true); + expect(accepted).toMatchObject({ + realtimeNotificationMode: 'shared-exact', + notificationBrokers: 1, + notificationListenerConnections: 1, + notificationBrokerLeases: 3, + notificationBrokerSubscribers: 3, + postgresBackendPeak: 1, + pgPoolTotalClients: 1 + }); + + input.postWarmupSnapshots = input.memorySnapshots.map((snapshot) => ({ + ...snapshot, + notificationBrokerSubscribers: 2 + })); + const rejected = scoreRun(input); + expect(rejected.accepted).toBe(false); + expect(rejected.failures).toContain( + 'shared realtime broker residency or listener-role attestation is not exact' + ); + }); + + it('binds required cache admission to both live telemetry and pinned provenance', () => { + const input = qualifyingInput(); + input.gates = { + ...input.gates, + requiredCacheAdmissionMode: 'preserve-resident' + }; + input.provenance = { + ...provenance, + memoryPolicy: { + ...provenance.memoryPolicy!, + graphileCacheAdmissionMode: 'preserve-resident' + } + }; + input.memorySnapshots = input.memorySnapshots.map((snapshot) => ({ + ...snapshot, + cacheAdmissionMode: 'preserve-resident' as const + })); + input.postWarmupSnapshots = input.memorySnapshots; + const accepted = scoreRun(input); + expect(accepted.accepted).toBe(true); + expect(accepted.cacheAdmissionMode).toBe('preserve-resident'); + + input.postWarmupSnapshots = input.memorySnapshots.map((snapshot) => ({ + ...snapshot, + cacheAdmissionMode: 'evict-idle' as const + })); + const liveMismatch = scoreRun(input); + expect(liveMismatch.accepted).toBe(false); + expect(liveMismatch.failures).toContain( + 'live Graphile cache admission mode=evict-idle, required preserve-resident' + ); + + input.postWarmupSnapshots = input.memorySnapshots; + input.provenance.memoryPolicy!.graphileCacheAdmissionMode = 'evict-idle'; + const pinnedMismatch = scoreRun(input); + expect(pinnedMismatch.accepted).toBe(false); + expect(pinnedMismatch.failures).toContain( + 'pinned Graphile cache admission mode=evict-idle, required preserve-resident' + ); + }); + + const densityRun = ( + arm: string, + configuredTenants: number, + accepted: boolean, + peakRssDensity: number, + repetition = 1, + expectedMatrixRepetitions = 1, + heapMiB = 1024 + ): DensityRunResult => ({ + schemaVersion: 6, + runKind: 'matrix', + evidenceMode: 'qualification', + qualificationCohortSha256: 'a'.repeat(64), + arm, + repetition, + expectedMatrixRepetitions, + accepted, + configuredCustomers: configuredTenants, + qualifiedCustomers: accepted ? configuredTenants : 0, + qualifiedTenants: accepted ? configuredTenants : 0, + tenantsPerConfiguredOldSpaceGiB: accepted + ? configuredTenants / (heapMiB / 1024) + : 0, + tenantsPerPeakRssGiB: accepted ? peakRssDensity : null, + customersPerAlignedServiceGiB: accepted ? peakRssDensity : null, + customersPerServiceMemoryUpperBoundGiB: accepted ? peakRssDensity : null, + heapMiB, + configuredTenants + } as unknown as DensityRunResult); + + it('requires an all-repetition pass and a higher failure to establish capacity', () => { + const runs = [ + densityRun('scoped-introspection', 1, true, 2, 1, 2), + densityRun('scoped-introspection', 1, true, 2.1, 2, 2), + densityRun('scoped-introspection', 2, true, 3, 1, 2), + densityRun('scoped-introspection', 2, true, 3.1, 2, 2), + densityRun('scoped-introspection', 3, false, 0, 1, 2), + densityRun('scoped-introspection', 3, false, 0, 2, 2) + ]; + expect(summarizeCapacityBoundaries(runs)[0]).toMatchObject({ + highestAllRepetitionsPass: 2, + lowestGreaterFail: 3, + monotonicQualification: true, + capacityBoundaryReached: true, + incompleteTenantCounts: [] + }); + + runs.pop(); + expect(summarizeCapacityBoundaries(runs)[0]).toMatchObject({ + capacityBoundaryReached: false, + incompleteTenantCounts: [3] + }); + }); + + it('rejects non-monotonic and duplicate repetition boundaries', () => { + const nonMonotonic = [ + densityRun('scoped-introspection', 1, false, 0), + densityRun('scoped-introspection', 2, true, 2), + densityRun('scoped-introspection', 3, false, 0) + ]; + expect(summarizeCapacityBoundaries(nonMonotonic)[0]).toMatchObject({ + highestAllRepetitionsPass: 2, + monotonicQualification: false, + capacityBoundaryReached: false + }); + + const duplicateRepetition = [ + densityRun('scoped-introspection', 1, true, 1, 1, 2), + densityRun('scoped-introspection', 1, true, 1, 1, 2), + densityRun('scoped-introspection', 2, false, 0, 1, 2), + densityRun('scoped-introspection', 2, false, 0, 2, 2) + ]; + expect(summarizeCapacityBoundaries(duplicateRepetition)[0]).toMatchObject({ + capacityBoundaryReached: false, + incompleteTenantCounts: [1] + }); + }); + + it('decides improvement from actual service memory and keeps heap metrics diagnostic', () => { + const baseline = [ + densityRun('cache-governor-stock', 1, true, 1), + densityRun('cache-governor-stock', 2, false, 0), + densityRun('cache-governor-stock', 3, false, 0) + ]; + const candidate = [ + densityRun('scoped-introspection', 1, true, 1), + densityRun('scoped-introspection', 2, true, 1.3), + densityRun('scoped-introspection', 3, false, 0) + ]; + expect(compareDensity(baseline, candidate, gates)).toMatchObject({ + materiallyBetter: true, + everyHeapAddsTenants: true, + capacityBoundariesComplete: true, + pairedMatrixComplete: true, + configuredOldSpaceMedianImprovement: 1, + configuredOldSpaceNonRegression: true, + peakRssNonRegression: true + }); + expect(compareDensity(baseline, candidate, gates).peakRssMedianImprovement) + .toBeCloseTo(0.3, 10); + + candidate[1].tenantsPerPeakRssGiB = 0.9; + expect(compareDensity(baseline, candidate, gates)).toMatchObject({ + materiallyBetter: true, + peakRssNonRegression: false + }); + + candidate[1].customersPerAlignedServiceGiB = 0.9; + candidate[1].customersPerServiceMemoryUpperBoundGiB = 0.9; + expect(compareDensity(baseline, candidate, gates)).toMatchObject({ + materiallyBetter: false, + alignedServiceNonRegression: false, + serviceMemoryUpperBoundNonRegression: false + }); + }); + + it('requires the additional-customer gate in every paired repetition', () => { + const matrix = ( + arm: string, + capacities: [number, number] + ): DensityRunResult[] => [1, 2, 3, 4].flatMap((count) => [1, 2].map((repetition) => + densityRun( + arm, + count, + count <= capacities[repetition - 1], + count <= capacities[repetition - 1] ? count : 0, + repetition, + 2 + ) + )); + + const baseline = matrix('cache-governor-stock', [1, 2]); + const aggregateOnlyImprovement = matrix('scoped-introspection', [2, 2]); + const aggregateComparison = compareDensity( + baseline, + aggregateOnlyImprovement, + gates + ); + expect(aggregateComparison.baselineBoundaries[0].highestAllRepetitionsPass).toBe(1); + expect(aggregateComparison.candidateBoundaries[0].highestAllRepetitionsPass).toBe(2); + expect(aggregateComparison.capacityBoundariesComplete).toBe(true); + expect(aggregateComparison.everyHeapAddsTenants).toBe(false); + expect(aggregateComparison.materiallyBetter).toBe(false); + + const everyRepetitionImproves = matrix('scoped-introspection', [2, 3]); + expect(compareDensity(baseline, everyRepetitionImproves, gates)).toMatchObject({ + everyHeapAddsTenants: true, + materiallyBetter: true + }); + }); + + it('rejects an unbracketed per-repetition capacity even when the aggregate boundary exists', () => { + const baseline = [1, 2, 3].flatMap((count) => [1, 2].map((repetition) => + densityRun( + 'cache-governor-stock', + count, + count <= repetition, + count <= repetition ? count : 0, + repetition, + 2 + ) + )); + const candidate = [1, 2, 3].flatMap((count) => [1, 2].map((repetition) => + densityRun( + 'scoped-introspection', + count, + count <= repetition + 1, + count <= repetition + 1 ? count : 0, + repetition, + 2 + ) + )); + expect(summarizeCapacityBoundaries(candidate)[0].capacityBoundaryReached).toBe(true); + expect(compareDensity(baseline, candidate, gates)).toMatchObject({ + everyHeapAddsTenants: false, + materiallyBetter: false + }); + }); + + it('does not call an unbracketed or incomplete matrix materially better', () => { + const baseline = [ + densityRun('cache-governor-stock', 1, true, 1), + densityRun('cache-governor-stock', 2, false, 0) + ]; + const candidate = [ + densityRun('scoped-introspection', 1, true, 1), + densityRun('scoped-introspection', 2, true, 1.3) + ]; + expect(compareDensity(baseline, candidate, gates)).toMatchObject({ + materiallyBetter: false, + capacityBoundariesComplete: false, + pairedMatrixComplete: true + }); + expect(compareDensity(baseline, candidate.slice(0, 1), gates)).toMatchObject({ + materiallyBetter: false, + pairedMatrixComplete: false + }); + }); +}); diff --git a/packages/perf-harness/src/catalog-bench.ts b/packages/perf-harness/src/catalog-bench.ts new file mode 100644 index 0000000000..90fa5f91a2 --- /dev/null +++ b/packages/perf-harness/src/catalog-bench.ts @@ -0,0 +1,3996 @@ +import { execFileSync, spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { getHeapStatistics } from 'node:v8'; + +import { execute, grafast } from 'grafast'; +import { withPgClientFromPgService } from 'graphile-build-pg'; +import { + createGraphileInstance, + type GraphileCacheEntry +} from 'graphile-cache'; +import { + ConstructivePreset, + createGrafastCacheLimitsPreset, + makePgService +} from 'graphile-settings'; +import { + type ExecutionResult, + lexicographicSortSchema, + parse, + printSchema} from 'graphql'; +import { Pool } from 'pg'; +import { getPgEnvOptions } from 'pg-env'; + +import { + nodeFlagsForV8Profile, + replaceMaxOldSpaceSize, + tokenizeNodeOptions +} from './process'; +import type { IntrospectionMode, NodeV8Profile } from './types'; + +const BUILD_TRANSIENT_SAMPLE_INTERVAL_MS = 5; +const BACKEND_MEMORY_SAMPLE_INTERVAL_MS = 10; +const BACKEND_MEMORY_MAX_CONCLUSIVE_GAP_MS = 50; +const BACKEND_MEMORY_SAMPLER_START_TIMEOUT_MS = 5_000; +const BACKEND_MEMORY_SAMPLER_STOP_TIMEOUT_MS = 5_000; +const BACKEND_MEMORY_SAMPLER_TERM_TIMEOUT_MS = 1_000; +const BACKEND_MEMORY_SAMPLER_KILL_TIMEOUT_MS = 1_000; +const BACKEND_START_IDENTITY_TOLERANCE_MS = 1_500; +const BACKEND_RETIREMENT_POLL_INTERVAL_MS = 10; +const BACKEND_RETIREMENT_TIMEOUT_MS = 5_000; +const GIB = 1024 ** 3; +const MIB_PER_GIB = 1024; +const LINUX_CGROUP_V2_DENSITY_AUTHORITY = 'linux-cgroup-v2-memory.current'; +const DESTROYED_BACKEND_LOWER_BOUND_LIMITATION = + 'Without a PostgreSQL pre-destroy acknowledgement, sampling may stop before the ' + + 'backend records its terminal VmHWM; every sampled peak is therefore a diagnostic ' + + 'lower bound, even when cadence and process identity are conclusive.'; +const DOCKER_DESKTOP_BACKEND_SAMPLER_LIMITATION = + 'Docker Desktop transports procfs samples across a Linux VM boundary. The backend ' + + 'trace remains diagnostic, and service-density authority must come from a ' + + 'separately validated Linux cgroup-v2 memory.current measurement.'; +const BACKEND_MEMORY_SAMPLER_SCRIPT = String.raw`set -eu +status_file=$1 +stat_file=$2 +proc_stat_file=$3 +expected_pid=$4 +expected_backend_start_epoch_ms=$5 +start_tolerance_ms=$6 +interval_seconds=$7 + +if [ -x /usr/bin/awk ]; then + awk_command=/usr/bin/awk +elif [ -x /bin/awk ]; then + awk_command=/bin/awk +else + printf 'catalog backend sampler requires /usr/bin/awk or /bin/awk\n' >&2 + exit 43 +fi +if [ -x /usr/bin/sleep ]; then + sleep_command=/usr/bin/sleep +elif [ -x /bin/sleep ]; then + sleep_command=/bin/sleep +else + printf 'catalog backend sampler requires /usr/bin/sleep or /bin/sleep\n' >&2 + exit 43 +fi +if [ -x /usr/bin/getconf ]; then + getconf_command=/usr/bin/getconf +elif [ -x /bin/getconf ]; then + getconf_command=/bin/getconf +else + printf 'catalog backend sampler requires /usr/bin/getconf or /bin/getconf\n' >&2 + exit 43 +fi + +clock_ticks=$($getconf_command CLK_TCK) +case "$clock_ticks" in + ''|*[!0-9]*) printf 'invalid CLK_TCK value\n' >&2; exit 43 ;; +esac + +if [ ! -r "$stat_file" ] || [ ! -r "$proc_stat_file" ]; then + printf 'PostgreSQL backend procfs identity files are unavailable\n' >&2 + exit 42 +fi +initial_identity=$($awk_command -v stat_file="$stat_file" -v proc_stat_file="$proc_stat_file" ' + FILENAME == stat_file { + line=$0 + sub(/^.*\) /, "", line) + count=split(line, fields, / +/) + if (count >= 20) start_ticks=fields[20] + } + FILENAME == proc_stat_file && $1 == "btime" { boot_time=$2 } + END { + if (start_ticks !~ /^[0-9]+$/ || boot_time !~ /^[0-9]+$/) exit 42 + printf "%s %s\n", start_ticks, boot_time + } +' "$stat_file" "$proc_stat_file") +set -- $initial_identity +expected_start_ticks=$1 +expected_boot_time_epoch_seconds=$2 + +sampler_pid= +cleanup_sampler() { + trap - EXIT + if [ -n "$sampler_pid" ]; then + kill "$sampler_pid" 2>/dev/null || true + wait "$sampler_pid" 2>/dev/null || true + sampler_pid= + fi +} +trap cleanup_sampler EXIT +trap 'exit 143' HUP INT TERM + +report_gone() { + IFS=' ' read -r uptime_seconds _ < /proc/uptime || uptime_seconds=0 + printf 'gone\t%s\n' "$uptime_seconds" +} + +sample_backend() { + if [ ! -r "$status_file" ] || [ ! -r "$stat_file" ]; then + report_gone + return 1 + fi + IFS=' ' read -r uptime_seconds _ < /proc/uptime + set +e + $awk_command \ + -v uptime_seconds="$uptime_seconds" \ + -v expected_pid="$expected_pid" \ + -v expected_start_ticks="$expected_start_ticks" \ + -v expected_boot_time="$expected_boot_time_epoch_seconds" \ + -v expected_backend_start_epoch_ms="$expected_backend_start_epoch_ms" \ + -v start_tolerance_ms="$start_tolerance_ms" \ + -v clock_ticks="$clock_ticks" \ + -v status_file="$status_file" \ + -v stat_file="$stat_file" \ + -v proc_stat_file="$proc_stat_file" ' + FILENAME == status_file && /^Name:/ { process_name=$2 } + FILENAME == status_file && /^NSpid:/ { namespace_pid=$NF } + FILENAME == status_file && /^VmRSS:/ { rss=$2 } + FILENAME == status_file && /^VmHWM:/ { hwm=$2 } + FILENAME == stat_file { + line=$0 + sub(/^.*\) /, "", line) + count=split(line, fields, / +/) + if (count >= 20) start_ticks=fields[20] + } + FILENAME == proc_stat_file && $1 == "btime" { boot_time=$2 } + END { + proc_start_epoch_ms=(boot_time + (start_ticks / clock_ticks)) * 1000 + start_delta_ms=proc_start_epoch_ms - expected_backend_start_epoch_ms + if (start_delta_ms < 0) start_delta_ms=-start_delta_ms + if ( + process_name !~ /^postgres/ + || namespace_pid != expected_pid + || rss !~ /^[0-9]+$/ + || hwm !~ /^[0-9]+$/ + || start_ticks !~ /^[0-9]+$/ + || start_ticks != expected_start_ticks + || boot_time != expected_boot_time + || clock_ticks !~ /^[0-9]+$/ + || start_delta_ms > start_tolerance_ms + ) exit 42 + printf "sample\t%s\t%s\t%s\t%s\t%.0f\t%s\t%s\n", \ + uptime_seconds, rss, hwm, start_ticks, proc_start_epoch_ms, \ + boot_time, clock_ticks + } + ' "$status_file" "$stat_file" "$proc_stat_file" + sample_status=$? + set -e + if [ "$sample_status" -ne 0 ]; then + if [ ! -r "$status_file" ] || [ ! -r "$stat_file" ]; then + report_gone + return 1 + fi + printf 'immutable PostgreSQL backend procfs identity validation failed\n' >&2 + exit 42 + fi +} + +sample_backend +( + while "$sleep_command" "$interval_seconds"; do + sample_backend || break + done +) & +sampler_pid=$! +IFS= read -r _ || true +kill "$sampler_pid" 2>/dev/null || true +set +e +wait "$sampler_pid" 2>/dev/null +sampler_status=$? +set -e +sampler_pid= +if [ "$sampler_status" -ne 0 ] && [ "$sampler_status" -ne 143 ]; then + exit "$sampler_status" +fi +set +e +sample_backend +final_sample_status=$? +set -e +if [ "$final_sample_status" -ne 0 ] && [ "$final_sample_status" -ne 1 ]; then + exit "$final_sample_status" +fi +`; + +export type CatalogScopedCatalogTypes = 'all' | 'dependency-closure'; +export type CatalogIntrospectionClientReleaseMode = 'reuse' | 'destroy'; +export type CatalogBackendSamplerMode = 'off' | 'diagnostic-lower-bound'; + +export interface CatalogBenchConfig { + version: 1; + database: string; + mode: IntrospectionMode; + scopedCatalogTypes: CatalogScopedCatalogTypes | null; + introspectionClientReleaseMode: CatalogIntrospectionClientReleaseMode; + postgresBackendSamplerMode: CatalogBackendSamplerMode; + releaseBuildStateAfterValidation: boolean; + schemas: string[]; + /** Ordered exposed schemas per instance; omitted for the legacy one-schema path. */ + schemaSets?: string[][]; + /** Explicit scoped-introspection dependency allowlist for schemaSets mode. */ + allowedDependencySchemas?: string[]; + checkpoints: number[]; + expectedTokens: string[] | null; + heapMiB: number; + repetition: number; + settleMs: number; + warmOperationsPerInstance: number; + warmOperationReplayPasses: number; + grafastCacheLimits: CatalogGrafastCacheLimits; + postgresContainer: string | null; + commit: string | null; + worktreeDirty: boolean | null; + sourceStateSha256: string | null; + lockfileSha256: string | null; + executedEntrySha256: string; + v8Profile: NodeV8Profile; + nodeOptions: string; + nodeOptionsArgv: string[]; + nodeExecArgv: string[]; + effectiveNodeRuntimeFlags: string[]; +} + +export interface CatalogGrafastCacheLimits { + queryCacheMaxLength: number | null; + operationsCacheMaxLength: number | null; + operationOperationPlansCacheMaxLength: number | null; +} + +export interface CatalogMemorySnapshot { + instances: number; + heapUsedBytes: number; + heapDeltaBytes: number; + rssBytes: number; + rssDeltaBytes: number; + externalBytes: number; + externalDeltaBytes: number; + processPeakRssBytes: number; + processPeakRssDeltaBytes: number; + postgresBackendRssBytes: number | null; + postgresBackendRssDeltaBytes: number | null; + postgresBackendHighWaterBytes: number | null; + postgresBackendHighWaterDeltaBytes: number | null; +} + +export interface CatalogBenchProgress { + version: 1; + status: 'in-progress' | 'complete'; + mode: IntrospectionMode; + scopedCatalogTypes: CatalogScopedCatalogTypes | null; + introspectionClientReleaseMode: CatalogIntrospectionClientReleaseMode; + postgresBackendSamplerMode: CatalogBackendSamplerMode; + releaseBuildStateAfterValidation: boolean; + repetition: number; + heapMiB: number; + v8Profile: NodeV8Profile; + nodeOptions: string; + nodeOptionsArgv: string[]; + nodeExecArgv: string[]; + effectiveNodeRuntimeFlags: string[]; + targetInstances: number; + completedInstances: number; + configuredCheckpoints: number[]; + completedCheckpoints: number[]; + buildsCompleted: number; + canariesCompleted: number; + mismatchViolations: number; + crossTenantViolations: number; + lastSnapshot: CatalogMemorySnapshot; + updatedAt: string; +} + +export interface CatalogTenantProxyDensityPoint { + residentSurfaceInstances: number; + fullTenantProxyGroups: number; + remainderSurfaceInstances: number; + configuredOldSpaceMiB: number; + absolutePeakProcessRssBytes: number; + groupsPerConfiguredOldSpaceGiB: number; + groupsPerAbsolutePeakProcessRssGiB: number; +} + +export interface CatalogBackendPidTransition { + introspectionBackendPid: number; + introspectionBackendStartEpochMs: number; + steadyBackendPid: number; + steadyBackendStartEpochMs: number; + introspectionBackendRetired: boolean; +} + +export interface CatalogBuildSample extends CatalogBackendPidTransition { + instance: number; + schema: string; + buildMs: number; + queryMs: number; + token: string | null; + sdlBytes: number; + sdlSha256: string; + queryFields: string[]; + warmOperations: number; + warmOperationLatencyP50Ms: number | null; + warmOperationLatencyP99Ms: number | null; + warmOperationErrors: number; + warmOperationReturnedStrings: number; + warmOperationExactMatches: number; + warmOperationMismatchViolations: number; + warmOperationCrossTenantViolations: number; + warmOperationCorrectnessConclusive: boolean; + warmOperationCorrectnessPassed: boolean; + warmOperationReplayPasses: number; + warmOperationReplayExecutions: number; + warmOperationReplayLatencyP50Ms: number | null; + warmOperationReplayLatencyP99Ms: number | null; + warmOperationReplayErrors: number; + warmOperationReplayReturnedStrings: number; + warmOperationReplayExactMatches: number; + warmOperationReplayMismatchViolations: number; + warmOperationReplayCrossTenantViolations: number; + warmOperationReplayCorrectnessConclusive: boolean; + warmOperationReplayCorrectnessPassed: boolean; + buildBaselineHeapUsedBytes: number; + buildBaselineRssBytes: number; + sampledBuildPeakHeapUsedBytes: number; + sampledBuildPeakHeapDeltaBytes: number; + sampledBuildPeakRssBytes: number; + sampledBuildPeakRssDeltaBytes: number; + processBuildPeakRssBytes: number; + processBuildPeakRssDeltaBytes: number; + buildTransientSampleCount: number; + postgresIntrospectionBackendMemoryLowerBound: + CatalogBackendIntrospectionMemoryLowerBoundMeasurement | null; +} + +export interface CatalogCanarySample { + phase: 'initial' | 'checkpoint'; + residentInstances: number; + instance: number; + schema: string; + expected: string; + actual: string | null; + returnedString: boolean; + exactMatch: boolean; + matchedOtherTenant: boolean; +} + +export interface CatalogBenchResult { + version: 1; + status: 'performance-only'; + database: string; + mode: IntrospectionMode; + scopedCatalogTypes: CatalogScopedCatalogTypes | null; + introspectionClientReleaseMode: CatalogIntrospectionClientReleaseMode; + postgresBackendSamplerMode: CatalogBackendSamplerMode; + releaseBuildStateAfterValidation: boolean; + /** Present only for the explicit multi-schema surface mode. */ + schemaSets?: string[][]; + /** Present only for the explicit multi-schema surface mode. */ + allowedDependencySchemas?: string[]; + repetition: number; + heapMiB: number; + commit: string | null; + worktreeDirty: boolean | null; + sourceStateSha256: string | null; + lockfileSha256: string | null; + executedEntrySha256: string; + v8Profile: NodeV8Profile; + nodeOptions: string; + nodeOptionsArgv: string[]; + nodeExecArgv: string[]; + effectiveNodeRuntimeFlags: string[]; + node: string; + v8: string; + effectiveV8HeapLimitBytes: number; + platform: string; + architecture: string; + startedAt: string; + endedAt: string; + catalog: { + classes: number; + attributes: number; + procs: number; + types: number; + namespaces: number; + }; + runtimeRole: { + name: string; + superuser: boolean; + bypassRls: boolean; + createRole: boolean; + ownsDatabase: boolean; + canCreateInDatabase: boolean; + ownsRequestedSchema: boolean; + canCreateInRequestedSchema: boolean; + }; + catalogWarmth: 'shared-server-not-reset'; + grafastCacheWarmth: { + operationsPerInstance: number; + cacheLimits: CatalogGrafastCacheLimits; + sourceMode: 'grafast-source'; + sourceSetSha256: string | null; + operationExecutions: number; + latencyP50Ms: number | null; + latencyP99Ms: number | null; + errors: number; + returnedStrings: number; + exactMatches: number; + mismatchViolations: number; + crossTenantViolations: number; + correctnessConclusive: boolean; + correctnessPassed: boolean; + replay: { + passesPerInstance: number; + operationExecutions: number; + latencyP50Ms: number | null; + latencyP99Ms: number | null; + errors: number; + returnedStrings: number; + exactMatches: number; + mismatchViolations: number; + crossTenantViolations: number; + correctnessConclusive: boolean; + correctnessPassed: boolean; + }; + }; + buildTransientSampling: { + approximate: true; + intervalMs: number; + limitation: string; + maxSampledHeapDeltaBytes: number; + maxSampledRssDeltaBytes: number; + maxProcessPeakRssDeltaBytes: number; + }; + postgresBackendMeasurement: { + initialBackendPid: number; + initialBackendStartEpochMs: number; + finalSteadyBackendPid: number; + finalSteadyBackendStartEpochMs: number; + expectedRetirementChecks: number; + completedRetirementChecks: number; + allExpectedRetirementsProven: boolean; + steadyBackendRss: { + measured: boolean; + samplePhase: 'shared-introspection-and-steady' | 'post-introspection-replacement'; + deltaBasis: 'initial-backend' | 'replacement-acquisition'; + }; + introspectionBackendMemory: { + sampledLowerBoundMeasured: boolean; + sharedSnapshotMeasured: boolean; + semantics: + | 'diagnostic-lower-bound-without-pre-destroy-acknowledgement' + | 'post-build-shared-backend-snapshot' + | 'unavailable'; + measurementMethod: + | 'dedicated-identity-bound-procfs-sampler' + | 'post-build-shared-backend-procfs' + | 'unavailable'; + expectedBuildMeasurements: number; + completedBuildMeasurements: number; + allBuildCadenceChecksConclusive: boolean; + backendSamplerAuthority: 'diagnostic-only'; + serviceDensityMemoryAuthority: + `separately-validated-${typeof LINUX_CGROUP_V2_DENSITY_AUTHORITY}`; + limitation: string | null; + }; + }; + fixtureFingerprint: string; + builds: CatalogBuildSample[]; + canaries: CatalogCanarySample[]; + snapshots: CatalogMemorySnapshot[]; + heapSlopeBytesPerInstance: number; + rssSlopeBytesPerInstance: number; + allSdlHashesEqualWithinArm: boolean; + tokenCanariesConclusive: boolean; + tokenCanariesPassed: boolean; + tokenMismatchViolations: number; + crossTenantTokenViolations: number; + bleedViolations: number; +} + +export interface CatalogWarmthCliOptions { + warmOperationsPerInstance: number; + warmOperationReplayPasses: number; + grafastCacheLimits: CatalogGrafastCacheLimits; +} + +export interface CatalogSchemaLayout { + /** Compatibility labels: one primary schema name per resident instance. */ + schemas: string[]; + /** Null preserves the legacy one-schema-per-instance configuration shape. */ + schemaSets: string[][] | null; + /** Null preserves legacy makePgService behavior. */ + allowedDependencySchemas: string[] | null; +} + +export interface CatalogWarmOperationResult { + latenciesMs: number[]; + errors: number; + returnedStrings: number; + exactMatches: number; + mismatchViolations: number; + crossTenantViolations: number; + correctnessConclusive: boolean; + correctnessPassed: boolean; +} + +export interface BuildTransientSample { + baselineHeapUsedBytes: number; + baselineRssBytes: number; + sampledPeakHeapUsedBytes: number; + sampledPeakHeapDeltaBytes: number; + sampledPeakRssBytes: number; + sampledPeakRssDeltaBytes: number; + processPeakRssBytes: number; + processPeakRssDeltaBytes: number; + sampleCount: number; +} + +export interface CatalogBackendMemoryPoint { + monotonicMs: number; + rssBytes: number; + highWaterBytes: number; + procStartTicks: number; + procStartEpochMs: number; + bootTimeEpochSeconds: number; + clockTicksPerSecond: number; +} + +interface BackendMemory { + rssBytes: number; + highWaterBytes: number; +} + +export type CatalogBackendMemorySamplerSource = + | 'linux-host-container-procfs' + | 'docker-container-procfs-diagnostic' + | 'local-linux-procfs'; + +export interface CatalogBackendIdentity { + pid: number; + backendStartEpochMs: number; +} + +export interface CatalogDockerContainerIdentity { + requestedName: string; + immutableId: string; + startedAt: string; + initHostPid: number; +} + +export interface CatalogBackendIntrospectionMemoryLowerBoundMeasurement { + backendPid: number; + backendStartEpochMs: number; + baselineRssBytes: number; + baselineHighWaterBytes: number; + sampledPeakRssLowerBoundBytes: number; + sampledHighWaterLowerBoundBytes: number; + sampledPeakRssDeltaLowerBoundBytes: number; + sampledHighWaterDeltaLowerBoundBytes: number; + sampleCount: number; + targetExitedBeforeStop: boolean; + targetExitedAtMonotonicMs: number | null; + timing: { + configuredIntervalMs: number; + maximumConclusiveGapMs: number; + firstSampleMonotonicMs: number; + lastSampleMonotonicMs: number; + maximumObservedGapMs: number | null; + samplerStartedAt: string; + samplerReadyAt: string; + buildStartedAt: string; + buildCompletedAt: string; + samplerStoppedAt: string; + buildDurationMs: number; + samplerDurationMs: number; + coveredBuildWindow: boolean; + cadenceConclusive: boolean; + samplerLaunchToReadyMs: number; + samplerStopRequestToCloseMs: number; + }; + observerEffect: { + samplerProcessCount: 1; + correctionApplied: false; + pairedComparisonSupported: true; + pairedComparisonFlag: '--postgres-backend-sampler'; + measuredLaunchToReadyMs: number; + measuredStopRequestToCloseMs: number; + limitation: string; + }; + provenance: { + samplerProcess: 'dedicated-external-procfs-loop'; + samplerPid: number; + source: CatalogBackendMemorySamplerSource; + postgresContainer: string | null; + containerIdentity: CatalogDockerContainerIdentity | null; + clientPlatform: string; + clientArchitecture: string; + backendIdentity: { + sqlBackendStartEpochMs: number; + procStartTicks: number; + procStartEpochMs: number; + bootTimeEpochSeconds: number; + clockTicksPerSecond: number; + toleranceMs: number; + }; + dockerInitialExecEnvironment: + | 'not-applicable' + | 'may-inherit-container-config-before-env-i'; + samplerShellEnvironment: 'env-i-path-only'; + hostEnvironmentVariableNames: string[]; + semantics: 'diagnostic-lower-bound-without-pre-destroy-acknowledgement'; + backendSamplerAuthority: 'diagnostic-only'; + serviceDensityMemoryAuthority: + `separately-validated-${typeof LINUX_CGROUP_V2_DENSITY_AUTHORITY}`; + limitation: string | null; + }; +} + +export interface CatalogBackendMemorySamplerHandle { + stop(input: { + buildStartedAt: string; + buildCompletedAt: string; + buildDurationMs: number; + }): Promise; +} + +export interface CatalogBackendPidLifecycleDependencies { + waitForRetirement(identity: CatalogBackendIdentity): Promise; + acquireBackendIdentity(): Promise; +} + +const flag = (args: string[], name: string): string | undefined => { + const index = args.indexOf(`--${name}`); + return index >= 0 ? args[index + 1] : undefined; +}; + +const requireFlag = (args: string[], name: string): string => { + const value = flag(args, name); + if (!value) throw new Error(`catalog-bench requires --${name}`); + return value; +}; + +const parsePositiveInteger = (value: string, label: string): number => { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${label} must be a positive integer`); + } + return parsed; +}; + +const strictOptionalFlag = (args: string[], name: string): string | undefined => { + const flagName = `--${name}`; + const indexes = args.flatMap((value, index) => value === flagName ? [index] : []); + if (indexes.length > 1) throw new Error(`${flagName} may only be specified once`); + if (indexes.length === 0) return undefined; + const value = args[indexes[0] + 1]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flagName} requires a value`); + } + return value; +}; + +export const validateCatalogV8Profile: ( + value: unknown +) => asserts value is NodeV8Profile = (value) => { + if ( + value !== 'stock' + && value !== 'optimize-for-size' + && value !== 'baseline-optimize-for-size' + && value !== 'jitless-optimize-for-size' + ) { + throw new Error( + "v8Profile must be 'stock', 'optimize-for-size', " + + "'baseline-optimize-for-size', or 'jitless-optimize-for-size'" + ); + } +}; + +export const parseCatalogV8Profile = (args: string[]): NodeV8Profile => { + const value = strictOptionalFlag(args, 'v8-profile') ?? 'stock'; + validateCatalogV8Profile(value); + return value; +}; + +interface CatalogRuntimeFlags { + nodeOptions: string; + nodeOptionsArgv: string[]; + nodeExecArgv: string[]; + effectiveNodeRuntimeFlags: string[]; +} + +const CATALOG_MANAGED_V8_OPTION = + /^--(?:no[-_])?(?:jitless|optimize[-_]for[-_]size|max[-_]opt)(?:=.*)?$/; +const CATALOG_MAX_OLD_SPACE_OPTION = + /^--max(?:-|_)old(?:-|_)space(?:-|_)size(?:=.*)?$/; + +export const validateCatalogRuntimeFlags = ( + config: Pick< + CatalogBenchConfig, + | 'heapMiB' + | 'v8Profile' + | 'nodeOptions' + | 'nodeOptionsArgv' + | 'nodeExecArgv' + | 'effectiveNodeRuntimeFlags' + >, + actual: CatalogRuntimeFlags = { + nodeOptions: process.env.NODE_OPTIONS ?? '', + nodeOptionsArgv: tokenizeNodeOptions(process.env.NODE_OPTIONS ?? ''), + nodeExecArgv: [...process.execArgv], + effectiveNodeRuntimeFlags: [ + ...tokenizeNodeOptions(process.env.NODE_OPTIONS ?? ''), + ...process.execArgv + ] + } +): void => { + validateCatalogV8Profile(config.v8Profile); + const expectedExecArgv = [ + ...nodeFlagsForV8Profile(config.v8Profile), + '--expose-gc' + ]; + const configuredNodeOptionsArgv = tokenizeNodeOptions(config.nodeOptions); + const expectedEffective = [ + ...config.nodeOptionsArgv, + ...config.nodeExecArgv + ]; + const maxOldSpace = config.nodeOptionsArgv.filter((option) => + CATALOG_MAX_OLD_SPACE_OPTION.test(option) + ); + const managedInNodeOptions = config.nodeOptionsArgv.some((option) => + CATALOG_MANAGED_V8_OPTION.test(option) + ); + if ( + JSON.stringify(config.nodeOptionsArgv) !== JSON.stringify(configuredNodeOptionsArgv) + || JSON.stringify(config.nodeExecArgv) !== JSON.stringify(expectedExecArgv) + || JSON.stringify(config.effectiveNodeRuntimeFlags) !== JSON.stringify(expectedEffective) + || maxOldSpace.length !== 1 + || maxOldSpace[0] !== `--max-old-space-size=${config.heapMiB}` + || managedInNodeOptions + ) { + throw new Error('catalog-bench configured Node runtime flags are inconsistent'); + } + for (const [label, configured, observed] of [ + ['NODE_OPTIONS', config.nodeOptions, actual.nodeOptions], + ['NODE_OPTIONS argv', config.nodeOptionsArgv, actual.nodeOptionsArgv], + ['process.execArgv', config.nodeExecArgv, actual.nodeExecArgv], + [ + 'effective Node runtime flags', + config.effectiveNodeRuntimeFlags, + actual.effectiveNodeRuntimeFlags + ] + ] as const) { + if (JSON.stringify(configured) !== JSON.stringify(observed)) { + throw new Error(`catalog-bench ${label} does not match the pinned worker config`); + } + } +}; + +const parseStrictInteger = ( + value: string, + label: string, + allowZero: boolean +): number => { + if (!/^(0|[1-9][0-9]*)$/.test(value)) { + throw new Error(`${label} must be ${allowZero ? 'a non-negative' : 'a positive'} integer`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) { + throw new Error(`${label} must be ${allowZero ? 'a non-negative' : 'a positive'} safe integer`); + } + return parsed; +}; + +export const validateCatalogScopedCatalogTypes = ( + mode: IntrospectionMode, + value: CatalogScopedCatalogTypes | null +): void => { + if (mode !== 'stock' && mode !== 'scoped-required') { + throw new Error("catalog-bench mode must be 'stock' or 'scoped-required'"); + } + if (mode === 'stock') { + if (value !== null) { + throw new Error('scopedCatalogTypes must be null for stock introspection'); + } + return; + } + if (value !== 'all' && value !== 'dependency-closure') { + throw new Error( + "scopedCatalogTypes must be 'all' or 'dependency-closure' for scoped-required introspection" + ); + } +}; + +export const parseCatalogScopedCatalogTypes = ( + args: string[], + mode: IntrospectionMode +): CatalogScopedCatalogTypes | null => { + const value = strictOptionalFlag(args, 'scoped-catalog-types'); + if (mode === 'stock') { + if (value !== undefined) { + throw new Error('--scoped-catalog-types requires --mode scoped-required'); + } + return null; + } + const parsed = value ?? 'all'; + if (parsed !== 'all' && parsed !== 'dependency-closure') { + throw new Error( + "--scoped-catalog-types must be 'all' or 'dependency-closure'" + ); + } + return parsed; +}; + +export const validateCatalogIntrospectionClientReleaseMode: ( + value: unknown +) => asserts value is CatalogIntrospectionClientReleaseMode = (value) => { + if (value !== 'reuse' && value !== 'destroy') { + throw new Error( + "introspectionClientReleaseMode must be 'reuse' or 'destroy'" + ); + } +}; + +export const parseCatalogIntrospectionClientReleaseMode = ( + args: string[] +): CatalogIntrospectionClientReleaseMode => { + const value = strictOptionalFlag(args, 'introspection-client-release-mode') + ?? 'reuse'; + validateCatalogIntrospectionClientReleaseMode(value); + return value; +}; + +export const validateCatalogBackendSamplerMode: ( + value: unknown +) => asserts value is CatalogBackendSamplerMode = (value) => { + if (value !== 'off' && value !== 'diagnostic-lower-bound') { + throw new Error( + "postgresBackendSamplerMode must be 'off' or 'diagnostic-lower-bound'" + ); + } +}; + +export const parseCatalogBackendSamplerMode = ( + args: string[] +): CatalogBackendSamplerMode => { + const value = strictOptionalFlag(args, 'postgres-backend-sampler') + ?? 'diagnostic-lower-bound'; + validateCatalogBackendSamplerMode(value); + return value; +}; + +export const catalogIntrospectionBuildIdentity = ( + mode: IntrospectionMode, + scopedCatalogTypes: CatalogScopedCatalogTypes | null, + releaseBuildStateAfterValidation = false, + introspectionClientReleaseMode: CatalogIntrospectionClientReleaseMode = 'reuse' +): string => { + validateCatalogScopedCatalogTypes(mode, scopedCatalogTypes); + validateCatalogIntrospectionClientReleaseMode(introspectionClientReleaseMode); + return `${mode}:scoped-catalog-types=${scopedCatalogTypes ?? 'not-applicable'}` + + `:release-build-state=${releaseBuildStateAfterValidation}` + + `:introspection-client-release=${introspectionClientReleaseMode}`; +}; + +export const resolveCatalogBackendPidAfterBuild = async ( + introspectionClientReleaseMode: CatalogIntrospectionClientReleaseMode, + introspectionBackendIdentity: CatalogBackendIdentity, + dependencies: CatalogBackendPidLifecycleDependencies +): Promise => { + validateCatalogIntrospectionClientReleaseMode(introspectionClientReleaseMode); + validateCatalogBackendIdentity(introspectionBackendIdentity); + if (introspectionClientReleaseMode === 'destroy') { + await dependencies.waitForRetirement(introspectionBackendIdentity); + } + const steadyBackendIdentity = await dependencies.acquireBackendIdentity(); + validateCatalogBackendIdentity(steadyBackendIdentity); + if ( + introspectionClientReleaseMode === 'destroy' + && steadyBackendIdentity.pid === introspectionBackendIdentity.pid + ) { + throw new Error( + `destroyed PostgreSQL introspection backend ${introspectionBackendIdentity.pid} was reused` + ); + } + if ( + introspectionClientReleaseMode === 'reuse' + && ( + steadyBackendIdentity.pid !== introspectionBackendIdentity.pid + || steadyBackendIdentity.backendStartEpochMs + !== introspectionBackendIdentity.backendStartEpochMs + ) + ) { + throw new Error( + `PostgreSQL benchmark backend identity changed from ` + + `${introspectionBackendIdentity.pid}@${introspectionBackendIdentity.backendStartEpochMs} ` + + `to ${steadyBackendIdentity.pid}@${steadyBackendIdentity.backendStartEpochMs}` + ); + } + return { + introspectionBackendPid: introspectionBackendIdentity.pid, + introspectionBackendStartEpochMs: + introspectionBackendIdentity.backendStartEpochMs, + steadyBackendPid: steadyBackendIdentity.pid, + steadyBackendStartEpochMs: steadyBackendIdentity.backendStartEpochMs, + introspectionBackendRetired: introspectionClientReleaseMode === 'destroy' + }; +}; + +export const parseCatalogBuildStateRetirement = (args: string[]): boolean => { + const flagName = '--release-build-state-after-validation'; + const count = args.filter((value) => value === flagName).length; + if (count > 1) throw new Error(`${flagName} may only be specified once`); + return count === 1; +}; + +export const parseCatalogTenantProxySurfaces = (args: string[]): number | null => { + const value = strictOptionalFlag(args, 'tenant-proxy-surfaces'); + return value === undefined + ? null + : parseStrictInteger(value, 'tenant-proxy-surfaces', false); +}; + +export const parseCatalogWarmthCliOptions = ( + args: string[] +): CatalogWarmthCliOptions => { + const warmOperations = strictOptionalFlag(args, 'warm-operations-per-instance'); + const replayPasses = strictOptionalFlag(args, 'warm-operation-replay-passes'); + const queryCacheMax = strictOptionalFlag(args, 'grafast-query-cache-max'); + const operationsCacheMax = strictOptionalFlag(args, 'grafast-operations-cache-max'); + const operationPlansCacheMax = strictOptionalFlag( + args, + 'grafast-operation-plans-cache-max' + ); + const options: CatalogWarmthCliOptions = { + warmOperationsPerInstance: warmOperations === undefined + ? 0 + : parseStrictInteger( + warmOperations, + 'warm-operations-per-instance', + true + ), + warmOperationReplayPasses: replayPasses === undefined + ? 0 + : parseStrictInteger( + replayPasses, + 'warm-operation-replay-passes', + true + ), + grafastCacheLimits: { + queryCacheMaxLength: queryCacheMax === undefined + ? null + : parseStrictInteger(queryCacheMax, 'grafast-query-cache-max', false), + operationsCacheMaxLength: operationsCacheMax === undefined + ? null + : parseStrictInteger(operationsCacheMax, 'grafast-operations-cache-max', false), + operationOperationPlansCacheMaxLength: operationPlansCacheMax === undefined + ? null + : parseStrictInteger( + operationPlansCacheMax, + 'grafast-operation-plans-cache-max', + false + ) + } + }; + validateCatalogWarmthConfig(options); + return options; +}; + +export const validateCatalogWarmthConfig = ( + options: CatalogWarmthCliOptions +): void => { + if ( + !Number.isSafeInteger(options.warmOperationsPerInstance) + || options.warmOperationsPerInstance < 0 + ) { + throw new Error('warmOperationsPerInstance must be a non-negative safe integer'); + } + if ( + !Number.isSafeInteger(options.warmOperationReplayPasses) + || options.warmOperationReplayPasses < 0 + ) { + throw new Error('warmOperationReplayPasses must be a non-negative safe integer'); + } + if ( + options.warmOperationReplayPasses > 0 + && options.warmOperationsPerInstance === 0 + ) { + throw new Error( + 'warmOperationReplayPasses requires warmOperationsPerInstance to be greater than zero' + ); + } + if (!Number.isSafeInteger( + options.warmOperationsPerInstance * options.warmOperationReplayPasses + )) { + throw new Error('warm operation replay execution count must be a safe integer'); + } + if (!options.grafastCacheLimits || typeof options.grafastCacheLimits !== 'object') { + throw new Error('grafastCacheLimits must define all three cache limit fields'); + } + for (const [key, value] of Object.entries(options.grafastCacheLimits)) { + if (value !== null && (!Number.isSafeInteger(value) || value < 2)) { + throw new Error(`grafastCacheLimits.${key} must be null or a safe integer of at least 2`); + } + } + const requiredKeys: Array = [ + 'queryCacheMaxLength', + 'operationsCacheMaxLength', + 'operationOperationPlansCacheMaxLength' + ]; + if (requiredKeys.some((key) => !(key in options.grafastCacheLimits))) { + throw new Error('grafastCacheLimits must define all three cache limit fields'); + } +}; + +const configuredGrafastCacheLimits = ( + limits: CatalogGrafastCacheLimits +): { + queryCacheMaxLength?: number; + operationsCacheMaxLength?: number; + operationOperationPlansCacheMaxLength?: number; +} => ({ + ...(limits.queryCacheMaxLength === null + ? {} + : { queryCacheMaxLength: limits.queryCacheMaxLength }), + ...(limits.operationsCacheMaxLength === null + ? {} + : { operationsCacheMaxLength: limits.operationsCacheMaxLength }), + ...(limits.operationOperationPlansCacheMaxLength === null + ? {} + : { + operationOperationPlansCacheMaxLength: + limits.operationOperationPlansCacheMaxLength + }) +}); + +const parseList = (value: string): string[] => value + .split(',') + .map((item) => item.trim()) + .filter(Boolean); + +const validateSchemaNames = ( + value: unknown, + label: string, + allowEmpty = false +): string[] => { + if (!Array.isArray(value) || (!allowEmpty && value.length === 0)) { + throw new Error(`${label} must contain at least one schema name`); + } + if (value.length === 0) return []; + const names = value.map((name, index) => { + if ( + typeof name !== 'string' + || name.length === 0 + || name.trim() !== name + || name.includes('\0') + ) { + throw new Error(`${label}[${index}] must be a nonempty exact schema name`); + } + return name; + }); + if (new Set(names).size !== names.length) { + throw new Error(`${label} must contain unique schema names`); + } + return names; +}; + +const parseStrictSchemaList = ( + value: string, + label: string, + allowEmpty = false +): string[] => { + if (allowEmpty && value.length === 0) return []; + const raw = value.split(','); + if (raw.some((name) => name.trim().length === 0)) { + throw new Error(`${label} must not contain empty schema names`); + } + return validateSchemaNames(raw.map((name) => name.trim()), label); +}; + +export const parseCatalogSchemaLayout = ( + args: string[], + maxInstances: number +): CatalogSchemaLayout => { + const legacyValue = strictOptionalFlag(args, 'schemas'); + const surfaceValue = strictOptionalFlag(args, 'surface-schemas'); + const dependencyValue = strictOptionalFlag(args, 'allowed-dependency-schemas'); + if (legacyValue !== undefined && surfaceValue !== undefined) { + throw new Error('--schemas and --surface-schemas are mutually exclusive'); + } + if (surfaceValue === undefined) { + if (dependencyValue !== undefined) { + throw new Error('--allowed-dependency-schemas requires --surface-schemas'); + } + if (legacyValue === undefined) { + throw new Error('catalog-bench requires --schemas or --surface-schemas'); + } + const schemas = parseList(legacyValue); + if (schemas.length !== maxInstances || new Set(schemas).size !== schemas.length) { + throw new Error(`--schemas must contain exactly ${maxInstances} unique entries`); + } + return { schemas, schemaSets: null, allowedDependencySchemas: null }; + } + if (maxInstances !== 1) { + throw new Error('--surface-schemas requires exactly one resident instance'); + } + if (dependencyValue === undefined) { + throw new Error('--surface-schemas requires --allowed-dependency-schemas'); + } + const surfaceSchemas = parseStrictSchemaList(surfaceValue, '--surface-schemas'); + const allowedDependencySchemas = parseStrictSchemaList( + dependencyValue, + '--allowed-dependency-schemas', + true + ); + const overlap = surfaceSchemas.filter((schema) => + allowedDependencySchemas.includes(schema) + ); + if (overlap.length > 0) { + throw new Error( + `surface and dependency schema lists must not overlap: ${overlap.join(', ')}` + ); + } + return { + schemas: [surfaceSchemas[0]], + schemaSets: [surfaceSchemas], + allowedDependencySchemas + }; +}; + +export const resolveCatalogSchemaLayout = ( + config: Pick< + CatalogBenchConfig, + 'schemas' | 'schemaSets' | 'allowedDependencySchemas' | 'checkpoints' + > +): CatalogSchemaLayout => { + const maxInstances = Math.max(...config.checkpoints); + if (!Number.isSafeInteger(maxInstances) || maxInstances <= 0) { + throw new Error('checkpoints must contain a positive resident instance count'); + } + const schemas = validateSchemaNames(config.schemas, 'schemas'); + if (config.schemaSets === undefined) { + if (config.allowedDependencySchemas !== undefined) { + throw new Error('allowedDependencySchemas requires schemaSets'); + } + if (schemas.length !== maxInstances) { + throw new Error(`schemas must contain exactly ${maxInstances} entries`); + } + return { schemas, schemaSets: null, allowedDependencySchemas: null }; + } + if ( + !Array.isArray(config.schemaSets) + || maxInstances !== 1 + || config.schemaSets.length !== 1 + || schemas.length !== 1 + ) { + throw new Error('schemaSets mode requires exactly one resident instance'); + } + const schemaSet = validateSchemaNames(config.schemaSets[0], 'schemaSets[0]'); + if (schemas[0] !== schemaSet[0]) { + throw new Error('schemas[0] must equal the first ordered schemaSets[0] entry'); + } + const allowedDependencySchemas = validateSchemaNames( + config.allowedDependencySchemas, + 'allowedDependencySchemas', + true + ); + const overlap = schemaSet.filter((schema) => allowedDependencySchemas.includes(schema)); + if (overlap.length > 0) { + throw new Error( + `schemaSets and allowedDependencySchemas must not overlap: ${overlap.join(', ')}` + ); + } + return { + schemas, + schemaSets: [schemaSet], + allowedDependencySchemas + }; +}; + +export const catalogSchemaContractIdentity = ( + schemas: string[], + allowedDependencySchemas: string[] +): string => { + const exposed = validateSchemaNames(schemas, 'schemas'); + const dependencies = validateSchemaNames( + allowedDependencySchemas, + 'allowedDependencySchemas', + true + ); + const overlap = exposed.filter((schema) => dependencies.includes(schema)); + if (overlap.length > 0) { + throw new Error(`schema contract lists must not overlap: ${overlap.join(', ')}`); + } + return createHash('sha256').update(JSON.stringify({ + schemas: exposed, + allowedDependencySchemas: dependencies + })).digest('hex'); +}; + +const parseCheckpoints = (value: string): number[] => { + const parsed = parseList(value).map((item) => parsePositiveInteger(item, 'instances')); + return [...new Set(parsed)].sort((a, b) => a - b); +}; + +const median = (values: number[]): number => { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; +}; + +export const catalogPercentile = ( + values: readonly number[], + probability: number +): number | null => { + if (values.length === 0) return null; + if (!Number.isFinite(probability) || probability <= 0 || probability > 1) { + throw new Error('percentile probability must be greater than zero and at most one'); + } + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * probability) - 1)]; +}; + +export const makeCatalogWarmOperationSource = (operationIndex: number): string => { + if (!Number.isSafeInteger(operationIndex) || operationIndex <= 0) { + throw new Error('operationIndex must be a positive safe integer'); + } + return `query CatalogWarm${operationIndex} { warmTenantToken: tenantToken }`; +}; + +export const projectCatalogTenantDensity = (input: { + tenantProxySurfaces: number; + configuredOldSpaceMiB: number; + snapshot: Pick< + CatalogMemorySnapshot, + 'instances' | 'processPeakRssBytes' | 'processPeakRssDeltaBytes' + >; +}): CatalogTenantProxyDensityPoint => { + const { + tenantProxySurfaces, + configuredOldSpaceMiB, + snapshot + } = input; + if (!Number.isSafeInteger(tenantProxySurfaces) || tenantProxySurfaces <= 0) { + throw new Error('tenantProxySurfaces must be a positive safe integer'); + } + if (!Number.isSafeInteger(configuredOldSpaceMiB) || configuredOldSpaceMiB <= 0) { + throw new Error('configuredOldSpaceMiB must be a positive safe integer'); + } + if (!Number.isSafeInteger(snapshot.instances) || snapshot.instances < 0) { + throw new Error('snapshot.instances must be a non-negative safe integer'); + } + if (!Number.isFinite(snapshot.processPeakRssBytes) || snapshot.processPeakRssBytes <= 0) { + throw new Error('snapshot.processPeakRssBytes must be a positive finite number'); + } + const fullTenantProxyGroups = Math.floor(snapshot.instances / tenantProxySurfaces); + return { + residentSurfaceInstances: snapshot.instances, + fullTenantProxyGroups, + remainderSurfaceInstances: snapshot.instances % tenantProxySurfaces, + configuredOldSpaceMiB, + absolutePeakProcessRssBytes: snapshot.processPeakRssBytes, + groupsPerConfiguredOldSpaceGiB: + fullTenantProxyGroups / (configuredOldSpaceMiB / MIB_PER_GIB), + groupsPerAbsolutePeakProcessRssGiB: + fullTenantProxyGroups / (snapshot.processPeakRssBytes / GIB) + }; +}; + +export const catalogProgressPath = (resultFile: string): string => + path.join(path.dirname(resultFile), 'progress.json'); + +/** + * Persist a small checkpoint without serializing resident schemas or build + * samples. The same-directory rename is atomic, so an OOM can leave either the + * preceding valid checkpoint or the new one, never a truncated JSON artifact. + */ +export const writeCatalogProgress = ( + resultFile: string, + progress: CatalogBenchProgress +): void => { + const progressFile = catalogProgressPath(resultFile); + const temporaryFile = `${progressFile}.${process.pid}.tmp`; + fs.mkdirSync(path.dirname(progressFile), { recursive: true }); + fs.writeFileSync(temporaryFile, `${JSON.stringify(progress, null, 2)}\n`, 'utf8'); + fs.renameSync(temporaryFile, progressFile); +}; + +const maxOrNull = (values: Array): number | null => { + const measured = values.filter((value): value is number => value !== null); + return measured.length > 0 ? Math.max(...measured) : null; +}; + +const medianOrNull = (values: Array): number | null => { + const measured = values.filter((value): value is number => value !== null); + return measured.length > 0 ? median(measured) : null; +}; + +const sha256File = (file: string): string => + createHash('sha256').update(fs.readFileSync(file)).digest('hex'); + +const GIT_PROVENANCE_MAX_BUFFER_BYTES = 64 * 1024 ** 2; + +const readGitProvenance = (): { + commit: string | null; + worktreeDirty: boolean | null; + sourceStateSha256: string | null; +} => { + try { + const commit = execFileSync('git', ['rev-parse', 'HEAD'], { + encoding: 'utf8', + maxBuffer: GIT_PROVENANCE_MAX_BUFFER_BYTES + }).trim(); + const status = execFileSync( + 'git', + ['status', '--porcelain=v1', '--untracked-files=all'], + { + encoding: 'utf8', + maxBuffer: GIT_PROVENANCE_MAX_BUFFER_BYTES + } + ); + const hash = createHash('sha256').update(commit).update('\0').update(status); + hash.update(execFileSync('git', ['diff', '--binary', 'HEAD'], { + encoding: 'buffer', + maxBuffer: GIT_PROVENANCE_MAX_BUFFER_BYTES + })); + const untracked = execFileSync( + 'git', + ['ls-files', '--others', '--exclude-standard', '-z'], + { + encoding: 'buffer', + maxBuffer: GIT_PROVENANCE_MAX_BUFFER_BYTES + } + ).toString('utf8').split('\0').filter(Boolean).sort(); + for (const relativeFile of untracked) { + hash.update('\0').update(relativeFile).update('\0'); + hash.update(fs.readFileSync(path.resolve(relativeFile))); + } + return { + commit, + worktreeDirty: status.length > 0, + sourceStateSha256: hash.digest('hex') + }; + } catch { + return { commit: null, worktreeDirty: null, sourceStateSha256: null }; + } +}; + +const slope = (points: { x: number; y: number }[]): number => { + if (points.length < 2) return 0; + const meanX = points.reduce((sum, point) => sum + point.x, 0) / points.length; + const meanY = points.reduce((sum, point) => sum + point.y, 0) / points.length; + const numerator = points.reduce( + (sum, point) => sum + (point.x - meanX) * (point.y - meanY), + 0 + ); + const denominator = points.reduce( + (sum, point) => sum + (point.x - meanX) ** 2, + 0 + ); + return denominator === 0 ? 0 : numerator / denominator; +}; + +const forceGc = async (settleMs: number): Promise => { + if (typeof global.gc !== 'function') { + throw new Error('catalog-bench worker requires Node --expose-gc'); + } + for (let index = 0; index < 3; index++) { + global.gc(); + await new Promise((resolve) => setImmediate(resolve)); + } + if (settleMs > 0) { + await new Promise((resolve) => setTimeout(resolve, settleMs)); + global.gc(); + } +}; + +interface TransientMemoryPoint { + heapUsedBytes: number; + rssBytes: number; + processPeakRssBytes: number; +} + +export const summarizeBuildTransientSamples = ( + baseline: TransientMemoryPoint, + samples: readonly TransientMemoryPoint[] +): BuildTransientSample => { + if (samples.length === 0) throw new Error('build transient sampling requires a sample'); + const sampledPeakHeapUsedBytes = Math.max(...samples.map((sample) => sample.heapUsedBytes)); + const sampledPeakRssBytes = Math.max(...samples.map((sample) => sample.rssBytes)); + const processPeakRssBytes = Math.max(...samples.map((sample) => sample.processPeakRssBytes)); + return { + baselineHeapUsedBytes: baseline.heapUsedBytes, + baselineRssBytes: baseline.rssBytes, + sampledPeakHeapUsedBytes, + sampledPeakHeapDeltaBytes: Math.max( + 0, + sampledPeakHeapUsedBytes - baseline.heapUsedBytes + ), + sampledPeakRssBytes, + sampledPeakRssDeltaBytes: Math.max(0, sampledPeakRssBytes - baseline.rssBytes), + processPeakRssBytes, + processPeakRssDeltaBytes: Math.max( + 0, + processPeakRssBytes - baseline.processPeakRssBytes + ), + sampleCount: samples.length + }; +}; + +const readTransientMemoryPoint = (): TransientMemoryPoint => { + const memory = process.memoryUsage(); + return { + heapUsedBytes: memory.heapUsed, + rssBytes: memory.rss, + processPeakRssBytes: process.resourceUsage().maxRSS * 1024 + }; +}; + +const measureBuildTransient = async ( + operation: () => Promise +): Promise<{ value: T; transient: BuildTransientSample }> => { + const baseline = readTransientMemoryPoint(); + const samples: TransientMemoryPoint[] = [baseline]; + const sample = () => samples.push(readTransientMemoryPoint()); + const timer = setInterval(sample, BUILD_TRANSIENT_SAMPLE_INTERVAL_MS); + timer.unref(); + try { + const value = await operation(); + sample(); + return { + value, + transient: summarizeBuildTransientSamples(baseline, samples) + }; + } finally { + clearInterval(timer); + } +}; + +const validateCatalogBackendPid = (backendPid: number): void => { + if (!Number.isSafeInteger(backendPid) || backendPid <= 0) { + throw new Error('PostgreSQL backend PID must be a positive safe integer'); + } +}; + +export const validateCatalogBackendIdentity = ( + identity: CatalogBackendIdentity +): void => { + validateCatalogBackendPid(identity.pid); + if ( + !Number.isSafeInteger(identity.backendStartEpochMs) + || identity.backendStartEpochMs <= 0 + ) { + throw new Error( + 'PostgreSQL backend start timestamp must be a positive safe epoch millisecond' + ); + } +}; + +export const validateCatalogPostgresContainer = (container: string): void => { + if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(container)) { + throw new Error(`invalid PostgreSQL container name '${container}'`); + } +}; + +export const parseCatalogDockerContainerIdentity = ( + output: string, + requestedName: string +): CatalogDockerContainerIdentity => { + validateCatalogPostgresContainer(requestedName); + const fields = output.trim().split('\t'); + if (fields.length !== 3 || !/^[a-f0-9]{64}$/i.test(fields[0])) { + throw new Error('Docker inspect did not return an immutable container ID'); + } + if (!Number.isFinite(Date.parse(fields[1]))) { + throw new Error('Docker inspect did not return a valid container start timestamp'); + } + const initHostPid = Number(fields[2]); + if (!Number.isSafeInteger(initHostPid) || initHostPid <= 0) { + throw new Error('Docker inspect did not return a positive container init PID'); + } + return { + requestedName, + immutableId: fields[0].toLowerCase(), + startedAt: new Date(fields[1]).toISOString(), + initHostPid + }; +}; + +export const assertCatalogDockerContainerIdentity = ( + expected: CatalogDockerContainerIdentity, + actual: CatalogDockerContainerIdentity +): void => { + if ( + actual.requestedName !== expected.requestedName + || actual.immutableId !== expected.immutableId + || actual.startedAt !== expected.startedAt + || actual.initHostPid !== expected.initHostPid + ) { + throw new Error( + `PostgreSQL container '${expected.requestedName}' changed immutable identity ` + + 'during backend sampling' + ); + } +}; + +const parseProcStatusKiB = (value: string | undefined, label: string): number => { + if (value === undefined || !/^\d+$/.test(value)) { + throw new Error(`PostgreSQL backend status did not contain a valid ${label}`); + } + const bytes = Number(value) * 1024; + if (!Number.isSafeInteger(bytes)) { + throw new Error(`PostgreSQL backend ${label} exceeds the safe integer range`); + } + return bytes; +}; + +export const parseCatalogBackendProcStatus = ( + status: string, + expectedPid: number +): BackendMemory => { + validateCatalogBackendPid(expectedPid); + const fields = new Map(); + for (const line of status.split(/\r?\n/)) { + const match = /^([A-Za-z]+):\s*(.*?)\s*$/.exec(line); + if (match) fields.set(match[1], match[2]); + } + const processName = fields.get('Name')?.split(/\s+/)[0]; + const namespacePids = fields.get('NSpid')?.split(/\s+/).filter(Boolean) ?? []; + const namespacePid = Number(namespacePids.at(-1)); + if (!processName?.startsWith('postgres') || namespacePid !== expectedPid) { + throw new Error( + `PostgreSQL backend status identity did not match exact PID ${expectedPid}` + ); + } + return { + rssBytes: parseProcStatusKiB(fields.get('VmRSS')?.split(/\s+/)[0], 'VmRSS'), + highWaterBytes: parseProcStatusKiB(fields.get('VmHWM')?.split(/\s+/)[0], 'VmHWM') + }; +}; + +const maxCatalogBackendSampleGapMs = ( + samples: readonly CatalogBackendMemoryPoint[] +): number | null => { + if (samples.length < 2) return null; + let maximum = 0; + for (let index = 1; index < samples.length; index++) { + const gap = samples[index].monotonicMs - samples[index - 1].monotonicMs; + if (!Number.isFinite(gap) || gap < 0) { + throw new Error('PostgreSQL backend sampler timestamps must be monotonic'); + } + maximum = Math.max(maximum, gap); + } + return maximum; +}; + +export const summarizeCatalogBackendMemorySamples = (input: { + backendIdentity: CatalogBackendIdentity; + samplerPid: number; + source: CatalogBackendMemorySamplerSource; + postgresContainer: string | null; + containerIdentity?: CatalogDockerContainerIdentity | null; + hostEnvironmentVariableNames?: string[]; + samples: readonly CatalogBackendMemoryPoint[]; + targetExitedBeforeStop: boolean; + targetExitedAtMonotonicMs?: number | null; + samplerStartedAt: string; + samplerReadyAt: string; + buildStartedAt: string; + buildCompletedAt: string; + samplerStopRequestedAt: string; + samplerStoppedAt: string; + buildDurationMs: number; + clientPlatform?: string; + clientArchitecture?: string; +}): CatalogBackendIntrospectionMemoryLowerBoundMeasurement => { + validateCatalogBackendIdentity(input.backendIdentity); + if (!Number.isSafeInteger(input.samplerPid) || input.samplerPid <= 0) { + throw new Error('PostgreSQL backend sampler PID must be a positive safe integer'); + } + if (input.samples.length === 0) { + throw new Error('PostgreSQL backend sampler produced no memory samples'); + } + for (const sample of input.samples) { + if ( + !Number.isFinite(sample.monotonicMs) + || sample.monotonicMs < 0 + || !Number.isSafeInteger(sample.rssBytes) + || sample.rssBytes <= 0 + || !Number.isSafeInteger(sample.highWaterBytes) + || sample.highWaterBytes < sample.rssBytes + || !Number.isSafeInteger(sample.procStartTicks) + || sample.procStartTicks <= 0 + || !Number.isSafeInteger(sample.procStartEpochMs) + || sample.procStartEpochMs <= 0 + || !Number.isSafeInteger(sample.bootTimeEpochSeconds) + || sample.bootTimeEpochSeconds <= 0 + || !Number.isSafeInteger(sample.clockTicksPerSecond) + || sample.clockTicksPerSecond <= 0 + ) { + throw new Error('PostgreSQL backend sampler produced an invalid memory sample'); + } + } + if (!Number.isFinite(input.buildDurationMs) || input.buildDurationMs < 0) { + throw new Error('PostgreSQL backend sampler requires a finite build duration'); + } + const baseline = input.samples[0]; + for (const sample of input.samples) { + if ( + sample.procStartTicks !== baseline.procStartTicks + || sample.bootTimeEpochSeconds !== baseline.bootTimeEpochSeconds + || sample.clockTicksPerSecond !== baseline.clockTicksPerSecond + || Math.abs( + sample.procStartEpochMs - input.backendIdentity.backendStartEpochMs + ) > BACKEND_START_IDENTITY_TOLERANCE_MS + ) { + throw new Error( + 'PostgreSQL backend sampler observed a changed or mismatched process start identity' + ); + } + } + const timestamps = { + samplerStarted: Date.parse(input.samplerStartedAt), + samplerReady: Date.parse(input.samplerReadyAt), + buildStarted: Date.parse(input.buildStartedAt), + buildCompleted: Date.parse(input.buildCompletedAt), + samplerStopRequested: Date.parse(input.samplerStopRequestedAt), + samplerStopped: Date.parse(input.samplerStoppedAt) + }; + if (Object.values(timestamps).some((value) => !Number.isFinite(value))) { + throw new Error('PostgreSQL backend sampler timing contains an invalid timestamp'); + } + const coveredBuildWindow = + timestamps.samplerStarted <= timestamps.samplerReady + && timestamps.samplerReady <= timestamps.buildStarted + && timestamps.buildStarted <= timestamps.buildCompleted + && timestamps.buildCompleted <= timestamps.samplerStopRequested + && timestamps.samplerStopRequested <= timestamps.samplerStopped + && timestamps.buildCompleted <= timestamps.samplerStopped; + if (input.targetExitedBeforeStop !== (input.targetExitedAtMonotonicMs != null)) { + throw new Error('PostgreSQL backend sampler target-exit provenance is inconsistent'); + } + if ( + input.targetExitedAtMonotonicMs !== undefined + && input.targetExitedAtMonotonicMs !== null + && ( + !Number.isFinite(input.targetExitedAtMonotonicMs) + || input.targetExitedAtMonotonicMs < input.samples.at(-1)!.monotonicMs + ) + ) { + throw new Error('PostgreSQL backend sampler target-exit time is invalid'); + } + const observedTimingPoints = input.targetExitedAtMonotonicMs === undefined + || input.targetExitedAtMonotonicMs === null + ? input.samples + : [ + ...input.samples, + { + ...input.samples.at(-1)!, + monotonicMs: input.targetExitedAtMonotonicMs + } + ]; + const maximumObservedGapMs = maxCatalogBackendSampleGapMs(observedTimingPoints); + const cadenceConclusive = input.samples.length >= 2 + && coveredBuildWindow + && maximumObservedGapMs !== null + && maximumObservedGapMs <= BACKEND_MEMORY_MAX_CONCLUSIVE_GAP_MS; + const limitations: string[] = []; + limitations.push(DESTROYED_BACKEND_LOWER_BOUND_LIMITATION); + if (!cadenceConclusive) { + limitations.push( + 'The identity-bound sampler did not cover the build with at least two samples ' + + `and a maximum observed gap of ${BACKEND_MEMORY_MAX_CONCLUSIVE_GAP_MS}ms.` + ); + } + if (input.source === 'docker-container-procfs-diagnostic') { + limitations.push(DOCKER_DESKTOP_BACKEND_SAMPLER_LIMITATION); + } + const sampledPeakRssLowerBoundBytes = Math.max( + ...input.samples.map((sample) => sample.rssBytes) + ); + const sampledHighWaterLowerBoundBytes = Math.max( + ...input.samples.map((sample) => sample.highWaterBytes) + ); + const samplerLaunchToReadyMs = Math.max( + 0, + timestamps.samplerReady - timestamps.samplerStarted + ); + const samplerStopRequestToCloseMs = Math.max( + 0, + timestamps.samplerStopped - timestamps.samplerStopRequested + ); + return { + backendPid: input.backendIdentity.pid, + backendStartEpochMs: input.backendIdentity.backendStartEpochMs, + baselineRssBytes: baseline.rssBytes, + baselineHighWaterBytes: baseline.highWaterBytes, + sampledPeakRssLowerBoundBytes, + sampledHighWaterLowerBoundBytes, + sampledPeakRssDeltaLowerBoundBytes: Math.max( + 0, + sampledPeakRssLowerBoundBytes - baseline.rssBytes + ), + sampledHighWaterDeltaLowerBoundBytes: Math.max( + 0, + sampledHighWaterLowerBoundBytes - baseline.highWaterBytes + ), + sampleCount: input.samples.length, + targetExitedBeforeStop: input.targetExitedBeforeStop, + targetExitedAtMonotonicMs: input.targetExitedAtMonotonicMs ?? null, + timing: { + configuredIntervalMs: BACKEND_MEMORY_SAMPLE_INTERVAL_MS, + maximumConclusiveGapMs: BACKEND_MEMORY_MAX_CONCLUSIVE_GAP_MS, + firstSampleMonotonicMs: baseline.monotonicMs, + lastSampleMonotonicMs: input.samples.at(-1)!.monotonicMs, + maximumObservedGapMs, + samplerStartedAt: input.samplerStartedAt, + samplerReadyAt: input.samplerReadyAt, + buildStartedAt: input.buildStartedAt, + buildCompletedAt: input.buildCompletedAt, + samplerStoppedAt: input.samplerStoppedAt, + buildDurationMs: input.buildDurationMs, + samplerDurationMs: Math.max( + 0, + timestamps.samplerStopped - timestamps.samplerStarted + ), + coveredBuildWindow, + cadenceConclusive, + samplerLaunchToReadyMs, + samplerStopRequestToCloseMs + }, + observerEffect: { + samplerProcessCount: 1, + correctionApplied: false, + pairedComparisonSupported: true, + pairedComparisonFlag: '--postgres-backend-sampler', + measuredLaunchToReadyMs: samplerLaunchToReadyMs, + measuredStopRequestToCloseMs: samplerStopRequestToCloseMs, + limitation: 'Only sampler launch and shutdown wall time is measured; sampling ' + + 'CPU/I/O interference is not corrected. Compare paired runs using ' + + "'--postgres-backend-sampler off' and " + + "'--postgres-backend-sampler diagnostic-lower-bound'." + }, + provenance: { + samplerProcess: 'dedicated-external-procfs-loop', + samplerPid: input.samplerPid, + source: input.source, + postgresContainer: input.postgresContainer, + containerIdentity: input.containerIdentity ?? null, + clientPlatform: input.clientPlatform ?? os.platform(), + clientArchitecture: input.clientArchitecture ?? os.arch(), + backendIdentity: { + sqlBackendStartEpochMs: input.backendIdentity.backendStartEpochMs, + procStartTicks: baseline.procStartTicks, + procStartEpochMs: baseline.procStartEpochMs, + bootTimeEpochSeconds: baseline.bootTimeEpochSeconds, + clockTicksPerSecond: baseline.clockTicksPerSecond, + toleranceMs: BACKEND_START_IDENTITY_TOLERANCE_MS + }, + dockerInitialExecEnvironment: + input.source === 'docker-container-procfs-diagnostic' + ? 'may-inherit-container-config-before-env-i' + : 'not-applicable', + samplerShellEnvironment: 'env-i-path-only', + hostEnvironmentVariableNames: + [...(input.hostEnvironmentVariableNames ?? [])].sort(), + semantics: 'diagnostic-lower-bound-without-pre-destroy-acknowledgement', + backendSamplerAuthority: 'diagnostic-only', + serviceDensityMemoryAuthority: + `separately-validated-${LINUX_CGROUP_V2_DENSITY_AUTHORITY}`, + limitation: limitations.length === 0 ? null : limitations.join(' ') + } + }; +}; + +export const catalogBackendSamplerEnvironment = ( + environment: NodeJS.ProcessEnv = process.env +): NodeJS.ProcessEnv => { + const allowed = [ + 'PATH', + 'HOME', + 'LANG', + 'LC_ALL', + 'TZ', + 'TMPDIR', + 'DOCKER_HOST', + 'DOCKER_CONTEXT', + 'DOCKER_TLS_VERIFY', + 'DOCKER_CERT_PATH', + 'DOCKER_CONFIG', + 'XDG_RUNTIME_DIR' + ]; + return Object.fromEntries(allowed.flatMap((name) => { + const value = environment[name]; + return value === undefined ? [] : [[name, value]]; + })); +}; + +const readCatalogDockerContainerIdentity = ( + requestedName: string +): CatalogDockerContainerIdentity => { + validateCatalogPostgresContainer(requestedName); + const output = execFileSync( + 'docker', + [ + 'inspect', + '--format', + '{{.Id}}\t{{.State.StartedAt}}\t{{.State.Pid}}', + requestedName + ], + { + encoding: 'utf8', + env: catalogBackendSamplerEnvironment() + } + ); + return parseCatalogDockerContainerIdentity(output, requestedName); +}; + +export interface CatalogBackendSamplerLaunchSpec { + command: string; + args: string[]; + environment: NodeJS.ProcessEnv; + hostEnvironmentVariableNames: string[]; + source: CatalogBackendMemorySamplerSource; + statusPath: string; + statPath: string; + procStatPath: string; +} + +const catalogSamplerShellArgs = ( + statusPath: string, + statPath: string, + procStatPath: string, + backendIdentity: CatalogBackendIdentity +): string[] => [ + '/bin/sh', + '-c', + BACKEND_MEMORY_SAMPLER_SCRIPT, + 'catalog-backend-sampler', + statusPath, + statPath, + procStatPath, + String(backendIdentity.pid), + String(backendIdentity.backendStartEpochMs), + String(BACKEND_START_IDENTITY_TOLERANCE_MS), + String(BACKEND_MEMORY_SAMPLE_INTERVAL_MS / 1000) +]; + +export const makeCatalogBackendSamplerLaunchSpec = (input: { + backendIdentity: CatalogBackendIdentity; + containerIdentity?: CatalogDockerContainerIdentity | null; + clientPlatform?: string; + environment?: NodeJS.ProcessEnv; +}): CatalogBackendSamplerLaunchSpec | null => { + validateCatalogBackendIdentity(input.backendIdentity); + const clientPlatform = input.clientPlatform ?? os.platform(); + const environment = catalogBackendSamplerEnvironment(input.environment); + const hostEnvironmentVariableNames = Object.keys(environment).sort(); + const containerIdentity = input.containerIdentity ?? null; + if (containerIdentity) { + const containerProcRoot = `/proc/${containerIdentity.initHostPid}/root/proc`; + const hostStatusPath = `${containerProcRoot}/${input.backendIdentity.pid}/status`; + const hostStatPath = `${containerProcRoot}/${input.backendIdentity.pid}/stat`; + const hostProcStatPath = `${containerProcRoot}/stat`; + if ( + clientPlatform === 'linux' + && fs.existsSync(hostStatusPath) + && fs.existsSync(hostStatPath) + && fs.existsSync(hostProcStatPath) + ) { + return { + command: fs.existsSync('/usr/bin/env') ? '/usr/bin/env' : '/bin/env', + args: [ + '-i', + 'PATH=/usr/bin:/bin', + ...catalogSamplerShellArgs( + hostStatusPath, + hostStatPath, + hostProcStatPath, + input.backendIdentity + ) + ], + environment, + hostEnvironmentVariableNames, + source: 'linux-host-container-procfs', + statusPath: hostStatusPath, + statPath: hostStatPath, + procStatPath: hostProcStatPath + }; + } + const statusPath = `/proc/${input.backendIdentity.pid}/status`; + const statPath = `/proc/${input.backendIdentity.pid}/stat`; + return { + command: 'docker', + args: [ + 'exec', + '-i', + containerIdentity.immutableId, + '/usr/bin/env', + '-i', + 'PATH=/usr/bin:/bin', + ...catalogSamplerShellArgs( + statusPath, + statPath, + '/proc/stat', + input.backendIdentity + ) + ], + environment, + hostEnvironmentVariableNames, + source: 'docker-container-procfs-diagnostic', + statusPath, + statPath, + procStatPath: '/proc/stat' + }; + } + const statusPath = `/proc/${input.backendIdentity.pid}/status`; + const statPath = `/proc/${input.backendIdentity.pid}/stat`; + if ( + clientPlatform !== 'linux' + || !fs.existsSync(statusPath) + || !fs.existsSync(statPath) + || !fs.existsSync('/proc/stat') + ) return null; + return { + command: fs.existsSync('/usr/bin/env') ? '/usr/bin/env' : '/bin/env', + args: [ + '-i', + 'PATH=/usr/bin:/bin', + ...catalogSamplerShellArgs( + statusPath, + statPath, + '/proc/stat', + input.backendIdentity + ) + ], + environment, + hostEnvironmentVariableNames, + source: 'local-linux-procfs', + statusPath, + statPath, + procStatPath: '/proc/stat' + }; +}; + +const parseBackendSamplerOutputLine = ( + line: string, + expectedBackendIdentity: CatalogBackendIdentity +): CatalogBackendMemoryPoint | { targetExitedAtMonotonicMs: number } => { + const fields = line.split('\t'); + if (fields[0] === 'gone' && fields.length === 2) { + const targetExitedAtMonotonicMs = Number(fields[1]) * 1000; + if (!Number.isFinite(targetExitedAtMonotonicMs) || targetExitedAtMonotonicMs < 0) { + throw new Error(`invalid PostgreSQL backend sampler output '${line}'`); + } + return { targetExitedAtMonotonicMs }; + } + if (fields[0] !== 'sample' || fields.length !== 8) { + throw new Error(`unexpected PostgreSQL backend sampler output '${line}'`); + } + const monotonicMs = Number(fields[1]) * 1000; + const rssBytes = Number(fields[2]) * 1024; + const highWaterBytes = Number(fields[3]) * 1024; + const procStartTicks = Number(fields[4]); + const procStartEpochMs = Number(fields[5]); + const bootTimeEpochSeconds = Number(fields[6]); + const clockTicksPerSecond = Number(fields[7]); + if ( + !Number.isFinite(monotonicMs) + || monotonicMs < 0 + || !Number.isSafeInteger(rssBytes) + || rssBytes <= 0 + || !Number.isSafeInteger(highWaterBytes) + || highWaterBytes < rssBytes + || !Number.isSafeInteger(procStartTicks) + || procStartTicks <= 0 + || !Number.isSafeInteger(procStartEpochMs) + || procStartEpochMs <= 0 + || !Number.isSafeInteger(bootTimeEpochSeconds) + || bootTimeEpochSeconds <= 0 + || !Number.isSafeInteger(clockTicksPerSecond) + || clockTicksPerSecond <= 0 + || Math.abs(procStartEpochMs - expectedBackendIdentity.backendStartEpochMs) + > BACKEND_START_IDENTITY_TOLERANCE_MS + ) { + throw new Error(`invalid PostgreSQL backend sampler output '${line}'`); + } + return { + monotonicMs, + rssBytes, + highWaterBytes, + procStartTicks, + procStartEpochMs, + bootTimeEpochSeconds, + clockTicksPerSecond + }; +}; + +const withCatalogBackendSamplerTimeout = async ( + operation: Promise, + timeoutMs: number, + message: string +): Promise => new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(message)), timeoutMs); + operation.then((value) => { + clearTimeout(timer); + resolve(value); + }, (error) => { + clearTimeout(timer); + reject(error); + }); +}); + +export type CatalogBackendSamplerStopOutcome = + | 'already-exited' + | 'graceful' + | 'sigterm' + | 'sigkill'; + +export const stopCatalogBackendSamplerProcessTree = async (input: { + requestGracefulStop(): void; + waitForTreeExit(timeoutMs: number): Promise; + signalProcessGroup(signal: 'SIGTERM' | 'SIGKILL'): void; + gracefulTimeoutMs?: number; + termTimeoutMs?: number; + killTimeoutMs?: number; +}): Promise => { + if (await input.waitForTreeExit(0)) return 'already-exited'; + let gracefulStopError: unknown; + try { + input.requestGracefulStop(); + } catch (error) { + gracefulStopError = error; + } + const finish = (outcome: CatalogBackendSamplerStopOutcome) => { + if (gracefulStopError !== undefined) throw gracefulStopError; + return outcome; + }; + if (await input.waitForTreeExit( + input.gracefulTimeoutMs ?? BACKEND_MEMORY_SAMPLER_STOP_TIMEOUT_MS + )) return finish('graceful'); + input.signalProcessGroup('SIGTERM'); + if (await input.waitForTreeExit( + input.termTimeoutMs ?? BACKEND_MEMORY_SAMPLER_TERM_TIMEOUT_MS + )) return finish('sigterm'); + input.signalProcessGroup('SIGKILL'); + if (await input.waitForTreeExit( + input.killTimeoutMs ?? BACKEND_MEMORY_SAMPLER_KILL_TIMEOUT_MS + )) return finish('sigkill'); + throw new Error('PostgreSQL backend sampler process tree survived SIGKILL'); +}; + +const startCatalogBackendMemorySampler = async ( + container: string | null, + backendIdentity: CatalogBackendIdentity +): Promise => { + validateCatalogBackendIdentity(backendIdentity); + const containerIdentity = container === null + ? null + : readCatalogDockerContainerIdentity(container); + const launch = makeCatalogBackendSamplerLaunchSpec({ + backendIdentity, + containerIdentity + }); + if (!launch) return null; + + const samplerStartedAt = new Date().toISOString(); + const detached = process.platform !== 'win32'; + const child = spawn(launch.command, launch.args, { + detached, + env: launch.environment, + stdio: ['pipe', 'pipe', 'pipe'] + }); + const samplerPid = child.pid; + if (!samplerPid) { + child.kill('SIGKILL'); + throw new Error('PostgreSQL backend sampler did not receive a process PID'); + } + const samples: CatalogBackendMemoryPoint[] = []; + let targetExitedBeforeStop = false; + let targetExitedAtMonotonicMs: number | null = null; + let stdoutBuffer = ''; + let stderr = ''; + let stopRequested = false; + let stdinEnded = false; + let childClosed = false; + let exitCode: number | null = null; + let exitSignal: NodeJS.Signals | null = null; + let processError: Error | null = null; + let protocolError: Error | null = null; + let resolveReady!: () => void; + let rejectReady!: (error: Error) => void; + const ready = new Promise((resolve, reject) => { + resolveReady = resolve; + rejectReady = reject; + }); + let resolveClosed!: () => void; + const closed = new Promise((resolve) => { + resolveClosed = resolve; + }); + const requestGracefulStop = (): void => { + stopRequested = true; + if (!stdinEnded) { + stdinEnded = true; + child.stdin.end('stop\n'); + } + }; + const signalProcessGroup = (signal: 'SIGTERM' | 'SIGKILL'): void => { + try { + if (detached) process.kill(-samplerPid, signal); + else child.kill(signal); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; + } + }; + const processGroupExists = (): boolean => { + if (!detached) return !childClosed; + try { + process.kill(-samplerPid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== 'ESRCH'; + } + }; + const waitForTreeExit = async (timeoutMs: number): Promise => { + const deadline = performance.now() + timeoutMs; + while (true) { + if (childClosed && !processGroupExists()) return true; + if (performance.now() >= deadline) return false; + await Promise.race([ + closed, + new Promise((resolve) => { + const timer = setTimeout(resolve, Math.min(10, timeoutMs)); + timer.unref(); + }) + ]); + } + }; + let cleanupPromise: Promise | null = null; + const cleanupTree = (): Promise => { + cleanupPromise ??= stopCatalogBackendSamplerProcessTree({ + requestGracefulStop, + waitForTreeExit, + signalProcessGroup + }); + return cleanupPromise; + }; + const failProtocol = (error: Error): void => { + protocolError ??= error; + rejectReady(error); + requestGracefulStop(); + }; + const consumeLine = (rawLine: string): void => { + const line = rawLine.trim(); + if (!line) return; + try { + const parsed = parseBackendSamplerOutputLine(line, backendIdentity); + if ('targetExitedAtMonotonicMs' in parsed) { + targetExitedBeforeStop = true; + targetExitedAtMonotonicMs ??= parsed.targetExitedAtMonotonicMs; + } else { + const baseline = samples[0]; + if (baseline && ( + parsed.procStartTicks !== baseline.procStartTicks + || parsed.bootTimeEpochSeconds !== baseline.bootTimeEpochSeconds + || parsed.clockTicksPerSecond !== baseline.clockTicksPerSecond + )) { + throw new Error('PostgreSQL backend procfs start identity changed mid-sample'); + } + samples.push(parsed); + if (samples.length === 1) resolveReady(); + } + } catch (error) { + failProtocol(error instanceof Error ? error : new Error(String(error))); + } + }; + child.stdout.on('data', (chunk: Buffer | string) => { + stdoutBuffer += chunk.toString(); + const lines = stdoutBuffer.split('\n'); + stdoutBuffer = lines.pop() ?? ''; + lines.forEach(consumeLine); + }); + child.stderr.on('data', (chunk: Buffer | string) => { + if (stderr.length < 4_096) stderr += chunk.toString().slice(0, 4_096 - stderr.length); + }); + child.stdin.on('error', (error) => { + if (!stopRequested) failProtocol(error); + }); + child.once('error', (error) => { + processError = error; + rejectReady(error); + }); + child.once('close', (code, signal) => { + if (stdoutBuffer.trim()) consumeLine(stdoutBuffer); + childClosed = true; + exitCode = code; + exitSignal = signal; + if (samples.length === 0 && !processError && !protocolError) { + const detail = stderr.trim() ? `: ${stderr.trim()}` : ''; + rejectReady(new Error( + `PostgreSQL backend sampler exited code=${code} signal=${signal}${detail}` + )); + } + resolveClosed(); + }); + try { + await withCatalogBackendSamplerTimeout( + ready, + BACKEND_MEMORY_SAMPLER_START_TIMEOUT_MS, + `PostgreSQL backend sampler for PID ${backendIdentity.pid} did not produce a baseline` + ); + } catch (error) { + try { + await cleanupTree(); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + 'PostgreSQL backend sampler startup and cleanup both failed' + ); + } + throw error; + } + const samplerReadyAt = new Date().toISOString(); + let stopPromise: + Promise | null = null; + return { + stop(input) { + if (stopPromise) return stopPromise; + stopPromise = (async () => { + const samplerStopRequestedAt = new Date().toISOString(); + const stopOutcome = await cleanupTree(); + const samplerStoppedAt = new Date().toISOString(); + if (containerIdentity) { + assertCatalogDockerContainerIdentity( + containerIdentity, + readCatalogDockerContainerIdentity(containerIdentity.requestedName) + ); + } + if (protocolError) throw protocolError; + if (processError) throw processError; + if (exitCode !== 0 || exitSignal !== null) { + const detail = stderr.trim() ? `: ${stderr.trim()}` : ''; + throw new Error( + `PostgreSQL backend sampler exited code=${exitCode} ` + + `signal=${exitSignal} cleanup=${stopOutcome}${detail}` + ); + } + const measurement = summarizeCatalogBackendMemorySamples({ + backendIdentity, + samplerPid, + source: launch.source, + postgresContainer: container, + containerIdentity, + hostEnvironmentVariableNames: launch.hostEnvironmentVariableNames, + samples, + targetExitedBeforeStop, + targetExitedAtMonotonicMs, + samplerStartedAt, + samplerReadyAt, + buildStartedAt: input.buildStartedAt, + buildCompletedAt: input.buildCompletedAt, + samplerStopRequestedAt, + samplerStoppedAt, + buildDurationMs: input.buildDurationMs + }); + if (!measurement.timing.cadenceConclusive) { + throw new Error( + `PostgreSQL introspection backend ${backendIdentity.pid} sampling cadence ` + + `was inconclusive: ${measurement.provenance.limitation}` + ); + } + return measurement; + })(); + return stopPromise; + } + }; +}; + +export const measureCatalogBuildWithBackendSampler = async (input: { + startSampler(): Promise; + build(): Promise; +}): Promise<{ + value: T; + backendMemoryLowerBound: + CatalogBackendIntrospectionMemoryLowerBoundMeasurement | null; + buildDurationMs: number; + }> => { + const sampler = await input.startSampler(); + const buildStartedAt = new Date().toISOString(); + const started = performance.now(); + let value: T | undefined; + let buildError: unknown; + try { + value = await input.build(); + } catch (error) { + buildError = error; + } + const buildDurationMs = performance.now() - started; + const buildCompletedAt = new Date().toISOString(); + let backendMemoryLowerBound: + CatalogBackendIntrospectionMemoryLowerBoundMeasurement | null = null; + let samplerError: unknown; + if (sampler) { + try { + backendMemoryLowerBound = await sampler.stop({ + buildStartedAt, + buildCompletedAt, + buildDurationMs + }); + } catch (error) { + samplerError = error; + } + } + if (buildError !== undefined && samplerError !== undefined) { + throw new AggregateError( + [buildError, samplerError], + 'Graphile build and PostgreSQL introspection backend sampling both failed' + ); + } + if (samplerError !== undefined) throw samplerError; + if (buildError !== undefined) throw buildError; + return { value: value!, backendMemoryLowerBound, buildDurationMs }; +}; + +const readBackendMemory = ( + container: string | null, + backendPid: number +): BackendMemory | null => { + validateCatalogBackendPid(backendPid); + const statusPath = `/proc/${backendPid}/status`; + if (!container) { + if (os.platform() !== 'linux' || !fs.existsSync(statusPath)) return null; + try { + return parseCatalogBackendProcStatus( + fs.readFileSync(statusPath, 'utf8'), + backendPid + ); + } catch { + return null; + } + } + validateCatalogPostgresContainer(container); + const output = execFileSync( + 'docker', + [ + 'exec', + container, + 'cat', + statusPath + ], + { encoding: 'utf8' } + ); + return parseCatalogBackendProcStatus(output, backendPid); +}; + +interface CatalogExecutionContext { + schema: Awaited>; + resolvedPreset: ReturnType; + contextValue: Record; +} + +const getCatalogExecutionContext = async ( + entry: GraphileCacheEntry +): Promise => { + const schema = await entry.pgl.getSchema(); + const resolvedPreset = entry.pgl.getResolvedPreset(); + type BenchPgService = Parameters[0] & { + withPgClientKey?: string; + }; + const pgService = ( + resolvedPreset.pgServices as readonly BenchPgService[] | undefined + )?.[0]; + if (!pgService) throw new Error('built PostGraphile instance has no PostgreSQL service'); + const contextValue: Record = { pgSettings: {} }; + contextValue[pgService.withPgClientKey ?? 'withPgClient'] = withPgClientFromPgService.bind( + null, + pgService + ); + return { schema, resolvedPreset, contextValue }; +}; + +const executeTokenQuery = async ( + entry: GraphileCacheEntry +): Promise<{ token: string | null; elapsedMs: number }> => { + const { schema, resolvedPreset, contextValue } = await getCatalogExecutionContext(entry); + const started = performance.now(); + const result = await execute({ + schema, + document: parse('{ tenantToken }'), + contextValue, + resolvedPreset + }) as ExecutionResult<{ tenantToken?: unknown }>; + const elapsedMs = performance.now() - started; + if (result.errors?.length) { + throw new AggregateError(result.errors, 'tenant token query failed'); + } + const data = result.data as { tenantToken?: unknown } | null | undefined; + return { + token: typeof data?.tenantToken === 'string' ? data.tenantToken : null, + elapsedMs + }; +}; + +const executeWarmOperations = async ( + entry: GraphileCacheEntry, + sources: readonly string[], + passes: number, + expectedToken: string, + allExpectedTokens: readonly string[] +): Promise => { + const { schema, resolvedPreset, contextValue } = await getCatalogExecutionContext(entry); + const latenciesMs: number[] = []; + let errors = 0; + let returnedStrings = 0; + let exactMatches = 0; + let mismatchViolations = 0; + let crossTenantViolations = 0; + for (let pass = 0; pass < passes; pass++) { + for (const source of sources) { + const started = performance.now(); + try { + const result = await grafast({ + schema, + source, + contextValue, + resolvedPreset + }) as ExecutionResult<{ warmTenantToken?: unknown }>; + latenciesMs.push(performance.now() - started); + if (result.errors?.length) { + errors++; + continue; + } + const token = result.data?.warmTenantToken; + if (typeof token !== 'string') continue; + returnedStrings++; + if (token === expectedToken) { + exactMatches++; + } else { + mismatchViolations++; + if (allExpectedTokens.some((candidate) => candidate === token)) { + crossTenantViolations++; + } + } + } catch { + latenciesMs.push(performance.now() - started); + errors++; + } + } + } + const executionCount = sources.length * passes; + const correctnessConclusive = errors === 0 && returnedStrings === executionCount; + return { + latenciesMs, + errors, + returnedStrings, + exactMatches, + mismatchViolations, + crossTenantViolations, + correctnessConclusive, + correctnessPassed: correctnessConclusive && exactMatches === executionCount + }; +}; + +const emptyWarmOperationResult = (): CatalogWarmOperationResult => ({ + latenciesMs: [], + errors: 0, + returnedStrings: 0, + exactMatches: 0, + mismatchViolations: 0, + crossTenantViolations: 0, + correctnessConclusive: true, + correctnessPassed: true +}); + +const checkTokenCanary = async ( + entry: GraphileCacheEntry, + instanceIndex: number, + phase: CatalogCanarySample['phase'], + residentInstances: number, + schemas: string[], + expectedTokens: string[] | null, + canaries: CatalogCanarySample[] +): Promise<{ token: string | null; elapsedMs: number }> => { + const query = await executeTokenQuery(entry); + if (!expectedTokens) return query; + const expected = expectedTokens[instanceIndex]; + const matchedOtherTenant = query.token !== null && expectedTokens.some( + (candidate, tokenIndex) => tokenIndex !== instanceIndex && candidate === query.token + ); + canaries.push({ + phase, + residentInstances, + instance: instanceIndex + 1, + schema: schemas[instanceIndex], + expected, + actual: query.token, + returnedString: query.token !== null, + exactMatch: query.token === expected, + matchedOtherTenant + }); + return query; +}; + +const memorySnapshot = ( + instances: number, + baseline: NodeJS.MemoryUsage, + baselinePeakRssBytes: number, + baselineBackend: BackendMemory | null, + backend: BackendMemory | null, + reportBackendHighWater: boolean +): CatalogMemorySnapshot => { + const memory = process.memoryUsage(); + const processPeakRssBytes = process.resourceUsage().maxRSS * 1024; + return { + instances, + heapUsedBytes: memory.heapUsed, + heapDeltaBytes: memory.heapUsed - baseline.heapUsed, + rssBytes: memory.rss, + rssDeltaBytes: memory.rss - baseline.rss, + externalBytes: memory.external, + externalDeltaBytes: memory.external - baseline.external, + processPeakRssBytes, + processPeakRssDeltaBytes: Math.max(0, processPeakRssBytes - baselinePeakRssBytes), + postgresBackendRssBytes: backend?.rssBytes ?? null, + postgresBackendRssDeltaBytes: backend && baselineBackend + ? backend.rssBytes - baselineBackend.rssBytes + : null, + postgresBackendHighWaterBytes: reportBackendHighWater + ? backend?.highWaterBytes ?? null + : null, + postgresBackendHighWaterDeltaBytes: reportBackendHighWater + && backend + && baselineBackend + ? Math.max(0, backend.highWaterBytes - baselineBackend.highWaterBytes) + : null + }; +}; + +const baselineMemorySnapshot = ( + baseline: NodeJS.MemoryUsage, + baselinePeakRssBytes: number, + backend: BackendMemory | null, + reportBackendHighWater: boolean +): CatalogMemorySnapshot => ({ + instances: 0, + heapUsedBytes: baseline.heapUsed, + heapDeltaBytes: 0, + rssBytes: baseline.rss, + rssDeltaBytes: 0, + externalBytes: baseline.external, + externalDeltaBytes: 0, + processPeakRssBytes: baselinePeakRssBytes, + processPeakRssDeltaBytes: 0, + postgresBackendRssBytes: backend?.rssBytes ?? null, + postgresBackendRssDeltaBytes: backend ? 0 : null, + postgresBackendHighWaterBytes: reportBackendHighWater + ? backend?.highWaterBytes ?? null + : null, + postgresBackendHighWaterDeltaBytes: reportBackendHighWater && backend ? 0 : null +}); + +const readCatalogBackendIdentity = async ( + pool: Pool +): Promise => { + const result = await pool.query<{ + backend_pid: number; + backend_start_epoch_ms: string; + }>(`select + activity.pid as backend_pid, + floor(pg_catalog.extract(epoch from activity.backend_start) * 1000)::bigint::text + as backend_start_epoch_ms + from pg_catalog.pg_stat_activity as activity + where activity.pid = pg_catalog.pg_backend_pid()`); + const row = result.rows[0]; + const identity = { + pid: row?.backend_pid, + backendStartEpochMs: Number(row?.backend_start_epoch_ms) + }; + validateCatalogBackendIdentity(identity); + return identity; +}; + +const waitForBackendPidRetirement = async ( + controlPool: Pool, + backendIdentity: CatalogBackendIdentity +): Promise => { + validateCatalogBackendIdentity(backendIdentity); + const deadline = performance.now() + BACKEND_RETIREMENT_TIMEOUT_MS; + while (true) { + const result = await controlPool.query<{ backend_exists: boolean }>( + `select exists ( + select 1 + from pg_catalog.pg_stat_activity + where pid = $1 + and floor(pg_catalog.extract(epoch from backend_start) * 1000)::bigint = $2 + ) as backend_exists`, + [backendIdentity.pid, backendIdentity.backendStartEpochMs] + ); + if (result.rows[0]?.backend_exists === false) return; + if (performance.now() >= deadline) { + throw new Error( + `PostgreSQL introspection backend ${backendIdentity.pid}` + + `@${backendIdentity.backendStartEpochMs} did not retire within ` + + `${BACKEND_RETIREMENT_TIMEOUT_MS}ms` + ); + } + await new Promise((resolve) => { + const timer = setTimeout(resolve, BACKEND_RETIREMENT_POLL_INTERVAL_MS); + timer.unref(); + }); + } +}; + +const assertBackendIdentity = async ( + pool: Pool, + expected: CatalogBackendIdentity +): Promise => { + const actual = await readCatalogBackendIdentity(pool); + if ( + actual.pid !== expected.pid + || actual.backendStartEpochMs !== expected.backendStartEpochMs + ) { + throw new Error( + `PostgreSQL benchmark backend identity changed from ` + + `${expected.pid}@${expected.backendStartEpochMs} to ` + + `${actual.pid}@${actual.backendStartEpochMs}` + ); + } +}; + +const cleanupEntries = async (entries: GraphileCacheEntry[]): Promise => { + for (const entry of entries.reverse()) { + await entry.pgl.release(); + } +}; + +export const runCatalogBenchWorker = async ( + configFile: string, + resultFile: string +): Promise => { + const config = JSON.parse(fs.readFileSync(configFile, 'utf8')) as CatalogBenchConfig; + validateCatalogRuntimeFlags(config); + if (!Object.prototype.hasOwnProperty.call(config, 'scopedCatalogTypes')) { + config.scopedCatalogTypes = config.mode === 'scoped-required' ? 'all' : null; + } + validateCatalogScopedCatalogTypes(config.mode, config.scopedCatalogTypes); + config.introspectionClientReleaseMode ??= 'reuse'; + validateCatalogIntrospectionClientReleaseMode( + config.introspectionClientReleaseMode + ); + config.postgresBackendSamplerMode ??= 'diagnostic-lower-bound'; + validateCatalogBackendSamplerMode(config.postgresBackendSamplerMode); + config.releaseBuildStateAfterValidation ??= false; + if (typeof config.releaseBuildStateAfterValidation !== 'boolean') { + throw new Error('releaseBuildStateAfterValidation must be boolean'); + } + const schemaLayout = resolveCatalogSchemaLayout(config); + const schemaSets = schemaLayout.schemaSets + ?? schemaLayout.schemas.map((schema) => [schema]); + const requestedSchemaNames = [...new Set([ + ...schemaSets.flat(), + ...(schemaLayout.allowedDependencySchemas ?? []) + ])]; + config.warmOperationReplayPasses ??= 0; + validateCatalogWarmthConfig({ + warmOperationsPerInstance: config.warmOperationsPerInstance, + warmOperationReplayPasses: config.warmOperationReplayPasses, + grafastCacheLimits: config.grafastCacheLimits + }); + if ( + config.warmOperationsPerInstance > 0 + && ( + config.expectedTokens === null + || config.expectedTokens.length !== Math.max(...config.checkpoints) + ) + ) { + throw new Error( + 'catalog-bench warmth requires one expected token per instance' + ); + } + const startedAt = new Date().toISOString(); + const connectionOptions = { + ...getPgEnvOptions({ database: config.database }), + max: 1, + idleTimeoutMillis: 0 + }; + const controlPool = new Pool(connectionOptions); + const pool = new Pool(connectionOptions); + const entries: GraphileCacheEntry[] = []; + let backendPid = 0; + let backendIdentity: CatalogBackendIdentity | null = null; + try { + const metadata = await controlPool.query<{ + classes: string; + attributes: string; + procs: string; + types: string; + namespaces: string; + database_oid: string; + max_class_oid: string; + max_attribute_relation_oid: string; + max_proc_oid: string; + max_type_oid: string; + proc_signature_hash: string; + pg_version: string; + server_version_num: string; + jit: string; + }>(`select + (select count(*) from pg_catalog.pg_class)::text as classes, + (select count(*) from pg_catalog.pg_attribute)::text as attributes, + (select count(*) from pg_catalog.pg_proc)::text as procs, + (select count(*) from pg_catalog.pg_type)::text as types, + (select count(*) from pg_catalog.pg_namespace)::text as namespaces, + (select oid::text from pg_catalog.pg_database where datname = current_database()) as database_oid, + (select max(oid)::text from pg_catalog.pg_class) as max_class_oid, + (select max(attrelid)::text from pg_catalog.pg_attribute) as max_attribute_relation_oid, + (select max(oid)::text from pg_catalog.pg_proc) as max_proc_oid, + (select max(oid)::text from pg_catalog.pg_type) as max_type_oid, + (select md5(coalesce(string_agg( + md5(row( + pg_proc.oid, + pg_proc.pronamespace, + pg_proc.proname, + pg_proc.proowner, + pg_proc.prolang, + pg_proc.prokind, + pg_proc.prosecdef, + pg_proc.proleakproof, + pg_proc.proisstrict, + pg_proc.proretset, + pg_proc.provolatile, + pg_proc.proparallel, + pg_proc.pronargs, + pg_proc.pronargdefaults, + pg_proc.prorettype, + pg_proc.proargtypes, + pg_proc.proallargtypes, + pg_proc.proargmodes, + pg_proc.proargnames, + pg_proc.proconfig, + pg_proc.proacl, + pg_catalog.obj_description(pg_proc.oid, 'pg_proc') + )::text), + '' order by pg_proc.oid + ), '')) from pg_catalog.pg_proc) as proc_signature_hash, + version() as pg_version, + current_setting('server_version_num') as server_version_num, + current_setting('jit') as jit`); + const roleResult = await controlPool.query<{ + rolname: string; + rolsuper: boolean; + rolbypassrls: boolean; + rolcreaterole: boolean; + owns_database: boolean; + can_create_in_database: boolean; + }>(`select + pg_roles.rolname, + pg_roles.rolsuper, + pg_roles.rolbypassrls, + pg_roles.rolcreaterole, + pg_database.datdba = pg_roles.oid as owns_database, + pg_catalog.has_database_privilege( + pg_roles.rolname, + pg_database.oid, + 'CREATE' + ) as can_create_in_database + from pg_catalog.pg_roles + inner join pg_catalog.pg_database + on pg_database.datname = pg_catalog.current_database() + where pg_roles.rolname = current_user`); + const schemaSafety = await controlPool.query<{ + requested_schema_count: string; + owns_schema: boolean | null; + can_create: boolean | null; + }>(`select + count(*)::text as requested_schema_count, + bool_or(pg_catalog.pg_get_userbyid(pg_namespace.nspowner) = current_user) as owns_schema, + bool_or(pg_catalog.has_schema_privilege(current_user, pg_namespace.oid, 'CREATE')) as can_create + from pg_catalog.pg_namespace + where pg_namespace.nspname = any($1::text[])`, [requestedSchemaNames]); + const role = roleResult.rows[0]; + const schemaRole = schemaSafety.rows[0]; + if (!role) throw new Error('runtime role metadata was not returned'); + if ( + role.rolsuper + || role.rolbypassrls + || role.rolcreaterole + || role.owns_database + || role.can_create_in_database + ) { + throw new Error(`unsafe benchmark runtime role '${role.rolname}'`); + } + if (Number(schemaRole.requested_schema_count) !== requestedSchemaNames.length) { + throw new Error('one or more requested schemas do not exist'); + } + if (schemaRole.owns_schema || schemaRole.can_create) { + throw new Error(`runtime role '${role.rolname}' owns or can create in a requested schema`); + } + + backendIdentity = await readCatalogBackendIdentity(pool); + backendPid = backendIdentity.pid; + const initialBackendIdentity = { ...backendIdentity }; + + await forceGc(config.settleMs); + const baseline = process.memoryUsage(); + const baselinePeakRssBytes = process.resourceUsage().maxRSS * 1024; + const baselineBackend = readBackendMemory(config.postgresContainer, backendPid); + let steadyBackendBaseline = baselineBackend; + const reportIntrospectionBackendHighWater = + config.introspectionClientReleaseMode === 'reuse'; + let completedRetirementChecks = 0; + const snapshots: CatalogMemorySnapshot[] = [baselineMemorySnapshot( + baseline, + baselinePeakRssBytes, + baselineBackend, + reportIntrospectionBackendHighWater + )]; + const builds: CatalogBuildSample[] = []; + const canaries: CatalogCanarySample[] = []; + const allWarmOperationLatenciesMs: number[] = []; + const allWarmOperationReplayLatenciesMs: number[] = []; + const maxInstances = Math.max(...config.checkpoints); + const persistProgress = (status: CatalogBenchProgress['status']): void => { + const lastSnapshot = snapshots[snapshots.length - 1]; + writeCatalogProgress(resultFile, { + version: 1, + status, + mode: config.mode, + scopedCatalogTypes: config.scopedCatalogTypes, + introspectionClientReleaseMode: + config.introspectionClientReleaseMode, + postgresBackendSamplerMode: config.postgresBackendSamplerMode, + releaseBuildStateAfterValidation: + config.releaseBuildStateAfterValidation, + repetition: config.repetition, + heapMiB: config.heapMiB, + v8Profile: config.v8Profile, + nodeOptions: config.nodeOptions, + nodeOptionsArgv: [...config.nodeOptionsArgv], + nodeExecArgv: [...process.execArgv], + effectiveNodeRuntimeFlags: [ + ...config.nodeOptionsArgv, + ...process.execArgv + ], + targetInstances: maxInstances, + completedInstances: builds.length, + configuredCheckpoints: [...config.checkpoints], + completedCheckpoints: snapshots + .map((snapshot) => snapshot.instances) + .filter((instances) => instances > 0), + buildsCompleted: builds.length, + canariesCompleted: canaries.length, + mismatchViolations: canaries.filter((canary) => !canary.exactMatch).length, + crossTenantViolations: canaries.filter( + (canary) => canary.matchedOtherTenant + ).length, + lastSnapshot, + updatedAt: new Date().toISOString() + }); + }; + persistProgress('in-progress'); + const warmOperationSources = Array.from( + { length: config.warmOperationsPerInstance }, + (_, operationIndex) => makeCatalogWarmOperationSource(operationIndex + 1) + ); + const grafastCacheLimits = configuredGrafastCacheLimits(config.grafastCacheLimits); + const hasGrafastCacheLimits = Object.keys(grafastCacheLimits).length > 0; + const presetExtensions = hasGrafastCacheLimits + ? [ConstructivePreset, createGrafastCacheLimitsPreset(grafastCacheLimits)] + : [ConstructivePreset]; + const cacheLimitIdentity = hasGrafastCacheLimits + ? createHash('sha256').update(JSON.stringify(config.grafastCacheLimits)).digest('hex') + : null; + const introspectionBuildIdentity = catalogIntrospectionBuildIdentity( + config.mode, + config.scopedCatalogTypes, + config.releaseBuildStateAfterValidation, + config.introspectionClientReleaseMode + ); + + for (let index = 0; index < maxInstances; index++) { + const schemaName = schemaLayout.schemas[index]; + const instanceSchemas = schemaSets[index]; + const pgService = makePgService({ + pool, + schemas: instanceSchemas, + introspectionMode: config.mode, + introspectionClientReleaseMode: + config.introspectionClientReleaseMode, + ...(schemaLayout.allowedDependencySchemas === null + ? {} + : { + introspectionAllowedDependencySchemas: + schemaLayout.allowedDependencySchemas + }), + ...(config.scopedCatalogTypes === null + ? {} + : { introspectionScopedCatalogTypes: config.scopedCatalogTypes }) + }); + const preset = { + extends: presetExtensions, + schema: { + releaseBuildStateAfterValidation: + config.releaseBuildStateAfterValidation + }, + pgServices: [pgService] + }; + const schemaContractIdentity = schemaLayout.schemaSets === null + ? schemaName + : catalogSchemaContractIdentity( + instanceSchemas, + schemaLayout.allowedDependencySchemas! + ); + const buildCacheIdentity = schemaLayout.schemaSets === null + ? schemaName + : schemaContractIdentity; + await forceGc(0); + const sampledBuild = await measureCatalogBuildWithBackendSampler({ + startSampler: () => config.introspectionClientReleaseMode === 'destroy' + && config.postgresBackendSamplerMode === 'diagnostic-lower-bound' + ? startCatalogBackendMemorySampler( + config.postgresContainer, + backendIdentity! + ) + : Promise.resolve(null), + build: () => measureBuildTransient(() => createGraphileInstance({ + preset, + cacheKey: cacheLimitIdentity + ? `${introspectionBuildIdentity}:${cacheLimitIdentity}:${buildCacheIdentity}` + : `${introspectionBuildIdentity}:${buildCacheIdentity}`, + serviceKey: schemaLayout.schemaSets === null + ? schemaName + : `catalog:${schemaContractIdentity}` + })) + }); + const measuredBuild = sampledBuild.value; + const buildMs = sampledBuild.buildDurationMs; + const entry = measuredBuild.value; + entries.push(entry); + // The dedicated sampler is stopped and awaited by the helper before the + // destroyed PID is checked or a replacement checkout can be acquired. + const backendTransition = await resolveCatalogBackendPidAfterBuild( + config.introspectionClientReleaseMode, + backendIdentity, + { + waitForRetirement: (identity) => + waitForBackendPidRetirement(controlPool, identity), + acquireBackendIdentity: () => readCatalogBackendIdentity(pool) + } + ); + if ( + sampledBuild.backendMemoryLowerBound + && ( + sampledBuild.backendMemoryLowerBound.backendPid + !== backendTransition.introspectionBackendPid + || sampledBuild.backendMemoryLowerBound.backendStartEpochMs + !== backendTransition.introspectionBackendStartEpochMs + ) + ) { + throw new Error( + 'PostgreSQL introspection backend sampler identity did not match the ' + + 'retirement lifecycle identity' + ); + } + backendPid = backendTransition.steadyBackendPid; + backendIdentity = { + pid: backendTransition.steadyBackendPid, + backendStartEpochMs: backendTransition.steadyBackendStartEpochMs + }; + if (backendTransition.introspectionBackendRetired) { + completedRetirementChecks++; + steadyBackendBaseline = readBackendMemory( + config.postgresContainer, + backendPid + ); + } + + const query = await checkTokenCanary( + entry, + index, + 'initial', + index + 1, + config.schemas, + config.expectedTokens, + canaries + ); + const warmOperations = config.warmOperationsPerInstance > 0 + ? await executeWarmOperations( + entry, + warmOperationSources, + 1, + config.expectedTokens![index], + config.expectedTokens! + ) + : emptyWarmOperationResult(); + const warmOperationReplay = config.warmOperationReplayPasses > 0 + ? await executeWarmOperations( + entry, + warmOperationSources, + config.warmOperationReplayPasses, + config.expectedTokens![index], + config.expectedTokens! + ) + : emptyWarmOperationResult(); + allWarmOperationLatenciesMs.push(...warmOperations.latenciesMs); + allWarmOperationReplayLatenciesMs.push(...warmOperationReplay.latenciesMs); + await assertBackendIdentity(pool, backendIdentity); + const schema = await entry.pgl.getSchema(); + const sdl = printSchema(lexicographicSortSchema(schema)); + builds.push({ + instance: index + 1, + schema: schemaName, + ...backendTransition, + buildMs, + queryMs: query.elapsedMs, + token: query.token, + sdlBytes: Buffer.byteLength(sdl), + sdlSha256: createHash('sha256').update(sdl).digest('hex'), + queryFields: Object.keys(schema.getQueryType()?.getFields() ?? {}).sort(), + warmOperations: config.warmOperationsPerInstance, + warmOperationLatencyP50Ms: catalogPercentile(warmOperations.latenciesMs, 0.5), + warmOperationLatencyP99Ms: catalogPercentile(warmOperations.latenciesMs, 0.99), + warmOperationErrors: warmOperations.errors, + warmOperationReturnedStrings: warmOperations.returnedStrings, + warmOperationExactMatches: warmOperations.exactMatches, + warmOperationMismatchViolations: warmOperations.mismatchViolations, + warmOperationCrossTenantViolations: warmOperations.crossTenantViolations, + warmOperationCorrectnessConclusive: warmOperations.correctnessConclusive, + warmOperationCorrectnessPassed: warmOperations.correctnessPassed, + warmOperationReplayPasses: config.warmOperationReplayPasses, + warmOperationReplayExecutions: + config.warmOperationsPerInstance * config.warmOperationReplayPasses, + warmOperationReplayLatencyP50Ms: catalogPercentile( + warmOperationReplay.latenciesMs, + 0.5 + ), + warmOperationReplayLatencyP99Ms: catalogPercentile( + warmOperationReplay.latenciesMs, + 0.99 + ), + warmOperationReplayErrors: warmOperationReplay.errors, + warmOperationReplayReturnedStrings: warmOperationReplay.returnedStrings, + warmOperationReplayExactMatches: warmOperationReplay.exactMatches, + warmOperationReplayMismatchViolations: + warmOperationReplay.mismatchViolations, + warmOperationReplayCrossTenantViolations: + warmOperationReplay.crossTenantViolations, + warmOperationReplayCorrectnessConclusive: + warmOperationReplay.correctnessConclusive, + warmOperationReplayCorrectnessPassed: + warmOperationReplay.correctnessPassed, + buildBaselineHeapUsedBytes: measuredBuild.transient.baselineHeapUsedBytes, + buildBaselineRssBytes: measuredBuild.transient.baselineRssBytes, + sampledBuildPeakHeapUsedBytes: measuredBuild.transient.sampledPeakHeapUsedBytes, + sampledBuildPeakHeapDeltaBytes: measuredBuild.transient.sampledPeakHeapDeltaBytes, + sampledBuildPeakRssBytes: measuredBuild.transient.sampledPeakRssBytes, + sampledBuildPeakRssDeltaBytes: measuredBuild.transient.sampledPeakRssDeltaBytes, + processBuildPeakRssBytes: measuredBuild.transient.processPeakRssBytes, + processBuildPeakRssDeltaBytes: measuredBuild.transient.processPeakRssDeltaBytes, + buildTransientSampleCount: measuredBuild.transient.sampleCount, + postgresIntrospectionBackendMemoryLowerBound: + sampledBuild.backendMemoryLowerBound + }); + + if (config.checkpoints.includes(index + 1)) { + for (let residentIndex = 0; residentIndex < entries.length; residentIndex++) { + await checkTokenCanary( + entries[residentIndex], + residentIndex, + 'checkpoint', + index + 1, + config.schemas, + config.expectedTokens, + canaries + ); + } + await assertBackendIdentity(pool, backendIdentity); + await forceGc(config.settleMs); + snapshots.push(memorySnapshot( + index + 1, + baseline, + baselinePeakRssBytes, + steadyBackendBaseline, + readBackendMemory(config.postgresContainer, backendPid), + reportIntrospectionBackendHighWater + )); + // Serialize only a compact post-GC checkpoint after the measurement. + // If the next build OOMs, the parent can still recover the last + // conclusively resident point and bracket the capacity boundary. + persistProgress('in-progress'); + } + } + + const catalogRow = metadata.rows[0]; + const hashes = new Set(builds.map((build) => build.sdlSha256)); + const tokenMismatchViolations = canaries.filter((canary) => !canary.exactMatch).length; + const crossTenantTokenViolations = canaries.filter( + (canary) => canary.matchedOtherTenant + ).length; + const expectedCanaryCount = config.expectedTokens + ? maxInstances + config.checkpoints.reduce((sum, checkpoint) => sum + checkpoint, 0) + : 0; + const tokenCanariesConclusive = config.expectedTokens !== null + && canaries.length === expectedCanaryCount + && canaries.every((canary) => canary.returnedString); + const warmOperationExecutions = builds.reduce( + (sum, build) => sum + build.warmOperations, + 0 + ); + const warmOperationErrors = builds.reduce( + (sum, build) => sum + build.warmOperationErrors, + 0 + ); + const warmOperationReturnedStrings = builds.reduce( + (sum, build) => sum + build.warmOperationReturnedStrings, + 0 + ); + const warmOperationExactMatches = builds.reduce( + (sum, build) => sum + build.warmOperationExactMatches, + 0 + ); + const warmOperationMismatchViolations = builds.reduce( + (sum, build) => sum + build.warmOperationMismatchViolations, + 0 + ); + const warmOperationCrossTenantViolations = builds.reduce( + (sum, build) => sum + build.warmOperationCrossTenantViolations, + 0 + ); + const warmOperationCorrectnessConclusive = config.warmOperationsPerInstance === 0 + || ( + warmOperationErrors === 0 + && warmOperationReturnedStrings === warmOperationExecutions + ); + const warmOperationCorrectnessPassed = warmOperationCorrectnessConclusive + && warmOperationExactMatches === warmOperationExecutions; + const warmOperationReplayExecutions = builds.reduce( + (sum, build) => sum + build.warmOperationReplayExecutions, + 0 + ); + const warmOperationReplayErrors = builds.reduce( + (sum, build) => sum + build.warmOperationReplayErrors, + 0 + ); + const warmOperationReplayReturnedStrings = builds.reduce( + (sum, build) => sum + build.warmOperationReplayReturnedStrings, + 0 + ); + const warmOperationReplayExactMatches = builds.reduce( + (sum, build) => sum + build.warmOperationReplayExactMatches, + 0 + ); + const warmOperationReplayMismatchViolations = builds.reduce( + (sum, build) => sum + build.warmOperationReplayMismatchViolations, + 0 + ); + const warmOperationReplayCrossTenantViolations = builds.reduce( + (sum, build) => sum + build.warmOperationReplayCrossTenantViolations, + 0 + ); + const warmOperationReplayCorrectnessConclusive = + config.warmOperationReplayPasses === 0 + || ( + warmOperationReplayErrors === 0 + && warmOperationReplayReturnedStrings === warmOperationReplayExecutions + ); + const warmOperationReplayCorrectnessPassed = + warmOperationReplayCorrectnessConclusive + && warmOperationReplayExactMatches === warmOperationReplayExecutions; + const fixtureFingerprint = createHash('sha256').update(JSON.stringify({ + database: config.database, + schemas: config.schemas, + ...(schemaLayout.schemaSets === null + ? {} + : { + schemaSets: schemaLayout.schemaSets, + allowedDependencySchemas: schemaLayout.allowedDependencySchemas + }), + classes: catalogRow.classes, + attributes: catalogRow.attributes, + procs: catalogRow.procs, + types: catalogRow.types, + namespaces: catalogRow.namespaces, + databaseOid: catalogRow.database_oid, + maxClassOid: catalogRow.max_class_oid, + maxAttributeRelationOid: catalogRow.max_attribute_relation_oid, + maxProcOid: catalogRow.max_proc_oid, + maxTypeOid: catalogRow.max_type_oid, + procSignatureHash: catalogRow.proc_signature_hash, + pgVersion: catalogRow.pg_version, + serverVersionNum: catalogRow.server_version_num, + jit: catalogRow.jit + })).digest('hex'); + const introspectionBackendMeasurements = builds.flatMap((build) => + build.postgresIntrospectionBackendMemoryLowerBound + ? [build.postgresIntrospectionBackendMemoryLowerBound] + : [] + ); + const expectedIntrospectionBackendMeasurements = + config.introspectionClientReleaseMode === 'destroy' + && config.postgresBackendSamplerMode === 'diagnostic-lower-bound' + ? builds.length + : 0; + const allIntrospectionBackendCadenceChecksConclusive = + expectedIntrospectionBackendMeasurements === 0 + || ( + introspectionBackendMeasurements.length === builds.length + && introspectionBackendMeasurements.every( + (measurement) => measurement.timing.cadenceConclusive + ) + ); + const introspectionMeasurementLimitations = [...new Set( + introspectionBackendMeasurements.flatMap((measurement) => + measurement.provenance.limitation + ? [measurement.provenance.limitation] + : [] + ) + )]; + if ( + expectedIntrospectionBackendMeasurements > 0 + && introspectionBackendMeasurements.length !== 0 + && introspectionBackendMeasurements.length !== builds.length + ) { + throw new Error( + 'PostgreSQL introspection backend measurement was only recorded for ' + + `${introspectionBackendMeasurements.length} of ${builds.length} builds` + ); + } + const result: CatalogBenchResult = { + version: 1, + status: 'performance-only', + database: config.database, + mode: config.mode, + scopedCatalogTypes: config.scopedCatalogTypes, + introspectionClientReleaseMode: + config.introspectionClientReleaseMode, + postgresBackendSamplerMode: config.postgresBackendSamplerMode, + releaseBuildStateAfterValidation: + config.releaseBuildStateAfterValidation, + ...(schemaLayout.schemaSets === null + ? {} + : { + schemaSets: schemaLayout.schemaSets, + allowedDependencySchemas: schemaLayout.allowedDependencySchemas + }), + repetition: config.repetition, + heapMiB: config.heapMiB, + commit: config.commit, + worktreeDirty: config.worktreeDirty, + sourceStateSha256: config.sourceStateSha256, + lockfileSha256: config.lockfileSha256, + executedEntrySha256: config.executedEntrySha256, + v8Profile: config.v8Profile, + nodeOptions: config.nodeOptions, + nodeOptionsArgv: [...config.nodeOptionsArgv], + nodeExecArgv: [...process.execArgv], + effectiveNodeRuntimeFlags: [ + ...config.nodeOptionsArgv, + ...process.execArgv + ], + node: process.version, + v8: process.versions.v8, + effectiveV8HeapLimitBytes: getHeapStatistics().heap_size_limit, + platform: os.platform(), + architecture: os.arch(), + startedAt, + endedAt: new Date().toISOString(), + catalog: { + classes: Number(catalogRow.classes), + attributes: Number(catalogRow.attributes), + procs: Number(catalogRow.procs), + types: Number(catalogRow.types), + namespaces: Number(catalogRow.namespaces) + }, + runtimeRole: { + name: role.rolname, + superuser: role.rolsuper, + bypassRls: role.rolbypassrls, + createRole: role.rolcreaterole, + ownsDatabase: role.owns_database, + canCreateInDatabase: role.can_create_in_database, + ownsRequestedSchema: schemaRole.owns_schema, + canCreateInRequestedSchema: schemaRole.can_create + }, + catalogWarmth: 'shared-server-not-reset', + grafastCacheWarmth: { + operationsPerInstance: config.warmOperationsPerInstance, + cacheLimits: config.grafastCacheLimits, + sourceMode: 'grafast-source', + sourceSetSha256: warmOperationSources.length === 0 + ? null + : createHash('sha256').update(warmOperationSources.join('\0')).digest('hex'), + operationExecutions: warmOperationExecutions, + latencyP50Ms: catalogPercentile(allWarmOperationLatenciesMs, 0.5), + latencyP99Ms: catalogPercentile(allWarmOperationLatenciesMs, 0.99), + errors: warmOperationErrors, + returnedStrings: warmOperationReturnedStrings, + exactMatches: warmOperationExactMatches, + mismatchViolations: warmOperationMismatchViolations, + crossTenantViolations: warmOperationCrossTenantViolations, + correctnessConclusive: warmOperationCorrectnessConclusive, + correctnessPassed: warmOperationCorrectnessPassed, + replay: { + passesPerInstance: config.warmOperationReplayPasses, + operationExecutions: warmOperationReplayExecutions, + latencyP50Ms: catalogPercentile(allWarmOperationReplayLatenciesMs, 0.5), + latencyP99Ms: catalogPercentile(allWarmOperationReplayLatenciesMs, 0.99), + errors: warmOperationReplayErrors, + returnedStrings: warmOperationReplayReturnedStrings, + exactMatches: warmOperationReplayExactMatches, + mismatchViolations: warmOperationReplayMismatchViolations, + crossTenantViolations: warmOperationReplayCrossTenantViolations, + correctnessConclusive: warmOperationReplayCorrectnessConclusive, + correctnessPassed: warmOperationReplayCorrectnessPassed + } + }, + buildTransientSampling: { + approximate: true, + intervalMs: BUILD_TRANSIENT_SAMPLE_INTERVAL_MS, + limitation: 'Event-loop sampling can miss synchronous heap/RSS peaks; process RSS high-water is also captured.', + maxSampledHeapDeltaBytes: Math.max( + 0, + ...builds.map((build) => build.sampledBuildPeakHeapDeltaBytes) + ), + maxSampledRssDeltaBytes: Math.max( + 0, + ...builds.map((build) => build.sampledBuildPeakRssDeltaBytes) + ), + maxProcessPeakRssDeltaBytes: Math.max( + 0, + ...builds.map((build) => build.processBuildPeakRssDeltaBytes) + ) + }, + postgresBackendMeasurement: { + initialBackendPid: initialBackendIdentity.pid, + initialBackendStartEpochMs: + initialBackendIdentity.backendStartEpochMs, + finalSteadyBackendPid: backendPid, + finalSteadyBackendStartEpochMs: backendIdentity.backendStartEpochMs, + expectedRetirementChecks: + config.introspectionClientReleaseMode === 'destroy' + ? builds.length + : 0, + completedRetirementChecks, + allExpectedRetirementsProven: + completedRetirementChecks === ( + config.introspectionClientReleaseMode === 'destroy' + ? builds.length + : 0 + ), + steadyBackendRss: { + measured: baselineBackend !== null, + samplePhase: config.introspectionClientReleaseMode === 'destroy' + ? 'post-introspection-replacement' + : 'shared-introspection-and-steady', + deltaBasis: config.introspectionClientReleaseMode === 'destroy' + ? 'replacement-acquisition' + : 'initial-backend' + }, + introspectionBackendMemory: { + sampledLowerBoundMeasured: + config.introspectionClientReleaseMode === 'destroy' + ? introspectionBackendMeasurements.length === builds.length + : false, + sharedSnapshotMeasured: + config.introspectionClientReleaseMode === 'reuse' + && baselineBackend !== null + && reportIntrospectionBackendHighWater, + semantics: config.introspectionClientReleaseMode === 'destroy' + ? introspectionBackendMeasurements.length === builds.length + ? 'diagnostic-lower-bound-without-pre-destroy-acknowledgement' + : 'unavailable' + : baselineBackend !== null + ? 'post-build-shared-backend-snapshot' + : 'unavailable', + measurementMethod: config.introspectionClientReleaseMode === 'destroy' + ? introspectionBackendMeasurements.length === builds.length + ? 'dedicated-identity-bound-procfs-sampler' + : 'unavailable' + : baselineBackend !== null + ? 'post-build-shared-backend-procfs' + : 'unavailable', + expectedBuildMeasurements: expectedIntrospectionBackendMeasurements, + completedBuildMeasurements: introspectionBackendMeasurements.length, + allBuildCadenceChecksConclusive: + allIntrospectionBackendCadenceChecksConclusive, + backendSamplerAuthority: 'diagnostic-only', + serviceDensityMemoryAuthority: + `separately-validated-${LINUX_CGROUP_V2_DENSITY_AUTHORITY}`, + limitation: config.introspectionClientReleaseMode === 'destroy' + ? introspectionBackendMeasurements.length === 0 + ? config.postgresBackendSamplerMode === 'off' + ? 'Backend sampler disabled for an observer-effect comparison; no ' + + 'replacement-backend value was substituted.' + : 'No identity-bound PostgreSQL backend procfs target was available; ' + + 'no replacement-backend value was substituted.' + : introspectionMeasurementLimitations.length === 0 + ? null + : introspectionMeasurementLimitations.join(' ') + : baselineBackend === null + ? 'No identity-bound PostgreSQL backend procfs target was available.' + : 'This is a post-build snapshot of a reused backend, not a destroyed ' + + 'backend peak or a service-memory authority.' + } + }, + fixtureFingerprint, + builds, + canaries, + snapshots, + heapSlopeBytesPerInstance: slope( + snapshots.map((snapshot) => ({ x: snapshot.instances, y: snapshot.heapDeltaBytes })) + ), + rssSlopeBytesPerInstance: slope( + snapshots.map((snapshot) => ({ x: snapshot.instances, y: snapshot.rssDeltaBytes })) + ), + allSdlHashesEqualWithinArm: hashes.size === 1, + tokenCanariesConclusive, + tokenCanariesPassed: tokenCanariesConclusive && tokenMismatchViolations === 0, + tokenMismatchViolations, + crossTenantTokenViolations, + bleedViolations: tokenMismatchViolations + crossTenantTokenViolations + }; + fs.mkdirSync(path.dirname(resultFile), { recursive: true }); + fs.writeFileSync(resultFile, `${JSON.stringify(result, null, 2)}\n`, 'utf8'); + persistProgress('complete'); + } finally { + await cleanupEntries(entries); + await pool.end(); + await controlPool.end(); + } +}; + +const waitForChild = ( + command: string, + commandArgs: string[], + logFile: string, + env: NodeJS.ProcessEnv +): Promise => new Promise((resolve, reject) => { + const log = fs.createWriteStream(logFile, { flags: 'w' }); + const child = spawn(command, commandArgs, { + env, + stdio: ['ignore', 'pipe', 'pipe'] + }); + child.stdout?.pipe(log); + child.stderr?.pipe(log); + child.once('error', reject); + child.once('exit', (code, signal) => { + log.end(); + if (code === 0) resolve(); + else reject(new Error(`catalog worker exited code=${code} signal=${signal}; see ${logFile}`)); + }); +}); + +export const runCatalogBench = async (args: string[]): Promise => { + const database = requireFlag(args, 'database'); + const mode = requireFlag(args, 'mode') as IntrospectionMode; + if (mode !== 'stock' && mode !== 'scoped-required') { + throw new Error("--mode must be 'stock' or 'scoped-required'"); + } + const scopedCatalogTypes = parseCatalogScopedCatalogTypes(args, mode); + const introspectionClientReleaseMode = + parseCatalogIntrospectionClientReleaseMode(args); + const postgresBackendSamplerMode = parseCatalogBackendSamplerMode(args); + const releaseBuildStateAfterValidation = parseCatalogBuildStateRetirement(args); + const v8Profile = parseCatalogV8Profile(args); + const checkpoints = parseCheckpoints(flag(args, 'instances') ?? '1'); + const maxInstances = Math.max(...checkpoints); + const schemaLayout = parseCatalogSchemaLayout(args, maxInstances); + const schemas = schemaLayout.schemas; + const tokenFlag = flag(args, 'expected-tokens'); + const expectedTokens = tokenFlag ? parseList(tokenFlag) : null; + if (expectedTokens && ( + expectedTokens.length !== maxInstances + || new Set(expectedTokens).size !== expectedTokens.length + )) { + throw new Error('--expected-tokens must contain one unique value per instance'); + } + const warmth = parseCatalogWarmthCliOptions(args); + const tenantProxySurfaces = parseCatalogTenantProxySurfaces(args); + if (warmth.warmOperationsPerInstance > 0 && expectedTokens === null) { + throw new Error( + '--expected-tokens is required when --warm-operations-per-instance is greater than zero' + ); + } + const heapMiB = parsePositiveInteger(flag(args, 'heap-mib') ?? '2048', 'heap-mib'); + const nodeOptions = replaceMaxOldSpaceSize(process.env.NODE_OPTIONS, heapMiB); + const nodeOptionsArgv = tokenizeNodeOptions(nodeOptions); + const nodeExecArgv = [ + ...nodeFlagsForV8Profile(v8Profile), + '--expose-gc' + ]; + const effectiveNodeRuntimeFlags = [...nodeOptionsArgv, ...nodeExecArgv]; + const repetitions = parsePositiveInteger(flag(args, 'repetitions') ?? '3', 'repetitions'); + const settleMs = parsePositiveInteger(flag(args, 'settle-ms') ?? '100', 'settle-ms'); + const outputRoot = path.resolve(requireFlag(args, 'out')); + const postgresContainer = flag(args, 'postgres-container') ?? null; + if (fs.existsSync(outputRoot) && fs.readdirSync(outputRoot).length > 0) { + throw new Error(`catalog-bench refuses to overwrite nonempty output directory '${outputRoot}'`); + } + const provenance = readGitProvenance(); + const lockfile = path.resolve('pnpm-lock.yaml'); + const lockfileSha256 = fs.existsSync(lockfile) ? sha256File(lockfile) : null; + const executedEntrySha256 = sha256File(process.argv[1]); + fs.mkdirSync(outputRoot, { recursive: true }); + const results: CatalogBenchResult[] = []; + + for (let repetition = 1; repetition <= repetitions; repetition++) { + const repetitionDir = path.join(outputRoot, `rep-${repetition}`); + fs.mkdirSync(repetitionDir, { recursive: true }); + const config: CatalogBenchConfig = { + version: 1, + database, + mode, + scopedCatalogTypes, + introspectionClientReleaseMode, + postgresBackendSamplerMode, + releaseBuildStateAfterValidation, + schemas: schemas.slice(0, maxInstances), + ...(schemaLayout.schemaSets === null + ? {} + : { + schemaSets: schemaLayout.schemaSets, + allowedDependencySchemas: schemaLayout.allowedDependencySchemas! + }), + checkpoints, + expectedTokens: expectedTokens?.slice(0, maxInstances) ?? null, + heapMiB, + repetition, + settleMs, + warmOperationsPerInstance: warmth.warmOperationsPerInstance, + warmOperationReplayPasses: warmth.warmOperationReplayPasses, + grafastCacheLimits: warmth.grafastCacheLimits, + postgresContainer, + commit: provenance.commit, + worktreeDirty: provenance.worktreeDirty, + sourceStateSha256: provenance.sourceStateSha256, + lockfileSha256, + executedEntrySha256, + v8Profile, + nodeOptions, + nodeOptionsArgv, + nodeExecArgv, + effectiveNodeRuntimeFlags + }; + const configFile = path.join(repetitionDir, 'config.json'); + const resultFile = path.join(repetitionDir, 'result.json'); + fs.writeFileSync(configFile, `${JSON.stringify(config, null, 2)}\n`, 'utf8'); + await waitForChild( + process.execPath, + [ + ...nodeExecArgv, + process.argv[1], + '__catalog-worker', + '--config', + configFile, + '--result', + resultFile + ], + path.join(repetitionDir, 'worker.log'), + { + ...process.env, + NODE_ENV: 'production', + GRAPHILE_ENV: 'production', + NODE_OPTIONS: nodeOptions + } + ); + results.push(JSON.parse(fs.readFileSync(resultFile, 'utf8')) as CatalogBenchResult); + } + + const finalCheckpoint = maxInstances; + const finalSnapshots = results.map((result) => + result.snapshots.find((snapshot) => snapshot.instances === finalCheckpoint)! + ); + const postgresSampledHighWaterLowerBoundByRepetition = results.map((result) => + result.introspectionClientReleaseMode === 'destroy' + ? maxOrNull(result.builds.map( + (build) => build.postgresIntrospectionBackendMemoryLowerBound + ?.sampledHighWaterLowerBoundBytes ?? null + )) + : null + ); + const postgresSampledHighWaterDeltaLowerBoundByRepetition = results.map( + (result) => result.introspectionClientReleaseMode === 'destroy' + ? maxOrNull(result.builds.map( + (build) => build.postgresIntrospectionBackendMemoryLowerBound + ?.sampledHighWaterDeltaLowerBoundBytes ?? null + )) + : null + ); + const postgresSharedBackendSnapshotHighWaterByRepetition = results.map( + (result) => result.introspectionClientReleaseMode === 'reuse' + ? maxOrNull(result.snapshots.map( + (snapshot) => snapshot.postgresBackendHighWaterBytes + )) + : null + ); + const postgresSteadyRssByRepetition = finalSnapshots.map( + (snapshot) => snapshot.postgresBackendRssBytes + ); + const postgresSteadyRssDeltaByRepetition = finalSnapshots.map( + (snapshot) => snapshot.postgresBackendRssDeltaBytes + ); + const fixtureFingerprints = [...new Set(results.map((result) => result.fixtureFingerprint))]; + const effectiveV8HeapLimitBytes = results.map( + (result) => result.effectiveV8HeapLimitBytes + ); + const runtimeFlagsConsistent = results.every((result) => + result.v8Profile === v8Profile + && result.nodeOptions === nodeOptions + && JSON.stringify(result.nodeOptionsArgv) === JSON.stringify(nodeOptionsArgv) + && JSON.stringify(result.nodeExecArgv) === JSON.stringify(nodeExecArgv) + && JSON.stringify(result.effectiveNodeRuntimeFlags) + === JSON.stringify(effectiveNodeRuntimeFlags) + ); + if (!runtimeFlagsConsistent) { + throw new Error('catalog-bench worker runtime-flag provenance is inconsistent'); + } + const tenantDensityByRepetition = tenantProxySurfaces === null + ? null + : results.map((result, index) => ({ + repetition: result.repetition, + ...projectCatalogTenantDensity({ + tenantProxySurfaces, + configuredOldSpaceMiB: heapMiB, + snapshot: finalSnapshots[index] + }) + })); + const groupsPerConfiguredOldSpaceGiB = tenantDensityByRepetition?.map( + (density) => density.groupsPerConfiguredOldSpaceGiB + ) ?? []; + const groupsPerAbsolutePeakProcessRssGiB = tenantDensityByRepetition?.map( + (density) => density.groupsPerAbsolutePeakProcessRssGiB + ) ?? []; + const summary = { + version: 1, + status: 'performance-only', + mode, + scopedCatalogTypes, + introspectionClientReleaseMode, + postgresBackendSamplerMode, + releaseBuildStateAfterValidation, + v8Profile, + ...(schemaLayout.schemaSets === null + ? {} + : { + schemaSets: schemaLayout.schemaSets, + allowedDependencySchemas: schemaLayout.allowedDependencySchemas + }), + database, + heapMiB, + repetitions, + checkpoints, + v8Heap: { + configuredMaxOldSpaceMiB: heapMiB, + effectiveHeapLimitBytes: effectiveV8HeapLimitBytes, + effectiveHeapLimitConsistent: new Set(effectiveV8HeapLimitBytes).size === 1 + }, + nodeRuntimeFlags: { + nodeOptions, + nodeOptionsArgv, + nodeExecArgv, + effectiveNodeRuntimeFlags, + consistent: runtimeFlagsConsistent + }, + tenantDensityProjection: tenantProxySurfaces === null + ? null + : { + kind: 'synthetic-surface-instance-equivalent', + measuredCompleteTenants: false, + capacityBoundaryReached: false, + tenantProxySurfaces, + checkpoint: 'final-scheduled-checkpoint', + perRepetition: tenantDensityByRepetition, + medianGroupsPerConfiguredOldSpaceGiB: median( + groupsPerConfiguredOldSpaceGiB + ), + worstCaseGroupsPerConfiguredOldSpaceGiB: Math.min( + ...groupsPerConfiguredOldSpaceGiB + ), + medianGroupsPerAbsolutePeakProcessRssGiB: median( + groupsPerAbsolutePeakProcessRssGiB + ), + worstCaseGroupsPerAbsolutePeakProcessRssGiB: Math.min( + ...groupsPerAbsolutePeakProcessRssGiB + ) + }, + grafastCacheWarmth: { + operationsPerInstance: warmth.warmOperationsPerInstance, + cacheLimits: warmth.grafastCacheLimits, + sourceMode: 'grafast-source', + operationExecutions: results.map( + (result) => result.grafastCacheWarmth.operationExecutions + ), + latencyP50Ms: results.map((result) => result.grafastCacheWarmth.latencyP50Ms), + medianLatencyP50Ms: medianOrNull( + results.map((result) => result.grafastCacheWarmth.latencyP50Ms) + ), + latencyP99Ms: results.map((result) => result.grafastCacheWarmth.latencyP99Ms), + medianLatencyP99Ms: medianOrNull( + results.map((result) => result.grafastCacheWarmth.latencyP99Ms) + ), + correctnessConclusive: results.every( + (result) => result.grafastCacheWarmth.correctnessConclusive + ), + correctnessPassed: results.every( + (result) => result.grafastCacheWarmth.correctnessPassed + ), + errors: results.reduce( + (sum, result) => sum + result.grafastCacheWarmth.errors, + 0 + ), + mismatchViolations: results.reduce( + (sum, result) => sum + result.grafastCacheWarmth.mismatchViolations, + 0 + ), + crossTenantViolations: results.reduce( + (sum, result) => sum + result.grafastCacheWarmth.crossTenantViolations, + 0 + ), + replay: { + passesPerInstance: warmth.warmOperationReplayPasses, + sourceSet: 'same-exact-sources-as-population', + operationExecutions: results.map( + (result) => result.grafastCacheWarmth.replay.operationExecutions + ), + latencyP50Ms: results.map( + (result) => result.grafastCacheWarmth.replay.latencyP50Ms + ), + medianLatencyP50Ms: medianOrNull( + results.map((result) => result.grafastCacheWarmth.replay.latencyP50Ms) + ), + latencyP99Ms: results.map( + (result) => result.grafastCacheWarmth.replay.latencyP99Ms + ), + medianLatencyP99Ms: medianOrNull( + results.map((result) => result.grafastCacheWarmth.replay.latencyP99Ms) + ), + correctnessConclusive: results.every( + (result) => result.grafastCacheWarmth.replay.correctnessConclusive + ), + correctnessPassed: results.every( + (result) => result.grafastCacheWarmth.replay.correctnessPassed + ), + errors: results.reduce( + (sum, result) => sum + result.grafastCacheWarmth.replay.errors, + 0 + ), + mismatchViolations: results.reduce( + (sum, result) => + sum + result.grafastCacheWarmth.replay.mismatchViolations, + 0 + ), + crossTenantViolations: results.reduce( + (sum, result) => + sum + result.grafastCacheWarmth.replay.crossTenantViolations, + 0 + ) + } + }, + buildTransientSampling: { + approximate: true, + intervalMs: BUILD_TRANSIENT_SAMPLE_INTERVAL_MS, + limitation: 'Event-loop sampling can miss synchronous heap/RSS peaks; process RSS high-water is also captured.', + maxSampledHeapDeltaBytes: results.map( + (result) => result.buildTransientSampling.maxSampledHeapDeltaBytes + ), + medianMaxSampledHeapDeltaBytes: median( + results.map((result) => result.buildTransientSampling.maxSampledHeapDeltaBytes) + ), + maxSampledRssDeltaBytes: results.map( + (result) => result.buildTransientSampling.maxSampledRssDeltaBytes + ), + medianMaxSampledRssDeltaBytes: median( + results.map((result) => result.buildTransientSampling.maxSampledRssDeltaBytes) + ), + maxProcessPeakRssDeltaBytes: results.map( + (result) => result.buildTransientSampling.maxProcessPeakRssDeltaBytes + ), + medianMaxProcessPeakRssDeltaBytes: median( + results.map((result) => result.buildTransientSampling.maxProcessPeakRssDeltaBytes) + ) + }, + postgresBackendSamplerObserverEffect: { + mode: postgresBackendSamplerMode, + correctionApplied: false, + pairedComparisonSupported: true, + comparisonValues: ['off', 'diagnostic-lower-bound'], + measuredLaunchToReadyMs: results.map((result) => result.builds.flatMap( + (build) => build.postgresIntrospectionBackendMemoryLowerBound + ? [build.postgresIntrospectionBackendMemoryLowerBound + .observerEffect.measuredLaunchToReadyMs] + : [] + )), + measuredStopRequestToCloseMs: results.map((result) => result.builds.flatMap( + (build) => build.postgresIntrospectionBackendMemoryLowerBound + ? [build.postgresIntrospectionBackendMemoryLowerBound + .observerEffect.measuredStopRequestToCloseMs] + : [] + )), + limitation: 'Only sampler launch and shutdown wall time is recorded; sampling ' + + 'CPU/I/O interference is not corrected. Use paired runs with ' + + "'--postgres-backend-sampler off' and " + + "'--postgres-backend-sampler diagnostic-lower-bound'." + }, + measurementProtocol: { + process: 'fresh-node-process-per-repetition', + heap: 'three-forced-gc-cycles-after-resident-reprobe', + build: 'forced-gc-resident-baseline-then-createGraphileInstance-through-schema-readiness', + buildTransient: 'approximate-five-millisecond-event-loop-sampling-plus-process-rss-high-water', + operationWarmth: warmth.warmOperationsPerInstance > 0 + ? 'distinct-named-source-queries-through-grafast' + : 'disabled', + operationReplay: warmth.warmOperationReplayPasses > 0 + ? 'exact-population-source-set-replayed-through-grafast' + : 'disabled', + postgres: results.every( + (result) => ( + result.postgresBackendMeasurement.introspectionBackendMemory + .sampledLowerBoundMeasured + || result.postgresBackendMeasurement.introspectionBackendMemory + .sharedSnapshotMeasured + ) + ) + ? introspectionClientReleaseMode === 'destroy' + ? 'dedicated-identity-bound-procfs-diagnostic-lower-bound-before-retirement' + : 'shared-introspection-backend-post-build-procfs-baseline-relative' + : 'not-measured-no-replacement-backend-substitution', + postgresBackendSamplerAuthority: 'diagnostic-only', + postgresDensityMemoryAuthority: + `separately-validated-${LINUX_CGROUP_V2_DENSITY_AUTHORITY}`, + catalogWarmth: 'shared-server-not-reset', + historicalMethodologyComparable: false + }, + provenance: { + commit: provenance.commit, + worktreeDirty: provenance.worktreeDirty, + sourceStateSha256: provenance.sourceStateSha256, + lockfileSha256, + executedEntrySha256, + v8Profile, + nodeOptions, + nodeOptionsArgv, + nodeExecArgv, + effectiveNodeRuntimeFlags, + benchmarkConfiguration: { + scopedCatalogTypes, + introspectionClientReleaseMode, + postgresBackendSamplerMode, + releaseBuildStateAfterValidation, + ...(schemaLayout.schemaSets === null + ? {} + : { + schemaSets: schemaLayout.schemaSets, + allowedDependencySchemas: schemaLayout.allowedDependencySchemas + }), + warmOperationsPerInstance: warmth.warmOperationsPerInstance, + warmOperationReplayPasses: warmth.warmOperationReplayPasses, + grafastCacheLimits: warmth.grafastCacheLimits, + tenantProxySurfaces, + v8Profile + } + }, + catalog: results[0].catalog, + fixtureFingerprint: fixtureFingerprints.length === 1 ? fixtureFingerprints[0] : null, + fixtureFingerprintConsistent: fixtureFingerprints.length === 1, + freshProcessFirstBuildReadyMs: results.map((result) => result.builds[0].buildMs), + medianFreshProcessFirstBuildReadyMs: median( + results.map((result) => result.builds[0].buildMs) + ), + freshProcessFirstQueryMs: results.map((result) => result.builds[0].queryMs), + medianFreshProcessFirstQueryMs: median(results.map((result) => result.builds[0].queryMs)), + finalForcedGcHeapDeltaBytes: finalSnapshots.map((snapshot) => snapshot.heapDeltaBytes), + medianFinalForcedGcHeapDeltaBytes: median( + finalSnapshots.map((snapshot) => snapshot.heapDeltaBytes) + ), + finalRssDeltaBytes: finalSnapshots.map((snapshot) => snapshot.rssDeltaBytes), + medianFinalRssDeltaBytes: median(finalSnapshots.map((snapshot) => snapshot.rssDeltaBytes)), + medianHeapSlopeBytesPerInstance: median( + results.map((result) => result.heapSlopeBytesPerInstance) + ), + medianRssSlopeBytesPerInstance: median( + results.map((result) => result.rssSlopeBytesPerInstance) + ), + peakProcessRssBytes: Math.max(...results.flatMap((result) => + result.snapshots.map((snapshot) => snapshot.processPeakRssBytes) + )), + medianPeakProcessRssDeltaBytes: median(results.map((result) => + Math.max(...result.snapshots.map((snapshot) => snapshot.processPeakRssDeltaBytes)) + )), + peakProcessRssDeltaBytes: Math.max(...results.flatMap((result) => + result.snapshots.map((snapshot) => snapshot.processPeakRssDeltaBytes) + )), + postgresMemoryMeasured: postgresSteadyRssByRepetition.every( + (value) => value !== null + ), + postgresSteadyBackendRssBytes: postgresSteadyRssByRepetition, + medianPostgresSteadyBackendRssBytes: medianOrNull( + postgresSteadyRssByRepetition + ), + peakPostgresSteadyBackendRssBytes: maxOrNull( + postgresSteadyRssByRepetition + ), + postgresSteadyBackendRssDeltaBytes: + postgresSteadyRssDeltaByRepetition, + medianPostgresSteadyBackendRssDeltaBytes: medianOrNull( + postgresSteadyRssDeltaByRepetition + ), + postgresIntrospectionBackendSampledLowerBoundMeasured: + postgresSampledHighWaterLowerBoundByRepetition.every( + (value) => value !== null + ), + postgresIntrospectionBackendSampledLowerBoundSemantics: + 'diagnostic-lower-bound-without-pre-destroy-acknowledgement', + postgresIntrospectionBackendSampledLowerBoundLimitation: [...new Set( + results.flatMap((result) => { + const limitation = + result.postgresBackendMeasurement.introspectionBackendMemory.limitation; + return limitation ? [limitation] : []; + }) + )].join(' ') || null, + postgresBackendLifecycle: results.map( + (result) => result.postgresBackendMeasurement + ), + postgresIntrospectionBackendSampledHighWaterLowerBoundBytes: + postgresSampledHighWaterLowerBoundByRepetition, + peakPostgresIntrospectionBackendSampledHighWaterLowerBoundBytes: maxOrNull( + postgresSampledHighWaterLowerBoundByRepetition + ), + postgresIntrospectionBackendSampledHighWaterDeltaLowerBoundBytes: + postgresSampledHighWaterDeltaLowerBoundByRepetition, + medianPostgresIntrospectionBackendSampledHighWaterDeltaLowerBoundBytes: + medianOrNull(postgresSampledHighWaterDeltaLowerBoundByRepetition), + peakPostgresIntrospectionBackendSampledHighWaterDeltaLowerBoundBytes: + maxOrNull(postgresSampledHighWaterDeltaLowerBoundByRepetition), + postgresSharedBackendSnapshotHighWaterBytes: + postgresSharedBackendSnapshotHighWaterByRepetition, + peakPostgresSharedBackendSnapshotHighWaterBytes: maxOrNull( + postgresSharedBackendSnapshotHighWaterByRepetition + ), + allSdlHashesEqualWithinArm: results.every( + (result) => result.allSdlHashesEqualWithinArm + ), + tokenCanariesConclusive: results.every((result) => result.tokenCanariesConclusive), + tokenCanariesPassed: results.every((result) => result.tokenCanariesPassed), + tokenMismatchViolations: results.reduce( + (sum, result) => sum + result.tokenMismatchViolations, + 0 + ), + crossTenantTokenViolations: results.reduce( + (sum, result) => sum + result.crossTenantTokenViolations, + 0 + ), + bleedViolations: results.reduce((sum, result) => sum + result.bleedViolations, 0), + resultFiles: results.map((_, index) => `rep-${index + 1}/result.json`) + }; + fs.writeFileSync( + path.join(outputRoot, 'summary.json'), + `${JSON.stringify(summary, null, 2)}\n`, + 'utf8' + ); + process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); +}; diff --git a/packages/perf-harness/src/config.ts b/packages/perf-harness/src/config.ts new file mode 100644 index 0000000000..6a810b39e9 --- /dev/null +++ b/packages/perf-harness/src/config.ts @@ -0,0 +1,1207 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import type { + AcceptanceGates, + ArmPlan, + DensityPlanV1, + FleetV1, + TenantTarget, + WorkloadPlan +} from './types'; + +const RESERVED_PORTS = new Set([3000, 3001, 3002, 5432, 9000]); +const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]']); + +export const DEFAULT_RUN_ORDER_SEED = 'graphile-density-v1'; +export const DEFAULT_SOAK_ARM = 'scoped-introspection'; + +export const soakArmName = ( + plan: Pick +): string => plan.soak?.arm ?? DEFAULT_SOAK_ARM; + +export const hasExactHostileValidationEvidence = ( + plan: Pick +): boolean => { + const evidence = plan.qualification?.hostileValidationEvidence; + if (!evidence) return false; + const expectedArms = plan.arms.map((arm) => arm.name).sort(); + if (JSON.stringify(Object.keys(evidence).sort()) !== JSON.stringify(expectedArms)) { + return false; + } + return expectedArms.every((arm) => { + const binding = evidence[arm]; + return binding?.version === 1 + && binding.kind === 'exact-runtime-hostile-validation-v1' + && typeof binding.artifactFile === 'string' + && binding.artifactFile.length > 0 + && /^[a-f0-9]{64}$/.test(binding.artifactSha256) + && /^sha256:[a-f0-9]{64}$/.test(binding.runtimeArtifactFingerprint) + && /^sha256:[a-f0-9]{64}$/.test(binding.configurationFingerprint); + }); +}; + +const requirePositive = (value: unknown, label: string): void => { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) { + throw new Error(`${label} must be positive`); + } +}; + +const requireNonNegative = (value: unknown, label: string): void => { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new Error(`${label} must be non-negative`); + } +}; + +const requireBoolean = (value: unknown, label: string): void => { + if (typeof value !== 'boolean') throw new Error(`${label} must be boolean`); +}; + +export const validateAcceptanceGates = (gates: AcceptanceGates): void => { + if (!gates || typeof gates !== 'object' || Array.isArray(gates)) { + throw new Error('plan.gates is missing'); + } + requireNonNegative(gates.maxErrorRate, 'plan.gates.maxErrorRate'); + if (gates.maxErrorRate > 1) throw new Error('plan.gates.maxErrorRate must be at most 1'); + requirePositive(gates.maxP99Ms, 'plan.gates.maxP99Ms'); + requireNonNegative( + gates.maxPostWarmupHeapGrowthMiBPerHour, + 'plan.gates.maxPostWarmupHeapGrowthMiBPerHour' + ); + requireNonNegative( + gates.minMedianDensityImprovement, + 'plan.gates.minMedianDensityImprovement' + ); + requireNonNegative( + gates.minAdditionalTenantsEveryRun, + 'plan.gates.minAdditionalTenantsEveryRun' + ); + if (!Number.isSafeInteger(gates.minAdditionalTenantsEveryRun)) { + throw new Error('plan.gates.minAdditionalTenantsEveryRun must be a safe integer'); + } + if (gates.maxAlignedMemorySampleGapMs != null) { + requirePositive( + gates.maxAlignedMemorySampleGapMs, + 'plan.gates.maxAlignedMemorySampleGapMs' + ); + } + if (gates.minAlignedMemoryCoverageRatio != null) { + requirePositive( + gates.minAlignedMemoryCoverageRatio, + 'plan.gates.minAlignedMemoryCoverageRatio' + ); + if (gates.minAlignedMemoryCoverageRatio > 1) { + throw new Error('plan.gates.minAlignedMemoryCoverageRatio must be at most 1'); + } + } + const booleans: Array = [ + 'requireZeroBleed', + 'requireNoPostWarmupEvictions', + 'requireNoPostWarmupBuildRefusals', + 'requireNoPostWarmupBuilds', + 'requirePostgresMemoryTelemetry', + 'requireFreshPostgresRunAttestation', + 'requireRetainedMemoryCheckpoints', + 'requirePhysicalDatabaseTelemetry', + 'requireConclusiveCanaries', + 'requireCompletePeriodicCanaryCoverage', + 'requireConclusiveOperationOracles', + 'requireExplicitCustomerTopology' + ]; + for (const key of booleans) requireBoolean(gates[key], `plan.gates.${key}`); + if ( + gates.requiredCacheAdmissionMode !== null + && gates.requiredCacheAdmissionMode !== 'evict-idle' + && gates.requiredCacheAdmissionMode !== 'preserve-resident' + ) { + throw new Error( + 'plan.gates.requiredCacheAdmissionMode must be null, evict-idle, or preserve-resident' + ); + } +}; + +const SHA256 = /^[a-f0-9]{64}$/i; +const NODE_V8_PROFILES = new Set([ + 'stock', + 'optimize-for-size', + 'baseline-optimize-for-size', + 'jitless-optimize-for-size' +]); +const MANAGED_V8_FLAG = + /^--(?:no[-_])?(?:jitless|optimize[-_]for[-_]size|max[-_]opt)(?:=.*)?$/; + +const fileSha256 = (value: Buffer): string => createHash('sha256').update(value).digest('hex'); + +const validateCountRamp: ( + counts: unknown, + label: string +) => asserts counts is number[] = (counts, label) => { + if (!Array.isArray(counts) || counts.length === 0) { + throw new Error(`${label} must be a nonempty array`); + } + let previous = 0; + for (const count of counts) { + requirePositive(count, label); + if (!Number.isSafeInteger(count)) throw new Error(`${label} must contain safe integers`); + if (count <= previous) throw new Error(`${label} must be strictly increasing`); + previous = count; + } +}; + +export const tenantCountsForHeap = ( + plan: Pick, + heapMiB: number +): number[] => { + const specific = plan.tenantCountsByHeapMiB?.[String(heapMiB)]; + const counts = specific ?? plan.tenantCounts; + if (!counts?.length) { + throw new Error(`no tenant-count ramp is configured for heapMiB=${heapMiB}`); + } + return [...counts]; +}; + +export const armEnvironmentForHeap = ( + arm: Pick, + heapMiB: number +): Record => ({ + ...(arm.env ?? {}), + ...(arm.envByHeapMiB?.[String(heapMiB)] ?? {}) +}); + +/** + * Resolve the arm-specific identities and route without copying credentials + * into a benchmark artifact. The runner and semantic evidence replay must use + * this same transformation or a result could be scored against a different + * build/pool contract from the one that was exercised. + */ +export const resolveTenants = ( + tenants: TenantTarget[], + arm: ArmPlan +): TenantTarget[] => tenants.map((tenant) => ({ + ...tenant, + databases: tenant.databases?.map((database) => ({ + ...database, + apis: database.apis.map((api) => ({ + ...api, + runtimePoolIdentity: + api.runtimePoolIdentities?.[arm.name] ?? api.runtimePoolIdentity + })) + })), + surfaces: tenant.surfaces.map((surface) => ({ + ...surface, + buildContract: surface.buildContracts?.[arm.name] ?? surface.buildContract, + url: resolveTemplate(surface.url, { + port: arm.port, + mode: arm.introspectionMode + }) + })) +})); + +export const validateWorkloadPlan = (workload: WorkloadPlan): void => { + if (!workload || typeof workload !== 'object') throw new Error('plan.workload is missing'); + requirePositive(workload.durationSec, 'workload.durationSec'); + const hasFixedRps = workload.rps != null; + const hasPerTenantRps = workload.rpsPerTenant != null; + if (hasFixedRps === hasPerTenantRps) { + throw new Error('workload must define exactly one of rps or rpsPerTenant'); + } + requirePositive( + hasFixedRps ? workload.rps : workload.rpsPerTenant, + hasFixedRps ? 'workload.rps' : 'workload.rpsPerTenant' + ); + requirePositive( + workload.minWorkloadRequestsPerSurface, + 'workload.minWorkloadRequestsPerSurface' + ); + if (!Number.isSafeInteger(workload.minWorkloadRequestsPerSurface)) { + throw new Error('workload.minWorkloadRequestsPerSurface must be a safe integer'); + } + requirePositive(workload.maxInFlight, 'workload.maxInFlight'); + if (!Number.isSafeInteger(workload.maxInFlight)) { + throw new Error('workload.maxInFlight must be a safe integer'); + } + requirePositive(workload.canaryIntervalSec, 'workload.canaryIntervalSec'); + if ( + workload.periodicCanarySchedule != null + && workload.periodicCanarySchedule !== 'full-sweep' + && workload.periodicCanarySchedule !== 'rotating-one' + ) { + throw new Error( + "workload.periodicCanarySchedule must be 'full-sweep' or 'rotating-one'" + ); + } + if (workload.canaryConcurrency != null) { + requirePositive(workload.canaryConcurrency, 'workload.canaryConcurrency'); + if (!Number.isSafeInteger(workload.canaryConcurrency)) { + throw new Error('workload.canaryConcurrency must be a safe integer'); + } + } + requirePositive(workload.requestTimeoutMs, 'workload.requestTimeoutMs'); + requirePositive(workload.warmupTimeoutMs, 'workload.warmupTimeoutMs'); + requirePositive( + workload.warmupTimeoutPerSurfaceMs, + 'workload.warmupTimeoutPerSurfaceMs' + ); + if (workload.warmupConcurrency != null) { + requirePositive(workload.warmupConcurrency, 'workload.warmupConcurrency'); + if (!Number.isSafeInteger(workload.warmupConcurrency)) { + throw new Error('workload.warmupConcurrency must be a safe integer'); + } + } +}; + +const assertJsonPathMatches = (value: unknown, label: string): void => { + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${label} must contain at least one typed JSON-path match`); + } + for (const [index, match] of value.entries()) { + if (!match || typeof match !== 'object') { + throw new Error(`${label}[${index}] must be an object`); + } + const record = match as Record; + if (typeof record.path !== 'string' || (record.path !== '' && !record.path.startsWith('/'))) { + throw new Error(`${label}[${index}].path must be an RFC 6901 JSON pointer`); + } + if (!Object.prototype.hasOwnProperty.call(record, 'value')) { + throw new Error(`${label}[${index}] must define value`); + } + } +}; + +const assertJsonPathInvariants = (value: unknown, label: string): void => { + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${label} must contain at least one JSON-path invariant`); + } + for (const [index, invariant] of value.entries()) { + if (!invariant || typeof invariant !== 'object') { + throw new Error(`${label}[${index}] must be an object`); + } + const record = invariant as Record; + if (typeof record.path !== 'string' || (record.path !== '' && !record.path.startsWith('/'))) { + throw new Error(`${label}[${index}].path must be an RFC 6901 JSON pointer`); + } + if (!Object.prototype.hasOwnProperty.call(record, 'everyEquals')) { + throw new Error(`${label}[${index}] must define everyEquals`); + } + if (!Number.isSafeInteger(record.min) || (record.min as number) <= 0) { + throw new Error(`${label}[${index}].min must be a positive safe integer`); + } + if ( + record.max != null + && ( + !Number.isSafeInteger(record.max) + || (record.max as number) < (record.min as number) + ) + ) { + throw new Error(`${label}[${index}].max must be a safe integer at least min`); + } + } +}; + +const GRAPHQL_VARIABLE_NAME = /^[_A-Za-z][_0-9A-Za-z]*$/; + +const assertResponseVariableBindings = ( + value: unknown, + staticVariables: unknown, + label: string +): void => { + if (value == null) return; + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + const entries = Object.entries(value as Record); + if (entries.length === 0) throw new Error(`${label} must not be empty`); + const configuredStaticVariables = staticVariables && typeof staticVariables === 'object' + && !Array.isArray(staticVariables) + ? staticVariables as Record + : {}; + for (const [name, pointer] of entries) { + if (!GRAPHQL_VARIABLE_NAME.test(name)) { + throw new Error(`${label} has invalid GraphQL variable '${name}'`); + } + if (Object.prototype.hasOwnProperty.call(configuredStaticVariables, name)) { + throw new Error(`${label}.${name} collides with a static variable`); + } + if (typeof pointer !== 'string' || (pointer !== '' && !pointer.startsWith('/'))) { + throw new Error(`${label}.${name} must be an RFC 6901 JSON pointer`); + } + } +}; + +const assertOptionalOperationOracle = ( + operation: TenantTarget['surfaces'][number]['warmup'], + label: string +): void => { + const hasRequired = operation.requiredMatches != null; + const hasForbidden = operation.forbiddenMatches != null; + if (hasRequired !== hasForbidden) { + throw new Error(`${label} must configure requiredMatches and forbiddenMatches together`); + } + if (hasRequired) { + assertJsonPathMatches(operation.requiredMatches, `${label}.requiredMatches`); + assertJsonPathMatches(operation.forbiddenMatches, `${label}.forbiddenMatches`); + } + if (operation.invariants != null) { + assertJsonPathInvariants(operation.invariants, `${label}.invariants`); + } + const verification = operation.postCoverageVerification; + if (verification != null) { + if (typeof verification.query !== 'string' || !verification.query.trim()) { + throw new Error(`${label}.postCoverageVerification has no query`); + } + assertJsonPathMatches( + verification.requiredMatches, + `${label}.postCoverageVerification.requiredMatches` + ); + assertJsonPathMatches( + verification.forbiddenMatches, + `${label}.postCoverageVerification.forbiddenMatches` + ); + if (verification.invariants != null) { + assertJsonPathInvariants( + verification.invariants, + `${label}.postCoverageVerification.invariants` + ); + } + assertResponseVariableBindings( + verification.variablesFromResponse, + verification.variables, + `${label}.postCoverageVerification.variablesFromResponse` + ); + } +}; + +const hasConclusiveOperationOracle = ( + operation: TenantTarget['surfaces'][number]['warmup'] +): boolean => Boolean( + (operation.requiredMatches?.length && operation.forbiddenMatches?.length) + || ( + operation.postCoverageVerification?.requiredMatches.length + && operation.postCoverageVerification.forbiddenMatches.length + ) +); + +const ENVIRONMENT_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; +const isExactJsonPointer = (value: unknown): value is string => + typeof value === 'string' + && value.startsWith('/') + && value.split('/').slice(1).every((segment) => + segment !== '*' && !/~(?:[^01]|$)/.test(segment) + ); +const FORBIDDEN_DRIVER_HEADERS = new Set([ + 'connection', + 'content-length', + 'host', + 'sec-websocket-accept', + 'sec-websocket-extensions', + 'sec-websocket-key', + 'sec-websocket-protocol', + 'sec-websocket-version', + 'transfer-encoding', + 'upgrade' +]); +const SENSITIVE_HEADERS = new Set(['authorization', 'cookie', 'proxy-authorization']); + +const assertRealtimeProbe = ( + surface: TenantTarget['surfaces'][number], + label: string +): void => { + if (!surface.realtime || typeof surface.realtime !== 'object') { + throw new Error(`${label} has no realtime probe`); + } + for (const operationName of ['subscription', 'prime'] as const) { + const operation = surface.realtime[operationName]; + if (!operation || typeof operation.query !== 'string' || !operation.query.trim()) { + throw new Error(`${label}.realtime.${operationName} has no query`); + } + assertJsonPathMatches( + operation.requiredMatches, + `${label}.realtime.${operationName}.requiredMatches` + ); + assertJsonPathMatches( + operation.forbiddenMatches, + `${label}.realtime.${operationName}.forbiddenMatches` + ); + } + const correlation = surface.realtime.correlation; + if ( + !correlation + || typeof correlation !== 'object' + || !/^[_A-Za-z][_0-9A-Za-z]*$/.test(correlation.primeVariable ?? '') + || !isExactJsonPointer(correlation.primeResponsePath) + || !isExactJsonPointer(correlation.subscriptionEventPath) + || !Object.prototype.hasOwnProperty.call( + surface.realtime.prime.variables ?? {}, + correlation.primeVariable + ) + ) { + throw new Error(`${label}.realtime.correlation is invalid`); + } + if ( + surface.realtime.prime.requiredMatches.some( + (match) => match.path === correlation.primeResponsePath + ) + || surface.realtime.subscription.requiredMatches.some( + (match) => match.path === correlation.subscriptionEventPath + ) + ) { + throw new Error( + `${label}.realtime.correlation paths must not carry a static required match` + ); + } + const inlineHeaders = new Set(); + for (const [name, value] of Object.entries(surface.headers ?? {})) { + const normalized = name.trim().toLowerCase(); + if (!normalized || typeof value !== 'string') { + throw new Error(`${label}.headers must contain nonempty string values`); + } + if (FORBIDDEN_DRIVER_HEADERS.has(normalized)) { + throw new Error(`${label}.headers cannot override '${normalized}'`); + } + if (SENSITIVE_HEADERS.has(normalized)) { + throw new Error( + `${label}.${normalized} must use realtime.headersFromEnvironment` + ); + } + if (inlineHeaders.has(normalized)) { + throw new Error(`${label}.headers contains duplicate '${normalized}'`); + } + inlineHeaders.add(normalized); + } + const environmentHeaders = new Set(); + const mappings = surface.realtime.headersFromEnvironment ?? {}; + if (!mappings || typeof mappings !== 'object' || Array.isArray(mappings)) { + throw new Error(`${label}.realtime.headersFromEnvironment must be an object`); + } + for (const [name, environmentName] of Object.entries(mappings)) { + const normalized = name.trim().toLowerCase(); + if ( + !normalized + || FORBIDDEN_DRIVER_HEADERS.has(normalized) + || typeof environmentName !== 'string' + || !ENVIRONMENT_NAME.test(environmentName) + ) { + throw new Error(`${label}.realtime.headersFromEnvironment is invalid`); + } + if (inlineHeaders.has(normalized) || environmentHeaders.has(normalized)) { + throw new Error(`${label} configures header '${normalized}' more than once`); + } + environmentHeaders.add(normalized); + } +}; + +export const assertIsolatedPort = (port: number, allowReserved = false): void => { + requirePositive(port, 'arm.port'); + if (!allowReserved && RESERVED_PORTS.has(port)) { + throw new Error(`refusing reserved shared-workspace port ${port}`); + } +}; + +export const assertLoopbackObservabilityUrl = (value: string, expectedPort: number): void => { + let url: URL; + try { + url = new URL(value); + } catch { + throw new Error(`memoryUrl is not a valid URL: ${value}`); + } + const valid = url.protocol === 'http:' + && LOOPBACK_HOSTS.has(url.hostname.toLowerCase()) + && Number(url.port) === expectedPort + && !url.username + && !url.password + && url.pathname === '/debug/memory' + && !url.search + && !url.hash; + if (!valid) { + throw new Error( + `memoryUrl must be the credential-free URL http://127.0.0.1:${expectedPort}/debug/memory ` + + '(localhost and ::1 are also accepted)' + ); + } +}; + +export const assertLoopbackRetainedHeapCheckpointUrl = ( + value: string, + expectedPort: number +): void => { + let url: URL; + try { + url = new URL(value); + } catch { + throw new Error(`retainedHeapCheckpointUrl is not a valid URL: ${value}`); + } + const valid = url.protocol === 'http:' + && LOOPBACK_HOSTS.has(url.hostname.toLowerCase()) + && Number(url.port) === expectedPort + && !url.username + && !url.password + && url.pathname === '/__cperf/retained-memory-checkpoint' + && !url.search + && !url.hash; + if (!valid) { + throw new Error( + 'retainedHeapCheckpointUrl must be the credential-free URL ' + + `http://127.0.0.1:${expectedPort}/__cperf/retained-memory-checkpoint ` + + '(localhost and ::1 are also accepted)' + ); + } +}; + +export const loadPlan = (file: string, allowReserved = false): DensityPlanV1 => { + const planPath = path.resolve(file); + const planBytes = fs.readFileSync(planPath); + const plan = JSON.parse(planBytes.toString('utf8')) as DensityPlanV1; + plan.sourceSha256 = fileSha256(planBytes); + if (plan.version !== 1) throw new Error('density plan version must be 1'); + if (!Array.isArray(plan.arms) || plan.arms.length === 0) throw new Error('plan.arms is empty'); + if (!Array.isArray(plan.heapMiB) || plan.heapMiB.length === 0) throw new Error('plan.heapMiB is empty'); + requirePositive(plan.repetitions, 'plan.repetitions'); + if (!Number.isInteger(plan.repetitions)) throw new Error('plan.repetitions must be an integer'); + validateWorkloadPlan(plan.workload); + validateAcceptanceGates(plan.gates); + if (!Array.isArray(plan.requiredCapabilities) || plan.requiredCapabilities.length === 0) { + throw new Error('plan.requiredCapabilities is empty'); + } + if (!Array.isArray(plan.requiredCanaries) || plan.requiredCanaries.length === 0) { + throw new Error('plan.requiredCanaries is empty'); + } + if (new Set(plan.heapMiB).size !== plan.heapMiB.length) { + throw new Error('heapMiB must not contain duplicates'); + } + for (const heap of plan.heapMiB) { + requirePositive(heap, 'heapMiB'); + if (!Number.isInteger(heap)) throw new Error('heapMiB must contain integers'); + validateCountRamp(tenantCountsForHeap(plan, heap), `tenant counts for heapMiB=${heap}`); + } + if (plan.qualification != null) { + if ( + typeof plan.qualification !== 'object' + || Array.isArray(plan.qualification) + || typeof plan.qualification.baselineArm !== 'string' + || !plan.qualification.baselineArm + ) { + throw new Error('plan.qualification.baselineArm must be a nonempty string'); + } + if (!plan.arms.some((arm) => arm.name === plan.qualification!.baselineArm)) { + throw new Error( + `plan.qualification.baselineArm '${plan.qualification.baselineArm}' is not configured` + ); + } + validateCountRamp( + plan.qualification.requiredHeapMiB, + 'plan.qualification.requiredHeapMiB' + ); + for (const heap of plan.qualification.requiredHeapMiB) { + if (!plan.heapMiB.includes(heap)) { + throw new Error(`qualification heap ${heap}MiB is not configured in plan.heapMiB`); + } + } + requirePositive( + plan.qualification.minimumRepetitions, + 'plan.qualification.minimumRepetitions' + ); + if (!Number.isSafeInteger(plan.qualification.minimumRepetitions)) { + throw new Error('plan.qualification.minimumRepetitions must be a safe integer'); + } + if (plan.repetitions < plan.qualification.minimumRepetitions) { + throw new Error( + `plan.repetitions=${plan.repetitions} is below qualification minimum=${plan.qualification.minimumRepetitions}` + ); + } + const hostileEvidence = plan.qualification.hostileValidationEvidence; + if (hostileEvidence != null) { + if (!hasExactHostileValidationEvidence(plan)) { + throw new Error( + 'plan.qualification.hostileValidationEvidence must bind every exact arm' + ); + } + for (const arm of plan.arms) { + const binding = hostileEvidence[arm.name]; + const artifactFile = path.resolve(path.dirname(planPath), binding.artifactFile); + const stat = fs.lstatSync(artifactFile); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new Error(`hostile validation artifact for '${arm.name}' is not a regular file`); + } + const bytes = fs.readFileSync(artifactFile); + if (fileSha256(bytes) !== binding.artifactSha256) { + throw new Error(`hostile validation artifact for '${arm.name}' has the wrong SHA-256`); + } + const report = JSON.parse(bytes.toString('utf8')) as Record; + if ( + report.version !== 1 + || report.kind !== binding.kind + || report.passed !== true + || report.arm !== arm.name + || report.runtimeArtifactFingerprint !== binding.runtimeArtifactFingerprint + || report.configurationFingerprint !== binding.configurationFingerprint + ) { + throw new Error( + `hostile validation artifact for '${arm.name}' does not bind its exact runtime/config` + ); + } + binding.artifactFile = artifactFile; + } + } + } + if (plan.soak != null) { + if (typeof plan.soak !== 'object' || Array.isArray(plan.soak)) { + throw new Error('plan.soak must be an object'); + } + requireBoolean(plan.soak.enabled, 'plan.soak.enabled'); + if (plan.soak.enabled) { + requirePositive(plan.soak.durationSec, 'plan.soak.durationSec'); + requirePositive(plan.soak.tenantCount, 'plan.soak.tenantCount'); + requirePositive(plan.soak.heapMiB, 'plan.soak.heapMiB'); + for (const [value, label] of [ + [plan.soak.durationSec, 'plan.soak.durationSec'], + [plan.soak.tenantCount, 'plan.soak.tenantCount'], + [plan.soak.heapMiB, 'plan.soak.heapMiB'] + ] as const) { + if (!Number.isSafeInteger(value)) throw new Error(`${label} must be a safe integer`); + } + if (!plan.heapMiB.includes(plan.soak.heapMiB)) { + throw new Error(`plan.soak.heapMiB=${plan.soak.heapMiB} is not configured`); + } + if (plan.soak.arm != null && ( + typeof plan.soak.arm !== 'string' || !plan.soak.arm.trim() + )) { + throw new Error('plan.soak.arm must be a nonempty string'); + } + const armName = soakArmName(plan); + if (!plan.arms.some((arm) => arm.name === armName)) { + throw new Error(`plan.soak.arm '${armName}' is not configured`); + } + } + } + if (plan.tenantCountsByHeapMiB != null) { + if ( + typeof plan.tenantCountsByHeapMiB !== 'object' + || Array.isArray(plan.tenantCountsByHeapMiB) + ) { + throw new Error('tenantCountsByHeapMiB must be an object'); + } + const configuredHeaps = new Set(plan.heapMiB.map(String)); + for (const [heap, counts] of Object.entries(plan.tenantCountsByHeapMiB)) { + if (!configuredHeaps.has(heap)) { + throw new Error(`tenantCountsByHeapMiB contains unconfigured heap '${heap}'`); + } + validateCountRamp(counts, `tenantCountsByHeapMiB.${heap}`); + } + } + plan.runOrderSeed ??= DEFAULT_RUN_ORDER_SEED; + if (!plan.runOrderSeed.trim()) throw new Error('runOrderSeed must not be empty'); + const armNames = new Set(); + const armPorts = new Set(); + for (const arm of plan.arms) { + if (!arm.name || armNames.has(arm.name)) throw new Error(`duplicate or empty arm name '${arm.name}'`); + if (armPorts.has(arm.port)) throw new Error(`duplicate arm port ${arm.port}`); + armNames.add(arm.name); + armPorts.add(arm.port); + arm.v8Profile ??= 'stock'; + if (!NODE_V8_PROFILES.has(arm.v8Profile)) { + throw new Error(`arm '${arm.name}' has unknown v8Profile '${arm.v8Profile}'`); + } + if (arm.command?.some((argument) => MANAGED_V8_FLAG.test(argument))) { + throw new Error( + `arm '${arm.name}' must configure managed V8 flags through v8Profile` + ); + } + for (const [heap, environment] of [ + ['default', arm.env], + ...Object.entries(arm.envByHeapMiB ?? {}) + ] as Array<[string, Record | undefined]>) { + const nodeOptions = environment?.NODE_OPTIONS?.split(/\s+/) ?? []; + if (nodeOptions.some((argument) => MANAGED_V8_FLAG.test(argument))) { + throw new Error( + `arm '${arm.name}' NODE_OPTIONS for ${heap} must configure managed V8 flags through v8Profile` + ); + } + } + if ( + arm.v8Profile !== 'stock' + && arm.command?.length + && !['node', 'node.exe'].includes(path.basename(arm.command[0]).toLowerCase()) + ) { + throw new Error(`arm '${arm.name}' non-stock v8Profile requires a Node command`); + } + assertIsolatedPort(arm.port, allowReserved); + if (!Array.isArray(arm.command) && !arm.readinessUrl) { + throw new Error(`arm '${arm.name}' needs command or readinessUrl`); + } + if (arm.command?.length && !arm.commit) { + throw new Error(`arm '${arm.name}' must pin commit for a spawned run`); + } + if (arm.command?.length && plan.gates.requireRetainedMemoryCheckpoints) { + if (!arm.command.includes('--expose-gc')) { + throw new Error(`arm '${arm.name}' must launch Node with --expose-gc`); + } + if (!arm.retainedHeapCheckpointUrl) { + throw new Error(`arm '${arm.name}' needs retainedHeapCheckpointUrl`); + } + const checkpointUrl = resolveTemplate(arm.retainedHeapCheckpointUrl!, { + heapMiB: plan.heapMiB[0], + port: arm.port, + artifactDir: plan.artifactDir, + mode: arm.introspectionMode, + tenantCount: tenantCountsForHeap(plan, plan.heapMiB[0])[0] + }); + assertLoopbackRetainedHeapCheckpointUrl(checkpointUrl, arm.port); + for (const heapMiB of plan.heapMiB) { + if ( + armEnvironmentForHeap(arm, heapMiB) + .GRAPHQL_CPERF_RETAINED_HEAP_ENABLED !== 'true' + ) { + throw new Error( + `arm '${arm.name}' must set GRAPHQL_CPERF_RETAINED_HEAP_ENABLED=true for heap ${heapMiB}` + ); + } + } + } + if (arm.entrySha256 && !SHA256.test(arm.entrySha256)) { + throw new Error(`arm '${arm.name}' entrySha256 must be a SHA-256 hex digest`); + } + if (arm.lockfileSha256 && !SHA256.test(arm.lockfileSha256)) { + throw new Error(`arm '${arm.name}' lockfileSha256 must be a SHA-256 hex digest`); + } + if (arm.envByHeapMiB != null) { + if (typeof arm.envByHeapMiB !== 'object' || Array.isArray(arm.envByHeapMiB)) { + throw new Error(`arm '${arm.name}' envByHeapMiB must be an object`); + } + const configuredHeaps = new Set(plan.heapMiB.map(String)); + for (const heap of configuredHeaps) { + if (!Object.prototype.hasOwnProperty.call(arm.envByHeapMiB, heap)) { + throw new Error(`arm '${arm.name}' envByHeapMiB is missing heap '${heap}'`); + } + } + for (const [heap, environment] of Object.entries(arm.envByHeapMiB)) { + if (!configuredHeaps.has(heap)) { + throw new Error(`arm '${arm.name}' envByHeapMiB contains unconfigured heap '${heap}'`); + } + if ( + !environment + || typeof environment !== 'object' + || Array.isArray(environment) + || Object.entries(environment).some(([key, value]) => + !key || typeof value !== 'string' + ) + ) { + throw new Error(`arm '${arm.name}' envByHeapMiB.${heap} must contain string values`); + } + } + } + if (plan.gates.requirePostgresMemoryTelemetry && !arm.postgresContainer) { + throw new Error(`arm '${arm.name}' needs postgresContainer for required PostgreSQL telemetry`); + } + if (plan.gates.requireFreshPostgresRunAttestation) { + const attestation = arm.postgresRunAttestation; + if (!attestation || !Array.isArray(attestation.command) || attestation.command.length === 0) { + throw new Error( + `arm '${arm.name}' needs postgresRunAttestation.command for fresh PostgreSQL evidence` + ); + } + if ( + attestation.command.some((part) => typeof part !== 'string' || !part) + || !Array.isArray(attestation.prepareCommand) + || attestation.prepareCommand.length === 0 + || attestation.prepareCommand.some((part) => typeof part !== 'string' || !part) + || ( + attestation.timeoutMs != null + && ( + !Number.isSafeInteger(attestation.timeoutMs) + || attestation.timeoutMs <= 0 + ) + ) + ) { + throw new Error(`arm '${arm.name}' has invalid postgresRunAttestation config`); + } + const requiredServerTemplates = [ + '{postgresManifestFile}', + '{postgresSecretsFile}', + '{postgresManifestSha256}', + '{postgresCloneId}' + ]; + const requiredPrepareTemplates = [ + '{postgresFixtureDir}', + '{arm}', + '{heapMiB}', + '{tenantCount}', + '{repetition}', + '{runOrderIndex}' + ]; + const requiredAuditTemplates = [ + '{postgresManifestFile}', + '{postgresSecretsFile}', + '{attestationFile}', + '{planSha256}', + '{fleetSha256}', + '{notBeforeEpochMs}' + ]; + if ( + !arm.command?.length + || requiredServerTemplates.some((template) => !arm.command!.includes(template)) + || requiredPrepareTemplates.some((template) => + !attestation.prepareCommand.includes(template) + ) + || requiredAuditTemplates.some((template) => + !attestation.command.includes(template) + ) + ) { + throw new Error( + `arm '${arm.name}' does not bind the fresh PostgreSQL fixture into its server command` + ); + } + } + } + plan.fleetFile = path.resolve(path.dirname(planPath), plan.fleetFile); + plan.artifactDir = path.resolve(path.dirname(planPath), plan.artifactDir); + return plan; +}; + +export const loadFleet = (file: string): FleetV1 => { + const fleetBytes = fs.readFileSync(path.resolve(file)); + const fleet = JSON.parse(fleetBytes.toString('utf8')) as FleetV1; + fleet.sourceSha256 = fileSha256(fleetBytes); + if (fleet.version !== 1) throw new Error('fleet version must be 1'); + if (!Array.isArray(fleet.tenants) || fleet.tenants.length === 0) { + throw new Error('fleet.tenants is empty'); + } + const tenantIds = new Set(); + for (const tenant of fleet.tenants) { + if (!tenant.id || tenantIds.has(tenant.id)) throw new Error(`duplicate or empty tenant id '${tenant.id}'`); + tenantIds.add(tenant.id); + if (!Array.isArray(tenant.surfaces) || tenant.surfaces.length === 0) { + throw new Error(`tenant '${tenant.id}' has no surfaces`); + } + const surfaceNames = new Set(); + for (const surface of tenant.surfaces) { + if (!surface.name || surfaceNames.has(surface.name)) { + throw new Error(`tenant '${tenant.id}' has duplicate or empty surface '${surface.name}'`); + } + surfaceNames.add(surface.name); + const armContracts = surface.buildContracts; + if (!surface.buildContract && !armContracts) { + throw new Error( + `tenant '${tenant.id}' surface '${surface.name}' has no buildContract or buildContracts` + ); + } + if (armContracts && ( + Object.keys(armContracts).length === 0 + || Object.values(armContracts).some((contract) => !contract) + )) { + throw new Error( + `tenant '${tenant.id}' surface '${surface.name}' has incomplete buildContracts` + ); + } + if (!surface.url || !surface.warmup || surface.operations.length === 0) { + throw new Error(`tenant '${tenant.id}' surface '${surface.name}' is incomplete`); + } + if (!surface.warmup.name || !surface.warmup.capability || !surface.warmup.query) { + throw new Error(`tenant '${tenant.id}' surface '${surface.name}' has an incomplete warmup`); + } + assertOptionalOperationOracle( + surface.warmup, + `tenant '${tenant.id}' surface '${surface.name}'.warmup` + ); + const operationNames = new Set(); + for (const operation of surface.operations) { + if (!operation.name || operationNames.has(operation.name)) { + throw new Error(`tenant '${tenant.id}' surface '${surface.name}' has duplicate or empty operation '${operation.name}'`); + } + operationNames.add(operation.name); + if (!operation.capability || !operation.query) { + throw new Error(`operation '${operation.name}' is incomplete`); + } + if (operation.weight != null && (!Number.isFinite(operation.weight) || operation.weight <= 0)) { + throw new Error(`operation '${operation.name}' weight must be positive`); + } + assertOptionalOperationOracle( + operation, + `tenant '${tenant.id}' surface '${surface.name}' operation '${operation.name}'` + ); + } + if (!Array.isArray(surface.canaries) || surface.canaries.length === 0) { + throw new Error(`tenant '${tenant.id}' surface '${surface.name}' has no isolation canaries`); + } + const canaryNames = new Set(); + for (const canary of surface.canaries) { + if (!canary.name || canaryNames.has(canary.name)) { + throw new Error(`tenant '${tenant.id}' surface '${surface.name}' has duplicate or empty canary '${canary.name}'`); + } + canaryNames.add(canary.name); + if (!canary.query) throw new Error(`canary '${canary.name}' has no query`); + assertJsonPathMatches( + canary.forbiddenMatches, + `canary '${canary.name}'.forbiddenMatches` + ); + assertJsonPathMatches( + canary.requiredMatches, + `canary '${canary.name}'.requiredMatches` + ); + if (canary.invariants != null) { + assertJsonPathInvariants( + canary.invariants, + `canary '${canary.name}'.invariants` + ); + } + } + if (surface.realtime) { + assertRealtimeProbe( + surface, + `tenant '${tenant.id}' surface '${surface.name}'` + ); + } + } + validateCustomerTopology(tenant); + } + return fleet; +}; + +export const validateCustomerTopology = (customer: TenantTarget): void => { + if (customer.databases == null) return; + if (!Array.isArray(customer.databases) || customer.databases.length === 0) { + throw new Error(`customer '${customer.id}' has an empty database topology`); + } + const configuredSurfaces = new Set(customer.surfaces.map((surface) => surface.name)); + const mappedSurfaces = new Set(); + const databaseIds = new Set(); + const apiIds = new Set(); + for (const database of customer.databases) { + if (!database.id || databaseIds.has(database.id)) { + throw new Error(`customer '${customer.id}' has duplicate or empty database id '${database.id}'`); + } + databaseIds.add(database.id); + if (!database.physicalDatabase?.trim()) { + throw new Error(`customer '${customer.id}' database '${database.id}' has no physical database`); + } + if (!Array.isArray(database.apis) || database.apis.length === 0) { + throw new Error(`customer '${customer.id}' database '${database.id}' has no APIs`); + } + for (const api of database.apis) { + if (!api.id || apiIds.has(api.id)) { + throw new Error(`customer '${customer.id}' has duplicate or empty API id '${api.id}'`); + } + apiIds.add(api.id); + if (!api.runtimePoolIdentity && !api.runtimePoolIdentities) { + throw new Error(`customer '${customer.id}' API '${api.id}' has no runtime pool identity`); + } + if (api.runtimePoolIdentities && ( + Object.keys(api.runtimePoolIdentities).length === 0 + || Object.values(api.runtimePoolIdentities).some((identity) => !identity) + )) { + throw new Error(`customer '${customer.id}' API '${api.id}' has incomplete runtime pool identities`); + } + if ( + !Array.isArray(api.physicalSchemas) + || api.physicalSchemas.length === 0 + || api.physicalSchemas.some((schema) => typeof schema !== 'string' || !schema) + || new Set(api.physicalSchemas).size !== api.physicalSchemas.length + ) { + throw new Error(`customer '${customer.id}' API '${api.id}' has invalid physical schemas`); + } + if ( + !Array.isArray(api.routingLabels) + || api.routingLabels.length === 0 + || api.routingLabels.some((label) => typeof label !== 'string' || !label) + || new Set(api.routingLabels).size !== api.routingLabels.length + ) { + throw new Error(`customer '${customer.id}' API '${api.id}' has invalid routing labels`); + } + if (typeof api.realtime !== 'boolean') { + throw new Error(`customer '${customer.id}' API '${api.id}' has no explicit realtime flag`); + } + if ( + !Array.isArray(api.surfaces) + || api.surfaces.length === 0 + || new Set(api.surfaces).size !== api.surfaces.length + ) { + throw new Error(`customer '${customer.id}' API '${api.id}' has invalid surfaces`); + } + for (const surface of api.surfaces) { + if (!configuredSurfaces.has(surface)) { + throw new Error(`customer '${customer.id}' API '${api.id}' maps unknown surface '${surface}'`); + } + if (mappedSurfaces.has(surface)) { + throw new Error(`customer '${customer.id}' maps surface '${surface}' more than once`); + } + const configuredSurface = customer.surfaces.find((candidate) => + candidate.name === surface + ); + if (api.realtime !== Boolean(configuredSurface?.realtime)) { + throw new Error( + `customer '${customer.id}' API '${api.id}' realtime topology disagrees with surface '${surface}'` + ); + } + mappedSurfaces.add(surface); + } + } + } + const missingSurfaces = [...configuredSurfaces].filter((surface) => !mappedSurfaces.has(surface)); + if (missingSurfaces.length > 0) { + throw new Error( + `customer '${customer.id}' topology omits surfaces: ${missingSurfaces.join(', ')}` + ); + } +}; + +export const validateCoverage = (plan: DensityPlanV1, fleet: FleetV1): void => { + const failures: string[] = []; + const databaseOwners = new Map(); + const apiOwners = new Map(); + const matrixCounts = plan.heapMiB?.flatMap((heap) => tenantCountsForHeap(plan, heap)) + ?? plan.tenantCounts + ?? []; + const maxTenantCount = Math.max(0, ...matrixCounts, plan.soak?.tenantCount ?? 0); + if (fleet.tenants.length < maxTenantCount) { + failures.push(`fleet has ${fleet.tenants.length} tenants but the matrix requests ${maxTenantCount}`); + } + if ( + plan.gates?.requireCompletePeriodicCanaryCoverage + && (plan.workload.periodicCanarySchedule ?? 'full-sweep') === 'rotating-one' + ) { + const timedRounds = Math.max( + 0, + Math.ceil(plan.workload.durationSec / plan.workload.canaryIntervalSec) - 1 + ); + const selectedFleet = fleet.tenants.slice(0, maxTenantCount || fleet.tenants.length); + const maxConfiguredCanaries = Math.max( + 0, + ...selectedFleet.flatMap((tenant) => + tenant.surfaces.map((surface) => surface.canaries.length) + ) + ); + if (timedRounds < maxConfiguredCanaries) { + failures.push( + `rotating periodic canary schedule has ${timedRounds} timed rounds but ` + + `a qualifying surface configures ${maxConfiguredCanaries} canaries` + ); + } + } + for (const tenant of fleet.tenants) { + if (plan.gates?.requireExplicitCustomerTopology && !tenant.databases) { + failures.push(`${tenant.id} has no explicit customer -> database -> API topology`); + } + const capabilities = new Set(tenant.surfaces.flatMap((surface) => + surface.operations.map((operation) => operation.capability) + )); + const missingCapabilities = plan.requiredCapabilities.filter((capability) => + !capabilities.has(capability) + ); + if (missingCapabilities.length > 0) { + failures.push(`${tenant.id} has no operations for capabilities: ${missingCapabilities.join(', ')}`); + } + for (const surface of tenant.surfaces) { + if (plan.gates?.requireConclusiveOperationOracles) { + if (!hasConclusiveOperationOracle(surface.warmup)) { + failures.push( + `${tenant.id}/${surface.name} warmup has no conclusive response oracle` + ); + } + const missingOperationOracles = surface.operations + .filter((operation) => !hasConclusiveOperationOracle(operation)) + .map((operation) => operation.name); + if (missingOperationOracles.length > 0) { + failures.push( + `${tenant.id}/${surface.name} operations lack conclusive response oracles: ` + + missingOperationOracles.join(', ') + ); + } + } + if (surface.buildContracts) { + const missingArms = (plan.arms ?? []) + .filter((arm) => !surface.buildContracts?.[arm.name]) + .map((arm) => arm.name); + if (missingArms.length > 0) { + failures.push( + `${tenant.id}/${surface.name} lacks exact build contracts for arms: ${missingArms.join(', ')}` + ); + } + } + const canaries = new Set(surface.canaries.map((canary) => canary.name)); + const missing = plan.requiredCanaries.filter((canary) => !canaries.has(canary)); + if (missing.length > 0) { + failures.push(`${tenant.id}/${surface.name} lacks canaries: ${missing.join(', ')}`); + } + } + for (const database of tenant.databases ?? []) { + const databaseOwner = databaseOwners.get(database.id); + if (databaseOwner && databaseOwner !== tenant.id) { + failures.push( + `logical database id '${database.id}' is reused across customers '${databaseOwner}' and '${tenant.id}'` + ); + } else { + databaseOwners.set(database.id, tenant.id); + } + for (const api of database.apis) { + const apiOwner = apiOwners.get(api.id); + if (apiOwner && apiOwner !== tenant.id) { + failures.push( + `API id '${api.id}' is reused across customers '${apiOwner}' and '${tenant.id}'` + ); + } else { + apiOwners.set(api.id, tenant.id); + } + if (api.runtimePoolIdentities) { + const missingArms = (plan.arms ?? []) + .filter((arm) => !api.runtimePoolIdentities?.[arm.name]) + .map((arm) => arm.name); + if (missingArms.length > 0) { + failures.push( + `${tenant.id}/${database.id}/${api.id} lacks exact runtime pool identities for arms: ${missingArms.join(', ')}` + ); + } + } + } + } + } + const arms = plan.arms?.length + ? plan.arms.map((arm) => arm.name) + : ['default']; + for (const armName of arms) { + const owners = new Map(); + const poolOwners = new Map(); + for (const tenant of fleet.tenants) { + for (const surface of tenant.surfaces) { + const identity = armName === 'default' + ? surface.buildContract + : surface.buildContracts?.[armName] ?? surface.buildContract; + if (!identity) continue; + const owner = owners.get(identity); + if (owner && owner !== tenant.id) { + failures.push( + `build contract '${identity}' for arm '${armName}' is reused across tenants '${owner}' and '${tenant.id}'` + ); + } else { + owners.set(identity, tenant.id); + } + } + for (const database of tenant.databases ?? []) { + for (const api of database.apis) { + const identity = armName === 'default' + ? api.runtimePoolIdentity + : api.runtimePoolIdentities?.[armName] ?? api.runtimePoolIdentity; + if (!identity) continue; + const owner = poolOwners.get(identity); + if (owner && owner !== tenant.id) { + failures.push( + `runtime pool identity '${identity}' for arm '${armName}' is reused across customers '${owner}' and '${tenant.id}'` + ); + } else { + poolOwners.set(identity, tenant.id); + } + } + } + } + } + if (failures.length > 0) { + throw new Error(`density fixture is not qualification-complete:\n- ${failures.join('\n- ')}`); + } +}; + +export const resolveTemplate = ( + value: string, + vars: Record +): string => value.replace(/\{([a-zA-Z][a-zA-Z0-9]*)\}/g, (_match, key: string) => { + if (!(key in vars)) throw new Error(`unknown template variable '${key}'`); + return String(vars[key]); +}); diff --git a/packages/perf-harness/src/evidence.ts b/packages/perf-harness/src/evidence.ts new file mode 100644 index 0000000000..664aed4e78 --- /dev/null +++ b/packages/perf-harness/src/evidence.ts @@ -0,0 +1,1781 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; + +import { + DEFAULT_RUN_ORDER_SEED, + resolveTenants +} from './config'; +import { resolveOfferedLoad, resolveWarmupTimeoutMs } from './http'; +import { normalizeRetainedMemoryCheckpoint } from './memory'; +import { summarizeRealtimeReceiptEvidence } from './realtime-evidence'; +import { normalizePostgresRunAttestation } from './run-attestation'; +import { scoreRun, type ScoreInput } from './score'; +import type { RealtimeDriverSnapshot } from './realtime'; +import type { + ArmProvenance, + CanaryResult, + CanaryScheduleSummary, + DensityPlanV1, + DensityRunResult, + FleetV1, + MemorySnapshot, + NodeRssSnapshot, + PostgresMemorySnapshot, + PostgresRunAttestationEvidence, + RealtimeDeliveryCoverage, + ResolvedOfferedLoad, + RetainedMemoryCheckpointPair, + RequestSample +} from './types'; + +const SHA256 = /^[a-f0-9]{64}$/; +const SANITIZED_EXECUTION_ERROR = /^[A-Z][A-Z0-9_]*:sha256:[a-f0-9]{64}$/; + +/** + * These files contain every variable input to scoreRun that is not supplied by + * the exact plan and fleet bytes. score-context.json deliberately contains no + * fleet, operation, header, environment, or tenant credential material. + */ +export const RESULT_RAW_EVIDENCE_FILES = [ + 'memory.json', + 'postgres-memory.json', + 'canaries.json', + 'canary-schedule.json', + 'requests.ndjson', + 'workload-progress.json', + 'retained-memory.json', + 'realtime-driver.json', + 'score-context.json' +] as const; + +const SCORE_CONTEXT_KEYS = [ + 'version', + 'planSha256', + 'fleetSha256', + 'campaignId', + 'scheduleSha256', + 'previousResultPayloadSha256', + 'evidenceMode', + 'runKind', + 'arm', + 'heapMiB', + 'configuredCustomers', + 'repetition', + 'runOrderIndex', + 'notBeforeEpochMs', + 'startedAt', + 'endedAt', + 'configuredDurationSec', + 'serverExit', + 'externalServer', + 'executionErrors', + 'provenance', + 'provenanceErrors', + 'postgresRunAttestation' +] as const; + +export interface DensityScoreContextV1 { + version: 1; + planSha256: string; + fleetSha256: string; + campaignId: string; + scheduleSha256: string; + previousResultPayloadSha256: string | null; + evidenceMode: 'qualification' | 'diagnostic'; + runKind: 'matrix' | 'soak'; + arm: string; + heapMiB: number; + configuredCustomers: number; + repetition: number; + runOrderIndex: number; + notBeforeEpochMs: number; + startedAt: string; + endedAt: string; + configuredDurationSec: number; + serverExit: DensityRunResult['serverExit']; + externalServer: boolean; + executionErrors: string[]; + provenance: ArmProvenance | null; + provenanceErrors: string[]; + postgresRunAttestation: PostgresRunAttestationEvidence | null; +} + +export interface DensityScoreContextMetadata { + planSha256: string; + fleetSha256: string; + campaignId: string; + scheduleSha256: string; + previousResultPayloadSha256: string | null; + notBeforeEpochMs: number; + /** Compared in-memory only and never serialized into score-context.json. */ + knownRuntimeSecretValues?: readonly string[]; +} + +interface MemoryEvidence { + snapshots: MemorySnapshot[]; + osSnapshots: NodeRssSnapshot[]; + errors: string[]; + warmupIndex: number; + osWarmupIndex: number; + osPeakRssBytes: number | null; +} + +interface PostgresMemoryEvidence { + snapshots: PostgresMemorySnapshot[]; + errors: string[]; +} + +interface WorkloadProgressEvidence { + warmedSurfaces: Array<{ tenantId: string; surfaces: string[] }>; + warmupLatencies: number[]; + samples: number; + canaries: number; + canarySchedule: CanaryScheduleSummary | null; + offeredLoad: ResolvedOfferedLoad | null; + resolvedWarmupTimeoutMs: number | null; + workloadDurationMs: number | null; +} + +interface RealtimeEvidenceEntry { + phase: string; + timestamp: string; + snapshot: RealtimeDriverSnapshot; +} + +const sha256 = (value: string | Buffer): string => createHash('sha256') + .update(value) + .digest('hex'); + +const sourceSha256 = (value: DensityPlanV1 | FleetV1): string => + value.sourceSha256 ?? sha256(JSON.stringify(value)); + +const requireRecord = (value: unknown, label: string): Record => { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value as Record; +}; + +const requireExactKeys = ( + value: Record, + expected: readonly string[], + label: string +): void => { + const actual = Object.keys(value); + const missing = expected.filter((key) => !Object.prototype.hasOwnProperty.call(value, key)); + const unexpected = actual.filter((key) => !expected.includes(key)); + if (missing.length > 0 || unexpected.length > 0) { + throw new Error( + `${label} has an invalid shape; missing=${missing.join(',') || 'none'}; ` + + `unexpected=${unexpected.join(',') || 'none'}` + ); + } +}; + +const requireStringArray = (value: unknown, label: string): string[] => { + if (!Array.isArray(value) || value.some((item) => typeof item !== 'string')) { + throw new Error(`${label} must be a string array`); + } + return value; +}; + +const requireSanitizedExecutionErrors = (value: unknown, label: string): string[] => { + const errors = requireStringArray(value, label); + if (errors.some((error) => !SANITIZED_EXECUTION_ERROR.test(error))) { + throw new Error(`${label} must contain only code-and-SHA-256 evidence`); + } + return errors; +}; + +const requireFinite = (value: unknown, label: string): number => { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`${label} must be finite`); + } + return value; +}; + +const requireSafeInteger = ( + value: unknown, + label: string, + minimum = 0 +): number => { + if (!Number.isSafeInteger(value) || (value as number) < minimum) { + throw new Error(`${label} must be a safe integer >= ${minimum}`); + } + return value as number; +}; + +const requireCanonicalTimestamp = (value: unknown, label: string): string => { + if (typeof value !== 'string') throw new Error(`${label} must be a timestamp`); + const parsed = Date.parse(value); + if (!Number.isFinite(parsed) || new Date(parsed).toISOString() !== value) { + throw new Error(`${label} must be a canonical ISO timestamp`); + } + return value; +}; + +const requireBoolean = (value: unknown, label: string): boolean => { + if (typeof value !== 'boolean') throw new Error(`${label} must be boolean`); + return value; +}; + +const requireNonEmptyString = (value: unknown, label: string): string => { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`${label} must be a nonempty string`); + } + return value; +}; + +const requireAllowedKeys = ( + value: Record, + allowed: readonly string[], + label: string +): void => { + const unexpected = Object.keys(value).filter((key) => !allowed.includes(key)); + if (unexpected.length > 0) { + throw new Error(`${label} has unexpected fields: ${unexpected.join(',')}`); + } +}; + +const requireNullableNonNegativeFinite = (value: unknown, label: string): void => { + if (value == null) return; + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new Error(`${label} must be null or a finite non-negative number`); + } +}; + +/** Read one immutable regular file without following a final symlink. */ +export const readRegularEvidenceFile = (file: string): Buffer => { + const before = fs.lstatSync(file); + if (before.isSymbolicLink() || !before.isFile()) { + throw new Error(`evidence is not a regular non-symlink file: ${path.basename(file)}`); + } + const descriptor = fs.openSync( + file, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) + ); + try { + const opened = fs.fstatSync(descriptor); + if (!opened.isFile() || opened.dev !== before.dev || opened.ino !== before.ino) { + throw new Error(`evidence changed while opening: ${path.basename(file)}`); + } + const bytes = fs.readFileSync(descriptor); + const after = fs.fstatSync(descriptor); + if ( + after.dev !== opened.dev + || after.ino !== opened.ino + || after.size !== opened.size + || after.mtimeMs !== opened.mtimeMs + || after.ctimeMs !== opened.ctimeMs + ) { + throw new Error(`evidence changed while reading: ${path.basename(file)}`); + } + return bytes; + } finally { + fs.closeSync(descriptor); + } +}; + +const parseJsonEvidence = (artifactDir: string, name: string): unknown => { + try { + return JSON.parse(readRegularEvidenceFile(path.join(artifactDir, name)).toString('utf8')); + } catch (error) { + throw new Error( + `invalid ${name}: ${error instanceof Error ? error.message : String(error)}` + ); + } +}; + +const artifactNamesForResult = (result: DensityRunResult): string[] => [ + ...RESULT_RAW_EVIDENCE_FILES, + ...(result.postgresRunAttestation ? ['postgres-run-attestation.json'] : []) +]; + +const resultPayload = (result: DensityRunResult): Omit => { + const payload = { ...result }; + delete payload.evidenceBinding; + return payload; +}; + +const resultPayloadSha256 = (result: DensityRunResult): string => + sha256(JSON.stringify(resultPayload(result))); + +export const bindResultEvidence = (result: DensityRunResult): void => { + const artifacts = artifactNamesForResult(result).map((name) => ({ + name, + sha256: sha256(readRegularEvidenceFile(path.join(result.artifactDir, name))) + })); + result.evidenceBinding = { + version: 2, + algorithm: 'sha256', + resultPayloadSha256: resultPayloadSha256(result), + artifacts + }; +}; + +export const validateResultEvidenceBinding = ( + result: DensityRunResult, + label: string +): void => { + const binding = result.evidenceBinding; + if ( + !binding + || binding.version !== 2 + || binding.algorithm !== 'sha256' + || !SHA256.test(binding.resultPayloadSha256) + || !Array.isArray(binding.artifacts) + ) { + throw new Error(`${label} evidence binding is missing or invalid`); + } + if (binding.resultPayloadSha256 !== resultPayloadSha256(result)) { + throw new Error(`${label} result payload does not match its evidence binding`); + } + const expectedNames = artifactNamesForResult(result); + const byName = new Map(binding.artifacts.map((artifact) => [artifact.name, artifact])); + if (byName.size !== expectedNames.length || binding.artifacts.length !== expectedNames.length) { + throw new Error(`${label} raw evidence binding is incomplete or duplicated`); + } + for (const name of expectedNames) { + const artifact = byName.get(name); + if (!artifact || !SHA256.test(artifact.sha256)) { + throw new Error(`${label} raw evidence binding is missing ${name}`); + } + let bytes: Buffer; + try { + bytes = readRegularEvidenceFile(path.join(result.artifactDir, name)); + } catch { + throw new Error(`${label} raw evidence file is unavailable or unsafe: ${name}`); + } + if (sha256(bytes) !== artifact.sha256) { + throw new Error(`${label} raw evidence file does not match: ${name}`); + } + } +}; + +const SENSITIVE_ENVIRONMENT_NAME = /(?:^|_)(?:API_KEY|AUTHORIZATION|COOKIE|DATABASE_URL|DSN|PASSWORD|PASSWD|PGURL|PRIVATE_KEY|SECRET|TOKEN)$/i; +const SENSITIVE_COMMAND_FLAG = /^--(?:[a-z0-9]+[-_])*(?:api[-_]?key|authorization|cookie|database[-_]?url|dsn|password|passwd|private[-_]?key|secret|token)(?:[-_][a-z0-9]+)*(?:=|$)/i; +const SAFE_SECRET_FILE_FLAG = /^--(?:[a-z0-9]+[-_])*(?:credential|secret)s?(?:[-_]file)?$/i; +const SENSITIVE_ASSIGNMENT = /^(?:DATABASE_URL|PGPASSWORD|PGURL|[^=]*(?:PASSWORD|PASSWD|PRIVATE_KEY|SECRET|TOKEN))=/i; +const USERINFO_URL = /\b(?:https?|postgres(?:ql)?|wss?):\/\/[^\s/:@]+:[^\s/@]+@/i; +const BEARER_VALUE = /^Bearer\s+\S+/i; +const SENSITIVE_QUERY_PARAMETER = /[?&](?:api[-_]?key|authorization|cookie|password|passwd|private[-_]?key|secret|token)=[^&#\s]+/i; +const SENSITIVE_HEADER_VALUE = /^(?:authorization|cookie|proxy-authorization)\s*:\s*\S+/i; +const SENSITIVE_PROVENANCE_KEY = /(?:^|[-_])(?:api[-_]?key|authorization|cookie|database[-_]?url|dsn|password|passwd|pgurl|private[-_]?key|secret|token)(?:$|[-_])/i; +const SAFE_SECRET_REFERENCE_KEY = /(?:file|path)$/i; + +const knownRuntimeSecrets = ( + environment: Readonly> = process.env +): string[] => [...new Set(Object.entries(environment) + .filter(([name, value]) => SENSITIVE_ENVIRONMENT_NAME.test(name) && Boolean(value)) + .map(([_name, value]) => value!))]; + +/** + * Provenance must describe how the process was started, but it must never turn + * into a second credential store. Secret files and environment-variable names + * are safe; literal credential arguments and URL userinfo are rejected. + */ +export const assertScoreContextCredentialSafe = ( + context: DensityScoreContextV1, + secretValues: readonly string[] = knownRuntimeSecrets() +): void => { + const command = context.provenance?.command ?? []; + const sensitiveString = (value: string): boolean => + USERINFO_URL.test(value) + || BEARER_VALUE.test(value) + || SENSITIVE_QUERY_PARAMETER.test(value) + || SENSITIVE_HEADER_VALUE.test(value) + || SENSITIVE_ASSIGNMENT.test(value) + || secretValues.some((secret) => secret.length > 0 && value.includes(secret)); + const inspectStrings = (value: unknown, pathValue: string, ancestors = new Set()): void => { + if (typeof value === 'string') { + if (sensitiveString(value)) { + throw new Error(`score-context provenance contains credential material at ${pathValue}`); + } + return; + } + if (!value || typeof value !== 'object') return; + if (ancestors.has(value)) throw new Error('score-context provenance contains a cycle'); + ancestors.add(value); + if (Array.isArray(value)) { + value.forEach((item, index) => inspectStrings(item, `${pathValue}[${index}]`, ancestors)); + } else { + for (const [key, item] of Object.entries(value)) { + const normalizedKey = key.replace(/([a-z0-9])([A-Z])/g, '$1-$2'); + if ( + typeof item === 'string' + && item.length > 0 + && SENSITIVE_PROVENANCE_KEY.test(normalizedKey) + && !SAFE_SECRET_REFERENCE_KEY.test(normalizedKey) + ) { + throw new Error( + `score-context provenance contains credential material at ${pathValue}.${key}` + ); + } + inspectStrings(item, `${pathValue}.${key}`, ancestors); + } + } + ancestors.delete(value); + }; + inspectStrings( + context.provenance == null + ? null + : { ...context.provenance, command: [] }, + 'provenance' + ); + for (let index = 0; index < command.length; index += 1) { + const argument = command[index]; + const sensitiveFlag = SENSITIVE_COMMAND_FLAG.test(argument) + && !SAFE_SECRET_FILE_FLAG.test(argument); + if ( + sensitiveString(argument) + || (sensitiveFlag && argument.includes('=')) + || (sensitiveFlag && command[index + 1] != null) + ) { + throw new Error('score-context provenance command contains credential material'); + } + } +}; + +export const scoreContextFromInput = ( + input: ScoreInput, + metadata: DensityScoreContextMetadata +): DensityScoreContextV1 => { + requireSanitizedExecutionErrors(input.executionErrors, 'ScoreInput.executionErrors'); + const context: DensityScoreContextV1 = { + version: 1, + planSha256: metadata.planSha256, + fleetSha256: metadata.fleetSha256, + campaignId: metadata.campaignId, + scheduleSha256: metadata.scheduleSha256, + previousResultPayloadSha256: metadata.previousResultPayloadSha256, + evidenceMode: input.evidenceMode, + runKind: input.runKind, + arm: input.arm, + heapMiB: input.heapMiB, + configuredCustomers: input.tenants.length, + repetition: input.repetition, + runOrderIndex: input.runOrderIndex, + notBeforeEpochMs: metadata.notBeforeEpochMs, + startedAt: input.startedAt, + endedAt: input.endedAt, + configuredDurationSec: input.configuredDurationSec, + serverExit: input.serverExit ? { ...input.serverExit } : null, + externalServer: input.externalServer, + executionErrors: [...input.executionErrors], + provenance: input.provenance == null ? null : { ...input.provenance }, + provenanceErrors: [...input.provenanceErrors], + postgresRunAttestation: input.postgresRunAttestation == null + ? null + : { ...input.postgresRunAttestation } + }; + assertScoreContextCredentialSafe( + context, + metadata.knownRuntimeSecretValues ?? knownRuntimeSecrets() + ); + return context; +}; + +export const writeScoreContext = ( + artifactDir: string, + input: ScoreInput, + metadata: DensityScoreContextMetadata +): DensityScoreContextV1 => { + const context = scoreContextFromInput(input, metadata); + fs.writeFileSync( + path.join(artifactDir, 'score-context.json'), + `${JSON.stringify(context, null, 2)}\n`, + { encoding: 'utf8', flag: 'wx' } + ); + return context; +}; + +export const readScoreContextEvidence = (artifactDir: string): DensityScoreContextV1 => { + const record = requireRecord( + parseJsonEvidence(artifactDir, 'score-context.json'), + 'score-context.json' + ); + requireExactKeys(record, SCORE_CONTEXT_KEYS, 'score-context.json'); + if (record.version !== 1) throw new Error('score-context.json version must be 1'); + if (typeof record.planSha256 !== 'string' || !SHA256.test(record.planSha256)) { + throw new Error('score-context.json planSha256 is invalid'); + } + if (typeof record.fleetSha256 !== 'string' || !SHA256.test(record.fleetSha256)) { + throw new Error('score-context.json fleetSha256 is invalid'); + } + for (const key of ['campaignId', 'scheduleSha256'] as const) { + if (typeof record[key] !== 'string' || !SHA256.test(record[key] as string)) { + throw new Error(`score-context.json ${key} is invalid`); + } + } + if ( + record.previousResultPayloadSha256 != null + && ( + typeof record.previousResultPayloadSha256 !== 'string' + || !SHA256.test(record.previousResultPayloadSha256) + ) + ) { + throw new Error('score-context.json previousResultPayloadSha256 is invalid'); + } + if (!['qualification', 'diagnostic'].includes(String(record.evidenceMode))) { + throw new Error('score-context.json evidenceMode is invalid'); + } + if (!['matrix', 'soak'].includes(String(record.runKind))) { + throw new Error('score-context.json runKind is invalid'); + } + if (typeof record.arm !== 'string' || record.arm.length === 0) { + throw new Error('score-context.json arm is invalid'); + } + requireSafeInteger(record.heapMiB, 'score-context.json heapMiB', 1); + requireSafeInteger( + record.configuredCustomers, + 'score-context.json configuredCustomers', + 1 + ); + requireSafeInteger(record.repetition, 'score-context.json repetition', 1); + requireSafeInteger(record.runOrderIndex, 'score-context.json runOrderIndex', 1); + requireSafeInteger(record.notBeforeEpochMs, 'score-context.json notBeforeEpochMs', 1); + for (const key of ['startedAt', 'endedAt'] as const) { + requireCanonicalTimestamp(record[key], `score-context.json ${key}`); + } + if (Date.parse(record.endedAt as string) < Date.parse(record.startedAt as string)) { + throw new Error('score-context.json endedAt precedes startedAt'); + } + requireFinite(record.configuredDurationSec, 'score-context.json configuredDurationSec'); + if (typeof record.externalServer !== 'boolean') { + throw new Error('score-context.json externalServer is invalid'); + } + requireSanitizedExecutionErrors( + record.executionErrors, + 'score-context.json executionErrors' + ); + requireStringArray(record.provenanceErrors, 'score-context.json provenanceErrors'); + if (record.provenance != null) requireRecord(record.provenance, 'score-context.json provenance'); + if (record.postgresRunAttestation != null) { + requireRecord( + record.postgresRunAttestation, + 'score-context.json postgresRunAttestation' + ); + } + if (record.serverExit != null) { + const exit = requireRecord(record.serverExit, 'score-context.json serverExit'); + requireExactKeys(exit, ['code', 'signal'], 'score-context.json serverExit'); + if (exit.code != null && !Number.isSafeInteger(exit.code)) { + throw new Error('score-context.json serverExit.code is invalid'); + } + if (exit.signal != null && typeof exit.signal !== 'string') { + throw new Error('score-context.json serverExit.signal is invalid'); + } + } + const context = record as unknown as DensityScoreContextV1; + assertScoreContextCredentialSafe(context); + return context; +}; + +const parseMemoryEvidence = (artifactDir: string): MemoryEvidence => { + const record = requireRecord(parseJsonEvidence(artifactDir, 'memory.json'), 'memory.json'); + requireExactKeys( + record, + ['snapshots', 'osSnapshots', 'errors', 'warmupIndex', 'osWarmupIndex', 'osPeakRssBytes'], + 'memory.json' + ); + if (!Array.isArray(record.snapshots) || !Array.isArray(record.osSnapshots)) { + throw new Error('memory.json snapshots are invalid'); + } + const memoryRequiredKeys = [ + 'timestamp', + 'pid', + 'nodeEnv', + 'heapLimitBytes', + 'heapUsedBytes', + 'rssBytes', + 'processPeakRssBytes', + 'cacheSize', + 'residentBuildContracts', + 'evictions', + 'buildRefusals', + 'buildsStarted', + 'buildsSucceeded', + 'buildMaxMs', + 'pgPoolCacheSize', + 'pgPoolLeasedPools', + 'pgPoolActiveLeases', + 'pgPoolCapacityEvictions', + 'pgPoolCapacityRefusals', + 'pgPoolDisposalFailures', + 'cacheCountersAvailable', + 'buildCountersAvailable' + ] as const; + const optionalNumericKeys = [ + 'cacheConfiguredMax', + 'cacheBudgetCapacity', + 'cacheInstanceHeapBytes', + 'pgPoolTotalClients', + 'pgPoolIdleClients', + 'pgPoolWaitingClients', + 'runtimePoolRequestedMaxUses', + 'runtimePoolEffectiveMaxUses', + 'runtimePoolExpectedPools', + 'runtimePoolObservedPools', + 'runtimePoolTotalClients', + 'runtimePoolIdleClients', + 'runtimePoolWaitingClients', + 'postgresBackendTotal', + 'postgresBackendActive', + 'postgresBackendIdle', + 'postgresBackendIdleInTransaction', + 'physicalDatabases', + 'unexpectedPostgresDatabases', + 'realtimeManagersExpected', + 'realtimeManagersActive', + 'realtimeTransportsExpected', + 'realtimeTransportsActive', + 'notificationBrokers', + 'notificationListenerConnections', + 'notificationBrokerLeases', + 'notificationBrokerTopics', + 'notificationBrokerSubscribers', + 'notificationBrokerQueueOverflows', + 'notificationBrokerFatalFailures', + 'notificationAuditIdentities', + 'notificationAuditsHealthy', + 'notificationAuditsFailed', + 'notificationAuditsStale', + 'notificationAuditAttempts', + 'notificationAuditFailures', + 'notificationAuditActiveDatabaseTargets', + 'notificationAuditDatabaseConflicts' + ] as const; + const requiredNumericKeys = [ + 'heapLimitBytes', + 'heapUsedBytes', + 'rssBytes', + 'processPeakRssBytes', + 'cacheSize', + 'evictions', + 'buildRefusals', + 'buildsStarted', + 'buildsSucceeded', + 'buildMaxMs', + 'pgPoolCacheSize', + 'pgPoolLeasedPools', + 'pgPoolActiveLeases', + 'pgPoolCapacityEvictions', + 'pgPoolCapacityRefusals', + 'pgPoolDisposalFailures' + ] as const; + const optionalBooleanKeys = [ + 'runtimePoolTelemetryAvailable', + 'runtimePoolEffectiveMaxUsesKnown', + 'runtimePoolMaxUsesExact', + 'postgresContainerDedicated' + ] as const; + const allowedMemoryKeys = [ + ...memoryRequiredKeys, + ...optionalNumericKeys, + ...optionalBooleanKeys, + 'cacheCalibrationId', + 'cacheAdmissionMode', + 'residentBuildContractFingerprints', + 'runtimePoolTelemetryScope', + 'realtimeNotificationMode', + 'raw' + ]; + const snapshots = record.snapshots.map((raw, index): MemorySnapshot => { + const label = `memory.json snapshots[${index}]`; + const snapshot = requireRecord(raw, label); + requireAllowedKeys(snapshot, allowedMemoryKeys, label); + for (const key of memoryRequiredKeys) { + if (!Object.prototype.hasOwnProperty.call(snapshot, key)) { + throw new Error(`${label} is missing ${key}`); + } + } + requireCanonicalTimestamp(snapshot.timestamp, `${label}.timestamp`); + if (snapshot.pid != null) requireSafeInteger(snapshot.pid, `${label}.pid`, 1); + if (snapshot.nodeEnv != null && typeof snapshot.nodeEnv !== 'string') { + throw new Error(`${label}.nodeEnv is invalid`); + } + for (const key of [...requiredNumericKeys, ...optionalNumericKeys]) { + requireNullableNonNegativeFinite(snapshot[key], `${label}.${key}`); + } + if ( + snapshot.residentBuildContracts != null + && ( + !Array.isArray(snapshot.residentBuildContracts) + || snapshot.residentBuildContracts.some((item) => + typeof item !== 'string' || item.length === 0) + || new Set(snapshot.residentBuildContracts).size + !== snapshot.residentBuildContracts.length + ) + ) { + throw new Error(`${label}.residentBuildContracts is invalid`); + } + if ( + snapshot.residentBuildContractFingerprints != null + && ( + !Array.isArray(snapshot.residentBuildContractFingerprints) + || snapshot.residentBuildContractFingerprints.some((item) => + typeof item !== 'string' || item.length === 0) + || new Set(snapshot.residentBuildContractFingerprints).size + !== snapshot.residentBuildContractFingerprints.length + ) + ) { + throw new Error(`${label}.residentBuildContractFingerprints is invalid`); + } + requireBoolean(snapshot.cacheCountersAvailable, `${label}.cacheCountersAvailable`); + requireBoolean(snapshot.buildCountersAvailable, `${label}.buildCountersAvailable`); + for (const key of optionalBooleanKeys) { + if (snapshot[key] != null && typeof snapshot[key] !== 'boolean') { + throw new Error(`${label}.${key} is invalid`); + } + } + if ( + snapshot.cacheCalibrationId != null + && typeof snapshot.cacheCalibrationId !== 'string' + ) { + throw new Error(`${label}.cacheCalibrationId is invalid`); + } + if ( + snapshot.cacheAdmissionMode != null + && !['evict-idle', 'preserve-resident'].includes(String(snapshot.cacheAdmissionMode)) + ) { + throw new Error(`${label}.cacheAdmissionMode is invalid`); + } + if ( + snapshot.runtimePoolTelemetryScope != null + && snapshot.runtimePoolTelemetryScope !== 'runtime-only-exact-identities' + ) { + throw new Error(`${label}.runtimePoolTelemetryScope is invalid`); + } + if ( + snapshot.realtimeNotificationMode != null + && !['dedicated', 'shared-exact'].includes(String(snapshot.realtimeNotificationMode)) + ) { + throw new Error(`${label}.realtimeNotificationMode is invalid`); + } + return snapshot as unknown as MemorySnapshot; + }); + const osSnapshots = record.osSnapshots.map((raw, index): NodeRssSnapshot => { + const label = `memory.json osSnapshots[${index}]`; + const snapshot = requireRecord(raw, label); + requireExactKeys(snapshot, ['timestamp', 'pid', 'source', 'rssBytes'], label); + requireCanonicalTimestamp(snapshot.timestamp, `${label}.timestamp`); + requireSafeInteger(snapshot.pid, `${label}.pid`, 1); + requireSafeInteger(snapshot.rssBytes, `${label}.rssBytes`, 1); + if (!['proc', 'authenticated-endpoint'].includes(String(snapshot.source))) { + throw new Error(`${label}.source is invalid`); + } + return snapshot as unknown as NodeRssSnapshot; + }); + const errors = requireStringArray(record.errors, 'memory.json errors'); + const warmupIndex = requireSafeInteger(record.warmupIndex, 'memory.json warmupIndex', -1); + const osWarmupIndex = requireSafeInteger( + record.osWarmupIndex, + 'memory.json osWarmupIndex', + -1 + ); + if (warmupIndex > snapshots.length || osWarmupIndex > osSnapshots.length) { + throw new Error('memory.json warmup index is out of range'); + } + if (record.osPeakRssBytes != null) { + requireSafeInteger(record.osPeakRssBytes, 'memory.json osPeakRssBytes', 1); + if ( + osSnapshots.length > 0 + && record.osPeakRssBytes !== Math.max(...osSnapshots.map((snapshot) => snapshot.rssBytes)) + ) { + throw new Error('memory.json osPeakRssBytes does not match its raw snapshots'); + } + } + return { + snapshots, + osSnapshots, + errors, + warmupIndex, + osWarmupIndex, + osPeakRssBytes: record.osPeakRssBytes as number | null + }; +}; + +const parsePostgresMemoryEvidence = (artifactDir: string): PostgresMemoryEvidence => { + const record = requireRecord( + parseJsonEvidence(artifactDir, 'postgres-memory.json'), + 'postgres-memory.json' + ); + requireExactKeys(record, ['snapshots', 'errors'], 'postgres-memory.json'); + if (!Array.isArray(record.snapshots)) { + throw new Error('postgres-memory.json snapshots are invalid'); + } + const snapshots = record.snapshots.map((raw, index): PostgresMemorySnapshot => { + const label = `postgres-memory.json snapshots[${index}]`; + const snapshot = requireRecord(raw, label); + requireAllowedKeys(snapshot, [ + 'timestamp', + 'containerId', + 'cgroupIdentitySha256', + 'usedBytes', + 'limitBytes', + 'source', + 'workingSetBytes', + 'sampleStartedAt', + 'sampleEndedAt', + 'sampleDurationMs', + 'cgroupV2', + 'raw' + ], label); + for (const key of ['timestamp', 'usedBytes', 'limitBytes', 'raw']) { + if (!Object.prototype.hasOwnProperty.call(snapshot, key)) { + throw new Error(`${label} is missing ${key}`); + } + } + requireCanonicalTimestamp(snapshot.timestamp, `${label}.timestamp`); + requireSafeInteger(snapshot.usedBytes, `${label}.usedBytes`); + requireSafeInteger(snapshot.limitBytes, `${label}.limitBytes`); + if (typeof snapshot.raw !== 'string') throw new Error(`${label}.raw is invalid`); + if (snapshot.containerId != null && ( + typeof snapshot.containerId !== 'string' || !/^[a-f0-9]{64}$/.test(snapshot.containerId) + )) { + throw new Error(`${label}.containerId is invalid`); + } + if (snapshot.cgroupIdentitySha256 != null && ( + typeof snapshot.cgroupIdentitySha256 !== 'string' + || !/^sha256:[a-f0-9]{64}$/.test(snapshot.cgroupIdentitySha256) + )) { + throw new Error(`${label}.cgroupIdentitySha256 is invalid`); + } + if (snapshot.source != null && !['cgroup-v2', 'docker-stats'].includes(String(snapshot.source))) { + throw new Error(`${label}.source is invalid`); + } + if (snapshot.workingSetBytes != null) { + requireSafeInteger(snapshot.workingSetBytes, `${label}.workingSetBytes`); + } + if ( + snapshot.sampleStartedAt != null + || snapshot.sampleEndedAt != null + || snapshot.sampleDurationMs != null + ) { + const startedAt = requireCanonicalTimestamp( + snapshot.sampleStartedAt, + `${label}.sampleStartedAt` + ); + const endedAt = requireCanonicalTimestamp( + snapshot.sampleEndedAt, + `${label}.sampleEndedAt` + ); + const durationMs = requireSafeInteger( + snapshot.sampleDurationMs, + `${label}.sampleDurationMs` + ); + if (Date.parse(endedAt) < Date.parse(startedAt)) { + throw new Error(`${label} sample chronology is invalid`); + } + if (Math.abs((Date.parse(endedAt) - Date.parse(startedAt)) - durationMs) > 1) { + throw new Error(`${label}.sampleDurationMs is inconsistent`); + } + } + if (snapshot.cgroupV2 != null) { + const cgroup = requireRecord(snapshot.cgroupV2, `${label}.cgroupV2`); + requireExactKeys( + cgroup, + ['currentBytes', 'peakBytes', 'maxBytes', 'stat', 'events'], + `${label}.cgroupV2` + ); + requireSafeInteger(cgroup.currentBytes, `${label}.cgroupV2.currentBytes`); + if (cgroup.peakBytes != null) { + requireSafeInteger(cgroup.peakBytes, `${label}.cgroupV2.peakBytes`); + } + if (cgroup.maxBytes != null) { + requireSafeInteger(cgroup.maxBytes, `${label}.cgroupV2.maxBytes`); + } + for (const field of ['stat', 'events'] as const) { + const values = requireRecord(cgroup[field], `${label}.cgroupV2.${field}`); + for (const [key, value] of Object.entries(values)) { + if (!key) throw new Error(`${label}.cgroupV2.${field} has an empty key`); + requireSafeInteger(value, `${label}.cgroupV2.${field}.${key}`); + } + } + if (snapshot.source !== 'cgroup-v2' || snapshot.usedBytes !== cgroup.currentBytes) { + throw new Error(`${label} cgroup-v2 source does not match current bytes`); + } + } else if (snapshot.source === 'cgroup-v2') { + throw new Error(`${label} cgroup-v2 source has no cgroup payload`); + } + return snapshot as unknown as PostgresMemorySnapshot; + }); + return { + snapshots, + errors: requireStringArray(record.errors, 'postgres-memory.json errors') + }; +}; + +function parseCanarySchedule( + raw: unknown, + label: string +): CanaryScheduleSummary | null { + if (raw == null) return null; + const schedule = requireRecord(raw, label); + requireExactKeys(schedule, [ + 'schedule', + 'intervalMs', + 'durationMs', + 'canaryConcurrency', + 'startedAt', + 'deadlineAt', + 'planned', + 'started', + 'completed', + 'missed', + 'overlapped', + 'deadlineLate', + 'checksPlanned', + 'checksStarted', + 'checksCompleted', + 'rounds' + ], label); + if (!['full-sweep', 'rotating-one'].includes(String(schedule.schedule))) { + throw new Error(`${label}.schedule is invalid`); + } + const intervalMs = requireSafeInteger(schedule.intervalMs, `${label}.intervalMs`, 1); + const durationMs = requireSafeInteger(schedule.durationMs, `${label}.durationMs`, 1); + requireSafeInteger(schedule.canaryConcurrency, `${label}.canaryConcurrency`, 1); + const startedAt = requireCanonicalTimestamp(schedule.startedAt, `${label}.startedAt`); + const deadlineAt = requireCanonicalTimestamp(schedule.deadlineAt, `${label}.deadlineAt`); + if (Date.parse(deadlineAt) - Date.parse(startedAt) !== durationMs) { + throw new Error(`${label}.deadlineAt does not match durationMs`); + } + const counterKeys = [ + 'planned', + 'started', + 'completed', + 'missed', + 'overlapped', + 'deadlineLate', + 'checksPlanned', + 'checksStarted', + 'checksCompleted' + ] as const; + for (const key of counterKeys) requireSafeInteger(schedule[key], `${label}.${key}`); + if (!Array.isArray(schedule.rounds)) throw new Error(`${label}.rounds must be an array`); + const rounds = schedule.rounds.map((rawRound, index) => { + const roundLabel = `${label}.rounds[${index}]`; + const round = requireRecord(rawRound, roundLabel); + requireExactKeys(round, [ + 'periodicRound', + 'plannedAt', + 'startedAt', + 'completedAt', + 'targetsPlanned', + 'targetsStarted', + 'targetsCompleted', + 'checksPlanned', + 'checksStarted', + 'checksCompleted', + 'overlapped', + 'deadlineLate', + 'startDelayMs', + 'durationMs' + ], roundLabel); + const periodicRound = requireSafeInteger( + round.periodicRound, + `${roundLabel}.periodicRound`, + 1 + ); + if (periodicRound !== index + 1) { + throw new Error(`${roundLabel}.periodicRound is not contiguous`); + } + const plannedAt = requireCanonicalTimestamp(round.plannedAt, `${roundLabel}.plannedAt`); + if (Date.parse(plannedAt) !== Date.parse(startedAt) + periodicRound * intervalMs) { + throw new Error(`${roundLabel}.plannedAt does not match its schedule slot`); + } + for (const key of [ + 'targetsPlanned', + 'targetsStarted', + 'targetsCompleted', + 'checksPlanned', + 'checksStarted', + 'checksCompleted' + ] as const) { + requireSafeInteger(round[key], `${roundLabel}.${key}`); + } + requireBoolean(round.overlapped, `${roundLabel}.overlapped`); + requireBoolean(round.deadlineLate, `${roundLabel}.deadlineLate`); + const roundStartedAt = round.startedAt == null + ? null + : requireCanonicalTimestamp(round.startedAt, `${roundLabel}.startedAt`); + const roundCompletedAt = round.completedAt == null + ? null + : requireCanonicalTimestamp(round.completedAt, `${roundLabel}.completedAt`); + if (round.startDelayMs != null) { + requireFinite(round.startDelayMs, `${roundLabel}.startDelayMs`); + if ((round.startDelayMs as number) < 0) { + throw new Error(`${roundLabel}.startDelayMs must be non-negative`); + } + } + if (round.durationMs != null) { + requireFinite(round.durationMs, `${roundLabel}.durationMs`); + if ((round.durationMs as number) < 0) { + throw new Error(`${roundLabel}.durationMs must be non-negative`); + } + } + if ( + (roundStartedAt == null) !== (round.startDelayMs == null) + || (roundCompletedAt == null) !== (round.durationMs == null) + || (roundCompletedAt != null && roundStartedAt == null) + || ( + roundStartedAt != null + && Date.parse(roundStartedAt) < Date.parse(plannedAt) + ) + || ( + roundCompletedAt != null + && Date.parse(roundCompletedAt) < Date.parse(roundStartedAt!) + ) + ) { + throw new Error(`${roundLabel} chronology is invalid`); + } + return round; + }); + const aggregate = rounds.reduce<{ + started: number; + completed: number; + overlapped: number; + deadlineLate: number; + checksPlanned: number; + checksStarted: number; + checksCompleted: number; + }>((summary, round) => ({ + started: summary.started + (round.startedAt == null ? 0 : 1), + completed: summary.completed + (round.completedAt == null ? 0 : 1), + overlapped: summary.overlapped + (round.overlapped ? 1 : 0), + deadlineLate: summary.deadlineLate + (round.deadlineLate ? 1 : 0), + checksPlanned: summary.checksPlanned + Number(round.checksPlanned), + checksStarted: summary.checksStarted + Number(round.checksStarted), + checksCompleted: summary.checksCompleted + Number(round.checksCompleted) + }), { + started: 0, + completed: 0, + overlapped: 0, + deadlineLate: 0, + checksPlanned: 0, + checksStarted: 0, + checksCompleted: 0 + }); + if ( + Number(schedule.planned) !== rounds.length + || Number(schedule.started) !== aggregate.started + || Number(schedule.completed) !== aggregate.completed + || Number(schedule.missed) !== rounds.length - aggregate.completed + || Number(schedule.overlapped) !== aggregate.overlapped + || Number(schedule.deadlineLate) !== aggregate.deadlineLate + || Number(schedule.checksPlanned) !== aggregate.checksPlanned + || Number(schedule.checksStarted) !== aggregate.checksStarted + || Number(schedule.checksCompleted) !== aggregate.checksCompleted + ) { + throw new Error(`${label} aggregate counters do not match its rounds`); + } + return schedule as unknown as CanaryScheduleSummary; +} + +const parseWorkloadProgress = (artifactDir: string): WorkloadProgressEvidence => { + const record = requireRecord( + parseJsonEvidence(artifactDir, 'workload-progress.json'), + 'workload-progress.json' + ); + requireExactKeys(record, [ + 'warmedSurfaces', + 'warmupLatencies', + 'samples', + 'canaries', + 'canarySchedule', + 'offeredLoad', + 'resolvedWarmupTimeoutMs', + 'workloadDurationMs' + ], 'workload-progress.json'); + if (!Array.isArray(record.warmedSurfaces)) { + throw new Error('workload-progress.json warmedSurfaces is invalid'); + } + const warmedSurfaces = record.warmedSurfaces.map((raw, index) => { + const entry = requireRecord(raw, `workload-progress.json warmedSurfaces[${index}]`); + requireExactKeys( + entry, + ['tenantId', 'surfaces'], + `workload-progress.json warmedSurfaces[${index}]` + ); + if (typeof entry.tenantId !== 'string' || !entry.tenantId) { + throw new Error(`workload-progress.json warmedSurfaces[${index}].tenantId is invalid`); + } + const surfaces = requireStringArray( + entry.surfaces, + `workload-progress.json warmedSurfaces[${index}].surfaces` + ); + if (new Set(surfaces).size !== surfaces.length) { + throw new Error(`workload-progress.json warmedSurfaces[${index}] is duplicated`); + } + return { tenantId: entry.tenantId, surfaces }; + }); + if (new Set(warmedSurfaces.map((entry) => entry.tenantId)).size !== warmedSurfaces.length) { + throw new Error('workload-progress.json contains duplicate tenant warmup entries'); + } + if (!Array.isArray(record.warmupLatencies)) { + throw new Error('workload-progress.json warmupLatencies is invalid'); + } + const warmupLatencies = record.warmupLatencies.map((value, index) => + requireFinite(value, `workload-progress.json warmupLatencies[${index}]`)); + const samples = requireSafeInteger(record.samples, 'workload-progress.json samples'); + const canaries = requireSafeInteger(record.canaries, 'workload-progress.json canaries'); + let offeredLoad: ResolvedOfferedLoad | null = null; + if (record.offeredLoad != null) { + const resolved = requireRecord( + record.offeredLoad, + 'workload-progress.json offeredLoad' + ); + requireExactKeys(resolved, [ + 'mode', + 'configuredRps', + 'tenantCount', + 'totalRps', + 'rpsPerTenant' + ], 'workload-progress.json offeredLoad'); + if (!['fixed-total', 'per-tenant'].includes(String(resolved.mode))) { + throw new Error('workload-progress.json offeredLoad.mode is invalid'); + } + requireFinite(resolved.configuredRps, 'workload-progress.json offeredLoad.configuredRps'); + requireSafeInteger(resolved.tenantCount, 'workload-progress.json offeredLoad.tenantCount', 1); + requireFinite(resolved.totalRps, 'workload-progress.json offeredLoad.totalRps'); + requireFinite(resolved.rpsPerTenant, 'workload-progress.json offeredLoad.rpsPerTenant'); + if ( + (resolved.configuredRps as number) <= 0 + || (resolved.totalRps as number) <= 0 + || (resolved.rpsPerTenant as number) <= 0 + ) { + throw new Error('workload-progress.json offeredLoad rates must be positive'); + } + offeredLoad = resolved as unknown as ResolvedOfferedLoad; + } + if (record.resolvedWarmupTimeoutMs != null) { + requireSafeInteger( + record.resolvedWarmupTimeoutMs, + 'workload-progress.json resolvedWarmupTimeoutMs', + 1 + ); + } + if (record.workloadDurationMs != null) { + requireFinite(record.workloadDurationMs, 'workload-progress.json workloadDurationMs'); + } + return { + warmedSurfaces, + warmupLatencies, + samples, + canaries, + canarySchedule: parseCanarySchedule( + record.canarySchedule, + 'workload-progress.json canarySchedule' + ), + offeredLoad, + resolvedWarmupTimeoutMs: record.resolvedWarmupTimeoutMs as number | null, + workloadDurationMs: record.workloadDurationMs as number | null + }; +}; + +const parseRequestSample = (raw: unknown, label: string): RequestSample => { + const sample = requireRecord(raw, label); + requireAllowedKeys(sample, [ + 'tenantId', + 'surface', + 'operation', + 'capability', + 'latencyMs', + 'status', + 'ok', + 'phase', + 'scheduledAtMs', + 'errorCode', + 'oracleConfigured', + 'oracleConclusive', + 'oracleViolation', + 'oracleUnavailable', + 'postCoverageVerification' + ], label); + for (const key of [ + 'tenantId', + 'surface', + 'operation', + 'capability', + 'latencyMs', + 'status', + 'ok', + 'phase' + ]) { + if (!Object.prototype.hasOwnProperty.call(sample, key)) { + throw new Error(`${label} is missing ${key}`); + } + } + for (const key of ['tenantId', 'surface', 'operation', 'capability'] as const) { + requireNonEmptyString(sample[key], `${label}.${key}`); + } + const latencyMs = requireFinite(sample.latencyMs, `${label}.latencyMs`); + if (latencyMs < 0) throw new Error(`${label}.latencyMs must be non-negative`); + const status = requireSafeInteger(sample.status, `${label}.status`); + if (status > 599) throw new Error(`${label}.status is invalid`); + requireBoolean(sample.ok, `${label}.ok`); + if (!['coverage', 'workload'].includes(String(sample.phase))) { + throw new Error(`${label}.phase is invalid`); + } + if (sample.scheduledAtMs != null) { + requireFinite(sample.scheduledAtMs, `${label}.scheduledAtMs`); + } + if (sample.errorCode != null && typeof sample.errorCode !== 'string') { + throw new Error(`${label}.errorCode is invalid`); + } + for (const key of [ + 'oracleConfigured', + 'oracleConclusive', + 'oracleViolation', + 'oracleUnavailable', + 'postCoverageVerification' + ] as const) { + if (sample[key] != null && typeof sample[key] !== 'boolean') { + throw new Error(`${label}.${key} is invalid`); + } + } + return sample as unknown as RequestSample; +}; + +const parseCanaryResult = (raw: unknown, label: string): CanaryResult => { + const canary = requireRecord(raw, label); + requireAllowedKeys(canary, [ + 'tenantId', + 'surface', + 'canary', + 'phase', + 'periodicRound', + 'scheduledAt', + 'startedAt', + 'completedAt', + 'latencyMs', + 'conclusive', + 'violation', + 'detail' + ], label); + for (const key of [ + 'tenantId', + 'surface', + 'canary', + 'phase', + 'scheduledAt', + 'startedAt', + 'completedAt', + 'latencyMs', + 'conclusive', + 'violation' + ]) { + if (!Object.prototype.hasOwnProperty.call(canary, key)) { + throw new Error(`${label} is missing ${key}`); + } + } + for (const key of ['tenantId', 'surface', 'canary'] as const) { + requireNonEmptyString(canary[key], `${label}.${key}`); + } + if (!['initial', 'periodic', 'final'].includes(String(canary.phase))) { + throw new Error(`${label}.phase is invalid`); + } + if (canary.phase === 'periodic') { + requireSafeInteger(canary.periodicRound, `${label}.periodicRound`, 1); + } else if (canary.periodicRound != null) { + throw new Error(`${label}.periodicRound is only valid for periodic canaries`); + } + const scheduledAt = requireCanonicalTimestamp(canary.scheduledAt, `${label}.scheduledAt`); + const startedAt = requireCanonicalTimestamp(canary.startedAt, `${label}.startedAt`); + const completedAt = requireCanonicalTimestamp(canary.completedAt, `${label}.completedAt`); + if ( + Date.parse(startedAt) < Date.parse(scheduledAt) + || Date.parse(completedAt) < Date.parse(startedAt) + ) { + throw new Error(`${label} chronology is invalid`); + } + const latencyMs = requireFinite(canary.latencyMs, `${label}.latencyMs`); + if (latencyMs < 0) throw new Error(`${label}.latencyMs must be non-negative`); + requireBoolean(canary.conclusive, `${label}.conclusive`); + requireBoolean(canary.violation, `${label}.violation`); + if (canary.detail != null && typeof canary.detail !== 'string') { + throw new Error(`${label}.detail is invalid`); + } + return canary as unknown as CanaryResult; +}; + +const parseRequests = (artifactDir: string): RequestSample[] => { + const text = readRegularEvidenceFile(path.join(artifactDir, 'requests.ndjson')) + .toString('utf8'); + return text.split('\n').map((line) => line.trim()).filter(Boolean).map((line, index) => { + try { + return parseRequestSample( + JSON.parse(line), + `requests.ndjson line ${index + 1}` + ); + } catch (error) { + throw new Error( + `invalid requests.ndjson line ${index + 1}: ` + + `${error instanceof Error ? error.message : String(error)}` + ); + } + }); +}; + +const realtimeSurfaceKey = (surface: RealtimeDriverSnapshot['surfaces'][number]): string => + `${surface.tenantId}\0${surface.surface}\0${surface.route}`; + +const deriveRealtimeCoverage = ( + snapshot: RealtimeDriverSnapshot, + label: string +): RealtimeDeliveryCoverage | null => { + const reported = snapshot.timedCoverage; + if (reported == null) return null; + const summary = summarizeRealtimeReceiptEvidence({ + deliveryIntervalMs: snapshot.deliveryIntervalMs, + workloadStartedAt: reported.workloadStartedAt, + workloadDeadlineAt: reported.workloadDeadlineAt, + workloadEndedAt: reported.workloadEndedAt, + surfaces: snapshot.surfaces.map((surface) => ({ + tenantId: surface.tenantId, + surface: surface.surface, + route: surface.route, + expectedRecurringRounds: surface.timedRoundsExpected, + startedRecurringRounds: surface.timedRoundsStarted, + verifiedRecurringRounds: surface.timedRoundsVerified, + deadlineLateRecurringRounds: surface.timedRoundsDeadlineLate, + receipts: surface.correlationReceipts + })) + }); + const structurallyInvalid = summary.failures.find((failure) => + failure.startsWith('duplicate realtime surface:') + || failure.startsWith('invalid realtime receipt sequence:') + || failure.startsWith('invalid realtime receipt digest:') + || failure.startsWith('reused realtime receipt digest:') + || failure.startsWith('invalid realtime prime digest:') + || failure.startsWith('invalid realtime event digest:')); + if (structurallyInvalid) { + throw new Error(`${label} contains invalid receipt evidence: ${structurallyInvalid}`); + } + if (!isDeepStrictEqual(summary.coverage, reported)) { + throw new Error(`${label} reported coverage does not match its raw receipts`); + } + return summary.coverage; +}; + +const assertRealtimeHistoryAppendOnly = ( + previous: RealtimeDriverSnapshot, + current: RealtimeDriverSnapshot, + label: string +): void => { + const previousBySurface = new Map(previous.surfaces.map((surface) => [ + realtimeSurfaceKey(surface), + surface + ])); + const currentBySurface = new Map(current.surfaces.map((surface) => [ + realtimeSurfaceKey(surface), + surface + ])); + if ( + previousBySurface.size !== previous.surfaces.length + || currentBySurface.size !== current.surfaces.length + || previousBySurface.size !== currentBySurface.size + || [...previousBySurface.keys()].some((key) => !currentBySurface.has(key)) + ) { + throw new Error(`${label} realtime surface set changed`); + } + const monotonicSnapshotCounters: Array = [ + 'deliveryEvents', + 'deliveryRoundsStarted', + 'deliveryRoundsVerified' + ]; + for (const key of monotonicSnapshotCounters) { + if ((current[key] as number) < (previous[key] as number)) { + throw new Error(`${label} realtime aggregate counter regressed: ${key}`); + } + } + if (current.deliveryIntervalMs !== previous.deliveryIntervalMs) { + throw new Error(`${label} realtime delivery interval changed`); + } + for (const [key, prior] of previousBySurface) { + const next = currentBySurface.get(key)!; + for (const counter of [ + 'deliveryEvents', + 'deliveryRoundsStarted', + 'deliveryRoundsVerified', + 'timedRoundsExpected', + 'timedRoundsStarted', + 'timedRoundsVerified', + 'timedRoundsDeadlineLate' + ] as const) { + if (next[counter] < prior[counter]) { + throw new Error(`${label} realtime surface counter regressed: ${counter}`); + } + } + if ( + prior.correlationReceipts.length > next.correlationReceipts.length + || !prior.correlationReceipts.every((receipt, index) => + isDeepStrictEqual(receipt, next.correlationReceipts[index])) + ) { + throw new Error(`${label} realtime receipt history is not append-only`); + } + } + if ( + previous.errors.length > current.errors.length + || !previous.errors.every((error, index) => error === current.errors[index]) + ) { + throw new Error(`${label} realtime error history is not append-only`); + } +}; + +export const readRealtimeCoverageEvidence = ( + artifactDir: string +): RealtimeDeliveryCoverage | null => { + const raw = parseJsonEvidence(artifactDir, 'realtime-driver.json'); + if (!Array.isArray(raw)) throw new Error('realtime-driver.json must be an array'); + const entries = raw.map((value, index) => { + const entry = requireRecord(value, `realtime-driver.json[${index}]`); + requireExactKeys(entry, ['phase', 'timestamp', 'snapshot'], `realtime-driver.json[${index}]`); + if (typeof entry.phase !== 'string' || typeof entry.timestamp !== 'string') { + throw new Error(`realtime-driver.json[${index}] metadata is invalid`); + } + if ( + !Number.isFinite(Date.parse(entry.timestamp)) + || new Date(Date.parse(entry.timestamp)).toISOString() !== entry.timestamp + ) { + throw new Error(`realtime-driver.json[${index}] timestamp is invalid`); + } + const snapshot = requireRecord(entry.snapshot, `realtime-driver.json[${index}].snapshot`); + if (!Object.prototype.hasOwnProperty.call(snapshot, 'timedCoverage')) { + throw new Error(`realtime-driver.json[${index}].snapshot has no timedCoverage`); + } + return entry as unknown as RealtimeEvidenceEntry; + }); + const completed = entries.filter((entry) => entry.phase === 'timed-coverage-complete'); + if (completed.length > 1) { + throw new Error('realtime-driver.json has duplicate timed-coverage-complete records'); + } + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]; + deriveRealtimeCoverage(entry.snapshot, `realtime-driver.json[${index}]`); + if (index > 0) { + if (Date.parse(entry.timestamp) < Date.parse(entries[index - 1].timestamp)) { + throw new Error('realtime-driver.json timestamps regressed'); + } + assertRealtimeHistoryAppendOnly( + entries[index - 1].snapshot, + entry.snapshot, + `realtime-driver.json[${index}]` + ); + } + } + if (completed.length === 0) return null; + const completedIndex = entries.indexOf(completed[0]); + const coverage = deriveRealtimeCoverage( + completed[0].snapshot, + `realtime-driver.json[${completedIndex}]` + ); + if (!coverage || coverage.workloadEndedAt == null) { + throw new Error('timed-coverage-complete is not a terminal coverage transition'); + } + for (const [index, entry] of entries.entries()) { + if (index <= completedIndex || entry.snapshot.timedCoverage == null) continue; + const later = deriveRealtimeCoverage(entry.snapshot, `realtime-driver.json[${index}]`); + if (!isDeepStrictEqual(later, coverage)) { + throw new Error('timed realtime evidence changed after terminal coverage transition'); + } + } + return coverage; +}; + +const assertArtifactDirectory = (artifactDir: string, plan: DensityPlanV1): void => { + const root = fs.realpathSync(plan.artifactDir); + const stat = fs.lstatSync(artifactDir); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error('result artifactDir is not a regular directory'); + } + const realArtifactDir = fs.realpathSync(artifactDir); + const relative = path.relative(root, realArtifactDir); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative) || path.dirname(relative) !== '.') { + throw new Error('result artifactDir is outside the configured artifact root'); + } +}; + +const assertContextMatchesResult = ( + context: DensityScoreContextV1, + result: DensityRunResult, + plan: DensityPlanV1, + fleet: FleetV1 +): void => { + const planSha256 = sourceSha256(plan); + const fleetSha256 = sourceSha256(fleet); + const pairs: Array<[unknown, unknown, string]> = [ + [context.planSha256, planSha256, 'plan SHA-256'], + [context.fleetSha256, fleetSha256, 'fleet SHA-256'], + [context.campaignId, result.campaignId, 'campaign identity'], + [context.scheduleSha256, result.scheduleSha256, 'schedule SHA-256'], + [ + context.previousResultPayloadSha256, + result.previousResultPayloadSha256, + 'previous result payload SHA-256' + ], + [context.evidenceMode, result.evidenceMode, 'evidence mode'], + [context.runKind, result.runKind, 'run kind'], + [context.arm, result.arm, 'arm'], + [context.heapMiB, result.heapMiB, 'heap'], + [context.configuredCustomers, result.configuredCustomers, 'configured customers'], + [context.repetition, result.repetition, 'repetition'], + [context.runOrderIndex, result.runOrderIndex, 'run order'], + [context.startedAt, result.startedAt, 'start timestamp'], + [context.endedAt, result.endedAt, 'end timestamp'], + [context.serverExit, result.serverExit, 'server exit'], + [context.provenance, result.provenance, 'provenance'], + [context.provenanceErrors, result.provenanceErrors, 'provenance errors'], + [ + context.postgresRunAttestation, + result.postgresRunAttestation ?? null, + 'PostgreSQL run attestation' + ] + ]; + for (const [left, right, label] of pairs) { + if (!isDeepStrictEqual(left, right)) { + throw new Error(`score-context.json ${label} does not match the result/plan`); + } + } +}; + +const validatePostgresAttestation = ( + context: DensityScoreContextV1, + artifactDir: string +): void => { + const evidence = context.postgresRunAttestation; + if (!evidence) return; + const artifactPath = path.join(artifactDir, 'postgres-run-attestation.json'); + if (path.resolve(evidence.artifactPath) !== path.resolve(artifactPath)) { + throw new Error('PostgreSQL run attestation points outside the result artifact'); + } + const raw = parseJsonEvidence(artifactDir, 'postgres-run-attestation.json'); + const normalized = normalizePostgresRunAttestation(raw, { + arm: context.arm, + heapMiB: context.heapMiB, + tenantCount: context.configuredCustomers, + repetition: context.repetition, + runOrderIndex: context.runOrderIndex, + planSha256: context.planSha256, + fleetSha256: context.fleetSha256, + notBeforeEpochMs: context.notBeforeEpochMs, + artifactDir + }, artifactPath); + if (!isDeepStrictEqual(normalized, evidence)) { + throw new Error('PostgreSQL run attestation does not normalize to score-context evidence'); + } +}; + +export const reconstructScoreInput = ( + result: DensityRunResult, + plan: DensityPlanV1, + fleet: FleetV1 +): ScoreInput => { + assertArtifactDirectory(result.artifactDir, plan); + const context = readScoreContextEvidence(result.artifactDir); + assertContextMatchesResult(context, result, plan, fleet); + validatePostgresAttestation(context, result.artifactDir); + const arm = plan.arms.find((candidate) => candidate.name === context.arm); + if (!arm) throw new Error(`score-context.json uses unknown arm '${context.arm}'`); + if (context.externalServer !== !arm.command?.length) { + throw new Error('score-context.json external-server state contradicts the arm plan'); + } + if (context.evidenceMode === 'qualification') { + const configuredDurationSec = context.runKind === 'soak' + ? plan.soak?.durationSec + : plan.workload.durationSec; + if (configuredDurationSec == null || context.configuredDurationSec !== configuredDurationSec) { + throw new Error('score-context.json qualification duration contradicts the plan'); + } + } + if (context.configuredCustomers > fleet.tenants.length) { + throw new Error('score-context.json configured customer count exceeds the fleet'); + } + const memory = parseMemoryEvidence(result.artifactDir); + const postgres = parsePostgresMemoryEvidence(result.artifactDir); + const samples = parseRequests(result.artifactDir); + const canariesRaw = parseJsonEvidence(result.artifactDir, 'canaries.json'); + if (!Array.isArray(canariesRaw)) throw new Error('canaries.json must be an array'); + const canaries = canariesRaw.map((canary, index) => + parseCanaryResult(canary, `canaries.json[${index}]`)); + const canarySchedule = parseCanarySchedule( + parseJsonEvidence(result.artifactDir, 'canary-schedule.json'), + 'canary-schedule.json' + ); + const workload = parseWorkloadProgress(result.artifactDir); + const retainedRaw = requireRecord( + parseJsonEvidence(result.artifactDir, 'retained-memory.json'), + 'retained-memory.json' + ); + requireExactKeys(retainedRaw, ['baseline', 'final', 'errors'], 'retained-memory.json'); + requireStringArray(retainedRaw.errors, 'retained-memory.json errors'); + const retainedMemory: RetainedMemoryCheckpointPair = { + baseline: retainedRaw.baseline == null + ? null + : normalizeRetainedMemoryCheckpoint(retainedRaw.baseline), + final: retainedRaw.final == null + ? null + : normalizeRetainedMemoryCheckpoint(retainedRaw.final), + errors: retainedRaw.errors as string[] + }; + if ( + (retainedRaw.baseline != null && retainedMemory.baseline == null) + || (retainedRaw.final != null && retainedMemory.final == null) + || ( + retainedMemory.baseline != null + && !isDeepStrictEqual(retainedMemory.baseline, retainedRaw.baseline) + ) + || ( + retainedMemory.final != null + && !isDeepStrictEqual(retainedMemory.final, retainedRaw.final) + ) + ) { + throw new Error('retained-memory.json checkpoint shape is invalid'); + } + if ( + workload.samples !== samples.length + || workload.canaries !== canaries.length + || !isDeepStrictEqual(workload.canarySchedule, canarySchedule) + ) { + throw new Error('workload-progress.json counters or canary schedule are inconsistent'); + } + if ( + context.executionErrors.length === 0 + && ( + workload.offeredLoad == null + || workload.resolvedWarmupTimeoutMs == null + || workload.workloadDurationMs == null + ) + ) { + throw new Error('successful run evidence has incomplete workload progress'); + } + const tenants = resolveTenants( + fleet.tenants.slice(0, context.configuredCustomers), + arm + ); + const selectedWorkload = context.evidenceMode === 'diagnostic' + && context.configuredDurationSec === 5 + ? { + ...plan.workload, + durationSec: 5, + ...(plan.workload.rps != null + ? { rps: Math.min(plan.workload.rps, 5), rpsPerTenant: undefined } + : { + rps: undefined, + rpsPerTenant: Math.min( + plan.workload.rpsPerTenant!, + 5 / context.configuredCustomers + ) + }) + } + : { ...plan.workload, durationSec: context.configuredDurationSec }; + const offeredLoad = resolveOfferedLoad(selectedWorkload, context.configuredCustomers); + const surfaceCount = tenants.reduce((sum, tenant) => sum + tenant.surfaces.length, 0); + const resolvedWarmupTimeoutMs = resolveWarmupTimeoutMs(plan.workload, surfaceCount); + if ( + workload.offeredLoad != null + && !isDeepStrictEqual(workload.offeredLoad, offeredLoad) + ) { + throw new Error('workload-progress.json offered load contradicts the plan/fleet'); + } + if ( + workload.resolvedWarmupTimeoutMs != null + && workload.resolvedWarmupTimeoutMs !== resolvedWarmupTimeoutMs + ) { + throw new Error('workload-progress.json warmup timeout contradicts the plan/fleet'); + } + const tenantById = new Map(tenants.map((tenant) => [tenant.id, tenant])); + const warmedSurfaces = new Map>(); + for (const entry of workload.warmedSurfaces) { + const tenant = tenantById.get(entry.tenantId); + if (!tenant) throw new Error(`warmup evidence contains unknown tenant '${entry.tenantId}'`); + const configured = new Set(tenant.surfaces.map((surface) => surface.name)); + if (entry.surfaces.some((surface) => !configured.has(surface))) { + throw new Error(`warmup evidence contains an unknown surface for '${entry.tenantId}'`); + } + warmedSurfaces.set(entry.tenantId, new Set(entry.surfaces)); + } + const planSha256 = sourceSha256(plan); + const fleetSha256 = sourceSha256(fleet); + const realtimeDeliveryCoverage = readRealtimeCoverageEvidence(result.artifactDir); + if (context.executionErrors.length === 0 && realtimeDeliveryCoverage == null) { + throw new Error('successful run evidence has no timed realtime terminal transition'); + } + return { + arm: arm.name, + evidenceMode: context.evidenceMode, + campaignId: context.campaignId, + scheduleSha256: context.scheduleSha256, + previousResultPayloadSha256: context.previousResultPayloadSha256, + qualificationCohortSha256: sha256(`${planSha256}\0${fleetSha256}`), + commit: arm.commit, + introspectionMode: arm.introspectionMode, + heapMiB: context.heapMiB, + repetition: context.repetition, + expectedMatrixRepetitions: plan.repetitions, + runKind: context.runKind, + runOrderSeed: plan.runOrderSeed ?? DEFAULT_RUN_ORDER_SEED, + runOrderIndex: context.runOrderIndex, + startedAt: context.startedAt, + endedAt: context.endedAt, + configuredDurationSec: context.configuredDurationSec, + workloadDurationMs: workload.workloadDurationMs ?? 0, + artifactDir: result.artifactDir, + tenants, + warmedSurfaces, + warmupLatencies: workload.warmupLatencies, + resolvedWarmupTimeoutMs, + offeredLoad, + canaryIntervalSec: plan.workload.canaryIntervalSec, + periodicCanarySchedule: plan.workload.periodicCanarySchedule ?? 'full-sweep', + canarySchedule, + minWorkloadRequestsPerSurface: plan.workload.minWorkloadRequestsPerSurface, + samples, + canaries, + memorySnapshots: memory.snapshots, + postWarmupSnapshots: memory.snapshots.slice(Math.max(0, memory.warmupIndex)), + postWarmupNodeRssSnapshots: memory.osSnapshots.slice( + Math.max(0, memory.osWarmupIndex) + ), + retainedMemory, + memorySampleErrors: memory.errors, + postgresSnapshots: postgres.snapshots, + postgresSampleErrors: postgres.errors, + missedArrivals: samples.filter( + (sample) => sample.errorCode === 'LOAD_GENERATOR_MISSED_ARRIVAL' + ).length, + requiredCapabilities: [...plan.requiredCapabilities], + requiredCanaries: [...plan.requiredCanaries], + gates: plan.gates, + serverExit: context.serverExit, + provenance: context.provenance, + provenanceErrors: context.provenanceErrors, + postgresRunAttestation: context.postgresRunAttestation, + realtimeDeliveryCoverage, + externalServer: context.externalServer, + executionErrors: context.executionErrors + }; +}; + +export const assertResultSemanticReplay = ( + result: DensityRunResult, + plan: DensityPlanV1, + fleet: FleetV1, + label: string +): void => { + let replayed: DensityRunResult; + try { + replayed = scoreRun(reconstructScoreInput(result, plan, fleet)); + } catch (error) { + throw new Error( + `${label} semantic replay failed: ` + + `${error instanceof Error ? error.message : String(error)}` + ); + } + if (!isDeepStrictEqual(resultPayload(result), replayed)) { + throw new Error(`${label} result does not match semantic replay of raw evidence`); + } +}; diff --git a/packages/perf-harness/src/http.ts b/packages/perf-harness/src/http.ts new file mode 100644 index 0000000000..5a0c3be511 --- /dev/null +++ b/packages/perf-harness/src/http.ts @@ -0,0 +1,888 @@ +import { isDeepStrictEqual } from 'node:util'; + +import type { + CanaryRoundSummary, + CanaryResult, + CanaryScheduleSummary, + GraphqlOperation, + GraphqlSurface, + IsolationCanary, + JsonPathInvariant, + JsonPathMatch, + PeriodicCanarySchedule, + RequestSample, + ResolvedOfferedLoad, + TenantTarget, + WorkloadPlan +} from './types'; + +export interface WorkloadResult { + samples: RequestSample[]; + canaries: CanaryResult[]; + canarySchedule: CanaryScheduleSummary; + warmedSurfaces: Map>; + warmupLatencies: number[]; + capabilities: Set; + capabilitiesByTenantSurface: Map>; + missedArrivals: number; + workloadDurationMs: number; + offeredLoad: ResolvedOfferedLoad; + resolvedWarmupTimeoutMs: number; + warmupSurfaceCount: number; + warmupConcurrency: number; +} + +export interface WorkloadCapture { + samples: RequestSample[]; + canaries: CanaryResult[]; + canarySchedule: CanaryScheduleSummary | null; + warmedSurfaces: Map>; + warmupLatencies: number[]; + capabilities: Set; + capabilitiesByTenantSurface: Map>; +} + +export const createWorkloadCapture = (): WorkloadCapture => ({ + samples: [], + canaries: [], + canarySchedule: null, + warmedSurfaces: new Map(), + warmupLatencies: [], + capabilities: new Set(), + capabilitiesByTenantSurface: new Map() +}); + +export const resolveOfferedLoad = ( + plan: Pick, + tenantCount: number +): ResolvedOfferedLoad => { + if (!Number.isSafeInteger(tenantCount) || tenantCount <= 0) { + throw new Error('tenantCount must be a positive safe integer'); + } + const fixed = plan.rps; + const perTenant = plan.rpsPerTenant; + if ((fixed == null) === (perTenant == null)) { + throw new Error('workload must define exactly one of rps or rpsPerTenant'); + } + const configuredRps = fixed ?? perTenant!; + if (!Number.isFinite(configuredRps) || configuredRps <= 0) { + throw new Error('configured workload RPS must be positive'); + } + const totalRps = fixed ?? perTenant! * tenantCount; + if (!Number.isFinite(totalRps) || totalRps <= 0) { + throw new Error('resolved workload RPS must be positive'); + } + return { + mode: fixed == null ? 'per-tenant' : 'fixed-total', + configuredRps, + tenantCount, + totalRps, + rpsPerTenant: fixed == null ? perTenant! : fixed / tenantCount + }; +}; + +export const resolveWarmupTimeoutMs = ( + plan: Pick< + WorkloadPlan, + 'warmupTimeoutMs' | 'warmupTimeoutPerSurfaceMs' | 'warmupConcurrency' + >, + surfaceCount: number +): number => { + if (!Number.isSafeInteger(surfaceCount) || surfaceCount <= 0) { + throw new Error('warmup surface count must be a positive safe integer'); + } + const concurrency = plan.warmupConcurrency ?? 1; + if (!Number.isSafeInteger(concurrency) || concurrency <= 0) { + throw new Error('warmup concurrency must be a positive safe integer'); + } + if (!Number.isFinite(plan.warmupTimeoutMs) || plan.warmupTimeoutMs <= 0) { + throw new Error('warmupTimeoutMs must be positive'); + } + if ( + !Number.isFinite(plan.warmupTimeoutPerSurfaceMs) + || plan.warmupTimeoutPerSurfaceMs <= 0 + ) { + throw new Error('warmupTimeoutPerSurfaceMs must be positive'); + } + const waves = Math.ceil(surfaceCount / concurrency); + return Math.max(plan.warmupTimeoutMs, waves * plan.warmupTimeoutPerSurfaceMs); +}; + +interface GraphqlResponse { + status: number; + latencyMs: number; + body: unknown; + text: string; + ok: boolean; + errorCode?: string; + retryAfterMs: number; + oracleConfigured: boolean; + oracleConclusive: boolean; + oracleViolation: boolean; + oracleUnavailable: boolean; + postCoverageVerification?: boolean; +} + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +const decodePointerSegment = (segment: string): string => + segment.replace(/~1/g, '/').replace(/~0/g, '~'); + +/** Resolve an RFC 6901 pointer; `*` selects every child at that segment. */ +export const jsonPointerValues = (root: unknown, pointer: string): unknown[] => { + if (pointer === '') return [root]; + if (!pointer.startsWith('/')) return []; + let values: unknown[] = [root]; + for (const rawSegment of pointer.slice(1).split('/')) { + const segment = decodePointerSegment(rawSegment); + const next: unknown[] = []; + for (const value of values) { + if (segment === '*') { + if (Array.isArray(value)) next.push(...value); + else if (value && typeof value === 'object') next.push(...Object.values(value)); + continue; + } + if (Array.isArray(value)) { + const index = /^(0|[1-9]\d*)$/.test(segment) ? Number(segment) : -1; + if (index >= 0 && index < value.length) next.push(value[index]); + } else if ( + value && + typeof value === 'object' && + Object.prototype.hasOwnProperty.call(value, segment) + ) { + next.push((value as Record)[segment]); + } + } + values = next; + if (values.length === 0) break; + } + return values; +}; + +const matchesJsonPath = (body: unknown, match: JsonPathMatch): boolean => + jsonPointerValues(body, match.path).some((value) => isDeepStrictEqual(value, match.value)); + +const evaluateInvariant = ( + body: unknown, + invariant: JsonPathInvariant +): 'missing' | 'unexpected' | null => { + const values = jsonPointerValues(body, invariant.path); + if (values.length < invariant.min) return 'missing'; + if (invariant.max != null && values.length > invariant.max) return 'unexpected'; + return values.every((value) => isDeepStrictEqual(value, invariant.everyEquals)) + ? null + : 'unexpected'; +}; + +const applyResponseOracle = ( + response: Omit< + GraphqlResponse, + 'oracleConfigured' | 'oracleConclusive' | 'oracleViolation' | 'oracleUnavailable' + >, + operation: Pick +): GraphqlResponse => { + const configured = operation.requiredMatches != null + || operation.forbiddenMatches != null + || operation.invariants != null; + if (!configured) { + return { + ...response, + oracleConfigured: false, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: false + }; + } + if ( + !operation.requiredMatches?.length + || !operation.forbiddenMatches?.length + || (operation.invariants != null && operation.invariants.length === 0) + ) { + return { + ...response, + ok: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_INVALID', + oracleConfigured: true, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: false + }; + } + const forbidden = operation.forbiddenMatches.find((match) => + matchesJsonPath(response.body, match) + ); + const missing = operation.requiredMatches.find((match) => + !matchesJsonPath(response.body, match) + ); + const unexpectedInvariant = operation.invariants?.find((invariant) => + evaluateInvariant(response.body, invariant) === 'unexpected' + ); + const missingInvariant = operation.invariants?.find((invariant) => + evaluateInvariant(response.body, invariant) === 'missing' + ); + if (forbidden) { + return { + ...response, + ok: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_FORBIDDEN', + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: true, + oracleUnavailable: false + }; + } + if (unexpectedInvariant) { + return { + ...response, + ok: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_INVARIANT_UNEXPECTED', + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: true, + oracleUnavailable: false + }; + } + if (!response.ok) { + return { + ...response, + oracleConfigured: true, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: true + }; + } + if (missing || missingInvariant) { + return { + ...response, + ok: false, + errorCode: missingInvariant + ? 'GRAPHQL_OPERATION_ORACLE_INVARIANT_MISSING' + : 'GRAPHQL_OPERATION_ORACLE_MISSING', + oracleConfigured: true, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: false + }; + } + return { + ...response, + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: false, + oracleUnavailable: false + }; +}; + +export const mapWithConcurrency = async ( + items: readonly T[], + concurrency: number, + worker: (item: T, index: number) => Promise +): Promise => { + if (items.length === 0) return; + let cursor = 0; + const workers = Array.from({ length: Math.min(items.length, Math.max(1, concurrency)) }, async () => { + while (cursor < items.length) { + const index = cursor++; + await worker(items[index], index); + } + }); + await Promise.all(workers); +}; + +const requestGraphql = async ( + surface: GraphqlSurface, + operation: Pick< + GraphqlOperation, + 'query' | 'variables' | 'requiredMatches' | 'forbiddenMatches' | 'invariants' + >, + timeoutMs: number +): Promise => { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + const startedAt = performance.now(); + try { + const response = await fetch(surface.url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(surface.headers ?? {}) + }, + body: JSON.stringify({ query: operation.query, variables: operation.variables ?? {} }), + signal: controller.signal + }); + const text = await response.text(); + let body: any = null; + try { + body = JSON.parse(text); + } catch { + body = null; + } + const graphqlError = Array.isArray(body?.errors) && body.errors.length > 0; + const errorCode = graphqlError + ? body.errors[0]?.extensions?.code ?? 'GRAPHQL_ERROR' + : undefined; + const retryAfter = Number.parseInt(response.headers.get('retry-after') ?? '', 10); + return applyResponseOracle({ + status: response.status, + latencyMs: performance.now() - startedAt, + body, + text, + ok: response.ok && !graphqlError, + errorCode, + retryAfterMs: Number.isFinite(retryAfter) ? retryAfter * 1000 : 0 + }, operation); + } catch (error) { + return applyResponseOracle({ + status: 0, + latencyMs: performance.now() - startedAt, + body: null, + text: error instanceof Error ? error.message : String(error), + ok: false, + errorCode: error instanceof Error && error.name === 'AbortError' ? 'TIMEOUT' : 'NETWORK_ERROR', + retryAfterMs: 0 + }, operation); + } finally { + clearTimeout(timer); + } +}; + +const warmSurface = async ( + surface: GraphqlSurface, + deadline: number, + requestTimeoutMs: number +): Promise => { + let last: GraphqlResponse | null = null; + while (Date.now() < deadline) { + const remainingMs = Math.max(1, deadline - Date.now()); + last = await requestGraphql( + surface, + surface.warmup, + Math.min(requestTimeoutMs, remainingMs) + ); + if (last.ok) return last; + if (last.status !== 503) return last; + const retryDelayMs = Math.min( + Math.max(100, last.retryAfterMs), + Math.max(0, deadline - Date.now()) + ); + if (retryDelayMs > 0) await sleep(retryDelayMs); + } + return last ?? { + status: 0, + latencyMs: 0, + body: null, + text: 'global warmup deadline elapsed before this surface could start', + ok: false, + errorCode: 'WARMUP_DEADLINE', + retryAfterMs: 0, + oracleConfigured: surface.warmup.requiredMatches != null + || surface.warmup.forbiddenMatches != null + || surface.warmup.invariants != null, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: true + }; +}; + +const runCanary = async ( + tenant: TenantTarget, + surface: GraphqlSurface, + configuredCanaries: readonly IsolationCanary[], + timeoutMs: number, + metadata: { + phase: CanaryResult['phase']; + periodicRound?: number; + scheduledAt: string; + onCheckStarted?: () => void; + } +): Promise => { + const results: CanaryResult[] = []; + // A physical-density surface may have one pool slot permanently leased by + // realtime. Queueing every hostile probe at once behind the remaining slot + // lets the validation rig create head-of-line blocking for customer traffic. + // Submit probes one at a time so workload requests can interleave while the + // exact same fail-closed canary set is still exercised. + for (const canary of configuredCanaries) { + metadata.onCheckStarted?.(); + const startedAt = new Date().toISOString(); + const response = await requestGraphql(surface, canary, timeoutMs); + const completedAt = new Date().toISOString(); + const evidence = { + tenantId: tenant.id, + surface: surface.name, + canary: canary.name, + phase: metadata.phase, + ...(metadata.periodicRound != null + ? { periodicRound: metadata.periodicRound } + : {}), + scheduledAt: metadata.scheduledAt, + startedAt, + completedAt, + latencyMs: response.latencyMs + }; + results.push({ + ...evidence, + conclusive: response.oracleConclusive, + violation: response.oracleViolation, + ...(!response.ok ? { + detail: response.errorCode ?? `HTTP_${response.status}` + } : {}) + }); + } + return results; +}; + +/** + * Timed slots are strictly inside the workload window. A 15-minute workload + * with a 60-second interval therefore has rounds 1..14, never one at t=0 or + * exactly at the deadline (those boundaries belong to the full sweeps). + */ +export const periodicCanaryRoundCount = ( + durationMs: number, + intervalMs: number +): number => { + if (!Number.isFinite(durationMs) || durationMs < 0) { + throw new Error('canary schedule duration must be non-negative'); + } + if (!Number.isFinite(intervalMs) || intervalMs <= 0) { + throw new Error('canary schedule interval must be positive'); + } + return Math.max(0, Math.ceil(durationMs / intervalMs) - 1); +}; + +const stableOffset = (namespace: string, values: readonly string[], count: number): number => { + if (!Number.isSafeInteger(count) || count <= 0) return 0; + let hash = 0x811c9dc5; + for (const character of [namespace, ...values].join('\0')) { + hash ^= character.charCodeAt(0); + hash = Math.imul(hash, 0x01000193); + } + return (hash >>> 0) % count; +}; + +export const deterministicCanaryOffset = ( + tenantId: string, + surfaceName: string, + canaryCount: number +): number => stableOffset('canary', [tenantId, surfaceName], canaryCount); + +/** Return the selected zero-based canary index for a one-based periodic round. */ +export const rotatingCanaryIndex = ( + tenantId: string, + surfaceName: string, + canaryCount: number, + periodicRound: number +): number => { + if (!Number.isSafeInteger(canaryCount) || canaryCount <= 0) { + throw new Error('rotating canary selection requires at least one canary'); + } + if (!Number.isSafeInteger(periodicRound) || periodicRound <= 0) { + throw new Error('periodic canary round must be a positive safe integer'); + } + return ( + deterministicCanaryOffset(tenantId, surfaceName, canaryCount) + + periodicRound - 1 + ) % canaryCount; +}; + +const weightedOperations = (surface: GraphqlSurface): GraphqlOperation[] => { + const expanded: GraphqlOperation[] = []; + for (const operation of surface.operations) { + const weight = Math.max(1, Math.round((operation.weight ?? 1) * 10)); + for (let index = 0; index < weight; index++) expanded.push(operation); + } + return expanded; +}; + +/** + * Give each tenant/surface a reproducible position in its weighted operation + * schedule. Starting every surface at index zero creates fleet-wide operation + * waves that exaggerate one query shape at a time instead of exercising a + * mixed customer workload. + */ +export const deterministicOperationOffset = ( + tenantId: string, + surfaceName: string, + operationCount: number +): number => { + if (!Number.isSafeInteger(operationCount) || operationCount <= 0) return 0; + let hash = 0x811c9dc5; + for (const character of `${tenantId}\0${surfaceName}`) { + hash ^= character.charCodeAt(0); + hash = Math.imul(hash, 0x01000193); + } + return (hash >>> 0) % operationCount; +}; + +export const runWorkload = async ( + tenants: TenantTarget[], + plan: WorkloadPlan, + onWarmBoundary?: () => void | Promise, + capture: WorkloadCapture = createWorkloadCapture() +): Promise => { + const { + warmedSurfaces, + warmupLatencies, + capabilities, + capabilitiesByTenantSurface, + samples, + canaries + } = capture; + const surfaceTargets = tenants.flatMap((tenant) => tenant.surfaces.map((surface) => { + const operations = weightedOperations(surface); + return { + tenant, + surface, + operations, + cursor: deterministicOperationOffset(tenant.id, surface.name, operations.length) + }; + })); + + const warmupConcurrency = plan.warmupConcurrency ?? 1; + const resolvedWarmupTimeoutMs = resolveWarmupTimeoutMs(plan, surfaceTargets.length); + const warmupDeadline = Date.now() + resolvedWarmupTimeoutMs; + await mapWithConcurrency(surfaceTargets, warmupConcurrency, async ({ tenant, surface }) => { + const response = await warmSurface( + surface, + warmupDeadline, + plan.requestTimeoutMs + ); + warmupLatencies.push(response.latencyMs); + if (response.ok) { + const warmed = warmedSurfaces.get(tenant.id) ?? new Set(); + warmed.add(surface.name); + warmedSurfaces.set(tenant.id, warmed); + } + }); + let missedArrivals = 0; + + const recordResponse = ( + target: typeof surfaceTargets[number], + operation: GraphqlOperation, + response: GraphqlResponse, + phase: RequestSample['phase'], + scheduledAtMs?: number + ): void => { + if (response.ok) { + capabilities.add(operation.capability); + const capabilityKey = `${target.tenant.id}/${target.surface.name}`; + const localCapabilities = capabilitiesByTenantSurface.get(capabilityKey) ?? new Set(); + localCapabilities.add(operation.capability); + capabilitiesByTenantSurface.set(capabilityKey, localCapabilities); + } + samples.push({ + tenantId: target.tenant.id, + surface: target.surface.name, + operation: operation.name, + capability: operation.capability, + // Workload latency starts at the scheduled open-loop arrival, not when + // fetch happened to begin. This includes scheduler/event-loop delay and + // prevents coordinated omission from making a saturated arm look fast. + latencyMs: phase === 'workload' && scheduledAtMs != null + ? Math.max(response.latencyMs, performance.now() - scheduledAtMs) + : response.latencyMs, + status: response.status, + ok: response.ok, + phase, + oracleConfigured: response.oracleConfigured, + oracleConclusive: response.oracleConclusive, + oracleViolation: response.oracleViolation, + oracleUnavailable: response.oracleUnavailable, + ...(response.postCoverageVerification + ? { postCoverageVerification: true } + : {}), + ...(scheduledAtMs != null ? { scheduledAtMs } : {}), + ...(!response.ok ? { errorCode: response.errorCode ?? `HTTP_${response.status}` } : {}) + }); + }; + + const coverage = surfaceTargets.flatMap((target) => + target.surface.operations.map((operation) => ({ target, operation })) + ); + await mapWithConcurrency(coverage, warmupConcurrency, async ({ target, operation }) => { + const primary = await requestGraphql( + target.surface, + operation, + plan.requestTimeoutMs + ); + if (!primary.ok || !operation.postCoverageVerification) { + recordResponse(target, operation, primary, 'coverage'); + return; + } + const responseVariables: Record = {}; + for (const [name, pointer] of Object.entries( + operation.postCoverageVerification.variablesFromResponse ?? {} + )) { + const values = jsonPointerValues(primary.body, pointer); + if (values.length !== 1) { + recordResponse(target, operation, { + ...primary, + ok: false, + errorCode: values.length === 0 + ? 'GRAPHQL_POST_COVERAGE_VARIABLE_MISSING' + : 'GRAPHQL_POST_COVERAGE_VARIABLE_AMBIGUOUS', + oracleConfigured: true, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: false, + postCoverageVerification: true + }, 'coverage'); + return; + } + responseVariables[name] = values[0]; + } + const verification = await requestGraphql( + target.surface, + { + ...operation.postCoverageVerification, + variables: { + ...(operation.postCoverageVerification.variables ?? {}), + ...responseVariables + } + }, + plan.requestTimeoutMs + ); + recordResponse(target, operation, { + ...verification, + latencyMs: primary.latencyMs + verification.latencyMs, + postCoverageVerification: true + }, 'coverage'); + }); + + const canaryConcurrency = plan.canaryConcurrency + ?? Math.min(plan.maxInFlight, warmupConcurrency); + const runFullCanarySweep = async ( + phase: 'initial' | 'final' + ): Promise => { + const scheduledAt = new Date().toISOString(); + await mapWithConcurrency( + surfaceTargets, + canaryConcurrency, + async ({ tenant, surface }) => { + canaries.push(...await runCanary( + tenant, + surface, + surface.canaries, + plan.requestTimeoutMs, + { phase, scheduledAt } + )); + } + ); + }; + await runFullCanarySweep('initial'); + // The warm boundary is the first point at which every resident surface has + // been built, every configured capability has been exercised, and the + // initial isolation sweep has completed. The caller may perform additional + // awaited setup (for example, establishing realtime transports) before it + // marks memory warm. A callback failure must prevent timed traffic. + await onWarmBoundary?.(); + + const startedAt = performance.now(); + const startedWallMs = Date.now(); + const durationMs = plan.durationSec * 1000; + const deadline = startedAt + durationMs; + const deadlineWallMs = startedWallMs + durationMs; + const canaryIntervalMs = plan.canaryIntervalSec * 1000; + const periodicSchedule: PeriodicCanarySchedule = + plan.periodicCanarySchedule ?? 'full-sweep'; + const plannedPeriodicRounds = periodicCanaryRoundCount( + durationMs, + canaryIntervalMs + ); + const checksPerFullSweep = surfaceTargets.reduce( + (sum, target) => sum + target.surface.canaries.length, + 0 + ); + const checksPerRound = periodicSchedule === 'rotating-one' + ? surfaceTargets.length + : checksPerFullSweep; + const canarySchedule: CanaryScheduleSummary = { + schedule: periodicSchedule, + intervalMs: canaryIntervalMs, + durationMs, + canaryConcurrency, + startedAt: new Date(startedWallMs).toISOString(), + deadlineAt: new Date(deadlineWallMs).toISOString(), + planned: plannedPeriodicRounds, + started: 0, + completed: 0, + missed: plannedPeriodicRounds, + overlapped: 0, + deadlineLate: 0, + checksPlanned: plannedPeriodicRounds * checksPerRound, + checksStarted: 0, + checksCompleted: 0, + rounds: Array.from({ length: plannedPeriodicRounds }, (_unused, index): CanaryRoundSummary => ({ + periodicRound: index + 1, + plannedAt: new Date(startedWallMs + (index + 1) * canaryIntervalMs).toISOString(), + startedAt: null, + completedAt: null, + targetsPlanned: surfaceTargets.length, + targetsStarted: 0, + targetsCompleted: 0, + checksPlanned: checksPerRound, + checksStarted: 0, + checksCompleted: 0, + overlapped: false, + deadlineLate: false, + startDelayMs: null, + durationMs: null + })) + }; + capture.canarySchedule = canarySchedule; + + // This is a finite serialized schedule, not a setInterval callback. If a + // round overlaps the next slot it is recorded and drained before the next + // round starts; no validation round disappears behind a boolean guard. Each + // request has requestTimeoutMs, so the finite set of planned probes also + // gives the post-deadline drain a deterministic upper bound. + const periodicCanaries = (async (): Promise => { + let previousCompletedAt = startedAt; + for (const round of canarySchedule.rounds) { + const plannedAt = startedAt + round.periodicRound * canaryIntervalMs; + const waitMs = plannedAt - performance.now(); + if (waitMs > 0) await sleep(waitMs); + const roundStartedAt = performance.now(); + round.overlapped = round.periodicRound > 1 && previousCompletedAt > plannedAt; + round.startedAt = new Date().toISOString(); + round.startDelayMs = Math.max(0, roundStartedAt - plannedAt); + canarySchedule.started++; + if (round.overlapped) canarySchedule.overlapped++; + + await mapWithConcurrency( + surfaceTargets, + canaryConcurrency, + async ({ tenant, surface }) => { + round.targetsStarted++; + const selectedCanaries = periodicSchedule === 'rotating-one' + ? [surface.canaries[rotatingCanaryIndex( + tenant.id, + surface.name, + surface.canaries.length, + round.periodicRound + )]] + : surface.canaries; + const results = await runCanary( + tenant, + surface, + selectedCanaries, + plan.requestTimeoutMs, + { + phase: 'periodic', + periodicRound: round.periodicRound, + scheduledAt: round.plannedAt, + onCheckStarted: () => { + round.checksStarted++; + canarySchedule.checksStarted++; + } + } + ); + canaries.push(...results); + round.checksCompleted += results.length; + canarySchedule.checksCompleted += results.length; + round.targetsCompleted++; + } + ); + + previousCompletedAt = performance.now(); + round.completedAt = new Date().toISOString(); + round.durationMs = previousCompletedAt - roundStartedAt; + round.deadlineLate = previousCompletedAt > deadline; + canarySchedule.completed++; + canarySchedule.missed = canarySchedule.planned - canarySchedule.completed; + if (round.deadlineLate) canarySchedule.deadlineLate++; + } + })(); + + const offeredLoad = resolveOfferedLoad(plan, tenants.length); + const intervalMs = 1000 / offeredLoad.totalRps; + let nextAt = startedAt; + let sequence = 0; + const inFlight = new Set>(); + + const nextScheduledOperation = (): { + target: typeof surfaceTargets[number]; + operation: GraphqlOperation; + } => { + const target = surfaceTargets[sequence % surfaceTargets.length]; + sequence++; + const operation = target.operations[target.cursor % target.operations.length]; + target.cursor++; + return { target, operation }; + }; + + const dispatch = ( + target: typeof surfaceTargets[number], + operation: GraphqlOperation, + scheduledAtMs: number + ): void => { + const pending = requestGraphql(target.surface, operation, plan.requestTimeoutMs) + .then((response) => recordResponse( + target, + operation, + response, + 'workload', + scheduledAtMs + )) + .finally(() => inFlight.delete(pending)); + inFlight.add(pending); + }; + + while (performance.now() < deadline) { + const now = performance.now(); + if (now >= nextAt) { + // Advance directly to the next future arrival. Overdue arrivals become + // explicit failed samples, so saturation/event-loop stalls cannot hide + // latency through coordinated omission and never trigger a catch-up burst. + const due = Math.floor((now - nextAt) / intervalMs) + 1; + const canDispatchLatest = inFlight.size < plan.maxInFlight; + for (let slot = 0; slot < due; slot++) { + const scheduledAtMs = nextAt + slot * intervalMs; + const { target, operation } = nextScheduledOperation(); + if (canDispatchLatest && slot === due - 1) { + dispatch(target, operation, scheduledAtMs); + continue; + } + missedArrivals++; + recordResponse(target, operation, { + status: 0, + latencyMs: Math.max(plan.requestTimeoutMs, now - scheduledAtMs), + body: null, + text: 'scheduled arrival missed before dispatch', + ok: false, + errorCode: 'LOAD_GENERATOR_MISSED_ARRIVAL', + retryAfterMs: 0, + oracleConfigured: operation.requiredMatches != null + || operation.forbiddenMatches != null + || operation.invariants != null, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: true + }, 'workload', scheduledAtMs); + } + nextAt += due * intervalMs; + continue; + } + await sleep(Math.min(20, Math.max(1, nextAt - now))); + } + const workloadDurationMs = performance.now() - startedAt; + await Promise.all(inFlight); + await periodicCanaries; + await runFullCanarySweep('final'); + + return { + samples, + canaries, + canarySchedule, + warmedSurfaces, + warmupLatencies, + capabilities, + capabilitiesByTenantSurface, + missedArrivals, + workloadDurationMs, + offeredLoad, + resolvedWarmupTimeoutMs, + warmupSurfaceCount: surfaceTargets.length, + warmupConcurrency + }; +}; diff --git a/packages/perf-harness/src/index.ts b/packages/perf-harness/src/index.ts new file mode 100644 index 0000000000..8b7aacf941 --- /dev/null +++ b/packages/perf-harness/src/index.ts @@ -0,0 +1,127 @@ +#!/usr/bin/env node +import { runCatalogBench, runCatalogBenchWorker } from './catalog-bench'; +import { loadFleet, loadPlan, validateCoverage } from './config'; +import { writeReport } from './report'; +import { runDensityPlan } from './run'; + +const parseList = (value: string | undefined): string[] | undefined => value + ? value.split(',').map((item) => item.trim()).filter(Boolean) + : undefined; + +const parseNumbers = (value: string | undefined): number[] | undefined => parseList(value)?.map((item) => { + const parsed = Number(item); + if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`invalid positive integer '${item}'`); + return parsed; +}); + +const parsePositiveInteger = (value: string | undefined, label: string): number | undefined => { + if (value == null) return undefined; + const parsed = Number.parseInt(value, 10); + if (!Number.isInteger(parsed) || parsed <= 0 || String(parsed) !== value) { + throw new Error(`${label} must be a positive integer`); + } + return parsed; +}; + +const flag = (name: string): string | undefined => { + const index = process.argv.indexOf(`--${name}`); + return index >= 0 ? process.argv[index + 1] : undefined; +}; + +const hasFlag = (name: string): boolean => process.argv.includes(`--${name}`); + +const usage = (): void => { + process.stdout.write(`cperf — local Graphile tenant-density research harness + + cperf validate --plan [--allow-reserved-ports] + cperf run --plan [--arm a,b] [--heaps 1024,2048] [--tenants 1,5] [--repetitions 3] [--smoke] + cperf report --plan --results --out + cperf catalog-bench --database --mode stock|scoped-required --schemas a,b --instances 1,2 --out + cperf catalog-bench --database --mode stock|scoped-required --surface-schemas a,b --allowed-dependency-schemas deps,private --instances 1 --out + [--scoped-catalog-types all|dependency-closure] + [--introspection-client-release-mode reuse|destroy] + [--release-build-state-after-validation] + [--v8-profile stock|optimize-for-size|baseline-optimize-for-size|jitless-optimize-for-size] + [--warm-operations-per-instance 500] [--expected-tokens token-a,token-b] + [--warm-operation-replay-passes 3] + [--grafast-query-cache-max 8] [--grafast-operations-cache-max 8] + [--grafast-operation-plans-cache-max 8] + [--tenant-proxy-surfaces 5] + +Full runs honor the plan's 15-minute matrix and optional two-hour soak. --smoke +forces one five-second run and can never satisfy the qualification gates. +`); +}; + +const main = async (): Promise => { + const command = process.argv[2]; + if (command === '__catalog-worker') { + await runCatalogBenchWorker( + requireFlagForWorker('config'), + requireFlagForWorker('result') + ); + return 0; + } + if (command === 'catalog-bench') { + await runCatalogBench(process.argv.slice(3)); + return 0; + } + const planFile = flag('plan'); + if (!command || !planFile || hasFlag('help')) { + usage(); + return command && hasFlag('help') ? 0 : 1; + } + const plan = loadPlan(planFile, hasFlag('allow-reserved-ports')); + const fleet = loadFleet(plan.fleetFile); + validateCoverage(plan, fleet); + if (command === 'validate') { + process.stdout.write( + `valid plan: ${plan.arms.length} arms, ${fleet.tenants.length} tenants, ` + + `${plan.heapMiB.length} heaps, ${plan.repetitions} repetitions\n` + ); + return 0; + } + if (command === 'run') { + await runDensityPlan(plan, fleet, { + arms: parseList(flag('arm')), + heaps: parseNumbers(flag('heaps')), + tenantCounts: parseNumbers(flag('tenants')), + repetitions: parsePositiveInteger(flag('repetitions'), 'repetitions'), + smoke: hasFlag('smoke') + }); + return 0; + } + if (command === 'report') { + const results = flag('results'); + const output = flag('out'); + if (!results || !output) throw new Error('report requires --results and --out'); + writeReport(results, output, plan, fleet); + return 0; + } + usage(); + return 1; +}; + +const requireFlagForWorker = (name: string): string => { + const value = flag(name); + if (!value) throw new Error(`catalog worker requires --${name}`); + return value; +}; + +void main().then((code) => { + process.exitCode = code; +}, (error) => { + process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`); + process.exitCode = 1; +}); + +export * from './catalog-bench'; +export * from './config'; +export * from './http'; +export * from './memory'; +export * from './postgres'; +export * from './report'; +export * from './run'; +export * from './run-attestation'; +export * from './score'; +export * from './types'; diff --git a/packages/perf-harness/src/memory.ts b/packages/perf-harness/src/memory.ts new file mode 100644 index 0000000000..7b95ddbbd2 --- /dev/null +++ b/packages/perf-harness/src/memory.ts @@ -0,0 +1,650 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import type { + MemorySnapshot, + NodeRssSnapshot, + RetainedMemoryCheckpoint, + RetainedMemoryGuard, + RetainedMemorySample +} from './types'; + +const finiteNumber = (value: unknown): number | null => typeof value === 'number' + && Number.isFinite(value) + ? value + : null; + +const positiveNumber = (value: unknown): number | null => { + const parsed = finiteNumber(value); + return parsed != null && parsed > 0 ? parsed : null; +}; + +const sumNumbers = (value: unknown): number | null => { + if (typeof value === 'number' && Number.isFinite(value)) return value; + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const numbers = Object.values(value).map(sumNumbers); + return numbers.every((item): item is number => item != null) + ? numbers.reduce((sum, item) => sum + item, 0) + : null; +}; + +const stringArray = (value: unknown): string[] | null => Array.isArray(value) + && value.every((item) => typeof item === 'string') + ? [...value] + : null; + +const booleanValue = (value: unknown): boolean | null => typeof value === 'boolean' + ? value + : null; + +const cacheAdmissionMode = ( + value: unknown +): MemorySnapshot['cacheAdmissionMode'] => value === 'evict-idle' + || value === 'preserve-resident' + ? value + : null; + +const realtimeNotificationMode = ( + value: unknown +): MemorySnapshot['realtimeNotificationMode'] => value === 'dedicated' + || value === 'shared-exact' + ? value + : null; + +const runtimePoolTelemetryScope = ( + value: unknown +): MemorySnapshot['runtimePoolTelemetryScope'] => + value === 'runtime-only-exact-identities' ? value : null; + +const maxUsesValue = (value: unknown): number | null => + Number.isSafeInteger(value) && (value as number) > 0 ? value as number : null; + +/** Convert Node's process.resourceUsage().maxRSS KiB value to bytes. */ +const resourcePeakRssBytes = (raw: any): number | null => { + const maxRssKiB = positiveNumber(raw?.resourceUsage?.maxRSS); + return maxRssKiB == null ? null : maxRssKiB * 1024; +}; + +export const normalizeMemorySnapshot = (raw: any): MemorySnapshot => ({ + timestamp: typeof raw?.timestamp === 'string' ? raw.timestamp : new Date().toISOString(), + pid: Number.isSafeInteger(raw?.pid) && raw.pid > 0 ? raw.pid : null, + nodeEnv: typeof raw?.nodeEnv === 'string' ? raw.nodeEnv : null, + heapLimitBytes: positiveNumber(raw?.v8?.heapStatistics?.heap_size_limit), + heapUsedBytes: finiteNumber(raw?.memory?.heapUsedBytes), + rssBytes: positiveNumber(raw?.memory?.rssBytes), + processPeakRssBytes: resourcePeakRssBytes(raw), + cacheSize: finiteNumber(raw?.graphileCache?.size), + cacheConfiguredMax: finiteNumber(raw?.graphileCache?.max), + cacheBudgetCapacity: finiteNumber(raw?.graphileCache?.budgetCapacity), + cacheInstanceHeapBytes: finiteNumber(raw?.graphileCache?.instanceHeapBytes), + cacheCalibrationId: typeof raw?.graphileCache?.calibration?.id === 'string' + ? raw.graphileCache.calibration.id + : null, + cacheAdmissionMode: cacheAdmissionMode(raw?.graphileCache?.admissionMode), + residentBuildContractFingerprints: stringArray( + raw?.physicalDatabaseFixture?.contractEvidence + ?.residentGraphileBuildFingerprints + ), + residentBuildContracts: stringArray(raw?.graphileCache?.keys), + evictions: sumNumbers(raw?.graphileCacheCounters?.evictions), + buildRefusals: sumNumbers(raw?.graphileCacheCounters?.buildRefusals), + buildsStarted: finiteNumber(raw?.graphileGovernor?.buildsStarted + ?? raw?.graphileBuilds?.started), + buildsSucceeded: finiteNumber(raw?.graphileBuilds?.succeeded), + buildMaxMs: finiteNumber(raw?.graphileBuilds?.maxMs), + pgPoolCacheSize: finiteNumber(raw?.pgCache?.size), + pgPoolLeasedPools: finiteNumber(raw?.pgCache?.leasedPools), + pgPoolActiveLeases: finiteNumber(raw?.pgCache?.activeLeases), + pgPoolCapacityEvictions: finiteNumber(raw?.pgCache?.capacityEvictions), + pgPoolCapacityRefusals: finiteNumber(raw?.pgCache?.capacityRefusals), + pgPoolDisposalFailures: finiteNumber(raw?.pgCache?.disposalFailures), + pgPoolTotalClients: finiteNumber( + raw?.pgCache?.totalClients + ?? raw?.physicalDatabaseFixture?.pools?.totalClients + ), + pgPoolIdleClients: finiteNumber( + raw?.pgCache?.idleClients + ?? raw?.physicalDatabaseFixture?.pools?.idleClients + ), + pgPoolWaitingClients: finiteNumber( + raw?.pgCache?.waitingClients + ?? raw?.physicalDatabaseFixture?.pools?.waitingClients + ), + runtimePoolTelemetryScope: runtimePoolTelemetryScope( + raw?.physicalDatabaseFixture?.pools?.scope + ), + runtimePoolTelemetryAvailable: booleanValue( + raw?.physicalDatabaseFixture?.pools?.available + ), + runtimePoolRequestedMaxUses: maxUsesValue( + raw?.physicalDatabaseFixture?.pools?.requestedMaxUses + ), + runtimePoolEffectiveMaxUses: maxUsesValue( + raw?.physicalDatabaseFixture?.pools?.effectiveMaxUses + ), + runtimePoolEffectiveMaxUsesKnown: booleanValue( + raw?.physicalDatabaseFixture?.pools?.effectiveMaxUsesKnown + ), + runtimePoolMaxUsesExact: booleanValue( + raw?.physicalDatabaseFixture?.pools?.maxUsesExact + ), + runtimePoolExpectedPools: finiteNumber( + raw?.physicalDatabaseFixture?.pools?.expectedPools + ), + runtimePoolObservedPools: finiteNumber( + raw?.physicalDatabaseFixture?.pools?.observedPools + ), + runtimePoolTotalClients: finiteNumber( + raw?.physicalDatabaseFixture?.pools?.totalClients + ), + runtimePoolIdleClients: finiteNumber( + raw?.physicalDatabaseFixture?.pools?.idleClients + ), + runtimePoolWaitingClients: finiteNumber( + raw?.physicalDatabaseFixture?.pools?.waitingClients + ), + postgresBackendTotal: finiteNumber(raw?.physicalDatabaseFixture?.backends?.total), + postgresBackendActive: finiteNumber(raw?.physicalDatabaseFixture?.backends?.active), + postgresBackendIdle: finiteNumber(raw?.physicalDatabaseFixture?.backends?.idle), + postgresBackendIdleInTransaction: finiteNumber( + raw?.physicalDatabaseFixture?.backends?.idleInTransaction + ), + physicalDatabases: finiteNumber(raw?.physicalDatabaseFixture?.physicalDatabases), + postgresContainerDedicated: booleanValue( + raw?.physicalDatabaseFixture?.containerScope?.dedicated + ), + unexpectedPostgresDatabases: finiteNumber( + raw?.physicalDatabaseFixture?.containerScope?.unexpectedDatabases + ), + realtimeManagersExpected: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.managersExpected + ), + realtimeManagersActive: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.managersActive + ), + realtimeTransportsExpected: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.transportsExpected + ), + realtimeTransportsActive: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.transportsActive + ), + realtimeNotificationMode: realtimeNotificationMode( + raw?.physicalDatabaseFixture?.realtime?.notificationMode + ), + notificationBrokers: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.brokers + ), + notificationListenerConnections: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.listenerConnections + ), + notificationBrokerLeases: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.leases + ), + notificationBrokerTopics: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.topics + ), + notificationBrokerSubscribers: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.subscribers + ), + notificationBrokerQueueOverflows: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.queueOverflows + ), + notificationBrokerFatalFailures: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.fatalFailures + ), + notificationAuditIdentities: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.identities + ), + notificationAuditsHealthy: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.healthy + ), + notificationAuditsFailed: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.failed + ), + notificationAuditsStale: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.stale + ), + notificationAuditAttempts: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.catalogAuditAttempts + ), + notificationAuditFailures: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.catalogAuditFailures + ), + notificationAuditActiveDatabaseTargets: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.activeDatabaseTargets + ), + notificationAuditDatabaseConflicts: finiteNumber( + raw?.physicalDatabaseFixture?.realtime + ?.notificationRoleAudits?.databaseConfigurationConflicts + ), + cacheCountersAvailable: sumNumbers(raw?.graphileCacheCounters?.evictions) != null + && sumNumbers(raw?.graphileCacheCounters?.buildRefusals) != null, + buildCountersAvailable: finiteNumber( + raw?.graphileGovernor?.buildsStarted ?? raw?.graphileBuilds?.started + ) != null, + raw +}); + +const normalizeRetainedMemorySample = (raw: any): RetainedMemorySample | null => { + const heapUsedBytes = finiteNumber(raw?.heapUsedBytes); + const externalBytes = finiteNumber(raw?.externalBytes); + const arrayBuffersBytes = finiteNumber(raw?.arrayBuffersBytes); + const rssBytes = positiveNumber(raw?.rssBytes); + if ( + typeof raw?.timestamp !== 'string' + || typeof raw?.monotonicNs !== 'string' + || !/^\d+$/.test(raw.monotonicNs) + || heapUsedBytes == null + || heapUsedBytes < 0 + || externalBytes == null + || externalBytes < 0 + || arrayBuffersBytes == null + || arrayBuffersBytes < 0 + || rssBytes == null + ) return null; + return { + timestamp: raw.timestamp, + monotonicNs: raw.monotonicNs, + heapUsedBytes, + externalBytes, + arrayBuffersBytes, + rssBytes + }; +}; + +const normalizeRetainedMemoryGuard = (raw: any): RetainedMemoryGuard | null => { + if ( + !Number.isSafeInteger(raw?.pid) + || raw.pid <= 0 + || !Number.isSafeInteger(raw?.graphileInFlight) + || raw.graphileInFlight < 0 + || !Array.isArray(raw?.residentBuildContracts) + || raw.residentBuildContracts.some((value: unknown) => typeof value !== 'string') + || typeof raw?.stateSha256 !== 'string' + || !/^sha256:[a-f0-9]{64}$/.test(raw.stateSha256) + || !raw?.state + || typeof raw.state !== 'object' + || Array.isArray(raw.state) + ) return null; + return { + pid: raw.pid, + graphileInFlight: raw.graphileInFlight, + residentBuildContracts: [...raw.residentBuildContracts], + stateSha256: raw.stateSha256, + state: raw.state + }; +}; + +export const normalizeRetainedMemoryCheckpoint = ( + raw: any +): RetainedMemoryCheckpoint | null => { + const samples: Array = Array.isArray(raw?.samples) + ? raw.samples.map(normalizeRetainedMemorySample) + : []; + const guardBefore = normalizeRetainedMemoryGuard(raw?.guardBefore); + const guardAfter = normalizeRetainedMemoryGuard(raw?.guardAfter); + if ( + raw?.version !== 1 + || typeof raw?.fixture !== 'string' + || !Number.isSafeInteger(raw?.pid) + || raw.pid <= 0 + || !Number.isSafeInteger(raw?.gcRounds) + || raw.gcRounds < 5 + || raw.gcRounds > 8 + || !Number.isSafeInteger(raw?.stableSampleCount) + || raw.stableSampleCount !== 3 + || typeof raw?.stable !== 'boolean' + || samples.length !== raw.gcRounds + || samples.some((sample) => sample == null) + || !guardBefore + || !guardAfter + || !Array.isArray(raw?.errors) + || raw.errors.some((error: unknown) => typeof error !== 'string') + ) return null; + return { + version: 1, + fixture: raw.fixture, + pid: raw.pid, + gcRounds: raw.gcRounds, + stableSampleCount: 3, + stable: raw.stable, + samples: samples as RetainedMemorySample[], + guardBefore, + guardAfter, + errors: [...raw.errors] + }; +}; + +export interface LinuxProcessMemory { + rssBytes: number | null; + peakRssBytes: number | null; +} + +const statusKiB = (status: string, field: 'VmRSS' | 'VmHWM'): number | null => { + const match = new RegExp(`^${field}:\\s+(\\d+)\\s+kB$`, 'm').exec(status); + return match ? Number(match[1]) * 1024 : null; +}; + +/** Read current and cumulative peak RSS for one exact Linux process. */ +export const readLinuxProcessMemory = ( + pid: number, + procRoot = '/proc', + onError?: (message: string) => void +): LinuxProcessMemory | null => { + try { + const status = fs.readFileSync(path.join(procRoot, String(pid), 'status'), 'utf8'); + const rssBytes = statusKiB(status, 'VmRSS'); + const peakRssBytes = statusKiB(status, 'VmHWM'); + if (rssBytes == null) onError?.(`OS RSS proc status for pid ${pid} omitted VmRSS`); + if (peakRssBytes == null) onError?.(`OS RSS proc status for pid ${pid} omitted VmHWM`); + if (rssBytes == null && peakRssBytes == null) return null; + return { rssBytes, peakRssBytes }; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + onError?.(`OS RSS proc read failed for pid ${pid}: ${detail}`); + return null; + } +}; + +export interface MemorySamplerOptions { + intervalMs?: number; + osSampleIntervalMs?: number; + expectedPid?: number | null; + expectedHeapLimitBytes?: number | null; + procRoot?: string; + /** Auto uses /proc on Linux and the authenticated memory endpoint elsewhere. */ + currentRssSource?: 'auto' | 'proc' | 'authenticated-endpoint'; + /** Ephemeral request headers; callers must never persist bearer credentials. */ + headers?: Readonly>; +} + +export interface MemorySampler { + snapshots: MemorySnapshot[]; + /** High-frequency, harness-timestamped current RSS samples. */ + osSnapshots: NodeRssSnapshot[]; + errors: string[]; + ready: Promise; + markWarmupComplete(): Promise; + stop(): Promise; + warmupIndex: number; + osWarmupIndex: number; + osPeakRssBytes: number | null; +} + +const samplerOptions = (options: number | MemorySamplerOptions): Required> & Pick & { + currentRssSource: 'proc' | 'authenticated-endpoint'; + headers: Readonly>; +} => { + if (typeof options === 'number') { + const currentRssSource = process.platform === 'linux' + ? 'proc' + : 'authenticated-endpoint'; + return { + intervalMs: options, + osSampleIntervalMs: Math.min( + currentRssSource === 'proc' ? 100 : 250, + options + ), + expectedPid: null, + expectedHeapLimitBytes: null, + procRoot: '/proc', + currentRssSource, + headers: Object.freeze({}) + }; + } + const requestedSource = options.currentRssSource ?? 'auto'; + if (!['auto', 'proc', 'authenticated-endpoint'].includes(requestedSource)) { + throw new Error(`unknown current RSS source '${String(requestedSource)}'`); + } + const currentRssSource = requestedSource === 'auto' + ? (options.procRoot != null || process.platform === 'linux' + ? 'proc' + : 'authenticated-endpoint') + : requestedSource; + return { + intervalMs: options.intervalMs ?? 1000, + osSampleIntervalMs: options.osSampleIntervalMs + ?? (currentRssSource === 'proc' ? 100 : 250), + expectedPid: options.expectedPid ?? null, + expectedHeapLimitBytes: options.expectedHeapLimitBytes ?? null, + procRoot: options.procRoot ?? '/proc', + currentRssSource, + headers: Object.freeze({ ...(options.headers ?? {}) }) + }; +}; + +const hasBearerAuthorization = ( + headers: Readonly> +): boolean => Object.entries(headers).some(([name, value]) => + name.toLowerCase() === 'authorization' && /^Bearer\s+\S+$/.test(value) +); + +export const startMemorySampler = ( + url: string, + options: number | MemorySamplerOptions = {} +): MemorySampler => { + const resolved = samplerOptions(options); + if (!Number.isFinite(resolved.intervalMs) || resolved.intervalMs <= 0) { + throw new Error(`memory sample interval must be positive, received ${resolved.intervalMs}`); + } + if (!Number.isFinite(resolved.osSampleIntervalMs) || resolved.osSampleIntervalMs <= 0) { + throw new Error(`OS memory sample interval must be positive, received ${resolved.osSampleIntervalMs}`); + } + + const snapshots: MemorySnapshot[] = []; + const osSnapshots: NodeRssSnapshot[] = []; + const errors: string[] = []; + const observedErrors = new Set(); + let stopped = false; + let inFlight: Promise | null = null; + let osInFlight: Promise | null = null; + let warmupIndex = -1; + let osWarmupIndex = -1; + let osPeakRssBytes: number | null = null; + + const recordError = (message: string): void => { + if (observedErrors.has(message)) return; + observedErrors.add(message); + errors.push(message); + }; + + const validateIdentity = (snapshot: MemorySnapshot): void => { + if (resolved.expectedPid == null) recordError('expected server pid is unavailable'); + else if (snapshot.pid == null) recordError('memory endpoint pid is unavailable'); + else if (snapshot.pid !== resolved.expectedPid) { + recordError(`memory endpoint pid mismatch: expected ${resolved.expectedPid}, observed ${snapshot.pid}`); + } + if (snapshot.nodeEnv !== 'production') { + recordError(`memory endpoint NODE_ENV must be production, observed ${snapshot.nodeEnv ?? 'unknown'}`); + } + if (resolved.expectedHeapLimitBytes == null) recordError('expected V8 heap limit is unavailable'); + else if (snapshot.heapLimitBytes == null) recordError('memory endpoint V8 heap limit is unavailable'); + else if (snapshot.heapLimitBytes !== resolved.expectedHeapLimitBytes) { + recordError( + `V8 heap limit mismatch: expected ${resolved.expectedHeapLimitBytes}, observed ${snapshot.heapLimitBytes}` + ); + } + }; + + const validate = (snapshot: MemorySnapshot): void => { + validateIdentity(snapshot); + if (snapshot.heapUsedBytes == null) recordError('memory endpoint heap usage is unavailable'); + if (snapshot.rssBytes == null) recordError('memory endpoint RSS is unavailable'); + if (snapshot.processPeakRssBytes == null && osPeakRssBytes == null) { + recordError('process peak RSS is unavailable'); + } + if ( + snapshot.pgPoolCacheSize == null + || snapshot.pgPoolLeasedPools == null + || snapshot.pgPoolActiveLeases == null + || snapshot.pgPoolCapacityEvictions == null + || snapshot.pgPoolCapacityRefusals == null + || snapshot.pgPoolDisposalFailures == null + ) { + recordError('PostgreSQL pool-cache telemetry is unavailable'); + } + }; + + const sampleOs = async (): Promise => { + if (resolved.expectedPid == null) return; + if (resolved.currentRssSource === 'authenticated-endpoint') { + if (!hasBearerAuthorization(resolved.headers)) { + recordError('authenticated memory-endpoint RSS sampling requires bearer authorization'); + return; + } + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5_000); + const startedAtMs = Date.now(); + try { + const response = await fetch(url, { + signal: controller.signal, + headers: resolved.headers + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const snapshot = normalizeMemorySnapshot(await response.json()); + const endedAtMs = Date.now(); + validateIdentity(snapshot); + if (snapshot.pid !== resolved.expectedPid || snapshot.rssBytes == null) { + recordError('authenticated memory-endpoint current RSS sample is unavailable'); + return; + } + osSnapshots.push({ + timestamp: new Date( + startedAtMs + ((endedAtMs - startedAtMs) / 2) + ).toISOString(), + pid: resolved.expectedPid, + source: 'authenticated-endpoint', + rssBytes: snapshot.rssBytes + }); + if (snapshot.processPeakRssBytes != null) { + osPeakRssBytes = Math.max( + osPeakRssBytes ?? 0, + snapshot.processPeakRssBytes + ); + } + } catch (error) { + recordError( + `authenticated memory-endpoint RSS sample failed: ${error instanceof Error ? error.message : String(error)}` + ); + } finally { + clearTimeout(timeout); + } + return; + } + const startedAtMs = Date.now(); + const processMemory = readLinuxProcessMemory( + resolved.expectedPid, + resolved.procRoot, + recordError + ); + const endedAtMs = Date.now(); + if (processMemory?.rssBytes != null) { + osSnapshots.push({ + timestamp: new Date(startedAtMs + ((endedAtMs - startedAtMs) / 2)).toISOString(), + pid: resolved.expectedPid, + source: 'proc', + rssBytes: processMemory.rssBytes + }); + } + if (processMemory?.peakRssBytes != null) { + osPeakRssBytes = Math.max(osPeakRssBytes ?? 0, processMemory.peakRssBytes); + } else if (processMemory?.rssBytes != null) { + osPeakRssBytes = Math.max(osPeakRssBytes ?? 0, processMemory.rssBytes); + } + const lastSnapshot = snapshots[snapshots.length - 1]; + if (lastSnapshot && osPeakRssBytes != null) { + lastSnapshot.processPeakRssBytes = Math.max( + lastSnapshot.processPeakRssBytes ?? 0, + osPeakRssBytes + ); + } + }; + + const runOsSample = (): Promise => { + if (osInFlight) return osInFlight; + let pending: Promise; + pending = sampleOs().finally(() => { + if (osInFlight === pending) osInFlight = null; + }); + osInFlight = pending; + return pending; + }; + + const sample = async (): Promise => { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5_000); + try { + await runOsSample(); + const response = await fetch(url, { + signal: controller.signal, + headers: resolved.headers + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const snapshot = normalizeMemorySnapshot(await response.json()); + if (osPeakRssBytes != null) { + snapshot.processPeakRssBytes = Math.max(snapshot.processPeakRssBytes ?? 0, osPeakRssBytes); + } + validate(snapshot); + snapshots.push(snapshot); + } catch (error) { + recordError(error instanceof Error ? error.message : String(error)); + } finally { + clearTimeout(timeout); + } + }; + + const runSample = (): Promise => { + if (inFlight) return inFlight; + let pending: Promise; + pending = sample().finally(() => { + if (inFlight === pending) inFlight = null; + }); + inFlight = pending; + return pending; + }; + + const timer = setInterval(() => { + void runSample(); + }, resolved.intervalMs); + const osTimer = setInterval(() => { + void runOsSample(); + }, resolved.osSampleIntervalMs); + const ready = runSample(); + + return { + snapshots, + osSnapshots, + errors, + ready, + get warmupIndex() { + return warmupIndex; + }, + get osWarmupIndex() { + return osWarmupIndex; + }, + get osPeakRssBytes() { + return osPeakRssBytes; + }, + async markWarmupComplete(): Promise { + if (inFlight) await inFlight; + if (osInFlight) await osInFlight; + warmupIndex = snapshots.length; + osWarmupIndex = osSnapshots.length; + await runOsSample(); + await runSample(); + }, + async stop(): Promise { + if (stopped) return; + stopped = true; + clearInterval(timer); + clearInterval(osTimer); + if (inFlight) await inFlight; + if (osInFlight) await osInFlight; + await runOsSample(); + await runSample(); + } + }; +}; diff --git a/packages/perf-harness/src/postgres.ts b/packages/perf-harness/src/postgres.ts new file mode 100644 index 0000000000..45c05638ab --- /dev/null +++ b/packages/perf-harness/src/postgres.ts @@ -0,0 +1,466 @@ +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import type { PostgresMemorySnapshot } from './types'; + +const UNIT_BYTES: Record = { + B: 1, + KiB: 1024, + MiB: 1024 ** 2, + GiB: 1024 ** 3, + TiB: 1024 ** 4 +}; + +const CGROUP_FILE_MARKER = '__CPERF_CGROUP_FILE__ '; +const CGROUP_FILES = [ + 'memory.current', + 'memory.peak', + 'memory.max', + 'memory.stat', + 'memory.events' +] as const; + +const CGROUP_V2_SCRIPT = ` +set -eu +base=/sys/fs/cgroup +if [ ! -r "$base/memory.current" ]; then + relative=$(awk -F: '$1 == "0" { print $3; exit }' /proc/self/cgroup) + if [ -n "$relative" ] && [ -r "$base$relative/memory.current" ]; then + base="$base$relative" + fi +fi +for file in memory.current memory.peak memory.max memory.stat memory.events; do + if [ -r "$base/$file" ]; then + printf '${CGROUP_FILE_MARKER}%s\n' "$file" + sed -n '1,256p' "$base/$file" + fi +done +`; + +const CGROUP_IDENTITY_SCRIPT = [ + 'set -eu', + 'test -r /sys/fs/cgroup/memory.current', + 'test -r /sys/fs/cgroup/memory.events', + 'printf "membership="', + 'cat /proc/1/cgroup', + 'printf "mount="', + 'stat -c "%d:%i" /sys/fs/cgroup' +].join('\n'); + +const CONTAINER_ID = /^[a-f0-9]{64}$/; +const PREFIXED_SHA256 = /^sha256:[a-f0-9]{64}$/; + +const canonicalStringSha256 = (value: string): string => `sha256:${createHash('sha256') + .update(JSON.stringify(value)) + .digest('hex')}`; + +export const parseDockerBytes = (value: string): number | null => { + const match = /^\s*([\d.]+)\s*(B|KiB|MiB|GiB|TiB)\s*$/.exec(value); + if (!match) return null; + const parsed = Number.parseFloat(match[1]); + return Number.isFinite(parsed) ? Math.round(parsed * UNIT_BYTES[match[2]]) : null; +}; + +const parseNonNegativeInteger = (value: string | undefined): number | null => { + if (!value || !/^\d+$/.test(value.trim())) return null; + const parsed = Number(value.trim()); + return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : null; +}; + +export const parseCgroupKeyValues = (value: string): Record => { + const result: Record = {}; + for (const line of value.split(/\r?\n/)) { + const match = /^([^\s]+)\s+(\d+)$/.exec(line.trim()); + if (!match) continue; + const parsed = Number(match[2]); + if (Number.isSafeInteger(parsed) && parsed >= 0) result[match[1]] = parsed; + } + return result; +}; + +const cgroupSections = (raw: string): Map => { + const sections = new Map(); + let current: string | null = null; + for (const line of raw.split(/\r?\n/)) { + if (line.startsWith(CGROUP_FILE_MARKER)) { + const file = line.slice(CGROUP_FILE_MARKER.length).trim(); + current = (CGROUP_FILES as readonly string[]).includes(file) ? file : null; + if (current && !sections.has(current)) sections.set(current, []); + } else if (current) { + sections.get(current)!.push(line); + } + } + return new Map([...sections].map(([file, lines]) => [file, lines.join('\n').trim()])); +}; + +export interface ParsedCgroupV2Memory { + currentBytes: number; + peakBytes: number | null; + maxBytes: number | null; + stat: Record; + events: Record; +} + +export const parseCgroupV2Memory = (raw: string): ParsedCgroupV2Memory | null => { + const sections = cgroupSections(raw); + const currentBytes = parseNonNegativeInteger(sections.get('memory.current')); + if (currentBytes == null) return null; + const maxRaw = sections.get('memory.max')?.trim(); + return { + currentBytes, + peakBytes: parseNonNegativeInteger(sections.get('memory.peak')), + maxBytes: maxRaw === 'max' ? null : parseNonNegativeInteger(maxRaw), + stat: parseCgroupKeyValues(sections.get('memory.stat') ?? ''), + events: parseCgroupKeyValues(sections.get('memory.events') ?? '') + }; +}; + +const execFileText = ( + command: string, + args: string[], + timeout: number +): Promise => new Promise((resolve, reject) => { + execFile(command, args, { timeout }, (error, stdout, stderr) => { + if (error) { + const detail = String(stderr).trim(); + reject(new Error(detail ? `${error.message}: ${detail}` : error.message)); + return; + } + resolve(String(stdout)); + }); +}); + +interface DockerContainerIdentity { + id: string; + startedAt: string; + cgroupIdentitySha256: string; +} + +const inspectDockerContainer = async (container: string): Promise<{ + id: string; + startedAt: string; +}> => { + const raw = await execFileText('docker', ['inspect', container], 10_000); + const records = JSON.parse(raw) as Array<{ + Id?: unknown; + State?: { Running?: unknown; StartedAt?: unknown }; + }>; + const record = Array.isArray(records) && records.length === 1 ? records[0] : null; + const startedAtMs = Date.parse( + typeof record?.State?.StartedAt === 'string' ? record.State.StartedAt : '' + ); + if ( + !record + || !CONTAINER_ID.test(String(record.Id ?? '')) + || record.State?.Running !== true + || !Number.isSafeInteger(startedAtMs) + ) { + throw new Error('PostgreSQL sampler container identity is invalid'); + } + return { + id: String(record.Id), + startedAt: new Date(startedAtMs).toISOString() + }; +}; + +const inspectContainerCgroupIdentity = async (containerId: string): Promise => { + const raw = await execFileText('docker', [ + 'exec', + containerId, + '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin', + 'sh', '-ceu', CGROUP_IDENTITY_SCRIPT + ], 10_000); + return canonicalStringSha256(raw.trim()); +}; + +const resolveDockerContainerIdentity = async ( + container: string, + expected: Pick< + PostgresMemorySamplerOptions, + 'expectedContainerId' | 'expectedContainerStartedAt' | 'expectedCgroupIdentitySha256' + > +): Promise => { + const inspected = await inspectDockerContainer(container); + if (expected.expectedContainerId && inspected.id !== expected.expectedContainerId) { + throw new Error( + `PostgreSQL sampler container ID mismatch: expected ${expected.expectedContainerId}, observed ${inspected.id}` + ); + } + if ( + expected.expectedContainerStartedAt + && inspected.startedAt !== expected.expectedContainerStartedAt + ) { + throw new Error( + 'PostgreSQL sampler container start time does not match run attestation' + ); + } + const cgroupIdentitySha256 = await inspectContainerCgroupIdentity(inspected.id); + if ( + expected.expectedCgroupIdentitySha256 + && cgroupIdentitySha256 !== expected.expectedCgroupIdentitySha256 + ) { + throw new Error('PostgreSQL sampler cgroup identity does not match run attestation'); + } + return { ...inspected, cgroupIdentitySha256 }; +}; + +const dockerWorkingSet = async (container: string): Promise<{ + usedBytes: number; + limitBytes: number; + raw: string; +}> => { + const raw = (await execFileText( + 'docker', + ['stats', '--no-stream', '--format', '{{.MemUsage}}', container], + 10_000 + )).trim(); + const parts = raw.split('/').map((part) => part.trim()); + const usedBytes = parseDockerBytes(parts[0] ?? ''); + const limitBytes = parseDockerBytes(parts[1] ?? ''); + if (usedBytes == null || limitBytes == null) { + throw new Error(`unrecognized docker memory value '${raw}'`); + } + return { usedBytes, limitBytes, raw }; +}; + +const dockerCgroupV2 = async (container: string): Promise<{ + parsed: ParsedCgroupV2Memory; + raw: string; +}> => { + const raw = await execFileText( + 'docker', + ['exec', container, 'sh', '-c', CGROUP_V2_SCRIPT], + 10_000 + ); + const parsed = parseCgroupV2Memory(raw); + if (!parsed) throw new Error('container does not expose readable cgroup-v2 memory.current'); + return { parsed, raw }; +}; + +type CgroupReader = () => Promise<{ parsed: ParsedCgroupV2Memory; raw: string }>; + +const readHostCgroupV2 = async (base: string): Promise<{ + parsed: ParsedCgroupV2Memory; + raw: string; +}> => { + const sections: string[] = []; + for (const file of CGROUP_FILES) { + try { + const value = await fs.readFile(path.join(base, file), 'utf8'); + sections.push(`${CGROUP_FILE_MARKER}${file}\n${value.trim()}\n`); + } catch (error) { + if (file === 'memory.current') throw error; + } + } + const raw = sections.join(''); + const parsed = parseCgroupV2Memory(raw); + if (!parsed) throw new Error('host does not expose readable cgroup-v2 memory.current'); + return { parsed, raw }; +}; + +const resolveCgroupReader = async (container: string): Promise => { + if (process.platform === 'linux') { + try { + const pidRaw = (await execFileText( + 'docker', + ['inspect', '--format', '{{.State.Pid}}', container], + 10_000 + )).trim(); + const pid = Number(pidRaw); + if (!Number.isSafeInteger(pid) || pid <= 0) { + throw new Error(`invalid container pid '${pidRaw}'`); + } + const membership = await fs.readFile(`/proc/${pid}/cgroup`, 'utf8'); + const relative = membership + .split(/\r?\n/) + .map((line) => /^0::(.+)$/.exec(line)?.[1]) + .find(Boolean); + if (!relative) throw new Error('container process has no cgroup-v2 membership'); + const cgroupRoot = path.resolve('/sys/fs/cgroup'); + const base = path.resolve(cgroupRoot, `.${relative}`); + if (base !== cgroupRoot && !base.startsWith(`${cgroupRoot}${path.sep}`)) { + throw new Error('container cgroup path escaped the cgroup-v2 root'); + } + await readHostCgroupV2(base); + return () => readHostCgroupV2(base); + } catch { + // Docker Desktop and rootless engines may hide the host cgroup. The + // container namespace fallback remains correct, though less frequent. + } + } + return () => dockerCgroupV2(container); +}; + +export interface PostgresMemorySamplerOptions { + intervalMs?: number; + requireCgroupV2?: boolean; + expectedContainerId?: string; + expectedContainerStartedAt?: string; + expectedCgroupIdentitySha256?: string; +} + +export interface PostgresMemorySampler { + snapshots: PostgresMemorySnapshot[]; + errors: string[]; + ready: Promise; + stop(): Promise; +} + +export const startPostgresMemorySampler = ( + container: string, + options: number | PostgresMemorySamplerOptions = {} +): PostgresMemorySampler => { + const intervalMs = typeof options === 'number' ? options : options.intervalMs ?? 250; + const requireCgroupV2 = typeof options === 'number' + ? false + : options.requireCgroupV2 ?? false; + const identityOptions: PostgresMemorySamplerOptions = typeof options === 'number' + ? {} + : options; + if ( + identityOptions.expectedContainerId != null + && !CONTAINER_ID.test(identityOptions.expectedContainerId) + ) { + throw new Error('expected PostgreSQL container ID must be a 64-character digest'); + } + if ( + identityOptions.expectedCgroupIdentitySha256 != null + && !PREFIXED_SHA256.test(identityOptions.expectedCgroupIdentitySha256) + ) { + throw new Error('expected PostgreSQL cgroup identity must be a prefixed SHA-256'); + } + if (!Number.isFinite(intervalMs) || intervalMs <= 0) { + throw new Error(`PostgreSQL sample interval must be positive, received ${intervalMs}`); + } + const snapshots: PostgresMemorySnapshot[] = []; + const errors: string[] = []; + let cgroupErrorRecorded = false; + let cgroupReader: CgroupReader | null = null; + let latestWorkingSet: Awaited> | null = null; + let latestWorkingSetError: string | null = null; + let workingSetInFlight: Promise | null = null; + let inFlight: Promise | null = null; + let stopped = false; + let immutableIdentity: DockerContainerIdentity | null = null; + const exactContainer = (): string => { + if (!immutableIdentity) throw new Error('PostgreSQL sampler identity is unavailable'); + return immutableIdentity.id; + }; + const revalidateIdentity = async (): Promise => { + if (!immutableIdentity) throw new Error('PostgreSQL sampler identity is unavailable'); + const observed = await resolveDockerContainerIdentity(immutableIdentity.id, { + expectedContainerId: immutableIdentity.id, + expectedContainerStartedAt: immutableIdentity.startedAt, + expectedCgroupIdentitySha256: immutableIdentity.cgroupIdentitySha256 + }); + if ( + observed.id !== immutableIdentity.id + || observed.startedAt !== immutableIdentity.startedAt + || observed.cgroupIdentitySha256 !== immutableIdentity.cgroupIdentitySha256 + ) { + throw new Error('PostgreSQL sampler immutable identity changed during the run'); + } + }; + const sampleWorkingSet = (): Promise => { + if (workingSetInFlight) return workingSetInFlight; + let pending: Promise; + pending = dockerWorkingSet(exactContainer()).then((snapshot) => { + latestWorkingSet = snapshot; + latestWorkingSetError = null; + }, (error) => { + // Docker's cache-subtracted working set is diagnostic when the raw + // cgroup-v2 reader is healthy. Keep its failure in the sample payload, + // but do not disqualify an otherwise complete raw-memory measurement. + latestWorkingSetError = error instanceof Error ? error.message : String(error); + }).finally(() => { + if (workingSetInFlight === pending) workingSetInFlight = null; + }); + workingSetInFlight = pending; + return pending; + }; + const sample = async (): Promise => { + const startedAtMs = Date.now(); + let cgroup: Awaited> | null = null; + let cgroupError: string | null = null; + try { + cgroup = await cgroupReader!(); + } catch (error) { + cgroupError = error instanceof Error ? error.message : String(error); + if (requireCgroupV2 && !cgroupErrorRecorded) { + cgroupErrorRecorded = true; + errors.push(`cgroup-v2 telemetry unavailable: ${cgroupError}`); + } + } + const endedAtMs = Date.now(); + const workingSet = latestWorkingSet; + if (!cgroup && !workingSet) { + errors.push('PostgreSQL memory telemetry produced neither cgroup nor Docker data'); + return; + } + const midpointMs = startedAtMs + ((endedAtMs - startedAtMs) / 2); + snapshots.push({ + timestamp: new Date(midpointMs).toISOString(), + containerId: immutableIdentity!.id, + cgroupIdentitySha256: immutableIdentity!.cgroupIdentitySha256, + sampleStartedAt: new Date(startedAtMs).toISOString(), + sampleEndedAt: new Date(endedAtMs).toISOString(), + sampleDurationMs: endedAtMs - startedAtMs, + usedBytes: cgroup?.parsed.currentBytes ?? workingSet!.usedBytes, + ...(workingSet ? { workingSetBytes: workingSet.usedBytes } : {}), + limitBytes: cgroup?.parsed.maxBytes ?? workingSet?.limitBytes ?? 0, + source: cgroup ? 'cgroup-v2' : 'docker-stats', + ...(cgroup ? { cgroupV2: cgroup.parsed } : {}), + raw: JSON.stringify({ + dockerStats: workingSet?.raw ?? null, + dockerStatsError: latestWorkingSetError, + cgroupV2: cgroup?.raw ?? null, + cgroupError + }) + }); + }; + const runSample = (): Promise => { + if (inFlight) return inFlight; + let pending: Promise; + pending = sample().finally(() => { + if (inFlight === pending) inFlight = null; + }); + inFlight = pending; + return pending; + }; + let timer: ReturnType | null = null; + let workingSetTimer: ReturnType | null = null; + const ready = (async () => { + immutableIdentity = await resolveDockerContainerIdentity(container, identityOptions); + cgroupReader = await resolveCgroupReader(immutableIdentity.id); + await sampleWorkingSet(); + await runSample(); + if (stopped) return; + timer = setInterval(() => { + void runSample(); + }, intervalMs); + timer.unref?.(); + workingSetTimer = setInterval(() => { + void sampleWorkingSet(); + }, Math.max(1_000, intervalMs * 4)); + workingSetTimer.unref?.(); + })(); + return { + snapshots, + errors, + ready, + async stop(): Promise { + if (stopped) return; + stopped = true; + if (timer) clearInterval(timer); + if (workingSetTimer) clearInterval(workingSetTimer); + await ready; + if (inFlight) await inFlight; + if (workingSetInFlight) await workingSetInFlight; + await sampleWorkingSet(); + await runSample(); + await revalidateIdentity(); + } + }; +}; diff --git a/packages/perf-harness/src/process.ts b/packages/perf-harness/src/process.ts new file mode 100644 index 0000000000..d8f33503ad --- /dev/null +++ b/packages/perf-harness/src/process.ts @@ -0,0 +1,434 @@ +import { type ChildProcess, spawn, spawnSync } from 'node:child_process'; +import { createHash, randomBytes } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { armEnvironmentForHeap, resolveTemplate } from './config'; +import type { ArmPlan, ArmProvenance, NodeV8Profile } from './types'; + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); +const MAX_OLD_SPACE_OPTION = /^--max(?:-|_)old(?:-|_)space(?:-|_)size(?:=(.*))?$/; +const MANAGED_V8_OPTION = + /^--(?:no[-_])?(?:jitless|optimize[-_]for[-_]size|max[-_]opt)(?:=.*)?$/; +const V8_PROFILE_FLAGS: Readonly> = Object.freeze({ + stock: Object.freeze([]), + 'optimize-for-size': Object.freeze(['--optimize-for-size']), + 'baseline-optimize-for-size': Object.freeze([ + '--max-opt=1', + '--optimize-for-size' + ]), + 'jitless-optimize-for-size': Object.freeze(['--jitless', '--optimize-for-size']) +}); + +export interface ArmProcess { + pid: number | null; + external: boolean; + exit: { code: number | null; signal: NodeJS.Signals | null } | null; + expectedHeapLimitBytes: number | null; + observabilityHeaders: Readonly>; + provenance: ArmProvenance; + provenanceErrors: string[]; + stop(): Promise; +} + +interface ProvenanceResult { + provenance: ArmProvenance; + errors: string[]; +} + +/** Ephemeral credentials for one spawned arm; never serialize this object. */ +export const createObservabilityHeaders = (): Readonly> => Object.freeze({ + Authorization: `Bearer ${randomBytes(32).toString('base64url')}` +}); + +const sha256 = (value: string | Buffer): string => createHash('sha256').update(value).digest('hex'); + +export const tokenizeNodeOptions = (input: string): string[] => { + const tokens: string[] = []; + let token = ''; + let quote: '"' | "'" | null = null; + let escaped = false; + let started = false; + + for (const character of input) { + if (escaped) { + token += character; + escaped = false; + started = true; + } else if (character === '\\') { + escaped = true; + started = true; + } else if (quote) { + if (character === quote) quote = null; + else token += character; + } else if (character === '"' || character === "'") { + quote = character; + started = true; + } else if (/\s/.test(character)) { + if (started) { + tokens.push(token); + token = ''; + started = false; + } + } else { + token += character; + started = true; + } + } + if (escaped || quote) throw new Error('NODE_OPTIONS contains an unterminated escape or quote'); + if (started) tokens.push(token); + return tokens; +}; + +const quoteNodeOption = (option: string): string => { + if (/^[^\s"'\\]+$/.test(option)) return option; + return `"${option.replace(/(["\\])/g, '\\$1')}"`; +}; + +export const nodeFlagsForV8Profile = (profile: NodeV8Profile): string[] => { + const flags = V8_PROFILE_FLAGS[profile]; + if (!flags) throw new Error(`unknown Node V8 profile '${profile}'`); + return [...flags]; +}; + +/** Remove every inherited old-space flag before installing the requested limit. */ +export const replaceMaxOldSpaceSize = (nodeOptions: string | undefined, heapMiB: number): string => { + if (!Number.isSafeInteger(heapMiB) || heapMiB <= 0) { + throw new Error(`heapMiB must be a positive integer, received ${heapMiB}`); + } + const input = nodeOptions?.trim() ? tokenizeNodeOptions(nodeOptions) : []; + const retained: string[] = []; + for (let index = 0; index < input.length; index += 1) { + const match = MAX_OLD_SPACE_OPTION.exec(input[index]); + if (MANAGED_V8_OPTION.test(input[index])) continue; + if (!match) { + retained.push(input[index]); + continue; + } + if (match[1] === undefined && index + 1 < input.length && /^\d+$/.test(input[index + 1])) { + index += 1; + } + } + retained.push(`--max-old-space-size=${heapMiB}`); + return retained.map(quoteNodeOption).join(' '); +}; + +export const expectedHeapLimitForNodeOptions = ( + nodeOptions: string, + nodeExecutable = process.execPath, + directNodeFlags: readonly string[] = [] +): number => { + const result = spawnSync( + nodeExecutable, + [ + ...directNodeFlags, + '-e', + 'process.stdout.write(String(require("node:v8").getHeapStatistics().heap_size_limit))' + ], + { + encoding: 'utf8', + env: { ...process.env, NODE_OPTIONS: nodeOptions }, + timeout: 15_000 + } + ); + const limit = Number(result.stdout?.trim()); + if (result.status !== 0 || !Number.isSafeInteger(limit) || limit <= 0) { + const detail = result.error?.message || result.stderr?.trim() || `exit=${result.status}`; + throw new Error(`could not resolve expected V8 heap limit: ${detail}`); + } + return limit; +}; + +const gitOutput = (cwd: string, args: string[]): string | null => { + const result = spawnSync('git', ['-C', cwd, ...args], { + encoding: 'utf8', + timeout: 30_000 + }); + return result.status === 0 ? result.stdout.trimEnd() : null; +}; + +const regularFile = (file: string): boolean => { + try { + return fs.statSync(file).isFile(); + } catch { + return false; + } +}; + +const resolveEntryPath = (command: string[], cwd: string): string | null => { + const executable = path.basename(command[0] ?? '').toLowerCase(); + const candidates = executable === 'node' || executable === 'node.exe' + ? command.slice(1).filter((part) => !part.startsWith('-')) + : command; + for (const candidate of candidates) { + const resolved = path.isAbsolute(candidate) ? candidate : path.resolve(cwd, candidate); + if (regularFile(resolved)) return fs.realpathSync(resolved); + } + return null; +}; + +const directNodeExecArgv = (command: string[], cwd: string): string[] => { + const executable = path.basename(command[0] ?? '').toLowerCase(); + if (executable !== 'node' && executable !== 'node.exe') return []; + for (let index = 1; index < command.length; index++) { + const candidate = command[index]; + if (candidate.startsWith('-')) continue; + const resolved = path.isAbsolute(candidate) ? candidate : path.resolve(cwd, candidate); + if (regularFile(resolved)) return command.slice(1, index); + } + return command.slice(1).filter((argument) => argument.startsWith('-')); +}; + +interface NodeRuntimeProvenance { + v8Profile: NodeV8Profile; + nodeOptions: string | null; + nodeOptionsArgv: string[]; + nodeExecArgv: string[]; +} + +export const collectArmProvenance = ( + cwd: string, + command: string[], + serverPid: number | null, + runtime: NodeRuntimeProvenance = { + v8Profile: 'stock', + nodeOptions: null, + nodeOptionsArgv: [], + nodeExecArgv: directNodeExecArgv(command, cwd) + } +): ProvenanceResult => { + const errors: string[] = []; + const repoRoot = gitOutput(cwd, ['rev-parse', '--show-toplevel']); + const gitHead = gitOutput(cwd, ['rev-parse', 'HEAD']); + const gitStatus = gitOutput(cwd, ['status', '--porcelain=v1', '--untracked-files=all']); + if (!repoRoot) errors.push(`could not resolve git worktree root for ${cwd}`); + if (!gitHead) errors.push(`could not resolve git HEAD for ${cwd}`); + if (gitStatus == null) errors.push(`could not resolve git status for ${cwd}`); + + const lockfileCandidate = repoRoot ? path.join(repoRoot, 'pnpm-lock.yaml') : null; + const lockfilePath = lockfileCandidate && regularFile(lockfileCandidate) + ? fs.realpathSync(lockfileCandidate) + : null; + if (!lockfilePath) errors.push('workspace pnpm-lock.yaml was not found'); + + const entryPath = resolveEntryPath(command, cwd); + if (!entryPath) errors.push(`could not resolve an executed entry from command: ${command.join(' ')}`); + + return { + provenance: { + cwd, + command: [...command], + gitHead, + worktreeDirty: gitStatus == null ? null : gitStatus.length > 0, + gitStatusSha256: gitStatus == null ? null : sha256(gitStatus), + lockfilePath, + lockfileSha256: lockfilePath ? sha256(fs.readFileSync(lockfilePath)) : null, + entryPath, + entrySha256: entryPath ? sha256(fs.readFileSync(entryPath)) : null, + serverPid, + v8Profile: runtime.v8Profile, + nodeOptions: runtime.nodeOptions, + nodeOptionsArgv: [...runtime.nodeOptionsArgv], + nodeExecArgv: [...runtime.nodeExecArgv], + effectiveNodeRuntimeFlags: [ + ...runtime.nodeOptionsArgv, + ...runtime.nodeExecArgv + ], + planSha256: null, + fleetSha256: null, + node: process.version, + v8: process.versions.v8, + platform: process.platform, + architecture: process.arch, + runOrderSeed: null, + runOrderIndex: null, + memoryPolicy: null + }, + errors + }; +}; + +const assertPinnedProvenance = (arm: ArmPlan, provenance: ArmProvenance): void => { + if (arm.commit && !provenance.gitHead?.startsWith(arm.commit)) { + throw new Error(`arm commit mismatch: expected ${arm.commit}, observed ${provenance.gitHead ?? 'unknown'}`); + } + if (arm.lockfileSha256 && provenance.lockfileSha256 !== arm.lockfileSha256) { + throw new Error( + `arm lockfile mismatch: expected ${arm.lockfileSha256}, observed ${provenance.lockfileSha256 ?? 'unknown'}` + ); + } + if (arm.entrySha256 && provenance.entrySha256 !== arm.entrySha256) { + throw new Error( + `arm entry mismatch: expected ${arm.entrySha256}, observed ${provenance.entrySha256 ?? 'unknown'}` + ); + } +}; + +const waitForReady = async ( + url: string, + timeoutMs: number, + child?: ChildProcess, + getChildError?: () => Error | null +): Promise => { + const deadline = Date.now() + timeoutMs; + let lastError = 'not ready'; + while (Date.now() < deadline) { + const childError = getChildError?.(); + if (childError) throw new Error(`server process failed before readiness: ${childError.message}`); + if (child?.exitCode != null || child?.signalCode != null) { + throw new Error(`server exited before readiness: code=${child.exitCode} signal=${child.signalCode}`); + } + try { + const response = await fetch(url); + if (response.ok) return; + lastError = `HTTP ${response.status}`; + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + await sleep(250); + } + throw new Error(`server readiness timed out after ${timeoutMs}ms: ${lastError}`); +}; + +export const startArmProcess = async ( + arm: ArmPlan, + heapMiB: number, + artifactDir: string, + tenantCount: number, + attestedPostgresVariables: Record = {} +): Promise => { + const vars = { + heapMiB, + port: arm.port, + artifactDir, + mode: arm.introspectionMode, + tenantCount, + ...attestedPostgresVariables + }; + const readinessUrl = resolveTemplate(arm.readinessUrl, vars); + const cwd = path.resolve(arm.cwd ? resolveTemplate(arm.cwd, vars) : process.cwd()); + if (!arm.command?.length) { + await waitForReady(readinessUrl, arm.startupTimeoutMs ?? 120_000); + const collected = collectArmProvenance(cwd, [], null); + return { + pid: null, + external: true, + exit: null, + expectedHeapLimitBytes: null, + observabilityHeaders: Object.freeze({}), + provenance: collected.provenance, + provenanceErrors: collected.errors, + stop: async () => undefined + }; + } + + const configuredCommand = arm.command.map((part) => resolveTemplate(part, vars)); + const v8Profile = arm.v8Profile ?? 'stock'; + const profileFlags = nodeFlagsForV8Profile(v8Profile); + if (configuredCommand.some((argument) => MANAGED_V8_OPTION.test(argument))) { + throw new Error('managed V8 flags must be selected through v8Profile'); + } + const isNodeCommand = ['node', 'node.exe'].includes( + path.basename(configuredCommand[0]).toLowerCase() + ); + if (profileFlags.length > 0 && !isNodeCommand) { + throw new Error(`v8Profile '${v8Profile}' requires a Node command`); + } + const command = isNodeCommand + ? [configuredCommand[0], ...profileFlags, ...configuredCommand.slice(1)] + : configuredCommand; + const armEnvironment = armEnvironmentForHeap(arm, heapMiB); + const nodeOptions = replaceMaxOldSpaceSize( + armEnvironment.NODE_OPTIONS ?? process.env.NODE_OPTIONS, + heapMiB + ); + const nodeExecutable = ['node', 'node.exe'].includes(path.basename(command[0]).toLowerCase()) + ? command[0] + : process.execPath; + const expectedHeapLimitBytes = expectedHeapLimitForNodeOptions( + nodeOptions, + nodeExecutable, + profileFlags + ); + const nodeOptionsArgv = tokenizeNodeOptions(nodeOptions); + const collected = collectArmProvenance(cwd, command, null, { + v8Profile, + nodeOptions, + nodeOptionsArgv, + nodeExecArgv: directNodeExecArgv(command, cwd) + }); + assertPinnedProvenance(arm, collected.provenance); + const observabilityHeaders = createObservabilityHeaders(); + const observabilityToken = observabilityHeaders.Authorization.slice('Bearer '.length); + + fs.mkdirSync(artifactDir, { recursive: true }); + const logStream = fs.createWriteStream(path.join(artifactDir, 'server.log'), { flags: 'a' }); + const samplerDir = path.join(artifactDir, 'debug-sampler'); + const child = spawn(command[0], command.slice(1), { + cwd, + env: { + ...process.env, + ...armEnvironment, + NODE_ENV: 'production', + GRAPHILE_CACHE_TTL_MS: armEnvironment.GRAPHILE_CACHE_TTL_MS ?? '21600000', + NODE_OPTIONS: nodeOptions, + GRAPHILE_INTROSPECTION_MODE: arm.introspectionMode, + GRAPHQL_OBSERVABILITY_ENABLED: 'true', + GRAPHQL_OBSERVABILITY_TOKEN: observabilityToken, + GRAPHQL_DEBUG_SAMPLER_ENABLED: 'true', + GRAPHQL_DEBUG_SAMPLER_INTERVAL_MS: '1000', + GRAPHQL_DEBUG_SAMPLER_DIR: samplerDir + }, + stdio: ['ignore', 'pipe', 'pipe'] + }); + child.stdout?.pipe(logStream); + child.stderr?.pipe(logStream); + collected.provenance.serverPid = child.pid ?? null; + + let exit: ArmProcess['exit'] = null; + let childError: Error | null = null; + child.once('error', (error) => { + childError = error; + }); + child.once('exit', (code, signal) => { + exit = { code, signal }; + logStream.end(); + }); + const stopChild = async (): Promise => { + if (exit) return; + if (child.pid == null) { + logStream.end(); + return; + } + child.kill('SIGTERM'); + const deadline = Date.now() + 15_000; + while (!exit && Date.now() < deadline) await sleep(100); + if (!exit) child.kill('SIGKILL'); + const killDeadline = Date.now() + 2_000; + while (!exit && Date.now() < killDeadline) await sleep(50); + }; + try { + await waitForReady( + readinessUrl, + arm.startupTimeoutMs ?? 120_000, + child, + () => childError + ); + } catch (error) { + await stopChild(); + throw error; + } + + return { + pid: child.pid ?? null, + external: false, + expectedHeapLimitBytes, + observabilityHeaders, + provenance: collected.provenance, + provenanceErrors: collected.errors, + get exit() { + return exit; + }, + stop: stopChild + }; +}; diff --git a/packages/perf-harness/src/realtime-evidence.ts b/packages/perf-harness/src/realtime-evidence.ts new file mode 100644 index 0000000000..1cf02dadf6 --- /dev/null +++ b/packages/perf-harness/src/realtime-evidence.ts @@ -0,0 +1,243 @@ +import { createHash } from 'node:crypto'; + +import type { + RealtimeCorrelationReceipt, + RealtimeDeliveryCoverage, + RealtimeDeliverySurfaceCoverage +} from './types'; + +const SHA256 = /^[a-f0-9]{64}$/; +const EMPTY_SHA256 = createHash('sha256').update('').digest('hex'); + +const timestamp = (value: string | null): number => { + if (value == null) return Number.NaN; + const parsed = Date.parse(value); + return Number.isFinite(parsed) && new Date(parsed).toISOString() === value + ? parsed + : Number.NaN; +}; + +const orderedDigestSha256 = (digests: string[]): string => digests.length === 0 + ? EMPTY_SHA256 + : createHash('sha256').update(digests.join('\n')).digest('hex'); + +export interface RealtimeReceiptSurfaceEvidence { + tenantId: string; + surface: string; + route: string; + expectedRecurringRounds: number; + startedRecurringRounds: number; + verifiedRecurringRounds: number; + deadlineLateRecurringRounds: number; + receipts: RealtimeCorrelationReceipt[]; +} + +export interface RealtimeReceiptEvidenceInput { + deliveryIntervalMs: number; + workloadStartedAt: string; + workloadDeadlineAt: string; + workloadEndedAt: string | null; + surfaces: RealtimeReceiptSurfaceEvidence[]; +} + +export interface RealtimeReceiptEvidenceSummary { + coverage: RealtimeDeliveryCoverage; + failures: string[]; +} + +const receiptIsVerified = (receipt: RealtimeCorrelationReceipt): boolean => { + const deadlineAt = timestamp(receipt.deadlineAt); + const issuedAt = timestamp(receipt.issuedAt); + const primeAt = timestamp(receipt.primeResponseAt); + const eventAt = timestamp(receipt.eventAt); + return Number.isSafeInteger(receipt.sequence) + && receipt.sequence > 0 + && SHA256.test(receipt.issuedSha256) + && receipt.primeResponseSha256 === receipt.issuedSha256 + && receipt.eventSha256 === receipt.issuedSha256 + && Number.isFinite(deadlineAt) + && Number.isFinite(issuedAt) + && Number.isFinite(primeAt) + && Number.isFinite(eventAt) + && issuedAt <= primeAt + && issuedAt <= eventAt + && primeAt <= deadlineAt + && eventAt <= deadlineAt; +}; + +const percentile = (values: number[], fraction: number): number => { + if (values.length === 0) return 0; + const sorted = [...values].sort((left, right) => left - right); + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)]; +}; + +export const summarizeRealtimeReceiptEvidence = ( + input: RealtimeReceiptEvidenceInput +): RealtimeReceiptEvidenceSummary => { + const failures: string[] = []; + const workloadStartedAtMs = timestamp(input.workloadStartedAt); + const workloadDeadlineAtMs = timestamp(input.workloadDeadlineAt); + const workloadEndedAtMs = timestamp(input.workloadEndedAt); + const globalDigests = new Set(); + const surfaceKeys = new Set(); + const allPrimeLatencies: number[] = []; + const allDeliveryLatencies: number[] = []; + const surfaces: RealtimeDeliverySurfaceCoverage[] = input.surfaces.map((surface) => { + const key = `${surface.tenantId}\0${surface.surface}`; + if (surfaceKeys.has(key)) failures.push(`duplicate realtime surface: ${surface.tenantId}/${surface.surface}`); + surfaceKeys.add(key); + const sequences = new Set(); + for (let index = 0; index < surface.receipts.length; index += 1) { + const receipt = surface.receipts[index]; + if ( + !Number.isSafeInteger(receipt.sequence) + || receipt.sequence !== index + 1 + || sequences.has(receipt.sequence) + ) failures.push(`invalid realtime receipt sequence: ${surface.tenantId}/${surface.surface}`); + sequences.add(receipt.sequence); + if (!SHA256.test(receipt.issuedSha256)) { + failures.push(`invalid realtime receipt digest: ${surface.tenantId}/${surface.surface}`); + } else if (globalDigests.has(receipt.issuedSha256)) { + failures.push(`reused realtime receipt digest: ${surface.tenantId}/${surface.surface}`); + } + if ( + receipt.primeResponseSha256 != null + && !SHA256.test(receipt.primeResponseSha256) + ) failures.push(`invalid realtime prime digest: ${surface.tenantId}/${surface.surface}`); + if (receipt.eventSha256 != null && !SHA256.test(receipt.eventSha256)) { + failures.push(`invalid realtime event digest: ${surface.tenantId}/${surface.surface}`); + } + globalDigests.add(receipt.issuedSha256); + } + const timed = surface.receipts.filter((receipt) => receipt.timed); + const verified = timed.filter((receipt, index) => { + const scheduledAt = workloadStartedAtMs + (index + 1) * input.deliveryIntervalMs; + const slotDeadline = Math.min( + workloadDeadlineAtMs, + scheduledAt + input.deliveryIntervalMs + ); + const issuedAt = timestamp(receipt.issuedAt); + const receiptDeadline = timestamp(receipt.deadlineAt); + const scheduleBound = Number.isFinite(scheduledAt) + && Number.isFinite(slotDeadline) + && Number.isFinite(issuedAt) + && Number.isFinite(receiptDeadline) + // Node timers and wall-clock serialization can differ by a few + // milliseconds. This tolerance cannot extend the externally derived + // slot deadline and therefore cannot bless a late delivery. + && issuedAt >= scheduledAt - 5 + && issuedAt < slotDeadline + && receiptDeadline >= issuedAt + && receiptDeadline <= slotDeadline + && receiptDeadline <= workloadDeadlineAtMs; + if (!scheduleBound) { + failures.push( + `invalid realtime receipt schedule deadline: ${surface.tenantId}/${surface.surface}` + ); + } + return scheduleBound && receiptIsVerified(receipt); + }); + if (timed.length !== surface.startedRecurringRounds) { + failures.push(`realtime receipt count mismatch: ${surface.tenantId}/${surface.surface}`); + } + if (verified.length !== surface.verifiedRecurringRounds) { + failures.push(`realtime verified receipt count mismatch: ${surface.tenantId}/${surface.surface}`); + } + if ( + !Number.isSafeInteger(surface.expectedRecurringRounds) + || !Number.isSafeInteger(surface.startedRecurringRounds) + || !Number.isSafeInteger(surface.verifiedRecurringRounds) + || !Number.isSafeInteger(surface.deadlineLateRecurringRounds) + || surface.expectedRecurringRounds < 0 + || surface.startedRecurringRounds < 0 + || surface.verifiedRecurringRounds < 0 + || surface.deadlineLateRecurringRounds < 0 + ) failures.push(`invalid realtime counters: ${surface.tenantId}/${surface.surface}`); + const primeLatencies = verified.map((receipt) => + timestamp(receipt.primeResponseAt) - timestamp(receipt.issuedAt) + ); + const deliveryLatencies = verified.map((receipt) => + timestamp(receipt.eventAt) - timestamp(receipt.issuedAt) + ); + allPrimeLatencies.push(...primeLatencies); + allDeliveryLatencies.push(...deliveryLatencies); + return { + tenantId: surface.tenantId, + surface: surface.surface, + route: surface.route, + expectedRecurringRounds: surface.expectedRecurringRounds, + startedRecurringRounds: surface.startedRecurringRounds, + verifiedRecurringRounds: surface.verifiedRecurringRounds, + issuedCorrelationSha256: orderedDigestSha256( + timed.map((receipt) => receipt.issuedSha256) + ), + verifiedCorrelationSha256: orderedDigestSha256( + verified.map((receipt) => receipt.issuedSha256) + ), + primeRequests: timed.length, + primeResponseP99Ms: percentile(primeLatencies, 0.99), + deliveryP99Ms: percentile(deliveryLatencies, 0.99) + }; + }); + const expectedRecurringRounds = surfaces.reduce( + (sum, surface) => sum + surface.expectedRecurringRounds, + 0 + ); + const startedRecurringRounds = surfaces.reduce( + (sum, surface) => sum + surface.startedRecurringRounds, + 0 + ); + const verifiedRecurringRounds = surfaces.reduce( + (sum, surface) => sum + surface.verifiedRecurringRounds, + 0 + ); + const deadlineLateRecurringRounds = input.surfaces.reduce( + (sum, surface) => sum + surface.deadlineLateRecurringRounds, + 0 + ); + const primeRequests = surfaces.reduce( + (sum, surface) => sum + surface.primeRequests, + 0 + ); + if ( + !Number.isSafeInteger(input.deliveryIntervalMs) + || input.deliveryIntervalMs <= 0 + || !Number.isFinite(workloadStartedAtMs) + || !Number.isFinite(workloadDeadlineAtMs) + || ( + input.workloadEndedAt != null + && !Number.isFinite(workloadEndedAtMs) + ) + ) failures.push('invalid realtime coverage window'); + if (surfaces.length > 0 && expectedRecurringRounds === 0) { + failures.push('realtime coverage has no recurring rounds'); + } + const complete = failures.length === 0 + && input.workloadEndedAt != null + && workloadEndedAtMs >= workloadDeadlineAtMs + && startedRecurringRounds === expectedRecurringRounds + && verifiedRecurringRounds === expectedRecurringRounds + && deadlineLateRecurringRounds === 0 + && surfaces.every((surface) => + surface.issuedCorrelationSha256 === surface.verifiedCorrelationSha256 + ); + return { + coverage: { + version: 2, + deliveryIntervalMs: input.deliveryIntervalMs, + workloadStartedAt: input.workloadStartedAt, + workloadDeadlineAt: input.workloadDeadlineAt, + workloadEndedAt: input.workloadEndedAt, + expectedRecurringRounds, + startedRecurringRounds, + verifiedRecurringRounds, + deadlineLateRecurringRounds, + primeRequests, + primeResponseP99Ms: percentile(allPrimeLatencies, 0.99), + deliveryP99Ms: percentile(allDeliveryLatencies, 0.99), + complete, + surfaces + }, + failures + }; +}; diff --git a/packages/perf-harness/src/realtime.ts b/packages/perf-harness/src/realtime.ts new file mode 100644 index 0000000000..e4a91008dc --- /dev/null +++ b/packages/perf-harness/src/realtime.ts @@ -0,0 +1,764 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { isDeepStrictEqual } from 'node:util'; + +import { createClient } from 'graphql-ws'; +import { WebSocket } from 'ws'; + +import { jsonPointerValues, mapWithConcurrency } from './http'; +import { summarizeRealtimeReceiptEvidence } from './realtime-evidence'; +import type { + GraphqlSurface, + JsonPathMatch, + RealtimeCorrelationReceipt, + RealtimeDeliveryCoverage, + RealtimeGraphqlOperation, + TenantTarget +} from './types'; + +interface RealtimeSink { + next(value: any): void; + error(error: unknown): void; + complete(): void; +} + +interface DriverRealtimeClient { + subscribe( + payload: { query: string; variables?: Record }, + sink: RealtimeSink + ): () => void; + dispose(): Promise; +} + +export interface RealtimeClientFactoryInput { + url: string; + headers: Readonly>; + onConnected(): void; + onClosed(): void; + onError(): void; +} + +export type RealtimeClientFactory = ( + input: RealtimeClientFactoryInput +) => DriverRealtimeClient; + +export interface RealtimeDriverDependencies { + clientFactory?: RealtimeClientFactory; + fetch?: typeof fetch; + environment?: Readonly>; + sleep?: (ms: number) => Promise; + correlationFactory?: (surfaceKey: string, sequence: number) => string; +} + +export interface RealtimeDriverOptions { + concurrency: number; + timeoutMs: number; + deliveryIntervalMs?: number; +} + +export interface RealtimeDriverSnapshot { + expected: number; + active: number; + verified: number; + deliveryIntervalMs: number; + deliveryEvents: number; + deliveryRoundsStarted: number; + deliveryRoundsVerified: number; + deliveryRoundsPending: number; + timedCoverage: RealtimeDeliveryCoverage | null; + errors: string[]; + surfaces: Array<{ + tenantId: string; + surface: string; + route: string; + active: boolean; + verified: boolean; + deliveryEvents: number; + deliveryRoundsStarted: number; + deliveryRoundsVerified: number; + deliveryRoundPending: boolean; + timedRoundsExpected: number; + timedRoundsStarted: number; + timedRoundsVerified: number; + timedRoundsDeadlineLate: number; + correlationReceipts: RealtimeCorrelationReceipt[]; + }>; +} + +interface RealtimeTargetState { + tenantId: string; + surface: GraphqlSurface; + key: string; + route: string; + active: boolean; + verified: boolean; + deliveryEvents: number; + deliveryRoundsStarted: number; + deliveryRoundsVerified: number; + deliveryRoundPending: boolean; + timedRoundsExpected: number; + timedRoundsStarted: number; + timedRoundsVerified: number; + timedRoundsDeadlineLate: number; + correlationSequence: number; + pendingCorrelation: { + value: string; + sha256: string; + receipt: RealtimeCorrelationReceipt; + } | null; + correlationReceipts: RealtimeCorrelationReceipt[]; + client: DriverRealtimeClient | null; + unsubscribe: (() => void) | null; + errors: Set; +} + +const DEFAULT_SLEEP = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); +const DEFAULT_DELIVERY_INTERVAL_MS = 60_000; + +const matches = (body: unknown, match: JsonPathMatch): boolean => + jsonPointerValues(body, match.path).some((value) => + isDeepStrictEqual(value, match.value) + ); + +const exactCorrelationValue = ( + body: unknown, + path: string, + expected: string +): string | null => { + const selected = jsonPointerValues(body, path); + return selected.length === 1 + && typeof selected[0] === 'string' + && isDeepStrictEqual(selected[0], expected) + ? selected[0] + : null; +}; + +const firstForbiddenMatch = ( + body: unknown, + operation: RealtimeGraphqlOperation +): JsonPathMatch | undefined => operation.forbiddenMatches.find((match) => + matches(body, match) +); + +const firstMissingMatch = ( + body: unknown, + operation: RealtimeGraphqlOperation +): JsonPathMatch | undefined => operation.requiredMatches.find((match) => + !matches(body, match) +); + +export const realtimeWebSocketUrl = (surfaceUrl: string): string => { + const parsed = new URL(surfaceUrl); + if ( + !['http:', 'https:'].includes(parsed.protocol) + || parsed.username + || parsed.password + || parsed.search + || parsed.hash + ) { + throw new Error('CPERF_REALTIME_SURFACE_URL_INVALID'); + } + parsed.protocol = parsed.protocol === 'https:' ? 'wss:' : 'ws:'; + return parsed.toString(); +}; + +const surfaceRoute = (surfaceUrl: string): string => { + const parsed = new URL(surfaceUrl); + return parsed.pathname; +}; + +export const realtimeHeaders = ( + surface: GraphqlSurface, + environment: Readonly> = process.env +): Record => { + const headers: Record = { ...(surface.headers ?? {}) }; + for (const [name, environmentName] of Object.entries( + surface.realtime?.headersFromEnvironment ?? {} + )) { + const value = environment[environmentName]; + if (!value) { + throw new Error( + `CPERF_REALTIME_HEADER_ENV_MISSING:${surface.name}:${environmentName}` + ); + } + headers[name] = value; + } + return headers; +}; + +const defaultClientFactory: RealtimeClientFactory = ({ + url, + headers, + onConnected, + onClosed, + onError +}) => { + class HeaderWebSocket extends WebSocket { + constructor(address: string | URL, protocols?: string | string[]) { + super(address, protocols, { headers }); + } + } + const client = createClient({ + url, + webSocketImpl: HeaderWebSocket, + retryAttempts: 0, + connectionAckWaitTimeout: 10_000, + on: { + connected: onConnected, + closed: onClosed, + error: onError + } + }); + return { + subscribe: (payload, sink) => client.subscribe(payload, sink), + dispose: async () => { await client.dispose(); } + }; +}; + +const stateFailure = (state: RealtimeTargetState): string | null => + state.errors.values().next().value ?? null; + +export interface RealtimeDriver { + startAndVerify(): Promise; + beginTimedCoverage(durationMs: number): void; + finishTimedCoverage(): Promise; + verifyDeliveryNow(): Promise; + assertHealthy(): void; + snapshot(): RealtimeDriverSnapshot; + dispose(): Promise; +} + +export const createRealtimeDriver = ( + tenants: TenantTarget[], + options: RealtimeDriverOptions, + dependencies: RealtimeDriverDependencies = {} +): RealtimeDriver => { + if (!Number.isSafeInteger(options.concurrency) || options.concurrency <= 0) { + throw new Error('CPERF_REALTIME_CONCURRENCY_INVALID'); + } + if (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0) { + throw new Error('CPERF_REALTIME_TIMEOUT_INVALID'); + } + const deliveryIntervalMs = options.deliveryIntervalMs + ?? DEFAULT_DELIVERY_INTERVAL_MS; + if ( + !Number.isSafeInteger(deliveryIntervalMs) + || deliveryIntervalMs <= 0 + ) { + throw new Error('CPERF_REALTIME_DELIVERY_INTERVAL_INVALID'); + } + const clientFactory = dependencies.clientFactory ?? defaultClientFactory; + const fetchImpl = dependencies.fetch ?? fetch; + const environment = dependencies.environment ?? process.env; + const sleep = dependencies.sleep ?? DEFAULT_SLEEP; + const correlationFactory = dependencies.correlationFactory + ?? ((_surfaceKey: string, sequence: number) => + `cperf-realtime-v1:${sequence}:${randomUUID()}`); + const states: RealtimeTargetState[] = tenants.flatMap((tenant) => + tenant.surfaces.filter((surface) => surface.realtime).map( + (surface): RealtimeTargetState => ({ + tenantId: tenant.id, + surface, + key: `${tenant.id}/${surface.name}`, + route: surfaceRoute(surface.url), + active: false, + verified: false, + deliveryEvents: 0, + deliveryRoundsStarted: 0, + deliveryRoundsVerified: 0, + deliveryRoundPending: false, + timedRoundsExpected: 0, + timedRoundsStarted: 0, + timedRoundsVerified: 0, + timedRoundsDeadlineLate: 0, + correlationSequence: 0, + pendingCorrelation: null, + correlationReceipts: [], + client: null, + unsubscribe: null, + errors: new Set() + }) + ) + ); + let started = false; + let disposing = false; + let disposed = false; + let deliveryTimer: ReturnType | null = null; + let deliveryRound: Promise | null = null; + let timedCoverage: { + startedAtMs: number; + deadlineAtMs: number; + endedAtMs: number | null; + expectedRounds: number; + nextRound: number; + } | null = null; + const primeAbortControllers = new Set(); + const issuedCorrelations = new Set(); + + const recordError = (state: RealtimeTargetState, code: string): void => { + if (!disposing) state.errors.add(`${code}:${state.key}`); + }; + + const assertState = (state: RealtimeTargetState): void => { + const failure = stateFailure(state); + if (failure) throw new Error(failure); + }; + + const waitUntil = async ( + state: RealtimeTargetState, + predicate: () => boolean, + deadline: number, + timeoutCode: string + ): Promise => { + while (!predicate() && Date.now() < deadline) { + if (disposing || disposed) throw new Error('CPERF_REALTIME_DISPOSED'); + assertState(state); + await sleep(Math.min(25, Math.max(1, deadline - Date.now()))); + } + assertState(state); + if (!predicate()) throw new Error(`${timeoutCode}:${state.key}`); + }; + + const startState = async (state: RealtimeTargetState): Promise => { + const probe = state.surface.realtime!; + const headers = realtimeHeaders(state.surface, environment); + state.client = clientFactory({ + url: realtimeWebSocketUrl(state.surface.url), + headers, + onConnected: () => { state.active = true; }, + onClosed: () => { + state.active = false; + recordError(state, 'CPERF_REALTIME_TRANSPORT_DROPPED'); + }, + onError: () => recordError(state, 'CPERF_REALTIME_TRANSPORT_ERROR') + }); + state.unsubscribe = state.client.subscribe( + { query: probe.subscription.query, variables: probe.subscription.variables }, + { + next: (value) => { + if (Array.isArray(value?.errors) && value.errors.length > 0) { + recordError(state, 'CPERF_REALTIME_GRAPHQL_ERROR'); + return; + } + if (firstForbiddenMatch(value, probe.subscription)) { + recordError(state, 'CPERF_REALTIME_FOREIGN_PAYLOAD'); + return; + } + if (firstMissingMatch(value, probe.subscription)) { + recordError(state, 'CPERF_REALTIME_EVENT_INVARIANT_FAILED'); + return; + } + const pending = state.pendingCorrelation; + const verifiedCorrelation = pending && exactCorrelationValue( + value, + probe.correlation.subscriptionEventPath, + pending.value + ); + if (!pending || !verifiedCorrelation) { + // Cursor-backed delivery is at-least-once, so a valid replay for + // this exact tenant/database may arrive before the event caused by + // this round's fresh nonce. Permanent identity violations above + // still fail closed, but an old event cannot satisfy this round. + return; + } + pending.receipt.eventAt = new Date().toISOString(); + pending.receipt.eventSha256 = createHash('sha256') + .update(verifiedCorrelation) + .digest('hex'); + state.pendingCorrelation = null; + state.deliveryEvents++; + state.verified = true; + }, + error: () => recordError(state, 'CPERF_REALTIME_GRAPHQL_ERROR'), + complete: () => recordError(state, 'CPERF_REALTIME_SUBSCRIPTION_ENDED') + } + ); + await waitUntil( + state, + () => state.active, + Date.now() + options.timeoutMs, + 'CPERF_REALTIME_CONNECT_TIMEOUT' + ); + }; + + const primeOnce = async ( + state: RealtimeTargetState, + deadline: number, + correlation: string, + receipt: RealtimeCorrelationReceipt + ): Promise => { + const probe = state.surface.realtime!; + const controller = new AbortController(); + primeAbortControllers.add(controller); + const timeout = setTimeout( + () => controller.abort(), + Math.max(1, deadline - Date.now()) + ); + try { + const response = await fetchImpl(state.surface.url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...realtimeHeaders(state.surface, environment) + }, + body: JSON.stringify({ + query: probe.prime.query, + variables: { + ...(probe.prime.variables ?? {}), + [probe.correlation.primeVariable]: correlation + } + }), + signal: controller.signal + }); + const body = await response.json().catch((): null => null); + if (!response.ok || Array.isArray((body as any)?.errors)) { + throw new Error(`CPERF_REALTIME_PRIME_FAILED:${state.key}:HTTP_${response.status}`); + } + if (firstForbiddenMatch(body, probe.prime)) { + throw new Error(`CPERF_REALTIME_PRIME_FOREIGN_PAYLOAD:${state.key}`); + } + if (firstMissingMatch(body, probe.prime)) { + throw new Error(`CPERF_REALTIME_PRIME_INCONCLUSIVE:${state.key}`); + } + const responseCorrelation = exactCorrelationValue( + body, + probe.correlation.primeResponsePath, + correlation + ); + if (!responseCorrelation) { + throw new Error(`CPERF_REALTIME_PRIME_CORRELATION_MISMATCH:${state.key}`); + } + if (receipt.primeResponseAt == null) { + receipt.primeResponseAt = new Date().toISOString(); + receipt.primeResponseSha256 = createHash('sha256') + .update(responseCorrelation) + .digest('hex'); + } + } finally { + clearTimeout(timeout); + primeAbortControllers.delete(controller); + } + }; + + const verifyStateDelivery = async ( + state: RealtimeTargetState, + deadline = Date.now() + options.timeoutMs, + timed = false + ): Promise => { + const requiredEventCount = state.deliveryEvents + 1; + const correlation = correlationFactory(state.key, ++state.correlationSequence); + if ( + typeof correlation !== 'string' + || correlation.length < 24 + || correlation.length > 1024 + ) { + throw new Error(`CPERF_REALTIME_CORRELATION_INVALID:${state.key}`); + } + const correlationSha256 = createHash('sha256').update(correlation).digest('hex'); + if (issuedCorrelations.has(correlationSha256)) { + throw new Error(`CPERF_REALTIME_CORRELATION_REUSED:${state.key}`); + } + issuedCorrelations.add(correlationSha256); + const receipt: RealtimeCorrelationReceipt = { + sequence: state.correlationSequence, + timed, + deadlineAt: new Date(deadline).toISOString(), + issuedAt: new Date().toISOString(), + issuedSha256: correlationSha256, + primeResponseAt: null, + primeResponseSha256: null, + eventAt: null, + eventSha256: null + }; + state.correlationReceipts.push(receipt); + state.pendingCorrelation = { + value: correlation, + sha256: correlationSha256, + receipt + }; + state.deliveryRoundsStarted++; + if (timed) state.timedRoundsStarted++; + state.deliveryRoundPending = true; + try { + assertState(state); + await primeOnce(state, deadline, correlation, receipt); + await waitUntil( + state, + () => state.deliveryEvents >= requiredEventCount, + deadline, + 'CPERF_REALTIME_EVENT_TIMEOUT' + ); + state.deliveryRoundsVerified++; + if (timed) { + state.timedRoundsVerified++; + if (Date.now() > deadline) state.timedRoundsDeadlineLate++; + } + } finally { + if (state.pendingCorrelation?.sha256 === correlationSha256) { + state.pendingCorrelation = null; + } + state.deliveryRoundPending = false; + } + }; + + const runDeliveryRound = async ( + deadline = Date.now() + options.timeoutMs, + timed = false + ): Promise => { + const failures: Error[] = []; + await mapWithConcurrency(states, options.concurrency, async (state) => { + try { + await verifyStateDelivery( + state, + timed ? Math.min(deadline, Date.now() + options.timeoutMs) : deadline, + timed + ); + } catch (error) { + const failure = error instanceof Error + ? error + : new Error(`CPERF_REALTIME_DELIVERY_FAILED:${state.key}`); + if (!disposing) state.errors.add(failure.message); + failures.push(failure); + } + }); + if (failures.length > 0) throw failures[0]; + }; + + const clearDeliveryTimer = (): void => { + if (deliveryTimer) clearTimeout(deliveryTimer); + deliveryTimer = null; + }; + + const scheduleDeliveryRound = (): void => { + if ( + disposing + || disposed + || states.length === 0 + || deliveryTimer + || deliveryRound + ) return; + const coverage = timedCoverage; + if (!coverage || coverage.nextRound > coverage.expectedRounds) return; + const scheduledAt = coverage.startedAtMs + + coverage.nextRound * deliveryIntervalMs; + deliveryTimer = setTimeout(() => { + deliveryTimer = null; + const current = timedCoverage; + if (!current || current.nextRound > current.expectedRounds) return; + current.nextRound++; + const deadline = Math.min( + current.deadlineAtMs, + scheduledAt + deliveryIntervalMs + ); + if (Date.now() >= deadline) { + for (const state of states) { + state.timedRoundsStarted++; + state.timedRoundsDeadlineLate++; + } + scheduleDeliveryRound(); + return; + } + void launchDeliveryRound(deadline, true).catch((): void => undefined); + }, Math.max(0, scheduledAt - Date.now())); + }; + + const launchDeliveryRound = ( + deadline = Date.now() + options.timeoutMs, + timed = false + ): Promise => { + if (deliveryRound) return deliveryRound; + if (disposing || disposed) { + return Promise.reject(new Error('CPERF_REALTIME_DISPOSED')); + } + const round = runDeliveryRound(deadline, timed); + deliveryRound = round; + void round.then( + () => { + if (deliveryRound === round) deliveryRound = null; + scheduleDeliveryRound(); + }, + () => { + if (deliveryRound === round) deliveryRound = null; + } + ); + return round; + }; + + const coverageSnapshot = (): RealtimeDeliveryCoverage | null => { + if (!timedCoverage) return null; + return summarizeRealtimeReceiptEvidence({ + deliveryIntervalMs, + workloadStartedAt: new Date(timedCoverage.startedAtMs).toISOString(), + workloadDeadlineAt: new Date(timedCoverage.deadlineAtMs).toISOString(), + workloadEndedAt: timedCoverage.endedAtMs == null + ? null + : new Date(timedCoverage.endedAtMs).toISOString(), + surfaces: states.map((state) => ({ + tenantId: state.tenantId, + surface: state.surface.name, + route: state.route, + expectedRecurringRounds: state.timedRoundsExpected, + startedRecurringRounds: state.timedRoundsStarted, + verifiedRecurringRounds: state.timedRoundsVerified, + deadlineLateRecurringRounds: state.timedRoundsDeadlineLate, + receipts: state.correlationReceipts + })) + }).coverage; + }; + + const snapshot = (): RealtimeDriverSnapshot => ({ + expected: states.length, + active: states.filter((state) => state.active).length, + verified: states.filter((state) => state.verified).length, + deliveryIntervalMs, + deliveryEvents: states.reduce((sum, state) => sum + state.deliveryEvents, 0), + deliveryRoundsStarted: states.reduce( + (sum, state) => sum + state.deliveryRoundsStarted, + 0 + ), + deliveryRoundsVerified: states.reduce( + (sum, state) => sum + state.deliveryRoundsVerified, + 0 + ), + deliveryRoundsPending: states.filter((state) => + state.deliveryRoundPending + ).length, + timedCoverage: coverageSnapshot(), + errors: states.flatMap((state) => [...state.errors]).sort(), + surfaces: states.map((state) => ({ + tenantId: state.tenantId, + surface: state.surface.name, + route: state.route, + active: state.active, + verified: state.verified, + deliveryEvents: state.deliveryEvents, + deliveryRoundsStarted: state.deliveryRoundsStarted, + deliveryRoundsVerified: state.deliveryRoundsVerified, + deliveryRoundPending: state.deliveryRoundPending, + timedRoundsExpected: state.timedRoundsExpected, + timedRoundsStarted: state.timedRoundsStarted, + timedRoundsVerified: state.timedRoundsVerified, + timedRoundsDeadlineLate: state.timedRoundsDeadlineLate, + correlationReceipts: state.correlationReceipts.map((receipt) => ({ + ...receipt + })) + })) + }); + + const assertHealthy = (): void => { + const current = snapshot(); + if (current.errors.length > 0) throw new Error(current.errors[0]); + if ( + current.active !== current.expected + || current.verified !== current.expected + ) { + throw new Error( + `CPERF_REALTIME_NOT_HEALTHY:${current.active}:${current.verified}:${current.expected}` + ); + } + }; + + return { + async startAndVerify(): Promise { + if (started) throw new Error('CPERF_REALTIME_ALREADY_STARTED'); + started = true; + const failures: Error[] = []; + await mapWithConcurrency(states, options.concurrency, async (state) => { + try { + await startState(state); + } catch (error) { + failures.push(error instanceof Error ? error : new Error(String(error))); + } + }); + if (failures.length > 0) throw failures[0]; + await runDeliveryRound(); + assertHealthy(); + }, + beginTimedCoverage(durationMs: number): void { + if (!started) throw new Error('CPERF_REALTIME_NOT_STARTED'); + if (timedCoverage) throw new Error('CPERF_REALTIME_TIMED_COVERAGE_ALREADY_STARTED'); + if (!Number.isSafeInteger(durationMs) || durationMs <= 0) { + throw new Error('CPERF_REALTIME_TIMED_DURATION_INVALID'); + } + const startedAtMs = Date.now(); + const expectedRounds = Math.max( + 0, + Math.ceil(durationMs / deliveryIntervalMs) - 1 + ); + timedCoverage = { + startedAtMs, + deadlineAtMs: startedAtMs + durationMs, + endedAtMs: null, + expectedRounds, + nextRound: 1 + }; + for (const state of states) state.timedRoundsExpected = expectedRounds; + scheduleDeliveryRound(); + }, + async finishTimedCoverage(): Promise { + if (!timedCoverage) throw new Error('CPERF_REALTIME_TIMED_COVERAGE_NOT_STARTED'); + clearDeliveryTimer(); + if (deliveryRound) await deliveryRound; + clearDeliveryTimer(); + while (timedCoverage.nextRound <= timedCoverage.expectedRounds) { + timedCoverage.nextRound++; + for (const state of states) { + state.timedRoundsStarted++; + state.timedRoundsDeadlineLate++; + } + } + timedCoverage.endedAtMs = Date.now(); + const coverage = coverageSnapshot()!; + return coverage; + }, + async verifyDeliveryNow(): Promise { + if (!started) throw new Error('CPERF_REALTIME_NOT_STARTED'); + if (disposing || disposed) throw new Error('CPERF_REALTIME_DISPOSED'); + clearDeliveryTimer(); + const pendingRound = deliveryRound; + if (pendingRound) { + await pendingRound; + } else { + assertHealthy(); + await launchDeliveryRound(); + } + assertHealthy(); + }, + assertHealthy, + snapshot, + async dispose(): Promise { + if (disposed) return; + disposing = true; + clearDeliveryTimer(); + for (const controller of primeAbortControllers) controller.abort(); + const pendingRound = deliveryRound; + if (pendingRound) { + try { + await pendingRound; + } catch { + // Disposal intentionally aborts an in-flight prime or event wait. + } + } + for (const state of states) { + try { + state.unsubscribe?.(); + } catch { + state.errors.add(`CPERF_REALTIME_UNSUBSCRIBE_FAILED:${state.key}`); + } + } + const results = await Promise.allSettled(states.map((state) => state.client?.dispose())); + results.forEach((result, index) => { + if (result.status === 'rejected') { + states[index].errors.add(`CPERF_REALTIME_DISPOSE_FAILED:${states[index].key}`); + } + }); + disposed = true; + for (const state of states) state.active = false; + const disposalFailure = states.flatMap((state) => [...state.errors]).find((error) => + error.startsWith('CPERF_REALTIME_UNSUBSCRIBE_FAILED:') + || error.startsWith('CPERF_REALTIME_DISPOSE_FAILED:') + ); + if (disposalFailure) throw new Error(disposalFailure); + } + }; +}; diff --git a/packages/perf-harness/src/report.ts b/packages/perf-harness/src/report.ts new file mode 100644 index 0000000000..e9d9ac8297 --- /dev/null +++ b/packages/perf-harness/src/report.ts @@ -0,0 +1,1006 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { + DEFAULT_RUN_ORDER_SEED, + hasExactHostileValidationEvidence, + resolveTemplate, + soakArmName, + tenantCountsForHeap +} from './config'; +import { + assertResultSemanticReplay, + bindResultEvidence, + readRegularEvidenceFile, + RESULT_RAW_EVIDENCE_FILES, + validateResultEvidenceBinding +} from './evidence'; +import { postgresRunIdentityClaims } from './run-attestation'; +import { + buildRunSchedule, + scheduleJobsForPlan, + scheduleManifestSha256, + type CampaignScheduleJob, + type CampaignScheduleManifestV1 +} from './schedule'; +import { compareDensity, percentile, summarizeCapacityBoundaries } from './score'; +import type { DensityPlanV1, DensityRunResult, FleetV1 } from './types'; + +const GIB = 1024 ** 3; + +const SHA256 = /^[a-f0-9]{64}$/; + +const sha256 = (value: string | Buffer): string => createHash('sha256') + .update(value) + .digest('hex'); + +export { bindResultEvidence, RESULT_RAW_EVIDENCE_FILES }; + +const RESULT_V6_REQUIRED_KEYS = ` +schemaVersion runKind evidenceMode campaignId scheduleSha256 previousResultPayloadSha256 +qualificationCohortSha256 arm commit +introspectionMode heapMiB configuredCustomers configuredTenants fleetShape +repetition expectedMatrixRepetitions runOrderSeed runOrderIndex startedAt endedAt +durationSec warmupMaxMs resolvedWarmupTimeoutMs offeredLoad requests +coverageRequests workloadRequests errors customerWorkloadRps periodicValidationRps realtimeValidationRps +combinedHttpRps achievedRps missedArrivals errorRate p50Ms p95Ms p99Ms +peakHeapBytes peakRssBytes observedHeapLimitBytes residentInstances +expectedResidentInstances cacheConfiguredMax cacheBudgetCapacity cacheInstanceHeapBytes +cacheCalibrationId cacheAdmissionMode warmObservedHeapDeltaPerInstanceBytes +postWarmupHeapGrowthMiBPerHour rawPostWarmupHeapGrowthMiBPerHour +retainedHeapGrowthMiBPerHour retainedExternalGrowthMiBPerHour +retainedMemoryDurationSec retainedHeapBaselineBytes retainedHeapFinalBytes +retainedExternalBaselineBytes retainedExternalFinalBytes retainedMemoryCheckpointErrors +postWarmupEvictions postWarmupBuildRefusals postWarmupBuilds pgPoolCacheSize +pgPoolLeasedPools pgPoolActiveLeases postWarmupPgPoolCapacityEvictions +postWarmupPgPoolCapacityRefusals postWarmupPgPoolDisposalFailures coldBuildMaxMs +memorySampleErrors postgresBaselineBytes postgresWarmBoundaryBytes postgresPeakBytes +postgresWorkingSetPeakBytes postgresCgroupV2PeakBytes postgresCgroupV2Samples +postgresOomEvents postgresBackendPeak residentPhysicalDatabases postgresContainerDedicated +unexpectedPostgresDatabases pgPoolTotalClients pgPoolIdleClients pgPoolWaitingClients +runtimePoolRequestedMaxUses runtimePoolEffectiveMaxUses runtimePoolExpectedPools +runtimePoolObservedPools runtimePoolTotalClients runtimePoolIdleClients +runtimePoolWaitingClients +residentRealtimeManagers residentRealtimeTransports realtimeNotificationMode +realtimeDeliveryCoverage notificationBrokers notificationListenerConnections +notificationBrokerLeases notificationBrokerTopics notificationBrokerSubscribers +notificationBrokerQueueOverflows notificationBrokerFatalFailures notificationAuditIdentities +notificationAuditsHealthy notificationAuditsFailed notificationAuditsStale +notificationAuditAttempts notificationAuditFailures notificationAuditActiveDatabaseTargets +notificationAuditDatabaseConflicts postgresColdBuildSpikeBytes postgresSampleErrors +alignedServicePeakBytes alignedServicePeakNodeRssBytes alignedServicePeakPostgresBytes +alignedServicePeakTimestamp alignedServiceMemorySamples alignedServiceMemoryMaxSkewMs +alignedServiceMemoryCoverageRatio alignedServiceMemoryCoveredDurationMs +alignedServiceMemoryExpectedDurationMs alignedServiceMemoryMaxGapMs +serviceMemoryUpperBoundBytes serviceMemoryUpperBoundPostgresSource capabilitiesExercised +missingCapabilities missingCanaries canarySchedule canaryChecks canaryInconclusive +bleedViolations operationOracleChecks operationOracleInconclusive operationOracleViolations +missingOperationOracles tenants qualifiedCustomers qualifiedTenants +tenantsPerConfiguredOldSpaceGiB tenantsPerPeakRssGiB customersPerAlignedServiceGiB +customersPerServiceMemoryUpperBoundGiB configuredCustomersPerAlignedServiceGiB +configuredCustomersPerServiceMemoryUpperBoundGiB accepted failures serverExit provenance +provenanceErrors postgresRunAttestation evidenceBinding artifactDir +`.trim().split(/\s+/); + +export const readResults = (file: string): unknown[] => fs.readFileSync(file, 'utf8') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .map((line, index) => { + try { + return JSON.parse(line); + } catch (error) { + throw new Error( + `invalid result JSONL record ${index + 1}: ${error instanceof Error ? error.message : String(error)}` + ); + } + }); + +const finite = (value: unknown): value is number => typeof value === 'number' + && Number.isFinite(value); + +const closeEnough = (left: number, right: number): boolean => + Math.abs(left - right) <= Math.max(1e-9, Math.abs(right) * 1e-9); + +export interface ResultMatrixValidation { + complete: boolean; + expectedCoordinates: number; + observedCoordinates: number; + missing: string[]; + duplicates: string[]; + diagnostic: string[]; + soakExpected: boolean; + soakObserved: number; + soakComplete: boolean; +} + +const resultCoordinate = (result: Pick< +DensityRunResult, +'arm' | 'heapMiB' | 'configuredTenants' | 'repetition' +>): string => [ + result.arm, + result.heapMiB, + result.configuredTenants, + result.repetition +].join('/'); + +const expectedMatrixCoordinates = (plan: DensityPlanV1): string[] => plan.arms.flatMap( + (arm) => plan.heapMiB.flatMap((heapMiB) => tenantCountsForHeap(plan, heapMiB).flatMap( + (configuredTenants) => Array.from({ length: plan.repetitions }, (_unused, index) => [ + arm.name, + heapMiB, + configuredTenants, + index + 1 + ].join('/')) + )) +); + +interface CampaignEvidence { + manifest: CampaignScheduleManifestV1; + scheduleSha256: string; + evidenceMode: 'qualification' | 'diagnostic'; + qualificationBlockers: string[]; +} + +const canonicalIsoTimestamp = (value: unknown): value is string => { + if (typeof value !== 'string') return false; + const parsed = Date.parse(value); + return Number.isFinite(parsed) && new Date(parsed).toISOString() === value; +}; + +const requireCampaignJob = (value: unknown, label: string): CampaignScheduleJob => { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + const record = value as Record; + const expectedKeys = [ + 'runKind', + 'arm', + 'heapMiB', + 'tenantCount', + 'repetition', + 'orderIndex' + ]; + if ( + Object.keys(record).length !== expectedKeys.length + || expectedKeys.some((key) => !Object.prototype.hasOwnProperty.call(record, key)) + ) { + throw new Error(`${label} has an invalid shape`); + } + if ( + !['matrix', 'soak'].includes(String(record.runKind)) + || typeof record.arm !== 'string' + || record.arm.length === 0 + || !Number.isSafeInteger(record.heapMiB) + || (record.heapMiB as number) <= 0 + || !Number.isSafeInteger(record.tenantCount) + || (record.tenantCount as number) <= 0 + || !Number.isSafeInteger(record.repetition) + || (record.repetition as number) <= 0 + || !Number.isSafeInteger(record.orderIndex) + || (record.orderIndex as number) <= 0 + ) { + throw new Error(`${label} is invalid`); + } + return record as unknown as CampaignScheduleJob; +}; + +const readCampaignEvidence = ( + plan: DensityPlanV1, + campaignId: string +): CampaignEvidence => { + if (!SHA256.test(campaignId)) throw new Error('campaign identity is invalid'); + const root = fs.realpathSync(plan.artifactDir); + const file = path.join(root, `campaign-${campaignId}.json`); + const relative = path.relative(root, path.resolve(file)); + if (relative.startsWith('..') || path.isAbsolute(relative)) { + throw new Error('campaign manifest escaped the configured artifact root'); + } + let raw: unknown; + try { + raw = JSON.parse(readRegularEvidenceFile(file).toString('utf8')); + } catch (error) { + throw new Error( + `campaign manifest is unavailable or invalid: ` + + `${error instanceof Error ? error.message : String(error)}` + ); + } + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error('campaign manifest must be an object'); + } + const record = raw as Record; + const expectedKeys = [ + 'version', + 'campaignId', + 'campaignStartedAt', + 'runOrderSeed', + 'planSha256', + 'fleetSha256', + 'node', + 'v8', + 'platform', + 'architecture', + 'jobs', + 'scheduleSha256', + 'evidenceMode', + 'qualificationBlockers' + ]; + if ( + Object.keys(record).length !== expectedKeys.length + || expectedKeys.some((key) => !Object.prototype.hasOwnProperty.call(record, key)) + ) { + throw new Error('campaign manifest has an invalid shape'); + } + if ( + record.version !== 1 + || record.campaignId !== campaignId + || !canonicalIsoTimestamp(record.campaignStartedAt) + || typeof record.runOrderSeed !== 'string' + || record.runOrderSeed.length === 0 + || typeof record.planSha256 !== 'string' + || !SHA256.test(record.planSha256) + || typeof record.fleetSha256 !== 'string' + || !SHA256.test(record.fleetSha256) + || typeof record.node !== 'string' + || record.node.length === 0 + || typeof record.v8 !== 'string' + || record.v8.length === 0 + || typeof record.platform !== 'string' + || record.platform.length === 0 + || typeof record.architecture !== 'string' + || record.architecture.length === 0 + || typeof record.scheduleSha256 !== 'string' + || !SHA256.test(record.scheduleSha256) + || !['qualification', 'diagnostic'].includes(String(record.evidenceMode)) + || !Array.isArray(record.qualificationBlockers) + || record.qualificationBlockers.some((blocker) => + typeof blocker !== 'string' || blocker.length === 0) + || new Set(record.qualificationBlockers).size !== record.qualificationBlockers.length + || !Array.isArray(record.jobs) + || record.jobs.length === 0 + ) { + throw new Error('campaign manifest is invalid'); + } + const jobs = record.jobs.map((job, index) => + requireCampaignJob(job, `campaign manifest job ${index + 1}`)); + if (jobs.some((job, index) => job.orderIndex !== index + 1)) { + throw new Error('campaign manifest run order is not contiguous'); + } + const manifest: CampaignScheduleManifestV1 = { + version: 1, + campaignId, + campaignStartedAt: record.campaignStartedAt as string, + runOrderSeed: record.runOrderSeed as string, + planSha256: record.planSha256 as string, + fleetSha256: record.fleetSha256 as string, + node: record.node as string, + v8: record.v8 as string, + platform: record.platform as NodeJS.Platform, + architecture: record.architecture as string, + jobs + }; + if (scheduleManifestSha256(manifest) !== record.scheduleSha256) { + throw new Error('campaign manifest does not match its schedule SHA-256'); + } + return { + manifest, + scheduleSha256: record.scheduleSha256, + evidenceMode: record.evidenceMode as CampaignEvidence['evidenceMode'], + qualificationBlockers: record.qualificationBlockers as string[] + }; +}; + +const exactHostileEvidenceAvailable = (plan: DensityPlanV1): boolean => { + if (!hasExactHostileValidationEvidence(plan)) return false; + return plan.arms.every((arm) => { + const binding = plan.qualification!.hostileValidationEvidence![arm.name]; + if (!path.isAbsolute(binding.artifactFile)) return false; + try { + const bytes = readRegularEvidenceFile(binding.artifactFile); + if (sha256(bytes) !== binding.artifactSha256) return false; + const raw = JSON.parse(bytes.toString('utf8')) as Record; + return raw.version === 1 + && raw.kind === binding.kind + && raw.passed === true + && raw.arm === arm.name + && raw.runtimeArtifactFingerprint === binding.runtimeArtifactFingerprint + && raw.configurationFingerprint === binding.configurationFingerprint; + } catch { + return false; + } + }); +}; + +export const validateResultSet = ( + input: unknown[], + plan: DensityPlanV1, + fleet: FleetV1 +): { results: DensityRunResult[]; matrix: ResultMatrixValidation } => { + if (input.length === 0) throw new Error('result set contains no campaign records'); + const planSha256 = plan.sourceSha256 ?? sha256(JSON.stringify(plan)); + const fleetSha256 = fleet.sourceSha256 ?? sha256(JSON.stringify(fleet)); + const cohortSha256 = sha256(`${planSha256}\0${fleetSha256}`); + const armByName = new Map(plan.arms.map((arm) => [arm.name, arm])); + const expected = new Set(expectedMatrixCoordinates(plan)); + const firstRecord = input[0] && typeof input[0] === 'object' && !Array.isArray(input[0]) + ? input[0] as Record + : null; + if (!firstRecord || typeof firstRecord.campaignId !== 'string') { + throw new Error('result record 1 has no campaign identity'); + } + const campaign = readCampaignEvidence(plan, firstRecord.campaignId); + const canonicalMatrix = buildRunSchedule( + plan, + plan.arms, + plan.heapMiB, + plan.repetitions + ); + const canonicalJobs = scheduleJobsForPlan(plan, canonicalMatrix, true); + const canonicalQualificationSchedule = JSON.stringify(campaign.manifest.jobs) + === JSON.stringify(canonicalJobs); + const hostileEvidenceReady = exactHostileEvidenceAvailable(plan); + if ( + campaign.manifest.planSha256 !== planSha256 + || campaign.manifest.fleetSha256 !== fleetSha256 + || campaign.manifest.runOrderSeed !== (plan.runOrderSeed ?? DEFAULT_RUN_ORDER_SEED) + ) { + throw new Error('campaign manifest does not match the plan/fleet cohort'); + } + if (campaign.evidenceMode === 'qualification') { + if (campaign.manifest.platform !== 'linux') { + throw new Error('qualification campaign was not executed on Linux'); + } + if (campaign.qualificationBlockers.length > 0) { + throw new Error('qualification campaign contains prerequisite blockers'); + } + if (!canonicalQualificationSchedule) { + throw new Error('qualification campaign schedule is not the exact configured schedule'); + } + if (!hostileEvidenceReady) { + throw new Error( + 'qualification campaign lacks exact-runtime hostile validation evidence' + ); + } + } + let previousResultPayloadSha256: string | null = null; + let previousEndedAtMs = Date.parse(campaign.manifest.campaignStartedAt); + const seen = new Map(); + const diagnostic: string[] = []; + let soakObserved = 0; + const results = input.map((raw, index): DensityRunResult => { + const label = `result record ${index + 1}`; + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error(`${label} must be an object`); + } + const rawRecord = raw as Record; + const missingKeys = RESULT_V6_REQUIRED_KEYS.filter((key) => + !Object.prototype.hasOwnProperty.call(rawRecord, key) + ); + const unexpectedKeys = Object.keys(rawRecord).filter((key) => + !RESULT_V6_REQUIRED_KEYS.includes(key) + ); + if (missingKeys.length > 0 || unexpectedKeys.length > 0) { + throw new Error( + `${label} does not match the complete result-v6 shape; ` + + `missing=${missingKeys.join(',') || 'none'}; ` + + `unexpected=${unexpectedKeys.join(',') || 'none'}` + ); + } + const result = raw as DensityRunResult; + const arm = armByName.get(result.arm); + const scheduled = campaign.manifest.jobs[index]; + if (result.schemaVersion !== 6) throw new Error(`${label} schemaVersion must be 6`); + if (!scheduled) throw new Error(`${label} exceeds the campaign schedule`); + if ( + result.campaignId !== campaign.manifest.campaignId + || result.scheduleSha256 !== campaign.scheduleSha256 + || result.evidenceMode !== campaign.evidenceMode + || result.previousResultPayloadSha256 !== previousResultPayloadSha256 + || result.runOrderIndex !== index + 1 + || scheduled.orderIndex !== result.runOrderIndex + || scheduled.runKind !== result.runKind + || scheduled.arm !== result.arm + || scheduled.heapMiB !== result.heapMiB + || scheduled.tenantCount !== result.configuredTenants + || scheduled.repetition !== result.repetition + ) { + throw new Error(`${label} does not match the campaign schedule or result chain`); + } + if ( + !canonicalIsoTimestamp(result.startedAt) + || !canonicalIsoTimestamp(result.endedAt) + || Date.parse(result.startedAt) < previousEndedAtMs + || Date.parse(result.endedAt) < Date.parse(result.startedAt) + ) { + throw new Error(`${label} campaign chronology is invalid or overlapping`); + } + if (typeof result.artifactDir !== 'string' || result.artifactDir.length === 0) { + throw new Error(`${label} artifactDir is invalid`); + } + validateResultEvidenceBinding(result, label); + previousResultPayloadSha256 = result.evidenceBinding!.resultPayloadSha256; + previousEndedAtMs = Date.parse(result.endedAt); + if (!arm) throw new Error(`${label} uses unconfigured arm '${String(result.arm)}'`); + assertResultSemanticReplay(result, plan, fleet, label); + if (!['matrix', 'soak'].includes(result.runKind)) throw new Error(`${label} runKind is invalid`); + if (!['qualification', 'diagnostic'].includes(result.evidenceMode)) { + throw new Error(`${label} evidenceMode is invalid`); + } + if (result.qualificationCohortSha256 !== cohortSha256) { + throw new Error(`${label} qualification cohort does not match plan/fleet bytes`); + } + if ( + !Number.isSafeInteger(result.heapMiB) + || !plan.heapMiB.includes(result.heapMiB) + || !Number.isSafeInteger(result.configuredTenants) + || result.configuredTenants <= 0 + || result.configuredTenants !== result.configuredCustomers + || result.configuredTenants > fleet.tenants.length + || !Number.isSafeInteger(result.repetition) + || result.repetition <= 0 + || result.expectedMatrixRepetitions !== plan.repetitions + ) { + throw new Error(`${label} matrix coordinate is invalid`); + } + if (result.runKind === 'matrix') { + if (!tenantCountsForHeap(plan, result.heapMiB).includes(result.configuredTenants)) { + throw new Error(`${label} tenant count is not configured for heap ${result.heapMiB}`); + } + const coordinate = resultCoordinate(result); + seen.set(coordinate, (seen.get(coordinate) ?? 0) + 1); + if (result.evidenceMode !== 'qualification') diagnostic.push(coordinate); + } else { + const soak = plan.soak; + if (!soak?.enabled) { + throw new Error(`${label} contains soak evidence but plan.soak is not enabled`); + } + soakObserved++; + const expectedSoakArm = soakArmName(plan); + if ( + result.arm !== expectedSoakArm + || result.heapMiB !== soak.heapMiB + || result.configuredTenants !== soak.tenantCount + || result.repetition !== plan.repetitions + 1 + || ( + result.evidenceMode === 'qualification' + && result.runOrderIndex !== expected.size + 1 + ) + ) { + throw new Error(`${label} does not match the configured soak coordinate`); + } + if ( + result.accepted + && ( + !finite(result.durationSec) + || result.durationSec < soak.durationSec * 0.99 + || result.durationSec > soak.durationSec * 1.01 + ) + ) { + throw new Error(`${label} accepted soak duration does not match plan.soak.durationSec`); + } + } + const provenance = result.provenance; + if ( + !provenance + || provenance.planSha256 !== planSha256 + || provenance.fleetSha256 !== fleetSha256 + || provenance.runOrderSeed !== (plan.runOrderSeed ?? DEFAULT_RUN_ORDER_SEED) + || provenance.runOrderIndex !== result.runOrderIndex + || provenance.worktreeDirty !== false + || !provenance.gitHead + || !provenance.gitStatusSha256 + || !provenance.entrySha256 + || !provenance.lockfileSha256 + || !provenance.node + || !provenance.v8 + || provenance.node !== campaign.manifest.node + || provenance.v8 !== campaign.manifest.v8 + || provenance.platform !== campaign.manifest.platform + || provenance.architecture !== campaign.manifest.architecture + ) { + throw new Error(`${label} provenance is incomplete or does not match the plan/fleet cohort`); + } + if ( + (arm.commit && !provenance.gitHead.startsWith(arm.commit)) + || (arm.entrySha256 && provenance.entrySha256 !== arm.entrySha256) + || (arm.lockfileSha256 && provenance.lockfileSha256 !== arm.lockfileSha256) + || result.commit !== (arm.commit ?? null) + || result.introspectionMode !== arm.introspectionMode + ) { + throw new Error(`${label} arm provenance does not match its configured arm`); + } + if (!Array.isArray(result.provenanceErrors) || result.provenanceErrors.length > 0) { + throw new Error(`${label} contains provenance validation errors`); + } + if ( + typeof result.accepted !== 'boolean' + || !Array.isArray(result.failures) + || !Number.isSafeInteger(result.qualifiedCustomers) + || result.qualifiedCustomers < 0 + || result.qualifiedCustomers > result.configuredTenants + || result.qualifiedCustomers !== result.qualifiedTenants + || result.accepted !== (result.failures.length === 0) + || (result.accepted && ( + result.failures.length > 0 || result.qualifiedCustomers !== result.configuredTenants + )) + || (!result.accepted && result.qualifiedCustomers !== 0) + ) { + throw new Error(`${label} acceptance fields are internally inconsistent`); + } + const expectedOldSpaceDensity = result.qualifiedCustomers / (result.heapMiB / 1024); + if (!finite(result.tenantsPerConfiguredOldSpaceGiB) + || !closeEnough(result.tenantsPerConfiguredOldSpaceGiB, expectedOldSpaceDensity)) { + throw new Error(`${label} configured-old-space density is inconsistent`); + } + const densityPairs: Array<[number | null, number | null]> = [ + [result.alignedServicePeakBytes, result.customersPerAlignedServiceGiB], + [result.serviceMemoryUpperBoundBytes, result.customersPerServiceMemoryUpperBoundGiB], + [result.peakRssBytes, result.tenantsPerPeakRssGiB] + ]; + for (const [bytes, density] of densityPairs) { + if (bytes == null) { + if (density != null) throw new Error(`${label} density exists without its memory denominator`); + } else if ( + !finite(bytes) + || bytes <= 0 + || !finite(density) + || !closeEnough(density, result.qualifiedCustomers / (bytes / GIB)) + ) { + throw new Error(`${label} density does not match qualified customers and memory bytes`); + } + } + const configuredDensityPairs: Array<[number | null, number | null]> = [ + [result.alignedServicePeakBytes, result.configuredCustomersPerAlignedServiceGiB], + [ + result.serviceMemoryUpperBoundBytes, + result.configuredCustomersPerServiceMemoryUpperBoundGiB + ] + ]; + for (const [bytes, density] of configuredDensityPairs) { + if (bytes == null) { + if (density != null) throw new Error(`${label} diagnostic density has no denominator`); + } else if ( + !finite(bytes) + || bytes <= 0 + || !finite(density) + || !closeEnough(density, result.configuredCustomers / (bytes / GIB)) + ) { + throw new Error(`${label} configured-customer diagnostic density is inconsistent`); + } + } + if ( + result.requests !== result.workloadRequests + || result.achievedRps !== result.customerWorkloadRps + || !closeEnough( + result.combinedHttpRps, + result.customerWorkloadRps + + result.periodicValidationRps + + result.realtimeValidationRps + ) + || !finite(result.errorRate) + || !closeEnough( + result.errorRate, + result.requests > 0 ? result.errors / result.requests : 1 + ) + ) { + throw new Error(`${label} workload counters or rates are internally inconsistent`); + } + const realtime = result.realtimeDeliveryCoverage; + if (realtime != null) { + const startedAtMs = Date.parse(realtime.workloadStartedAt); + const deadlineAtMs = Date.parse(realtime.workloadDeadlineAt); + const endedAtMs = realtime.workloadEndedAt == null + ? NaN + : Date.parse(realtime.workloadEndedAt); + const expectedRoundsPerSurface = Math.max( + 0, + Math.ceil((deadlineAtMs - startedAtMs) / realtime.deliveryIntervalMs) - 1 + ); + const aggregate = realtime.surfaces.reduce((summary, surface) => ({ + expected: summary.expected + surface.expectedRecurringRounds, + started: summary.started + surface.startedRecurringRounds, + verified: summary.verified + surface.verifiedRecurringRounds, + primeRequests: summary.primeRequests + surface.primeRequests + }), { expected: 0, started: 0, verified: 0, primeRequests: 0 }); + const complete = ( + Number.isFinite(endedAtMs) + && endedAtMs >= deadlineAtMs + && realtime.startedRecurringRounds === realtime.expectedRecurringRounds + && realtime.verifiedRecurringRounds === realtime.expectedRecurringRounds + && realtime.deadlineLateRecurringRounds === 0 + && realtime.surfaces.every((surface) => + surface.issuedCorrelationSha256 === surface.verifiedCorrelationSha256 + ) + ); + const surfaceKeys = realtime.surfaces.map((surface) => + `${surface.tenantId}\0${surface.surface}\0${surface.route}` + ); + const expectedRealtimeSurfaceKeys = fleet.tenants + .slice(0, result.configuredTenants) + .flatMap((tenant) => tenant.surfaces + .filter((surface) => surface.realtime != null) + .map((surface) => { + const url = resolveTemplate(surface.url, { + port: arm.port, + mode: arm.introspectionMode + }); + return `${tenant.id}\0${surface.name}\0${new URL(url).pathname}`; + })) + .sort(); + if ( + realtime.version !== 2 + || !Number.isSafeInteger(realtime.deliveryIntervalMs) + || realtime.deliveryIntervalMs <= 0 + || !Number.isSafeInteger(realtime.primeRequests) + || realtime.primeRequests < 0 + || !finite(realtime.primeResponseP99Ms) + || realtime.primeResponseP99Ms < 0 + || !finite(realtime.deliveryP99Ms) + || realtime.deliveryP99Ms < 0 + || !Number.isFinite(startedAtMs) + || !Number.isFinite(deadlineAtMs) + || deadlineAtMs <= startedAtMs + || !Array.isArray(realtime.surfaces) + || new Set(surfaceKeys).size !== surfaceKeys.length + || JSON.stringify([...surfaceKeys].sort()) + !== JSON.stringify(expectedRealtimeSurfaceKeys) + || realtime.surfaces.some((surface) => + surface.expectedRecurringRounds !== expectedRoundsPerSurface + || !Number.isSafeInteger(surface.startedRecurringRounds) + || !Number.isSafeInteger(surface.verifiedRecurringRounds) + || surface.startedRecurringRounds < surface.verifiedRecurringRounds + || !Number.isSafeInteger(surface.primeRequests) + || surface.primeRequests < 0 + || !finite(surface.primeResponseP99Ms) + || surface.primeResponseP99Ms < 0 + || !finite(surface.deliveryP99Ms) + || surface.deliveryP99Ms < 0 + || !/^[a-f0-9]{64}$/.test(surface.issuedCorrelationSha256) + || !/^[a-f0-9]{64}$/.test(surface.verifiedCorrelationSha256) + ) + || aggregate.expected !== realtime.expectedRecurringRounds + || aggregate.started !== realtime.startedRecurringRounds + || aggregate.verified !== realtime.verifiedRecurringRounds + || aggregate.primeRequests !== realtime.primeRequests + || !closeEnough( + result.realtimeValidationRps, + result.durationSec > 0 ? realtime.primeRequests / result.durationSec : 0 + ) + || realtime.complete !== complete + || (result.accepted && !complete) + ) { + throw new Error(`${label} recurring realtime coverage is inconsistent`); + } + } else if (result.accepted) { + throw new Error(`${label} accepted without recurring realtime coverage evidence`); + } + if ( + !Array.isArray(result.tenants) + || (result.accepted && result.tenants.length !== result.configuredTenants) + || (!result.accepted && ![0, result.configuredTenants].includes(result.tenants.length)) + ) { + throw new Error(`${label} does not contain one scored result per configured customer`); + } + for (const tenant of result.tenants) { + if ( + !Array.isArray(tenant.surfaces) + || tenant.surfaces.length !== tenant.surfacesConfigured + || (tenant.qualified && !tenant.surfaces.every((surface) => surface.qualified)) + ) { + throw new Error(`${label} customer/surface qualification evidence is inconsistent`); + } + } + if (plan.gates.requireFreshPostgresRunAttestation) { + const attestation = result.postgresRunAttestation; + if ( + !attestation + || attestation.planSha256 !== `sha256:${planSha256}` + || attestation.fleetSha256 !== `sha256:${fleetSha256}` + || attestation.arm !== result.arm + || attestation.heapMiB !== result.heapMiB + || attestation.tenantCount !== result.configuredTenants + || attestation.repetition !== result.repetition + || attestation.runOrderIndex !== result.runOrderIndex + ) { + throw new Error(`${label} PostgreSQL attestation does not match its matrix coordinate`); + } + } + return result; + }); + const missing = [...expected].filter((coordinate) => !seen.has(coordinate)).sort(); + const duplicates = [...seen] + .filter(([_coordinate, count]) => count !== 1) + .map(([coordinate]) => coordinate) + .sort(); + const soakExpected = plan.soak?.enabled === true; + const soakResults = results.filter((result) => result.runKind === 'soak'); + const soakComplete = !soakExpected + ? soakObserved === 0 + : soakObserved === 1 + && soakResults[0].evidenceMode === 'qualification' + && soakResults[0].accepted; + return { + results, + matrix: { + complete: plan.qualification != null + && missing.length === 0 + && duplicates.length === 0 + && diagnostic.length === 0 + && seen.size === expected.size + && input.length === campaign.manifest.jobs.length + && canonicalQualificationSchedule + && campaign.evidenceMode === 'qualification' + && campaign.manifest.platform === 'linux' + && campaign.qualificationBlockers.length === 0 + && hostileEvidenceReady + && soakComplete, + expectedCoordinates: expected.size, + observedCoordinates: seen.size, + missing, + duplicates, + diagnostic: diagnostic.sort(), + soakExpected, + soakObserved, + soakComplete + } + }; +}; + +const formatNumber = (value: number | null, digits = 2): string => value == null + ? 'n/a' + : Number.isFinite(value) ? value.toFixed(digits) : String(value); + +const medianNullable = (values: Array): number | null => { + const available = values.filter((value): value is number => value != null); + return available.length > 0 ? percentile(available, 0.5) : null; +}; + +const toMiB = (value: number | null): number | null => value == null + ? null + : value / 1024 ** 2; + +export const rejectDuplicatePostgresRunEpochs = ( + input: DensityRunResult[] +): DensityRunResult[] => { + const counts = new Map(); + for (const run of input) { + const evidence = run.postgresRunAttestation; + if (!evidence) continue; + for (const claim of postgresRunIdentityClaims(evidence)) { + counts.set(claim, (counts.get(claim) ?? 0) + 1); + } + } + const duplicates = new Set([...counts] + .filter(([_claim, count]) => count > 1) + .map(([claim]) => claim)); + return input.map((run) => { + const evidence = run.postgresRunAttestation; + if (!evidence) return run; + const reused = postgresRunIdentityClaims(evidence) + .filter((claim) => duplicates.has(claim)); + if (reused.length === 0) return run; + const failure = `PostgreSQL container/clone identities reused across matrix: ${reused.join(', ')}`; + return { + ...run, + accepted: false, + qualifiedCustomers: 0, + qualifiedTenants: 0, + tenantsPerConfiguredOldSpaceGiB: 0, + tenantsPerPeakRssGiB: 0, + customersPerAlignedServiceGiB: 0, + customersPerServiceMemoryUpperBoundGiB: 0, + failures: run.failures.includes(failure) + ? run.failures + : [...run.failures, failure] + }; + }); +}; + +export const renderReport = ( + inputResults: unknown[], + plan: DensityPlanV1, + fleet: FleetV1 +): string => { + const validated = validateResultSet(inputResults, plan, fleet); + const results = rejectDuplicatePostgresRunEpochs(validated.results); + const matrixResults = results.filter((result) => result.runKind === 'matrix'); + const soakResults = results.filter((result) => result.runKind === 'soak'); + const soakComplete = !validated.matrix.soakExpected + ? soakResults.length === 0 + : soakResults.length === 1 + && soakResults[0].evidenceMode === 'qualification' + && soakResults[0].accepted; + const qualificationEvidenceComplete = validated.matrix.complete && soakComplete; + const gates = plan.gates; + const groups = new Map(); + for (const result of matrixResults) { + const key = `${result.arm}|${result.heapMiB}|${result.configuredTenants}`; + const group = groups.get(key) ?? []; + group.push(result); + groups.set(key, group); + } + const lines = [ + '# Graphile customer-density results', + '', + `Generated: ${new Date().toISOString()}`, + '', + `Evidence mode: **${qualificationEvidenceComplete ? 'qualification' : 'diagnostic'}**. Full configured matrix: ${validated.matrix.observedCoordinates}/${validated.matrix.expectedCoordinates} coordinates; missing=${validated.matrix.missing.length}; duplicates=${validated.matrix.duplicates.length}; diagnostic-only=${validated.matrix.diagnostic.length}; configured soak=${validated.matrix.soakExpected ? `${validated.matrix.soakObserved}/1, accepted=${soakComplete ? 'yes' : 'no'}` : 'disabled'}.`, + '', + 'A customer counts only when every declared GraphQL surface and realtime transport stays resident and serves the full qualification workload, all isolation canaries are conclusive, bleed is zero, error rate and p99 meet their gates, required capabilities ran, PostgreSQL telemetry completed, and post-warmup Graphile and PostgreSQL pool eviction/refusal/build/disposal counters remain unchanged. The primary memory denominator is the maximum post-warmup time-aligned sum of current Node RSS and raw PostgreSQL cgroup-v2 memory charge when available; the all-phase high-water upper bound, Docker working set, configured V8, and Node-only RSS remain diagnostics.', + '', + 'Qualifying physical-database runs use a full live DDL/ACL audit before the Graphile timer starts. That audit intentionally warms PostgreSQL catalogs, so the build column is post-attestation warm-catalog latency, not a pristine-catalog cold-build claim. Every arm receives the same audit, and reused container/clone epochs are rejected across the complete result set.', + '', + '| Arm | Old-space MiB | Customers | Physical DBs | Dedicated PG | Runs | Accepted | Warm observed heap delta MiB/instance | Post-attestation build ms | PG baseline MiB | PG warm-boundary MiB | PG spike MiB | PG raw peak MiB | PG working-set peak MiB | Node peak RSS MiB | Aligned Node+PG peak MiB | Conservative service upper bound MiB | Offered RPS | Customer workload RPS | Periodic validation RPS | Realtime validation RPS | Combined HTTP RPS | workload p99 ms | PG pools | PG active leases | PG backends | Pool clients | Realtime managers | Realtime transports | Qualified customers/aligned service GiB | Qualified customers/service upper-bound GiB | Configured customers/aligned service GiB (diagnostic) | Configured customers/service upper-bound GiB (diagnostic) | Customers/configured old-space GiB | Customers/Node peak RSS GiB |', + '|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|' + ]; + for (const group of [...groups.values()].sort((a, b) => { + const left = `${a[0].arm}:${a[0].heapMiB}:${a[0].configuredTenants}`; + const right = `${b[0].arm}:${b[0].heapMiB}:${b[0].configuredTenants}`; + return left.localeCompare(right); + })) { + const first = group[0]; + lines.push([ + `| ${first.arm}`, + first.heapMiB, + first.configuredTenants, + formatNumber(medianNullable(group.map( + (run) => run.residentPhysicalDatabases ?? null + )), 0), + group.every((run) => run.postgresContainerDedicated === true) ? 'yes' : 'no', + group.length, + group.filter((run) => run.accepted).length, + formatNumber(toMiB(medianNullable(group.map( + (run) => run.warmObservedHeapDeltaPerInstanceBytes + ))), 1), + formatNumber(medianNullable(group.map((run) => run.coldBuildMaxMs)), 1), + formatNumber(toMiB(medianNullable(group.map((run) => run.postgresBaselineBytes))), 1), + formatNumber(toMiB(medianNullable(group.map( + (run) => run.postgresWarmBoundaryBytes + ))), 1), + formatNumber(toMiB(medianNullable(group.map((run) => run.postgresColdBuildSpikeBytes))), 1), + formatNumber(toMiB(medianNullable(group.map((run) => run.postgresPeakBytes))), 1), + formatNumber(toMiB(medianNullable(group.map( + (run) => run.postgresWorkingSetPeakBytes ?? null + ))), 1), + formatNumber(toMiB(medianNullable(group.map((run) => run.peakRssBytes))), 1), + formatNumber(toMiB(medianNullable(group.map((run) => run.alignedServicePeakBytes))), 1), + formatNumber(toMiB(medianNullable(group.map((run) => run.serviceMemoryUpperBoundBytes))), 1), + formatNumber(percentile(group.map((run) => run.offeredLoad.totalRps), 0.5), 1), + formatNumber(percentile(group.map( + (run) => run.customerWorkloadRps ?? run.achievedRps + ), 0.5), 1), + formatNumber(percentile(group.map( + (run) => run.periodicValidationRps ?? 0 + ), 0.5), 1), + formatNumber(percentile(group.map( + (run) => run.realtimeValidationRps ?? 0 + ), 0.5), 1), + formatNumber(percentile(group.map((run) => + run.combinedHttpRps + ?? (run.customerWorkloadRps ?? run.achievedRps) + + (run.periodicValidationRps ?? 0) + + (run.realtimeValidationRps ?? 0) + ), 0.5), 1), + formatNumber(percentile(group.map((run) => run.p99Ms), 0.5), 1), + formatNumber(medianNullable(group.map((run) => run.pgPoolCacheSize)), 0), + formatNumber(medianNullable(group.map((run) => run.pgPoolActiveLeases)), 0), + formatNumber(medianNullable(group.map((run) => run.postgresBackendPeak ?? null)), 0), + formatNumber(medianNullable(group.map((run) => run.pgPoolTotalClients ?? null)), 0), + formatNumber(medianNullable(group.map((run) => run.residentRealtimeManagers ?? null)), 0), + formatNumber(medianNullable(group.map((run) => run.residentRealtimeTransports ?? null)), 0), + formatNumber(medianNullable(group.map((run) => run.customersPerAlignedServiceGiB))), + formatNumber(medianNullable(group.map( + (run) => run.customersPerServiceMemoryUpperBoundGiB + ))), + formatNumber(medianNullable(group.map( + (run) => run.configuredCustomersPerAlignedServiceGiB + ))), + formatNumber(medianNullable(group.map( + (run) => run.configuredCustomersPerServiceMemoryUpperBoundGiB + ))), + formatNumber(percentile(group.map( + (run) => run.tenantsPerConfiguredOldSpaceGiB + ), 0.5)), + `${formatNumber(medianNullable(group.map((run) => run.tenantsPerPeakRssGiB)))} |` + ].join(' | ')); + } + + const boundaries = summarizeCapacityBoundaries(matrixResults); + lines.push( + '', + '## Capacity boundaries', + '', + '| Arm | Old-space MiB | Highest all-repetitions customer pass | Lowest greater fail | Monotonic | Boundary reached | Incomplete counts | Customers/aligned service GiB | Customers/service upper-bound GiB | Customers/configured old-space GiB | Customers/Node peak RSS GiB |', + '|---|---:|---:|---:|---:|---:|---|---:|---:|---:|---:|', + ...boundaries.map((boundary) => [ + `| ${boundary.arm}`, + boundary.heapMiB, + boundary.highestAllRepetitionsPass ?? 'n/a', + boundary.lowestGreaterFail ?? 'n/a', + boundary.monotonicQualification ? 'yes' : 'no', + boundary.capacityBoundaryReached ? 'yes' : 'no', + boundary.incompleteTenantCounts.join(',') || 'none', + formatNumber(boundary.medianCustomersPerAlignedServiceGiB), + formatNumber(boundary.medianCustomersPerServiceMemoryUpperBoundGiB), + formatNumber(boundary.medianTenantsPerConfiguredOldSpaceGiB), + `${formatNumber(boundary.medianTenantsPerPeakRssGiB)} |` + ].join(' | ')), + '', + '## Candidate decisions', + '', + ); + const baselineArm = plan.qualification?.baselineArm ?? plan.arms[0]?.name; + const baseline = matrixResults.filter((result) => result.arm === baselineArm); + const candidateArms = plan.arms + .map((arm) => arm.name) + .filter((arm) => + arm !== baselineArm && matrixResults.some((result) => result.arm === arm) + ); + if (!plan.qualification) { + lines.push('This plan is diagnostic-only; it has no qualification contract.', ''); + } else if (candidateArms.length === 0) { + lines.push('No configured candidate arms were executed.', ''); + } else { + for (const candidateArm of candidateArms) { + const candidate = matrixResults.filter((result) => result.arm === candidateArm); + const comparison = compareDensity(baseline, candidate, gates); + const materiallyBetter = qualificationEvidenceComplete && comparison.materiallyBetter; + lines.push( + `### ${candidateArm} vs ${baselineArm}`, + '', + `Materially better: **${materiallyBetter ? 'yes' : 'no'}**. Median aligned Node+PostgreSQL density improvement: ${formatNumber(comparison.alignedServiceMedianImprovement * 100, 1)}%; conservative service upper-bound density improvement: ${formatNumber(comparison.serviceMemoryUpperBoundMedianImprovement * 100, 1)}%; both actual service-memory measures avoid per-heap regression: ${comparison.alignedServiceNonRegression && comparison.serviceMemoryUpperBoundNonRegression ? 'yes' : 'no'}; configured-old-space diagnostic improvement: ${formatNumber(comparison.configuredOldSpaceMedianImprovement * 100, 1)}%; Node-only peak-RSS diagnostic improvement: ${formatNumber(comparison.peakRssMedianImprovement * 100, 1)}%; every paired heap adds the required customer count: ${comparison.everyHeapAddsTenants ? 'yes' : 'no'}; capacity boundaries are complete: ${comparison.capacityBoundariesComplete ? 'yes' : 'no'}; matrices are exactly paired: ${comparison.pairedMatrixComplete ? 'yes' : 'no'}; full configured qualification evidence, including soak when enabled, is present: ${qualificationEvidenceComplete ? 'yes' : 'no'}.`, + '' + ); + } + } + lines.push( + 'Failed and incomplete runs remain in the denominator. Missing arms, heaps, tenant counts, repetitions, short smoke workloads, unavailable memory telemetry, and inconclusive canaries are not treated as passing evidence.', + '', + '## Soak runs', + '', + 'Soak records validate the selected maximum-density candidate over time; they are excluded from every matrix median, capacity boundary, and candidate comparison above.', + '', + '| Arm | Old-space MiB | Customers | Duration sec | Accepted | workload p99 ms | Aligned Node+PG peak MiB | Customers/aligned service GiB | Heap growth MiB/hour |', + '|---|---:|---:|---:|---:|---:|---:|---:|---:|' + ); + if (soakResults.length === 0) { + lines.push('| none | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a |'); + } else { + for (const result of soakResults) { + lines.push([ + `| ${result.arm}`, + result.heapMiB, + result.configuredTenants, + formatNumber(result.durationSec, 0), + result.accepted ? 'yes' : 'no', + formatNumber(result.p99Ms, 1), + formatNumber(toMiB(result.alignedServicePeakBytes), 1), + formatNumber(result.customersPerAlignedServiceGiB), + `${formatNumber(result.retainedHeapGrowthMiBPerHour)} |` + ].join(' | ')); + } + } + lines.push( + '', + '## Run failures', + '' + ); + const failed = results.filter((result) => !result.accepted); + if (failed.length === 0) lines.push('None.'); + else for (const result of failed) { + lines.push(`- ${result.arm} h${result.heapMiB} t${result.configuredTenants} r${result.repetition}: ${result.failures.join('; ')}`); + } + return `${lines.join('\n')}\n`; +}; + +export const writeReport = ( + resultsFile: string, + outputFile: string, + plan: DensityPlanV1, + fleet: FleetV1 +): void => { + const report = renderReport(readResults(resultsFile), plan, fleet); + fs.mkdirSync(path.dirname(path.resolve(outputFile)), { recursive: true }); + fs.writeFileSync(path.resolve(outputFile), report, 'utf8'); +}; diff --git a/packages/perf-harness/src/run-attestation.ts b/packages/perf-harness/src/run-attestation.ts new file mode 100644 index 0000000000..9c195b4d9a --- /dev/null +++ b/packages/perf-harness/src/run-attestation.ts @@ -0,0 +1,364 @@ +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { resolveTemplate } from './config'; +import type { + ArmPlan, + PostgresRunAttestationEvidence +} from './types'; + +const SHA256 = /^sha256:[a-f0-9]{64}$/; +const CONTAINER_ID = /^[a-f0-9]{64}$/; +const KIND = 'physical-density-measurement-attestation-v1'; +const COMMAND_KILL_GRACE_MS = 2_000; + +export interface RunAttestationContext { + arm: string; + heapMiB: number; + tenantCount: number; + repetition: number; + runOrderIndex: number; + planSha256: string; + fleetSha256: string; + notBeforeEpochMs: number; + artifactDir: string; +} + +export const postgresRunIdentityClaims = ( + evidence: PostgresRunAttestationEvidence +): string[] => [ + `epoch:${evidence.epochId}`, + `container:${evidence.containerId}`, + `cgroup:${evidence.cgroupIdentitySha256}`, + `postgres-system:${evidence.postgresSystemIdentifier}`, + `clone:${evidence.cloneId}`, + `clone-attestation-set:${evidence.cloneAttestationSetSha256}`, + `clone-nonce-set:${evidence.cloneNonceSetSha256}` +]; + +const canonicalize = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + const record = value as Record; + return Object.fromEntries(Object.keys(record).sort().map((key) => [ + key, + canonicalize(record[key]) + ])); +}; + +const canonicalSha256 = (value: unknown): string => `sha256:${createHash('sha256') + .update(JSON.stringify(canonicalize(value))) + .digest('hex')}`; + +const readRegularFile = (file: string): Buffer => { + const before = fs.lstatSync(file); + if (before.isSymbolicLink() || !before.isFile()) { + throw new Error('PostgreSQL run evidence must be a regular non-symlink file'); + } + const descriptor = fs.openSync( + file, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) + ); + try { + const opened = fs.fstatSync(descriptor); + if ( + !opened.isFile() + || opened.dev !== before.dev + || opened.ino !== before.ino + ) { + throw new Error('PostgreSQL run evidence changed while it was opened'); + } + return fs.readFileSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +}; + +const fileSha256 = (file: string): string => `sha256:${createHash('sha256') + .update(readRegularFile(file)) + .digest('hex')}`; + +const runCommand = async ( + command: string[], + cwd: string, + timeoutMs: number +): Promise => { + if (command.length === 0) throw new Error('PostgreSQL run command is empty'); + await new Promise((resolve, reject) => { + const child = spawn(command[0], command.slice(1), { + cwd, + env: process.env, + detached: process.platform !== 'win32', + stdio: 'ignore' + }); + let settled = false; + let timedOut = false; + let forceTimer: NodeJS.Timeout | null = null; + const signalTree = (signal: NodeJS.Signals): void => { + if (child.pid && process.platform !== 'win32') { + try { + process.kill(-child.pid, signal); + return; + } catch { + // Fall through to the direct child as a best-effort Windows/fork + // fallback. The child exit remains the completion boundary. + } + } + child.kill(signal); + }; + const clearTimers = (): void => { + clearTimeout(timer); + if (forceTimer) clearTimeout(forceTimer); + }; + const timer = setTimeout(() => { + if (settled) return; + timedOut = true; + signalTree('SIGTERM'); + forceTimer = setTimeout(() => signalTree('SIGKILL'), COMMAND_KILL_GRACE_MS); + }, timeoutMs); + child.once('error', (error) => { + if (settled) return; + settled = true; + clearTimers(); + reject(error); + }); + child.once('exit', (code, signal) => { + if (settled) return; + settled = true; + clearTimers(); + if (timedOut) reject(new Error('PostgreSQL run command timed out')); + else if (code === 0 && signal == null) resolve(); + else reject(new Error( + `PostgreSQL run command failed: code=${code ?? 'null'} signal=${signal ?? 'null'}` + )); + }); + }); +}; + +const requireRecord = (value: unknown, label: string): Record => { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`PostgreSQL run attestation ${label} is invalid`); + } + return value as Record; +}; + +export const normalizePostgresRunAttestation = ( + raw: unknown, + context: RunAttestationContext, + artifactPath: string, + artifactSha256 = fileSha256(artifactPath) +): PostgresRunAttestationEvidence => { + const envelope = requireRecord(raw, 'envelope'); + const payload = requireRecord(envelope.payload, 'payload'); + const run = requireRecord(payload.run, 'run binding'); + const freshness = requireRecord(payload.freshness, 'freshness'); + const immutableEpoch = requireRecord(payload.immutableEpoch, 'immutable epoch'); + const container = requireRecord(payload.container, 'container'); + const cgroup = requireRecord(payload.cgroup, 'cgroup'); + const postgres = requireRecord(payload.postgres, 'PostgreSQL cluster'); + const customerAudits = payload.customerAudits; + const provisionClone = requireRecord(payload.provisionClone, 'provision clone'); + const orderedCustomerAudits = Array.isArray(customerAudits) + ? [...customerAudits].sort((left, right) => + String(left?.customerId).localeCompare(String(right?.customerId))) + : []; + const cloneNonceSetSha256 = canonicalSha256(orderedCustomerAudits.map((audit) => ({ + customerId: audit.customerId, + cloneNonceSha256: audit.cloneNonceSha256 + }))); + const liveContractSetSha256 = canonicalSha256(orderedCustomerAudits.map((audit) => ({ + customerId: audit.customerId, + databaseContractFingerprint: audit.databaseContractFingerprint, + structuralFingerprint: audit.structuralFingerprints?.combined?.sha256 + }))); + const observedAtMs = Date.parse(payload.observedAt ?? ''); + const containerStartedAtMs = Date.parse(container.startedAt ?? ''); + const postgresStartedAtMs = Date.parse(postgres.postmasterStartedAt ?? ''); + if ( + envelope.version !== 1 + || envelope.kind !== KIND + || !SHA256.test(envelope.payloadSha256 ?? '') + || canonicalSha256(payload) !== envelope.payloadSha256 + || run.arm !== context.arm + || run.heapMiB !== context.heapMiB + || run.customerCount !== context.tenantCount + || run.repetition !== context.repetition + || run.runOrderIndex !== context.runOrderIndex + || run.planSha256 !== `sha256:${context.planSha256}` + || run.fleetSha256 !== `sha256:${context.fleetSha256}` + || !SHA256.test(payload.epochId ?? '') + || payload.epochId !== canonicalSha256(immutableEpoch) + || !CONTAINER_ID.test(container.id ?? '') + || typeof container.startedAt !== 'string' + || !Number.isSafeInteger(observedAtMs) + || !Number.isSafeInteger(containerStartedAtMs) + || !Number.isSafeInteger(postgresStartedAtMs) + || observedAtMs < context.notBeforeEpochMs + || containerStartedAtMs > observedAtMs + || postgresStartedAtMs > observedAtMs + || !SHA256.test(cgroup.identitySha256 ?? '') + || cgroup.version !== 1 + || cgroup.source !== 'container-cgroup-v2' + || typeof freshness.freshContainerForRun !== 'boolean' + || freshness.freshContainerForRun + !== (containerStartedAtMs >= context.notBeforeEpochMs) + || freshness.cgroupV2Verified !== true + || freshness.notBeforeEpochMs !== context.notBeforeEpochMs + || freshness.startToleranceMs !== 0 + || payload.catalogCacheState !== 'warmed-by-live-contract-audit' + || provisionClone.purpose !== 'measurement' + || provisionClone.version !== 1 + || typeof provisionClone.id !== 'string' + || !provisionClone.id + || !SHA256.test(provisionClone.attestationSetSha256 ?? '') + || !SHA256.test(payload.manifestSha256 ?? '') + || !SHA256.test(payload.containerTemplateSha256 ?? '') + || !SHA256.test(payload.canonicalDatabaseContractFingerprint ?? '') + || !/^\d+$/.test(postgres.systemIdentifier ?? '') + || !Array.isArray(customerAudits) + || customerAudits.length !== context.tenantCount + || new Set(customerAudits.map((audit: any) => audit?.customerId)).size + !== context.tenantCount + || customerAudits.some((audit: any) => + typeof audit?.customerId !== 'string' + || !audit.customerId + || !SHA256.test(audit?.databaseContractFingerprint ?? '') + || !SHA256.test(audit?.structuralFingerprints?.combined?.sha256 ?? '') + || !SHA256.test(audit?.cloneAttestationSha256 ?? '') + || !SHA256.test(audit?.cloneNonceSha256 ?? '') + ) + || immutableEpoch.dockerContainerId !== container.id + || immutableEpoch.dockerStartedAt !== container.startedAt + || !SHA256.test(immutableEpoch.containerConfigurationSha256 ?? '') + || immutableEpoch.cgroupIdentitySha256 !== cgroup.identitySha256 + || immutableEpoch.postgresSystemIdentifier !== postgres.systemIdentifier + || immutableEpoch.postgresStartedAt !== postgres.postmasterStartedAt + || immutableEpoch.cloneId !== provisionClone.id + || immutableEpoch.cloneAttestationSetSha256 + !== provisionClone.attestationSetSha256 + || immutableEpoch.cloneNonceSetSha256 !== cloneNonceSetSha256 + || immutableEpoch.liveContractSetSha256 !== liveContractSetSha256 + || !SHA256.test(immutableEpoch.cloneNonceSetSha256 ?? '') + || !SHA256.test(immutableEpoch.liveContractSetSha256 ?? '') + ) { + throw new Error('PostgreSQL run attestation failed exact validation'); + } + return { + version: 1, + kind: KIND, + artifactPath, + artifactSha256, + payloadSha256: envelope.payloadSha256, + epochId: payload.epochId, + arm: run.arm, + heapMiB: run.heapMiB, + tenantCount: run.customerCount, + repetition: run.repetition, + runOrderIndex: run.runOrderIndex, + planSha256: run.planSha256, + fleetSha256: run.fleetSha256, + containerId: container.id, + containerStartedAt: container.startedAt, + cgroupIdentitySha256: cgroup.identitySha256, + containerConfigurationSha256: + immutableEpoch.containerConfigurationSha256, + postgresSystemIdentifier: immutableEpoch.postgresSystemIdentifier, + postgresStartedAt: immutableEpoch.postgresStartedAt, + cloneId: provisionClone.id, + cloneAttestationSetSha256: immutableEpoch.cloneAttestationSetSha256, + cloneNonceSetSha256: immutableEpoch.cloneNonceSetSha256, + liveContractSetSha256: immutableEpoch.liveContractSetSha256, + manifestSha256: payload.manifestSha256, + containerTemplateSha256: payload.containerTemplateSha256, + canonicalDatabaseContractFingerprint: + payload.canonicalDatabaseContractFingerprint, + freshContainerForRun: freshness.freshContainerForRun, + cgroupV2Verified: freshness.cgroupV2Verified, + liveCustomerContractsAudited: customerAudits.length, + catalogCacheState: payload.catalogCacheState + }; +}; + +export const collectPostgresRunAttestation = async ( + arm: ArmPlan, + context: RunAttestationContext +): Promise => { + const configured = arm.postgresRunAttestation; + if (!configured) return null; + const artifactPath = path.join( + context.artifactDir, + 'postgres-run-attestation.json' + ); + const postgresFixtureDir = path.join(context.artifactDir, 'postgres-fixture'); + const postgresManifestFile = path.join(postgresFixtureDir, 'provision.json'); + const postgresSecretsFile = path.join(postgresFixtureDir, 'runtime-secrets.json'); + if (fs.existsSync(artifactPath)) { + throw new Error('PostgreSQL run attestation artifact already exists'); + } + const variables = { + arm: context.arm, + heapMiB: context.heapMiB, + tenantCount: context.tenantCount, + repetition: context.repetition, + runOrderIndex: context.runOrderIndex, + planSha256: `sha256:${context.planSha256}`, + fleetSha256: `sha256:${context.fleetSha256}`, + notBeforeEpochMs: context.notBeforeEpochMs, + artifactDir: context.artifactDir, + attestationFile: artifactPath, + postgresFixtureDir, + postgresManifestFile, + postgresSecretsFile, + port: arm.port, + mode: arm.introspectionMode + }; + const cwd = path.resolve(arm.cwd + ? resolveTemplate(arm.cwd, variables) + : process.cwd()); + const timeoutMs = configured.timeoutMs ?? 900_000; + if (configured.prepareCommand?.length) { + await runCommand( + configured.prepareCommand.map((part) => resolveTemplate(part, variables)), + cwd, + timeoutMs + ); + } + await runCommand( + configured.command.map((part) => resolveTemplate(part, variables)), + cwd, + timeoutMs + ); + const stat = fs.lstatSync(artifactPath); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new Error('PostgreSQL run attestation must be a regular non-symlink file'); + } + const manifestStat = fs.lstatSync(postgresManifestFile); + const secretsStat = fs.lstatSync(postgresSecretsFile); + if ( + manifestStat.isSymbolicLink() + || !manifestStat.isFile() + || secretsStat.isSymbolicLink() + || !secretsStat.isFile() + || (secretsStat.mode & 0o777) !== 0o600 + ) { + throw new Error('PostgreSQL run fixture inputs failed private-file validation'); + } + const artifactBytes = readRegularFile(artifactPath); + const artifactSha256 = `sha256:${createHash('sha256') + .update(artifactBytes) + .digest('hex')}`; + const raw = JSON.parse(artifactBytes.toString('utf8')) as unknown; + const evidence = normalizePostgresRunAttestation( + raw, + context, + artifactPath, + artifactSha256 + ); + const manifestSha256 = fileSha256(postgresManifestFile); + if (manifestSha256 !== evidence.manifestSha256) { + throw new Error('PostgreSQL run manifest does not match its attestation'); + } + return evidence; +}; diff --git a/packages/perf-harness/src/run.ts b/packages/perf-harness/src/run.ts new file mode 100644 index 0000000000..07a746cccb --- /dev/null +++ b/packages/perf-harness/src/run.ts @@ -0,0 +1,856 @@ +import { createHash, randomBytes } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { + armEnvironmentForHeap, + assertLoopbackObservabilityUrl, + assertLoopbackRetainedHeapCheckpointUrl, + DEFAULT_RUN_ORDER_SEED, + hasExactHostileValidationEvidence, + resolveTenants, + resolveTemplate, + soakArmName, + tenantCountsForHeap, + validateCoverage +} from './config'; +import { + createWorkloadCapture, + resolveOfferedLoad, + resolveWarmupTimeoutMs, + runWorkload, + type WorkloadCapture, + type WorkloadResult +} from './http'; +import { bindResultEvidence, writeScoreContext } from './evidence'; +import { + normalizeRetainedMemoryCheckpoint, + startMemorySampler +} from './memory'; +import { startPostgresMemorySampler } from './postgres'; +import { startArmProcess } from './process'; +import { createRealtimeDriver, type RealtimeDriverSnapshot } from './realtime'; +import { + buildRunSchedule, + sameRunSchedule, + scheduleJobsForPlan, + scheduleManifestSha256, + type CampaignScheduleManifestV1 +} from './schedule'; +import { + collectPostgresRunAttestation, + postgresRunIdentityClaims +} from './run-attestation'; +import { + scoreRun, + summarizeCapacityBoundaries, + type ScoreInput +} from './score'; +import type { + ArmPlan, + ArmProvenance, + DensityPlanV1, + DensityRunResult, + FleetV1, + PostgresRunAttestationEvidence, + ResolvedMemoryPolicy, + RetainedMemoryCheckpoint, + RetainedMemoryCheckpointPair, + RealtimeDeliveryCoverage, + WorkloadPlan +} from './types'; + +export interface RunSelection { + arms?: string[]; + heaps?: number[]; + tenantCounts?: number[]; + repetitions?: number; + smoke?: boolean; +} + +export { buildRunSchedule } from './schedule'; + +interface RunContext { + expectedMatrixRepetitions: number; + runKind: 'matrix' | 'soak'; + evidenceMode: 'qualification' | 'diagnostic'; + campaignId: string; + scheduleSha256: string; + previousResultPayloadSha256: string | null; + qualificationCohortSha256: string; + runOrderSeed: string; + runOrderIndex: number; + planSha256: string; + fleetSha256: string; + notBeforeEpochMs: number; + claimPostgresRunIdentity( + evidence: PostgresRunAttestationEvidence + ): string | null; +} + +const sha256 = (value: string): string => createHash('sha256').update(value).digest('hex'); + +const executionErrorEvidence = (error: unknown): string => { + const message = error instanceof Error ? error.message : String(error); + const code = message.match(/(?:^|\b)([A-Z][A-Z0-9_]{2,})(?=\b|:)/)?.[1] + ?? 'CPERF_EXECUTION_FAILED'; + return `${code}:sha256:${sha256(message)}`; +}; + +const writeResult = ( + root: string, + result: DensityRunResult, + input: ScoreInput, + context: Pick< + RunContext, + | 'planSha256' + | 'fleetSha256' + | 'notBeforeEpochMs' + | 'campaignId' + | 'scheduleSha256' + | 'previousResultPayloadSha256' + > +): void => { + fs.mkdirSync(result.artifactDir, { recursive: true }); + writeScoreContext(result.artifactDir, input, context); + bindResultEvidence(result); + fs.writeFileSync( + path.join(result.artifactDir, 'result.json'), + `${JSON.stringify(result, null, 2)}\n`, + 'utf8' + ); + fs.mkdirSync(root, { recursive: true }); + const serialized = `${JSON.stringify(result)}\n`; + fs.appendFileSync(path.join(root, 'results.ndjson'), serialized, 'utf8'); + fs.appendFileSync( + path.join(root, `results-${context.campaignId}.ndjson`), + serialized, + 'utf8' + ); +}; + +const writeJson = (file: string, value: unknown): void => { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, 'utf8'); +}; + +const writeExclusiveJson = (file: string, value: unknown): void => { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, { + encoding: 'utf8', + flag: 'wx' + }); +}; + +const invokePostWarmupHook = async ( + arm: ArmPlan, + heapMiB: number, + tenantCount: number, + artifactDir: string, + headers: Readonly> +): Promise => { + if (!arm.postWarmupUrl) return; + const url = resolveTemplate(arm.postWarmupUrl, { + heapMiB, + port: arm.port, + artifactDir, + mode: arm.introspectionMode, + tenantCount + }); + const parsed = new URL(url); + if ( + parsed.protocol !== 'http:' + || !['127.0.0.1', 'localhost', '::1'].includes(parsed.hostname) + || Number(parsed.port) !== arm.port + || parsed.username + || parsed.password + || parsed.search + || parsed.hash + ) { + throw new Error(`postWarmupUrl must be an authenticated loopback URL on port ${arm.port}`); + } + const response = await fetch(url, { + method: 'POST', + headers, + signal: AbortSignal.timeout(60_000) + }); + const responseText = await response.text(); + if (!response.ok) { + throw new Error( + `post-warmup hook failed with HTTP ${response.status}: ${responseText.slice(0, 512)}` + ); + } + let responseBody: unknown = responseText; + try { + responseBody = responseText ? JSON.parse(responseText) : null; + } catch { + // The hook contract permits a diagnostic text response. + } + writeJson(path.join(artifactDir, 'post-warmup-hook.json'), { + timestamp: new Date().toISOString(), + url: parsed.pathname, + response: responseBody + }); +}; + +const invokeRetainedMemoryCheckpoint = async ( + arm: ArmPlan, + heapMiB: number, + tenantCount: number, + headers: Readonly>, + errors: string[] +): Promise => { + if (!arm.retainedHeapCheckpointUrl) return null; + const url = resolveTemplate(arm.retainedHeapCheckpointUrl, { + heapMiB, + port: arm.port, + artifactDir: '', + mode: arm.introspectionMode, + tenantCount + }); + assertLoopbackRetainedHeapCheckpointUrl(url, arm.port); + try { + const response = await fetch(url, { + method: 'POST', + headers, + signal: AbortSignal.timeout(60_000) + }); + const responseText = await response.text(); + let body: unknown = null; + try { + body = responseText ? JSON.parse(responseText) : null; + } catch { + errors.push('retained-memory checkpoint returned non-JSON data'); + } + const checkpoint = normalizeRetainedMemoryCheckpoint(body); + if (!checkpoint) { + const serverMessage = typeof (body as any)?.error?.message === 'string' + ? `: ${(body as any).error.message}` + : ''; + errors.push( + `retained-memory checkpoint response was invalid (HTTP ${response.status})${serverMessage}` + ); + return null; + } + if (!response.ok) { + errors.push(`retained-memory checkpoint failed with HTTP ${response.status}`); + } + return checkpoint; + } catch (error) { + errors.push( + `retained-memory checkpoint request failed: ${error instanceof Error ? error.message : String(error)}` + ); + return null; + } +}; + +const resolvedMemoryPolicy = ( + arm: ArmPlan, + heapMiB: number, + expectedV8HeapLimitBytes: number | null +): ResolvedMemoryPolicy => { + const env = { ...process.env, ...armEnvironmentForHeap(arm, heapMiB) }; + const value = (name: string): string | null => env[name]?.trim() || null; + return { + configuredMaxOldSpaceMiB: heapMiB, + expectedV8HeapLimitBytes, + graphileCacheMax: value('GRAPHILE_CACHE_MAX'), + graphileCacheInstanceHeapBytes: value('GRAPHILE_CACHE_INSTANCE_HEAP_BYTES'), + graphileCacheServerReserveBytes: value('GRAPHILE_CACHE_SERVER_RESERVE_BYTES'), + graphileCacheBuildReserveBytes: value('GRAPHILE_CACHE_BUILD_RESERVE_BYTES'), + graphileCacheRssLimitBytes: value('GRAPHILE_CACHE_RSS_LIMIT_BYTES'), + graphileCacheRssBuildReserveBytes: value('GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES'), + graphileCacheCalibrationId: value('GRAPHILE_CACHE_CALIBRATION_ID'), + graphileCacheAdmissionMode: value('GRAPHILE_CACHE_ADMISSION_MODE'), + graphileBuildMaxConcurrency: value('GRAPHILE_BUILD_MAX_CONCURRENCY') + }; +}; + +const contextualProvenance = ( + provenance: ArmProvenance, + arm: ArmPlan, + heapMiB: number, + expectedV8HeapLimitBytes: number | null, + context: RunContext +): ArmProvenance => ({ + ...provenance, + planSha256: context.planSha256, + fleetSha256: context.fleetSha256, + runOrderSeed: context.runOrderSeed, + runOrderIndex: context.runOrderIndex, + memoryPolicy: resolvedMemoryPolicy(arm, heapMiB, expectedV8HeapLimitBytes) +}); + +const persistPartialArtifacts = ( + artifactDir: string, + memory: ReturnType | null, + postgresMemory: ReturnType | null, + capture: WorkloadCapture, + workloadResult: WorkloadResult | null, + retainedMemory: RetainedMemoryCheckpointPair +): void => { + writeJson(path.join(artifactDir, 'memory.json'), { + snapshots: memory?.snapshots ?? [], + osSnapshots: memory?.osSnapshots ?? [], + errors: memory?.errors ?? [], + warmupIndex: memory?.warmupIndex ?? -1, + osWarmupIndex: memory?.osWarmupIndex ?? -1, + osPeakRssBytes: memory?.osPeakRssBytes ?? null + }); + writeJson(path.join(artifactDir, 'postgres-memory.json'), { + snapshots: postgresMemory?.snapshots ?? [], + errors: postgresMemory?.errors ?? [] + }); + writeJson(path.join(artifactDir, 'canaries.json'), capture.canaries); + writeJson(path.join(artifactDir, 'canary-schedule.json'), capture.canarySchedule); + fs.writeFileSync( + path.join(artifactDir, 'requests.ndjson'), + capture.samples.length > 0 + ? `${capture.samples.map((sample) => JSON.stringify(sample)).join('\n')}\n` + : '', + 'utf8' + ); + writeJson(path.join(artifactDir, 'workload-progress.json'), { + warmedSurfaces: [...capture.warmedSurfaces].map(([tenantId, surfaces]) => ({ + tenantId, + surfaces: [...surfaces].sort() + })), + warmupLatencies: capture.warmupLatencies, + samples: capture.samples.length, + canaries: capture.canaries.length, + canarySchedule: capture.canarySchedule, + offeredLoad: workloadResult?.offeredLoad ?? null, + resolvedWarmupTimeoutMs: workloadResult?.resolvedWarmupTimeoutMs ?? null, + workloadDurationMs: workloadResult?.workloadDurationMs ?? null + }); + writeJson(path.join(artifactDir, 'retained-memory.json'), retainedMemory); +}; + +const artifactName = ( + arm: ArmPlan, + heapMiB: number, + tenantCount: number, + repetition: number, + suffix = 'matrix' +): string => [suffix, arm.name, `h${heapMiB}`, `t${tenantCount}`, `r${repetition}`] + .map((part) => part.replace(/[^a-zA-Z0-9_.-]+/g, '-')) + .join('-'); + +const runOne = async ( + plan: DensityPlanV1, + fleet: FleetV1, + arm: ArmPlan, + heapMiB: number, + tenantCount: number, + repetition: number, + workload: WorkloadPlan, + context: RunContext, + suffix = 'matrix' +): Promise => { + const runId = `${new Date().toISOString().replace(/[:.]/g, '-')}-${process.pid}`; + const artifactDir = path.join( + plan.artifactDir, + `${artifactName(arm, heapMiB, tenantCount, repetition, suffix)}-${runId}` + ); + fs.mkdirSync(artifactDir, { recursive: true }); + let server: Awaited> | null = null; + let memory: ReturnType | null = null; + let postgresMemory: ReturnType | null = null; + let workloadResult: WorkloadResult | null = null; + let provenance: ArmProvenance | null = null; + let postgresRunAttestation: PostgresRunAttestationEvidence | null = null; + let realtimeDeliveryCoverage: RealtimeDeliveryCoverage | null = null; + const realtimeEvidence: Array<{ + phase: string; + timestamp: string; + snapshot: RealtimeDriverSnapshot; + }> = []; + const retainedMemory: RetainedMemoryCheckpointPair = { + baseline: null, + final: null, + errors: [] + }; + const capture = createWorkloadCapture(); + const tenants = resolveTenants(fleet.tenants.slice(0, tenantCount), arm); + const realtime = createRealtimeDriver(tenants, { + // Keep connection/prime transients bounded independently of schema-build + // concurrency; one-at-a-time setup is unnecessarily slow at 500+ surfaces. + concurrency: Math.min(8, workload.maxInFlight), + timeoutMs: workload.requestTimeoutMs + }); + const recordRealtime = (phase: string): void => { + realtimeEvidence.push({ + phase, + timestamp: new Date().toISOString(), + snapshot: realtime.snapshot() + }); + writeJson(path.join(artifactDir, 'realtime-driver.json'), realtimeEvidence); + }; + const startedAt = new Date().toISOString(); + const makeScoreInput = ( + endedAt: string, + executionErrors: string[] + ): ScoreInput => { + const memorySnapshots = memory?.snapshots ?? []; + const osSnapshots = memory?.osSnapshots ?? []; + return { + arm: arm.name, + evidenceMode: context.evidenceMode, + campaignId: context.campaignId, + scheduleSha256: context.scheduleSha256, + previousResultPayloadSha256: context.previousResultPayloadSha256, + qualificationCohortSha256: context.qualificationCohortSha256, + commit: arm.commit, + introspectionMode: arm.introspectionMode, + heapMiB, + repetition, + expectedMatrixRepetitions: context.expectedMatrixRepetitions, + runKind: context.runKind, + runOrderSeed: context.runOrderSeed, + runOrderIndex: context.runOrderIndex, + startedAt, + endedAt, + configuredDurationSec: workload.durationSec, + workloadDurationMs: workloadResult?.workloadDurationMs ?? 0, + artifactDir, + tenants, + warmedSurfaces: capture.warmedSurfaces, + warmupLatencies: capture.warmupLatencies, + resolvedWarmupTimeoutMs: workloadResult?.resolvedWarmupTimeoutMs + ?? resolveWarmupTimeoutMs( + workload, + tenants.reduce((sum, tenant) => sum + tenant.surfaces.length, 0) + ), + offeredLoad: workloadResult?.offeredLoad + ?? resolveOfferedLoad(workload, tenants.length), + canaryIntervalSec: workload.canaryIntervalSec, + periodicCanarySchedule: workload.periodicCanarySchedule ?? 'full-sweep', + canarySchedule: capture.canarySchedule, + minWorkloadRequestsPerSurface: workload.minWorkloadRequestsPerSurface, + samples: capture.samples, + canaries: capture.canaries, + memorySnapshots, + postWarmupSnapshots: memorySnapshots.slice( + Math.max(0, memory?.warmupIndex ?? -1) + ), + postWarmupNodeRssSnapshots: osSnapshots.slice( + Math.max(0, memory?.osWarmupIndex ?? -1) + ), + retainedMemory, + memorySampleErrors: memory?.errors ?? [], + postgresSnapshots: postgresMemory?.snapshots ?? [], + postgresSampleErrors: postgresMemory?.errors ?? [], + missedArrivals: capture.samples.filter( + (sample) => sample.errorCode === 'LOAD_GENERATOR_MISSED_ARRIVAL' + ).length, + requiredCapabilities: plan.requiredCapabilities, + requiredCanaries: plan.requiredCanaries, + gates: plan.gates, + serverExit: server?.exit ?? null, + provenance, + provenanceErrors: (server?.provenanceErrors ?? []).map( + executionErrorEvidence + ), + postgresRunAttestation, + realtimeDeliveryCoverage, + externalServer: server?.external ?? false, + executionErrors + }; + }; + try { + if (plan.gates.requireFreshPostgresRunAttestation) { + postgresRunAttestation = await collectPostgresRunAttestation(arm, { + arm: arm.name, + heapMiB, + tenantCount, + repetition, + runOrderIndex: context.runOrderIndex, + planSha256: context.planSha256, + fleetSha256: context.fleetSha256, + notBeforeEpochMs: context.notBeforeEpochMs, + artifactDir + }); + if (postgresRunAttestation) { + const reusedIdentity = context.claimPostgresRunIdentity( + postgresRunAttestation + ); + if (reusedIdentity) { + throw new Error( + `PostgreSQL run identity was reused: ${reusedIdentity}` + ); + } + } + } + const memoryUrl = resolveTemplate(arm.memoryUrl, { + heapMiB, + port: arm.port, + artifactDir, + mode: arm.introspectionMode + }); + assertLoopbackObservabilityUrl(memoryUrl, arm.port); + server = await startArmProcess( + arm, + heapMiB, + artifactDir, + tenantCount, + postgresRunAttestation ? { + postgresFixtureDir: path.join(artifactDir, 'postgres-fixture'), + postgresManifestFile: path.join( + artifactDir, + 'postgres-fixture', + 'provision.json' + ), + postgresSecretsFile: path.join( + artifactDir, + 'postgres-fixture', + 'runtime-secrets.json' + ), + postgresManifestSha256: postgresRunAttestation.manifestSha256, + postgresCloneId: postgresRunAttestation.cloneId + } : {} + ); + provenance = contextualProvenance( + server.provenance, + arm, + heapMiB, + server.expectedHeapLimitBytes, + context + ); + writeJson(path.join(artifactDir, 'provenance.json'), { + ...provenance, + expectedHeapLimitBytes: server.expectedHeapLimitBytes, + errors: server.provenanceErrors + }); + memory = startMemorySampler(memoryUrl, { + expectedPid: server.pid, + expectedHeapLimitBytes: server.expectedHeapLimitBytes, + currentRssSource: context.evidenceMode === 'qualification' ? 'proc' : 'auto', + headers: server.observabilityHeaders + }); + if (arm.postgresContainer) { + postgresMemory = startPostgresMemorySampler(arm.postgresContainer, { + requireCgroupV2: arm.requirePostgresCgroupV2, + ...(postgresRunAttestation ? { + expectedContainerId: postgresRunAttestation.containerId, + expectedContainerStartedAt: postgresRunAttestation.containerStartedAt, + expectedCgroupIdentitySha256: postgresRunAttestation.cgroupIdentitySha256 + } : {}) + }); + } + await Promise.all([memory.ready, postgresMemory?.ready]); + workloadResult = await runWorkload( + tenants, + workload, + async () => { + // runWorkload invokes this at the final pre-load boundary, after its + // schema warmups, capability coverage, and initial hostile canaries. + // The driver owns all graphql-ws client objects, so their heap/RSS is + // outside the measured server child. The server hook only verifies + // that every exact-route inbound connection and manager is resident. + await realtime.startAndVerify(); + recordRealtime('verified-before-baseline'); + await invokePostWarmupHook( + arm, + heapMiB, + tenantCount, + artifactDir, + server!.observabilityHeaders + ); + realtime.assertHealthy(); + if (arm.retainedHeapCheckpointUrl) { + retainedMemory.baseline = await invokeRetainedMemoryCheckpoint( + arm, + heapMiB, + tenantCount, + server!.observabilityHeaders, + retainedMemory.errors + ); + } + // Begin natural post-warm RSS/heap accounting after the benchmark-only + // baseline GC. The sampler itself and the PostgreSQL sampler remain + // live throughout both bookends. + await memory!.markWarmupComplete(); + realtime.beginTimedCoverage(workload.durationSec * 1000); + }, + capture + ); + realtimeDeliveryCoverage = await realtime.finishTimedCoverage(); + recordRealtime('timed-coverage-complete'); + await realtime.verifyDeliveryNow(); + realtime.assertHealthy(); + recordRealtime('healthy-after-workload'); + if (arm.retainedHeapCheckpointUrl) { + retainedMemory.final = await invokeRetainedMemoryCheckpoint( + arm, + heapMiB, + tenantCount, + server.observabilityHeaders, + retainedMemory.errors + ); + } + realtime.assertHealthy(); + recordRealtime('healthy-after-final-checkpoint'); + await memory.stop(); + if (postgresMemory) await postgresMemory.stop(); + realtime.assertHealthy(); + recordRealtime('healthy-before-disposal'); + await realtime.dispose(); + recordRealtime('disposed'); + persistPartialArtifacts( + artifactDir, + memory, + postgresMemory, + capture, + workloadResult, + retainedMemory + ); + const endedAt = new Date().toISOString(); + const scoreInput = makeScoreInput(endedAt, []); + const result = scoreRun(scoreInput); + writeResult(plan.artifactDir, result, scoreInput, context); + return result; + } catch (error) { + const executionErrors = [executionErrorEvidence(error)]; + recordRealtime('failed'); + if (memory) { + try { + await memory.stop(); + } catch (stopError) { + memory.errors.push(stopError instanceof Error ? stopError.message : String(stopError)); + } + } + if (postgresMemory) { + try { + await postgresMemory.stop(); + } catch (stopError) { + postgresMemory.errors.push( + stopError instanceof Error ? stopError.message : String(stopError) + ); + } + } + try { + await realtime.dispose(); + recordRealtime('disposed-after-failure'); + } catch (disposeError) { + executionErrors.push(executionErrorEvidence(disposeError)); + recordRealtime('dispose-failed'); + } + persistPartialArtifacts( + artifactDir, + memory, + postgresMemory, + capture, + workloadResult, + retainedMemory + ); + const scoreInput = makeScoreInput( + new Date().toISOString(), + executionErrors + ); + const result = scoreRun(scoreInput); + writeResult(plan.artifactDir, result, scoreInput, context); + return result; + } finally { + if (memory) await memory.stop(); + if (postgresMemory) await postgresMemory.stop(); + try { + await realtime.dispose(); + } catch { + // Disposal was already attempted and recorded in the main path. + } + await server?.stop(); + } +}; + +export const runDensityPlan = async ( + plan: DensityPlanV1, + fleet: FleetV1, + selection: RunSelection = {} +): Promise => { + validateCoverage(plan, fleet); + const arms = plan.arms.filter((arm) => !selection.arms || selection.arms.includes(arm.name)); + if (arms.length === 0) throw new Error('run selection contains no known arms'); + const heaps = selection.heaps ?? (selection.smoke ? plan.heapMiB.slice(0, 1) : plan.heapMiB); + const repetitions = selection.smoke ? 1 : selection.repetitions ?? plan.repetitions; + if (!Number.isInteger(repetitions) || repetitions <= 0) { + throw new Error('run repetitions must be a positive integer'); + } + const tenantCountsOverride = selection.tenantCounts + ?? (selection.smoke ? tenantCountsForHeap(plan, heaps[0]).slice(0, 1) : undefined); + const schedule = buildRunSchedule( + plan, + arms, + heaps, + repetitions, + tenantCountsOverride + ); + const configuredSchedule = buildRunSchedule( + plan, + plan.arms, + plan.heapMiB, + plan.repetitions + ); + const exactConfiguredMatrix = sameRunSchedule(schedule, configuredSchedule); + for (const job of schedule) { + if (job.tenantCount > fleet.tenants.length) { + throw new Error(`fleet has ${fleet.tenants.length} tenants, requested ${job.tenantCount}`); + } + } + const results: DensityRunResult[] = []; + const runOrderSeed = plan.runOrderSeed ?? DEFAULT_RUN_ORDER_SEED; + const planSha256 = plan.sourceSha256 ?? sha256(JSON.stringify(plan)); + const fleetSha256 = fleet.sourceSha256 ?? sha256(JSON.stringify(fleet)); + const qualificationCohortSha256 = sha256(`${planSha256}\0${fleetSha256}`); + const hostileEvidenceReady = hasExactHostileValidationEvidence(plan); + const evidenceMode = plan.qualification + && exactConfiguredMatrix + && !selection.smoke + && process.platform === 'linux' + && hostileEvidenceReady + ? 'qualification' + : 'diagnostic'; + const campaignId = randomBytes(32).toString('hex'); + const campaignStartedAt = new Date().toISOString(); + const scheduleManifest: CampaignScheduleManifestV1 = { + version: 1, + campaignId, + campaignStartedAt, + runOrderSeed, + planSha256, + fleetSha256, + node: process.version, + v8: process.versions.v8, + platform: process.platform, + architecture: process.arch, + jobs: scheduleJobsForPlan(plan, schedule, !selection.smoke) + }; + const scheduleSha256 = scheduleManifestSha256(scheduleManifest); + writeExclusiveJson(path.join(plan.artifactDir, `campaign-${campaignId}.json`), { + ...scheduleManifest, + scheduleSha256, + evidenceMode, + qualificationBlockers: [ + ...(!plan.qualification ? ['qualification-plan-missing'] : []), + ...(!exactConfiguredMatrix ? ['noncanonical-or-partial-schedule'] : []), + ...(selection.smoke ? ['smoke-run'] : []), + ...(process.platform !== 'linux' ? ['linux-required'] : []), + ...(!hostileEvidenceReady ? ['exact-hostile-validation-evidence-required'] : []) + ] + }); + if (evidenceMode === 'diagnostic' && plan.qualification) { + process.stdout.write( + `[cperf] campaign=${campaignId} diagnostic qualification prerequisites were not met\n` + ); + } + const claimedPostgresRunIdentities = new Set(); + const claimPostgresRunIdentity = ( + evidence: PostgresRunAttestationEvidence + ): string | null => { + const claims = postgresRunIdentityClaims(evidence); + const reused = claims.find((claim) => claimedPostgresRunIdentities.has(claim)); + if (reused) return reused; + for (const claim of claims) claimedPostgresRunIdentities.add(claim); + return null; + }; + + let previousResultPayloadSha256: string | null = null; + for (const job of schedule) { + const workload: WorkloadPlan = selection.smoke + ? { + ...plan.workload, + durationSec: 5, + ...(plan.workload.rps != null + ? { rps: Math.min(plan.workload.rps, 5), rpsPerTenant: undefined } + : { + rps: undefined, + rpsPerTenant: Math.min(plan.workload.rpsPerTenant!, 5 / job.tenantCount) + }) + } + : plan.workload; + const result = await runOne( + plan, + fleet, + job.arm, + job.heapMiB, + job.tenantCount, + job.repetition, + workload, + { + expectedMatrixRepetitions: plan.repetitions, + runKind: 'matrix', + evidenceMode, + campaignId, + scheduleSha256, + previousResultPayloadSha256, + qualificationCohortSha256, + runOrderSeed, + runOrderIndex: job.orderIndex, + planSha256, + fleetSha256, + notBeforeEpochMs: Date.now(), + claimPostgresRunIdentity + } + ); + results.push(result); + previousResultPayloadSha256 = result.evidenceBinding?.resultPayloadSha256 ?? null; + if (!previousResultPayloadSha256) { + throw new Error('persisted result is missing its evidence payload binding'); + } + process.stdout.write( + `[cperf] order=${job.orderIndex} ${job.arm.name} heap=${job.heapMiB} ` + + `customers=${job.tenantCount} run=${job.repetition} accepted=${result.accepted} ` + + `customersPerAlignedServiceGiB=${result.customersPerAlignedServiceGiB?.toFixed(2) ?? 'n/a'}\n` + ); + } + + if (!selection.smoke && plan.soak?.enabled) { + const configuredSoakArm = soakArmName(plan); + const candidate = arms.find((arm) => arm.name === configuredSoakArm); + if (!candidate) { + throw new Error(`soak enabled but arm '${configuredSoakArm}' is not selected`); + } + results.push(await runOne( + plan, + fleet, + candidate, + plan.soak.heapMiB, + plan.soak.tenantCount, + plan.repetitions + 1, + { ...plan.workload, durationSec: plan.soak.durationSec }, + { + expectedMatrixRepetitions: plan.repetitions, + runKind: 'soak', + evidenceMode, + campaignId, + scheduleSha256, + previousResultPayloadSha256, + qualificationCohortSha256, + runOrderSeed, + runOrderIndex: schedule.length + 1, + planSha256, + fleetSha256, + notBeforeEpochMs: Date.now(), + claimPostgresRunIdentity + }, + 'soak' + )); + previousResultPayloadSha256 = results[results.length - 1] + .evidenceBinding?.resultPayloadSha256 ?? null; + if (!previousResultPayloadSha256) { + throw new Error('persisted soak result is missing its evidence payload binding'); + } + } + writeJson( + path.join(plan.artifactDir, `capacity-boundaries-${campaignId}.json`), + { + version: 1, + runOrderSeed, + planSha256, + fleetSha256, + campaignId, + scheduleSha256, + boundaries: summarizeCapacityBoundaries(results) + } + ); + return results; +}; diff --git a/packages/perf-harness/src/schedule.ts b/packages/perf-harness/src/schedule.ts new file mode 100644 index 0000000000..c1bcfb8f08 --- /dev/null +++ b/packages/perf-harness/src/schedule.ts @@ -0,0 +1,133 @@ +import { createHash } from 'node:crypto'; + +import { + DEFAULT_RUN_ORDER_SEED, + soakArmName, + tenantCountsForHeap +} from './config'; +import type { ArmPlan, DensityPlanV1 } from './types'; + +export interface DensityRunJob { + arm: ArmPlan; + heapMiB: number; + tenantCount: number; + repetition: number; + orderIndex: number; +} + +export interface CampaignScheduleJob { + runKind: 'matrix' | 'soak'; + arm: string; + heapMiB: number; + tenantCount: number; + repetition: number; + orderIndex: number; +} + +export interface CampaignScheduleManifestV1 { + version: 1; + campaignId: string; + campaignStartedAt: string; + runOrderSeed: string; + planSha256: string; + fleetSha256: string; + node: string; + v8: string; + platform: NodeJS.Platform; + architecture: string; + jobs: CampaignScheduleJob[]; +} + +const sha256 = (value: string): string => createHash('sha256').update(value).digest('hex'); + +const deterministicArmOrder = ( + arms: ArmPlan[], + seed: string, + repetition: number, + heapMiB: number, + tenantCount: number +): ArmPlan[] => [...arms].sort((left, right) => { + const prefix = `${seed}\0${repetition}\0${heapMiB}\0${tenantCount}\0`; + return sha256(`${prefix}${left.name}`).localeCompare(sha256(`${prefix}${right.name}`)); +}); + +export const buildRunSchedule = ( + plan: DensityPlanV1, + arms: ArmPlan[], + heaps: number[], + repetitions: number, + tenantCountsOverride?: number[] +): DensityRunJob[] => { + const jobs: DensityRunJob[] = []; + const seed = plan.runOrderSeed ?? DEFAULT_RUN_ORDER_SEED; + for (let repetition = 1; repetition <= repetitions; repetition++) { + for (const heapMiB of heaps) { + const counts = tenantCountsOverride ?? tenantCountsForHeap(plan, heapMiB); + for (const tenantCount of counts) { + for (const arm of deterministicArmOrder( + arms, + seed, + repetition, + heapMiB, + tenantCount + )) { + jobs.push({ + arm, + heapMiB, + tenantCount, + repetition, + orderIndex: jobs.length + 1 + }); + } + } + } + } + return jobs; +}; + +export const scheduleJobsForPlan = ( + plan: DensityPlanV1, + matrix: DensityRunJob[], + includeSoak: boolean +): CampaignScheduleJob[] => { + const jobs: CampaignScheduleJob[] = matrix.map((job) => ({ + runKind: 'matrix', + arm: job.arm.name, + heapMiB: job.heapMiB, + tenantCount: job.tenantCount, + repetition: job.repetition, + orderIndex: job.orderIndex + })); + if (includeSoak && plan.soak?.enabled) { + jobs.push({ + runKind: 'soak', + arm: soakArmName(plan), + heapMiB: plan.soak.heapMiB, + tenantCount: plan.soak.tenantCount, + repetition: plan.repetitions + 1, + orderIndex: matrix.length + 1 + }); + } + return jobs; +}; + +export const scheduleManifestSha256 = ( + manifest: CampaignScheduleManifestV1 +): string => sha256(JSON.stringify(manifest)); + +export const sameRunSchedule = ( + left: DensityRunJob[], + right: DensityRunJob[] +): boolean => JSON.stringify(left.map((job) => [ + job.arm.name, + job.heapMiB, + job.tenantCount, + job.repetition, + job.orderIndex +])) === JSON.stringify(right.map((job) => [ + job.arm.name, + job.heapMiB, + job.tenantCount, + job.repetition, + job.orderIndex +])); diff --git a/packages/perf-harness/src/score.ts b/packages/perf-harness/src/score.ts new file mode 100644 index 0000000000..bad31b7c8c --- /dev/null +++ b/packages/perf-harness/src/score.ts @@ -0,0 +1,2634 @@ +import { createHash } from 'node:crypto'; + +import { periodicCanaryRoundCount, rotatingCanaryIndex } from './http'; +import type { + AcceptanceGates, + ArmProvenance, + CanaryResult, + CanaryScheduleSummary, + CustomerFleetShape, + DensityCapacityBoundary, + DensityRunResult, + MemorySnapshot, + NodeRssSnapshot, + PeriodicCanarySchedule, + PostgresMemorySnapshot, + PostgresRunAttestationEvidence, + RequestSample, + RealtimeDeliveryCoverage, + ResolvedOfferedLoad, + RetainedMemoryCheckpoint, + RetainedMemoryCheckpointPair, + RetainedMemoryGuard, + SurfaceResult, + TenantResult, + TenantTarget +} from './types'; + +const GIB = 1024 ** 3; +const MIB = 1024 ** 2; +const DEFAULT_MEMORY_ALIGNMENT_SKEW_MS = 500; +export const DEFAULT_MAX_ALIGNED_MEMORY_SAMPLE_GAP_MS = 1_000; +export const DEFAULT_MIN_ALIGNED_MEMORY_COVERAGE_RATIO = 0.99; + +export interface AlignedServiceMemoryPeak { + bytes: number; + nodeRssBytes: number; + postgresBytes: number; + timestamp: string; + samples: number; + maxSkewMs: number; +} + +interface AlignedServiceMemorySample { + bytes: number; + nodeRssBytes: number; + postgresBytes: number; + timestamp: string; + timeMs: number; + skewMs: number; +} + +export interface AlignedServiceMemoryCoverage { + expectedDurationMs: number; + coveredDurationMs: number; + coverageRatio: number; + maxGapMs: number; + maxPairedSampleGapMs: number; + maxNodeSampleGapMs: number; + maxPostgresSampleGapMs: number; + firstSampleTimestamp: string | null; + lastSampleTimestamp: string | null; +} + +const alignedServiceMemorySamples = ( + memorySnapshots: Array>, + postgresSnapshots: PostgresMemorySnapshot[], + allowedSkewMs: number +): AlignedServiceMemorySample[] => { + const postgres = postgresSnapshots + .map((snapshot) => ({ snapshot, timeMs: Date.parse(snapshot.timestamp) })) + .filter(({ snapshot, timeMs }) => + Number.isFinite(timeMs) + && Number.isFinite(snapshot.usedBytes) + && snapshot.usedBytes >= 0 + ) + .sort((left, right) => left.timeMs - right.timeMs); + if (postgres.length === 0) return []; + + let postgresIndex = 0; + const samples: AlignedServiceMemorySample[] = []; + const node = memorySnapshots + .map((snapshot) => ({ snapshot, timeMs: Date.parse(snapshot.timestamp) })) + .filter(({ snapshot, timeMs }) => + Number.isFinite(timeMs) + && snapshot.rssBytes != null + && Number.isFinite(snapshot.rssBytes) + && snapshot.rssBytes > 0 + ) + .sort((left, right) => left.timeMs - right.timeMs); + + for (const { snapshot, timeMs } of node) { + while ( + postgresIndex + 1 < postgres.length + && Math.abs(postgres[postgresIndex + 1].timeMs - timeMs) + <= Math.abs(postgres[postgresIndex].timeMs - timeMs) + ) postgresIndex += 1; + const candidate = postgres[postgresIndex]; + const skewMs = Math.abs(candidate.timeMs - timeMs); + if (skewMs > allowedSkewMs) continue; + samples.push({ + bytes: snapshot.rssBytes! + candidate.snapshot.usedBytes, + nodeRssBytes: snapshot.rssBytes!, + postgresBytes: candidate.snapshot.usedBytes, + timestamp: snapshot.timestamp, + timeMs, + skewMs + }); + } + return samples; +}; + +export const alignedServiceMemoryCoverage = ( + memorySnapshots: Array>, + postgresSnapshots: PostgresMemorySnapshot[], + expectedStartMs: number, + expectedDurationMs: number, + allowedSkewMs = DEFAULT_MEMORY_ALIGNMENT_SKEW_MS +): AlignedServiceMemoryCoverage | null => { + if (!Number.isFinite(allowedSkewMs) || allowedSkewMs < 0) { + throw new Error(`memory alignment skew must be non-negative, received ${allowedSkewMs}`); + } + if (!Number.isFinite(expectedStartMs) || !Number.isFinite(expectedDurationMs) + || expectedDurationMs <= 0) return null; + const expectedEndMs = expectedStartMs + expectedDurationMs; + const samples = alignedServiceMemorySamples( + memorySnapshots, + postgresSnapshots, + allowedSkewMs + ).filter((sample) => + sample.timeMs >= expectedStartMs - allowedSkewMs + && sample.timeMs <= expectedEndMs + allowedSkewMs + ); + if (samples.length === 0) return null; + const first = samples[0]; + const last = samples.at(-1)!; + const coveredStartMs = Math.max(expectedStartMs, first.timeMs); + const coveredEndMs = Math.min(expectedEndMs, last.timeMs); + const coveredDurationMs = Math.max(0, coveredEndMs - coveredStartMs); + const cadenceGap = (timestamps: number[]): number => { + const times = [...new Set(timestamps + .filter((timeMs) => + Number.isFinite(timeMs) + && timeMs >= expectedStartMs - allowedSkewMs + && timeMs <= expectedEndMs + allowedSkewMs + ) + .map((timeMs) => Math.max(expectedStartMs, Math.min(expectedEndMs, timeMs))))] + .sort((left, right) => left - right); + if (times.length === 0) return expectedDurationMs; + let gap = Math.max(times[0] - expectedStartMs, expectedEndMs - times.at(-1)!); + for (let index = 1; index < times.length; index += 1) { + gap = Math.max(gap, times[index] - times[index - 1]); + } + return gap; + }; + const maxPairedSampleGapMs = cadenceGap(samples.map((sample) => sample.timeMs)); + const maxNodeSampleGapMs = cadenceGap(memorySnapshots + .filter((snapshot) => Number.isFinite(snapshot.rssBytes) && snapshot.rssBytes! > 0) + .map((snapshot) => Date.parse(snapshot.timestamp))); + const maxPostgresSampleGapMs = cadenceGap(postgresSnapshots + .filter((snapshot) => Number.isFinite(snapshot.usedBytes) && snapshot.usedBytes >= 0) + .map((snapshot) => Date.parse(snapshot.timestamp))); + return { + expectedDurationMs, + coveredDurationMs, + coverageRatio: coveredDurationMs / expectedDurationMs, + maxGapMs: Math.max( + maxPairedSampleGapMs, + maxNodeSampleGapMs, + maxPostgresSampleGapMs + ), + maxPairedSampleGapMs, + maxNodeSampleGapMs, + maxPostgresSampleGapMs, + firstSampleTimestamp: first.timestamp, + lastSampleTimestamp: last.timestamp + }; +}; + +/** + * Pair each current Node RSS sample with the nearest PostgreSQL cgroup sample. + * Cumulative process HWM is deliberately excluded because adding a historical + * Node peak to current PostgreSQL usage would not be a simultaneous service + * footprint. + */ +export const alignedServiceMemoryPeak = ( + memorySnapshots: Array>, + postgresSnapshots: PostgresMemorySnapshot[], + allowedSkewMs = DEFAULT_MEMORY_ALIGNMENT_SKEW_MS +): AlignedServiceMemoryPeak | null => { + if (!Number.isFinite(allowedSkewMs) || allowedSkewMs < 0) { + throw new Error(`memory alignment skew must be non-negative, received ${allowedSkewMs}`); + } + const aligned = alignedServiceMemorySamples( + memorySnapshots, + postgresSnapshots, + allowedSkewMs + ); + if (aligned.length === 0) return null; + let peak: Omit | null = null; + for (const sample of aligned) { + if (!peak || sample.bytes > peak.bytes) { + peak = { + bytes: sample.bytes, + nodeRssBytes: sample.nodeRssBytes, + postgresBytes: sample.postgresBytes, + timestamp: sample.timestamp + }; + } + } + return peak ? { + ...peak, + samples: aligned.length, + maxSkewMs: Math.max(...aligned.map((sample) => sample.skewMs)) + } : null; +}; + +export const percentile = (values: number[], fraction: number): number => { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)]; +}; + +export const summarizeCustomerFleet = ( + customers: TenantTarget[] +): CustomerFleetShape => { + const physicalDatabases = new Set(); + const routingLabels = new Set(); + const buildContracts = new Set(); + const runtimePoolIdentities = new Set(); + let logicalDatabases = 0; + let apis = 0; + let realtimeApis = 0; + let surfaces = 0; + let physicalSchemaBindings = 0; + for (const customer of customers) { + surfaces += customer.surfaces.length; + for (const surface of customer.surfaces) buildContracts.add(surface.buildContract); + for (const database of customer.databases ?? []) { + logicalDatabases += 1; + physicalDatabases.add(database.physicalDatabase); + for (const api of database.apis) { + apis += 1; + if (api.realtime) realtimeApis += 1; + physicalSchemaBindings += api.physicalSchemas.length; + for (const label of api.routingLabels) routingLabels.add(label); + runtimePoolIdentities.add(api.runtimePoolIdentity); + } + } + } + return { + topologyComplete: customers.every((customer) => customer.databases != null), + customers: customers.length, + logicalDatabases, + physicalDatabases: physicalDatabases.size, + apis, + realtimeApis, + surfaces, + physicalSchemaBindings, + routingLabels: routingLabels.size, + uniqueBuildContracts: buildContracts.size, + uniqueRuntimePoolIdentities: runtimePoolIdentities.size + }; +}; + +const counterDelta = ( + snapshots: MemorySnapshot[], + field: 'evictions' | 'buildRefusals' | 'buildsStarted' +): number | null => { + if (snapshots.length < 2) return null; + const available = field === 'buildsStarted' + ? snapshots.every((snapshot) => snapshot.buildCountersAvailable) + : snapshots.every((snapshot) => snapshot.cacheCountersAvailable); + if (!available) return null; + const values = snapshots.map((snapshot) => snapshot[field]); + if (values.some((value) => !Number.isSafeInteger(value) || value! < 0)) return null; + if (values.some((value, index) => index > 0 && value! < values[index - 1]!)) { + return null; + } + return values.at(-1)! - values[0]!; +}; + +type PgPoolCounterField = + | 'pgPoolCapacityEvictions' + | 'pgPoolCapacityRefusals' + | 'pgPoolDisposalFailures'; + +const pgPoolCounterDelta = ( + snapshots: MemorySnapshot[], + field: PgPoolCounterField +): number | null => { + if (snapshots.length < 2) return null; + const values = snapshots.map((snapshot) => snapshot[field]); + if (values.some((value) => !Number.isSafeInteger(value) || value! < 0)) return null; + if (values.some((value, index) => index > 0 && value! < values[index - 1]!)) { + return null; + } + return values.at(-1)! - values[0]!; +}; + +export const heapGrowthMiBPerHour = (snapshots: MemorySnapshot[]): number | null => { + if (snapshots.length < 2 || snapshots.some((snapshot) => snapshot.heapUsedBytes == null)) { + return null; + } + const points = snapshots.map((snapshot) => ({ + x: new Date(snapshot.timestamp).getTime() / 3_600_000, + y: snapshot.heapUsedBytes! / MIB + })); + const meanX = points.reduce((sum, point) => sum + point.x, 0) / points.length; + const meanY = points.reduce((sum, point) => sum + point.y, 0) / points.length; + const numerator = points.reduce((sum, point) => sum + (point.x - meanX) * (point.y - meanY), 0); + const denominator = points.reduce((sum, point) => sum + (point.x - meanX) ** 2, 0); + return denominator === 0 ? null : numerator / denominator; +}; + +export interface RetainedMemoryGrowthSummary { + heapMiBPerHour: number | null; + externalMiBPerHour: number | null; + durationSec: number | null; + heapBaselineBytes: number | null; + heapFinalBytes: number | null; + externalBaselineBytes: number | null; + externalFinalBytes: number | null; + errors: string[]; +} + +const canonicalJson = (value: unknown): string => { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value && typeof value === 'object') { + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => + `${JSON.stringify(key)}:${canonicalJson(record[key])}` + ).join(',')}}`; + } + return JSON.stringify(value); +}; + +const stateSha256 = (value: unknown): string => + `sha256:${createHash('sha256').update(canonicalJson(value)).digest('hex')}`; + +const asRecord = (value: unknown): Record | null => value + && typeof value === 'object' + && !Array.isArray(value) + ? value as Record + : null; + +const lifecycleCounter = ( + counters: Record, + name: string +): number | null => Number.isSafeInteger(counters[name]) + && (counters[name] as number) >= 0 + ? counters[name] as number + : null; + +/** + * The physical fixture hashes its complete residency and monotonic counter + * state. A single GC checkpoint must remain byte-identical, but normal HTTP + * traffic advances its started/completed counters between the two bookends. + * Permit only a balanced monotonic HTTP delta; every other field remains an + * exact topology/counter comparison. WebSocket lifecycle changes are rejected + * because qualifying transports must remain continuously resident. + */ +const validateCrossWorkloadGuardState = ( + baseline: RetainedMemoryGuard, + final: RetainedMemoryGuard, + errors: string[] +): void => { + const baselineState = asRecord(baseline.state); + const finalState = asRecord(final.state); + if (!baselineState || !finalState) { + errors.push('retained-memory workload guard state is invalid'); + return; + } + const baselineCounters = asRecord(baselineState.cacheCounters); + const finalCounters = asRecord(finalState.cacheCounters); + // Legacy/non-physical checkpoint producers do not expose handler lifecycle + // counters, so retain their former exact-state requirement. + if (!baselineCounters && !finalCounters) { + if (baseline.stateSha256 !== final.stateSha256) { + errors.push('retained-memory residency or counters changed across the workload'); + } + return; + } + if (!baselineCounters || !finalCounters) { + errors.push('retained-memory handler counters changed shape across the workload'); + return; + } + const names = [ + 'httpRequestsStarted', + 'httpRequestsCompleted', + 'websocketUpgradesStarted', + 'websocketUpgradesCompleted' + ] as const; + const before = Object.fromEntries(names.map((name) => [ + name, + lifecycleCounter(baselineCounters, name) + ])) as Record<(typeof names)[number], number | null>; + const after = Object.fromEntries(names.map((name) => [ + name, + lifecycleCounter(finalCounters, name) + ])) as Record<(typeof names)[number], number | null>; + if (names.some((name) => before[name] == null || after[name] == null)) { + errors.push('retained-memory handler counters are invalid'); + return; + } + const delta = Object.fromEntries(names.map((name) => [ + name, + after[name]! - before[name]! + ])) as Record<(typeof names)[number], number>; + if (names.some((name) => delta[name] < 0)) { + errors.push('retained-memory handler counters regressed across the workload'); + } + if (delta.httpRequestsStarted !== delta.httpRequestsCompleted) { + errors.push( + `retained-memory HTTP handler delta is unbalanced: started=${delta.httpRequestsStarted}, completed=${delta.httpRequestsCompleted}` + ); + } + if (delta.websocketUpgradesStarted !== 0 || delta.websocketUpgradesCompleted !== 0) { + errors.push( + `retained-memory WebSocket lifecycle changed across the workload: started=${delta.websocketUpgradesStarted}, completed=${delta.websocketUpgradesCompleted}` + ); + } + const withoutHttpLifecycle = ( + state: Record + ): Record => { + const counters = { ...asRecord(state.cacheCounters) }; + delete counters.httpRequestsStarted; + delete counters.httpRequestsCompleted; + return { ...state, cacheCounters: counters }; + }; + if ( + stateSha256(withoutHttpLifecycle(baselineState)) + !== stateSha256(withoutHttpLifecycle(finalState)) + ) { + errors.push('retained-memory residency or non-HTTP counters changed across the workload'); + } +}; + +const stableTail = (checkpoint: RetainedMemoryCheckpoint) => + checkpoint.samples.slice(-checkpoint.stableSampleCount); + +const medianNumber = (values: number[]): number => percentile(values, 0.5); + +const validateCheckpoint = ( + label: string, + checkpoint: RetainedMemoryCheckpoint | null, + expectedPid: number | null, + errors: string[] +): checkpoint is RetainedMemoryCheckpoint => { + if (!checkpoint) { + errors.push(`${label} retained-memory checkpoint is unavailable`); + return false; + } + let structurallyUsable = true; + if (!checkpoint.stable) errors.push(`${label} retained-memory checkpoint is unstable`); + errors.push(...checkpoint.errors.map((error) => `${label}: ${error}`)); + if (checkpoint.samples.length < 5 || checkpoint.samples.length > 8) { + errors.push(`${label} retained-memory checkpoint has invalid GC sample count`); + structurallyUsable = false; + } + if (checkpoint.stableSampleCount !== 3) { + errors.push(`${label} retained-memory checkpoint must use three stable samples`); + structurallyUsable = false; + } + if ( + checkpoint.pid !== checkpoint.guardBefore.pid + || checkpoint.pid !== checkpoint.guardAfter.pid + || (expectedPid != null && checkpoint.pid !== expectedPid) + ) { + errors.push(`${label} retained-memory checkpoint PID mismatch`); + } + for (const [guardLabel, guard] of [ + ['before', checkpoint.guardBefore], + ['after', checkpoint.guardAfter] + ] as const) { + if (guard.graphileInFlight !== 0) { + errors.push(`${label} retained-memory ${guardLabel} guard has in-flight Graphile work`); + } + if (stateSha256(guard.state) !== guard.stateSha256) { + errors.push(`${label} retained-memory ${guardLabel} state hash mismatch`); + } + const state = guard.state as Record; + const stateContracts = Array.isArray(state.residentBuildContracts) + ? state.residentBuildContracts + : null; + if ( + state.pid !== guard.pid + || state.graphileInFlight !== guard.graphileInFlight + || !stateContracts + || stateContracts.length !== guard.residentBuildContracts.length + || guard.residentBuildContracts.some( + (contract, index) => stateContracts[index] !== contract + ) + ) { + errors.push(`${label} retained-memory ${guardLabel} guard summary mismatch`); + } + } + if (checkpoint.guardBefore.stateSha256 !== checkpoint.guardAfter.stateSha256) { + errors.push(`${label} retained-memory residency or counters changed during GC`); + } + const tail = stableTail(checkpoint); + for (const field of ['heapUsedBytes', 'externalBytes'] as const) { + const values = tail.map((sample) => sample[field]); + const spread = Math.max(...values) - Math.min(...values); + const threshold = Math.max(MIB, Math.ceil(Math.max(...values) * 0.0025)); + if (spread > threshold) { + errors.push(`${label} retained ${field} samples did not converge`); + } + } + const monotonic = checkpoint.samples.map((sample) => { + try { + return BigInt(sample.monotonicNs); + } catch { + return null; + } + }); + if ( + monotonic.some((value) => value == null) + || monotonic.some((value, index) => + index > 0 && value! <= monotonic[index - 1]! + ) + ) { + errors.push(`${label} retained-memory monotonic timestamps are invalid`); + structurallyUsable = false; + } + return structurallyUsable; +}; + +export const retainedMemoryGrowth = ( + checkpoints: RetainedMemoryCheckpointPair, + expectedPid: number | null = null, + expectedResidentBuildContracts: ReadonlySet | null = null, + requireStableResidentBuildFingerprints = false +): RetainedMemoryGrowthSummary => { + const errors = [...checkpoints.errors]; + const baselineValid = validateCheckpoint( + 'baseline', checkpoints.baseline, expectedPid, errors + ); + const finalValid = validateCheckpoint( + 'final', checkpoints.final, expectedPid, errors + ); + if (!baselineValid || !finalValid) { + return { + heapMiBPerHour: null, + externalMiBPerHour: null, + durationSec: null, + heapBaselineBytes: null, + heapFinalBytes: null, + externalBaselineBytes: null, + externalFinalBytes: null, + errors + }; + } + const baseline = checkpoints.baseline!; + const final = checkpoints.final!; + if (baseline.fixture !== final.fixture) { + errors.push('retained-memory checkpoint fixture changed'); + } + validateCrossWorkloadGuardState( + baseline.guardAfter, + final.guardBefore, + errors + ); + if (expectedResidentBuildContracts) { + const expected = [...expectedResidentBuildContracts].sort(); + for (const [label, checkpoint] of [ + ['baseline', baseline], + ['final', final] + ] as const) { + const stateFingerprints = checkpoint.guardAfter.state + .residentBuildContractFingerprints; + const stableFingerprints = Array.isArray(stateFingerprints) + && stateFingerprints.every((value) => typeof value === 'string') + ? stateFingerprints as string[] + : null; + const resident = [ + ...(stableFingerprints ?? checkpoint.guardAfter.residentBuildContracts) + ].sort(); + if ( + (requireStableResidentBuildFingerprints && stableFingerprints == null) + || new Set(resident).size !== resident.length + || expected.length !== resident.length + || expected.some((contract, index) => contract !== resident[index]) + ) { + errors.push(`${label} retained-memory residency set mismatch`); + } + } + } + const baselineTail = stableTail(baseline); + const finalTail = stableTail(final); + const baselineNs = BigInt(baseline.samples.at(-1)!.monotonicNs); + const finalNs = BigInt(final.samples.at(-1)!.monotonicNs); + const durationSec = Number(finalNs - baselineNs) / 1e9; + if (!Number.isFinite(durationSec) || durationSec <= 0) { + errors.push('retained-memory checkpoint duration is invalid'); + } + const heapBaselineValues = baselineTail.map((sample) => sample.heapUsedBytes); + const heapFinalValues = finalTail.map((sample) => sample.heapUsedBytes); + const externalBaselineValues = baselineTail.map((sample) => sample.externalBytes); + const externalFinalValues = finalTail.map((sample) => sample.externalBytes); + const heapBaselineBytes = medianNumber(heapBaselineValues); + const heapFinalBytes = medianNumber(heapFinalValues); + const externalBaselineBytes = medianNumber(externalBaselineValues); + const externalFinalBytes = medianNumber(externalFinalValues); + const durationHours = durationSec / 3_600; + return { + heapMiBPerHour: durationHours > 0 + ? (Math.max(...heapFinalValues) - Math.min(...heapBaselineValues)) / MIB + / durationHours + : null, + externalMiBPerHour: durationHours > 0 + ? (Math.max(...externalFinalValues) - Math.min(...externalBaselineValues)) / MIB + / durationHours + : null, + durationSec: durationSec > 0 ? durationSec : null, + heapBaselineBytes, + heapFinalBytes, + externalBaselineBytes, + externalFinalBytes, + errors + }; +}; + +const tenantResult = ( + tenant: TenantTarget, + samples: RequestSample[], + canaries: CanaryResult[], + warmed: Set, + requiredCapabilities: string[], + minWorkloadRequestsPerSurface: number, + gates: AcceptanceGates +): TenantResult => { + const localSamples = samples.filter((sample) => sample.tenantId === tenant.id); + const workloadSamples = localSamples.filter((sample) => sample.phase === 'workload'); + const successfulCoverage = localSamples.filter((sample) => sample.ok); + const localCanaries = canaries.filter((canary) => canary.tenantId === tenant.id); + const errors = workloadSamples.filter((sample) => !sample.ok).length; + const errorRate = workloadSamples.length > 0 ? errors / workloadSamples.length : 1; + const p99Ms = percentile(workloadSamples.map((sample) => sample.latencyMs), 0.99); + const canaryInconclusive = localCanaries.filter((canary) => !canary.conclusive).length; + const bleedViolations = localCanaries.filter((canary) => canary.violation).length; + const successfulOperationKeys = new Set(successfulCoverage.map((sample) => + `${sample.surface}/${sample.operation}` + )); + const successfulCapabilities = new Set(successfulCoverage.map((sample) => sample.capability)); + const successfulCapabilityKeys = new Set(successfulCoverage.map((sample) => + `${sample.surface}/${sample.capability}` + )); + const operationOracleSamples = localSamples.filter((sample) => + sample.oracleConfigured === true + ); + const operationOracleInconclusive = operationOracleSamples.filter((sample) => + sample.oracleUnavailable !== true + && sample.oracleConclusive !== true + ).length; + const operationOracleViolations = operationOracleSamples.filter((sample) => + sample.oracleViolation === true + ).length; + const coverageOracleSamplesByKey = new Map(); + for (const sample of localSamples.filter((candidate) => candidate.phase === 'coverage')) { + const key = `${sample.surface}/${sample.operation}`; + const evidence = coverageOracleSamplesByKey.get(key) ?? []; + evidence.push(sample); + coverageOracleSamplesByKey.set(key, evidence); + } + const conclusiveCoverageOracleKeys = new Set([...coverageOracleSamplesByKey] + .filter(([, evidence]) => evidence.length === 1 && ( + evidence[0].ok + && evidence[0].oracleConfigured === true + && evidence[0].oracleConclusive === true + && evidence[0].oracleViolation !== true + )) + .map(([key]) => key)); + const surfaceResults: SurfaceResult[] = tenant.surfaces.map((surface) => { + const surfaceSamples = localSamples.filter((sample) => sample.surface === surface.name); + const surfaceWorkload = surfaceSamples.filter((sample) => sample.phase === 'workload'); + const surfaceSuccessfulWorkload = surfaceWorkload.filter((sample) => + sample.ok && sample.errorCode !== 'LOAD_GENERATOR_MISSED_ARRIVAL' + ); + const surfaceCoverage = surfaceSamples.filter((sample) => sample.ok); + const operationNames = surface.operations.map((operation) => operation.name); + const exercisedOperations = new Set(surfaceCoverage.map((sample) => sample.operation)); + const missingOperations = operationNames.filter((operation) => + !exercisedOperations.has(operation) + ); + const configuredCapabilities = [...new Set(surface.operations.map( + (operation) => operation.capability + ))]; + const exercisedCapabilities = new Set(surfaceCoverage.map((sample) => sample.capability)); + const missingCapabilities = configuredCapabilities.filter((capability) => + !exercisedCapabilities.has(capability) + ); + const missingOperationOracles = gates.requireConclusiveOperationOracles + ? operationNames.filter((operation) => + !conclusiveCoverageOracleKeys.has(`${surface.name}/${operation}`) + ) + : []; + const surfaceCanaries = localCanaries.filter((canary) => canary.surface === surface.name); + const surfaceOracleSamples = surfaceSamples.filter((sample) => + sample.oracleConfigured === true + ); + const surfaceOracleInconclusive = surfaceOracleSamples.filter((sample) => + sample.oracleUnavailable !== true && sample.oracleConclusive !== true + ).length; + const surfaceOracleViolations = surfaceOracleSamples.filter((sample) => + sample.oracleViolation === true + ).length; + const surfaceErrors = surfaceWorkload.filter((sample) => !sample.ok).length; + const surfaceErrorRate = surfaceWorkload.length > 0 + ? surfaceErrors / surfaceWorkload.length + : 1; + const surfaceP99Ms = percentile(surfaceWorkload.map((sample) => sample.latencyMs), 0.99); + const surfaceCanaryInconclusive = surfaceCanaries.filter( + (canary) => !canary.conclusive + ).length; + const surfaceBleedViolations = surfaceCanaries.filter( + (canary) => canary.violation + ).length; + const surfaceQualified = warmed.has(surface.name) + && surfaceSuccessfulWorkload.length >= minWorkloadRequestsPerSurface + && missingOperations.length === 0 + && missingCapabilities.length === 0 + && missingOperationOracles.length === 0 + && surfaceErrorRate <= gates.maxErrorRate + && surfaceP99Ms <= gates.maxP99Ms + && (!gates.requireConclusiveCanaries || surfaceCanaryInconclusive === 0) + && (!gates.requireZeroBleed || surfaceBleedViolations === 0) + && ( + !gates.requireConclusiveOperationOracles + || (surfaceOracleInconclusive === 0 && surfaceOracleViolations === 0) + ); + return { + surface: surface.name, + warmed: warmed.has(surface.name), + workloadRequests: surfaceWorkload.length, + successfulWorkloadRequests: surfaceSuccessfulWorkload.length, + errors: surfaceErrors, + errorRate: surfaceErrorRate, + p99Ms: surfaceP99Ms, + operationsConfigured: operationNames.length, + operationsExercised: operationNames.length - missingOperations.length, + canaryChecks: surfaceCanaries.length, + canaryInconclusive: surfaceCanaryInconclusive, + bleedViolations: surfaceBleedViolations, + operationOracleChecks: surfaceOracleSamples.length, + operationOracleInconclusive: surfaceOracleInconclusive, + operationOracleViolations: surfaceOracleViolations, + missingOperations, + missingCapabilities, + missingOperationOracles, + qualified: surfaceQualified + }; + }); + const trafficSurfaceNames = new Set(surfaceResults + .filter((surface) => surface.successfulWorkloadRequests >= minWorkloadRequestsPerSurface) + .map((surface) => surface.surface)); + const missingSurfaces = surfaceResults + .filter((surface) => !surface.warmed || !trafficSurfaceNames.has(surface.surface)) + .map((surface) => surface.surface); + const configuredOperations = tenant.surfaces.flatMap((surface) => + surface.operations.map((operation) => `${surface.name}/${operation.name}`) + ); + const missingOperations = configuredOperations.filter((operation) => + !successfulOperationKeys.has(operation) + ); + const missingOperationOracles = gates.requireConclusiveOperationOracles + ? configuredOperations.filter((operation) => + !conclusiveCoverageOracleKeys.has(operation) + ) + : []; + const configuredCapabilities = [...new Set(tenant.surfaces.flatMap((surface) => + surface.operations.map((operation) => `${surface.name}/${operation.capability}`) + ))]; + const missingCapabilities = [ + ...configuredCapabilities.filter((capability) => !successfulCapabilityKeys.has(capability)), + ...requiredCapabilities + .filter((capability) => !successfulCapabilities.has(capability)) + .map((capability) => `required/${capability}`) + ]; + const qualified = warmed.size === tenant.surfaces.length + && surfaceResults.every((surface) => surface.qualified) + && missingSurfaces.length === 0 + && missingOperations.length === 0 + && missingCapabilities.length === 0 + && missingOperationOracles.length === 0 + && errorRate <= gates.maxErrorRate + && p99Ms <= gates.maxP99Ms + && (!gates.requireConclusiveCanaries || canaryInconclusive === 0) + && (!gates.requireZeroBleed || bleedViolations === 0) + && ( + !gates.requireConclusiveOperationOracles + || ( + operationOracleInconclusive === 0 + && operationOracleViolations === 0 + ) + ); + return { + tenantId: tenant.id, + surfacesConfigured: tenant.surfaces.length, + surfacesWarmed: warmed.size, + surfacesWithTraffic: trafficSurfaceNames.size, + operationsConfigured: configuredOperations.length, + operationsExercised: configuredOperations.length - missingOperations.length, + requests: workloadSamples.length, + errors, + errorRate, + p99Ms, + canaryChecks: localCanaries.length, + canaryInconclusive, + bleedViolations, + operationOracleChecks: operationOracleSamples.length, + operationOracleInconclusive, + operationOracleViolations, + missingSurfaces, + missingOperations, + missingCapabilities, + missingOperationOracles, + surfaces: surfaceResults, + qualified + }; +}; + +export interface ScoreInput { + arm: string; + evidenceMode: 'qualification' | 'diagnostic'; + campaignId: string; + scheduleSha256: string; + previousResultPayloadSha256: string | null; + qualificationCohortSha256: string; + commit?: string; + introspectionMode: 'stock' | 'scoped-required'; + heapMiB: number; + repetition: number; + expectedMatrixRepetitions: number; + runKind: 'matrix' | 'soak'; + runOrderSeed: string; + runOrderIndex: number; + startedAt: string; + endedAt: string; + configuredDurationSec: number; + workloadDurationMs: number; + artifactDir: string; + tenants: TenantTarget[]; + warmedSurfaces: Map>; + warmupLatencies: number[]; + resolvedWarmupTimeoutMs: number; + offeredLoad: ResolvedOfferedLoad; + canaryIntervalSec: number; + periodicCanarySchedule: PeriodicCanarySchedule; + canarySchedule: CanaryScheduleSummary | null; + minWorkloadRequestsPerSurface: number; + samples: RequestSample[]; + canaries: CanaryResult[]; + memorySnapshots: MemorySnapshot[]; + postWarmupNodeRssSnapshots: NodeRssSnapshot[]; + memorySampleErrors: string[]; + retainedMemory: RetainedMemoryCheckpointPair; + postgresSnapshots: PostgresMemorySnapshot[]; + postgresSampleErrors: string[]; + postWarmupSnapshots: MemorySnapshot[]; + missedArrivals: number; + requiredCapabilities: string[]; + requiredCanaries: string[]; + gates: AcceptanceGates; + serverExit: DensityRunResult['serverExit']; + provenance: ArmProvenance | null; + provenanceErrors: string[]; + postgresRunAttestation?: PostgresRunAttestationEvidence | null; + realtimeDeliveryCoverage: RealtimeDeliveryCoverage | null; + externalServer: boolean; + executionErrors: string[]; +} + +const SHA256 = /^[a-f0-9]{64}$/; +const EMPTY_SHA256 = createHash('sha256').update('').digest('hex'); + +const validateRealtimeDeliveryCoverage = (input: ScoreInput): string[] => { + const coverage = input.realtimeDeliveryCoverage; + if (!coverage) return ['coverage record is unavailable']; + const failures: string[] = []; + const startedAtMs = Date.parse(coverage.workloadStartedAt); + const deadlineAtMs = Date.parse(coverage.workloadDeadlineAt); + const endedAtMs = coverage.workloadEndedAt == null + ? Number.NaN + : Date.parse(coverage.workloadEndedAt); + if ( + coverage.version !== 2 + || !Number.isSafeInteger(coverage.deliveryIntervalMs) + || coverage.deliveryIntervalMs <= 0 + || !Number.isFinite(startedAtMs) + || !Number.isFinite(deadlineAtMs) + || !Number.isFinite(endedAtMs) + || deadlineAtMs - startedAtMs !== input.configuredDurationSec * 1000 + || endedAtMs < deadlineAtMs + ) failures.push('coverage timing is invalid'); + + const expectedRoundsPerSurface = Math.max( + 0, + Math.ceil( + input.configuredDurationSec * 1000 / coverage.deliveryIntervalMs + ) - 1 + ); + const expectedSurfaces = new Map(); + for (const tenant of input.tenants) { + for (const surface of tenant.surfaces.filter((candidate) => candidate.realtime)) { + expectedSurfaces.set( + `${tenant.id}\0${surface.name}`, + new URL(surface.url).pathname + ); + } + } + const observed = new Set(); + if (expectedSurfaces.size > 0 && expectedRoundsPerSurface === 0) { + failures.push('no recurring round fits inside the workload'); + } + for (const surface of coverage.surfaces) { + const key = `${surface.tenantId}\0${surface.surface}`; + if (observed.has(key)) failures.push(`duplicate surface ${surface.tenantId}/${surface.surface}`); + observed.add(key); + if (expectedSurfaces.get(key) !== surface.route) { + failures.push(`unexpected route ${surface.tenantId}/${surface.surface}`); + } + if ( + surface.expectedRecurringRounds !== expectedRoundsPerSurface + || surface.startedRecurringRounds !== expectedRoundsPerSurface + || surface.verifiedRecurringRounds !== expectedRoundsPerSurface + || surface.primeRequests !== expectedRoundsPerSurface + ) failures.push(`incomplete surface ${surface.tenantId}/${surface.surface}`); + if ( + !SHA256.test(surface.issuedCorrelationSha256) + || !SHA256.test(surface.verifiedCorrelationSha256) + || surface.issuedCorrelationSha256 !== surface.verifiedCorrelationSha256 + || ( + expectedRoundsPerSurface === 0 + && surface.issuedCorrelationSha256 !== EMPTY_SHA256 + ) + || !Number.isFinite(surface.primeResponseP99Ms) + || surface.primeResponseP99Ms < 0 + || !Number.isFinite(surface.deliveryP99Ms) + || surface.deliveryP99Ms < 0 + ) failures.push(`correlation mismatch ${surface.tenantId}/${surface.surface}`); + } + if ( + observed.size !== expectedSurfaces.size + || [...expectedSurfaces.keys()].some((key) => !observed.has(key)) + ) failures.push('configured realtime surface set is incomplete'); + + const expectedTotal = expectedRoundsPerSurface * expectedSurfaces.size; + if ( + coverage.expectedRecurringRounds !== expectedTotal + || coverage.startedRecurringRounds !== expectedTotal + || coverage.verifiedRecurringRounds !== expectedTotal + || coverage.deadlineLateRecurringRounds !== 0 + || coverage.primeRequests !== expectedTotal + || !Number.isFinite(coverage.primeResponseP99Ms) + || coverage.primeResponseP99Ms < 0 + || !Number.isFinite(coverage.deliveryP99Ms) + || coverage.deliveryP99Ms < 0 + || coverage.complete !== true + ) failures.push('aggregate recurring delivery counters are incomplete'); + return failures; +}; + +type CanaryTuple = readonly [ + tenantId: string, + surface: string, + canary: string, + phase: CanaryResult['phase'], + periodicRound: number | null +]; + +const canaryTupleKey = (tuple: CanaryTuple): string => JSON.stringify(tuple); + +const increment = (counts: Map, key: string): void => { + counts.set(key, (counts.get(key) ?? 0) + 1); +}; + +const shortList = (values: string[]): string => { + const limit = 8; + return values.length <= limit + ? values.join(', ') + : `${values.slice(0, limit).join(', ')} (+${values.length - limit} more)`; +}; + +/** + * Validate isolation evidence from the fleet contract, not from artifact + * counters. JSON-encoded tuples preserve boundaries even when tenant, surface, + * or canary names themselves contain slashes. + */ +const validateStrictCanarySchedule = (input: ScoreInput): string[] => { + if (!input.gates.requireCompletePeriodicCanaryCoverage) return []; + const failures: string[] = []; + const summary = input.canarySchedule; + const expectedRoundCount = periodicCanaryRoundCount( + input.configuredDurationSec * 1000, + input.canaryIntervalSec * 1000 + ); + const expected = new Map(); + const configuredCanaries = new Map(); + const expectedChecksByTargetRound = new Map(); + const expectedTargetsPerRound = input.tenants.reduce( + (sum, tenant) => sum + tenant.surfaces.length, + 0 + ); + + for (const tenant of input.tenants) { + for (const surface of tenant.surfaces) { + for (const canary of surface.canaries) { + configuredCanaries.set( + JSON.stringify([tenant.id, surface.name, canary.name]), + [tenant.id, surface.name, canary.name] + ); + for (const phase of ['initial', 'final'] as const) { + const tuple: CanaryTuple = [tenant.id, surface.name, canary.name, phase, null]; + expected.set(canaryTupleKey(tuple), tuple); + } + } + for (let periodicRound = 1; periodicRound <= expectedRoundCount; periodicRound++) { + const selected = input.periodicCanarySchedule === 'rotating-one' + ? [surface.canaries[rotatingCanaryIndex( + tenant.id, + surface.name, + surface.canaries.length, + periodicRound + )]] + : surface.canaries; + expectedChecksByTargetRound.set( + JSON.stringify([tenant.id, surface.name, periodicRound]), + selected.length + ); + for (const canary of selected) { + const tuple: CanaryTuple = [ + tenant.id, + surface.name, + canary.name, + 'periodic', + periodicRound + ]; + expected.set(canaryTupleKey(tuple), tuple); + } + } + } + } + + const actualCounts = new Map(); + const periodicCoverage = new Set(); + const actualChecksByTargetRound = new Map(); + for (const result of input.canaries) { + const round = result.phase === 'periodic' + ? result.periodicRound ?? null + : null; + increment(actualCounts, canaryTupleKey([ + result.tenantId, + result.surface, + result.canary, + result.phase, + round + ])); + if (result.phase === 'periodic' && result.periodicRound != null) { + periodicCoverage.add(JSON.stringify([ + result.tenantId, + result.surface, + result.canary + ])); + increment(actualChecksByTargetRound, JSON.stringify([ + result.tenantId, + result.surface, + result.periodicRound + ])); + } + const scheduledAt = Date.parse(result.scheduledAt); + const startedAt = Date.parse(result.startedAt); + const completedAt = Date.parse(result.completedAt); + if ( + (result.phase === 'periodic') !== (result.periodicRound != null) + || (result.periodicRound != null && ( + !Number.isSafeInteger(result.periodicRound) + || result.periodicRound <= 0 + )) + || !Number.isFinite(scheduledAt) + || !Number.isFinite(startedAt) + || !Number.isFinite(completedAt) + || startedAt + 2 < scheduledAt + || completedAt < startedAt + || !Number.isFinite(result.latencyMs) + || result.latencyMs < 0 + ) { + failures.push( + `canary evidence has invalid timing ${JSON.stringify([ + result.tenantId, + result.surface, + result.canary, + result.phase, + round + ])}` + ); + } + } + + const missing = [...expected].filter(([key]) => !actualCounts.has(key)).map(([key]) => key); + const duplicates = [...expected].filter(([key]) => (actualCounts.get(key) ?? 0) !== 1) + .filter(([key]) => actualCounts.has(key)) + .map(([key]) => `${key} x${actualCounts.get(key)}`); + const unexpected = [...actualCounts].filter(([key]) => !expected.has(key)) + .map(([key, count]) => `${key} x${count}`); + if (missing.length > 0) failures.push(`missing exact canary evidence: ${shortList(missing)}`); + if (duplicates.length > 0) { + failures.push(`duplicate exact canary evidence: ${shortList(duplicates)}`); + } + if (unexpected.length > 0) { + failures.push(`unexpected canary evidence: ${shortList(unexpected)}`); + } + + const missingPeriodicCoverage = [...configuredCanaries] + .filter(([key]) => !periodicCoverage.has(key)) + .map(([key]) => key); + if (missingPeriodicCoverage.length > 0) { + failures.push( + `periodic canary coverage is incomplete: ${shortList(missingPeriodicCoverage)}` + ); + } + const targetRoundMismatches = [...expectedChecksByTargetRound] + .filter(([key, count]) => (actualChecksByTargetRound.get(key) ?? 0) !== count) + .map(([key, count]) => + `${key} expected=${count} actual=${actualChecksByTargetRound.get(key) ?? 0}` + ); + if (targetRoundMismatches.length > 0) { + failures.push( + `periodic target/round evidence mismatch: ${shortList(targetRoundMismatches)}` + ); + } + + if (!summary) { + failures.push('periodic canary schedule summary is unavailable'); + return failures; + } + if (summary.schedule !== input.periodicCanarySchedule) { + failures.push( + `periodic canary schedule=${summary.schedule}, expected ${input.periodicCanarySchedule}` + ); + } + if ( + summary.intervalMs !== input.canaryIntervalSec * 1000 + || summary.durationMs !== input.configuredDurationSec * 1000 + ) { + failures.push('periodic canary schedule timing does not match the workload plan'); + } + if ( + summary.planned !== expectedRoundCount + || summary.started !== expectedRoundCount + || summary.completed !== expectedRoundCount + || summary.missed !== 0 + ) { + failures.push( + `periodic canary rounds planned=${summary.planned} started=${summary.started} ` + + `completed=${summary.completed} missed=${summary.missed}, expected ${expectedRoundCount}` + ); + } + const expectedChecks = [...expected.values()].filter((tuple) => tuple[3] === 'periodic').length; + if ( + summary.checksPlanned !== expectedChecks + || summary.checksStarted !== expectedChecks + || summary.checksCompleted !== expectedChecks + ) { + failures.push( + `periodic canary checks planned=${summary.checksPlanned} ` + + `started=${summary.checksStarted} completed=${summary.checksCompleted}, ` + + `expected ${expectedChecks}` + ); + } + + const scheduleStart = Date.parse(summary.startedAt); + const scheduleDeadline = Date.parse(summary.deadlineAt); + if ( + !Number.isFinite(scheduleStart) + || !Number.isFinite(scheduleDeadline) + || scheduleDeadline !== scheduleStart + summary.durationMs + ) { + failures.push('periodic canary schedule boundary timestamps are invalid'); + } + const wrongPeriodicSlots = input.canaries.filter((result) => + result.phase === 'periodic' + && result.periodicRound != null + && Date.parse(result.scheduledAt) + !== scheduleStart + result.periodicRound * summary.intervalMs + ); + if (wrongPeriodicSlots.length > 0) { + failures.push( + `periodic canary evidence has wrong scheduled slots: ${shortList( + wrongPeriodicSlots.map((result) => JSON.stringify([ + result.tenantId, + result.surface, + result.canary, + result.periodicRound + ])) + )}` + ); + } + const roundsByNumber = new Map(summary.rounds.map((round) => [round.periodicRound, round])); + if (summary.rounds.length !== expectedRoundCount || roundsByNumber.size !== expectedRoundCount) { + failures.push('periodic canary round summaries are incomplete or duplicated'); + } + let recomputedDeadlineLate = 0; + const recomputedOverlapped = summary.rounds.filter((round) => round.overlapped).length; + if (summary.overlapped !== recomputedOverlapped) { + failures.push('periodic canary overlap count does not match round summaries'); + } + for (let periodicRound = 1; periodicRound <= expectedRoundCount; periodicRound++) { + const round = roundsByNumber.get(periodicRound); + if (!round) continue; + const plannedAt = Date.parse(round.plannedAt); + const startedAt = round.startedAt == null ? NaN : Date.parse(round.startedAt); + const completedAt = round.completedAt == null ? NaN : Date.parse(round.completedAt); + const expectedRoundChecks = [...expectedChecksByTargetRound] + .filter(([key]) => (JSON.parse(key) as [string, string, number])[2] === periodicRound) + .reduce((sum, [, count]) => sum + count, 0); + if ( + !Number.isFinite(plannedAt) + || plannedAt !== scheduleStart + periodicRound * summary.intervalMs + || !Number.isFinite(startedAt) + || !Number.isFinite(completedAt) + || completedAt < startedAt + || round.targetsPlanned !== expectedTargetsPerRound + || round.targetsStarted !== round.targetsPlanned + || round.targetsCompleted !== round.targetsPlanned + || round.checksPlanned !== expectedRoundChecks + || round.checksStarted !== expectedRoundChecks + || round.checksCompleted !== expectedRoundChecks + ) { + failures.push(`periodic canary round ${periodicRound} summary is incomplete`); + } + if (completedAt > scheduleDeadline || round.deadlineLate) recomputedDeadlineLate++; + } + const periodicResultsLate = input.canaries.filter((result) => + result.phase === 'periodic' + && Date.parse(result.completedAt) > scheduleDeadline + ).length; + if ( + summary.deadlineLate !== 0 + || recomputedDeadlineLate !== 0 + || periodicResultsLate !== 0 + ) { + failures.push( + `periodic canary rounds completed after deadline=${Math.max( + summary.deadlineLate, + recomputedDeadlineLate + )}; late checks=${periodicResultsLate}` + ); + } + return failures; +}; + +export const scoreRun = (input: ScoreInput): DensityRunResult => { + const coverageSamples = input.samples.filter((sample) => sample.phase === 'coverage'); + const workloadSamples = input.samples.filter((sample) => sample.phase === 'workload'); + const latencies = workloadSamples.map((sample) => sample.latencyMs); + const errors = workloadSamples.filter((sample) => !sample.ok).length; + const errorRate = workloadSamples.length > 0 ? errors / workloadSamples.length : 1; + const tenantResults = input.tenants.map((tenant) => tenantResult( + tenant, + input.samples, + input.canaries, + input.warmedSurfaces.get(tenant.id) ?? new Set(), + input.requiredCapabilities, + input.minWorkloadRequestsPerSurface, + input.gates + )); + const fleetShape = summarizeCustomerFleet(input.tenants); + const evictions = counterDelta(input.postWarmupSnapshots, 'evictions'); + const buildRefusals = counterDelta(input.postWarmupSnapshots, 'buildRefusals'); + const postWarmupBuilds = counterDelta(input.postWarmupSnapshots, 'buildsStarted'); + const postWarmupPgPoolCapacityEvictions = pgPoolCounterDelta( + input.postWarmupSnapshots, + 'pgPoolCapacityEvictions' + ); + const postWarmupPgPoolCapacityRefusals = pgPoolCounterDelta( + input.postWarmupSnapshots, + 'pgPoolCapacityRefusals' + ); + const postWarmupPgPoolDisposalFailures = pgPoolCounterDelta( + input.postWarmupSnapshots, + 'pgPoolDisposalFailures' + ); + const completePgPoolTelemetry = input.postWarmupSnapshots.length > 0 + && input.postWarmupSnapshots.every((snapshot) => + snapshot.pgPoolCacheSize != null + && snapshot.pgPoolLeasedPools != null + && snapshot.pgPoolActiveLeases != null + && snapshot.pgPoolCapacityEvictions != null + && snapshot.pgPoolCapacityRefusals != null + && snapshot.pgPoolDisposalFailures != null + ); + const pgPoolCacheSize = completePgPoolTelemetry + ? Math.max(...input.postWarmupSnapshots.map((snapshot) => snapshot.pgPoolCacheSize!)) + : null; + const pgPoolLeasedPools = completePgPoolTelemetry + ? Math.max(...input.postWarmupSnapshots.map((snapshot) => snapshot.pgPoolLeasedPools!)) + : null; + const pgPoolActiveLeases = completePgPoolTelemetry + ? Math.max(...input.postWarmupSnapshots.map((snapshot) => snapshot.pgPoolActiveLeases!)) + : null; + const maximumCompleteValue = (key: keyof MemorySnapshot): number | null => { + const values = input.postWarmupSnapshots + .map((snapshot) => snapshot[key]) + .filter((value): value is number => typeof value === 'number'); + return values.length === input.postWarmupSnapshots.length && values.length > 0 + ? Math.max(...values) + : null; + }; + const minimumCompleteValue = (key: keyof MemorySnapshot): number | null => { + const values = input.postWarmupSnapshots + .map((snapshot) => snapshot[key]) + .filter((value): value is number => typeof value === 'number'); + return values.length === input.postWarmupSnapshots.length && values.length > 0 + ? Math.min(...values) + : null; + }; + const postgresBackendPeak = maximumCompleteValue('postgresBackendTotal'); + const physicalDatabaseValues = input.postWarmupSnapshots + .map((snapshot) => snapshot.physicalDatabases) + .filter((value): value is number => typeof value === 'number'); + const residentPhysicalDatabases = physicalDatabaseValues.length + === input.postWarmupSnapshots.length && physicalDatabaseValues.length > 0 + ? Math.min(...physicalDatabaseValues) + : null; + const postgresContainerScopeValues = input.postWarmupSnapshots + .map((snapshot) => snapshot.postgresContainerDedicated) + .filter((value): value is boolean => typeof value === 'boolean'); + const postgresContainerDedicated = postgresContainerScopeValues.length + === input.postWarmupSnapshots.length && postgresContainerScopeValues.length > 0 + ? postgresContainerScopeValues.every(Boolean) + : null; + const unexpectedPostgresDatabases = maximumCompleteValue( + 'unexpectedPostgresDatabases' + ); + const pgPoolTotalClients = maximumCompleteValue('pgPoolTotalClients'); + const pgPoolIdleClients = maximumCompleteValue('pgPoolIdleClients'); + const pgPoolWaitingClients = maximumCompleteValue('pgPoolWaitingClients'); + const completeRuntimePoolTelemetry = input.postWarmupSnapshots.length > 0 + && input.postWarmupSnapshots.every((snapshot) => + snapshot.runtimePoolTelemetryScope === 'runtime-only-exact-identities' + && snapshot.runtimePoolTelemetryAvailable === true + && snapshot.runtimePoolEffectiveMaxUsesKnown === true + && snapshot.runtimePoolMaxUsesExact === true + && snapshot.runtimePoolExpectedPools === fleetShape.apis + && snapshot.runtimePoolObservedPools === fleetShape.apis + && Number.isSafeInteger(snapshot.runtimePoolTotalClients) + && snapshot.runtimePoolTotalClients! >= 0 + && Number.isSafeInteger(snapshot.runtimePoolIdleClients) + && snapshot.runtimePoolIdleClients! >= 0 + && Number.isSafeInteger(snapshot.runtimePoolWaitingClients) + && snapshot.runtimePoolWaitingClients! >= 0 + && ( + snapshot.runtimePoolRequestedMaxUses == null + || ( + Number.isSafeInteger(snapshot.runtimePoolRequestedMaxUses) + && snapshot.runtimePoolRequestedMaxUses > 0 + ) + ) + && snapshot.runtimePoolEffectiveMaxUses + === snapshot.runtimePoolRequestedMaxUses + ); + const requestedMaxUsesValues = completeRuntimePoolTelemetry + ? [...new Set(input.postWarmupSnapshots.map( + (snapshot) => snapshot.runtimePoolRequestedMaxUses ?? null + ))] + : []; + const effectiveMaxUsesValues = completeRuntimePoolTelemetry + ? [...new Set(input.postWarmupSnapshots.map( + (snapshot) => snapshot.runtimePoolEffectiveMaxUses ?? null + ))] + : []; + const runtimePoolRequestedMaxUses = requestedMaxUsesValues.length === 1 + ? requestedMaxUsesValues[0] + : null; + const runtimePoolEffectiveMaxUses = effectiveMaxUsesValues.length === 1 + ? effectiveMaxUsesValues[0] + : null; + const runtimePoolExpectedPools = completeRuntimePoolTelemetry + ? minimumCompleteValue('runtimePoolExpectedPools') + : null; + const runtimePoolObservedPools = completeRuntimePoolTelemetry + ? minimumCompleteValue('runtimePoolObservedPools') + : null; + const runtimePoolTotalClients = completeRuntimePoolTelemetry + ? maximumCompleteValue('runtimePoolTotalClients') + : null; + const runtimePoolIdleClients = completeRuntimePoolTelemetry + ? maximumCompleteValue('runtimePoolIdleClients') + : null; + const runtimePoolWaitingClients = completeRuntimePoolTelemetry + ? maximumCompleteValue('runtimePoolWaitingClients') + : null; + const residentRealtimeManagers = minimumCompleteValue('realtimeManagersActive'); + const residentRealtimeTransports = minimumCompleteValue('realtimeTransportsActive'); + const notificationModes = input.postWarmupSnapshots + .map((snapshot) => snapshot.realtimeNotificationMode) + .filter((value): value is 'dedicated' | 'shared-exact' => + value === 'dedicated' || value === 'shared-exact' + ); + const realtimeNotificationMode = notificationModes.length + === input.postWarmupSnapshots.length + && notificationModes.length > 0 + && new Set(notificationModes).size === 1 + ? notificationModes[0] + : null; + const notificationBrokers = minimumCompleteValue('notificationBrokers'); + const notificationListenerConnections = minimumCompleteValue( + 'notificationListenerConnections' + ); + const notificationBrokerLeases = minimumCompleteValue('notificationBrokerLeases'); + const notificationBrokerTopics = minimumCompleteValue('notificationBrokerTopics'); + const notificationBrokerSubscribers = minimumCompleteValue( + 'notificationBrokerSubscribers' + ); + const notificationBrokerQueueOverflows = maximumCompleteValue( + 'notificationBrokerQueueOverflows' + ); + const notificationBrokerFatalFailures = maximumCompleteValue( + 'notificationBrokerFatalFailures' + ); + const notificationAuditIdentities = minimumCompleteValue( + 'notificationAuditIdentities' + ); + const notificationAuditsHealthy = minimumCompleteValue( + 'notificationAuditsHealthy' + ); + const notificationAuditsFailed = maximumCompleteValue('notificationAuditsFailed'); + const notificationAuditsStale = maximumCompleteValue('notificationAuditsStale'); + const notificationAuditAttempts = maximumCompleteValue('notificationAuditAttempts'); + const notificationAuditFailures = maximumCompleteValue('notificationAuditFailures'); + const notificationAuditActiveDatabaseTargets = minimumCompleteValue( + 'notificationAuditActiveDatabaseTargets' + ); + const notificationAuditDatabaseConflicts = maximumCompleteValue( + 'notificationAuditDatabaseConflicts' + ); + const cacheConfiguredMax = minimumCompleteValue('cacheConfiguredMax'); + const cacheBudgetCapacity = minimumCompleteValue('cacheBudgetCapacity'); + const cacheInstanceHeapBytes = maximumCompleteValue('cacheInstanceHeapBytes'); + const cacheCalibrationIds = input.postWarmupSnapshots + .map((snapshot) => snapshot.cacheCalibrationId) + .filter((value): value is string => typeof value === 'string' && value.length > 0); + const cacheCalibrationId = cacheCalibrationIds.length + === input.postWarmupSnapshots.length + && new Set(cacheCalibrationIds).size === 1 + ? cacheCalibrationIds[0] + : null; + const cacheAdmissionModes = input.postWarmupSnapshots + .map((snapshot) => snapshot.cacheAdmissionMode) + .filter((value): value is NonNullable => + value === 'evict-idle' || value === 'preserve-resident' + ); + const cacheAdmissionMode = cacheAdmissionModes.length + === input.postWarmupSnapshots.length + && cacheAdmissionModes.length > 0 + && new Set(cacheAdmissionModes).size === 1 + ? cacheAdmissionModes[0] + : null; + const rawHeapGrowth = heapGrowthMiBPerHour(input.postWarmupSnapshots); + const postgresPeakBytes = input.postgresSnapshots.length + ? Math.max(...input.postgresSnapshots.map((snapshot) => snapshot.usedBytes)) + : null; + const postgresBaselineBytes = input.postgresSnapshots[0]?.usedBytes ?? null; + const postgresWorkingSetValues = input.postgresSnapshots + .map((snapshot) => snapshot.workingSetBytes) + .filter((value): value is number => value != null); + const postgresWorkingSetPeakBytes = postgresWorkingSetValues.length > 0 + ? Math.max(...postgresWorkingSetValues) + : null; + const postgresCgroupV2Samples = input.postgresSnapshots.filter( + (snapshot) => snapshot.source === 'cgroup-v2' && snapshot.cgroupV2 != null + ).length; + const completePostgresCgroupV2CurrentTelemetry = input.postgresSnapshots.length > 0 + && input.postgresSnapshots.every((snapshot) => + snapshot.source === 'cgroup-v2' + && snapshot.cgroupV2 != null + && Number.isSafeInteger(snapshot.cgroupV2.currentBytes) + && snapshot.cgroupV2.currentBytes >= 0 + && snapshot.usedBytes === snapshot.cgroupV2.currentBytes + ); + const postgresCgroupPeakValues = input.postgresSnapshots + .map((snapshot) => snapshot.cgroupV2?.peakBytes) + .filter((value): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 + ); + const postgresCgroupV2PeakBytes = postgresCgroupPeakValues.length > 0 + ? Math.max(...postgresCgroupPeakValues) + : null; + const completePostgresCgroupV2PeakTelemetry = completePostgresCgroupV2CurrentTelemetry + && postgresCgroupPeakValues.length === input.postgresSnapshots.length; + const firstPostgresEvents = input.postgresSnapshots[0]?.cgroupV2?.events; + const lastPostgresEvents = input.postgresSnapshots.at(-1)?.cgroupV2?.events; + const completePostgresOomEvents = firstPostgresEvents + && lastPostgresEvents + && typeof firstPostgresEvents.oom === 'number' + && typeof firstPostgresEvents.oom_kill === 'number' + && typeof lastPostgresEvents.oom === 'number' + && typeof lastPostgresEvents.oom_kill === 'number'; + const postgresOomEvents = completePostgresOomEvents + ? Math.max(0, lastPostgresEvents.oom - firstPostgresEvents.oom) + + Math.max( + 0, + lastPostgresEvents.oom_kill - firstPostgresEvents.oom_kill + ) + : null; + const postWarmupStartedAtMs = input.postWarmupSnapshots[0] + ? Date.parse(input.postWarmupSnapshots[0].timestamp) + : Number.NaN; + const coldBuildPostgresSnapshots = Number.isFinite(postWarmupStartedAtMs) + ? input.postgresSnapshots.filter( + (snapshot) => Date.parse(snapshot.timestamp) <= postWarmupStartedAtMs + ) + : []; + const postgresColdBuildPeakBytes = coldBuildPostgresSnapshots.length > 0 + ? Math.max(...coldBuildPostgresSnapshots.map((snapshot) => snapshot.usedBytes)) + : null; + const postgresColdBuildSpikeBytes = postgresColdBuildPeakBytes != null + && input.postgresSnapshots.length >= 2 + ? Math.max(0, postgresColdBuildPeakBytes - input.postgresSnapshots[0].usedBytes) + : null; + const postWarmupPostgresSnapshots = Number.isFinite(postWarmupStartedAtMs) + ? input.postgresSnapshots.filter( + // Retain the nearest preceding cgroup sample for sub-second alignment. + (snapshot) => Date.parse(snapshot.timestamp) >= postWarmupStartedAtMs - 1_000 + ) + : []; + const alignedPostgresSnapshots = input.evidenceMode === 'qualification' + ? postWarmupPostgresSnapshots.filter( + (snapshot) => snapshot.source === 'cgroup-v2' && snapshot.cgroupV2 != null + ).map((snapshot) => ({ + ...snapshot, + usedBytes: snapshot.cgroupV2!.currentBytes + })) + : postWarmupPostgresSnapshots; + const postgresWarmBoundaryBytes = Number.isFinite(postWarmupStartedAtMs) + && input.postgresSnapshots.length > 0 + ? input.postgresSnapshots.reduce((nearest, snapshot) => + Math.abs(Date.parse(snapshot.timestamp) - postWarmupStartedAtMs) + < Math.abs(Date.parse(nearest.timestamp) - postWarmupStartedAtMs) + ? snapshot + : nearest + ).usedBytes + : null; + const alignedServicePeak = alignedServiceMemoryPeak( + input.postWarmupNodeRssSnapshots, + alignedPostgresSnapshots + ); + const alignedServiceCoverage = alignedServiceMemoryCoverage( + input.postWarmupNodeRssSnapshots, + alignedPostgresSnapshots, + postWarmupStartedAtMs, + input.workloadDurationMs + ); + const expectedResidentInstances = new Set(input.tenants.flatMap((tenant) => + tenant.surfaces.map((surface) => surface.buildContract) + )).size; + const expectedResidentBuildContracts = new Set(input.tenants.flatMap((tenant) => + tenant.surfaces.map((surface) => surface.buildContract) + )); + const retainedGrowth = retainedMemoryGrowth( + input.retainedMemory, + input.provenance?.serverPid ?? null, + expectedResidentBuildContracts, + input.gates.requirePhysicalDatabaseTelemetry + ); + const residentCounts = input.postWarmupSnapshots + .map((snapshot) => snapshot.cacheSize) + .filter((value): value is number => value != null); + const residentInstances = residentCounts.length === input.postWarmupSnapshots.length + && residentCounts.length > 0 + ? Math.min(...residentCounts) + : null; + const baselineHeapBytes = input.memorySnapshots[0]?.heapUsedBytes; + const warmHeapBytes = input.postWarmupSnapshots[0]?.heapUsedBytes; + const warmCacheSize = input.postWarmupSnapshots[0]?.cacheSize; + const warmObservedHeapDeltaPerInstanceBytes = baselineHeapBytes != null + && warmHeapBytes != null + && warmCacheSize > 0 + ? Math.max(0, warmHeapBytes - baselineHeapBytes) / warmCacheSize + : null; + const successfulSamples = input.samples.filter((sample) => sample.ok); + const capabilitiesExercised = [...new Set(successfulSamples.map((sample) => sample.capability))] + .sort(); + const missingCapabilities = tenantResults.flatMap((tenant) => + tenant.missingCapabilities.map((capability) => `${tenant.tenantId}/${capability}`) + ); + const checkedCanaries = new Set(input.canaries.map((canary) => + JSON.stringify([canary.tenantId, canary.surface, canary.canary]) + )); + const missingCanaries = input.tenants.flatMap((tenant) => tenant.surfaces.flatMap((surface) => + input.requiredCanaries + .filter((canary) => !checkedCanaries.has(JSON.stringify([ + tenant.id, + surface.name, + canary + ]))) + .map((canary) => `${tenant.id}/${surface.name}/${canary}`) + )); + const operationOracleChecks = tenantResults.reduce( + (sum, tenant) => sum + tenant.operationOracleChecks, + 0 + ); + const operationOracleInconclusive = tenantResults.reduce( + (sum, tenant) => sum + tenant.operationOracleInconclusive, + 0 + ); + const operationOracleViolations = tenantResults.reduce( + (sum, tenant) => sum + tenant.operationOracleViolations, + 0 + ); + const missingOperationOracles = tenantResults.flatMap((tenant) => + tenant.missingOperationOracles.map((operation) => + `${tenant.tenantId}/${operation}` + ) + ); + const failures: string[] = validateStrictCanarySchedule(input); + if (!/^[a-f0-9]{64}$/.test(input.campaignId)) { + failures.push('campaign identity is unavailable or invalid'); + } + if (!/^[a-f0-9]{64}$/.test(input.scheduleSha256)) { + failures.push('campaign schedule binding is unavailable or invalid'); + } + if ( + input.previousResultPayloadSha256 != null + && !/^[a-f0-9]{64}$/.test(input.previousResultPayloadSha256) + ) { + failures.push('campaign result-chain pointer is invalid'); + } + if (input.configuredDurationSec < 900) failures.push('workload shorter than the 15-minute qualification floor'); + if (input.workloadDurationMs < input.configuredDurationSec * 1000 * 0.99) { + failures.push(`measured workload duration ${(input.workloadDurationMs / 1000).toFixed(2)}s is short`); + } + if (input.memorySnapshots.length === 0) failures.push('memory snapshots unavailable'); + if (input.memorySampleErrors.length > 0) failures.push(`memory sampler errors=${input.memorySampleErrors.length}`); + if (input.provenanceErrors.length > 0) { + failures.push(`provenance validation errors: ${input.provenanceErrors.join('; ')}`); + } + if (!input.provenance) { + failures.push('server provenance unavailable'); + } else { + const missingProvenance = [ + !input.provenance.cwd ? 'cwd' : null, + input.provenance.command.length === 0 ? 'command' : null, + !input.provenance.gitHead ? 'gitHead' : null, + input.provenance.worktreeDirty !== false ? 'cleanWorktree' : null, + !input.provenance.gitStatusSha256 ? 'gitStatusSha256' : null, + !input.provenance.lockfileSha256 ? 'lockfileSha256' : null, + !input.provenance.entrySha256 ? 'entrySha256' : null, + input.provenance.serverPid == null ? 'serverPid' : null, + !input.provenance.v8Profile ? 'v8Profile' : null, + !input.provenance.nodeOptions ? 'nodeOptions' : null, + !Array.isArray(input.provenance.nodeOptionsArgv) ? 'nodeOptionsArgv' : null, + !Array.isArray(input.provenance.nodeExecArgv) ? 'nodeExecArgv' : null, + !Array.isArray(input.provenance.effectiveNodeRuntimeFlags) + ? 'effectiveNodeRuntimeFlags' + : null, + !input.provenance.planSha256 ? 'planSha256' : null, + !input.provenance.fleetSha256 ? 'fleetSha256' : null, + !input.provenance.node ? 'node' : null, + !input.provenance.v8 ? 'v8' : null, + !input.provenance.runOrderSeed ? 'runOrderSeed' : null, + !input.provenance.memoryPolicy ? 'memoryPolicy' : null + ].filter((value): value is string => value != null); + if (missingProvenance.length > 0) { + failures.push(`server provenance incomplete: ${missingProvenance.join(', ')}`); + } + if ( + input.evidenceMode === 'qualification' + && ( + input.provenance.platform !== 'linux' + || input.postWarmupNodeRssSnapshots.length === 0 + || input.postWarmupNodeRssSnapshots.some((snapshot) => + snapshot.source !== 'proc' + || snapshot.pid !== input.provenance!.serverPid + ) + ) + ) { + failures.push('qualification requires exact-PID Linux /proc RSS evidence'); + } + const expectedProfileFlags: Record = { + stock: [], + 'optimize-for-size': ['--optimize-for-size'], + 'baseline-optimize-for-size': ['--max-opt=1', '--optimize-for-size'], + 'jitless-optimize-for-size': ['--jitless', '--optimize-for-size'] + }; + const expectedFlags = expectedProfileFlags[input.provenance.v8Profile]; + const managedFlags = input.provenance.nodeExecArgv.filter((flag) => + flag === '--jitless' + || flag === '--optimize-for-size' + || flag === '--max-opt=1' + ); + const expectedEffectiveFlags = [ + ...input.provenance.nodeOptionsArgv, + ...input.provenance.nodeExecArgv + ]; + if ( + !expectedFlags + || JSON.stringify(managedFlags) !== JSON.stringify(expectedFlags) + || input.provenance.nodeOptionsArgv.some((flag) => + flag === '--jitless' + || flag === '--optimize-for-size' + || flag === '--max-opt=1' + ) + || JSON.stringify(input.provenance.effectiveNodeRuntimeFlags) + !== JSON.stringify(expectedEffectiveFlags) + ) { + failures.push('server V8 runtime-flag provenance is inconsistent'); + } + } + if (input.gates.requireFreshPostgresRunAttestation) { + const attestation = input.postgresRunAttestation; + const provenanceCommand = input.provenance?.command ?? []; + const argumentAfter = (flag: string): string | null => { + const index = provenanceCommand.indexOf(flag); + return index >= 0 ? provenanceCommand[index + 1] ?? null : null; + }; + const exactRunBinding = attestation != null + && attestation.arm === input.arm + && attestation.heapMiB === input.heapMiB + && attestation.tenantCount === input.tenants.length + && attestation.repetition === input.repetition + && attestation.runOrderIndex === input.runOrderIndex + && attestation.planSha256 === `sha256:${input.provenance?.planSha256}` + && attestation.fleetSha256 === `sha256:${input.provenance?.fleetSha256}` + && argumentAfter('--expected-manifest-sha256') === attestation.manifestSha256 + && argumentAfter('--clone-id') === attestation.cloneId; + if (!attestation) { + failures.push('fresh PostgreSQL run attestation unavailable'); + } else if ( + !exactRunBinding + || attestation.freshContainerForRun !== true + || attestation.cgroupV2Verified !== true + || attestation.liveCustomerContractsAudited !== input.tenants.length + || attestation.catalogCacheState !== 'warmed-by-live-contract-audit' + || !/^sha256:[a-f0-9]{64}$/.test(attestation.containerConfigurationSha256) + || !/^sha256:[a-f0-9]{64}$/.test(attestation.cloneAttestationSetSha256) + || !/^sha256:[a-f0-9]{64}$/.test(attestation.cloneNonceSetSha256) + || !/^sha256:[a-f0-9]{64}$/.test(attestation.liveContractSetSha256) + ) { + failures.push('fresh PostgreSQL run attestation is incomplete or mismatched'); + } + if ( + attestation + && ( + input.postgresSnapshots.length === 0 + || input.postgresSnapshots.some((snapshot) => + snapshot.containerId !== attestation.containerId + || snapshot.cgroupIdentitySha256 !== attestation.cgroupIdentitySha256 + ) + ) + ) { + failures.push('PostgreSQL memory samples do not match the attested immutable container'); + } + } + if (input.gates.requiredCacheAdmissionMode) { + const requiredMode = input.gates.requiredCacheAdmissionMode; + const pinnedMode = input.provenance?.memoryPolicy + ?.graphileCacheAdmissionMode ?? null; + if (cacheAdmissionMode !== requiredMode) { + failures.push( + `live Graphile cache admission mode=${cacheAdmissionMode ?? 'unknown'}, required ${requiredMode}` + ); + } + if (pinnedMode !== requiredMode) { + failures.push( + `pinned Graphile cache admission mode=${pinnedMode ?? 'unknown'}, required ${requiredMode}` + ); + } + } + if (input.gates.requirePostgresMemoryTelemetry && input.postgresSnapshots.length < 2) failures.push('PostgreSQL memory telemetry unavailable'); + if (input.gates.requirePostgresMemoryTelemetry && input.postgresSampleErrors.length > 0) failures.push(`PostgreSQL memory sampler errors=${input.postgresSampleErrors.length}`); + if ( + input.gates.requirePostgresMemoryTelemetry + && input.evidenceMode === 'qualification' + && !completePostgresCgroupV2CurrentTelemetry + ) { + failures.push('PostgreSQL cgroup-v2 telemetry was incomplete'); + } + if ( + input.gates.requirePostgresMemoryTelemetry + && input.evidenceMode === 'qualification' + && !completePostgresCgroupV2PeakTelemetry + ) { + failures.push( + 'PostgreSQL cgroup-v2 memory.peak telemetry unavailable for conservative denominator' + ); + } + if (postgresOomEvents != null && postgresOomEvents > 0) { + failures.push(`PostgreSQL cgroup recorded OOM events=${postgresOomEvents}`); + } + if ( + input.gates.requirePostgresMemoryTelemetry + && postgresCgroupV2Samples === input.postgresSnapshots.length + && postgresCgroupV2Samples > 0 + && postgresOomEvents == null + ) { + failures.push('PostgreSQL cgroup OOM event telemetry unavailable'); + } + if ( + input.gates.requirePostgresMemoryTelemetry + && (!alignedServicePeak || alignedServicePeak.samples < 2) + ) { + failures.push('aligned Node and PostgreSQL service-memory telemetry unavailable'); + } + if (input.gates.requirePostgresMemoryTelemetry && input.evidenceMode === 'qualification') { + const maxGapMs = input.gates.maxAlignedMemorySampleGapMs + ?? DEFAULT_MAX_ALIGNED_MEMORY_SAMPLE_GAP_MS; + const minCoverageRatio = input.gates.minAlignedMemoryCoverageRatio + ?? DEFAULT_MIN_ALIGNED_MEMORY_COVERAGE_RATIO; + if (!alignedServiceCoverage) { + failures.push('aligned service-memory workload coverage unavailable'); + } else { + if (alignedServiceCoverage.maxGapMs > maxGapMs) { + failures.push( + `aligned service-memory maximum sample gap ${alignedServiceCoverage.maxGapMs.toFixed(0)}ms exceeds ${maxGapMs}ms` + ); + } + if (alignedServiceCoverage.coverageRatio < minCoverageRatio) { + failures.push( + `aligned service-memory workload coverage ${(alignedServiceCoverage.coverageRatio * 100).toFixed(2)}% is below ${(minCoverageRatio * 100).toFixed(2)}%` + ); + } + } + } + if (residentInstances == null || residentInstances < expectedResidentInstances) { + failures.push(`resident Graphile instances=${residentInstances ?? 'unknown'}, expected at least ${expectedResidentInstances}`); + } + const expectedBuildContracts = new Set(input.tenants.flatMap((tenant) => + tenant.surfaces.map((surface) => surface.buildContract) + )); + const identityUnavailable = input.postWarmupSnapshots.some((snapshot) => + input.gates.requirePhysicalDatabaseTelemetry + ? snapshot.residentBuildContractFingerprints == null + : snapshot.residentBuildContracts == null + ); + const missingResidentBuilds = new Set(); + for (const snapshot of input.postWarmupSnapshots) { + const resident = new Set( + (input.gates.requirePhysicalDatabaseTelemetry + ? snapshot.residentBuildContractFingerprints + : snapshot.residentBuildContracts) + ?? [] + ); + for (const contract of expectedBuildContracts) { + if (!resident.has(contract)) missingResidentBuilds.add(contract); + } + } + if (identityUnavailable || input.postWarmupSnapshots.length === 0) { + failures.push(input.gates.requirePhysicalDatabaseTelemetry + ? 'resident Graphile build-contract fingerprints unavailable' + : 'resident Graphile build-contract identities unavailable'); + } else if (missingResidentBuilds.size > 0) { + failures.push(input.gates.requirePhysicalDatabaseTelemetry + ? `resident Graphile build fingerprints missing: ${[...missingResidentBuilds].join(', ')}` + : `resident Graphile build contracts missing: ${[...missingResidentBuilds].join(', ')}`); + } + if (input.missedArrivals > 0) { + failures.push(`load generator missed scheduled arrivals=${input.missedArrivals}`); + } + if (errorRate > input.gates.maxErrorRate) failures.push(`error rate ${errorRate} exceeds ${input.gates.maxErrorRate}`); + if (percentile(latencies, 0.99) > input.gates.maxP99Ms) failures.push(`p99 exceeds ${input.gates.maxP99Ms}ms`); + if (input.gates.requireNoPostWarmupEvictions && evictions !== 0) failures.push(`post-warmup evictions=${evictions ?? 'unknown'}`); + if (input.gates.requireNoPostWarmupBuildRefusals && buildRefusals !== 0) failures.push(`post-warmup build refusals=${buildRefusals ?? 'unknown'}`); + if (input.gates.requireNoPostWarmupBuilds && postWarmupBuilds !== 0) failures.push(`post-warmup builds=${postWarmupBuilds ?? 'unknown'}`); + if (!completePgPoolTelemetry) failures.push('PostgreSQL pool-cache telemetry unavailable'); + const expectedRealtimeApis = fleetShape.realtimeApis; + if (input.gates.requirePhysicalDatabaseTelemetry) { + const expectedCalibrationId = input.provenance?.memoryPolicy + ?.graphileCacheCalibrationId ?? null; + if ( + !expectedCalibrationId + || cacheCalibrationId !== expectedCalibrationId + ) { + failures.push( + `Graphile cache calibration identity=${cacheCalibrationId ?? 'unknown'}, expected ${expectedCalibrationId ?? 'pinned provenance identity'}` + ); + } + if (cacheConfiguredMax == null || cacheConfiguredMax < expectedResidentInstances) { + failures.push( + `Graphile configured cache max=${cacheConfiguredMax ?? 'unknown'}, expected at least ${expectedResidentInstances}` + ); + } + if (cacheBudgetCapacity == null || cacheBudgetCapacity < expectedResidentInstances) { + failures.push( + `Graphile heap budget capacity=${cacheBudgetCapacity ?? 'unknown'}, expected at least ${expectedResidentInstances}` + ); + } + if (postgresContainerDedicated !== true) { + failures.push( + `dedicated PostgreSQL container scope not proven; unexpected databases=${unexpectedPostgresDatabases ?? 'unknown'}` + ); + } + if ( + residentPhysicalDatabases == null + || physicalDatabaseValues.some((value) => value !== fleetShape.physicalDatabases) + ) { + failures.push( + `resident physical databases=${residentPhysicalDatabases ?? 'unknown'}, expected exactly ${fleetShape.physicalDatabases}` + ); + } + if (postgresBackendPeak == null) { + failures.push('physical PostgreSQL backend telemetry unavailable'); + } + if ( + pgPoolTotalClients == null + || pgPoolIdleClients == null + || pgPoolWaitingClients == null + ) { + failures.push('physical PostgreSQL pool-client telemetry unavailable'); + } + if ( + !completeRuntimePoolTelemetry + || requestedMaxUsesValues.length !== 1 + || effectiveMaxUsesValues.length !== 1 + || runtimePoolExpectedPools !== fleetShape.apis + || runtimePoolObservedPools !== fleetShape.apis + ) { + failures.push( + `exact runtime PostgreSQL pool telemetry unavailable or inconsistent; observed=${runtimePoolObservedPools ?? 'unknown'}, expected=${fleetShape.apis}` + ); + } else if ( + runtimePoolRequestedMaxUses === 1 + && runtimePoolEffectiveMaxUses === 1 + && input.postWarmupSnapshots.some((snapshot) => + snapshot.runtimePoolIdleClients !== 0 + ) + ) { + failures.push( + 'runtime PostgreSQL maxUses=1 retained idle clients after warmup' + ); + } + if (expectedRealtimeApis > 0 && realtimeNotificationMode === 'shared-exact') { + const expectedBrokers = fleetShape.physicalDatabases; + const exactSharedRealtime = input.postWarmupSnapshots.length > 0 + && input.postWarmupSnapshots.every((snapshot) => + snapshot.realtimeNotificationMode === 'shared-exact' + && snapshot.notificationBrokers === expectedBrokers + && snapshot.notificationListenerConnections === expectedBrokers + && snapshot.notificationBrokerLeases === expectedRealtimeApis + && snapshot.notificationBrokerTopics === expectedRealtimeApis + && snapshot.notificationBrokerSubscribers === expectedRealtimeApis + && snapshot.notificationBrokerQueueOverflows === 0 + && snapshot.notificationBrokerFatalFailures === 0 + && snapshot.notificationAuditIdentities === expectedBrokers + && snapshot.notificationAuditsHealthy === expectedBrokers + && snapshot.notificationAuditsFailed === 0 + && snapshot.notificationAuditsStale === 0 + && snapshot.notificationAuditAttempts != null + && snapshot.notificationAuditAttempts >= expectedRealtimeApis + && snapshot.notificationAuditFailures === 0 + && snapshot.notificationAuditActiveDatabaseTargets === expectedBrokers + && snapshot.notificationAuditDatabaseConflicts === 0 + ); + if (!exactSharedRealtime) { + failures.push( + 'shared realtime broker residency or listener-role attestation is not exact' + ); + } + } else if (expectedRealtimeApis > 0) { + if (realtimeNotificationMode !== 'dedicated') { + failures.push('realtime notification mode telemetry unavailable or inconsistent'); + } + if (postgresBackendPeak != null && postgresBackendPeak < expectedRealtimeApis) { + failures.push( + `physical PostgreSQL backends=${postgresBackendPeak}, expected at least ${expectedRealtimeApis} dedicated realtime APIs` + ); + } + if (pgPoolTotalClients != null && pgPoolTotalClients < expectedRealtimeApis) { + failures.push( + `physical PostgreSQL pool clients=${pgPoolTotalClients}, expected at least ${expectedRealtimeApis} dedicated realtime APIs` + ); + } + } + } + if (expectedRealtimeApis > 0) { + const expectedManagers = minimumCompleteValue('realtimeManagersExpected'); + const expectedTransports = minimumCompleteValue('realtimeTransportsExpected'); + if ( + expectedManagers == null + || expectedManagers < expectedRealtimeApis + || residentRealtimeManagers == null + || residentRealtimeManagers < expectedManagers + ) { + failures.push( + `resident realtime managers=${residentRealtimeManagers ?? 'unknown'}, expected ${expectedManagers ?? expectedRealtimeApis}` + ); + } + if ( + expectedTransports == null + || expectedTransports < expectedRealtimeApis + || residentRealtimeTransports == null + || residentRealtimeTransports < expectedTransports + ) { + failures.push( + `resident realtime transports=${residentRealtimeTransports ?? 'unknown'}, expected ${expectedTransports ?? expectedRealtimeApis}` + ); + } + } + if (postWarmupPgPoolCapacityEvictions !== 0) { + failures.push( + `post-warmup PostgreSQL pool capacity evictions=${postWarmupPgPoolCapacityEvictions ?? 'unknown'}` + ); + } + if (postWarmupPgPoolCapacityRefusals !== 0) { + failures.push( + `post-warmup PostgreSQL pool capacity refusals=${postWarmupPgPoolCapacityRefusals ?? 'unknown'}` + ); + } + if (postWarmupPgPoolDisposalFailures !== 0) { + failures.push( + `post-warmup PostgreSQL pool disposal failures=${postWarmupPgPoolDisposalFailures ?? 'unknown'}` + ); + } + if (input.gates.requireRetainedMemoryCheckpoints) { + if (retainedGrowth.errors.length > 0) { + failures.push( + `retained-memory checkpoint errors: ${retainedGrowth.errors.join('; ')}` + ); + } + if (retainedGrowth.heapMiBPerHour == null) { + failures.push('retained heap growth could not be measured'); + } else if ( + retainedGrowth.heapMiBPerHour + > input.gates.maxPostWarmupHeapGrowthMiBPerHour + ) { + failures.push( + `retained heap growth ${retainedGrowth.heapMiBPerHour.toFixed(2)}MiB/hour exceeds ${input.gates.maxPostWarmupHeapGrowthMiBPerHour}` + ); + } + if (retainedGrowth.externalMiBPerHour == null) { + failures.push('retained external-memory growth could not be measured'); + } else if ( + retainedGrowth.externalMiBPerHour + > input.gates.maxPostWarmupHeapGrowthMiBPerHour + ) { + failures.push( + `retained external-memory growth ${retainedGrowth.externalMiBPerHour.toFixed(2)}MiB/hour exceeds ${input.gates.maxPostWarmupHeapGrowthMiBPerHour}` + ); + } + } else if (rawHeapGrowth == null) { + failures.push('post-warmup heap growth could not be measured'); + } else if ( + rawHeapGrowth > input.gates.maxPostWarmupHeapGrowthMiBPerHour + ) { + failures.push( + `heap growth ${rawHeapGrowth.toFixed(2)}MiB/hour exceeds ${input.gates.maxPostWarmupHeapGrowthMiBPerHour}` + ); + } + if (input.gates.requireConclusiveCanaries && input.canaries.some((canary) => !canary.conclusive)) failures.push('isolation canary was inconclusive'); + if (input.gates.requireZeroBleed && input.canaries.some((canary) => canary.violation)) failures.push('cross-tenant bleed detected'); + if (input.gates.requireConclusiveOperationOracles) { + if (operationOracleInconclusive > 0) { + failures.push( + `GraphQL operation response oracles inconclusive=${operationOracleInconclusive}` + ); + } + if (operationOracleViolations > 0) { + failures.push( + `GraphQL operation response oracle violations=${operationOracleViolations}` + ); + } + if (missingOperationOracles.length > 0) { + failures.push( + `missing GraphQL operation response oracles: ${shortList(missingOperationOracles)}` + ); + } + } + if (missingCapabilities.length > 0) failures.push(`missing capabilities: ${missingCapabilities.join(', ')}`); + if (missingCanaries.length > 0) failures.push(`missing canaries: ${missingCanaries.join(', ')}`); + if (tenantResults.some((tenant) => !tenant.qualified)) failures.push('one or more complete tenants failed qualification'); + if (input.serverExit) failures.push(`server exited code=${input.serverExit.code} signal=${input.serverExit.signal}`); + const realtimeFailures = validateRealtimeDeliveryCoverage(input); + if (realtimeFailures.length > 0) { + failures.push( + `deadline-bounded recurring realtime delivery coverage is incomplete: ${realtimeFailures.join('; ')}` + ); + } else if ( + input.realtimeDeliveryCoverage + && ( + input.realtimeDeliveryCoverage.deliveryP99Ms > input.gates.maxP99Ms + || input.realtimeDeliveryCoverage.primeResponseP99Ms > input.gates.maxP99Ms + ) + ) { + failures.push( + `realtime prime or delivery p99 exceeds ${input.gates.maxP99Ms}ms` + ); + } + if (input.externalServer) { + failures.push('external server reuse cannot qualify as a fresh-arm run'); + } + failures.push(...input.executionErrors.map((error) => `execution failed: ${error}`)); + + const heapValues = input.memorySnapshots + .map((snapshot) => snapshot.heapUsedBytes) + .filter((value): value is number => value != null); + const peakHeapBytes = heapValues.length === input.memorySnapshots.length && heapValues.length > 0 + ? Math.max(...heapValues) + : null; + const peakRssValues = input.memorySnapshots + .map((snapshot) => snapshot.processPeakRssBytes) + .filter((value): value is number => value != null); + const peakRssBytes = peakRssValues.length > 0 + ? Math.max(...peakRssValues) + : null; + const serviceMemoryUpperBoundPostgresBytes = completePostgresCgroupV2PeakTelemetry + ? postgresCgroupV2PeakBytes + : input.evidenceMode === 'diagnostic' + ? postgresPeakBytes + : null; + const serviceMemoryUpperBoundPostgresSource = completePostgresCgroupV2PeakTelemetry + ? 'cgroup-v2-memory.peak' as const + : input.evidenceMode === 'diagnostic' && postgresPeakBytes != null + ? 'sampled-current-diagnostic' as const + : null; + const serviceMemoryUpperBoundBytes = peakRssBytes != null + && serviceMemoryUpperBoundPostgresBytes != null + ? peakRssBytes + serviceMemoryUpperBoundPostgresBytes + : null; + const buildMaxValues = input.memorySnapshots + .map((snapshot) => snapshot.buildMaxMs) + .filter((value): value is number => value != null); + const buildMaxMs = buildMaxValues.length === input.memorySnapshots.length && buildMaxValues.length > 0 + ? Math.max(...buildMaxValues) + : null; + const elapsedSec = input.workloadDurationMs / 1000; + const heapLimits = [...new Set(input.memorySnapshots + .map((snapshot) => snapshot.heapLimitBytes) + .filter((value): value is number => value != null))]; + const observedHeapLimitBytes = heapLimits.length === 1 ? heapLimits[0] : null; + if (peakHeapBytes == null) failures.push('heap-used telemetry unavailable'); + if (peakRssBytes == null) failures.push('OS process peak RSS telemetry unavailable'); + if (input.evidenceMode === 'qualification' && serviceMemoryUpperBoundBytes == null) { + failures.push('conservative service-memory upper bound unavailable'); + } + if (observedHeapLimitBytes == null) failures.push('effective V8 heap limit telemetry unavailable or inconsistent'); + const globallyQualified = failures.length === 0; + const qualifiedCustomers = globallyQualified + ? tenantResults.filter((tenant) => tenant.qualified).length + : 0; + const dispatchedWorkloadRequests = workloadSamples.filter((sample) => + sample.phase === 'workload' + && sample.errorCode !== 'LOAD_GENERATOR_MISSED_ARRIVAL' + ).length; + const periodicValidationRequests = input.canaries.filter( + (canary) => canary.phase === 'periodic' + ).length; + const customerWorkloadRps = elapsedSec > 0 + ? dispatchedWorkloadRequests / elapsedSec + : 0; + const periodicValidationRps = elapsedSec > 0 + ? periodicValidationRequests / elapsedSec + : 0; + const realtimeValidationRps = elapsedSec > 0 + ? (input.realtimeDeliveryCoverage?.primeRequests ?? 0) / elapsedSec + : 0; + + return { + schemaVersion: 6, + runKind: input.runKind, + evidenceMode: input.evidenceMode, + campaignId: input.campaignId, + scheduleSha256: input.scheduleSha256, + previousResultPayloadSha256: input.previousResultPayloadSha256, + qualificationCohortSha256: input.qualificationCohortSha256, + arm: input.arm, + commit: input.commit ?? null, + introspectionMode: input.introspectionMode, + heapMiB: input.heapMiB, + configuredCustomers: input.tenants.length, + configuredTenants: input.tenants.length, + fleetShape, + repetition: input.repetition, + expectedMatrixRepetitions: input.expectedMatrixRepetitions, + runOrderSeed: input.runOrderSeed, + runOrderIndex: input.runOrderIndex, + startedAt: input.startedAt, + endedAt: input.endedAt, + durationSec: elapsedSec, + warmupMaxMs: percentile(input.warmupLatencies, 1), + resolvedWarmupTimeoutMs: input.resolvedWarmupTimeoutMs, + offeredLoad: input.offeredLoad, + requests: workloadSamples.length, + coverageRequests: coverageSamples.length, + workloadRequests: workloadSamples.length, + errors, + customerWorkloadRps, + periodicValidationRps, + realtimeValidationRps, + combinedHttpRps: + customerWorkloadRps + periodicValidationRps + realtimeValidationRps, + achievedRps: customerWorkloadRps, + missedArrivals: input.missedArrivals, + errorRate, + p50Ms: percentile(latencies, 0.5), + p95Ms: percentile(latencies, 0.95), + p99Ms: percentile(latencies, 0.99), + peakHeapBytes, + peakRssBytes, + observedHeapLimitBytes, + residentInstances, + expectedResidentInstances, + cacheConfiguredMax, + cacheBudgetCapacity, + cacheInstanceHeapBytes, + cacheCalibrationId, + cacheAdmissionMode, + warmObservedHeapDeltaPerInstanceBytes, + postWarmupHeapGrowthMiBPerHour: rawHeapGrowth, + rawPostWarmupHeapGrowthMiBPerHour: rawHeapGrowth, + retainedHeapGrowthMiBPerHour: retainedGrowth.heapMiBPerHour, + retainedExternalGrowthMiBPerHour: retainedGrowth.externalMiBPerHour, + retainedMemoryDurationSec: retainedGrowth.durationSec, + retainedHeapBaselineBytes: retainedGrowth.heapBaselineBytes, + retainedHeapFinalBytes: retainedGrowth.heapFinalBytes, + retainedExternalBaselineBytes: retainedGrowth.externalBaselineBytes, + retainedExternalFinalBytes: retainedGrowth.externalFinalBytes, + retainedMemoryCheckpointErrors: retainedGrowth.errors, + postWarmupEvictions: evictions, + postWarmupBuildRefusals: buildRefusals, + postWarmupBuilds, + pgPoolCacheSize, + pgPoolLeasedPools, + pgPoolActiveLeases, + postWarmupPgPoolCapacityEvictions, + postWarmupPgPoolCapacityRefusals, + postWarmupPgPoolDisposalFailures, + coldBuildMaxMs: buildMaxMs, + memorySampleErrors: input.memorySampleErrors, + postgresBaselineBytes, + postgresWarmBoundaryBytes, + postgresPeakBytes, + postgresWorkingSetPeakBytes, + postgresCgroupV2PeakBytes, + postgresCgroupV2Samples, + postgresOomEvents, + postgresBackendPeak, + residentPhysicalDatabases, + postgresContainerDedicated, + unexpectedPostgresDatabases, + pgPoolTotalClients, + pgPoolIdleClients, + pgPoolWaitingClients, + runtimePoolRequestedMaxUses, + runtimePoolEffectiveMaxUses, + runtimePoolExpectedPools, + runtimePoolObservedPools, + runtimePoolTotalClients, + runtimePoolIdleClients, + runtimePoolWaitingClients, + residentRealtimeManagers, + residentRealtimeTransports, + realtimeNotificationMode, + realtimeDeliveryCoverage: input.realtimeDeliveryCoverage, + notificationBrokers, + notificationListenerConnections, + notificationBrokerLeases, + notificationBrokerTopics, + notificationBrokerSubscribers, + notificationBrokerQueueOverflows, + notificationBrokerFatalFailures, + notificationAuditIdentities, + notificationAuditsHealthy, + notificationAuditsFailed, + notificationAuditsStale, + notificationAuditAttempts, + notificationAuditFailures, + notificationAuditActiveDatabaseTargets, + notificationAuditDatabaseConflicts, + postgresColdBuildSpikeBytes, + postgresSampleErrors: input.postgresSampleErrors, + alignedServicePeakBytes: alignedServicePeak?.bytes ?? null, + alignedServicePeakNodeRssBytes: alignedServicePeak?.nodeRssBytes ?? null, + alignedServicePeakPostgresBytes: alignedServicePeak?.postgresBytes ?? null, + alignedServicePeakTimestamp: alignedServicePeak?.timestamp ?? null, + alignedServiceMemorySamples: alignedServicePeak?.samples ?? 0, + alignedServiceMemoryMaxSkewMs: alignedServicePeak?.maxSkewMs ?? null, + alignedServiceMemoryCoverageRatio: alignedServiceCoverage?.coverageRatio ?? null, + alignedServiceMemoryCoveredDurationMs: alignedServiceCoverage?.coveredDurationMs ?? null, + alignedServiceMemoryExpectedDurationMs: alignedServiceCoverage?.expectedDurationMs ?? null, + alignedServiceMemoryMaxGapMs: alignedServiceCoverage?.maxGapMs ?? null, + serviceMemoryUpperBoundBytes, + serviceMemoryUpperBoundPostgresSource, + capabilitiesExercised, + missingCapabilities, + missingCanaries, + canarySchedule: input.canarySchedule, + canaryChecks: input.canaries.length, + canaryInconclusive: input.canaries.filter((canary) => !canary.conclusive).length, + bleedViolations: input.canaries.filter((canary) => canary.violation).length, + operationOracleChecks, + operationOracleInconclusive, + operationOracleViolations, + missingOperationOracles, + tenants: tenantResults, + qualifiedCustomers, + qualifiedTenants: qualifiedCustomers, + tenantsPerConfiguredOldSpaceGiB: qualifiedCustomers / (input.heapMiB / 1024), + tenantsPerPeakRssGiB: peakRssBytes && peakRssBytes > 0 + ? qualifiedCustomers / (peakRssBytes / GIB) + : null, + customersPerAlignedServiceGiB: alignedServicePeak && alignedServicePeak.bytes > 0 + ? qualifiedCustomers / (alignedServicePeak.bytes / GIB) + : null, + customersPerServiceMemoryUpperBoundGiB: + serviceMemoryUpperBoundBytes && serviceMemoryUpperBoundBytes > 0 + ? qualifiedCustomers / (serviceMemoryUpperBoundBytes / GIB) + : null, + configuredCustomersPerAlignedServiceGiB: + alignedServicePeak && alignedServicePeak.bytes > 0 + ? input.tenants.length / (alignedServicePeak.bytes / GIB) + : null, + configuredCustomersPerServiceMemoryUpperBoundGiB: + serviceMemoryUpperBoundBytes && serviceMemoryUpperBoundBytes > 0 + ? input.tenants.length / (serviceMemoryUpperBoundBytes / GIB) + : null, + accepted: globallyQualified, + failures, + serverExit: input.serverExit, + provenance: input.provenance, + provenanceErrors: input.provenanceErrors, + postgresRunAttestation: input.postgresRunAttestation ?? null, + artifactDir: input.artifactDir + }; +}; + +const median = (values: number[]): number => percentile(values, 0.5); + +export const summarizeCapacityBoundaries = ( + runs: DensityRunResult[] +): DensityCapacityBoundary[] => { + const groups = new Map(); + for (const run of runs) { + if (run.runKind === 'soak') continue; + const key = `${run.arm}\0${run.heapMiB}`; + const group = groups.get(key) ?? []; + group.push(run); + groups.set(key, group); + } + return [...groups.values()].map((group) => { + const { arm, heapMiB } = group[0]; + const expectedRepetitions = Math.max( + ...group.map((run) => run.expectedMatrixRepetitions ?? 1) + ); + const byCount = new Map(); + for (const run of group) { + const local = byCount.get(run.configuredTenants) ?? []; + local.push(run); + byCount.set(run.configuredTenants, local); + } + const testedTenantCounts = [...byCount.keys()].sort((a, b) => a - b); + const completeCounts = testedTenantCounts.filter((count) => { + const countRuns = byCount.get(count)!; + const repetitions = new Set(countRuns.map((run) => run.repetition)); + return countRuns.length === expectedRepetitions + && repetitions.size === expectedRepetitions + && countRuns.every((run) => run.evidenceMode === 'qualification') + && new Set(countRuns.map((run) => run.qualificationCohortSha256)).size === 1 + && countRuns.every((run) => run.expectedMatrixRepetitions === expectedRepetitions) + && Array.from( + { length: expectedRepetitions }, + (_unused, index) => index + 1 + ).every((repetition) => repetitions.has(repetition)); + }); + const incompleteTenantCounts = testedTenantCounts.filter( + (count) => !completeCounts.includes(count) + ); + const passingCounts = completeCounts.filter((count) => + byCount.get(count)!.every((run) => run.accepted) + ); + const highestAllRepetitionsPass = passingCounts.length > 0 + ? Math.max(...passingCounts) + : null; + const failingCounts = completeCounts.filter((count) => + byCount.get(count)!.some((run) => !run.accepted) + ); + const greaterFailures = highestAllRepetitionsPass == null + ? [] + : failingCounts.filter((count) => count > highestAllRepetitionsPass); + const monotonicQualification = highestAllRepetitionsPass == null + ? failingCounts.length === completeCounts.length + : failingCounts.every((count) => count > highestAllRepetitionsPass); + const lowestGreaterFail = greaterFailures.length > 0 + ? Math.min(...greaterFailures) + : null; + const boundaryRuns = highestAllRepetitionsPass == null + ? [] + : byCount.get(highestAllRepetitionsPass)!; + const peakRssDensities = boundaryRuns + .map((run) => run.tenantsPerPeakRssGiB) + .filter((value): value is number => value != null); + const alignedServiceDensities = boundaryRuns + .map((run) => run.customersPerAlignedServiceGiB) + .filter((value): value is number => value != null); + const serviceUpperBoundDensities = boundaryRuns + .map((run) => run.customersPerServiceMemoryUpperBoundGiB) + .filter((value): value is number => value != null); + return { + arm, + heapMiB, + expectedRepetitions, + testedTenantCounts, + incompleteTenantCounts, + highestAllRepetitionsPass, + lowestGreaterFail, + monotonicQualification, + capacityBoundaryReached: + highestAllRepetitionsPass != null + && lowestGreaterFail != null + && monotonicQualification + && incompleteTenantCounts.length === 0, + medianTenantsPerConfiguredOldSpaceGiB: boundaryRuns.length > 0 + ? median(boundaryRuns.map((run) => run.tenantsPerConfiguredOldSpaceGiB)) + : null, + medianTenantsPerPeakRssGiB: + boundaryRuns.length > 0 && peakRssDensities.length === boundaryRuns.length + ? median(peakRssDensities) + : null, + medianCustomersPerAlignedServiceGiB: + boundaryRuns.length > 0 && alignedServiceDensities.length === boundaryRuns.length + ? median(alignedServiceDensities) + : null, + medianCustomersPerServiceMemoryUpperBoundGiB: + boundaryRuns.length > 0 && serviceUpperBoundDensities.length === boundaryRuns.length + ? median(serviceUpperBoundDensities) + : null + }; + }).sort((a, b) => a.arm.localeCompare(b.arm) || a.heapMiB - b.heapMiB); +}; + +const relativeImprovement = (baseline: number, candidate: number): number => + baseline === 0 ? (candidate > 0 ? Infinity : 0) : (candidate - baseline) / baseline; + +const bracketedPassingCountForRepetition = ( + runs: DensityRunResult[], + arm: string, + heapMiB: number, + repetition: number +): number | null => { + const coordinate = runs.filter((run) => + run.runKind !== 'soak' + && run.arm === arm + && run.heapMiB === heapMiB + && run.repetition === repetition + ); + const byCount = new Map(); + for (const run of coordinate) { + if (byCount.has(run.configuredTenants)) return null; + if (run.evidenceMode !== 'qualification') return null; + byCount.set(run.configuredTenants, run); + } + const counts = [...byCount.keys()].sort((left, right) => left - right); + const passing = counts.filter((count) => byCount.get(count)!.accepted); + if (passing.length === 0) return null; + const highestPassing = Math.max(...passing); + if (counts.some((count) => count < highestPassing && !byCount.get(count)!.accepted)) { + return null; + } + const greaterFailures = counts.filter((count) => + count > highestPassing && !byCount.get(count)!.accepted + ); + return greaterFailures.length > 0 ? highestPassing : null; +}; + +export const compareDensity = ( + baseline: DensityRunResult[], + candidate: DensityRunResult[], + gates: AcceptanceGates +): { + materiallyBetter: boolean; + configuredOldSpaceMedianImprovement: number; + peakRssMedianImprovement: number; + alignedServiceMedianImprovement: number; + serviceMemoryUpperBoundMedianImprovement: number; + configuredOldSpaceNonRegression: boolean; + peakRssNonRegression: boolean; + alignedServiceNonRegression: boolean; + serviceMemoryUpperBoundNonRegression: boolean; + everyHeapAddsTenants: boolean; + capacityBoundariesComplete: boolean; + pairedMatrixComplete: boolean; + baselineBoundaries: DensityCapacityBoundary[]; + candidateBoundaries: DensityCapacityBoundary[]; +} => { + const baselineBoundaries = summarizeCapacityBoundaries(baseline); + const candidateBoundaries = summarizeCapacityBoundaries(candidate); + if (baseline.length === 0 || candidate.length === 0) { + return { + materiallyBetter: false, + configuredOldSpaceMedianImprovement: 0, + peakRssMedianImprovement: 0, + alignedServiceMedianImprovement: 0, + serviceMemoryUpperBoundMedianImprovement: 0, + configuredOldSpaceNonRegression: false, + peakRssNonRegression: false, + alignedServiceNonRegression: false, + serviceMemoryUpperBoundNonRegression: false, + everyHeapAddsTenants: false, + capacityBoundariesComplete: false, + pairedMatrixComplete: false, + baselineBoundaries, + candidateBoundaries + }; + } + const matrixKeys = (runs: DensityRunResult[]): string[] => runs + .filter((run) => run.runKind !== 'soak') + .map((run) => [ + run.qualificationCohortSha256, + run.evidenceMode, + run.heapMiB, + run.configuredTenants, + run.repetition + ].join(':')) + .sort(); + const pairedMatrixComplete = JSON.stringify(matrixKeys(baseline)) === JSON.stringify(matrixKeys(candidate)); + const baselineByHeap = new Map(baselineBoundaries.map((boundary) => [ + boundary.heapMiB, + boundary + ])); + const candidateByHeap = new Map(candidateBoundaries.map((boundary) => [ + boundary.heapMiB, + boundary + ])); + const pairedHeaps = [...candidateByHeap.keys()].filter((heap) => baselineByHeap.has(heap)); + const capacityBoundariesComplete = pairedMatrixComplete + && pairedHeaps.length === baselineByHeap.size + && pairedHeaps.length === candidateByHeap.size + && [...baselineBoundaries, ...candidateBoundaries].every( + (boundary) => boundary.capacityBoundaryReached + ); + const densityPairs = pairedHeaps.map((heap) => ({ + baseline: baselineByHeap.get(heap)!, + candidate: candidateByHeap.get(heap)! + })); + const completeDensityPairs = densityPairs.filter(({ baseline: prior, candidate: next }) => + prior.medianTenantsPerConfiguredOldSpaceGiB != null + && next.medianTenantsPerConfiguredOldSpaceGiB != null + && prior.medianTenantsPerPeakRssGiB != null + && next.medianTenantsPerPeakRssGiB != null + && prior.medianCustomersPerAlignedServiceGiB != null + && next.medianCustomersPerAlignedServiceGiB != null + && prior.medianCustomersPerServiceMemoryUpperBoundGiB != null + && next.medianCustomersPerServiceMemoryUpperBoundGiB != null + ); + const configuredImprovements = completeDensityPairs.map(({ baseline: prior, candidate: next }) => + relativeImprovement( + prior.medianTenantsPerConfiguredOldSpaceGiB!, + next.medianTenantsPerConfiguredOldSpaceGiB! + ) + ); + const peakRssImprovements = completeDensityPairs.map(({ baseline: prior, candidate: next }) => + relativeImprovement( + prior.medianTenantsPerPeakRssGiB!, + next.medianTenantsPerPeakRssGiB! + ) + ); + const alignedServiceImprovements = completeDensityPairs.map(({ + baseline: prior, + candidate: next + }) => relativeImprovement( + prior.medianCustomersPerAlignedServiceGiB!, + next.medianCustomersPerAlignedServiceGiB! + )); + const serviceMemoryUpperBoundImprovements = completeDensityPairs.map(({ + baseline: prior, + candidate: next + }) => relativeImprovement( + prior.medianCustomersPerServiceMemoryUpperBoundGiB!, + next.medianCustomersPerServiceMemoryUpperBoundGiB! + )); + const metricsComplete = completeDensityPairs.length === densityPairs.length + && densityPairs.length > 0; + const configuredOldSpaceMedianImprovement = metricsComplete + ? median(configuredImprovements) + : 0; + const peakRssMedianImprovement = metricsComplete ? median(peakRssImprovements) : 0; + const alignedServiceMedianImprovement = metricsComplete + ? median(alignedServiceImprovements) + : 0; + const serviceMemoryUpperBoundMedianImprovement = metricsComplete + ? median(serviceMemoryUpperBoundImprovements) + : 0; + const configuredOldSpaceNonRegression = metricsComplete + && configuredImprovements.every((improvement) => improvement >= 0); + const peakRssNonRegression = metricsComplete + && peakRssImprovements.every((improvement) => improvement >= 0); + const alignedServiceNonRegression = metricsComplete + && alignedServiceImprovements.every((improvement) => improvement >= 0); + const serviceMemoryUpperBoundNonRegression = metricsComplete + && serviceMemoryUpperBoundImprovements.every((improvement) => improvement >= 0); + const everyHeapAddsTenants = capacityBoundariesComplete + && densityPairs.every(({ baseline: prior, candidate: next }) => { + if (prior.expectedRepetitions !== next.expectedRepetitions) return false; + return Array.from( + { length: prior.expectedRepetitions }, + (_unused, index) => index + 1 + ).every((repetition) => { + const priorCapacity = bracketedPassingCountForRepetition( + baseline, + prior.arm, + prior.heapMiB, + repetition + ); + const nextCapacity = bracketedPassingCountForRepetition( + candidate, + next.arm, + next.heapMiB, + repetition + ); + return priorCapacity != null + && nextCapacity != null + && nextCapacity >= priorCapacity + gates.minAdditionalTenantsEveryRun; + }); + }); + return { + materiallyBetter: everyHeapAddsTenants + && alignedServiceNonRegression + && serviceMemoryUpperBoundNonRegression + && alignedServiceMedianImprovement >= gates.minMedianDensityImprovement + && serviceMemoryUpperBoundMedianImprovement >= gates.minMedianDensityImprovement, + configuredOldSpaceMedianImprovement, + peakRssMedianImprovement, + alignedServiceMedianImprovement, + serviceMemoryUpperBoundMedianImprovement, + configuredOldSpaceNonRegression, + peakRssNonRegression, + alignedServiceNonRegression, + serviceMemoryUpperBoundNonRegression, + everyHeapAddsTenants, + capacityBoundariesComplete, + pairedMatrixComplete, + baselineBoundaries, + candidateBoundaries + }; +}; diff --git a/packages/perf-harness/src/types.ts b/packages/perf-harness/src/types.ts new file mode 100644 index 0000000000..8b9a2fdb60 --- /dev/null +++ b/packages/perf-harness/src/types.ts @@ -0,0 +1,962 @@ +export type IntrospectionMode = 'stock' | 'scoped-required'; +export type CacheAdmissionMode = 'evict-idle' | 'preserve-resident'; +export type NodeV8Profile = + | 'stock' + | 'optimize-for-size' + | 'baseline-optimize-for-size' + | 'jitless-optimize-for-size'; + +export interface GraphqlResponseOracle { + /** Every match must be present in the response for the operation to count. */ + requiredMatches: JsonPathMatch[]; + /** Any match is an isolation violation, even when required matches are present. */ + forbiddenMatches: JsonPathMatch[]; + /** Exhaustive assertions over every value selected by a wildcard-capable pointer. */ + invariants?: JsonPathInvariant[]; +} + +export interface GraphqlPostCoverageVerification extends GraphqlResponseOracle { + query: string; + variables?: Record; + /** GraphQL variable name -> JSON pointer in the primary operation response. */ + variablesFromResponse?: Record; +} + +export interface GraphqlOperation { + name: string; + capability: string; + weight?: number; + query: string; + variables?: Record; + /** Optional fail-closed response oracle, evaluated for every invocation. */ + requiredMatches?: JsonPathMatch[]; + /** Must be configured together with requiredMatches. */ + forbiddenMatches?: JsonPathMatch[]; + /** Cardinality-bounded assertions evaluated for every invocation. */ + invariants?: JsonPathInvariant[]; + /** + * An untimed verification query run after this operation during coverage. + * This is for mutations whose production payload cannot echo a database- + * stamped value; it is never injected into the production GraphQL API. + */ + postCoverageVerification?: GraphqlPostCoverageVerification; +} + +export type JsonValue = + | null + | boolean + | number + | string + | JsonValue[] + | { [key: string]: JsonValue }; + +/** RFC 6901 JSON pointer; `*` may select every array/object child. */ +export interface JsonPathMatch { + path: string; + value: JsonValue; +} + +/** + * Every selected value must equal `everyEquals`, and the selection cardinality + * must stay within the inclusive bounds. A positive `min` makes empty + * collection responses fail closed instead of vacuously passing. + */ +export interface JsonPathInvariant { + path: string; + everyEquals: JsonValue; + min: number; + max?: number; +} + +export interface IsolationCanary { + name: string; + query: string; + variables?: Record; + forbiddenMatches: JsonPathMatch[]; + requiredMatches: JsonPathMatch[]; + invariants?: JsonPathInvariant[]; +} + +export interface RealtimeGraphqlOperation { + query: string; + variables?: Record; + /** Every match must be present for the response/event to be conclusive. */ + requiredMatches: JsonPathMatch[]; + /** Any match is an isolation failure. */ + forbiddenMatches: JsonPathMatch[]; +} + +export type RealtimeSubscriptionOperation = RealtimeGraphqlOperation; + +export interface RealtimeRoundTripCorrelation { + /** Top-level GraphQL variable replaced with a fresh opaque nonce per round. */ + primeVariable: string; + /** Exact JSON pointer where the prime mutation must return that nonce. */ + primeResponsePath: string; + /** Exact JSON pointer where the subscription event must return that nonce. */ + subscriptionEventPath: string; +} + +export interface RealtimeProbe { + subscription: RealtimeSubscriptionOperation; + prime: RealtimeGraphqlOperation; + /** Proves every delivery round was caused by its own fresh mutation. */ + correlation: RealtimeRoundTripCorrelation; + /** + * Header name -> environment variable name. This lets the driver authenticate + * both the HTTP prime and WebSocket upgrade without serializing credentials + * into a fleet or artifact. + */ + headersFromEnvironment?: Record; +} + +export interface GraphqlSurface { + name: string; + /** Default opaque identity when every arm is expected to produce the same build. */ + buildContract: string; + /** + * Exact opaque identity by arm name. When present, every configured arm must + * have an entry and the default is never used for that arm. + */ + buildContracts?: Record; + url: string; + headers?: Record; + warmup: GraphqlOperation; + operations: GraphqlOperation[]; + canaries: IsolationCanary[]; + /** Driver-owned subscription used to prove this exact surface stays live. */ + realtime?: RealtimeProbe; +} + +export interface CustomerApiTopology { + /** Stable control-plane API identity, never a host/service routing label. */ + id: string; + /** Opaque credential-sensitive runtime pool identity. */ + runtimePoolIdentity: string; + /** Exact opaque pool identity by arm when arms use different credentials. */ + runtimePoolIdentities?: Record; + /** Ordered physical schemas compiled into this exact API build. */ + physicalSchemas: string[]; + /** Host/service labels are reported for routing coverage, never isolation. */ + routingLabels: string[]; + /** Whether qualification must keep a realtime transport resident for this API. */ + realtime: boolean; + /** Names from TenantTarget.surfaces served by this API. */ + surfaces: string[]; +} + +export interface CustomerDatabaseTopology { + /** Stable logical database identity used by the Graphile build contract. */ + id: string; + /** Credential-free physical database label for fleet-shape accounting. */ + physicalDatabase: string; + apis: CustomerApiTopology[]; +} + +export interface TenantTarget { + id: string; + /** + * Explicit customer -> logical database -> API mapping. Legacy diagnostic + * fleets may omit it, but a qualifying plan can require it fail-closed. + */ + databases?: CustomerDatabaseTopology[]; + surfaces: GraphqlSurface[]; +} + +export interface FleetV1 { + version: 1; + tenants: TenantTarget[]; + /** Populated by loadFleet; not part of the fleet JSON contract. */ + sourceSha256?: string; +} + +export interface CustomerFleetShape { + topologyComplete: boolean; + customers: number; + logicalDatabases: number; + physicalDatabases: number; + apis: number; + realtimeApis: number; + surfaces: number; + physicalSchemaBindings: number; + routingLabels: number; + uniqueBuildContracts: number; + uniqueRuntimePoolIdentities: number; +} + +export interface ArmPlan { + name: 'origin-main' | 'runtime-boundary-stock' | 'cache-governor-stock' | 'scoped-introspection' | string; + commit?: string; + cwd?: string; + command?: string[]; + port: number; + readinessUrl: string; + memoryUrl: string; + /** + * Authenticated loopback endpoint that performs a benchmark-only full-GC + * checkpoint. Spawned qualification arms must expose this explicitly. + */ + retainedHeapCheckpointUrl?: string; + /** + * Optional authenticated loopback hook invoked after every configured + * surface has warmed, but before post-warmup memory accounting starts. + * Physical-density fixtures use this to assert server-side realtime residency. + */ + postWarmupUrl?: string; + /** Dedicated PostgreSQL container used for cold-build memory telemetry. */ + postgresContainer?: string; + /** + * Outside-process live database/ACL audit bound to one fresh container and + * full matrix coordinate. An optional prepare command must create that run's + * container/clone after the harness establishes its not-before boundary. + */ + postgresRunAttestation?: { + command: string[]; + prepareCommand: string[]; + timeoutMs?: number; + }; + /** Fail the run unless raw cgroup-v2 PostgreSQL memory telemetry is present. */ + requirePostgresCgroupV2?: boolean; + introspectionMode: IntrospectionMode; + /** Closed set of benchmarked V8 flag combinations; defaults to stock. */ + v8Profile?: NodeV8Profile; + env?: Record; + /** + * Heap-specific environment overrides. This is intentionally explicit in + * the plan so governor calibration cannot silently reuse one reserve across + * materially different V8 pressure points. + */ + envByHeapMiB?: Record>; + startupTimeoutMs?: number; + /** Optional pin for the built JavaScript entry executed by command. */ + entrySha256?: string; + /** Optional pin for the workspace pnpm-lock.yaml. */ + lockfileSha256?: string; +} + +export interface AcceptanceGates { + maxErrorRate: number; + maxP99Ms: number; + maxPostWarmupHeapGrowthMiBPerHour: number; + minMedianDensityImprovement: number; + minAdditionalTenantsEveryRun: number; + /** Maximum uncovered boundary or internal gap in aligned service-memory telemetry. */ + maxAlignedMemorySampleGapMs?: number; + /** Minimum fraction of the post-warm workload covered by aligned samples. */ + minAlignedMemoryCoverageRatio?: number; + requireZeroBleed: boolean; + requireNoPostWarmupEvictions: boolean; + requireNoPostWarmupBuildRefusals: boolean; + requireNoPostWarmupBuilds: boolean; + requirePostgresMemoryTelemetry: boolean; + /** Require a unique, fresh, live-audited PostgreSQL epoch for every run. */ + requireFreshPostgresRunAttestation: boolean; + /** Require authenticated forced-GC bookends for retained-memory gating. */ + requireRetainedMemoryCheckpoints: boolean; + /** Require physical database, backend, and concrete pg.Pool client counts. */ + requirePhysicalDatabaseTelemetry: boolean; + requireConclusiveCanaries: boolean; + /** + * Require exact initial/final sweeps plus complete, deadline-bounded + * periodic coverage of every configured canary. + */ + requireCompletePeriodicCanaryCoverage: boolean; + /** Require exact response or post-coverage evidence for every operation. */ + requireConclusiveOperationOracles: boolean; + requireExplicitCustomerTopology: boolean; + /** Require the live cache and pinned process environment to use this mode. */ + requiredCacheAdmissionMode: CacheAdmissionMode | null; +} + +export interface DensityQualificationPlan { + /** Arm used as the denominator for every configured candidate comparison. */ + baselineArm: string; + /** Mandatory curve checkpoints; extra configured checkpoints are permitted. */ + requiredHeapMiB: number[]; + /** Every configured matrix point must contain at least this many repetitions. */ + minimumRepetitions: number; + /** + * One real induced-hostile report for every exact arm runtime/configuration. + * Passive GraphQL identity probes do not satisfy this publication boundary. + */ + hostileValidationEvidence?: Record; +} + +export interface ExactHostileValidationBinding { + version: 1; + kind: 'exact-runtime-hostile-validation-v1'; + artifactFile: string; + /** SHA-256 over the exact artifact bytes, without a prefix. */ + artifactSha256: string; + runtimeArtifactFingerprint: string; + configurationFingerprint: string; +} + +export type PeriodicCanarySchedule = 'full-sweep' | 'rotating-one'; + +export interface WorkloadPlan { + durationSec: number; + /** Fixed process-wide offered load. Mutually exclusive with rpsPerTenant. */ + rps?: number; + /** Offered load multiplied by the number of tenants in this run. */ + rpsPerTenant?: number; + /** Every surface must receive at least this many workload-phase requests. */ + minWorkloadRequestsPerSurface: number; + requestTimeoutMs: number; + maxInFlight: number; + canaryIntervalSec: number; + /** Defaults to the legacy full-fleet/full-canary periodic sweep. */ + periodicCanarySchedule?: PeriodicCanarySchedule; + /** Parallelism across surfaces; probes within one surface stay sequential. */ + canaryConcurrency?: number; + /** Minimum whole-fleet warmup allowance. */ + warmupTimeoutMs: number; + /** Additional scaling budget, applied once per warmup-concurrency wave. */ + warmupTimeoutPerSurfaceMs: number; + /** Maximum simultaneous schema warmups; defaults to one. */ + warmupConcurrency?: number; +} + +export interface DensityPlanV1 { + version: 1; + fleetFile: string; + artifactDir: string; + arms: ArmPlan[]; + heapMiB: number[]; + /** Legacy count ramp used for every heap unless a heap-specific ramp exists. */ + tenantCounts?: number[]; + /** Heap-specific ramps, keyed by configured old-space MiB. */ + tenantCountsByHeapMiB?: Record; + repetitions: number; + /** Reproducible arm interleaving seed. */ + runOrderSeed?: string; + requiredCapabilities: string[]; + requiredCanaries: string[]; + workload: WorkloadPlan; + gates: AcceptanceGates; + /** Omit for diagnostic-only plans that cannot make a qualification claim. */ + qualification?: DensityQualificationPlan; + soak?: { + enabled: boolean; + /** Candidate arm to soak; defaults to scoped-introspection for compatibility. */ + arm?: string; + durationSec: number; + tenantCount: number; + heapMiB: number; + }; + /** Populated by loadPlan; not part of the plan JSON contract. */ + sourceSha256?: string; +} + +export interface RequestSample { + tenantId: string; + surface: string; + operation: string; + capability: string; + latencyMs: number; + status: number; + ok: boolean; + phase: 'coverage' | 'workload'; + scheduledAtMs?: number; + errorCode?: string; + /** True when this request was checked against a configured response oracle. */ + oracleConfigured?: boolean; + /** True only when every required match was observed. */ + oracleConclusive?: boolean; + /** True when at least one forbidden match was observed. */ + oracleViolation?: boolean; + /** Transport/HTTP/GraphQL failure prevented evidence evaluation. */ + oracleUnavailable?: boolean; + /** Whether an untimed post-coverage side-effect verification was used. */ + postCoverageVerification?: boolean; +} + +export interface CanaryResult { + tenantId: string; + surface: string; + canary: string; + phase: 'initial' | 'periodic' | 'final'; + /** One-based planned round number, present only for periodic probes. */ + periodicRound?: number; + scheduledAt: string; + startedAt: string; + completedAt: string; + latencyMs: number; + conclusive: boolean; + violation: boolean; + detail?: string; +} + +export interface CanaryRoundSummary { + /** One-based planned round number. */ + periodicRound: number; + plannedAt: string; + startedAt: string | null; + completedAt: string | null; + targetsPlanned: number; + targetsStarted: number; + targetsCompleted: number; + checksPlanned: number; + checksStarted: number; + checksCompleted: number; + /** The preceding serialized round was still running at this round's slot. */ + overlapped: boolean; + /** This round completed after the workload deadline. */ + deadlineLate: boolean; + startDelayMs: number | null; + durationMs: number | null; +} + +export interface CanaryScheduleSummary { + schedule: PeriodicCanarySchedule; + intervalMs: number; + durationMs: number; + canaryConcurrency: number; + startedAt: string; + deadlineAt: string; + /** Round counts. */ + planned: number; + started: number; + completed: number; + missed: number; + overlapped: number; + deadlineLate: number; + /** Probe counts across periodic rounds only. */ + checksPlanned: number; + checksStarted: number; + checksCompleted: number; + rounds: CanaryRoundSummary[]; +} + +export interface MemorySnapshot { + timestamp: string; + pid: number | null; + nodeEnv: string | null; + heapLimitBytes: number | null; + heapUsedBytes: number | null; + rssBytes: number | null; + processPeakRssBytes: number | null; + cacheSize: number | null; + cacheConfiguredMax?: number | null; + cacheBudgetCapacity?: number | null; + cacheInstanceHeapBytes?: number | null; + cacheCalibrationId?: string | null; + cacheAdmissionMode?: CacheAdmissionMode | null; + /** Stable credential-free evidence for cross-process fleet comparison. */ + residentBuildContractFingerprints?: string[] | null; + /** Process-local keyed identities used only for same-process accounting. */ + residentBuildContracts: string[] | null; + evictions: number | null; + buildRefusals: number | null; + buildsStarted: number | null; + buildsSucceeded: number | null; + buildMaxMs: number | null; + pgPoolCacheSize: number | null; + pgPoolLeasedPools: number | null; + pgPoolActiveLeases: number | null; + pgPoolCapacityEvictions: number | null; + pgPoolCapacityRefusals: number | null; + pgPoolDisposalFailures: number | null; + pgPoolTotalClients?: number | null; + pgPoolIdleClients?: number | null; + pgPoolWaitingClients?: number | null; + runtimePoolTelemetryScope?: 'runtime-only-exact-identities' | null; + runtimePoolTelemetryAvailable?: boolean | null; + runtimePoolRequestedMaxUses?: number | null; + runtimePoolEffectiveMaxUses?: number | null; + runtimePoolEffectiveMaxUsesKnown?: boolean | null; + runtimePoolMaxUsesExact?: boolean | null; + runtimePoolExpectedPools?: number | null; + runtimePoolObservedPools?: number | null; + runtimePoolTotalClients?: number | null; + runtimePoolIdleClients?: number | null; + runtimePoolWaitingClients?: number | null; + postgresBackendTotal?: number | null; + postgresBackendActive?: number | null; + postgresBackendIdle?: number | null; + postgresBackendIdleInTransaction?: number | null; + physicalDatabases?: number | null; + postgresContainerDedicated?: boolean | null; + unexpectedPostgresDatabases?: number | null; + realtimeManagersExpected?: number | null; + realtimeManagersActive?: number | null; + realtimeTransportsExpected?: number | null; + realtimeTransportsActive?: number | null; + realtimeNotificationMode?: 'dedicated' | 'shared-exact' | null; + notificationBrokers?: number | null; + notificationListenerConnections?: number | null; + notificationBrokerLeases?: number | null; + notificationBrokerTopics?: number | null; + notificationBrokerSubscribers?: number | null; + notificationBrokerQueueOverflows?: number | null; + notificationBrokerFatalFailures?: number | null; + notificationAuditIdentities?: number | null; + notificationAuditsHealthy?: number | null; + notificationAuditsFailed?: number | null; + notificationAuditsStale?: number | null; + notificationAuditAttempts?: number | null; + notificationAuditFailures?: number | null; + notificationAuditActiveDatabaseTargets?: number | null; + notificationAuditDatabaseConflicts?: number | null; + cacheCountersAvailable: boolean; + buildCountersAvailable: boolean; + raw?: unknown; +} + +/** + * High-frequency, harness-timestamped current RSS sample for the exact server + * PID. Linux reads /proc; other hosts use the authenticated memory endpoint. + */ +export interface NodeRssSnapshot { + timestamp: string; + /** Exact child PID read by the harness. */ + pid: number; + source: 'proc' | 'authenticated-endpoint'; + rssBytes: number; +} + +export interface RetainedMemorySample { + timestamp: string; + /** Monotonic process time serialized as decimal nanoseconds. */ + monotonicNs: string; + heapUsedBytes: number; + externalBytes: number; + arrayBuffersBytes: number; + rssBytes: number; +} + +export interface RetainedMemoryGuard { + pid: number; + graphileInFlight: number; + residentBuildContracts: string[]; + stateSha256: string; + /** Credential-free residency and monotonic process-counter state. */ + state: Record; +} + +export interface RetainedMemoryCheckpoint { + version: 1; + fixture: string; + pid: number; + gcRounds: number; + stableSampleCount: number; + stable: boolean; + samples: RetainedMemorySample[]; + guardBefore: RetainedMemoryGuard; + guardAfter: RetainedMemoryGuard; + errors: string[]; +} + +export interface RetainedMemoryCheckpointPair { + baseline: RetainedMemoryCheckpoint | null; + final: RetainedMemoryCheckpoint | null; + errors: string[]; +} + +export interface PostgresMemorySnapshot { + timestamp: string; + /** Immutable 64-character Docker ID sampled for this record. */ + containerId?: string; + /** Attested container cgroup identity revalidated by the sampler. */ + cgroupIdentitySha256?: string; + /** Raw cgroup-v2 charge when available; Docker working set otherwise. */ + usedBytes: number; + limitBytes: number; + source?: 'cgroup-v2' | 'docker-stats'; + workingSetBytes?: number; + sampleStartedAt?: string; + sampleEndedAt?: string; + sampleDurationMs?: number; + cgroupV2?: { + currentBytes: number; + peakBytes: number | null; + maxBytes: number | null; + stat: Record; + events: Record; + }; + raw: string; +} + +export interface SurfaceResult { + surface: string; + warmed: boolean; + workloadRequests: number; + successfulWorkloadRequests: number; + errors: number; + errorRate: number; + p99Ms: number; + operationsConfigured: number; + operationsExercised: number; + canaryChecks: number; + canaryInconclusive: number; + bleedViolations: number; + operationOracleChecks: number; + operationOracleInconclusive: number; + operationOracleViolations: number; + missingOperations: string[]; + missingCapabilities: string[]; + missingOperationOracles: string[]; + qualified: boolean; +} + +export interface TenantResult { + tenantId: string; + surfacesConfigured: number; + surfacesWarmed: number; + surfacesWithTraffic: number; + operationsConfigured: number; + operationsExercised: number; + requests: number; + errors: number; + errorRate: number; + p99Ms: number; + canaryChecks: number; + canaryInconclusive: number; + bleedViolations: number; + operationOracleChecks: number; + operationOracleInconclusive: number; + operationOracleViolations: number; + missingSurfaces: string[]; + missingOperations: string[]; + missingCapabilities: string[]; + missingOperationOracles: string[]; + surfaces: SurfaceResult[]; + qualified: boolean; +} + +export interface ResolvedOfferedLoad { + mode: 'fixed-total' | 'per-tenant'; + configuredRps: number; + tenantCount: number; + totalRps: number; + rpsPerTenant: number; +} + +export interface RealtimeDeliverySurfaceCoverage { + tenantId: string; + surface: string; + route: string; + expectedRecurringRounds: number; + startedRecurringRounds: number; + verifiedRecurringRounds: number; + issuedCorrelationSha256: string; + verifiedCorrelationSha256: string; + primeRequests: number; + primeResponseP99Ms: number; + deliveryP99Ms: number; +} + +export interface RealtimeDeliveryCoverage { + version: 2; + deliveryIntervalMs: number; + workloadStartedAt: string; + workloadDeadlineAt: string; + workloadEndedAt: string | null; + expectedRecurringRounds: number; + startedRecurringRounds: number; + verifiedRecurringRounds: number; + deadlineLateRecurringRounds: number; + primeRequests: number; + primeResponseP99Ms: number; + deliveryP99Ms: number; + complete: boolean; + surfaces: RealtimeDeliverySurfaceCoverage[]; +} + +/** Credential-free proof that one fresh prime nonce reached one subscription. */ +export interface RealtimeCorrelationReceipt { + sequence: number; + timed: boolean; + deadlineAt: string; + issuedAt: string; + issuedSha256: string; + primeResponseAt: string | null; + primeResponseSha256: string | null; + eventAt: string | null; + eventSha256: string | null; +} + +export interface DensityResultEvidenceBinding { + version: 2; + algorithm: 'sha256'; + resultPayloadSha256: string; + artifacts: Array<{ + name: string; + sha256: string; + }>; +} + +export interface ResolvedMemoryPolicy { + configuredMaxOldSpaceMiB: number; + expectedV8HeapLimitBytes: number | null; + graphileCacheMax: string | null; + graphileCacheInstanceHeapBytes: string | null; + graphileCacheServerReserveBytes: string | null; + graphileCacheBuildReserveBytes: string | null; + graphileCacheRssLimitBytes: string | null; + graphileCacheRssBuildReserveBytes: string | null; + graphileCacheCalibrationId: string | null; + graphileCacheAdmissionMode: string | null; + graphileBuildMaxConcurrency: string | null; +} + +export interface ArmProvenance { + cwd: string | null; + command: string[]; + gitHead: string | null; + worktreeDirty: boolean | null; + gitStatusSha256: string | null; + lockfilePath: string | null; + lockfileSha256: string | null; + entryPath: string | null; + entrySha256: string | null; + serverPid: number | null; + /** Named allowlisted profile from the plan. */ + v8Profile: NodeV8Profile; + /** Exact sanitized NODE_OPTIONS string installed for the child. */ + nodeOptions: string | null; + /** Exact tokenization of nodeOptions. */ + nodeOptionsArgv: string[]; + /** Exact direct Node flags before the executed entry file. */ + nodeExecArgv: string[]; + /** NODE_OPTIONS followed by direct flags, in effective precedence order. */ + effectiveNodeRuntimeFlags: string[]; + planSha256: string | null; + fleetSha256: string | null; + node: string; + v8: string; + platform: NodeJS.Platform; + architecture: string; + runOrderSeed: string | null; + runOrderIndex: number | null; + memoryPolicy: ResolvedMemoryPolicy | null; +} + +export interface PostgresRunAttestationEvidence { + version: 1; + kind: 'physical-density-measurement-attestation-v1'; + artifactPath: string; + artifactSha256: string; + payloadSha256: string; + epochId: string; + arm: string; + heapMiB: number; + tenantCount: number; + repetition: number; + runOrderIndex: number; + planSha256: string; + fleetSha256: string; + containerId: string; + containerStartedAt: string; + cgroupIdentitySha256: string; + containerConfigurationSha256: string; + postgresSystemIdentifier: string; + postgresStartedAt: string; + cloneId: string; + cloneAttestationSetSha256: string; + cloneNonceSetSha256: string; + liveContractSetSha256: string; + manifestSha256: string; + containerTemplateSha256: string; + canonicalDatabaseContractFingerprint: string; + freshContainerForRun: boolean; + cgroupV2Verified: boolean; + liveCustomerContractsAudited: number; + /** Full pre-run audit intentionally warms PostgreSQL's catalogs. */ + catalogCacheState: 'warmed-by-live-contract-audit'; +} + +export interface DensityRunResult { + schemaVersion: 6; + runKind: 'matrix' | 'soak'; + /** Only a full, unmodified configured matrix may carry qualification evidence. */ + evidenceMode: 'qualification' | 'diagnostic'; + /** Random per-invocation campaign identity; never derived from plan bytes. */ + campaignId: string; + /** Hash of the exact ordered schedule manifest for this invocation. */ + scheduleSha256: string; + /** Hash-chain pointer to the prior result payload in exact run order. */ + previousResultPayloadSha256: string | null; + /** SHA-256 over the exact plan and fleet byte identities. */ + qualificationCohortSha256: string; + arm: string; + commit: string | null; + introspectionMode: IntrospectionMode; + heapMiB: number; + /** Complete customer bundles selected from the explicit fleet manifest. */ + configuredCustomers: number; + /** @deprecated Compatibility alias for configuredCustomers. */ + configuredTenants: number; + fleetShape: CustomerFleetShape; + repetition: number; + expectedMatrixRepetitions: number; + runOrderSeed: string; + runOrderIndex: number; + startedAt: string; + endedAt: string; + durationSec: number; + warmupMaxMs: number; + resolvedWarmupTimeoutMs: number; + offeredLoad: ResolvedOfferedLoad; + requests: number; + coverageRequests: number; + workloadRequests: number; + errors: number; + /** Dispatched customer workload requests divided by measured load duration. */ + customerWorkloadRps: number; + /** Completed periodic isolation probes divided by measured load duration. */ + periodicValidationRps: number; + /** Timed realtime prime mutations divided by measured load duration. */ + realtimeValidationRps: number; + /** Customer workload plus periodic and realtime validation HTTP requests per second. */ + combinedHttpRps: number; + /** @deprecated Compatibility alias for customerWorkloadRps. */ + achievedRps: number; + missedArrivals: number; + errorRate: number; + p50Ms: number; + p95Ms: number; + p99Ms: number; + peakHeapBytes: number | null; + peakRssBytes: number | null; + observedHeapLimitBytes: number | null; + residentInstances: number | null; + expectedResidentInstances: number; + cacheConfiguredMax: number | null; + cacheBudgetCapacity: number | null; + cacheInstanceHeapBytes: number | null; + cacheCalibrationId: string | null; + cacheAdmissionMode: CacheAdmissionMode | null; + warmObservedHeapDeltaPerInstanceBytes: number | null; + /** Raw heapUsed OLS trend; diagnostic only because normal GC is sawtoothed. */ + postWarmupHeapGrowthMiBPerHour: number | null; + rawPostWarmupHeapGrowthMiBPerHour: number | null; + retainedHeapGrowthMiBPerHour: number | null; + retainedExternalGrowthMiBPerHour: number | null; + retainedMemoryDurationSec: number | null; + retainedHeapBaselineBytes: number | null; + retainedHeapFinalBytes: number | null; + retainedExternalBaselineBytes: number | null; + retainedExternalFinalBytes: number | null; + retainedMemoryCheckpointErrors: string[]; + postWarmupEvictions: number | null; + postWarmupBuildRefusals: number | null; + postWarmupBuilds: number | null; + pgPoolCacheSize: number | null; + pgPoolLeasedPools: number | null; + pgPoolActiveLeases: number | null; + postWarmupPgPoolCapacityEvictions: number | null; + postWarmupPgPoolCapacityRefusals: number | null; + postWarmupPgPoolDisposalFailures: number | null; + coldBuildMaxMs: number | null; + memorySampleErrors: string[]; + postgresBaselineBytes: number | null; + postgresWarmBoundaryBytes: number | null; + postgresPeakBytes: number | null; + postgresWorkingSetPeakBytes: number | null; + postgresCgroupV2PeakBytes: number | null; + postgresCgroupV2Samples: number; + postgresOomEvents: number | null; + postgresBackendPeak: number | null; + residentPhysicalDatabases: number | null; + postgresContainerDedicated: boolean | null; + unexpectedPostgresDatabases: number | null; + pgPoolTotalClients: number | null; + pgPoolIdleClients: number | null; + pgPoolWaitingClients: number | null; + runtimePoolRequestedMaxUses: number | null; + runtimePoolEffectiveMaxUses: number | null; + runtimePoolExpectedPools: number | null; + runtimePoolObservedPools: number | null; + runtimePoolTotalClients: number | null; + runtimePoolIdleClients: number | null; + runtimePoolWaitingClients: number | null; + residentRealtimeManagers: number | null; + residentRealtimeTransports: number | null; + realtimeNotificationMode: 'dedicated' | 'shared-exact' | null; + /** Deadline-bounded fresh-event coverage during the timed workload. */ + realtimeDeliveryCoverage?: RealtimeDeliveryCoverage | null; + notificationBrokers: number | null; + notificationListenerConnections: number | null; + notificationBrokerLeases: number | null; + notificationBrokerTopics: number | null; + notificationBrokerSubscribers: number | null; + notificationBrokerQueueOverflows: number | null; + notificationBrokerFatalFailures: number | null; + notificationAuditIdentities: number | null; + notificationAuditsHealthy: number | null; + notificationAuditsFailed: number | null; + notificationAuditsStale: number | null; + notificationAuditAttempts: number | null; + notificationAuditFailures: number | null; + notificationAuditActiveDatabaseTargets: number | null; + notificationAuditDatabaseConflicts: number | null; + postgresColdBuildSpikeBytes: number | null; + postgresSampleErrors: string[]; + /** Maximum near-simultaneous Node current RSS + PostgreSQL cgroup usage. */ + alignedServicePeakBytes: number | null; + alignedServicePeakNodeRssBytes: number | null; + alignedServicePeakPostgresBytes: number | null; + alignedServicePeakTimestamp: string | null; + alignedServiceMemorySamples: number; + alignedServiceMemoryMaxSkewMs: number | null; + alignedServiceMemoryCoverageRatio?: number | null; + alignedServiceMemoryCoveredDurationMs?: number | null; + alignedServiceMemoryExpectedDurationMs?: number | null; + alignedServiceMemoryMaxGapMs?: number | null; + /** Conservative non-simultaneous upper bound: Node RSS HWM + PostgreSQL peak. */ + serviceMemoryUpperBoundBytes: number | null; + serviceMemoryUpperBoundPostgresSource?: + | 'cgroup-v2-memory.peak' + | 'sampled-current-diagnostic' + | null; + capabilitiesExercised: string[]; + missingCapabilities: string[]; + missingCanaries: string[]; + canarySchedule: CanaryScheduleSummary | null; + canaryChecks: number; + canaryInconclusive: number; + bleedViolations: number; + operationOracleChecks: number; + operationOracleInconclusive: number; + operationOracleViolations: number; + missingOperationOracles: string[]; + tenants: TenantResult[]; + qualifiedCustomers: number; + /** @deprecated Compatibility alias for qualifiedCustomers. */ + qualifiedTenants: number; + tenantsPerConfiguredOldSpaceGiB: number; + tenantsPerPeakRssGiB: number | null; + customersPerAlignedServiceGiB: number | null; + customersPerServiceMemoryUpperBoundGiB: number | null; + /** Diagnostic only: configured customers divided by aligned service memory. */ + configuredCustomersPerAlignedServiceGiB: number | null; + /** Diagnostic only: configured customers divided by the service upper bound. */ + configuredCustomersPerServiceMemoryUpperBoundGiB: number | null; + accepted: boolean; + failures: string[]; + serverExit: { code: number | null; signal: NodeJS.Signals | null } | null; + provenance: ArmProvenance | null; + provenanceErrors: string[]; + postgresRunAttestation?: PostgresRunAttestationEvidence | null; + /** Hash binding over the complete result payload and persisted raw evidence. */ + evidenceBinding?: DensityResultEvidenceBinding; + artifactDir: string; +} + +export interface DensityCapacityBoundary { + arm: string; + heapMiB: number; + expectedRepetitions: number; + testedTenantCounts: number[]; + incompleteTenantCounts: number[]; + highestAllRepetitionsPass: number | null; + lowestGreaterFail: number | null; + /** False when a lower tenant count failed but a higher count passed. */ + monotonicQualification: boolean; + capacityBoundaryReached: boolean; + medianTenantsPerConfiguredOldSpaceGiB: number | null; + medianTenantsPerPeakRssGiB: number | null; + medianCustomersPerAlignedServiceGiB: number | null; + medianCustomersPerServiceMemoryUpperBoundGiB: number | null; +} diff --git a/packages/perf-harness/tsconfig.esm.json b/packages/perf-harness/tsconfig.esm.json new file mode 100644 index 0000000000..6bff62dc07 --- /dev/null +++ b/packages/perf-harness/tsconfig.esm.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "ESNext", + "moduleResolution": "bundler", + "outDir": "dist/esm" + } +} diff --git a/packages/perf-harness/tsconfig.json b/packages/perf-harness/tsconfig.json new file mode 100644 index 0000000000..9c8a7d7c10 --- /dev/null +++ b/packages/perf-harness/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"] +} diff --git a/packages/server-utils/src/__tests__/lru.test.ts b/packages/server-utils/src/__tests__/lru.test.ts new file mode 100644 index 0000000000..ba391ef3a2 --- /dev/null +++ b/packages/server-utils/src/__tests__/lru.test.ts @@ -0,0 +1,97 @@ +import { + configureSvcCache, + DEFAULT_SVC_CACHE_MAX_ENTRIES, + getSvcCacheStats, + resetSvcCacheCounters, + resolveSvcCacheMaxEntries, + svcCache +} from '../lru'; + +describe('routing service cache', () => { + beforeEach(() => { + svcCache.clear(); + configureSvcCache({ maxEntries: DEFAULT_SVC_CACHE_MAX_ENTRIES }); + resetSvcCacheCounters(); + }); + + afterEach(() => { + svcCache.clear(); + configureSvcCache({ maxEntries: DEFAULT_SVC_CACHE_MAX_ENTRIES }); + resetSvcCacheCounters(); + }); + + it('uses at least the required resident capacity by default', () => { + expect(resolveSvcCacheMaxEntries({ minimumEntries: 8 })).toBe( + DEFAULT_SVC_CACHE_MAX_ENTRIES + ); + expect(resolveSvcCacheMaxEntries({ minimumEntries: 2048 })).toBe(2048); + }); + + it('rejects an explicit capacity below the required resident floor', () => { + expect(() => resolveSvcCacheMaxEntries({ + maxEntries: 63, + minimumEntries: 64 + })).toThrow('must be at least the required minimum (64)'); + }); + + it.each([0, -1, 1.5, Number.NaN])( + 'rejects invalid capacity %s', + (maxEntries) => { + expect(() => resolveSvcCacheMaxEntries({ maxEntries })).toThrow( + 'must be a positive safe integer' + ); + } + ); + + it('reports lookups, capacity eviction, and current residency', () => { + configureSvcCache({ maxEntries: 2 }); + svcCache.set('label-a', { apiId: 'api-a' }); + svcCache.set('label-b', { apiId: 'api-b' }); + + expect(svcCache.get('label-a')).toEqual({ apiId: 'api-a' }); + expect(svcCache.get('missing')).toBeUndefined(); + svcCache.set('label-c', { apiId: 'api-c' }); + + expect(svcCache.has('label-a')).toBe(true); + expect(svcCache.has('label-b')).toBe(false); + expect(svcCache.has('label-c')).toBe(true); + expect(getSvcCacheStats()).toMatchObject({ + size: 2, + max: 2, + hits: 1, + misses: 1, + evictions: 1, + evictionsByReason: { + capacity: 1, + ttl: 0 + } + }); + }); + + it('does not classify explicit invalidation as cache pressure eviction', () => { + configureSvcCache({ maxEntries: 2 }); + svcCache.set('label-a', { apiId: 'api-a' }); + svcCache.delete('label-a'); + svcCache.set('label-b', { apiId: 'api-b' }); + svcCache.clear(); + + expect(getSvcCacheStats()).toMatchObject({ + size: 0, + evictions: 0, + evictionsByReason: { + capacity: 0, + ttl: 0 + } + }); + }); + + it('refuses to resize a live process cache instead of silently evicting metadata', () => { + svcCache.set('label-a', { apiId: 'api-a' }); + + expect(() => configureSvcCache({ maxEntries: 2048 })).toThrow( + 'cannot be reconfigured while routing metadata is resident' + ); + expect(svcCache.peek('label-a')).toEqual({ apiId: 'api-a' }); + expect(svcCache.max).toBe(DEFAULT_SVC_CACHE_MAX_ENTRIES); + }); +}); diff --git a/packages/server-utils/src/lru.ts b/packages/server-utils/src/lru.ts index 7f07b2ff53..e7c3b8281c 100644 --- a/packages/server-utils/src/lru.ts +++ b/packages/server-utils/src/lru.ts @@ -1,21 +1,204 @@ import { Logger } from '@pgpmjs/logger'; import { LRUCache } from 'lru-cache'; -const log = new Logger('pg-cache'); +const log = new Logger('routing-service-cache'); const ONE_HOUR_IN_MS = 1000 * 60 * 60; const ONE_DAY = ONE_HOUR_IN_MS * 24; const ONE_YEAR = ONE_DAY * 366; export const SVC_CACHE_TTL_MS = ONE_YEAR; +export const DEFAULT_SVC_CACHE_MAX_ENTRIES = 1024; -// --- Service Cache --- -// Keep max aligned with PG_CACHE_MAX and GRAPHILE_CACHE_MAX (default: 50) -export const svcCache = new LRUCache({ - max: 50, - ttl: SVC_CACHE_TTL_MS, - updateAgeOnGet: true, - dispose: (_, key) => { - log.debug(`Disposing service[${key}]`); +export type SvcCacheEvictionReason = 'capacity' | 'ttl'; + +export interface SvcCacheStats { + size: number; + max: number; + ttlMs: number; + hits: number; + misses: number; + evictions: number; + evictionsByReason: Record; + oldestKeyAgeMs: number | null; + keys: string[]; +} + +export interface ConfigureSvcCacheOptions { + /** Exact operator ceiling. Omit to use the safe process default. */ + maxEntries?: number; + /** Capacity floor imposed by the caller, such as resident Graphile capacity. */ + minimumEntries?: number; +} + +const assertPositiveSafeInteger = (value: number, label: string): void => { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${label} must be a positive safe integer`); + } +}; + +export const resolveSvcCacheMaxEntries = ({ + maxEntries, + minimumEntries = 1 +}: ConfigureSvcCacheOptions = {}): number => { + assertPositiveSafeInteger(minimumEntries, 'svcCache minimumEntries'); + if (maxEntries === undefined) { + return Math.max(DEFAULT_SVC_CACHE_MAX_ENTRIES, minimumEntries); + } + + assertPositiveSafeInteger(maxEntries, 'svcCache maxEntries'); + if (maxEntries < minimumEntries) { + throw new Error( + `svcCache maxEntries (${maxEntries}) must be at least the required minimum (${minimumEntries})` + ); + } + return maxEntries; +}; + +/** + * Process-wide routing-label metadata cache. + * + * This cache deliberately owns only resolved routing metadata. Capacity or TTL + * eviction never disposes a PostGraphile instance; a later request simply + * resolves the label again and reuses the independently keyed Graphile build. + */ +class RoutingServiceCache { + private cache: LRUCache; + private hits = 0; + private misses = 0; + private readonly evictionsByReason: Record = { + capacity: 0, + ttl: 0 + }; + + constructor(maxEntries: number) { + this.cache = this.createCache(maxEntries); + } + + private createCache(maxEntries: number): LRUCache { + return new LRUCache({ + max: maxEntries, + ttl: SVC_CACHE_TTL_MS, + updateAgeOnGet: true, + dispose: (_, key, reason) => { + if (reason === 'evict') { + this.evictionsByReason.capacity++; + log.debug(`Evicting routing metadata[${key}] (capacity)`); + } else if (reason === 'expire') { + this.evictionsByReason.ttl++; + log.debug(`Evicting routing metadata[${key}] (ttl)`); + } + } + }); } -}); \ No newline at end of file + + configure(maxEntries: number): void { + assertPositiveSafeInteger(maxEntries, 'svcCache maxEntries'); + if (maxEntries === this.cache.max) return; + if (this.cache.size > 0) { + throw new Error( + 'svcCache cannot be reconfigured while routing metadata is resident; clear it first' + ); + } + this.cache = this.createCache(maxEntries); + } + + get size(): number { + return this.cache.size; + } + + get max(): number { + return this.cache.max; + } + + get(key: string): T | undefined { + const value = this.cache.get(key); + if (value === undefined) this.misses++; + else this.hits++; + return value; + } + + /** Existence inspection is intentionally not counted as a request lookup. */ + has(key: string): boolean { + return this.cache.has(key); + } + + /** Non-mutating inspection that does not affect LRU age or lookup counters. */ + peek(key: string): T | undefined { + return this.cache.peek(key); + } + + set(key: string, value: T): this { + this.cache.set(key, value); + return this; + } + + delete(key: string): boolean { + return this.cache.delete(key); + } + + clear(): void { + this.cache.clear(); + } + + keys(): IterableIterator { + return this.cache.keys(); + } + + entries(): IterableIterator<[string, T]> { + return this.cache.entries(); + } + + getRemainingTTL(key: string): number { + return this.cache.getRemainingTTL(key); + } + + resetCounters(): void { + this.hits = 0; + this.misses = 0; + this.evictionsByReason.capacity = 0; + this.evictionsByReason.ttl = 0; + } + + getStats(maxKeys = 200): SvcCacheStats { + assertPositiveSafeInteger(maxKeys, 'svcCache stats maxKeys'); + let minRemaining = Infinity; + for (const key of this.cache.keys()) { + const remaining = this.cache.getRemainingTTL(key); + if (remaining < minRemaining) minRemaining = remaining; + } + const evictionsByReason = { ...this.evictionsByReason }; + + return { + size: this.cache.size, + max: this.cache.max, + ttlMs: SVC_CACHE_TTL_MS, + hits: this.hits, + misses: this.misses, + evictions: evictionsByReason.capacity + evictionsByReason.ttl, + evictionsByReason, + oldestKeyAgeMs: Number.isFinite(minRemaining) + ? Math.max(0, SVC_CACHE_TTL_MS - minRemaining) + : null, + keys: [...this.cache.keys()].slice(0, maxKeys) + }; + } +} + +export const svcCache = new RoutingServiceCache( + DEFAULT_SVC_CACHE_MAX_ENTRIES +); + +export const configureSvcCache = ( + options: ConfigureSvcCacheOptions = {} +): SvcCacheStats => { + svcCache.configure(resolveSvcCacheMaxEntries(options)); + return svcCache.getStats(); +}; + +export const getSvcCacheStats = (maxKeys = 200): SvcCacheStats => + svcCache.getStats(maxKeys); + +export const resetSvcCacheCounters = (): void => { + svcCache.resetCounters(); +}; diff --git a/patches/@dataplan__pg@1.0.3.patch b/patches/@dataplan__pg@1.0.3.patch new file mode 100644 index 0000000000..b9339962ab --- /dev/null +++ b/patches/@dataplan__pg@1.0.3.patch @@ -0,0 +1,179 @@ +diff --git a/dist/adaptors/pg.js b/dist/adaptors/pg.js +--- a/dist/adaptors/pg.js ++++ b/dist/adaptors/pg.js +@@ -26,6 +26,15 @@ const cacheSizeFromEnv = process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE + */ + const PREPARED_STATEMENT_CACHE_SIZE = !!cacheSizeFromEnv || cacheSizeFromEnv === 0 ? cacheSizeFromEnv : 100; + const $$isSetup = Symbol("isConfiguredForDataplanPg"); ++const REUSABLE_CLIENT_RELEASE_MODES = Object.freeze(["reuse"]); ++const DESTROYABLE_CLIENT_RELEASE_MODES = Object.freeze(["reuse", "destroy"]); ++function getClientReleaseMode(options) { ++ const mode = options?.clientReleaseMode ?? "reuse"; ++ if (mode !== "reuse" && mode !== "destroy") { ++ throw new Error(`Unsupported PostgreSQL client release mode '${mode}'`); ++ } ++ return mode; ++} + /** + * \> JIT compilation is beneficial primarily for long-running CPU-bound + * \> queries. Frequently these will be analytical queries. For short +@@ -253,24 +262,37 @@ async function makeNodePostgresWithPgClient_inner(pgClient, pgSettings, callback + * Returns a `withPgClient` for the given `Pool` instance. + */ + function makePgAdaptorWithPgClient(pool, release = () => { }) { +- const withPgClient = async (pgSettings, callback) => { ++ const withPgClient = async (pgSettings, callback, options) => { ++ const clientReleaseMode = getClientReleaseMode(options); ++ const supportsExactClientDestruction = typeof PgPool === "function" && pool instanceof PgPool; ++ if (clientReleaseMode === "destroy" && !supportsExactClientDestruction) { ++ throw new Error("Exact PostgreSQL client destruction requires a node-postgres Pool"); ++ } + const pgClient = await pool.connect(); +- if (!pgClient[$$isSetup]) { +- pgClient[$$isSetup] = true; +- if (!DONT_DISABLE_JIT) { +- // We don't actually disable JIT, it's the optimization that's expensive so we disable that. +- pgClient.query("set jit_optimize_above_cost = -1;").catch((e) => { +- console.error(`Error occurred applying @dataplan/pg global Postgres settings: ${e}`); +- }); ++ try { ++ if (!pgClient[$$isSetup]) { ++ pgClient[$$isSetup] = true; ++ if (!DONT_DISABLE_JIT) { ++ // We don't actually disable JIT, it's the optimization that's expensive so we disable that. ++ pgClient.query("set jit_optimize_above_cost = -1;").catch((e) => { ++ console.error(`Error occurred applying @dataplan/pg global Postgres settings: ${e}`); ++ }); ++ } + } +- } +- try { + return await makeNodePostgresWithPgClient_inner(pgClient, pgSettings, callback, false, false); + } + finally { + // NOTE: have decided not to `RESET ALL` here; otherwise timezone,jit,etc will reset +- pgClient.release(); ++ if (clientReleaseMode === "destroy") { ++ pgClient.release(true); ++ } ++ else { ++ pgClient.release(); ++ } + } + }; ++ withPgClient.supportedClientReleaseModes = typeof PgPool === "function" && pool instanceof PgPool ++ ? DESTROYABLE_CLIENT_RELEASE_MODES ++ : REUSABLE_CLIENT_RELEASE_MODES; + let released = false; + const releaseOnce = () => { +@@ -292,11 +314,16 @@ function makePgAdaptorWithPgClient(pool, release = () => { }) { + */ + function makeWithPgClientViaPgClientAlreadyInTransaction(pgClient, alreadyInTransaction = false) { + const release = () => { }; +- const withPgClient = async (pgSettings, callback) => { ++ const withPgClient = async (pgSettings, callback, options) => { ++ const clientReleaseMode = getClientReleaseMode(options); ++ if (clientReleaseMode !== "reuse") { ++ throw new Error("Cannot destroy a caller-owned PostgreSQL client"); ++ } + return makeNodePostgresWithPgClient_inner(pgClient, pgSettings, callback, + // Ensure only one withPgClient can run at a time, since we only have on pgClient. + true, alreadyInTransaction); + }; ++ withPgClient.supportedClientReleaseModes = REUSABLE_CLIENT_RELEASE_MODES; + let released = false; + const releaseOnce = () => { + if (released) { +diff --git a/dist/executor.d.ts b/dist/executor.d.ts +--- a/dist/executor.d.ts ++++ b/dist/executor.d.ts +@@ -62,8 +62,13 @@ export interface PgClient { + query(opts: PgClientQuery): Promise>; + withTransaction(callback: (client: this) => Promise): Promise; + } ++export type PgClientReleaseMode = "reuse" | "destroy"; ++export interface WithPgClientUseOptions { ++ clientReleaseMode?: PgClientReleaseMode; ++} + export interface WithPgClient { +- (pgSettings: Record | null, callback: (client: TPgClient) => T | Promise): Promise; ++ (pgSettings: Record | null, callback: (client: TPgClient) => T | Promise, options?: WithPgClientUseOptions): Promise; ++ supportedClientReleaseModes?: readonly PgClientReleaseMode[]; + release?(): PromiseOrDirect; + } + export type PgExecutorContext = { +diff --git a/dist/index.js b/dist/index.js +--- a/dist/index.js ++++ b/dist/index.js +@@ -1,2 +1,3 @@ + "use strict"; + Object.defineProperty(exports, "__esModule", { value: true }); ++exports.exactClientReleaseCapability = "dataplan-pg-exact-client-destroy-v1"; +diff --git a/dist/index.d.ts b/dist/index.d.ts +--- a/dist/index.d.ts ++++ b/dist/index.d.ts +@@ -1,3 +1,4 @@ + import type { GrafastSubscriber } from "grafast"; ++export declare const exactClientReleaseCapability: "dataplan-pg-exact-client-destroy-v1"; + export { sql } from "pg-sql2"; + import type { ObjectFromPgCodecAttributes, PgCodecAttribute, PgCodecAttributeExtensions, PgCodecAttributes, PgCodecAttributeVia, PgCodecAttributeViaExplicit, PgEnumCodecSpec, PgRecordTypeCodecSpec } from "./codecs.ts"; +@@ -39,6 +39,7 @@ import type { SideEffectWithPgClientStepCallback } from "./steps/withPgClient.ts + import { loadManyWithPgClient, loadOneWithPgClient, sideEffectWithPgClient, SideEffectWithPgClientStep, sideEffectWithPgClientTransaction, withPgClient, withPgClientTransaction } from "./steps/withPgClient.ts"; + import { assertPgClassSingleStep } from "./utils.ts"; + export type { GetPgCodecAttributes, GetPgRegistryCodecRelations, GetPgRegistryCodecs, GetPgRegistrySources, GetPgResourceAttributes, GetPgResourceCodec, GetPgResourceRegistry, GetPgResourceRelations, GetPgResourceUniques, KeysOfType, MakePgServiceOptions, ObjectFromPgCodecAttributes, PgAdaptor, PgBox, PgCircle, PgClassSingleStep, PgClient, PgClientQuery, PgClientResult, PgCodec, PgCodecAnyScalar, PgCodecAttribute, PgCodecAttributeExtensions, PgCodecAttributes, PgCodecAttributeVia, PgCodecAttributeViaExplicit, PgCodecExtensions, PgCodecList, PgCodecPolymorphism, PgCodecPolymorphismRelational, PgCodecPolymorphismRelationalTypeSpec, PgCodecPolymorphismSingle, PgCodecPolymorphismSingleTypeAttributeSpec, PgCodecPolymorphismSingleTypeSpec, PgCodecPolymorphismUnion, PgCodecRef, PgCodecRefExtensions, PgCodecRefPath, PgCodecRefPathEntry, PgCodecRefs, PgCodecRelation, PgCodecRelationConfig, PgCodecRelationExtensions, PgCodecWithAttributes, PgConditionCapableParent, PgConditionLike, PgDecode, PgDeleteSingleQueryBuilder, PgEncode, PgEnumCodec, PgEnumCodecSpec, PgEnumValue, PgExecutorContext, PgExecutorContextPlans, PgExecutorInput, PgExecutorMutationOptions, PgExecutorOptions, PgFunctionResourceOptions, PgGroupDetails, PgGroupSpec, PgHavingConditionSpec, PgHStore, PgInsertSingleQueryBuilder, PgInterval, PgLine, PgLockableParameter, PgLockCallback, PgLseg, PgOrderSpec, PgPath, PgPoint, PgPolygon, PgRecordTypeCodecSpec, PgRefDefinition, PgRefDefinitionExtensions, PgRefDefinitions, PgRegistry, PgRegistryBuilder, PgResourceExtensions, PgResourceOptions, PgResourceParameter, PgResourceUnique, PgResourceUniqueExtensions, PgRootStep, PgSelectArgumentDigest, PgSelectArgumentRuntimeValue, PgSelectArgumentSpec, PgSelectIdentifierSpec, PgSelectMode, PgSelectOptions, PgSelectParsedCursorStep, PgSelectQueryBuilder, PgSelectQueryBuilderCallback, PgSelectSinglePlanOptions, PgTypedStep, PgUnionAllQueryBuilder, PgUnionAllQueryBuilderCallback, PgUnionAllStepCondition, PgUnionAllStepConfig, PgUnionAllStepConfigAttributes, PgUnionAllStepMember, PgUnionAllStepOrder, PgUpdateSingleQueryBuilder, PgWhereConditionSpec, PlanByUniques, SideEffectWithPgClientStepCallback, TuplePlanMap, WithPgClient, }; ++export type { PgClientReleaseMode, WithPgClientUseOptions } from "./executor.ts"; + export { assertPgClassSingleStep, domainOfCodec, enumCodec, generatePgParameterAnalysis, getCodecByPgCatalogTypeName, getInnerCodec, getWithPgClientFromPgService, isEnumCodec, LIST_TYPES, listOfCodec, loadManyWithPgClient, loadOneWithPgClient, pgResourceOptions as makePgResourceOptions, makeRegistry, makeRegistryBuilder, PgBooleanFilter, pgClassExpression, PgClassExpressionStep, PgClassFilter, PgCondition, PgContextPlugin, PgCursorStep, pgDeleteSingle, PgDeleteSingleStep, PgExecutor, pgFromExpression, pgFromExpressionRuntime, pgInsertSingle, PgInsertSingleStep, PgManyFilter, PgOrFilter, PgResource, pgResourceOptions, pgSelect, pgSelectFromRecord, pgSelectFromRecords, PgSelectRowsStep, pgSelectSingleFromRecord, PgSelectSingleStep, PgSelectStep, PgTempTable, pgUnionAll, PgUnionAllRowsStep, PgUnionAllSingleStep, PgUnionAllStep, pgUpdateSingle, PgUpdateSingleStep, pgValidateParsedCursor, PgValidateParsedCursorStep, pgWhereConditionSpecListToSQL, rangeOfCodec, recordCodec, sideEffectWithPgClient, SideEffectWithPgClientStep, sideEffectWithPgClientTransaction, sqlFromArgDigests, sqlValueWithCodec, toPg, ToPgStep, TYPES, withPgClient, withPgClientFromPgService, withPgClientTransaction, withSuperuserPgClientFromPgService, }; + export { version } from "./version.ts"; + declare global { +diff --git a/dist/pgServices.d.ts b/dist/pgServices.d.ts +--- a/dist/pgServices.d.ts ++++ b/dist/pgServices.d.ts +@@ -1,4 +1,4 @@ +-import type { PgClient, WithPgClient } from "./executor.ts"; ++import type { PgClient, WithPgClient, WithPgClientUseOptions } from "./executor.ts"; + type PromiseOrDirect = T | PromiseLike; + /** @experimental */ + export interface PgAdaptor { +@@ -14,6 +14,6 @@ export declare function isPromiseLike(t: T | Promise | PromiseLike): t + * config, caching it to make future lookups faster. + */ + export declare function getWithPgClientFromPgService(config: GraphileConfig.PgServiceConfiguration): PromiseOrDirect>; +-export declare function withPgClientFromPgService(config: GraphileConfig.PgServiceConfiguration, pgSettings: Record | null, callback: (client: PgClient) => T | Promise): Promise; ++export declare function withPgClientFromPgService(config: GraphileConfig.PgServiceConfiguration, pgSettings: Record | null, callback: (client: PgClient) => T | Promise, options?: WithPgClientUseOptions): Promise; + export declare function withSuperuserPgClientFromPgService(config: GraphileConfig.PgServiceConfiguration, pgSettings: Record | null, callback: (client: PgClient) => T | Promise): Promise; + export {}; +diff --git a/dist/pgServices.js b/dist/pgServices.js +--- a/dist/pgServices.js ++++ b/dist/pgServices.js +@@ -38,6 +38,7 @@ function getWithPgClientFromPgService(config) { + } + const originalWithPgClient = await factory(config.adaptorSettings); + const withPgClient = ((...args) => originalWithPgClient.apply(null, args)); ++ withPgClient.supportedClientReleaseModes = originalWithPgClient.supportedClientReleaseModes; + const cachedValue = { + withPgClient, + retainers: 1, +@@ -70,13 +71,21 @@ function getWithPgClientFromPgService(config) { + return promise.then((v) => v.withPgClient); + } + } +-async function withPgClientFromPgService(config, pgSettings, callback) { ++async function withPgClientFromPgService(config, pgSettings, callback, options) { + const withPgClientFromPgService = getWithPgClientFromPgService(config); + const withPgClient = isPromiseLike(withPgClientFromPgService) + ? await withPgClientFromPgService + : withPgClientFromPgService; + try { +- return await withPgClient(pgSettings, callback); ++ const clientReleaseMode = options?.clientReleaseMode ?? "reuse"; ++ if (clientReleaseMode !== "reuse" && clientReleaseMode !== "destroy") { ++ throw new Error(`Unsupported PostgreSQL client release mode '${clientReleaseMode}' for service '${config.name}'`); ++ } ++ if (clientReleaseMode === "destroy" && ++ !withPgClient.supportedClientReleaseModes?.includes("destroy")) { ++ throw new Error(`PostgreSQL service '${config.name}' does not support exact client destruction`); ++ } ++ return await withPgClient(pgSettings, callback, options); + } + finally { + withPgClient.release(); diff --git a/patches/@graphile-contrib__pg-many-to-many@2.0.0-rc.2.patch b/patches/@graphile-contrib__pg-many-to-many@2.0.0-rc.2.patch new file mode 100644 index 0000000000..95d546b2be --- /dev/null +++ b/patches/@graphile-contrib__pg-many-to-many@2.0.0-rc.2.patch @@ -0,0 +1,15 @@ +diff --git a/dist/PgManyToManyRelationPlugin.js b/dist/PgManyToManyRelationPlugin.js +index 983bc745b4707efb8e3ade5b25b88b9e2cd114e7..d362f41b307e68b28e0737559cd6c40bb95e3109 100644 +--- a/dist/PgManyToManyRelationPlugin.js ++++ b/dist/PgManyToManyRelationPlugin.js +@@ -265,7 +265,9 @@ exports.PgManyToManyRelationPlugin = { + }, + hooks: { + build(build) { +- build.pgManyToManyRealtionshipsByResource = new Map(); ++ const relationshipsByResource = new Map(); ++ build.pgManyToManyRealtionshipsByResource = relationshipsByResource; ++ build.registerBuildStateDisposer(() => relationshipsByResource.clear()); + return build; + }, + init(_, build, _context) { diff --git a/patches/graphile-build-pg.patch b/patches/graphile-build-pg.patch new file mode 100644 index 0000000000..3e50c9fae0 --- /dev/null +++ b/patches/graphile-build-pg.patch @@ -0,0 +1,402 @@ +diff --git a/dist/index.js b/dist/index.js +--- a/dist/index.js ++++ b/dist/index.js +@@ -1,2 +1,3 @@ + "use strict"; + Object.defineProperty(exports, "__esModule", { value: true }); ++exports.introspectionClientReleaseCapability = "graphile-build-pg-exact-client-destroy-v1"; +diff --git a/dist/index.d.ts b/dist/index.d.ts +--- a/dist/index.d.ts ++++ b/dist/index.d.ts +@@ -1,2 +1,3 @@ + import type { PgRegistry } from "@dataplan/pg"; + import type { PartitionExpose } from "./interfaces.ts"; ++export declare const introspectionClientReleaseCapability: "graphile-build-pg-exact-client-destroy-v1"; +diff --git a/dist/plugins/PgBasicsPlugin.js b/dist/plugins/PgBasicsPlugin.js +index 97d175d650d0a2cec51932a3da004563c6815500..3766b3f998f2f128e7669765aac309dff6d09f6d 100644 +--- a/dist/plugins/PgBasicsPlugin.js ++++ b/dist/plugins/PgBasicsPlugin.js +@@ -220,6 +220,7 @@ exports.PgBasicsPlugin = { + build.pgCodecs = pgRegistry.pgCodecs; + build.pgRelations = pgRegistry.pgRelations; + const pgCodecMetaLookup = (0, inputUtils_ts_1.getCodecMetaLookupFromInput)(build.input); ++ build.registerBuildStateDisposer(() => pgCodecMetaLookup.clear()); + const getGraphQLTypeNameByPgCodec = (codec, situation) => { + if (codec.arrayOfCodec) { + throw new Error("Do not use getGraphQLTypeNameByPgCodec with an array type, find the underlying type instead"); +diff --git a/dist/plugins/PgCodecsPlugin.js b/dist/plugins/PgCodecsPlugin.js +index fe7110a577eae139451d2b431b9f9a9d4e2a5337..eed17c45df4a188c5b23ffc68dd626e096f0fe96 100644 +--- a/dist/plugins/PgCodecsPlugin.js ++++ b/dist/plugins/PgCodecsPlugin.js +@@ -421,6 +421,135 @@ exports.PgCodecsPlugin = { + event.pgCodec = (0, graphile_build_1.EXPORTABLE)((codecName, innerCodec, rangeOfCodec, spec, sqlIdent) => rangeOfCodec(innerCodec, codecName, sqlIdent, spec), [codecName, innerCodec, pg_1.rangeOfCodec, spec, sqlIdent], `${codecName}Codec`); + return; + } ++ // Multirange types are represented in GraphQL as lists of the ++ // corresponding range type. PostgreSQL's multirange text format ++ // is deliberately not treated as an array: its range entries are ++ // not quoted, so commas inside a range are not array delimiters. ++ if (type.typtype === "m") { ++ const range = await info.helpers.pgIntrospection.getRangeByType(serviceName, type._id); ++ if (!range?.rngtypid) { ++ throw new Error(`Failed to get range entry related to multirange '${type._id}'`); ++ } ++ const rangeCodec = (await info.helpers.pgCodecs.getCodecFromType(serviceName, range.rngtypid)); ++ if (!rangeCodec?.rangeOfCodec) { ++ throw new Error(`Failed to build the range codec related to multirange '${type._id}'`); ++ } ++ const typeName = type.typname; ++ const codecName = info.inflection.typeCodecName({ ++ pgType: type, ++ serviceName, ++ }); ++ const { tags, description } = type.getTagsAndDescription(); ++ const extensions = { ++ oid: type._id, ++ pg: { ++ serviceName, ++ schemaName: namespaceName, ++ name: typeName, ++ }, ++ listItemNonNull: true, ++ ...(Object.keys(tags).length > 0 ? { tags } : null), ++ }; ++ await info.process("pgCodecs_listOfCodec_extensions", { ++ serviceName, ++ pgType: type, ++ innerCodec: rangeCodec, ++ extensions, ++ }); ++ // A PostgreSQL multirange cannot contain null ranges. ++ extensions.listItemNonNull = true; ++ const sqlIdent = info.helpers.pgBasics.identifier(namespaceName, typeName); ++ (0, utils_ts_1.exportNameHint)(sqlIdent, `${codecName}Identifier`); ++ const spec = { ++ name: codecName, ++ identifier: sqlIdent, ++ extensions, ++ ...(description ? { description } : null), ++ }; ++ event.pgCodec = (0, graphile_build_1.EXPORTABLE)((listOfCodec, rangeCodec, spec) => { ++ const listCodec = listOfCodec(rangeCodec, spec); ++ const parseCastedList = listCodec.fromPg; ++ const hasCastedRepresentation = typeof listCodec.castFromPg === "function"; ++ return { ++ ...listCodec, ++ fromPg(value) { ++ if (hasCastedRepresentation) { ++ return parseCastedList(value); ++ } ++ if (typeof value !== "string" || ++ value.length < 2 || ++ value[0] !== "{" || ++ value[value.length - 1] !== "}") { ++ throw new Error(`Invalid PostgreSQL multirange value '${String(value)}'`); ++ } ++ const body = value.slice(1, -1); ++ if (body.length === 0) { ++ return []; ++ } ++ const encodedRanges = []; ++ let start = 0; ++ let depth = 0; ++ let quoted = false; ++ let escaped = false; ++ for (let index = 0; index < body.length; index++) { ++ const character = body[index]; ++ if (quoted) { ++ if (escaped) { ++ escaped = false; ++ } ++ else if (character === "\\") { ++ escaped = true; ++ } ++ else if (character === '"') { ++ quoted = false; ++ } ++ } ++ else if (character === '"') { ++ quoted = true; ++ } ++ else if (character === "[" || character === "(") { ++ depth++; ++ } ++ else if (character === "]" || character === ")") { ++ depth--; ++ if (depth < 0) { ++ throw new Error(`Invalid PostgreSQL multirange value '${value}'`); ++ } ++ } ++ else if (character === "," && depth === 0) { ++ const encodedRange = body.slice(start, index).trim(); ++ if (!encodedRange) { ++ throw new Error(`Invalid PostgreSQL multirange value '${value}'`); ++ } ++ encodedRanges.push(encodedRange); ++ start = index + 1; ++ } ++ } ++ if (quoted || escaped || depth !== 0) { ++ throw new Error(`Invalid PostgreSQL multirange value '${value}'`); ++ } ++ const encodedRange = body.slice(start).trim(); ++ if (!encodedRange) { ++ throw new Error(`Invalid PostgreSQL multirange value '${value}'`); ++ } ++ encodedRanges.push(encodedRange); ++ return encodedRanges.map((entry) => rangeCodec.fromPg(entry)); ++ }, ++ toPg(value) { ++ if (!Array.isArray(value)) { ++ throw new Error("PostgreSQL multirange input must be a list of ranges"); ++ } ++ return `{${value.map((entry) => { ++ if (entry == null) { ++ throw new Error("PostgreSQL multirange input cannot contain a null range"); ++ } ++ return rangeCodec.toPg(entry); ++ }).join(",")}}`; ++ }, ++ }; ++ }, [pg_1.listOfCodec, rangeCodec, spec], `${codecName}Codec`); ++ return; ++ } + // Domains are wrappers under an underlying type + if (type.typtype === "d") { + const { typnotnull: notNull, typbasetype: baseTypeOid, typtypmod: baseTypeModifier, typndims: _numberOfArrayDimensions, typcollation: _domainCollation, typdefaultbin: _defaultValueNodeTree, } = type; +@@ -533,7 +662,9 @@ exports.PgCodecsPlugin = { + schema: { + hooks: { + build(build) { +- build.allPgCodecs = new Set(); ++ const allPgCodecs = new Set(); ++ build.allPgCodecs = allPgCodecs; ++ build.registerBuildStateDisposer(() => allPgCodecs.clear()); + function walkCodec(codec) { + if (build.allPgCodecs.has(codec)) { + return; +diff --git a/dist/plugins/PgIntrospectionPlugin.d.ts b/dist/plugins/PgIntrospectionPlugin.d.ts +index 821629b96574c90ff58804be3aa3b98c4443c7a0..3c85017d8a765c4368febfd766e5082b0609b5a1 100644 +--- a/dist/plugins/PgIntrospectionPlugin.d.ts ++++ b/dist/plugins/PgIntrospectionPlugin.d.ts +@@ -1,6 +1,22 @@ + import { PgExecutor } from "@dataplan/pg"; + import type { PromiseOrDirect } from "grafast"; + import type { Introspection, PgAttribute, PgAuthMembers, PgClass, PgConstraint, PgDepend, PgDescription, PgEnum, PgExtension, PgIndex, PgInherits, PgLanguage, PgNamespace, PgProc, PgRange, PgRoles, PgType } from "pg-introspection"; ++declare global { ++ namespace GraphileConfig { ++ interface PgServiceConfiguration { ++ /** Selects the catalog query used during this service's gather phase. */ ++ introspectionMode?: "stock" | "scoped-required"; ++ /** Catalog types retained by scoped introspection; defaults to all for compatibility. */ ++ introspectionScopedCatalogTypes?: "all" | "dependency-closure"; ++ /** Non-root schemas that scoped dependency closure may retain. */ ++ introspectionAllowedDependencySchemas?: readonly string[]; ++ /** Exact installed extension names whose optional capability metadata is required. */ ++ introspectionCapabilityExtensions?: readonly string[]; ++ /** How to release the exact client after its catalog query. */ ++ introspectionClientReleaseMode?: "reuse" | "destroy"; ++ } ++ } ++} + export type PgEntityWithId = PgNamespace | PgClass | PgConstraint | PgProc | PgRoles | PgType | PgEnum | PgExtension | PgExtension | PgIndex | PgLanguage; + declare global { + namespace GraphileBuild { +diff --git a/dist/plugins/PgIntrospectionPlugin.js b/dist/plugins/PgIntrospectionPlugin.js +index a1597b1dc9a4f76d3debc80f5ef151f4e58bc2cd..e97d2aecfdbe6e94eeb2182b3f0e3f94f9a57cab 100644 +--- a/dist/plugins/PgIntrospectionPlugin.js ++++ b/dist/plugins/PgIntrospectionPlugin.js +@@ -42,6 +42,118 @@ function makeGetEntities(loc) { + return list; + }; + } ++function getIntrospectionQuery(pgService) { ++ const mode = pgService.introspectionMode ?? "stock"; ++ const configuredCatalogTypes = pgService.introspectionScopedCatalogTypes; ++ const configuredCapabilityExtensions = pgService.introspectionCapabilityExtensions; ++ const scopedCatalogTypes = configuredCatalogTypes ?? "all"; ++ if (scopedCatalogTypes !== "all" && scopedCatalogTypes !== "dependency-closure") { ++ throw new Error(`Unsupported scoped catalog type policy '${scopedCatalogTypes}' for service '${pgService.name}'`); ++ } ++ if (mode === "stock") { ++ if (configuredCatalogTypes !== undefined) { ++ throw new Error(`Scoped catalog type policy is only valid with scoped-required introspection for service '${pgService.name}'`); ++ } ++ if (configuredCapabilityExtensions !== undefined) { ++ throw new Error(`Scoped extension capabilities are only valid with scoped-required introspection for service '${pgService.name}'`); ++ } ++ return { ++ query: { text: (0, pg_introspection_1.makeIntrospectionQuery)() }, ++ requiredSchemas: null, ++ allowedSchemas: null, ++ scopedCatalogTypes: null, ++ }; ++ } ++ if (mode === "scoped-required") { ++ const requiredSchemas = pgService.schemas ?? []; ++ const dependencySchemas = pgService.introspectionAllowedDependencySchemas ?? []; ++ const capabilityExtensions = configuredCapabilityExtensions ?? []; ++ return { ++ query: (0, pg_introspection_1.makeSchemaScopedIntrospectionQuery)(requiredSchemas, { ++ catalogTypes: scopedCatalogTypes, ++ capabilityExtensions, ++ }), ++ requiredSchemas, ++ allowedSchemas: [...new Set([...requiredSchemas, ...dependencySchemas, "pg_catalog"])], ++ scopedCatalogTypes, ++ }; ++ } ++ throw new Error(`Unsupported PostgreSQL introspection mode '${mode}' for service '${pgService.name}'`); ++} ++function assertScopedNamespaces(introspection, requiredSchemas, allowedSchemas, serviceName) { ++ if (requiredSchemas === null) { ++ return; ++ } ++ const found = new Set(introspection.namespaces.map((namespace) => namespace.nspname)); ++ const missing = requiredSchemas.filter((schema) => !found.has(schema)); ++ if (missing.length > 0) { ++ throw new Error(`Schema-scoped introspection for service '${serviceName}' did not find required schema(s): ${missing.join(", ")}`); ++ } ++ const allowed = new Set(allowedSchemas); ++ const unexpected = [...found].filter((schema) => !allowed.has(schema)); ++ if (unexpected.length > 0) { ++ throw new Error(`Schema-scoped introspection for service '${serviceName}' crossed into unapproved dependency schema(s): ${unexpected.join(", ")}`); ++ } ++} ++function assertDependencyClosureTypes(introspection, scopedCatalogTypes, serviceName) { ++ if (scopedCatalogTypes !== "dependency-closure") { ++ return; ++ } ++ const retainedTypeOids = new Set(introspection.types.map((type) => String(type._id))); ++ const requireType = (oid, objectKind, objectContext, field) => { ++ if (oid === null || oid === undefined || String(oid) === "0") { ++ return; ++ } ++ const normalizedOid = String(oid); ++ // pg-introspection deliberately removes extension-owned composite ++ // resources from the public arrays after constructing its lookup maps. ++ // Array and user-object dependencies still resolve through that exact ++ // lookup, so validate the same runtime resolution surface instead of ++ // rejecting a coherent extension-pruned result. ++ const resolvesThroughIntrospection = retainedTypeOids.has(normalizedOid) || ++ introspection._lookups?.typeById?.has(normalizedOid) === true; ++ if (!resolvesThroughIntrospection) { ++ throw new Error(`Dependency-closure introspection for service '${serviceName}' retained ${objectKind} '${objectContext}' field '${field}' referencing missing pg_type OID '${normalizedOid}'`); ++ } ++ }; ++ const requireTypes = (oids, objectKind, objectContext, field) => { ++ for (const oid of oids ?? []) { ++ requireType(oid, objectKind, objectContext, field); ++ } ++ }; ++ for (const entity of introspection.classes) { ++ const context = `${entity.relname} (${entity._id})`; ++ requireType(entity.reltype, "pg_class", context, "reltype"); ++ requireType(entity.reloftype, "pg_class", context, "reloftype"); ++ } ++ for (const entity of introspection.attributes) { ++ requireType(entity.atttypid, "pg_attribute", `${entity.attrelid}.${entity.attname}`, "atttypid"); ++ } ++ for (const entity of introspection.constraints) { ++ requireType(entity.contypid, "pg_constraint", `${entity.conname} (${entity._id})`, "contypid"); ++ } ++ for (const entity of introspection.procs) { ++ const context = `${entity.proname} (${entity._id})`; ++ requireType(entity.prorettype, "pg_proc", context, "prorettype"); ++ requireTypes(entity.proargtypes, "pg_proc", context, "proargtypes"); ++ requireTypes(entity.proallargtypes, "pg_proc", context, "proallargtypes"); ++ } ++ for (const entity of introspection.types) { ++ const context = `${entity.typname} (${entity._id})`; ++ requireType(entity.typbasetype, "pg_type", context, "typbasetype"); ++ requireType(entity.typelem, "pg_type", context, "typelem"); ++ requireType(entity.typarray, "pg_type", context, "typarray"); ++ } ++ for (const entity of introspection.enums) { ++ requireType(entity.enumtypid, "pg_enum", `${entity.enumlabel} (${entity._id})`, "enumtypid"); ++ } ++ for (const entity of introspection.ranges) { ++ const context = `range ${entity.rngtypid ?? "unknown"}`; ++ requireType(entity.rngtypid, "pg_range", context, "rngtypid"); ++ requireType(entity.rngsubtype, "pg_range", context, "rngsubtype"); ++ requireType(entity.rngmultitypid, "pg_range", context, "rngmultitypid"); ++ } ++} + exports.PgIntrospectionPlugin = { + name: "PgIntrospectionPlugin", + description: "Introspects PostgreSQL databases and makes the results available to other plugins", +@@ -171,8 +283,8 @@ exports.PgIntrospectionPlugin = { + return type?.getEnumValues() ?? []; + }, + async getRangeByType(info, serviceName, typeId) { +- const type = await info.helpers.pgIntrospection.getType(serviceName, typeId); +- return type?.getRange(); ++ const relevant = await getDb(info, serviceName); ++ return relevant.introspection.ranges.find((range) => range.rngtypid === typeId || range.rngmultitypid === typeId); + }, + async getExtensionByName(info, serviceName, extensionName) { + const relevant = await getDb(info, serviceName); +@@ -198,11 +310,20 @@ exports.PgIntrospectionPlugin = { + info.cache.introspectionResultsPromise = null; + }); + const rawIntrospections = await introspectionPromise; +- const introspections = rawIntrospections.map(({ pgService, introspectionText }) => ({ +- pgService, ++ // Scoped mode keeps raw text only in this gather's async frame. ++ // Clear only the exact promise we consumed so every later gather ++ // re-queries, while concurrent helpers still share parsed state. ++ if (rawIntrospections.some(({ requiredSchemas }) => requiredSchemas !== null) && ++ info.cache.introspectionResultsPromise === introspectionPromise) { ++ info.cache.introspectionResultsPromise = null; ++ } ++ const introspections = rawIntrospections.map(({ pgService, introspectionText, requiredSchemas, allowedSchemas, scopedCatalogTypes }) => { + // IMPORTANT: parseIntrospectionResults must NOT be cached, because other plugins mutate it. +- introspection: (0, pg_introspection_1.parseIntrospectionResults)(introspectionText), +- })); ++ const introspection = (0, pg_introspection_1.parseIntrospectionResults)(introspectionText); ++ assertScopedNamespaces(introspection, requiredSchemas, allowedSchemas, pgService.name); ++ assertDependencyClosureTypes(introspection, scopedCatalogTypes, pgService.name); ++ return { pgService, introspection }; ++ }); + // Store the resolved state, so access during announcements doesn't cause the system to hang + info.state.getIntrospectionPromise = introspections; + // Announce the introspection results. +@@ -416,13 +537,15 @@ function introspectPgServices(pgServices) { + seenPgSettingsKeys.set(pgSettingsKey, i); + } + // Do the introspection +- const introspectionQuery = (0, pg_introspection_1.makeIntrospectionQuery)(); ++ const { query, requiredSchemas, allowedSchemas, scopedCatalogTypes } = getIntrospectionQuery(pgService); ++ const clientReleaseMode = pgService.introspectionClientReleaseMode ?? "reuse"; + const { rows: [row], } = await (0, pg_1.withPgClientFromPgService)(pgService, pgService.pgSettingsForIntrospection ?? null, (client) => client.query({ +- text: introspectionQuery, ++ text: query.text, ++ values: query.values, +- })); ++ }), clientReleaseMode === "reuse" ? undefined : { clientReleaseMode }); + if (!row) { + throw new Error("Introspection failed"); + } +- return { pgService, introspectionText: row.introspection }; ++ return { pgService, introspectionText: row.introspection, requiredSchemas, allowedSchemas, scopedCatalogTypes }; + })); + } +diff --git a/dist/plugins/PgPolymorphismPlugin.js b/dist/plugins/PgPolymorphismPlugin.js +index b1892244aebf936cb697d9e664e6b3c2432e8c0e..6907c0d70640bebd87c93377c3e30f901b18efa9 100644 +--- a/dist/plugins/PgPolymorphismPlugin.js ++++ b/dist/plugins/PgPolymorphismPlugin.js +@@ -554,6 +554,14 @@ exports.PgPolymorphismPlugin = { + } + } + const pgCodecByPolymorphicUnionModeTypeName = Object.create(null); ++ build.registerBuildStateDisposer(() => { ++ for (const key of Object.keys(pgResourcesByPolymorphicTypeName)) { ++ delete pgResourcesByPolymorphicTypeName[key]; ++ } ++ for (const key of Object.keys(pgCodecByPolymorphicUnionModeTypeName)) { ++ delete pgCodecByPolymorphicUnionModeTypeName[key]; ++ } ++ }); + for (const codec of Object.values(pgRegistry.pgCodecs)) { + if (!codec.polymorphism) + continue; diff --git a/patches/graphile-build@5.0.2.patch b/patches/graphile-build@5.0.2.patch new file mode 100644 index 0000000000..de2bf336e9 --- /dev/null +++ b/patches/graphile-build@5.0.2.patch @@ -0,0 +1,423 @@ +diff --git a/dist/SchemaBuilder.js b/dist/SchemaBuilder.js +index 551b884b615a094c50dec2a2c4acec5a4b303bcf..2b199700aa366e5f36c2bdc33ce8dbfc1e09998b 100644 +--- a/dist/SchemaBuilder.js ++++ b/dist/SchemaBuilder.js +@@ -182,6 +182,9 @@ class SchemaBuilder extends events_1.EventEmitter { + if (validationErrors.length) { + throw new AggregateError(validationErrors, `Schema construction failed due to ${validationErrors.length} validation failure(s). First failure was: ${String(validationErrors[0])}`); + } ++ if (this.options.releaseBuildStateAfterValidation === true) { ++ (0, makeNewBuild_ts_1.releaseBuildState)(build); ++ } + return schema; + } + } +diff --git a/dist/behavior.d.ts b/dist/behavior.d.ts +index ad1c639575544039b66298af8f63f9202a5d775d..54a8862e576da6046fe84583ef9e946f3644e3d1 100644 +--- a/dist/behavior.d.ts ++++ b/dist/behavior.d.ts +@@ -9,6 +9,7 @@ export type BehaviorDynamicMethods = { + [entityType in keyof GraphileBuild.BehaviorEntities as `${entityType}Behavior`]: (entity: GraphileBuild.BehaviorEntities[entityType]) => string; + }; + export declare class Behavior { ++ private _retirementState; + private behaviorEntities; + private behaviorRegistry; + behaviorEntityTypes: (keyof GraphileBuild.BehaviorEntities)[]; +@@ -55,6 +56,9 @@ export declare class Behavior { + private validateBehavior; + _defaultBehaviorByEntityTypeCache: Map<"string", string>; + getDefaultBehaviorFor(entityType: keyof GraphileBuild.BehaviorEntities): string; ++ /** @internal */ ++ releaseBuildState(): boolean; ++ private assertBuildStateLive; + } + export declare function joinBehaviors(strings: ReadonlyArray): GraphileBuild.BehaviorString; + export declare function joinResolvedBehaviors(behaviors: ReadonlyArray): ResolvedBehavior; +diff --git a/dist/behavior.js b/dist/behavior.js +index 80d7ef5543ac45e3922ff9d16413d3372e4d0f1d..42e7f46ae9057924ebb9e42619ee7c97913c4ebd 100644 +--- a/dist/behavior.js ++++ b/dist/behavior.js +@@ -10,6 +10,12 @@ const NULL_BEHAVIOR = Object.freeze({ + behaviorString: "", + stack: Object.freeze([]), + }); ++const BUILD_STATE_RELEASED_ERROR_CODE = "GRAPHILE_BUILD_STATE_RELEASED"; ++function buildStateReleasedError(api) { ++ const error = new Error(`Cannot use ${api} after Graphile build state has been released`); ++ error.code = BUILD_STATE_RELEASED_ERROR_CODE; ++ return error; ++} + const getEntityBehaviorHooks = (plugin, type) => { + const val = plugin.schema?.entityBehavior; + if (!val) +@@ -87,6 +93,7 @@ const getEntityBehaviorOverrideHooks = (plugin) => { + }; + class Behavior { + constructor(resolvedPreset, build) { ++ this._retirementState = { released: false }; + this.behaviorEntityTypes = []; + this._defaultBehaviorByEntityTypeCache = new Map(); + this.resolvedPreset = resolvedPreset; +@@ -202,6 +209,7 @@ class Behavior { + this[`${entityType}Behavior`] = (entity) => this.getBehaviorForEntity(entityType, entity).behaviorString; + } + assertEntity(entityType) { ++ this.assertBuildStateLive("build.behavior"); + if (entityType === "string") { + throw new Error(`Runtime behaviors cannot be attached to strings, please use 'behavior.stringMatches' directly.`); + } +@@ -216,6 +224,7 @@ class Behavior { + * @param filter - the behavior the plugin specifies + */ + entityMatches(entityType, entity, filter) { ++ this.assertBuildStateLive("build.behavior.entityMatches"); + if (!this.behaviorRegistry[filter]) { + // DIAGNOSTIC: enable for all filters + /* +@@ -317,6 +326,7 @@ class Behavior { + return behavior; + } + getPreferencesAppliedBehaviors(entityType, inferredBehavior, overrideBehavior) { ++ this.assertBuildStateLive("build.behavior.getPreferencesAppliedBehaviors"); + const defaultBehavior = this.getDefaultBehaviorFor(entityType); + const inferredBehaviorWithPreferencesApplied = multiplyBehavior(defaultBehavior, inferredBehavior.behaviorString, entityType); + const behaviorString = joinBehaviors([ +@@ -342,6 +352,7 @@ class Behavior { + } + /** @deprecated Please use entityMatches or stringMatches instead */ + matches(localBehaviorSpecsString, filter, defaultBehavior = "") { ++ this.assertBuildStateLive("build.behavior.matches"); + let err; + try { + throw new Error("Deprecated call happened here"); +@@ -369,6 +380,7 @@ class Behavior { + resolveBehavior(entityType, initialBehavior, + // Misnomer; also allows strings or nothings + callbacks, ...args) { ++ this.assertBuildStateLive("build.behavior.resolveBehavior"); + let behaviorString = initialBehavior.behaviorString; + const stack = [...initialBehavior.stack]; + for (const [source, rawG] of callbacks) { +@@ -425,6 +437,7 @@ class Behavior { + } + } + getDefaultBehaviorFor(entityType) { ++ this.assertBuildStateLive("build.behavior.getDefaultBehaviorFor"); + if (!this._defaultBehaviorByEntityTypeCache.has(entityType)) { + const supportedBehaviors = new Set(); + for (const [behaviorString, spec] of Object.entries(this.behaviorRegistry)) { +@@ -455,6 +468,38 @@ class Behavior { + } + return this._defaultBehaviorByEntityTypeCache.get(entityType); + } ++ assertBuildStateLive(api) { ++ if (this._retirementState.released) { ++ throw buildStateReleasedError(api); ++ } ++ } ++ releaseBuildState() { ++ if (this._retirementState.released) { ++ return false; ++ } ++ this._retirementState.released = true; ++ this._defaultBehaviorByEntityTypeCache.clear(); ++ this.behaviorEntityTypes.length = 0; ++ for (const key of Object.keys(this.behaviorEntities)) { ++ const behaviorEntity = this.behaviorEntities[key]; ++ behaviorEntity.listCache.clear(); ++ behaviorEntity.inferredCache.clear(); ++ behaviorEntity.overrideCache.clear(); ++ behaviorEntity.fullCache.clear(); ++ behaviorEntity.inferredBehaviorCallbacks.length = 0; ++ behaviorEntity.overrideBehaviorCallbacks.length = 0; ++ for (const behaviorString of Object.keys(behaviorEntity.behaviorStrings)) { ++ delete behaviorEntity.behaviorStrings[behaviorString]; ++ } ++ } ++ for (const key of Object.keys(this.behaviorRegistry)) { ++ delete this.behaviorRegistry[key]; ++ } ++ if (Array.isArray(this.globalDefaultBehavior.stack)) { ++ this.globalDefaultBehavior.stack.length = 0; ++ } ++ return true; ++ } + } + exports.Behavior = Behavior; + /** +diff --git a/dist/global.d.ts b/dist/global.d.ts +index d53e2d8648fe42b5e9811dc83455061a8d75e1a8..ace058ae10a3f4c5a9861a19431ab0fc44d520d0 100644 +--- a/dist/global.d.ts ++++ b/dist/global.d.ts +@@ -67,6 +67,14 @@ declare global { + muteWarnings?: boolean; + } + interface SchemaOptions { ++ /** ++ * Release schema-construction-only state after successful schema ++ * validation. Materialized schema execution remains available, but ++ * late plugin access to retired build APIs fails closed. ++ * ++ * @experimental ++ */ ++ releaseBuildStateAfterValidation?: boolean; + /** + * A behavior string to prepend to all behavior checks, can be overriden + * but is a great way to disable things by default and then re-enable +@@ -211,6 +219,18 @@ declare global { + * influence what types/fields/etc are added to the GraphQL schema. + */ + input: BuildInput; ++ /** ++ * Register synchronous cleanup for plugin-owned state that is used ++ * only while constructing the schema. Disposers run once, in reverse ++ * registration order, after successful schema validation when ++ * `releaseBuildStateAfterValidation` is enabled. ++ * ++ * Disposers must capture the exact state they own and must not call ++ * other build APIs, which fail closed while disposal is in progress. ++ * ++ * @experimental ++ */ ++ registerBuildStateDisposer(disposer: () => void): void; + /** + * Returns true if `Build.versions` contains an entry for `packageName` + * compatible with the version range `range`, false otherwise. +diff --git a/dist/index.d.ts b/dist/index.d.ts +index f6d0d7fd6d772f8433b3890f25b79369b565963f..57992c592558a61d231f5a92bced97d40b2ea246 100644 +--- a/dist/index.d.ts ++++ b/dist/index.d.ts +@@ -1,6 +1,7 @@ + import "./global.ts"; + import "./interfaces.ts"; + export { isValidBehaviorString } from "./behavior.ts"; ++export { BUILD_STATE_RELEASED_ERROR_CODE, isBuildStateReleased, releaseBuildState } from "./makeNewBuild.ts"; + import { AddNodeInterfaceToSuitableTypesPlugin, BuiltinScalarConnectionsPlugin, ClientMutationIdDescriptionPlugin, CommonBehaviorsPlugin, CommonTypesPlugin, CursorTypePlugin, MinifySchemaPlugin, MutationPayloadQueryPlugin, MutationPlugin, NodeAccessorPlugin, NodeIdCodecBase64JSONPlugin, NodeIdCodecPipeStringPlugin, NodePlugin, PageInfoStartEndCursorPlugin, QueryPlugin, QueryQueryPlugin, RegisterQueryNodePlugin, StreamDeferPlugin, SubscriptionPlugin, SwallowErrorsPlugin, TrimEmptyDescriptionsPlugin } from "./plugins/index.ts"; + import SchemaBuilder from "./SchemaBuilder.ts"; + export { camelCase, constantCase, constantCaseAll, EXPORTABLE, EXPORTABLE_ARRAY_CLONE, EXPORTABLE_OBJECT_CLONE, formatInsideUnderscores, gatherConfig, pluralize, singularize, upperCamelCase, upperFirst, } from "./utils.ts"; +diff --git a/dist/index.js b/dist/index.js +index 544e9961824999d10f191510c05a0bd1edb02349..6015ec06aff2ed5ec802f1e57bd082afdebbe3ab 100644 +--- a/dist/index.js ++++ b/dist/index.js +@@ -1,6 +1,6 @@ + "use strict"; + Object.defineProperty(exports, "__esModule", { value: true }); +-exports.version = exports.defaultPreset = exports.TrimEmptyDescriptionsPlugin = exports.SwallowErrorsPlugin = exports.SubscriptionPlugin = exports.StreamDeferPlugin = exports.RegisterQueryNodePlugin = exports.QueryQueryPlugin = exports.QueryPlugin = exports.PageInfoStartEndCursorPlugin = exports.NodePlugin = exports.NodeIdCodecPipeStringPlugin = exports.NodeIdCodecBase64JSONPlugin = exports.NodeAccessorPlugin = exports.MutationPlugin = exports.MutationPayloadQueryPlugin = exports.MinifySchemaPlugin = exports.CursorTypePlugin = exports.CommonTypesPlugin = exports.CommonBehaviorsPlugin = exports.ClientMutationIdDescriptionPlugin = exports.BuiltinScalarConnectionsPlugin = exports.AddNodeInterfaceToSuitableTypesPlugin = exports.buildSchema = exports.getBuilder = exports.watchGather = exports.gather = exports.buildInflection = exports.SchemaBuilder = exports.upperFirst = exports.upperCamelCase = exports.singularize = exports.pluralize = exports.gatherConfig = exports.formatInsideUnderscores = exports.EXPORTABLE_OBJECT_CLONE = exports.EXPORTABLE_ARRAY_CLONE = exports.EXPORTABLE = exports.constantCaseAll = exports.constantCase = exports.camelCase = exports.isValidBehaviorString = void 0; ++exports.version = exports.defaultPreset = exports.TrimEmptyDescriptionsPlugin = exports.SwallowErrorsPlugin = exports.SubscriptionPlugin = exports.StreamDeferPlugin = exports.RegisterQueryNodePlugin = exports.QueryQueryPlugin = exports.QueryPlugin = exports.PageInfoStartEndCursorPlugin = exports.NodePlugin = exports.NodeIdCodecPipeStringPlugin = exports.NodeIdCodecBase64JSONPlugin = exports.NodeAccessorPlugin = exports.MutationPlugin = exports.MutationPayloadQueryPlugin = exports.MinifySchemaPlugin = exports.CursorTypePlugin = exports.CommonTypesPlugin = exports.CommonBehaviorsPlugin = exports.ClientMutationIdDescriptionPlugin = exports.BuiltinScalarConnectionsPlugin = exports.AddNodeInterfaceToSuitableTypesPlugin = exports.buildSchema = exports.getBuilder = exports.watchGather = exports.gather = exports.buildInflection = exports.SchemaBuilder = exports.upperFirst = exports.upperCamelCase = exports.singularize = exports.pluralize = exports.gatherConfig = exports.formatInsideUnderscores = exports.EXPORTABLE_OBJECT_CLONE = exports.EXPORTABLE_ARRAY_CLONE = exports.EXPORTABLE = exports.constantCaseAll = exports.constantCase = exports.camelCase = exports.releaseBuildState = exports.isBuildStateReleased = exports.BUILD_STATE_RELEASED_ERROR_CODE = exports.isValidBehaviorString = void 0; + exports.makeSchema = makeSchema; + exports.watchSchema = watchSchema; + const tslib_1 = require("tslib"); +@@ -11,6 +11,10 @@ const graphql_1 = require("grafast/graphql"); + const graphile_config_1 = require("graphile-config"); + var behavior_ts_1 = require("./behavior.js"); + Object.defineProperty(exports, "isValidBehaviorString", { enumerable: true, get: function () { return behavior_ts_1.isValidBehaviorString; } }); ++var makeNewBuild_ts_1 = require("./makeNewBuild.js"); ++Object.defineProperty(exports, "BUILD_STATE_RELEASED_ERROR_CODE", { enumerable: true, get: function () { return makeNewBuild_ts_1.BUILD_STATE_RELEASED_ERROR_CODE; } }); ++Object.defineProperty(exports, "isBuildStateReleased", { enumerable: true, get: function () { return makeNewBuild_ts_1.isBuildStateReleased; } }); ++Object.defineProperty(exports, "releaseBuildState", { enumerable: true, get: function () { return makeNewBuild_ts_1.releaseBuildState; } }); + const extend_ts_1 = tslib_1.__importDefault(require("./extend.js")); + const inflection_ts_1 = require("./inflection.js"); + const index_ts_1 = require("./plugins/index.js"); +diff --git a/dist/makeNewBuild.d.ts b/dist/makeNewBuild.d.ts +index c294c17ab5486a4a8c7b7d448d5b921f4af5c2ff..5665d2b4a0173a24ec88694ff9be1c69fbed9d02 100644 +--- a/dist/makeNewBuild.d.ts ++++ b/dist/makeNewBuild.d.ts +@@ -1,5 +1,8 @@ + import "./global.ts"; + import type SchemaBuilder from "./SchemaBuilder.ts"; ++export declare const BUILD_STATE_RELEASED_ERROR_CODE: "GRAPHILE_BUILD_STATE_RELEASED"; ++export declare function isBuildStateReleased(build: GraphileBuild.Build): boolean; ++export declare function releaseBuildState(build: GraphileBuild.Build): boolean; + /** + * Makes a new 'Build' object suitable to be passed through the 'build' hook. + */ +diff --git a/dist/makeNewBuild.js b/dist/makeNewBuild.js +index 5a873bb569247840eeb110a754095236849fc35a..00b098b4c38b1e2dc36b9457c3af3e11ad887a8c 100644 +--- a/dist/makeNewBuild.js ++++ b/dist/makeNewBuild.js +@@ -1,5 +1,8 @@ + "use strict"; + Object.defineProperty(exports, "__esModule", { value: true }); ++exports.BUILD_STATE_RELEASED_ERROR_CODE = void 0; ++exports.isBuildStateReleased = isBuildStateReleased; ++exports.releaseBuildState = releaseBuildState; + exports.default = makeNewBuild; + const tslib_1 = require("tslib"); + require("./global.js"); +@@ -11,6 +14,57 @@ const extend_ts_1 = tslib_1.__importStar(require("./extend.js")); + const utils_ts_1 = require("./utils.js"); + const version_ts_1 = require("./version.js"); + const BUILTINS = ["Int", "Float", "Boolean", "ID", "String"]; ++exports.BUILD_STATE_RELEASED_ERROR_CODE = "GRAPHILE_BUILD_STATE_RELEASED"; ++const buildRetirementStates = new WeakMap(); ++function buildStateReleasedError(api) { ++ const error = new Error(`Cannot use ${api} after Graphile build state has been released`); ++ error.code = exports.BUILD_STATE_RELEASED_ERROR_CODE; ++ return error; ++} ++function assertBuildStateLive(build, api) { ++ if (isBuildStateReleased(build)) { ++ throw buildStateReleasedError(api); ++ } ++} ++function isBuildStateReleased(build) { ++ const state = buildRetirementStates.get(build); ++ return state !== undefined && state.status !== "live"; ++} ++function releaseBuildState(build) { ++ const state = buildRetirementStates.get(build); ++ if (state === undefined || state.status !== "live") { ++ return false; ++ } ++ state.status = "releasing"; ++ const errors = []; ++ const disposers = state.disposers.splice(0).reverse(); ++ for (const disposer of disposers) { ++ try { ++ disposer(); ++ } ++ catch (error) { ++ errors.push(error); ++ } ++ } ++ try { ++ state.releaseCore(); ++ } ++ catch (error) { ++ errors.push(error); ++ } ++ try { ++ build.behavior.releaseBuildState(); ++ } ++ catch (error) { ++ errors.push(error); ++ } ++ state.status = "released"; ++ state.releaseCore = null; ++ if (errors.length > 0) { ++ throw new AggregateError(errors, `Failed to release Graphile build state (${errors.length} disposal error${errors.length === 1 ? "" : "s"})`); ++ } ++ return true; ++} + /** Have we warned the user they're using the 5-arg deprecated registerObjectType call? */ + let registerObjectType5argsDeprecatedWarned = false; + /** +@@ -18,15 +72,16 @@ let registerObjectType5argsDeprecatedWarned = false; + */ + function makeNewBuild(builder, input, inflection) { + const lib = builder.resolvedPreset.lib ?? {}; +- const building = new Set(); +- const allTypes = { ++ let inputState = input; ++ let building = new Set(); ++ let allTypes = { + Int: graphql_1.GraphQLInt, + Float: graphql_1.GraphQLFloat, + String: graphql_1.GraphQLString, + Boolean: graphql_1.GraphQLBoolean, + ID: graphql_1.GraphQLID, + }; +- const allTypesSources = { ++ let allTypesSources = { + Int: "GraphQL Built-in", + Float: "GraphQL Built-in", + String: "GraphQL Built-in", +@@ -36,10 +91,11 @@ function makeNewBuild(builder, input, inflection) { + /** + * Where the type factories are; so we don't construct types until they're needed. + */ +- const typeRegistry = Object.create(null); +- const scopeByType = new Map(); ++ let typeRegistry = Object.create(null); ++ let scopeByType = new Map(); + // TODO: allow registering a previously constructed type. + function register(klass, typeName, scope, specGenerator, origin) { ++ assertBuildStateLive(build, "build.register*"); + if (!this.status.isBuildPhaseComplete || this.status.isInitPhaseComplete) { + throw new Error("Types may only be registered in the 'init' phase"); + } +@@ -83,7 +139,17 @@ function makeNewBuild(builder, input, inflection) { + graphql: lib.graphql.version, + "graphile-build": version_ts_1.version, + }, +- input, ++ get input() { ++ assertBuildStateLive(build, "build.input"); ++ return inputState; ++ }, ++ registerBuildStateDisposer(disposer) { ++ assertBuildStateLive(build, "build.registerBuildStateDisposer"); ++ if (typeof disposer !== "function") { ++ throw new TypeError("build.registerBuildStateDisposer requires a function"); ++ } ++ retirementState.disposers.push(disposer); ++ }, + hasVersion(packageName, range, options = { includePrerelease: true }) { + const packageVersion = this.versions[packageName]; + if (!packageVersion) +@@ -119,9 +185,13 @@ function makeNewBuild(builder, input, inflection) { + } + }, + getAllTypes() { ++ assertBuildStateLive(build, "build.getAllTypes"); + return allTypes; + }, +- scopeByType, ++ get scopeByType() { ++ assertBuildStateLive(build, "build.scopeByType"); ++ return scopeByType; ++ }, + inflection, + handleRecoverableError(e) { + e["recoverable"] = true; +@@ -186,6 +256,7 @@ function makeNewBuild(builder, input, inflection) { + }, + __postInitAssertions: [], + assertTypeName(typeName, deferrable = false) { ++ assertBuildStateLive(build, "build.assertTypeName"); + if (!this.status.isBuildPhaseComplete) { + throw new Error("Must not call build.assertTypeName before 'build' phase is complete; use 'init' phase instead"); + } +@@ -207,6 +278,7 @@ function makeNewBuild(builder, input, inflection) { + } + }, + getTypeMetaByName(typeName) { ++ assertBuildStateLive(build, "build.getTypeMetaByName"); + if (!this.status.isBuildPhaseComplete) { + throw new Error("Must not call build.getTypeMetaByName before 'build' phase is complete; use 'init' phase instead"); + } +@@ -253,6 +325,7 @@ function makeNewBuild(builder, input, inflection) { + return null; + }, + getTypeByName(typeName) { ++ assertBuildStateLive(build, "build.getTypeByName"); + if (currentTypeDetails && !BUILTINS.includes(typeName)) { + throw new Error(`Error in spec callback for ${currentTypeDetails.klass.name} '${currentTypeDetails.typeName}'; the callback made a call to \`build.getTypeByName(${JSON.stringify(typeName)})\` (directly or indirectly) - this is the wrong time for such a call \ + to occur since it can lead to circular dependence. To fix this, ensure that any \ +@@ -377,6 +450,25 @@ style for these configuration options (e.g. change \`interfaces: \ + }, + _pluginMeta: Object.create(null), + }; ++ const retirementState = { ++ status: "live", ++ disposers: [], ++ releaseCore() { ++ inputState = undefined; ++ building.clear(); ++ building = undefined; ++ allTypes = undefined; ++ allTypesSources = undefined; ++ typeRegistry = undefined; ++ scopeByType.clear(); ++ scopeByType = undefined; ++ build.__postInitAssertions.length = 0; ++ for (const key of Object.keys(build._pluginMeta)) { ++ delete build._pluginMeta[key]; ++ } ++ }, ++ }; ++ buildRetirementStates.set(build, retirementState); + return build; + } + let currentTypeDetails = null; diff --git a/patches/pg-introspection@1.0.1.patch b/patches/pg-introspection@1.0.1.patch new file mode 100644 index 0000000000..c12fa79220 --- /dev/null +++ b/patches/pg-introspection@1.0.1.patch @@ -0,0 +1,599 @@ +diff --git a/dist/index.d.ts b/dist/index.d.ts +index 8b378fb1edcdf727c35d86360b38baf042b29476..fb49ae92221e6052f48a702c9632ef4a439ea449 100644 +--- a/dist/index.d.ts ++++ b/dist/index.d.ts +@@ -1,5 +1,5 @@ + import type { Introspection, PgAttribute, PgAuthMembers, PgClass, PgConstraint, PgDatabase, PgDepend, PgDescription, PgEntity, PgEnum, PgExtension, PgIndex, PgInherits, PgLanguage, PgNamespace, PgProc, PgProcArgument, PgRange, PgRoles, PgType } from "./introspection.ts"; +-export { makeIntrospectionQuery } from "./introspection.ts"; ++export { makeIntrospectionQuery, makeSchemaScopedIntrospectionQuery, type SchemaScopedCatalogTypes, type SchemaScopedIntrospectionOptions } from "./introspection.ts"; + import type { AclObject } from "./acl.ts"; + import { aclContainsRole, entityPermissions, expandRoles, resolvePermissions } from "./acl.ts"; + import type { PgSmartTagsAndDescription, PgSmartTagsDict } from "./smartComments.ts"; +diff --git a/dist/index.js b/dist/index.js +index e74695ac2c2469f423637837ac454c26f1ea03d9..81044f8b02061d6db8716fd9f9bd65bac70974f9 100644 +--- a/dist/index.js ++++ b/dist/index.js +@@ -1,10 +1,11 @@ + "use strict"; + Object.defineProperty(exports, "__esModule", { value: true }); +-exports.resolvePermissions = exports.expandRoles = exports.entityPermissions = exports.aclContainsRole = exports.parseSmartComment = exports.reservedWords = exports.makeIntrospectionQuery = void 0; ++exports.resolvePermissions = exports.expandRoles = exports.entityPermissions = exports.aclContainsRole = exports.parseSmartComment = exports.reservedWords = exports.makeSchemaScopedIntrospectionQuery = exports.makeIntrospectionQuery = void 0; + exports.parseIntrospectionResults = parseIntrospectionResults; + const tslib_1 = require("tslib"); + var introspection_ts_1 = require("./introspection.js"); + Object.defineProperty(exports, "makeIntrospectionQuery", { enumerable: true, get: function () { return introspection_ts_1.makeIntrospectionQuery; } }); ++Object.defineProperty(exports, "makeSchemaScopedIntrospectionQuery", { enumerable: true, get: function () { return introspection_ts_1.makeSchemaScopedIntrospectionQuery; } }); + const acl_ts_1 = require("./acl.js"); + Object.defineProperty(exports, "aclContainsRole", { enumerable: true, get: function () { return acl_ts_1.aclContainsRole; } }); + Object.defineProperty(exports, "entityPermissions", { enumerable: true, get: function () { return acl_ts_1.entityPermissions; } }); +diff --git a/dist/introspection.d.ts b/dist/introspection.d.ts +index d4d1dd52e4e53dcbbd13c206b62309991ab38d2d..3461e09ca361c5ac7af991844c7d80595416cf1f 100644 +--- a/dist/introspection.d.ts ++++ b/dist/introspection.d.ts +@@ -1224,5 +1224,20 @@ export type PgEntity = PgDatabase | PgNamespace | PgClass | PgAttribute | PgCons + * Builds a PostgreSQL introspection SQL query to return an object with the same shape as `Introspection` above. + */ + export declare const makeIntrospectionQuery: () => string; ++/** ++ * Builds a parameterized introspection query scoped to the requested schemas ++ * and the transitive object dependencies required by their objects. ++ */ ++export type SchemaScopedCatalogTypes = "all" | "dependency-closure"; ++export interface SchemaScopedIntrospectionOptions { ++ /** Catalog type retention policy. Defaults to the compatibility-safe `all`. */ ++ catalogTypes?: SchemaScopedCatalogTypes; ++ /** Exact installed extension names whose optional capability metadata is required. */ ++ capabilityExtensions?: readonly string[]; ++} ++export declare const makeSchemaScopedIntrospectionQuery: (schemas: readonly string[], options?: SchemaScopedIntrospectionOptions) => { ++ text: string; ++ values: [string[], string[]]; ++}; + export {}; + //# sourceMappingURL=introspection.d.ts.map +diff --git a/dist/introspection.js b/dist/introspection.js +index b21ef287be6d529c1801bba6b7c2096758849884..19bfae9ba9dadac29a947e7d893aaa81a331ebeb 100644 +--- a/dist/introspection.js ++++ b/dist/introspection.js +@@ -5,13 +5,341 @@ + * DO NOT EDIT! + */ + Object.defineProperty(exports, "__esModule", { value: true }); +-exports.makeIntrospectionQuery = void 0; ++exports.makeSchemaScopedIntrospectionQuery = exports.makeIntrospectionQuery = void 0; ++const STOCK_NAMESPACE_PREDICATE = "nspname <> 'information_schema'"; ++const STOCK_OBJECT_NAMESPACE_PREDICATE = "in (select namespaces._id from namespaces where nspname <> 'information_schema' and nspname not like 'pg\\_%')"; ++const SCOPED_CTES = `recursive ++ requested_schema_names(schema_name) as ( ++ select distinct requested.schema_name ++ from pg_catalog.unnest($1::text[]) as requested(schema_name) ++ ), ++ ++ capability_extension_names(extension_name) as ( ++ select distinct capability.extension_name ++ from pg_catalog.unnest($2::text[]) as capability(extension_name) ++ ), ++ ++ requested_namespaces as ( ++ select pg_namespace.oid as _id, pg_namespace.nspname ++ from pg_catalog.pg_namespace ++ inner join requested_schema_names ++ on requested_schema_names.schema_name = pg_namespace.nspname ++ ), ++ ++ root_objects(object_class, object_id) as ( ++ select 'pg_catalog.pg_class'::regclass::oid, pg_class.oid ++ from pg_catalog.pg_class ++ where pg_class.relnamespace in (select requested_namespaces._id from requested_namespaces) ++ ++ union ++ ++ select 'pg_catalog.pg_constraint'::regclass::oid, pg_constraint.oid ++ from pg_catalog.pg_constraint ++ where pg_constraint.connamespace in (select requested_namespaces._id from requested_namespaces) ++ ++ union ++ ++ select 'pg_catalog.pg_proc'::regclass::oid, pg_proc.oid ++ from pg_catalog.pg_proc ++ where pg_proc.pronamespace in (select requested_namespaces._id from requested_namespaces) ++ and pg_proc.prorettype operator(pg_catalog.<>) 2279 ++ ++ union ++ ++ select 'pg_catalog.pg_type'::regclass::oid, pg_type.oid ++ from pg_catalog.pg_type ++ where pg_type.typnamespace in (select requested_namespaces._id from requested_namespaces) ++ ), ++ ++ object_closure(object_class, object_id) as ( ++ select root_objects.object_class, root_objects.object_id ++ from root_objects ++ ++ union ++ ++ select dependency.object_class, dependency.object_id ++ from object_closure ++ cross join lateral ( ++ select ++ 'pg_catalog.pg_type'::regclass::oid as object_class, ++ pg_class.reltype as object_id ++ from pg_catalog.pg_class ++ where object_closure.object_class = 'pg_catalog.pg_class'::regclass ++ and pg_class.oid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_type'::regclass::oid, pg_class.reloftype ++ from pg_catalog.pg_class ++ where object_closure.object_class = 'pg_catalog.pg_class'::regclass ++ and pg_class.oid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_type'::regclass::oid, pg_attribute.atttypid ++ from pg_catalog.pg_attribute ++ where object_closure.object_class = 'pg_catalog.pg_class'::regclass ++ and pg_attribute.attrelid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_constraint'::regclass::oid, pg_constraint.oid ++ from pg_catalog.pg_constraint ++ where object_closure.object_class = 'pg_catalog.pg_class'::regclass ++ and pg_constraint.conrelid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_class'::regclass::oid, pg_index.indexrelid ++ from pg_catalog.pg_index ++ where object_closure.object_class = 'pg_catalog.pg_class'::regclass ++ and pg_index.indrelid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_class'::regclass::oid, pg_inherits.inhparent ++ from pg_catalog.pg_inherits ++ where object_closure.object_class = 'pg_catalog.pg_class'::regclass ++ and pg_inherits.inhrelid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_class'::regclass::oid, constraint_class.oid ++ from pg_catalog.pg_constraint ++ cross join lateral pg_catalog.unnest( ++ array[ ++ pg_constraint.conrelid, ++ pg_constraint.confrelid, ++ pg_constraint.conindid ++ ]::oid[] ++ ) as constraint_class(oid) ++ where object_closure.object_class = 'pg_catalog.pg_constraint'::regclass ++ and pg_constraint.oid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_type'::regclass::oid, pg_constraint.contypid ++ from pg_catalog.pg_constraint ++ where object_closure.object_class = 'pg_catalog.pg_constraint'::regclass ++ and pg_constraint.oid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_constraint'::regclass::oid, pg_constraint.conparentid ++ from pg_catalog.pg_constraint ++ where object_closure.object_class = 'pg_catalog.pg_constraint'::regclass ++ and pg_constraint.oid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_type'::regclass::oid, procedure_type.oid ++ from pg_catalog.pg_proc ++ cross join lateral pg_catalog.unnest( ++ coalesce(pg_proc.proallargtypes, pg_proc.proargtypes::oid[]) ++ || array[pg_proc.prorettype]::oid[] ++ ) as procedure_type(oid) ++ where object_closure.object_class = 'pg_catalog.pg_proc'::regclass ++ and pg_proc.oid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_type'::regclass::oid, dependency_type.oid ++ from pg_catalog.pg_type ++ cross join lateral pg_catalog.unnest( ++ array[ ++ pg_type.typbasetype, ++ pg_type.typelem, ++ pg_type.typarray ++ ]::oid[] ++ ) as dependency_type(oid) ++ where object_closure.object_class = 'pg_catalog.pg_type'::regclass ++ and pg_type.oid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_class'::regclass::oid, pg_type.typrelid ++ from pg_catalog.pg_type ++ where object_closure.object_class = 'pg_catalog.pg_type'::regclass ++ and pg_type.oid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_constraint'::regclass::oid, pg_constraint.oid ++ from pg_catalog.pg_constraint ++ where object_closure.object_class = 'pg_catalog.pg_type'::regclass ++ and pg_constraint.contypid = object_closure.object_id ++ ++ union all ++ ++ select 'pg_catalog.pg_type'::regclass::oid, range_type.oid ++ from pg_catalog.pg_range ++ cross join lateral pg_catalog.unnest( ++ array[ ++ pg_range.rngtypid, ++ pg_range.rngsubtype, ++ pg_range.rngmultitypid ++ ]::oid[] ++ ) as range_type(oid) ++ where object_closure.object_class = 'pg_catalog.pg_type'::regclass ++ and object_closure.object_id in (pg_range.rngtypid, pg_range.rngmultitypid) ++ ) as dependency ++ where dependency.object_id operator(pg_catalog.<>) 0 ++ ), ++ ++ retained_index_metadata(indexrelid, indclass, indcollation) as ( ++ select pg_index.indexrelid, pg_index.indclass, pg_index.indcollation ++ from object_closure ++ inner join pg_catalog.pg_class retained_index ++ on object_closure.object_class = 'pg_catalog.pg_class'::regclass ++ and retained_index.oid = object_closure.object_id ++ and retained_index.relkind in ('i', 'I') ++ inner join pg_catalog.pg_index ++ on pg_index.indexrelid = retained_index.oid ++ ), ++ ++ retained_index_opclasses(_id, opcfamily) as ( ++ select pg_opclass.oid, pg_opclass.opcfamily ++ from retained_index_metadata ++ cross join lateral pg_catalog.unnest( ++ retained_index_metadata.indclass::oid[] ++ ) as index_opclass(_id) ++ inner join pg_catalog.pg_opclass ++ on pg_opclass.oid = index_opclass._id ++ ), ++ ++ retained_index_support_objects(object_class, object_id) as ( ++ select 'pg_catalog.pg_opclass'::regclass::oid, retained_index_opclasses._id ++ from retained_index_opclasses ++ ++ union ++ ++ select 'pg_catalog.pg_opfamily'::regclass::oid, retained_index_opclasses.opcfamily ++ from retained_index_opclasses ++ ++ union ++ ++ select 'pg_catalog.pg_operator'::regclass::oid, pg_amop.amopopr ++ from retained_index_opclasses ++ inner join pg_catalog.pg_amop ++ on pg_amop.amopfamily = retained_index_opclasses.opcfamily ++ ++ union ++ ++ select 'pg_catalog.pg_proc'::regclass::oid, pg_amproc.amproc ++ from retained_index_opclasses ++ inner join pg_catalog.pg_amproc ++ on pg_amproc.amprocfamily = retained_index_opclasses.opcfamily ++ ++ union ++ ++ select 'pg_catalog.pg_collation'::regclass::oid, index_collation._id ++ from retained_index_metadata ++ cross join lateral pg_catalog.unnest( ++ retained_index_metadata.indcollation::oid[] ++ ) as index_collation(_id) ++ where index_collation._id operator(pg_catalog.<>) 0 ++ ), ++ ++ installed_extensions(_id, extnamespace) as ( ++ select pg_extension.oid, pg_extension.extnamespace ++ from pg_catalog.pg_extension ++ where pg_extension.extname in ( ++ select capability_extension_names.extension_name ++ from capability_extension_names ++ ) ++ or exists ( ++ select 1 ++ from object_closure ++ inner join pg_catalog.pg_depend ++ on pg_depend.classid = object_closure.object_class ++ and pg_depend.objid = object_closure.object_id ++ and pg_depend.refclassid = 'pg_catalog.pg_extension'::regclass ++ and pg_depend.refobjid = pg_extension.oid ++ and pg_depend.deptype = 'e' ++ ) ++ or exists ( ++ select 1 ++ from retained_index_support_objects ++ inner join pg_catalog.pg_depend ++ on pg_depend.classid = retained_index_support_objects.object_class ++ and pg_depend.objid = retained_index_support_objects.object_id ++ and pg_depend.refclassid = 'pg_catalog.pg_extension'::regclass ++ and pg_depend.refobjid = pg_extension.oid ++ and pg_depend.deptype = 'e' ++ ) ++ or exists ( ++ select 1 ++ from object_closure ++ inner join pg_catalog.pg_class retained_index ++ on object_closure.object_class = 'pg_catalog.pg_class'::regclass ++ and retained_index.oid = object_closure.object_id ++ and retained_index.relkind = 'i' ++ inner join pg_catalog.pg_depend ++ on pg_depend.classid = 'pg_catalog.pg_am'::regclass ++ and pg_depend.objid = retained_index.relam ++ and pg_depend.refclassid = 'pg_catalog.pg_extension'::regclass ++ and pg_depend.refobjid = pg_extension.oid ++ and pg_depend.deptype = 'e' ++ ) ++ ), ++ ++ scoped_namespaces(_id) as ( ++ select requested_namespaces._id ++ from requested_namespaces ++ ++ union ++ ++ select pg_class.relnamespace ++ from object_closure ++ inner join pg_catalog.pg_class ++ on object_closure.object_class = 'pg_catalog.pg_class'::regclass ++ and pg_class.oid = object_closure.object_id ++ ++ union ++ ++ select pg_constraint.connamespace ++ from object_closure ++ inner join pg_catalog.pg_constraint ++ on object_closure.object_class = 'pg_catalog.pg_constraint'::regclass ++ and pg_constraint.oid = object_closure.object_id ++ ++ union ++ ++ select pg_proc.pronamespace ++ from object_closure ++ inner join pg_catalog.pg_proc ++ on object_closure.object_class = 'pg_catalog.pg_proc'::regclass ++ and pg_proc.oid = object_closure.object_id ++ ++ union ++ ++ select pg_type.typnamespace ++ from object_closure ++ inner join pg_catalog.pg_type ++ on object_closure.object_class = 'pg_catalog.pg_type'::regclass ++ and pg_type.oid = object_closure.object_id ++ ++ union ++ ++ select installed_extensions.extnamespace ++ from installed_extensions ++ where installed_extensions.extnamespace operator(pg_catalog.<>) 0 ++ ++ union ++ ++ select pg_namespace.oid ++ from pg_catalog.pg_namespace ++ where pg_namespace.nspname = 'pg_catalog' ++ ), ++ ++`; + // We might want this to take options in future, so we've made it a function. + /** + * Builds a PostgreSQL introspection SQL query to return an object with the same shape as `Introspection` above. + */ +-const makeIntrospectionQuery = () => `\ ++const buildIntrospectionQuery = (scope) => `\ + with ++${scope?.ctes ?? ""}\ + database as ( + select pg_database.oid as _id, * + from pg_catalog.pg_database +@@ -21,14 +349,14 @@ with + namespaces as ( + select pg_namespace.oid as _id, * + from pg_catalog.pg_namespace +- where nspname <> 'information_schema' ++ where ${scope?.namespacePredicate ?? STOCK_NAMESPACE_PREDICATE} + ), + + classes as ( + select pg_class.oid as _id, *, + pg_catalog.pg_relation_is_updatable(oid, true)::bit(8)::int4 as "updatable_mask" + from pg_catalog.pg_class +- where relnamespace in (select namespaces._id from namespaces where nspname <> 'information_schema' and nspname not like 'pg\\_%') ++ where ${scope?.classPredicate ?? `relnamespace ${STOCK_OBJECT_NAMESPACE_PREDICATE}`} + ), + + attributes as ( +@@ -40,32 +368,34 @@ with + constraints as ( + select pg_constraint.oid as _id, * + from pg_catalog.pg_constraint +- where connamespace in (select namespaces._id from namespaces where nspname <> 'information_schema' and nspname not like 'pg\\_%') ++ where ${scope?.constraintPredicate ?? `connamespace ${STOCK_OBJECT_NAMESPACE_PREDICATE}`} + ), + + procs as ( + select pg_proc.oid as _id, * + from pg_catalog.pg_proc +- where pronamespace in (select namespaces._id from namespaces where nspname <> 'information_schema' and nspname not like 'pg\\_%') ++ where ${scope?.procPredicate ?? `pronamespace ${STOCK_OBJECT_NAMESPACE_PREDICATE}`} + and prorettype operator(pg_catalog.<>) 2279 + ), + + roles as ( + select pg_roles.oid as _id, * + from pg_catalog.pg_roles ++${scope?.rolePredicate ? ` where ${scope.rolePredicate} ++` : ""}\ + ), + + auth_members as ( + select * + from pg_catalog.pg_auth_members +- where roleid in (select roles._id from roles) ++ where ${scope?.authMemberPredicate ?? "roleid in (select roles._id from roles)"} + ), + + types as ( + select pg_type.oid as _id, * + from pg_catalog.pg_type +- where (typnamespace in (select namespaces._id from namespaces where nspname <> 'information_schema' and nspname not like 'pg\\_%')) +- or (typnamespace = 'pg_catalog'::regnamespace) ++ where ${scope?.typePredicate ?? `(typnamespace ${STOCK_OBJECT_NAMESPACE_PREDICATE}) ++ or (typnamespace = 'pg_catalog'::regnamespace)`} + ), + + enums as ( +@@ -77,6 +407,8 @@ with + extensions as ( + select pg_extension.oid as _id, * + from pg_catalog.pg_extension ++${scope?.extensionPredicate ? ` where ${scope.extensionPredicate} ++` : ""}\ + ), + + indexes as ( +@@ -94,6 +426,8 @@ with + languages as ( + select pg_language.oid as _id, * + from pg_catalog.pg_language ++${scope?.languagePredicate ? ` where ${scope.languagePredicate} ++` : ""}\ + ), + + policies as ( +@@ -141,7 +475,7 @@ with + am as ( + select pg_am.oid as _id, * + from pg_catalog.pg_am +- where true ++ where ${scope?.accessMethodPredicate ?? "true"} + ) + select json_build_object( + 'database', +@@ -221,5 +555,66 @@ select json_build_object( + 1 + )::text as introspection + `; ++const makeIntrospectionQuery = () => buildIntrospectionQuery(); + exports.makeIntrospectionQuery = makeIntrospectionQuery; ++/** ++ * Builds a parameterized introspection query scoped to the requested schemas ++ * and the transitive object dependencies required by their objects. ++ */ ++const makeSchemaScopedIntrospectionQuery = (schemas, options = {}) => { ++ if (!Array.isArray(schemas) || schemas.length === 0) { ++ throw new Error("Schema-scoped introspection requires at least one schema"); ++ } ++ if (options === null || typeof options !== "object" || Array.isArray(options)) { ++ throw new Error("Schema-scoped introspection options must be an object"); ++ } ++ const unsupportedOptions = Object.keys(options).filter((key) => key !== "catalogTypes" && key !== "capabilityExtensions"); ++ if (unsupportedOptions.length > 0) { ++ throw new Error(`Unsupported schema-scoped introspection option(s): ${unsupportedOptions.join(", ")}`); ++ } ++ const catalogTypes = options.catalogTypes ?? "all"; ++ if (catalogTypes !== "all" && catalogTypes !== "dependency-closure") { ++ throw new Error(`Unsupported schema-scoped catalog type policy '${catalogTypes}'`); ++ } ++ const capabilityExtensions = options.capabilityExtensions ?? []; ++ if (!Array.isArray(capabilityExtensions)) { ++ throw new Error("Schema-scoped introspection capabilityExtensions must be an array"); ++ } ++ const normalizedCapabilityExtensions = Array.from(new Set(capabilityExtensions.map((extension) => { ++ if (typeof extension !== "string" || extension.length === 0 || extension.trim() !== extension || extension.includes("\0")) { ++ throw new Error("Schema-scoped introspection capabilityExtensions must contain exact non-empty extension names"); ++ } ++ return extension; ++ }))); ++ const normalized = Array.from(new Set(schemas.map((schema) => { ++ if (typeof schema !== "string" || schema.length === 0) { ++ throw new Error("Schema-scoped introspection schemas must be non-empty strings"); ++ } ++ if (schema.includes("\0")) { ++ throw new Error("Schema-scoped introspection schemas must not contain NUL bytes"); ++ } ++ if (schema === "information_schema" || schema.startsWith("pg_")) { ++ throw new Error(`Schema-scoped introspection cannot expose system schema '${schema}'`); ++ } ++ return schema; ++ }))); ++ const dependencyClosureTypePredicate = "pg_type.oid = any (array(select object_id from object_closure where object_class = 'pg_catalog.pg_type'::regclass))"; ++ return { ++ text: buildIntrospectionQuery({ ++ ctes: SCOPED_CTES, ++ namespacePredicate: "pg_namespace.oid = any (array(select scoped_namespaces._id from scoped_namespaces))", ++ classPredicate: "pg_class.oid = any (array(select object_id from object_closure where object_class = 'pg_catalog.pg_class'::regclass))", ++ constraintPredicate: "pg_constraint.oid = any (array(select object_id from object_closure where object_class = 'pg_catalog.pg_constraint'::regclass))", ++ procPredicate: "pg_proc.oid = any (array(select object_id from object_closure where object_class = 'pg_catalog.pg_proc'::regclass))", ++ typePredicate: catalogTypes === "all" ++ ? `${dependencyClosureTypePredicate} or pg_type.typnamespace = 'pg_catalog'::regnamespace` ++ : dependencyClosureTypePredicate, ++ extensionPredicate: "pg_extension.oid = any (array(select installed_extensions._id from installed_extensions))", ++ languagePredicate: "true", ++ accessMethodPredicate: "true", ++ }), ++ values: [normalized, normalizedCapabilityExtensions], ++ }; ++}; ++exports.makeSchemaScopedIntrospectionQuery = makeSchemaScopedIntrospectionQuery; + //# sourceMappingURL=introspection.js.map +diff --git a/dist/augmentIntrospection.js b/dist/augmentIntrospection.js +--- a/dist/augmentIntrospection.js ++++ b/dist/augmentIntrospection.js +@@ -7,16 +7,28 @@ const smartComments_ts_1 = require("./smartComments.js"); + /** + * Only suitable for functions that accept no arguments. + */ +-function memo(fn) { +- let cache; +- let called = false; +- return () => { +- if (!called) { +- cache = fn(); +- called = true; ++function createMemoDispatcher() { ++ const resolvers = []; ++ const values = []; ++ const resolved = []; ++ // Bound functions retain only their integer slot. Resolver state lives in ++ // compact per-introspection arrays instead of one wrapper closure context ++ // per helper. An arrow keeps the original helpers non-constructable. ++ const dispatchMemo = (index) => { ++ if (resolved[index] !== true) { ++ const value = resolvers[index](); ++ values[index] = value; ++ resolved[index] = true; ++ // A resolved helper no longer needs to retain its resolver closure. ++ resolvers[index] = undefined; + } +- return cache; ++ return values[index]; + }; ++ return (resolver) => { ++ const index = resolvers.length; ++ resolvers.push(resolver); ++ return dispatchMemo.bind(undefined, index); ++ }; + } + function del(toDelete, collection, attr) { + for (let i = collection.length - 1; i >= 0; i--) { +@@ -176,5 +188,6 @@ function augmentIntrospection(introspectionResultsString, includeExtensionResour + } + /** @internal */ + function augmentIntrospectionParsed(introspection, includeExtensionResources = false) { ++ const memo = createMemoDispatcher(); + introspection._lookups = createLookups(introspection); + introspection._caches = createCaches(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d8cea89353..67a3289f5a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,23 @@ overrides: packageExtensionsChecksum: sha256-x8B4zkJ4KLRX+yspUWxuggXWlz6zrBLSIh72pNhpPiE= +patchedDependencies: + '@dataplan/pg@1.0.3': + hash: 1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb + path: patches/@dataplan__pg@1.0.3.patch + '@graphile-contrib/pg-many-to-many@2.0.0-rc.2': + hash: 71202d598d0d80e79890b8fbd8a1fc60cee20622dca60e59a0a6e994b6ab92c8 + path: patches/@graphile-contrib__pg-many-to-many@2.0.0-rc.2.patch + graphile-build-pg@5.0.2: + hash: 869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624 + path: patches/graphile-build-pg.patch + graphile-build@5.0.2: + hash: f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6 + path: patches/graphile-build@5.0.2.patch + pg-introspection@1.0.1: + hash: 987dfb1b6491b9423e890d57722efdb618efce72183de5dd25c3fde11189fbc4 + path: patches/pg-introspection@1.0.1.patch + importers: .: @@ -343,22 +360,22 @@ importers: version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 graphile-utils: specifier: 5.0.0 - version: 5.0.0(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1) + version: 5.0.0(421ca45f0ba699c78ec7eea2d25ec7d4) graphql: specifier: 16.13.0 version: 16.13.0 postgraphile: specifier: 5.0.3 - version: 5.0.3(bc368b92b12e127d4eb1f1e143433708) + version: 5.0.3(8e984f3e9cfc73b4b77b478748651d25) devDependencies: '@types/node': specifier: ^22.19.11 @@ -372,16 +389,19 @@ importers: dependencies: '@dataplan/pg': specifier: 1.0.3 - version: 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + version: 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + '@pgsql/quotes': + specifier: ^18.1.0 + version: 18.2.1 grafast: specifier: 1.0.2 version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -393,7 +413,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(bc368b92b12e127d4eb1f1e143433708) + version: 5.0.3(8e984f3e9cfc73b4b77b478748651d25) devDependencies: '@types/node': specifier: ^22.19.11 @@ -434,7 +454,7 @@ importers: version: link:../../postgres/pg-cache/dist postgraphile: specifier: 5.0.3 - version: 5.0.3(02c18e7c6179c4ae54031f8cdca16ab5) + version: 5.0.3(2fb27169ec8b91d10ab754f66c750f53) devDependencies: '@types/express': specifier: ^5.0.6 @@ -454,13 +474,13 @@ importers: dependencies: '@dataplan/pg': specifier: 1.0.3 - version: 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + version: 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -475,7 +495,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(f0ca433f04a3f28b00a29892b7510c70) + version: 5.0.3(65ecfcf3543e7c87a3dabce321c8b0d1) devDependencies: '@types/node': specifier: ^22.19.11 @@ -498,7 +518,7 @@ importers: version: link:../../postgres/query-builder/dist '@dataplan/pg': specifier: 1.0.3 - version: 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + version: 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) '@pgpmjs/logger': specifier: workspace:^ version: link:../../pgpm/logger/dist @@ -507,10 +527,10 @@ importers: version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -548,16 +568,16 @@ importers: dependencies: '@dataplan/pg': specifier: 1.0.3 - version: 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + version: 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) grafast: specifier: 1.0.2 version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -569,7 +589,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(f0a861a74cc4311fffaf615b439fe994) + version: 5.0.3(df995cf62d46f35f4829b81730cdf6da) devDependencies: '@types/node': specifier: ^22.19.11 @@ -595,7 +615,10 @@ importers: dependencies: '@dataplan/pg': specifier: 1.0.3 - version: 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + version: 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + '@pgsql/quotes': + specifier: ^18.1.0 + version: 18.2.1 accept-language-parser: specifier: ^1.5.0 version: 1.5.0 @@ -604,10 +627,10 @@ importers: version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -619,7 +642,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(f0a861a74cc4311fffaf615b439fe994) + version: 5.0.3(df995cf62d46f35f4829b81730cdf6da) devDependencies: '@types/accept-language-parser': specifier: ^1.5.4 @@ -657,16 +680,19 @@ importers: version: link:../../packages/llm-env/dist '@dataplan/pg': specifier: 1.0.3 - version: 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + version: 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + '@pgsql/quotes': + specifier: ^18.1.0 + version: 18.2.1 grafast: specifier: 1.0.2 version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-cache: specifier: workspace:^ version: link:../graphile-cache/dist @@ -675,7 +701,7 @@ importers: version: 1.0.1 graphile-utils: specifier: 5.0.0 - version: 5.0.0(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1) + version: 5.0.0(421ca45f0ba699c78ec7eea2d25ec7d4) graphql: specifier: 16.13.0 version: 16.13.0 @@ -684,7 +710,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(bc368b92b12e127d4eb1f1e143433708) + version: 5.0.3(8e984f3e9cfc73b4b77b478748651d25) devDependencies: '@types/node': specifier: ^22.19.11 @@ -710,22 +736,22 @@ importers: dependencies: '@dataplan/pg': specifier: 1.0.3 - version: 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + version: 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) grafast: specifier: 1.0.2 version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 graphile-connection-filter: specifier: ^1.13.0 - version: 1.14.0(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(postgraphile@5.0.3(bc368b92b12e127d4eb1f1e143433708)) + version: 1.14.0(1cb56828c00532c0532fac2b893bbf72) graphql: specifier: 16.13.0 version: 16.13.0 @@ -734,7 +760,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(bc368b92b12e127d4eb1f1e143433708) + version: 5.0.3(8e984f3e9cfc73b4b77b478748651d25) devDependencies: '@types/node': specifier: ^22.19.11 @@ -754,10 +780,10 @@ importers: dependencies: graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -766,7 +792,7 @@ importers: version: 16.13.0 postgraphile: specifier: 5.0.3 - version: 5.0.3(f0a861a74cc4311fffaf615b439fe994) + version: 5.0.3(df995cf62d46f35f4829b81730cdf6da) devDependencies: '@types/node': specifier: ^22.19.11 @@ -780,22 +806,22 @@ importers: dependencies: '@dataplan/pg': specifier: 1.0.3 - version: 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + version: 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) grafast: specifier: 1.0.2 version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 graphile-connection-filter: specifier: ^1.13.0 - version: 1.14.0(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(postgraphile@5.0.3(bc368b92b12e127d4eb1f1e143433708)) + version: 1.14.0(1cb56828c00532c0532fac2b893bbf72) graphile-plugin-utils: specifier: workspace:^ version: link:../graphile-plugin-utils/dist @@ -807,7 +833,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(bc368b92b12e127d4eb1f1e143433708) + version: 5.0.3(8e984f3e9cfc73b4b77b478748651d25) devDependencies: '@types/node': specifier: ^22.19.11 @@ -827,13 +853,13 @@ importers: dependencies: '@dataplan/pg': specifier: 1.0.3 - version: 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + version: 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -845,7 +871,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(f0a861a74cc4311fffaf615b439fe994) + version: 5.0.3(df995cf62d46f35f4829b81730cdf6da) devDependencies: '@types/node': specifier: ^22.19.11 @@ -859,22 +885,22 @@ importers: dependencies: '@dataplan/pg': specifier: 1.0.3 - version: 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + version: 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) grafast: specifier: 1.0.2 version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 graphile-connection-filter: specifier: ^1.13.0 - version: 1.14.0(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(postgraphile@5.0.3(f0ca433f04a3f28b00a29892b7510c70)) + version: 1.14.0(3cdce88baf7cc47d1e7a2ba13375cd59) graphql: specifier: 16.13.0 version: 16.13.0 @@ -883,7 +909,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(f0ca433f04a3f28b00a29892b7510c70) + version: 5.0.3(65ecfcf3543e7c87a3dabce321c8b0d1) devDependencies: '@types/geojson': specifier: ^7946.0.14 @@ -921,16 +947,16 @@ importers: version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 graphile-utils: specifier: 5.0.0 - version: 5.0.0(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1) + version: 5.0.0(421ca45f0ba699c78ec7eea2d25ec7d4) graphql: specifier: 16.13.0 version: 16.13.0 @@ -939,7 +965,7 @@ importers: version: 11.2.7 postgraphile: specifier: 5.0.3 - version: 5.0.3(bc368b92b12e127d4eb1f1e143433708) + version: 5.0.3(8e984f3e9cfc73b4b77b478748651d25) devDependencies: '@constructive-io/s3-utils': specifier: workspace:^ @@ -959,10 +985,10 @@ importers: version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -977,7 +1003,7 @@ importers: version: 8.21.0 postgraphile: specifier: 5.0.3 - version: 5.0.3(02c18e7c6179c4ae54031f8cdca16ab5) + version: 5.0.3(2fb27169ec8b91d10ab754f66c750f53) devDependencies: '@types/pg': specifier: ^8.20.0 @@ -1003,22 +1029,22 @@ importers: version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 graphile-utils: specifier: 5.0.0 - version: 5.0.0(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1) + version: 5.0.0(421ca45f0ba699c78ec7eea2d25ec7d4) graphql: specifier: 16.13.0 version: 16.13.0 postgraphile: specifier: 5.0.3 - version: 5.0.3(bc368b92b12e127d4eb1f1e143433708) + version: 5.0.3(8e984f3e9cfc73b4b77b478748651d25) devDependencies: '@types/node': specifier: ^22.19.11 @@ -1035,10 +1061,10 @@ importers: version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -1084,7 +1110,7 @@ importers: version: link:../../postgres/pgsql-test/dist postgraphile: specifier: 5.0.3 - version: 5.0.3(02c18e7c6179c4ae54031f8cdca16ab5) + version: 5.0.3(2fb27169ec8b91d10ab754f66c750f53) publishDirectory: dist graphile/graphile-schema: @@ -1094,7 +1120,7 @@ importers: version: 4.3.1 graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -1126,13 +1152,16 @@ importers: dependencies: '@dataplan/pg': specifier: 1.0.3 - version: 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + version: 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + '@pgsql/quotes': + specifier: ^18.1.0 + version: 18.2.1 graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -1147,7 +1176,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(f0ca433f04a3f28b00a29892b7510c70) + version: 5.0.3(65ecfcf3543e7c87a3dabce321c8b0d1) devDependencies: '@types/node': specifier: ^22.19.11 @@ -1200,10 +1229,10 @@ importers: version: 1.0.0(grafast@1.0.2(graphql@16.13.0)) '@dataplan/pg': specifier: 1.0.3 - version: 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + version: 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) '@graphile-contrib/pg-many-to-many': specifier: 2.0.0-rc.2 - version: 2.0.0-rc.2 + version: 2.0.0-rc.2(patch_hash=71202d598d0d80e79890b8fbd8a1fc60cee20622dca60e59a0a6e994b6ab92c8) '@pgpmjs/logger': specifier: workspace:^ version: link:../../pgpm/logger/dist @@ -1230,10 +1259,10 @@ importers: version: link:../graphile-bucket-provisioner-plugin/dist graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-bulk-mutations: specifier: workspace:^ version: link:../graphile-bulk-mutations/dist @@ -1278,7 +1307,7 @@ importers: version: link:../graphile-upload-plugin/dist graphile-utils: specifier: 5.0.1 - version: 5.0.1(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1) + version: 5.0.1(421ca45f0ba699c78ec7eea2d25ec7d4) graphql: specifier: 16.13.0 version: 16.13.0 @@ -1299,7 +1328,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(02c18e7c6179c4ae54031f8cdca16ab5) + version: 5.0.3(2fb27169ec8b91d10ab754f66c750f53) request-ip: specifier: ^3.3.0 version: 3.3.0 @@ -1340,10 +1369,10 @@ importers: version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -1381,10 +1410,10 @@ importers: version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -1402,7 +1431,7 @@ importers: version: link:../../postgres/pgsql-test/dist postgraphile: specifier: 5.0.3 - version: 5.0.3(02c18e7c6179c4ae54031f8cdca16ab5) + version: 5.0.3(2fb27169ec8b91d10ab754f66c750f53) devDependencies: '@types/pg': specifier: ^8.20.0 @@ -1419,10 +1448,10 @@ importers: dependencies: graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -1431,7 +1460,7 @@ importers: version: 16.13.0 postgraphile: specifier: 5.0.3 - version: 5.0.3(f0ca433f04a3f28b00a29892b7510c70) + version: 5.0.3(65ecfcf3543e7c87a3dabce321c8b0d1) devDependencies: '@types/node': specifier: ^22.19.11 @@ -1570,10 +1599,10 @@ importers: version: 1.0.0(@types/node@25.9.1)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(use-sync-external-store@1.6.0(react@19.2.5))(ws@8.20.1) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-cache: specifier: workspace:^ version: link:../../graphile/graphile-cache/dist @@ -1585,7 +1614,7 @@ importers: version: link:../../graphile/graphile-settings/dist graphile-utils: specifier: 5.0.1 - version: 5.0.1(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1) + version: 5.0.1(421ca45f0ba699c78ec7eea2d25ec7d4) graphql: specifier: 16.13.0 version: 16.13.0 @@ -1603,7 +1632,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(02c18e7c6179c4ae54031f8cdca16ab5) + version: 5.0.3(2fb27169ec8b91d10ab754f66c750f53) devDependencies: '@types/express': specifier: ^5.0.6 @@ -1676,7 +1705,7 @@ importers: version: link:../../postgres/pg-env/dist postgraphile: specifier: 5.0.3 - version: 5.0.3(02c18e7c6179c4ae54031f8cdca16ab5) + version: 5.0.3(2fb27169ec8b91d10ab754f66c750f53) devDependencies: '@types/express': specifier: ^5.0.6 @@ -1820,7 +1849,7 @@ importers: version: 1.0.2(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -1841,7 +1870,7 @@ importers: version: 11.2.7 postgraphile: specifier: 5.0.3 - version: 5.0.3(02c18e7c6179c4ae54031f8cdca16ab5) + version: 5.0.3(2fb27169ec8b91d10ab754f66c750f53) devDependencies: makage: specifier: ^0.3.0 @@ -1893,10 +1922,10 @@ importers: version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -1920,7 +1949,7 @@ importers: version: 8.20.0 postgraphile: specifier: 5.0.3 - version: 5.0.3(4b1f938b62eff937cb4c255aac53ddd9) + version: 5.0.3(260a0a3f37f6479930862ca6ba480e4c) ws: specifier: ^8.20.0 version: 8.20.1 @@ -1973,7 +2002,7 @@ importers: version: link:../../packages/url-domains/dist '@graphile-contrib/pg-many-to-many': specifier: 2.0.0-rc.2 - version: 2.0.0-rc.2 + version: 2.0.0-rc.2(patch_hash=71202d598d0d80e79890b8fbd8a1fc60cee20622dca60e59a0a6e994b6ab92c8) '@pgpmjs/env': specifier: workspace:^ version: link:../../pgpm/env/dist @@ -2006,10 +2035,10 @@ importers: version: 1.0.0(@types/node@25.9.1)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(use-sync-external-store@1.6.0(react@19.2.5))(ws@8.20.1) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-cache: specifier: workspace:^ version: link:../../graphile/graphile-cache/dist @@ -2019,12 +2048,15 @@ importers: graphile-function-bindings: specifier: workspace:^ version: link:../../graphile/graphile-function-bindings/dist + graphile-realtime-subscriptions: + specifier: workspace:^ + version: link:../../graphile/graphile-realtime-subscriptions/dist graphile-settings: specifier: workspace:^ version: link:../../graphile/graphile-settings/dist graphile-utils: specifier: 5.0.1 - version: 5.0.1(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1) + version: 5.0.1(421ca45f0ba699c78ec7eea2d25ec7d4) graphql: specifier: 16.13.0 version: 16.13.0 @@ -2051,7 +2083,7 @@ importers: version: 5.0.1 postgraphile: specifier: 5.0.3 - version: 5.0.3(02c18e7c6179c4ae54031f8cdca16ab5) + version: 5.0.3(2fb27169ec8b91d10ab754f66c750f53) request-ip: specifier: ^3.3.0 version: 3.3.0 @@ -2092,6 +2124,9 @@ importers: nodemon: specifier: ^3.1.14 version: 3.1.14 + pg-introspection: + specifier: 1.0.1 + version: 1.0.1(patch_hash=987dfb1b6491b9423e890d57722efdb618efce72183de5dd25c3fde11189fbc4) supertest: specifier: ^7.2.2 version: 7.2.2 @@ -2202,10 +2237,10 @@ importers: version: 1.0.2(graphql@16.13.0) graphile-build: specifier: 5.0.2 - version: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + version: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-build-pg: specifier: 5.0.2 - version: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: specifier: 1.0.1 version: 1.0.1 @@ -2229,7 +2264,7 @@ importers: version: link:../../postgres/pgsql-test/dist postgraphile: specifier: 5.0.3 - version: 5.0.3(02c18e7c6179c4ae54031f8cdca16ab5) + version: 5.0.3(2fb27169ec8b91d10ab754f66c750f53) typescript: specifier: ^5.0.0 version: 5.9.3 @@ -2526,6 +2561,9 @@ importers: '@pgpmjs/types': specifier: workspace:^ version: link:../../pgpm/types/dist + '@pgsql/quotes': + specifier: ^18.2.0 + version: 18.2.1 lru-cache: specifier: ^11.2.7 version: 11.2.7 @@ -2617,6 +2655,52 @@ importers: version: 0.3.0 publishDirectory: dist + packages/perf-harness: + dependencies: + grafast: + specifier: 1.0.2 + version: 1.0.2(graphql@16.13.0) + graphile-build-pg: + specifier: 5.0.2 + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + graphile-cache: + specifier: workspace:^ + version: link:../../graphile/graphile-cache/dist + graphile-settings: + specifier: workspace:^ + version: link:../../graphile/graphile-settings/dist + graphql: + specifier: 16.13.0 + version: 16.13.0 + graphql-ws: + specifier: ^6.0.8 + version: 6.0.8(graphql@16.13.0)(ws@8.20.1) + pg: + specifier: ^8.21.0 + version: 8.21.0 + pg-env: + specifier: workspace:^ + version: link:../../postgres/pg-env/dist + ws: + specifier: ^8.20.0 + version: 8.20.1 + devDependencies: + '@types/node': + specifier: ^22.19.11 + version: 22.19.19 + '@types/pg': + specifier: ^8.20.0 + version: 8.20.0 + '@types/ws': + specifier: ^8.18.1 + version: 8.18.1 + makage: + specifier: ^0.3.0 + version: 0.3.0 + ts-node: + specifier: ^10.9.2 + version: 10.9.2(@types/node@22.19.19)(typescript@5.9.3) + packages/postmaster: dependencies: 12factor-env: @@ -11683,7 +11767,7 @@ snapshots: grafast: 1.0.2(graphql@16.13.0) tslib: 2.8.1 - '@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0)': + '@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0)': dependencies: '@dataplan/json': 1.0.0(grafast@1.0.2(graphql@16.13.0)) '@graphile/lru': 5.0.0 @@ -11703,7 +11787,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)': + '@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)': dependencies: '@dataplan/json': 1.0.0(grafast@1.0.2(graphql@16.13.0)) '@graphile/lru': 5.0.0 @@ -12114,7 +12198,7 @@ snapshots: - supports-color - utf-8-validate - '@graphile-contrib/pg-many-to-many@2.0.0-rc.2': {} + '@graphile-contrib/pg-many-to-many@2.0.0-rc.2(patch_hash=71202d598d0d80e79890b8fbd8a1fc60cee20622dca60e59a0a6e994b6ab92c8)': {} '@graphile/lru@5.0.0': dependencies: @@ -15869,17 +15953,17 @@ snapshots: - supports-color - use-sync-external-store - graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0)(tamedevil@0.1.1): + graphile-build-pg@5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0)(tamedevil@0.1.1): dependencies: - '@dataplan/pg': 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0) + '@dataplan/pg': 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0) '@types/node': 22.19.19 debug: 4.4.3(supports-color@5.5.0) grafast: 1.0.2(graphql@16.13.0) - graphile-build: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + graphile-build: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-config: 1.0.1 graphql: 16.13.0 jsonwebtoken: 9.0.3 - pg-introspection: 1.0.1 + pg-introspection: 1.0.1(patch_hash=987dfb1b6491b9423e890d57722efdb618efce72183de5dd25c3fde11189fbc4) pg-sql2: 5.0.1 tamedevil: 0.1.1 tslib: 2.8.1 @@ -15888,17 +15972,17 @@ snapshots: transitivePeerDependencies: - supports-color - graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1): + graphile-build-pg@5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1): dependencies: - '@dataplan/pg': 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + '@dataplan/pg': 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) '@types/node': 22.19.19 debug: 4.4.3(supports-color@5.5.0) grafast: 1.0.2(graphql@16.13.0) - graphile-build: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + graphile-build: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-config: 1.0.1 graphql: 16.13.0 jsonwebtoken: 9.0.3 - pg-introspection: 1.0.1 + pg-introspection: 1.0.1(patch_hash=987dfb1b6491b9423e890d57722efdb618efce72183de5dd25c3fde11189fbc4) pg-sql2: 5.0.1 tamedevil: 0.1.1 tslib: 2.8.1 @@ -15907,7 +15991,7 @@ snapshots: transitivePeerDependencies: - supports-color - graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0): + graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0): dependencies: '@types/node': 22.19.19 '@types/pluralize': 0.0.33 @@ -15937,71 +16021,71 @@ snapshots: transitivePeerDependencies: - supports-color - graphile-connection-filter@1.14.0(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(postgraphile@5.0.3(bc368b92b12e127d4eb1f1e143433708)): + graphile-connection-filter@1.14.0(1cb56828c00532c0532fac2b893bbf72): dependencies: - '@dataplan/pg': 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) - graphile-build: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) - graphile-build-pg: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + '@dataplan/pg': 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + graphile-build: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + graphile-build-pg: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: 1.0.1 graphql: 16.13.0 pg-sql2: 5.0.1 - postgraphile: 5.0.3(bc368b92b12e127d4eb1f1e143433708) + postgraphile: 5.0.3(8e984f3e9cfc73b4b77b478748651d25) - graphile-connection-filter@1.14.0(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(postgraphile@5.0.3(f0ca433f04a3f28b00a29892b7510c70)): + graphile-connection-filter@1.14.0(3cdce88baf7cc47d1e7a2ba13375cd59): dependencies: - '@dataplan/pg': 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) - graphile-build: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) - graphile-build-pg: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + '@dataplan/pg': 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + graphile-build: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + graphile-build-pg: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: 1.0.1 graphql: 16.13.0 pg-sql2: 5.0.1 - postgraphile: 5.0.3(f0ca433f04a3f28b00a29892b7510c70) + postgraphile: 5.0.3(65ecfcf3543e7c87a3dabce321c8b0d1) - graphile-utils@5.0.0(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1): + graphile-utils@5.0.0(421ca45f0ba699c78ec7eea2d25ec7d4): dependencies: - '@dataplan/pg': 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + '@dataplan/pg': 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) debug: 4.4.3(supports-color@5.5.0) grafast: 1.0.2(graphql@16.13.0) - graphile-build: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + graphile-build: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-config: 1.0.1 graphql: 16.13.0 json5: 2.2.3 tamedevil: 0.1.1 tslib: 2.8.1 optionalDependencies: - graphile-build-pg: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + graphile-build-pg: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) transitivePeerDependencies: - supports-color - graphile-utils@5.0.1(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1): + graphile-utils@5.0.1(421ca45f0ba699c78ec7eea2d25ec7d4): dependencies: - '@dataplan/pg': 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0) + '@dataplan/pg': 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) debug: 4.4.3(supports-color@5.5.0) grafast: 1.0.2(graphql@16.13.0) - graphile-build: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + graphile-build: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-config: 1.0.1 graphql: 16.13.0 json5: 2.2.3 tamedevil: 0.1.1 tslib: 2.8.1 optionalDependencies: - graphile-build-pg: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0)(tamedevil@0.1.1) + graphile-build-pg: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) transitivePeerDependencies: - supports-color - graphile-utils@5.0.1(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1): + graphile-utils@5.0.1(8232bac9ba396693c72a53ebcaff3457): dependencies: - '@dataplan/pg': 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + '@dataplan/pg': 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0) debug: 4.4.3(supports-color@5.5.0) grafast: 1.0.2(graphql@16.13.0) - graphile-build: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + graphile-build: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) graphile-config: 1.0.1 graphql: 16.13.0 json5: 2.2.3 tamedevil: 0.1.1 tslib: 2.8.1 optionalDependencies: - graphile-build-pg: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + graphile-build-pg: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0)(tamedevil@0.1.1) transitivePeerDependencies: - supports-color @@ -17881,7 +17965,7 @@ snapshots: pg-int8@1.0.1: {} - pg-introspection@1.0.1: + pg-introspection@1.0.1(patch_hash=987dfb1b6491b9423e890d57722efdb618efce72183de5dd25c3fde11189fbc4): dependencies: tslib: 2.8.1 @@ -18019,24 +18103,24 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - postgraphile@5.0.3(02c18e7c6179c4ae54031f8cdca16ab5): + postgraphile@5.0.3(260a0a3f37f6479930862ca6ba480e4c): dependencies: '@dataplan/json': 1.0.0(grafast@1.0.2(graphql@16.13.0)) - '@dataplan/pg': 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + '@dataplan/pg': 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0) '@graphile/lru': 5.0.0 '@types/node': 22.19.19 '@types/pg': 8.20.0 debug: 4.4.3(supports-color@5.5.0) grafast: 1.0.2(graphql@16.13.0) grafserv: 1.0.0(@types/node@25.9.1)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(use-sync-external-store@1.6.0(react@19.2.5))(ws@8.20.1) - graphile-build: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) - graphile-build-pg: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + graphile-build: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + graphile-build-pg: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0)(tamedevil@0.1.1) graphile-config: 1.0.1 - graphile-utils: 5.0.1(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1) + graphile-utils: 5.0.1(8232bac9ba396693c72a53ebcaff3457) graphql: 16.13.0 iterall: 1.3.0 jsonwebtoken: 9.0.3 - pg: 8.21.0 + pg: 8.20.0 pg-sql2: 5.0.1 tamedevil: 0.1.1 tslib: 2.8.1 @@ -18046,24 +18130,24 @@ snapshots: - supports-color - utf-8-validate - postgraphile@5.0.3(4b1f938b62eff937cb4c255aac53ddd9): + postgraphile@5.0.3(2fb27169ec8b91d10ab754f66c750f53): dependencies: '@dataplan/json': 1.0.0(grafast@1.0.2(graphql@16.13.0)) - '@dataplan/pg': 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0) + '@dataplan/pg': 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) '@graphile/lru': 5.0.0 '@types/node': 22.19.19 '@types/pg': 8.20.0 debug: 4.4.3(supports-color@5.5.0) grafast: 1.0.2(graphql@16.13.0) grafserv: 1.0.0(@types/node@25.9.1)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(use-sync-external-store@1.6.0(react@19.2.5))(ws@8.20.1) - graphile-build: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) - graphile-build-pg: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0)(tamedevil@0.1.1) + graphile-build: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + graphile-build-pg: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: 1.0.1 - graphile-utils: 5.0.1(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.20.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1) + graphile-utils: 5.0.1(421ca45f0ba699c78ec7eea2d25ec7d4) graphql: 16.13.0 iterall: 1.3.0 jsonwebtoken: 9.0.3 - pg: 8.20.0 + pg: 8.21.0 pg-sql2: 5.0.1 tamedevil: 0.1.1 tslib: 2.8.1 @@ -18073,20 +18157,20 @@ snapshots: - supports-color - utf-8-validate - postgraphile@5.0.3(bc368b92b12e127d4eb1f1e143433708): + postgraphile@5.0.3(65ecfcf3543e7c87a3dabce321c8b0d1): dependencies: '@dataplan/json': 1.0.0(grafast@1.0.2(graphql@16.13.0)) - '@dataplan/pg': 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + '@dataplan/pg': 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) '@graphile/lru': 5.0.0 '@types/node': 22.19.19 '@types/pg': 8.20.0 debug: 4.4.3(supports-color@5.5.0) grafast: 1.0.2(graphql@16.13.0) - grafserv: 1.0.0(@types/node@22.19.15)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(use-sync-external-store@1.6.0(react@19.2.5))(ws@8.20.1) - graphile-build: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) - graphile-build-pg: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + grafserv: 1.0.0(@types/node@22.19.11)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(use-sync-external-store@1.6.0(react@19.2.5))(ws@8.20.1) + graphile-build: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + graphile-build-pg: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: 1.0.1 - graphile-utils: 5.0.1(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1) + graphile-utils: 5.0.1(421ca45f0ba699c78ec7eea2d25ec7d4) graphql: 16.13.0 iterall: 1.3.0 jsonwebtoken: 9.0.3 @@ -18100,20 +18184,20 @@ snapshots: - supports-color - utf-8-validate - postgraphile@5.0.3(f0a861a74cc4311fffaf615b439fe994): + postgraphile@5.0.3(8e984f3e9cfc73b4b77b478748651d25): dependencies: '@dataplan/json': 1.0.0(grafast@1.0.2(graphql@16.13.0)) - '@dataplan/pg': 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + '@dataplan/pg': 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) '@graphile/lru': 5.0.0 '@types/node': 22.19.19 '@types/pg': 8.20.0 debug: 4.4.3(supports-color@5.5.0) grafast: 1.0.2(graphql@16.13.0) - grafserv: 1.0.0(@types/node@22.19.19)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(use-sync-external-store@1.6.0(react@19.2.5))(ws@8.20.1) - graphile-build: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) - graphile-build-pg: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + grafserv: 1.0.0(@types/node@22.19.15)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(use-sync-external-store@1.6.0(react@19.2.5))(ws@8.20.1) + graphile-build: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + graphile-build-pg: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: 1.0.1 - graphile-utils: 5.0.1(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1) + graphile-utils: 5.0.1(421ca45f0ba699c78ec7eea2d25ec7d4) graphql: 16.13.0 iterall: 1.3.0 jsonwebtoken: 9.0.3 @@ -18127,20 +18211,20 @@ snapshots: - supports-color - utf-8-validate - postgraphile@5.0.3(f0ca433f04a3f28b00a29892b7510c70): + postgraphile@5.0.3(df995cf62d46f35f4829b81730cdf6da): dependencies: '@dataplan/json': 1.0.0(grafast@1.0.2(graphql@16.13.0)) - '@dataplan/pg': 1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) + '@dataplan/pg': 1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0) '@graphile/lru': 5.0.0 '@types/node': 22.19.19 '@types/pg': 8.20.0 debug: 4.4.3(supports-color@5.5.0) grafast: 1.0.2(graphql@16.13.0) - grafserv: 1.0.0(@types/node@22.19.11)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(use-sync-external-store@1.6.0(react@19.2.5))(ws@8.20.1) - graphile-build: 5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) - graphile-build-pg: 5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + grafserv: 1.0.0(@types/node@22.19.19)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(react-dom@19.2.5(react@19.2.5))(react@19.2.5)(use-sync-external-store@1.6.0(react@19.2.5))(ws@8.20.1) + graphile-build: 5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0) + graphile-build-pg: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) graphile-config: 1.0.1 - graphile-utils: 5.0.1(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build-pg@5.0.2(@dataplan/pg@1.0.3(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.0.2(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(tamedevil@0.1.1) + graphile-utils: 5.0.1(421ca45f0ba699c78ec7eea2d25ec7d4) graphql: 16.13.0 iterall: 1.3.0 jsonwebtoken: 9.0.3 @@ -19139,6 +19223,24 @@ snapshots: v8-compile-cache-lib: 3.0.1 yn: 3.1.1 + ts-node@10.9.2(@types/node@22.19.19)(typescript@5.9.3): + dependencies: + '@cspotcode/source-map-support': 0.8.1 + '@tsconfig/node10': 1.0.12 + '@tsconfig/node12': 1.0.11 + '@tsconfig/node14': 1.0.3 + '@tsconfig/node16': 1.0.4 + '@types/node': 22.19.19 + acorn: 8.15.0 + acorn-walk: 8.3.4 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + typescript: 5.9.3 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3): dependencies: '@cspotcode/source-map-support': 0.8.1 diff --git a/postgres/pg-cache/README.md b/postgres/pg-cache/README.md index 8888375971..a2a0be4619 100644 --- a/postgres/pg-cache/README.md +++ b/postgres/pg-cache/README.md @@ -22,8 +22,11 @@ npm install pg-cache ## Features -- LRU cache for PostgreSQL connection pools +- Lease-aware LRU registry for PostgreSQL connection pools +- Fail-closed capacity admission for long-lived production consumers - Automatic pool cleanup and disposal +- Pool identity, lease, and disposal observability +- Checkout queue/sanitation timing and fast-path counters - Extensible cleanup callback system - Service cache for general use - Graceful shutdown handling @@ -52,6 +55,105 @@ const result = await pool.query('SELECT NOW()'); const samePool = getPgPool({ database: 'mydb' }); // Returns cached pool ``` +`getPgPool()` remains the synchronous compatibility API. A component that keeps +a pool across requests or asynchronous lifecycle boundaries should hold a lease: + +```typescript +import { acquirePgPool, PgPoolCapacityError } from 'pg-cache'; + +try { + const lease = acquirePgPool( + { database: 'tenant_a', user: 'graphql_runtime' }, + { purpose: 'runtime', sanitizeOnCheckout: true } + ); + + // Retain lease.pool for the owning handler or request lifecycle. + // release() is idempotent and must run only after the owner has drained. + lease.release(); +} catch (error) { + if (error instanceof PgPoolCapacityError) { + // Map error.code === 'PG_POOL_CAPACITY' to HTTP 503 and Retry-After. + } +} +``` + +Acquisition is synchronous and atomic. Reusing an existing exact identity costs +no new slot; a new identity evicts only an unleased pool. If every slot is +leased, admission throws before constructing a pool or ending an existing one. + +Sanitized default-driver pools issue `DISCARD ALL` before every reused checkout +and clear node-postgres/Graphile prepared-statement bookkeeping. The first +checkout of a brand-new factory-owned connection skips that redundant round +trip only when its trusted startup baseline is pinned and no other `connect` +listener could have changed the session. `getPgCheckoutSanitizerStats()` exposes +checkout wait, queue, sanitation, failure, and virgin-fast-path counters. +For alternate pool factories, pg-cache replaces direct `pool.query()` calls +with a sanitized checkout/query/release cycle; a custom sanitized pool must +therefore provide a replaceable `query()` method and Promise-based client +queries. Query failures destroy the checked-out client before the error is +returned through either the Promise or callback API. + +Pool sizing accepts `max`, `idleTimeoutMillis`, `connectionTimeoutMillis`, +`allowExitOnIdle`, and native pg-pool `maxUses`. Set `maxUses: 1` to retire a +client after every checkout, or leave it unset/use `0` for unlimited reuse. +The equivalent environment setting is `PG_POOL_MAX_USES`; it accepts only a +canonical decimal integer, so alternate numeric spellings, negative, +fractional, and unsafe-integer values fail before a pool identity is published. +Because `maxUses` changes connection lifecycle and latency, it participates in +the opaque exact-pool identity and should be benchmarked under the real request +rate before production use. + +Exact-pool and physical-target identities use a process-random keyed HMAC. They +are stable for the lifetime of one pool registry, but intentionally differ in +another process; this prevents an emitted identity from becoming an offline +password verifier. Connection and pool identity inputs must be primitive, +canonical data, and node-postgres password callbacks are rejected because a +callback's captured credential cannot be represented without risking an alias. + +### Shared Notification Broker + +`acquirePgNotificationBroker(listenerPgConfig, { topics })` leases one +process-local LISTEN connection per opaque, versioned pool identity. The +identity includes the canonical connection, credentials, pool, driver, +sanitation, and data-only TLS settings. The broker LISTENs only to each lease's +exact channel allowlist, rejects identifiers PostgreSQL would truncate past 63 +UTF-8 bytes, and awaits UNLISTEN plus connection and pool-lease release. +The final lease destroys its listener client after UNLISTEN, so an inactive +database does not retain an idle PostgreSQL backend until the pool timeout. + +Production broker acquisition performs a fresh, read-only catalog audit on the +same pinned client before each listener lease is admitted, so a pool with +`max: 1` cannot deadlock waiting for a second checkout. The attested lease's +narrow `revalidateRole()` capability serializes TTL audits on that client and +never exposes general SQL access. A conforming login may +CONNECT only to that exact database and has no role memberships, privileged +role attributes, database CREATE/TEMP, or effective privileges on non-system +schemas, relations, routines, and sequences. The returned audit contains only +the role, database, stable violation codes, and audit version; callers must keep +the separate `PgConfig` credentials in their secret store. Use +`normalizePgNotificationRoleContracts()` to reject one login spanning physical +databases or multiple listener logins targeting one physical database. +The live contract test is opt-in with +`PG_CACHE_RUN_NOTIFICATION_ROLE_INTEGRATION=1` and uses the standard `PG*` +connection settings for a pre-provisioned conforming notification login. + +Every role-audit, `LISTEN`, and `UNLISTEN` command has the same bounded deadline +as the listener pool's `connectionTimeoutMillis`. Configure it through the +listener's `pool.connectionTimeoutMillis`, or through +`PG_POOL_CONNECTION_TIMEOUT_MS` when the pool setting is omitted; the default is +5 seconds. The broker rejects zero, fractional, negative, or setTimeout-unsafe +values before publishing an identity. A command that exceeds the deadline +fatally closes every lease and destroys the pinned client, so TTL refresh and +shutdown cannot wait forever on an abandoned driver query. + +Each GraphQL subscriber has a fixed 256-message queue. A slow subscriber that +overflows is failed independently; a listener connection error fails every +lease and is never reconnected while any failed owner remains. The caller must +provide a dedicated least-privilege listener login and remains responsible for +the deployment's certificate and role policy. `lease.terminated` and +`getPgNotificationBrokerStats()` expose failure and lifecycle state without +revealing connection credentials. + ### Direct Cache Access ```typescript @@ -103,7 +205,7 @@ const service = svcCache.get('my-service'); ```typescript import { close, teardownPgPools } from 'pg-cache'; -// In your shutdown handler +// The executable owns process signals; importing pg-cache never installs one. process.on('SIGTERM', async () => { await close(); // or teardownPgPools() process.exit(0); @@ -119,14 +221,28 @@ The main PostgreSQL pool cache instance. - `get(key: string): Pool | undefined` - Get a cached pool - `set(key: string, pool: Pool): void` - Cache a pool - `has(key: string): boolean` - Check if a pool is cached -- `delete(key: string): void` - Remove and dispose a pool -- `clear(): void` - Remove and dispose all pools +- `delete(key: string): void` - Remove an unleased pool +- `clear(): void` - Remove all currently unleased pools +- `acquire(key: string, factory: () => Pool): PgPoolLease` - Atomically acquire a lease +- `getStats(): PgPoolCacheStats` - Read capacity and lifecycle counters - `registerCleanupCallback(callback: (key: string) => void): () => void` - Register a cleanup callback ### getPgPool(config: Partial): Pool Get or create a cached PostgreSQL pool using the provided configuration. +### acquirePgPool(config, options): PgPoolLease + +Get or create the exact pool identity and protect it from TTL/LRU disposal until +the returned idempotent `release()` is called. + +### Capacity + +`PG_CACHE_MAX` limits lazy pool identities, not eagerly allocated connections. +The default is 2064: two identities for each of 1024 database-per-tenant Graphile +contracts plus a 16-identity operational reserve. `PG_CACHE_TTL_MS` applies only +while an identity has zero leases. + ### svcCache A general-purpose LRU cache for services and objects. diff --git a/postgres/pg-cache/src/__tests__/driver.test.ts b/postgres/pg-cache/src/__tests__/driver.test.ts index dd155d538f..eacdd9e116 100644 --- a/postgres/pg-cache/src/__tests__/driver.test.ts +++ b/postgres/pg-cache/src/__tests__/driver.test.ts @@ -3,13 +3,17 @@ // lets an alternate backend (e.g. PGlite) plug in without any change to pgpm / // pgsql-* — and guarantees the default path is untouched when nothing registers. -import { randomUUID } from 'crypto'; +import { createHash, randomUUID } from 'crypto'; import pg from 'pg'; import { + acquirePgPool, defaultPgPoolFactory, getActivePgPoolFactory, getPgPool, + getPgPoolConfig, + getPgPoolDriverIdentity, + getPgPoolIdentity, hasPgPoolFactory, PgPoolFactory, registerPgPoolFactory @@ -55,10 +59,10 @@ describe('pg-cache pool-factory seam', () => { expect(factory).toHaveBeenCalledTimes(1); expect(pool).toBe(mock); - pgCache.delete(cfg.database); + pgCache.delete(getPgPoolIdentity(cfg)); }); - it('caches by database: a second call reuses the pool and does not re-invoke the factory', () => { + it('caches by exact pool identity: an identical call reuses the pool', () => { const cfg = freshConfig(); const factory = jest.fn(() => createMockPool()); registerPgPoolFactory(factory); @@ -69,7 +73,324 @@ describe('pg-cache pool-factory seam', () => { expect(first).toBe(second); expect(factory).toHaveBeenCalledTimes(1); - pgCache.delete(cfg.database); + pgCache.delete(getPgPoolIdentity(cfg)); + }); + + it('acquires an idempotently releasable lease for the exact pool identity', () => { + const cfg = freshConfig(); + const mock = createMockPool(); + const factory = jest.fn(() => mock); + registerPgPoolFactory(factory); + + const first = acquirePgPool(cfg, { purpose: 'runtime' }); + const second = acquirePgPool(cfg, { purpose: 'runtime' }); + + expect(first.identity).toBe(getPgPoolIdentity(cfg, { purpose: 'runtime' })); + expect(first.pool).toBe(mock); + expect(second.pool).toBe(mock); + expect(factory).toHaveBeenCalledTimes(1); + + first.release(); + first.release(); + second.release(); + pgCache.delete(first.identity); + }); + + it('separates credentials, purpose, and sanitation mode for the same database', () => { + const cfg = freshConfig(); + const factory = jest.fn(() => createMockPool()); + registerPgPoolFactory(factory); + + const control = getPgPool(cfg, { purpose: 'control' }); + const runtime = getPgPool({ ...cfg, user: 'runtime' }, { + purpose: 'runtime', + sanitizeOnCheckout: true + }); + const unsanitizedRuntime = getPgPool({ ...cfg, user: 'runtime' }, { + purpose: 'runtime' + }); + + expect(control).not.toBe(runtime); + expect(runtime).not.toBe(unsanitizedRuntime); + expect(factory).toHaveBeenCalledTimes(3); + + pgCache.delete(getPgPoolIdentity(cfg, { purpose: 'control' })); + pgCache.delete(getPgPoolIdentity({ ...cfg, user: 'runtime' }, { + purpose: 'runtime', + sanitizeOnCheckout: true + })); + pgCache.delete(getPgPoolIdentity({ ...cfg, user: 'runtime' }, { purpose: 'runtime' })); + }); + + it('normalizes maxUses into the exact pool identity', () => { + const cfg = freshConfig(); + const unlimited = getPgPoolIdentity(cfg); + const explicitUnlimited = getPgPoolIdentity({ ...cfg, pool: { maxUses: 0 } }); + const singleUse = getPgPoolIdentity({ ...cfg, pool: { maxUses: 1 } }); + const doubleUse = getPgPoolIdentity({ ...cfg, pool: { maxUses: 2 } }); + + expect(explicitUnlimited).toBe(unlimited); + expect(singleUse).not.toBe(unlimited); + expect(doubleUse).not.toBe(singleUse); + }); + + it('uses a process-keyed identity instead of an offline password verifier', () => { + const cfg = { + ...freshConfig(), + pool: { + max: 3, + idleTimeoutMillis: 1234, + connectionTimeoutMillis: 5678, + allowExitOnIdle: true + } + }; + const identity = getPgPoolIdentity(cfg, { + purpose: 'runtime', + sanitizeOnCheckout: true + }); + const unkeyedInput = JSON.stringify({ + version: 1, + driver: getPgPoolDriverIdentity(), + host: cfg.host, + port: cfg.port, + database: cfg.database, + user: cfg.user, + password: cfg.password, + ssl: null, + pool: { + max: 3, + maxUses: null, + idleTimeoutMillis: 1234, + connectionTimeoutMillis: 5678, + allowExitOnIdle: true + }, + purpose: 'runtime', + sanitizeOnCheckout: true + }); + const offlineDigest = `pg:v1:${createHash('sha256') + .update(unkeyedInput) + .digest('hex')}`; + + expect(getPgPoolIdentity(cfg, { + purpose: 'runtime', + sanitizeOnCheckout: true + })).toBe(identity); + expect(identity).toMatch(/^pg:v1:[a-f0-9]{64}$/); + expect(identity).not.toBe(offlineDigest); + }); + + it('rejects password callbacks before they can alias a pool identity', () => { + const cfg = freshConfig(); + const first = async () => 'first-secret'; + const second = async () => 'second-secret'; + const factory = jest.fn(() => createMockPool()); + registerPgPoolFactory(factory); + + for (const password of [first, second]) { + const invalid = { + ...cfg, + password: password as unknown as string + }; + expect(() => getPgPoolIdentity(invalid)).toThrow( + 'pg.password must be a string' + ); + expect(() => getPgPool(invalid)).toThrow( + 'pg.password must be a string' + ); + } + expect(factory).not.toHaveBeenCalled(); + }); + + it('rejects noncanonical exact-identity inputs', () => { + const cfg = freshConfig(); + const accessorSsl = {} as Record; + Object.defineProperty(accessorSsl, 'ca', { get: () => 'dynamic-ca' }); + const symbolSsl = { ca: 'tenant-ca' } as Record; + symbolSsl[Symbol('hidden')] = 'untracked'; + const sparseCa = new Array(2); + sparseCa[1] = 'tenant-ca'; + const undefinedSsl: { ca: undefined } = { ca: undefined }; + + expect(() => getPgPoolIdentity({ + ...cfg, + port: '5432' as unknown as number + })).toThrow('pg.port must be a safe integer'); + expect(() => getPgPoolIdentity({ + ...cfg, + pool: { max: '2' as unknown as number } + })).toThrow('pool.max must be a safe integer'); + expect(() => getPgPoolIdentity(cfg, { + purpose: {} as unknown as string + })).toThrow('pg pool purpose must be a non-empty string'); + expect(() => getPgPoolIdentity(cfg, { + sanitizeOnCheckout: 'false' as unknown as boolean + })).toThrow('pg pool sanitizeOnCheckout must be a boolean'); + expect(() => getPgPoolIdentity({ + ...cfg, + ssl: accessorSsl as never + })).toThrow('pg.ssl.ca must be a data property'); + expect(() => getPgPoolIdentity({ + ...cfg, + ssl: symbolSsl as never + })).toThrow('pg.ssl must not contain symbol properties'); + expect(() => getPgPoolIdentity({ + ...cfg, + ssl: { ca: sparseCa } as never + })).toThrow('pg.ssl.ca must be a dense array without custom properties'); + expect(() => getPgPoolIdentity({ + ...cfg, + ssl: undefinedSsl as never + })).toThrow('pg.ssl.ca must not be undefined'); + }); + + it('parses PG_POOL_MAX_USES as an unlimited sentinel or positive safe integer', () => { + const previous = process.env.PG_POOL_MAX_USES; + try { + process.env.PG_POOL_MAX_USES = '0'; + expect(getPgPoolConfig().maxUses).toBeUndefined(); + + process.env.PG_POOL_MAX_USES = '17'; + expect(getPgPoolConfig().maxUses).toBe(17); + + for (const invalid of [ + '-1', + '1.5', + '01', + '1e2', + '0x10', + ' 1', + '1 ', + ' ', + 'not-a-number', + '9007199254740992' + ]) { + process.env.PG_POOL_MAX_USES = invalid; + expect(() => getPgPoolConfig()).toThrow( + 'PG_POOL_MAX_USES must be 0 or a positive safe integer' + ); + } + } finally { + if (previous === undefined) delete process.env.PG_POOL_MAX_USES; + else process.env.PG_POOL_MAX_USES = previous; + } + }); + + it('validates explicit maxUses overrides before constructing an identity or pool', () => { + expect(getPgPoolConfig({ maxUses: 0 }).maxUses).toBeUndefined(); + expect(getPgPoolConfig({ maxUses: 23 }).maxUses).toBe(23); + + for (const invalid of [-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, 9007199254740992]) { + expect(() => getPgPoolConfig({ maxUses: invalid })).toThrow( + 'pool.maxUses must be 0 or a positive safe integer' + ); + } + for (const invalid of [true, null, {}]) { + expect(() => getPgPoolConfig({ + maxUses: invalid as unknown as number + })).toThrow('pool.maxUses must be 0 or a positive safe integer'); + } + }); + + it('replaces a sanitized custom factory query that bypasses connect', async () => { + const cfg = freshConfig(); + const queryResult = { rows: [{ value: 42 }] }; + const client = { + query: jest.fn(async (text: string) => text === 'SELECT $1::int AS value' + ? queryResult + : { rows: [] }), + release: jest.fn() + }; + const bypassingQuery = jest.fn(async () => ({ rows: [{ value: -1 }] })); + const connect = jest.fn(async () => client); + const pool = { + query: bypassingQuery, + connect, + end: jest.fn(async (): Promise => undefined) + } as unknown as pg.Pool; + const factory = jest.fn(() => pool); + registerPgPoolFactory(factory); + const options = { purpose: 'runtime', sanitizeOnCheckout: true } as const; + const identity = getPgPoolIdentity(cfg, options); + + try { + const sanitizedPool = getPgPool(cfg, options); + + await expect(sanitizedPool.query( + 'SELECT $1::int AS value', + [42] + )).resolves.toBe(queryResult); + expect(bypassingQuery).not.toHaveBeenCalled(); + expect(connect).toHaveBeenCalledTimes(1); + expect(client.query).toHaveBeenNthCalledWith(1, 'DISCARD ALL'); + expect(client.query).toHaveBeenNthCalledWith( + 2, + 'SET search_path TO pg_catalog; SET row_security TO on; SET jit_optimize_above_cost TO -1' + ); + expect(client.query).toHaveBeenNthCalledWith(3, 'SELECT $1::int AS value', [42]); + expect(client.release).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith(); + } finally { + pgCache.delete(identity); + await pgCache.waitForDisposals(); + } + }); + + it('separates TLS trust contracts for the same database and role', () => { + const cfg = freshConfig(); + + const verified = getPgPoolIdentity({ + ...cfg, + ssl: { ca: 'tenant-ca', rejectUnauthorized: true, servername: 'db.internal' } + }); + const insecure = getPgPoolIdentity({ + ...cfg, + ssl: { ca: 'tenant-ca', rejectUnauthorized: false, servername: 'db.internal' } + }); + const plaintext = getPgPoolIdentity(cfg); + + expect(verified).not.toBe(insecure); + expect(verified).not.toBe(plaintext); + expect(insecure).not.toBe(plaintext); + }); + + it('canonicalizes TLS data and rejects identity inputs JSON would omit', () => { + const cfg = freshConfig(); + const first = getPgPoolIdentity({ + ...cfg, + ssl: { ca: 'tenant-ca', rejectUnauthorized: true } + }); + const second = getPgPoolIdentity({ + ...cfg, + ssl: { rejectUnauthorized: true, ca: 'tenant-ca' } + }); + + expect(first).toBe(second); + expect(() => getPgPoolIdentity({ + ...cfg, + ssl: { checkServerIdentity: (): undefined => undefined } as any + })).toThrow('pg.ssl.checkServerIdentity must contain only deterministic data values'); + + const bufferIdentity = getPgPoolIdentity({ + ...cfg, + ssl: { ca: Buffer.from('tenant-ca') } + }); + const mimickedBufferIdentity = getPgPoolIdentity({ + ...cfg, + ssl: { + ca: { + bufferSha256: 'b60c1883ea3c4bf71a5959468ac16f36e2aa4f5c8702ca157fbbae61415f2f10' + } + } as any + }); + expect(bufferIdentity).not.toBe(mimickedBufferIdentity); + }); + + it('uses opaque identities that never disclose credentials', () => { + const cfg = { ...freshConfig(), password: 'top-secret-password' }; + const identity = getPgPoolIdentity(cfg, { purpose: 'runtime' }); + expect(identity).toMatch(/^pg:v1:[a-f0-9]{64}$/); + expect(identity).not.toContain(cfg.user); + expect(identity).not.toContain(cfg.password); }); it('falls back to defaultPgPoolFactory when nothing is registered', () => { @@ -78,7 +399,7 @@ describe('pg-cache pool-factory seam', () => { // query runs, so this is safe without a live server. const pool = getPgPool(cfg); expect(pool).toBeInstanceOf(pg.Pool); - pgCache.delete(cfg.database); + pgCache.delete(getPgPoolIdentity(cfg)); }); it('defaultPgPoolFactory returns a pg.Pool', () => { @@ -86,4 +407,60 @@ describe('pg-cache pool-factory seam', () => { expect(pool).toBeInstanceOf(pg.Pool); return pool.end(); }); + + it('passes credentials as discrete fields instead of reparsing them as a URI', async () => { + const cfg = { + ...freshConfig(), + user: 'runtime@tenant', + password: 'x@evil.example/other?sslmode=require', + database: 'tenant/database' + }; + const pool = defaultPgPoolFactory(cfg); + const options = (pool as pg.Pool & { options: pg.PoolConfig }).options; + + expect(options.host).toBe(cfg.host); + expect(options.port).toBe(cfg.port); + expect(options.database).toBe(cfg.database); + expect(options.user).toBe(cfg.user); + expect(options.password).toBe(cfg.password); + await pool.end(); + }); + + it('passes maxUses to the native pg.Pool driver', async () => { + const pool = defaultPgPoolFactory({ ...freshConfig(), pool: { maxUses: 1 } }); + const options = (pool as pg.Pool & { options: pg.PoolConfig }).options; + + expect(options.maxUses).toBe(1); + await pool.end(); + }); + + it('passes the exact TLS contract to node-postgres', async () => { + const ssl = { + ca: 'tenant-ca', + cert: 'runtime-cert', + key: 'runtime-key', + rejectUnauthorized: true, + servername: 'db.internal', + minVersion: 'TLSv1.2' as const + }; + const pool = defaultPgPoolFactory({ ...freshConfig(), ssl }); + const options = (pool as pg.Pool & { options: pg.PoolConfig }).options; + + expect(options.ssl).toEqual(ssl); + await pool.end(); + }); + + it('pins the trusted baseline in sanitized node-postgres startup options', async () => { + const pool = defaultPgPoolFactory(freshConfig(), { + purpose: 'runtime', + sanitizeOnCheckout: true + }); + const options = (pool as pg.Pool & { options: pg.PoolConfig }).options; + + expect(options.options).toContain('-c search_path=pg_catalog'); + expect(options.options).toContain('-c row_security=on'); + expect(options.options).toContain('-c jit_optimize_above_cost=-1'); + expect(pool.query).toBe(pg.Pool.prototype.query); + await pool.end(); + }); }); diff --git a/postgres/pg-cache/src/__tests__/lru.test.ts b/postgres/pg-cache/src/__tests__/lru.test.ts index a68afa616f..1a10500d88 100644 --- a/postgres/pg-cache/src/__tests__/lru.test.ts +++ b/postgres/pg-cache/src/__tests__/lru.test.ts @@ -1,15 +1,26 @@ -// Guards against the pg-cache close() resource leak fixed in feat/observability. -// -// Previously, close() reset this.closed = false after shutdown, allowing -// set() to silently accept new pools that were never cleaned up. The module- -// level closePromise also reset to null, enabling double-shutdown. -// -// These tests lock the fix: close() is final, set() rejects, and repeated -// close() calls are idempotent. See pg-cache-close-leak.md for full details. - import pg from 'pg'; -import { PgPoolCacheManager } from '../lru'; +import { + DEFAULT_PG_CACHE_MAX, + PG_CACHE_GRAPHILE_CONTRACT_CAPACITY, + PG_CACHE_OPERATIONAL_RESERVE, + PgPoolCacheManager, + PgPoolCapacityError +} from '../lru'; + +describe('process lifecycle ownership', () => { + it('does not install process signal handlers from a library import', () => { + const beforeSigterm = process.listenerCount('SIGTERM'); + const beforeSigint = process.listenerCount('SIGINT'); + + jest.isolateModules(() => { + jest.requireActual('../lru'); + }); + + expect(process.listenerCount('SIGTERM')).toBe(beforeSigterm); + expect(process.listenerCount('SIGINT')).toBe(beforeSigint); + }); +}); // Minimal mock — we only need pool.end() and pool.ended const createMockPool = (): pg.Pool => { @@ -45,8 +56,11 @@ describe('PgPoolCacheManager', () => { }); describe('configuration', () => { - it('uses env-var defaults (max=50) when no overrides given', () => { - expect(cache.config.max).toBe(50); + it('reserves two identities per supported Graphile contract plus operations', () => { + expect(DEFAULT_PG_CACHE_MAX).toBe( + PG_CACHE_GRAPHILE_CONTRACT_CAPACITY * 2 + PG_CACHE_OPERATIONAL_RESERVE + ); + expect(cache.config.max).toBe(2064); }); it('accepts constructor overrides', () => { @@ -90,6 +104,157 @@ describe('PgPoolCacheManager', () => { }); }); + describe('leases and fail-closed admission', () => { + it('counts an existing exact identity as zero new slots', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const pool = createMockPool(); + const factory = jest.fn(() => pool); + + const first = small.acquire('runtime-a', factory); + const second = small.acquire('runtime-a', factory); + + expect(first.pool).toBe(pool); + expect(second.pool).toBe(pool); + expect(factory).toHaveBeenCalledTimes(1); + expect(small.getStats()).toMatchObject({ + size: 1, + leasedPools: 1, + activeLeases: 2, + leasesAcquired: 2 + }); + + first.release(); + first.release(); + expect(small.getStats().activeLeases).toBe(1); + second.release(); + await small.close(); + }); + + it('refuses before constructing or ending when every slot is leased', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const firstPool = createMockPool(); + const first = small.acquire('runtime-a', () => firstPool); + const rejectedFactory = jest.fn(() => createMockPool()); + + let capacityError: PgPoolCapacityError | undefined; + try { + small.acquire('runtime-b', rejectedFactory); + } catch (error) { + capacityError = error as PgPoolCapacityError; + } + + expect(capacityError).toBeInstanceOf(PgPoolCapacityError); + expect(capacityError).toMatchObject({ + code: 'PG_POOL_CAPACITY', + retryAfterSeconds: 15, + max: 1, + size: 1, + leased: 1 + }); + expect(rejectedFactory).not.toHaveBeenCalled(); + expect(firstPool.end).not.toHaveBeenCalled(); + expect(small.getStats().capacityRefusals).toBe(1); + + first.release(); + await small.close(); + }); + + it('evicts only the least-recent zero-lease identity', async () => { + const small = new PgPoolCacheManager({ max: 2 }); + const leasedPool = createMockPool(); + const idlePool = createMockPool(); + const replacementPool = createMockPool(); + const lease = small.acquire('leased', () => leasedPool); + small.set('idle', idlePool); + + small.set('replacement', replacementPool); + await small.waitForDisposals(); + + expect(small.has('leased')).toBe(true); + expect(leasedPool.end).not.toHaveBeenCalled(); + expect(small.has('idle')).toBe(false); + expect(idlePool.end).toHaveBeenCalledTimes(1); + expect(small.has('replacement')).toBe(true); + + lease.release(); + await small.close(); + }); + + it('keeps an expired leased identity until release', async () => { + jest.useFakeTimers(); + const small = new PgPoolCacheManager({ max: 1, ttl: 50 }); + const pool = createMockPool(); + const lease = small.acquire('runtime', () => pool); + try { + jest.advanceTimersByTime(51); + expect(small.has('runtime')).toBe(true); + expect(pool.end).not.toHaveBeenCalled(); + + lease.release(); + await small.waitForDisposals(); + + expect(small.has('runtime')).toBe(false); + expect(pool.end).toHaveBeenCalledTimes(1); + expect(small.getStats().ttlExpirations).toBe(1); + } finally { + jest.useRealTimers(); + await small.close(); + } + }); + + it('deterministically gives the final slot to the first synchronous acquisition', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const firstFactory = jest.fn(() => createMockPool()); + const secondFactory = jest.fn(() => createMockPool()); + + const outcomes = await Promise.allSettled([ + Promise.resolve().then(() => small.acquire('first', firstFactory)), + Promise.resolve().then(() => small.acquire('second', secondFactory)) + ]); + + expect(outcomes[0].status).toBe('fulfilled'); + expect(outcomes[1].status).toBe('rejected'); + expect((outcomes[1] as PromiseRejectedResult).reason).toBeInstanceOf( + PgPoolCapacityError + ); + expect(firstFactory).toHaveBeenCalledTimes(1); + expect(secondFactory).not.toHaveBeenCalled(); + + if (outcomes[0].status === 'fulfilled') outcomes[0].value.release(); + await small.close(); + }); + + it('rolls back its reservation if pool construction fails', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const retained = createMockPool(); + small.set('retained', retained); + + expect(() => small.acquire('broken', () => { + throw new Error('factory failed'); + })).toThrow('factory failed'); + + expect(small.has('retained')).toBe(true); + expect(retained.end).not.toHaveBeenCalled(); + expect(small.getStats()).toMatchObject({ size: 1, reservations: 0 }); + await small.close(); + }); + + it('does not end a physical pool retained under another exact identity', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const sharedPool = createMockPool(); + + small.set('identity-a', sharedPool); + small.set('identity-b', sharedPool); + await small.waitForDisposals(); + expect(sharedPool.end).not.toHaveBeenCalled(); + + small.delete('identity-b'); + await small.waitForDisposals(); + expect(sharedPool.end).toHaveBeenCalledTimes(1); + await small.close(); + }); + }); + describe('close() lifecycle', () => { it('set() after close() succeeds (cache re-opens for restart)', async () => { const pool1 = createMockPool(); diff --git a/postgres/pg-cache/src/__tests__/notification-broker.integration.test.ts b/postgres/pg-cache/src/__tests__/notification-broker.integration.test.ts new file mode 100644 index 0000000000..a7a8ef585a --- /dev/null +++ b/postgres/pg-cache/src/__tests__/notification-broker.integration.test.ts @@ -0,0 +1,154 @@ +import type pg from 'pg'; +import { getPgEnvOptions, type PgConfig } from 'pg-env'; + +import { teardownPgPools } from '../lru'; +import { + acquirePgNotificationBroker, + getPgNotificationBrokerStats, + PgNotificationTopicError, + teardownPgNotificationBrokers +} from '../notification-broker'; +import { defaultPgPoolFactory, getPgPool } from '../pg'; + +// Production acquisition always audits the login on its pinned listener, so +// this test requires the dedicated least-privilege notification fixture. +const describeWithPostgres = + process.env.PG_CACHE_RUN_NOTIFICATION_ROLE_INTEGRATION === '1' + ? describe + : describe.skip; + +describeWithPostgres('notification broker against PostgreSQL', () => { + let observerPool: pg.Pool; + let listenerPgConfig: PgConfig & { pool: { max: number } }; + + beforeAll(() => { + listenerPgConfig = { + ...getPgEnvOptions(), + pool: { max: 1 } + }; + observerPool = defaultPgPoolFactory( + { ...listenerPgConfig, pool: { max: 1 } }, + { + purpose: 'notification-broker-integration-observer', + sanitizeOnCheckout: false + } + ) as pg.Pool; + }); + + afterAll(async () => { + await teardownPgNotificationBrokers(); + await teardownPgPools(); + await observerPool?.end(); + }); + + it('shares one LISTEN backend across three isolated generation leases and releases it', async () => { + const nonce = `${process.pid.toString(36)}_${Date.now().toString(36)}`; + const topics = [ + `pg_cache_it_${nonce}_a`, + `pg_cache_it_${nonce}_b`, + `pg_cache_it_${nonce}_c` + ]; + const listenQueries = topics.map((topic) => `LISTEN "${topic}"`); + + const first = await acquirePgNotificationBroker(listenerPgConfig, { + topics: [topics[0]] + }); + const second = await acquirePgNotificationBroker(listenerPgConfig, { + topics: [topics[1]] + }); + const third = await acquirePgNotificationBroker(listenerPgConfig, { + topics: [topics[2]] + }); + const brokerPool = getPgPool(listenerPgConfig, { + purpose: 'notification-broker', + sanitizeOnCheckout: true + }); + + expect(new Set([first.identity, second.identity, third.identity]).size).toBe(1); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 1, + listenerConnections: 1, + leases: 3, + topics: 3 + }); + expect(brokerPool.totalCount).toBe(1); + expect(brokerPool.idleCount).toBe(0); + + const activeListeners = await observerPool.query<{ + pid: number; + query: string; + }>(` + SELECT pid, query + FROM pg_stat_activity + WHERE datname = current_database() + AND usename = current_user + AND pid <> pg_backend_pid() + AND query = ANY($1::text[]) + `, [listenQueries]); + expect(activeListeners.rows).toEqual([ + { pid: expect.any(Number), query: listenQueries[2] } + ]); + const listenerPid = activeListeners.rows[0].pid; + + expect(() => first.subscribe(topics[1])).toThrow(PgNotificationTopicError); + const firstStream = first.subscribe(topics[0]); + const secondStream = second.subscribe(topics[1]); + const thirdStream = third.subscribe(topics[2]); + let firstResolved = false; + let thirdResolved = false; + const firstNext = firstStream.next().then((result) => { + firstResolved = true; + return result; + }); + const secondNext = secondStream.next(); + const thirdNext = thirdStream.next().then((result) => { + thirdResolved = true; + return result; + }); + + await observerPool.query('SELECT pg_notify($1, $2)', [topics[1], 'for-second']); + await expect(secondNext).resolves.toEqual({ done: false, value: 'for-second' }); + // Delivery to every lease happens synchronously inside one notification + // callback, so these flags prove the second topic did not reach its peers. + expect(firstResolved).toBe(false); + expect(thirdResolved).toBe(false); + + await observerPool.query('SELECT pg_notify($1, $2)', [topics[0], 'for-first']); + await observerPool.query('SELECT pg_notify($1, $2)', [topics[2], 'for-third']); + await expect(firstNext).resolves.toEqual({ done: false, value: 'for-first' }); + await expect(thirdNext).resolves.toEqual({ done: false, value: 'for-third' }); + + await second.release(); + await first.release(); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 1, + listenerConnections: 1, + leases: 1, + topics: 1 + }); + expect(brokerPool.idleCount).toBe(0); + + await third.release(); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + topics: 0 + }); + expect(brokerPool.totalCount).toBe(0); + expect(brokerPool.idleCount).toBe(0); + + let releasedListenerRows: Array<{ pid: number }> = []; + for (let attempt = 0; attempt < 50; attempt++) { + const releasedListener = await observerPool.query<{ pid: number }>(` + SELECT pid + FROM pg_stat_activity + WHERE pid = $1 + `, [listenerPid]); + releasedListenerRows = releasedListener.rows; + if (releasedListenerRows.length === 0) break; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + expect(releasedListenerRows).toEqual([]); + }); +}); diff --git a/postgres/pg-cache/src/__tests__/notification-broker.test.ts b/postgres/pg-cache/src/__tests__/notification-broker.test.ts new file mode 100644 index 0000000000..b179908f25 --- /dev/null +++ b/postgres/pg-cache/src/__tests__/notification-broker.test.ts @@ -0,0 +1,855 @@ +import { EventEmitter } from 'node:events'; + +import { + DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS, + getPgNotificationBrokerIdentity, + getPgNotificationDatabaseIdentity, + PgNotificationBrokerFailedError, + PgNotificationBrokerRegistry, + PgNotificationConnectionSource, + PgNotificationOperationTimeoutError, + PgNotificationQueueOverflowError, + PgNotificationTopicError +} from '../notification-broker'; +import { + PG_NOTIFICATION_ROLE_AUDIT_SQL, + UnsafePgNotificationRoleError +} from '../notification-role'; + +const roleContract = { + role: 'tenant_a_notify', + database: 'tenant_a' +}; + +const safeRoleAuditRow = { + expected_role: roleContract.role, + session_role: roleContract.role, + active_role: roleContract.role, + active_database: roleContract.database, + rolcanlogin: true, + rolinherit: false, + rolsuper: false, + rolbypassrls: false, + rolcreaterole: false, + rolcreatedb: false, + rolreplication: false, + membership_count: 0, + target_database_exists: true, + target_connect: true, + other_database_connect_count: 0, + target_database_owner: false, + target_database_create: false, + target_database_temp: false, + schema_owner_count: 0, + schema_create_count: 0, + schema_usage_count: 0, + relation_privilege_count: 0, + function_privilege_count: 0, + sequence_privilege_count: 0 +}; + +class MockNotificationClient extends EventEmitter { + readonly queries: string[] = []; + roleAuditRow: Record | undefined = safeRoleAuditRow; + readonly query = jest.fn(async ( + text: string, + _values?: readonly unknown[] + ): Promise => { + this.queries.push(text); + if (text === PG_NOTIFICATION_ROLE_AUDIT_SQL) { + return { rows: this.roleAuditRow ? [this.roleAuditRow] : [] }; + } + return { rows: [] }; + }); + readonly release = jest.fn(async (_error?: Error | boolean): Promise => {}); + + notification(channel: string, payload?: string): void { + this.emit('notification', { channel, payload }); + } +} + +const createSource = (client = new MockNotificationClient()) => { + const source: PgNotificationConnectionSource = { + connect: jest.fn(async () => client), + release: jest.fn(async () => {}) + }; + return { client, source }; +}; + +const deferred = () => { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +const flushMicrotasks = async (): Promise => { + for (let index = 0; index < 20; index++) await Promise.resolve(); +}; + +describe('PgNotificationBrokerRegistry', () => { + it('shares one dedicated listener and reference-counts exact topics', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const sourceFactory = jest.fn(() => source); + + const [first, second] = await Promise.all([ + registry.acquireForTests('opaque-a', sourceFactory, ['tenant.a', 'shared']), + registry.acquireForTests('opaque-a', sourceFactory, ['shared', 'tenant.b']) + ]); + + expect(sourceFactory).toHaveBeenCalledTimes(1); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(client.queries).toEqual([ + 'LISTEN "tenant.a"', + 'LISTEN "shared"', + 'LISTEN "tenant.b"' + ]); + expect(registry.stats()).toMatchObject({ + brokers: 1, + listenerConnections: 1, + leases: 2, + topics: 3 + }); + + await first.release(); + expect(client.queries).toContain('UNLISTEN "tenant.a"'); + expect(client.queries).not.toContain('UNLISTEN "shared"'); + expect(client.release).not.toHaveBeenCalled(); + + await second.release(); + expect(client.queries.slice(-2)).toEqual([ + 'UNLISTEN "shared"', + 'UNLISTEN "tenant.b"' + ]); + expect(client.release).toHaveBeenCalledWith(true); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ brokers: 0, leases: 0, topics: 0 }); + }); + + it('audits three generations on the one pinned listener before admission', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const sourceFactory = jest.fn(() => source); + + const first = await registry.acquireAttestedForTests( + 'opaque-attested', sourceFactory, ['tenant.a'], roleContract + ); + const second = await registry.acquireAttestedForTests( + 'opaque-attested', sourceFactory, ['tenant.b'], roleContract + ); + const third = await registry.acquireAttestedForTests( + 'opaque-attested', sourceFactory, ['tenant.c'], roleContract + ); + + expect(sourceFactory).toHaveBeenCalledTimes(1); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(client.queries.filter( + (query) => query === PG_NOTIFICATION_ROLE_AUDIT_SQL + )).toHaveLength(3); + expect(client.queries).toEqual([ + 'BEGIN READ ONLY', + 'SET LOCAL jit TO off', + PG_NOTIFICATION_ROLE_AUDIT_SQL, + 'COMMIT', + 'LISTEN "tenant.a"', + 'BEGIN READ ONLY', + 'SET LOCAL jit TO off', + PG_NOTIFICATION_ROLE_AUDIT_SQL, + 'COMMIT', + 'LISTEN "tenant.b"', + 'BEGIN READ ONLY', + 'SET LOCAL jit TO off', + PG_NOTIFICATION_ROLE_AUDIT_SQL, + 'COMMIT', + 'LISTEN "tenant.c"' + ]); + expect(first.roleAudit).toMatchObject({ ...roleContract, safe: true }); + expect(second.roleAudit).toMatchObject({ ...roleContract, safe: true }); + expect(third.roleAudit).toMatchObject({ ...roleContract, safe: true }); + expect(registry.stats()).toMatchObject({ + listenerConnections: 1, + leases: 3, + roleAuditAttempts: 3, + roleAuditFailures: 0 + }); + + await Promise.all([first.release(), second.release(), third.release()]); + }); + + it('serializes concurrent admission audits without another connection', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const firstCatalogAudit = deferred(); + let catalogAuditsStarted = 0; + let activeCatalogAudits = 0; + let peakCatalogAudits = 0; + client.query.mockImplementation(async (text: string) => { + client.queries.push(text); + if (text === PG_NOTIFICATION_ROLE_AUDIT_SQL) { + catalogAuditsStarted++; + activeCatalogAudits++; + peakCatalogAudits = Math.max(peakCatalogAudits, activeCatalogAudits); + if (catalogAuditsStarted === 1) await firstCatalogAudit.promise; + activeCatalogAudits--; + return { rows: [safeRoleAuditRow] }; + } + return { rows: [] }; + }); + + const acquisitions = [ + registry.acquireAttestedForTests( + 'opaque-attested', () => source, ['a'], roleContract + ), + registry.acquireAttestedForTests( + 'opaque-attested', () => source, ['b'], roleContract + ), + registry.acquireAttestedForTests( + 'opaque-attested', () => source, ['c'], roleContract + ) + ]; + await flushMicrotasks(); + expect(catalogAuditsStarted).toBe(1); + expect(source.connect).toHaveBeenCalledTimes(1); + + firstCatalogAudit.resolve(); + const leases = await Promise.all(acquisitions); + expect(catalogAuditsStarted).toBe(3); + expect(peakCatalogAudits).toBe(1); + expect(source.connect).toHaveBeenCalledTimes(1); + await Promise.all(leases.map((lease) => lease.release())); + }); + + it('bounds a never-resolving admission audit and destroys its client', async () => { + jest.useFakeTimers(); + try { + const registry = new PgNotificationBrokerRegistry(4, 25); + const { client, source } = createSource(); + client.query.mockImplementation((text: string) => { + client.queries.push(text); + if (text === 'BEGIN READ ONLY') return new Promise(() => undefined); + return Promise.resolve({ rows: [] }); + }); + + const acquiring = registry.acquireAttestedForTests( + 'opaque-timeout', + () => source, + ['a'], + roleContract + ); + const rejected = expect(acquiring).rejects.toBeInstanceOf( + PgNotificationOperationTimeoutError + ); + await flushMicrotasks(); + await jest.advanceTimersByTimeAsync(25); + await rejected; + + expect(client.queries).toEqual(['BEGIN READ ONLY']); + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + fatalFailures: 1, + roleAuditAttempts: 1, + roleAuditFailures: 1 + }); + } finally { + jest.useRealTimers(); + } + }); + + it('revalidates on the pinned listener and fails every lease closed on drift', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const first = await registry.acquireAttestedForTests( + 'opaque-attested', () => source, ['a'], roleContract + ); + const second = await registry.acquireAttestedForTests( + 'opaque-attested', () => source, ['b'], roleContract + ); + + await expect(first.revalidateRole()).resolves.toMatchObject({ safe: true }); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + roleAuditAttempts: 3, + roleAuditFailures: 0 + }); + + const firstNext = first.subscribe('a').next(); + const secondNext = second.subscribe('b').next(); + client.roleAuditRow = { ...safeRoleAuditRow, rolsuper: true }; + await expect(second.revalidateRole()).rejects.toBeInstanceOf( + UnsafePgNotificationRoleError + ); + await expect(firstNext).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + await expect(secondNext).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + await expect(first.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await expect(second.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(registry.stats()).toMatchObject({ + listenerConnections: 0, + leases: 2, + fatalFailures: 1, + roleAuditAttempts: 4, + roleAuditFailures: 1 + }); + + await Promise.all([first.release(), second.release()]); + }); + + it('bounds a never-resolving TTL role refresh on the pinned listener', async () => { + jest.useFakeTimers(); + try { + const registry = new PgNotificationBrokerRegistry(4, 25); + const { client, source } = createSource(); + const lease = await registry.acquireAttestedForTests( + 'opaque-refresh-timeout', + () => source, + ['a'], + roleContract + ); + client.query.mockImplementation((text: string) => { + client.queries.push(text); + if (text === 'BEGIN READ ONLY') return new Promise(() => undefined); + return Promise.resolve({ rows: [] }); + }); + + const refreshing = lease.revalidateRole(); + const rejected = expect(refreshing).rejects.toBeInstanceOf( + PgNotificationOperationTimeoutError + ); + await flushMicrotasks(); + await jest.advanceTimersByTimeAsync(25); + await rejected; + await expect(lease.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(registry.stats()).toMatchObject({ + listenerConnections: 0, + leases: 1, + fatalFailures: 1, + roleAuditAttempts: 2, + roleAuditFailures: 1 + }); + await lease.release(); + } finally { + jest.useRealTimers(); + } + }); + + it('uses exact topic equality for prefix and quoted-identifier channels', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const hostileButValid = 'tenant"; UNLISTEN *;--'; + const lease = await registry.acquireForTests( + 'opaque-a', + () => source, + ['tenant', 'tenant.longer', hostileButValid] + ); + const exact = lease.subscribe('tenant'); + const longer = lease.subscribe('tenant.longer'); + const hostile = lease.subscribe(hostileButValid); + + expect(() => lease.subscribe('ten')).toThrow(PgNotificationTopicError); + expect(client.queries).toContain( + 'LISTEN "tenant""; UNLISTEN *;--"' + ); + + client.notification('ten', 'wrong-prefix'); + client.notification('tenant.longer', 'longer'); + client.notification(hostileButValid, 'quoted'); + client.notification('tenant', 'exact'); + + await expect(exact.next()).resolves.toEqual({ done: false, value: 'exact' }); + await expect(longer.next()).resolves.toEqual({ done: false, value: 'longer' }); + await expect(hostile.next()).resolves.toEqual({ done: false, value: 'quoted' }); + expect(registry.stats().ignoredNotifications).toBe(1); + await lease.release(); + }); + + it('rejects channels PostgreSQL would truncate, including multi-byte Unicode', async () => { + const registry = new PgNotificationBrokerRegistry(); + const { source } = createSource(); + + const ascii63 = 'a'.repeat(63); + const unicode63 = '界'.repeat(21); + const lease = await registry.acquireForTests( + 'opaque-a', + () => source, + [ascii63, unicode63] + ); + expect(lease.topics).toEqual([ascii63, unicode63]); + + await expect( + registry.acquireForTests('opaque-b', () => source, ['a'.repeat(64)]) + ).rejects.toBeInstanceOf(PgNotificationTopicError); + await expect( + registry.acquireForTests('opaque-b', () => source, ['界'.repeat(22)]) + ).rejects.toBeInstanceOf(PgNotificationTopicError); + await expect( + registry.acquireForTests( + 'opaque-b', + () => source, + [`bad${String.fromCharCode(0xd800)}`] + ) + ).rejects.toBeInstanceOf(PgNotificationTopicError); + await lease.release(); + }); + + it('does not normalize canonically equivalent Unicode topics', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const composed = 'réaltime'; + const decomposed = 're\u0301altime'; + const lease = await registry.acquireForTests( + 'opaque-a', + () => source, + [composed, decomposed] + ); + const composedStream = lease.subscribe(composed); + const decomposedStream = lease.subscribe(decomposed); + + client.notification(composed, 'composed-only'); + client.notification(decomposed, 'decomposed-only'); + + await expect(composedStream.next()).resolves.toMatchObject({ value: 'composed-only' }); + await expect(decomposedStream.next()).resolves.toMatchObject({ value: 'decomposed-only' }); + await lease.release(); + }); + + it('fans out only to subscribers for the exact allowed topic', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const first = await registry.acquireForTests('opaque-a', () => source, ['a']); + const second = await registry.acquireForTests('opaque-a', () => source, ['a', 'b']); + const firstA = first.subscribe('a'); + const secondA = second.subscribe('a'); + const secondB = second.subscribe('b'); + + client.notification('a', 'for-a'); + client.notification('b', 'for-b'); + + await expect(firstA.next()).resolves.toMatchObject({ value: 'for-a' }); + await expect(secondA.next()).resolves.toMatchObject({ value: 'for-a' }); + await expect(secondB.next()).resolves.toMatchObject({ value: 'for-b' }); + await Promise.all([first.release(), second.release()]); + }); + + it('fails only the slow subscriber when its bounded queue overflows', async () => { + const registry = new PgNotificationBrokerRegistry(1); + const { client, source } = createSource(); + const lease = await registry.acquireForTests('opaque-a', () => source, ['events']); + const slow = lease.subscribe('events'); + const fast = lease.subscribe('events'); + + const fastFirst = fast.next(); + client.notification('events', 'one'); + const fastSecond = fast.next(); + client.notification('events', 'two'); + + await expect(fastFirst).resolves.toMatchObject({ value: 'one' }); + await expect(fastSecond).resolves.toMatchObject({ value: 'two' }); + await expect(slow.next()).rejects.toBeInstanceOf(PgNotificationQueueOverflowError); + expect(registry.stats()).toMatchObject({ subscribers: 1, queueOverflows: 1 }); + await lease.release(); + }); + + it('fails every active subscriber and never silently reconnects', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const sourceFactory = jest.fn(() => source); + const first = await registry.acquireForTests('opaque-a', sourceFactory, ['a']); + const second = await registry.acquireForTests('opaque-a', sourceFactory, ['b']); + const firstNext = first.subscribe('a').next(); + const secondNext = second.subscribe('b').next(); + + client.emit('error', new Error('socket lost')); + + await expect(firstNext).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + await expect(secondNext).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + await expect(first.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await expect( + registry.acquireForTests('opaque-a', sourceFactory, ['a']) + ).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith(expect.any(PgNotificationBrokerFailedError)); + + await Promise.all([first.release(), second.release()]); + const replacement = createSource(); + const explicitReplacement = await registry.acquireForTests( + 'opaque-a', + () => replacement.source, + ['a'] + ); + expect(replacement.source.connect).toHaveBeenCalledTimes(1); + await explicitReplacement.release(); + }); + + it('bounds a never-resolving LISTEN and fails admission closed', async () => { + jest.useFakeTimers(); + try { + const registry = new PgNotificationBrokerRegistry(4, 25); + const { client, source } = createSource(); + client.query.mockImplementation((text: string) => { + client.queries.push(text); + if (text.startsWith('LISTEN')) return new Promise(() => undefined); + return Promise.resolve({ rows: [] }); + }); + + const acquiring = registry.acquireForTests('opaque-listen-timeout', () => source, ['a']); + const rejected = expect(acquiring).rejects.toMatchObject({ + code: 'PG_NOTIFICATION_BROKER_FAILED', + cause: { code: 'PG_NOTIFICATION_OPERATION_TIMEOUT' } + }); + await flushMicrotasks(); + await jest.advanceTimersByTimeAsync(25); + await rejected; + + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + fatalFailures: 1 + }); + } finally { + jest.useRealTimers(); + } + }); + + it('fails closed when a listener emits a malformed notification', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const lease = await registry.acquireForTests('opaque-a', () => source, ['a']); + const next = lease.subscribe('a').next(); + + client.emit('notification', { channel: 'a', payload: { hostile: true } }); + + await expect(next).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + await expect(lease.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await lease.release(); + }); + + it('makes double release idempotent and awaits UNLISTEN plus both releases', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const unlisten = deferred(); + const clientReleased = deferred(); + const sourceReleased = deferred(); + client.query.mockImplementation(async (text: string) => { + client.queries.push(text); + if (text.startsWith('UNLISTEN')) await unlisten.promise; + return { rows: [] }; + }); + client.release.mockImplementation(async () => clientReleased.promise); + (source.release as jest.Mock).mockImplementation(async () => sourceReleased.promise); + const lease = await registry.acquireForTests('opaque-a', () => source, ['a']); + + const firstRelease = lease.release(); + const secondRelease = lease.release(); + expect(firstRelease).toBe(secondRelease); + await flushMicrotasks(); + expect(client.queries).toContain('UNLISTEN "a"'); + + let settled = false; + void firstRelease.then(() => { + settled = true; + }); + unlisten.resolve(); + await flushMicrotasks(); + expect(settled).toBe(false); + clientReleased.resolve(); + await flushMicrotasks(); + expect(settled).toBe(false); + sourceReleased.resolve(); + await firstRelease; + await expect(lease.terminated).resolves.toBeNull(); + expect(settled).toBe(true); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + }); + + it('serializes a final release against a concurrent new acquisition', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const firstSource = createSource(); + const unlisten = deferred(); + firstSource.client.query.mockImplementation(async (text: string) => { + firstSource.client.queries.push(text); + if (text.startsWith('UNLISTEN')) await unlisten.promise; + return { rows: [] }; + }); + const first = await registry.acquireForTests( + 'opaque-a', + () => firstSource.source, + ['a'] + ); + const releasing = first.release(); + await flushMicrotasks(); + + const secondSource = createSource(); + const acquiring = registry.acquireForTests( + 'opaque-a', + () => secondSource.source, + ['a'] + ); + await flushMicrotasks(); + expect(secondSource.source.connect).not.toHaveBeenCalled(); + + unlisten.resolve(); + await releasing; + const second = await acquiring; + expect(secondSource.source.connect).toHaveBeenCalledTimes(1); + await second.release(); + }); + + it('makes concurrent registry close calls await the same teardown', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { source } = createSource(); + const sourceReleased = deferred(); + (source.release as jest.Mock).mockImplementation(async () => sourceReleased.promise); + await registry.acquireForTests('opaque-a', () => source, ['a']); + + const firstClose = registry.close(); + const secondClose = registry.close(); + let firstSettled = false; + let secondSettled = false; + void firstClose.then(() => { + firstSettled = true; + }); + void secondClose.then(() => { + secondSettled = true; + }); + await flushMicrotasks(); + + expect(firstSettled).toBe(false); + expect(secondSettled).toBe(false); + sourceReleased.resolve(); + await Promise.all([firstClose, secondClose]); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ brokers: 0, leases: 0 }); + }); + + it('bounds a never-resolving UNLISTEN so teardown cannot hang', async () => { + jest.useFakeTimers(); + try { + const registry = new PgNotificationBrokerRegistry(4, 25); + const { client, source } = createSource(); + await registry.acquireForTests('opaque-unlisten-timeout', () => source, ['a']); + client.query.mockImplementation((text: string) => { + client.queries.push(text); + if (text.startsWith('UNLISTEN')) return new Promise(() => undefined); + return Promise.resolve({ rows: [] }); + }); + + const closing = registry.close(); + const rejected = expect(closing).rejects.toMatchObject({ + code: 'PG_NOTIFICATION_BROKER_FAILED', + cause: { code: 'PG_NOTIFICATION_OPERATION_TIMEOUT' } + }); + await flushMicrotasks(); + await jest.advanceTimersByTimeAsync(25); + await rejected; + + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + fatalFailures: 1 + }); + } finally { + jest.useRealTimers(); + } + }); + + it('drains an in-flight acquisition before registry close resolves', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const connected = deferred(); + const sourceReleased = deferred(); + (source.connect as jest.Mock).mockImplementation(async () => connected.promise); + (source.release as jest.Mock).mockImplementation(async () => sourceReleased.promise); + const acquiring = registry.acquireForTests('opaque-a', () => source, ['a']); + await flushMicrotasks(); + expect(source.connect).toHaveBeenCalledTimes(1); + + const closing = registry.close(); + let closeSettled = false; + void closing.then(() => { + closeSettled = true; + }); + connected.resolve(client); + await flushMicrotasks(); + const closeSettledBeforeSourceRelease = closeSettled; + const issuedListenDuringClose = client.queries.includes('LISTEN "a"'); + sourceReleased.resolve(); + await expect(acquiring).rejects.toThrow( + 'PostgreSQL notification broker registry is closed' + ); + await closing; + expect(closeSettledBeforeSourceRelease).toBe(false); + expect(issuedListenDuringClose).toBe(false); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ brokers: 0, leases: 0 }); + }); + + it('UNLISTENs a provisional topic when close races an in-flight LISTEN', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const listened = deferred(); + client.query.mockImplementation(async (text: string) => { + client.queries.push(text); + if (text === 'LISTEN "a"') await listened.promise; + return { rows: [] }; + }); + const acquiring = registry.acquireForTests('opaque-a', () => source, ['a']); + await flushMicrotasks(); + expect(client.queries).toEqual(['LISTEN "a"']); + + const closing = registry.close(); + listened.resolve(); + await expect(acquiring).rejects.toThrow( + 'PostgreSQL notification broker registry is closed' + ); + await closing; + + expect(client.queries).toEqual(['LISTEN "a"', 'UNLISTEN *']); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + }); + + it('reports a failed UNLISTEN only after finishing registry teardown', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + client.query.mockImplementation(async (text: string) => { + client.queries.push(text); + if (text.startsWith('UNLISTEN')) throw new Error('unlisten failed'); + return { rows: [] }; + }); + await registry.acquireForTests('opaque-a', () => source, ['a']); + + await expect(registry.close()).rejects.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ brokers: 0, leases: 0 }); + }); +}); + +describe('getPgNotificationBrokerIdentity', () => { + const baseConfig = { + host: 'db.internal', + port: 5432, + database: 'customer', + user: 'listener', + password: 'secret' + }; + + it('is versioned, opaque, stable, and includes the canonical SSL contract', () => { + const first = getPgNotificationBrokerIdentity({ + ...baseConfig, + ssl: { rejectUnauthorized: true, ca: 'ca-one' }, + pool: { connectionTimeoutMillis: DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS } + }); + const reordered = getPgNotificationBrokerIdentity({ + ...baseConfig, + ssl: { ca: 'ca-one', rejectUnauthorized: true }, + pool: { connectionTimeoutMillis: DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS } + }); + const changedTls = getPgNotificationBrokerIdentity({ + ...baseConfig, + ssl: { rejectUnauthorized: false, ca: 'ca-one' }, + pool: { connectionTimeoutMillis: DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS } + }); + const changedDeadline = getPgNotificationBrokerIdentity({ + ...baseConfig, + ssl: { rejectUnauthorized: true, ca: 'ca-one' }, + pool: { + connectionTimeoutMillis: + DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS + 1 + } + }); + + expect(first).toBe(reordered); + expect(first).not.toBe(changedTls); + expect(first).not.toBe(changedDeadline); + expect(first).toMatch(/^pg-notification-broker:v1:pg:v1:[a-f0-9]{64}$/); + expect(first).not.toContain('listener'); + expect(first).not.toContain('secret'); + }); + + it.each([0, -1, 1.5, 2_147_483_648])( + 'rejects invalid notification operation timeout %p before identity publication', + (connectionTimeoutMillis) => { + expect(() => getPgNotificationBrokerIdentity({ + ...baseConfig, + pool: { connectionTimeoutMillis } + })).toThrow('notification operation timeout'); + } + ); + + it('uses one credential-free identity for the same physical database target', () => { + const first = getPgNotificationDatabaseIdentity({ + ...baseConfig, + ssl: { rejectUnauthorized: true, ca: 'ca-one' }, + pool: { max: 2 } + }); + const rotated = getPgNotificationDatabaseIdentity({ + ...baseConfig, + user: 'rotated-listener', + password: 'rotated-secret', + ssl: { ca: 'different-ca', rejectUnauthorized: false }, + pool: { max: 20, idleTimeoutMillis: 99_000 } + }); + const otherHost = getPgNotificationDatabaseIdentity({ + ...baseConfig, + host: 'other-db.internal' + }); + const otherPort = getPgNotificationDatabaseIdentity({ + ...baseConfig, + port: 5433 + }); + const otherDatabase = getPgNotificationDatabaseIdentity({ + ...baseConfig, + database: 'other-customer', + ssl: { rejectUnauthorized: true, ca: 'ca-one' } + }); + + expect(first).toBe(rotated); + expect(first).not.toBe(otherHost); + expect(first).not.toBe(otherPort); + expect(first).not.toBe(otherDatabase); + expect(first).toMatch(/^pg-notification-database:v1:pg-target:v1:[a-f0-9]{64}$/); + expect(first).not.toContain('listener'); + expect(first).not.toContain('secret'); + }); +}); diff --git a/postgres/pg-cache/src/__tests__/notification-role.integration.test.ts b/postgres/pg-cache/src/__tests__/notification-role.integration.test.ts new file mode 100644 index 0000000000..587e07c28d --- /dev/null +++ b/postgres/pg-cache/src/__tests__/notification-role.integration.test.ts @@ -0,0 +1,120 @@ +import type pg from 'pg'; +import { getPgEnvOptions } from 'pg-env'; + +import { teardownPgPools } from '../lru'; +import { + acquirePgNotificationBroker, + getPgNotificationBrokerStats, + teardownPgNotificationBrokers +} from '../notification-broker'; +import { + assertPgNotificationRole, + auditPgNotificationRole +} from '../notification-role'; +import { defaultPgPoolFactory, getPgPool } from '../pg'; + +const describeWithNotificationRole = + process.env.PG_CACHE_RUN_NOTIFICATION_ROLE_INTEGRATION === '1' + ? describe + : describe.skip; + +describeWithNotificationRole('dedicated notification role against PostgreSQL', () => { + const pgConfig = getPgEnvOptions(); + let pool: pg.Pool; + + beforeAll(() => { + pool = defaultPgPoolFactory( + { ...pgConfig, pool: { max: 1 } }, + { purpose: 'notification-role-integration', sanitizeOnCheckout: true } + ) as pg.Pool; + }); + + afterAll(async () => { + await teardownPgNotificationBrokers(); + await teardownPgPools(); + await pool?.end(); + }); + + it('accepts only the exact credential-free role/database contract', async () => { + const audit = await assertPgNotificationRole(pool, { + role: pgConfig.user, + database: pgConfig.database + }); + + expect(audit).toMatchObject({ + role: pgConfig.user, + database: pgConfig.database, + safe: true, + violations: [] + }); + expect(Object.keys(audit).sort()).toEqual([ + 'database', + 'role', + 'safe', + 'version', + 'violations' + ]); + expect(audit).not.toHaveProperty('password'); + expect(audit).not.toHaveProperty('host'); + + const wrongRole = await auditPgNotificationRole(pool, { + role: `wrong_${process.pid}`, + database: pgConfig.database + }); + expect(wrongRole).toMatchObject({ + safe: false, + violations: expect.arrayContaining(['LOGIN_ROLE_MISMATCH']) + }); + + const wrongDatabase = await auditPgNotificationRole(pool, { + role: pgConfig.user, + database: `wrong_${process.pid}` + }); + expect(wrongDatabase).toMatchObject({ + safe: false, + violations: expect.arrayContaining([ + 'DATABASE_MISMATCH', + 'TARGET_DATABASE_MISSING', + 'TARGET_CONNECT_REQUIRED', + 'CROSS_DATABASE_CONNECT' + ]) + }); + }); + + it('retains enough privilege for isolated LISTEN and NOTIFY delivery', async () => { + const nonce = `${process.pid}_${Date.now().toString(36)}`; + const topics = [0, 1, 2].map((index) => `notify_role_it_${nonce}_${index}`); + const listenerConfig = { ...pgConfig, pool: { max: 1 } }; + const statsBefore = getPgNotificationBrokerStats(); + const [first, second, third] = await Promise.all(topics.map((topic) => + acquirePgNotificationBroker(listenerConfig, { topics: [topic] }) + )); + const brokerPool = getPgPool(listenerConfig, { + purpose: 'notification-broker', + sanitizeOnCheckout: true + }); + await first.revalidateRole(); + const next = second.subscribe(topics[1]).next(); + + await pool.query('SELECT pg_notify($1, $2)', [topics[1], 'safe-listener']); + await expect(next).resolves.toEqual({ done: false, value: 'safe-listener' }); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 1, + listenerConnections: 1, + leases: 3, + topics: 3, + roleAuditAttempts: statsBefore.roleAuditAttempts + 4, + roleAuditFailures: statsBefore.roleAuditFailures + }); + expect(brokerPool.totalCount).toBe(1); + expect(brokerPool.idleCount).toBe(0); + + await Promise.all([first.release(), second.release(), third.release()]); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + topics: 0 + }); + }); +}); diff --git a/postgres/pg-cache/src/__tests__/notification-role.test.ts b/postgres/pg-cache/src/__tests__/notification-role.test.ts new file mode 100644 index 0000000000..365cd220d7 --- /dev/null +++ b/postgres/pg-cache/src/__tests__/notification-role.test.ts @@ -0,0 +1,262 @@ +import type { Pool } from 'pg'; + +import { + assertPgNotificationRole, + assertPgNotificationRoleClient, + auditPgNotificationRole, + auditPgNotificationRoleClient, + normalizePgNotificationRoleContracts, + PG_NOTIFICATION_ROLE_AUDIT_SQL, + PG_NOTIFICATION_ROLE_AUDIT_VERSION, + type PgNotificationRoleClient, + PgNotificationRoleContractError, + type PgNotificationRoleViolationCode, + UnsafePgNotificationRoleError +} from '../notification-role'; + +const contract = { + role: 'tenant_001_notification', + database: 'tenant_001' +}; + +const safeRow = { + expected_role: contract.role, + session_role: contract.role, + active_role: contract.role, + active_database: contract.database, + rolcanlogin: true, + rolinherit: false, + rolsuper: false, + rolbypassrls: false, + rolcreaterole: false, + rolcreatedb: false, + rolreplication: false, + membership_count: 0, + target_database_exists: true, + target_connect: true, + other_database_connect_count: 0, + target_database_owner: false, + target_database_create: false, + target_database_temp: false, + schema_owner_count: 0, + schema_create_count: 0, + schema_usage_count: 0, + relation_privilege_count: 0, + function_privilege_count: 0, + sequence_privilege_count: 0 +}; + +const poolWithRow = (row: Record | undefined) => { + const client = { + query: jest.fn(async (query: string) => query === PG_NOTIFICATION_ROLE_AUDIT_SQL + ? { rows: row ? [row] : [] } + : { rows: [] }), + release: jest.fn() + }; + return { + pool: { connect: jest.fn(async () => client) } as unknown as Pool, + client + }; +}; + +describe('PostgreSQL notification-role audit', () => { + it('returns a frozen credential-free attestation for an exact safe login', async () => { + const { pool, client } = poolWithRow(safeRow); + const audit = await assertPgNotificationRole( + pool, + { ...contract, password: 'must-not-escape' } as typeof contract + ); + + expect(audit).toEqual({ + version: PG_NOTIFICATION_ROLE_AUDIT_VERSION, + ...contract, + safe: true, + violations: [] + }); + expect(Object.isFrozen(audit)).toBe(true); + expect(Object.isFrozen(audit.violations)).toBe(true); + expect(JSON.stringify(audit)).not.toContain('must-not-escape'); + expect(client.query).toHaveBeenNthCalledWith( + 1, + 'BEGIN READ ONLY' + ); + expect(client.query).toHaveBeenNthCalledWith(2, 'SET LOCAL jit TO off'); + expect(client.query).toHaveBeenNthCalledWith(3, PG_NOTIFICATION_ROLE_AUDIT_SQL, [ + contract.role, + contract.database + ]); + expect(client.query).toHaveBeenNthCalledWith(4, 'COMMIT'); + expect(client.release).toHaveBeenCalledWith(false); + }); + + it('audits an already-owned listener client without releasing it', async () => { + const { client } = poolWithRow(safeRow); + + await expect(assertPgNotificationRoleClient( + client as unknown as PgNotificationRoleClient, + contract + )).resolves + .toMatchObject({ ...contract, safe: true }); + expect(client.release).not.toHaveBeenCalled(); + }); + + it('rolls back a failed pinned-client audit without taking ownership of release', async () => { + const failure = new Error('catalog unavailable'); + const client = { + query: jest.fn() + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce({ rows: [] }), + release: jest.fn() + }; + + await expect(auditPgNotificationRoleClient(client, contract)).rejects.toBe(failure); + expect(client.query).toHaveBeenNthCalledWith(4, 'ROLLBACK'); + expect(client.release).not.toHaveBeenCalled(); + }); + + it.each<[ + keyof typeof safeRow, + unknown, + PgNotificationRoleViolationCode + ]>([ + ['session_role', 'different_login', 'LOGIN_ROLE_MISMATCH'], + ['active_role', 'set_role_target', 'CURRENT_ROLE_MISMATCH'], + ['active_database', 'different_database', 'DATABASE_MISMATCH'], + ['rolcanlogin', false, 'LOGIN_REQUIRED'], + ['rolinherit', true, 'NOINHERIT_REQUIRED'], + ['rolsuper', true, 'SUPERUSER'], + ['rolbypassrls', true, 'BYPASSRLS'], + ['rolcreaterole', true, 'CREATEROLE'], + ['rolcreatedb', true, 'CREATEDB'], + ['rolreplication', true, 'REPLICATION'], + ['membership_count', 1, 'ROLE_MEMBERSHIP'], + ['target_database_exists', false, 'TARGET_DATABASE_MISSING'], + ['target_connect', false, 'TARGET_CONNECT_REQUIRED'], + ['other_database_connect_count', 1, 'CROSS_DATABASE_CONNECT'], + ['target_database_owner', true, 'DATABASE_OWNER'], + ['target_database_create', true, 'DATABASE_CREATE'], + ['target_database_temp', true, 'DATABASE_TEMP'], + ['schema_owner_count', 1, 'SCHEMA_OWNER'], + ['schema_create_count', 1, 'SCHEMA_CREATE'], + ['schema_usage_count', 1, 'SCHEMA_USAGE'], + ['relation_privilege_count', 1, 'RELATION_PRIVILEGE'], + ['function_privilege_count', 1, 'FUNCTION_PRIVILEGE'], + ['sequence_privilege_count', 1, 'SEQUENCE_PRIVILEGE'] + ])('maps %s to its stable violation code', async (field, unsafeValue, code) => { + const { pool } = poolWithRow({ ...safeRow, [field]: unsafeValue }); + const audit = await auditPgNotificationRole(pool, contract); + + expect(audit.safe).toBe(false); + expect(audit.violations).toContain(code); + await expect(assertPgNotificationRole( + poolWithRow({ ...safeRow, [field]: unsafeValue }).pool, + contract + )).rejects.toMatchObject({ + code: 'PG_NOTIFICATION_ROLE_UNSAFE', + audit: expect.objectContaining({ violations: expect.arrayContaining([code]) }) + }); + }); + + it('fails closed when the catalog audit returns no role row', async () => { + const { pool } = poolWithRow(undefined); + const audit = await auditPgNotificationRole(pool, contract); + + expect(audit).toMatchObject({ safe: false, violations: ['AUDIT_NO_RESULT'] }); + await expect(assertPgNotificationRole(poolWithRow(undefined).pool, contract)) + .rejects.toBeInstanceOf(UnsafePgNotificationRoleError); + }); + + it('rolls back and destroys the client when the catalog query fails', async () => { + const failure = new Error('catalog unavailable'); + const client = { + query: jest.fn() + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce({ rows: [] }), + release: jest.fn() + }; + const pool = { connect: jest.fn(async () => client) } as unknown as Pool; + + await expect(auditPgNotificationRole(pool, contract)).rejects.toBe(failure); + expect(client.query).toHaveBeenNthCalledWith(4, 'ROLLBACK'); + expect(client.release).toHaveBeenCalledWith(true); + }); + + it('audits exact database scope, membership edges, and every prohibited ACL class', () => { + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'membership.member = r.oid OR membership.roleid = r.oid' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'database_record.datname <> $2::text' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("'CONNECT'"); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("'CREATE'"); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("'TEMP'"); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain('schema_record.nspowner = r.oid'); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("'USAGE'"); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'pg_catalog.has_table_privilege' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'pg_catalog.has_any_column_privilege' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'pg_catalog.has_function_privilege' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'pg_catalog.has_sequence_privilege' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("n.nspname !~ '^pg_'"); + }); +}); + +describe('notification-role fleet contract', () => { + it('collapses exact generation duplicates and returns a deterministic frozen mapping', () => { + const normalized = normalizePgNotificationRoleContracts([ + { role: 'notify_b', database: 'tenant_b' }, + { ...contract }, + { ...contract } + ]); + + expect(normalized).toEqual([ + contract, + { role: 'notify_b', database: 'tenant_b' } + ]); + expect(Object.isFrozen(normalized)).toBe(true); + expect(normalized.every(Object.isFrozen)).toBe(true); + }); + + it('rejects multiple logins for one database and one login spanning databases', () => { + expect(() => normalizePgNotificationRoleContracts([ + contract, + { role: 'another_notification', database: contract.database } + ])).toThrow('maps to multiple login roles'); + expect(() => normalizePgNotificationRoleContracts([ + contract, + { role: contract.role, database: 'tenant_002' } + ])).toThrow('maps to multiple databases'); + }); + + const malformedContracts: Array<{ + contracts: readonly { role: string; database: string }[]; + }> = [ + { contracts: [] }, + { contracts: [{ role: '', database: 'tenant_001' }] }, + { contracts: [{ role: 'notify', database: '' }] }, + { contracts: [{ role: 'n'.repeat(64), database: 'tenant_001' }] }, + { + contracts: [{ + role: 'notify', + database: `bad${String.fromCharCode(0xd800)}` + }] + } + ]; + + it.each(malformedContracts)('rejects malformed contract input', ({ contracts }) => { + expect(() => normalizePgNotificationRoleContracts(contracts)) + .toThrow(PgNotificationRoleContractError); + }); +}); diff --git a/postgres/pg-cache/src/__tests__/sanitizer.integration.test.ts b/postgres/pg-cache/src/__tests__/sanitizer.integration.test.ts new file mode 100644 index 0000000000..30272f5d74 --- /dev/null +++ b/postgres/pg-cache/src/__tests__/sanitizer.integration.test.ts @@ -0,0 +1,103 @@ +import type pg from 'pg'; + +import { defaultPgPoolFactory, getPgCheckoutSanitizerStats } from '../pg'; + +const describeWithPostgres = process.env.PG_CACHE_RUN_PG_INTEGRATION === '1' + ? describe + : describe.skip; + +describeWithPostgres('runtime checkout sanitation against PostgreSQL', () => { + let pool: pg.Pool; + + beforeAll(() => { + pool = defaultPgPoolFactory( + { pool: { max: 1 } }, + { purpose: 'runtime', sanitizeOnCheckout: true } + ) as pg.Pool; + }); + + afterAll(async () => { + await pool?.end(); + }); + + it('restores the startup baseline and clears poisoned prepared state', async () => { + const poisoned = await pool.connect(); + const poisonedPid = await poisoned.query<{ pid: number }>( + 'SELECT pg_catalog.pg_backend_pid()::integer AS pid' + ); + await poisoned.query('SET search_path TO public'); + await poisoned.query('SET row_security TO off'); + await poisoned.query('SET jit_optimize_above_cost TO 123'); + await poisoned.query("SET application_name TO 'poisoned-tenant-session'"); + await poisoned.query({ name: 'tenant_cache_canary', text: 'SELECT 1 AS value' }); + poisoned.release(); + expect(getPgCheckoutSanitizerStats(pool)).toMatchObject({ + virginFastPathCheckouts: 1, + sanitizedReuseCheckouts: 0 + }); + + const clean = await pool.connect(); + try { + const cleanPid = await clean.query<{ pid: number }>( + 'SELECT pg_catalog.pg_backend_pid()::integer AS pid' + ); + expect(cleanPid.rows[0]?.pid).toBe(poisonedPid.rows[0]?.pid); + const settings = await clean.query<{ + search_path: string; + row_security: string; + jit_optimize_above_cost: string; + application_name: string; + }>(` + SELECT + current_setting('search_path') AS search_path, + current_setting('row_security') AS row_security, + current_setting('jit_optimize_above_cost') AS jit_optimize_above_cost, + current_setting('application_name') AS application_name + `); + expect(settings.rows[0]).toMatchObject({ + search_path: 'pg_catalog', + row_security: 'on', + jit_optimize_above_cost: '-1' + }); + expect(settings.rows[0].application_name).not.toBe('poisoned-tenant-session'); + + await expect(clean.query({ + name: 'tenant_cache_canary', + text: 'SELECT 2 AS value' + })).resolves.toMatchObject({ rows: [{ value: 2 }] }); + expect(getPgCheckoutSanitizerStats(pool)).toMatchObject({ + virginFastPathCheckouts: 1, + sanitizedReuseCheckouts: 1, + sanitationFailures: 0 + }); + } finally { + clean.release(); + } + }); + + it('proves maxUses=1 rotates the PostgreSQL backend instead of reusing it', async () => { + const rotatingPool = defaultPgPoolFactory( + { pool: { max: 1, maxUses: 1 } }, + { purpose: 'runtime-max-uses-one', sanitizeOnCheckout: true } + ) as pg.Pool; + try { + const first = await rotatingPool.connect(); + const firstPid = await first.query<{ pid: number }>( + 'SELECT pg_catalog.pg_backend_pid()::integer AS pid' + ); + first.release(); + + const second = await rotatingPool.connect(); + try { + const secondPid = await second.query<{ pid: number }>( + 'SELECT pg_catalog.pg_backend_pid()::integer AS pid' + ); + expect(secondPid.rows[0]?.pid).not.toBe(firstPid.rows[0]?.pid); + } finally { + second.release(); + } + } finally { + await rotatingPool.end(); + } + }); +}); diff --git a/postgres/pg-cache/src/__tests__/sanitizer.test.ts b/postgres/pg-cache/src/__tests__/sanitizer.test.ts new file mode 100644 index 0000000000..78a4890722 --- /dev/null +++ b/postgres/pg-cache/src/__tests__/sanitizer.test.ts @@ -0,0 +1,285 @@ +import { EventEmitter } from 'node:events'; + +import type pg from 'pg'; + +import { + getPgCheckoutSanitizerStats, + installCheckoutSanitizer, + sanitizePgClient +} from '../pg'; + +const mockClient = () => ({ + query: jest.fn(async () => ({ rows: [] as unknown[] })), + release: jest.fn(), + connection: { + parsedStatements: { tenant_query: 'select 1' }, + _graphilePreparedStatementCache: { reset: jest.fn() } + } +}) as unknown as pg.PoolClient & { + connection: { + parsedStatements: Record; + _graphilePreparedStatementCache?: { reset: jest.Mock }; + }; +}; + +describe('runtime checkout sanitation', () => { + it('discards server state and clears both prepared-statement caches', async () => { + const client = mockClient(); + + await expect(sanitizePgClient(client)).resolves.toBe(client); + + expect(client.query).toHaveBeenNthCalledWith(1, 'DISCARD ALL'); + expect(client.query).toHaveBeenNthCalledWith( + 2, + 'SET search_path TO pg_catalog; SET row_security TO on; SET jit_optimize_above_cost TO -1' + ); + expect(client.connection.parsedStatements).toEqual({}); + expect(client.connection).not.toHaveProperty('_graphilePreparedStatementCache'); + expect(client.release).not.toHaveBeenCalled(); + }); + + it('does not run Dataplan LRU disposers after DISCARD ALL', async () => { + const client = mockClient(); + const reset = client.connection._graphilePreparedStatementCache!.reset; + + await sanitizePgClient(client, true); + + expect(reset).not.toHaveBeenCalled(); + expect(client.query).toHaveBeenCalledTimes(1); + expect(client.query).toHaveBeenCalledWith('DISCARD ALL'); + }); + + it('uses one checkout query when DISCARD restores a pinned startup baseline', async () => { + const client = mockClient(); + + await expect(sanitizePgClient(client, true)).resolves.toBe(client); + + expect(client.query).toHaveBeenCalledTimes(1); + expect(client.query).toHaveBeenCalledWith('DISCARD ALL'); + expect(client.connection.parsedStatements).toEqual({}); + }); + + it('destroys a connection when DISCARD ALL fails', async () => { + const client = mockClient(); + (client.query as jest.Mock).mockRejectedValueOnce(new Error('idle in transaction')); + + await expect(sanitizePgClient(client)).rejects.toThrow('idle in transaction'); + expect(client.release).toHaveBeenCalledWith(true); + }); + + it('destroys a connection when restoring the trusted baseline fails', async () => { + const client = mockClient(); + (client.query as jest.Mock) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(new Error('baseline rejected')); + + await expect(sanitizePgClient(client)).rejects.toThrow('baseline rejected'); + expect(client.release).toHaveBeenCalledWith(true); + }); + + it('skips DISCARD only for a factory-marked virgin with no competing connect hook', async () => { + const client = mockClient(); + let connected = false; + const pool = Object.assign(new EventEmitter(), { + waitingCount: 0, + query: jest.fn(), + connect: jest.fn(async () => { + if (!connected) { + connected = true; + pool.emit('connect', client); + } + return client; + }) + }) as unknown as pg.Pool; + + installCheckoutSanitizer(pool, true, true); + + await expect(pool.connect()).resolves.toBe(client); + expect(client.query).not.toHaveBeenCalled(); + expect(getPgCheckoutSanitizerStats(pool)).toMatchObject({ + checkoutAttempts: 1, + virginFastPathCheckouts: 1, + sanitizedReuseCheckouts: 0 + }); + + await expect(pool.connect()).resolves.toBe(client); + expect(client.query).toHaveBeenCalledTimes(1); + expect(client.query).toHaveBeenCalledWith('DISCARD ALL'); + expect(getPgCheckoutSanitizerStats(pool)).toMatchObject({ + checkoutAttempts: 2, + virginFastPathCheckouts: 1, + sanitizedReuseCheckouts: 1, + sanitationFailures: 0 + }); + }); + + it('fully sanitizes a virgin after a self-removing connect hook can touch it', async () => { + const client = mockClient(); + let connected = false; + const pool = Object.assign(new EventEmitter(), { + waitingCount: 0, + query: jest.fn(), + connect: jest.fn(async () => { + if (!connected) { + connected = true; + pool.emit('connect', client); + } + return client; + }) + }) as unknown as pg.Pool; + + installCheckoutSanitizer(pool, true, true); + pool.prependOnceListener('connect', () => undefined); + + await expect(pool.connect()).resolves.toBe(client); + expect(client.query).toHaveBeenCalledWith('DISCARD ALL'); + expect(getPgCheckoutSanitizerStats(pool)).toMatchObject({ + virginFastPathCheckouts: 0, + sanitizedReuseCheckouts: 1 + }); + }); + + it('fully sanitizes a minimal custom pool without EventEmitter methods', async () => { + const client = mockClient(); + const pool = { + waitingCount: 0, + query: jest.fn(), + connect: jest.fn(async () => client) + } as unknown as pg.Pool; + + installCheckoutSanitizer(pool); + + await expect(pool.connect()).resolves.toBe(client); + expect(client.query).toHaveBeenNthCalledWith(1, 'DISCARD ALL'); + expect(client.query).toHaveBeenNthCalledWith( + 2, + 'SET search_path TO pg_catalog; SET row_security TO on; SET jit_optimize_above_cost TO -1' + ); + expect(getPgCheckoutSanitizerStats(pool)).toMatchObject({ + checkoutAttempts: 1, + virginFastPathCheckouts: 0, + sanitizedReuseCheckouts: 1 + }); + }); + + it('routes a custom pool query through one sanitized checkout', async () => { + const client = mockClient(); + const queryResult = { rows: [{ value: 7 }] }; + (client.query as jest.Mock) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce(queryResult); + const bypassingQuery = jest.fn(async () => ({ rows: [{ value: -1 }] })); + const connect = jest.fn(async () => client); + const pool = { + waitingCount: 0, + query: bypassingQuery, + connect + } as unknown as pg.Pool; + + installCheckoutSanitizer(pool); + + await expect(pool.query('SELECT $1::int AS value', [7])).resolves.toBe(queryResult); + expect(bypassingQuery).not.toHaveBeenCalled(); + expect(connect).toHaveBeenCalledTimes(1); + expect(client.query).toHaveBeenNthCalledWith(1, 'DISCARD ALL'); + expect(client.query).toHaveBeenNthCalledWith( + 2, + 'SET search_path TO pg_catalog; SET row_security TO on; SET jit_optimize_above_cost TO -1' + ); + expect(client.query).toHaveBeenNthCalledWith(3, 'SELECT $1::int AS value', [7]); + expect(client.release).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith(); + }); + + it('preserves callback queries without executing the custom pool bypass', async () => { + const client = mockClient(); + const queryResult = { rows: [{ value: 9 }] }; + (client.query as jest.Mock) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce(queryResult); + const bypassingQuery = jest.fn(); + const pool = { + waitingCount: 0, + query: bypassingQuery, + connect: jest.fn(async () => client) + } as unknown as pg.Pool; + + installCheckoutSanitizer(pool); + + const callbackResult = await new Promise((resolve, reject) => { + const returned = pool.query( + 'SELECT $1::int AS value', + [9], + (error, result) => error ? reject(error) : resolve(result) + ); + expect(returned).toBeUndefined(); + }); + + expect(callbackResult).toBe(queryResult); + expect(bypassingQuery).not.toHaveBeenCalled(); + expect(client.query).toHaveBeenCalledTimes(3); + expect(client.query).toHaveBeenNthCalledWith(3, 'SELECT $1::int AS value', [9]); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it('destroys the checked-out custom client when a direct query fails', async () => { + const client = mockClient(); + const queryError = new Error('query rejected'); + (client.query as jest.Mock) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(queryError); + const bypassingQuery = jest.fn(); + const pool = { + waitingCount: 0, + query: bypassingQuery, + connect: jest.fn(async () => client) + } as unknown as pg.Pool; + + installCheckoutSanitizer(pool); + + await expect(pool.query('SELECT broken')).rejects.toBe(queryError); + expect(bypassingQuery).not.toHaveBeenCalled(); + expect(client.query).toHaveBeenCalledTimes(3); + expect(client.query).toHaveBeenNthCalledWith(3, 'SELECT broken'); + expect(client.release).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith(queryError); + }); + + it('does not release twice when direct-query sanitation fails', async () => { + const client = mockClient(); + const sanitationError = new Error('discard rejected'); + (client.query as jest.Mock).mockRejectedValueOnce(sanitationError); + const bypassingQuery = jest.fn(); + const pool = { + waitingCount: 0, + query: bypassingQuery, + connect: jest.fn(async () => client) + } as unknown as pg.Pool; + + installCheckoutSanitizer(pool); + + await expect(pool.query('SELECT unsafe')).rejects.toBe(sanitationError); + expect(bypassingQuery).not.toHaveBeenCalled(); + expect(client.query).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith(true); + }); + + it('rejects a custom sanitized pool whose query method cannot be replaced', () => { + const pool = { + waitingCount: 0, + connect: jest.fn() + } as unknown as pg.Pool; + Object.defineProperty(pool, 'query', { + value: jest.fn(), + writable: false + }); + + expect(() => installCheckoutSanitizer(pool)).toThrow( + 'A sanitized custom PostgreSQL pool must expose a replaceable query() method' + ); + }); +}); diff --git a/postgres/pg-cache/src/driver.ts b/postgres/pg-cache/src/driver.ts index 9a6c22ffb7..16b715ae40 100644 --- a/postgres/pg-cache/src/driver.ts +++ b/postgres/pg-cache/src/driver.ts @@ -14,10 +14,15 @@ import type { PgConfig, PgPoolConfig } from 'pg-env'; * `end()` (plus an `ended` flag for disposal), so a factory may return anything * implementing that subset — `QueryablePool`. A real `pg.Pool` structurally * satisfies it, so the default path is unchanged and fully backward-compatible. + * When checkout sanitation is requested, pg-cache replaces a custom pool's + * `query()` method so direct queries also use a sanitized `connect()`/`release()` + * cycle. Custom pools must therefore expose a replaceable `query()` property, + * and connected clients must implement the Promise-based contract below. */ export interface QueryableClient { query(text: string, values?: any[]): Promise; - release(...args: any[]): void; + /** A truthy error argument must permanently discard this client. */ + release(error?: Error | boolean): void; } export interface QueryablePool { @@ -27,10 +32,17 @@ export interface QueryablePool { } export type PgPoolFactory = ( - config: Partial & { pool?: PgPoolConfig } + config: Partial & { pool?: PgPoolConfig }, + options?: PgPoolFactoryOptions ) => pg.Pool | QueryablePool; +export interface PgPoolFactoryOptions { + purpose: string; + sanitizeOnCheckout: boolean; +} + let activeFactory: PgPoolFactory | undefined; +let driverGeneration = 0; /** * Register the factory `getPgPool` uses to build new pools. Pass `undefined` @@ -42,6 +54,7 @@ let activeFactory: PgPoolFactory | undefined; */ export const registerPgPoolFactory = (factory: PgPoolFactory | undefined): void => { activeFactory = factory; + driverGeneration++; }; /** The currently-registered factory, or `undefined` when using the default. */ @@ -49,3 +62,7 @@ export const getActivePgPoolFactory = (): PgPoolFactory | undefined => activeFac /** Whether a non-default pool factory is currently registered. */ export const hasPgPoolFactory = (): boolean => activeFactory !== undefined; + +/** Stable until the active factory registration changes. */ +export const getPgPoolDriverIdentity = (): string => + activeFactory ? `registered:${driverGeneration}` : 'node-postgres'; diff --git a/postgres/pg-cache/src/index.ts b/postgres/pg-cache/src/index.ts index 286748297a..21f071edf1 100644 --- a/postgres/pg-cache/src/index.ts +++ b/postgres/pg-cache/src/index.ts @@ -1,23 +1,91 @@ // Main exports from pg-cache package export { getActivePgPoolFactory, + getPgPoolDriverIdentity, hasPgPoolFactory, registerPgPoolFactory } from './driver'; -export { +export { close, + DEFAULT_PG_CACHE_MAX, getPgCacheConfig, - pgCache, - PgPoolCacheManager, + getPgCacheStats, + PG_CACHE_GRAPHILE_CONTRACT_CAPACITY, + PG_CACHE_OPERATIONAL_RESERVE, + PG_POOL_CAPACITY_ERROR_CODE, + pgCache, + PgPoolCacheManager, + PgPoolCapacityError, teardownPgPools } from './lru'; export { + acquirePgNotificationBroker, + assertValidPgNotificationTopic, + DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS, + getPgNotificationBrokerIdentity, + getPgNotificationBrokerStats, + getPgNotificationDatabaseIdentity, + PG_NOTIFICATION_BROKER_FAILED_ERROR_CODE, + PG_NOTIFICATION_BROKER_IDENTITY_VERSION, + PG_NOTIFICATION_DATABASE_IDENTITY_VERSION, + PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE, + PG_NOTIFICATION_OPERATION_TIMEOUT_ERROR_CODE, + PG_NOTIFICATION_QUEUE_CAPACITY, + PG_NOTIFICATION_QUEUE_OVERFLOW_ERROR_CODE, + PG_NOTIFICATION_TOPIC_ERROR_CODE, + PgNotificationBrokerFailedError, + PgNotificationLeaseReleasedError, + PgNotificationOperationTimeoutError, + PgNotificationQueueOverflowError, + PgNotificationTopicError, + teardownPgNotificationBrokers +} from './notification-broker'; +export { + assertPgNotificationRole, + assertPgNotificationRoleClient, + auditPgNotificationRole, + auditPgNotificationRoleClient, + normalizePgNotificationRoleContracts, + PG_NOTIFICATION_ROLE_AUDIT_SQL, + PG_NOTIFICATION_ROLE_AUDIT_VERSION, + PG_NOTIFICATION_ROLE_CONTRACT_ERROR_CODE, + PG_NOTIFICATION_ROLE_UNSAFE_ERROR_CODE, + PgNotificationRoleContractError, + UnsafePgNotificationRoleError +} from './notification-role'; +export { + acquirePgPool, buildConnectionString, + clearPreparedStatementBookkeeping, defaultPgPoolFactory, + getPgCheckoutSanitizerStats, getPgPool, - getPgPoolConfig + getPgPoolConfig, + getPgPoolIdentity, + installCheckoutSanitizer, + sanitizePgClient } from './pg'; // Re-export types -export type { PgPoolFactory, QueryableClient, QueryablePool } from './driver'; -export type { PgCacheConfig, PoolCleanupCallback } from './lru'; \ No newline at end of file +export type { PgPoolFactory, PgPoolFactoryOptions, QueryableClient, QueryablePool } from './driver'; +export type { + PgCacheConfig, + PgPoolCacheStats, + PgPoolDisposalReason, + PgPoolLease, + PoolCleanupCallback +} from './lru'; +export type { + AcquirePgNotificationBrokerOptions, + PgAttestedNotificationBrokerLease, + PgNotificationBrokerLease, + PgNotificationBrokerStats, + PgNotificationListenerConfig +} from './notification-broker'; +export type { + PgNotificationRoleAudit, + PgNotificationRoleClient, + PgNotificationRoleContract, + PgNotificationRoleViolationCode +} from './notification-role'; +export type { GetPgPoolOptions, PgCheckoutSanitizerStats } from './pg'; diff --git a/postgres/pg-cache/src/lru.ts b/postgres/pg-cache/src/lru.ts index 633dc6388e..de328f5a95 100644 --- a/postgres/pg-cache/src/lru.ts +++ b/postgres/pg-cache/src/lru.ts @@ -1,6 +1,5 @@ import { Logger } from '@pgpmjs/logger'; import { parseEnvNumber } from '12factor-env'; -import { LRUCache } from 'lru-cache'; import pg from 'pg'; const log = new Logger('pg-cache'); @@ -9,58 +8,141 @@ const ONE_HOUR_IN_MS = 1000 * 60 * 60; const ONE_DAY = ONE_HOUR_IN_MS * 24; const ONE_YEAR = ONE_DAY * 366; -// Kubernetes sends only SIGTERM on pod shutdown -const SYS_EVENTS = ['SIGTERM']; +// One runtime and one control identity per database-per-tenant Graphile +// contract, plus room for routing, diagnostics, listeners, and build overlap. +export const PG_CACHE_GRAPHILE_CONTRACT_CAPACITY = 1024; +export const PG_CACHE_OPERATIONAL_RESERVE = 16; +export const DEFAULT_PG_CACHE_MAX = + PG_CACHE_GRAPHILE_CONTRACT_CAPACITY * 2 + PG_CACHE_OPERATIONAL_RESERVE; type PgPoolKey = string; +type PoolFactory = () => pg.Pool; -// Cleanup callback type - called when a pg pool is disposed +export type PgPoolDisposalReason = + | 'capacity' + | 'ttl' + | 'delete' + | 'clear' + | 'close' + | 'replace'; + +// Called only when an identity is actually removed from the registry. export type PoolCleanupCallback = (pgPoolKey: string) => void; -// --- Cache Configuration --- +export interface PgPoolLease { + pool: pg.Pool; + identity: string; + /** Idempotently release this exact ownership claim. */ + release(): void; +} export interface PgCacheConfig { - /** Maximum number of pools in the LRU cache (env: PG_CACHE_MAX, default: 50) */ + /** Maximum number of lazy pool identities retained by this process. */ max: number; - /** TTL for cached pools in ms (default: ONE_YEAR) */ + /** Idle identity TTL in milliseconds. Leased identities never expire. */ ttl: number; } -/** - * Read cache configuration from environment variables. - * - * Supports: - * - PG_CACHE_MAX: Maximum number of pools (default: 50) - * - PG_CACHE_TTL_MS: TTL in milliseconds (default: ONE_YEAR) - */ +export interface PgPoolCacheStats { + size: number; + max: number; + ttl: number; + leasedPools: number; + idlePools: number; + activeLeases: number; + reservations: number; + pendingDisposals: number; + hits: number; + misses: number; + poolsCreated: number; + leasesAcquired: number; + leasesReleased: number; + capacityEvictions: number; + ttlExpirations: number; + capacityRefusals: number; + disposalsStarted: number; + disposalsCompleted: number; + disposalFailures: number; +} + +interface PgPoolCacheCounters { + hits: number; + misses: number; + poolsCreated: number; + leasesAcquired: number; + leasesReleased: number; + capacityEvictions: number; + ttlExpirations: number; + capacityRefusals: number; + disposalsStarted: number; + disposalsCompleted: number; + disposalFailures: number; +} + +interface SlotReservation { + key: PgPoolKey; + victims: ManagedPgPool[]; +} + +export const PG_POOL_CAPACITY_ERROR_CODE = 'PG_POOL_CAPACITY'; + +/** Fail-closed pool admission error suitable for a stable HTTP 503 mapping. */ +export class PgPoolCapacityError extends Error { + readonly code = PG_POOL_CAPACITY_ERROR_CODE; + readonly retryAfterSeconds = 15; + + constructor( + readonly max: number, + readonly size: number, + readonly leased: number + ) { + super( + `PostgreSQL pool capacity exhausted: ${size}/${max} identities are retained ` + + `and ${leased} are leased` + ); + this.name = 'PgPoolCapacityError'; + } +} + +/** Read cache configuration without allocating any pools or connections. */ export function getPgCacheConfig(): PgCacheConfig { return { - max: parseEnvNumber(process.env.PG_CACHE_MAX) ?? 50, + max: parseEnvNumber(process.env.PG_CACHE_MAX) ?? DEFAULT_PG_CACHE_MAX, ttl: parseEnvNumber(process.env.PG_CACHE_TTL_MS) ?? ONE_YEAR, }; } class ManagedPgPool { public isDisposed = false; + public leaseCount = 0; + public lastAccessOrder = 0; + public expiresAt = 0; private disposePromise: Promise | null = null; - constructor(public readonly pool: pg.Pool, public readonly key: string) {} + constructor( + public readonly pool: pg.Pool, + public readonly key: string + ) {} + + touch(order: number, now: number, ttl: number): void { + this.lastAccessOrder = order; + this.expiresAt = now + ttl; + } + + isExpired(now: number): boolean { + return now >= this.expiresAt; + } async dispose(): Promise { if (this.isDisposed) return this.disposePromise; this.isDisposed = true; this.disposePromise = (async () => { - try { - if (!this.pool.ended) { - await this.pool.end(); - log.success(`pg.Pool ${this.key} ended.`); - } else { - log.info(`pg.Pool ${this.key} already ended.`); - } - } catch (err) { - log.error(`Error ending pg.Pool ${this.key}: ${(err as Error).message}`); - throw err; + if (!this.pool.ended) { + await this.pool.end(); + log.success(`pg.Pool ${this.key} ended.`); + } else { + log.info(`pg.Pool ${this.key} already ended.`); } })(); @@ -68,37 +150,55 @@ class ManagedPgPool { } } +/** + * A lease-aware, lazy pool registry. + * + * JavaScript executes acquisition synchronously, including slot reservation and + * factory invocation. Two callers therefore cannot both claim the final slot. + * Pools may finish ending asynchronously after a zero-lease identity is removed. + */ export class PgPoolCacheManager { - private cleanupTasks: Promise[] = []; + private readonly records = new Map(); + private readonly cleanupTasks = new Set>(); + private readonly cleanupCallbacks = new Set(); + private readonly reservedKeys = new Set(); + private reservations = 0; + private accessOrder = 0; private closed = false; - private cleanupCallbacks: Set = new Set(); readonly config: PgCacheConfig; - private readonly pgCache: LRUCache; + private readonly counters: PgPoolCacheCounters = { + hits: 0, + misses: 0, + poolsCreated: 0, + leasesAcquired: 0, + leasesReleased: 0, + capacityEvictions: 0, + ttlExpirations: 0, + capacityRefusals: 0, + disposalsStarted: 0, + disposalsCompleted: 0, + disposalFailures: 0 + }; constructor(config?: Partial) { const defaults = getPgCacheConfig(); this.config = { ...defaults, ...config }; + if (!Number.isSafeInteger(this.config.max) || this.config.max <= 0) { + throw new Error('pg-cache max must be a positive safe integer'); + } + if (!Number.isFinite(this.config.ttl) || this.config.ttl <= 0) { + throw new Error('pg-cache ttl must be a positive number'); + } + } - this.pgCache = new LRUCache({ - max: this.config.max, - ttl: this.config.ttl, - updateAgeOnGet: true, - dispose: (managedPool, key, reason) => { - log.debug(`Disposing pg pool [${key}] (${reason})`); - this.notifyCleanup(key); - this.disposePool(managedPool); - } - }); + get size(): number { + return this.records.size; } - // Register a cleanup callback to be called when pools are disposed registerCleanupCallback(callback: PoolCleanupCallback): () => void { this.cleanupCallbacks.add(callback); - // Return unregister function - return () => { - this.cleanupCallbacks.delete(callback); - }; + return () => this.cleanupCallbacks.delete(callback); } get(key: PgPoolKey): pg.Pool | undefined { @@ -106,73 +206,315 @@ export class PgPoolCacheManager { log.warn(`Cache is closed, ignoring get(${key})`); return undefined; } - return this.pgCache.get(key)?.pool; + const managedPool = this.getLiveRecord(key, true); + if (!managedPool) { + this.counters.misses++; + return undefined; + } + this.counters.hits++; + return managedPool.pool; } has(key: PgPoolKey): boolean { - return this.pgCache.has(key); + if (this.closed) return false; + return Boolean(this.getLiveRecord(key, false)); } + /** + * Legacy direct insertion. Prefer getOrCreate/acquire so capacity is checked + * before the caller constructs a pool. + */ set(key: PgPoolKey, pool: pg.Pool): void { - if (this.closed) throw new Error(`Cannot add to cache after it has been closed (key: ${key})`); - this.pgCache.set(key, new ManagedPgPool(pool, key)); + this.assertOpen(key); + const existing = this.records.get(key); + if (existing?.pool === pool) { + this.touch(existing); + return; + } + if (existing?.leaseCount) { + throw new Error(`Cannot replace leased pg pool identity ${key}`); + } + if (existing) this.removeRecord(existing, 'replace'); + + const reservation = this.reserveSlot(key); + this.commitReservation(reservation, pool, 0); } - delete(key: PgPoolKey): void { - const managedPool = this.pgCache.get(key); - const existed = this.pgCache.delete(key); - if (!existed && managedPool) { - this.notifyCleanup(key); - this.disposePool(managedPool); + /** Atomically capacity-check, synchronously construct, and cache an idle pool. */ + getOrCreate(key: PgPoolKey, factory: PoolFactory): pg.Pool { + this.assertOpen(key); + const existing = this.getLiveRecord(key, true); + if (existing) { + this.counters.hits++; + return existing.pool; } + + this.counters.misses++; + return this.createWithReservation(key, factory, 0).pool; } + /** + * Atomically get/create and lease an exact identity. A leased identity cannot + * be selected by capacity or TTL eviction until every lease is released. + */ + acquire(key: PgPoolKey, factory: PoolFactory): PgPoolLease { + this.assertOpen(key); + let managedPool = this.getLiveRecord(key, true); + if (managedPool) { + this.counters.hits++; + managedPool.leaseCount++; + } else { + this.counters.misses++; + managedPool = this.createWithReservation(key, factory, 1); + } + this.counters.leasesAcquired++; + return this.makeLease(managedPool); + } + + /** Explicit deletion never interrupts a lease; callers may retry after release. */ + delete(key: PgPoolKey): void { + const managedPool = this.records.get(key); + if (!managedPool || managedPool.leaseCount > 0) return; + this.removeRecord(managedPool, 'delete'); + } + + /** Clear every currently unleased identity. */ clear(): void { - const entries = [...this.pgCache.entries()]; - this.pgCache.clear(); - for (const [key, managedPool] of entries) { - this.notifyCleanup(key); - this.disposePool(managedPool); + for (const managedPool of [...this.records.values()]) { + if (managedPool.leaseCount === 0) this.removeRecord(managedPool, 'clear'); } } async close(): Promise { if (this.closed) return; this.closed = true; - this.clear(); + // Explicit process teardown is the only operation that may override leases. + for (const managedPool of [...this.records.values()]) { + this.removeRecord(managedPool, 'close'); + } await this.waitForDisposals(); - // Re-open the cache so it can accept new entries if the process - // survives the shutdown signal (e.g. during provisioning or restart). + // Preserve the established restart/provisioning behavior. this.closed = false; } async waitForDisposals(): Promise { - if (this.cleanupTasks.length === 0) return; - const tasks = [...this.cleanupTasks]; - this.cleanupTasks = []; - await Promise.allSettled(tasks); + while (this.cleanupTasks.size > 0) { + await Promise.allSettled([...this.cleanupTasks]); + } + } + + getStats(): PgPoolCacheStats { + let leasedPools = 0; + let activeLeases = 0; + for (const managedPool of this.records.values()) { + if (managedPool.leaseCount > 0) leasedPools++; + activeLeases += managedPool.leaseCount; + } + return { + size: this.records.size, + max: this.config.max, + ttl: this.config.ttl, + leasedPools, + idlePools: this.records.size - leasedPools, + activeLeases, + reservations: this.reservations, + pendingDisposals: this.cleanupTasks.size, + ...this.counters + }; + } + + private assertOpen(key: PgPoolKey): void { + if (this.closed) { + throw new Error(`Cannot access pg cache while it is closed (key: ${key})`); + } + } + + private touch(managedPool: ManagedPgPool): void { + managedPool.touch(++this.accessOrder, Date.now(), this.config.ttl); + } + + private getLiveRecord(key: PgPoolKey, updateAge: boolean): ManagedPgPool | undefined { + const managedPool = this.records.get(key); + if (!managedPool) return undefined; + if (managedPool.leaseCount === 0 && managedPool.isExpired(Date.now())) { + this.removeRecord(managedPool, 'ttl'); + return undefined; + } + if (updateAge) this.touch(managedPool); + return managedPool; + } + + private idleRecordsByAge(): ManagedPgPool[] { + return [...this.records.values()] + .filter((managedPool) => managedPool.leaseCount === 0) + .sort((a, b) => a.lastAccessOrder - b.lastAccessOrder); + } + + private reserveSlot(key: PgPoolKey): SlotReservation { + if (this.reservedKeys.has(key)) { + throw new Error(`Re-entrant pg pool acquisition for identity ${key}`); + } + + const overflow = Math.max( + 0, + this.records.size + this.reservations + 1 - this.config.max + ); + const candidates = this.idleRecordsByAge(); + if (candidates.length < overflow) { + this.counters.capacityRefusals++; + throw new PgPoolCapacityError( + this.config.max, + this.records.size + this.reservations, + this.countLeasedPools() + ); + } + + const victims = candidates.slice(0, overflow); + for (const victim of victims) this.records.delete(victim.key); + this.reservations++; + this.reservedKeys.add(key); + return { key, victims }; + } + + private rollbackReservation(reservation: SlotReservation): void { + this.reservations = Math.max(0, this.reservations - 1); + this.reservedKeys.delete(reservation.key); + for (const victim of reservation.victims) { + this.records.set(victim.key, victim); + } + } + + private commitReservation( + reservation: SlotReservation, + pool: pg.Pool, + leaseCount: number + ): ManagedPgPool { + const managedPool = new ManagedPgPool(pool, reservation.key); + managedPool.leaseCount = leaseCount; + this.touch(managedPool); + this.records.set(reservation.key, managedPool); + this.reservations = Math.max(0, this.reservations - 1); + this.reservedKeys.delete(reservation.key); + this.counters.poolsCreated++; + + for (const victim of reservation.victims) { + this.counters.capacityEvictions++; + this.disposeRemovedRecord(victim); + } + return managedPool; + } + + private createWithReservation( + key: PgPoolKey, + factory: PoolFactory, + leaseCount: number + ): ManagedPgPool { + const reservation = this.reserveSlot(key); + let pool: pg.Pool; + try { + pool = factory(); + } catch (error) { + this.rollbackReservation(reservation); + throw error; + } + return this.commitReservation(reservation, pool, leaseCount); + } + + private makeLease(managedPool: ManagedPgPool): PgPoolLease { + let released = false; + return { + pool: managedPool.pool, + identity: managedPool.key, + release: () => { + if (released) return; + released = true; + this.counters.leasesReleased++; + managedPool.leaseCount = Math.max(0, managedPool.leaseCount - 1); + + // close() may already have detached this record. + if (this.records.get(managedPool.key) !== managedPool) return; + if (managedPool.leaseCount > 0) return; + if (managedPool.isExpired(Date.now())) { + this.removeRecord(managedPool, 'ttl'); + return; + } + this.enforceCapacity(); + } + }; + } + + private enforceCapacity(): void { + while (this.records.size > this.config.max) { + const victim = this.idleRecordsByAge()[0]; + if (!victim) return; + this.removeRecord(victim, 'capacity'); + } + } + + private countLeasedPools(): number { + let leased = 0; + for (const managedPool of this.records.values()) { + if (managedPool.leaseCount > 0) leased++; + } + return leased; + } + + private removeRecord( + managedPool: ManagedPgPool, + reason: PgPoolDisposalReason + ): void { + if (this.records.get(managedPool.key) !== managedPool) return; + this.records.delete(managedPool.key); + if (reason === 'capacity') this.counters.capacityEvictions++; + if (reason === 'ttl') this.counters.ttlExpirations++; + this.disposeRemovedRecord(managedPool); + } + + private disposeRemovedRecord(managedPool: ManagedPgPool): void { + this.notifyCleanup(managedPool.key); + + // Alternate drivers may intentionally return one physical pool for multiple + // exact identities. Never end it while another retained identity owns it. + if ([...this.records.values()].some((entry) => entry.pool === managedPool.pool)) { + return; + } + if (managedPool.isDisposed) return; + + this.counters.disposalsStarted++; + let task: Promise; + task = managedPool.dispose() + .then(() => { + this.counters.disposalsCompleted++; + }) + .catch((error) => { + this.counters.disposalFailures++; + log.error( + `Error ending pg.Pool ${managedPool.key}: ${(error as Error).message}` + ); + }) + .finally(() => this.cleanupTasks.delete(task)); + this.cleanupTasks.add(task); } private notifyCleanup(pgPoolKey: string): void { this.cleanupCallbacks.forEach(callback => { try { callback(pgPoolKey); - } catch (err) { - log.error(`Error in cleanup callback for pool ${pgPoolKey}: ${(err as Error).message}`); + } catch (error) { + log.error( + `Error in cleanup callback for pool ${pgPoolKey}: ${(error as Error).message}` + ); } }); } - - private disposePool(managedPool: ManagedPgPool): void { - if (managedPool.isDisposed) return; - const task = managedPool.dispose(); - this.cleanupTasks.push(task); - } } -// Create the singleton instance +// Process-wide registry. Its large capacity is only a key limit; pools and +// PostgreSQL connections remain lazily allocated on first use. export const pgCache = new PgPoolCacheManager(); +export const getPgCacheStats = (): PgPoolCacheStats => pgCache.getStats(); + // --- Graceful Shutdown --- const closePromise: { promise: Promise | null } = { promise: null }; @@ -185,7 +527,6 @@ export const close = async (verbose = false): Promise => { await pgCache.close(); if (verbose) log.success('PG cache disposed.'); } finally { - // Reset so close() can be called again if the process survives. closePromise.promise = null; } })(); @@ -193,13 +534,6 @@ export const close = async (verbose = false): Promise => { return closePromise.promise; }; -SYS_EVENTS.forEach(event => { - process.on(event, () => { - log.info(`Received ${event}`); - close(); - }); -}); - export const teardownPgPools = async (verbose = false): Promise => { return close(verbose); }; diff --git a/postgres/pg-cache/src/notification-broker.ts b/postgres/pg-cache/src/notification-broker.ts new file mode 100644 index 0000000000..cbb2e3afed --- /dev/null +++ b/postgres/pg-cache/src/notification-broker.ts @@ -0,0 +1,1087 @@ +import type { PgConfig, PgPoolConfig } from 'pg-env'; + +import { + assertPgNotificationRoleClient, + type PgNotificationRoleAudit, + type PgNotificationRoleClient, + type PgNotificationRoleContract +} from './notification-role'; +import { + acquirePgPool, + getPgDatabaseTargetIdentity, + getPgPoolConfig, + getPgPoolIdentity +} from './pg'; + +export const PG_NOTIFICATION_BROKER_IDENTITY_VERSION = 'pg-notification-broker:v1'; +export const PG_NOTIFICATION_DATABASE_IDENTITY_VERSION = 'pg-notification-database:v1'; +export const PG_NOTIFICATION_QUEUE_CAPACITY = 256; +export const DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS = 5_000; + +export const PG_NOTIFICATION_TOPIC_ERROR_CODE = 'PG_NOTIFICATION_TOPIC_INVALID'; +export const PG_NOTIFICATION_BROKER_FAILED_ERROR_CODE = 'PG_NOTIFICATION_BROKER_FAILED'; +export const PG_NOTIFICATION_QUEUE_OVERFLOW_ERROR_CODE = 'PG_NOTIFICATION_QUEUE_OVERFLOW'; +export const PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE = 'PG_NOTIFICATION_LEASE_RELEASED'; +export const PG_NOTIFICATION_OPERATION_TIMEOUT_ERROR_CODE = + 'PG_NOTIFICATION_OPERATION_TIMEOUT'; + +type PromiseOrDirect = T | Promise; + +export interface PgNotification { + channel: string; + payload?: string; +} + +export interface PgNotificationClient { + query(text: string, values?: readonly unknown[]): Promise; + on(event: string, listener: (...args: any[]) => void): unknown; + off(event: string, listener: (...args: any[]) => void): unknown; + release(error?: Error | boolean): PromiseOrDirect; +} + +export interface PgNotificationConnectionSource { + connect(): Promise; + release(): PromiseOrDirect; +} + +export interface PgNotificationBrokerLease { + /** Versioned digest of the complete listener connection contract. */ + readonly identity: string; + /** Frozen, exact PostgreSQL channels this lease may subscribe to. */ + readonly topics: readonly string[]; + /** Resolves on fatal broker failure or with null after graceful release. */ + readonly terminated: Promise; + subscribe(topic: string): AsyncIterableIterator; + /** Idempotent and awaited through UNLISTEN and connection release. */ + release(): Promise; +} + +/** + * A production lease whose login was audited on the same pinned PostgreSQL + * client before admission. Arbitrary SQL and the client itself stay private. + */ +export interface PgAttestedNotificationBrokerLease +extends PgNotificationBrokerLease { + readonly roleAudit: PgNotificationRoleAudit; + revalidateRole(): Promise; +} + +export interface AcquirePgNotificationBrokerOptions { + /** Every channel this generation may observe. Prefix matching is never used. */ + topics: readonly string[]; +} + +export type PgNotificationListenerConfig = PgConfig & { pool?: PgPoolConfig }; + +export interface PgNotificationBrokerStats { + brokers: number; + listenerConnections: number; + leases: number; + topics: number; + subscribers: number; + acquisitions: number; + releases: number; + notifications: number; + ignoredNotifications: number; + queueOverflows: number; + fatalFailures: number; + roleAuditAttempts: number; + roleAuditFailures: number; +} + +interface MutableBrokerCounters { + acquisitions: number; + releases: number; + notifications: number; + ignoredNotifications: number; + queueOverflows: number; + fatalFailures: number; + roleAuditAttempts: number; + roleAuditFailures: number; +} + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(error: unknown): void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +export class PgNotificationTopicError extends Error { + readonly code = PG_NOTIFICATION_TOPIC_ERROR_CODE; + + constructor(readonly topic: unknown, reason: string) { + super(`Invalid PostgreSQL notification topic: ${reason}`); + this.name = 'PgNotificationTopicError'; + } +} + +export class PgNotificationBrokerFailedError extends Error { + readonly code = PG_NOTIFICATION_BROKER_FAILED_ERROR_CODE; + + constructor(reason: unknown) { + const cause = reason instanceof Error ? reason : new Error(String(reason)); + super('PostgreSQL notification broker failed; all subscribers were terminated', { + cause + }); + this.name = 'PgNotificationBrokerFailedError'; + } +} + +export class PgNotificationQueueOverflowError extends Error { + readonly code = PG_NOTIFICATION_QUEUE_OVERFLOW_ERROR_CODE; + + constructor( + readonly topic: string, + readonly capacity: number + ) { + super( + `PostgreSQL notification subscriber queue for ${JSON.stringify(topic)} ` + + `exceeded its fixed capacity of ${capacity}` + ); + this.name = 'PgNotificationQueueOverflowError'; + } +} + +export class PgNotificationLeaseReleasedError extends Error { + readonly code = PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE; + + constructor() { + super('PostgreSQL notification broker lease has been released'); + this.name = 'PgNotificationLeaseReleasedError'; + } +} + +export class PgNotificationOperationTimeoutError extends Error { + readonly code = PG_NOTIFICATION_OPERATION_TIMEOUT_ERROR_CODE; + + constructor( + readonly operation: 'role-audit' | 'listen' | 'unlisten', + readonly timeoutMs: number + ) { + super( + `PostgreSQL notification ${operation} exceeded its fixed ${timeoutMs}ms deadline` + ); + this.name = 'PgNotificationOperationTimeoutError'; + } +} + +class BrokerClosedError extends Error {} + +const containsUnpairedSurrogate = (value: string): boolean => { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return true; + index++; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +}; + +/** + * PostgreSQL identifiers are limited to 63 UTF-8 bytes. PostgreSQL truncates + * longer identifiers, so accepting them here could collapse distinct tenant + * topics onto one physical LISTEN channel. + */ +export function assertValidPgNotificationTopic(topic: unknown): asserts topic is string { + if (typeof topic !== 'string') { + throw new PgNotificationTopicError(topic, 'the topic must be a string'); + } + if (topic.length === 0) { + throw new PgNotificationTopicError(topic, 'the topic must not be empty'); + } + if (topic.includes('\0')) { + throw new PgNotificationTopicError(topic, 'NUL bytes are not allowed'); + } + if (containsUnpairedSurrogate(topic)) { + throw new PgNotificationTopicError(topic, 'unpaired UTF-16 surrogates are not allowed'); + } + const bytes = Buffer.byteLength(topic, 'utf8'); + if (bytes > 63) { + throw new PgNotificationTopicError( + topic, + `the UTF-8 encoding is ${bytes} bytes; PostgreSQL allows at most 63` + ); + } +} + +const normalizeTopics = (topics: readonly string[]): readonly string[] => { + if (!Array.isArray(topics) || topics.length === 0) { + throw new PgNotificationTopicError(topics, 'at least one exact topic is required'); + } + for (const topic of topics) assertValidPgNotificationTopic(topic); + return Object.freeze([...new Set(topics)]); +}; + +const quoteIdentifier = (identifier: string): string => + `"${identifier.replace(/"/g, '""')}"`; + +class BoundedNotificationQueue implements AsyncIterableIterator { + private readonly buffered: string[] = []; + private readonly waiting: Deferred>[] = []; + private terminal: 'open' | 'complete' | 'failed' = 'open'; + private failure: Error | null = null; + + constructor( + private readonly topic: string, + private readonly capacity: number, + private readonly onClose: () => void, + private readonly onOverflow: () => void + ) {} + + [Symbol.asyncIterator](): AsyncIterableIterator { + return this; + } + + next(): Promise> { + const buffered = this.buffered.shift(); + if (buffered !== undefined) { + return Promise.resolve({ done: false, value: buffered }); + } + if (this.terminal === 'failed') return Promise.reject(this.failure); + if (this.terminal === 'complete') { + return Promise.resolve({ done: true, value: undefined }); + } + + const result = deferred>(); + this.waiting.push(result); + return result.promise; + } + + return(value?: unknown): Promise> { + this.complete(); + return Promise.resolve({ done: true, value: value as string }); + } + + throw(error?: unknown): Promise> { + const failure = error instanceof Error ? error : new Error(String(error)); + this.fail(failure); + return Promise.reject(failure); + } + + push(payload: string): void { + if (this.terminal !== 'open') return; + const waiter = this.waiting.shift(); + if (waiter) { + waiter.resolve({ done: false, value: payload }); + return; + } + if (this.buffered.length >= this.capacity) { + this.onOverflow(); + this.fail(new PgNotificationQueueOverflowError(this.topic, this.capacity)); + return; + } + this.buffered.push(payload); + } + + complete(): void { + if (this.terminal !== 'open') return; + this.terminal = 'complete'; + this.buffered.length = 0; + for (const waiter of this.waiting.splice(0)) { + waiter.resolve({ done: true, value: undefined }); + } + this.onClose(); + } + + fail(error: Error): void { + if (this.terminal !== 'open') return; + this.terminal = 'failed'; + this.failure = error; + this.buffered.length = 0; + for (const waiter of this.waiting.splice(0)) waiter.reject(error); + this.onClose(); + } +} + +type BrokerState = 'new' | 'active' | 'failed' | 'closing' | 'closed'; +type ConnectionSourceFactory = () => PromiseOrDirect; +type NotificationOperation = PgNotificationOperationTimeoutError['operation']; + +const MAX_TIMER_DELAY_MS = 2_147_483_647; + +const assertNotificationOperationTimeoutMs = (timeoutMs: number): number => { + if ( + !Number.isSafeInteger(timeoutMs) + || timeoutMs <= 0 + || timeoutMs > MAX_TIMER_DELAY_MS + ) { + throw new TypeError( + 'PostgreSQL notification operation timeout must be an integer ' + + `between 1 and ${MAX_TIMER_DELAY_MS}` + ); + } + return timeoutMs; +}; + +const getNotificationOperationTimeoutMs = ( + listenerPgConfig: PgNotificationListenerConfig +): number => { + // Preserve this API's narrower deadline contract and stable error before the + // generic pool validator runs as part of identity construction. + const configured = listenerPgConfig.pool?.connectionTimeoutMillis; + return assertNotificationOperationTimeoutMs( + configured ?? getPgPoolConfig(listenerPgConfig.pool).connectionTimeoutMillis + ?? DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS + ); +}; + +class NotificationBrokerLease implements PgAttestedNotificationBrokerLease { + readonly topics: readonly string[]; + readonly terminated: Promise; + private readonly allowedTopics: ReadonlySet; + private readonly termination = deferred(); + private readonly queues = new Map>(); + private audit: PgNotificationRoleAudit | null = null; + private released = false; + private releasePromise: Promise | null = null; + + constructor( + readonly identity: string, + topics: readonly string[], + private readonly broker: NotificationBrokerRecord, + private readonly queueCapacity: number, + private readonly counters: MutableBrokerCounters, + readonly roleContract: Readonly | null + ) { + this.topics = topics; + this.terminated = this.termination.promise; + this.allowedTopics = new Set(topics); + } + + get subscriberCount(): number { + let count = 0; + for (const topicQueues of this.queues.values()) count += topicQueues.size; + return count; + } + + get isReleased(): boolean { + return this.released; + } + + get roleAudit(): PgNotificationRoleAudit { + if (!this.audit) { + throw new Error('PostgreSQL notification broker lease is not role-attested'); + } + return this.audit; + } + + setRoleAudit(audit: PgNotificationRoleAudit): void { + this.audit = audit; + } + + revalidateRole(): Promise { + if (this.released) return Promise.reject(new PgNotificationLeaseReleasedError()); + if (!this.roleContract) { + return Promise.reject( + new Error('PostgreSQL notification broker lease is not role-attested') + ); + } + return this.broker.revalidateLeaseRole(this); + } + + subscribe(topic: string): AsyncIterableIterator { + if (this.released) throw new PgNotificationLeaseReleasedError(); + this.broker.assertAvailable(); + if (!this.allowedTopics.has(topic)) { + throw new PgNotificationTopicError( + topic, + 'the topic is not in this lease\'s exact allowlist' + ); + } + + let topicQueues = this.queues.get(topic); + if (!topicQueues) { + topicQueues = new Set(); + this.queues.set(topic, topicQueues); + } + let queue!: BoundedNotificationQueue; + queue = new BoundedNotificationQueue( + topic, + this.queueCapacity, + () => { + topicQueues!.delete(queue); + if (topicQueues!.size === 0) this.queues.delete(topic); + }, + () => { + this.counters.queueOverflows++; + } + ); + topicQueues.add(queue); + return queue; + } + + dispatch(topic: string, payload: string): void { + const queues = this.queues.get(topic); + if (!queues) return; + for (const queue of [...queues]) queue.push(payload); + } + + fail(error: PgNotificationBrokerFailedError): void { + this.termination.resolve(error); + for (const queues of [...this.queues.values()]) { + for (const queue of [...queues]) queue.fail(error); + } + } + + release(): Promise { + if (this.releasePromise) return this.releasePromise; + this.released = true; + for (const queues of [...this.queues.values()]) { + for (const queue of [...queues]) queue.complete(); + } + this.releasePromise = this.broker.releaseLease(this); + void this.releasePromise.then( + () => this.termination.resolve(null), + (error) => this.termination.resolve( + error instanceof PgNotificationBrokerFailedError + ? error + : new PgNotificationBrokerFailedError(error) + ) + ); + return this.releasePromise; + } +} + +class NotificationBrokerRecord { + private state: BrokerState = 'new'; + private acceptingLeases = true; + private operation: Promise = Promise.resolve(); + private source: PgNotificationConnectionSource | null = null; + private client: PgNotificationClient | null = null; + private clientCleanup: Promise | null = null; + private sourceCleanup: Promise | null = null; + private fatalError: PgNotificationBrokerFailedError | null = null; + private readonly leases = new Set(); + private readonly topicReferences = new Map(); + /** Includes provisional LISTENs whose lease admission has not committed yet. */ + private readonly listenedTopics = new Set(); + + private readonly onNotification = (notification: PgNotification): void => { + if (this.state !== 'active') return; + if ( + !notification + || typeof notification.channel !== 'string' + || ( + notification.payload !== undefined + && typeof notification.payload !== 'string' + ) + ) { + this.markFailed(new Error('PostgreSQL listener emitted a malformed notification')); + return; + } + if (!this.topicReferences.has(notification.channel)) { + this.counters.ignoredNotifications++; + return; + } + this.counters.notifications++; + const payload = notification.payload ?? ''; + for (const lease of [...this.leases]) { + lease.dispatch(notification.channel, payload); + } + }; + + private readonly onClientError = (error: unknown): void => { + this.markFailed(error); + }; + + private readonly onClientEnd = (): void => { + this.markFailed(new Error('PostgreSQL notification listener connection ended')); + }; + + constructor( + readonly identity: string, + private readonly sourcePromise: Promise, + private readonly queueCapacity: number, + private readonly operationTimeoutMs: number, + private readonly counters: MutableBrokerCounters, + private readonly onTerminal: (record: NotificationBrokerRecord) => void + ) {} + + get snapshot(): Pick< + PgNotificationBrokerStats, + 'listenerConnections' | 'leases' | 'topics' | 'subscribers' + > { + let subscribers = 0; + for (const lease of this.leases) subscribers += lease.subscriberCount; + return { + listenerConnections: this.client ? 1 : 0, + leases: this.leases.size, + topics: this.topicReferences.size, + subscribers + }; + } + + assertAvailable(): void { + if (this.state === 'failed') throw this.fatalError!; + if (this.state !== 'active') throw new PgNotificationLeaseReleasedError(); + } + + async acquire( + topics: readonly string[], + roleContract: Readonly | null = null + ): Promise { + const lease = new NotificationBrokerLease( + this.identity, + topics, + this, + this.queueCapacity, + this.counters, + roleContract + ); + await this.enqueue(async () => { + if (!this.acceptingLeases) throw new BrokerClosedError(); + if (this.state === 'failed') throw this.fatalError!; + if (this.state === 'closing' || this.state === 'closed') { + throw new BrokerClosedError(); + } + const client = await this.ensureClient(); + if (!this.acceptingLeases) throw new BrokerClosedError(); + if (roleContract) { + lease.setRoleAudit(await this.auditRole(client, roleContract)); + } + if (!this.acceptingLeases) throw new BrokerClosedError(); + for (const topic of topics) { + if ((this.topicReferences.get(topic) ?? 0) === 0) { + await this.executeListenerQuery(client, `LISTEN ${quoteIdentifier(topic)}`); + this.listenedTopics.add(topic); + } + } + if (!this.acceptingLeases || this.state !== 'active') { + if (this.fatalError) throw this.fatalError; + throw new BrokerClosedError(); + } + for (const topic of topics) { + this.topicReferences.set(topic, (this.topicReferences.get(topic) ?? 0) + 1); + } + this.leases.add(lease); + this.counters.acquisitions++; + }); + return lease; + } + + async revalidateLeaseRole( + lease: NotificationBrokerLease + ): Promise { + return this.enqueue(async () => { + if (lease.isReleased || !this.leases.has(lease)) { + throw new PgNotificationLeaseReleasedError(); + } + if (this.state === 'failed') throw this.fatalError!; + if (this.state !== 'active' || !this.client || !lease.roleContract) { + throw new PgNotificationLeaseReleasedError(); + } + const audit = await this.auditRole(this.client, lease.roleContract); + lease.setRoleAudit(audit); + return audit; + }); + } + + async releaseLease(lease: NotificationBrokerLease): Promise { + return this.enqueue(async () => { + if (!this.leases.delete(lease)) return; + this.counters.releases++; + + const topicsToUnlisten: string[] = []; + for (const topic of lease.topics) { + const next = (this.topicReferences.get(topic) ?? 0) - 1; + if (next <= 0) { + this.topicReferences.delete(topic); + topicsToUnlisten.push(topic); + } else { + this.topicReferences.set(topic, next); + } + } + + let releaseError: Error | null = null; + if (this.state === 'active' && this.client) { + for (const topic of topicsToUnlisten) { + try { + await this.executeListenerQuery( + this.client, + `UNLISTEN ${quoteIdentifier(topic)}` + ); + this.listenedTopics.delete(topic); + } catch (error) { + releaseError = this.fatalError + ?? new PgNotificationBrokerFailedError(error); + break; + } + } + } + + if (this.leases.size === 0) await this.closeUnused(); + if (releaseError) throw releaseError; + }); + } + + async closeAll(): Promise { + this.acceptingLeases = false; + // Cross the serialized-operation barrier before snapshotting leases. This + // either rejects an acquisition already waiting on connect/LISTEN or makes + // its completed lease visible to the release snapshot below. + await this.enqueue((): void => undefined); + const releases = [...this.leases].map((lease) => lease.release()); + const releaseResults = await Promise.allSettled(releases); + let finalCleanupError: unknown; + let finalCleanupFailed = false; + try { + await this.enqueue(() => this.closeUnused()); + } catch (error) { + finalCleanupError = error; + finalCleanupFailed = true; + } + const failedRelease = releaseResults.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ); + if (failedRelease) throw failedRelease.reason; + if (finalCleanupFailed) throw finalCleanupError; + } + + async closeIfUnused(): Promise { + await this.enqueue(() => this.closeUnused()); + } + + private enqueue(operation: () => PromiseOrDirect): Promise { + const pending = this.operation.then(operation, operation); + this.operation = pending.then( + (): void => undefined, + (): void => undefined + ); + return pending; + } + + private async ensureClient(): Promise { + if (this.client) return this.client; + try { + this.source = await this.sourcePromise; + if (this.fatalError) throw this.fatalError; + const client = await this.source.connect(); + this.client = client; + client.on('notification', this.onNotification); + client.on('error', this.onClientError); + client.on('end', this.onClientEnd); + this.state = 'active'; + return client; + } catch (error) { + this.markFailed(error); + await this.clientCleanup; + throw this.fatalError!; + } + } + + private async executeListenerQuery( + client: PgNotificationClient, + text: string + ): Promise { + try { + await this.runWithOperationDeadline( + text.startsWith('UNLISTEN') ? 'unlisten' : 'listen', + () => client.query(text) + ); + if (this.state === 'failed') throw this.fatalError!; + } catch (error) { + this.markFailed(error); + await this.clientCleanup; + throw this.fatalError!; + } + } + + private async auditRole( + client: PgNotificationClient, + contract: Readonly + ): Promise { + this.counters.roleAuditAttempts++; + try { + const audit = await this.runWithOperationDeadline( + 'role-audit', + () => assertPgNotificationRoleClient( + client as unknown as PgNotificationRoleClient, + contract + ) + ); + if (this.state === 'failed') throw this.fatalError!; + return audit; + } catch (error) { + this.counters.roleAuditFailures++; + this.markFailed(error); + await this.clientCleanup; + // Preserve the stable unsafe-role error for startup and attestation + // diagnostics. Active leases separately observe the broker-failed latch. + throw error; + } + } + + private async runWithOperationDeadline( + operation: NotificationOperation, + task: () => PromiseOrDirect + ): Promise { + let timer: ReturnType | null = null; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + const error = new PgNotificationOperationTimeoutError( + operation, + this.operationTimeoutMs + ); + // Latch failure and start client destruction at the exact deadline. The + // driver promise remains observed below, so a later rejection is safe. + this.markFailed(error); + reject(error); + }, this.operationTimeoutMs); + timer.unref?.(); + }); + // Promise.race installs a rejection handler on the driver query. If the + // deadline wins, destroying the client may settle that abandoned query + // later without producing an unhandled rejection. + const operationPromise = Promise.resolve().then(task); + try { + return await Promise.race([operationPromise, timeout]); + } finally { + if (timer) clearTimeout(timer); + } + } + + private markFailed(reason: unknown): void { + if ( + this.state === 'failed' + || this.state === 'closing' + || this.state === 'closed' + ) return; + this.state = 'failed'; + this.fatalError = reason instanceof PgNotificationBrokerFailedError + ? reason + : new PgNotificationBrokerFailedError(reason); + this.counters.fatalFailures++; + for (const lease of [...this.leases]) lease.fail(this.fatalError); + + const client = this.client; + this.client = null; + if (client) this.clientCleanup = this.releaseClient(client, this.fatalError); + } + + private async releaseClient( + client: PgNotificationClient, + error?: Error, + destroy = false + ): Promise { + client.off('notification', this.onNotification); + client.off('end', this.onClientEnd); + try { + await client.release(error ?? (destroy ? true : undefined)); + } catch (releaseError) { + // The broker is already failed or closing. Source cleanup below remains + // mandatory, and the original delivery failure is the useful error. + if (!error) throw releaseError; + } finally { + client.off('error', this.onClientError); + } + } + + private async closeUnused(): Promise { + if (this.leases.size > 0 || this.state === 'closed') return; + + let cleanupError: unknown; + let cleanupFailed = false; + if (this.client && this.listenedTopics.size > 0) { + try { + // This also covers a shutdown racing between a successful LISTEN and + // lease admission, where no committed topic reference exists yet. + await this.executeListenerQuery( + this.client, + 'UNLISTEN *' + ); + this.listenedTopics.clear(); + } catch (error) { + cleanupError = error; + cleanupFailed = true; + } + } + if (this.state !== 'failed') this.state = 'closing'; + + const client = this.client; + this.client = null; + if (client) { + const releaseError = cleanupFailed + ? new PgNotificationBrokerFailedError(cleanupError) + : undefined; + // Once the last exact-generation lease is gone, retaining an idle + // listener backend only delays PostgreSQL memory reclamation. Destroy it + // after UNLISTEN; the identity-only pool can create a fresh client later. + this.clientCleanup = this.releaseClient(client, releaseError, true); + } + try { + if (this.clientCleanup) await this.clientCleanup; + } catch (error) { + cleanupError = error; + cleanupFailed = true; + } + + if (this.source && !this.sourceCleanup) { + const source = this.source; + this.source = null; + this.sourceCleanup = Promise.resolve(source.release()); + } + try { + if (this.sourceCleanup) await this.sourceCleanup; + } catch (error) { + if (!cleanupFailed) cleanupError = error; + cleanupFailed = true; + } + + this.state = 'closed'; + this.onTerminal(this); + if (cleanupFailed) throw cleanupError; + } +} + +/** + * Registry implementation exposed for deterministic unit tests. Production + * callers must use acquirePgNotificationBroker so identity and pool ownership + * always come from the canonical PgConfig path. + * + * @internal + */ +export class PgNotificationBrokerRegistry { + private readonly records = new Map(); + private closed = false; + private closePromise: Promise | null = null; + private readonly counters: MutableBrokerCounters = { + acquisitions: 0, + releases: 0, + notifications: 0, + ignoredNotifications: 0, + queueOverflows: 0, + fatalFailures: 0, + roleAuditAttempts: 0, + roleAuditFailures: 0 + }; + + constructor( + private readonly queueCapacity = PG_NOTIFICATION_QUEUE_CAPACITY, + private readonly defaultOperationTimeoutMs = + DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS + ) { + if (!Number.isSafeInteger(queueCapacity) || queueCapacity <= 0) { + throw new Error('PostgreSQL notification queue capacity must be a positive safe integer'); + } + assertNotificationOperationTimeoutMs(defaultOperationTimeoutMs); + } + + async acquireForTests( + identity: string, + sourceFactory: ConnectionSourceFactory, + topics: readonly string[], + operationTimeoutMs = this.defaultOperationTimeoutMs + ): Promise { + return this.acquireInternal( + identity, + sourceFactory, + topics, + null, + operationTimeoutMs + ); + } + + /** @internal Exercise production attestation without constructing PgConfig. */ + async acquireAttestedForTests( + identity: string, + sourceFactory: ConnectionSourceFactory, + topics: readonly string[], + roleContract: PgNotificationRoleContract, + operationTimeoutMs = this.defaultOperationTimeoutMs + ): Promise { + return this.acquireInternal( + identity, + sourceFactory, + topics, + roleContract, + operationTimeoutMs + ); + } + + private async acquireInternal( + identity: string, + sourceFactory: ConnectionSourceFactory, + topics: readonly string[], + roleContract: PgNotificationRoleContract | null, + operationTimeoutMs: number + ): Promise { + if (this.closed) throw new Error('PostgreSQL notification broker registry is closed'); + if (typeof identity !== 'string' || identity.length === 0) { + throw new Error('PostgreSQL notification broker identity must be a non-empty string'); + } + const normalizedTopics = normalizeTopics(topics); + const normalizedOperationTimeoutMs = assertNotificationOperationTimeoutMs( + operationTimeoutMs + ); + + for (;;) { + let record = this.records.get(identity); + if (!record) { + const sourcePromise = Promise.resolve(sourceFactory()); + // Acquisition consumes this immediately, but guard the small interval + // before its serialized operation attaches a rejection handler. + void sourcePromise.catch(() => {}); + record = new NotificationBrokerRecord( + identity, + sourcePromise, + this.queueCapacity, + normalizedOperationTimeoutMs, + this.counters, + (terminal) => { + if (this.records.get(identity) === terminal) this.records.delete(identity); + } + ); + this.records.set(identity, record); + } + try { + const lease = await record.acquire(normalizedTopics, roleContract); + if (this.closed) { + await lease.release(); + throw new Error('PostgreSQL notification broker registry is closed'); + } + return lease; + } catch (error) { + if (error instanceof BrokerClosedError && !this.closed) continue; + // A failed broker remains pinned until every existing owner explicitly + // releases it. This prevents an acquisition attempt from silently + // replacing a listener after a possible notification gap. + await record.closeIfUnused(); + if (error instanceof BrokerClosedError && this.closed) { + throw new Error('PostgreSQL notification broker registry is closed'); + } + throw error; + } + } + } + + stats(): PgNotificationBrokerStats { + let listenerConnections = 0; + let leases = 0; + let topics = 0; + let subscribers = 0; + for (const record of this.records.values()) { + const snapshot = record.snapshot; + listenerConnections += snapshot.listenerConnections; + leases += snapshot.leases; + topics += snapshot.topics; + subscribers += snapshot.subscribers; + } + return { + brokers: this.records.size, + listenerConnections, + leases, + topics, + subscribers, + ...this.counters + }; + } + + close(): Promise { + if (this.closePromise) return this.closePromise; + this.closed = true; + this.closePromise = (async () => { + const closeResults = await Promise.allSettled( + [...this.records.values()].map((record) => record.closeAll()) + ); + this.records.clear(); + const failedClose = closeResults.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ); + if (failedClose) throw failedClose.reason; + })(); + return this.closePromise; + } +} + +let brokerRegistry = new PgNotificationBrokerRegistry(); +let brokerTeardownTail: Promise = Promise.resolve(); + +/** Opaque identity over the complete canonical listener pool contract. */ +export const getPgNotificationBrokerIdentity = ( + listenerPgConfig: PgNotificationListenerConfig +): string => { + // The operation deadline is represented by the pool connection timeout in + // the identity below. Validate it before publishing an apparently usable key. + getNotificationOperationTimeoutMs(listenerPgConfig); + const poolIdentity = getPgPoolIdentity(listenerPgConfig, { + purpose: 'notification-broker', + sanitizeOnCheckout: true + }); + return `${PG_NOTIFICATION_BROKER_IDENTITY_VERSION}:${poolIdentity}`; +}; + +/** + * Opaque identity for one physical database target, deliberately excluding + * credentials, TLS policy, pool sizing, and checkout behavior. Those inputs + * split listener pools, but must not let two active listener contracts silently + * fragment one database's broker. + */ +export const getPgNotificationDatabaseIdentity = ( + listenerPgConfig: PgNotificationListenerConfig +): string => { + const targetIdentity = getPgDatabaseTargetIdentity(listenerPgConfig); + return `${PG_NOTIFICATION_DATABASE_IDENTITY_VERSION}:${targetIdentity}`; +}; + +/** + * Acquire a generation lease over one process-local listener. The supplied + * config must name the dedicated least-privilege notification login; this API + * never falls back to a request runtime or control-plane credential. + */ +export const acquirePgNotificationBroker = async ( + listenerPgConfig: PgNotificationListenerConfig, + options: AcquirePgNotificationBrokerOptions +): Promise => { + const operationTimeoutMs = getNotificationOperationTimeoutMs(listenerPgConfig); + const identity = getPgNotificationBrokerIdentity(listenerPgConfig); + return brokerRegistry.acquireAttestedForTests( + identity, + () => { + const poolLease = acquirePgPool(listenerPgConfig, { + purpose: 'notification-broker', + sanitizeOnCheckout: true + }); + return { + connect: () => poolLease.pool.connect() as Promise, + release: () => poolLease.release() + }; + }, + options.topics, + { + role: listenerPgConfig.user, + database: listenerPgConfig.database + }, + operationTimeoutMs + ); +}; + +export const getPgNotificationBrokerStats = (): PgNotificationBrokerStats => + brokerRegistry.stats(); + +/** Await every UNLISTEN and checked-out connection release, then reset. */ +export const teardownPgNotificationBrokers = (): Promise => { + const closing = brokerRegistry; + brokerRegistry = new PgNotificationBrokerRegistry(); + const teardown = brokerTeardownTail.then(() => closing.close()); + // A later teardown must wait until this registry has fully drained even when + // this caller observes a cleanup failure. + brokerTeardownTail = teardown.then( + (): void => undefined, + (): void => undefined + ); + return teardown; +}; diff --git a/postgres/pg-cache/src/notification-role.ts b/postgres/pg-cache/src/notification-role.ts new file mode 100644 index 0000000000..cd6398c917 --- /dev/null +++ b/postgres/pg-cache/src/notification-role.ts @@ -0,0 +1,419 @@ +import type { Pool, PoolClient, QueryResult } from 'pg'; + +export const PG_NOTIFICATION_ROLE_AUDIT_VERSION = 'pg-notification-role:v1'; +export const PG_NOTIFICATION_ROLE_UNSAFE_ERROR_CODE = 'PG_NOTIFICATION_ROLE_UNSAFE'; +export const PG_NOTIFICATION_ROLE_CONTRACT_ERROR_CODE = + 'PG_NOTIFICATION_ROLE_CONTRACT_INVALID'; + +export type PgNotificationRoleViolationCode = + | 'LOGIN_ROLE_MISMATCH' + | 'CURRENT_ROLE_MISMATCH' + | 'DATABASE_MISMATCH' + | 'LOGIN_REQUIRED' + | 'NOINHERIT_REQUIRED' + | 'SUPERUSER' + | 'BYPASSRLS' + | 'CREATEROLE' + | 'CREATEDB' + | 'REPLICATION' + | 'ROLE_MEMBERSHIP' + | 'TARGET_DATABASE_MISSING' + | 'TARGET_CONNECT_REQUIRED' + | 'CROSS_DATABASE_CONNECT' + | 'DATABASE_OWNER' + | 'DATABASE_CREATE' + | 'DATABASE_TEMP' + | 'SCHEMA_OWNER' + | 'SCHEMA_CREATE' + | 'SCHEMA_USAGE' + | 'RELATION_PRIVILEGE' + | 'FUNCTION_PRIVILEGE' + | 'SEQUENCE_PRIVILEGE' + | 'AUDIT_NO_RESULT'; + +/** Credential-free identity expected from one dedicated listener login. */ +export interface PgNotificationRoleContract { + role: string; + database: string; +} + +/** Safe to persist in diagnostics: connection secrets/config are never copied. */ +export interface PgNotificationRoleAudit { + version: typeof PG_NOTIFICATION_ROLE_AUDIT_VERSION; + role: string; + database: string; + safe: boolean; + violations: readonly PgNotificationRoleViolationCode[]; +} + +/** Catalog-query capability used by the broker's pinned LISTEN client. */ +export type PgNotificationRoleClient = Pick; + +interface PgNotificationRoleAuditRow { + expected_role: string; + session_role: string; + active_role: string; + active_database: string; + rolcanlogin: boolean; + rolinherit: boolean; + rolsuper: boolean; + rolbypassrls: boolean; + rolcreaterole: boolean; + rolcreatedb: boolean; + rolreplication: boolean; + membership_count: number; + target_database_exists: boolean; + target_connect: boolean; + other_database_connect_count: number; + target_database_owner: boolean; + target_database_create: boolean; + target_database_temp: boolean; + schema_owner_count: number; + schema_create_count: number; + schema_usage_count: number; + relation_privilege_count: number; + function_privilege_count: number; + sequence_privilege_count: number; +} + +/** + * Audit only the session login's effective privileges. PostgreSQL system + * schemas/objects are excluded because ordinary logins necessarily use the + * catalog; every non-system schema and object remains in scope. + */ +export const PG_NOTIFICATION_ROLE_AUDIT_SQL = ` +WITH login_role AS MATERIALIZED ( + SELECT r.oid, r.rolname, r.rolcanlogin, r.rolinherit, r.rolsuper, + r.rolbypassrls, r.rolcreaterole, r.rolcreatedb, r.rolreplication + FROM pg_catalog.pg_roles r + WHERE r.rolname = session_user +), target_database AS MATERIALIZED ( + SELECT d.oid, d.datname, d.datdba + FROM pg_catalog.pg_database d + WHERE d.datname = $2::text +), application_schemas AS MATERIALIZED ( + SELECT n.oid, n.nspname, n.nspowner + FROM pg_catalog.pg_namespace n + WHERE n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_' +) +SELECT $1::text AS expected_role, + session_user AS session_role, + current_user AS active_role, + pg_catalog.current_database() AS active_database, + r.rolcanlogin, + r.rolinherit, + r.rolsuper, + r.rolbypassrls, + r.rolcreaterole, + r.rolcreatedb, + r.rolreplication, + ( + SELECT count(*)::int + FROM pg_catalog.pg_auth_members membership + WHERE membership.member = r.oid OR membership.roleid = r.oid + ) AS membership_count, + (target.oid IS NOT NULL) AS target_database_exists, + COALESCE( + pg_catalog.has_database_privilege(r.rolname, target.oid, 'CONNECT'), + false + ) AS target_connect, + ( + SELECT count(*)::int + FROM pg_catalog.pg_database database_record + WHERE database_record.datname <> $2::text + AND pg_catalog.has_database_privilege( + r.rolname, + database_record.oid, + 'CONNECT' + ) + ) AS other_database_connect_count, + COALESCE(target.datdba = r.oid, false) AS target_database_owner, + COALESCE( + pg_catalog.has_database_privilege(r.rolname, target.oid, 'CREATE'), + false + ) AS target_database_create, + COALESCE( + pg_catalog.has_database_privilege(r.rolname, target.oid, 'TEMP'), + false + ) AS target_database_temp, + ( + SELECT count(*)::int + FROM application_schemas schema_record + WHERE schema_record.nspowner = r.oid + ) AS schema_owner_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + WHERE pg_catalog.has_schema_privilege( + r.rolname, + schema_record.oid, + 'CREATE' + ) + ) AS schema_create_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + WHERE pg_catalog.has_schema_privilege( + r.rolname, + schema_record.oid, + 'USAGE' + ) + ) AS schema_usage_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + INNER JOIN pg_catalog.pg_class relation + ON relation.relnamespace = schema_record.oid + WHERE CASE WHEN relation.relkind IN ('r', 'p', 'v', 'm', 'f') + THEN pg_catalog.has_table_privilege( + r.rolname, + relation.oid, + 'SELECT,INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER' + ) + OR pg_catalog.has_any_column_privilege( + r.rolname, + relation.oid, + 'SELECT,INSERT,UPDATE,REFERENCES' + ) + ELSE false + END + ) AS relation_privilege_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + INNER JOIN pg_catalog.pg_proc routine + ON routine.pronamespace = schema_record.oid + WHERE pg_catalog.has_function_privilege( + r.rolname, + routine.oid, + 'EXECUTE' + ) + ) AS function_privilege_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + INNER JOIN pg_catalog.pg_class sequence_record + ON sequence_record.relnamespace = schema_record.oid + WHERE CASE WHEN sequence_record.relkind = 'S' + THEN pg_catalog.has_sequence_privilege( + r.rolname, + sequence_record.oid, + 'USAGE,SELECT,UPDATE' + ) + ELSE false + END + ) AS sequence_privilege_count +FROM login_role r +LEFT JOIN target_database target ON true +`; + +const containsUnpairedSurrogate = (value: string): boolean => { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return true; + index++; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +}; + +const assertIdentifier = (kind: 'role' | 'database', value: unknown): string => { + if (typeof value !== 'string' || value.length === 0) { + throw new PgNotificationRoleContractError(`${kind} must be a non-empty string`); + } + if (value.includes('\0') || containsUnpairedSurrogate(value)) { + throw new PgNotificationRoleContractError(`${kind} is not a valid PostgreSQL name`); + } + const bytes = Buffer.byteLength(value, 'utf8'); + if (bytes > 63) { + throw new PgNotificationRoleContractError( + `${kind} is ${bytes} UTF-8 bytes; PostgreSQL allows at most 63` + ); + } + return value; +}; + +const normalizeContract = ( + contract: PgNotificationRoleContract +): Readonly => Object.freeze({ + role: assertIdentifier('role', contract?.role), + database: assertIdentifier('database', contract?.database) +}); + +export class PgNotificationRoleContractError extends Error { + readonly code = PG_NOTIFICATION_ROLE_CONTRACT_ERROR_CODE; + + constructor(reason: string) { + super(`Invalid PostgreSQL notification-role contract: ${reason}`); + this.name = 'PgNotificationRoleContractError'; + } +} + +export class UnsafePgNotificationRoleError extends Error { + readonly code = PG_NOTIFICATION_ROLE_UNSAFE_ERROR_CODE; + + constructor( + readonly audit: PgNotificationRoleAudit + ) { + super( + `PostgreSQL notification role ${JSON.stringify(audit.role)} for database ` + + `${JSON.stringify(audit.database)} is unsafe: ${audit.violations.join(',')}` + ); + this.name = 'UnsafePgNotificationRoleError'; + } +} + +/** + * Enforce a one-to-one role/database mapping without accepting connection + * config. Exact duplicate pairs are collapsed for multi-generation reuse. + */ +export const normalizePgNotificationRoleContracts = ( + contracts: readonly PgNotificationRoleContract[] +): readonly Readonly[] => { + if (!Array.isArray(contracts) || contracts.length === 0) { + throw new PgNotificationRoleContractError('at least one role/database pair is required'); + } + const byDatabase = new Map(); + const byRole = new Map(); + const unique = new Map>(); + for (const candidate of contracts) { + const contract = normalizeContract(candidate); + const databaseRole = byDatabase.get(contract.database); + if (databaseRole && databaseRole !== contract.role) { + throw new PgNotificationRoleContractError( + `database ${JSON.stringify(contract.database)} maps to multiple login roles` + ); + } + const roleDatabase = byRole.get(contract.role); + if (roleDatabase && roleDatabase !== contract.database) { + throw new PgNotificationRoleContractError( + `login role ${JSON.stringify(contract.role)} maps to multiple databases` + ); + } + byDatabase.set(contract.database, contract.role); + byRole.set(contract.role, contract.database); + unique.set(`${contract.database}\0${contract.role}`, contract); + } + return Object.freeze( + [...unique.values()].sort((left, right) => { + if (left.database !== right.database) { + return left.database < right.database ? -1 : 1; + } + if (left.role === right.role) return 0; + return left.role < right.role ? -1 : 1; + }) + ); +}; + +const violationCodes = ( + row: PgNotificationRoleAuditRow | undefined, + contract: Readonly +): PgNotificationRoleViolationCode[] => { + if (!row) return ['AUDIT_NO_RESULT']; + const violations: PgNotificationRoleViolationCode[] = []; + if (row.session_role !== contract.role) violations.push('LOGIN_ROLE_MISMATCH'); + if (row.active_role !== row.session_role) violations.push('CURRENT_ROLE_MISMATCH'); + if (row.active_database !== contract.database) violations.push('DATABASE_MISMATCH'); + if (!row.rolcanlogin) violations.push('LOGIN_REQUIRED'); + if (row.rolinherit) violations.push('NOINHERIT_REQUIRED'); + if (row.rolsuper) violations.push('SUPERUSER'); + if (row.rolbypassrls) violations.push('BYPASSRLS'); + if (row.rolcreaterole) violations.push('CREATEROLE'); + if (row.rolcreatedb) violations.push('CREATEDB'); + if (row.rolreplication) violations.push('REPLICATION'); + if (row.membership_count > 0) violations.push('ROLE_MEMBERSHIP'); + if (!row.target_database_exists) violations.push('TARGET_DATABASE_MISSING'); + if (!row.target_connect) violations.push('TARGET_CONNECT_REQUIRED'); + if (row.other_database_connect_count > 0) violations.push('CROSS_DATABASE_CONNECT'); + if (row.target_database_owner) violations.push('DATABASE_OWNER'); + if (row.target_database_create) violations.push('DATABASE_CREATE'); + if (row.target_database_temp) violations.push('DATABASE_TEMP'); + if (row.schema_owner_count > 0) violations.push('SCHEMA_OWNER'); + if (row.schema_create_count > 0) violations.push('SCHEMA_CREATE'); + if (row.schema_usage_count > 0) violations.push('SCHEMA_USAGE'); + if (row.relation_privilege_count > 0) violations.push('RELATION_PRIVILEGE'); + if (row.function_privilege_count > 0) violations.push('FUNCTION_PRIVILEGE'); + if (row.sequence_privilege_count > 0) violations.push('SEQUENCE_PRIVILEGE'); + return violations; +}; + +/** Execute one fresh audit on an already-owned client without releasing it. */ +export const auditPgNotificationRoleClient = async ( + client: PgNotificationRoleClient, + candidate: PgNotificationRoleContract +): Promise => { + const contract = normalizeContract(candidate); + let inTransaction = false; + let result: QueryResult; + try { + await client.query('BEGIN READ ONLY'); + inTransaction = true; + await client.query('SET LOCAL jit TO off'); + result = await client.query( + PG_NOTIFICATION_ROLE_AUDIT_SQL, + [contract.role, contract.database] + ); + await client.query('COMMIT'); + inTransaction = false; + } catch (error) { + if (inTransaction) { + try { + await client.query('ROLLBACK'); + } catch { + // Preserve the catalog-audit failure; the owning broker destroys the client. + } + } + throw error; + } + + const violations = Object.freeze(violationCodes(result.rows[0], contract)); + return Object.freeze({ + version: PG_NOTIFICATION_ROLE_AUDIT_VERSION, + role: contract.role, + database: contract.database, + safe: violations.length === 0, + violations + }); +}; + +/** Execute one fresh, read-only catalog audit. Successful results are not cached. */ +export const auditPgNotificationRole = async ( + pool: Pool, + candidate: PgNotificationRoleContract +): Promise => { + const client: PoolClient = await pool.connect(); + let destroyClient = false; + try { + return await auditPgNotificationRoleClient(client, candidate); + } catch (error) { + destroyClient = true; + throw error; + } finally { + client.release(destroyClient); + } +}; + +/** Fail closed on a pinned client without exposing general query access. */ +export const assertPgNotificationRoleClient = async ( + client: PgNotificationRoleClient, + contract: PgNotificationRoleContract +): Promise => { + const audit = await auditPgNotificationRoleClient(client, contract); + if (!audit.safe) throw new UnsafePgNotificationRoleError(audit); + return audit; +}; + +/** Fail closed with a stable code while retaining a credential-free audit. */ +export const assertPgNotificationRole = async ( + pool: Pool, + contract: PgNotificationRoleContract +): Promise => { + const audit = await auditPgNotificationRole(pool, contract); + if (!audit.safe) throw new UnsafePgNotificationRoleError(audit); + return audit; +}; diff --git a/postgres/pg-cache/src/pg.ts b/postgres/pg-cache/src/pg.ts index 08920e924d..888c6b1dd5 100644 --- a/postgres/pg-cache/src/pg.ts +++ b/postgres/pg-cache/src/pg.ts @@ -1,49 +1,719 @@ +import { createHash, createHmac, randomBytes } from 'node:crypto'; +import type { EventEmitter } from 'node:events'; +import { performance } from 'node:perf_hooks'; + import { Logger } from '@pgpmjs/logger'; import { parseEnvNumber } from '12factor-env'; import pg from 'pg'; import { getPgEnvOptions, PgConfig, PgPoolConfig } from 'pg-env'; -import { getActivePgPoolFactory, PgPoolFactory } from './driver'; -import { pgCache } from './lru'; +import { getActivePgPoolFactory, getPgPoolDriverIdentity, PgPoolFactory } from './driver'; +import { pgCache, type PgPoolLease } from './lru'; const log = new Logger('pg-cache'); +export interface GetPgPoolOptions { + /** Separates pools used by different trust boundaries. */ + purpose?: string; + /** Reset all server and driver session state before every checkout. */ + sanitizeOnCheckout?: boolean; +} + +interface NodePostgresConnection { + parsedStatements?: Record; + _graphilePreparedStatementCache?: unknown; +} + +interface NodePostgresClient extends pg.PoolClient { + connection?: NodePostgresConnection; +} + +type PoolQueryCallback = (error: Error | undefined, result?: unknown) => void; + +const requireSanitizableClient = (value: unknown): NodePostgresClient => { + const client = value as Partial | null | undefined; + if ( + client + && typeof client.query === 'function' + && typeof client.release === 'function' + ) { + return client as NodePostgresClient; + } + if (client && typeof client.release === 'function') { + try { + client.release(true); + } catch { + // The factory contract is already invalid; never hand this client out. + } + } + throw new TypeError( + 'A sanitized PostgreSQL pool must return clients with callable query() and release() methods' + ); +}; + +export interface PgCheckoutSanitizerStats { + checkoutAttempts: number; + checkoutFailures: number; + queuedCheckouts: number; + virginFastPathCheckouts: number; + sanitizedReuseCheckouts: number; + sanitationFailures: number; + checkoutWaitMsTotal: number; + checkoutWaitMsMax: number; + sanitationMsTotal: number; + sanitationMsMax: number; +} + +const makeCheckoutSanitizerStats = (): PgCheckoutSanitizerStats => ({ + checkoutAttempts: 0, + checkoutFailures: 0, + queuedCheckouts: 0, + virginFastPathCheckouts: 0, + sanitizedReuseCheckouts: 0, + sanitationFailures: 0, + checkoutWaitMsTotal: 0, + checkoutWaitMsMax: 0, + sanitationMsTotal: 0, + sanitationMsMax: 0 +}); + +const aggregateCheckoutSanitizerStats = makeCheckoutSanitizerStats(); +const poolCheckoutSanitizerStats = new WeakMap(); + +const recordCount = ( + stats: PgCheckoutSanitizerStats, + key: 'checkoutAttempts' + | 'checkoutFailures' + | 'queuedCheckouts' + | 'virginFastPathCheckouts' + | 'sanitizedReuseCheckouts' + | 'sanitationFailures' +): void => { + stats[key]++; + aggregateCheckoutSanitizerStats[key]++; +}; + +const recordDuration = ( + stats: PgCheckoutSanitizerStats, + totalKey: 'checkoutWaitMsTotal' | 'sanitationMsTotal', + maxKey: 'checkoutWaitMsMax' | 'sanitationMsMax', + durationMs: number +): void => { + stats[totalKey] += durationMs; + stats[maxKey] = Math.max(stats[maxKey], durationMs); + aggregateCheckoutSanitizerStats[totalKey] += durationMs; + aggregateCheckoutSanitizerStats[maxKey] = Math.max( + aggregateCheckoutSanitizerStats[maxKey], + durationMs + ); +}; + +/** Aggregate checkout/sanitation telemetry, or telemetry for one exact pool. */ +export const getPgCheckoutSanitizerStats = ( + pool?: pg.Pool +): Readonly => ({ + ...(pool ? poolCheckoutSanitizerStats.get(pool) : aggregateCheckoutSanitizerStats) + ?? makeCheckoutSanitizerStats() +}); + +const normalizePoolOptions = (options: GetPgPoolOptions = {}) => ({ + purpose: options.purpose ?? 'default', + sanitizeOnCheckout: options.sanitizeOnCheckout ?? false +}); + +// Pool identities may be emitted through diagnostics and cache lifecycle logs. +// A plain digest over a known connection shape would let that digest act as an +// offline password verifier. Keep the key private and process-local: identities +// remain deterministic for this registry's lifetime but are intentionally not +// portable evidence across processes. +const pgIdentityHmacKey = randomBytes(32); + +const hmacIdentity = (prefix: string, identity: string): string => + `${prefix}:${createHmac('sha256', pgIdentityHmacKey) + .update(identity) + .digest('hex')}`; + +const requireIdentityString = (value: unknown, path: string): string => { + if (typeof value !== 'string') { + throw new TypeError(`${path} must be a string`); + } + return value; +}; + +const requireIdentityInteger = ( + value: unknown, + path: string, + minimum: number, + maximum = Number.MAX_SAFE_INTEGER +): number => { + if ( + typeof value !== 'number' + || !Number.isSafeInteger(value) + || value < minimum + || value > maximum + ) { + throw new TypeError( + `${path} must be a safe integer between ${minimum} and ${maximum}` + ); + } + return value; +}; + +const normalizeIdentityOptions = ( + options: GetPgPoolOptions = {} +): Required => { + const purpose = options.purpose ?? 'default'; + const sanitizeOnCheckout = options.sanitizeOnCheckout ?? false; + if (typeof purpose !== 'string' || purpose.length === 0) { + throw new TypeError('pg pool purpose must be a non-empty string'); + } + if (typeof sanitizeOnCheckout !== 'boolean') { + throw new TypeError('pg pool sanitizeOnCheckout must be a boolean'); + } + return { purpose, sanitizeOnCheckout }; +}; + +const canonicalizeIdentityValue = ( + value: unknown, + path: string, + ancestors = new Set() +): unknown => { + if ( + value === null + || typeof value === 'string' + || typeof value === 'boolean' + ) { + return value; + } + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + throw new TypeError(`${path} must not contain a non-finite number`); + } + return Object.is(value, -0) ? ['number', '-0'] : value; + } + if (Buffer.isBuffer(value)) { + return ['buffer-sha256', createHash('sha256').update(value).digest('hex')]; + } + if (Array.isArray(value)) { + if (ancestors.has(value)) throw new TypeError(`${path} must not be cyclic`); + const ownKeys = Reflect.ownKeys(value); + if ( + ownKeys.some((key) => typeof key !== 'string') + || ownKeys.some((key) => key !== 'length' && !/^(?:0|[1-9]\d*)$/.test(key as string)) + || value.some((_entry, index) => !Object.prototype.hasOwnProperty.call(value, index)) + || Object.keys(value).length !== value.length + ) { + throw new TypeError(`${path} must be a dense array without custom properties`); + } + ancestors.add(value); + const result = value.map((entry, index) => { + if (entry === undefined) { + throw new TypeError(`${path}[${index}] must not be undefined`); + } + return canonicalizeIdentityValue(entry, `${path}[${index}]`, ancestors); + }); + ancestors.delete(value); + return ['array', result]; + } + if (typeof value === 'object') { + const record = value as Record; + const prototype = Object.getPrototypeOf(record); + if (prototype !== Object.prototype && prototype !== null) { + throw new TypeError(`${path} must contain only data values`); + } + if (ancestors.has(record)) throw new TypeError(`${path} must not be cyclic`); + ancestors.add(record); + const result: Array<[string, unknown]> = []; + const ownKeys = Reflect.ownKeys(record); + if (ownKeys.some((key) => typeof key !== 'string')) { + throw new TypeError(`${path} must not contain symbol properties`); + } + for (const key of (ownKeys as string[]).sort()) { + const descriptor = Object.getOwnPropertyDescriptor(record, key); + if (!descriptor || !('value' in descriptor)) { + throw new TypeError(`${path}.${key} must be a data property`); + } + const entry = descriptor.value; + if (entry === undefined) { + throw new TypeError(`${path}.${key} must not be undefined`); + } + result.push([ + key, + canonicalizeIdentityValue(entry, `${path}.${key}`, ancestors) + ]); + } + ancestors.delete(record); + return ['object', result]; + } + throw new TypeError(`${path} must contain only deterministic data values`); +}; + export const buildConnectionString = ( user: string, password: string, host: string, port: string | number, database: string -): string => - `postgres://${user}:${password}@${host}:${port}/${database}`; +): string => { + const encodedHost = host.includes(':') && !host.startsWith('[') + ? `[${host}]` + : encodeURIComponent(host); + return `postgres://${encodeURIComponent(user)}:${encodeURIComponent(password)}` + + `@${encodedHost}:${port}/${encodeURIComponent(database)}`; +}; /** * Read per-pool configuration from environment variables. * * Supports: * - PG_POOL_MAX: Maximum clients per pool (default: 5) + * - PG_POOL_MAX_USES: Retire a client after this many checkouts (0/unset: unlimited) * - PG_POOL_IDLE_TIMEOUT_MS: Close idle clients after ms (default: 30000) * - PG_POOL_CONNECTION_TIMEOUT_MS: Fail connect() after ms (default: 5000) */ +const normalizeMaxUses = ( + value: number | string | undefined, + source: 'pool.maxUses' | 'PG_POOL_MAX_USES' +): number | undefined => { + if (value === undefined || value === '') { + return undefined; + } + if (typeof value !== 'number' && typeof value !== 'string') { + throw new TypeError(`${source} must be 0 or a positive safe integer`); + } + if (typeof value === 'string' && !/^(?:0|[1-9]\d*)$/.test(value)) { + throw new TypeError(`${source} must be 0 or a positive safe integer`); + } + const parsed = typeof value === 'number' ? value : Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 0) { + throw new TypeError(`${source} must be 0 or a positive safe integer`); + } + return parsed === 0 ? undefined : parsed; +}; + export function getPgPoolConfig(overrides?: PgPoolConfig): pg.PoolConfig { - return { + const maxUses = overrides?.maxUses !== undefined + ? normalizeMaxUses(overrides.maxUses, 'pool.maxUses') + : normalizeMaxUses(process.env.PG_POOL_MAX_USES, 'PG_POOL_MAX_USES'); + const pool = { max: overrides?.max ?? parseEnvNumber(process.env.PG_POOL_MAX) ?? 5, + ...(maxUses !== undefined && { maxUses }), idleTimeoutMillis: overrides?.idleTimeoutMillis ?? parseEnvNumber(process.env.PG_POOL_IDLE_TIMEOUT_MS) ?? 30000, connectionTimeoutMillis: overrides?.connectionTimeoutMillis ?? parseEnvNumber(process.env.PG_POOL_CONNECTION_TIMEOUT_MS) ?? 5000, ...(overrides?.allowExitOnIdle !== undefined && { allowExitOnIdle: overrides.allowExitOnIdle }), }; + requireIdentityInteger(pool.max, 'pool.max', 1); + if (pool.maxUses !== undefined) { + requireIdentityInteger(pool.maxUses, 'pool.maxUses', 1); + } + requireIdentityInteger(pool.idleTimeoutMillis, 'pool.idleTimeoutMillis', 0); + requireIdentityInteger( + pool.connectionTimeoutMillis, + 'pool.connectionTimeoutMillis', + 0 + ); + if ( + pool.allowExitOnIdle !== undefined + && typeof pool.allowExitOnIdle !== 'boolean' + ) { + throw new TypeError('pool.allowExitOnIdle must be a boolean'); + } + return pool; +} + +const normalizeIdentityConfig = ( + pgConfig: Partial & { pool?: PgPoolConfig } +): { + config: PgConfig; + ssl: unknown; +} => { + const config = getPgEnvOptions(pgConfig); + requireIdentityString(config.host, 'pg.host'); + requireIdentityInteger(config.port, 'pg.port', 1, 65_535); + requireIdentityString(config.database, 'pg.database'); + requireIdentityString(config.user, 'pg.user'); + // node-postgres also accepts password callbacks at runtime, despite the + // narrower public PgConfig type. A callback's captured secret cannot be + // represented exactly, so accepting it could alias two security principals. + requireIdentityString(config.password, 'pg.password'); + return { + config, + ssl: canonicalizeIdentityValue(config.ssl ?? null, 'pg.ssl') + }; +}; + +/** + * Opaque identity for the exact connection and checkout contract. + * + * The digest deliberately includes credentials: two roles that happen to + * connect to the same database must never share a pool. Only the digest is + * exposed, so passwords cannot leak through cache keys or logs. + */ +export function getPgPoolIdentity( + pgConfig: Partial & { pool?: PgPoolConfig }, + options: GetPgPoolOptions = {} +): string { + const { config, ssl } = normalizeIdentityConfig(pgConfig); + const pool = getPgPoolConfig(pgConfig.pool); + const normalizedOptions = normalizeIdentityOptions(options); + const driver = requireIdentityString( + getPgPoolDriverIdentity(), + 'pg driver identity' + ); + const identity = JSON.stringify({ + version: 1, + driver, + host: config.host, + port: config.port, + database: config.database, + user: config.user, + password: config.password, + ssl, + pool: { + max: pool.max, + maxUses: pool.maxUses ?? null, + idleTimeoutMillis: pool.idleTimeoutMillis, + connectionTimeoutMillis: pool.connectionTimeoutMillis, + allowExitOnIdle: pool.allowExitOnIdle ?? false + }, + ...normalizedOptions + }); + return hmacIdentity('pg:v1', identity); +} + +/** + * Opaque identity for one configured physical PostgreSQL database target. + * + * Credentials, TLS policy, pool sizing, and checkout behavior deliberately do + * not participate: those inputs must split connection pools, but they must not + * let two active listener contracts evade a one-target reservation. + */ +export function getPgDatabaseTargetIdentity( + pgConfig: Partial +): string { + const { config } = normalizeIdentityConfig(pgConfig); + const driver = requireIdentityString( + getPgPoolDriverIdentity(), + 'pg driver identity' + ); + const identity = JSON.stringify({ + version: 1, + driver, + host: config.host, + port: config.port, + database: config.database + }); + return hmacIdentity('pg-target:v1', identity); +} + +/** Clear client-side state whose server-side counterpart DISCARD ALL removed. */ +export function clearPreparedStatementBookkeeping(client: NodePostgresClient): void { + const connection = client.connection; + if (!connection) return; + + if (connection.parsedStatements) { + for (const statementName of Object.keys(connection.parsedStatements)) { + delete connection.parsedStatements[statementName]; + } + } + // Dataplan's LRU disposer issues asynchronous DEALLOCATE queries. DISCARD + // ALL has already removed every server-side prepared statement, so invoking + // that disposer here would race those cleanup queries with the next tenant + // transaction on this client. Drop the now-invalid client-side LRU instead; + // Dataplan will create a fresh one on demand. + delete connection._graphilePreparedStatementCache; +} + +const SANITIZED_SESSION_BASELINE = [ + 'SET search_path TO pg_catalog', + 'SET row_security TO on' +] as string[]; + +const sanitizedStartupOptions = (): string => { + const settings = [ + '-c search_path=pg_catalog', + '-c row_security=on' + ]; + if (process.env.DATAPLAN_PG_DONT_DISABLE_JIT !== '1') { + settings.push('-c jit_optimize_above_cost=-1'); + } + return settings.join(' '); +}; + +const sanitizedSessionBaseline = (): string => { + const statements = [...SANITIZED_SESSION_BASELINE]; + if (process.env.DATAPLAN_PG_DONT_DISABLE_JIT !== '1') { + statements.push('SET jit_optimize_above_cost TO -1'); + } + return statements.join('; '); +}; + +/** + * Reset a checked-out connection before it crosses a request boundary. + * A failed reset destroys the connection; it is never returned to a caller. + */ +export async function sanitizePgClient( + client: NodePostgresClient, + baselineRestoredByDiscard = false +): Promise { + try { + await client.query('DISCARD ALL'); + clearPreparedStatementBookkeeping(client); + if (!baselineRestoredByDiscard) { + // Custom drivers may not support PostgreSQL startup options. Restore all + // trusted defaults in one simple-query round trip after DISCARD ALL. + await client.query(sanitizedSessionBaseline()); + } + return client; + } catch (error) { + client.release(true); + throw error; + } +} + +/** Sanitize Promise/callback checkouts and every custom-pool direct query. */ +export function installCheckoutSanitizer( + pool: pg.Pool, + baselineRestoredByDiscard = false, + /** @internal Only the default factory may assert this startup contract. */ + factoryOwnedVirginFastPath = false +): pg.Pool { + if (typeof pool.connect !== 'function' || typeof pool.query !== 'function') { + throw new TypeError( + 'A sanitized PostgreSQL pool must expose callable connect() and query() methods' + ); + } + const stats = makeCheckoutSanitizerStats(); + poolCheckoutSanitizerStats.set(pool, stats); + const virginClients = new WeakSet(); + const eventPool = pool as unknown as Partial; + const canProveFactoryListenerContract = + typeof eventPool.rawListeners === 'function' + && typeof eventPool.on === 'function' + && typeof eventPool.prependListener === 'function'; + const connectListeners = (): Function[] => canProveFactoryListenerContract + ? eventPool.rawListeners!('connect') + : []; + // A non-EventEmitter custom QueryablePool can still use full sanitation, but + // it can never qualify for the default node-postgres virgin fast path. + let factoryContractContaminated = factoryOwnedVirginFastPath + && (!canProveFactoryListenerContract || connectListeners().length > 0); + const markFactoryOwnedVirgin = (client: pg.PoolClient): void => { + const listeners = connectListeners(); + if ( + !factoryContractContaminated + && listeners.length === 1 + && listeners[0] === markFactoryOwnedVirgin + ) { + virginClients.add(client); + } + }; + if (factoryOwnedVirginFastPath && canProveFactoryListenerContract) { + // Remember that an untrusted connect hook has existed even if it is a + // self-removing once/prependOnce listener and disappears during emission. + eventPool.on!('newListener', (eventName, listener) => { + if (eventName === 'connect' && listener !== markFactoryOwnedVirgin) { + factoryContractContaminated = true; + } + }); + // This listener is installed before the lazy pool can open a connection. + // A second connect listener disables the fast path because that listener + // could mutate session state before the checkout reaches this wrapper. + eventPool.prependListener!('connect', markFactoryOwnedVirgin); + } + const originalConnect = pool.connect.bind(pool); + const sanitizedConnect = async (): Promise => { + recordCount(stats, 'checkoutAttempts'); + const checkoutStartedAt = performance.now(); + const waitingBefore = pool.waitingCount; + const pending = originalConnect(); + if (pool.waitingCount > waitingBefore) recordCount(stats, 'queuedCheckouts'); + + let client: pg.PoolClient; + try { + client = requireSanitizableClient(await pending); + } catch (error) { + recordDuration( + stats, + 'checkoutWaitMsTotal', + 'checkoutWaitMsMax', + performance.now() - checkoutStartedAt + ); + recordCount(stats, 'checkoutFailures'); + throw error; + } + recordDuration( + stats, + 'checkoutWaitMsTotal', + 'checkoutWaitMsMax', + performance.now() - checkoutStartedAt + ); + + const virgin = virginClients.delete(client); + const markerIsExclusive = factoryOwnedVirginFastPath + && !factoryContractContaminated + && connectListeners().length === 1 + && connectListeners()[0] === markFactoryOwnedVirgin; + if (virgin && markerIsExclusive) { + // The default factory supplies the trusted baseline in the startup + // packet. With no other connect listener and no prior checkout, neither + // server nor driver state exists to discard. + clearPreparedStatementBookkeeping(client as NodePostgresClient); + recordCount(stats, 'virginFastPathCheckouts'); + return client; + } + + const sanitationStartedAt = performance.now(); + try { + const sanitized = await sanitizePgClient( + client as NodePostgresClient, + baselineRestoredByDiscard + ); + recordCount(stats, 'sanitizedReuseCheckouts'); + return sanitized; + } catch (error) { + recordCount(stats, 'sanitationFailures'); + throw error; + } finally { + recordDuration( + stats, + 'sanitationMsTotal', + 'sanitationMsMax', + performance.now() - sanitationStartedAt + ); + } + }; + + pool.connect = ((callback?: ( + error: Error | undefined, + client: pg.PoolClient | undefined, + done: ((release?: boolean | Error) => void) | undefined + ) => void) => { + const pending = sanitizedConnect(); + if (!callback) return pending; + pending.then( + (client) => callback(undefined, client, client.release.bind(client)), + (error) => callback(error as Error, undefined, undefined) + ); + }) as typeof pool.connect; + + if (!factoryOwnedVirginFastPath) { + // node-postgres' own pool.query dynamically calls this.connect(), so the + // default factory already reaches the sanitizer while retaining the full + // native query contract. An arbitrary QueryablePool may bypass connect() + // in its query implementation, so never trust that method in sanitized + // mode: acquire the sanitized client here and execute the query once. + const sanitizedQuery = ((...args: unknown[]) => { + if (typeof args[0] === 'function') { + const callback = args[0] as PoolQueryCallback; + queueMicrotask(() => callback( + new TypeError('Passing a function as the first parameter to pool.query is not supported') + )); + return undefined; + } + + const callback = typeof args[args.length - 1] === 'function' + ? args.pop() as PoolQueryCallback + : undefined; + const execute = async (): Promise => { + const client = await sanitizedConnect() as NodePostgresClient; + let settled = false; + const canObserveErrors = + typeof client.once === 'function' + && typeof client.removeListener === 'function'; + + return new Promise((resolve, reject) => { + const removeErrorListener = (): void => { + if (!canObserveErrors) return; + try { + client.removeListener('error', onClientError); + } catch { + // A broken optional EventEmitter surface must not prevent release. + } + }; + const releaseAfterError = (error: unknown): void => { + client.release(error instanceof Error ? error : true); + }; + const fail = (error: unknown): void => { + if (settled) return; + settled = true; + removeErrorListener(); + try { + releaseAfterError(error); + } catch { + // Preserve the query/connection error; release errors cannot make + // an already unsafe client eligible for reuse. + } + reject(error); + }; + const succeed = (result: unknown): void => { + if (settled) return; + settled = true; + removeErrorListener(); + try { + client.release(); + resolve(result); + } catch (error) { + reject(error); + } + }; + const onClientError = (error: Error): void => fail(error); + + try { + if (canObserveErrors) client.once('error', onClientError); + Promise.resolve((client.query as (...queryArgs: unknown[]) => unknown)(...args)) + .then(succeed, fail); + } catch (error) { + fail(error); + } + }); + }; + + const pending = execute(); + if (!callback) return pending; + pending.then( + (result) => callback(undefined, result), + (error) => callback(error as Error) + ); + return undefined; + }) as typeof pool.query; + + try { + pool.query = sanitizedQuery; + } catch { + throw new TypeError( + 'A sanitized custom PostgreSQL pool must expose a replaceable query() method' + ); + } + if (pool.query !== sanitizedQuery) { + throw new TypeError( + 'A sanitized custom PostgreSQL pool must expose a replaceable query() method' + ); + } + } + + return pool; } /** * Default pool factory: builds a real `pg.Pool` over TCP. This is the behavior * used whenever no alternate driver is registered (see `./driver`). */ -export const defaultPgPoolFactory: PgPoolFactory = (pgConfig): pg.Pool => { - const config = getPgEnvOptions(pgConfig); - const { user, password, host, port, database } = config; - const connectionString = buildConnectionString(user, password, host, port, database); +export const defaultPgPoolFactory: PgPoolFactory = (pgConfig, options): pg.Pool => { + const { config } = normalizeIdentityConfig(pgConfig); + normalizeIdentityOptions(options); + const { user, password, host, port, database, ssl } = config; const poolConfig = getPgPoolConfig(pgConfig.pool); - const pgPool = new pg.Pool({ connectionString, ...poolConfig }); + const pgPool = new pg.Pool({ + host, + port: Number(port), + database, + user, + password, + ...(ssl !== undefined && { ssl }), + ...(options?.sanitizeOnCheckout && { options: sanitizedStartupOptions() }), + ...poolConfig + }); /** * IMPORTANT: Pool-level error handler for idle connection errors. @@ -97,23 +767,57 @@ export const defaultPgPoolFactory: PgPoolFactory = (pgConfig): pg.Pool => { } }); - return pgPool; + // DISCARD ALL restores startup parameters. Pinning the security baseline in + // the startup packet makes the default driver a one-round-trip checkout; + // custom drivers use the explicit post-DISCARD fallback above. + return options?.sanitizeOnCheckout + ? installCheckoutSanitizer(pgPool, true, true) + : pgPool; }; -export const getPgPool = (pgConfig: Partial & { pool?: PgPoolConfig }): pg.Pool => { - const config = getPgEnvOptions(pgConfig); - const { database } = config; - if (pgCache.has(database)) { - const cached = pgCache.get(database); - if (cached) return cached; - } - +const createPgPool = ( + pgConfig: Partial & { pool?: PgPoolConfig }, + normalizedOptions: ReturnType +): pg.Pool => { // Route through the registered driver (default = pg.Pool over TCP). A custom // factory may return any QueryablePool (e.g. an in-process PGlite pool); it is // treated as a pg.Pool since that is the only surface consumers use. const factory = getActivePgPoolFactory() ?? defaultPgPoolFactory; - const pgPool = factory(pgConfig) as pg.Pool; - - pgCache.set(database, pgPool); + const pgPool = factory(pgConfig, normalizedOptions) as pg.Pool; + if (normalizedOptions.sanitizeOnCheckout && factory !== defaultPgPoolFactory) { + installCheckoutSanitizer(pgPool); + } return pgPool; }; + +/** Synchronously get or create an unleased pool for backwards compatibility. */ +export const getPgPool = ( + pgConfig: Partial & { pool?: PgPoolConfig }, + options: GetPgPoolOptions = {} +): pg.Pool => { + const normalizedOptions = normalizePoolOptions(options); + const poolIdentity = getPgPoolIdentity(pgConfig, normalizedOptions); + return pgCache.getOrCreate( + poolIdentity, + () => createPgPool(pgConfig, normalizedOptions) + ); +}; + +/** + * Atomically get/create and lease the exact connection identity. + * + * Callers that retain a pool beyond the current stack frame should use this + * production API and release only after their final request or long-lived + * resource has drained. + */ +export const acquirePgPool = ( + pgConfig: Partial & { pool?: PgPoolConfig }, + options: GetPgPoolOptions = {} +): PgPoolLease => { + const normalizedOptions = normalizePoolOptions(options); + const poolIdentity = getPgPoolIdentity(pgConfig, normalizedOptions); + return pgCache.acquire( + poolIdentity, + () => createPgPool(pgConfig, normalizedOptions) + ); +}; diff --git a/postgres/pg-env/README.md b/postgres/pg-env/README.md index dcf748b4c1..db6168a16e 100644 --- a/postgres/pg-env/README.md +++ b/postgres/pg-env/README.md @@ -97,9 +97,31 @@ interface PgConfig { user: string; password: string; database: string; + ssl?: boolean | PgSslOptions; } ``` +`ssl` is a data-only subset of Node TLS options (`ca`, `cert`, `key`, +`passphrase`, hostname verification, protocol bounds, and ciphers). Callback +and pre-opened-socket TLS options are deliberately excluded so connection pool +identities can account for the complete trust contract deterministically. + +#### `PgPoolConfig` + +```typescript +interface PgPoolConfig { + max?: number; + maxUses?: number; + idleTimeoutMillis?: number; + connectionTimeoutMillis?: number; + allowExitOnIdle?: boolean; +} +``` + +`maxUses` is passed to native pg-pool by `pg-cache`. Its +`PG_POOL_MAX_USES` parser treats `0` or an unset value as unlimited reuse and +accepts only canonical positive decimal safe integers otherwise. + ### Functions - `getPgEnvOptions(overrides?: Partial): PgConfig` - Get config from environment with overrides diff --git a/postgres/pg-env/src/index.ts b/postgres/pg-env/src/index.ts index 118de7b4d5..e4622cf002 100644 --- a/postgres/pg-env/src/index.ts +++ b/postgres/pg-env/src/index.ts @@ -4,4 +4,10 @@ export { getSpawnEnvWithPg, toPgEnvVars} from './env'; export { getPgClientCommand,PgClientTool } from './pg-client'; -export { defaultPgConfig,PgConfig,PgPoolConfig } from './pg-config'; \ No newline at end of file +export { + defaultPgConfig, + PgConfig, + PgPoolConfig, + PgSslConfig, + PgSslOptions +} from './pg-config'; diff --git a/postgres/pg-env/src/pg-config.ts b/postgres/pg-env/src/pg-config.ts index 7ed78ce5cd..ec7b05f430 100644 --- a/postgres/pg-env/src/pg-config.ts +++ b/postgres/pg-env/src/pg-config.ts @@ -1,9 +1,33 @@ +import type { SecureVersion } from 'node:tls'; + +/** + * Serializable TLS options supported by the shared PostgreSQL connection + * contract. Keeping this surface data-only is intentional: pool identities + * must account for every TLS input, which callback and socket objects cannot + * do deterministically. + */ +export interface PgSslOptions { + ca?: string | Buffer | Array; + cert?: string | Buffer | Array; + key?: string | Buffer | Array; + passphrase?: string; + rejectUnauthorized?: boolean; + servername?: string; + minVersion?: SecureVersion; + maxVersion?: SecureVersion; + ciphers?: string; +} + +export type PgSslConfig = boolean | PgSslOptions; + export interface PgConfig { host: string; port: number; user: string; password: string; database: string; + /** TLS settings passed directly to node-postgres. */ + ssl?: PgSslConfig; } /** @@ -15,6 +39,8 @@ export interface PgConfig { export interface PgPoolConfig { /** Maximum number of clients in the pool (env: PG_POOL_MAX, default: 5) */ max?: number; + /** Retire a client after this many checkouts (env: PG_POOL_MAX_USES, 0/unset: unlimited) */ + maxUses?: number; /** Close idle clients after this many ms (env: PG_POOL_IDLE_TIMEOUT_MS, default: 30000) */ idleTimeoutMillis?: number; /** Reject pool.connect() after this many ms (env: PG_POOL_CONNECTION_TIMEOUT_MS, default: 5000) */ @@ -29,4 +55,4 @@ export const defaultPgConfig: PgConfig = { user: 'postgres', password: 'password', database: 'postgres' -}; \ No newline at end of file +}; diff --git a/postgres/pg-query-context/README.md b/postgres/pg-query-context/README.md index 7bed4c693b..3235016f51 100644 --- a/postgres/pg-query-context/README.md +++ b/postgres/pg-query-context/README.md @@ -24,7 +24,8 @@ npm install pg-query-context ## Features -* Sets session-level context (e.g., role, user ID) using `set_config`. +* Sets the complete transaction-local context (e.g., role, user ID) with one + parameterized `set_config` batch. * Automatically wraps execution in a transaction (`BEGIN`/`COMMIT`). * Automatically rolls back on error. * Supports both `Pool` and `Client` from `pg`. @@ -91,7 +92,12 @@ const user = await withPgClient( | `pool` | `Pool` | ✅ | The PostgreSQL pool to acquire a client from | | `context` | `Record` | ✅ | Session variables set via `set_config` | | `fn` | `(client: PoolClient) => T` | ✅ | Callback receiving the connected client | -| `opts` | `{ skipTransaction?: boolean }` | ❌ | Skip BEGIN/COMMIT wrapping (e.g., inside existing txn) | +| `opts` | `{ skipTransaction?: boolean }` | ❌ | Skip BEGIN/COMMIT only when no context is supplied; pooled transaction-local context fails closed | + +`set_config(..., true)` is transaction-local. A checked-out PostgreSQL client may +use the one-query API with `skipTransaction` inside a transaction managed by its +caller, but a pool cannot prove that transaction ownership. Both pooled APIs +therefore reject `skipTransaction` when the context is non-empty. ## Example with `express` diff --git a/postgres/pg-query-context/src/__tests__/index.test.ts b/postgres/pg-query-context/src/__tests__/index.test.ts new file mode 100644 index 0000000000..294b6a555c --- /dev/null +++ b/postgres/pg-query-context/src/__tests__/index.test.ts @@ -0,0 +1,151 @@ +import type { Pool, PoolClient } from 'pg'; + +import pgQueryContext, { + UNSAFE_POOLED_CONTEXT_ERROR_CODE, + UnsafePooledContextError, + withPgClient +} from '../index'; + +const SETTINGS_SQL = + 'SELECT pg_catalog.set_config(setting->>0, setting->>1, true) ' + + 'FROM pg_catalog.json_array_elements($1::json) AS setting'; + +const makePool = () => { + const client = { + query: jest.fn(async () => ({ rows: [] as unknown[] })), + release: jest.fn() + } as unknown as PoolClient; + const pool = { + connect: jest.fn(async () => client) + } as unknown as Pool; + return { client, pool }; +}; + +describe('pg query context', () => { + it('applies the complete ordered context in one parameterized round trip', async () => { + const { client, pool } = makePool(); + const context = { + 'jwt.claims.user_id': '', + role: 'tenant_runtime', + transaction_read_only: 'off', + search_path: 'pg_catalog, "tenant_api"', + row_security: 'on' + }; + const callback = jest.fn(async () => 'ok'); + + await expect(withPgClient(pool, context, callback)).resolves.toBe('ok'); + + expect(client.query).toHaveBeenNthCalledWith(1, 'BEGIN'); + expect(client.query).toHaveBeenNthCalledWith(2, SETTINGS_SQL, [ + JSON.stringify(Object.entries(context)) + ]); + expect(client.query).toHaveBeenNthCalledWith(3, 'COMMIT'); + expect(callback).toHaveBeenCalledWith(client); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it('does not issue a context query for an empty context', async () => { + const { client, pool } = makePool(); + + await withPgClient(pool, {}, async (): Promise => undefined); + + expect(client.query).toHaveBeenCalledTimes(2); + expect(client.query).toHaveBeenNthCalledWith(1, 'BEGIN'); + expect(client.query).toHaveBeenNthCalledWith(2, 'COMMIT'); + }); + + it('fails closed instead of coercing non-string security settings', async () => { + const { client, pool } = makePool(); + + await expect(withPgClient( + pool, + { 'jwt.claims.user_id': null } as unknown as Record, + async (): Promise => undefined + )).rejects.toThrow( + "PostgreSQL context setting 'jwt.claims.user_id' must be a string" + ); + + expect(client.query).toHaveBeenNthCalledWith(1, 'BEGIN'); + expect(client.query).toHaveBeenNthCalledWith(2, 'ROLLBACK'); + expect(client.query).toHaveBeenCalledTimes(2); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it('rolls back and releases when the batched context is rejected', async () => { + const { client, pool } = makePool(); + (client.query as jest.Mock) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(new Error('invalid role')) + .mockResolvedValueOnce({ rows: [] }); + + await expect(withPgClient( + pool, + { role: 'missing_role' }, + async (): Promise => undefined + )).rejects.toThrow('invalid role'); + + expect(client.query).toHaveBeenNthCalledWith(3, 'ROLLBACK'); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it('uses the same single context batch for the one-query API', async () => { + const { client, pool } = makePool(); + (client.query as jest.Mock).mockImplementation(async (query: unknown) => ({ + rows: [query === 'SELECT tenant_id FROM documents' ? { tenant_id: 'a' } : undefined] + .filter(Boolean) + })); + + await pgQueryContext({ + client: pool, + context: { role: 'tenant_runtime', 'jwt.claims.tenant_id': 'a' }, + query: 'SELECT tenant_id FROM documents' + }); + + expect(client.query).toHaveBeenNthCalledWith(2, SETTINGS_SQL, [ + JSON.stringify([ + ['role', 'tenant_runtime'], + ['jwt.claims.tenant_id', 'a'] + ]) + ]); + expect(client.query).toHaveBeenCalledTimes(4); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it('rejects transaction-local context through a pool without a transaction', async () => { + const { pool } = makePool(); + + await expect(withPgClient( + pool, + { role: 'tenant_runtime' }, + async (): Promise => undefined, + { skipTransaction: true } + )).rejects.toMatchObject({ + name: UnsafePooledContextError.name, + code: UNSAFE_POOLED_CONTEXT_ERROR_CODE + }); + + expect(pool.connect).not.toHaveBeenCalled(); + + await expect(pgQueryContext({ + client: pool, + context: { 'jwt.claims.tenant_id': 'tenant-a' }, + query: 'SELECT 1', + skipTransaction: true + })).rejects.toBeInstanceOf(UnsafePooledContextError); + expect(pool.connect).not.toHaveBeenCalled(); + }); + + it('allows transaction-free pooled execution only when no context is requested', async () => { + const { client, pool } = makePool(); + + await expect(withPgClient( + pool, + {}, + async () => 'ok', + { skipTransaction: true } + )).resolves.toBe('ok'); + + expect(client.query).not.toHaveBeenCalled(); + expect(client.release).toHaveBeenCalledTimes(1); + }); +}); diff --git a/postgres/pg-query-context/src/__tests__/integration.test.ts b/postgres/pg-query-context/src/__tests__/integration.test.ts new file mode 100644 index 0000000000..3019d4b329 --- /dev/null +++ b/postgres/pg-query-context/src/__tests__/integration.test.ts @@ -0,0 +1,131 @@ +import { Pool, PoolClient } from 'pg'; + +import { withPgClient } from '../index'; + +const describeWithPostgres = process.env.PG_QUERY_CONTEXT_RUN_PG_INTEGRATION === '1' + ? describe + : describe.skip; + +interface SessionState { + role: string; + transaction_read_only: string; + search_path: string; + row_security: string; + user_id: string | null; +} + +async function readSessionState(client: PoolClient): Promise { + const result = await client.query(` + SELECT + current_setting('role') AS role, + current_setting('transaction_read_only') AS transaction_read_only, + current_setting('search_path') AS search_path, + current_setting('row_security') AS row_security, + current_setting('jwt.claims.user_id', true) AS user_id + `); + return result.rows[0]; +} + +describeWithPostgres('transaction-local PostgreSQL context', () => { + let pool: Pool; + let runtimeRole: string; + + beforeAll(async () => { + pool = new Pool({ max: 1 }); + const client = await pool.connect(); + try { + const identity = await client.query<{ current_user: string }>( + 'SELECT current_user' + ); + runtimeRole = identity.rows[0].current_user; + + // Deliberately establish a visibly different session baseline. With a + // one-client pool, every assertion below observes the same backend. + await client.query('RESET ROLE'); + await client.query('SET transaction_read_only TO off'); + await client.query('SET search_path TO public'); + await client.query('SET row_security TO off'); + await client.query( + "SELECT pg_catalog.set_config('jwt.claims.user_id', 'baseline-user', false)" + ); + } finally { + client.release(); + } + }); + + afterAll(async () => { + if (!pool) return; + const client = await pool.connect(); + try { + await client.query('RESET ROLE'); + await client.query('RESET ALL'); + } finally { + client.release(); + await pool.end(); + } + }); + + it('applies every security setting locally and restores the session after commit and rollback', async () => { + const committedInside = await withPgClient(pool, { + role: runtimeRole, + transaction_read_only: 'on', + search_path: 'pg_catalog', + row_security: 'on', + 'jwt.claims.user_id': '' + }, readSessionState); + + expect(committedInside).toEqual({ + role: runtimeRole, + transaction_read_only: 'on', + search_path: 'pg_catalog', + row_security: 'on', + user_id: '' + }); + + const afterCommitClient = await pool.connect(); + try { + await expect(readSessionState(afterCommitClient)).resolves.toEqual({ + role: 'none', + transaction_read_only: 'off', + search_path: 'public', + row_security: 'off', + user_id: 'baseline-user' + }); + } finally { + afterCommitClient.release(); + } + + let rolledBackInside: SessionState | undefined; + await expect(withPgClient(pool, { + role: runtimeRole, + transaction_read_only: 'on', + search_path: 'pg_catalog', + row_security: 'on', + 'jwt.claims.user_id': 'rollback-canary' + }, async (client) => { + rolledBackInside = await readSessionState(client); + throw new Error('force rollback'); + })).rejects.toThrow('force rollback'); + + expect(rolledBackInside).toEqual({ + role: runtimeRole, + transaction_read_only: 'on', + search_path: 'pg_catalog', + row_security: 'on', + user_id: 'rollback-canary' + }); + + const afterRollbackClient = await pool.connect(); + try { + await expect(readSessionState(afterRollbackClient)).resolves.toEqual({ + role: 'none', + transaction_read_only: 'off', + search_path: 'public', + row_security: 'off', + user_id: 'baseline-user' + }); + } finally { + afterRollbackClient.release(); + } + }); +}); diff --git a/postgres/pg-query-context/src/index.ts b/postgres/pg-query-context/src/index.ts index 188d8670f2..7a228c6c15 100644 --- a/postgres/pg-query-context/src/index.ts +++ b/postgres/pg-query-context/src/index.ts @@ -2,18 +2,54 @@ import { ClientBase, Pool, PoolClient, QueryResult } from 'pg'; // --- Internal helpers --- -function setContext(ctx: Record): { query: string; values: string[] }[] { - return Object.keys(ctx || {}).reduce<{ query: string; values: string[] }[]>((m, el) => { - m.push({ query: 'SELECT set_config($1, $2, true)', values: [el, ctx[el]] }); - return m; - }, []); +export const UNSAFE_POOLED_CONTEXT_ERROR_CODE = + 'PG_QUERY_CONTEXT_UNSAFE_POOLED_CONTEXT'; + +export class UnsafePooledContextError extends Error { + readonly code = UNSAFE_POOLED_CONTEXT_ERROR_CODE; + + constructor() { + super( + 'Transaction-local PostgreSQL context cannot be applied through a pool ' + + 'when skipTransaction is enabled' + ); + this.name = 'UnsafePooledContextError'; + } +} + +function assertContextHasTransaction( + usesPool: boolean, + skipTransaction: boolean, + context: Record +): void { + if (usesPool && skipTransaction && Object.keys(context).length > 0) { + throw new UnsafePooledContextError(); + } } async function execContext(client: ClientBase, ctx: Record): Promise { - const local = setContext(ctx); - for (const { query, values } of local) { - await client.query(query, values); + const entries = Object.entries(ctx || {}); + if (entries.length === 0) return; + + for (const [key, value] of entries) { + // This API establishes the request's security boundary. Runtime callers + // can still bypass TypeScript, so reject ambiguous null/object values + // instead of silently installing literal "null" or "[object Object]" + // session settings. + if (typeof value !== 'string') { + throw new TypeError(`PostgreSQL context setting '${key}' must be a string`); + } } + + // Apply the complete request context in one parameterized round trip. The + // array preserves insertion order (including role, read-only, search_path, + // and every explicitly-empty security claim), while set_config(..., true) + // keeps every value transaction-local exactly as the former per-key loop did. + await client.query( + 'SELECT pg_catalog.set_config(setting->>0, setting->>1, true) ' + + 'FROM pg_catalog.json_array_elements($1::json) AS setting', + [JSON.stringify(entries)] + ); } // --- Single-query API (original) --- @@ -31,6 +67,8 @@ async function pgQueryContext({ client, context = {}, query = '', variables = [] const shouldRelease = isPool; let pgClient: ClientBase | PoolClient | null = null; + assertContextHasTransaction(isPool, skipTransaction, context); + try { pgClient = isPool ? await (client as Pool).connect() : client as ClientBase; @@ -80,6 +118,7 @@ export async function withPgClient( fn: (client: PoolClient) => Promise, opts: WithPgClientOptions = {}, ): Promise { + assertContextHasTransaction(true, opts.skipTransaction === true, context); const client = await pool.connect(); try { if (!opts.skipTransaction) { diff --git a/research/graphile-density/IMPLEMENTATION-NOTES.md b/research/graphile-density/IMPLEMENTATION-NOTES.md new file mode 100644 index 0000000000..90962d3387 --- /dev/null +++ b/research/graphile-density/IMPLEMENTATION-NOTES.md @@ -0,0 +1,44 @@ +# Implementation notes and unknowns ledger + +## Map retained + +- Optimize complete warm tenants per GiB, not isolated schema build size. +- Keep one PostGraphile instance per exact physical tenant/API build contract. +- Treat RLS as row-level defense in depth; do not use it to justify SQL schema rewriting. +- Keep work local, use a fresh `origin/main` worktree, and never modify `constructive-db`. + +## Territory-driven deviations + +- #1332's SQL-text substitution was replaced by a parameterized query generator and explicit Graphile service option. +- #1333/#1334 were not ported. Dedicated physical-schema instances make BM25 and other `regclass`-like values ordinary physical names and remove the rewrite coverage problem. +- #1335's blueprint-oriented result model was replaced. A run counts tenants only after every declared build contract is resident, every surface is warm, all required operations and canaries ran, and the 15-minute telemetry gates pass. +- Reverse inheritance closure was removed after a partition fixture showed it would traverse from a shared parent into unrelated tenant child schemas. +- The checked-in fleet is an intentionally failing template because real tokens, credentials, APIs, S3, realtime events, and hostile probes do not exist safely inside this repository alone. + +## Unknowns ledger + +| Type | Item | Resolution/status | +|---|---|---| +| Known known | SQL rewrite routes objects before RLS evaluates rows. | Rewrite pooling rejected. | +| Known known | `SET LOCAL` restores the prior session value at transaction end. | Checkout uses `DISCARD ALL`; each request seeds the full security-GUC set including empty claims, role, and read-only state. | +| Known known | Graphile/node-postgres retain client-side prepared-statement bookkeeping. | Both installed adaptor structures are cleared after `DISCARD ALL`; sanitation failure destroys the client. | +| Known known | Actual scoped retained heap and cold-build cost at 62,298 `pg_class`. | Clean three-repetition single-surface median: 6.55 MiB retained heap, 47.77 MiB final RSS delta, and 130.34 ms cold build. Stock was 449.42 MiB, 1,106.94 MiB, and 3,495.95 ms. Complete-customer density remains unmeasured. | +| Known unknown | Full least-privilege grants and RLS/FORCE-RLS behavior for deployed tenant schemas. | The disposable A/B/C production-shaped fixture passed all 56 hostile checks, but it cannot prove every real tenant table/policy. A production-schema policy manifest remains a blocker. | +| Unknown found | Parent→child inheritance closure can include unrelated tenant partitions. | Removed; small byte-equivalence regression passed. | +| Unknown found | Storage and public-key plugins had null-context/fallback paths. | Constructive now preloads an immutable exact-build storage snapshot, the generic fallback is exact/request-scoped/fail-closed, and public-key plans require request settings and the native Graphile transaction client. Remaining deliberate system/build lanes are recorded in `PLUGIN-SQL-AUDIT.md`. | +| Unknown found | Schema-level checks did not catch a runtime/request role that owns an individual table and therefore bypasses RLS. | Top-of-stack safety query now rejects relation, sequence, view, function, and type ownership in exposed schemas. | +| Unknown found | Cache instance samples are supported but not automatically measured by the server. | Operators must feed a benchmarked `GRAPHILE_CACHE_INSTANCE_HEAP_BYTES`; automatic self-tuning is not claimed safe. | +| Unknown found | Docker memory telemetry cannot attribute a sub-second spike to one PostgreSQL backend. | Harness records a coarse dedicated-container delta and labels it accordingly. | +| Unknown found | Capability labels cannot prove semantic coverage by themselves. | The completed fleet and hostile query definitions require human review; the placeholder fails validation rather than producing a score. | +| Unknown found | Legacy Node module resolution in `@constructive-io/graphql-query` loses the PostGraphile adaptor augmentation after the scoped service wrapper is emitted. | The package now uses the same NodeNext/bundler split as the newer Graphile query package; CJS, ESM, tests, and the monorepo build pass without a service-type cast. | +| Unknown found | Route and security settings are fresh but are read by independent statements with no common revision. | Exact identities prevent arbitrary A/B pool aliasing, but handover/revocation can leave one stale HTTP request and an accepted WebSocket can retain stale authorization indefinitely. An atomic `TenantSecurityContractV1` plus socket/generation retirement is a production blocker. | +| Unknown found | Graphile caller plugins are unrestricted in-process code; a fingerprint is not a sandbox or code signature. | Production now rejects non-empty caller presets/plugins by default. Any explicit trust opt-in requires pinned code review and full requalification; built-ins and the runtime credential resolver remain TCB. | +| Unknown found | CAPTCHA admission trusted a client-controlled operation label and did not cover every transport/body shape. | Admission now classifies root mutation fields from the selected AST, parses supported HTTP bodies first, rejects ambiguous/unclassifiable inputs, blocks protected WebSocket mutations, and fails closed on a missing production/strict secret. The protected-field allowlist and Google hostname/action/timeout policy still need release ownership. | + +## Conservative continuation policy + +Do not enable `scoped-required` in production until upstream review, the atomic +security-contract gap, production-schema policy proof, and the full fixture +matrix pass. Do not configure the governor directly from the 6.55 MiB +single-surface result. Any failed or missing telemetry, semantic capability, +surface canary, resident instance, or paired matrix point remains a failed run. diff --git a/research/graphile-density/ORIGINAL-STACK.md b/research/graphile-density/ORIGINAL-STACK.md new file mode 100644 index 0000000000..88d7e41cac --- /dev/null +++ b/research/graphile-density/ORIGINAL-STACK.md @@ -0,0 +1,16 @@ +# Original PR stack provenance + +Fetched on 2026-07-31 after fetching `origin/main`. The local worktree base is `a10ea246fcc45b025713024e131fafb908171149`. + +| PR | GitHub head | Logical base branch | Disposition | +|---|---|---|---| +| #1330 | `6786cc1f77bc5a6b9412b0d5ba4cbb3031de76d1` | `main` | Reimplemented selectively. Its four-commit PR history contains the plugin fixes, quoting migration, lockfile normalization, and a final description-only cleanup; comparing only its head against `cdf155e8` hides the earlier useful commits. | +| #1331 | `161c067c6616a5e121e6757b8921ff28e743179d` | `feat/scale-s1-plugin-fixes` | Hardened and separated from runtime identity work. | +| #1332 | `b8d0c20a20c8a5363e22d8f4285a73d86202b2b5` | `feat/scale-s2-cache-hardening` | SQL substitution implementation rejected and replaced. | +| #1333 | `269b23c2ff82b0b6295cb1822f5affddfbc511cb` | `feat/scale-s3-introspection-filter` | Rejected as a production design. | +| #1334 | `b3d0021b80896baaafaee04c16bd08da35ef2d26` | `feat/scale-s4-pooling-core` | Rejected as a production design. | +| #1335 | `de35e09d67702b6ce3b4ed77e743e449fdc5ea2f` | `feat/scale-s5-pooling-integration` | Harness concepts refreshed; old results retained only as historical claims. | + +The old stack is not one simple linear range: #1330 ends one commit after `cdf155e8`, while #1331 also has `cdf155e8` as its parent and #1332–#1335 form the linear five-commit chain beginning at #1331. Review therefore uses two range-diffs: #1330's full PR range against local branch 1, and #1331–#1335 against local branches 2–5. Lockfile normalization is reviewed independently rather than replayed. + +No branch or PR was pushed or modified. diff --git a/research/graphile-density/PLUGIN-SQL-AUDIT.md b/research/graphile-density/PLUGIN-SQL-AUDIT.md new file mode 100644 index 0000000000..c52614471e --- /dev/null +++ b/research/graphile-density/PLUGIN-SQL-AUDIT.md @@ -0,0 +1,38 @@ +# Graphile plugin SQL and request-scope audit + +This audit covers product packages under `graphile/*/src`, excluding test helpers. Its security model is the refreshed one: each tenant/API build uses its physical schemas and its own exact pool/build identity, so there is no SQL schema rewrite to make plugin SQL “poolable.” Identifiers still need correct quoting, values need binds, and runtime SQL needs the complete request settings. + +Graphile plugins are unrestricted Node.js code, not declarative SQL fragments. +They can use the configured runtime service for raw SQL, open their own +connections, or access process I/O. Production therefore rejects every non-empty +caller `extends`/`preset` by default with +`GRAPHILE_CALLER_PRESET_NOT_TRUSTED`. An explicit +`trustCallerPresetsInProduction: true` opt-in admits that code into the trusted +computing base; the exact build fingerprint separates its cache identity but +does not sandbox it, sign it, or attest mutable closure state. + +| Package/path family | SQL path | Result | +|---|---|---| +| `graphile-i18n` | Raw runtime translation query | Fixed: schema/table/type/column identifiers use `@pgsql/quotes`; values are bound; execution uses request `pgSettings`; locale type state is per build. | +| `graphile-llm` agent discovery | Raw control-metadata query and cache | Fixed: database ID is required, query-filtered, and part of the cache key. Missing identity fails closed. | +| `graphile-llm` RAG/metering | Raw chunk search and usage SQL | Fixed/retained: chunk relations are schema-qualified with quoted identifiers; vector and limits are values; runtime calls carry request `pgSettings`. | +| `graphile-search` BM25 | `pg-sql2` expressions with index-name bind values | Fixed: `to_bm25query` receives the physical schema-qualified index name. BM25 remains enabled because no tenant-schema rewrite occurs. | +| `graphile-search` tsvector/trigram/vector | `pg-sql2` expressions | Acceptable by construction: catalog-derived relation/column/type names use `sql.identifier`; request terms, vectors, limits, and thresholds use values. Must still run in the capability fixture. | +| `graphile-ltree`, `graphile-postgis`, `graphile-connection-filter`, `graphile-pg-aggregates` | Generated `pg-sql2` fragments | Acceptable by construction: dynamic identifiers use `sql.identifier`, while user inputs use `sql.value`. Must still run against quoted physical schemas. | +| `graphile-bulk-mutations` | Generated mutation plans and relation fragments | No tenant routing substitution exists; identifiers come from the introspected build. Full insert/update/upsert/delete behavior remains an integration gate. | +| `graphile-history` | Raw SELECT/UPDATE/INSERT SQL | Request-scoped and parameterized; schema/table/columns are quoted from the current build's codec/tag metadata. Integration coverage remains required. | +| `graphile-function-bindings` | QueryBuilder insert into configured invocation relation | Runtime inserts are request-scoped; schema/table and columns originate in the database-scoped compute-module configuration and values are bound. The optional generic gather fallback has one build-lane `withPgClientFromPgService(pgService, null, ...)` call against the exact build service, but Constructive supplies an authoritative control-plane binding snapshot and does not enter that branch. Integration coverage remains required. | +| `graphile-bucket-provisioner-plugin` | Raw storage metadata and bucket SQL | Metadata and bucket authorization run with request settings and are database-ID filtered; table identifiers are qualified with `@pgsql/quotes`. One request-triggered system-lane call records `physical_name` after the request-scoped lookup and S3 provision: the exact qualified table, authorized bucket UUID, physical pool, and `physical_name IS NULL` guard bound the write, while the baseline role supplies the deliberate bookkeeping privilege. S3 side effects and those grants remain an explicit integration gate. | +| `graphile-presigned-url-plugin` | Raw metadata, bucket, and file SQL | Fixed: Constructive supplies an immutable exact-build control-plane snapshot, including an authoritative empty list when storage is absent. The generic package fallback now carries the exact request `pgSettings`; missing context/settings, database identity, module metadata, bucket visibility, and lookup errors all fail before signing, and a metadata failure cannot select process-global S3 configuration. Global S3 values may fill nullable fields only after a tenant module and persisted physical bucket coordinate have resolved. One deliberate system-lane call remains to record a newly provisioned bucket's `physical_name`, with the same exact-table/UUID/null-guard constraints as the provisioner plugin. Hostile A/B/C storage and S3-side-effect tests remain a production gate. | +| `graphile-settings/PublicKeySignature` | Three raw auth-function paths | Fixed: every plan reads Grafast `pgSettings`, copies the complete request GUC map, explicitly overrides only `role` to `anonymous`, and fails closed if either request settings or `withPgClient` is absent. Queries now use the native Graphile client API inside the transaction established by `withPgClient`, so there is no nested manual transaction or null-context checkout. Identifier validation and qualification remain in place; real auth-function/RLS integration is still required. | +| `graphile-meta` | Build-time Graphile metadata collection | Current `main` already replaced the old module-global table array with schema/build-local state during extraction to `graphile-meta`; no additional port was needed. | + +The post-fix null-context inventory is three deliberate product-source lanes: + +- `graphile-presigned-url-plugin` performs one request-triggered system write that records an already-authorized bucket's persisted physical coordinate. +- `graphile-bucket-provisioner-plugin` uses the same system write pattern for explicit and automatic provisioning. +- `graphile-function-bindings` has one schema-gather fallback against the exact build's PostgreSQL service; Constructive's preloaded control-plane snapshot bypasses it. + +No request metadata, file/bucket authorization, signing, public-key auth, history, search, i18n, LLM/RAG, realtime visibility, or function-invocation path uses a null context. Test utilities use null or synthetic settings by design and are outside the product-source inventory. + +Prepared-statement sanitation was checked against the installed `@dataplan/pg` adaptor: it stores its LRU at `connection._graphilePreparedStatementCache` and node-postgres stores names at `connection.parsedStatements`, which are the two client-side structures cleared after `DISCARD ALL`. The performance cost of discarding prepared plans on every checkout remains a benchmark question, not a claimed free safety measure. diff --git a/research/graphile-density/RANGE-DIFF.md b/research/graphile-density/RANGE-DIFF.md new file mode 100644 index 0000000000..9cc0c09d73 --- /dev/null +++ b/research/graphile-density/RANGE-DIFF.md @@ -0,0 +1,45 @@ +# Old PR stack to local research stack + +The old commits and current-main implementations are different enough that `git range-diff` correctly reports delete/add pairs rather than pretending they are textual rebases. The semantic mapping below comes from that result plus per-file review. + +## Commands and raw correspondence + +```text +git range-diff \ + ed31ed2aa63fcd2d42acec6f27e672dd300a3959..6786cc1f77bc5a6b9412b0d5ba4cbb3031de76d1 \ + a10ea246fcc45b025713024e131fafb908171149..7ef0502666d71d33bd7275bd27588698a205f3ee + +1: ad54f1fbd < -: --------- tenant-isolation/correctness fixes +2: c96a87c84 < -: --------- whole-lockfile normalization +3: cdf155e84 < -: --------- @pgsql/quotes conversion +4: 6786cc1f7 < -: --------- description punctuation cleanup +-: --------- > 1: 7ef050266 current-main plugin scope/quoting implementation +``` + +```text +git range-diff \ + cdf155e84370153b6f5db9a5e7061efd8b9c0329..de35e09d67702b6ce3b4ed77e743e449fdc5ea2f \ + 7ef0502666d71d33bd7275bd27588698a205f3ee..848bfe2e7 + +1: 161c067c6 < -: --------- old cache/governor +2: b8d0c20a2 < -: --------- old introspection text filter +3: 269b23c2f < -: --------- blueprint rewrite core +4: b3d0021b8 < -: --------- blueprint pooling integration +5: de35e09d6 < -: --------- old cperf/scale validation +-: --------- > 1: de92be5a9 runtime boundary +-: --------- > 2: 5fd90ee2b hardened cache governor +-: --------- > 3: f41e480d5 parameterized scoped introspection +-: --------- > 4: 848bfe2e7 refreshed cperf and final audit hardening +``` + +## Semantic disposition + +| Old work | Local replacement | Per-file conclusion | +|---|---|---| +| #1330 | `7ef050266` | Reapplied quoting and tenant/build scoping in the current i18n, LLM/RAG, search/BM25, and cache APIs. The current `graphile-meta` WeakMap/build boundary supersedes its old global-cache edits. The lockfile rewrite and punctuation-only cleanup were intentionally omitted. | +| #1331 | `de92be5a9` + `5fd90ee2b` | Split identity/security from memory policy. Pool/build identities, runtime credentials, GUC initialization, and checkout reset live below the hardened governor rather than being implicit cache-key behavior. | +| #1332 | `f41e480d5` plus the top-branch closure regression | Replaced query-string substitution with `{ text, values }`, an explicit service mode, required-schema assertions, and dependency closure. A partition fixture then removed unsafe parent-to-unrelated-child expansion. | +| #1333/#1334 | none | Intentionally absent. No SQL rewrite seam or blueprint-sharing flag exists in the local stack. Dedicated instances compile against their actual physical schemas. | +| #1335 | `848bfe2e7` | Rebuilt around complete tenants and all surfaces, four fresh-process arms, hostile canaries, mandatory telemetry, paired density scoring, and immutable artifacts. Old blueprint figures remain labeled historical. | + +This is a reconstruction on `origin/main`, not a claim that old patches replayed cleanly. Review should compare the behavior and trust boundaries above, then use the focused files listed in `PLUGIN-SQL-AUDIT.md`, `UPSTREAM-REVIEW.md`, and `REPORT.md`. diff --git a/research/graphile-density/REPORT.md b/research/graphile-density/REPORT.md new file mode 100644 index 0000000000..2ba795208a --- /dev/null +++ b/research/graphile-density/REPORT.md @@ -0,0 +1,163 @@ +# Graphile tenant-density research spike + +The useful non-rewrite work has been rebuilt on current `origin/main` and the +memory result is now measured: on a clean three-repetition 62,298-`pg_class` +fixture, scoped dependency introspection reduced median retained heap from +449.42 MiB to 6.55 MiB and cold build time from 3,495.95 ms to 130.34 ms. The +latest disposable PostgreSQL 18 A/B/C hostile run passed all 56 checks with zero +cross-tenant tokens. The default remains stock introspection and blueprint/SQL +rewrite pooling is absent. + +This is still not production-qualified. A final audit found a real +route/security revision race: exact identities prevent arbitrary cross-tenant +pool aliasing, but a domain handover or revocation can leave an in-flight HTTP +request on the old authorization snapshot, and an accepted WebSocket can retain +it indefinitely. `SECURITY-AUDIT.md` records the required atomic versioned +contract and the remaining release gates. + +## Local stack + +| Branch | Commit at report time | Outcome | +|---|---|---| +| `research/graphile-density-01-plugin-scope` | `7ef0502666d71d33bd7275bd27588698a205f3ee` | Reapplied i18n, LLM/RAG, BM25, quoting, and cache-scope fixes to current plugin architecture; current `graphile-meta` already has build-local state. | +| `research/graphile-density-02-runtime-boundary` | `de92be5a9f459079f75e94f23d35abcb92365d16` | Added exact pool/build identities, optional least-privilege runtime credentials, complete request-GUC initialization, checkout sanitation, and runtime-role safety checks. | +| `research/graphile-density-03-cache-governor` | `5fd90ee2b7aa5c52984cb53269ab4e1ec16422f0` | Hardened disposal, draining, build coalescing/admission, heap budgeting, stable 503 refusal codes, timers, and debug counters. | +| `research/graphile-density-04-scoped-introspection` | `f41e480d55dfee99a68567dc12145b40f5356bd7` | Replaced SQL text substitution with a bind-parameter query API and fail-closed Graphile service mode; stock remains the default. | +| `research/graphile-density-05-cperf` | this report's branch | Refreshed the harness around complete tenants, physical build contracts, hostile per-surface canaries, fresh processes, four arms, immutable run directories, and fail-closed scoring. The audit also extends runtime safety from schema ownership to relation/sequence/view/function/type ownership. | +| `research/graphile-density-06-measured-optimization` | current local branch | Measured the production-shaped catalog, released build-only state, hardened routing/credentials/plugins/storage/auth/GUC boundaries, and ran the complete A/B/C hostile gate. | + +All branches and artifacts are local. No push, PR edit, deployment, or `constructive-db` change was made. + +## Original PR disposition + +| PR | Disposition | Reason | +|---|---|---| +| #1330 | Reimplemented | The SQL quoting and tenant-scoped plugin fixes remain valid. The large old lockfile normalization was not replayed; current package manifests only add dependencies actually used. | +| #1331 | Hardened and split | Cache pressure work remains valuable, but pool/build identity, credentials, GUCs, and checkout sanitation were made an earlier trust boundary rather than mixed into eviction policy. | +| #1332 | Replaced | The old implementation substituted text inside a generated catalog query and could silently fall back. The candidate preserves `makeIntrospectionQuery()`, passes names as values, computes dependency closure, asserts requested schemas, and has no scoped-mode fallback. | +| #1333/#1334 | Rejected for production | Rewriting qualified SQL makes tenant routing depend on exhaustive interception of generated plans, plugins, raw SQL, prepared statements, metadata, functions, sequences, and extension-specific bind values. RLS does not prove that routing layer correct. | +| #1335 | Refreshed with corrected provenance | Process isolation, open-loop load, canaries, and artifacts were retained. Blueprint-specific baselines and success claims are historical and are not accepted as evidence for the dedicated-instance candidate. | + +The exact remote heads and unusual #1330/#1331 ancestry are recorded in `ORIGINAL-STACK.md`. + +## Decisions on the raised concerns + +Zhi's isolation concern is valid, so the aggressive rewrite design is out. Every resident instance compiles against ordered physical schemas, and host/service names are routing labels rather than cache identities. RLS remains valuable for row filtering, but metadata, schema objects, functions, sequences, owner/BYPASS roles, and privileged paths have their own hostile gates. + +The owner check now covers objects as well as schemas: a runtime login or reachable request role that owns a relation, sequence, view, function, or type in an exposed schema is rejected. Admission also rejects `SECURITY DEFINER`, owner-rights views, foreign/materialized views, unsafe stored-expression dependencies, unexpected inherited/`SET ROLE` paths, and privileges on unapproved objects. The real PostgreSQL integration suite created these unsafe grants and proved that they are rejected. + +Dan's API-separation concern is addressed by isolating `makeSchemaScopedIntrospectionQuery(schemas): { text, values }` from Constructive policy. Graphile selects it through a service option; Constructive only supplies the explicit `stock|scoped-required` setting. It is not presented as a general Graphile plugin because the narrow upstream seam belongs beside `makeIntrospectionQuery()` and the gather layer. + +BM25 stays enabled. Its query builder passes the physical schema-qualified index name, so there is no rewrite exclusion or canonical index alias. `SET LOCAL` does restore the prior session state, which is why request initialization alone is insufficient: a reused checkout runs `DISCARD ALL`, clears node-postgres and `@dataplan/pg` prepared bookkeeping, and destroys the connection if reset fails. The hostile fixture proved this on the same backend for A, B, and C. + +The plugin audit closed the earlier storage and `PublicKeySignature` request-context gaps. Constructive now supplies one immutable exact-build storage snapshot; the generic presigned fallback is database/API filtered, request-scoped, ambiguity-failing, and non-sliding. Public-key plans require request `pgSettings` and use the native Graphile transaction client. Arbitrary plugins remain unsandboxed Node code, so production rejects all caller presets by default; an explicit trust opt-in makes that code part of the process and database trusted computing base. See `PLUGIN-SQL-AUDIT.md`. + +The remaining authorization gap is control-plane atomicity rather than SQL rewrite. Route, RLS, auth, feature, CORS, public-key, and WebAuthn fields are read through separate statements with no shared revision. A handover can allow one stale HTTP request to the former tenant, while a WebSocket can retain a stale route/session until the generation is retired. Exact pool/build identities keep the data path on that captured tenant and prevent an A/B mixture, but production needs a revisioned, single-snapshot contract and long-lived transport revalidation. + +## Evidence produced + +The clean production-shaped catalog run used 62,298 `pg_class` rows, 346,369 +attributes, 8,496 procedures, 23,709 types, and 4,036 namespaces. Across three +fresh-process repetitions, stock introspection retained a median 471,256,024 +bytes (449.42 MiB) after forced GC, ended 1,160,708,096 bytes (1,106.94 MiB) +above the RSS baseline, and reached first-build readiness in 3,495.95 ms. Scoped +dependency introspection retained 6,869,560 bytes (6.55 MiB), ended 50,085,888 +bytes (47.77 MiB) above the RSS baseline, and built in 130.34 ms. That is 68.60× +less retained heap, 23.17× less final RSS, and a 26.82× faster cold build. + +The clean arms had identical source/lockfile/entry provenance within their +comparison, identical catalog fingerprints, and zero recorded errors, +mismatches, or cross-tenant tokens. The broader comparison produced an identical +17,976-byte GraphQL SDL with SHA-256 +`5fb82f96153815b23820a9ccf10322a20c864e49605ef5781cd33422b3b31020`. +These are performance-only results for one complete Graphile surface, not a +complete-customer density qualification. + +The small disposable PostgreSQL fixtures also proved stock-query byte +stability, bind-only schema names, fail-closed missing schemas, cross-schema +type/FK closure, safe partition direction, and byte-identical Constructive SDL. +The latest full-capability A/B/C fixture then passed all 56 hostile checks on a +fresh PostgreSQL 18 database with three distinct least-privilege logins and +realtime-resident instances. It recorded same-backend prepared reset for every +tenant, serialized cold builds, ten alternating connection rounds, and zero +cross-tenant tokens. The disposable container was stopped and removed; existing +local PostgreSQL containers were not modified. + +The new cperf package compiles and its current unit/in-process integration suite passes. It rejects the checked-in placeholder with a precise list of the missing 47 tenants, capabilities, and per-surface canaries, so a five-second smoke or a mislabeled `__typename` query cannot become a qualifying result. Each real run records request samples, canary results, Node/cache telemetry, coarse PostgreSQL container telemetry, server logs, and an immutable scored result. + +## Validation performed + +The final top-of-stack root build completed all 119 participating workspace +packages (of 120 total). The GraphQL server's CJS/ESM build passed and its full +suite passed 399 tests with two skips. +GraphQL environment passed 42 tests, request context passed 41, pg-cache passed +119 with five environment-gated skips, cperf passed 233, and the focused +Graphile settings security/capability suites passed 22. Separate live PostgreSQL +sanitizer and runtime-role suites passed two checks each. + +The complete `graphile-llm` suite additionally passed 53 tests and failed 11 +live-provider cases because no Ollama endpoint was available; the changed +discovery/RAG SQL tests pass, but the unavailable model capability remains an +external integration gate rather than being silently skipped. The full Graphile +settings suite has one credential-gated cross-database BM25 case that cannot use +the passwordless local default; the focused changed suites and the independent +A/B/C BM25 fixture pass. + +The final admission pass also found and fixed a CAPTCHA bypass: admission now +classifies the selected mutation from its AST rather than trusting a +client-controlled operation label, rejects malformed/ambiguous/batched requests, +parses all supported HTTP body formats before admission, rejects protected +WebSocket mutations, and fails closed on a missing production/strict secret. +The focused HTTP/WebSocket suite passed all 36 tests. Caller Graphile presets are +now denied by default in production; the focused composition/contract suites +passed all 21 tests. + +The root lint command is not a usable acceptance gate on current `origin/main` +because ESLint 9 cannot find a flat `eslint.config.*`; this predates the spike +and was not papered over locally. + +The lockfile contains only the changed `pg-introspection` patch hash and the new cperf workspace importer with its TypeScript toolchain snapshot. No unrelated lockfile normalization from the old PR stack was replayed. + +## Historical results are not current evidence + +#1335 documents approximately 14.7 MiB retained heap per instance, 417 ms cold builds, a 17 MiB PostgreSQL spike, an 87× reduction, and high same-blueprint tenant density. Those numbers were collected with the rejected SQL-rewrite/blueprint-pooling system and are not reproduced here. Its five-hour soak also recorded one inconclusive isolation canary; the refreshed gate requires zero inconclusive checks, so that run would not qualify under this spike's rules. + +The safe dedicated-instance candidate now beats the old retained-heap and cold +build targets on the production-shaped single-surface fixture: 6.55 MiB and +130.34 ms. That does not reproduce the old blueprint density claim because the +accepted design intentionally keeps tenant/API instances separate. Complete +customers per GiB still requires the multi-surface ramp and soak. + +## Gates still open + +- Introduce an atomic `TenantSecurityContractV1` revision spanning route, + exposure, role, feature, and auth policy. Carry it through runtime/build and + WebSocket contracts, reject mismatches, retire resident generations, and close + stale subscriptions. The present multi-statement reads allow bounded stale + HTTP access and potentially unbounded stale WebSocket authorization after a + handover/revocation. +- Prove the intended RLS/FORCE-RLS policy manifest, request roles, runtime login, + and dependency-schema object allowlist against disposable copies of the real + production tenant schemas. Add an authoritative per-API `authRequired` + contract so missing auth metadata cannot silently become anonymous access. +- Run the four fresh-process arms at 1/2/4 GiB, every ramp point, three repetitions, 15 minutes each, then the two-hour maximum-density churn soak. A candidate passes only if every heap/repetition ramp adds at least one complete tenant and median maximum density improves at least 15%. +- Obtain Graphile maintainer review of the source-level introspection API and dependency closure; remove the temporary package dist patches before production. +- Re-run throughput and p99 on the final security code; authoritative metadata + and role admission add real PostgreSQL work per request and may not be removed + to improve the benchmark. Complete multipart storage byte roundtrips and the + intended provider gates. +- Feed the governor a conservative validated instance-cost value from the final + complete-customer runs. The 6.55 MiB single-surface result is not automatically + a safe production capacity setting. + +## Reviewer checklist + +1. Can any routing label, hostname, or service key affect cache isolation? It should not; only the exact build contract hash can. +2. What happens when scoped introspection misses a configured schema? The build fails; it never retries stock. +3. Does RLS make a wrong physical schema safe? No; the design avoids rewrite routing and tests non-row objects separately. +4. Can a resident tenant count after an eviction, rebuild, build refusal, missing telemetry, or inconclusive canary? No. +5. Are the old 87× and 14.7 MiB figures current evidence? No. The current clean + result is 68.60× retained-heap improvement and 6.55 MiB for one dedicated + production-shaped surface; complete-customer density is still pending. +6. Can a committed handover/revocation invalidate every in-flight HTTP and + WebSocket operation? Not yet; this is the production-blocking revision gap. diff --git a/research/graphile-density/SECURITY-AUDIT.md b/research/graphile-density/SECURITY-AUDIT.md new file mode 100644 index 0000000000..2470ac8847 --- /dev/null +++ b/research/graphile-density/SECURITY-AUDIT.md @@ -0,0 +1,154 @@ +# Graphile tenant-density security audit + +Audit date: 2026-08-02. Current branch: +`research/graphile-density-06-measured-optimization`. + +## Verdict + +Zhi's concern is correct for the old blueprint-pooling design, so that design is +not part of this candidate. The executable candidate does not rewrite runtime +SQL and does not share one PostGraphile instance between tenants. It builds one +instance against each exact physical database, login, API, ordered schema set, +role set, plugin/settings contract, and surface configuration. Scoped +introspection reduces catalog input during schema construction; it is not a +routing or authorization mechanism. + +The current data plane passed the latest disposable PostgreSQL 18 A/B/C hostile +gate: 56 of 56 checks passed, including generated/plugin SQL, metadata, +functions, sequences, owner/BYPASS rejection, poisoned GUCs, rollback, +same-backend prepared-statement reuse, schema drift, cache invalidation, +serialized builds, realtime-resident instances, and ten alternating connection +reuse rounds with zero cross-tenant tokens. That is strong evidence for the +exact physical boundary, but it is not a production approval. + +Production remains blocked on an atomic, versioned route-and-security contract. +Today a request resolves the route and then reads RLS/auth/features through +separate statements. Exact database/API identities prevent this from becoming +an arbitrary A-to-B pool/cache alias, but a domain handover or revocation can +leave one in-flight HTTP request using the old tenant snapshot. An accepted +WebSocket can retain the old route/session indefinitely because operations and +subscriptions are not checked against a control-plane revision. This is a real +stale-authorization defect and must be fixed before production. + +## Claim-by-claim disposition + +| Concern | Finding | Current protection and remaining limit | +|---|---|---| +| SQL is rewritten | True only of rejected #1333/#1334; false in the current runtime. | The current path gives the exact runtime pool and physical schema names directly to `makePgService`. There is no canonical-schema substitution, rewrite pool, or blueprint pool in runtime source. | +| RLS cannot prove SQL routing | Correct. | The candidate does not use RLS to justify routing. Dedicated logins/pools, schema/object ACLs, exact build identities, and runtime role audits enforce the physical boundary. RLS remains row-level defense in depth. | +| A missed identifier or raw plugin SQL can hit another tenant | Correct for rewrite pooling. | There is no rewrite coverage list to miss. Built-in raw SQL paths were audited and run against exact tenant pools. Arbitrary plugins are unsandboxed trusted code, so production now rejects all caller presets/plugins unless explicitly admitted. An admitted plugin remains part of the process and database trusted computing base. | +| RLS does not protect metadata, functions, sequences, indexes, or privileged code | Correct. | Runtime admission rejects cross-schema relation, sequence, function, and type privileges; object ownership; `SUPERUSER`, `BYPASSRLS`, `CREATEROLE`, `CREATEDB`, and replication; unexpected inherited/`SET ROLE` paths; `SECURITY DEFINER`; owner-rights views; foreign/materialized views; and unsafe stored-expression dependencies. PostgreSQL catalog names are still generally visible to connected roles, so the claim is non-use/non-exposure through GraphQL, not catalog-name confidentiality. | +| Fail-closed behavior is required | Correct. | Missing/ambiguous route rows, database/API IDs, physical schemas, roles, feature contracts, runtime credentials, scoped schemas, unsafe roles, protected preset overrides, untrusted production plugins, and invalid internal headers all fail closed. The remaining fail-closed gap is coherent revisioning across route/security reads and long-lived WebSockets. | +| Prepared statements can cross tenant state | Correct in a shared pool. | Pools are split by an opaque HMAC over endpoint, TLS, database, login, password, driver, pool settings, purpose, and sanitation mode. Reused checkouts run `DISCARD ALL`, then clear node-postgres and Graphile prepared-statement bookkeeping; reset failure destroys the client. The real same-backend A/B/C test passed. | +| The fingerprint groups tenants into one Graphile instance | False for the current candidate. | The HMAC build contract includes opaque pool identity, database/API IDs, ordered schemas, roles, resolved plugins/settings, compute/storage bindings, surface flags, and introspection mode. Each exact tenant/API contract gets its own instance. The hash is a cache identity, not a code signature or sandbox. | +| More separate APIs improve reuse | False under the secure design. | API ID participates in the build contract, so a separate API intentionally gets a separate instance. Density comes from lowering each dedicated instance's retained memory and governing builds, not weakening isolation through reuse. | +| BM25 is skipped | False in the current candidate; true of the old rewrite design's compatibility exclusions. | BM25 stays enabled and binds the physical schema-qualified index name. The A/B/C fixture built and exercised BM25 together with tsvector, trigram, vector, PostGIS, and ltree. | +| GUC values can revert to an earlier tenant | `SET LOCAL` restores the prior value after transaction end, so the concern is correct. | Every request writes the full security-GUC allowlist, including empty values, plus role, read-only state, `row_security`, and a pinned `search_path`. Checkout sanitation removes any earlier session and prepared state first. | +| Plugins that reference RLS/schema objects may bypass the design | Plugins are trusted code, so the concern is correct in principle. | Built-ins receive the exact pool, physical schemas, and request `pgSettings`; metadata loaders use same-database/API joins and quoted identifiers. Production caller presets are denied by default. Any explicit opt-in requires pinned dependencies, code review, and requalification because a plugin can open its own connection or use process I/O. | + +## Evidence + +### Latest hostile A/B/C execution + +The final rerun used a disposable `constructiveio/postgres-plus:18` container, a +fresh database, three distinct `LOGIN NOINHERIT` roles, forced-RLS tenant tables, +denied cross-database/schema privileges, exact per-tenant pools, scoped-required +introspection, and realtime-resident instances. It completed in 3.9 seconds and +recorded 56 passing checks with `crossTenantTokens: 0`. The fixture is +deliberately marked `customerQualified: false`: it proves hostile isolation, not +the 15-minute workload/provider/density gates. + +Evidence artifact: +`complete-tenant-fixture/generated/hostile-validation.json`. + +Separate real PostgreSQL integration tests passed 2 of 2 checkout-sanitizer +checks and 2 of 2 runtime-role checks. Those tests deliberately reused one +backend after poisoning session/prepared state and created actual inherited +owner, `BYPASSRLS`, object-ownership, `SECURITY DEFINER`, and cross-schema +relation/sequence/function/type violations that admission had to reject. + +### Relevant source boundaries + +- `graphql/server/src/middleware/graphile.ts` resolves one exact runtime pool, + passes physical schemas to `makePgService`, audits the role boundary on every + resident request, and keys the resident instance by the exact build contract. +- `graphql/server/src/middleware/graphile-build-contract.ts` builds and HMACs the + exact contract. Function source and exact in-process identity participate, but + mutable closure state and supply-chain integrity cannot be attested by a hash. +- `graphql/server/src/middleware/runtime-pg-config.ts` accepts only explicit + credential data, matches the routed physical database and network/TLS target, + and keeps raw credentials in a request-keyed `WeakMap`. +- `graphql/server/src/middleware/runtime-role-safety.ts` performs the catalog + privilege, ownership, role-reachability, privileged-object, and stored + dependency audit. Successful-result reuse defaults to zero milliseconds. +- `postgres/pg-cache/src/pg.ts` defines exact pool identity, executes + `DISCARD ALL`, clears both prepared-statement caches, pins the safe baseline, + and destroys a client on sanitation failure. +- `packages/express-context/src/pg-settings.ts` initializes every security GUC, + role, read-only state, `row_security`, and the allowlisted search path for each + Graphile transaction. +- `graphql/server/src/middleware/graphile-preset-composition.ts` rejects + production caller plugins by default and prevents admitted presets from + replacing server-owned PostgreSQL services, request context, transport/error + policy, build-state policy, or protected plugins. This is admission, not a + sandbox. +- `research/graphile-density/PLUGIN-SQL-AUDIT.md` records the built-in plugin/raw + SQL review and the remaining deliberate system/build lanes. + +### Performance result retained under the secure architecture + +On the clean three-repetition 62,298-`pg_class` single-surface fixture, stock +introspection retained a median 449.42 MiB heap, ended 1,106.94 MiB above the +RSS baseline, and built in 3,495.95 ms. Scoped dependency introspection retained +6.55 MiB heap, ended 47.77 MiB above the RSS baseline, and built in 130.34 ms: +68.60 times less retained heap, 23.17 times less final RSS, and a 26.82 times +faster cold build. The compared schema was byte-equivalent and the recorded +operations had zero errors, mismatches, or cross-tenant tokens. + +These are performance-only instance measurements, not a complete-customer +tenants-per-GiB qualification. Security hardening added authoritative metadata +reads and role admission work, so final throughput/p99 must be remeasured; no +security check may be removed to recover a benchmark. + +## Production blockers + +1. **Atomic security contract.** Add `TenantSecurityContractV1` with an immutable + revision. Resolve route plus routing-plane security/exposure fields in one + parameterized read-only snapshot. Publish tenant-local auth settings under + that revision, then atomically activate it in routing; absence or mismatch + must fail closed. Carry the revision through `ApiStructure`, runtime resolver + input, the Graphile build contract, and WebSocket admission. +2. **Revocation semantics for long-lived transports.** Recheck + `(selector, apiId, databaseId, revision)` before every WebSocket operation, + retire the resident generation on mismatch, and terminate existing + subscriptions through event-driven invalidation plus an authoritative + fallback. Decide and document whether an HTTP request owns an admission + snapshot or must recheck immediately before execution. +3. **Production database policy proof.** Against disposable production-shaped + databases, verify the exact runtime login and both request roles for every + exposed/dependency object, and assert the intended RLS/policy/FORCE-RLS + manifest for shared-row tables. The fixture proves the mechanism, not every + deployed tenant schema. +4. **Explicit auth-required contract.** `strictAuth=false` permits an API with no + RLS module to proceed anonymously. Public/no-auth APIs may be intentional, so + production needs an authoritative per-API `authRequired` field rather than a + process-wide guess; missing required auth metadata must fail closed. +5. **Upstream introspection review.** `scoped-required` depends on source-level + Graphile patches and dependency-closure semantics. It must stay off by + default until Graphile maintainers review the isolated API and the patches + are replaced by supported upstream code. +6. **Operational trust boundary.** The internal-header secret must be stripped + at public ingress and carried only over authenticated encrypted service hops. + `X-Meta-Schema` is a deliberate cross-tenant administration capability, + disabled by default, and must use a separate private ingress and safe roles + if enabled. Runtime credential resolution and every explicitly admitted + plugin remain trusted code. +7. **Release qualification.** Rerun the 15-minute repeated complete-customer + density matrix and two-hour churn soak on the final security code, including + multipart upload/storage byte roundtrips and intended external providers. + +Until blockers 1–5 are closed, the production decision is **no-go**. The +dedicated-instance/scoped-introspection architecture remains the right candidate +because its measured memory gain does not depend on SQL rewrite or weakened +tenant isolation. + diff --git a/research/graphile-density/UNIFORM-DENSITY-FIXTURE.md b/research/graphile-density/UNIFORM-DENSITY-FIXTURE.md new file mode 100644 index 0000000000..3aeb78f817 --- /dev/null +++ b/research/graphile-density/UNIFORM-DENSITY-FIXTURE.md @@ -0,0 +1,63 @@ +# Uniform Graphile density fixture + +`graphile_density_uniform_20260801_a` is a performance-only routing-canary +fixture. It measures Graphile memory density across a uniform 4,000-tenant +catalog; it does not prove database-enforced tenant isolation or qualify a +complete customer surface. The shared `gd_runtime_20260801_a` login can read +every tenant schema by design. + +The fixture was physically cloned from `graphile_density_20260801_a` without +modifying or dropping the source. Tenants 401 through 4,000 were populated in +100-tenant transactions, and 7,920 complete disposable noise tables were +removed in 100-table transactions. The exact `pg_class` accounting is: + +| Catalog bucket | Rows | +| --- | ---: | +| 4,000 tenant schemas, seven direct classes each | 28,000 | +| Remaining `gd_noise` tables, sequences, and indexes | 10,094 | +| Tenant, noise, and system TOAST tables/indexes | 22,808 | +| System classes outside `pg_toast` | 337 | +| **Total** | **61,239** | + +Each tenant has two tables, two identity sequences, three indexes, four owned +TOAST classes, one stable `tenant_token()` function, and nine PostgreSQL 18 +constraints (including cataloged `NOT NULL` constraints). Objects are owned by +`postgres`; the runtime role has schema `USAGE`, table `SELECT`, sequence +`SELECT, USAGE`, and function `EXECUTE`, but no schema/database `CREATE` and no +table write privileges. + +## Reproduce locally + +Run with an already configured PostgreSQL administrator environment; neither +script contains credentials: + +```bash +psql -X -v ON_ERROR_STOP=1 -d postgres \ + -f research/graphile-density/create-uniform-density-fixture.sql + +psql -X -v ON_ERROR_STOP=1 -d postgres \ + -f research/graphile-density/validate-uniform-density-fixture.sql +``` + +The creator fails when the fixed target already exists and never drops a +database. Its explicit `-v resume=1` path is limited to an existing clone that +still passes the asserted 61,239-row heterogeneous source shape before any +tenant DDL runs. + +## Recorded validation + +The final standalone validation completed all 40 least-privilege runtime +batches and reported: + +```text +database_name: graphile_density_uniform_20260801_a +pg_class_count: 61239 +tenant_schema_count: 4000 +distinct_tenant_shapes: 1 +logical_pg_class_fingerprint ec670a0d19a77919732f544d54bb34c9 +tenant_shape_fingerprint: 91910068fdc30af0dc304390ee3a605a +``` + +The logical class fingerprint normalizes OID-derived TOAST names and excludes +volatile physical statistics, so it records catalog shape rather than clone +OID allocation or post-benchmark `ANALYZE` state. diff --git a/research/graphile-density/UPSTREAM-REVIEW.md b/research/graphile-density/UPSTREAM-REVIEW.md new file mode 100644 index 0000000000..5b37372f9e --- /dev/null +++ b/research/graphile-density/UPSTREAM-REVIEW.md @@ -0,0 +1,51 @@ +# Upstream review packet: schema-scoped PostgreSQL introspection + +No upstream contact has been made. Production use of `scoped-required` remains blocked on Graphile maintainer review. + +## Proposed API + +Keep `makeIntrospectionQuery(): string` byte-for-byte unchanged and add: + +```ts +makeSchemaScopedIntrospectionQuery( + schemas: readonly string[] +): { text: string; values: [string[]] } +``` + +The requested names are carried only in `$1::text[]`. Empty, NUL-containing, `pg_*`, and `information_schema` scopes are rejected before SQL execution. The Graphile service option is `introspectionMode: 'stock' | 'scoped-required'`; scoped mode parses the result normally and then asserts that every requested service schema was found. There is no fallback to stock. + +## Dependency closure + +The recursive namespace graph follows dependencies required to parse selected objects: + +- foreign-key source relation → referenced relation; +- relation attribute → attribute type; +- function namespace → argument, OUT-argument, and return types; +- type → base, element, and array types; +- range type → subtype; +- inheritance child → parent. + +`pg_catalog` is returned in the namespace payload and its types remain available as in stock introspection. Inheritance deliberately does not follow parent → child: stock introspection emits `pg_inherits` rows only when the child class is selected, and reverse closure pulled a shared partition parent into unrelated tenant child schemas. The disposable partition regression preserved byte-identical SDL while returning only `density_shared` and `pg_catalog`. + +## Local evidence + +- Stock query stability: 7,332 bytes, SHA-256 `c0ed817b912f78e1ea68c70d89ff4b7f9cb4c02d88112a69ac4109d5b996e4c5`. +- Cross-schema FK/domain/enum/function fixture: stock and scoped Constructive SDL are both 22,538 bytes with SHA-256 `2d899a6f9abcea107987a0aa932f18dd8d1466bca3f4ddd340199316aea1f238`. +- Partition-parent fixture: stock and scoped SDL are both 20,201 bytes with SHA-256 `8f32cab18fa54c8d4c6afa7ab05bed73be770f98c29aa9b1bcd4f29e6f54d532`; hidden tenant child schemas are absent from scoped introspection. +- A missing requested schema fails with `Schema-scoped introspection for service 'main' did not find required schema(s): density_missing`. + +The exact evidence is in `artifacts/scoped-introspection-smoke.json`. These are small PostgreSQL 18.4 fixtures, not the required PostgreSQL 17+/61k-catalog benchmark. + +## Review questions + +1. Is namespace closure the right upstream seam, or should filtering occur by object OID after the stock query generator has expressed every catalog dependency? +2. Which additional OID dependencies must be closed for extensions, composite/domain types, cross-schema defaults/sequences, partitioned tables, procedures, policies, and future introspection versions? +3. Should referenced schemas be included in raw introspection but remain absent from the configured GraphQL surface, as this candidate does? +4. Should missing configured schemas fail in the gather layer, and how should watch-mode schema creation/deletion invalidate a cached failed gather? +5. Can the option live on `PgServiceConfiguration`, and should it take explicit schema names, a callback, or an upstream-defined scope object? +6. What PostgreSQL versions and extension catalogs should upstream CI cover, especially PostgreSQL 17+, PostGIS, vector, BM25, ltree, and partitioning? +7. Can upstream source generate both stock and scoped queries so Constructive can remove its temporary dist patches? + +## Requested upstream tests + +The upstream change should preserve the stock query byte, compare parsed introspection and emitted GraphQL SDL for the dependency cases above, prove bind-only schema names including quotes, fail on missing schemas, and benchmark catalog rows returned, PostgreSQL peak memory, cold-build time, and retained Node heap at the 61k catalog. Constructive's full plugin and hostile tenant matrix remains a separate downstream responsibility. diff --git a/research/graphile-density/artifacts/scoped-introspection-smoke.json b/research/graphile-density/artifacts/scoped-introspection-smoke.json new file mode 100644 index 0000000000..429fe85da7 --- /dev/null +++ b/research/graphile-density/artifacts/scoped-introspection-smoke.json @@ -0,0 +1,73 @@ +{ + "schemaVersion": 1, + "recordedAt": "2026-07-31T17:21:37Z", + "classification": "small disposable-fixture smoke; not a density benchmark or production acceptance run", + "environment": { + "postgresVersion": "18.4", + "database": "codex_graphile_density_20260731_rerun", + "databaseDroppedAfterRun": true, + "constructiveDbUsed": false + }, + "catalogFixture": { + "requestedSchemas": ["density_a"], + "features": [ + "cross-schema foreign key", + "shared enum", + "shared domain", + "identity sequences", + "SQL function" + ], + "stockQuery": { + "bytes": 7332, + "sha256": "c0ed817b912f78e1ea68c70d89ff4b7f9cb4c02d88112a69ac4109d5b996e4c5" + }, + "stockIntrospection": { + "bytes": 404110, + "namespaces": 6, + "classes": 8, + "procedures": 1, + "types": 481 + }, + "scopedIntrospection": { + "bytes": 403639, + "namespaces": 4, + "classes": 8, + "procedures": 1, + "types": 481, + "bindValues": [["density_a"]] + }, + "stockSdl": { + "bytes": 22538, + "sha256": "2d899a6f9abcea107987a0aa932f18dd8d1466bca3f4ddd340199316aea1f238" + }, + "scopedSdl": { + "bytes": 22538, + "sha256": "2d899a6f9abcea107987a0aa932f18dd8d1466bca3f4ddd340199316aea1f238" + }, + "sdlByteEquivalent": true, + "missingSchemaError": "Schema-scoped introspection for service 'main' did not find required schema(s): density_missing" + }, + "partitionFixture": { + "database": "codex_graphile_density_partition_20260731", + "databaseDroppedAfterRun": true, + "requestedSchema": "density_shared", + "hiddenChildSchemas": ["density_a", "density_b"], + "scopedNamespaces": ["density_shared", "pg_catalog"], + "scopedClasses": ["events", "events_pkey"], + "stockSdl": { + "bytes": 20201, + "sha256": "8f32cab18fa54c8d4c6afa7ab05bed73be770f98c29aa9b1bcd4f29e6f54d532" + }, + "scopedSdl": { + "bytes": 20201, + "sha256": "8f32cab18fa54c8d4c6afa7ab05bed73be770f98c29aa9b1bcd4f29e6f54d532" + }, + "sdlByteEquivalent": true, + "finding": "child-to-parent inheritance closure is required; parent-to-child closure over-expands into unrelated tenant partitions and was removed" + }, + "limitations": [ + "The catalog is tiny, so the byte reduction is not representative of the 61k-pg_class target.", + "This does not exercise RLS, runtime credentials, hostile tenants, every plugin, throughput, RSS density, or soak behavior.", + "The first fixture's forced database cleanup emitted expected 57P01 messages from adaptor clients after schema release; the database was confirmed absent afterward. The partition rerun used explicit pools and drained cleanly." + ] +} diff --git a/research/graphile-density/complete-tenant-fixture/.gitignore b/research/graphile-density/complete-tenant-fixture/.gitignore new file mode 100644 index 0000000000..df5372de97 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/.gitignore @@ -0,0 +1,3 @@ +/artifacts/ +/generated/ +/qualification-artifacts/ diff --git a/research/graphile-density/complete-tenant-fixture/README.md b/research/graphile-density/complete-tenant-fixture/README.md new file mode 100644 index 0000000000..2d6b52b526 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/README.md @@ -0,0 +1,192 @@ +# Complete-tenant A/B/C research fixture + +This fixture is the correctness half of the tenant-density spike. The uniform +catalog fixture finds the memory-capacity curve; this fixture asks whether a +candidate can keep a small, production-shaped A/B/C fleet complete and isolated. +Three tenants are too few to establish tenants per GiB, so no performance claim +may be derived from this fixture alone. + +The A/B/C database and generated GraphQL operation names were validated end to +end on 2026-08-02 against a disposable PostgreSQL 18 `postgres-plus` instance. +The latest-source hostile run passed all 56 checks, including all declared +capability operations, realtime-resident instances, same-backend sanitation, +schema drift/rebuild, and alternating connection reuse with zero cross-tenant +tokens. Its local evidence is `generated/hostile-validation.json`. Unsupported +or renamed fields still fail the run; nothing is silently skipped. This is an +offline hostile-isolation result, not complete-customer density or production +provider qualification. + +## Isolation model + +`schema.sql` creates three physical tenant schemas (`ctf_a`, `ctf_b`, `ctf_c`) +with identical objects, forced RLS policies, and distinct canary values. It +requires three distinct `LOGIN NOINHERIT` runtime roles. Each role receives +USAGE and object privileges for exactly one tenant schema, while shared access is +limited to the audited `ctf_extensions` and `jwt_private` dependency schemas. +Configured request roles must not reach parent roles through either `INHERIT` +or `SET`; the startup audit evaluates each request role as a separate execution +root, so a privilege or ownership path that appears only after `SET ROLE` fails +closed. +Each role also receives USAGE and EXECUTE on its own `ctf__realtime` cursor +schema, with PUBLIC and both foreign tenant roles denied. The fixture setup +asserts that ACL matrix before it succeeds. + +`server.cjs` creates one PostGraphile instance and one dedicated runtime pool +for each tenant. The default non-realtime lane uses `max=1`; realtime fails +closed unless the pool has at least two slots because its cursor manager keeps +one client resident. A live build contract includes the exact credential-sensitive, +process-keyed pool identity and physical schema, so host labels cannot alias cache +entries. Cross-process evidence uses a separate deterministic credential-free +contract fingerprint and proves its live role/database/schema mapping at runtime. +Runtime checkouts use `DISCARD ALL`; each request then sets the complete security-GUC +allowlist, role, read-only state, RLS state, and pinned search path. The hostile +probe deliberately reuses a named statement with different SQL after checkout +to verify both PostgreSQL and node-postgres prepared-statement bookkeeping were +cleared. + +Schema drift is deliberately outside the runtime boundary. The runtime roles +have no USAGE or EXECUTE access to `ctf_control`; the loopback control endpoint +uses the separately configured control-plane `PG*` login and a random in-memory +token. RLS remains defense in depth for rows, while schema ACLs and dedicated +logins enforce the physical routing boundary. + +The metadata canary covers GraphQL schema and introspection isolation. It does +not claim that tenant schema names are confidential inside PostgreSQL: +system-catalog object names are generally visible to connected roles, and +hiding those names would require a stronger database/process boundary. The +security claim here is that another tenant's objects cannot be used or exposed +through the GraphQL build, not that their catalog names cannot be observed. + +## What the offline lane covers + +The candidate fleet includes generated CRUD/function plans, i18n, deterministic +LLM/RAG, BM25, tsvector, trigram, pgvector, PostGIS, ltree, presigned-upload +metadata/signing, bulk mutations, realtime-tagged writes, and preloaded function +bindings. BM25 stays enabled because every instance compiles against its real +physical schema; there is no SQL schema rewrite. + +`hostile-validation.cjs` checks every declared canary plus dynamic session +poisoning, savepoint rollback, prepared-statement reset, schema drift and cache +invalidation, serialized cold builds, and alternating A/B/C connection reuse. +Every dynamic identity and authenticated control response must also return the +caller-supplied `current_database()` identity, and every tenant runs a negative +role-safety probe that must reject the control-plane role. +The exact roles and pools prevent cross-tenant session reuse by construction; +the reuse checks prove sanitation within each tenant pool and distinct build +contracts prove that pools cannot alias. Any unavailable or inconclusive probe +exits non-zero. + +Realtime mutations exercise the tagged database write and NOTIFY trigger. With +`--enable-realtime`, every cached instance keeps a cursor manager resident +against its exact tenant cursor schema and exposes a no-server Grafserv upgrade +handler. The outer fixture server selects that handler only after an exact +tenant path match, and disposal terminates that generation's long-lived sockets +before releasing its pool. The physical-density wrapper keeps one +`graphql-transport-ws` subscription resident per surface and requires a real, +tenant-specific event before the surface can count. + +## External-provider boundary + +The exact fixture currently injects a deterministic LLM and uses a signing-only +S3 client. Those paths exercise plugin and database integration but prove +neither model semantics nor an object-storage byte roundtrip. Consequently: + +- `--class offline-research` may pass local gates but always records + `customerQualified: false`. +- `--class production` currently fails with + `CTF_PRODUCTION_EQUIVALENCE_NOT_IMPLEMENTED`, even when provider arguments are + supplied. Missing arguments fail earlier with + `CTF_EXTERNAL_PROVIDER_GATES_UNSATISFIED`. +- Production support requires wiring and testing the intended Ollama-compatible + models plus disposable S3/MinIO PUT, HEAD/GET, and cleanup paths. The manifest + requires `--ollama-url`, `--embedding-model`, `--chat-model`, `--s3-endpoint`, + and `--s3-bucket`; provider credentials stay in the environment and never in + artifacts. + +## Disposable local setup + +Create the three runtime logins separately under an administrator. They must be +distinct, `LOGIN NOINHERIT`, and must not be superuser, `BYPASSRLS`, +`CREATEROLE`, `CREATEDB`, replication, a tenant-schema owner, or able to CREATE +in a tenant schema. The fixture intentionally does not create or alter roles. + +```bash +createdb graphile_complete_tenant_spike +psql --set=ON_ERROR_STOP=1 \ + --set=runtime_role_a=ctf_runtime_a \ + --set=runtime_role_b=ctf_runtime_b \ + --set=runtime_role_c=ctf_runtime_c \ + --dbname=graphile_complete_tenant_spike \ + --file=research/graphile-density/complete-tenant-fixture/schema.sql +``` + +Keep the ordinary `PGHOST`, `PGPORT`, `PGDATABASE`, `PGUSER`, and `PGPASSWORD` +pointed at the fixture owner/control login. Supply runtime credentials only in +`CTF_RUNTIME_A_PGPASSWORD`, `CTF_RUNTIME_B_PGPASSWORD`, and +`CTF_RUNTIME_C_PGPASSWORD`; `GRAPHQL_RUNTIME_PGPASSWORD` is an optional shared +password fallback. Role names are non-secret command arguments. + +Start the candidate and run the hostile gate with a control token of at least 32 +bytes: + +```bash +export CTF_CONTROL_TOKEN="$(openssl rand -hex 32)" +export GRAPHILE_CACHE_MAX=3 PG_CACHE_MAX=4 PG_POOL_MAX=1 PG_POOL_MAX_USES=0 +export DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE=100 + +node research/graphile-density/complete-tenant-fixture/server.cjs \ + --port 3391 --arm local-complete-tenant --mode scoped-required \ + --runtime-role-a ctf_runtime_a \ + --runtime-role-b ctf_runtime_b \ + --runtime-role-c ctf_runtime_c + +node research/graphile-density/complete-tenant-fixture/hostile-validation.cjs \ + --base-url http://127.0.0.1:3391 \ + --expected-physical-database-identity graphile_complete_tenant_spike \ + --arm local-complete-tenant --mode scoped-required +``` + +For the realtime-resident lane, add `--enable-realtime true +--runtime-pool-max 2`, keep process-global `PG_POOL_MAX=1`, and set +`PG_CACHE_MAX` high enough for the three dedicated runtime identities plus the +control identity. Runtime capacity is explicit per pool and must not leak into +the control-plane baseline. + +Generate credential-free cperf inputs only after the server reports exact, +unique `graphile:v1:` contracts: + +```bash +node research/graphile-density/complete-tenant-fixture/generate-inputs.cjs \ + --port 3391 --postgres-container postgres \ + --runtime-role-a ctf_runtime_a \ + --runtime-role-b ctf_runtime_b \ + --runtime-role-c ctf_runtime_c +``` + +The one-command research gate is explicitly offline and runs three 15-minute +repetitions at a 4-GiB V8 old-space setting, followed by the mandatory repository +suites. It is a completeness gate, not a capacity search: + +```bash +node research/graphile-density/complete-tenant-fixture/qualification-runner.cjs \ + --class offline-research \ + --postgres-container postgres \ + --runtime-role-a ctf_runtime_a \ + --runtime-role-b ctf_runtime_b \ + --runtime-role-c ctf_runtime_c +``` + +The perf harness rejects a dirty server provenance. Commit the local research +branches and confirm `git status --short` is empty before a qualifying run; the +fixture ignores only its generated inputs and run-artifact directories so those +outputs do not invalidate provenance. No commit or push is performed by these +scripts. + +Build the affected packages before starting the fixture. Its exact runtime +`dist` artifact fingerprint is part of every Graphile build contract and the +generated benchmark provenance; a stale build-contract API fails closed instead +of silently measuring old code. + +Before using any result, inspect `qualification.json`. The offline lane is valid +only when `localPassed` is true, and `customerQualified` must remain false until +the provider-backed production runner exists and passes. diff --git a/research/graphile-density/complete-tenant-fixture/coverage-manifest.json b/research/graphile-density/complete-tenant-fixture/coverage-manifest.json new file mode 100644 index 0000000000..fe6e4cce09 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/coverage-manifest.json @@ -0,0 +1,78 @@ +{ + "version": 1, + "fixture": "complete-tenant-abc-v1", + "qualificationClass": "offline-research-candidate", + "tenants": ["a", "b", "c"], + "surfacesPerTenant": ["api"], + "performanceClaimsAllowed": false, + "productionQualificationImplemented": false, + "runtimeIsolation": { + "model": "dedicated-login-and-pool-per-tenant", + "sharedRuntimePool": false, + "runtimeSchemaDriftControl": false, + "checkoutSanitationRequired": true + }, + "localCapabilities": [ + { "id": "graphile-generated", "status": "candidate-runtime-required", "evidence": "candidate connection and primary-key operations must compile and execute for A/B/C" }, + { "id": "i18n", "status": "candidate-runtime-required", "evidence": "candidate localeStrings operation must execute the parameterized i18n SQL path" }, + { "id": "llm-deterministic", "status": "offline-only", "evidence": "candidate embedText operation uses an injected deterministic 3-D provider" }, + { "id": "rag-deterministic", "status": "offline-only", "evidence": "candidate ragQuery operation uses deterministic embedding/chat plus pgvector chunk SQL" }, + { "id": "bm25", "status": "candidate-runtime-required", "evidence": "candidate BM25 filter and score operation; BM25 remains enabled" }, + { "id": "tsvector", "status": "candidate-runtime-required", "evidence": "candidate tsvector filter and rank operation" }, + { "id": "trigram", "status": "candidate-runtime-required", "evidence": "candidate pg_trgm filter and similarity operation" }, + { "id": "vector", "status": "candidate-runtime-required", "evidence": "candidate pgvector cosine filter and distance operation" }, + { "id": "postgis", "status": "candidate-runtime-required", "evidence": "candidate PostGIS codec operation using schema-qualified extension SQL" }, + { "id": "ltree", "status": "candidate-runtime-required", "evidence": "candidate ltree filter using schema-qualified operator and cast SQL" }, + { "id": "uploads-storage-presign-only", "status": "offline-only", "evidence": "candidate metadata insert and offline SigV4 signing path; no object bytes are transferred" }, + { "id": "bulk-mutations", "status": "candidate-runtime-required", "evidence": "candidate bulk upsert operation" }, + { "id": "realtime-tagged-write", "status": "candidate-runtime-required", "evidence": "candidate generated update must fire the fixture NOTIFY trigger; delivery is a separate mandatory suite" }, + { "id": "function-bindings", "status": "candidate-runtime-required", "evidence": "candidate preloaded binding must insert an RLS-visible invocation" }, + { "id": "security-session", "status": "candidate-runtime-required", "evidence": "protected controls poison and roll back state; the next exact-tenant checkout must sanitize and initialize every request GUC" } + ], + "hostileCanaries": [ + "cross-schema-identifiers", + "metadata", + "functions", + "sequences", + "prepared-statement-reuse", + "poisoned-gucs", + "rollback-savepoints", + "plugin-raw-sql", + "owner-bypass-role", + "schema-drift", + "cache-invalidation", + "concurrent-builds", + "connection-reuse" + ], + "metadataBoundary": "GraphQL schema/introspection isolation only; PostgreSQL system catalogs may reveal object names to any connected login", + "mandatoryRepositorySuites": [ + { + "id": "realtime-websocket-delivery", + "cwd": "graphile/graphile-realtime-test", + "command": ["pnpm", "exec", "jest", "--runInBand", "__tests__/realtime-websocket.integration.test.ts"] + }, + { + "id": "plugin-capability-closure", + "cwd": "graphile/graphile-settings", + "command": ["pnpm", "exec", "jest", "--runInBand", "__tests__/scoped-introspection-capability-closure.integration.test.ts"] + } + ], + "externalProviderGates": [ + { + "id": "ollama-real-semantic", + "status": "blocking", + "requiredArguments": ["ollama-url", "embedding-model", "chat-model"], + "offlineSurrogate": ["llm-deterministic", "rag-deterministic"] + }, + { + "id": "object-storage-byte-roundtrip", + "status": "blocking", + "requiredArguments": ["s3-endpoint", "s3-bucket"], + "offlineSurrogate": ["uploads-storage-presign-only"], + "repositorySuite": { + "cwd": "graphql/server-test", + "command": ["pnpm", "exec", "jest", "--runInBand", "__tests__/upload.integration.test.ts"] + } + } + ] +} diff --git a/research/graphile-density/complete-tenant-fixture/generate-inputs.cjs b/research/graphile-density/complete-tenant-fixture/generate-inputs.cjs new file mode 100644 index 0000000000..b872a4ca41 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/generate-inputs.cjs @@ -0,0 +1,260 @@ +'use strict'; + +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + FIXTURE_DIR, + REPO_ROOT, + TENANTS, + assertCredentialFree, + assertLoopbackBaseUrl, + makeFleet, + makePlan, + parseArgs, + parsePositiveInteger, + requireString, + validateIntrospectionClientReleaseMode, +} = require('./lib.cjs'); + +const fetchJson = async (url, fetchImpl = fetch) => { + const response = await fetchImpl(url); + if (!response.ok) throw new Error(`CTF_STATUS_HTTP_${response.status}`); + return response.json(); +}; + +const validateServerStatus = ( + status, + { arm, mode, introspectionClientReleaseMode = 'destroy' }, +) => { + validateIntrospectionClientReleaseMode(introspectionClientReleaseMode); + if (status?.version !== 1 || status?.fixture !== 'complete-tenant-abc-v1') { + throw new Error('CTF_SERVER_STATUS_IDENTITY_MISMATCH'); + } + if (status.arm !== arm) throw new Error(`CTF_SERVER_ARM_MISMATCH:${status.arm}`); + if (status.introspectionMode !== mode) { + throw new Error(`CTF_SERVER_MODE_MISMATCH:${status.introspectionMode}`); + } + if (status.introspectionClientReleaseMode !== introspectionClientReleaseMode) { + throw new Error( + 'CTF_SERVER_INTROSPECTION_CLIENT_RELEASE_MODE_MISMATCH:' + + `${status.introspectionClientReleaseMode ?? 'missing'}` + ); + } + if (status.releaseBuildStateAfterValidation !== true) { + throw new Error('CTF_SERVER_BUILD_STATE_RETIREMENT_REQUIRED'); + } + if ( + status.physicalIsolation !== 'dedicated-login-and-pool-per-tenant' + || status.sharedRuntimePool !== false + || status.runtimeSafety?.passed !== true + || status.runtimeSafety?.rolesDistinct !== true + ) { + throw new Error('CTF_SERVER_RUNTIME_BOUNDARY_UNSAFE'); + } + if (!/^sha256:[0-9a-f]{64}$/.test(status.runtimeArtifactFingerprint ?? '')) { + throw new Error('CTF_SERVER_RUNTIME_FINGERPRINT_INVALID'); + } + if ( + status.liveIdentityScope !== 'process-local-keyed-hmac-v1' + || !/^graphile-configuration:ctf:v1:[a-f0-9]{64}$/.test( + status.configurationIdentity ?? '' + ) + ) { + throw new Error('CTF_SERVER_CONFIGURATION_IDENTITY_INVALID'); + } + const contracts = status.buildContracts; + if (!contracts || typeof contracts !== 'object') { + throw new Error('CTF_SERVER_CONTRACTS_MISSING'); + } + const values = TENANTS.map((tenant) => contracts[tenant.id]); + if (values.some((value) => typeof value !== 'string' || !value.startsWith('graphile:v1:'))) { + throw new Error('CTF_SERVER_CONTRACT_INVALID'); + } + if (new Set(values).size !== TENANTS.length) { + throw new Error('CTF_SERVER_CONTRACT_COLLISION'); + } + if (typeof status.physicalDatabase !== 'string' || !status.physicalDatabase.trim()) { + throw new Error('CTF_SERVER_PHYSICAL_DATABASE_MISSING'); + } + const runtimePoolIdentities = status.runtimePoolIdentities; + const poolValues = TENANTS.map((tenant) => runtimePoolIdentities?.[tenant.id]); + if (poolValues.some((value) => + typeof value !== 'string' || !/^pg:v1:[a-f0-9]{64}$/i.test(value) + )) { + throw new Error('CTF_SERVER_POOL_IDENTITY_INVALID'); + } + if (new Set(poolValues).size !== TENANTS.length) { + throw new Error('CTF_SERVER_POOL_IDENTITY_COLLISION'); + } + const evidence = status.contractEvidence; + if ( + evidence?.version !== 1 + || evidence.credentialFree !== true + || evidence.configurationIdentity !== status.configurationIdentity + ) { + throw new Error('CTF_SERVER_CONTRACT_EVIDENCE_INVALID'); + } + assertCredentialFree(evidence); + for (const tenant of TENANTS) { + const pool = evidence.runtimePools?.[tenant.id]; + const build = evidence.graphileBuilds?.[tenant.id]; + const binding = status.runtimeBindings?.[tenant.id]; + if ( + !/^pg-contract-evidence:v1:[a-f0-9]{64}$/.test(pool?.fingerprint ?? '') + || !/^graphile-contract-evidence:v1:[a-f0-9]{64}$/.test( + build?.fingerprint ?? '' + ) + || pool?.input?.databaseName !== status.physicalDatabase + || pool?.input?.role !== binding?.role + || binding?.databaseId !== tenant.databaseId + || binding?.databaseName !== status.physicalDatabase + || JSON.stringify(binding?.schemas) !== JSON.stringify([tenant.schema]) + ) { + throw new Error(`CTF_SERVER_CONTRACT_EVIDENCE_INVALID:${tenant.id}`); + } + } + return Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + evidence.graphileBuilds[tenant.id].fingerprint, + ])); +}; + +const atomicWriteJson = (file, value) => { + const serialized = `${JSON.stringify(value, null, 2)}\n`; + assertCredentialFree(value); + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); + const temporary = `${file}.${process.pid}.tmp`; + fs.writeFileSync(temporary, serialized, { encoding: 'utf8', mode: 0o600 }); + fs.renameSync(temporary, file); +}; + +const currentCommit = () => execFileSync( + 'git', + ['rev-parse', 'HEAD'], + { cwd: REPO_ROOT, encoding: 'utf8' }, +).trim(); + +const validateGeneratedInputs = (planFile, fleetFile) => { + const perfConfigPath = path.join(REPO_ROOT, 'packages/perf-harness/dist/config.js'); + if (!fs.existsSync(perfConfigPath)) { + throw new Error('CTF_BUILD_ARTIFACT_MISSING:packages/perf-harness/dist/config.js'); + } + const { loadFleet, loadPlan, validateCoverage } = require(perfConfigPath); + const plan = loadPlan(planFile); + const fleet = loadFleet(fleetFile); + validateCoverage(plan, fleet); +}; + +const generateInputs = async ({ + arm = 'local-complete-tenant', + mode = 'scoped-required', + introspectionClientReleaseMode = 'destroy', + port = 3391, + baseUrl = `http://127.0.0.1:${port}`, + postgresContainer, + runtimeRoles, + durationSec = 900, + outputDir = path.join(FIXTURE_DIR, 'generated'), + commit = currentCommit(), + fetchImpl = fetch, + validate = true, +} = {}) => { + if (!postgresContainer) throw new Error('CTF_ARGUMENT_REQUIRED:postgres-container'); + if (!['stock', 'scoped-required'].includes(mode)) { + throw new Error(`CTF_INTROSPECTION_MODE_INVALID:${mode}`); + } + validateIntrospectionClientReleaseMode(introspectionClientReleaseMode); + const localBaseUrl = assertLoopbackBaseUrl(baseUrl); + const status = await fetchJson(`${localBaseUrl}/__ctf/status`, fetchImpl); + const buildContracts = validateServerStatus(status, { + arm, + mode, + introspectionClientReleaseMode, + }); + const fleet = makeFleet({ + arm, + port, + buildContracts, + runtimePoolIdentities: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + status.contractEvidence.runtimePools[tenant.id].fingerprint, + ])), + physicalDatabase: status.physicalDatabase, + }); + const plan = makePlan({ + arm, + port, + postgresContainer, + commit, + durationSec, + cwd: REPO_ROOT, + introspectionMode: mode, + introspectionClientReleaseMode, + runtimeRoles, + }); + assertCredentialFree(fleet); + assertCredentialFree(plan); + fs.mkdirSync(outputDir, { recursive: true, mode: 0o700 }); + const fleetFile = path.join(outputDir, 'fleet.json'); + const planFile = path.join(outputDir, 'plan.json'); + atomicWriteJson(fleetFile, fleet); + atomicWriteJson(planFile, plan); + if (validate) validateGeneratedInputs(planFile, fleetFile); + const provenance = { + version: 1, + generatedAt: new Date().toISOString(), + arm, + mode, + introspectionClientReleaseMode, + commit, + customerQualified: false, + reason: 'inputs-only; workload and external provider gates have not run', + files: { + fleet: path.relative(REPO_ROOT, fleetFile), + plan: path.relative(REPO_ROOT, planFile), + }, + }; + atomicWriteJson(path.join(outputDir, 'generation.json'), provenance); + return { fleetFile, planFile, provenance }; +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + const port = parsePositiveInteger(args.port ?? '3391', 'port'); + const runtimeRoles = Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + requireString(args, tenant.runtimeRoleArgument), + ])); + const result = await generateInputs({ + arm: requireString(args, 'arm', 'local-complete-tenant'), + mode: requireString(args, 'mode', 'scoped-required'), + introspectionClientReleaseMode: requireString( + args, + 'introspection-client-release-mode', + 'destroy', + ), + port, + baseUrl: requireString(args, 'base-url', `http://127.0.0.1:${port}`), + postgresContainer: requireString(args, 'postgres-container'), + runtimeRoles, + durationSec: parsePositiveInteger(args['duration-sec'] ?? '900', 'duration-sec'), + outputDir: path.resolve(requireString(args, 'output-dir', path.join(FIXTURE_DIR, 'generated'))), + }); + process.stdout.write(`${JSON.stringify(result.provenance)}\n`); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + atomicWriteJson, + generateInputs, + validateGeneratedInputs, + validateServerStatus, +}; diff --git a/research/graphile-density/complete-tenant-fixture/generate-inputs.test.cjs b/research/graphile-density/complete-tenant-fixture/generate-inputs.test.cjs new file mode 100644 index 0000000000..3bf06ecb39 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/generate-inputs.test.cjs @@ -0,0 +1,216 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); + +const { TENANTS } = require('./lib.cjs'); +const { + atomicWriteJson, + generateInputs, + validateServerStatus, +} = require('./generate-inputs.cjs'); + +const contracts = () => Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `graphile:v1:${tenant.id.repeat(64)}`, +])); + +const poolIdentities = () => Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `pg:v1:${tenant.id.repeat(64)}`, +])); + +const configurationIdentity = + `graphile-configuration:ctf:v1:${'e'.repeat(64)}`; + +const runtimeBindings = () => Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + databaseId: tenant.databaseId, + databaseName: 'graphile_complete_tenant_spike', + role: `ctf_runtime_${tenant.id}`, + schemas: [tenant.schema], + }, +])); + +const contractEvidence = () => ({ + version: 1, + credentialFree: true, + configurationIdentity, + realtimeListener: null, + runtimePools: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + version: 1, + fingerprint: `pg-contract-evidence:v1:${tenant.id.repeat(64)}`, + input: { + databaseName: 'graphile_complete_tenant_spike', + role: `ctf_runtime_${tenant.id}`, + }, + }, + ])), + graphileBuilds: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + version: 1, + fingerprint: `graphile-contract-evidence:v1:${tenant.id.repeat(64)}`, + input: {}, + }, + ])), + residentGraphileBuildFingerprints: [], +}); + +const evidenceContracts = () => Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `graphile-contract-evidence:v1:${tenant.id.repeat(64)}`, +])); + +const evidencePoolIdentities = () => Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `pg-contract-evidence:v1:${tenant.id.repeat(64)}`, +])); + +const status = (overrides = {}) => ({ + version: 1, + fixture: 'complete-tenant-abc-v1', + arm: 'fixture-arm', + introspectionMode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: true, + physicalIsolation: 'dedicated-login-and-pool-per-tenant', + sharedRuntimePool: false, + runtimeSafety: { passed: true, rolesDistinct: true }, + runtimeArtifactFingerprint: `sha256:${'f'.repeat(64)}`, + configurationIdentity, + liveIdentityScope: 'process-local-keyed-hmac-v1', + physicalDatabase: 'graphile_complete_tenant_spike', + runtimePoolIdentities: poolIdentities(), + runtimeBindings: runtimeBindings(), + buildContracts: contracts(), + contractEvidence: contractEvidence(), + ...overrides, +}); + +test('status validation requires strict physical isolation and unique exact contracts', () => { + assert.deepEqual( + validateServerStatus(status(), { arm: 'fixture-arm', mode: 'scoped-required' }), + evidenceContracts(), + ); + assert.throws( + () => validateServerStatus(status({ introspectionMode: 'stock' }), { + arm: 'fixture-arm', + mode: 'scoped-required', + }), + /CTF_SERVER_MODE_MISMATCH:stock/, + ); + assert.throws( + () => validateServerStatus(status({ introspectionClientReleaseMode: 'reuse' }), { + arm: 'fixture-arm', + mode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + }), + /CTF_SERVER_INTROSPECTION_CLIENT_RELEASE_MODE_MISMATCH:reuse/, + ); + assert.throws( + () => validateServerStatus(status({ sharedRuntimePool: true }), { + arm: 'fixture-arm', + mode: 'scoped-required', + }), + /CTF_SERVER_RUNTIME_BOUNDARY_UNSAFE/, + ); + assert.throws( + () => validateServerStatus(status({ releaseBuildStateAfterValidation: false }), { + arm: 'fixture-arm', + mode: 'scoped-required', + }), + /CTF_SERVER_BUILD_STATE_RETIREMENT_REQUIRED/, + ); + const collided = contracts(); + collided.c = collided.a; + assert.throws( + () => validateServerStatus(status({ buildContracts: collided }), { + arm: 'fixture-arm', + mode: 'scoped-required', + }), + /CTF_SERVER_CONTRACT_COLLISION/, + ); + const unresolved = contracts(); + unresolved.b = 'ctf:unresolved:v1:b'; + assert.throws( + () => validateServerStatus(status({ buildContracts: unresolved }), { + arm: 'fixture-arm', + mode: 'scoped-required', + }), + /CTF_SERVER_CONTRACT_INVALID/, + ); +}); + +test('input generation writes exact credential-free contracts atomically', async (context) => { + const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctf-inputs-')); + context.after(() => fs.rmSync(outputDir, { recursive: true, force: true })); + const runtimeRoles = { a: 'ctf_runtime_a', b: 'ctf_runtime_b', c: 'ctf_runtime_c' }; + const result = await generateInputs({ + arm: 'fixture-arm', + mode: 'scoped-required', + port: 3391, + postgresContainer: 'ctf-postgres', + runtimeRoles, + durationSec: 60, + outputDir, + commit: '0123456789abcdef', + validate: false, + fetchImpl: async () => ({ ok: true, json: async () => status() }), + }); + const fleet = JSON.parse(fs.readFileSync(result.fleetFile, 'utf8')); + const plan = JSON.parse(fs.readFileSync(result.planFile, 'utf8')); + assert.equal(fleet.tenants[0].surfaces[0].buildContract, evidenceContracts().a); + assert.equal( + fleet.tenants[0].databases[0].apis[0].runtimePoolIdentity, + evidencePoolIdentities().a, + ); + assert.equal(plan.arms[0].env.PG_POOL_MAX, '1'); + assert.equal(plan.arms[0].env.PG_POOL_MAX_USES, '0'); + assert.equal(plan.arms[0].env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE, '100'); + const releaseModeIndex = plan.arms[0].command.indexOf( + '--introspection-client-release-mode', + ); + assert.ok(releaseModeIndex >= 0); + assert.equal(plan.arms[0].command[releaseModeIndex + 1], 'destroy'); + assert.equal(result.provenance.introspectionClientReleaseMode, 'destroy'); + assert.equal(result.provenance.customerQualified, false); + assert.doesNotMatch( + fs.readdirSync(outputDir).map((file) => fs.readFileSync(path.join(outputDir, file), 'utf8')).join('\n'), + /password|secretAccessKey|authorization|bearer/i, + ); +}); + +test('input generation rejects unsupported introspection client release modes', async () => { + await assert.rejects(() => generateInputs({ + introspectionClientReleaseMode: 'best-effort', + postgresContainer: 'ctf-postgres', + runtimeRoles: { a: 'ctf_runtime_a', b: 'ctf_runtime_b', c: 'ctf_runtime_c' }, + validate: false, + }), /CTF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:best-effort/); +}); + +test('atomic artifact writes reject credential markers', (context) => { + const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctf-atomic-')); + context.after(() => fs.rmSync(outputDir, { recursive: true, force: true })); + assert.throws( + () => atomicWriteJson(path.join(outputDir, 'unsafe.json'), { password: 'value' }), + /CTF_ARTIFACT_CONTAINS_CREDENTIAL_MARKER/, + ); + assert.equal(fs.existsSync(path.join(outputDir, 'unsafe.json')), false); + assert.doesNotThrow(() => atomicWriteJson(path.join(outputDir, 'safe-error.json'), { + failure: 'CTF_RUNTIME_PASSWORD_REQUIRED:CTF_RUNTIME_A_PGPASSWORD', + })); + assert.throws( + () => atomicWriteJson(path.join(outputDir, 'unsafe-url.json'), { + failure: 'postgres://runtime:credential@127.0.0.1/fixture', + }), + /CTF_ARTIFACT_CONTAINS_CREDENTIAL_MARKER/, + ); +}); diff --git a/research/graphile-density/complete-tenant-fixture/hostile-validation.cjs b/research/graphile-density/complete-tenant-fixture/hostile-validation.cjs new file mode 100644 index 0000000000..abeccb0b34 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/hostile-validation.cjs @@ -0,0 +1,589 @@ +'use strict'; + +const path = require('node:path'); + +const { + FIXTURE_DIR, + TENANTS, + assertLoopbackBaseUrl, + evaluateCanaryResponse, + makeFleet, + parseArgs, + requireString, +} = require('./lib.cjs'); +const { atomicWriteJson, validateServerStatus } = require('./generate-inputs.cjs'); + +const DIAGNOSTIC_TEXT_LIMIT = 512; +const CUSTOMER_ID_PATTERN = /^[a-z0-9-]+$/; + +const requirePhysicalDatabaseIdentity = (value) => { + if (typeof value !== 'string' || !value.trim()) { + throw new Error('CTF_EXPECTED_PHYSICAL_DATABASE_IDENTITY_REQUIRED'); + } + return value.trim(); +}; + +const assertCustomerPathPrefix = (pathPrefix, expectedCustomerId) => { + if ( + typeof expectedCustomerId !== 'string' + || !CUSTOMER_ID_PATTERN.test(expectedCustomerId) + || pathPrefix !== `/customer/${expectedCustomerId}` + ) { + throw new Error('CTF_CUSTOMER_PATH_PREFIX_MISMATCH'); + } + return pathPrefix; +}; + +const requestUrl = (baseUrl, pathPrefix, pathname) => + `${baseUrl}${pathPrefix}${pathname}`; + +const collectDiagnosticSecrets = (value, secrets = [], seen = new Set()) => { + if (typeof value === 'string') { + if (value.length >= 4) secrets.push(value); + return secrets; + } + if (!value || typeof value !== 'object' || seen.has(value)) return secrets; + seen.add(value); + if (Array.isArray(value)) { + for (const entry of value) collectDiagnosticSecrets(entry, secrets, seen); + } else { + for (const entry of Object.values(value)) collectDiagnosticSecrets(entry, secrets, seen); + } + return secrets; +}; + +const redactDiagnosticText = (value, secrets = []) => { + let text = typeof value === 'string' ? value : String(value ?? ''); + text = text + .replace(/\bbearer\s+[^\s,'"}]+/gi, 'Bearer [REDACTED]') + .replace(/\b(postgres(?:ql)?):\/\/[^@\s/]+@/gi, '$1://[REDACTED]@') + .replace( + /((?:password|passwd|pwd|token|secret|api[-_]?key|authorization)\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;]+)/gi, + '$1[REDACTED]', + ); + for (const secret of [...new Set(secrets)].sort((left, right) => right.length - left.length)) { + text = text.split(secret).join('[REDACTED]'); + } + return text.replace(/\s+/g, ' ').trim().slice(0, DIAGNOSTIC_TEXT_LIMIT); +}; + +const diagnosticCode = (value, fallback) => { + const normalized = redactDiagnosticText(value ?? fallback) + .replace(/[^A-Za-z0-9_.-]+/g, '_') + .slice(0, 80); + return normalized || fallback; +}; + +const diagnosticRequestTarget = (value) => { + try { + const parsed = new URL(value); + return `${parsed.protocol}//${parsed.host}${parsed.pathname}`; + } catch { + return 'invalid-url'; + } +}; + +const graphqlOperationName = (operation) => { + const declaredName = typeof operation?.name === 'string' ? operation.name : null; + const parsedName = typeof operation?.query === 'string' + ? /\b(?:query|mutation|subscription)\s+([_A-Za-z][_0-9A-Za-z]*)/.exec(operation.query)?.[1] + : null; + return diagnosticCode(declaredName ?? parsedName, 'anonymous-operation'); +}; + +const requestJson = async (url, { + method = 'GET', + body, + token, + headers = {}, + fetchImpl = fetch, +} = {}) => { + const secrets = collectDiagnosticSecrets({ body, token, headers }); + const target = diagnosticRequestTarget(url); + let response; + try { + response = await fetchImpl(url, { + method, + redirect: 'error', + headers: { + ...(body === undefined ? {} : { 'content-type': 'application/json' }), + ...(token ? { authorization: `Bearer ${token}` } : {}), + ...headers, + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + } catch (error) { + const detail = redactDiagnosticText( + error instanceof Error ? error.message : error, + secrets, + ) || 'transport-error'; + throw new Error(`CTF_HTTP_REQUEST_FAILED:${target}:${detail}`); + } + let parsed = null; + try { + parsed = await response.json(); + } catch { + parsed = null; + } + if (!response.ok) { + const code = diagnosticCode(parsed?.error?.code, `HTTP_${response.status}`); + const detail = redactDiagnosticText(parsed?.error?.message ?? '', secrets); + throw new Error( + `CTF_HTTP_FAILURE:${code}:status=${response.status}:target=${target}` + + (detail ? `:message=${detail}` : ''), + ); + } + return parsed; +}; + +const postGraphql = async ( + baseUrl, + pathPrefix, + tenantId, + operation, + fetchImpl = fetch, +) => { + const body = await requestJson(requestUrl( + baseUrl, + pathPrefix, + `/tenant/${tenantId}/graphql`, + ), { + method: 'POST', + headers: { 'accept-language': 'es' }, + body: { + query: operation.query, + variables: operation.variables ?? {}, + }, + fetchImpl, + }); + if (Array.isArray(body?.errors) && body.errors.length > 0) { + const first = body.errors[0] ?? {}; + const code = diagnosticCode(first?.extensions?.code, 'GRAPHQL_ERROR'); + const operationName = graphqlOperationName(operation); + const secrets = collectDiagnosticSecrets(operation?.variables); + const detail = redactDiagnosticText(first?.message ?? '', secrets) || 'no-message'; + const errorPath = Array.isArray(first?.path) + ? first.path.map((entry) => diagnosticCode(entry, 'unknown')).join('.') + : 'none'; + const location = Array.isArray(first?.locations) && first.locations.length > 0 + ? `${Number(first.locations[0]?.line) || 0}:${Number(first.locations[0]?.column) || 0}` + : 'none'; + throw new Error( + `CTF_GRAPHQL_FAILURE:${diagnosticCode(tenantId, 'unknown-tenant')}:${code}` + + `:operation=${operationName}:path=${errorPath}:location=${location}` + + `:errors=${body.errors.length}:message=${detail}`, + ); + } + if (!body || typeof body !== 'object' || !('data' in body)) { + throw new Error(`CTF_GRAPHQL_RESPONSE_INVALID:${tenantId}`); + } + return body; +}; + +const assertPhysicalDatabaseIdentity = ( + observed, + expectedPhysicalDatabaseIdentity, + label, +) => { + const expected = requirePhysicalDatabaseIdentity(expectedPhysicalDatabaseIdentity); + if (observed !== expected) { + throw new Error( + `CTF_PHYSICAL_DATABASE_IDENTITY_MISMATCH:${diagnosticCode(label, 'unknown')}`, + ); + } +}; + +const control = async ( + baseUrl, + pathPrefix, + token, + action, + tenant, + expectedPhysicalDatabaseIdentity, + fetchImpl, +) => { + const response = await requestJson(requestUrl(baseUrl, pathPrefix, '/__ctf/control'), { + method: 'POST', + token, + body: { action, ...(tenant ? { tenant } : {}) }, + fetchImpl, + }); + assertPhysicalDatabaseIdentity( + response?.physicalDatabaseIdentity, + expectedPhysicalDatabaseIdentity, + `${action}:${tenant ?? 'fleet'}`, + ); + return response; +}; + +const identityOperation = { + query: 'query HostileTenantIdentity { tenantIdentity requestIdentity physicalDatabaseIdentity }', +}; + +const physicalIdentityOperation = { + query: 'query HostilePhysicalDatabaseIdentity { physicalDatabaseIdentity }', +}; + +const assertIdentity = (tenant, body, expectedPhysicalDatabaseIdentity) => { + const expectedRequestIdentity = `${tenant.token}:${tenant.databaseId}`; + if ( + body?.data?.tenantIdentity !== tenant.token + || body?.data?.requestIdentity !== expectedRequestIdentity + ) { + throw new Error(`CTF_TENANT_IDENTITY_MISMATCH:${tenant.id}`); + } + assertPhysicalDatabaseIdentity( + body?.data?.physicalDatabaseIdentity, + expectedPhysicalDatabaseIdentity, + `graphql-identity:${tenant.id}`, + ); + const serialized = JSON.stringify(body); + for (const other of TENANTS) { + if (other.id !== tenant.id && serialized.includes(other.token)) { + throw new Error(`CTF_CROSS_TENANT_TOKEN:${tenant.id}:${other.id}`); + } + } +}; + +const runHostileValidation = async ({ + baseUrl = 'http://127.0.0.1:3391', + pathPrefix = '', + expectedCustomerId, + expectedPhysicalDatabaseIdentity, + controlToken, + arm = 'local-complete-tenant', + mode = 'scoped-required', + fetchImpl = fetch, + outputFile, +} = {}) => { + if (typeof controlToken !== 'string' || Buffer.byteLength(controlToken) < 32) { + throw new Error('CTF_CONTROL_TOKEN_REQUIRED'); + } + expectedPhysicalDatabaseIdentity = requirePhysicalDatabaseIdentity( + expectedPhysicalDatabaseIdentity, + ); + baseUrl = assertLoopbackBaseUrl(baseUrl); + if (pathPrefix !== '' || expectedCustomerId !== undefined) { + pathPrefix = assertCustomerPathPrefix(pathPrefix, expectedCustomerId); + } + const startedAt = new Date().toISOString(); + const checks = []; + const record = (name, detail = {}) => checks.push({ name, passed: true, ...detail }); + const status = await requestJson(requestUrl(baseUrl, pathPrefix, '/__ctf/status'), { + fetchImpl, + }); + const contracts = validateServerStatus(status, { arm, mode }); + assertPhysicalDatabaseIdentity( + status.physicalDatabase, + expectedPhysicalDatabaseIdentity, + 'status', + ); + if (status.controlAvailable !== true) throw new Error('CTF_CONTROL_ENDPOINT_UNAVAILABLE'); + record('runtime-boundary', { + physicalIsolation: status.physicalIsolation, + sharedRuntimePool: status.sharedRuntimePool, + }); + + const fleet = makeFleet({ arm, buildContracts: contracts }); + for (const tenantTarget of fleet.tenants) { + const tenantId = tenantTarget.id.slice('complete-tenant-'.length); + for (const canary of tenantTarget.surfaces[0].canaries) { + const response = await postGraphql( + baseUrl, + pathPrefix, + tenantId, + canary, + fetchImpl, + ); + const result = evaluateCanaryResponse(canary, response); + if (!result.conclusive || result.violation) { + throw new Error( + `CTF_CANARY_FAILED:${tenantId}:${canary.name}:${result.detail ?? 'inconclusive'}`, + ); + } + const physicalResponse = await postGraphql( + baseUrl, + pathPrefix, + tenantId, + physicalIdentityOperation, + fetchImpl, + ); + assertPhysicalDatabaseIdentity( + physicalResponse?.data?.physicalDatabaseIdentity, + expectedPhysicalDatabaseIdentity, + `canary:${tenantId}:${canary.name}`, + ); + record(`canary:${tenantId}:${canary.name}`); + } + } + + for (const tenant of TENANTS) { + const poisoned = await control( + baseUrl, + pathPrefix, + controlToken, + 'poison', + tenant.id, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + if (poisoned?.ok !== true) throw new Error(`CTF_POISON_PROBE_FAILED:${tenant.id}`); + const response = await postGraphql( + baseUrl, + pathPrefix, + tenant.id, + identityOperation, + fetchImpl, + ); + assertIdentity(tenant, response, expectedPhysicalDatabaseIdentity); + record(`checkout-sanitization:${tenant.id}`); + + const rollback = await control( + baseUrl, + pathPrefix, + controlToken, + 'rollback-savepoint', + tenant.id, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + if (rollback?.ok !== true || rollback.observed !== tenant.databaseId) { + throw new Error(`CTF_ROLLBACK_SAVEPOINT_FAILED:${tenant.id}`); + } + const afterRollback = await postGraphql( + baseUrl, + pathPrefix, + tenant.id, + identityOperation, + fetchImpl, + ); + assertIdentity(tenant, afterRollback, expectedPhysicalDatabaseIdentity); + record(`rollback-savepoint:${tenant.id}`); + + const prepared = await control( + baseUrl, + pathPrefix, + controlToken, + 'prepared-reset', + tenant.id, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + if ( + prepared?.ok !== true + || prepared.first !== tenant.token + || prepared.second !== `${tenant.token}:${tenant.databaseId}` + || prepared.runtimeRole !== status.runtimeBindings?.[tenant.id]?.role + || prepared.backend?.exact !== true + || prepared.backend?.expected !== ( + status.runtimePoolMaxUses === 1 ? 'rotated-client' : 'same-client' + ) + || prepared.backend?.observed !== prepared.backend?.expected + || !Number.isSafeInteger(prepared.backend?.firstBackendPid) + || !Number.isSafeInteger(prepared.backend?.secondBackendPid) + ) { + throw new Error(`CTF_PREPARED_RESET_FAILED:${tenant.id}`); + } + record(`prepared-statement-reset:${tenant.id}`, { + backendBehavior: prepared.backend.observed, + firstBackendPid: prepared.backend.firstBackendPid, + secondBackendPid: prepared.backend.secondBackendPid, + }); + + const badRole = await control( + baseUrl, + pathPrefix, + controlToken, + 'bad-role-expected-failure', + tenant.id, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + if ( + badRole?.ok !== true + || badRole.rejectedCode !== 'GRAPHILE_UNSAFE_RUNTIME_ROLE' + ) { + throw new Error(`CTF_BAD_ROLE_ACCEPTED:${tenant.id}`); + } + record(`bad-role-expected-failure:${tenant.id}`); + } + + const driftTenant = TENANTS[0]; + let driftApplied = false; + try { + const applied = await control( + baseUrl, + pathPrefix, + controlToken, + 'drift-apply', + driftTenant.id, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + if (applied?.ok !== true) throw new Error('CTF_SCHEMA_DRIFT_APPLY_FAILED'); + driftApplied = true; + const driftResponse = await postGraphql(baseUrl, pathPrefix, driftTenant.id, { + query: 'query DriftApplied { schemaEpoch physicalDatabaseIdentity __type(name: "Document") { fields { name } } }', + }, fetchImpl); + const driftFields = driftResponse?.data?.__type?.fields?.map((field) => field.name) ?? []; + if (driftResponse?.data?.schemaEpoch !== 2 || !driftFields.includes('driftProbe')) { + throw new Error('CTF_SCHEMA_DRIFT_NOT_REBUILT'); + } + assertPhysicalDatabaseIdentity( + driftResponse?.data?.physicalDatabaseIdentity, + expectedPhysicalDatabaseIdentity, + 'schema-drift-applied', + ); + record('schema-drift-apply-and-rebuild'); + } finally { + if (driftApplied) { + const reverted = await control( + baseUrl, + pathPrefix, + controlToken, + 'drift-revert', + driftTenant.id, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + if (reverted?.ok !== true) throw new Error('CTF_SCHEMA_DRIFT_REVERT_FAILED'); + } + } + const revertedResponse = await postGraphql(baseUrl, pathPrefix, driftTenant.id, { + query: 'query DriftReverted { schemaEpoch physicalDatabaseIdentity __type(name: "Document") { fields { name } } }', + }, fetchImpl); + const revertedFields = revertedResponse?.data?.__type?.fields?.map((field) => field.name) ?? []; + if (revertedResponse?.data?.schemaEpoch !== 1 || revertedFields.includes('driftProbe')) { + throw new Error('CTF_SCHEMA_DRIFT_REVERT_NOT_REBUILT'); + } + assertPhysicalDatabaseIdentity( + revertedResponse?.data?.physicalDatabaseIdentity, + expectedPhysicalDatabaseIdentity, + 'schema-drift-reverted', + ); + record('schema-drift-revert-and-rebuild'); + + const beforeConcurrent = await requestJson( + requestUrl(baseUrl, pathPrefix, '/__ctf/status'), + { fetchImpl }, + ); + assertPhysicalDatabaseIdentity( + beforeConcurrent?.physicalDatabase, + expectedPhysicalDatabaseIdentity, + 'before-concurrent-rebuild-status', + ); + await control( + baseUrl, + pathPrefix, + controlToken, + 'invalidate-all', + null, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + await Promise.all(TENANTS.map(async (tenant) => { + const response = await postGraphql( + baseUrl, + pathPrefix, + tenant.id, + identityOperation, + fetchImpl, + ); + assertIdentity(tenant, response, expectedPhysicalDatabaseIdentity); + })); + const afterConcurrent = await requestJson( + requestUrl(baseUrl, pathPrefix, '/__ctf/status'), + { fetchImpl }, + ); + assertPhysicalDatabaseIdentity( + afterConcurrent?.physicalDatabase, + expectedPhysicalDatabaseIdentity, + 'after-concurrent-rebuild-status', + ); + if (afterConcurrent?.builds?.maxConcurrent !== 1) { + throw new Error(`CTF_BUILD_SERIALIZATION_FAILED:${afterConcurrent?.builds?.maxConcurrent}`); + } + for (const tenant of TENANTS) { + const before = beforeConcurrent?.builds?.byTenant?.[tenant.id] ?? 0; + const after = afterConcurrent?.builds?.byTenant?.[tenant.id] ?? 0; + if (after !== before + 1) { + throw new Error(`CTF_CONCURRENT_REBUILD_COUNT_FAILED:${tenant.id}:${before}:${after}`); + } + } + record('concurrent-build-serialization', { maxConcurrentBuilds: 1 }); + + for (let iteration = 0; iteration < 10; iteration += 1) { + for (const tenant of TENANTS) { + const response = await postGraphql( + baseUrl, + pathPrefix, + tenant.id, + identityOperation, + fetchImpl, + ); + assertIdentity(tenant, response, expectedPhysicalDatabaseIdentity); + } + } + record('prepared-and-connection-reuse', { rounds: 10, crossTenantTokens: 0 }); + + const report = { + version: 2, + fixture: 'complete-tenant-abc-v1', + startedAt, + endedAt: new Date().toISOString(), + arm, + mode, + pathPrefix, + expectedCustomerId: expectedCustomerId ?? null, + physicalDatabaseIdentity: expectedPhysicalDatabaseIdentity, + passed: true, + customerQualified: false, + customerQualificationReason: 'hostile validation alone does not satisfy provider and workload gates', + checks, + }; + if (outputFile) atomicWriteJson(outputFile, report); + return report; +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + const output = args.output + ? path.resolve(requireString(args, 'output')) + : path.join(FIXTURE_DIR, 'generated', 'hostile-validation.json'); + const report = await runHostileValidation({ + baseUrl: requireString(args, 'base-url', 'http://127.0.0.1:3391'), + pathPrefix: args['path-prefix'] === undefined + ? '' + : requireString(args, 'path-prefix'), + expectedCustomerId: args['customer-id'], + expectedPhysicalDatabaseIdentity: requireString( + args, + 'expected-physical-database-identity', + ), + controlToken: process.env.CTF_CONTROL_TOKEN, + arm: requireString(args, 'arm', 'local-complete-tenant'), + mode: requireString(args, 'mode', 'scoped-required'), + outputFile: output, + }); + process.stdout.write(`${JSON.stringify(report)}\n`); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + assertCustomerPathPrefix, + assertIdentity, + assertPhysicalDatabaseIdentity, + control, + diagnosticRequestTarget, + identityOperation, + postGraphql, + redactDiagnosticText, + requestJson, + runHostileValidation, +}; diff --git a/research/graphile-density/complete-tenant-fixture/hostile-validation.test.cjs b/research/graphile-density/complete-tenant-fixture/hostile-validation.test.cjs new file mode 100644 index 0000000000..5992541e97 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/hostile-validation.test.cjs @@ -0,0 +1,187 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); + +const { TENANTS } = require('./lib.cjs'); +const { + assertCustomerPathPrefix, + assertIdentity, + assertPhysicalDatabaseIdentity, + postGraphql, + requestJson, + runHostileValidation, +} = require('./hostile-validation.cjs'); + +const response = (body, { ok = true, status = 200 } = {}) => ({ + ok, + status, + json: async () => body, +}); + +test('GraphQL helper rejects transport, GraphQL, and malformed success responses', async () => { + let requestOptions; + await requestJson('http://127.0.0.1/no-redirect', { + fetchImpl: async (_url, options) => { + requestOptions = options; + return response({ ok: true }); + }, + }); + assert.equal(requestOptions.redirect, 'error'); + await assert.rejects( + () => requestJson('http://127.0.0.1/failure', { + fetchImpl: async () => response({ error: { code: 'DENIED' } }, { ok: false, status: 403 }), + }), + /CTF_HTTP_FAILURE:DENIED/, + ); + await assert.rejects( + () => postGraphql('http://127.0.0.1', '', 'a', { query: 'query Test { x }' }, async () => + response({ errors: [{ message: 'unsupported field' }] }) + ), + /CTF_GRAPHQL_FAILURE:a:GRAPHQL_ERROR:operation=Test:path=none:location=none:errors=1:message=unsupported field/, + ); + await assert.rejects( + () => postGraphql('http://127.0.0.1', '', 'a', { query: 'query Test { x }' }, async () => + response({ ok: true }) + ), + /CTF_GRAPHQL_RESPONSE_INVALID:a/, + ); +}); + +test('failure diagnostics identify the operation and redact request secrets', async () => { + const variableSecret = 'fixture-variable-secret-value'; + const bearerSecret = 'fixture-bearer-credential-value'; + const databaseSecret = 'fixture-database-password'; + let graphQlError; + try { + await postGraphql('http://127.0.0.1', '', 'a', { + name: 'localized-post-read', + query: 'query LocalizedPostRead($value: String!) { x(value: $value) }', + variables: { value: variableSecret }, + }, async () => response({ + errors: [{ + message: `invalid ${variableSecret}; Bearer ${bearerSecret}; postgresql://runtime:${databaseSecret}@127.0.0.1/db`, + path: ['posts', 'nodes', 0, 'localeStrings'], + locations: [{ line: 2, column: 7 }], + extensions: { code: 'GRAPHQL_VALIDATION_FAILED' }, + }], + })); + } catch (error) { + graphQlError = error; + } + assert.ok(graphQlError instanceof Error); + assert.match( + graphQlError.message, + /operation=localized-post-read:path=posts\.nodes\.0\.localeStrings:location=2:7/, + ); + assert.match(graphQlError.message, /\[REDACTED\]/); + assert.doesNotMatch( + graphQlError.message, + new RegExp([variableSecret, bearerSecret, databaseSecret].join('|')), + ); + + const controlSecret = 'fixture-control-token-value'; + let httpError; + try { + await requestJson( + 'http://runtime:fixture-url-password@127.0.0.1/failure?token=fixture-query-token', + { + token: controlSecret, + fetchImpl: async () => response({ + error: { code: 'DENIED', message: `authorization=${controlSecret}` }, + }, { ok: false, status: 403 }), + }, + ); + } catch (error) { + httpError = error; + } + assert.ok(httpError instanceof Error); + assert.match(httpError.message, /CTF_HTTP_FAILURE:DENIED:status=403:target=http:\/\/127\.0\.0\.1\/failure/); + assert.doesNotMatch( + httpError.message, + /fixture-control-token-value|fixture-url-password|fixture-query-token/, + ); +}); + +test('identity oracle detects wrong identities and any foreign tenant token', () => { + const tenant = TENANTS[0]; + const physicalDatabaseIdentity = 'fixture_database_a'; + assert.doesNotThrow(() => assertIdentity(tenant, { + data: { + tenantIdentity: tenant.token, + requestIdentity: `${tenant.token}:${tenant.databaseId}`, + physicalDatabaseIdentity, + }, + }, physicalDatabaseIdentity)); + assert.throws( + () => assertIdentity(tenant, { + data: { + tenantIdentity: tenant.token, + requestIdentity: `${tenant.token}:${tenant.databaseId}`, + physicalDatabaseIdentity, + unexpected: TENANTS[1].token, + }, + }, physicalDatabaseIdentity), + /CTF_CROSS_TENANT_TOKEN:a:b/, + ); + assert.throws( + () => assertIdentity(tenant, { + data: { + tenantIdentity: 'guc-mismatch', + requestIdentity: null, + physicalDatabaseIdentity, + }, + }, physicalDatabaseIdentity), + /CTF_TENANT_IDENTITY_MISMATCH:a/, + ); + assert.throws( + () => assertIdentity(tenant, { + data: { + tenantIdentity: tenant.token, + requestIdentity: `${tenant.token}:${tenant.databaseId}`, + physicalDatabaseIdentity: 'fixture_database_b', + }, + }, physicalDatabaseIdentity), + /CTF_PHYSICAL_DATABASE_IDENTITY_MISMATCH:graphql-identity_a/, + ); +}); + +test('mounted hostile routes require the exact manifest customer path', () => { + assert.equal( + assertCustomerPathPrefix( + '/customer/physical-customer-0001', + 'physical-customer-0001', + ), + '/customer/physical-customer-0001', + ); + for (const value of [ + '/customer/physical-customer-0002', + '/customer/physical-customer-0001/', + '/customer/%70hysical-customer-0001', + '/customer/physical-customer-0001?tenant=other', + ]) { + assert.throws( + () => assertCustomerPathPrefix(value, 'physical-customer-0001'), + /CTF_CUSTOMER_PATH_PREFIX_MISMATCH/, + ); + } +}); + +test('physical identity is mandatory before hostile validation performs I/O', async () => { + assert.throws( + () => assertPhysicalDatabaseIdentity('database-a', undefined, 'probe'), + /CTF_EXPECTED_PHYSICAL_DATABASE_IDENTITY_REQUIRED/, + ); + let requested = false; + await assert.rejects(() => runHostileValidation({ + baseUrl: 'http://127.0.0.1:3391', + pathPrefix: '/customer/physical-customer-0001', + expectedCustomerId: 'physical-customer-0001', + controlToken: 'c'.repeat(32), + fetchImpl: async () => { + requested = true; + return response({}); + }, + }), /CTF_EXPECTED_PHYSICAL_DATABASE_IDENTITY_REQUIRED/); + assert.equal(requested, false); +}); diff --git a/research/graphile-density/complete-tenant-fixture/lib.cjs b/research/graphile-density/complete-tenant-fixture/lib.cjs new file mode 100644 index 0000000000..d79535ccb9 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/lib.cjs @@ -0,0 +1,718 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const FIXTURE_DIR = __dirname; +const REPO_ROOT = path.resolve(FIXTURE_DIR, '../../..'); +const LOOPBACK_URL_HOSTS = new Set(['127.0.0.1', '[::1]', 'localhost']); + +const TENANTS = Object.freeze([ + Object.freeze({ + id: 'a', + schema: 'ctf_a', + runtimeRoleArgument: 'runtime-role-a', + runtimePasswordEnvironment: 'CTF_RUNTIME_A_PGPASSWORD', + token: 'tenant-a-canary', + databaseId: '10000000-0000-4000-8000-00000000000a', + apiId: '20000000-0000-4000-8000-00000000000a', + metadataField: 'metadataA', + foreignSchema: 'ctf_b', + foreignToken: 'tenant-b-canary', + }), + Object.freeze({ + id: 'b', + schema: 'ctf_b', + runtimeRoleArgument: 'runtime-role-b', + runtimePasswordEnvironment: 'CTF_RUNTIME_B_PGPASSWORD', + token: 'tenant-b-canary', + databaseId: '10000000-0000-4000-8000-00000000000b', + apiId: '20000000-0000-4000-8000-00000000000b', + metadataField: 'metadataB', + foreignSchema: 'ctf_c', + foreignToken: 'tenant-c-canary', + }), + Object.freeze({ + id: 'c', + schema: 'ctf_c', + runtimeRoleArgument: 'runtime-role-c', + runtimePasswordEnvironment: 'CTF_RUNTIME_C_PGPASSWORD', + token: 'tenant-c-canary', + databaseId: '10000000-0000-4000-8000-00000000000c', + apiId: '20000000-0000-4000-8000-00000000000c', + metadataField: 'metadataC', + foreignSchema: 'ctf_a', + foreignToken: 'tenant-a-canary', + }), +]); + +const REQUIRED_CAPABILITIES = Object.freeze([ + 'graphile-generated', + 'i18n', + 'llm-deterministic', + 'rag-deterministic', + 'bm25', + 'tsvector', + 'trigram', + 'vector', + 'postgis', + 'ltree', + 'uploads-storage-presign-only', + 'bulk-mutations', + 'realtime-tagged-write', + 'function-bindings', + 'security-session', +]); + +const REQUIRED_CANARIES = Object.freeze([ + 'cross-schema-identifiers', + 'metadata', + 'functions', + 'sequences', + 'prepared-statement-reuse', + 'poisoned-gucs', + 'rollback-savepoints', + 'plugin-raw-sql', + 'owner-bypass-role', + 'schema-drift', + 'cache-invalidation', + 'concurrent-builds', + 'connection-reuse', +]); + +const parseArgs = (argv) => { + const result = { positional: [] }; + for (let index = 0; index < argv.length; index += 1) { + const value = argv[index]; + if (!value.startsWith('--')) { + result.positional.push(value); + continue; + } + const key = value.slice(2); + const next = argv[index + 1]; + if (!next || next.startsWith('--')) { + result[key] = true; + } else { + result[key] = next; + index += 1; + } + } + return result; +}; + +const requireString = (args, name, fallback) => { + const value = args[name] ?? fallback; + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`CTF_ARGUMENT_REQUIRED:${name}`); + } + return value.trim(); +}; + +const assertLoopbackBaseUrl = (value) => { + let parsed; + try { + parsed = new URL(value); + } catch { + throw new Error('CTF_LOOPBACK_BASE_URL_REQUIRED'); + } + if ( + parsed.protocol !== 'http:' + || !LOOPBACK_URL_HOSTS.has(parsed.hostname) + || parsed.username + || parsed.password + || (parsed.pathname !== '/' && parsed.pathname !== '') + || parsed.search + || parsed.hash + ) { + throw new Error('CTF_LOOPBACK_BASE_URL_REQUIRED'); + } + return parsed.origin; +}; + +const parsePositiveInteger = (value, label) => { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`CTF_INVALID_POSITIVE_INTEGER:${label}`); + } + return parsed; +}; + +const validateIntrospectionClientReleaseMode = (value = 'destroy') => { + if (value !== 'reuse' && value !== 'destroy') { + throw new Error(`CTF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:${value}`); + } + return value; +}; + +const fileSha256 = (file) => crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); + +const readManifest = () => { + const manifest = JSON.parse(fs.readFileSync(path.join(FIXTURE_DIR, 'coverage-manifest.json'), 'utf8')); + validateManifest(manifest); + return manifest; +}; + +const uniqueStrings = (values, label) => { + if (!Array.isArray(values) || values.length === 0 || values.some((value) => typeof value !== 'string' || !value)) { + throw new Error(`CTF_MANIFEST_INVALID:${label}`); + } + if (new Set(values).size !== values.length) throw new Error(`CTF_MANIFEST_DUPLICATE:${label}`); +}; + +const validateManifest = (manifest) => { + if (!manifest || manifest.version !== 1 || manifest.fixture !== 'complete-tenant-abc-v1') { + throw new Error('CTF_MANIFEST_INVALID:identity'); + } + if ( + manifest.performanceClaimsAllowed !== false + || manifest.productionQualificationImplemented !== false + || manifest.runtimeIsolation?.model !== 'dedicated-login-and-pool-per-tenant' + || manifest.runtimeIsolation?.sharedRuntimePool !== false + || manifest.runtimeIsolation?.runtimeSchemaDriftControl !== false + || manifest.runtimeIsolation?.checkoutSanitationRequired !== true + ) { + throw new Error('CTF_MANIFEST_INVALID:failClosedBoundary'); + } + uniqueStrings(manifest.tenants, 'tenants'); + uniqueStrings(manifest.surfacesPerTenant, 'surfacesPerTenant'); + uniqueStrings(manifest.hostileCanaries, 'hostileCanaries'); + const capabilities = manifest.localCapabilities?.map((entry) => entry.id); + uniqueStrings(capabilities, 'localCapabilities'); + for (const capability of REQUIRED_CAPABILITIES) { + if (!capabilities.includes(capability)) throw new Error(`CTF_MANIFEST_MISSING_CAPABILITY:${capability}`); + } + for (const canary of REQUIRED_CANARIES) { + if (!manifest.hostileCanaries.includes(canary)) throw new Error(`CTF_MANIFEST_MISSING_CANARY:${canary}`); + } + for (const gate of manifest.externalProviderGates ?? []) { + if (gate.status !== 'blocking' || !Array.isArray(gate.requiredArguments) || gate.requiredArguments.length === 0) { + throw new Error(`CTF_MANIFEST_INVALID_GATE:${gate.id ?? 'unknown'}`); + } + } +}; + +const assertProviderGates = (manifest, qualificationClass, args) => { + if (qualificationClass === 'offline-research') { + return { + customerQualified: false, + unresolved: manifest.externalProviderGates.map((gate) => gate.id), + }; + } + if (qualificationClass !== 'production') { + throw new Error(`CTF_UNKNOWN_QUALIFICATION_CLASS:${qualificationClass}`); + } + const missing = []; + for (const gate of manifest.externalProviderGates) { + for (const argument of gate.requiredArguments) { + if (typeof args[argument] !== 'string' || !args[argument].trim()) { + missing.push(`${gate.id}:${argument}`); + } + } + } + if (missing.length > 0) { + throw new Error(`CTF_EXTERNAL_PROVIDER_GATES_UNSATISFIED:${missing.join(',')}`); + } + return { customerQualified: true, unresolved: [] }; +}; + +const operation = (name, capability, query, variables, weight = 1) => ({ + name, + capability, + weight, + query, + ...(variables ? { variables } : {}), +}); + +const canary = (name, query, requiredMatches, forbiddenMatches, variables) => ({ + name, + query, + ...(variables ? { variables } : {}), + requiredMatches, + forbiddenMatches, +}); + +const operationsFor = (tenant) => [ + operation( + 'generated-document-read', + 'graphile-generated', + 'query GeneratedDocumentRead { documents(first: 1) { nodes { id tenantId title } } }', + ), + operation( + 'localized-post-read', + 'i18n', + 'query LocalizedPostRead { posts(first: 1, where: { id: { equalTo: 1 } }) { nodes { tenantId localeStrings { langCode title body } } } }', + ), + operation( + 'deterministic-embed', + 'llm-deterministic', + 'query DeterministicEmbed { embedText(text: "tenant fixture") { vector dimensions } }', + ), + operation( + 'deterministic-rag', + 'rag-deterministic', + 'query DeterministicRag { ragQuery(prompt: "machine learning tenant fixture", contextLimit: 2) { answer tokensUsed sources { content similarity tableName parentId } } }', + undefined, + 0.5, + ), + operation( + 'bm25-search', + 'bm25', + 'query Bm25Search { documents(where: { bm25Body: { query: "machine learning intelligence" } }) { nodes { tenantId title bodyBm25Score } } }', + ), + operation( + 'tsvector-search', + 'tsvector', + 'query TsvectorSearch { documents(where: { tsvTsv: "machine learning" }) { nodes { tenantId title tsvRank } } }', + ), + operation( + 'trigram-search', + 'trigram', + 'query TrigramSearch { documents(where: { trgmTitle: { value: "Machne Lerning", threshold: 0.05 } }) { nodes { tenantId title titleTrgmSimilarity } } }', + ), + operation( + 'vector-search', + 'vector', + 'query VectorSearch { documents(where: { vectorEmbedding: { vector: [1, 0, 0], metric: COSINE } }) { nodes { tenantId title embeddingVectorDistance } } }', + ), + operation( + 'postgis-read', + 'postgis', + 'query PostgisRead { documents(first: 1) { nodes { tenantId location { geojson } } } }', + ), + operation( + 'ltree-filter', + 'ltree', + 'query LtreeFilter { documents(where: { path: { within: "/root" } }) { nodes { tenantId title path } } }', + ), + operation( + 'presigned-upload', + 'uploads-storage-presign-only', + 'mutation PresignedUpload($input: UploadAppFileInput!) { uploadAppFile(input: $input) { fileId key deduplicated expiresAt uploadUrl } }', + { + input: { + bucketKey: 'private', + contentHash: crypto.createHash('sha256').update(`complete-tenant-${tenant.id}`).digest('hex'), + contentType: 'text/plain', + size: 32, + filename: `${tenant.id}.txt`, + }, + }, + 0.25, + ), + operation( + 'bulk-upsert', + 'bulk-mutations', + 'mutation BulkUpsert($name: String!) { bulkUpsertBulkItems(input: { values: [{ name: $name, quantity: 1 }], onConflict: { constraint: BULK_ITEMS_NAME_KEY } }) { affectedCount } }', + { name: `${tenant.token}-bulk` }, + 0.5, + ), + operation( + 'realtime-tagged-update', + 'realtime-tagged-write', + 'mutation RealtimeTaggedUpdate($payload: String!) { updateRealtimeItem(input: { id: 1, realtimeItemPatch: { payload: $payload } }) { realtimeItem { id tenantId payload } } }', + { payload: `${tenant.token}-realtime` }, + 0.5, + ), + operation( + 'bound-function-invocation', + 'function-bindings', + 'mutation BoundFunctionInvocation($payload: JSON!) { fixtureTask(input: { payload: $payload }) { invocationId status } }', + { payload: { tenant: tenant.id, source: 'complete-tenant-fixture' } }, + 0.25, + ), + operation( + 'security-context-read', + 'security-session', + 'query SecurityContextRead { requestIdentity }', + undefined, + 0.25, + ), +]; + +const tokenMatch = (pathValue, value) => ({ path: pathValue, value }); + +const canariesFor = (tenant) => { + const ownIdentity = `${tenant.token}:${tenant.databaseId}`; + const identityQuery = 'query TenantIdentity { tenantIdentity }'; + return [ + canary( + 'cross-schema-identifiers', + 'query CrossSchemaIdentifier($schemaName: String!) { foreignAccessState(targetSchema: $schemaName) }', + [tokenMatch('/data/foreignAccessState', 'acl-denied')], + [tokenMatch('/data/foreignAccessState', 'visible')], + { schemaName: tenant.foreignSchema }, + ), + canary( + 'metadata', + 'query MetadataIsolation { __type(name: "Query") { fields { name } } }', + [tokenMatch('/data/__type/fields/*/name', tenant.metadataField)], + TENANTS.filter((candidate) => candidate.id !== tenant.id) + .map((candidate) => tokenMatch('/data/__type/fields/*/name', candidate.metadataField)), + ), + canary( + 'functions', + identityQuery, + [tokenMatch('/data/tenantIdentity', tenant.token)], + [tokenMatch('/data/tenantIdentity', tenant.foreignToken)], + ), + canary( + 'sequences', + 'mutation SequenceIsolation($secret: String!) { createTenantCanary(input: { tenantCanary: { secret: $secret } }) { tenantCanary { tenantId secret } } }', + [tokenMatch('/data/createTenantCanary/tenantCanary/tenantId', tenant.token)], + [tokenMatch('/data/createTenantCanary/tenantCanary/tenantId', tenant.foreignToken)], + { secret: `${tenant.token}-sequence` }, + ), + canary( + 'prepared-statement-reuse', + identityQuery, + [tokenMatch('/data/tenantIdentity', tenant.token)], + [tokenMatch('/data/tenantIdentity', tenant.foreignToken)], + ), + canary( + 'poisoned-gucs', + 'query RequestIdentity { requestIdentity }', + [tokenMatch('/data/requestIdentity', ownIdentity)], + [tokenMatch('/data/requestIdentity', `${tenant.foreignToken}:${tenant.databaseId}`)], + ), + canary( + 'rollback-savepoints', + 'query RequestIdentityAfterRollbackProbe { requestIdentity }', + [tokenMatch('/data/requestIdentity', ownIdentity)], + [tokenMatch('/data/requestIdentity', `poisoned-savepoint:${tenant.databaseId}`)], + ), + canary( + 'plugin-raw-sql', + 'query I18nRawSql { posts(first: 1, where: { id: { equalTo: 1 } }) { nodes { localeStrings { title } } } }', + [tokenMatch('/data/posts/nodes/0/localeStrings/title', `${tenant.token} español`)], + [tokenMatch('/data/posts/nodes/0/localeStrings/title', `${tenant.foreignToken} español`)], + ), + canary( + 'owner-bypass-role', + 'query RuntimeRoleSafety { runtimeRoleSafe }', + [tokenMatch('/data/runtimeRoleSafe', true)], + [tokenMatch('/data/runtimeRoleSafe', false)], + ), + canary( + 'schema-drift', + 'query SchemaEpoch { schemaEpoch }', + [tokenMatch('/data/schemaEpoch', 1)], + [tokenMatch('/data/schemaEpoch', 0)], + ), + canary( + 'cache-invalidation', + identityQuery, + [tokenMatch('/data/tenantIdentity', tenant.token)], + [tokenMatch('/data/tenantIdentity', tenant.foreignToken)], + ), + canary( + 'concurrent-builds', + identityQuery, + [tokenMatch('/data/tenantIdentity', tenant.token)], + [tokenMatch('/data/tenantIdentity', tenant.foreignToken)], + ), + canary( + 'connection-reuse', + 'query RequestIdentity { requestIdentity }', + [tokenMatch('/data/requestIdentity', ownIdentity)], + [tokenMatch('/data/requestIdentity', `${tenant.foreignToken}:${tenant.databaseId}`)], + ), + ]; +}; + +const fallbackBuildContract = (arm, tenant) => + `ctf:unresolved:v1:${arm}:${tenant.id}:api`; + +const fallbackPoolIdentity = (arm, tenant) => + `ctf:unresolved-pool:v1:${arm}:${tenant.id}:api`; + +const makeFleet = ({ + arm = 'local-complete-tenant', + port = 3391, + buildContracts = {}, + runtimePoolIdentities = {}, + physicalDatabase = 'ctf-unresolved-physical-database', +} = {}) => ({ + version: 1, + tenants: TENANTS.map((tenant) => ({ + id: `complete-tenant-${tenant.id}`, + databases: [{ + id: tenant.databaseId, + physicalDatabase, + apis: [{ + id: tenant.apiId, + runtimePoolIdentity: runtimePoolIdentities[tenant.id] + ?? fallbackPoolIdentity(arm, tenant), + runtimePoolIdentities: { + [arm]: runtimePoolIdentities[tenant.id] + ?? fallbackPoolIdentity(arm, tenant), + }, + physicalSchemas: [tenant.schema], + routingLabels: [`ctf-${tenant.id}-api`], + realtime: false, + surfaces: ['api'], + }], + }], + surfaces: [{ + name: 'api', + buildContract: buildContracts[tenant.id] ?? fallbackBuildContract(arm, tenant), + buildContracts: { + [arm]: buildContracts[tenant.id] ?? fallbackBuildContract(arm, tenant), + }, + url: `http://127.0.0.1:{port}/tenant/${tenant.id}/graphql`, + headers: { 'accept-language': 'es' }, + warmup: operation('warm-tenant-identity', 'graphile-generated', 'query WarmTenantIdentity { tenantIdentity }'), + operations: operationsFor(tenant), + canaries: canariesFor(tenant), + }], + })), +}); + +const makePlan = ({ + arm = 'local-complete-tenant', + port = 3391, + postgresContainer, + commit, + durationSec = 900, + cwd = REPO_ROOT, + introspectionMode = 'scoped-required', + introspectionClientReleaseMode = 'destroy', + runtimeRoles, +} = {}) => { + if (!postgresContainer) throw new Error('CTF_ARGUMENT_REQUIRED:postgres-container'); + if (!commit) throw new Error('CTF_ARGUMENT_REQUIRED:commit'); + if (!['stock', 'scoped-required'].includes(introspectionMode)) { + throw new Error(`CTF_INTROSPECTION_MODE_INVALID:${introspectionMode}`); + } + validateIntrospectionClientReleaseMode(introspectionClientReleaseMode); + for (const tenant of TENANTS) { + if (typeof runtimeRoles?.[tenant.id] !== 'string' || !runtimeRoles[tenant.id].trim()) { + throw new Error(`CTF_ARGUMENT_REQUIRED:${tenant.runtimeRoleArgument}`); + } + } + return { + version: 1, + fleetFile: 'fleet.json', + artifactDir: '../artifacts', + arms: [{ + name: arm, + commit, + cwd, + command: [ + 'node', + path.join(FIXTURE_DIR, 'server.cjs'), + '--port', + '{port}', + '--arm', + arm, + '--mode', + '{mode}', + '--introspection-client-release-mode', + introspectionClientReleaseMode, + '--runtime-pool-max', + '1', + '--runtime-pool-max-uses', + 'unlimited', + ...TENANTS.flatMap((tenant) => [ + `--${tenant.runtimeRoleArgument}`, + runtimeRoles[tenant.id], + ]), + ], + port, + readinessUrl: `http://127.0.0.1:{port}/healthz`, + memoryUrl: `http://127.0.0.1:{port}/debug/memory`, + postgresContainer, + introspectionMode, + entrySha256: fileSha256(path.join(FIXTURE_DIR, 'server.cjs')), + lockfileSha256: fileSha256(path.join(REPO_ROOT, 'pnpm-lock.yaml')), + env: { + GRAPHILE_CACHE_MAX: '3', + GRAPHILE_CACHE_INSTANCE_HEAP_BYTES: String(64 * 1024 * 1024), + GRAPHILE_CACHE_SERVER_RESERVE_BYTES: String(256 * 1024 * 1024), + GRAPHILE_CACHE_BUILD_RESERVE_BYTES: String(768 * 1024 * 1024), + GRAPHILE_BUILD_MAX_CONCURRENCY: '1', + GRAPHILE_BUILD_CONCURRENCY: '1', + GRAPHILE_BUILD_QUEUE_MAX: '8', + PG_CACHE_MAX: '4', + PG_POOL_MAX: '1', + PG_POOL_MAX_USES: '0', + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: '100', + }, + }], + heapMiB: [4096], + tenantCountsByHeapMiB: { 4096: [3] }, + repetitions: 3, + runOrderSeed: 'complete-tenant-abc-v1', + requiredCapabilities: [...REQUIRED_CAPABILITIES], + requiredCanaries: [...REQUIRED_CANARIES], + workload: { + durationSec, + rpsPerTenant: 1, + minWorkloadRequestsPerSurface: 30, + requestTimeoutMs: 30000, + maxInFlight: 32, + canaryIntervalSec: 60, + warmupTimeoutMs: 180000, + warmupTimeoutPerSurfaceMs: 30000, + warmupConcurrency: 3, + }, + gates: { + maxErrorRate: 0.005, + maxP99Ms: 150, + maxPostWarmupHeapGrowthMiBPerHour: 5, + minMedianDensityImprovement: 0.15, + minAdditionalTenantsEveryRun: 1, + requireZeroBleed: true, + requireNoPostWarmupEvictions: true, + requireNoPostWarmupBuildRefusals: true, + requireNoPostWarmupBuilds: true, + requirePostgresMemoryTelemetry: true, + requireFreshPostgresRunAttestation: false, + requireRetainedMemoryCheckpoints: false, + requirePhysicalDatabaseTelemetry: false, + requireConclusiveCanaries: true, + requireCompletePeriodicCanaryCoverage: true, + requireConclusiveOperationOracles: false, + requireExplicitCustomerTopology: true, + requiredCacheAdmissionMode: 'evict-idle', + }, + }; +}; + +const decodePointerSegment = (segment) => segment + .replace(/~1/g, '/') + .replace(/~0/g, '~'); + +const jsonPointerValues = (root, pointer) => { + if (pointer === '') return [root]; + if (typeof pointer !== 'string' || !pointer.startsWith('/')) return []; + let values = [root]; + for (const rawSegment of pointer.slice(1).split('/')) { + const segment = decodePointerSegment(rawSegment); + const next = []; + for (const value of values) { + if (segment === '*') { + if (Array.isArray(value)) next.push(...value); + else if (value && typeof value === 'object') next.push(...Object.values(value)); + } else if (Array.isArray(value) && /^(0|[1-9]\d*)$/.test(segment)) { + const index = Number(segment); + if (index < value.length) next.push(value[index]); + } else if ( + value + && typeof value === 'object' + && Object.prototype.hasOwnProperty.call(value, segment) + ) { + next.push(value[segment]); + } + } + values = next; + if (values.length === 0) break; + } + return values; +}; + +const deepEqualJson = (left, right) => { + if (Object.is(left, right)) return true; + if (Array.isArray(left) && Array.isArray(right)) { + return left.length === right.length + && left.every((value, index) => deepEqualJson(value, right[index])); + } + if ( + left + && right + && typeof left === 'object' + && typeof right === 'object' + && !Array.isArray(left) + && !Array.isArray(right) + ) { + const leftKeys = Object.keys(left).sort(); + const rightKeys = Object.keys(right).sort(); + return deepEqualJson(leftKeys, rightKeys) + && leftKeys.every((key) => deepEqualJson(left[key], right[key])); + } + return false; +}; + +const evaluateCanaryResponse = (canary, responseBody) => { + const forbidden = canary.forbiddenMatches.find((match) => + jsonPointerValues(responseBody, match.path).some((value) => + deepEqualJson(value, match.value) + ) + ); + const missing = canary.requiredMatches.find((match) => + !jsonPointerValues(responseBody, match.path).some((value) => + deepEqualJson(value, match.value) + ) + ); + return { + conclusive: !missing, + violation: Boolean(forbidden), + ...(forbidden ? { detail: `forbidden match at '${forbidden.path}' was returned` } : {}), + ...(!forbidden && missing + ? { detail: `required match at '${missing.path}' was absent` } + : {}), + }; +}; + +const assertCredentialFree = (value) => { + const reject = () => { + throw new Error('CTF_ARTIFACT_CONTAINS_CREDENTIAL_MARKER'); + }; + const visit = (candidate, ancestors = new Set()) => { + if (typeof candidate === 'string') { + if ( + /\bbearer\s+[a-z0-9._~+/-]{16,}/i.test(candidate) + || /postgres(?:ql)?:\/\/[^:@/\s]+:[^@/\s]+@/i.test(candidate) + ) reject(); + return; + } + if (!candidate || typeof candidate !== 'object') return; + if (ancestors.has(candidate)) return; + const nextAncestors = new Set(ancestors).add(candidate); + if (Array.isArray(candidate)) { + for (const entry of candidate) visit(entry, nextAncestors); + return; + } + for (const [key, entry] of Object.entries(candidate)) { + if (/password|secretAccessKey|authorization|controlToken|observabilityToken|accessKeyId/i.test(key)) { + reject(); + } + visit(entry, nextAncestors); + } + }; + if (typeof value === 'string') { + try { + visit(JSON.parse(value)); + } catch (error) { + if (error?.message === 'CTF_ARTIFACT_CONTAINS_CREDENTIAL_MARKER') throw error; + visit(value); + } + } else { + visit(value); + } +}; + +module.exports = { + FIXTURE_DIR, + REPO_ROOT, + REQUIRED_CANARIES, + REQUIRED_CAPABILITIES, + TENANTS, + assertCredentialFree, + assertLoopbackBaseUrl, + assertProviderGates, + canariesFor, + evaluateCanaryResponse, + fallbackBuildContract, + fileSha256, + makeFleet, + makePlan, + operationsFor, + parseArgs, + parsePositiveInteger, + jsonPointerValues, + readManifest, + requireString, + validateIntrospectionClientReleaseMode, + validateManifest, +}; diff --git a/research/graphile-density/complete-tenant-fixture/lib.test.cjs b/research/graphile-density/complete-tenant-fixture/lib.test.cjs new file mode 100644 index 0000000000..efdcbb22b5 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/lib.test.cjs @@ -0,0 +1,165 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); + +const { + REQUIRED_CANARIES, + REQUIRED_CAPABILITIES, + TENANTS, + assertLoopbackBaseUrl, + assertProviderGates, + evaluateCanaryResponse, + jsonPointerValues, + makeFleet, + makePlan, + operationsFor, + readManifest, +} = require('./lib.cjs'); + +test('manifest is complete and production provider gates fail closed', () => { + const manifest = readManifest(); + assert.throws( + () => assertProviderGates(manifest, 'production', {}), + /CTF_EXTERNAL_PROVIDER_GATES_UNSATISFIED/, + ); + assert.deepEqual(assertProviderGates(manifest, 'offline-research', {}), { + customerQualified: false, + unresolved: ['ollama-real-semantic', 'object-storage-byte-roundtrip'], + }); +}); + +test('each A/B/C fleet member covers every capability and hostile canary', () => { + const fleet = makeFleet({ arm: 'test-arm', port: 3391 }); + assert.equal(fleet.tenants.length, TENANTS.length); + + const contracts = new Set(); + for (const tenant of fleet.tenants) { + assert.equal(tenant.databases.length, 1); + assert.equal(tenant.databases[0].apis.length, 1); + assert.deepEqual(tenant.databases[0].apis[0].surfaces, ['api']); + assert.equal(tenant.surfaces.length, 1); + const surface = tenant.surfaces[0]; + assert.equal(contracts.has(surface.buildContract), false); + contracts.add(surface.buildContract); + assert.deepEqual( + [...new Set(surface.operations.map((entry) => entry.capability))].sort(), + [...REQUIRED_CAPABILITIES].sort(), + ); + assert.deepEqual( + surface.canaries.map((entry) => entry.name).sort(), + [...REQUIRED_CANARIES].sort(), + ); + for (const entry of surface.canaries) { + assert.ok(entry.requiredMatches.length > 0, `${entry.name} has a positive oracle`); + assert.ok(entry.forbiddenMatches.length > 0, `${entry.name} has a negative oracle`); + } + } +}); + +test('fixture operations match the NoUniqueLookup GraphQL surface', () => { + const tenant = TENANTS[0]; + const operations = operationsFor(tenant); + const localizedRead = operations.find((entry) => entry.name === 'localized-post-read'); + const realtimeUpdate = operations.find((entry) => entry.name === 'realtime-tagged-update'); + const rawSqlCanary = makeFleet().tenants[0].surfaces[0].canaries + .find((entry) => entry.name === 'plugin-raw-sql'); + + assert.match(localizedRead.query, /posts\(first: 1, where: \{ id: \{ equalTo: 1 \} \}\)/); + assert.match(realtimeUpdate.query, /updateRealtimeItem\(input:/); + assert.doesNotMatch( + JSON.stringify({ operations, rawSqlCanary }), + /postById|updateRealtimeItemById/, + ); + assert.deepEqual(rawSqlCanary.requiredMatches, [{ + path: '/data/posts/nodes/0/localeStrings/title', + value: `${tenant.token} español`, + }]); +}); + +test('generated fleet is credential-free and does not call offline surrogates production', () => { + const serialized = JSON.stringify(makeFleet({ arm: 'test-arm', port: 3391 })); + assert.doesNotMatch(serialized, /password|secretAccessKey|authorization|bearer/i); + assert.match(serialized, /llm-deterministic/); + assert.match(serialized, /uploads-storage-presign-only/); +}); + +test('exact server build contracts replace unresolved fixture placeholders', () => { + const contracts = Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `graphile:v1:${tenant.id.repeat(64)}`, + ])); + const fleet = makeFleet({ arm: 'exact-arm', port: 3391, buildContracts: contracts }); + for (const tenant of fleet.tenants) { + const tenantId = tenant.id.slice('complete-tenant-'.length); + const surface = tenant.surfaces[0]; + assert.equal(surface.buildContract, contracts[tenantId]); + assert.equal(surface.buildContracts['exact-arm'], contracts[tenantId]); + assert.doesNotMatch(surface.buildContract, /^ctf:unresolved:/); + } +}); + +test('canary pointer matching is positive, negative, and wildcard aware', () => { + const response = { + data: { + values: [{ name: 'metadataA' }, { name: 'safe' }], + }, + }; + assert.deepEqual(jsonPointerValues(response, '/data/values/*/name'), ['metadataA', 'safe']); + assert.deepEqual(evaluateCanaryResponse({ + requiredMatches: [{ path: '/data/values/*/name', value: 'metadataA' }], + forbiddenMatches: [{ path: '/data/values/*/name', value: 'metadataB' }], + }, response), { conclusive: true, violation: false }); + assert.deepEqual(evaluateCanaryResponse({ + requiredMatches: [{ path: '/data/values/*/name', value: 'missing' }], + forbiddenMatches: [], + }, response), { + conclusive: false, + violation: false, + detail: "required match at '/data/values/*/name' was absent", + }); +}); + +test('plan carries non-secret role names and no runtime credentials', () => { + const runtimeRoles = { a: 'ctf_runtime_a', b: 'ctf_runtime_b', c: 'ctf_runtime_c' }; + const plan = makePlan({ + arm: 'exact-arm', + postgresContainer: 'ctf-postgres', + commit: '0123456789abcdef', + durationSec: 60, + runtimeRoles, + }); + const command = plan.arms[0].command; + for (const tenant of TENANTS) { + assert.ok(command.includes(`--${tenant.runtimeRoleArgument}`)); + assert.ok(command.includes(runtimeRoles[tenant.id])); + } + const releaseModeIndex = command.indexOf('--introspection-client-release-mode'); + assert.ok(releaseModeIndex >= 0); + assert.equal(command[releaseModeIndex + 1], 'destroy'); + assert.throws(() => makePlan({ + arm: 'invalid-release-arm', + postgresContainer: 'ctf-postgres', + commit: '0123456789abcdef', + introspectionClientReleaseMode: 'best-effort', + runtimeRoles, + }), /CTF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:best-effort/); + assert.doesNotMatch(JSON.stringify(plan), /password|secretAccessKey|authorization|bearer/i); +}); + +test('control-bearing fixture URLs are restricted to credential-free loopback HTTP', () => { + assert.equal(assertLoopbackBaseUrl('http://127.0.0.1:3391'), 'http://127.0.0.1:3391'); + assert.equal(assertLoopbackBaseUrl('http://[::1]:3391'), 'http://[::1]:3391'); + assert.throws( + () => assertLoopbackBaseUrl('https://example.com'), + /CTF_LOOPBACK_BASE_URL_REQUIRED/, + ); + assert.throws( + () => assertLoopbackBaseUrl('http://user:credential@127.0.0.1:3391'), + /CTF_LOOPBACK_BASE_URL_REQUIRED/, + ); + assert.throws( + () => assertLoopbackBaseUrl('http://127.0.0.1:3391/prefix'), + /CTF_LOOPBACK_BASE_URL_REQUIRED/, + ); +}); diff --git a/research/graphile-density/complete-tenant-fixture/qualification-runner.cjs b/research/graphile-density/complete-tenant-fixture/qualification-runner.cjs new file mode 100644 index 0000000000..9f3a36ceb5 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/qualification-runner.cjs @@ -0,0 +1,350 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { spawn } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + FIXTURE_DIR, + REPO_ROOT, + TENANTS, + assertProviderGates, + parseArgs, + parsePositiveInteger, + readManifest, + requireString, +} = require('./lib.cjs'); +const { atomicWriteJson, generateInputs } = require('./generate-inputs.cjs'); +const { runHostileValidation } = require('./hostile-validation.cjs'); +const { createFixtureServer, parseServerOptions } = require('./server.cjs'); + +const IN_PROCESS_ENVIRONMENT_KEYS = Object.freeze([ + 'NODE_ENV', + 'DATABASE_URL', + 'PGHOST', + 'PGPORT', + 'PGDATABASE', + 'PGUSER', + 'PGPASSWORD', + 'PGSSLMODE', + 'PGSSLROOTCERT', + 'PGSSLCERT', + 'PGSSLKEY', + 'GRAPHQL_RUNTIME_PGPASSWORD', + ...TENANTS.flatMap((tenant) => [ + tenant.runtimePasswordEnvironment, + `CTF_RUNTIME_${tenant.id.toUpperCase()}_PGUSER`, + ]), + 'CTF_CONTROL_TOKEN', + 'GRAPHQL_OBSERVABILITY_ENABLED', + 'GRAPHQL_OBSERVABILITY_TOKEN', + 'GRAPHILE_CACHE_MAX', + 'GRAPHILE_CACHE_INSTANCE_HEAP_BYTES', + 'GRAPHILE_CACHE_SERVER_RESERVE_BYTES', + 'GRAPHILE_CACHE_BUILD_RESERVE_BYTES', + 'GRAPHILE_BUILD_MAX_CONCURRENCY', + 'GRAPHILE_BUILD_CONCURRENCY', + 'GRAPHILE_BUILD_QUEUE_MAX', + 'PG_CACHE_MAX', + 'PG_POOL_MAX', + 'PG_POOL_MAX_USES', + 'DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE', +]); + +const installProcessEnvironment = ( + source, + keys = IN_PROCESS_ENVIRONMENT_KEYS, +) => { + const previous = new Map(); + for (const key of keys) { + previous.set(key, Object.prototype.hasOwnProperty.call(process.env, key) + ? { present: true, value: process.env[key] } + : { present: false }); + const value = source[key]; + if (value === undefined || value === null) delete process.env[key]; + else process.env[key] = String(value); + } + let restored = false; + return () => { + if (restored) return; + restored = true; + for (const [key, state] of previous) { + if (state.present) process.env[key] = state.value; + else delete process.env[key]; + } + }; +}; + +const runCommand = (command, cwd, environment = process.env) => new Promise((resolve, reject) => { + const child = spawn(command[0], command.slice(1), { + cwd, + env: environment, + stdio: 'inherit', + }); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (code === 0) resolve(); + else reject(new Error( + `CTF_COMMAND_FAILED:${command[0]}:code=${code ?? 'null'}:signal=${signal ?? 'null'}`, + )); + }); +}); + +const runRepositorySuites = async (manifest, environment) => { + const results = []; + for (const suite of manifest.mandatoryRepositorySuites) { + await runCommand(suite.command, path.join(REPO_ROOT, suite.cwd), environment); + results.push({ id: suite.id, passed: true }); + } + return results; +}; + +const summarizeQualification = ({ + qualificationClass, + providerState, + hostilePassed, + repositorySuites, + densityResults, + productionEquivalent = false, + error = null, +}) => { + const localPassed = error === null + && hostilePassed + && repositorySuites.length > 0 + && repositorySuites.every((suite) => suite.passed) + && densityResults.length > 0 + && densityResults.every((result) => result.accepted === true); + const customerQualified = qualificationClass === 'production' + && productionEquivalent === true + && providerState.customerQualified === true + && localPassed; + return { + localPassed, + customerQualified, + unresolvedExternalGates: [...providerState.unresolved], + ...(error ? { failure: error instanceof Error ? error.message : String(error) } : {}), + }; +}; + +const runQualification = async ({ + qualificationClass, + arm, + mode, + port, + postgresContainer, + runtimeRoles, + durationSec, + outputDir, + providerArguments = {}, + environment = process.env, +} = {}) => { + const manifest = readManifest(); + fs.mkdirSync(outputDir, { recursive: true, mode: 0o700 }); + const reportFile = path.join(outputDir, 'qualification.json'); + let providerState = { + customerQualified: false, + unresolved: manifest.externalProviderGates.map((gate) => gate.id), + }; + let failure = null; + try { + providerState = assertProviderGates( + manifest, + qualificationClass, + providerArguments, + ); + if (qualificationClass === 'production') { + throw new Error( + 'CTF_PRODUCTION_EQUIVALENCE_NOT_IMPLEMENTED:the exact fixture still uses deterministic LLM and signing-only storage paths', + ); + } + } catch (error) { + failure = error; + } + const controlToken = crypto.randomBytes(32).toString('hex'); + const observabilityToken = crypto.randomBytes(32).toString('hex'); + const runEnvironment = { + ...environment, + NODE_ENV: 'production', + CTF_CONTROL_TOKEN: controlToken, + GRAPHQL_OBSERVABILITY_ENABLED: 'true', + GRAPHQL_OBSERVABILITY_TOKEN: observabilityToken, + GRAPHILE_CACHE_MAX: '3', + GRAPHILE_CACHE_INSTANCE_HEAP_BYTES: String(64 * 1024 * 1024), + GRAPHILE_CACHE_SERVER_RESERVE_BYTES: String(256 * 1024 * 1024), + GRAPHILE_CACHE_BUILD_RESERVE_BYTES: String(768 * 1024 * 1024), + GRAPHILE_BUILD_MAX_CONCURRENCY: '1', + GRAPHILE_BUILD_CONCURRENCY: '1', + GRAPHILE_BUILD_QUEUE_MAX: '8', + PG_CACHE_MAX: '4', + PG_POOL_MAX: '1', + PG_POOL_MAX_USES: '0', + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: '100', + }; + const serverArgs = [ + '--host', + '127.0.0.1', + '--port', + String(port), + '--arm', + arm, + '--mode', + mode, + '--runtime-pool-max', + '1', + '--runtime-pool-max-uses', + 'unlimited', + ...TENANTS.flatMap((tenant) => [ + `--${tenant.runtimeRoleArgument}`, + runtimeRoles[tenant.id], + ]), + ]; + let server = null; + let hostilePassed = false; + let repositorySuites = []; + let densityResults = []; + let generated = null; + let localExecutionStarted = false; + let restoreProcessEnvironment = null; + try { + if (!failure) { + // The server loads pg-cache and graphile-cache lazily, but those modules + // snapshot governor and pool settings from process.env at first require. + // Install the exact run environment before that load and retain it for + // the child cperf arm, which inherits runtime credentials from this + // process without ever writing them into the generated plan. + restoreProcessEnvironment = installProcessEnvironment(runEnvironment); + localExecutionStarted = true; + server = await createFixtureServer(parseServerOptions(serverArgs, runEnvironment), runEnvironment); + await server.listen(); + generated = await generateInputs({ + arm, + mode, + port, + postgresContainer, + runtimeRoles, + durationSec, + outputDir: path.join(outputDir, 'generated'), + }); + const generatedFleet = JSON.parse(fs.readFileSync(generated.fleetFile, 'utf8')); + const expectedPhysicalDatabaseIdentity = + generatedFleet.tenants?.[0]?.databases?.[0]?.physicalDatabase; + await runHostileValidation({ + baseUrl: `http://127.0.0.1:${port}`, + expectedPhysicalDatabaseIdentity, + controlToken, + arm, + mode, + outputFile: path.join(outputDir, 'hostile-validation.json'), + }); + hostilePassed = true; + await server.close(); + server = null; + + repositorySuites = await runRepositorySuites(manifest, runEnvironment); + const perfHarness = require(path.join(REPO_ROOT, 'packages/perf-harness/dist/index.js')); + const plan = perfHarness.loadPlan(generated.planFile); + const fleet = perfHarness.loadFleet(generated.fleetFile); + perfHarness.validateCoverage(plan, fleet); + densityResults = await perfHarness.runDensityPlan(plan, fleet); + } + } catch (error) { + failure ??= error; + } finally { + try { + if (server) await server.close().catch(() => undefined); + } finally { + restoreProcessEnvironment?.(); + } + } + + const summary = summarizeQualification({ + qualificationClass, + providerState, + hostilePassed, + repositorySuites, + densityResults, + productionEquivalent: false, + error: failure, + }); + const report = { + version: 1, + fixture: manifest.fixture, + qualificationClass, + startedLocally: localExecutionStarted, + productionEquivalent: false, + endedAt: new Date().toISOString(), + arm, + mode, + durationSec, + ...summary, + providerGates: manifest.externalProviderGates.map((gate) => ({ + id: gate.id, + passed: false, + blocking: true, + })), + hostileValidation: { passed: hostilePassed }, + repositorySuites, + densityRuns: densityResults.map((result) => ({ + arm: result.arm, + heapMiB: result.heapMiB, + configuredTenants: result.configuredTenants, + repetition: result.repetition, + accepted: result.accepted, + artifactDir: path.relative(REPO_ROOT, result.artifactDir), + })), + generatedInputs: generated ? { + plan: path.relative(REPO_ROOT, generated.planFile), + fleet: path.relative(REPO_ROOT, generated.fleetFile), + } : null, + }; + atomicWriteJson(reportFile, report); + if (failure) throw failure; + if (!summary.localPassed) throw new Error('CTF_OFFLINE_RESEARCH_GATES_FAILED'); + return { reportFile, report }; +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + const qualificationClass = requireString(args, 'class', 'production'); + const runtimeRoles = Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + requireString(args, tenant.runtimeRoleArgument), + ])); + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const result = await runQualification({ + qualificationClass, + arm: requireString(args, 'arm', 'local-complete-tenant'), + mode: requireString(args, 'mode', 'scoped-required'), + port: parsePositiveInteger(args.port ?? '3391', 'port'), + postgresContainer: requireString(args, 'postgres-container'), + runtimeRoles, + durationSec: parsePositiveInteger(args['duration-sec'] ?? '900', 'duration-sec'), + outputDir: path.resolve(requireString( + args, + 'output-dir', + path.join(FIXTURE_DIR, 'qualification-artifacts', timestamp), + )), + providerArguments: args, + }); + process.stdout.write(`${JSON.stringify({ + reportFile: result.reportFile, + localPassed: result.report.localPassed, + customerQualified: result.report.customerQualified, + })}\n`); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + IN_PROCESS_ENVIRONMENT_KEYS, + installProcessEnvironment, + runCommand, + runQualification, + summarizeQualification, +}; diff --git a/research/graphile-density/complete-tenant-fixture/qualification-runner.test.cjs b/research/graphile-density/complete-tenant-fixture/qualification-runner.test.cjs new file mode 100644 index 0000000000..51b84d9dbd --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/qualification-runner.test.cjs @@ -0,0 +1,74 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); + +const { + installProcessEnvironment, + runQualification, + summarizeQualification, +} = require('./qualification-runner.cjs'); + +test('offline success remains explicitly unqualified for customers', () => { + const localEvidence = { + hostilePassed: true, + repositorySuites: [{ id: 'suite', passed: true }], + densityResults: [{ accepted: true }], + }; + assert.deepEqual(summarizeQualification({ + qualificationClass: 'offline-research', + providerState: { customerQualified: false, unresolved: ['provider'] }, + ...localEvidence, + }), { + localPassed: true, + customerQualified: false, + unresolvedExternalGates: ['provider'], + }); + assert.equal(summarizeQualification({ + qualificationClass: 'production', + providerState: { customerQualified: true, unresolved: [] }, + ...localEvidence, + }).customerQualified, false); + assert.equal(summarizeQualification({ + qualificationClass: 'production', + providerState: { customerQualified: true, unresolved: [] }, + productionEquivalent: true, + ...localEvidence, + }).customerQualified, true); +}); + +test('temporary in-process environment installation is exactly reversible', () => { + const key = `CTF_TEST_ENV_${process.pid}`; + delete process.env[key]; + const restore = installProcessEnvironment({ [key]: 'fixture-value' }, [key]); + assert.equal(process.env[key], 'fixture-value'); + restore(); + assert.equal(Object.prototype.hasOwnProperty.call(process.env, key), false); + restore(); +}); + +test('production preflight failure still writes a fail-closed report', async (context) => { + const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctf-qualification-')); + context.after(() => fs.rmSync(outputDir, { recursive: true, force: true })); + await assert.rejects(() => runQualification({ + qualificationClass: 'production', + arm: 'fixture-arm', + mode: 'scoped-required', + port: 3391, + postgresContainer: 'ctf-postgres', + runtimeRoles: { a: 'ctf_runtime_a', b: 'ctf_runtime_b', c: 'ctf_runtime_c' }, + durationSec: 60, + outputDir, + providerArguments: {}, + environment: {}, + }), /CTF_EXTERNAL_PROVIDER_GATES_UNSATISFIED/); + const report = JSON.parse(fs.readFileSync(path.join(outputDir, 'qualification.json'), 'utf8')); + assert.equal(report.localPassed, false); + assert.equal(report.customerQualified, false); + assert.equal(report.startedLocally, false); + assert.match(report.failure, /^CTF_EXTERNAL_PROVIDER_GATES_UNSATISFIED:/); + assert.equal(report.generatedInputs, null); +}); diff --git a/research/graphile-density/complete-tenant-fixture/schema.sql b/research/graphile-density/complete-tenant-fixture/schema.sql new file mode 100644 index 0000000000..8f8660e087 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/schema.sql @@ -0,0 +1,709 @@ +\set ON_ERROR_STOP on + +\if :{?runtime_role_a} +\else + \echo 'CTF_SETUP_RUNTIME_ROLE_REQUIRED: pass --set=runtime_role_a=' + \quit 3 +\endif +\if :{?runtime_role_b} +\else + \echo 'CTF_SETUP_RUNTIME_ROLE_REQUIRED: pass --set=runtime_role_b=' + \quit 3 +\endif +\if :{?runtime_role_c} +\else + \echo 'CTF_SETUP_RUNTIME_ROLE_REQUIRED: pass --set=runtime_role_c=' + \quit 3 +\endif + +-- psql deliberately does not interpolate variables inside dollar-quoted PL/pgSQL +-- bodies. Materialize the three safely quoted values as data before entering any +-- DO block, then read them through pg_temp below. +CREATE TEMP TABLE ctf_runtime_roles ( + ordinal smallint PRIMARY KEY, + role_name name NOT NULL UNIQUE +); +INSERT INTO ctf_runtime_roles (ordinal, role_name) VALUES + (1, :'runtime_role_a'), + (2, :'runtime_role_b'), + (3, :'runtime_role_c'); + +DO $roles$ +DECLARE + runtime_roles text[]; + runtime_role text; + role_record pg_catalog.pg_roles%ROWTYPE; +BEGIN + SELECT pg_catalog.array_agg(role_name::text ORDER BY ordinal) + INTO runtime_roles + FROM pg_temp.ctf_runtime_roles; + IF pg_catalog.array_length(runtime_roles, 1) <> 3 + OR (SELECT pg_catalog.count(DISTINCT value) FROM pg_catalog.unnest(runtime_roles) AS value) <> 3 THEN + RAISE EXCEPTION 'CTF_RUNTIME_ROLES_MUST_BE_DISTINCT'; + END IF; + + FOREACH runtime_role IN ARRAY runtime_roles + LOOP + SELECT * INTO role_record + FROM pg_catalog.pg_roles + WHERE rolname = runtime_role; + IF NOT FOUND THEN + RAISE EXCEPTION 'CTF_RUNTIME_ROLE_NOT_FOUND:%', runtime_role; + END IF; + IF NOT role_record.rolcanlogin + OR role_record.rolinherit + OR role_record.rolsuper + OR role_record.rolbypassrls + OR role_record.rolcreaterole + OR role_record.rolcreatedb + OR role_record.rolreplication THEN + RAISE EXCEPTION 'CTF_RUNTIME_ROLE_UNSAFE:%', runtime_role; + END IF; + END LOOP; +END +$roles$; + +CREATE SCHEMA ctf_extensions; +CREATE EXTENSION vector WITH SCHEMA ctf_extensions; +CREATE EXTENSION pg_trgm WITH SCHEMA ctf_extensions; +CREATE EXTENSION pg_textsearch WITH SCHEMA ctf_extensions; +CREATE EXTENSION ltree WITH SCHEMA ctf_extensions; +CREATE EXTENSION postgis WITH SCHEMA ctf_extensions; + +-- A shared notification login must be able to CONNECT and LISTEN without +-- inheriting PostgreSQL's default PUBLIC access to application metadata or +-- extension routines. Runtime roles receive the exact extension capabilities +-- they need explicitly below. +REVOKE ALL ON SCHEMA public FROM PUBLIC; +REVOKE ALL ON SCHEMA ctf_extensions FROM PUBLIC; +REVOKE ALL ON ALL TABLES IN SCHEMA ctf_extensions FROM PUBLIC; +REVOKE ALL ON ALL SEQUENCES IN SCHEMA ctf_extensions FROM PUBLIC; +REVOKE ALL ON ALL FUNCTIONS IN SCHEMA ctf_extensions FROM PUBLIC; + +-- PostGIS creates these compatibility views with PostgreSQL's historical +-- owner-rights default. They are dependency metadata, not part of a tenant API; +-- keep any future access under the caller's privileges and remove the PUBLIC +-- read grant before approving ctf_extensions as a runtime dependency schema. +ALTER VIEW ctf_extensions.geometry_columns SET (security_invoker = true); +ALTER VIEW ctf_extensions.geography_columns SET (security_invoker = true); +REVOKE ALL ON ctf_extensions.geometry_columns FROM PUBLIC; +REVOKE ALL ON ctf_extensions.geography_columns FROM PUBLIC; + +DO $fixture$ +DECLARE + extension_name text; + extension_schema text; +BEGIN + FOREACH extension_name IN ARRAY ARRAY['vector', 'pg_trgm', 'pg_textsearch', 'ltree', 'postgis'] + LOOP + SELECT n.nspname + INTO extension_schema + FROM pg_catalog.pg_extension e + JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace + WHERE e.extname = extension_name; + IF extension_schema IS DISTINCT FROM 'ctf_extensions' THEN + RAISE EXCEPTION 'CTF_EXTENSION_SCHEMA_MISMATCH:%:%', extension_name, extension_schema; + END IF; + END LOOP; +END +$fixture$; + +SET search_path TO pg_catalog, ctf_extensions; + +CREATE SCHEMA jwt_private; +REVOKE ALL ON SCHEMA jwt_private FROM PUBLIC; + +CREATE FUNCTION jwt_private.current_database_id() +RETURNS uuid +LANGUAGE sql +STABLE +SET search_path = pg_catalog +AS $function$ + SELECT nullif(current_setting('jwt.claims.database_id', true), '')::uuid +$function$; +REVOKE ALL ON FUNCTION jwt_private.current_database_id() FROM PUBLIC; + +CREATE SCHEMA ctf_control; +REVOKE ALL ON SCHEMA ctf_control FROM PUBLIC; + +CREATE PROCEDURE pg_temp.create_complete_tenant( + schema_name text, + tenant_token text, + database_id uuid, + runtime_role text, + metadata_function text, + storage_module_id uuid, + bucket_id uuid, + binding_id uuid, + definition_id uuid +) +LANGUAGE plpgsql +AS $procedure$ +DECLARE + table_name text; + fn_body text; + realtime_schema_name text; +BEGIN + IF schema_name NOT IN ('ctf_a', 'ctf_b', 'ctf_c') THEN + RAISE EXCEPTION 'CTF_UNKNOWN_TENANT_SCHEMA:%', schema_name; + END IF; + + EXECUTE format('CREATE SCHEMA %I', schema_name); + EXECUTE format('REVOKE ALL ON SCHEMA %I FROM PUBLIC', schema_name); + + realtime_schema_name := schema_name || '_realtime'; + EXECUTE format('CREATE SCHEMA %I', realtime_schema_name); + EXECUTE format('REVOKE ALL ON SCHEMA %I FROM PUBLIC', realtime_schema_name); + EXECUTE format( + 'CREATE FUNCTION %I.touch_listener(node_id text) RETURNS void LANGUAGE sql VOLATILE SECURITY INVOKER SET search_path = pg_catalog AS %L', + realtime_schema_name, + 'SELECT NULL::void' + ); + EXECUTE format( + 'CREATE FUNCTION %I.drain_changes(node_id text, batch_limit integer) RETURNS SETOF jsonb LANGUAGE sql VOLATILE SECURITY INVOKER SET search_path = pg_catalog AS %L', + realtime_schema_name, + 'SELECT NULL::jsonb WHERE false' + ); + EXECUTE format( + 'CREATE FUNCTION %I.cleanup_ephemeral(node_id text) RETURNS void LANGUAGE sql VOLATILE SECURITY INVOKER SET search_path = pg_catalog AS %L', + realtime_schema_name, + 'SELECT NULL::void' + ); + EXECUTE format( + 'REVOKE ALL ON ALL FUNCTIONS IN SCHEMA %I FROM PUBLIC', + realtime_schema_name + ); + EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', realtime_schema_name, runtime_role); + EXECUTE format( + 'GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', + realtime_schema_name, + runtime_role + ); + + EXECUTE format($sql$ + CREATE TABLE %I.tenant_canary ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + secret text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() + ) + $sql$, schema_name, database_id, tenant_token); + + EXECUTE format($sql$ + CREATE TABLE %I.documents ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + title text NOT NULL, + body text NOT NULL, + tsv tsvector NOT NULL, + embedding ctf_extensions.vector(3) NOT NULL, + location ctf_extensions.geometry(Point, 4326) NOT NULL, + path ctf_extensions.ltree NOT NULL, + attachment text + ) + $sql$, schema_name, database_id, tenant_token); + EXECUTE format('COMMENT ON COLUMN %I.documents.attachment IS %L', schema_name, E'@upload'); + EXECUTE format('CREATE INDEX documents_tsv_idx ON %I.documents USING gin (tsv)', schema_name); + EXECUTE format( + 'CREATE INDEX documents_embedding_idx ON %I.documents USING ivfflat (embedding ctf_extensions.vector_cosine_ops) WITH (lists = 1)', + schema_name + ); + EXECUTE format('CREATE INDEX documents_body_bm25_idx ON %I.documents USING bm25 (body) WITH (text_config = %L)', schema_name, 'english'); + EXECUTE format('CREATE INDEX documents_title_trgm_idx ON %I.documents USING gin (title ctf_extensions.gin_trgm_ops)', schema_name); + EXECUTE format('CREATE INDEX documents_location_idx ON %I.documents USING gist (location)', schema_name); + EXECUTE format('CREATE INDEX documents_path_idx ON %I.documents USING gist (path)', schema_name); + + EXECUTE format($sql$ + INSERT INTO %I.documents (id, title, body, tsv, embedding, location, path, attachment) + VALUES ( + 1, + %L, + %L, + to_tsvector('english', %L), + '[1,0,0]'::ctf_extensions.vector, + ctf_extensions.st_setsrid(ctf_extensions.st_makepoint(106.7, 10.8), 4326), + 'root.%s'::ctf_extensions.ltree, + 'fixture://%s/document.txt' + ) + $sql$, + schema_name, + tenant_token || ' Machine Learning', + tenant_token || ' machine learning artificial intelligence', + tenant_token || ' machine learning artificial intelligence', + right(schema_name, 1), + tenant_token + ); + + EXECUTE format($sql$ + CREATE TABLE %I.posts ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + title text NOT NULL, + body text + ) + $sql$, schema_name, database_id, tenant_token); + EXECUTE format('COMMENT ON TABLE %I.posts IS %L', schema_name, E'@i18n posts_translations'); + EXECUTE format($sql$ + CREATE TABLE %I.posts_translations ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + post_id integer NOT NULL REFERENCES %I.posts(id) ON DELETE CASCADE, + lang_code text NOT NULL, + title text NOT NULL, + body text, + UNIQUE (post_id, lang_code) + ) + $sql$, schema_name, database_id, schema_name); + EXECUTE format('INSERT INTO %I.posts (id, title, body) VALUES (1, %L, %L)', schema_name, tenant_token, tenant_token || ' base body'); + EXECUTE format( + 'INSERT INTO %I.posts_translations (post_id, lang_code, title, body) VALUES (1, %L, %L, %L), (1, %L, %L, %L)', + schema_name, + 'en', tenant_token || ' English', tenant_token || ' English body', + 'es', tenant_token || ' español', tenant_token || ' cuerpo español' + ); + + EXECUTE format($sql$ + CREATE TABLE %I.articles ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + title text NOT NULL, + body text NOT NULL, + embedding ctf_extensions.vector(3) NOT NULL + ) + $sql$, schema_name, database_id, tenant_token); + EXECUTE format( + 'COMMENT ON TABLE %I.articles IS %L', + schema_name, + E'@hasChunks {"chunksTable":"articles_chunks","parentFk":"parent_id","parentPk":"id","embeddingField":"embedding","contentField":"content"}' + ); + EXECUTE format($sql$ + CREATE TABLE %I.articles_chunks ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + parent_id integer NOT NULL REFERENCES %I.articles(id) ON DELETE CASCADE, + content text NOT NULL, + embedding ctf_extensions.vector(3) NOT NULL + ) + $sql$, schema_name, database_id, schema_name); + EXECUTE format('CREATE INDEX articles_embedding_idx ON %I.articles USING hnsw (embedding ctf_extensions.vector_cosine_ops)', schema_name); + EXECUTE format('CREATE INDEX articles_chunks_embedding_idx ON %I.articles_chunks USING hnsw (embedding ctf_extensions.vector_cosine_ops)', schema_name); + EXECUTE format( + 'INSERT INTO %I.articles (id, title, body, embedding) VALUES (1, %L, %L, %L::ctf_extensions.vector)', + schema_name, tenant_token || ' article', tenant_token || ' machine learning article', '[1,0,0]' + ); + EXECUTE format( + 'INSERT INTO %I.articles_chunks (id, parent_id, content, embedding) VALUES (1, 1, %L, %L::ctf_extensions.vector)', + schema_name, tenant_token || ' machine learning tenant fixture context', '[0.99,0.01,0]' + ); + + EXECUTE format($sql$ + CREATE TABLE %I.bulk_items ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + name text NOT NULL, + quantity integer NOT NULL DEFAULT 0, + CONSTRAINT bulk_items_name_key UNIQUE (name) + ) + $sql$, schema_name, database_id, tenant_token); + EXECUTE format('COMMENT ON TABLE %I.bulk_items IS %L', schema_name, E'@behavior +bulkInsert +bulkUpsert +bulkUpdate +bulkDelete'); + + EXECUTE format($sql$ + CREATE TABLE %I.realtime_items ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + payload text NOT NULL + ) + $sql$, schema_name, database_id, tenant_token); + EXECUTE format('COMMENT ON TABLE %I.realtime_items IS %L', schema_name, E'@realtime'); + EXECUTE format('INSERT INTO %I.realtime_items (id, payload) VALUES (1, %L)', schema_name, tenant_token || '-initial'); + + fn_body := format($body$ + BEGIN + IF TG_OP = 'DELETE' THEN + PERFORM pg_catalog.pg_notify(%L, TG_OP || ':' || OLD.id::text); + RETURN OLD; + END IF; + PERFORM pg_catalog.pg_notify(%L, TG_OP || ':' || NEW.id::text); + RETURN NEW; + END + $body$, + 'realtime:' || schema_name || '.realtime_items', + 'realtime:' || schema_name || '.realtime_items' + ); + EXECUTE format( + 'CREATE FUNCTION %I.notify_realtime_item() RETURNS trigger LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog AS %L', + schema_name, + fn_body + ); + EXECUTE format( + 'CREATE TRIGGER realtime_items_notify AFTER INSERT OR UPDATE OR DELETE ON %I.realtime_items FOR EACH ROW EXECUTE FUNCTION %I.notify_realtime_item()', + schema_name, + schema_name + ); + + EXECUTE format($sql$ + CREATE TABLE %I.app_buckets ( + id uuid PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + key text NOT NULL UNIQUE, + type text NOT NULL, + is_public boolean NOT NULL DEFAULT false, + owner_id uuid, + allowed_mime_types text[], + max_file_size integer, + allow_custom_keys boolean NOT NULL DEFAULT false, + physical_name text + ) + $sql$, schema_name, database_id, tenant_token); + EXECUTE format('COMMENT ON TABLE %I.app_buckets IS %L', schema_name, E'@storageBuckets'); + EXECUTE format($sql$ + CREATE TABLE %I.app_files ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + bucket_id uuid NOT NULL REFERENCES %I.app_buckets(id), + key text NOT NULL, + content_hash text NOT NULL, + mime_type text NOT NULL, + size integer NOT NULL, + filename text, + is_public boolean NOT NULL DEFAULT false, + previous_version_id uuid REFERENCES %I.app_files(id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (bucket_id, content_hash) + ) + $sql$, schema_name, database_id, tenant_token, schema_name, schema_name); + EXECUTE format('COMMENT ON TABLE %I.app_files IS %L', schema_name, E'@storageFiles'); + EXECUTE format( + 'INSERT INTO %I.app_buckets (id, key, type, allowed_mime_types, max_file_size, physical_name) VALUES (%L::uuid, %L, %L, ARRAY[%L], 1048576, NULL)', + schema_name, bucket_id, 'private', 'private', 'text/plain' + ); + + EXECUTE format($sql$ + CREATE TABLE %I.function_invocations ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + task_identifier text NOT NULL, + function_definition_id uuid NOT NULL, + api_binding_id uuid NOT NULL, + status text NOT NULL, + payload jsonb, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() + ) + $sql$, schema_name, database_id, tenant_token); + + EXECUTE format($sql$ + CREATE TABLE %I.schema_state ( + id integer PRIMARY KEY CHECK (id = 1), + database_id uuid NOT NULL DEFAULT %L::uuid, + epoch integer NOT NULL + ) + $sql$, schema_name, database_id); + EXECUTE format('INSERT INTO %I.schema_state (id, epoch) VALUES (1, 1)', schema_name); + + FOREACH table_name IN ARRAY ARRAY[ + 'tenant_canary', + 'documents', + 'posts', + 'posts_translations', + 'articles', + 'articles_chunks', + 'bulk_items', + 'realtime_items', + 'app_buckets', + 'app_files', + 'function_invocations', + 'schema_state' + ] + LOOP + EXECUTE format('ALTER TABLE %I.%I ENABLE ROW LEVEL SECURITY', schema_name, table_name); + EXECUTE format('ALTER TABLE %I.%I FORCE ROW LEVEL SECURITY', schema_name, table_name); + EXECUTE format( + 'CREATE POLICY tenant_guard ON %I.%I USING (database_id::text = nullif(current_setting(%L, true), %L)) WITH CHECK (database_id::text = nullif(current_setting(%L, true), %L))', + schema_name, + table_name, + 'jwt.claims.database_id', + '', + 'jwt.claims.database_id', + '' + ); + END LOOP; + + fn_body := format('SELECT CASE WHEN nullif(current_setting(%L, true), %L) = %L THEN %L::text ELSE %L::text END', 'jwt.claims.database_id', '', database_id::text, tenant_token, 'guc-mismatch'); + EXECUTE format('CREATE FUNCTION %I.tenant_identity() RETURNS text LANGUAGE sql STABLE SECURITY INVOKER SET search_path = pg_catalog AS %L', schema_name, fn_body); + EXECUTE format( + 'CREATE FUNCTION %I.physical_database_identity() RETURNS text LANGUAGE sql STABLE PARALLEL SAFE SECURITY INVOKER SET search_path = pg_catalog AS %L', + schema_name, + 'SELECT pg_catalog.current_database()::text' + ); + EXECUTE format('CREATE FUNCTION %I.%I() RETURNS text LANGUAGE sql STABLE SECURITY INVOKER SET search_path = pg_catalog AS %L', schema_name, metadata_function, format('SELECT %L::text', tenant_token)); + EXECUTE format( + 'CREATE FUNCTION %I.request_identity() RETURNS text LANGUAGE sql STABLE SECURITY INVOKER SET search_path = pg_catalog AS %L', + schema_name, + format('SELECT %L || %L || nullif(current_setting(%L, true), %L)', tenant_token, ':', 'jwt.claims.database_id', '') + ); + fn_body := format($body$ + DECLARE + observed text; + BEGIN + BEGIN + PERFORM set_config('jwt.claims.database_id', 'poisoned-savepoint', true); + RAISE EXCEPTION 'fixture subtransaction rollback'; + EXCEPTION WHEN OTHERS THEN + NULL; + END; + observed := nullif(current_setting('jwt.claims.database_id', true), ''); + RETURN observed; + END + $body$); + EXECUTE format('CREATE FUNCTION %I.savepoint_identity() RETURNS text LANGUAGE plpgsql VOLATILE SECURITY INVOKER SET search_path = pg_catalog AS %L', schema_name, fn_body); + EXECUTE format('COMMENT ON FUNCTION %I.savepoint_identity() IS %L', schema_name, E'@behavior -*'); + fn_body := format($body$ + BEGIN + PERFORM set_config('jwt.claims.database_id', %L, false); + PERFORM set_config('jwt.claims.user_id', %L, false); + RETURN 'poisoned'; + END + $body$, 'ffffffff-ffff-4fff-8fff-ffffffffffff', 'poisoned-user'); + EXECUTE format('CREATE FUNCTION %I.poison_session() RETURNS text LANGUAGE plpgsql VOLATILE SECURITY INVOKER SET search_path = pg_catalog AS %L', schema_name, fn_body); + EXECUTE format('COMMENT ON FUNCTION %I.poison_session() IS %L', schema_name, E'@behavior -*'); + fn_body := format($body$ + DECLARE + row_count integer; + BEGIN + IF target_schema NOT IN ('ctf_a', 'ctf_b', 'ctf_c') THEN + RAISE EXCEPTION 'CTF_FOREIGN_SCHEMA_NOT_ALLOWED:%%', target_schema; + END IF; + BEGIN + EXECUTE format('SELECT count(*)::integer FROM %%I.documents', target_schema) INTO row_count; + EXCEPTION WHEN insufficient_privilege THEN + RETURN 'acl-denied'; + END; + RETURN CASE WHEN row_count = 0 THEN 'rls-empty' ELSE 'visible' END; + END + $body$); + EXECUTE format('CREATE FUNCTION %I.foreign_access_state(target_schema text) RETURNS text LANGUAGE plpgsql STABLE SECURITY INVOKER SET search_path = pg_catalog AS %L', schema_name, fn_body); + fn_body := format($body$ + SELECT NOT r.rolsuper + AND NOT r.rolbypassrls + AND NOT r.rolcreaterole + AND NOT pg_has_role(session_user, n.nspowner, 'MEMBER') + AND NOT has_schema_privilege(session_user, %L, 'CREATE') + FROM pg_roles r + JOIN pg_namespace n ON n.nspname = %L + WHERE r.rolname = session_user + $body$, schema_name, schema_name); + EXECUTE format('CREATE FUNCTION %I.runtime_role_safe() RETURNS boolean LANGUAGE sql STABLE SECURITY INVOKER SET search_path = pg_catalog AS %L', schema_name, fn_body); + EXECUTE format( + 'CREATE FUNCTION %I.schema_epoch() RETURNS integer LANGUAGE sql STABLE SECURITY INVOKER SET search_path = pg_catalog AS %L', + schema_name, + format('SELECT epoch FROM %I.schema_state WHERE id = 1', schema_name) + ); + + EXECUTE format('REVOKE ALL ON ALL FUNCTIONS IN SCHEMA %I FROM PUBLIC', schema_name); + EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', schema_name, runtime_role); + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA %I TO %I', schema_name, runtime_role); + EXECUTE format('GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA %I TO %I', schema_name, runtime_role); + EXECUTE format('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', schema_name, runtime_role); +END +$procedure$; + +CALL pg_temp.create_complete_tenant( + 'ctf_a', + 'tenant-a-canary', + '10000000-0000-4000-8000-00000000000a', + :'runtime_role_a', + 'metadata_a', + '30000000-0000-4000-8000-00000000000a', + '40000000-0000-4000-8000-00000000000a', + '50000000-0000-4000-8000-00000000000a', + '60000000-0000-4000-8000-00000000000a' +); +CALL pg_temp.create_complete_tenant( + 'ctf_b', + 'tenant-b-canary', + '10000000-0000-4000-8000-00000000000b', + :'runtime_role_b', + 'metadata_b', + '30000000-0000-4000-8000-00000000000b', + '40000000-0000-4000-8000-00000000000b', + '50000000-0000-4000-8000-00000000000b', + '60000000-0000-4000-8000-00000000000b' +); +CALL pg_temp.create_complete_tenant( + 'ctf_c', + 'tenant-c-canary', + '10000000-0000-4000-8000-00000000000c', + :'runtime_role_c', + 'metadata_c', + '30000000-0000-4000-8000-00000000000c', + '40000000-0000-4000-8000-00000000000c', + '50000000-0000-4000-8000-00000000000c', + '60000000-0000-4000-8000-00000000000c' +); + +DO $realtime_isolation$ +DECLARE + runtime_record record; + target_ordinal integer; + target_schema text; + function_signature text; + function_oid oid; + should_have_access boolean; +BEGIN + FOR runtime_record IN + SELECT ordinal, role_name::text + FROM pg_temp.ctf_runtime_roles + ORDER BY ordinal + LOOP + FOR target_ordinal IN 1..3 + LOOP + target_schema := format( + 'ctf_%s_realtime', + chr(ascii('a') + target_ordinal - 1) + ); + should_have_access := runtime_record.ordinal = target_ordinal; + + IF pg_catalog.has_schema_privilege( + runtime_record.role_name, + target_schema, + 'USAGE' + ) IS DISTINCT FROM should_have_access THEN + RAISE EXCEPTION 'CTF_REALTIME_SCHEMA_ISOLATION_FAILED:%:%', + runtime_record.role_name, + target_schema; + END IF; + IF pg_catalog.has_schema_privilege( + runtime_record.role_name, + target_schema, + 'CREATE' + ) THEN + RAISE EXCEPTION 'CTF_REALTIME_SCHEMA_CREATE_FORBIDDEN:%:%', + runtime_record.role_name, + target_schema; + END IF; + + FOREACH function_signature IN ARRAY ARRAY[ + 'touch_listener(text)', + 'drain_changes(text,integer)', + 'cleanup_ephemeral(text)' + ] + LOOP + function_oid := pg_catalog.to_regprocedure( + format('%I.%s', target_schema, function_signature) + ); + IF function_oid IS NULL THEN + RAISE EXCEPTION 'CTF_REALTIME_FUNCTION_MISSING:%:%', + target_schema, + function_signature; + END IF; + IF pg_catalog.has_function_privilege( + runtime_record.role_name, + function_oid, + 'EXECUTE' + ) IS DISTINCT FROM should_have_access THEN + RAISE EXCEPTION 'CTF_REALTIME_FUNCTION_ISOLATION_FAILED:%:%:%', + runtime_record.role_name, + target_schema, + function_signature; + END IF; + END LOOP; + END LOOP; + END LOOP; +END +$realtime_isolation$; + +CREATE FUNCTION ctf_control.apply_schema_drift(target_schema text) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog +AS $function$ +DECLARE + target_database_id uuid; +BEGIN + target_database_id := CASE target_schema + WHEN 'ctf_a' THEN '10000000-0000-4000-8000-00000000000a'::uuid + WHEN 'ctf_b' THEN '10000000-0000-4000-8000-00000000000b'::uuid + WHEN 'ctf_c' THEN '10000000-0000-4000-8000-00000000000c'::uuid + ELSE NULL + END; + IF target_database_id IS NULL THEN + RAISE EXCEPTION 'CTF_DRIFT_SCHEMA_NOT_ALLOWED:%', target_schema; + END IF; + PERFORM pg_catalog.set_config( + 'jwt.claims.database_id', + target_database_id::text, + true + ); + EXECUTE format('ALTER TABLE %I.documents ADD COLUMN IF NOT EXISTS drift_probe text NOT NULL DEFAULT %L', target_schema, 'drift-applied'); + EXECUTE format('UPDATE %I.schema_state SET epoch = 2 WHERE id = 1', target_schema); +END +$function$; + +CREATE FUNCTION ctf_control.revert_schema_drift(target_schema text) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog +AS $function$ +DECLARE + target_database_id uuid; +BEGIN + target_database_id := CASE target_schema + WHEN 'ctf_a' THEN '10000000-0000-4000-8000-00000000000a'::uuid + WHEN 'ctf_b' THEN '10000000-0000-4000-8000-00000000000b'::uuid + WHEN 'ctf_c' THEN '10000000-0000-4000-8000-00000000000c'::uuid + ELSE NULL + END; + IF target_database_id IS NULL THEN + RAISE EXCEPTION 'CTF_DRIFT_SCHEMA_NOT_ALLOWED:%', target_schema; + END IF; + PERFORM pg_catalog.set_config( + 'jwt.claims.database_id', + target_database_id::text, + true + ); + EXECUTE format('ALTER TABLE %I.documents DROP COLUMN IF EXISTS drift_probe', target_schema); + EXECUTE format('UPDATE %I.schema_state SET epoch = 1 WHERE id = 1', target_schema); +END +$function$; + +REVOKE ALL ON ALL FUNCTIONS IN SCHEMA ctf_control FROM PUBLIC; + +DO $grants$ +DECLARE + runtime_role text; +BEGIN + FOR runtime_role IN + SELECT role_name::text + FROM pg_temp.ctf_runtime_roles + ORDER BY ordinal + LOOP + EXECUTE format('GRANT USAGE ON SCHEMA ctf_extensions, jwt_private TO %I', runtime_role); + EXECUTE format( + 'GRANT SELECT ON ALL TABLES IN SCHEMA ctf_extensions TO %I', + runtime_role + ); + EXECUTE format( + 'GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA ctf_extensions TO %I', + runtime_role + ); + EXECUTE format( + 'GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA ctf_extensions TO %I', + runtime_role + ); + EXECUTE format('GRANT EXECUTE ON FUNCTION jwt_private.current_database_id() TO %I', runtime_role); + END LOOP; +END +$grants$; + +RESET search_path; diff --git a/research/graphile-density/complete-tenant-fixture/schema.test.cjs b/research/graphile-density/complete-tenant-fixture/schema.test.cjs new file mode 100644 index 0000000000..2f58e78a37 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/schema.test.cjs @@ -0,0 +1,95 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); + +const sql = fs.readFileSync(path.join(__dirname, 'schema.sql'), 'utf8'); + +test('schema requires three distinct pre-existing least-privilege logins', () => { + for (const suffix of ['a', 'b', 'c']) { + assert.match(sql, new RegExp(`\\{\\?runtime_role_${suffix}\\}`)); + } + assert.match(sql, /CTF_RUNTIME_ROLES_MUST_BE_DISTINCT/); + assert.match(sql, /NOT role_record\.rolcanlogin/); + assert.match(sql, /role_record\.rolinherit/); + assert.match(sql, /role_record\.rolsuper/); + assert.match(sql, /role_record\.rolbypassrls/); + assert.match(sql, /CREATE TEMP TABLE ctf_runtime_roles/); + assert.doesNotMatch(sql, /CREATE TEMP TABLE ctf_runtime_roles[\s\S]*?ON COMMIT DROP/); + assert.match(sql, /FROM pg_temp\.ctf_runtime_roles/); + const doBlocks = [...sql.matchAll(/DO (\$[^$]+\$)([\s\S]*?)\1;/g)]; + assert.ok(doBlocks.length >= 3); + for (const [, , body] of doBlocks) { + assert.doesNotMatch(body, /:'runtime_role_[abc]'/); + } +}); + +test('runtime grants stay tenant-local and exclude drift control', () => { + assert.match(sql, /GRANT USAGE ON SCHEMA %I TO %I/); + assert.match(sql, /GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA %I TO %I/); + assert.doesNotMatch(sql, /GRANT[^;]+ctf_control[^;]+runtime_role/is); + assert.doesNotMatch(sql, /GRANT[^;]+apply_schema_drift/is); + assert.doesNotMatch(sql, /GRANT[^;]+revert_schema_drift/is); + assert.match(sql, /REVOKE ALL ON ALL FUNCTIONS IN SCHEMA ctf_control FROM PUBLIC/); +}); + +test('realtime startup functions are isolated in per-tenant least-privilege schemas', () => { + assert.match(sql, /realtime_schema_name := schema_name \|\| '_realtime'/); + assert.match(sql, /CREATE FUNCTION %I\.touch_listener\(node_id text\)/); + assert.match(sql, /CREATE FUNCTION %I\.drain_changes\(node_id text, batch_limit integer\) RETURNS SETOF jsonb/); + assert.match(sql, /CREATE FUNCTION %I\.cleanup_ephemeral\(node_id text\)/); + assert.match(sql, /REVOKE ALL ON SCHEMA %I FROM PUBLIC/); + assert.match(sql, /REVOKE ALL ON ALL FUNCTIONS IN SCHEMA %I FROM PUBLIC/); + assert.match(sql, /GRANT USAGE ON SCHEMA %I TO %I/); + assert.match(sql, /GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I/); + assert.match(sql, /CTF_REALTIME_SCHEMA_ISOLATION_FAILED/); + assert.match(sql, /CTF_REALTIME_FUNCTION_ISOLATION_FAILED/); + assert.match(sql, /CTF_REALTIME_SCHEMA_CREATE_FORBIDDEN/); +}); + +test('PostGIS dependency views execute with invoker rights and no public grant', () => { + for (const view of ['geometry_columns', 'geography_columns']) { + assert.match( + sql, + new RegExp(`ALTER VIEW ctf_extensions\\.${view} SET \\(security_invoker = true\\)`), + ); + assert.match(sql, new RegExp(`REVOKE ALL ON ctf_extensions\\.${view} FROM PUBLIC`)); + } +}); + +test('extension defaults do not grant the notification-only role application access', () => { + assert.match(sql, /REVOKE ALL ON SCHEMA public FROM PUBLIC/); + assert.match(sql, /REVOKE ALL ON SCHEMA ctf_extensions FROM PUBLIC/); + assert.match(sql, /REVOKE ALL ON ALL FUNCTIONS IN SCHEMA ctf_extensions FROM PUBLIC/); + assert.match(sql, /GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA ctf_extensions TO %I/); +}); + +test('session mutators are volatile and drift mutators restore an RLS identity', () => { + assert.match(sql, /savepoint_identity\(\) RETURNS text LANGUAGE plpgsql VOLATILE/); + assert.match(sql, /poison_session\(\) RETURNS text LANGUAGE plpgsql VOLATILE/); + assert.match( + sql, + /physical_database_identity\(\) RETURNS text LANGUAGE sql STABLE PARALLEL SAFE SECURITY INVOKER SET search_path = pg_catalog/, + ); + assert.match(sql, /SELECT pg_catalog\.current_database\(\)::text/); + assert.equal((sql.match(/PERFORM pg_catalog\.set_config\(/g) ?? []).length, 2); + assert.equal((sql.match(/SECURITY DEFINER\nSET search_path = pg_catalog/g) ?? []).length, 2); +}); + +test('the first upload must provision a physical bucket through the system lane', () => { + assert.match( + sql, + /INSERT INTO %I\.app_buckets \(id, key, type, allowed_mime_types, max_file_size, physical_name\) VALUES \(%L::uuid, %L, %L, ARRAY\[%L\], 1048576, NULL\)/, + ); + for (const table of ['app_buckets', 'app_files']) { + assert.ok(sql.includes(`'${table}'`)); + } + assert.match(sql, /ALTER TABLE %I\.%I FORCE ROW LEVEL SECURITY/); +}); + +test('realtime trigger returns the correct transition row for DELETE and writes', () => { + assert.match(sql, /IF TG_OP = 'DELETE' THEN[\s\S]+RETURN OLD;[\s\S]+RETURN NEW;/); + assert.doesNotMatch(sql, /coalesce\(NEW, OLD\)/i); +}); diff --git a/research/graphile-density/complete-tenant-fixture/server.cjs b/research/graphile-density/complete-tenant-fixture/server.cjs new file mode 100644 index 0000000000..08988d09d4 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/server.cjs @@ -0,0 +1,2007 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const { createRequire } = require('node:module'); +const path = require('node:path'); + +const { + REPO_ROOT, + TENANTS, + parseArgs, + parsePositiveInteger, + requireString, +} = require('./lib.cjs'); + +const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost']); +const RUNTIME_DEPENDENCY_SCHEMAS = Object.freeze(['ctf_extensions', 'jwt_private']); +const INTROSPECTION_DEPENDENCY_SCHEMAS = Object.freeze(['ctf_extensions']); +const realtimeSchemaFor = (tenant) => `${tenant.schema}_realtime`; +const runtimeDependencySchemasFor = (tenant, enableRealtime) => [ + ...RUNTIME_DEPENDENCY_SCHEMAS, + ...(enableRealtime ? [realtimeSchemaFor(tenant)] : []), +]; +const matchTenantUpgradePath = (rawUrl, pathPrefix = '') => { + if ( + typeof rawUrl !== 'string' + || typeof pathPrefix !== 'string' + || rawUrl.includes('?') + || (pathPrefix !== '' && (!pathPrefix.startsWith('/') || pathPrefix.endsWith('/'))) + || pathPrefix.includes('..') + ) return null; + const basePath = `${pathPrefix}/tenant/`; + if (!rawUrl.startsWith(basePath)) return null; + const parts = rawUrl.slice(basePath.length).split('/'); + return parts.length === 2 && parts[1] === 'graphql' && /^[a-z0-9-]+$/.test(parts[0]) + ? parts[0] + : null; +}; +const GRAFAST_CACHE_LIMITS = Object.freeze({ + queryCacheMaxLength: 8, + operationsCacheMaxLength: 8, + operationOperationPlansCacheMaxLength: 8, +}); +const RELEASE_BUILD_STATE_AFTER_VALIDATION = true; +const FEATURE_SETTINGS = Object.freeze({ + enableAggregates: false, + enablePostgis: true, + enableSearch: true, + enableDirectUploads: true, + enablePresignedUploads: true, + enableManyToMany: true, + enableConnectionFilter: true, + enableLtree: true, + enableLlm: true, + enableRealtime: true, + enableBulk: true, + enableI18n: true, + enableHistory: false, +}); +const RUNTIME_ARTIFACT_PATHS = Object.freeze([ + 'graphile/graphile-cache/dist/index.js', + 'graphile/graphile-cache/dist/create-instance.js', + 'graphile/graphile-cache/dist/graphile-cache.js', + 'graphile/graphile-cache/dist/http-adapter.js', + 'graphile/graphile-cache/dist/preset-services.js', + 'graphile/graphile-cache/dist/realtime-readiness.js', + 'graphile/graphile-realtime-subscriptions/dist/index.js', + 'graphile/graphile-realtime-subscriptions/dist/cursor-tracker.js', + 'graphile/graphile-realtime-subscriptions/dist/realtime-manager.js', + 'graphile/graphile-settings/dist/index.js', + 'postgres/pg-cache/dist/index.js', + 'packages/express-context/dist/index.js', + 'graphile/graphile-llm/dist/index.js', + 'graphile/graphile-function-bindings/dist/index.js', + 'graphile/graphile-presigned-url-plugin/dist/index.js', + 'graphql/server/dist/plugins/auth-cookie-plugin.js', + 'graphql/server/dist/middleware/graphile-build-contract.js', + 'graphql/server/dist/middleware/graphile-build-governor.js', + 'graphql/server/dist/middleware/runtime-role-safety.js', + 'graphql/server/dist/middleware/observability/graphile-build-stats.js', + 'graphql/server/dist/diagnostics/debug-memory-snapshot.js', +]); +const INSTALLED_RUNTIME_ARTIFACT_SPECS = Object.freeze([ + Object.freeze({ + label: 'installed:@dataplan/pg:dist/index.js', + resolveSpecifier: '@dataplan/pg', + relativePath: null, + markers: Object.freeze([ + 'exports.exactClientReleaseCapability = "dataplan-pg-exact-client-destroy-v1";', + ]), + }), + Object.freeze({ + label: 'installed:@dataplan/pg:dist/adaptors/pg.js', + resolveSpecifier: '@dataplan/pg/adaptors/pg', + relativePath: null, + markers: Object.freeze([ + 'const DESTROYABLE_CLIENT_RELEASE_MODES = Object.freeze(["reuse", "destroy"]);', + 'const supportsExactClientDestruction = typeof PgPool === "function" && pool instanceof PgPool;', + 'Exact PostgreSQL client destruction requires a node-postgres Pool', + 'pgClient.release(true);', + '? DESTROYABLE_CLIENT_RELEASE_MODES', + ]), + }), + Object.freeze({ + label: 'installed:@dataplan/pg:dist/pgServices.js', + resolveSpecifier: '@dataplan/pg', + relativePath: 'pgServices.js', + markers: Object.freeze([ + 'withPgClient.supportedClientReleaseModes = originalWithPgClient.supportedClientReleaseModes;', + 'const clientReleaseMode = options?.clientReleaseMode ?? "reuse";', + 'does not support exact client destruction', + ]), + }), + Object.freeze({ + label: 'installed:graphile-build-pg:dist/index.js', + resolveSpecifier: 'graphile-build-pg', + relativePath: null, + markers: Object.freeze([ + 'exports.introspectionClientReleaseCapability = "graphile-build-pg-exact-client-destroy-v1";', + ]), + }), + Object.freeze({ + label: 'installed:graphile-build-pg:dist/plugins/PgIntrospectionPlugin.js', + resolveSpecifier: 'graphile-build-pg', + relativePath: 'plugins/PgIntrospectionPlugin.js', + markers: Object.freeze([ + 'pgService.introspectionClientReleaseMode ?? "reuse"', + 'clientReleaseMode === "reuse" ? undefined : { clientReleaseMode }', + ]), + }), +]); + +const requireBuilt = (relativePath) => { + const absolutePath = path.join(REPO_ROOT, relativePath); + if (!fs.existsSync(absolutePath)) { + throw new Error(`CTF_BUILD_ARTIFACT_MISSING:${relativePath}`); + } + return require(absolutePath); +}; + +const readInstalledRuntimeArtifacts = () => { + const graphileSettingsEntry = path.join( + REPO_ROOT, + 'graphile/graphile-settings/dist/index.js', + ); + const graphileSettingsRequire = createRequire(graphileSettingsEntry); + let postgraphilePgEntry; + try { + postgraphilePgEntry = graphileSettingsRequire.resolve('postgraphile/adaptors/pg'); + } catch { + throw new Error('CTF_INSTALLED_RUNTIME_ARTIFACT_MISSING:postgraphile/adaptors/pg'); + } + const postgraphileRequire = createRequire(postgraphilePgEntry); + + return INSTALLED_RUNTIME_ARTIFACT_SPECS.map((spec) => { + let resolvedEntry; + try { + resolvedEntry = postgraphileRequire.resolve(spec.resolveSpecifier); + } catch { + throw new Error(`CTF_INSTALLED_RUNTIME_ARTIFACT_MISSING:${spec.label}`); + } + const absolutePath = spec.relativePath === null + ? resolvedEntry + : path.join(path.dirname(resolvedEntry), spec.relativePath); + if (!fs.existsSync(absolutePath)) { + throw new Error(`CTF_INSTALLED_RUNTIME_ARTIFACT_MISSING:${spec.label}`); + } + const bytes = fs.readFileSync(absolutePath); + const source = bytes.toString('utf8'); + spec.markers.forEach((marker, markerIndex) => { + if (!source.includes(marker)) { + throw new Error( + `CTF_INSTALLED_RUNTIME_MARKER_MISSING:${spec.label}:${markerIndex}` + ); + } + }); + return { bytes, spec }; + }); +}; + +const installedRuntimeArtifactManifest = () => readInstalledRuntimeArtifacts().map( + ({ bytes, spec }) => ({ + label: spec.label, + sha256: `sha256:${crypto.createHash('sha256').update(bytes).digest('hex')}`, + markerSetSha256: `sha256:${crypto.createHash('sha256') + .update(JSON.stringify(spec.markers)) + .digest('hex')}`, + markerCount: spec.markers.length, + }), +); + +const STATIC_REQUIRE_PATTERN = /\brequire(?:\.resolve)?\(\s*(['"])([^'"\r\n]+)\1\s*\)/g; + +const localDistRelativePath = (absolutePath) => { + let realPath; + try { + realPath = fs.realpathSync(absolutePath); + } catch { + return null; + } + const relativePath = path.relative(REPO_ROOT, realPath); + if ( + relativePath.startsWith('..') + || path.isAbsolute(relativePath) + || relativePath.split(path.sep).includes('node_modules') + || !relativePath.split(path.sep).includes('dist') + || !/\.(?:c|m)?js$/.test(relativePath) + ) return null; + return relativePath.split(path.sep).join('/'); +}; + +const staticRequireSpecifiers = (source) => { + const specifiers = new Set(); + for (const match of source.matchAll(STATIC_REQUIRE_PATTERN)) specifiers.add(match[2]); + return [...specifiers].sort(); +}; + +const resolvedLocalRuntimeArtifactManifest = () => { + const queue = RUNTIME_ARTIFACT_PATHS.map((relativePath) => { + const absolutePath = path.join(REPO_ROOT, relativePath); + if (!fs.existsSync(absolutePath)) { + throw new Error(`CTF_BUILD_ARTIFACT_MISSING:${relativePath}`); + } + return fs.realpathSync(absolutePath); + }); + const artifacts = new Map(); + while (queue.length > 0) { + const absolutePath = queue.shift(); + const relativePath = localDistRelativePath(absolutePath); + if (!relativePath || artifacts.has(relativePath)) continue; + const bytes = fs.readFileSync(absolutePath); + artifacts.set(relativePath, { + path: relativePath, + sha256: `sha256:${crypto.createHash('sha256').update(bytes).digest('hex')}`, + }); + const localRequire = createRequire(absolutePath); + for (const specifier of staticRequireSpecifiers(bytes.toString('utf8'))) { + let resolved; + try { + resolved = localRequire.resolve(specifier); + } catch { + // Generated source can contain documentation examples and optional + // package probes. Only successfully resolved JavaScript can belong to + // the concrete runtime closure for this installation. + continue; + } + if (localDistRelativePath(resolved)) queue.push(fs.realpathSync(resolved)); + } + } + return [...artifacts.values()].sort((left, right) => left.path.localeCompare(right.path)); +}; + +const loadInstalledDataplanPgAdaptor = () => { + const graphileSettingsEntry = path.join( + REPO_ROOT, + 'graphile/graphile-settings/dist/index.js', + ); + const graphileSettingsRequire = createRequire(graphileSettingsEntry); + let postgraphilePgEntry; + try { + postgraphilePgEntry = graphileSettingsRequire.resolve('postgraphile/adaptors/pg'); + } catch { + throw new Error('CTF_INSTALLED_RUNTIME_ARTIFACT_MISSING:postgraphile/adaptors/pg'); + } + const postgraphileRequire = createRequire(postgraphilePgEntry); + let adaptor; + try { + adaptor = postgraphileRequire('@dataplan/pg/adaptors/pg'); + } catch { + throw new Error( + 'CTF_INSTALLED_RUNTIME_ARTIFACT_MISSING:installed:@dataplan/pg:dist/adaptors/pg.js' + ); + } + if (typeof adaptor?.makePgAdaptorWithPgClient !== 'function') { + throw new Error('CTF_DATAPLAN_PREPARED_STATEMENT_ATTESTATION_UNAVAILABLE'); + } + return adaptor; +}; + +let cachedRuntimeArtifactManifest = null; +let cachedRuntimeArtifactFingerprint = null; +const runtimeArtifactManifest = () => { + if (!cachedRuntimeArtifactManifest) { + cachedRuntimeArtifactManifest = Object.freeze({ + version: 2, + roots: Object.freeze([...RUNTIME_ARTIFACT_PATHS]), + localDistClosure: Object.freeze(resolvedLocalRuntimeArtifactManifest()), + installedPatchedArtifacts: Object.freeze(installedRuntimeArtifactManifest()), + }); + } + return cachedRuntimeArtifactManifest; +}; + +const runtimeArtifactFingerprint = () => { + if (!cachedRuntimeArtifactFingerprint) { + cachedRuntimeArtifactFingerprint = `sha256:${crypto.createHash('sha256') + .update(JSON.stringify(runtimeArtifactManifest())) + .digest('hex')}`; + } + return cachedRuntimeArtifactFingerprint; +}; + +const parseBooleanArgument = (value, label, fallback = false) => { + if (value === undefined) return fallback; + if (value === true || value === 'true') return true; + if (value === 'false') return false; + throw new Error(`CTF_INVALID_BOOLEAN:${label}`); +}; + +const provisionAttestationSha256 = ({ + cloneId, + purpose, + customerId, + database, + nonce, +}) => { + const digest = crypto.createHash('sha256'); + for (const value of [ + 'physical-database-density-provision-attestation-v1', + cloneId, + purpose, + customerId, + database, + nonce, + ]) { + digest.update(value); + digest.update('\0'); + } + return `sha256:${digest.digest('hex')}`; +}; + +const validateExpectedProvisionAttestation = (value, customerId, database) => { + if (value == null) return null; + if ( + JSON.stringify(Object.keys(value).sort()) + !== JSON.stringify(['cloneId', 'purpose', 'sha256', 'version']) + || + value?.version !== 1 + || typeof value.cloneId !== 'string' + || !/^[a-z0-9][a-z0-9._-]{0,127}$/i.test(value.cloneId) + || (value.purpose !== 'hostile-preflight' && value.purpose !== 'measurement') + || typeof customerId !== 'string' + || !/^[a-z0-9-]+$/.test(customerId) + || typeof database !== 'string' + || !database + || !/^sha256:[a-f0-9]{64}$/.test(value.sha256 ?? '') + ) { + throw new Error('CTF_PROVISION_ATTESTATION_EXPECTATION_INVALID'); + } + return value; +}; + +const hostileControlEnabledFor = (runPurpose, expectedProvisionAttestation) => + expectedProvisionAttestation == null || runPurpose === 'hostile-preflight'; + +const CONTROL_POOL_MAX = 1; +const PREPARED_STATEMENT_ATTESTATION_KIND = + 'loaded-dataplan-adaptor-behavior-v1'; + +const parseRuntimePoolMaxUses = (value, label = 'runtime-pool-max-uses') => { + if (value === 'unlimited') return null; + if (typeof value !== 'string' || !/^[1-9]\d*$/.test(value)) { + throw new Error(`CTF_INVALID_MAX_USES:${label}`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`CTF_INVALID_MAX_USES:${label}`); + } + return parsed; +}; + +const fixtureConfigurationIdentity = ({ + databaseName, + mode, + introspectionClientReleaseMode, + enableRealtime, + realtimeNotificationMode, + realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs, + runtimeFingerprint, +}) => { + if ( + typeof databaseName !== 'string' + || databaseName.length === 0 + || typeof runtimeFingerprint !== 'string' + || !/^sha256:[a-f0-9]{64}$/.test(runtimeFingerprint) + ) { + throw new Error('CTF_CONFIGURATION_IDENTITY_INPUT_INVALID'); + } + const input = { + version: 1, + fixture: 'complete-tenant-abc-v1', + databaseName, + mode, + introspectionClientReleaseMode, + enableRealtime, + realtimeNotificationMode: enableRealtime ? realtimeNotificationMode : null, + realtimeCursorPollIntervalMs: enableRealtime + ? realtimeCursorPollIntervalMs + : null, + realtimeCursorHeartbeatIntervalMs: enableRealtime + ? realtimeCursorHeartbeatIntervalMs + : null, + runtimeFingerprint, + featureSettings: FEATURE_SETTINGS, + grafastCache: GRAFAST_CACHE_LIMITS, + releaseBuildStateAfterValidation: RELEASE_BUILD_STATE_AFTER_VALIDATION, + }; + return `graphile-configuration:ctf:v1:${crypto.createHash('sha256') + .update(JSON.stringify(input)) + .digest('hex')}`; +}; + +const credentialFreeContractEvidence = (kind, input) => ({ + version: 1, + fingerprint: `${kind}:v1:${crypto.createHash('sha256') + .update(JSON.stringify(input)) + .digest('hex')}`, + input, +}); + +const runtimePoolContractEvidence = ({ + databaseName, + role, + poolMax, + poolMaxUses, + runtimeFingerprint, + purpose = 'runtime', + sanitizeOnCheckout = true, +}) => credentialFreeContractEvidence('pg-contract-evidence', { + version: 1, + databaseName, + role, + pool: { + max: poolMax, + maxUses: poolMaxUses, + }, + purpose, + sanitizeOnCheckout, + runtimeFingerprint, +}); + +const preparedResetBackendEvidence = ( + firstBackendPid, + secondBackendPid, + runtimePoolMaxUses, +) => { + const pidsValid = Number.isSafeInteger(firstBackendPid) + && firstBackendPid > 0 + && Number.isSafeInteger(secondBackendPid) + && secondBackendPid > 0; + const observed = !pidsValid + ? 'invalid' + : firstBackendPid === secondBackendPid + ? 'same-client' + : 'rotated-client'; + const expected = runtimePoolMaxUses === null + ? 'same-client' + : runtimePoolMaxUses === 1 + ? 'rotated-client' + : 'unsupported'; + return { + firstBackendPid: pidsValid ? firstBackendPid : null, + secondBackendPid: pidsValid ? secondBackendPid : null, + observed, + expected, + exact: pidsValid && expected !== 'unsupported' && observed === expected, + }; +}; + +const nativePoolMaxUses = (pool) => { + const value = pool?.options?.maxUses; + if (value === Number.POSITIVE_INFINITY) return { known: true, value: null }; + if (Number.isSafeInteger(value) && value > 0) return { known: true, value }; + return { known: false, value: null }; +}; + +const makeRuntimePoolStats = (pgCache, runtimePoolIdentities, requestedMaxUses) => { + const identities = [...runtimePoolIdentities]; + const distinctIdentities = new Set(identities); + const identitiesUnique = distinctIdentities.size === identities.length; + const recordsAvailable = pgCache?.records instanceof Map; + const records = recordsAvailable + ? identities.map((identity) => pgCache.records.get(identity) ?? null) + : identities.map(() => null); + const pools = records.map((record) => record?.pool ?? null); + const observedPoolObjects = pools.filter(Boolean); + const distinctPoolObjects = new Set(observedPoolObjects); + const poolObjectsUnique = distinctPoolObjects.size === observedPoolObjects.length; + const countsAvailable = pools.every((pool) => + pool + && typeof pool.totalCount === 'number' + && typeof pool.idleCount === 'number' + && typeof pool.waitingCount === 'number' + ); + const effective = pools.map(nativePoolMaxUses); + const effectiveKnown = effective.every((entry) => entry.known); + const effectiveValues = effectiveKnown + ? [...new Set(effective.map((entry) => entry.value))] + : []; + const effectiveMaxUsesKnown = effectiveValues.length === 1; + const effectiveMaxUses = effectiveMaxUsesKnown ? effectiveValues[0] : null; + const observedPools = distinctPoolObjects.size; + const available = recordsAvailable + && identitiesUnique + && poolObjectsUnique + && observedPools === identities.length + && countsAvailable + && effectiveMaxUsesKnown; + return { + scope: 'runtime-only-exact-identities', + available, + requestedMaxUses, + effectiveMaxUses, + effectiveMaxUsesKnown, + maxUsesExact: available && effectiveMaxUses === requestedMaxUses, + identitiesUnique, + poolObjectsUnique, + expectedPools: identities.length, + observedPools, + totalClients: available + ? pools.reduce((sum, pool) => sum + pool.totalCount, 0) + : null, + idleClients: available + ? pools.reduce((sum, pool) => sum + pool.idleCount, 0) + : null, + waitingClients: available + ? pools.reduce((sum, pool) => sum + pool.waitingCount, 0) + : null, + }; +}; + +const preparedStatementCacheRequestFromEnvironment = (environment) => { + const raw = environment.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE; + const normalized = typeof raw === 'string' ? raw.trim() : ''; + const requestedSize = normalized === '' ? 100 : Number(normalized); + if ( + !Number.isSafeInteger(requestedSize) + || requestedSize < 0 + || requestedSize > 10_000 + || (raw != null && ( + typeof raw !== 'string' + || String(requestedSize) !== raw + )) + ) { + throw new Error('CTF_PREPARED_STATEMENT_CACHE_SIZE_INVALID'); + } + return { + environmentValue: typeof raw === 'string' ? raw : null, + requestedSize, + environmentCanonical: typeof raw === 'string' + && String(requestedSize) === raw, + }; +}; + +const attestDataplanPreparedStatementCache = async (adaptor, request) => { + if (typeof adaptor?.makePgAdaptorWithPgClient !== 'function') { + throw new Error('CTF_DATAPLAN_PREPARED_STATEMENT_ATTESTATION_UNAVAILABLE'); + } + const requestedSize = request?.requestedSize; + if (!Number.isSafeInteger(requestedSize) || requestedSize < 0 || requestedSize > 10_000) { + throw new Error('CTF_PREPARED_STATEMENT_CACHE_SIZE_INVALID'); + } + + const parsedStatements = Object.create(null); + const namedQueries = []; + const deallocations = []; + let releases = 0; + const rawClient = { + connection: { parsedStatements }, + addListener() {}, + removeListener() {}, + escapeIdentifier(identifier) { + return `"${String(identifier).replaceAll('"', '""')}"`; + }, + query(query) { + if (typeof query === 'string') { + if (query.startsWith('deallocate ')) { + deallocations.push({ + afterNamedQueries: namedQueries.length, + sql: query, + }); + } + return Promise.resolve({ rows: [], rowCount: 0 }); + } + if (typeof query?.name === 'string') { + namedQueries.push(query.name); + parsedStatements[query.name] = query.text; + } + return Promise.resolve({ rows: [], rowCount: 0 }); + }, + release() { + releases += 1; + }, + }; + const pool = { connect: async () => rawClient }; + const withPgClient = adaptor.makePgAdaptorWithPgClient(pool); + const queryCount = Math.max(1, requestedSize + 1); + await withPgClient(null, async (client) => { + for (let index = 0; index < queryCount; index += 1) { + await client.query({ + text: `select ${index}`, + name: `ctf_prepared_cache_attestation_${index}`, + values: [], + arrayMode: false, + }); + } + }); + // Dataplan's LRU disposer intentionally performs DEALLOCATE asynchronously. + // One turn lets its bookkeeping settle before this proof is published. + await new Promise((resolve) => setImmediate(resolve)); + + const firstEvictionAfterNamedQueries = deallocations[0]?.afterNamedQueries ?? null; + const effectiveSize = namedQueries.length === 0 + ? 0 + : firstEvictionAfterNamedQueries; + const expectedFirstName = 'ctf_prepared_cache_attestation_0'; + const exact = releases === 1 + && ( + requestedSize === 0 + ? namedQueries.length === 0 + && deallocations.length === 0 + && rawClient.connection._graphilePreparedStatementCache == null + : namedQueries.length === queryCount + && deallocations.length === 1 + && firstEvictionAfterNamedQueries === requestedSize + && deallocations[0].sql === `deallocate ${rawClient.escapeIdentifier(expectedFirstName)}` + && rawClient.connection._graphilePreparedStatementCache != null + ); + return { + ...request, + attestation: PREPARED_STATEMENT_ATTESTATION_KIND, + effectiveSize, + effectiveSizeKnown: effectiveSize != null, + exact, + namedQueriesObserved: namedQueries.length, + firstEvictionAfterNamedQueries, + }; +}; + +const parseServerOptions = (argv, environment = process.env) => { + const args = parseArgs(argv); + const host = requireString(args, 'host', '127.0.0.1'); + if (!LOOPBACK_HOSTS.has(host)) throw new Error('CTF_SERVER_LOOPBACK_REQUIRED'); + const mode = requireString(args, 'mode', 'scoped-required'); + if (!['stock', 'scoped-required'].includes(mode)) { + throw new Error(`CTF_INTROSPECTION_MODE_INVALID:${mode}`); + } + const introspectionClientReleaseMode = requireString( + args, + 'introspection-client-release-mode', + 'destroy', + ); + if (!['reuse', 'destroy'].includes(introspectionClientReleaseMode)) { + throw new Error( + `CTF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:${introspectionClientReleaseMode}` + ); + } + const runtimeRoles = Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + requireString( + args, + tenant.runtimeRoleArgument, + environment[`CTF_RUNTIME_${tenant.id.toUpperCase()}_PGUSER`], + ), + ])); + if (new Set(Object.values(runtimeRoles)).size !== TENANTS.length) { + throw new Error('CTF_RUNTIME_ROLES_MUST_BE_DISTINCT'); + } + for (const tenant of TENANTS) { + const password = environment[tenant.runtimePasswordEnvironment] + ?? environment.GRAPHQL_RUNTIME_PGPASSWORD; + if (typeof password !== 'string' || password.length === 0) { + throw new Error(`CTF_RUNTIME_PASSWORD_REQUIRED:${tenant.runtimePasswordEnvironment}`); + } + } + const runtimePoolMax = parsePositiveInteger( + args['runtime-pool-max'] ?? '1', + 'runtime-pool-max', + ); + const runtimePoolMaxUses = parseRuntimePoolMaxUses( + args['runtime-pool-max-uses'] ?? 'unlimited', + ); + const enableRealtime = parseBooleanArgument(args['enable-realtime'], 'enable-realtime'); + const realtimeNotificationMode = requireString( + args, + 'realtime-notification-mode', + 'dedicated', + ); + if (!['dedicated', 'shared-exact'].includes(realtimeNotificationMode)) { + throw new Error( + `CTF_REALTIME_NOTIFICATION_MODE_INVALID:${realtimeNotificationMode}` + ); + } + if (!enableRealtime && realtimeNotificationMode !== 'dedicated') { + throw new Error('CTF_SHARED_REALTIME_REQUIRES_REALTIME'); + } + // Dedicated mode pins the runtime pool's PgSubscriber connection. Shared + // exact mode uses a separate one-client notification pool, so max=1 remains + // a valid runtime density arm. + if (enableRealtime && realtimeNotificationMode === 'dedicated' && runtimePoolMax < 2) { + throw new Error('CTF_REALTIME_REQUIRES_RUNTIME_POOL_MAX_2'); + } + const notificationRole = realtimeNotificationMode === 'shared-exact' + ? requireString( + args, + 'notification-role', + environment.CTF_NOTIFICATION_PGUSER, + ) + : null; + let notificationPasswordAvailable = realtimeNotificationMode === 'shared-exact' + ? environment.CTF_NOTIFICATION_PGPASSWORD + : null; + if ( + realtimeNotificationMode === 'shared-exact' + && ( + typeof notificationPasswordAvailable !== 'string' + || notificationPasswordAvailable.length === 0 + ) + ) { + throw new Error('CTF_NOTIFICATION_PASSWORD_REQUIRED'); + } + if (notificationRole && new Set(Object.values(runtimeRoles)).has(notificationRole)) { + throw new Error('CTF_NOTIFICATION_ROLE_MUST_BE_DISTINCT'); + } + // Keep the listener credential out of the serializable options object. The + // closure exposes exactly one read and cannot enumerate the surrounding + // environment or any runtime-role credential. + const takeNotificationPassword = () => { + const value = notificationPasswordAvailable; + notificationPasswordAvailable = null; + if (realtimeNotificationMode === 'shared-exact' && !value) { + throw new Error('CTF_NOTIFICATION_PASSWORD_ALREADY_CONSUMED'); + } + return value; + }; + return { + host, + port: parsePositiveInteger(args.port ?? '3391', 'port'), + arm: requireString(args, 'arm', 'local-complete-tenant'), + mode, + introspectionClientReleaseMode, + runtimePoolMax, + runtimePoolMaxUses, + preparedStatementCacheRequest: + preparedStatementCacheRequestFromEnvironment(environment), + enableRealtime, + realtimeNotificationMode, + notificationRole, + takeNotificationPassword, + realtimeCursorPollIntervalMs: parsePositiveInteger( + args['realtime-cursor-poll-ms'] ?? '5000', + 'realtime-cursor-poll-ms', + ), + realtimeCursorHeartbeatIntervalMs: parsePositiveInteger( + args['realtime-cursor-heartbeat-ms'] ?? '30000', + 'realtime-cursor-heartbeat-ms', + ), + runtimeRoles, + controlToken: typeof environment.CTF_CONTROL_TOKEN === 'string' + ? environment.CTF_CONTROL_TOKEN + : '', + }; +}; + +const timingSafeTokenEqual = (candidate, expected) => { + if (!candidate || !expected) return false; + const actualBytes = Buffer.from(candidate); + const expectedBytes = Buffer.from(expected); + return actualBytes.length === expectedBytes.length + && crypto.timingSafeEqual(actualBytes, expectedBytes); +}; + +const bearerToken = (request) => { + const value = request.get('authorization'); + return value?.startsWith('Bearer ') ? value.slice('Bearer '.length) : ''; +}; + +const isLoopbackRequest = (request) => { + const address = request.socket?.remoteAddress ?? ''; + return address === '127.0.0.1' || address === '::1' || address === '::ffff:127.0.0.1'; +}; + +const languageCodes = (request) => { + const header = request.get('accept-language') ?? ''; + const parsed = header + .split(',') + .map((part) => part.trim().split(';')[0]?.toLowerCase()) + .filter(Boolean) + .map((part) => part.split('-')[0]); + return [...new Set([...parsed, 'en'])].slice(0, 8); +}; + +const storageModuleFor = (tenant) => ({ + id: `30000000-0000-4000-8000-00000000000${tenant.id}`, + bucketsQualifiedName: `"${tenant.schema}"."app_buckets"`, + filesQualifiedName: `"${tenant.schema}"."app_files"`, + schemaName: tenant.schema, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + scope: 'app', + entityTableId: null, + entityQualifiedName: null, + endpoint: null, + publicUrlPrefix: null, + provider: 'minio', + allowedOrigins: ['http://127.0.0.1'], + uploadUrlExpirySeconds: 900, + downloadUrlExpirySeconds: 3600, + defaultMaxFileSize: 1024 * 1024, + maxFilenameLength: 1024, + cacheTtlSeconds: 300, + hasPathShares: false, + maxBulkFiles: 100, + maxBulkTotalSize: 1024 * 1024, +}); + +const computeFor = (tenant) => { + const module = { + schemaName: tenant.schema, + bindingsTableName: 'fixture_preloaded_bindings', + definitionsTableName: 'fixture_preloaded_definitions', + invocationsSchemaName: tenant.schema, + invocationsTableName: 'function_invocations', + invocationsEntityField: 'database_id', + }; + return { + modules: [module], + bindings: [{ + bindingId: `50000000-0000-4000-8000-00000000000${tenant.id}`, + alias: 'fixture_task', + config: { graphql: true }, + functionDefinitionId: `60000000-0000-4000-8000-00000000000${tenant.id}`, + taskIdentifier: `ctf.fixture.${tenant.id}`, + description: 'Complete-tenant fixture task', + payloadArgs: null, + module, + }], + }; +}; + +const pluginComputeFor = (compute) => ({ + modules: compute.modules.map((module) => ({ + computeSchema: module.schemaName, + bindingsTable: module.bindingsTableName, + definitionsTable: module.definitionsTableName, + invocationsSchema: module.invocationsSchemaName, + invocationsTable: module.invocationsTableName, + invocationsEntityField: module.invocationsEntityField, + })), + bindings: compute.bindings.map((binding) => ({ + ...binding, + module: { + computeSchema: binding.module.schemaName, + bindingsTable: binding.module.bindingsTableName, + definitionsTable: binding.module.definitionsTableName, + invocationsSchema: binding.module.invocationsSchemaName, + invocationsTable: binding.module.invocationsTableName, + invocationsEntityField: binding.module.invocationsEntityField, + }, + })), +}); + +const deterministicLlmPlugin = () => ({ + // Downstream LLM plugins declare an ordering dependency on this canonical + // name. The production module plugin is deliberately replaced because this + // lane must not acquire an external provider during an offline run. + name: 'LlmModulePlugin', + version: '1.0.0', + schema: { + hooks: { + build(build) { + const embedder = async () => ({ embedding: [1, 0, 0], promptTokens: 5 }); + const chatCompleter = async (messages) => { + const prompt = messages.find((message) => message.role === 'user')?.content ?? ''; + return { + content: `Deterministic fixture answer: ${prompt}`, + usage: { + input: 10, + output: 10, + reasoning: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 20, + }, + }; + }; + return build.extend(build, { + llmEmbedder: embedder, + llmChatCompleter: chatCompleter, + llmEmbeddingModel: 'ctf-deterministic-3d-v1', + llmChatModel: 'ctf-deterministic-chat-v1', + }, 'Complete-tenant deterministic LLM provider'); + }, + }, + }, +}); + +const loadRuntime = () => { + const express = require(path.join(REPO_ROOT, 'graphql/server/node_modules/express')); + const { S3Client } = require(path.join( + REPO_ROOT, + 'graphql/server/node_modules/@aws-sdk/client-s3', + )); + return { + express, + S3Client, + ...requireBuilt('graphile/graphile-cache/dist/index.js'), + realtimeSubscriptions: requireBuilt( + 'graphile/graphile-realtime-subscriptions/dist/index.js' + ), + graphileSettings: requireBuilt('graphile/graphile-settings/dist/index.js'), + pgCacheApi: requireBuilt('postgres/pg-cache/dist/index.js'), + expressContext: requireBuilt('packages/express-context/dist/index.js'), + llm: requireBuilt('graphile/graphile-llm/dist/index.js'), + functionBindings: requireBuilt('graphile/graphile-function-bindings/dist/index.js'), + presigned: requireBuilt('graphile/graphile-presigned-url-plugin/dist/index.js'), + authCookie: requireBuilt('graphql/server/dist/plugins/auth-cookie-plugin.js'), + dataplanPgAdaptor: loadInstalledDataplanPgAdaptor(), + pgEnv: require(path.join(REPO_ROOT, 'graphql/server/node_modules/pg-env')), + buildContractApi: requireBuilt('graphql/server/dist/middleware/graphile-build-contract.js'), + buildGovernor: requireBuilt('graphql/server/dist/middleware/graphile-build-governor.js'), + roleSafety: requireBuilt('graphql/server/dist/middleware/runtime-role-safety.js'), + buildStats: requireBuilt('graphql/server/dist/middleware/observability/graphile-build-stats.js'), + debugMemory: requireBuilt('graphql/server/dist/diagnostics/debug-memory-snapshot.js'), + }; +}; + +const createFixtureServer = async (options, environment = process.env) => { + const runtime = loadRuntime(); + const preparedStatementCache = await attestDataplanPreparedStatementCache( + runtime.dataplanPgAdaptor, + options.preparedStatementCacheRequest, + ); + if (!preparedStatementCache.exact) { + throw new Error('CTF_DATAPLAN_PREPARED_STATEMENT_CACHE_MISMATCH'); + } + const { + createGraphileInstance, + deleteGraphileCacheEntry, + disposeUncachedEntry, + graphileCache, + invokeEntryHandler, + invokeEntryUpgradeHandler, + prepareCacheForBuild, + getGraphileRealtimeRoleAuditStats, + revalidateEntryRealtimeRole, + } = runtime; + const { + acquirePgPool, + getPgNotificationBrokerIdentity, + getPgNotificationBrokerStats, + getPgPoolIdentity, + pgCache, + teardownPgNotificationBrokers, + teardownPgPools, + } = runtime.pgCacheApi; + const { createGraphileBuildContract, hashGraphileBuildContract } = runtime.buildContractApi; + const { runGraphileBuild, getGraphileGovernorCounters } = runtime.buildGovernor; + const { ensureRuntimeRoleSafety, invalidateRuntimeRoleSafety } = runtime.roleSafety; + const { observeGraphileBuild, getGraphileBuildStats } = runtime.buildStats; + const { getDebugMemorySnapshot } = runtime.debugMemory; + const { buildPgSettings } = runtime.expressContext; + const { + createConstructivePreset, + createGrafastCacheLimitsPreset, + makePgService, + } = runtime.graphileSettings; + const { + createLlmRagPlugin, + createLlmTextMutationPlugin, + createLlmTextSearchPlugin, + } = runtime.llm; + const { createFunctionBindingsPlugin } = runtime.functionBindings; + const { PresignedUrlPreset } = runtime.presigned; + const { AuthCookiePlugin } = runtime.authCookie; + const { + ActivatableGenerationScopedRealtimeSubscriber, + RealtimeTopicCollector, + } = runtime.realtimeSubscriptions; + + const controlPgConfig = { + ...runtime.pgEnv.getPgEnvOptions({}), + // Control-plane queries are trusted and short-lived. Keep their pool shape + // constant so runtime-pool experiments cannot alter the measured baseline. + pool: { max: CONTROL_POOL_MAX, maxUses: 0 }, + }; + const runtimeFingerprint = runtimeArtifactFingerprint(); + const configurationIdentity = fixtureConfigurationIdentity({ + databaseName: controlPgConfig.database, + mode: options.mode, + introspectionClientReleaseMode: options.introspectionClientReleaseMode, + enableRealtime: options.enableRealtime, + realtimeNotificationMode: options.realtimeNotificationMode, + realtimeCursorPollIntervalMs: options.realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs: + options.realtimeCursorHeartbeatIntervalMs, + runtimeFingerprint, + }); + if (!/^graphile-configuration:ctf:v1:[a-f0-9]{64}$/.test(configurationIdentity)) { + throw new Error('CTF_CONFIGURATION_IDENTITY_INVALID'); + } + const runtimePoolOptions = { purpose: 'runtime', sanitizeOnCheckout: true }; + const notificationPassword = options.realtimeNotificationMode === 'shared-exact' + ? options.takeNotificationPassword() + : null; + const notificationPgConfig = options.realtimeNotificationMode === 'shared-exact' + ? { + host: controlPgConfig.host, + port: controlPgConfig.port, + database: controlPgConfig.database, + user: options.notificationRole, + password: notificationPassword, + // The notification broker is long-lived and must never inherit the + // runtime-only single-checkout experiment from process environment. + pool: { max: 1, maxUses: 0 }, + } + : null; + const realtimeListenerIdentity = notificationPgConfig + ? getPgNotificationBrokerIdentity(notificationPgConfig) + : null; + const realtimeListenerContractEvidence = notificationPgConfig + ? runtimePoolContractEvidence({ + databaseName: controlPgConfig.database, + role: options.notificationRole, + poolMax: 1, + poolMaxUses: null, + runtimeFingerprint, + purpose: 'notification-listener', + sanitizeOnCheckout: false, + }) + : null; + const expectedProvisionAttestation = validateExpectedProvisionAttestation( + options.provisionAttestation, + options.provisionCustomerId, + controlPgConfig.database, + ); + if ( + expectedProvisionAttestation + && ( + options.runPurpose !== expectedProvisionAttestation.purpose + || options.cloneId !== expectedProvisionAttestation.cloneId + ) + ) { + throw new Error('CTF_PROVISION_ATTESTATION_RUN_MISMATCH'); + } + const hostileControlEnabled = hostileControlEnabledFor( + options.runPurpose, + expectedProvisionAttestation, + ); + const tenantState = new Map(); + const buildContractEvidenceByLiveIdentity = new Map(); + + for (const tenant of TENANTS) { + const role = options.runtimeRoles[tenant.id]; + const password = environment[tenant.runtimePasswordEnvironment] + ?? environment.GRAPHQL_RUNTIME_PGPASSWORD; + const pgConfig = { + host: controlPgConfig.host, + port: controlPgConfig.port, + database: controlPgConfig.database, + user: role, + password, + pool: { + max: options.runtimePoolMax, + // An explicit zero is pg-cache's unlimited sentinel and prevents an + // ambient PG_POOL_MAX_USES value from contaminating the baseline arm. + maxUses: options.runtimePoolMaxUses ?? 0, + }, + }; + const poolIdentity = getPgPoolIdentity(pgConfig, runtimePoolOptions); + const poolContractEvidence = runtimePoolContractEvidence({ + databaseName: controlPgConfig.database, + role, + poolMax: options.runtimePoolMax, + poolMaxUses: options.runtimePoolMaxUses, + runtimeFingerprint, + }); + const storage = { modules: [storageModuleFor(tenant)] }; + const compute = computeFor(tenant); + const realtimeSchema = realtimeSchemaFor(tenant); + const runtimeDependencySchemas = runtimeDependencySchemasFor( + tenant, + options.enableRealtime, + ); + const contractInput = { + configurationIdentity, + poolIdentity, + databaseId: tenant.databaseId, + databaseName: controlPgConfig.database, + apiId: tenant.apiId, + schemas: [tenant.schema], + authenticatedRole: role, + anonymousRole: role, + pluginSettings: FEATURE_SETTINGS, + graphileSettings: { + releaseBuildStateAfterValidation: RELEASE_BUILD_STATE_AFTER_VALIDATION, + introspectionMode: options.mode, + introspectionClientReleaseMode: options.introspectionClientReleaseMode, + introspectionDependencySchemas: [...INTROSPECTION_DEPENDENCY_SCHEMAS], + grafastCache: GRAFAST_CACHE_LIMITS, + realtimeNotificationMode: options.realtimeNotificationMode, + realtimeCursorPollIntervalMs: options.realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs: + options.realtimeCursorHeartbeatIntervalMs, + fixturePluginConfiguration: { + authCookie: true, + llmProvider: 'deterministic-3d-v1', + storageProvider: 'offline-signing-only-v1', + functionBindings: 'preloaded-v1', + runtimeFingerprint, + }, + }, + compute, + storage, + isPublic: false, + // The complete-tenant lane defaults this off and delegates delivery to + // the mandatory graphql-ws integration suite. Physical-database research + // lanes may opt in after provisioning the required realtime cursor schema. + enableRealtime: options.enableRealtime, + realtimeSchema, + realtimeNotificationMode: options.realtimeNotificationMode, + realtimeListenerPoolIdentity: realtimeListenerIdentity ?? undefined, + realtimeNotificationRoleRevalidationMs: 60_000, + realtimeCursorPollIntervalMs: options.realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs: + options.realtimeCursorHeartbeatIntervalMs, + graphiql: false, + graphiqlOnGraphQLGET: false, + explain: false, + introspectionMode: options.mode, + introspectionClientReleaseMode: options.introspectionClientReleaseMode, + }; + const contract = createGraphileBuildContract(contractInput); + const evidenceContract = createGraphileBuildContract({ + ...contractInput, + poolIdentity: poolContractEvidence.fingerprint, + realtimeListenerPoolIdentity: realtimeListenerContractEvidence?.fingerprint, + }); + const buildContractEvidence = credentialFreeContractEvidence( + 'graphile-contract-evidence', + evidenceContract, + ); + if ( + contract.surface.graphiql !== false + || contract.surface.graphiqlOnGraphQLGET !== false + || contract.surface.realtimeSchema !== (options.enableRealtime ? realtimeSchema : null) + || contract.surface.realtimeNotificationMode + !== (options.enableRealtime ? options.realtimeNotificationMode : null) + || contract.surface.realtimeListenerPoolIdentity + !== (options.enableRealtime && options.realtimeNotificationMode === 'shared-exact' + ? realtimeListenerIdentity + : null) + ) { + throw new Error('CTF_BUILD_CONTRACT_SURFACE_FLAGS_UNSUPPORTED'); + } + const cacheKey = hashGraphileBuildContract(contract); + buildContractEvidenceByLiveIdentity.set( + cacheKey, + buildContractEvidence.fingerprint, + ); + tenantState.set(tenant.id, { + tenant, + role, + pgConfig, + poolIdentity, + storage, + compute, + realtimeSchema, + runtimeDependencySchemas, + realtimeListenerIdentity, + poolContractEvidence, + buildContractEvidence, + cacheKey, + }); + } + + const runtimePoolStats = () => makeRuntimePoolStats( + pgCache, + TENANTS.map((tenant) => tenantState.get(tenant.id).poolIdentity), + options.runtimePoolMaxUses, + ); + const runtimePoolObjects = () => TENANTS.map((tenant) => { + const identity = tenantState.get(tenant.id).poolIdentity; + return pgCache.records instanceof Map + ? pgCache.records.get(identity)?.pool ?? null + : null; + }); + + let activeBuilds = 0; + let maxConcurrentBuilds = 0; + const buildCounts = Object.fromEntries(TENANTS.map((tenant) => [tenant.id, 0])); + const buildGenerations = Object.fromEntries(TENANTS.map((tenant) => [tenant.id, 0])); + const inFlight = new Map(); + + const withLease = async (pgConfig, poolOptions, callback) => { + const lease = acquirePgPool(pgConfig, poolOptions); + try { + return await callback(lease.pool); + } finally { + lease.release(); + } + }; + + const readProvisionAttestation = async () => { + if (!expectedProvisionAttestation) return null; + const result = await withLease( + controlPgConfig, + { purpose: 'control', sanitizeOnCheckout: false }, + (pool) => pool.query(` + SELECT clone_id, + run_purpose, + customer_id, + attestation_nonce, + attestation_sha256, + pg_catalog.current_database()::text AS database + FROM ctf_provision_private.clone_attestation + WHERE singleton = true + `), + ); + if (result.rowCount !== 1) throw new Error('CTF_PROVISION_ATTESTATION_ROW_INVALID'); + const row = result.rows[0]; + if ( + typeof row.clone_id !== 'string' + || !/^[a-z0-9][a-z0-9._-]{0,127}$/i.test(row.clone_id) + || (row.run_purpose !== 'hostile-preflight' && row.run_purpose !== 'measurement') + || typeof row.customer_id !== 'string' + || !/^[a-z0-9-]+$/.test(row.customer_id) + || typeof row.database !== 'string' + || typeof row.attestation_nonce !== 'string' + || !/^[a-f0-9]{64}$/.test(row.attestation_nonce) + || !/^sha256:[a-f0-9]{64}$/.test(row.attestation_sha256 ?? '') + ) { + throw new Error('CTF_PROVISION_ATTESTATION_ROW_INVALID'); + } + const calculatedSha256 = provisionAttestationSha256({ + cloneId: row.clone_id, + purpose: row.run_purpose, + customerId: row.customer_id, + database: row.database, + nonce: row.attestation_nonce, + }); + if ( + row.clone_id !== expectedProvisionAttestation.cloneId + || row.run_purpose !== expectedProvisionAttestation.purpose + || row.customer_id !== options.provisionCustomerId + || row.database !== controlPgConfig.database + || row.attestation_sha256 !== expectedProvisionAttestation.sha256 + || calculatedSha256 !== row.attestation_sha256 + ) { + throw new Error('CTF_PROVISION_ATTESTATION_MISMATCH'); + } + return { + version: 1, + cloneId: row.clone_id, + purpose: row.run_purpose, + customerId: row.customer_id, + database: row.database, + sha256: calculatedSha256, + verified: true, + }; + }; + + // Refuse to publish any physical fixture until its opaque database nonce has + // been queried and matched to the credential-free manifest digest. + const initialProvisionAttestation = await readProvisionAttestation(); + + await Promise.all(TENANTS.map(async (tenant) => { + const state = tenantState.get(tenant.id); + await withLease(state.pgConfig, runtimePoolOptions, (pool) => + ensureRuntimeRoleSafety( + pool, + [state.role], + [tenant.schema], + state.runtimeDependencySchemas, + ) + ); + })); + + const s3Client = new runtime.S3Client({ + endpoint: 'http://127.0.0.1:9', + region: 'us-east-1', + forcePathStyle: true, + credentials: { + accessKeyId: 'ctf-offline-signing-only', + secretAccessKey: 'ctf-offline-signing-only', + }, + }); + + const makePreset = (state, pool, sharedRealtimeBuild = null) => { + const pluginCompute = pluginComputeFor(state.compute); + return { + extends: [ + createConstructivePreset({ + ...FEATURE_SETTINGS, + enableLlm: false, + enablePresignedUploads: false, + preloadedStorageModules: [], + ...(sharedRealtimeBuild ? { + realtimeSubscriptions: { + onTopicsDiscovered: sharedRealtimeBuild.topicCollector.collect, + }, + } : {}), + }), + PresignedUrlPreset({ + s3: { + client: s3Client, + bucket: `ctf-${state.tenant.id}-offline`, + endpoint: 'http://127.0.0.1:9', + region: 'us-east-1', + forcePathStyle: true, + }, + preloadedStorageModules: state.storage.modules, + }), + createGrafastCacheLimitsPreset(GRAFAST_CACHE_LIMITS), + ], + plugins: [ + AuthCookiePlugin, + deterministicLlmPlugin(), + createLlmTextSearchPlugin({ onQuotaExceeded: 'throw' }), + createLlmTextMutationPlugin(), + createLlmRagPlugin({ contextLimit: 2, maxTokens: 256 }), + createFunctionBindingsPlugin({ + apiId: state.tenant.apiId, + modules: pluginCompute.modules, + preloadedBindings: pluginCompute.bindings, + }), + ], + pgServices: [makePgService({ + pool, + schemas: [state.tenant.schema], + introspectionMode: options.mode, + introspectionClientReleaseMode: options.introspectionClientReleaseMode, + introspectionScopedCatalogTypes: options.mode === 'scoped-required' + ? 'dependency-closure' + : undefined, + introspectionAllowedDependencySchemas: [...INTROSPECTION_DEPENDENCY_SCHEMAS], + ...(sharedRealtimeBuild ? { + pubsub: false, + pgSubscriber: sharedRealtimeBuild.subscriber, + } : {}), + })], + schema: { + releaseBuildStateAfterValidation: RELEASE_BUILD_STATE_AFTER_VALIDATION, + }, + grafserv: { + graphqlPath: '/graphql', + graphiql: false, + graphiqlOnGraphQLGET: false, + websockets: options.enableRealtime, + }, + grafast: { + explain: false, + context: (requestContext) => { + const request = requestContext?.expressv4?.req; + const api = request?.api ?? { + dbname: controlPgConfig.database, + schema: [state.tenant.schema], + anonRole: state.role, + roleName: state.role, + databaseId: state.tenant.databaseId, + apiId: state.tenant.apiId, + isPublic: false, + }; + return { + pgSettings: buildPgSettings({ + api, + token: null, + requestId: request?.requestId ?? crypto.randomUUID(), + dependencySchemas: [...INTROSPECTION_DEPENDENCY_SCHEMAS], + }), + langCodes: request ? languageCodes(request) : ['es', 'en'], + }; + }, + }, + }; + }; + + const buildEntry = (state) => { + const resident = graphileCache.get(state.cacheKey); + if (resident && !resident.disposing) return Promise.resolve(resident); + const existing = inFlight.get(state.cacheKey); + if (existing) return existing; + + const buildGeneration = buildGenerations[state.tenant.id]; + const pending = runGraphileBuild(async () => { + await prepareCacheForBuild(); + const lease = acquirePgPool(state.pgConfig, runtimePoolOptions); + let entry = null; + let leaseOwnedByEntry = false; + const sharedRealtimeBuild = options.realtimeNotificationMode === 'shared-exact' + ? { + subscriber: new ActivatableGenerationScopedRealtimeSubscriber(), + topicCollector: new RealtimeTopicCollector(), + } + : null; + let sharedRealtimeOwnedByEntry = false; + activeBuilds += 1; + maxConcurrentBuilds = Math.max(maxConcurrentBuilds, activeBuilds); + buildCounts[state.tenant.id] += 1; + try { + await ensureRuntimeRoleSafety( + lease.pool, + [state.role], + [state.tenant.schema], + state.runtimeDependencySchemas, + ); + entry = await observeGraphileBuild({ + cacheKey: state.cacheKey, + serviceKey: `ctf-${state.tenant.id}-api`, + databaseId: state.tenant.databaseId, + }, () => createGraphileInstance({ + preset: makePreset(state, lease.pool, sharedRealtimeBuild), + cacheKey: state.cacheKey, + poolIdentity: state.poolIdentity, + poolLease: lease, + serviceKey: `ctf-${state.tenant.id}-api`, + databaseId: state.tenant.databaseId, + enableRealtime: options.enableRealtime, + enableWebsockets: options.enableRealtime, + realtimeSchema: state.realtimeSchema, + realtimeSourceSchemas: [state.tenant.schema], + realtimeCursorPollIntervalMs: options.realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs: + options.realtimeCursorHeartbeatIntervalMs, + ...(sharedRealtimeBuild && notificationPgConfig && realtimeListenerIdentity ? { + sharedRealtime: { + ...sharedRealtimeBuild, + listenerPgConfig: notificationPgConfig, + listenerIdentity: realtimeListenerIdentity, + roleRevalidationMs: 60_000, + }, + } : {}), + }), { enabled: true }); + sharedRealtimeOwnedByEntry = Boolean(sharedRealtimeBuild); + leaseOwnedByEntry = true; + if (buildGeneration !== buildGenerations[state.tenant.id]) { + throw new Error(`CTF_BUILD_INVALIDATED:${state.tenant.id}`); + } + graphileCache.set(state.cacheKey, entry); + if (graphileCache.get(state.cacheKey) !== entry) { + throw new Error(`CTF_CACHE_PUBLICATION_FAILED:${state.tenant.id}`); + } + return entry; + } catch (error) { + if (leaseOwnedByEntry && entry) { + await disposeUncachedEntry(entry, state.cacheKey).catch(() => undefined); + } else { + lease.release(); + } + throw error; + } finally { + if (sharedRealtimeBuild && !sharedRealtimeOwnedByEntry) { + await sharedRealtimeBuild.subscriber.release().catch(() => undefined); + } + activeBuilds -= 1; + } + }); + inFlight.set(state.cacheKey, pending); + void pending.finally(() => { + if (inFlight.get(state.cacheKey) === pending) inFlight.delete(state.cacheKey); + }).catch(() => undefined); + return pending; + }; + + const invalidateTenant = async (tenantId) => { + const state = tenantState.get(tenantId); + if (!state) throw new Error(`CTF_UNKNOWN_TENANT:${tenantId}`); + buildGenerations[tenantId] += 1; + const pending = inFlight.get(state.cacheKey); + if (pending) await pending.catch(() => undefined); + const lease = acquirePgPool(state.pgConfig, runtimePoolOptions); + invalidateRuntimeRoleSafety(lease.pool); + lease.release(); + await deleteGraphileCacheEntry(state.cacheKey); + }; + + const handleUpgrade = async (request, socket, head, { pathPrefix = '' } = {}) => { + if (!options.enableRealtime || request.aborted || socket.destroyed) return false; + const tenantId = matchTenantUpgradePath(request.url, pathPrefix); + if (!tenantId) return false; + const state = tenantState.get(tenantId); + if (!state) return false; + const protocols = String(request.headers['sec-websocket-protocol'] ?? '') + .split(',') + .map((value) => value.trim()); + if (!protocols.includes('graphql-transport-ws')) return false; + + request.api = { + apiId: state.tenant.apiId, + databaseId: state.tenant.databaseId, + dbname: controlPgConfig.database, + schema: [state.tenant.schema], + anonRole: state.role, + roleName: state.role, + isPublic: false, + databaseSettings: FEATURE_SETTINGS, + }; + request.token = null; + request.requestId = request.headers['x-request-id'] ?? crypto.randomUUID(); + + for (let attempt = 0; attempt < 2; attempt += 1) { + const entry = graphileCache.get(state.cacheKey) ?? await buildEntry(state); + await ensureRuntimeRoleSafety( + entry.poolLease.pool, + [state.role], + [state.tenant.schema], + state.runtimeDependencySchemas, + ); + await revalidateEntryRealtimeRole(entry); + if (invokeEntryUpgradeHandler(entry, request, socket, head)) return true; + if (request.aborted || socket.destroyed) return true; + } + return false; + }; + + const app = runtime.express(); + app.disable('x-powered-by'); + app.use(runtime.express.json({ limit: '256kb' })); + + app.get('/healthz', (_request, response) => { + const governor = getGraphileGovernorCounters(); + response.status(governor.restartRequired ? 503 : 200).json({ + status: governor.restartRequired ? 'unhealthy' : 'ok', + }); + }); + + app.get('/debug/memory', (request, response) => { + if (!isLoopbackRequest(request)) { + response.status(404).send('Not found'); + return; + } + const configuredToken = environment.GRAPHQL_OBSERVABILITY_TOKEN ?? ''; + if ( + environment.NODE_ENV !== 'development' + && !timingSafeTokenEqual(bearerToken(request), configuredToken) + ) { + response.status(401).json({ error: { code: 'CTF_OBSERVABILITY_UNAUTHORIZED' } }); + return; + } + response.json(getDebugMemorySnapshot()); + }); + + app.get('/__ctf/status', async (request, response, next) => { + if (!isLoopbackRequest(request)) { + response.status(404).send('Not found'); + return; + } + try { + const liveProvisionAttestation = await readProvisionAttestation(); + response.json({ + version: 1, + fixture: 'complete-tenant-abc-v1', + arm: options.arm, + introspectionMode: options.mode, + introspectionClientReleaseMode: options.introspectionClientReleaseMode, + releaseBuildStateAfterValidation: RELEASE_BUILD_STATE_AFTER_VALIDATION, + runtimeArtifactFingerprint: runtimeFingerprint, + configurationIdentity, + liveIdentityScope: 'process-local-keyed-hmac-v1', + physicalIsolation: 'dedicated-login-and-pool-per-tenant', + sharedRuntimePool: false, + runtimePoolMax: options.runtimePoolMax, + runtimePoolMaxUses: options.runtimePoolMaxUses, + runtimePools: runtimePoolStats(), + preparedStatementCache, + enableRealtime: options.enableRealtime, + realtimeNotificationMode: options.realtimeNotificationMode, + realtimeListenerIdentity, + realtimeCursorPollIntervalMs: options.realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs: + options.realtimeCursorHeartbeatIntervalMs, + realtimeNotificationBrokers: getPgNotificationBrokerStats(), + realtimeRoleAudits: getGraphileRealtimeRoleAuditStats(), + realtimeSchemas: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).realtimeSchema, + ])), + physicalDatabase: controlPgConfig.database, + runPurpose: options.runPurpose ?? null, + provisionAttestation: liveProvisionAttestation, + runtimePoolIdentities: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).poolIdentity, + ])), + runtimeBindings: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + databaseId: tenant.databaseId, + databaseName: controlPgConfig.database, + role: tenantState.get(tenant.id).role, + schemas: [tenant.schema], + }, + ])), + controlAvailable: hostileControlEnabled + && Buffer.byteLength(options.controlToken) >= 32, + buildContracts: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).cacheKey, + ])), + residentBuildContracts: [...graphileCache.keys()], + contractEvidence: { + version: 1, + credentialFree: true, + configurationIdentity, + realtimeListener: realtimeListenerContractEvidence, + runtimePools: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).poolContractEvidence, + ])), + graphileBuilds: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).buildContractEvidence, + ])), + residentGraphileBuildFingerprints: [...graphileCache.keys()] + .map((cacheKey) => buildContractEvidenceByLiveIdentity.get(cacheKey)) + .filter(Boolean), + }, + builds: { + active: activeBuilds, + maxConcurrent: maxConcurrentBuilds, + byTenant: { ...buildCounts }, + generations: { ...buildGenerations }, + inFlight: [...inFlight.keys()], + graphile: getGraphileBuildStats(), + }, + runtimeSafety: { + passed: true, + rolesDistinct: true, + dependencySchemasByTenant: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).runtimeDependencySchemas, + ])), + }, + }); + } catch (error) { + next(error); + } + }); + + app.post('/__ctf/control', async (request, response, next) => { + try { + if (!hostileControlEnabled) { + response.status(404).send('Not found'); + return; + } + if ( + !isLoopbackRequest(request) + || Buffer.byteLength(options.controlToken) < 32 + || !timingSafeTokenEqual(bearerToken(request), options.controlToken) + ) { + response.status(404).send('Not found'); + return; + } + const action = request.body?.action; + const tenantId = request.body?.tenant; + if (action === 'invalidate-all') { + await Promise.all(TENANTS.map((tenant) => invalidateTenant(tenant.id))); + const identity = await withLease( + controlPgConfig, + { purpose: 'control', sanitizeOnCheckout: false }, + (pool) => pool.query( + 'SELECT pg_catalog.current_database()::text AS physical_database_identity', + ), + ); + response.json({ + ok: true, + action, + physicalDatabaseIdentity: identity.rows[0]?.physical_database_identity, + }); + return; + } + const state = tenantState.get(tenantId); + if (!state) { + response.status(400).json({ error: { code: 'CTF_UNKNOWN_TENANT' } }); + return; + } + if (action === 'poison') { + const observed = await withLease(state.pgConfig, runtimePoolOptions, async (pool) => { + const client = await pool.connect(); + try { + const result = await client.query( + `SELECT ${state.tenant.schema}.poison_session() AS value, ` + + 'pg_catalog.current_database()::text AS physical_database_identity', + ); + return result.rows[0]; + } finally { + client.release(); + } + }); + response.json({ + ok: observed?.value === 'poisoned', + action, + tenant: tenantId, + physicalDatabaseIdentity: observed?.physical_database_identity, + }); + return; + } + if (action === 'rollback-savepoint') { + const observed = await withLease(state.pgConfig, runtimePoolOptions, async (pool) => { + const client = await pool.connect(); + try { + await client.query('SELECT pg_catalog.set_config($1, $2, false)', [ + 'jwt.claims.database_id', + state.tenant.databaseId, + ]); + const result = await client.query( + `SELECT ${state.tenant.schema}.savepoint_identity() AS value, ` + + 'pg_catalog.current_database()::text AS physical_database_identity', + ); + return result.rows[0]; + } finally { + client.release(); + } + }); + response.json({ + ok: observed?.value === state.tenant.databaseId, + action, + tenant: tenantId, + observed: observed?.value, + physicalDatabaseIdentity: observed?.physical_database_identity, + }); + return; + } + if (action === 'prepared-reset') { + const statementName = `ctf-prepared-reset-${tenantId}`; + const first = await withLease(state.pgConfig, runtimePoolOptions, async (pool) => { + const client = await pool.connect(); + try { + await client.query('SELECT pg_catalog.set_config($1, $2, false)', [ + 'jwt.claims.database_id', + state.tenant.databaseId, + ]); + const result = await client.query({ + name: statementName, + text: `SELECT ${state.tenant.schema}.tenant_identity() AS value, ` + + 'pg_catalog.current_database()::text AS physical_database_identity, ' + + 'pg_catalog.pg_backend_pid()::integer AS backend_pid, ' + + 'current_user::text AS runtime_role', + }); + return result.rows[0]; + } finally { + client.release(); + } + }); + const second = await withLease(state.pgConfig, runtimePoolOptions, async (pool) => { + const client = await pool.connect(); + try { + await client.query('SELECT pg_catalog.set_config($1, $2, false)', [ + 'jwt.claims.database_id', + state.tenant.databaseId, + ]); + const result = await client.query({ + name: statementName, + text: `SELECT ${state.tenant.schema}.request_identity() AS value, ` + + 'pg_catalog.current_database()::text AS physical_database_identity, ' + + 'pg_catalog.pg_backend_pid()::integer AS backend_pid, ' + + 'current_user::text AS runtime_role', + }); + return result.rows[0]; + } finally { + client.release(); + } + }); + const backend = preparedResetBackendEvidence( + first?.backend_pid, + second?.backend_pid, + options.runtimePoolMaxUses, + ); + response.json({ + ok: backend.exact + && first?.value === state.tenant.token + && second?.value === `${state.tenant.token}:${state.tenant.databaseId}` + && first?.physical_database_identity === second?.physical_database_identity + && first?.runtime_role === state.role + && second?.runtime_role === state.role, + action, + tenant: tenantId, + first: first?.value, + second: second?.value, + runtimeRole: first?.runtime_role, + backend, + physicalDatabaseIdentity: first?.physical_database_identity, + }); + return; + } + if (action === 'bad-role-expected-failure') { + const controlIdentity = await withLease( + controlPgConfig, + { purpose: 'control', sanitizeOnCheckout: false }, + (pool) => pool.query( + 'SELECT current_user::text AS role_name, ' + + 'pg_catalog.current_database()::text AS physical_database_identity', + ), + ); + let rejectedCode = null; + try { + await withLease(state.pgConfig, runtimePoolOptions, (pool) => + ensureRuntimeRoleSafety( + pool, + [state.role, controlIdentity.rows[0]?.role_name], + [state.tenant.schema], + state.runtimeDependencySchemas, + ) + ); + } catch (error) { + if (error?.code !== 'GRAPHILE_UNSAFE_RUNTIME_ROLE') throw error; + rejectedCode = error.code; + } + response.json({ + ok: rejectedCode === 'GRAPHILE_UNSAFE_RUNTIME_ROLE', + action, + tenant: tenantId, + rejectedCode, + physicalDatabaseIdentity: + controlIdentity.rows[0]?.physical_database_identity, + }); + return; + } + if (action === 'drift-apply' || action === 'drift-revert') { + const functionName = action === 'drift-apply' + ? 'apply_schema_drift' + : 'revert_schema_drift'; + const result = await withLease( + controlPgConfig, + { purpose: 'control', sanitizeOnCheckout: false }, + (pool) => pool.query( + `SELECT ctf_control.${functionName}($1), ` + + 'pg_catalog.current_database()::text AS physical_database_identity', + [state.tenant.schema], + ), + ); + await invalidateTenant(tenantId); + response.json({ + ok: true, + action, + tenant: tenantId, + physicalDatabaseIdentity: + result.rows[0]?.physical_database_identity, + }); + return; + } + response.status(400).json({ error: { code: 'CTF_UNKNOWN_CONTROL_ACTION' } }); + } catch (error) { + next(error); + } + }); + + for (const tenant of TENANTS) { + const state = tenantState.get(tenant.id); + app.use(`/tenant/${tenant.id}`, async (request, response, next) => { + try { + request.api = { + apiId: tenant.apiId, + databaseId: tenant.databaseId, + dbname: controlPgConfig.database, + schema: [tenant.schema], + anonRole: state.role, + roleName: state.role, + isPublic: false, + databaseSettings: FEATURE_SETTINGS, + }; + request.token = null; + request.requestId = request.get('x-request-id') ?? crypto.randomUUID(); + const entry = graphileCache.get(state.cacheKey) ?? await buildEntry(state); + await ensureRuntimeRoleSafety( + entry.poolLease.pool, + [state.role], + [tenant.schema], + state.runtimeDependencySchemas, + ); + await revalidateEntryRealtimeRole(entry); + if (!invokeEntryHandler(entry, request, response, next) && !response.headersSent) { + response.status(503).json({ error: { code: 'CTF_INSTANCE_ROTATING' } }); + } + } catch (error) { + next(error); + } + }); + } + + app.use((error, _request, response, _next) => { + const code = typeof error?.code === 'string' ? error.code : 'CTF_INTERNAL_ERROR'; + response.status(code === 'GRAPHILE_UNSAFE_RUNTIME_ROLE' ? 503 : 500).json({ + error: { code, message: error instanceof Error ? error.message : String(error) }, + }); + }); + + let httpServer = null; + let upgradeListener = null; + const listen = () => new Promise((resolve, reject) => { + httpServer = app.listen(options.port, options.host, () => resolve(httpServer)); + httpServer.once('error', reject); + if (options.enableRealtime) { + upgradeListener = (request, socket, head) => { + void handleUpgrade(request, socket, head) + .then((handled) => { + if (!handled && !socket.destroyed) socket.destroy(); + }) + .catch(() => socket.destroy()); + }; + httpServer.on('upgrade', upgradeListener); + } + }); + const close = async () => { + if (httpServer && upgradeListener) httpServer.off('upgrade', upgradeListener); + const closeServer = httpServer?.listening + ? new Promise((resolve) => httpServer.close(resolve)) + : Promise.resolve(); + await Promise.all(TENANTS.map((tenant) => + deleteGraphileCacheEntry(tenantState.get(tenant.id).cacheKey) + )); + await closeServer; + await teardownPgNotificationBrokers(); + await teardownPgPools(); + }; + + return { + app, + close, + handleUpgrade, + initialProvisionAttestation, + listen, + options: { + host: options.host, + port: options.port, + arm: options.arm, + mode: options.mode, + runtimeRoles: { ...options.runtimeRoles }, + }, + readProvisionAttestation, + contractEvidence: () => ({ + version: 1, + credentialFree: true, + configurationIdentity, + realtimeListener: realtimeListenerContractEvidence, + runtimePools: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).poolContractEvidence, + ])), + graphileBuilds: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).buildContractEvidence, + ])), + }), + buildContractFingerprintForLiveIdentity: (cacheKey) => + buildContractEvidenceByLiveIdentity.get(cacheKey) ?? null, + runtimePoolObjects, + runtimePoolStats, + }; +}; + +const main = async () => { + const options = parseServerOptions(process.argv.slice(2)); + const server = await createFixtureServer(options); + await server.listen(); + process.stdout.write( + `${JSON.stringify({ status: 'ready', host: options.host, port: options.port, arm: options.arm })}\n`, + ); + let closing = false; + const shutdown = async () => { + if (closing) return; + closing = true; + await server.close(); + }; + process.once('SIGTERM', () => void shutdown().finally(() => process.exit(0))); + process.once('SIGINT', () => void shutdown().finally(() => process.exit(130))); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + FEATURE_SETTINGS, + GRAFAST_CACHE_LIMITS, + INSTALLED_RUNTIME_ARTIFACT_SPECS, + INTROSPECTION_DEPENDENCY_SCHEMAS, + PREPARED_STATEMENT_ATTESTATION_KIND, + RELEASE_BUILD_STATE_AFTER_VALIDATION, + RUNTIME_ARTIFACT_PATHS, + RUNTIME_DEPENDENCY_SCHEMAS, + attestDataplanPreparedStatementCache, + createFixtureServer, + credentialFreeContractEvidence, + fixtureConfigurationIdentity, + loadInstalledDataplanPgAdaptor, + makeRuntimePoolStats, + parseServerOptions, + parseRuntimePoolMaxUses, + preparedResetBackendEvidence, + preparedStatementCacheRequestFromEnvironment, + matchTenantUpgradePath, + installedRuntimeArtifactManifest, + hostileControlEnabledFor, + realtimeSchemaFor, + provisionAttestationSha256, + resolvedLocalRuntimeArtifactManifest, + runtimeDependencySchemasFor, + runtimeArtifactManifest, + runtimeArtifactFingerprint, + runtimePoolContractEvidence, + timingSafeTokenEqual, +}; diff --git a/research/graphile-density/complete-tenant-fixture/server.test.cjs b/research/graphile-density/complete-tenant-fixture/server.test.cjs new file mode 100644 index 0000000000..605f187148 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/server.test.cjs @@ -0,0 +1,543 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const path = require('node:path'); +const test = require('node:test'); + +const { TENANTS } = require('./lib.cjs'); +const { + provisionAttestationSha256: provisionerAttestationSha256, +} = require('../physical-database-density/provision.cjs'); + +const { + INSTALLED_RUNTIME_ARTIFACT_SPECS, + PREPARED_STATEMENT_ATTESTATION_KIND, + RELEASE_BUILD_STATE_AFTER_VALIDATION, + RUNTIME_ARTIFACT_PATHS, + credentialFreeContractEvidence, + fixtureConfigurationIdentity, + hostileControlEnabledFor, + installedRuntimeArtifactManifest, + makeRuntimePoolStats, + matchTenantUpgradePath, + parseRuntimePoolMaxUses, + parseServerOptions, + preparedResetBackendEvidence, + preparedStatementCacheRequestFromEnvironment, + provisionAttestationSha256, + realtimeSchemaFor, + resolvedLocalRuntimeArtifactManifest, + runtimeArtifactManifest, + runtimeArtifactFingerprint, + runtimeDependencySchemasFor, + runtimePoolContractEvidence, + timingSafeTokenEqual, +} = require('./server.cjs'); + +const environment = () => ({ + CTF_RUNTIME_A_PGPASSWORD: 'runtime-a-value', + CTF_RUNTIME_B_PGPASSWORD: 'runtime-b-value', + CTF_RUNTIME_C_PGPASSWORD: 'runtime-c-value', +}); + +const roleArgs = (roles = ['ctf_runtime_a', 'ctf_runtime_b', 'ctf_runtime_c']) => [ + '--runtime-role-a', roles[0], + '--runtime-role-b', roles[1], + '--runtime-role-c', roles[2], +]; + +test('server accepts only loopback with three distinct credentialed runtime roles', () => { + const options = parseServerOptions([ + '--host', '127.0.0.1', + '--port', '3392', + '--mode', 'stock', + ...roleArgs(), + ], environment()); + assert.deepEqual(options.runtimeRoles, { + a: 'ctf_runtime_a', + b: 'ctf_runtime_b', + c: 'ctf_runtime_c', + }); + assert.equal(options.port, 3392); + assert.equal(options.mode, 'stock'); + assert.equal(options.introspectionClientReleaseMode, 'destroy'); + assert.equal(options.runtimePoolMax, 1); + assert.equal(options.runtimePoolMaxUses, null); + assert.equal(options.enableRealtime, false); + const serialized = JSON.stringify(options); + assert.doesNotMatch(serialized, /runtime-[abc]-value/); + + assert.throws( + () => parseServerOptions(['--host', '0.0.0.0', ...roleArgs()], environment()), + /CTF_SERVER_LOOPBACK_REQUIRED/, + ); + assert.throws( + () => parseServerOptions(roleArgs(['same', 'same', 'third']), environment()), + /CTF_RUNTIME_ROLES_MUST_BE_DISTINCT/, + ); + assert.throws( + () => parseServerOptions(roleArgs(), { + CTF_RUNTIME_A_PGPASSWORD: 'only-one', + }), + /CTF_RUNTIME_PASSWORD_REQUIRED:CTF_RUNTIME_B_PGPASSWORD/, + ); +}); + +test('server accepts explicit runtime pool capacity and realtime opt-in', () => { + const options = parseServerOptions([ + '--runtime-pool-max', '4', + '--runtime-pool-max-uses', '1', + '--enable-realtime', + ...roleArgs(), + ], environment()); + assert.equal(options.runtimePoolMax, 4); + assert.equal(options.runtimePoolMaxUses, 1); + assert.equal(options.enableRealtime, true); + + assert.throws( + () => parseServerOptions(['--runtime-pool-max', '0', ...roleArgs()], environment()), + /CTF_INVALID_POSITIVE_INTEGER:runtime-pool-max/, + ); + assert.throws( + () => parseServerOptions([ + '--runtime-pool-max-uses', '0', + ...roleArgs(), + ], environment()), + /CTF_INVALID_MAX_USES:runtime-pool-max-uses/, + ); + for (const value of ['01', '1e2', '0x1', ' 1', '1 ', '', true]) { + assert.throws( + () => parseRuntimePoolMaxUses(value), + /CTF_INVALID_MAX_USES:runtime-pool-max-uses/, + ); + } + assert.throws( + () => parseServerOptions(['--enable-realtime', 'sometimes', ...roleArgs()], environment()), + /CTF_INVALID_BOOLEAN:enable-realtime/, + ); + assert.throws( + () => parseServerOptions([ + '--runtime-pool-max', '1', + '--enable-realtime', + ...roleArgs(), + ], environment()), + /CTF_REALTIME_REQUIRES_RUNTIME_POOL_MAX_2/, + ); +}); + +test('fixture configuration and contract evidence are deterministic and credential-free', () => { + const input = { + databaseName: 'ctf_customer_0001', + mode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + enableRealtime: true, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 30_000, + runtimeFingerprint: `sha256:${'a'.repeat(64)}`, + }; + const configurationIdentity = fixtureConfigurationIdentity(input); + assert.match( + configurationIdentity, + /^graphile-configuration:ctf:v1:[a-f0-9]{64}$/, + ); + assert.equal(fixtureConfigurationIdentity({ ...input }), configurationIdentity); + assert.notEqual( + fixtureConfigurationIdentity({ ...input, databaseName: 'ctf_customer_0002' }), + configurationIdentity, + ); + + const pool = runtimePoolContractEvidence({ + databaseName: input.databaseName, + role: 'ctf_runtime_a', + poolMax: 1, + poolMaxUses: null, + runtimeFingerprint: input.runtimeFingerprint, + }); + assert.match(pool.fingerprint, /^pg-contract-evidence:v1:[a-f0-9]{64}$/); + assert.equal(JSON.stringify(pool).includes('runtime-password-value'), false); + assert.deepEqual( + credentialFreeContractEvidence('fixture-evidence', { a: 1 }), + credentialFreeContractEvidence('fixture-evidence', { a: 1 }), + ); + assert.throws( + () => fixtureConfigurationIdentity({ ...input, runtimeFingerprint: '' }), + /CTF_CONFIGURATION_IDENTITY_INPUT_INVALID/, + ); +}); + +test('prepared reset PID evidence distinguishes sanitation reuse from maxUses rotation', () => { + assert.deepEqual(preparedResetBackendEvidence(101, 101, null), { + firstBackendPid: 101, + secondBackendPid: 101, + observed: 'same-client', + expected: 'same-client', + exact: true, + }); + assert.deepEqual(preparedResetBackendEvidence(101, 202, 1), { + firstBackendPid: 101, + secondBackendPid: 202, + observed: 'rotated-client', + expected: 'rotated-client', + exact: true, + }); + assert.equal(preparedResetBackendEvidence(101, 202, null).exact, false); + assert.equal(preparedResetBackendEvidence(101, 101, 1).exact, false); + assert.deepEqual(preparedResetBackendEvidence(101, 202, 2), { + firstBackendPid: 101, + secondBackendPid: 202, + observed: 'rotated-client', + expected: 'unsupported', + exact: false, + }); +}); + +test('runtime pool telemetry follows exact runtime identities and proves native maxUses', () => { + const nativePool = (maxUses, totalCount, idleCount) => ({ + options: { maxUses }, + totalCount, + idleCount, + waitingCount: 0, + }); + const runtimeA = nativePool(1, 0, 0); + const runtimeB = nativePool(1, 1, 0); + const notification = nativePool(Number.POSITIVE_INFINITY, 1, 1); + const pgCache = { + records: new Map([ + ['runtime-a', { pool: runtimeA }], + ['runtime-b', { pool: runtimeB }], + ['notification', { pool: notification }], + ]), + }; + assert.deepEqual( + makeRuntimePoolStats(pgCache, ['runtime-a', 'runtime-b'], 1), + { + scope: 'runtime-only-exact-identities', + available: true, + requestedMaxUses: 1, + effectiveMaxUses: 1, + effectiveMaxUsesKnown: true, + maxUsesExact: true, + identitiesUnique: true, + poolObjectsUnique: true, + expectedPools: 2, + observedPools: 2, + totalClients: 1, + idleClients: 0, + waitingClients: 0, + }, + ); + + const unlimited = makeRuntimePoolStats( + { records: new Map([['runtime', { pool: notification }]]) }, + ['runtime'], + null, + ); + assert.equal(unlimited.available, true); + assert.equal(unlimited.effectiveMaxUsesKnown, true); + assert.equal(unlimited.effectiveMaxUses, null); + assert.equal(unlimited.maxUsesExact, true); + + const missing = makeRuntimePoolStats(pgCache, ['runtime-a', 'missing'], 1); + assert.equal(missing.available, false); + assert.equal(missing.observedPools, 1); + assert.equal(missing.maxUsesExact, false); + + const duplicateIdentity = makeRuntimePoolStats( + pgCache, + ['runtime-a', 'runtime-a'], + 1, + ); + assert.equal(duplicateIdentity.available, false); + assert.equal(duplicateIdentity.identitiesUnique, false); + assert.equal(duplicateIdentity.observedPools, 1); + assert.equal(duplicateIdentity.maxUsesExact, false); + + const duplicatePoolObject = makeRuntimePoolStats({ + records: new Map([ + ['runtime-a', { pool: runtimeA }], + ['runtime-b', { pool: runtimeA }], + ]), + }, ['runtime-a', 'runtime-b'], 1); + assert.equal(duplicatePoolObject.available, false); + assert.equal(duplicatePoolObject.identitiesUnique, true); + assert.equal(duplicatePoolObject.poolObjectsUnique, false); + assert.equal(duplicatePoolObject.observedPools, 1); +}); + +test('prepared statement cache request accepts only a canonical bounded integer', () => { + assert.deepEqual( + preparedStatementCacheRequestFromEnvironment({ + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: '0', + }), + { + environmentValue: '0', + requestedSize: 0, + environmentCanonical: true, + }, + ); + assert.deepEqual(preparedStatementCacheRequestFromEnvironment({}), { + environmentValue: null, + requestedSize: 100, + environmentCanonical: false, + }); + for (const value of ['01', '1e2', '-1', '10001', 'not-a-number', ' 1', '1 ']) { + assert.throws( + () => preparedStatementCacheRequestFromEnvironment({ + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: value, + }), + /CTF_PREPARED_STATEMENT_CACHE_SIZE_INVALID/, + ); + } +}); + +test('prepared statement cache telemetry attests the loaded Dataplan adaptor behavior', () => { + const serverFile = path.join(__dirname, 'server.cjs'); + const inspect = (size) => JSON.parse(execFileSync(process.execPath, ['-e', ` +const fixture = require(${JSON.stringify(serverFile)}); +(async () => { + const request = fixture.preparedStatementCacheRequestFromEnvironment(process.env); + const adaptor = fixture.loadInstalledDataplanPgAdaptor(); + const proof = await fixture.attestDataplanPreparedStatementCache(adaptor, request); + process.stdout.write(JSON.stringify(proof)); +})().catch((error) => { + process.stderr.write(String(error && error.stack || error)); + process.exit(1); +}); +`], { + cwd: path.resolve(__dirname, '../../..'), + encoding: 'utf8', + env: { + ...process.env, + GRAPHILE_ENV: 'production', + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: String(size), + }, + })); + + const disabled = inspect(0); + assert.equal(disabled.attestation, PREPARED_STATEMENT_ATTESTATION_KIND); + assert.equal(disabled.effectiveSizeKnown, true); + assert.equal(disabled.effectiveSize, 0); + assert.equal(disabled.exact, true); + assert.equal(disabled.namedQueriesObserved, 0); + assert.equal(disabled.firstEvictionAfterNamedQueries, null); + + const bounded = inspect(3); + assert.equal(bounded.attestation, PREPARED_STATEMENT_ATTESTATION_KIND); + assert.equal(bounded.effectiveSizeKnown, true); + assert.equal(bounded.effectiveSize, 3); + assert.equal(bounded.exact, true); + assert.equal(bounded.namedQueriesObserved, 4); + assert.equal(bounded.firstEvictionAfterNamedQueries, 3); +}); + +test('shared exact realtime permits a one-client runtime pool only with a distinct listener login', () => { + const options = parseServerOptions([ + '--runtime-pool-max', '1', + '--enable-realtime', + '--realtime-notification-mode', 'shared-exact', + '--notification-role', 'ctf_notification', + '--realtime-cursor-poll-ms', '30000', + ...roleArgs(), + ], { + ...environment(), + CTF_NOTIFICATION_PGPASSWORD: 'notification-password-value', + }); + assert.equal(options.runtimePoolMax, 1); + assert.equal(options.realtimeNotificationMode, 'shared-exact'); + assert.equal(options.notificationRole, 'ctf_notification'); + assert.equal(options.realtimeCursorPollIntervalMs, 30000); + const serialized = JSON.stringify(options); + assert.doesNotMatch(serialized, /notification-password-value/); + assert.equal(options.takeNotificationPassword(), 'notification-password-value'); + assert.throws( + () => options.takeNotificationPassword(), + /CTF_NOTIFICATION_PASSWORD_ALREADY_CONSUMED/, + ); + + assert.throws(() => parseServerOptions([ + '--runtime-pool-max', '1', + '--enable-realtime', + '--realtime-notification-mode', 'shared-exact', + '--notification-role', 'ctf_runtime_a', + ...roleArgs(), + ], { + ...environment(), + CTF_NOTIFICATION_PGPASSWORD: 'notification-password-value', + }), /CTF_NOTIFICATION_ROLE_MUST_BE_DISTINCT/); + assert.throws(() => parseServerOptions([ + '--runtime-pool-max', '1', + '--enable-realtime', + '--realtime-notification-mode', 'shared-exact', + '--notification-role', 'ctf_notification', + ...roleArgs(), + ], environment()), /CTF_NOTIFICATION_PASSWORD_REQUIRED/); +}); + +test('realtime cursor schemas and runtime safety allowlists remain tenant-exact', () => { + assert.deepEqual(TENANTS.map(realtimeSchemaFor), [ + 'ctf_a_realtime', + 'ctf_b_realtime', + 'ctf_c_realtime', + ]); + + for (const tenant of TENANTS) { + const disabled = runtimeDependencySchemasFor(tenant, false); + const enabled = runtimeDependencySchemasFor(tenant, true); + assert.deepEqual(disabled, ['ctf_extensions', 'jwt_private']); + assert.deepEqual(enabled, [ + 'ctf_extensions', + 'jwt_private', + realtimeSchemaFor(tenant), + ]); + for (const foreignTenant of TENANTS.filter((candidate) => candidate !== tenant)) { + assert.ok(!enabled.includes(realtimeSchemaFor(foreignTenant))); + } + } +}); + +test('websocket upgrade paths select one exact tenant and reject ambiguous routes', () => { + assert.equal(matchTenantUpgradePath('/tenant/a/graphql'), 'a'); + assert.equal( + matchTenantUpgradePath( + '/customer/physical-customer-0001/tenant/c/graphql', + '/customer/physical-customer-0001', + ), + 'c', + ); + assert.equal(matchTenantUpgradePath('/tenant/a/graphql?tenant=b'), null); + assert.equal(matchTenantUpgradePath('/tenant/a/graphql/extra'), null); + assert.equal(matchTenantUpgradePath('/tenant/%61/graphql'), null); + assert.equal(matchTenantUpgradePath('/tenant/a%2F..%2Fb/graphql'), null); + assert.equal(matchTenantUpgradePath('/tenant/a/graphql', '/customer/other'), null); + assert.equal(matchTenantUpgradePath('/tenant/a/graphql', '../customer'), null); +}); + +test('server rejects unknown introspection modes', () => { + assert.throws( + () => parseServerOptions(['--mode', 'fallback', ...roleArgs()], environment()), + /CTF_INTROSPECTION_MODE_INVALID:fallback/, + ); +}); + +test('server validates the introspection-client release mode', () => { + const options = parseServerOptions([ + '--introspection-client-release-mode', 'reuse', + ...roleArgs(), + ], environment()); + assert.equal(options.introspectionClientReleaseMode, 'reuse'); + assert.throws( + () => parseServerOptions([ + '--introspection-client-release-mode', 'best-effort', + ...roleArgs(), + ], environment()), + /CTF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:best-effort/, + ); +}); + +test('control token comparison is exact and timing safe for equal-length values', () => { + const token = 'a'.repeat(64); + assert.equal(timingSafeTokenEqual(token, token), true); + assert.equal(timingSafeTokenEqual(`${'a'.repeat(63)}b`, token), false); + assert.equal(timingSafeTokenEqual('short', token), false); + assert.equal(timingSafeTokenEqual('', token), false); +}); + +test('runtime fingerprint binds every executed built API artifact', () => { + assert.match(runtimeArtifactFingerprint(), /^sha256:[0-9a-f]{64}$/); + assert.equal(runtimeArtifactFingerprint(), runtimeArtifactFingerprint()); + const runtimeManifest = runtimeArtifactManifest(); + const localClosure = resolvedLocalRuntimeArtifactManifest(); + assert.equal(runtimeManifest.version, 2); + assert.ok(localClosure.length > RUNTIME_ARTIFACT_PATHS.length); + assert.deepEqual(runtimeManifest.localDistClosure, localClosure); + assert.ok(localClosure.some((entry) => + entry.path === 'graphile/graphile-settings/dist/presets/constructive-preset.js' + )); + assert.ok(localClosure.some((entry) => + entry.path === 'graphile/graphile-search/dist/index.js' + )); + assert.ok(localClosure.every((entry) => + !path.isAbsolute(entry.path) + && !entry.path.includes('node_modules') + && /^sha256:[0-9a-f]{64}$/.test(entry.sha256) + )); + + const expectedInstalledLabels = [ + 'installed:@dataplan/pg:dist/index.js', + 'installed:@dataplan/pg:dist/adaptors/pg.js', + 'installed:@dataplan/pg:dist/pgServices.js', + 'installed:graphile-build-pg:dist/index.js', + 'installed:graphile-build-pg:dist/plugins/PgIntrospectionPlugin.js', + ]; + const manifest = installedRuntimeArtifactManifest(); + assert.deepEqual( + manifest.map((entry) => entry.label), + expectedInstalledLabels, + ); + assert.deepEqual( + INSTALLED_RUNTIME_ARTIFACT_SPECS.map((entry) => entry.label), + expectedInstalledLabels, + ); + const installedSpecs = Object.fromEntries( + INSTALLED_RUNTIME_ARTIFACT_SPECS.map((entry) => [entry.label, entry]), + ); + assert.ok(installedSpecs['installed:@dataplan/pg:dist/index.js'].markers.includes( + 'exports.exactClientReleaseCapability = "dataplan-pg-exact-client-destroy-v1";', + )); + assert.ok(installedSpecs['installed:@dataplan/pg:dist/adaptors/pg.js'].markers.includes( + 'const supportsExactClientDestruction = typeof PgPool === "function" && pool instanceof PgPool;', + )); + assert.ok(installedSpecs['installed:@dataplan/pg:dist/adaptors/pg.js'].markers.includes( + 'Exact PostgreSQL client destruction requires a node-postgres Pool', + )); + assert.ok(installedSpecs['installed:graphile-build-pg:dist/index.js'].markers.includes( + 'exports.introspectionClientReleaseCapability = "graphile-build-pg-exact-client-destroy-v1";', + )); + assert.ok(manifest.every((entry) => /^sha256:[0-9a-f]{64}$/.test(entry.sha256))); + assert.ok(manifest.every( + (entry) => /^sha256:[0-9a-f]{64}$/.test(entry.markerSetSha256) + && entry.markerCount > 0 + )); + const serialized = JSON.stringify({ + specs: INSTALLED_RUNTIME_ARTIFACT_SPECS, + manifest, + }); + assert.equal(serialized.includes(process.cwd()), false); + assert.doesNotMatch(serialized, /password|credential|secret|authorization|bearer/i); +}); + +test('provisioned measurement servers cannot enable hostile controls', () => { + const attestation = { + version: 1, + cloneId: 'fixture-clone', + purpose: 'measurement', + sha256: `sha256:${'a'.repeat(64)}`, + }; + assert.equal(hostileControlEnabledFor('measurement', attestation), false); + assert.equal(hostileControlEnabledFor('hostile-preflight', { + ...attestation, + purpose: 'hostile-preflight', + }), true); + // The standalone complete fixture keeps its pre-existing local control lane; + // physical runs always carry a database-backed attestation. + assert.equal(hostileControlEnabledFor(undefined, null), true); +}); + +test('live server and provisioner compute the same context-bound attestation', () => { + const input = { + cloneId: 'fixture-clone', + customerId: 'physical-customer-0001', + database: 'pdc_fixture_db_0001', + nonce: 'a'.repeat(64), + }; + assert.equal( + provisionAttestationSha256({ ...input, purpose: 'measurement' }), + provisionerAttestationSha256({ ...input, runPurpose: 'measurement' }), + ); +}); + +test('complete fixture requires post-validation build-state retirement', () => { + assert.equal(RELEASE_BUILD_STATE_AFTER_VALIDATION, true); +}); diff --git a/research/graphile-density/create-uniform-density-fixture.sql b/research/graphile-density/create-uniform-density-fixture.sql new file mode 100644 index 0000000000..1bc3ba8d52 --- /dev/null +++ b/research/graphile-density/create-uniform-density-fixture.sql @@ -0,0 +1,796 @@ +\set ON_ERROR_STOP on +\pset pager off + +-- Rebuilds the local density fixture without modifying its source database. +-- This is a psql script, not generic SQL. Run it as a PostgreSQL administrator: +-- +-- psql -X -d postgres \ +-- -f research/graphile-density/create-uniform-density-fixture.sql +-- +-- The target name is intentionally fixed. The script fails closed when the +-- target already exists; it never replaces or drops a database. `-v resume=1` +-- is only for continuing a clone that passed the pristine-source assertions +-- but stopped before tenant DDL; those assertions run again before mutation. +-- +-- PERFORMANCE-ONLY ROUTING CANARY: the shared gd_runtime_20260801_a login has +-- SELECT/EXECUTE access across every tenant schema. This fixture brackets the +-- Graphile memory-density curve; it does not prove database-enforced tenant +-- isolation and cannot qualify a complete customer surface for production. + +\if :{?resume} +\else + \set resume 0 +\endif + +\echo 'Preflighting source, target, and runtime role' +\echo 'PERFORMANCE_ONLY_ROUTING_CANARY: this fixture is not a tenant-isolation proof' + +SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_database + WHERE datname = 'graphile_density_20260801_a' +) AS source_exists, +EXISTS ( + SELECT 1 FROM pg_catalog.pg_database + WHERE datname = 'graphile_density_uniform_20260801_a' +) AS target_exists, +EXISTS ( + SELECT 1 FROM pg_catalog.pg_roles + WHERE rolname = 'gd_runtime_20260801_a' +) AS runtime_role_exists +\gset + +\if :source_exists +\else + \echo 'GRAPHILE_DENSITY_SOURCE_MISSING: graphile_density_20260801_a' + \quit 3 +\endif + +\if :target_exists + \if :resume + \echo 'Resuming explicitly against the existing asserted-pristine clone' + \else + \echo 'GRAPHILE_DENSITY_TARGET_EXISTS: graphile_density_uniform_20260801_a' + \quit 4 + \endif +\endif + +\if :runtime_role_exists +\else + \echo 'GRAPHILE_DENSITY_RUNTIME_ROLE_MISSING: gd_runtime_20260801_a' + \quit 5 +\endif + +\echo 'Creating immutable physical clone graphile_density_uniform_20260801_a' + +\if :target_exists +\else + CREATE DATABASE graphile_density_uniform_20260801_a + WITH TEMPLATE graphile_density_20260801_a + OWNER postgres; +\endif + +\connect graphile_density_uniform_20260801_a + +SET client_min_messages = warning; +SET statement_timeout = 0; +SET lock_timeout = '30s'; +SET idle_in_transaction_session_timeout = '5min'; + +REVOKE CREATE ON DATABASE graphile_density_uniform_20260801_a + FROM PUBLIC, gd_runtime_20260801_a; +GRANT CONNECT ON DATABASE graphile_density_uniform_20260801_a + TO gd_runtime_20260801_a; + +\echo 'Validating the cloned heterogeneous source shape' + +DO $preflight$ +DECLARE + class_count integer; + tenant_schema_count integer; + full_schema_count integer; + function_only_schema_count integer; +BEGIN + SELECT count(*) INTO class_count FROM pg_catalog.pg_class; + IF class_count <> 61239 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_SOURCE_CLASS_COUNT_MISMATCH: expected 61239, got %', + class_count; + END IF; + + WITH tenant_shapes AS ( + SELECT namespace.nspname, + (SELECT count(*) + FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid) AS class_count, + (SELECT count(*) + FROM pg_catalog.pg_proc AS procedure + WHERE procedure.pronamespace = namespace.oid) AS proc_count, + (SELECT count(*) + FROM pg_catalog.pg_constraint AS constraint_row + WHERE constraint_row.connamespace = namespace.oid) AS constraint_count + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + ) + SELECT count(*), + count(*) FILTER ( + WHERE tenant_shapes.class_count = 7 + AND tenant_shapes.proc_count = 1 + AND tenant_shapes.constraint_count = 9 + ), + count(*) FILTER ( + WHERE tenant_shapes.class_count = 0 + AND tenant_shapes.proc_count = 1 + AND tenant_shapes.constraint_count = 0 + ) + INTO tenant_schema_count, full_schema_count, function_only_schema_count + FROM tenant_shapes; + + IF tenant_schema_count <> 2000 + OR full_schema_count <> 400 + OR function_only_schema_count <> 1600 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_SOURCE_SHAPE_MISMATCH: schemas %, full %, function-only %', + tenant_schema_count, full_schema_count, function_only_schema_count; + END IF; +END +$preflight$; + +-- DDL is deliberately split into 100-tenant transactions. A single +-- transaction would retain locks for tens of thousands of new relations and +-- can exhaust max_locks_per_transaction on an otherwise healthy local server. +CREATE PROCEDURE pg_temp.create_uniform_tenants( + batch_start integer, + batch_end integer +) +LANGUAGE plpgsql +AS $procedure$ +DECLARE + tenant_number integer; + tenant_suffix text; + tenant_schema text; + tenant_token text; +BEGIN + FOR tenant_number IN batch_start..batch_end LOOP + tenant_suffix := CASE + WHEN tenant_number < 1000 + THEN pg_catalog.lpad(tenant_number::text, 3, '0') + ELSE tenant_number::text + END; + tenant_schema := 'gd_t' || tenant_suffix || '_api'; + tenant_token := 'tenant-' || tenant_suffix || '-token'; + + IF tenant_number > 2000 THEN + EXECUTE pg_catalog.format( + 'CREATE SCHEMA %I AUTHORIZATION postgres', + tenant_schema + ); + END IF; + + EXECUTE pg_catalog.format( + 'CREATE OR REPLACE FUNCTION %I.tenant_token()' + ' RETURNS text LANGUAGE sql STABLE AS %L', + tenant_schema, + 'SELECT ' || pg_catalog.quote_literal(tenant_token) || '::text' + ); + + EXECUTE pg_catalog.format( + 'CREATE TABLE %I.tenant_canary (' + ' id bigint GENERATED ALWAYS AS IDENTITY,' + ' tenant_token text NOT NULL,' + ' CONSTRAINT tenant_canary_pkey PRIMARY KEY (id),' + ' CONSTRAINT tenant_canary_tenant_token_key UNIQUE (tenant_token)' + ')', + tenant_schema + ); + + EXECUTE pg_catalog.format( + 'CREATE TABLE %I.widget (' + ' id bigint GENERATED ALWAYS AS IDENTITY,' + ' canary_id bigint NOT NULL,' + ' label text NOT NULL,' + ' CONSTRAINT widget_pkey PRIMARY KEY (id),' + ' CONSTRAINT widget_canary_id_fkey FOREIGN KEY (canary_id)' + ' REFERENCES %I.tenant_canary(id)' + ')', + tenant_schema, + tenant_schema + ); + END LOOP; +END +$procedure$; + +SELECT pg_catalog.format( + 'CALL pg_temp.create_uniform_tenants(%s, %s)', + batch_start, + least(batch_start + 99, 4000) +) +FROM pg_catalog.generate_series(401, 4000, 100) AS batch(batch_start) +\gexec + +DROP PROCEDURE pg_temp.create_uniform_tenants(integer, integer); + +DO $intermediate_count$ +DECLARE + class_count integer; +BEGIN + SELECT count(*) INTO class_count FROM pg_catalog.pg_class; + IF class_count <> 100839 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_UNIFORM_CLASS_DELTA_MISMATCH: expected 100839, got %', + class_count; + END IF; +END +$intermediate_count$; + +\echo 'Normalizing ownership, grants, identity state, and canary rows' + +CREATE PROCEDURE pg_temp.normalize_uniform_tenants( + batch_start integer, + batch_end integer +) +LANGUAGE plpgsql +AS $procedure$ +DECLARE + tenant_number integer; + tenant_suffix text; + tenant_schema text; + tenant_token text; + widget_label text; +BEGIN + FOR tenant_number IN batch_start..batch_end LOOP + tenant_suffix := CASE + WHEN tenant_number < 1000 + THEN pg_catalog.lpad(tenant_number::text, 3, '0') + ELSE tenant_number::text + END; + tenant_schema := 'gd_t' || tenant_suffix || '_api'; + tenant_token := 'tenant-' || tenant_suffix || '-token'; + widget_label := 'tenant-' || tenant_suffix || '-widget'; + + EXECUTE pg_catalog.format('ALTER SCHEMA %I OWNER TO postgres', tenant_schema); + EXECUTE pg_catalog.format( + 'ALTER TABLE %I.tenant_canary OWNER TO postgres', tenant_schema + ); + EXECUTE pg_catalog.format( + 'ALTER TABLE %I.widget OWNER TO postgres', tenant_schema + ); + EXECUTE pg_catalog.format( + 'ALTER FUNCTION %I.tenant_token() OWNER TO postgres', tenant_schema + ); + + EXECUTE pg_catalog.format( + 'TRUNCATE TABLE %I.widget, %I.tenant_canary RESTART IDENTITY', + tenant_schema, + tenant_schema + ); + EXECUTE pg_catalog.format( + 'INSERT INTO %I.tenant_canary (tenant_token) VALUES (%L)', + tenant_schema, + tenant_token + ); + EXECUTE pg_catalog.format( + 'INSERT INTO %I.widget (canary_id, label) VALUES (1, %L)', + tenant_schema, + widget_label + ); + + EXECUTE pg_catalog.format( + 'REVOKE ALL PRIVILEGES ON SCHEMA %I' + ' FROM PUBLIC, postgres, gd_runtime_20260801_a', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT ALL PRIVILEGES ON SCHEMA %I TO postgres', tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT USAGE ON SCHEMA %I TO gd_runtime_20260801_a', tenant_schema + ); + + EXECUTE pg_catalog.format( + 'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I' + ' FROM PUBLIC, postgres, gd_runtime_20260801_a', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I TO postgres', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT SELECT ON ALL TABLES IN SCHEMA %I TO gd_runtime_20260801_a', + tenant_schema + ); + + EXECUTE pg_catalog.format( + 'REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA %I' + ' FROM PUBLIC, postgres, gd_runtime_20260801_a', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA %I TO postgres', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT SELECT, USAGE ON ALL SEQUENCES IN SCHEMA %I' + ' TO gd_runtime_20260801_a', + tenant_schema + ); + + EXECUTE pg_catalog.format( + 'REVOKE ALL PRIVILEGES ON FUNCTION %I.tenant_token()' + ' FROM PUBLIC, postgres, gd_runtime_20260801_a', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT ALL PRIVILEGES ON FUNCTION %I.tenant_token() TO postgres', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT EXECUTE ON FUNCTION %I.tenant_token()' + ' TO gd_runtime_20260801_a', + tenant_schema + ); + END LOOP; +END +$procedure$; + +SELECT pg_catalog.format( + 'CALL pg_temp.normalize_uniform_tenants(%s, %s)', + batch_start, + least(batch_start + 99, 4000) +) +FROM pg_catalog.generate_series(1, 4000, 100) AS batch(batch_start) +\gexec + +DROP PROCEDURE pg_temp.normalize_uniform_tenants(integer, integer); + +\echo 'Planning a footprint-exact reduction of disposable gd_noise tables' + +CREATE TEMPORARY TABLE noise_drop_plan ON COMMIT PRESERVE ROWS AS +WITH noise_tables AS ( + SELECT class.oid, class.relname, class.reltoastrelid, + 1 + + ( + SELECT count(*) + FROM pg_catalog.pg_index AS table_index + WHERE table_index.indrelid = class.oid + ) + + ( + SELECT count(*) + FROM pg_catalog.pg_depend AS sequence_dependency + JOIN pg_catalog.pg_class AS sequence + ON sequence.oid = sequence_dependency.objid + AND sequence.relkind = 'S' + WHERE sequence_dependency.classid = 'pg_catalog.pg_class'::regclass + AND sequence_dependency.refclassid = 'pg_catalog.pg_class'::regclass + AND sequence_dependency.refobjid = class.oid + AND sequence_dependency.deptype = 'i' + ) + + CASE WHEN class.reltoastrelid = 0 THEN 0 ELSE 1 END + + ( + SELECT count(*) + FROM pg_catalog.pg_index AS toast_index + WHERE toast_index.indrelid = class.reltoastrelid + ) AS class_footprint + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = class.relnamespace + WHERE namespace.nspname = 'gd_noise' + AND class.relkind = 'r' +), candidates AS ( + SELECT relname, class_footprint, + substring(relname FROM '^r_([0-9]+)$')::integer AS noise_number + FROM noise_tables + WHERE class_footprint = 5 +) +SELECT row_number() OVER (ORDER BY noise_number DESC) AS ordinal, + relname, + class_footprint +FROM candidates +ORDER BY noise_number DESC +LIMIT 7920; + +DO $drop_plan$ +DECLARE + candidate_count integer; + planned_footprint integer; +BEGIN + SELECT count(*), sum(class_footprint) + INTO candidate_count, planned_footprint + FROM pg_temp.noise_drop_plan; + + IF candidate_count <> 7920 OR planned_footprint <> 39600 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_NOISE_PLAN_MISMATCH: tables %, pg_class footprint %', + candidate_count, planned_footprint; + END IF; +END +$drop_plan$; + +CREATE PROCEDURE pg_temp.drop_noise_batch( + batch_start integer, + batch_end integer +) +LANGUAGE plpgsql +AS $procedure$ +DECLARE + candidate record; +BEGIN + FOR candidate IN + SELECT relname + FROM pg_temp.noise_drop_plan + WHERE ordinal BETWEEN batch_start AND batch_end + ORDER BY ordinal + LOOP + EXECUTE pg_catalog.format( + 'DROP TABLE gd_noise.%I CASCADE', + candidate.relname + ); + END LOOP; +END +$procedure$; + +SELECT pg_catalog.format( + 'CALL pg_temp.drop_noise_batch(%s, %s)', + batch_start, + least(batch_start + 99, 7920) +) +FROM pg_catalog.generate_series(1, 7920, 100) AS batch(batch_start) +\gexec + +DROP PROCEDURE pg_temp.drop_noise_batch(integer, integer); +DROP TABLE pg_temp.noise_drop_plan; + +\echo 'Analyzing the catalog tables used by Graphile introspection' + +ANALYZE pg_catalog.pg_namespace; +ANALYZE pg_catalog.pg_class; +ANALYZE pg_catalog.pg_attribute; +ANALYZE pg_catalog.pg_constraint; +ANALYZE pg_catalog.pg_proc; +ANALYZE pg_catalog.pg_depend; +ANALYZE pg_catalog.pg_index; + +\echo 'Hard-gating the uniform tenant shape and exact catalog count' + +DO $uniform_shape$ +DECLARE + class_count integer; + tenant_schema_count integer; + distinct_shape_count integer; + missing_schema_count integer; +BEGIN + SELECT count(*) INTO class_count FROM pg_catalog.pg_class; + IF class_count <> 61239 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_FINAL_CLASS_COUNT_MISMATCH: expected 61239, got %', + class_count; + END IF; + + WITH expected AS ( + SELECT 'gd_t' || + CASE WHEN tenant_number < 1000 + THEN pg_catalog.lpad(tenant_number::text, 3, '0') + ELSE tenant_number::text + END || '_api' AS nspname + FROM pg_catalog.generate_series(1, 4000) AS tenant(tenant_number) + ), actual AS ( + SELECT nspname + FROM pg_catalog.pg_namespace + WHERE nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + ), difference AS ( + (SELECT nspname FROM expected EXCEPT SELECT nspname FROM actual) + UNION ALL + (SELECT nspname FROM actual EXCEPT SELECT nspname FROM expected) + ) + SELECT (SELECT count(*) FROM actual), count(*) + INTO tenant_schema_count, missing_schema_count + FROM difference; + + IF tenant_schema_count <> 4000 OR missing_schema_count <> 0 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_SCHEMA_SET_MISMATCH: actual %, symmetric difference %', + tenant_schema_count, missing_schema_count; + END IF; + + WITH tenant_shapes AS ( + SELECT namespace.nspname, + pg_catalog.md5(pg_catalog.concat_ws('|', + namespace.nspowner::regrole::text, + coalesce(namespace.nspacl::text, ''), + ( + SELECT pg_catalog.string_agg( + pg_catalog.concat_ws(':', class.relname, class.relkind, + class.relowner::regrole::text, coalesce(class.relacl::text, '')), + ',' ORDER BY class.relname + ) + FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid + ), + ( + SELECT pg_catalog.string_agg( + pg_catalog.concat_ws(':', class.relname, attribute.attnum, + attribute.attname, + pg_catalog.format_type(attribute.atttypid, attribute.atttypmod), + attribute.attnotnull, attribute.attidentity, + attribute.attgenerated), + ',' ORDER BY class.relname, attribute.attnum + ) + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_attribute AS attribute + ON attribute.attrelid = class.oid + WHERE class.relnamespace = namespace.oid + AND class.relkind = 'r' + AND attribute.attnum > 0 + AND NOT attribute.attisdropped + ), + ( + SELECT pg_catalog.string_agg( + pg_catalog.concat_ws(':', constraint_row.conname, + constraint_row.contype, constraint_row.conkey::text, + constraint_row.confkey::text, + coalesce(referenced_class.relname, '')), + ',' ORDER BY constraint_row.conname + ) + FROM pg_catalog.pg_constraint AS constraint_row + LEFT JOIN pg_catalog.pg_class AS referenced_class + ON referenced_class.oid = constraint_row.confrelid + WHERE constraint_row.connamespace = namespace.oid + ), + ( + SELECT pg_catalog.string_agg( + pg_catalog.concat_ws(':', procedure.proname, + pg_catalog.pg_get_function_identity_arguments(procedure.oid), + pg_catalog.pg_get_function_result(procedure.oid), + procedure.provolatile, procedure.prosecdef, + procedure.proowner::regrole::text, + coalesce(procedure.proacl::text, '')), + ',' ORDER BY procedure.proname + ) + FROM pg_catalog.pg_proc AS procedure + WHERE procedure.pronamespace = namespace.oid + ), + ( + SELECT pg_catalog.string_agg( + pg_catalog.concat_ws(':', sequence.sequencename, + sequence.data_type, sequence.start_value, + sequence.min_value, sequence.max_value, + sequence.increment_by, sequence.cycle, + sequence.cache_size, sequence.last_value), + ',' ORDER BY sequence.sequencename + ) + FROM pg_catalog.pg_sequences AS sequence + WHERE sequence.schemaname = namespace.nspname + ) + )) AS shape_fingerprint + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + ) + SELECT count(DISTINCT shape_fingerprint) + INTO distinct_shape_count + FROM tenant_shapes; + + IF distinct_shape_count <> 1 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_NON_UNIFORM_SHAPE: % distinct fingerprints', + distinct_shape_count; + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND ( + (SELECT count(*) FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid) <> 7 + OR + (SELECT count(*) FROM pg_catalog.pg_proc AS procedure + WHERE procedure.pronamespace = namespace.oid) <> 1 + OR + (SELECT count(*) FROM pg_catalog.pg_constraint AS constraint_row + WHERE constraint_row.connamespace = namespace.oid) <> 9 + ) + ) THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_NON_UNIFORM_COUNTS: expected class/proc/constraint 7/1/9'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = class.relnamespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND class.relowner <> 'postgres'::regrole + ) OR EXISTS ( + SELECT 1 + FROM pg_catalog.pg_proc AS procedure + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = procedure.pronamespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND procedure.proowner <> 'postgres'::regrole + ) THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_OWNER_MISMATCH'; + END IF; +END +$uniform_shape$; + +\echo 'Validating all 4,000 surfaces under the runtime identity' + +CREATE PROCEDURE pg_temp.validate_runtime_batch( + batch_start integer, + batch_end integer +) +LANGUAGE plpgsql +AS $runtime_validation$ +DECLARE + tenant_number integer; + tenant_suffix text; + tenant_schema text; + expected_token text; + expected_label text; + function_token text; + table_token text; + widget_label text; + role_row record; +BEGIN + SELECT * INTO role_row + FROM pg_catalog.pg_roles + WHERE rolname = current_user; + + IF session_user <> 'gd_runtime_20260801_a' + OR current_user <> 'gd_runtime_20260801_a' + OR role_row.rolsuper + OR role_row.rolcreaterole + OR role_row.rolcreatedb + OR role_row.rolbypassrls THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_RUNTIME_ROLE_UNSAFE: %', current_user; + END IF; + + IF NOT pg_catalog.has_database_privilege( + current_user, current_database(), 'CONNECT' + ) OR pg_catalog.has_database_privilege( + current_user, current_database(), 'CREATE' + ) THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_RUNTIME_DATABASE_PRIVILEGE_MISMATCH'; + END IF; + + FOR tenant_number IN batch_start..batch_end LOOP + tenant_suffix := CASE + WHEN tenant_number < 1000 + THEN pg_catalog.lpad(tenant_number::text, 3, '0') + ELSE tenant_number::text + END; + tenant_schema := 'gd_t' || tenant_suffix || '_api'; + expected_token := 'tenant-' || tenant_suffix || '-token'; + expected_label := 'tenant-' || tenant_suffix || '-widget'; + + IF NOT pg_catalog.has_schema_privilege( + current_user, tenant_schema, 'USAGE' + ) OR pg_catalog.has_schema_privilege( + current_user, tenant_schema, 'CREATE' + ) THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_RUNTIME_SCHEMA_PRIVILEGE_MISMATCH: %', + tenant_schema; + END IF; + + IF NOT pg_catalog.has_table_privilege( + current_user, + pg_catalog.format('%I.tenant_canary', tenant_schema), + 'SELECT' + ) OR pg_catalog.has_table_privilege( + current_user, + pg_catalog.format('%I.tenant_canary', tenant_schema), + 'INSERT,UPDATE,DELETE' + ) THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_RUNTIME_TABLE_PRIVILEGE_MISMATCH: %', + tenant_schema; + END IF; + + IF NOT pg_catalog.has_function_privilege( + current_user, + pg_catalog.format('%I.tenant_token()', tenant_schema), + 'EXECUTE' + ) THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_RUNTIME_FUNCTION_PRIVILEGE_MISMATCH: %', + tenant_schema; + END IF; + + EXECUTE pg_catalog.format( + 'SELECT %I.tenant_token()', tenant_schema + ) INTO function_token; + EXECUTE pg_catalog.format( + 'SELECT tenant_token FROM %I.tenant_canary', tenant_schema + ) INTO table_token; + EXECUTE pg_catalog.format( + 'SELECT label FROM %I.widget', tenant_schema + ) INTO widget_label; + + IF function_token <> expected_token + OR table_token <> expected_token + OR widget_label <> expected_label THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_RUNTIME_CANARY_MISMATCH: schema %, function %, table %, widget %', + tenant_schema, function_token, table_token, widget_label; + END IF; + END LOOP; +END +$runtime_validation$; + +REVOKE ALL PRIVILEGES ON PROCEDURE + pg_temp.validate_runtime_batch(integer, integer) FROM PUBLIC; +GRANT EXECUTE ON PROCEDURE + pg_temp.validate_runtime_batch(integer, integer) + TO gd_runtime_20260801_a; + +SET SESSION AUTHORIZATION gd_runtime_20260801_a; + +SELECT pg_catalog.format( + 'CALL pg_temp.validate_runtime_batch(%s, %s)', + batch_start, + least(batch_start + 99, 4000) +) +FROM pg_catalog.generate_series(1, 4000, 100) AS batch(batch_start) +\gexec + +RESET SESSION AUTHORIZATION; + +DROP PROCEDURE pg_temp.validate_runtime_batch(integer, integer); + +\echo 'Recording deterministic logical catalog and tenant-shape fingerprints' + +WITH normalized_classes AS ( + SELECT pg_catalog.concat_ws('|', + CASE WHEN namespace.nspname = 'pg_toast' + THEN 'pg_toast.' + ELSE namespace.nspname || '.' || class.relname + END, + class.relkind, + class.relpersistence, + class.relowner::regrole::text, + coalesce(access_method.amname, ''), + class.relnatts, + class.relchecks, + class.relhasindex, + class.reltoastrelid <> 0, + class.relispartition, + coalesce(class.relacl::text, '') + ) AS logical_class + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = class.relnamespace + LEFT JOIN pg_catalog.pg_am AS access_method + ON access_method.oid = class.relam +), tenant_shapes AS ( + SELECT namespace.nspname, + pg_catalog.concat_ws('|', + (SELECT count(*) FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid), + (SELECT count(*) FROM pg_catalog.pg_proc AS procedure + WHERE procedure.pronamespace = namespace.oid), + (SELECT count(*) FROM pg_catalog.pg_constraint AS constraint_row + WHERE constraint_row.connamespace = namespace.oid), + (SELECT pg_catalog.string_agg( + class.relname || ':' || class.relkind::text, + ',' ORDER BY class.relname) + FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid) + ) AS logical_shape + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' +) +SELECT current_database() AS database_name, + (SELECT count(*) FROM pg_catalog.pg_class) AS pg_class_count, + (SELECT pg_catalog.md5(pg_catalog.string_agg( + logical_class, E'\n' ORDER BY logical_class)) + FROM normalized_classes) AS logical_pg_class_fingerprint, + (SELECT count(*) FROM tenant_shapes) AS tenant_schema_count, + (SELECT count(DISTINCT logical_shape) FROM tenant_shapes) + AS distinct_tenant_shapes, + (SELECT pg_catalog.md5(pg_catalog.string_agg( + nspname || '|' || logical_shape, E'\n' ORDER BY nspname)) + FROM tenant_shapes) AS tenant_shape_fingerprint; + +\echo 'Uniform density fixture completed successfully' diff --git a/research/graphile-density/fleet.example.json b/research/graphile-density/fleet.example.json new file mode 100644 index 0000000000..03ee4b58b3 --- /dev/null +++ b/research/graphile-density/fleet.example.json @@ -0,0 +1,82 @@ +{ + "version": 1, + "tenants": [ + { + "id": "replace-with-tenant-a", + "databases": [ + { + "id": "replace-with-tenant-a-database-id", + "physicalDatabase": "replace-with-tenant-a-physical-database", + "apis": [ + { + "id": "replace-with-tenant-a-api-id", + "runtimePoolIdentity": "replace-with-tenant-a-runtime-pool-identity", + "runtimePoolIdentities": { + "origin-main": "replace-with-origin-main-tenant-a-pool-identity", + "runtime-boundary-stock": "replace-with-runtime-stock-tenant-a-pool-identity", + "cache-governor-stock": "replace-with-governor-stock-tenant-a-pool-identity", + "scoped-introspection": "replace-with-scoped-tenant-a-pool-identity" + }, + "physicalSchemas": [ + "replace-with-tenant-a-api-schema" + ], + "routingLabels": [ + "replace-with-tenant-a-api-host-or-service-label" + ], + "realtime": false, + "surfaces": [ + "api" + ] + } + ] + } + ], + "surfaces": [ + { + "name": "api", + "buildContract": "", + "buildContracts": { + "origin-main": "replace-with-origin-main-tenant-a-api-build-contract", + "runtime-boundary-stock": "replace-with-runtime-stock-tenant-a-api-build-contract", + "cache-governor-stock": "replace-with-governor-stock-tenant-a-api-build-contract", + "scoped-introspection": "replace-with-scoped-tenant-a-api-build-contract" + }, + "url": "http://127.0.0.1:{port}/graphql", + "headers": { + "host": "replace-with-tenant-a-api-host" + }, + "warmup": { + "name": "warm-schema", + "capability": "graphile-generated", + "query": "query { __typename }" + }, + "operations": [ + { + "name": "replace-with-a-real-generated-plan", + "capability": "graphile-generated", + "query": "query { __typename }" + } + ], + "canaries": [ + { + "name": "cross-schema-identifiers", + "query": "query { __typename }", + "forbiddenMatches": [ + { + "path": "/data/tenantToken", + "value": "replace-with-tenant-b-token" + } + ], + "requiredMatches": [ + { + "path": "/data/tenantToken", + "value": "replace-with-tenant-a-token" + } + ] + } + ] + } + ] + } + ] +} diff --git a/research/graphile-density/four-arm-plan.example.json b/research/graphile-density/four-arm-plan.example.json new file mode 100644 index 0000000000..c4d1753db4 --- /dev/null +++ b/research/graphile-density/four-arm-plan.example.json @@ -0,0 +1,121 @@ +{ + "version": 1, + "fleetFile": "fleet.example.json", + "artifactDir": "../../graphile-density-artifacts", + "arms": [ + { + "name": "origin-main", + "commit": "a10ea246fcc45b025713024e131fafb908171149", + "cwd": "/ABSOLUTE/PATH/TO/origin-main-worktree", + "command": ["node", "packages/cli/dist/index.js", "server", "--port", "{port}", "--origin", "*", "--servicesApi"], + "port": 3341, + "readinessUrl": "http://127.0.0.1:{port}/healthz", + "memoryUrl": "http://127.0.0.1:{port}/debug/memory", + "postgresContainer": "REPLACE_WITH_DISPOSABLE_POSTGRES_CONTAINER", + "introspectionMode": "stock" + }, + { + "name": "runtime-boundary-stock", + "commit": "de92be5a9", + "cwd": "/ABSOLUTE/PATH/TO/runtime-boundary-worktree", + "command": ["node", "packages/cli/dist/index.js", "server", "--port", "{port}", "--origin", "*", "--servicesApi"], + "port": 3342, + "readinessUrl": "http://127.0.0.1:{port}/healthz", + "memoryUrl": "http://127.0.0.1:{port}/debug/memory", + "postgresContainer": "REPLACE_WITH_DISPOSABLE_POSTGRES_CONTAINER", + "introspectionMode": "stock" + }, + { + "name": "cache-governor-stock", + "commit": "5fd90ee2b", + "cwd": "/ABSOLUTE/PATH/TO/cache-governor-worktree", + "command": ["node", "packages/cli/dist/index.js", "server", "--port", "{port}", "--origin", "*", "--servicesApi"], + "port": 3343, + "readinessUrl": "http://127.0.0.1:{port}/healthz", + "memoryUrl": "http://127.0.0.1:{port}/debug/memory", + "postgresContainer": "REPLACE_WITH_DISPOSABLE_POSTGRES_CONTAINER", + "introspectionMode": "stock" + }, + { + "name": "scoped-introspection", + "commit": "f41e480d5", + "cwd": "/ABSOLUTE/PATH/TO/scoped-introspection-worktree", + "command": ["node", "packages/cli/dist/index.js", "server", "--port", "{port}", "--origin", "*", "--servicesApi"], + "port": 3344, + "readinessUrl": "http://127.0.0.1:{port}/healthz", + "memoryUrl": "http://127.0.0.1:{port}/debug/memory", + "postgresContainer": "REPLACE_WITH_DISPOSABLE_POSTGRES_CONTAINER", + "introspectionMode": "scoped-required" + } + ], + "heapMiB": [1024, 2048, 4096], + "tenantCountsByHeapMiB": { + "1024": [1, 2, 4, 8, 16, 32, 48, 64], + "2048": [1, 4, 8, 16, 32, 64, 96, 128], + "4096": [1, 8, 16, 32, 64, 128, 192, 256] + }, + "repetitions": 3, + "runOrderSeed": "graphile-density-qualification-v1", + "requiredCapabilities": [ + "graphile-generated", + "i18n", + "llm", + "rag", + "bm25", + "tsvector", + "trigram", + "vector", + "postgis", + "ltree", + "uploads-storage", + "bulk-mutations", + "realtime", + "function-bindings" + ], + "requiredCanaries": [ + "cross-schema-identifiers", + "metadata", + "functions", + "sequences", + "prepared-statement-reuse", + "poisoned-gucs", + "rollback-savepoints", + "plugin-raw-sql", + "owner-bypass-role", + "schema-drift", + "cache-invalidation", + "concurrent-builds", + "connection-reuse" + ], + "workload": { + "durationSec": 900, + "rpsPerTenant": 0.2, + "minWorkloadRequestsPerSurface": 10, + "requestTimeoutMs": 30000, + "maxInFlight": 128, + "canaryIntervalSec": 60, + "warmupTimeoutMs": 180000, + "warmupTimeoutPerSurfaceMs": 2000, + "warmupConcurrency": 1 + }, + "gates": { + "maxErrorRate": 0.005, + "maxP99Ms": 150, + "maxPostWarmupHeapGrowthMiBPerHour": 5, + "minMedianDensityImprovement": 0.15, + "minAdditionalTenantsEveryRun": 1, + "requireZeroBleed": true, + "requireNoPostWarmupEvictions": true, + "requireNoPostWarmupBuildRefusals": true, + "requireNoPostWarmupBuilds": true, + "requirePostgresMemoryTelemetry": true, + "requireConclusiveCanaries": true, + "requireExplicitCustomerTopology": true + }, + "soak": { + "enabled": true, + "durationSec": 7200, + "tenantCount": 48, + "heapMiB": 2048 + } +} diff --git a/research/graphile-density/physical-database-density/.gitignore b/research/graphile-density/physical-database-density/.gitignore new file mode 100644 index 0000000000..b7f1ad81a9 --- /dev/null +++ b/research/graphile-density/physical-database-density/.gitignore @@ -0,0 +1,2 @@ +.local/ +artifacts/ diff --git a/research/graphile-density/physical-database-density/HOSTILE-PREFLIGHT.md b/research/graphile-density/physical-database-density/HOSTILE-PREFLIGHT.md new file mode 100644 index 0000000000..3fb98db36c --- /dev/null +++ b/research/graphile-density/physical-database-density/HOSTILE-PREFLIGHT.md @@ -0,0 +1,62 @@ +# Physical hostile preflight + +Run this validation against a dedicated, unmeasured physical-fixture server before starting cperf. Provision and serve the hostile clone with the same explicit identity and purpose: + +```bash +node research/graphile-density/physical-database-density/provision.cjs \ + --prefix pdc_hostile \ + --customers 8 \ + --out-dir /absolute/path/to/hostile-clone \ + --maintenance-database postgres \ + --run-purpose hostile-preflight \ + --clone-id fresh-preflight-clone-20260802-a + +node --expose-gc research/graphile-density/physical-database-density/server.cjs \ + --manifest /absolute/path/to/hostile-clone/provision.json \ + --secrets /absolute/path/to/hostile-clone/runtime-secrets.json \ + --customers 8 \ + --arm physical-db-idle-1s \ + --mode scoped-required \ + --runtime-pool-max 2 \ + --enable-realtime true \ + --expected-database-contract 'sha256:' \ + --blueprint-compatibility 'sha256:' \ + --run-purpose hostile-preflight \ + --clone-id fresh-preflight-clone-20260802-a +``` + +The provisioner writes a distinct 256-bit nonce into a private schema in every physical database. The nonce never leaves PostgreSQL: the credential-free manifest contains only its context-bound digest, and both the child and aggregate status endpoints query the row live and recompute the digest before reporting `verified: true`. Runtime roles have no privilege on the private schema. + +The validator loads the credential-free manifest and the provisioner's private runtime-secrets file. `--secrets` must name a regular, non-symlink file owned by the current user with no group or other permission bits (normally mode `0600`). The parent reads that file once. Each isolated worker receives exactly the representative customer's three A/B/C passwords through its private process environment, with the selected surface replaced by the probe credential; no worker receives the secrets path or another customer's credential. Neither the secrets path, its contents, nor the authenticated `CTF_CONTROL_TOKEN` is written to the artifact. + +```bash +CTF_CONTROL_TOKEN='' node \ + research/graphile-density/physical-database-density/physical-hostile-preflight.cjs \ + --manifest /absolute/path/to/provision.json \ + --secrets /absolute/path/to/runtime-secrets.json \ + --base-url http://127.0.0.1:3410 \ + --arm physical-db-idle-1s \ + --mode scoped-required \ + --preflight-clone-id fresh-preflight-clone-20260802-a \ + --output /absolute/path/to/physical-hostile-preflight.json +``` + +The validator requires the server's exact hostile-preflight clone ID, live attestation set, manifest customer set, arm, introspection mode, canonical database contract, blueprint fingerprint, and runtime artifact fingerprint. Hostile startup also recomputes every customer's structural and database-contract fingerprints from the live database rather than echoing the provision manifest. It then walks customers sequentially through the exact `/customer/` mount, requires `current_database()` to match that customer's physical identity during every dynamic identity and control sequence, invalidates the whole fleet before rebuilding every surface, and records only credential-free hashes and outcomes. + +Fixture startup admission has one safe-role control followed by an exact 15-case unsafe matrix against a representative hostile physical database. The parent validator first reads the private attestation row and recomputes its context-bound digest, then a live catalog query proves that the temporary roles have exactly the intended profiles: `SUPERUSER`, `BYPASSRLS`, `CREATEROLE`, schema ownership, and schema `CREATE`. The control-plane login, `NODE_OPTIONS`, `NODE_PATH`, and generic Graphile environment are never forwarded to a probe child. Each child validates its exact environment key set, verifies `current_database()` and `current_user` with the selected runtime credential, and substitutes each profile independently into surfaces A, B, and C in a fresh process. All 15 fixture starts must fail with `GRAPHILE_UNSAFE_RUNTIME_ROLE` before the fixture's `buildEntry` or cache publication; the safe control must start successfully with zero builds and zero resident entries. Cleanup is attempted even after an ambiguous setup failure and a live catalog audit must find zero remaining probe roles and schemas. + +This matrix exercises `complete-tenant-fixture.createFixtureServer`, whose role audit is deliberately ahead of its custom build seam. It does not claim to execute the production `graphile()` middleware path. Every worker reports the exact loaded runtime-artifact fingerprint, and all 16 worker results must match the mounted hostile server before the aggregate artifact can pass. + +This command must never be imported by or invoked from the measured cperf process. Provision the measured clone separately with `--run-purpose measurement` and a different `--clone-id`, then pass those exact values to its server. A measurement-purpose server reports `controlAvailable: false` and refuses the entire hostile control endpoint even if a valid control token is accidentally present. The measured clone must reproduce the canonical structural and database-contract fingerprints, but it must not inherit the preflight clone's sessions, PostgreSQL cache state, mutations, nonce, or cgroup history. A passing preflight artifact is security evidence only, so it is explicitly marked `performanceEvidence: false` and `customerQualified: false`. + +The runtime fingerprint hashes the deterministic resolved closure of local `dist` JavaScript reached from the fixture's runtime roots, plus the exact patched installed Graphile artifacts. Changes behind an `index.js` export stub therefore invalidate the evidence without recording absolute paths, source bytes, or credentials. + +Focused tests: + +```bash +node --test \ + research/graphile-density/complete-tenant-fixture/schema.test.cjs \ + research/graphile-density/complete-tenant-fixture/hostile-validation.test.cjs \ + research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.test.cjs \ + research/graphile-density/physical-database-density/physical-hostile-preflight.test.cjs +``` diff --git a/research/graphile-density/physical-database-density/README.md b/research/graphile-density/physical-database-density/README.md new file mode 100644 index 0000000000..9e4a78dc23 --- /dev/null +++ b/research/graphile-density/physical-database-density/README.md @@ -0,0 +1,100 @@ +# Physical-database customer-density fixture + +This fixture measures complete customers per actual GiB consumed; it does not try to fit the service into 1 GiB. The 1, 2, and 4 GiB V8 settings are repeatable pressure points, while the primary denominator is the maximum post-warmup time-aligned sum of current Node RSS and the dedicated PostgreSQL container's raw cgroup-v2 memory charge. + +One logical customer owns one physical PostgreSQL database. Every database has the same three canonical GraphQL surfaces (`ctf_a`, `ctf_b`, and `ctf_c`), the same realtime schemas, build-visible dependency schemas, extension versions, and role-relative ACL shape. Each surface still gets its own least-privilege login, pool identity, and dedicated Graphile instance, so this is the secure production baseline rather than a shared-blueprint implementation. + +The provisioner emits two hashes. The database-contract hash covers normalized schema DDL and ACLs, extension versions, and runtime-role safety flags. The input preflight combines that hash with the exact fixture plugin/settings configuration, dependency closure, source, and built runtime artifact hash. That second hash proves only the structural prerequisites for a future no-rewrite blueprint experiment; it does not authorize sharing, and the fixture never rewrites SQL. + +## Run locally + +Use an existing disposable PostgreSQL 17 container with all fixture extensions available. Input generation only inspects and preflights that exact container. Each later cperf job deliberately removes it and creates a fresh replacement with the same immutable image, loopback port, cgroup resource limits, and narrowly validated `postgres -c name=value` settings; it never inherits data volumes and never accesses `constructive-db`. + +```bash +pnpm build + +PGHOST=127.0.0.1 \ +PGPORT=55432 \ +PGUSER=postgres \ +PGPASSWORD=local-admin-password \ +PGDATABASE=postgres \ +node research/graphile-density/physical-database-density/provision.cjs \ + --prefix pdc_density \ + --customers 64 \ + --out-dir research/graphile-density/physical-database-density/.local \ + --maintenance-database postgres +``` + +Provisioning fails if any target already exists. Replacing this exact disposable prefix requires both `--recreate` and `--yes`; no wildcard or workspace-wide deletion path exists. + +Before generating a density plan, derive the governor calibration from at least three clean, conclusive one-surface `catalog-bench` results produced with the ordered production schema set and explicit dependency allowlist. Every source must have build-state retirement enabled and must prove that its exact PostgreSQL introspection PID disappeared before a different steady-state PID was acquired. The tool takes the maximum measured retained heap, server baseline, build-transient heap, and build-transient RSS across repetitions, applies an explicit safety factor, and binds the retirement proof, source artifact hashes, and provisioned database-contract hash into one calibration identity. + +```bash +node research/graphile-density/physical-database-density/cache-calibration.cjs \ + --results /tmp/catalog/rep-1/result.json,/tmp/catalog/rep-2/result.json,/tmp/catalog/rep-3/result.json \ + --manifest research/graphile-density/physical-database-density/.local/provision.json \ + --safety-factor 1.25 \ + --out research/graphile-density/physical-database-density/.local/cache-calibration.json +``` + +Inconclusive, cross-tenant, legacy one-schema-layout, or scope-mismatched results are rejected. Dirty source state is retained explicitly as `sourceWorktreesClean: false`, which permits local diagnostic sizing but cannot turn the later run into qualifying evidence because cperf independently requires clean server provenance. The calibration is a governor sizing input, not a performance claim; the complete-customer workload still decides qualification. + +Generate the credential-free fleet and plan after provisioning. The secret file remains mode `0600`, and generated plans contain only its path. + +```bash +node research/graphile-density/physical-database-density/generate-inputs.cjs \ + --manifest research/graphile-density/physical-database-density/.local/provision.json \ + --secrets research/graphile-density/physical-database-density/.local/runtime-secrets.json \ + --out-dir research/graphile-density/physical-database-density/.local/inputs \ + --cache-calibration research/graphile-density/physical-database-density/.local/cache-calibration.json \ + --postgres-container postgres-density \ + --arm-profile density-tuning \ + --tenant-counts-by-heap-mib '1024:8,12,16;2048:16,24,32;4096:32,48,64' \ + --heaps 1024,2048,4096 \ + --repetitions 3 \ + --duration-sec 900 + +node packages/perf-harness/dist/index.js validate \ + --plan research/graphile-density/physical-database-density/.local/inputs/plan.json + +node packages/perf-harness/dist/index.js run \ + --plan research/graphile-density/physical-database-density/.local/inputs/plan.json +``` + +The container passed to `--postgres-container` is destructive, disposable fixture state. Generation captures its exact 64-character ID as the only unlabeled container the runner may remove; every replacement must carry the exact fixture, prefix, purpose, image, port, command, and resource-limit contract before it can be removed again. A same-named unrelated container fails closed. The source command may be the image default `postgres`, in which case generation adds and pins a sufficient `max_connections`; otherwise only the checked-in PostgreSQL setting allowlist is accepted, and explicit settings such as `max_connections` and `shared_buffers` are preserved and audited live. + +For each matrix coordinate the prepare wrapper reuses the validated private credential template, but live runtime-pool and Graphile cache identities use a process-random keyed HMAC and intentionally change between preflight and measurement. The fleet carries deterministic, credential-free pool and build-contract fingerprints instead; each process proves the one-to-one mapping from those fingerprints to its live role, database, schema, pool object, and resident cache entry. The wrapper writes a new `0600` secret file and credential-free manifest under that run's artifact directory, provisions a unique run-bound clone and nonce set into the fresh cluster, then runs the full live DDL/ACL/role/extension audit outside the measured Node process. Cperf starts the server with the attested per-run manifest path, manifest hash, and clone ID; a static preflight manifest cannot accidentally satisfy that binding. + +The `density-tuning` profile compares the dedicated-listener baseline with one exact notification broker per physical customer and one-client runtime pools. It isolates stock prepared statements, prepared statements disabled, native single-checkout client retirement (`maxUses=1`), and each V8 size profile before testing a cumulative single-checkout/size arm. Input preflight starts a fresh Node child for every arm, waits for that child to report readiness, validates every customer status plus representative shared realtime, and waits for the child to terminate before starting the next arm. This process boundary matters because Dataplan and pool modules may snapshot environment on first import; setting and restoring `process.env` around multiple in-process servers cannot prove arm isolation. Each child strips ambient Node preload/module-path hooks, runs the arm's exact V8 profile, behaviorally attests the loaded Dataplan prepared-statement cache, and keeps process-global `PG_POOL_MAX=1` plus `PG_POOL_MAX_USES=0`; arm-specific runtime capacity and `maxUses` travel only through exact server options. None is accepted from a microbenchmark alone. The default `idle` profile instead isolates PostgreSQL idle-client retention at 30, 5, and 1 seconds. Every heap checkpoint gets an explicit environment block containing the measured instance cost, server reserve, build reserve, RSS build reserve, calibrated budget capacity as the cache ceiling, `GRAPHILE_CACHE_ADMISSION_MODE=preserve-resident`, and the calibration identity. Input generation resolves Node's effective V8 heap limit and fails before starting runtime status collection when the calibrated resident-plus-next-build budget cannot admit all three surfaces for that heap's requested maximum. The plan records the required residents, calibrated capacity, remaining headroom, and stable boundary refusal reason/code; a 1 GiB checkpoint may still fail under real pressure or correctness gates, but it cannot fail merely because it inherited the old fixed 768 MiB reserve. + +Before scoring begins, every Graphile surface is warm, every configured capability has returned exact customer and physical-database evidence, every realtime manager is running, and one `graphql-transport-ws` subscription per surface has received its configured tenant-specific database event through the exact customer/tenant route. Fixture-only `BEFORE` triggers stamp read/search source rows and upload, bulk, function-binding, and realtime side effects with `current_database()`; i18n translations and RAG source content also carry a database-derived marker. Collection oracles assert every returned row plus nonempty one-row cardinality, and deterministic LLM/RAG, vector, PostGIS, ltree, and search results have operation-specific semantic assertions. + +Every presigned-upload invocation selects a fixture-only VOLATILE mutation sibling that returns `current_database()`, so timed workload calls carry direct physical-database evidence. Coverage then extracts the exact `fileId` returned by that invocation and verifies the stamped `appFiles` row by both ID and content hash, which prevents a reused fixture row from satisfying the check. Buckets start with `physical_name = NULL`, forcing the first upload through the plugin's `withPgClient(null)` provisioning lane while the fixture keeps forced RLS intact. Missing, ambiguous, or foreign evidence fails with stable oracle codes; none of these fixture-only fields or functions belongs to the production API design. + +Realtime verification requires permanent tenant and physical-database invariants plus a one-time prime payload; later legitimate payload changes remain valid, while another selected database is an explicit forbidden match. The subscription clients live in the perf-harness driver process, outside the measured server RSS; the server retains only its real websocket/session state and independently reports one accepted live connection per surface. The post-warmup hook asserts those server-side managers and connections instead of constructing load-generator clients. Driver credentials may be sourced from environment-variable names declared in the fleet, and neither their values nor resolved headers are written to evidence. Post-warmup samples then record Graphile residency and the live budget capacity/calibration identity, concrete `pg.Pool` clients, `pg_stat_activity` backends, raw cgroup-v2 `memory.current`, `memory.peak`, `memory.stat`, `memory.events`, and Docker working set. + +One generated plan owns the full customer-count ramp, and the highest count across all heaps must equal the source manifest's physical database count. Use `--tenant-counts` for one shared ramp or `--tenant-counts-by-heap-mib` for semicolon-separated heap-specific ramps; the latter lets each heap push its own density boundary without forcing the smallest heap to admit the largest fleet. Every matrix coordinate still gets a fresh container provisioned with exactly that coordinate's customer count, so lower-count samples cannot inherit unused databases or catalog cache. Keeping the complete ramp, every configured arm, and every repetition under one immutable plan/fleet cohort lets the report reject spliced or partial evidence while still bracketing the highest passing count with a greater failure. + +## Qualification conditions + +Use one dedicated PostgreSQL container for the measured Node process, with no unrelated databases or traffic. The memory endpoint enumerates `pg_database` and the score fails unless the container contains only the maintenance database and the selected physical customer databases, so extra provisioned-but-unserved databases and shared development databases both make the run non-qualifying. Every scheduled run gets a unique Docker ID, cgroup identity, PostgreSQL system identifier, clone ID, attestation set, and nonce set; reuse of any one identity rejects every affected result, including when separately generated result files are combined for reporting. On Linux the sampler reads the target container's cgroup directly from the host; Docker Desktop falls back to reads inside the container, so publishable numbers should be reproduced on a Linux cgroup-v2 host. + +The perf harness rejects a capacity point unless a greater customer count fails, all repetitions are present, every physical database and realtime transport remains resident, all request-path isolation canaries are conclusive, cross-customer results stay zero, and latency, error, eviction, build, pool, backend, OOM, and heap-growth gates pass. Each 15-minute measured run performs full request-path canary sweeps before and after timed traffic, plus 14 one-canary-per-surface rounds at 60-second intervals. The per-surface rotation is deterministic and staggered, covers all 14 configured canaries exactly once during the timed window, runs at concurrency 16 across surfaces, and never drops an overlapping round; incomplete or deadline-late validation fails the run. These passive probes do not count as induced hostile validation. The current generated physical plan deliberately omits `qualification.hostileValidationEvidence`, so cperf records diagnostic evidence until one immutable `exact-runtime-hostile-validation-v1` report is attached for every exact arm runtime and configuration. Configured V8 old space, Node-only RSS, Docker working set, and cumulative peaks remain diagnostics; customers per aligned Node-plus-PostgreSQL GiB is the decision metric once all qualification prerequisites exist. + +The harness also requires a clean pinned worktree for qualifying evidence. Until these local changes are reviewed and recorded on a local branch, smoke runs can validate mechanics but intentionally cannot count as performance evidence. A smoke still gets a fresh database epoch so it exercises the real lifecycle, but its five-second workload always receives zero qualified customers and cannot enter a capacity or density decision. + +## Offline checks + +```bash +node --test \ + research/graphile-density/physical-database-density/cache-calibration.test.cjs \ + research/graphile-density/physical-database-density/lib.test.cjs \ + research/graphile-density/physical-database-density/inputs.test.cjs \ + research/graphile-density/physical-database-density/prepare-measurement-run.test.cjs \ + research/graphile-density/physical-database-density/measurement-attestation.test.cjs \ + research/graphile-density/physical-database-density/server-realtime.test.cjs \ + research/graphile-density/physical-database-density/server-retained-memory.test.cjs + +pnpm --dir packages/perf-harness exec jest --runInBand +pnpm --dir packages/perf-harness build +``` diff --git a/research/graphile-density/physical-database-density/cache-calibration.cjs b/research/graphile-density/physical-database-density/cache-calibration.cjs new file mode 100644 index 0000000000..b3e0512700 --- /dev/null +++ b/research/graphile-density/physical-database-density/cache-calibration.cjs @@ -0,0 +1,390 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const CALIBRATION_KIND = 'graphile-cache-measured-calibration-v2'; +const SHA256 = /^sha256:[a-f0-9]{64}$/; + +const canonicalize = (value) => { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [ + key, + canonicalize(value[key]), + ])); +}; + +const sha256Canonical = (value) => `sha256:${crypto.createHash('sha256') + .update(JSON.stringify(canonicalize(value))) + .digest('hex')}`; + +const fileSha256 = (file) => `sha256:${crypto.createHash('sha256') + .update(fs.readFileSync(file)) + .digest('hex')}`; + +const positiveSafeInteger = (value, label) => { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`PDCF_CALIBRATION_POSITIVE_INTEGER_REQUIRED:${label}`); + } + return value; +}; + +const validateSafetyFactor = (value) => { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 1 || value > 3) { + throw new Error('PDCF_CALIBRATION_SAFETY_FACTOR_INVALID'); + } + return value; +}; + +const validateCatalogResult = (result, sourceFile) => { + if ( + !result + || result.version !== 1 + || result.status !== 'performance-only' + || !['stock', 'scoped-required'].includes(result.mode) + || result.introspectionClientReleaseMode !== 'destroy' + || result.releaseBuildStateAfterValidation !== true + || typeof result.worktreeDirty !== 'boolean' + || !Array.isArray(result.schemaSets) + || result.schemaSets.length !== 1 + || !Array.isArray(result.schemaSets[0]) + || result.schemaSets[0].length === 0 + || !Array.isArray(result.allowedDependencySchemas) + || !Array.isArray(result.builds) + || result.builds.length !== 1 + || !Array.isArray(result.snapshots) + || result.tokenCanariesConclusive !== true + || result.tokenCanariesPassed !== true + || result.bleedViolations !== 0 + || typeof result.fixtureFingerprint !== 'string' + || result.fixtureFingerprint.length === 0 + || !/^[a-f0-9]{64}$/.test(result.sourceStateSha256 ?? '') + || !/^[a-f0-9]{64}$/.test(result.executedEntrySha256 ?? '') + ) { + throw new Error(`PDCF_CALIBRATION_RESULT_NOT_CONCLUSIVE:${sourceFile}`); + } + const snapshot = result.snapshots.find((candidate) => candidate.instances === 1); + if (!snapshot) throw new Error(`PDCF_CALIBRATION_ONE_SURFACE_SNAPSHOT_REQUIRED:${sourceFile}`); + const build = result.builds[0]; + const introspectionBackendPid = positiveSafeInteger( + build.introspectionBackendPid, + `${sourceFile}:introspectionBackendPid`, + ); + const steadyBackendPid = positiveSafeInteger( + build.steadyBackendPid, + `${sourceFile}:steadyBackendPid`, + ); + if ( + build.introspectionBackendRetired !== true + || introspectionBackendPid === steadyBackendPid + || result.postgresBackendMeasurement?.expectedRetirementChecks !== result.builds.length + || result.postgresBackendMeasurement?.completedRetirementChecks !== result.builds.length + || result.postgresBackendMeasurement?.allExpectedRetirementsProven !== true + ) { + throw new Error(`PDCF_CALIBRATION_INTROSPECTION_RETIREMENT_UNPROVEN:${sourceFile}`); + } + if (!Number.isSafeInteger(build.buildTransientSampleCount) || build.buildTransientSampleCount <= 0) { + throw new Error(`PDCF_CALIBRATION_BUILD_SAMPLES_REQUIRED:${sourceFile}`); + } + const retainedHeapBytes = positiveSafeInteger( + Math.ceil(snapshot.heapDeltaBytes), + `${sourceFile}:retainedHeapBytes`, + ); + const serverBaselineHeapBytes = positiveSafeInteger( + Math.ceil(build.buildBaselineHeapUsedBytes), + `${sourceFile}:serverBaselineHeapBytes`, + ); + const buildTransientHeapBytes = positiveSafeInteger( + Math.ceil(Math.max( + build.sampledBuildPeakHeapDeltaBytes ?? 0, + retainedHeapBytes, + )), + `${sourceFile}:buildTransientHeapBytes`, + ); + const buildTransientRssBytes = positiveSafeInteger( + Math.ceil(Math.max( + build.sampledBuildPeakRssDeltaBytes ?? 0, + build.processBuildPeakRssDeltaBytes ?? 0, + )), + `${sourceFile}:buildTransientRssBytes`, + ); + return { + mode: result.mode, + introspectionClientReleaseMode: result.introspectionClientReleaseMode, + releaseBuildStateAfterValidation: result.releaseBuildStateAfterValidation, + introspectionBackendRetirement: { + conclusive: true, + introspectionBackendPid, + steadyBackendPid, + }, + fixtureFingerprint: result.fixtureFingerprint, + schemaSets: result.schemaSets, + allowedDependencySchemas: result.allowedDependencySchemas, + sourceStateSha256: result.sourceStateSha256, + executedEntrySha256: result.executedEntrySha256, + worktreeDirty: result.worktreeDirty, + retainedHeapBytes, + serverBaselineHeapBytes, + buildTransientHeapBytes, + buildTransientRssBytes, + }; +}; + +const deriveCacheCalibration = ({ + resultFiles, + databaseContractFingerprint, + safetyFactor = 1.25, +}) => { + if (!Array.isArray(resultFiles) || resultFiles.length < 3) { + throw new Error('PDCF_CALIBRATION_THREE_RESULTS_REQUIRED'); + } + if (!SHA256.test(databaseContractFingerprint ?? '')) { + throw new Error('PDCF_CALIBRATION_DATABASE_CONTRACT_REQUIRED'); + } + validateSafetyFactor(safetyFactor); + const sources = resultFiles.map((sourceFile) => { + const absolute = path.resolve(sourceFile); + const result = JSON.parse(fs.readFileSync(absolute, 'utf8')); + return { + fileSha256: fileSha256(absolute), + measurement: validateCatalogResult(result, absolute), + }; + }); + const modes = new Set(sources.map((source) => source.measurement.mode)); + const releaseModes = new Set(sources.map( + (source) => source.measurement.introspectionClientReleaseMode + )); + const buildStateRetirementModes = new Set(sources.map( + (source) => source.measurement.releaseBuildStateAfterValidation + )); + const fixtureFingerprints = new Set( + sources.map((source) => source.measurement.fixtureFingerprint) + ); + const schemaContracts = new Set(sources.map((source) => JSON.stringify({ + schemaSets: source.measurement.schemaSets, + allowedDependencySchemas: source.measurement.allowedDependencySchemas, + }))); + if ( + modes.size !== 1 + || releaseModes.size !== 1 + || !releaseModes.has('destroy') + || buildStateRetirementModes.size !== 1 + || !buildStateRetirementModes.has(true) + || fixtureFingerprints.size !== 1 + || schemaContracts.size !== 1 + ) { + throw new Error('PDCF_CALIBRATION_RESULT_SCOPE_MISMATCH'); + } + const maximum = (field) => Math.max(...sources.map( + (source) => source.measurement[field] + )); + const measured = { + repetitions: sources.length, + retainedHeapPerSurfaceBytes: maximum('retainedHeapBytes'), + serverBaselineHeapBytes: maximum('serverBaselineHeapBytes'), + buildTransientHeapBytes: maximum('buildTransientHeapBytes'), + buildTransientRssBytes: maximum('buildTransientRssBytes'), + }; + const configured = { + instanceHeapBytes: Math.ceil(measured.retainedHeapPerSurfaceBytes * safetyFactor), + serverReserveBytes: Math.ceil(measured.serverBaselineHeapBytes * safetyFactor), + buildReserveBytes: Math.ceil(measured.buildTransientHeapBytes * safetyFactor), + rssBuildReserveBytes: Math.ceil(measured.buildTransientRssBytes * safetyFactor), + }; + const identityPayload = { + kind: CALIBRATION_KIND, + databaseContractFingerprint, + introspectionMode: [...modes][0], + introspectionClientReleaseMode: [...releaseModes][0], + releaseBuildStateAfterValidation: [...buildStateRetirementModes][0], + introspectionBackendRetirementConclusive: sources.every( + (source) => source.measurement.introspectionBackendRetirement.conclusive === true + ), + fixtureFingerprint: [...fixtureFingerprints][0], + schemaContract: JSON.parse([...schemaContracts][0]), + safetyFactor, + measured, + configured, + sourceWorktreesClean: sources.every( + (source) => source.measurement.worktreeDirty === false + ), + sources: sources.map(({ fileSha256: sourceSha256, measurement }) => ({ + sourceSha256, + sourceStateSha256: measurement.sourceStateSha256, + executedEntrySha256: measurement.executedEntrySha256, + worktreeDirty: measurement.worktreeDirty, + introspectionBackendRetirement: measurement.introspectionBackendRetirement, + })), + }; + return { + version: 2, + ...identityPayload, + calibrationId: sha256Canonical(identityPayload), + }; +}; + +const validateCacheCalibration = ( + calibration, + { databaseContractFingerprint, introspectionMode } = {}, +) => { + if ( + !calibration + || calibration.version !== 2 + || calibration.kind !== CALIBRATION_KIND + || !SHA256.test(calibration.calibrationId ?? '') + || !SHA256.test(calibration.databaseContractFingerprint ?? '') + || !Array.isArray(calibration.sources) + || calibration.sources.length < 3 + || !['stock', 'scoped-required'].includes(calibration.introspectionMode) + || calibration.introspectionClientReleaseMode !== 'destroy' + || calibration.releaseBuildStateAfterValidation !== true + || calibration.introspectionBackendRetirementConclusive !== true + || typeof calibration.fixtureFingerprint !== 'string' + || calibration.fixtureFingerprint.length === 0 + || typeof calibration.sourceWorktreesClean !== 'boolean' + ) { + throw new Error('PDCF_CACHE_CALIBRATION_INVALID'); + } + validateSafetyFactor(calibration.safetyFactor); + for (const field of [ + 'retainedHeapPerSurfaceBytes', + 'serverBaselineHeapBytes', + 'buildTransientHeapBytes', + 'buildTransientRssBytes', + ]) positiveSafeInteger(calibration.measured?.[field], `measured.${field}`); + for (const field of [ + 'instanceHeapBytes', + 'serverReserveBytes', + 'buildReserveBytes', + 'rssBuildReserveBytes', + ]) positiveSafeInteger(calibration.configured?.[field], `configured.${field}`); + if (calibration.measured?.repetitions !== calibration.sources.length) { + throw new Error('PDCF_CACHE_CALIBRATION_REPETITION_MISMATCH'); + } + if (calibration.sources.some((source) => ( + !SHA256.test(source?.sourceSha256 ?? '') + || !/^[a-f0-9]{64}$/.test(source?.sourceStateSha256 ?? '') + || !/^[a-f0-9]{64}$/.test(source?.executedEntrySha256 ?? '') + || source?.introspectionBackendRetirement?.conclusive !== true + || !Number.isSafeInteger( + source?.introspectionBackendRetirement?.introspectionBackendPid + ) + || source.introspectionBackendRetirement.introspectionBackendPid <= 0 + || !Number.isSafeInteger(source?.introspectionBackendRetirement?.steadyBackendPid) + || source.introspectionBackendRetirement.steadyBackendPid <= 0 + || source.introspectionBackendRetirement.introspectionBackendPid + === source.introspectionBackendRetirement.steadyBackendPid + ))) { + throw new Error('PDCF_CACHE_CALIBRATION_SOURCE_INVALID'); + } + if ( + calibration.sourceWorktreesClean + !== calibration.sources.every((source) => source.worktreeDirty === false) + ) { + throw new Error('PDCF_CACHE_CALIBRATION_SOURCE_CLEANLINESS_MISMATCH'); + } + const expectedConfigured = { + instanceHeapBytes: Math.ceil( + calibration.measured.retainedHeapPerSurfaceBytes * calibration.safetyFactor + ), + serverReserveBytes: Math.ceil( + calibration.measured.serverBaselineHeapBytes * calibration.safetyFactor + ), + buildReserveBytes: Math.ceil( + calibration.measured.buildTransientHeapBytes * calibration.safetyFactor + ), + rssBuildReserveBytes: Math.ceil( + calibration.measured.buildTransientRssBytes * calibration.safetyFactor + ), + }; + if (JSON.stringify(expectedConfigured) !== JSON.stringify(calibration.configured)) { + throw new Error('PDCF_CACHE_CALIBRATION_FORMULA_MISMATCH'); + } + const { calibrationId: _calibrationId, version: _version, ...identityPayload } = calibration; + if (sha256Canonical(identityPayload) !== calibration.calibrationId) { + throw new Error('PDCF_CACHE_CALIBRATION_ID_MISMATCH'); + } + if ( + databaseContractFingerprint + && calibration.databaseContractFingerprint !== databaseContractFingerprint + ) { + throw new Error('PDCF_CACHE_CALIBRATION_DATABASE_CONTRACT_MISMATCH'); + } + if (introspectionMode && calibration.introspectionMode !== introspectionMode) { + throw new Error('PDCF_CACHE_CALIBRATION_MODE_MISMATCH'); + } + return calibration; +}; + +const computeCalibratedCapacity = (heapLimitBytes, configured) => { + positiveSafeInteger(heapLimitBytes, 'heapLimitBytes'); + const instance = positiveSafeInteger(configured?.instanceHeapBytes, 'instanceHeapBytes'); + const server = positiveSafeInteger(configured?.serverReserveBytes, 'serverReserveBytes'); + const build = positiveSafeInteger(configured?.buildReserveBytes, 'buildReserveBytes'); + if (server + build > heapLimitBytes) return 0; + const backingCapacity = Math.max( + 1024, + Math.min(65_536, Math.floor(heapLimitBytes / (256 * 1024))), + ); + const byResidency = Math.floor((heapLimitBytes - server) / instance); + const byRebuild = Math.floor((heapLimitBytes - server - build) / instance) + 1; + return Math.max(0, Math.min(backingCapacity, byResidency, byRebuild)); +}; + +const parseArgs = (argv) => { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const name = argv[index]; + if (!name.startsWith('--') || index + 1 >= argv.length) { + throw new Error(`PDCF_CALIBRATION_ARGUMENT_INVALID:${name}`); + } + result[name.slice(2)] = argv[++index]; + } + return result; +}; + +const main = () => { + const args = parseArgs(process.argv.slice(2)); + if (!args.results || !args.out) throw new Error('PDCF_CALIBRATION_ARGUMENTS_REQUIRED'); + if (Boolean(args.manifest) === Boolean(args['database-contract'])) { + throw new Error('PDCF_CALIBRATION_REQUIRES_ONE_DATABASE_CONTRACT_SOURCE'); + } + const manifest = args.manifest + ? JSON.parse(fs.readFileSync(path.resolve(args.manifest), 'utf8')) + : null; + const calibration = deriveCacheCalibration({ + resultFiles: args.results.split(',').map((value) => value.trim()).filter(Boolean), + databaseContractFingerprint: manifest?.canonicalDatabaseContractFingerprint + ?? args['database-contract'], + safetyFactor: Number(args['safety-factor'] ?? '1.25'), + }); + const output = path.resolve(args.out); + fs.mkdirSync(path.dirname(output), { recursive: true }); + if (fs.existsSync(output)) throw new Error(`PDCF_CALIBRATION_REFUSES_OVERWRITE:${output}`); + fs.writeFileSync(output, `${JSON.stringify(calibration, null, 2)}\n`, { mode: 0o644 }); + process.stdout.write(`${JSON.stringify({ + status: 'calibrated', + calibrationId: calibration.calibrationId, + output, + })}\n`); +}; + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} + +module.exports = { + CALIBRATION_KIND, + computeCalibratedCapacity, + deriveCacheCalibration, + sha256Canonical, + validateCacheCalibration, + validateCatalogResult, +}; diff --git a/research/graphile-density/physical-database-density/cache-calibration.test.cjs b/research/graphile-density/physical-database-density/cache-calibration.test.cjs new file mode 100644 index 0000000000..e0ed070557 --- /dev/null +++ b/research/graphile-density/physical-database-density/cache-calibration.test.cjs @@ -0,0 +1,146 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { + computeCalibratedCapacity, + deriveCacheCalibration, + validateCacheCalibration, +} = require('./cache-calibration.cjs'); + +const MIB = 1024 ** 2; +const databaseContractFingerprint = `sha256:${'d'.repeat(64)}`; + +const result = (ordinal) => ({ + version: 1, + status: 'performance-only', + mode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: true, + worktreeDirty: false, + schemaSets: [['ctf_a']], + allowedDependencySchemas: ['ctf_extensions'], + fixtureFingerprint: 'fixture-fingerprint-v1', + sourceStateSha256: String(ordinal).repeat(64), + executedEntrySha256: String(ordinal).repeat(64), + tokenCanariesConclusive: true, + tokenCanariesPassed: true, + bleedViolations: 0, + builds: [{ + introspectionBackendPid: 1000 + ordinal, + steadyBackendPid: 2000 + ordinal, + introspectionBackendRetired: true, + buildTransientSampleCount: 2, + buildBaselineHeapUsedBytes: (40 + ordinal) * MIB, + sampledBuildPeakHeapDeltaBytes: (80 + ordinal) * MIB, + sampledBuildPeakRssDeltaBytes: (90 + ordinal) * MIB, + processBuildPeakRssDeltaBytes: (100 + ordinal) * MIB, + }], + snapshots: [{ + instances: 1, + heapDeltaBytes: (10 + ordinal) * MIB, + }], + postgresBackendMeasurement: { + expectedRetirementChecks: 1, + completedRetirementChecks: 1, + allExpectedRetirementsProven: true, + }, +}); + +describe('physical density cache calibration', () => { + it('derives a safety-factored, source-bound calibration from three clean results', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-calibration-')); + const files = [1, 2, 3].map((ordinal) => { + const file = path.join(directory, `result-${ordinal}.json`); + fs.writeFileSync(file, JSON.stringify(result(ordinal))); + return file; + }); + const calibration = deriveCacheCalibration({ + resultFiles: files, + databaseContractFingerprint, + safetyFactor: 1.25, + }); + assert.equal(calibration.measured.repetitions, 3); + assert.equal(calibration.sourceWorktreesClean, true); + assert.equal(calibration.introspectionClientReleaseMode, 'destroy'); + assert.equal(calibration.releaseBuildStateAfterValidation, true); + assert.equal(calibration.introspectionBackendRetirementConclusive, true); + assert.equal(calibration.measured.retainedHeapPerSurfaceBytes, 13 * MIB); + assert.equal(calibration.configured.instanceHeapBytes, Math.ceil(13 * MIB * 1.25)); + assert.equal(calibration.configured.buildReserveBytes, Math.ceil(83 * MIB * 1.25)); + assert.equal(calibration.configured.rssBuildReserveBytes, Math.ceil(103 * MIB * 1.25)); + assert.equal(validateCacheCalibration(calibration, { + databaseContractFingerprint, + introspectionMode: 'scoped-required', + }), calibration); + assert.throws(() => validateCacheCalibration({ + ...calibration, + configured: { ...calibration.configured, buildReserveBytes: 1 }, + }), /CALIBRATION_FORMULA_MISMATCH|CALIBRATION_ID_MISMATCH/); + }); + + it('fails closed on inconclusive or mismatched source measurements', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-calibration-bad-')); + const values = [result(1), result(2), result(3)]; + values[1].tokenCanariesPassed = false; + const files = values.map((value, index) => { + const file = path.join(directory, `result-${index}.json`); + fs.writeFileSync(file, JSON.stringify(value)); + return file; + }); + assert.throws(() => deriveCacheCalibration({ + resultFiles: files, + databaseContractFingerprint, + }), /RESULT_NOT_CONCLUSIVE/); + assert.throws(() => deriveCacheCalibration({ + resultFiles: files.slice(0, 2), + databaseContractFingerprint, + }), /THREE_RESULTS_REQUIRED/); + }); + + it('fails closed when build-state or PostgreSQL introspection retirement is unproven', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-calibration-retire-')); + const writeResults = (values, label) => values.map((value, index) => { + const file = path.join(directory, `${label}-${index}.json`); + fs.writeFileSync(file, JSON.stringify(value)); + return file; + }); + const retained = [result(1), result(2), result(3)]; + retained[1].releaseBuildStateAfterValidation = false; + assert.throws(() => deriveCacheCalibration({ + resultFiles: writeResults(retained, 'retained'), + databaseContractFingerprint, + }), /RESULT_NOT_CONCLUSIVE/); + + const reused = [result(1), result(2), result(3)]; + reused[1].builds[0].steadyBackendPid = reused[1].builds[0].introspectionBackendPid; + assert.throws(() => deriveCacheCalibration({ + resultFiles: writeResults(reused, 'reused'), + databaseContractFingerprint, + }), /INTROSPECTION_RETIREMENT_UNPROVEN/); + + const unproven = [result(1), result(2), result(3)]; + unproven[1].postgresBackendMeasurement.allExpectedRetirementsProven = false; + assert.throws(() => deriveCacheCalibration({ + resultFiles: writeResults(unproven, 'unproven'), + databaseContractFingerprint, + }), /INTROSPECTION_RETIREMENT_UNPROVEN/); + }); + + it('computes admission capacity with both resident and next-build budgets', () => { + assert.equal(computeCalibratedCapacity(1024 * MIB, { + instanceHeapBytes: 16 * MIB, + serverReserveBytes: 64 * MIB, + buildReserveBytes: 128 * MIB, + }), 53); + assert.equal(computeCalibratedCapacity(1024 * MIB, { + instanceHeapBytes: 16 * MIB, + serverReserveBytes: 256 * MIB, + buildReserveBytes: 768 * MIB, + }), 1); + }); +}); diff --git a/research/graphile-density/physical-database-density/generate-inputs.cjs b/research/graphile-density/physical-database-density/generate-inputs.cjs new file mode 100644 index 0000000000..92aa92219b --- /dev/null +++ b/research/graphile-density/physical-database-density/generate-inputs.cjs @@ -0,0 +1,976 @@ +'use strict'; + +const { execFileSync, spawn } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + DEFAULT_IDLE_ARMS, + DENSITY_TUNING_ARMS, + FIXTURE_DIR, + PROCESS_GLOBAL_POOL_MAX, + REPO_ROOT, + atomicWriteJson, + loadProvision, + makeCacheCapacityProofByHeapMiB, + makeFleet, + makePlan, + cursorHeartbeatMsForArm, + cursorPollMsForArm, + notificationModeForArm, + preparedStatementCacheSizeForArm, + runtimePoolMaxForArm, + runtimePoolMaxUsesForArm, + validateCustomerCountMatrix, +} = require('./lib.cjs'); +const { + TENANTS, + parseArgs, + parsePositiveInteger, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); +const completeFixtureServer = require('../complete-tenant-fixture/server.cjs'); +const { validateCacheCalibration } = require('./cache-calibration.cjs'); +const { + captureContainerTemplate, + inspectDockerContainer, +} = require('./prepare-measurement-run.cjs'); + +const loadRealtimeDriver = () => require(path.join( + REPO_ROOT, + 'packages/perf-harness/dist/realtime.js', +)).createRealtimeDriver; + +const fileSha256 = (file) => crypto.createHash('sha256') + .update(fs.readFileSync(file)) + .digest('hex'); + +const expectedHeapLimitBytes = (heapMiB) => { + const output = execFileSync(process.execPath, [ + `--max-old-space-size=${heapMiB}`, + '-e', + 'process.stdout.write(String(require("node:v8").getHeapStatistics().heap_size_limit))', + ], { + encoding: 'utf8', + env: { ...process.env, NODE_OPTIONS: '' }, + }).trim(); + const value = Number(output); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`PDCF_EXPECTED_HEAP_LIMIT_INVALID:${heapMiB}:${output}`); + } + return value; +}; + +const canonicalize = (value) => { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [ + key, + canonicalize(value[key]), + ])); +}; + +const sha256Canonical = (value) => `sha256:${crypto.createHash('sha256') + .update(JSON.stringify(canonicalize(value))) + .digest('hex')}`; + +const makeArmPreflightEnvironment = (arm, environment = process.env) => { + const armEnvironment = { ...environment }; + armEnvironment.PG_POOL_IDLE_TIMEOUT_MS = String(arm.idleTimeoutMs); + armEnvironment.PG_POOL_MAX = String(PROCESS_GLOBAL_POOL_MAX); + // Keep process-global and notification pools reusable. The exact runtime + // pool gets its arm-specific maxUses through the explicit server option. + armEnvironment.PG_POOL_MAX_USES = '0'; + // This child receives the secret-file path and control-plane credentials. + // Ambient preloads and module-resolution paths may execute unreviewed code + // before the fixture can enforce its own boundary, so fail closed here. + armEnvironment.NODE_OPTIONS = ''; + delete armEnvironment.NODE_PATH; + const preparedStatementCacheSize = preparedStatementCacheSizeForArm(arm); + if (preparedStatementCacheSize == null) { + delete armEnvironment.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE; + } else { + armEnvironment.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE = + String(preparedStatementCacheSize); + } + return armEnvironment; +}; + +const V8_PROFILE_FLAGS = Object.freeze({ + stock: Object.freeze([]), + 'optimize-for-size': Object.freeze(['--optimize-for-size']), + 'baseline-optimize-for-size': Object.freeze([ + '--max-opt=1', + '--optimize-for-size', + ]), + 'jitless-optimize-for-size': Object.freeze([ + '--jitless', + '--optimize-for-size', + ]), +}); + +const v8FlagsForArm = (arm) => { + const profile = arm.v8Profile ?? 'stock'; + const flags = V8_PROFILE_FLAGS[profile]; + if (!flags) throw new Error(`PDCF_V8_PROFILE_INVALID:${profile}`); + return [...flags]; +}; + +const makeArmPreflightArgs = ({ + arm, + port, + manifestFile, + secretsFile, + customerCount, + mode, + provisionClone, +}) => [ + '--manifest', manifestFile, + '--secrets', secretsFile, + '--customers', String(customerCount), + '--host', '127.0.0.1', + '--port', String(port), + '--arm', arm.name, + '--mode', mode, + '--introspection-client-release-mode', 'destroy', + '--runtime-pool-max', String(runtimePoolMaxForArm(arm)), + '--runtime-pool-max-uses', runtimePoolMaxUsesForArm(arm) == null + ? 'unlimited' + : String(runtimePoolMaxUsesForArm(arm)), + '--realtime-notification-mode', notificationModeForArm(arm), + '--realtime-cursor-poll-ms', String(cursorPollMsForArm(arm)), + '--realtime-cursor-heartbeat-ms', String(cursorHeartbeatMsForArm(arm)), + '--enable-realtime', 'true', + '--run-purpose', provisionClone.purpose, + '--clone-id', provisionClone.id, +]; + +const waitForArmPreflightReady = ({ + child, + arm, + port, + customerCount, + timeoutMs = 600_000, +}) => new Promise((resolve, reject) => { + let buffer = ''; + let settled = false; + const finish = (error, value) => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.off('error', onError); + child.off('exit', onExit); + child.stdout?.off('data', onData); + child.stdout?.resume(); + if (error) reject(error); + else resolve(value); + }; + const onError = () => finish(new Error( + `PDCF_PREFLIGHT_CHILD_SPAWN_FAILED:${arm.name}` + )); + const onExit = (code, signal) => finish(new Error( + `PDCF_PREFLIGHT_CHILD_EXITED_BEFORE_READY:${arm.name}:${code ?? 'signal'}:${signal ?? 'none'}` + )); + const onData = (chunk) => { + buffer += chunk.toString('utf8'); + if (buffer.length > 64 * 1024) { + finish(new Error(`PDCF_PREFLIGHT_CHILD_READY_OUTPUT_EXCEEDED:${arm.name}`)); + return; + } + let newline = buffer.indexOf('\n'); + while (newline >= 0) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (line) { + let message; + try { + message = JSON.parse(line); + } catch { + message = null; + } + if (message?.status === 'ready') { + if ( + message.fixture !== 'physical-database-density-v1' + || message.host !== '127.0.0.1' + || message.port !== port + || message.arm !== arm.name + || message.customers !== customerCount + ) { + finish(new Error(`PDCF_PREFLIGHT_CHILD_READY_INVALID:${arm.name}`)); + } else { + finish(null, message); + } + return; + } + } + newline = buffer.indexOf('\n'); + } + }; + const timer = setTimeout(() => finish(new Error( + `PDCF_PREFLIGHT_CHILD_READY_TIMEOUT:${arm.name}:${timeoutMs}` + )), timeoutMs); + timer.unref?.(); + child.once('error', onError); + child.once('exit', onExit); + child.stdout?.on('data', onData); +}); + +const waitForChildExit = (child, timeoutMs) => { + if (child.exitCode != null || child.signalCode != null) { + return Promise.resolve(true); + } + return new Promise((resolve) => { + let settled = false; + const finish = (exited) => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.off('exit', onExit); + resolve(exited); + }; + const onExit = () => finish(true); + const timer = setTimeout(() => finish(false), timeoutMs); + timer.unref?.(); + child.once('exit', onExit); + }); +}; + +const terminateArmPreflightChild = async ({ + child, + arm, + timeoutMs = 15_000, +}) => { + if (child.exitCode != null || child.signalCode != null) return; + child.kill('SIGTERM'); + if (await waitForChildExit(child, timeoutMs)) return; + child.kill('SIGKILL'); + if (!await waitForChildExit(child, timeoutMs)) { + throw new Error(`PDCF_PREFLIGHT_CHILD_TERMINATION_TIMEOUT:${arm.name}`); + } +}; + +const startArmPreflightChild = async ({ + arm, + port, + manifestFile, + secretsFile, + customerCount, + mode, + provisionClone, + environment = process.env, + entryFile = path.join(FIXTURE_DIR, 'server.cjs'), + spawnImpl = spawn, + readinessTimeoutMs, +}) => { + const armEnvironment = makeArmPreflightEnvironment(arm, environment); + const child = spawnImpl(process.execPath, [ + ...v8FlagsForArm(arm), + '--expose-gc', + entryFile, + ...makeArmPreflightArgs({ + arm, + port, + manifestFile, + secretsFile, + customerCount, + mode, + provisionClone, + }), + ], { + cwd: REPO_ROOT, + env: armEnvironment, + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stderr?.resume(); + try { + await waitForArmPreflightReady({ + child, + arm, + port, + customerCount, + timeoutMs: readinessTimeoutMs, + }); + return child; + } catch (error) { + await terminateArmPreflightChild({ child, arm }); + throw error; + } +}; + +const parseIntegerList = (value, label) => { + const values = value.split(',').map((item) => parsePositiveInteger(item.trim(), label)); + if (values.length === 0 || new Set(values).size !== values.length) { + throw new Error(`PDCF_INVALID_LIST:${label}`); + } + return values.sort((left, right) => left - right); +}; + +const parseTenantCountsByHeapMiB = (value, heapMiB) => { + if (typeof value !== 'string' || !value.trim()) { + throw new Error('PDCF_TENANT_COUNTS_BY_HEAP_REQUIRED'); + } + const result = {}; + for (const entry of value.split(';')) { + const separator = entry.indexOf(':'); + if (separator <= 0 || separator === entry.length - 1) { + throw new Error('PDCF_TENANT_COUNTS_BY_HEAP_INVALID'); + } + const heap = parsePositiveInteger(entry.slice(0, separator).trim(), 'tenant-count-heap'); + const key = String(heap); + if (result[key]) throw new Error(`PDCF_TENANT_COUNTS_BY_HEAP_DUPLICATE:${heap}`); + result[key] = parseIntegerList( + entry.slice(separator + 1), + `tenant-counts-${heap}`, + ); + } + const configured = new Set(heapMiB.map(String)); + if ( + Object.keys(result).length !== configured.size + || Object.keys(result).some((heap) => !configured.has(heap)) + ) { + throw new Error('PDCF_TENANT_COUNTS_BY_HEAP_COVERAGE_INVALID'); + } + return result; +}; + +const validateChildStatus = (status, { arm, customer, mode }) => { + const expectedMaxUses = runtimePoolMaxUsesForArm(arm); + const expectedPreparedStatementCacheSize = preparedStatementCacheSizeForArm(arm); + const expectedPreparedNamedQueries = expectedPreparedStatementCacheSize === 0 + ? 0 + : expectedPreparedStatementCacheSize + 1; + const expectedPreparedFirstEviction = expectedPreparedStatementCacheSize === 0 + ? null + : expectedPreparedStatementCacheSize; + if ( + status?.version !== 1 + || status.fixture !== 'complete-tenant-abc-v1' + || status.arm !== arm.name + || status.introspectionMode !== mode + || status.introspectionClientReleaseMode !== 'destroy' + || status.releaseBuildStateAfterValidation !== true + || status.physicalDatabase !== customer.database + || status.runtimePoolMax !== runtimePoolMaxForArm(arm) + || status.runtimePoolMaxUses !== expectedMaxUses + || status.runtimePools?.scope !== 'runtime-only-exact-identities' + || status.runtimePools?.available !== true + || status.runtimePools?.requestedMaxUses !== expectedMaxUses + || status.runtimePools?.effectiveMaxUsesKnown !== true + || status.runtimePools?.effectiveMaxUses !== expectedMaxUses + || status.runtimePools?.maxUsesExact !== true + || status.runtimePools?.identitiesUnique !== true + || status.runtimePools?.poolObjectsUnique !== true + || status.runtimePools?.expectedPools !== TENANTS.length + || status.runtimePools?.observedPools !== TENANTS.length + || status.preparedStatementCache?.requestedSize + !== expectedPreparedStatementCacheSize + || status.preparedStatementCache?.effectiveSize + !== expectedPreparedStatementCacheSize + || status.preparedStatementCache?.environmentValue + !== String(expectedPreparedStatementCacheSize) + || status.preparedStatementCache?.environmentCanonical !== true + || status.preparedStatementCache?.attestation + !== completeFixtureServer.PREPARED_STATEMENT_ATTESTATION_KIND + || status.preparedStatementCache?.effectiveSizeKnown !== true + || status.preparedStatementCache?.exact !== true + || status.preparedStatementCache?.namedQueriesObserved + !== expectedPreparedNamedQueries + || status.preparedStatementCache?.firstEvictionAfterNamedQueries + !== expectedPreparedFirstEviction + || status.enableRealtime !== true + || status.realtimeNotificationMode !== notificationModeForArm(arm) + || status.realtimeCursorPollIntervalMs !== cursorPollMsForArm(arm) + || status.realtimeCursorHeartbeatIntervalMs !== cursorHeartbeatMsForArm(arm) + || status.runtimeSafety?.passed !== true + || status.liveIdentityScope !== 'process-local-keyed-hmac-v1' + || !/^graphile-configuration:ctf:v1:[a-f0-9]{64}$/.test( + status.configurationIdentity ?? '' + ) + || status.contractEvidence?.version !== 1 + || status.contractEvidence?.credentialFree !== true + || status.contractEvidence?.configurationIdentity + !== status.configurationIdentity + || !/^sha256:[a-f0-9]{64}$/.test(status.runtimeArtifactFingerprint ?? '') + ) { + throw new Error(`PDCF_PREFLIGHT_STATUS_INVALID:${arm.name}:${customer.id}`); + } + if ( + notificationModeForArm(arm) === 'shared-exact' + && !String(status.realtimeListenerIdentity ?? '') + .startsWith('pg-notification-broker:v1:pg:v1:') + ) { + throw new Error( + `PDCF_PREFLIGHT_LISTENER_IDENTITY_INVALID:${arm.name}:${customer.id}` + ); + } + for (const tenantId of ['a', 'b', 'c']) { + if (!String(status.runtimePoolIdentities?.[tenantId] ?? '').startsWith('pg:v1:')) { + throw new Error(`PDCF_PREFLIGHT_POOL_IDENTITY_INVALID:${arm.name}:${customer.id}:${tenantId}`); + } + if (!String(status.buildContracts?.[tenantId] ?? '').startsWith('graphile:v1:')) { + throw new Error(`PDCF_PREFLIGHT_BUILD_CONTRACT_INVALID:${arm.name}:${customer.id}:${tenantId}`); + } + const poolEvidence = status.contractEvidence?.runtimePools?.[tenantId]; + const buildEvidence = status.contractEvidence?.graphileBuilds?.[tenantId]; + const binding = status.runtimeBindings?.[tenantId]; + if ( + !/^pg-contract-evidence:v1:[a-f0-9]{64}$/.test( + poolEvidence?.fingerprint ?? '' + ) + || !/^graphile-contract-evidence:v1:[a-f0-9]{64}$/.test( + buildEvidence?.fingerprint ?? '' + ) + || poolEvidence?.input?.databaseName !== customer.database + || poolEvidence?.input?.role !== customer.roles?.[tenantId] + || binding?.databaseName !== customer.database + || binding?.role !== customer.roles?.[tenantId] + || JSON.stringify(binding?.schemas) !== JSON.stringify([`ctf_${tenantId}`]) + ) { + throw new Error( + `PDCF_PREFLIGHT_CONTRACT_EVIDENCE_INVALID:${arm.name}:${customer.id}:${tenantId}` + ); + } + if (status.realtimeSchemas?.[tenantId] !== `ctf_${tenantId}_realtime`) { + throw new Error(`PDCF_PREFLIGHT_REALTIME_SCHEMA_INVALID:${arm.name}:${customer.id}:${tenantId}`); + } + const expectedDependencies = [ + ...completeFixtureServer.RUNTIME_DEPENDENCY_SCHEMAS, + `ctf_${tenantId}_realtime`, + ]; + if ( + JSON.stringify(status.runtimeSafety?.dependencySchemasByTenant?.[tenantId]) + !== JSON.stringify(expectedDependencies) + ) { + throw new Error( + `PDCF_PREFLIGHT_RUNTIME_DEPENDENCIES_INVALID:${arm.name}:${customer.id}:${tenantId}` + ); + } + } + if (new Set(Object.values(status.runtimePoolIdentities)).size !== TENANTS.length) { + throw new Error(`PDCF_PREFLIGHT_POOL_IDENTITIES_NOT_UNIQUE:${arm.name}:${customer.id}`); + } + return status; +}; + +const assertUniqueRuntimePoolIdentities = ({ statuses, customers, arm }) => { + const identities = customers.flatMap((customer) => + Object.values(statuses?.[customer.id]?.runtimePoolIdentities ?? {}) + ); + if ( + identities.length !== customers.length * TENANTS.length + || new Set(identities).size !== identities.length + ) { + throw new Error(`PDCF_PREFLIGHT_POOL_IDENTITIES_NOT_UNIQUE:${arm.name}`); + } +}; + +const assertRepresentativeSharedRealtime = ({ + before, + after, + driverSnapshot, + arm, + customer, +}) => { + const expectedContracts = Object.values(before.buildContracts).sort(); + const residentContracts = [...(after.residentBuildContracts ?? [])].sort(); + const buildCounts = after.builds?.byTenant ?? {}; + if ( + driverSnapshot?.expected !== TENANTS.length + || driverSnapshot.active !== TENANTS.length + || driverSnapshot.verified !== TENANTS.length + || driverSnapshot.errors?.length !== 0 + || JSON.stringify(residentContracts) !== JSON.stringify(expectedContracts) + || TENANTS.some((tenant) => buildCounts[tenant.id] !== 1) + ) { + throw new Error( + `PDCF_SHARED_REALTIME_PREFLIGHT_INCOMPLETE:${arm.name}:${customer.id}` + ); + } + return { + customerId: customer.id, + surfacesBuilt: expectedContracts.length, + subscriptionsActive: driverSnapshot.active, + subscriptionsVerified: driverSnapshot.verified, + residentBuildContracts: residentContracts, + }; +}; + +const verifyRepresentativeSharedRealtime = async ({ + arm, + port, + customer, + status, + fetchImpl = fetch, + createRealtimeDriver = loadRealtimeDriver(), +}) => { + const oneCustomerFleet = makeFleet({ + manifest: { customers: [customer] }, + statuses: { [arm.name]: { [customer.id]: status } }, + arms: [arm], + port, + }); + const tenants = oneCustomerFleet.tenants.map((tenant) => ({ + ...tenant, + surfaces: tenant.surfaces.map((surface) => ({ + ...surface, + url: surface.url.replace('{port}', String(port)), + })), + })); + const driver = createRealtimeDriver(tenants, { + concurrency: TENANTS.length, + timeoutMs: 120_000, + }); + try { + await driver.startAndVerify(); + driver.assertHealthy(); + const response = await fetchImpl( + `http://127.0.0.1:${port}/customer/${customer.id}/__ctf/status` + ); + if (!response.ok) { + throw new Error( + `PDCF_SHARED_REALTIME_PREFLIGHT_STATUS_HTTP:${arm.name}:${response.status}` + ); + } + const after = await response.json(); + return assertRepresentativeSharedRealtime({ + before: status, + after, + driverSnapshot: driver.snapshot(), + arm, + customer, + }); + } finally { + await driver.dispose(); + } +}; + +const makeBlueprintCompatibility = ({ + manifest, + statuses, + mode, + arms = DEFAULT_IDLE_ARMS, +}) => { + const expectedCanonicalSchemas = [ + 'ctf_extensions', + ...TENANTS.flatMap((tenant) => [ + tenant.schema, + completeFixtureServer.realtimeSchemaFor(tenant), + ]), + 'jwt_private', + ]; + if (JSON.stringify(manifest.canonicalSchemas) !== JSON.stringify(expectedCanonicalSchemas)) { + throw new Error('PDCF_CANONICAL_SCHEMA_CLOSURE_MISMATCH'); + } + const canonicalDatabaseContractFingerprint = manifest.canonicalDatabaseContractFingerprint; + if (!/^sha256:[a-f0-9]{64}$/.test(canonicalDatabaseContractFingerprint ?? '')) { + throw new Error('PDCF_CANONICAL_DATABASE_CONTRACT_REQUIRED'); + } + if (manifest.customers.some( + (customer) => customer.databaseContractFingerprint + !== canonicalDatabaseContractFingerprint + )) { + throw new Error('PDCF_DATABASE_CONTRACT_MANIFEST_MISMATCH'); + } + for (const arm of arms.filter( + (candidate) => notificationModeForArm(candidate) === 'shared-exact' + )) { + const representative = manifest.customers[0]; + const evidence = statuses?.[arm.name]?.[representative.id] + ?.sharedRealtimePreflight; + if ( + evidence?.customerId !== representative.id + || evidence.surfacesBuilt !== TENANTS.length + || evidence.subscriptionsActive !== TENANTS.length + || evidence.subscriptionsVerified !== TENANTS.length + || !Array.isArray(evidence.residentBuildContracts) + || evidence.residentBuildContracts.length !== TENANTS.length + ) { + throw new Error(`PDCF_SHARED_REALTIME_PREFLIGHT_REQUIRED:${arm.name}`); + } + } + const runtimeArtifactFingerprints = new Set(Object.values(statuses).flatMap( + (armStatuses) => Object.values(armStatuses).map( + (status) => status.runtimeArtifactFingerprint + ) + )); + if (runtimeArtifactFingerprints.size !== 1) { + throw new Error('PDCF_RUNTIME_ARTIFACT_FINGERPRINT_MISMATCH'); + } + const runtimeArtifactFingerprint = [...runtimeArtifactFingerprints][0]; + const fixtureServerSha256 = `sha256:${fileSha256(path.join( + FIXTURE_DIR, + '../complete-tenant-fixture/server.cjs', + ))}`; + const pluginConfiguration = { + settingsSource: 'fixture-static-no-control-plane-overrides', + featureSettings: completeFixtureServer.FEATURE_SETTINGS, + grafastCacheLimits: completeFixtureServer.GRAFAST_CACHE_LIMITS, + introspectionDependencySchemas: + completeFixtureServer.INTROSPECTION_DEPENDENCY_SCHEMAS, + runtimeDependencySchemas: completeFixtureServer.RUNTIME_DEPENDENCY_SCHEMAS, + introspectionMode: mode, + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: + completeFixtureServer.RELEASE_BUILD_STATE_AFTER_VALIDATION, + runtimeProfiles: arms.map((arm) => ({ + name: arm.name, + runtimePoolMax: runtimePoolMaxForArm(arm), + runtimePoolMaxUses: runtimePoolMaxUsesForArm(arm), + realtimeNotificationMode: notificationModeForArm(arm), + realtimeCursorPollIntervalMs: cursorPollMsForArm(arm), + realtimeCursorHeartbeatIntervalMs: cursorHeartbeatMsForArm(arm), + preparedStatementCacheSize: preparedStatementCacheSizeForArm(arm), + v8Profile: arm.v8Profile ?? 'stock', + })), + enableRealtime: true, + tenantBindings: TENANTS.map((tenant) => ({ + id: tenant.id, + schema: tenant.schema, + realtimeSchema: completeFixtureServer.realtimeSchemaFor(tenant), + databaseId: tenant.databaseId, + apiId: tenant.apiId, + })), + fixtureServerSha256, + runtimeArtifactFingerprint, + }; + const pluginSettingsIdentity = sha256Canonical(pluginConfiguration); + const compatibility = { + version: 1, + scope: 'blueprint-prerequisites-only', + dedicatedInstancesRemainBaseline: true, + sqlRewriteEnabled: false, + releaseBuildStateAfterValidation: + completeFixtureServer.RELEASE_BUILD_STATE_AFTER_VALIDATION, + canonicalDatabaseContractFingerprint, + canonicalSchemas: manifest.canonicalSchemas, + pluginSettingsIdentity, + runtimeArtifactFingerprint, + }; + return { + ...compatibility, + sha256: sha256Canonical(compatibility), + }; +}; + +const collectArmStatuses = async ({ + arm, + port, + manifestFile, + secretsFile, + customerCount, + mode, + provisionClone, + environment = process.env, + verifySharedRealtime = verifyRepresentativeSharedRealtime, + fetchImpl = fetch, + entryFile, + spawnImpl, + readinessTimeoutMs, + terminationTimeoutMs, +}) => { + const { manifest } = loadProvision(manifestFile, secretsFile); + if ( + manifest.provisionClone?.id !== provisionClone.id + || manifest.provisionClone?.purpose !== provisionClone.purpose + ) { + throw new Error(`PDCF_PREFLIGHT_CLONE_MISMATCH:${arm.name}`); + } + const customers = manifest.customers.slice(0, customerCount); + if (customers.length !== customerCount) { + throw new Error( + `PDCF_PREFLIGHT_CUSTOMER_COUNT_INVALID:${arm.name}:${customerCount}:${customers.length}` + ); + } + let child = null; + try { + child = await startArmPreflightChild({ + arm, + port, + manifestFile, + secretsFile, + customerCount, + mode, + provisionClone, + environment, + entryFile, + spawnImpl, + readinessTimeoutMs, + }); + const statuses = Object.fromEntries(await Promise.all(customers.map(async (customer) => { + const response = await fetchImpl( + `http://127.0.0.1:${port}/customer/${customer.id}/__ctf/status` + ); + if (!response.ok) { + throw new Error(`PDCF_PREFLIGHT_STATUS_HTTP:${arm.name}:${customer.id}:${response.status}`); + } + const status = validateChildStatus(await response.json(), { arm, customer, mode }); + return [customer.id, status]; + }))); + assertUniqueRuntimePoolIdentities({ statuses, customers, arm }); + if (notificationModeForArm(arm) === 'shared-exact') { + const customer = customers[0]; + const evidence = await verifySharedRealtime({ + arm, + port, + customer, + status: statuses[customer.id], + }); + statuses[customer.id] = { + ...statuses[customer.id], + sharedRealtimePreflight: evidence, + }; + } + return statuses; + } finally { + if (child) await terminateArmPreflightChild({ + child, + arm, + timeoutMs: terminationTimeoutMs, + }); + } +}; + +const generateInputs = async ({ + manifestFile, + secretsFile, + outDir, + postgresContainer, + basePort, + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + repetitions, + durationSec, + mode, + cacheCalibrationFile, + arms = DEFAULT_IDLE_ARMS, + environment = process.env, + resolveHeapLimitBytes = expectedHeapLimitBytes, + collectStatuses = collectArmStatuses, + inspectContainer = inspectDockerContainer, +}) => { + const { manifest } = loadProvision(manifestFile, secretsFile); + const countMatrix = validateCustomerCountMatrix({ + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + }); + const cacheCalibration = validateCacheCalibration( + JSON.parse(fs.readFileSync(path.resolve(cacheCalibrationFile), 'utf8')), + { + databaseContractFingerprint: manifest.canonicalDatabaseContractFingerprint, + introspectionMode: mode, + }, + ); + if (Math.max(...countMatrix.all) !== manifest.customers.length) { + throw new Error( + `PDCF_MAX_COUNT_MANIFEST_REQUIRED:${countMatrix.all.join(',')}:${manifest.customers.length}` + ); + } + const maximumCustomers = Math.max(...countMatrix.all); + if (maximumCustomers > manifest.customers.length) { + throw new Error( + `PDCF_COUNT_RAMP_EXCEEDS_PROVISIONED:${maximumCustomers}:${manifest.customers.length}` + ); + } + // Capacity is a pure, calibrated prerequisite. Resolve it before starting + // any arm server so an impossible qualifying point cannot spend minutes on + // database/runtime status collection before failing. + const heapLimitBytesByHeapMiB = Object.fromEntries(heapMiB.map((value) => [ + String(value), + resolveHeapLimitBytes(value), + ])); + const cacheCapacityByHeapMiB = makeCacheCapacityProofByHeapMiB({ + cacheCalibration, + databaseContractFingerprint: manifest.canonicalDatabaseContractFingerprint, + introspectionMode: mode, + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + heapLimitBytesByHeapMiB, + }); + const statuses = {}; + for (let index = 0; index < arms.length; index += 1) { + const arm = arms[index]; + statuses[arm.name] = await collectStatuses({ + arm, + port: basePort + index, + manifestFile, + secretsFile, + customerCount: maximumCustomers, + mode, + provisionClone: manifest.provisionClone, + environment, + }); + } + const blueprintCompatibility = makeBlueprintCompatibility({ + manifest, + statuses, + mode, + arms, + }); + + const pgHost = environment.PGHOST ?? 'localhost'; + const pgPort = parsePositiveInteger( + String(environment.PGPORT ?? '5432'), + 'PGPORT', + ); + const containerTemplate = captureContainerTemplate({ + inspection: inspectContainer(postgresContainer), + container: postgresContainer, + prefix: manifest.prefix, + pgHost, + pgPort, + minimumMaxConnections: Math.max( + 100, + maximumCustomers * TENANTS.length * 3 + 16, + ), + }); + const containerTemplateFile = path.join(outDir, 'postgres-container-template.json'); + atomicWriteJson(containerTemplateFile, containerTemplate); + const containerTemplateSha256 = `sha256:${fileSha256(containerTemplateFile)}`; + + const entryFile = path.join(FIXTURE_DIR, 'server.cjs'); + const lockfile = path.join(REPO_ROOT, 'pnpm-lock.yaml'); + const commit = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: REPO_ROOT, + encoding: 'utf8', + }).trim(); + const fleet = makeFleet({ manifest, statuses, arms, port: basePort }); + const plan = makePlan({ + manifestFile: path.resolve(manifestFile), + secretsFile: path.resolve(secretsFile), + postgresContainer, + commit, + entrySha256: fileSha256(entryFile), + lockfileSha256: fileSha256(lockfile), + arms, + basePort, + heapMiB, + tenantCounts, + tenantCountsByHeapMiB, + repetitions, + durationSec, + introspectionMode: mode, + databaseContractFingerprint: + blueprintCompatibility.canonicalDatabaseContractFingerprint, + blueprintCompatibilityFingerprint: blueprintCompatibility.sha256, + manifestSha256: `sha256:${fileSha256(manifestFile)}`, + provisionClone: manifest.provisionClone, + cacheCalibration, + heapLimitBytesByHeapMiB, + cacheCapacityByHeapMiB, + postgresContainerTemplateFile: path.resolve(containerTemplateFile), + postgresContainerTemplateSha256: containerTemplateSha256, + }); + const fleetFile = path.join(outDir, 'fleet.json'); + const planFile = path.join(outDir, 'plan.json'); + const preflightFile = path.join(outDir, 'preflight-status.json'); + atomicWriteJson(fleetFile, fleet); + atomicWriteJson(planFile, plan); + atomicWriteJson(preflightFile, { + version: 1, + fixture: 'physical-database-density-v1', + canonicalStructuralFingerprint: + manifest.canonicalStructuralFingerprint?.combined?.sha256 ?? null, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint ?? null, + blueprintCompatibility, + cacheCalibration, + heapLimitBytesByHeapMiB, + cacheCapacityByHeapMiB, + statuses, + }); + return { + containerTemplateFile, + fleet, + fleetFile, + plan, + planFile, + preflightFile, + }; +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + const armProfile = requireString(args, 'arm-profile', 'idle'); + const arms = armProfile === 'idle' + ? DEFAULT_IDLE_ARMS + : armProfile === 'density-tuning' + ? DENSITY_TUNING_ARMS + : null; + if (!arms) throw new Error(`PDCF_ARM_PROFILE_INVALID:${armProfile}`); + const manifestFile = path.resolve(requireString(args, 'manifest')); + const secretsFile = path.resolve(requireString(args, 'secrets')); + const outDir = path.resolve(requireString( + args, + 'out-dir', + path.join(FIXTURE_DIR, '.local', 'inputs'), + )); + const heapMiB = parseIntegerList( + requireString(args, 'heaps', '1024,2048,4096'), + 'heaps', + ); + const tenantCountsByHeapMiB = args['tenant-counts-by-heap-mib'] == null + ? undefined + : parseTenantCountsByHeapMiB( + requireString(args, 'tenant-counts-by-heap-mib'), + heapMiB, + ); + const tenantCounts = args['tenant-counts'] == null + ? undefined + : parseIntegerList(requireString(args, 'tenant-counts'), 'tenant-counts'); + if ((tenantCounts == null) === (tenantCountsByHeapMiB == null)) { + throw new Error('PDCF_EXACTLY_ONE_TENANT_COUNT_MODE_REQUIRED'); + } + const result = await generateInputs({ + manifestFile, + secretsFile, + outDir, + postgresContainer: requireString(args, 'postgres-container'), + basePort: parsePositiveInteger(args['base-port'] ?? '3410', 'base-port'), + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + repetitions: parsePositiveInteger(args.repetitions ?? '3', 'repetitions'), + durationSec: parsePositiveInteger(args['duration-sec'] ?? '900', 'duration-sec'), + mode: requireString(args, 'mode', 'scoped-required'), + cacheCalibrationFile: path.resolve(requireString(args, 'cache-calibration')), + arms, + }); + process.stdout.write(`${JSON.stringify({ + status: 'generated', + fleetFile: result.fleetFile, + planFile: result.planFile, + preflightFile: result.preflightFile, + containerTemplateFile: result.containerTemplateFile, + })}\n`); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + collectArmStatuses, + assertUniqueRuntimePoolIdentities, + assertRepresentativeSharedRealtime, + generateInputs, + parseIntegerList, + parseTenantCountsByHeapMiB, + makeBlueprintCompatibility, + expectedHeapLimitBytes, + sha256Canonical, + makeArmPreflightArgs, + makeArmPreflightEnvironment, + startArmPreflightChild, + terminateArmPreflightChild, + validateChildStatus, + v8FlagsForArm, + verifyRepresentativeSharedRealtime, +}; diff --git a/research/graphile-density/physical-database-density/inputs.test.cjs b/research/graphile-density/physical-database-density/inputs.test.cjs new file mode 100644 index 0000000000..58a018443a --- /dev/null +++ b/research/graphile-density/physical-database-density/inputs.test.cjs @@ -0,0 +1,870 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { + DEFAULT_IDLE_ARMS, + loadProvision, + makeCustomers, + makeSecretResolver, +} = require('./lib.cjs'); +const { + assertRepresentativeSharedRealtime, + assertUniqueRuntimePoolIdentities, + collectArmStatuses, + generateInputs, + makeArmPreflightEnvironment, + makeBlueprintCompatibility, + parseIntegerList, + parseTenantCountsByHeapMiB, + startArmPreflightChild, + terminateArmPreflightChild, + validateChildStatus, + v8FlagsForArm, +} = require('./generate-inputs.cjs'); +const { + CALIBRATION_KIND, + sha256Canonical, +} = require('./cache-calibration.cjs'); +const { + aggregateRuntimePoolStats, + classifyDatabaseScope, + matchPhysicalUpgradeCustomer, + parseServerOptions, + runtimeEnvironmentFor, + tokenEqual, +} = require('./server.cjs'); + +const digest = (character) => `sha256:${character.repeat(64)}`; +const MIB = 1024 ** 2; + +const customer = makeCustomers('pdc_test', 1)[0]; +const childStatus = (armName, runtimeArtifactFingerprint = digest('a')) => ({ + version: 1, + fixture: 'complete-tenant-abc-v1', + arm: armName, + introspectionMode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: true, + physicalDatabase: customer.database, + runtimePoolMax: 2, + runtimePoolMaxUses: null, + runtimePools: { + scope: 'runtime-only-exact-identities', + available: true, + requestedMaxUses: null, + effectiveMaxUses: null, + effectiveMaxUsesKnown: true, + maxUsesExact: true, + identitiesUnique: true, + poolObjectsUnique: true, + expectedPools: 3, + observedPools: 3, + totalClients: 0, + idleClients: 0, + waitingClients: 0, + }, + preparedStatementCache: { + environmentValue: '100', + requestedSize: 100, + environmentCanonical: true, + attestation: 'loaded-dataplan-adaptor-behavior-v1', + effectiveSize: 100, + effectiveSizeKnown: true, + exact: true, + namedQueriesObserved: 101, + firstEvictionAfterNamedQueries: 100, + }, + enableRealtime: true, + realtimeNotificationMode: 'dedicated', + realtimeCursorPollIntervalMs: 5000, + realtimeCursorHeartbeatIntervalMs: 30000, + runtimeArtifactFingerprint, + configurationIdentity: `graphile-configuration:ctf:v1:${'e'.repeat(64)}`, + liveIdentityScope: 'process-local-keyed-hmac-v1', + runtimePoolIdentities: { + a: 'pg:v1:a', + b: 'pg:v1:b', + c: 'pg:v1:c', + }, + buildContracts: { + a: 'graphile:v1:a', + b: 'graphile:v1:b', + c: 'graphile:v1:c', + }, + runtimeBindings: Object.fromEntries(['a', 'b', 'c'].map((tenantId) => [ + tenantId, + { + databaseName: customer.database, + role: customer.roles[tenantId], + schemas: [`ctf_${tenantId}`], + }, + ])), + contractEvidence: { + version: 1, + credentialFree: true, + configurationIdentity: `graphile-configuration:ctf:v1:${'e'.repeat(64)}`, + runtimePools: Object.fromEntries(['a', 'b', 'c'].map((tenantId) => [ + tenantId, + { + version: 1, + fingerprint: `pg-contract-evidence:v1:${tenantId.repeat(64)}`, + input: { + databaseName: customer.database, + role: customer.roles[tenantId], + }, + }, + ])), + graphileBuilds: Object.fromEntries(['a', 'b', 'c'].map((tenantId) => [ + tenantId, + { + version: 1, + fingerprint: `graphile-contract-evidence:v1:${tenantId.repeat(64)}`, + input: {}, + }, + ])), + }, + realtimeSchemas: { + a: 'ctf_a_realtime', + b: 'ctf_b_realtime', + c: 'ctf_c_realtime', + }, + runtimeSafety: { + passed: true, + dependencySchemasByTenant: { + a: ['ctf_extensions', 'jwt_private', 'ctf_a_realtime'], + b: ['ctf_extensions', 'jwt_private', 'ctf_b_realtime'], + c: ['ctf_extensions', 'jwt_private', 'ctf_c_realtime'], + }, + }, +}); + +const manifest = { + version: 1, + fixture: 'physical-database-density-v1', + prefix: 'pdc_test', + canonicalSchemas: [ + 'ctf_extensions', + 'ctf_a', + 'ctf_a_realtime', + 'ctf_b', + 'ctf_b_realtime', + 'ctf_c', + 'ctf_c_realtime', + 'jwt_private', + ], + canonicalDatabaseContractFingerprint: digest('b'), + customers: [{ ...customer, databaseContractFingerprint: digest('b') }], +}; + +const makeInsufficientCalibration = () => { + const measured = { + repetitions: 3, + retainedHeapPerSurfaceBytes: 100 * MIB, + serverBaselineHeapBytes: 400 * MIB, + buildTransientHeapBytes: 400 * MIB, + buildTransientRssBytes: 100 * MIB, + }; + const safetyFactor = 1.25; + const payload = { + kind: CALIBRATION_KIND, + databaseContractFingerprint: manifest.canonicalDatabaseContractFingerprint, + introspectionMode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: true, + introspectionBackendRetirementConclusive: true, + fixtureFingerprint: 'fixture-v1', + schemaContract: { + schemaSets: [['ctf_a']], + allowedDependencySchemas: ['ctf_extensions'], + }, + safetyFactor, + measured, + configured: { + instanceHeapBytes: 125 * MIB, + serverReserveBytes: 500 * MIB, + buildReserveBytes: 500 * MIB, + rssBuildReserveBytes: 125 * MIB, + }, + sourceWorktreesClean: true, + sources: ['1', '2', '3'].map((value) => ({ + sourceSha256: digest(value), + sourceStateSha256: value.repeat(64), + executedEntrySha256: value.repeat(64), + worktreeDirty: false, + introspectionBackendRetirement: { + conclusive: true, + introspectionBackendPid: Number(value), + steadyBackendPid: Number(value) + 10, + }, + })), + }; + return { + version: 2, + ...payload, + calibrationId: sha256Canonical(payload), + }; +}; + +describe('physical database density inputs', () => { + it('maps every supported arm profile to the harness V8 flags', () => { + assert.deepEqual(v8FlagsForArm({ v8Profile: 'stock' }), []); + assert.deepEqual(v8FlagsForArm({ v8Profile: 'optimize-for-size' }), [ + '--optimize-for-size', + ]); + assert.deepEqual(v8FlagsForArm({ v8Profile: 'baseline-optimize-for-size' }), [ + '--max-opt=1', + '--optimize-for-size', + ]); + assert.deepEqual(v8FlagsForArm({ v8Profile: 'jitless-optimize-for-size' }), [ + '--jitless', + '--optimize-for-size', + ]); + assert.throws( + () => v8FlagsForArm({ v8Profile: 'ambient-flags' }), + /PDCF_V8_PROFILE_INVALID:ambient-flags/, + ); + }); + + it('preflights each arm in an isolated child and cleans up success and failure', async () => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-isolated-preflight-')); + const entryFile = path.join(temporary, 'fake-physical-server.cjs'); + const cleanupFile = path.join(temporary, 'cleanup.log'); + const preloadFile = path.join(temporary, 'ambient-preload.cjs'); + const preloadMarker = path.join(temporary, 'ambient-preload-ran'); + const parentPreparedCache = process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE; + const parentPoolMaxUses = process.env.PG_POOL_MAX_USES; + try { + fs.writeFileSync(entryFile, `'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const args = {}; +for (let index = 2; index < process.argv.length; index += 2) { + args[process.argv[index].slice(2)] = process.argv[index + 1]; +} +const outputDirectory = process.env.PDCF_TEST_OUTPUT_DIRECTORY; +fs.writeFileSync(path.join(outputDirectory, args.arm + '.json'), JSON.stringify({ + preparedStatementCacheSize: process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE ?? null, + processPoolMax: process.env.PG_POOL_MAX ?? null, + processPoolMaxUses: process.env.PG_POOL_MAX_USES ?? null, + runtimePoolMaxUses: args['runtime-pool-max-uses'], + nodeOptions: process.env.NODE_OPTIONS ?? null, + nodePath: process.env.NODE_PATH ?? null, + execArgv: process.execArgv, +})); +process.once('SIGTERM', () => { + fs.appendFileSync(path.join(outputDirectory, 'cleanup.log'), args.arm + '\\n'); + process.exit(0); +}); +process.stdout.write(JSON.stringify({ + status: 'ready', + fixture: 'physical-database-density-v1', + host: '127.0.0.1', + port: Number(args.port), + arm: args.arm, + customers: Number(args.customers), +}) + '\\n'); +setInterval(() => {}, 1000); +`); + fs.writeFileSync(preloadFile, `'use strict'; +require('node:fs').writeFileSync(process.env.PDCF_TEST_PRELOAD_MARKER, 'loaded'); +`); + process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE = 'parent-contamination'; + process.env.PG_POOL_MAX_USES = '77'; + const stockArm = { + name: 'isolated-stock', + idleTimeoutMs: 1000, + runtimePoolMax: 1, + }; + const noPrepareArm = { + ...stockArm, + name: 'isolated-no-prepare', + preparedStatementCacheSize: 0, + runtimePoolMaxUses: 1, + v8Profile: 'optimize-for-size', + }; + const common = { + manifestFile: '/not-read-by-fake/manifest.json', + secretsFile: '/not-read-by-fake/secrets.json', + customerCount: 1, + mode: 'scoped-required', + provisionClone: { id: 'measurement-clone', purpose: 'measurement' }, + environment: { + ...process.env, + PDCF_TEST_OUTPUT_DIRECTORY: temporary, + PDCF_TEST_PRELOAD_MARKER: preloadMarker, + NODE_OPTIONS: `--require=${preloadFile}`, + NODE_PATH: '/tmp/pdc-untrusted-node-path', + PG_POOL_MAX: '99', + }, + entryFile, + readinessTimeoutMs: 5000, + }; + const stockChild = await startArmPreflightChild({ + ...common, + arm: stockArm, + port: 3491, + }); + await terminateArmPreflightChild({ child: stockChild, arm: stockArm }); + const noPrepareChild = await startArmPreflightChild({ + ...common, + arm: noPrepareArm, + port: 3492, + }); + await terminateArmPreflightChild({ child: noPrepareChild, arm: noPrepareArm }); + + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(temporary, 'isolated-stock.json'), 'utf8')), + { + preparedStatementCacheSize: '100', + processPoolMax: '1', + processPoolMaxUses: '0', + runtimePoolMaxUses: 'unlimited', + nodeOptions: '', + nodePath: null, + execArgv: ['--expose-gc'], + }, + ); + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(temporary, 'isolated-no-prepare.json'), 'utf8')), + { + preparedStatementCacheSize: '0', + processPoolMax: '1', + processPoolMaxUses: '0', + runtimePoolMaxUses: '1', + nodeOptions: '', + nodePath: null, + execArgv: ['--optimize-for-size', '--expose-gc'], + }, + ); + assert.equal( + process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE, + 'parent-contamination', + ); + assert.equal(process.env.PG_POOL_MAX_USES, '77'); + assert.equal(fs.existsSync(preloadMarker), false); + assert.deepEqual( + fs.readFileSync(cleanupFile, 'utf8').trim().split('\n').sort(), + ['isolated-no-prepare', 'isolated-stock'], + ); + + const manifestFile = path.join(temporary, 'provision.json'); + const secretsFile = path.join(temporary, 'runtime-secrets.json'); + const provisionClone = { + version: 1, + id: 'measurement-clone', + purpose: 'measurement', + attestationSetSha256: digest('e'), + }; + fs.writeFileSync(manifestFile, JSON.stringify({ ...manifest, provisionClone })); + fs.writeFileSync(secretsFile, JSON.stringify({ + version: 1, + fixture: 'physical-database-density-v1', + runtimePasswords: Object.fromEntries(Object.values(customer.roles).map( + (role) => [role, `fixture-password-at-least-24-bytes-${role}`] + )), + notificationPasswords: { + [customer.notificationRole]: + `fixture-password-at-least-24-bytes-${customer.notificationRole}`, + }, + }), { mode: 0o600 }); + const failingArm = { ...stockArm, name: 'isolated-fetch-failure' }; + await assert.rejects(collectArmStatuses({ + arm: failingArm, + port: 3493, + manifestFile, + secretsFile, + customerCount: 1, + mode: 'scoped-required', + provisionClone, + environment: { + ...process.env, + PDCF_TEST_OUTPUT_DIRECTORY: temporary, + }, + entryFile, + readinessTimeoutMs: 5000, + terminationTimeoutMs: 5000, + fetchImpl: async () => { + throw new Error('injected status failure'); + }, + }), /injected status failure/); + assert.match(fs.readFileSync(cleanupFile, 'utf8'), /isolated-fetch-failure/); + assert.deepEqual( + makeArmPreflightEnvironment(noPrepareArm, { + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: 'stale', + NODE_OPTIONS: '--require=/tmp/ambient-preload.cjs', + NODE_PATH: '/tmp/ambient-node-path', + PG_POOL_MAX: '99', + PG_POOL_MAX_USES: '99', + }), + { + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: '0', + NODE_OPTIONS: '', + PG_POOL_IDLE_TIMEOUT_MS: '1000', + PG_POOL_MAX: '1', + PG_POOL_MAX_USES: '0', + }, + ); + } finally { + if (parentPreparedCache == null) { + delete process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE; + } else { + process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE = parentPreparedCache; + } + if (parentPoolMaxUses == null) delete process.env.PG_POOL_MAX_USES; + else process.env.PG_POOL_MAX_USES = parentPoolMaxUses; + fs.rmSync(temporary, { recursive: true, force: true }); + } + }); + + it('rejects insufficient calibrated capacity before collecting arm statuses', async () => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-capacity-preflight-')); + let statusCollectionStarted = false; + try { + const manifestFile = path.join(temporary, 'provision.json'); + const secretsFile = path.join(temporary, 'runtime-secrets.json'); + const calibrationFile = path.join(temporary, 'cache-calibration.json'); + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + fs.writeFileSync(secretsFile, JSON.stringify({ + version: 1, + fixture: 'physical-database-density-v1', + runtimePasswords: Object.fromEntries(Object.values(customer.roles).map( + (role) => [role, `fixture-password-at-least-24-bytes-${role}`] + )), + notificationPasswords: { + [customer.notificationRole]: + `fixture-password-at-least-24-bytes-${customer.notificationRole}`, + }, + }), { mode: 0o600 }); + fs.writeFileSync(calibrationFile, JSON.stringify(makeInsufficientCalibration())); + + await assert.rejects(generateInputs({ + manifestFile, + secretsFile, + outDir: path.join(temporary, 'inputs'), + postgresContainer: 'postgres-density', + basePort: 3410, + tenantCounts: [1], + heapMiB: [1024], + repetitions: 1, + durationSec: 5, + mode: 'scoped-required', + cacheCalibrationFile: calibrationFile, + resolveHeapLimitBytes: () => 1024 * MIB, + collectStatuses: async () => { + statusCollectionStarted = true; + throw new Error('status collection must not start'); + }, + }), /PDCF_CALIBRATED_CAPACITY_INSUFFICIENT:1024:1:3/); + assert.equal(statusCollectionStarted, false); + } finally { + fs.rmSync(temporary, { recursive: true, force: true }); + } + }); + + it('parses only loopback server options and binds non-secret compatibility hashes', () => { + const options = parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--host', '127.0.0.1', + '--runtime-pool-max', '2', + '--runtime-pool-max-uses', '1', + '--enable-realtime', 'true', + '--expected-database-contract', digest('b'), + '--blueprint-compatibility', digest('c'), + '--expected-manifest-sha256', digest('d'), + '--run-purpose', 'measurement', + '--clone-id', 'measurement-clone-test', + ]); + assert.equal(options.enableRealtime, true); + assert.equal(options.runtimePoolMax, 2); + assert.equal(options.runtimePoolMaxUses, 1); + assert.equal(options.introspectionClientReleaseMode, 'destroy'); + assert.equal(options.expectedDatabaseContractFingerprint, digest('b')); + assert.equal(options.blueprintCompatibilityFingerprint, digest('c')); + assert.equal(options.expectedManifestSha256, digest('d')); + assert.throws(() => parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--host', '0.0.0.0', + ]), /PDCF_SERVER_LOOPBACK_REQUIRED/); + assert.throws(() => parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--introspection-client-release-mode', 'best-effort', + ]), /PDCF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:best-effort/); + assert.throws(() => parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--runtime-pool-max-uses', '0', + '--run-purpose', 'measurement', + '--clone-id', 'measurement-clone-test', + ]), /PDCF_INVALID_MAX_USES:runtime-pool-max-uses/); + for (const value of ['01', '1e2', '0x1', ' 1', '1 ', '']) { + assert.throws(() => parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--runtime-pool-max-uses', value, + '--run-purpose', 'measurement', + '--clone-id', 'measurement-clone-test', + ]), /PDCF_INVALID_MAX_USES:runtime-pool-max-uses/); + } + }); + + it('maps one physical database to three exact least-privilege credentials', () => { + const rawSecrets = { + version: 1, + fixture: 'physical-database-density-v1', + runtimePasswords: Object.fromEntries( + Object.values(customer.roles).map((role) => [role, `long-fixture-secret-${role}`]) + ), + notificationPasswords: { + [customer.notificationRole]: `long-fixture-secret-${customer.notificationRole}`, + }, + }; + const secretResolver = makeSecretResolver(rawSecrets, manifest); + const environment = runtimeEnvironmentFor( + { PGHOST: 'fixture-host' }, + customer, + secretResolver, + true, + ); + assert.equal(environment.PGDATABASE, customer.database); + assert.equal(environment.PG_POOL_MAX_USES, '0'); + assert.equal(environment.CTF_RUNTIME_A_PGUSER, customer.roles.a); + assert.equal( + environment.CTF_RUNTIME_C_PGPASSWORD, + `long-fixture-secret-${customer.roles.c}`, + ); + assert.equal(environment.CTF_NOTIFICATION_PGUSER, customer.notificationRole); + assert.equal( + environment.CTF_NOTIFICATION_PGPASSWORD, + `long-fixture-secret-${customer.notificationRole}`, + ); + assert.equal(tokenEqual('same-token', 'same-token'), true); + assert.equal(tokenEqual('same-token', 'different-token'), false); + assert.doesNotMatch(JSON.stringify(secretResolver), /long-fixture-secret/); + }); + + it('aggregates only child runtime pools and preserves effective native maxUses evidence', () => { + const runtimeStats = (overrides = {}) => ({ + scope: 'runtime-only-exact-identities', + available: true, + requestedMaxUses: 1, + effectiveMaxUses: 1, + effectiveMaxUsesKnown: true, + maxUsesExact: true, + identitiesUnique: true, + poolObjectsUnique: true, + expectedPools: 3, + observedPools: 3, + totalClients: 1, + idleClients: 0, + waitingClients: 0, + ...overrides, + }); + const firstPools = [{}, {}, {}]; + const secondPools = [{}, {}, {}]; + const stats = aggregateRuntimePoolStats([ + { + child: { + runtimePoolStats: () => runtimeStats(), + runtimePoolObjects: () => firstPools, + }, + }, + { + child: { + runtimePoolStats: () => runtimeStats({ totalClients: 2 }), + runtimePoolObjects: () => secondPools, + }, + }, + ], 1); + assert.deepEqual(stats, { + scope: 'runtime-only-exact-identities', + available: true, + requestedMaxUses: 1, + effectiveMaxUses: 1, + effectiveMaxUsesKnown: true, + maxUsesExact: true, + identitiesUnique: true, + poolObjectsUnique: true, + expectedPools: 6, + observedPools: 6, + totalClients: 3, + idleClients: 0, + waitingClients: 0, + }); + + const mismatch = aggregateRuntimePoolStats([ + { + child: { + runtimePoolStats: () => runtimeStats({ + effectiveMaxUses: null, + maxUsesExact: false, + }), + runtimePoolObjects: () => [{}, {}, {}], + }, + }, + ], 1); + assert.equal(mismatch.available, false); + assert.equal(mismatch.maxUsesExact, false); + assert.equal(mismatch.totalClients, null); + + const sharedPool = {}; + const crossCustomerReuse = aggregateRuntimePoolStats([ + { + child: { + runtimePoolStats: () => runtimeStats(), + runtimePoolObjects: () => [sharedPool, {}, {}], + }, + }, + { + child: { + runtimePoolStats: () => runtimeStats(), + runtimePoolObjects: () => [sharedPool, {}, {}], + }, + }, + ], 1); + assert.equal(crossCustomerReuse.poolObjectsUnique, false); + assert.equal(crossCustomerReuse.available, false); + }); + + it('loads secrets only from a regular, non-symlink 0600 file', () => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-private-secrets-')); + try { + const manifestFile = path.join(temporary, 'provision.json'); + const secretsFile = path.join(temporary, 'runtime-secrets.json'); + const symlinkFile = path.join(temporary, 'runtime-secrets-link.json'); + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + const rawSecrets = { + version: 1, + fixture: 'physical-database-density-v1', + runtimePasswords: Object.fromEntries(Object.values(customer.roles).map( + (role) => [role, `fixture-password-at-least-24-bytes-${role}`] + )), + notificationPasswords: { + [customer.notificationRole]: + `fixture-password-at-least-24-bytes-${customer.notificationRole}`, + }, + }; + fs.writeFileSync(secretsFile, JSON.stringify(rawSecrets), { mode: 0o600 }); + const loaded = loadProvision(manifestFile, secretsFile); + assert.equal(loaded.manifest.fixture, 'physical-database-density-v1'); + assert.doesNotMatch( + JSON.stringify(loaded), + /fixture-password-at-least-24-bytes/, + ); + + fs.chmodSync(secretsFile, 0o640); + assert.throws( + () => loadProvision(manifestFile, secretsFile), + /PDCF_SECRETS_FILE_MODE_MUST_BE_0600/, + ); + fs.chmodSync(secretsFile, 0o600); + fs.symlinkSync(secretsFile, symlinkFile); + assert.throws( + () => loadProvision(manifestFile, symlinkFile), + /PDCF_SECRETS_FILE_MUST_BE_REGULAR/, + ); + } finally { + fs.rmSync(temporary, { recursive: true, force: true }); + } + }); + + it('routes websocket upgrades to one exact physical customer', () => { + assert.equal( + matchPhysicalUpgradeCustomer( + '/customer/physical-customer-0001/tenant/a/graphql', + ), + 'physical-customer-0001', + ); + assert.equal(matchPhysicalUpgradeCustomer( + '/customer/physical-customer-0001/tenant/a/graphql?customer=other', + ), null); + assert.equal(matchPhysicalUpgradeCustomer( + '/customer/%70hysical-customer-0001/tenant/a/graphql', + ), null); + assert.equal(matchPhysicalUpgradeCustomer( + '/customer/physical-customer-0001/tenant/a/graphql/extra', + ), null); + }); + + it('marks shared or incomplete PostgreSQL database sets non-qualifying', () => { + assert.deepEqual( + classifyDatabaseScope( + ['postgres', customer.database], + 'postgres', + [customer.database], + ), + { + dedicated: true, + databasesPresent: 2, + fixtureDatabasesExpected: 1, + fixtureDatabasesPresent: 1, + unexpectedDatabases: 0, + missingFixtureDatabases: 0, + unexpectedDatabaseSetSha256: null, + }, + ); + const shared = classifyDatabaseScope( + ['postgres', customer.database, 'unrelated_app'], + 'postgres', + [customer.database], + ); + assert.equal(shared.dedicated, false); + assert.equal(shared.unexpectedDatabases, 1); + assert.match(shared.unexpectedDatabaseSetSha256, /^sha256:[a-f0-9]{64}$/); + assert.equal(classifyDatabaseScope( + ['postgres'], + 'postgres', + [customer.database], + ).dedicated, false); + }); + + it('fails closed on status dependency drift and runtime artifact drift', () => { + const arm = DEFAULT_IDLE_ARMS[0]; + const status = childStatus(arm.name); + assert.equal(validateChildStatus(status, { + arm, + customer, + mode: 'scoped-required', + }), status); + assert.throws(() => validateChildStatus({ + ...status, + releaseBuildStateAfterValidation: false, + }, { arm, customer, mode: 'scoped-required' }), /PDCF_PREFLIGHT_STATUS_INVALID/); + assert.throws(() => validateChildStatus({ + ...status, + runtimeSafety: { + ...status.runtimeSafety, + dependencySchemasByTenant: { + ...status.runtimeSafety.dependencySchemasByTenant, + a: ['ctf_extensions', 'ctf_a_realtime'], + }, + }, + }, { arm, customer, mode: 'scoped-required' }), /RUNTIME_DEPENDENCIES_INVALID/); + assert.throws(() => validateChildStatus({ + ...status, + preparedStatementCache: { + ...status.preparedStatementCache, + attestation: 'environment-echo', + }, + }, { arm, customer, mode: 'scoped-required' }), /PDCF_PREFLIGHT_STATUS_INVALID/); + assert.throws(() => validateChildStatus({ + ...status, + runtimePoolIdentities: { + ...status.runtimePoolIdentities, + c: status.runtimePoolIdentities.a, + }, + }, { arm, customer, mode: 'scoped-required' }), /POOL_IDENTITIES_NOT_UNIQUE/); + + const secondCustomer = makeCustomers('pdc_test', 2)[1]; + assert.throws(() => assertUniqueRuntimePoolIdentities({ + arm, + customers: [customer, secondCustomer], + statuses: { + [customer.id]: status, + [secondCustomer.id]: { + ...status, + physicalDatabase: secondCustomer.database, + }, + }, + }), /POOL_IDENTITIES_NOT_UNIQUE/); + + const statuses = Object.fromEntries(DEFAULT_IDLE_ARMS.map((candidate, index) => [ + candidate.name, + { [customer.id]: childStatus(candidate.name, digest(index === 2 ? 'c' : 'a')) }, + ])); + assert.throws(() => makeBlueprintCompatibility({ + manifest, + statuses, + mode: 'scoped-required', + }), /RUNTIME_ARTIFACT_FINGERPRINT_MISMATCH/); + }); + + it('requires three built surfaces and three live verified shared subscriptions', () => { + const arm = { + name: 'physical-db-shared-stock', + realtimeNotificationMode: 'shared-exact', + }; + const before = childStatus(arm.name); + const after = { + ...before, + residentBuildContracts: Object.values(before.buildContracts), + builds: { byTenant: { a: 1, b: 1, c: 1 } }, + }; + const snapshot = { + expected: 3, + active: 3, + verified: 3, + errors: [], + }; + assert.deepEqual(assertRepresentativeSharedRealtime({ + before, + after, + driverSnapshot: snapshot, + arm, + customer, + }), { + customerId: customer.id, + surfacesBuilt: 3, + subscriptionsActive: 3, + subscriptionsVerified: 3, + residentBuildContracts: Object.values(before.buildContracts).sort(), + }); + assert.throws(() => assertRepresentativeSharedRealtime({ + before, + after, + driverSnapshot: { ...snapshot, active: 2 }, + arm, + customer, + }), /PDCF_SHARED_REALTIME_PREFLIGHT_INCOMPLETE/); + assert.throws(() => assertRepresentativeSharedRealtime({ + before, + after: { + ...after, + builds: { byTenant: { a: 1, b: 1, c: 0 } }, + }, + driverSnapshot: snapshot, + arm, + customer, + }), /PDCF_SHARED_REALTIME_PREFLIGHT_INCOMPLETE/); + }); + + it('emits a deterministic prerequisite fingerprint without enabling blueprint sharing', () => { + const statuses = Object.fromEntries(DEFAULT_IDLE_ARMS.map((arm) => [ + arm.name, + { [customer.id]: childStatus(arm.name) }, + ])); + const compatibility = makeBlueprintCompatibility({ + manifest, + statuses, + mode: 'scoped-required', + }); + assert.match(compatibility.sha256, /^sha256:[a-f0-9]{64}$/); + assert.equal(compatibility.scope, 'blueprint-prerequisites-only'); + assert.equal(compatibility.dedicatedInstancesRemainBaseline, true); + assert.equal(compatibility.sqlRewriteEnabled, false); + assert.equal(compatibility.releaseBuildStateAfterValidation, true); + assert.deepEqual(parseIntegerList('4,1,2', 'counts'), [1, 2, 4]); + assert.deepEqual( + parseTenantCountsByHeapMiB( + '1024:2,1;2048:4,2;4096:8,4', + [1024, 2048, 4096], + ), + { + '1024': [1, 2], + '2048': [2, 4], + '4096': [4, 8], + }, + ); + assert.throws(() => parseTenantCountsByHeapMiB( + '1024:1,2;2048:2,4', + [1024, 2048, 4096], + ), /PDCF_TENANT_COUNTS_BY_HEAP_COVERAGE_INVALID/); + }); +}); diff --git a/research/graphile-density/physical-database-density/lib.cjs b/research/graphile-density/physical-database-density/lib.cjs new file mode 100644 index 0000000000..0d4b4b1653 --- /dev/null +++ b/research/graphile-density/physical-database-density/lib.cjs @@ -0,0 +1,1357 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const completeFixture = require('../complete-tenant-fixture/lib.cjs'); +const { + computeCalibratedCapacity, + validateCacheCalibration, +} = require('./cache-calibration.cjs'); + +const FIXTURE_DIR = __dirname; +const REPO_ROOT = path.resolve(FIXTURE_DIR, '../../..'); +const PHYSICAL_DATABASE_CANARY = 'physical-database-routing'; +const FIXTURE_ID = 'physical-database-density-v1'; +const QUALIFYING_CACHE_ADMISSION_MODE = 'preserve-resident'; +const DEFAULT_PREPARED_STATEMENT_CACHE_SIZE = 100; +const PROCESS_GLOBAL_POOL_MAX = 1; +const DEFAULT_IDLE_ARMS = Object.freeze([ + Object.freeze({ name: 'physical-db-idle-30s', idleTimeoutMs: 30_000 }), + Object.freeze({ name: 'physical-db-idle-5s', idleTimeoutMs: 5_000 }), + Object.freeze({ name: 'physical-db-idle-1s', idleTimeoutMs: 1_000 }), +]); +const DENSITY_TUNING_ARMS = Object.freeze([ + Object.freeze({ + name: 'physical-db-dedicated-stock', + idleTimeoutMs: 1_000, + runtimePoolMax: 2, + realtimeNotificationMode: 'dedicated', + realtimeCursorPollIntervalMs: 5_000, + realtimeCursorHeartbeatIntervalMs: 30_000, + v8Profile: 'stock', + }), + Object.freeze({ + name: 'physical-db-shared-stock', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + v8Profile: 'stock', + }), + Object.freeze({ + name: 'physical-db-shared-no-prepare', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + preparedStatementCacheSize: 0, + v8Profile: 'stock', + }), + Object.freeze({ + name: 'physical-db-shared-maxuses-1', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + runtimePoolMaxUses: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + v8Profile: 'stock', + }), + Object.freeze({ + name: 'physical-db-shared-size', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + v8Profile: 'optimize-for-size', + }), + Object.freeze({ + name: 'physical-db-shared-baseline-size', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + v8Profile: 'baseline-optimize-for-size', + }), + Object.freeze({ + name: 'physical-db-shared-jitless-size', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + v8Profile: 'jitless-optimize-for-size', + }), + Object.freeze({ + name: 'physical-db-shared-maxuses-1-size', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + runtimePoolMaxUses: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + v8Profile: 'optimize-for-size', + }), +]); + +const runtimePoolMaxForArm = (arm) => arm.runtimePoolMax ?? 2; +const runtimePoolMaxUsesForArm = (arm) => arm.runtimePoolMaxUses ?? null; +const notificationModeForArm = (arm) => arm.realtimeNotificationMode ?? 'dedicated'; +const cursorPollMsForArm = (arm) => arm.realtimeCursorPollIntervalMs ?? 5_000; +const cursorHeartbeatMsForArm = (arm) => + arm.realtimeCursorHeartbeatIntervalMs ?? 30_000; +const preparedStatementCacheSizeForArm = (arm) => + arm.preparedStatementCacheSize ?? DEFAULT_PREPARED_STATEMENT_CACHE_SIZE; + +const validateCustomerCountRamp = (counts, label) => { + if ( + !Array.isArray(counts) + || counts.length === 0 + || counts.some((count) => !Number.isSafeInteger(count) || count <= 0) + || new Set(counts).size !== counts.length + || counts.some((count, index) => index > 0 && count <= counts[index - 1]) + ) { + throw new Error(`PDCF_PLAN_CUSTOMER_COUNT_RAMP_INVALID:${label}`); + } + return counts; +}; + +const customerCountsForHeap = ({ tenantCounts, tenantCountsByHeapMiB }, heap) => + tenantCountsByHeapMiB?.[String(heap)] ?? tenantCounts; + +const validateCustomerCountMatrix = ({ + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, +}) => { + if (!Array.isArray(heapMiB) || heapMiB.length === 0) { + throw new Error('PDCF_PLAN_HEAP_RAMP_REQUIRED'); + } + if (tenantCounts != null) { + validateCustomerCountRamp(tenantCounts, 'default'); + } + if (tenantCountsByHeapMiB != null) { + if ( + typeof tenantCountsByHeapMiB !== 'object' + || Array.isArray(tenantCountsByHeapMiB) + || tenantCountsByHeapMiB === null + ) { + throw new Error('PDCF_PLAN_CUSTOMER_COUNT_MATRIX_INVALID'); + } + const configuredHeaps = new Set(heapMiB.map(String)); + if (Object.keys(tenantCountsByHeapMiB).some((heap) => !configuredHeaps.has(heap))) { + throw new Error('PDCF_PLAN_CUSTOMER_COUNT_MATRIX_INVALID'); + } + } + const byHeap = Object.fromEntries(heapMiB.map((heap) => { + const counts = customerCountsForHeap({ tenantCounts, tenantCountsByHeapMiB }, heap); + return [String(heap), validateCustomerCountRamp(counts, String(heap))]; + })); + return { + byHeap, + all: [...new Set(Object.values(byHeap).flat())].sort((left, right) => left - right), + }; +}; +const RESIDENT_SUBSCRIPTION = ` +subscription PhysicalDensityRealtimeResident { + onRealtimeItemChanged { + event + overflow + realtimeItem { id tenantId physicalDatabaseIdentity payload } + } +} +`; +const REALTIME_PRIME_MUTATION = ` +mutation PhysicalDensityRealtimePrime($payload: String!) { + updateRealtimeItem(input: { id: 1, realtimeItemPatch: { payload: $payload } }) { + realtimeItem { id tenantId physicalDatabaseIdentity payload } + } +} +`; + +const strictIdentifier = (value, label) => { + if (typeof value !== 'string' || !/^[a-z][a-z0-9_]*$/.test(value) || value.length > 40) { + throw new Error(`PDCF_INVALID_IDENTIFIER:${label}`); + } + return value; +}; + +const customerSuffix = (ordinal) => String(ordinal).padStart(4, '0'); + +const makeCustomer = (prefix, ordinal) => { + const suffix = customerSuffix(ordinal); + const rolePrefix = `${prefix}_c${suffix}`; + return Object.freeze({ + id: `physical-customer-${suffix}`, + ordinal, + database: `${prefix}_db_${suffix}`, + // The schema-level canary returns current_database(), so the database name + // is both credential-free and conclusive under wrong-database routing. + physicalIdentity: `${prefix}_db_${suffix}`, + roles: Object.freeze(Object.fromEntries(completeFixture.TENANTS.map((tenant) => [ + tenant.id, + `${rolePrefix}_${tenant.id}`, + ]))), + // LISTEN is shared only by the three exact Graphile generations that + // target this physical database. It never executes GraphQL or reads an + // application schema. + notificationRole: `${rolePrefix}_notify`, + }); +}; + +const makeCustomers = (prefix, count) => { + strictIdentifier(prefix, 'prefix'); + if (!Number.isSafeInteger(count) || count <= 0 || count > 9999) { + throw new Error('PDCF_INVALID_CUSTOMER_COUNT'); + } + const customers = Array.from({ length: count }, (_unused, index) => + makeCustomer(prefix, index + 1) + ); + for (const customer of customers) { + strictIdentifier(customer.database, 'database'); + for (const role of Object.values(customer.roles)) strictIdentifier(role, 'role'); + strictIdentifier(customer.notificationRole, 'notification-role'); + } + return customers; +}; + +const validateProvisionManifest = (manifest) => { + if ( + !manifest + || manifest.version !== 1 + || manifest.fixture !== FIXTURE_ID + || typeof manifest.prefix !== 'string' + || !Array.isArray(manifest.customers) + || manifest.customers.length === 0 + ) { + throw new Error('PDCF_MANIFEST_INVALID'); + } + strictIdentifier(manifest.prefix, 'prefix'); + const ids = new Set(); + const databases = new Set(); + const physicalIdentities = new Set(); + const roles = new Set(); + for (const customer of manifest.customers) { + if ( + typeof customer?.id !== 'string' + || ids.has(customer.id) + || !Number.isSafeInteger(customer.ordinal) + || customer.ordinal <= 0 + || typeof customer.physicalIdentity !== 'string' + || !customer.physicalIdentity + ) { + throw new Error('PDCF_MANIFEST_CUSTOMER_INVALID'); + } + ids.add(customer.id); + strictIdentifier(customer.database, 'database'); + if (databases.has(customer.database)) throw new Error('PDCF_DATABASE_DUPLICATE'); + databases.add(customer.database); + if (customer.physicalIdentity !== customer.database) { + throw new Error(`PDCF_PHYSICAL_IDENTITY_DATABASE_MISMATCH:${customer.id}`); + } + if (physicalIdentities.has(customer.physicalIdentity)) { + throw new Error('PDCF_PHYSICAL_IDENTITY_DUPLICATE'); + } + physicalIdentities.add(customer.physicalIdentity); + for (const tenant of completeFixture.TENANTS) { + const role = strictIdentifier(customer.roles?.[tenant.id], 'role'); + if (roles.has(role)) throw new Error('PDCF_ROLE_DUPLICATE'); + roles.add(role); + } + const notificationRole = strictIdentifier( + customer.notificationRole, + 'notification-role', + ); + if (roles.has(notificationRole)) throw new Error('PDCF_ROLE_DUPLICATE'); + roles.add(notificationRole); + } + return manifest; +}; + +const validateMeasurementProvisionClone = (provisionClone) => { + if ( + !provisionClone + || provisionClone.version !== 1 + || typeof provisionClone.id !== 'string' + || !provisionClone.id.trim() + || provisionClone.purpose !== 'measurement' + || !/^sha256:[a-f0-9]{64}$/i.test(provisionClone.attestationSetSha256 ?? '') + ) { + throw new Error('PDCF_MEASUREMENT_PROVISION_CLONE_REQUIRED'); + } + return provisionClone; +}; + +const validateSecrets = (secrets, manifest) => { + if (!secrets || secrets.version !== 1 || secrets.fixture !== FIXTURE_ID) { + throw new Error('PDCF_SECRETS_INVALID'); + } + const requiredRoles = manifest.customers.flatMap((customer) => Object.values(customer.roles)); + for (const role of requiredRoles) { + const password = secrets.runtimePasswords?.[role]; + if (typeof password !== 'string' || Buffer.byteLength(password) < 24) { + throw new Error(`PDCF_RUNTIME_PASSWORD_INVALID:${role}`); + } + } + for (const customer of manifest.customers) { + const password = secrets.notificationPasswords?.[customer.notificationRole]; + if (typeof password !== 'string' || Buffer.byteLength(password) < 24) { + throw new Error(`PDCF_NOTIFICATION_PASSWORD_INVALID:${customer.notificationRole}`); + } + } + return secrets; +}; + +const readJson = (file) => JSON.parse(fs.readFileSync(path.resolve(file), 'utf8')); + +const readPrivateJson = (file) => { + const absolute = path.resolve(file); + let before; + try { + before = fs.lstatSync(absolute); + } catch { + throw new Error('PDCF_SECRETS_FILE_UNREADABLE'); + } + if (before.isSymbolicLink() || !before.isFile()) { + throw new Error('PDCF_SECRETS_FILE_MUST_BE_REGULAR'); + } + if ((before.mode & 0o777) !== 0o600) { + throw new Error('PDCF_SECRETS_FILE_MODE_MUST_BE_0600'); + } + + let descriptor; + try { + descriptor = fs.openSync( + absolute, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + const opened = fs.fstatSync(descriptor); + if ( + !opened.isFile() + || opened.dev !== before.dev + || opened.ino !== before.ino + || (opened.mode & 0o777) !== 0o600 + ) { + throw new Error('PDCF_SECRETS_FILE_CHANGED_DURING_OPEN'); + } + return JSON.parse(fs.readFileSync(descriptor, 'utf8')); + } catch (error) { + if (error instanceof Error && error.message.startsWith('PDCF_')) throw error; + throw new Error('PDCF_SECRETS_FILE_UNREADABLE'); + } finally { + if (descriptor != null) fs.closeSync(descriptor); + } +}; + +const makeSecretResolver = (rawSecrets, manifest) => { + const secrets = validateSecrets(rawSecrets, manifest); + const runtimePasswords = new Map(Object.entries(secrets.runtimePasswords)); + const notificationPasswords = new Map(Object.entries(secrets.notificationPasswords)); + return Object.freeze({ + runtimePasswordFor(role) { + const value = runtimePasswords.get(role); + if (typeof value !== 'string') throw new Error(`PDCF_RUNTIME_PASSWORD_REQUIRED:${role}`); + return value; + }, + notificationPasswordFor(role) { + const value = notificationPasswords.get(role); + if (typeof value !== 'string') { + throw new Error(`PDCF_NOTIFICATION_PASSWORD_REQUIRED:${role}`); + } + return value; + }, + toJSON() { + return { kind: 'physical-density-secret-resolver', redacted: true }; + }, + }); +}; + +const loadProvision = (manifestFile, secretsFile) => { + const absoluteManifest = path.resolve(manifestFile); + const absoluteSecrets = path.resolve(secretsFile); + const manifestStat = fs.statSync(absoluteManifest); + const secretsStat = fs.lstatSync(absoluteSecrets); + if (manifestStat.dev === secretsStat.dev && manifestStat.ino === secretsStat.ino) { + throw new Error('PDCF_MANIFEST_SECRETS_MUST_BE_DISTINCT'); + } + const manifest = validateProvisionManifest(readJson(absoluteManifest)); + const secretResolver = makeSecretResolver(readPrivateJson(absoluteSecrets), manifest); + return { manifest, secretResolver }; +}; + +const physicalCanary = (customer, otherCustomers) => ({ + name: PHYSICAL_DATABASE_CANARY, + query: 'query PhysicalDatabaseRouting { physicalDatabaseIdentity }', + requiredMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }], + // A one-customer mechanics run has no foreign database name to enumerate, + // but a null identity is still an explicit routing failure. Keep that + // negative oracle for every fleet, then add the concrete foreign identities + // when the density point contains multiple customers. + forbiddenMatches: [ + { + path: '/data/physicalDatabaseIdentity', + value: null, + }, + ...otherCustomers.map((candidate) => ({ + path: '/data/physicalDatabaseIdentity', + value: candidate.physicalIdentity, + })), + ], + invariants: [{ + path: '/data/physicalDatabaseIdentity', + everyEquals: customer.physicalIdentity, + min: 1, + max: 1, + }], +}); + +const statusFor = (statuses, armName, customerId) => { + const status = statuses?.[armName]?.[customerId]; + if (!status || status.physicalDatabase == null) { + throw new Error(`PDCF_STATUS_REQUIRED:${armName}:${customerId}`); + } + return status; +}; + +const runtimePoolContractFingerprintFor = (status, tenantId) => { + const fingerprint = status?.contractEvidence?.runtimePools?.[tenantId]?.fingerprint; + if (!/^pg-contract-evidence:v1:[a-f0-9]{64}$/.test(fingerprint ?? '')) { + throw new Error(`PDCF_RUNTIME_POOL_CONTRACT_EVIDENCE_MISSING:${tenantId}`); + } + return fingerprint; +}; + +const graphileBuildContractFingerprintFor = (status, tenantId) => { + const fingerprint = status?.contractEvidence?.graphileBuilds?.[tenantId]?.fingerprint; + if (!/^graphile-contract-evidence:v1:[a-f0-9]{64}$/.test(fingerprint ?? '')) { + throw new Error(`PDCF_GRAPHILE_BUILD_CONTRACT_EVIDENCE_MISSING:${tenantId}`); + } + return fingerprint; +}; + +const realtimeProbe = (customer, tenant, otherCustomers) => { + const payload = `${customer.physicalIdentity}:${tenant.id}:configured-placeholder`; + const foreignEventMatches = [ + ...completeFixture.TENANTS + .filter((candidate) => candidate.id !== tenant.id) + .map((candidate) => ({ + path: '/data/onRealtimeItemChanged/realtimeItem/tenantId', + value: candidate.token, + })), + ...otherCustomers.map((candidate) => ({ + path: '/data/onRealtimeItemChanged/realtimeItem/physicalDatabaseIdentity', + value: candidate.physicalIdentity, + })), + ]; + return { + subscription: { + query: RESIDENT_SUBSCRIPTION, + requiredMatches: [ + { + path: '/data/onRealtimeItemChanged/realtimeItem/tenantId', + value: tenant.token, + }, + { + path: '/data/onRealtimeItemChanged/realtimeItem/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + ], + forbiddenMatches: foreignEventMatches, + }, + prime: { + query: REALTIME_PRIME_MUTATION, + variables: { payload }, + requiredMatches: [ + { + path: '/data/updateRealtimeItem/realtimeItem/tenantId', + value: tenant.token, + }, + { + path: '/data/updateRealtimeItem/realtimeItem/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + ], + forbiddenMatches: [ + ...completeFixture.TENANTS + .filter((candidate) => candidate.id !== tenant.id) + .map((candidate) => ({ + path: '/data/updateRealtimeItem/realtimeItem/tenantId', + value: candidate.token, + })), + ...otherCustomers.map((candidate) => ({ + path: '/data/updateRealtimeItem/realtimeItem/physicalDatabaseIdentity', + value: candidate.physicalIdentity, + })), + ], + }, + correlation: { + primeVariable: 'payload', + primeResponsePath: '/data/updateRealtimeItem/realtimeItem/payload', + subscriptionEventPath: '/data/onRealtimeItemChanged/realtimeItem/payload', + }, + }; +}; + +const physicalIdentityMatches = (pathValue, customer, otherCustomers) => ({ + requiredMatches: [{ + path: pathValue, + value: customer.physicalIdentity, + }], + forbiddenMatches: [ + { path: pathValue, value: null }, + ...otherCustomers.map((candidate) => ({ + path: pathValue, + value: candidate.physicalIdentity, + })), + ], + invariants: [{ + path: pathValue, + everyEquals: customer.physicalIdentity, + min: 1, + max: 1, + }], +}); + +const exactInvariant = (pathValue, everyEquals, min = 1, max = 1) => ({ + path: pathValue, + everyEquals, + min, + max, +}); + +const physicalOperationsFor = (customer, tenant, otherCustomers) => { + const baseByName = new Map(completeFixture.operationsFor(tenant).map( + (candidate) => [candidate.name, candidate] + )); + const fromBase = (name, overrides) => { + const base = baseByName.get(name); + if (!base) throw new Error(`PDCF_OPERATION_REQUIRED:${name}`); + baseByName.delete(name); + return { ...base, ...overrides }; + }; + const expectedDocumentTitle = `${tenant.token} Machine Learning`; + const documentOracle = ( + extraRequired = [], + extraForbidden = [], + extraInvariants = [] + ) => ({ + ...physicalIdentityMatches( + '/data/documents/nodes/0/physicalDatabaseIdentity', + customer, + otherCustomers + ), + requiredMatches: [ + { + path: '/data/documents/nodes/0/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { path: '/data/documents/nodes/0/tenantId', value: tenant.token }, + { path: '/data/documents/nodes/0/title', value: expectedDocumentTitle }, + ...extraRequired, + ], + forbiddenMatches: [ + ...physicalIdentityMatches( + '/data/documents/nodes/0/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + ...completeFixture.TENANTS.filter((candidate) => candidate.id !== tenant.id) + .map((candidate) => ({ + path: '/data/documents/nodes/0/tenantId', + value: candidate.token, + })), + ...extraForbidden, + ], + invariants: [ + exactInvariant( + '/data/documents/nodes/*/physicalDatabaseIdentity', + customer.physicalIdentity + ), + exactInvariant('/data/documents/nodes/*/tenantId', tenant.token), + exactInvariant('/data/documents/nodes/*/title', expectedDocumentTitle), + ...extraInvariants, + ], + }); + const uploadContentHash = crypto.createHash('sha256') + .update(`${customer.physicalIdentity}:${tenant.id}:upload`) + .digest('hex'); + const operations = [ + fromBase('generated-document-read', { + query: 'query GeneratedDocumentRead { documents(first: 1) { nodes { id tenantId title physicalDatabaseIdentity } } }', + ...documentOracle(), + }), + fromBase('localized-post-read', { + query: 'query LocalizedPostRead { posts(first: 1, where: { id: { equalTo: 1 } }) { nodes { tenantId physicalDatabaseIdentity localeStrings { langCode title body } } } }', + requiredMatches: [ + { + path: '/data/posts/nodes/0/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/posts/nodes/0/localeStrings/title', + value: `${tenant.token} español @${customer.physicalIdentity}`, + }, + ], + forbiddenMatches: [ + ...physicalIdentityMatches( + '/data/posts/nodes/0/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + ...completeFixture.TENANTS.filter((candidate) => candidate.id !== tenant.id) + .map((candidate) => ({ + path: '/data/posts/nodes/0/localeStrings/title', + value: `${candidate.token} español @${customer.physicalIdentity}`, + })), + ], + invariants: [ + exactInvariant( + '/data/posts/nodes/*/physicalDatabaseIdentity', + customer.physicalIdentity + ), + exactInvariant( + '/data/posts/nodes/*/localeStrings/title', + `${tenant.token} español @${customer.physicalIdentity}` + ), + ], + }), + fromBase('deterministic-embed', { + query: 'query DeterministicEmbed { physicalDatabaseIdentity embedText(text: "tenant fixture") { vector dimensions } }', + requiredMatches: [ + { + path: '/data/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { path: '/data/embedText/vector', value: [1, 0, 0] }, + { path: '/data/embedText/dimensions', value: 3 }, + ], + forbiddenMatches: physicalIdentityMatches( + '/data/physicalDatabaseIdentity', customer, otherCustomers + ).forbiddenMatches, + invariants: [ + exactInvariant('/data/physicalDatabaseIdentity', customer.physicalIdentity), + exactInvariant('/data/embedText/vector', [1, 0, 0]), + exactInvariant('/data/embedText/dimensions', 3), + ], + }), + fromBase('deterministic-rag', { + query: 'query DeterministicRag { physicalDatabaseIdentity ragQuery(prompt: "machine learning tenant fixture", contextLimit: 2) { answer tokensUsed sources { content similarity tableName parentId } } }', + requiredMatches: [ + { + path: '/data/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/ragQuery/sources/*/content', + value: `${tenant.token} machine learning tenant fixture context @${customer.physicalIdentity}`, + }, + { + path: '/data/ragQuery/answer', + value: 'Deterministic fixture answer: machine learning tenant fixture', + }, + { path: '/data/ragQuery/tokensUsed', value: 20 }, + { path: '/data/ragQuery/sources/*/tableName', value: 'articles' }, + ], + forbiddenMatches: [ + ...physicalIdentityMatches( + '/data/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + ...otherCustomers.map((candidate) => ({ + path: '/data/ragQuery/sources/*/content', + value: `${tenant.token} machine learning tenant fixture context @${candidate.physicalIdentity}`, + })), + ], + invariants: [ + exactInvariant('/data/physicalDatabaseIdentity', customer.physicalIdentity), + exactInvariant( + '/data/ragQuery/sources/*/content', + `${tenant.token} machine learning tenant fixture context @${customer.physicalIdentity}` + ), + exactInvariant('/data/ragQuery/sources/*/tableName', 'articles'), + exactInvariant( + '/data/ragQuery/answer', + 'Deterministic fixture answer: machine learning tenant fixture' + ), + exactInvariant('/data/ragQuery/tokensUsed', 20), + ], + }), + fromBase('bm25-search', { + query: 'query Bm25Search { documents(where: { bm25Body: { query: "machine learning intelligence" } }) { nodes { tenantId title physicalDatabaseIdentity bodyBm25Score } } }', + ...documentOracle(), + }), + fromBase('tsvector-search', { + query: 'query TsvectorSearch { documents(where: { tsvTsv: "machine learning" }) { nodes { tenantId title physicalDatabaseIdentity tsvRank } } }', + ...documentOracle(), + }), + fromBase('trigram-search', { + query: 'query TrigramSearch { documents(where: { trgmTitle: { value: "Machne Lerning", threshold: 0.05 } }) { nodes { tenantId title physicalDatabaseIdentity titleTrgmSimilarity } } }', + ...documentOracle(), + }), + fromBase('vector-search', { + query: 'query VectorSearch { documents(where: { vectorEmbedding: { vector: [1, 0, 0], metric: COSINE } }) { nodes { tenantId title physicalDatabaseIdentity embeddingVectorDistance } } }', + ...documentOracle( + [{ path: '/data/documents/nodes/0/embeddingVectorDistance', value: 0 }], + [], + [exactInvariant('/data/documents/nodes/*/embeddingVectorDistance', 0)] + ), + }), + fromBase('postgis-read', { + query: 'query PostgisRead { documents(first: 1) { nodes { tenantId title physicalDatabaseIdentity location { geojson } } } }', + ...documentOracle( + [{ + path: '/data/documents/nodes/0/location/geojson', + value: { type: 'Point', coordinates: [106.7, 10.8] }, + }], + [], + [exactInvariant( + '/data/documents/nodes/*/location/geojson', + { type: 'Point', coordinates: [106.7, 10.8] } + )] + ), + }), + fromBase('ltree-filter', { + query: 'query LtreeFilter { documents(where: { path: { within: "/root" } }) { nodes { tenantId title physicalDatabaseIdentity path } } }', + ...documentOracle( + [{ path: '/data/documents/nodes/0/path', value: `/root/${tenant.id}` }], + [], + [exactInvariant('/data/documents/nodes/*/path', `/root/${tenant.id}`)] + ), + }), + fromBase('presigned-upload', { + query: 'mutation PresignedUpload($input: UploadAppFileInput!) { uploadAppFile(input: $input) { fileId key deduplicated expiresAt uploadUrl } physicalDatabaseMutationIdentity(input: {}) { result } }', + variables: { + input: { + bucketKey: 'private', + contentHash: uploadContentHash, + contentType: 'text/plain', + size: 32, + filename: `${customer.id}-${tenant.id}.txt`, + }, + }, + requiredMatches: [{ + path: '/data/physicalDatabaseMutationIdentity/result', + value: customer.physicalIdentity, + }], + forbiddenMatches: [ + { path: '/data/physicalDatabaseMutationIdentity/result', value: null }, + ...otherCustomers.map((candidate) => ({ + path: '/data/physicalDatabaseMutationIdentity/result', + value: candidate.physicalIdentity, + })), + ], + invariants: [exactInvariant( + '/data/physicalDatabaseMutationIdentity/result', + customer.physicalIdentity + )], + postCoverageVerification: { + query: 'query VerifyPresignedUpload($fileId: UUID!, $contentHash: String!) { appFiles(first: 1, where: { id: { equalTo: $fileId }, contentHash: { equalTo: $contentHash } }) { nodes { id tenantId contentHash physicalDatabaseIdentity } } }', + variables: { contentHash: uploadContentHash }, + variablesFromResponse: { + fileId: '/data/uploadAppFile/fileId', + }, + requiredMatches: [ + { + path: '/data/appFiles/nodes/0/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/appFiles/nodes/0/contentHash', + value: uploadContentHash, + }, + { path: '/data/appFiles/nodes/0/tenantId', value: tenant.token }, + ], + forbiddenMatches: physicalIdentityMatches( + '/data/appFiles/nodes/0/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + invariants: [ + exactInvariant( + '/data/appFiles/nodes/*/physicalDatabaseIdentity', + customer.physicalIdentity + ), + exactInvariant('/data/appFiles/nodes/*/tenantId', tenant.token), + exactInvariant('/data/appFiles/nodes/*/contentHash', uploadContentHash), + ], + }, + }), + fromBase('bulk-upsert', { + query: 'mutation BulkUpsert($name: String!) { bulkUpsertBulkItems(input: { values: [{ name: $name, quantity: 1 }], onConflict: { constraint: BULK_ITEMS_NAME_KEY } }) { affectedCount returning { tenantId name physicalDatabaseIdentity } } }', + variables: { name: `${customer.id}-${tenant.token}-bulk` }, + requiredMatches: [ + { + path: '/data/bulkUpsertBulkItems/returning/0/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/bulkUpsertBulkItems/returning/0/name', + value: `${customer.id}-${tenant.token}-bulk`, + }, + { + path: '/data/bulkUpsertBulkItems/returning/0/tenantId', + value: tenant.token, + }, + ], + forbiddenMatches: physicalIdentityMatches( + '/data/bulkUpsertBulkItems/returning/0/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + invariants: [ + exactInvariant( + '/data/bulkUpsertBulkItems/returning/*/physicalDatabaseIdentity', + customer.physicalIdentity + ), + exactInvariant( + '/data/bulkUpsertBulkItems/returning/*/tenantId', + tenant.token + ), + exactInvariant( + '/data/bulkUpsertBulkItems/returning/*/name', + `${customer.id}-${tenant.token}-bulk` + ), + ], + }), + fromBase('realtime-tagged-update', { + query: 'mutation RealtimeTaggedUpdate($payload: String!) { updateRealtimeItem(input: { id: 1, realtimeItemPatch: { payload: $payload } }) { realtimeItem { id tenantId physicalDatabaseIdentity payload } } }', + variables: { payload: `${customer.id}-${tenant.token}-realtime` }, + requiredMatches: [ + { + path: '/data/updateRealtimeItem/realtimeItem/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/updateRealtimeItem/realtimeItem/tenantId', + value: tenant.token, + }, + { + path: '/data/updateRealtimeItem/realtimeItem/payload', + value: `${customer.id}-${tenant.token}-realtime`, + }, + ], + forbiddenMatches: physicalIdentityMatches( + '/data/updateRealtimeItem/realtimeItem/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + }), + fromBase('bound-function-invocation', { + query: 'mutation BoundFunctionInvocation($payload: JSON!) { fixtureTask(input: { payload: $payload }) { invocationId status invocation { tenantId physicalDatabaseIdentity taskIdentifier } } }', + variables: { + payload: { + tenant: tenant.id, + customer: customer.id, + source: 'physical-database-density', + }, + }, + requiredMatches: [ + { + path: '/data/fixtureTask/invocation/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/fixtureTask/invocation/tenantId', + value: tenant.token, + }, + { + path: '/data/fixtureTask/invocation/taskIdentifier', + value: `ctf.fixture.${tenant.id}`, + }, + ], + forbiddenMatches: physicalIdentityMatches( + '/data/fixtureTask/invocation/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + }), + fromBase('security-context-read', { + query: 'query SecurityContextRead { physicalDatabaseIdentity requestIdentity }', + requiredMatches: [ + { + path: '/data/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/requestIdentity', + value: `${tenant.token}:${tenant.databaseId}`, + }, + ], + forbiddenMatches: [ + ...physicalIdentityMatches( + '/data/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + ...completeFixture.TENANTS.filter((candidate) => candidate.id !== tenant.id) + .map((candidate) => ({ + path: '/data/requestIdentity', + value: `${candidate.token}:${tenant.databaseId}`, + })), + ], + }), + ]; + if (baseByName.size > 0) { + throw new Error( + `PDCF_OPERATION_ORACLE_MISSING:${[...baseByName.keys()].sort().join(',')}` + ); + } + return operations; +}; + +const physicalCanariesFor = (customer, tenant, otherCustomers) => + completeFixture.canariesFor(tenant).map((candidate) => { + if (candidate.name !== 'plugin-raw-sql') return candidate; + const expectedTitle = `${tenant.token} español @${customer.physicalIdentity}`; + const databaseCandidates = [customer, ...otherCustomers]; + return { + ...candidate, + requiredMatches: [{ + path: '/data/posts/nodes/0/localeStrings/title', + value: expectedTitle, + }], + forbiddenMatches: [ + { path: '/data/posts/nodes/0/localeStrings/title', value: null }, + ...databaseCandidates.flatMap((databaseCustomer) => + completeFixture.TENANTS + .filter((candidateTenant) => ( + candidateTenant.id !== tenant.id + || databaseCustomer.id !== customer.id + )) + .map((candidateTenant) => ({ + path: '/data/posts/nodes/0/localeStrings/title', + value: `${candidateTenant.token} español @${databaseCustomer.physicalIdentity}`, + })) + ), + ], + invariants: [exactInvariant( + '/data/posts/nodes/*/localeStrings/title', + expectedTitle + )], + }; + }); + +const makeFleet = ({ manifest, statuses, arms = DEFAULT_IDLE_ARMS, port = 3410 }) => ({ + version: 1, + tenants: manifest.customers.map((customer) => { + const otherCustomers = manifest.customers.filter((candidate) => candidate.id !== customer.id); + const firstStatus = statusFor(statuses, arms[0].name, customer.id); + if (firstStatus.physicalDatabase !== customer.database) { + throw new Error(`PDCF_STATUS_DATABASE_MISMATCH:${customer.id}`); + } + return { + id: customer.id, + databases: [{ + id: `logical:${customer.id}`, + physicalDatabase: customer.database, + apis: completeFixture.TENANTS.map((tenant) => ({ + id: `api:${customer.id}:${tenant.id}`, + runtimePoolIdentity: runtimePoolContractFingerprintFor( + firstStatus, + tenant.id, + ), + runtimePoolIdentities: Object.fromEntries(arms.map((arm) => [ + arm.name, + runtimePoolContractFingerprintFor( + statusFor(statuses, arm.name, customer.id), + tenant.id, + ), + ])), + physicalSchemas: [tenant.schema], + routingLabels: [`${customer.id}-${tenant.id}`], + realtime: true, + surfaces: [`api-${tenant.id}`], + })), + }], + surfaces: completeFixture.TENANTS.map((tenant) => ({ + name: `api-${tenant.id}`, + buildContract: graphileBuildContractFingerprintFor(firstStatus, tenant.id), + buildContracts: Object.fromEntries(arms.map((arm) => [ + arm.name, + graphileBuildContractFingerprintFor( + statusFor(statuses, arm.name, customer.id), + tenant.id, + ), + ])), + url: `http://127.0.0.1:{port}/customer/${customer.id}/tenant/${tenant.id}/graphql`, + headers: { 'accept-language': 'es' }, + warmup: { + name: 'warm-physical-database-identity', + capability: 'graphile-generated', + query: 'query WarmPhysicalDatabaseIdentity { physicalDatabaseIdentity }', + ...physicalIdentityMatches( + '/data/physicalDatabaseIdentity', + customer, + otherCustomers + ), + }, + operations: physicalOperationsFor(customer, tenant, otherCustomers), + realtime: realtimeProbe(customer, tenant, otherCustomers), + canaries: [ + ...physicalCanariesFor(customer, tenant, otherCustomers), + physicalCanary(customer, otherCustomers), + ], + })), + }; + }), +}); + +const makeCacheCapacityProofByHeapMiB = ({ + cacheCalibration, + databaseContractFingerprint, + introspectionMode = 'scoped-required', + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + heapLimitBytesByHeapMiB, +}) => { + const countMatrix = validateCustomerCountMatrix({ + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + }); + const calibration = validateCacheCalibration(cacheCalibration, { + databaseContractFingerprint, + introspectionMode, + }); + return Object.fromEntries(heapMiB.map((configuredHeapMiB) => { + const requiredResidentInstances = Math.max( + ...countMatrix.byHeap[String(configuredHeapMiB)] + ) * completeFixture.TENANTS.length; + const heapLimitBytes = heapLimitBytesByHeapMiB?.[String(configuredHeapMiB)]; + if (!Number.isSafeInteger(heapLimitBytes) || heapLimitBytes <= 0) { + throw new Error(`PDCF_HEAP_LIMIT_REQUIRED:${configuredHeapMiB}`); + } + const budgetCapacity = computeCalibratedCapacity( + heapLimitBytes, + calibration.configured, + ); + if (budgetCapacity < requiredResidentInstances) { + throw new Error( + `PDCF_CALIBRATED_CAPACITY_INSUFFICIENT:${configuredHeapMiB}:${budgetCapacity}:${requiredResidentInstances}` + ); + } + return [String(configuredHeapMiB), { + calibrationId: calibration.calibrationId, + expectedHeapLimitBytes: heapLimitBytes, + budgetCapacity, + configuredResidentCapacity: budgetCapacity, + requiredResidentInstances, + residentHeadroomInstances: budgetCapacity - requiredResidentInstances, + admissionMode: QUALIFYING_CACHE_ADMISSION_MODE, + capacityRefusalReason: 'resident_capacity', + capacityResponseCode: 'GRAPHILE_BUILD_RESIDENT_CAPACITY', + preservesExistingResidentsAtCapacity: true, + safetyFactor: calibration.safetyFactor, + measured: calibration.measured, + configured: calibration.configured, + sourceResultSha256: calibration.sources.map((source) => source.sourceSha256), + }]; + })); +}; + +const makePlan = ({ + manifestFile, + secretsFile, + postgresContainer, + commit, + entrySha256, + lockfileSha256, + arms = DEFAULT_IDLE_ARMS, + basePort = 3410, + heapMiB = [1024, 2048, 4096], + tenantCounts, + tenantCountsByHeapMiB, + repetitions = 3, + durationSec = 900, + introspectionMode = 'scoped-required', + databaseContractFingerprint, + blueprintCompatibilityFingerprint, + manifestSha256, + provisionClone, + cacheCalibration, + heapLimitBytesByHeapMiB, + cacheCapacityByHeapMiB, + postgresContainerTemplateFile, + postgresContainerTemplateSha256, +}) => { + if ( + !postgresContainer + || !postgresContainerTemplateFile + || !/^sha256:[a-f0-9]{64}$/.test(postgresContainerTemplateSha256 ?? '') + || !commit + || !entrySha256 + || !lockfileSha256 + ) { + throw new Error('PDCF_PLAN_PROVENANCE_REQUIRED'); + } + validateMeasurementProvisionClone(provisionClone); + if ( + !/^sha256:[a-f0-9]{64}$/.test(databaseContractFingerprint ?? '') + || !/^sha256:[a-f0-9]{64}$/.test(blueprintCompatibilityFingerprint ?? '') + || !/^sha256:[a-f0-9]{64}$/.test(manifestSha256 ?? '') + ) { + throw new Error('PDCF_PLAN_COMPATIBILITY_FINGERPRINT_REQUIRED'); + } + const countMatrix = validateCustomerCountMatrix({ + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + }); + const calibration = validateCacheCalibration(cacheCalibration, { + databaseContractFingerprint, + introspectionMode, + }); + const computedCapacityByHeapMiB = makeCacheCapacityProofByHeapMiB({ + cacheCalibration: calibration, + databaseContractFingerprint, + introspectionMode, + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + heapLimitBytesByHeapMiB, + }); + if ( + cacheCapacityByHeapMiB + && JSON.stringify(cacheCapacityByHeapMiB) !== JSON.stringify(computedCapacityByHeapMiB) + ) { + throw new Error('PDCF_CACHE_CAPACITY_PROOF_MISMATCH'); + } + const calibrationByHeapMiB = cacheCapacityByHeapMiB ?? computedCapacityByHeapMiB; + return { + version: 1, + fleetFile: 'fleet.json', + artifactDir: '../artifacts', + arms: arms.map((arm, index) => { + const port = basePort + index; + const poolIdentitiesPerCustomer = completeFixture.TENANTS.length + + (notificationModeForArm(arm) === 'shared-exact' ? 1 : 0); + return { + name: arm.name, + commit, + cwd: REPO_ROOT, + command: [ + 'node', + '--expose-gc', + path.join(FIXTURE_DIR, 'server.cjs'), + '--manifest', '{postgresManifestFile}', + '--secrets', '{postgresSecretsFile}', + '--customers', '{tenantCount}', + '--host', '127.0.0.1', + '--port', '{port}', + '--arm', arm.name, + '--mode', '{mode}', + '--introspection-client-release-mode', 'destroy', + '--runtime-pool-max', String(runtimePoolMaxForArm(arm)), + '--runtime-pool-max-uses', runtimePoolMaxUsesForArm(arm) == null + ? 'unlimited' + : String(runtimePoolMaxUsesForArm(arm)), + '--realtime-notification-mode', notificationModeForArm(arm), + '--realtime-cursor-poll-ms', String(cursorPollMsForArm(arm)), + '--realtime-cursor-heartbeat-ms', String(cursorHeartbeatMsForArm(arm)), + '--enable-realtime', 'true', + '--expected-database-contract', databaseContractFingerprint, + '--blueprint-compatibility', blueprintCompatibilityFingerprint, + '--expected-manifest-sha256', '{postgresManifestSha256}', + '--run-purpose', 'measurement', + '--clone-id', '{postgresCloneId}', + ], + port, + readinessUrl: 'http://127.0.0.1:{port}/healthz', + memoryUrl: 'http://127.0.0.1:{port}/debug/memory', + retainedHeapCheckpointUrl: + 'http://127.0.0.1:{port}/__cperf/retained-memory-checkpoint', + postWarmupUrl: 'http://127.0.0.1:{port}/__cperf/post-warmup', + postgresContainer, + requirePostgresCgroupV2: true, + postgresRunAttestation: { + command: [ + 'node', + path.join(FIXTURE_DIR, 'measurement-attestation.cjs'), + '--manifest', '{postgresManifestFile}', + '--secrets', '{postgresSecretsFile}', + '--postgres-container', postgresContainer, + '--container-template', postgresContainerTemplateFile, + '--expected-container-template-sha256', + postgresContainerTemplateSha256, + '--arm', '{arm}', + '--heap-mib', '{heapMiB}', + '--customers', '{tenantCount}', + '--repetition', '{repetition}', + '--run-order-index', '{runOrderIndex}', + '--plan-sha256', '{planSha256}', + '--fleet-sha256', '{fleetSha256}', + '--not-before-epoch-ms', '{notBeforeEpochMs}', + '--out', '{attestationFile}', + ], + prepareCommand: [ + 'node', + path.join(FIXTURE_DIR, 'prepare-measurement-run.cjs'), + '--container-template', postgresContainerTemplateFile, + '--expected-container-template-sha256', + postgresContainerTemplateSha256, + '--manifest-template', manifestFile, + '--secrets-template', secretsFile, + '--expected-manifest-template-sha256', manifestSha256, + '--artifact-dir', '{postgresFixtureDir}', + '--arm', '{arm}', + '--heap-mib', '{heapMiB}', + '--customers', '{tenantCount}', + '--repetition', '{repetition}', + '--run-order-index', '{runOrderIndex}', + ], + timeoutMs: 900_000, + }, + introspectionMode, + v8Profile: arm.v8Profile ?? 'stock', + startupTimeoutMs: 600_000, + entrySha256, + lockfileSha256, + env: { + GRAPHILE_BUILD_MAX_CONCURRENCY: '1', + GRAPHILE_BUILD_CONCURRENCY: '1', + GRAPHILE_BUILD_QUEUE_MAX: '64', + // Keep ambient code-loading hooks out of the measured child too; + // the harness adds only the attested heap limit to NODE_OPTIONS. + NODE_OPTIONS: '', + NODE_PATH: '', + // Runtime pools are exact per surface. Shared realtime adds one exact + // notification pool identity per physical customer, even though that + // identity owns only one backend for all of the customer's surfaces. + PG_CACHE_MAX: String( + Math.max(...countMatrix.all) * poolIdentitiesPerCustomer + 8 + ), + PG_POOL_IDLE_TIMEOUT_MS: String(arm.idleTimeoutMs), + // Control and incidental pools must not change shape with the arm. + // Runtime pools receive their capacity through the explicit option. + PG_POOL_MAX: String(PROCESS_GLOBAL_POOL_MAX), + // maxUses is an exact runtime-pool option; ambient, control, and + // notification pools remain reusable in every arm. + PG_POOL_MAX_USES: '0', + ...(preparedStatementCacheSizeForArm(arm) == null ? {} : { + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: + String(preparedStatementCacheSizeForArm(arm)), + }), + }, + envByHeapMiB: Object.fromEntries(Object.entries(calibrationByHeapMiB).map( + ([configuredHeapMiB, proof]) => [configuredHeapMiB, { + GRAPHILE_CACHE_MAX: String(proof.configuredResidentCapacity), + GRAPHILE_CACHE_ADMISSION_MODE: proof.admissionMode, + GRAPHILE_CACHE_INSTANCE_HEAP_BYTES: + String(proof.configured.instanceHeapBytes), + GRAPHILE_CACHE_SERVER_RESERVE_BYTES: + String(proof.configured.serverReserveBytes), + GRAPHILE_CACHE_BUILD_RESERVE_BYTES: + String(proof.configured.buildReserveBytes), + GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES: + String(proof.configured.rssBuildReserveBytes), + GRAPHILE_CACHE_CALIBRATION_ID: proof.calibrationId, + GRAPHQL_CPERF_RETAINED_HEAP_ENABLED: 'true', + }] + )), + cacheCalibrationByHeapMiB: calibrationByHeapMiB, + }; + }), + heapMiB, + ...(tenantCounts == null ? {} : { tenantCounts }), + ...(tenantCountsByHeapMiB == null ? {} : { tenantCountsByHeapMiB }), + repetitions, + runOrderSeed: `${FIXTURE_ID}:${manifestFile}:${calibration.calibrationId}`, + cacheCalibration: calibration, + requiredCapabilities: [...completeFixture.REQUIRED_CAPABILITIES], + requiredCanaries: [ + ...completeFixture.REQUIRED_CANARIES, + PHYSICAL_DATABASE_CANARY, + ], + workload: { + durationSec, + rpsPerTenant: 0.2, + minWorkloadRequestsPerSurface: 10, + requestTimeoutMs: 30_000, + maxInFlight: 64, + canaryIntervalSec: 60, + periodicCanarySchedule: 'rotating-one', + canaryConcurrency: 16, + warmupTimeoutMs: 300_000, + warmupTimeoutPerSurfaceMs: 45_000, + warmupConcurrency: 1, + }, + gates: { + maxErrorRate: 0.005, + maxP99Ms: 150, + maxPostWarmupHeapGrowthMiBPerHour: 5, + minMedianDensityImprovement: 0.15, + minAdditionalTenantsEveryRun: 1, + requireZeroBleed: true, + requireNoPostWarmupEvictions: true, + requireNoPostWarmupBuildRefusals: true, + requireNoPostWarmupBuilds: true, + requirePostgresMemoryTelemetry: true, + requireFreshPostgresRunAttestation: true, + requirePhysicalDatabaseTelemetry: true, + requireConclusiveCanaries: true, + requireCompletePeriodicCanaryCoverage: true, + requireConclusiveOperationOracles: true, + requireExplicitCustomerTopology: true, + requireRetainedMemoryCheckpoints: true, + requiredCacheAdmissionMode: QUALIFYING_CACHE_ADMISSION_MODE, + }, + ...(repetitions >= 3 && [1024, 2048, 4096].every((heap) => heapMiB.includes(heap)) + ? { + qualification: { + baselineArm: arms[0].name, + requiredHeapMiB: [1024, 2048, 4096], + minimumRepetitions: 3, + }, + } + : {}), + }; +}; + +const atomicWriteJson = (file, value, mode = 0o644) => { + const absolute = path.resolve(file); + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + const temporary = `${absolute}.${process.pid}.${crypto.randomBytes(6).toString('hex')}.tmp`; + fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode }); + fs.renameSync(temporary, absolute); + fs.chmodSync(absolute, mode); +}; + +module.exports = { + DEFAULT_IDLE_ARMS, + DENSITY_TUNING_ARMS, + FIXTURE_DIR, + FIXTURE_ID, + PHYSICAL_DATABASE_CANARY, + PROCESS_GLOBAL_POOL_MAX, + QUALIFYING_CACHE_ADMISSION_MODE, + REPO_ROOT, + atomicWriteJson, + loadProvision, + makeSecretResolver, + makeCustomers, + makeFleet, + makeCacheCapacityProofByHeapMiB, + makePlan, + cursorHeartbeatMsForArm, + cursorPollMsForArm, + notificationModeForArm, + preparedStatementCacheSizeForArm, + runtimePoolMaxForArm, + runtimePoolMaxUsesForArm, + strictIdentifier, + validateCustomerCountMatrix, + validateCustomerCountRamp, + validateProvisionManifest, + validateSecrets, +}; diff --git a/research/graphile-density/physical-database-density/lib.test.cjs b/research/graphile-density/physical-database-density/lib.test.cjs new file mode 100644 index 0000000000..b6ee37cc29 --- /dev/null +++ b/research/graphile-density/physical-database-density/lib.test.cjs @@ -0,0 +1,702 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { + DEFAULT_IDLE_ARMS, + DENSITY_TUNING_ARMS, + PHYSICAL_DATABASE_CANARY, + makeCustomers, + makeCacheCapacityProofByHeapMiB, + makeFleet, + makePlan, + validateProvisionManifest, +} = require('./lib.cjs'); +const { + DEFAULT_CANONICAL_SCHEMAS, + normalizeSchemaDump, +} = require('./provision.cjs'); +const { + CALIBRATION_KIND, + sha256Canonical, +} = require('./cache-calibration.cjs'); + +const MIB = 1024 ** 2; +const databaseContractFingerprint = `sha256:${'d'.repeat(64)}`; +const calibrationPayload = { + kind: CALIBRATION_KIND, + databaseContractFingerprint, + introspectionMode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: true, + introspectionBackendRetirementConclusive: true, + fixtureFingerprint: 'fixture-v1', + schemaContract: { + schemaSets: [['ctf_a']], + allowedDependencySchemas: ['ctf_extensions'], + }, + safetyFactor: 1.25, + measured: { + repetitions: 3, + retainedHeapPerSurfaceBytes: 12 * MIB, + serverBaselineHeapBytes: 48 * MIB, + buildTransientHeapBytes: 96 * MIB, + buildTransientRssBytes: 96 * MIB, + }, + configured: { + instanceHeapBytes: 15 * MIB, + serverReserveBytes: 60 * MIB, + buildReserveBytes: 120 * MIB, + rssBuildReserveBytes: 120 * MIB, + }, + sourceWorktreesClean: true, + sources: ['1', '2', '3'].map((value) => ({ + sourceSha256: `sha256:${value.repeat(64)}`, + sourceStateSha256: value.repeat(64), + executedEntrySha256: value.repeat(64), + worktreeDirty: false, + introspectionBackendRetirement: { + conclusive: true, + introspectionBackendPid: Number(value), + steadyBackendPid: Number(value) + 10, + }, + })), +}; +const cacheCalibration = { + version: 2, + ...calibrationPayload, + calibrationId: sha256Canonical(calibrationPayload), +}; +const heapLimitBytesByHeapMiB = { + '1024': 1024 * MIB, + '2048': 2048 * MIB, + '4096': 4096 * MIB, +}; + +const manifest = (count = 2) => ({ + version: 1, + fixture: 'physical-database-density-v1', + prefix: 'pdc_test', + provisionClone: { + version: 1, + id: 'measurement-clone-test', + purpose: 'measurement', + attestationSetSha256: `sha256:${'9'.repeat(64)}`, + }, + customers: makeCustomers('pdc_test', count), +}); + +const statuses = (value) => Object.fromEntries(DEFAULT_IDLE_ARMS.map((arm) => [ + arm.name, + Object.fromEntries(value.customers.map((customer) => [customer.id, { + physicalDatabase: customer.database, + runtimePoolIdentities: { + a: `pg:v1:${arm.name}:${customer.id}:a`, + b: `pg:v1:${arm.name}:${customer.id}:b`, + c: `pg:v1:${arm.name}:${customer.id}:c`, + }, + buildContracts: { + a: `graphile:v1:${arm.name}:${customer.id}:a`, + b: `graphile:v1:${arm.name}:${customer.id}:b`, + c: `graphile:v1:${arm.name}:${customer.id}:c`, + }, + contractEvidence: { + runtimePools: Object.fromEntries(['a', 'b', 'c'].map((tenantId) => [ + tenantId, + { + fingerprint: `pg-contract-evidence:v1:${sha256Canonical({ + kind: 'pool', + arm: arm.name, + customer: customer.id, + tenantId, + }).slice('sha256:'.length)}`, + }, + ])), + graphileBuilds: Object.fromEntries(['a', 'b', 'c'].map((tenantId) => [ + tenantId, + { + fingerprint: `graphile-contract-evidence:v1:${sha256Canonical({ + kind: 'build', + arm: arm.name, + customer: customer.id, + tenantId, + }).slice('sha256:'.length)}`, + }, + ])), + }, + }])), +])); + +describe('physical database density fixture', () => { + it('stamps realtime physical identity inside PostgreSQL before every write', () => { + const sql = fs.readFileSync(path.join(__dirname, 'physical-identity.sql'), 'utf8'); + for (const schema of ['ctf_a', 'ctf_b', 'ctf_c']) { + assert.match( + sql, + new RegExp(`ALTER TABLE ${schema}\\.realtime_items[\\s\\S]*ADD COLUMN physical_database_identity text`) + ); + assert.match( + sql, + new RegExp(`BEFORE INSERT OR UPDATE ON ${schema}\\.realtime_items`) + ); + assert.match( + sql, + new RegExp(`CREATE FUNCTION ${schema}\\.stamp_realtime_physical_database_identity\\(\\)[\\s\\S]*NEW\\.physical_database_identity := pg_catalog\\.current_database\\(\\)::text`) + ); + } + for (const table of [ + 'documents', + 'posts', + 'posts_translations', + 'articles', + 'articles_chunks', + 'bulk_items', + 'app_files', + 'function_invocations', + ]) { + assert.match(sql, new RegExp(`'${table}'`)); + } + assert.match( + sql, + /NEW\.physical_database_identity := pg_catalog\.current_database\(\)::text/ + ); + assert.equal( + (sql.match(/physical_database_mutation_identity\(\)[\s\S]*?LANGUAGE sql[\s\S]*?VOLATILE/g) ?? []).length, + 3, + ); + assert.match( + sql, + /GRANT EXECUTE ON FUNCTION ctf_a\.physical_database_mutation_identity\(\) TO :"runtime_role_a"/, + ); + assert.match(sql, /posts_translations SET title = title \|\|/); + assert.match(sql, /articles_chunks SET content = content \|\|/); + }); + + it('generates distinct databases and cluster-wide least-privilege role identities', () => { + const customers = makeCustomers('pdc_test', 2); + assert.deepEqual(customers.map((customer) => customer.database), [ + 'pdc_test_db_0001', + 'pdc_test_db_0002', + ]); + assert.equal(new Set(customers.flatMap((customer) => Object.values(customer.roles))).size, 6); + assert.equal(new Set(customers.map((customer) => customer.notificationRole)).size, 2); + assert.ok(customers.every((customer) => + !Object.values(customer.roles).includes(customer.notificationRole) + )); + assert.equal(customers[0].physicalIdentity, customers[0].database); + assert.equal(validateProvisionManifest(manifest()).customers.length, 2); + }); + + it('binds every unique physical identity to its exact database label', () => { + const swapped = manifest(); + swapped.customers[0] = { + ...swapped.customers[0], + physicalIdentity: swapped.customers[1].database, + }; + assert.throws( + () => validateProvisionManifest(swapped), + /PDCF_PHYSICAL_IDENTITY_DATABASE_MISMATCH/, + ); + + const duplicate = manifest(); + duplicate.customers[1] = { + ...duplicate.customers[1], + database: duplicate.customers[0].database, + physicalIdentity: duplicate.customers[0].physicalIdentity, + }; + assert.throws( + () => validateProvisionManifest(duplicate), + /PDCF_DATABASE_DUPLICATE|PDCF_PHYSICAL_IDENTITY_DUPLICATE/, + ); + }); + + it('maps one complete customer to one physical database and three resident realtime APIs', () => { + const provision = manifest(); + const armStatuses = statuses(provision); + const fleet = makeFleet({ manifest: provision, statuses: armStatuses }); + assert.equal(fleet.tenants.length, 2); + assert.equal(fleet.tenants[0].databases.length, 1); + assert.equal(fleet.tenants[0].databases[0].physicalDatabase, 'pdc_test_db_0001'); + assert.equal(fleet.tenants[0].databases[0].apis.length, 3); + assert.ok(fleet.tenants[0].databases[0].apis.every((api) => api.realtime)); + assert.equal(fleet.tenants[0].surfaces.length, 3); + assert.ok(fleet.tenants[0].surfaces.every((surface) => + surface.realtime?.subscription?.query.includes('PhysicalDensityRealtimeResident') + && surface.realtime?.prime?.query.includes('PhysicalDensityRealtimePrime') + && surface.realtime.subscription.requiredMatches.length === 2 + && surface.realtime.subscription.forbiddenMatches.length === 3 + && surface.realtime.correlation.primeVariable === 'payload' + && surface.realtime.correlation.primeResponsePath + === '/data/updateRealtimeItem/realtimeItem/payload' + && surface.realtime.correlation.subscriptionEventPath + === '/data/onRealtimeItemChanged/realtimeItem/payload' + )); + assert.ok(fleet.tenants[0].surfaces.every((surface) => + surface.warmup.requiredMatches.some((match) => + match.value === 'pdc_test_db_0001' + ) + && surface.operations.every((operation) => { + const oracle = operation.postCoverageVerification ?? operation; + return oracle.requiredMatches.some((match) => + match.value === 'pdc_test_db_0001' + ) && oracle.forbiddenMatches.some((match) => + match.value === 'pdc_test_db_0002' + ); + }) + )); + const operations = new Map( + fleet.tenants[0].surfaces[0].operations.map((operation) => [operation.name, operation]) + ); + assert.match( + operations.get('deterministic-rag').query, + /physicalDatabaseIdentity.*ragQuery/ + ); + assert.ok( + operations.get('deterministic-rag').requiredMatches.some((match) => + match.path.includes('/sources/') + && match.value.endsWith('@pdc_test_db_0001') + ) + ); + assert.match(operations.get('bulk-upsert').query, /returning \{ tenantId name physicalDatabaseIdentity \}/); + assert.match(operations.get('realtime-tagged-update').query, /physicalDatabaseIdentity/); + assert.match(operations.get('bound-function-invocation').query, /invocation \{ tenantId physicalDatabaseIdentity/); + assert.match( + operations.get('presigned-upload').postCoverageVerification.query, + /appFiles.*physicalDatabaseIdentity/ + ); + assert.match( + operations.get('presigned-upload').query, + /physicalDatabaseMutationIdentity\(input: \{\}\) \{ result \}/, + ); + assert.deepEqual( + operations.get('presigned-upload').postCoverageVerification.variablesFromResponse, + { fileId: '/data/uploadAppFile/fileId' }, + ); + assert.match( + operations.get('presigned-upload').postCoverageVerification.query, + /id: \{ equalTo: \$fileId \}/, + ); + for (const operationName of [ + 'generated-document-read', + 'bm25-search', + 'tsvector-search', + 'trigram-search', + 'vector-search', + 'postgis-read', + 'ltree-filter', + ]) { + assert.ok( + operations.get(operationName).invariants.some((invariant) => + invariant.path === '/data/documents/nodes/*/physicalDatabaseIdentity' + && invariant.everyEquals === 'pdc_test_db_0001' + && invariant.min === 1 + && invariant.max === 1 + ), + ); + } + assert.ok(operations.get('deterministic-embed').requiredMatches.some((match) => + match.path === '/data/embedText/vector' + && JSON.stringify(match.value) === '[1,0,0]' + )); + assert.ok(operations.get('deterministic-rag').requiredMatches.some((match) => + match.path === '/data/ragQuery/answer' + && match.value === 'Deterministic fixture answer: machine learning tenant fixture' + )); + assert.ok(operations.get('postgis-read').requiredMatches.some((match) => + match.path.endsWith('/location/geojson') + && match.value.type === 'Point' + )); + assert.ok(operations.get('ltree-filter').requiredMatches.some((match) => + match.path.endsWith('/path') && match.value === '/root/a' + )); + const rawSqlCanary = fleet.tenants[0].surfaces[0].canaries.find( + (candidate) => candidate.name === 'plugin-raw-sql' + ); + assert.equal( + rawSqlCanary.requiredMatches[0].value, + 'tenant-a-canary español @pdc_test_db_0001', + ); + assert.deepEqual( + fleet.tenants[0].surfaces[0].realtime.subscription.requiredMatches, + [ + { + path: '/data/onRealtimeItemChanged/realtimeItem/tenantId', + value: 'tenant-a-canary', + }, + { + path: '/data/onRealtimeItemChanged/realtimeItem/physicalDatabaseIdentity', + value: 'pdc_test_db_0001', + }, + ] + ); + assert.ok( + fleet.tenants[0].surfaces[0].realtime.subscription.forbiddenMatches.some( + (match) => match.path.endsWith('/physicalDatabaseIdentity') + && match.value === 'pdc_test_db_0002' + ) + ); + const canary = fleet.tenants[0].surfaces[0].canaries.find( + (candidate) => candidate.name === PHYSICAL_DATABASE_CANARY + ); + assert.deepEqual(canary.requiredMatches, [{ + path: '/data/physicalDatabaseIdentity', + value: 'pdc_test_db_0001', + }]); + assert.deepEqual(canary.forbiddenMatches, [ + { + path: '/data/physicalDatabaseIdentity', + value: null, + }, + { + path: '/data/physicalDatabaseIdentity', + value: 'pdc_test_db_0002', + }, + ]); + assert.equal( + fleet.tenants[0].surfaces[0].buildContracts[DEFAULT_IDLE_ARMS[2].name], + armStatuses[DEFAULT_IDLE_ARMS[2].name]['physical-customer-0001'] + .contractEvidence.graphileBuilds.a.fingerprint + ); + }); + + it('keeps a conclusive negative routing oracle for a one-customer smoke', () => { + const provision = manifest(); + provision.customers = provision.customers.slice(0, 1); + const fleet = makeFleet({ manifest: provision, statuses: statuses(provision) }); + const canary = fleet.tenants[0].surfaces[0].canaries.find( + (candidate) => candidate.name === PHYSICAL_DATABASE_CANARY + ); + assert.deepEqual(canary.forbiddenMatches, [{ + path: '/data/physicalDatabaseIdentity', + value: null, + }]); + }); + + it('emits explicit 30s, 5s, and 1s arms with a post-warmup realtime hook', () => { + const plan = makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: manifest().provisionClone, + tenantCounts: [2], + cacheCalibration, + heapLimitBytesByHeapMiB, + }); + assert.deepEqual( + plan.arms.map((arm) => arm.env.PG_POOL_IDLE_TIMEOUT_MS), + ['30000', '5000', '1000'] + ); + assert.ok(plan.arms.every((arm) => arm.requirePostgresCgroupV2)); + assert.ok(plan.arms.every((arm) => arm.v8Profile === 'stock')); + assert.ok(plan.arms.every((arm) => arm.postWarmupUrl.endsWith('/__cperf/post-warmup'))); + assert.ok(plan.arms.every((arm) => + arm.retainedHeapCheckpointUrl + .endsWith('/__cperf/retained-memory-checkpoint') + )); + assert.ok(plan.arms.every((arm) => arm.command.includes('--expose-gc'))); + assert.ok(plan.arms.every((arm) => arm.command.includes('{tenantCount}'))); + assert.ok(plan.arms.every((arm) => { + const index = arm.command.indexOf('--introspection-client-release-mode'); + return index >= 0 && arm.command[index + 1] === 'destroy'; + })); + assert.ok(plan.arms.every((arm) => arm.command.includes(`sha256:${'d'.repeat(64)}`))); + assert.ok(plan.arms.every((arm) => { + const purposeIndex = arm.command.indexOf('--run-purpose'); + const cloneIndex = arm.command.indexOf('--clone-id'); + return purposeIndex >= 0 + && arm.command[purposeIndex + 1] === 'measurement' + && cloneIndex >= 0 + && arm.command[cloneIndex + 1] === '{postgresCloneId}'; + })); + assert.ok(plan.arms.every((arm) => + arm.command.includes('{postgresManifestFile}') + && arm.command.includes('{postgresSecretsFile}') + && arm.command.includes('{postgresManifestSha256}') + && arm.postgresRunAttestation.prepareCommand + .includes('{postgresFixtureDir}') + )); + assert.ok(plan.arms.every((arm) => + arm.envByHeapMiB['1024'].GRAPHILE_CACHE_CALIBRATION_ID + === cacheCalibration.calibrationId + )); + assert.ok(plan.arms.every((arm) => + Number(arm.envByHeapMiB['1024'].GRAPHILE_CACHE_MAX) + === arm.cacheCalibrationByHeapMiB['1024'].budgetCapacity + )); + assert.ok(plan.arms.every((arm) => + arm.envByHeapMiB['1024'].GRAPHILE_CACHE_ADMISSION_MODE === 'preserve-resident' + )); + assert.ok(plan.arms.every((arm) => + arm.envByHeapMiB['1024'].GRAPHQL_CPERF_RETAINED_HEAP_ENABLED === 'true' + )); + assert.equal(plan.gates.requireRetainedMemoryCheckpoints, true); + assert.equal(plan.gates.requireConclusiveOperationOracles, true); + assert.equal(plan.gates.requiredCacheAdmissionMode, 'preserve-resident'); + assert.equal(plan.gates.requireCompletePeriodicCanaryCoverage, true); + assert.equal(plan.workload.periodicCanarySchedule, 'rotating-one'); + assert.equal(plan.workload.canaryConcurrency, 16); + assert.equal( + Math.max( + 0, + Math.ceil(plan.workload.durationSec / plan.workload.canaryIntervalSec) - 1, + ), + 14, + ); + assert.equal(plan.requiredCanaries.length, 14); + const capacityProof = plan.arms[0].cacheCalibrationByHeapMiB['1024']; + assert.equal(capacityProof.requiredResidentInstances, 6); + assert.equal(capacityProof.configuredResidentCapacity, capacityProof.budgetCapacity); + assert.equal( + capacityProof.residentHeadroomInstances, + capacityProof.budgetCapacity - capacityProof.requiredResidentInstances, + ); + assert.equal(capacityProof.capacityRefusalReason, 'resident_capacity'); + assert.equal(capacityProof.capacityResponseCode, 'GRAPHILE_BUILD_RESIDENT_CAPACITY'); + assert.equal(capacityProof.preservesExistingResidentsAtCapacity, true); + const oversizedPayload = { + ...calibrationPayload, + measured: { + ...calibrationPayload.measured, + buildTransientHeapBytes: 720 * MIB, + }, + configured: { + ...calibrationPayload.configured, + buildReserveBytes: 900 * MIB, + }, + }; + const rampPlan = makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: manifest().provisionClone, + tenantCounts: [1, 2], + cacheCalibration, + heapLimitBytesByHeapMiB, + }); + assert.deepEqual(rampPlan.tenantCounts, [1, 2]); + assert.equal( + rampPlan.arms[0].cacheCalibrationByHeapMiB['1024'].requiredResidentInstances, + 6, + ); + const perHeapPlan = makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: manifest().provisionClone, + tenantCountsByHeapMiB: { + '1024': [1], + '2048': [1, 2], + }, + heapMiB: [1024, 2048], + cacheCalibration, + heapLimitBytesByHeapMiB, + }); + assert.deepEqual(perHeapPlan.tenantCountsByHeapMiB, { + '1024': [1], + '2048': [1, 2], + }); + assert.equal( + perHeapPlan.arms[0].cacheCalibrationByHeapMiB['1024'].requiredResidentInstances, + 3, + ); + assert.equal( + perHeapPlan.arms[0].cacheCalibrationByHeapMiB['2048'].requiredResidentInstances, + 6, + ); + assert.throws(() => makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: manifest().provisionClone, + tenantCounts: [2, 1], + cacheCalibration, + heapLimitBytesByHeapMiB, + }), /CUSTOMER_COUNT_RAMP_INVALID/); + assert.throws(() => makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: manifest().provisionClone, + tenantCounts: [2], + heapMiB: [1024], + cacheCalibration: { + version: 2, + ...oversizedPayload, + calibrationId: sha256Canonical(oversizedPayload), + }, + heapLimitBytesByHeapMiB: { '1024': 1024 * MIB }, + }), /CALIBRATED_CAPACITY_INSUFFICIENT/); + assert.throws(() => makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: { + ...manifest().provisionClone, + purpose: 'hostile-preflight', + }, + tenantCounts: [2], + cacheCalibration, + heapLimitBytesByHeapMiB, + }), /MEASUREMENT_PROVISION_CLONE_REQUIRED/); + }); + + it('emits secure shared-listener density arms with one-client runtime pools', () => { + const plan = makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: manifest().provisionClone, + tenantCounts: [2], + cacheCalibration, + heapLimitBytesByHeapMiB, + arms: DENSITY_TUNING_ARMS, + }); + const shared = plan.arms.filter((arm) => arm.name.includes('-shared-')); + assert.equal(shared.length, 7); + for (const arm of shared) { + const poolIndex = arm.command.indexOf('--runtime-pool-max'); + const maxUsesIndex = arm.command.indexOf('--runtime-pool-max-uses'); + const modeIndex = arm.command.indexOf('--realtime-notification-mode'); + const pollIndex = arm.command.indexOf('--realtime-cursor-poll-ms'); + assert.equal(arm.command[poolIndex + 1], '1'); + assert.equal( + arm.command[maxUsesIndex + 1], + arm.name.includes('-maxuses-1') ? '1' : 'unlimited', + ); + assert.equal(arm.env.PG_POOL_MAX_USES, '0'); + assert.equal(arm.env.PG_POOL_MAX, '1'); + assert.equal(arm.command[modeIndex + 1], 'shared-exact'); + assert.equal(arm.command[pollIndex + 1], '30000'); + assert.equal(arm.env.PG_CACHE_MAX, '16'); + } + assert.ok(plan.arms.every((arm) => arm.env.PG_POOL_MAX === '1')); + assert.ok(plan.arms.every((arm) => arm.env.NODE_OPTIONS === '')); + assert.ok(plan.arms.every((arm) => arm.env.NODE_PATH === '')); + assert.equal(plan.arms[0].env.PG_CACHE_MAX, '14'); + assert.deepEqual(plan.arms.map((arm) => arm.v8Profile), [ + 'stock', + 'stock', + 'stock', + 'stock', + 'optimize-for-size', + 'baseline-optimize-for-size', + 'jitless-optimize-for-size', + 'optimize-for-size', + ]); + assert.equal( + plan.arms.find((arm) => arm.name === 'physical-db-shared-stock') + .env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE, + '100', + ); + assert.equal( + plan.arms.find((arm) => arm.name === 'physical-db-shared-no-prepare') + .env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE, + '0', + ); + }); + + it('computes the qualifying capacity proof without runtime status inputs', () => { + const proof = makeCacheCapacityProofByHeapMiB({ + cacheCalibration, + databaseContractFingerprint, + tenantCounts: [2], + heapMiB: [1024], + heapLimitBytesByHeapMiB: { '1024': 1024 * MIB }, + })['1024']; + assert.equal(proof.admissionMode, 'preserve-resident'); + assert.equal(proof.requiredResidentInstances, 6); + assert.ok(proof.budgetCapacity > proof.requiredResidentInstances); + }); + + it('normalizes nondeterministic pg_dump guard and version lines before fingerprinting', () => { + const left = normalizeSchemaDump([ + '-- Dumped from database version 17.1', + '-- Dumped by pg_dump version 17.1', + '\\restrict random-left', + 'CREATE TABLE ctf_a.example(id integer);', + '\\unrestrict random-left', + ].join('\n')); + const right = normalizeSchemaDump([ + '-- Dumped from database version 17.2', + '-- Dumped by pg_dump version 17.2', + '\\restrict random-right', + 'CREATE TABLE ctf_a.example(id integer);', + '\\unrestrict random-right', + ].join('\n')); + assert.equal(left, right); + }); + + it('fingerprints build-visible dependency schemas and normalizes equivalent role ACLs', () => { + assert.ok(DEFAULT_CANONICAL_SCHEMAS.includes('ctf_extensions')); + assert.ok(DEFAULT_CANONICAL_SCHEMAS.includes('jwt_private')); + const left = normalizeSchemaDump( + 'GRANT SELECT ON TABLE ctf_a.item TO pdc_test_c0001_a;\n', + { pdc_test_c0001_a: '__runtime_a__' }, + ); + const right = normalizeSchemaDump( + 'GRANT SELECT ON TABLE ctf_a.item TO pdc_test_c0002_a;\n', + { pdc_test_c0002_a: '__runtime_a__' }, + ); + assert.equal(left, right); + assert.match(left, /__runtime_a__/); + }); +}); diff --git a/research/graphile-density/physical-database-density/measurement-attestation.cjs b/research/graphile-density/physical-database-density/measurement-attestation.cjs new file mode 100644 index 0000000000..d4a013e57d --- /dev/null +++ b/research/graphile-density/physical-database-density/measurement-attestation.cjs @@ -0,0 +1,558 @@ +'use strict'; + +const { execFileSync, spawnSync } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + FIXTURE_ID, + loadProvision, +} = require('./lib.cjs'); +const { + parseArgs, + parsePositiveInteger, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); +const { + inspectCustomerContract, + provisionAttestationSetSha256, +} = require('./provision.cjs'); +const { + buildLiveCloneAuditSql, + validateLiveCloneAudit, +} = require('./unsafe-runtime-startup-probe.cjs'); +const { + postgresSettingsFromCommand, + validateContainerTemplate, + validateRunningContainerAgainstTemplate, +} = require('./prepare-measurement-run.cjs'); + +const ATTESTATION_KIND = 'physical-density-measurement-attestation-v1'; +const SHA256 = /^sha256:[a-f0-9]{64}$/; +const CONTAINER_ID = /^[a-f0-9]{64}$/; +const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost']); +const START_TOLERANCE_MS = 0; + +const canonicalize = (value) => { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [ + key, + canonicalize(value[key]), + ])); +}; + +const canonicalSha256 = (value) => `sha256:${crypto.createHash('sha256') + .update(JSON.stringify(canonicalize(value))) + .digest('hex')}`; + +const readRegularFile = (file) => { + const absolute = path.resolve(file); + const before = fs.lstatSync(absolute); + if (before.isSymbolicLink() || !before.isFile()) { + throw new Error('PDCF_MEASUREMENT_EVIDENCE_FILE_INVALID'); + } + const descriptor = fs.openSync( + absolute, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + try { + const opened = fs.fstatSync(descriptor); + if ( + !opened.isFile() + || opened.dev !== before.dev + || opened.ino !== before.ino + ) { + throw new Error('PDCF_MEASUREMENT_EVIDENCE_FILE_INVALID'); + } + return fs.readFileSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +}; + +const bufferSha256 = (value) => `sha256:${crypto.createHash('sha256') + .update(value) + .digest('hex')}`; + +const writeImmutableJson = (file, value) => { + const absolute = path.resolve(file); + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + const temporary = `${absolute}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`; + try { + fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { + flag: 'wx', + mode: 0o644, + }); + fs.linkSync(temporary, absolute); + } finally { + try { fs.unlinkSync(temporary); } catch { /* Preserve the primary error. */ } + } +}; + +const exactCanonical = (left, right) => + JSON.stringify(canonicalize(left)) === JSON.stringify(canonicalize(right)); + +const runPsqlJson = ({ database, sql, environment = process.env }) => { + const result = spawnSync('psql', [ + '--no-psqlrc', + '--no-align', + '--tuples-only', + '--quiet', + '--set=ON_ERROR_STOP=1', + '--dbname', database, + ], { + cwd: __dirname, + env: environment, + encoding: 'utf8', + input: `${sql}\n`, + maxBuffer: 16 * 1024 * 1024, + timeout: 120_000, + }); + if (result.status !== 0) throw new Error('PDCF_MEASUREMENT_ATTESTATION_SQL_FAILED'); + const output = String(result.stdout ?? '').trim(); + if (!output || output.includes('\n')) { + throw new Error('PDCF_MEASUREMENT_ATTESTATION_SQL_RESULT_INVALID'); + } + try { + return JSON.parse(output); + } catch { + throw new Error('PDCF_MEASUREMENT_ATTESTATION_SQL_RESULT_INVALID'); + } +}; + +const inspectDockerContainer = (container) => { + const output = execFileSync('docker', ['inspect', container], { + encoding: 'utf8', + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }); + const records = JSON.parse(output); + if (!Array.isArray(records) || records.length !== 1) { + throw new Error('PDCF_MEASUREMENT_CONTAINER_INSPECT_INVALID'); + } + return records[0]; +}; + +const inspectContainerCgroup = (container) => { + const script = [ + 'set -eu', + 'test -r /sys/fs/cgroup/memory.current', + 'test -r /sys/fs/cgroup/memory.events', + 'printf "membership="', + 'cat /proc/1/cgroup', + 'printf "mount="', + 'stat -c "%d:%i" /sys/fs/cgroup', + ].join('\n'); + const dockerEnvironment = { + PATH: process.env.PATH ?? '/usr/bin:/bin', + ...(process.env.DOCKER_HOST ? { DOCKER_HOST: process.env.DOCKER_HOST } : {}), + ...(process.env.DOCKER_CONTEXT + ? { DOCKER_CONTEXT: process.env.DOCKER_CONTEXT } + : {}), + }; + const output = execFileSync('docker', [ + 'exec', + container, + '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin', + 'sh', '-ceu', script, + ], { + encoding: 'utf8', + timeout: 30_000, + env: dockerEnvironment, + }); + return { + version: 1, + source: 'container-cgroup-v2', + identitySha256: canonicalSha256(output.trim()), + }; +}; + +const clusterAuditSql = (settingNames = []) => ` +SELECT pg_catalog.jsonb_build_object( + 'systemIdentifier', system_identifier::text, + 'postmasterStartedAt', pg_catalog.pg_postmaster_start_time(), + 'serverVersionNum', pg_catalog.current_setting('server_version_num'), + 'databases', ( + SELECT pg_catalog.jsonb_agg(datname ORDER BY datname) + FROM pg_catalog.pg_database + WHERE NOT datistemplate + ), + 'settings', ( + SELECT pg_catalog.jsonb_object_agg( + name, + pg_catalog.jsonb_build_object('setting', setting, 'source', source) + ORDER BY name + ) + FROM pg_catalog.pg_settings + WHERE name = ANY(ARRAY[${settingNames.map((name) => `'${name}'`).join(', ')}]::text[]) + ) +)::text +FROM pg_catalog.pg_control_system(); +`; + +const validateContainer = ({ + inspection, + container, + environment, + notBeforeEpochMs, +}) => { + const id = inspection?.Id; + const name = String(inspection?.Name ?? '').replace(/^\//, ''); + const startedAt = Date.parse(inspection?.State?.StartedAt ?? ''); + const createdAt = Date.parse(inspection?.Created ?? ''); + if ( + !CONTAINER_ID.test(id ?? '') + || name !== container + || inspection?.State?.Running !== true + || !Number.isSafeInteger(startedAt) + || !Number.isSafeInteger(createdAt) + || startedAt < createdAt + ) { + throw new Error('PDCF_MEASUREMENT_CONTAINER_IDENTITY_INVALID'); + } + const pgHost = environment.PGHOST ?? 'localhost'; + const pgPort = String(environment.PGPORT ?? '5432'); + if (!LOOPBACK_HOSTS.has(pgHost)) { + throw new Error('PDCF_MEASUREMENT_POSTGRES_LOOPBACK_REQUIRED'); + } + const bindings = inspection?.NetworkSettings?.Ports?.['5432/tcp']; + if ( + !Array.isArray(bindings) + || !bindings.some((binding) => String(binding?.HostPort) === pgPort) + ) { + throw new Error('PDCF_MEASUREMENT_CONTAINER_PORT_MISMATCH'); + } + return { + id, + name, + imageId: inspection.Image, + createdAt: new Date(createdAt).toISOString(), + startedAt: new Date(startedAt).toISOString(), + freshForRun: startedAt >= notBeforeEpochMs, + }; +}; + +const validateRunBinding = (run) => { + if ( + typeof run?.arm !== 'string' + || !run.arm + || !Number.isSafeInteger(run.heapMiB) + || run.heapMiB <= 0 + || !Number.isSafeInteger(run.customerCount) + || run.customerCount <= 0 + || !Number.isSafeInteger(run.repetition) + || run.repetition <= 0 + || !Number.isSafeInteger(run.runOrderIndex) + || run.runOrderIndex <= 0 + || !SHA256.test(run.planSha256 ?? '') + || !SHA256.test(run.fleetSha256 ?? '') + ) { + throw new Error('PDCF_MEASUREMENT_RUN_BINDING_INVALID'); + } + return run; +}; + +const validateMeasurementAttestation = (attestation, expected = {}) => { + if ( + attestation?.version !== 1 + || attestation.kind !== ATTESTATION_KIND + || !attestation.payload + || !SHA256.test(attestation.payloadSha256 ?? '') + || canonicalSha256(attestation.payload) !== attestation.payloadSha256 + || !SHA256.test(attestation.payload.epochId ?? '') + || typeof attestation.payload.freshness?.freshContainerForRun !== 'boolean' + || attestation.payload.freshness?.cgroupV2Verified !== true + || attestation.payload.catalogCacheState !== 'warmed-by-live-contract-audit' + ) { + throw new Error('PDCF_MEASUREMENT_ATTESTATION_INVALID'); + } + validateRunBinding(attestation.payload.run); + for (const [key, value] of Object.entries(expected)) { + if (!exactCanonical(attestation.payload[key], value)) { + throw new Error(`PDCF_MEASUREMENT_ATTESTATION_MISMATCH:${key}`); + } + } + return attestation; +}; + +const attestMeasurementRun = ({ + manifestFile, + secretsFile, + postgresContainer, + containerTemplateFile, + expectedContainerTemplateSha256, + run, + notBeforeEpochMs, + outputFile, + environment = process.env, +}, dependencies = {}) => { + validateRunBinding(run); + if (!Number.isSafeInteger(notBeforeEpochMs) || notBeforeEpochMs <= 0) { + throw new Error('PDCF_MEASUREMENT_NOT_BEFORE_INVALID'); + } + const containerTemplateBytes = readRegularFile(containerTemplateFile); + if ( + !SHA256.test(expectedContainerTemplateSha256 ?? '') + || bufferSha256(containerTemplateBytes) !== expectedContainerTemplateSha256 + ) { + throw new Error('PDCF_MEASUREMENT_CONTAINER_TEMPLATE_MISMATCH'); + } + const containerTemplate = validateContainerTemplate(JSON.parse( + containerTemplateBytes.toString('utf8'), + )); + if (containerTemplate.containerName !== postgresContainer) { + throw new Error('PDCF_MEASUREMENT_CONTAINER_TEMPLATE_MISMATCH'); + } + const manifestBytesBeforeLoad = readRegularFile(manifestFile); + const provision = (dependencies.loadProvision ?? loadProvision)( + manifestFile, + secretsFile, + ); + const { manifest } = provision; + const manifestBytesAfterLoad = readRegularFile(manifestFile); + const manifestSha256 = bufferSha256(manifestBytesBeforeLoad); + if ( + bufferSha256(manifestBytesAfterLoad) !== manifestSha256 + || !exactCanonical( + JSON.parse(manifestBytesAfterLoad.toString('utf8')), + manifest, + ) + ) { + throw new Error('PDCF_MEASUREMENT_MANIFEST_CHANGED_DURING_AUDIT'); + } + if ( + manifest.provisionClone?.purpose !== 'measurement' + || run.customerCount !== manifest.customers.length + ) { + throw new Error('PDCF_MEASUREMENT_PROVISION_MISMATCH'); + } + const inspection = (dependencies.inspectDockerContainer ?? inspectDockerContainer)( + postgresContainer, + ); + validateRunningContainerAgainstTemplate(inspection, containerTemplate); + const container = validateContainer({ + inspection, + container: postgresContainer, + environment, + notBeforeEpochMs, + }); + const cgroup = (dependencies.inspectContainerCgroup ?? inspectContainerCgroup)( + postgresContainer, + ); + if (!SHA256.test(cgroup?.identitySha256 ?? '')) { + throw new Error('PDCF_MEASUREMENT_CGROUP_IDENTITY_INVALID'); + } + const queryJson = dependencies.runPsqlJson ?? runPsqlJson; + const maintenanceDatabase = environment.PGDATABASE ?? 'postgres'; + const expectedPostgresSettings = postgresSettingsFromCommand( + containerTemplate.postgresCommand, + ); + const cluster = queryJson({ + database: maintenanceDatabase, + sql: clusterAuditSql(Object.keys(expectedPostgresSettings).sort()), + environment, + }); + const postmasterStartedAtMs = Date.parse(cluster?.postmasterStartedAt ?? ''); + const livePostgresSettings = cluster?.settings; + const expectedSettingNames = Object.keys(expectedPostgresSettings).sort(); + if ( + typeof cluster?.systemIdentifier !== 'string' + || !/^\d+$/.test(cluster.systemIdentifier) + || !Number.isSafeInteger(postmasterStartedAtMs) + || Math.abs(postmasterStartedAtMs - Date.parse(container.startedAt)) > 120_000 + || !Array.isArray(cluster.databases) + || !livePostgresSettings + || JSON.stringify(Object.keys(livePostgresSettings).sort()) + !== JSON.stringify(expectedSettingNames) + || expectedSettingNames.some((name) => + typeof livePostgresSettings[name]?.setting !== 'string' + || livePostgresSettings[name].source !== 'command line' + ) + || Number(livePostgresSettings.max_connections?.setting) + !== Number(expectedPostgresSettings.max_connections) + ) { + throw new Error('PDCF_MEASUREMENT_CLUSTER_IDENTITY_INVALID'); + } + const expectedDatabases = [maintenanceDatabase, ...manifest.customers.map( + (customer) => customer.database + )].sort(); + if (JSON.stringify(cluster.databases) !== JSON.stringify(expectedDatabases)) { + throw new Error('PDCF_MEASUREMENT_DATABASE_INVENTORY_INVALID'); + } + + const inspectContract = dependencies.inspectCustomerContract + ?? inspectCustomerContract; + const customerAudits = manifest.customers.map((customer) => { + const liveContract = inspectContract({ + customer, + canonicalSchemas: manifest.canonicalSchemas, + environment, + }); + if ( + liveContract.databaseContractFingerprint !== customer.databaseContractFingerprint + || !exactCanonical( + liveContract.structuralFingerprints, + customer.structuralFingerprints, + ) + ) { + throw new Error(`PDCF_MEASUREMENT_LIVE_CONTRACT_MISMATCH:${customer.id}`); + } + const rawCloneAudit = queryJson({ + database: customer.database, + sql: buildLiveCloneAuditSql(), + environment, + }); + validateLiveCloneAudit(rawCloneAudit, { manifest, customer }); + return { + customerId: customer.id, + database: customer.database, + databaseContractFingerprint: liveContract.databaseContractFingerprint, + structuralFingerprints: liveContract.structuralFingerprints, + roleSafetyProfileSha256: canonicalSha256(liveContract.roleSafetyProfile), + notificationRoleSafetyProfileSha256: + canonicalSha256(liveContract.notificationRoleSafetyProfile), + extensionVersions: liveContract.extensionVersions, + cloneAttestationSha256: rawCloneAudit.sha256, + cloneNonceSha256: canonicalSha256(rawCloneAudit.nonce), + }; + }).sort((left, right) => left.customerId.localeCompare(right.customerId)); + const cloneAttestationSetSha256 = provisionAttestationSetSha256( + manifest.customers, + ); + if (cloneAttestationSetSha256 !== manifest.provisionClone.attestationSetSha256) { + throw new Error('PDCF_MEASUREMENT_CLONE_SET_MISMATCH'); + } + const immutableEpoch = { + dockerContainerId: container.id, + dockerStartedAt: container.startedAt, + containerConfigurationSha256: canonicalSha256({ + imageId: containerTemplate.imageId, + entrypoint: containerTemplate.entrypoint, + pgHost: containerTemplate.pgHost, + pgPort: containerTemplate.pgPort, + postgresCommand: containerTemplate.postgresCommand, + resourceLimits: containerTemplate.resourceLimits, + }), + cgroupIdentitySha256: cgroup.identitySha256, + postgresSystemIdentifier: cluster.systemIdentifier, + postgresStartedAt: new Date(postmasterStartedAtMs).toISOString(), + cloneId: manifest.provisionClone.id, + cloneAttestationSetSha256, + cloneNonceSetSha256: canonicalSha256(customerAudits.map((audit) => ({ + customerId: audit.customerId, + cloneNonceSha256: audit.cloneNonceSha256, + }))), + liveContractSetSha256: canonicalSha256(customerAudits.map((audit) => ({ + customerId: audit.customerId, + databaseContractFingerprint: audit.databaseContractFingerprint, + structuralFingerprint: audit.structuralFingerprints.combined.sha256, + }))), + }; + const payload = { + fixture: FIXTURE_ID, + observedAt: new Date().toISOString(), + run, + manifestSha256, + containerTemplateSha256: expectedContainerTemplateSha256, + provisionClone: manifest.provisionClone, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint, + canonicalStructuralFingerprint: + manifest.canonicalStructuralFingerprint?.combined?.sha256 ?? null, + container, + cgroup, + postgres: { + systemIdentifier: cluster.systemIdentifier, + postmasterStartedAt: new Date(postmasterStartedAtMs).toISOString(), + serverVersionNum: cluster.serverVersionNum, + databases: cluster.databases, + settings: cluster.settings, + }, + customerAudits, + immutableEpoch, + epochId: canonicalSha256(immutableEpoch), + freshness: { + freshContainerForRun: container.freshForRun, + cgroupV2Verified: true, + notBeforeEpochMs, + startToleranceMs: START_TOLERANCE_MS, + }, + // The required full pg_dump/ACL audit warms PostgreSQL catalogs before the + // Graphile timer starts. Results using this evidence must not call their + // build timing pristine-catalog cold start. + catalogCacheState: 'warmed-by-live-contract-audit', + }; + const attestation = validateMeasurementAttestation({ + version: 1, + kind: ATTESTATION_KIND, + payload, + payloadSha256: canonicalSha256(payload), + }); + if (outputFile) writeImmutableJson(outputFile, attestation); + return attestation; +}; + +const main = () => { + const args = parseArgs(process.argv.slice(2)); + const run = { + arm: requireString(args, 'arm'), + heapMiB: parsePositiveInteger(requireString(args, 'heap-mib'), 'heap-mib'), + customerCount: parsePositiveInteger( + requireString(args, 'customers'), + 'customers', + ), + repetition: parsePositiveInteger( + requireString(args, 'repetition'), + 'repetition', + ), + runOrderIndex: parsePositiveInteger( + requireString(args, 'run-order-index'), + 'run-order-index', + ), + planSha256: requireString(args, 'plan-sha256'), + fleetSha256: requireString(args, 'fleet-sha256'), + }; + const result = attestMeasurementRun({ + manifestFile: path.resolve(requireString(args, 'manifest')), + secretsFile: path.resolve(requireString(args, 'secrets')), + postgresContainer: requireString(args, 'postgres-container'), + containerTemplateFile: path.resolve(requireString(args, 'container-template')), + expectedContainerTemplateSha256: requireString( + args, + 'expected-container-template-sha256', + ), + run, + notBeforeEpochMs: Number(requireString(args, 'not-before-epoch-ms')), + outputFile: path.resolve(requireString(args, 'out')), + }); + process.stdout.write(`${JSON.stringify({ + status: 'attested', + epochId: result.payload.epochId, + payloadSha256: result.payloadSha256, + })}\n`); +}; + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} + +module.exports = { + ATTESTATION_KIND, + attestMeasurementRun, + canonicalSha256, + clusterAuditSql, + inspectContainerCgroup, + inspectDockerContainer, + runPsqlJson, + validateContainer, + validateMeasurementAttestation, + validateRunBinding, + writeImmutableJson, +}; diff --git a/research/graphile-density/physical-database-density/measurement-attestation.test.cjs b/research/graphile-density/physical-database-density/measurement-attestation.test.cjs new file mode 100644 index 0000000000..e7ff11b571 --- /dev/null +++ b/research/graphile-density/physical-database-density/measurement-attestation.test.cjs @@ -0,0 +1,175 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { + attestMeasurementRun, + validateMeasurementAttestation, +} = require('./measurement-attestation.cjs'); +const { + captureContainerTemplate, +} = require('./prepare-measurement-run.cjs'); +const { + provisionAttestationSetSha256, + provisionAttestationSha256, +} = require('./provision.cjs'); + +const digest = (character) => `sha256:${character.repeat(64)}`; +const fileSha256 = (file) => `sha256:${crypto.createHash('sha256') + .update(fs.readFileSync(file)) + .digest('hex')}`; + +const containerInspection = () => ({ + Id: '1'.repeat(64), + Name: '/postgres-density-exact', + Image: digest('2'), + Created: '2026-08-02T00:00:00.001Z', + State: { + Running: true, + StartedAt: '2026-08-02T00:00:00.010Z', + }, + Config: { + Cmd: ['postgres', '-c', 'max_connections=160'], + Labels: { + 'io.constructive.graphile-density.fixture': + 'physical-database-density-v1', + 'io.constructive.graphile-density.prefix': 'pdc_test', + 'io.constructive.graphile-density.purpose': 'measurement', + }, + }, + HostConfig: { + Memory: 1024 ** 3, + MemorySwap: 1024 ** 3, + NanoCpus: 1_000_000_000, + ShmSize: 128 * 1024 ** 2, + }, + NetworkSettings: { + Ports: { '5432/tcp': [{ HostIp: '127.0.0.1', HostPort: '55432' }] }, + }, +}); + +describe('physical measurement attestation', () => { + it('audits the live DDL/ACL contract outside Node and binds a fresh run epoch', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-measurement-')); + const manifestFile = path.join(directory, 'provision.json'); + const secretsFile = path.join(directory, 'runtime-secrets.json'); + const templateFile = path.join(directory, 'container-template.json'); + const outputFile = path.join(directory, 'attestation.json'); + const nonce = '3'.repeat(64); + const customer = { + id: 'physical-customer-0001', + database: 'pdc_test_db_0001', + provisionAttestation: { + version: 1, + cloneId: 'measurement-run-clone', + purpose: 'measurement', + sha256: provisionAttestationSha256({ + cloneId: 'measurement-run-clone', + runPurpose: 'measurement', + customerId: 'physical-customer-0001', + database: 'pdc_test_db_0001', + nonce, + }), + }, + databaseContractFingerprint: digest('4'), + structuralFingerprints: { combined: { sha256: digest('5'), bytes: 100 } }, + }; + const manifest = { + canonicalSchemas: ['ctf_a'], + canonicalDatabaseContractFingerprint: digest('4'), + canonicalStructuralFingerprint: customer.structuralFingerprints, + provisionClone: { + version: 1, + id: 'measurement-run-clone', + purpose: 'measurement', + attestationSetSha256: provisionAttestationSetSha256([customer]), + }, + customers: [customer], + }; + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + fs.writeFileSync(secretsFile, '{}', { mode: 0o600 }); + const template = captureContainerTemplate({ + inspection: containerInspection(), + container: 'postgres-density-exact', + prefix: 'pdc_test', + pgHost: '127.0.0.1', + pgPort: 55432, + minimumMaxConnections: 120, + }); + fs.writeFileSync(templateFile, JSON.stringify(template)); + const run = { + arm: 'candidate', + heapMiB: 2048, + customerCount: 1, + repetition: 1, + runOrderIndex: 2, + planSha256: digest('6'), + fleetSha256: digest('7'), + }; + const liveContract = { + databaseContractFingerprint: customer.databaseContractFingerprint, + structuralFingerprints: customer.structuralFingerprints, + roleSafetyProfile: { safe: true }, + notificationRoleSafetyProfile: { safe: true }, + extensionVersions: [{ name: 'vector', version: '1.0' }], + }; + const attestation = attestMeasurementRun({ + manifestFile, + secretsFile, + postgresContainer: 'postgres-density-exact', + containerTemplateFile: templateFile, + expectedContainerTemplateSha256: fileSha256(templateFile), + run, + notBeforeEpochMs: Date.parse('2026-08-02T00:00:00.000Z'), + outputFile, + environment: { + PGHOST: '127.0.0.1', + PGPORT: '55432', + PGDATABASE: 'postgres', + }, + }, { + loadProvision: () => ({ manifest }), + inspectDockerContainer: containerInspection, + inspectContainerCgroup: () => ({ + version: 1, + source: 'container-cgroup-v2', + identitySha256: digest('8'), + }), + inspectCustomerContract: () => liveContract, + runPsqlJson: ({ database }) => database === 'postgres' + ? { + systemIdentifier: '7421234567890123456', + postmasterStartedAt: '2026-08-02T00:00:00.020Z', + serverVersionNum: '170000', + databases: ['pdc_test_db_0001', 'postgres'], + settings: { + max_connections: { setting: '160', source: 'command line' }, + }, + } + : { + version: 1, + kind: 'unsafe-runtime-live-clone-audit-v1', + cloneId: customer.provisionAttestation.cloneId, + purpose: customer.provisionAttestation.purpose, + customerId: customer.id, + database: customer.database, + nonce, + sha256: customer.provisionAttestation.sha256, + }, + }); + assert.equal(attestation.payload.freshness.freshContainerForRun, true); + assert.equal(attestation.payload.customerAudits.length, 1); + assert.equal( + attestation.payload.immutableEpoch.cloneAttestationSetSha256, + manifest.provisionClone.attestationSetSha256, + ); + assert.match(attestation.payload.epochId, /^sha256:[a-f0-9]{64}$/); + assert.doesNotThrow(() => validateMeasurementAttestation(attestation)); + assert.deepEqual(JSON.parse(fs.readFileSync(outputFile, 'utf8')), attestation); + }); +}); diff --git a/research/graphile-density/physical-database-density/physical-hostile-preflight.cjs b/research/graphile-density/physical-database-density/physical-hostile-preflight.cjs new file mode 100644 index 0000000000..56ed0603a7 --- /dev/null +++ b/research/graphile-density/physical-database-density/physical-hostile-preflight.cjs @@ -0,0 +1,831 @@ +'use strict'; + +const { execFileSync } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + REPO_ROOT, + TENANTS, + assertCredentialFree, + assertLoopbackBaseUrl, + parseArgs, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); +const { + assertCustomerPathPrefix, + assertIdentity, + control, + identityOperation, + postGraphql, + requestJson, + runHostileValidation, +} = require('../complete-tenant-fixture/hostile-validation.cjs'); +const { + FIXTURE_ID, + atomicWriteJson, +} = require('./lib.cjs'); +const { provisionAttestationSetSha256 } = require('./provision.cjs'); +const { + ADMISSION_SCOPE, + CLEANUP_AUDIT_KIND, + PROBE_CAPABILITIES, + PROBE_KIND, + expectedAuditedProfiles, + loadPrivateProvision, + runUnsafeRuntimeStartupMatrix, +} = require('./unsafe-runtime-startup-probe.cjs'); + +const FIXTURE_DIR = __dirname; +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/; +const VALIDATOR_ENTRY_FILES = Object.freeze([ + 'research/graphile-density/physical-database-density/physical-hostile-preflight.cjs', + 'research/graphile-density/physical-database-density/lib.cjs', + 'research/graphile-density/complete-tenant-fixture/hostile-validation.cjs', + 'research/graphile-density/complete-tenant-fixture/generate-inputs.cjs', + 'research/graphile-density/complete-tenant-fixture/lib.cjs', + 'research/graphile-density/complete-tenant-fixture/server.cjs', + 'research/graphile-density/complete-tenant-fixture/schema.sql', + 'research/graphile-density/physical-database-density/server.cjs', + 'research/graphile-density/physical-database-density/physical-identity.sql', + 'research/graphile-density/physical-database-density/provision.cjs', + 'research/graphile-density/physical-database-density/provision-attestation.sql', + 'research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.cjs', +]); + +const sha256Buffer = (value) => `sha256:${crypto.createHash('sha256') + .update(value) + .digest('hex')}`; + +const fileSha256 = (file) => sha256Buffer(fs.readFileSync(file)); + +const canonicalize = (value) => { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [ + key, + canonicalize(value[key]), + ])); +}; + +const canonicalSha256 = (value) => sha256Buffer(JSON.stringify(canonicalize(value))); +const canonicalEqual = (left, right) => + JSON.stringify(canonicalize(left)) === JSON.stringify(canonicalize(right)); + +const exactKeys = (value, expected) => { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + return JSON.stringify(Object.keys(value).sort()) === JSON.stringify([...expected].sort()); +}; + +const requireSha256 = (value, code) => { + if (!SHA256_PATTERN.test(value ?? '')) throw new Error(code); + return value; +}; + +const requireArtifactLabel = (value, code) => { + if ( + typeof value !== 'string' + || !/^[a-z0-9][a-z0-9._-]{0,127}$/i.test(value) + ) { + throw new Error(code); + } + return value; +}; + +const assertOutputDoesNotAliasInputs = (outputFile, inputFiles) => { + if (!outputFile) return null; + const absoluteOutputFile = path.resolve(outputFile); + let outputStat = null; + try { + outputStat = fs.statSync(absoluteOutputFile); + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + for (const inputFile of inputFiles) { + const absoluteInputFile = path.resolve(inputFile); + if (absoluteInputFile === absoluteOutputFile) { + throw new Error('PDCF_HOSTILE_OUTPUT_ALIASES_INPUT'); + } + if (outputStat) { + const inputStat = fs.statSync(absoluteInputFile); + if (inputStat.dev === outputStat.dev && inputStat.ino === outputStat.ino) { + throw new Error('PDCF_HOSTILE_OUTPUT_ALIASES_INPUT'); + } + } + } + return absoluteOutputFile; +}; + +const expectedLiveProvisionAttestation = (manifest, customer) => ({ + version: 1, + cloneId: manifest.provisionClone.id, + purpose: manifest.provisionClone.purpose, + customerId: customer.id, + database: customer.database, + sha256: customer.provisionAttestation.sha256, + verified: true, +}); + +const validatePreflightManifest = (manifest, preflightCloneId) => { + if ( + !exactKeys( + manifest.provisionClone, + ['version', 'id', 'purpose', 'attestationSetSha256'], + ) + || + manifest.provisionClone?.version !== 1 + || manifest.provisionClone.id !== preflightCloneId + || manifest.provisionClone.purpose !== 'hostile-preflight' + || !SHA256_PATTERN.test(manifest.provisionClone.attestationSetSha256 ?? '') + || provisionAttestationSetSha256(manifest.customers) + !== manifest.provisionClone.attestationSetSha256 + || !SHA256_PATTERN.test( + manifest.canonicalStructuralFingerprint?.combined?.sha256 ?? '' + ) + || !SHA256_PATTERN.test(manifest.canonicalDatabaseContractFingerprint ?? '') + ) { + throw new Error('PDCF_HOSTILE_PREFLIGHT_MANIFEST_MISMATCH'); + } + for (const customer of manifest.customers) { + if ( + !exactKeys( + customer.provisionAttestation, + ['version', 'cloneId', 'purpose', 'sha256'], + ) + || + customer.provisionAttestation?.version !== 1 + || customer.provisionAttestation.cloneId !== preflightCloneId + || customer.provisionAttestation.purpose !== 'hostile-preflight' + || !SHA256_PATTERN.test(customer.provisionAttestation.sha256 ?? '') + || !SHA256_PATTERN.test(customer.structuralFingerprints?.combined?.sha256 ?? '') + || !SHA256_PATTERN.test(customer.databaseContractFingerprint ?? '') + ) { + throw new Error(`PDCF_HOSTILE_CUSTOMER_MANIFEST_INVALID:${customer.id}`); + } + } + return manifest; +}; + +const validatePhysicalStatus = (status, { + manifest, + arm, + mode, + preflightCloneId = manifest.provisionClone?.id, +}) => { + if ( + status?.version !== 1 + || status.fixture !== FIXTURE_ID + || status.arm !== arm + || status.introspectionMode !== mode + || status.introspectionClientReleaseMode !== 'destroy' + || status.runPurpose !== 'hostile-preflight' + || status.cloneId !== preflightCloneId + || status.provisionClone?.version !== 1 + || status.provisionClone.id !== preflightCloneId + || status.provisionClone.purpose !== 'hostile-preflight' + || status.provisionClone.attestationSetSha256 + !== manifest.provisionClone?.attestationSetSha256 + || status.provisionClone.verified !== true + || !canonicalEqual( + status.canonicalStructuralFingerprint, + manifest.canonicalStructuralFingerprint, + ) + || status.canonicalDatabaseContractFingerprint + !== manifest.canonicalDatabaseContractFingerprint + || status.realtime?.managersExpected !== manifest.customers.length * TENANTS.length + || status.realtime?.connectionsExpected !== manifest.customers.length * TENANTS.length + || status.realtime?.transportsExpected !== manifest.customers.length * TENANTS.length + || status.realtime?.notificationMode !== 'dedicated' + || status.runtimePoolMax !== 2 + || status.runtimePoolMaxUses !== null + ) { + throw new Error('PDCF_HOSTILE_PHYSICAL_STATUS_MISMATCH'); + } + requireSha256( + status.blueprintCompatibilityFingerprint, + 'PDCF_HOSTILE_BLUEPRINT_FINGERPRINT_REQUIRED', + ); + requireSha256( + manifest.canonicalDatabaseContractFingerprint, + 'PDCF_HOSTILE_CANONICAL_CONTRACT_REQUIRED', + ); + if (!Array.isArray(status.customers)) { + throw new Error('PDCF_HOSTILE_CUSTOMER_SET_MISMATCH'); + } + if ( + status.customers.length !== manifest.customers.length + || status.customers.some((observed, index) => { + const customer = manifest.customers[index]; + return observed.id !== customer.id + || observed.physicalDatabase !== customer.database + || !canonicalEqual( + observed.provisionAttestation, + expectedLiveProvisionAttestation(manifest, customer), + ) + || !canonicalEqual( + observed.structuralFingerprints, + customer.structuralFingerprints, + ) + || observed.canonicalStructuralFingerprint + !== customer.structuralFingerprints?.combined?.sha256 + || observed.databaseContractFingerprint !== customer.databaseContractFingerprint + || observed.contractVerification !== 'live-recomputed'; + }) + ) { + throw new Error('PDCF_HOSTILE_CUSTOMER_SET_MISMATCH'); + } + return status; +}; + +const validateChildStatus = (status, { customer, manifest, arm, mode }) => { + const tenantIds = TENANTS.map((tenant) => tenant.id); + if ( + status?.version !== 1 + || status.fixture !== 'complete-tenant-abc-v1' + || status.arm !== arm + || status.introspectionMode !== mode + || status.introspectionClientReleaseMode !== 'destroy' + || status.releaseBuildStateAfterValidation !== true + || status.physicalDatabase !== customer.physicalIdentity + || status.runPurpose !== 'hostile-preflight' + || !canonicalEqual( + status.provisionAttestation, + expectedLiveProvisionAttestation(manifest, customer), + ) + || status.physicalIsolation !== 'dedicated-login-and-pool-per-tenant' + || status.sharedRuntimePool !== false + || status.runtimePoolMax !== 2 + || status.runtimePoolMaxUses !== null + || status.enableRealtime !== true + || status.realtimeNotificationMode !== 'dedicated' + || status.realtimeCursorPollIntervalMs !== 5_000 + || status.realtimeCursorHeartbeatIntervalMs !== 30_000 + || !exactKeys(status.realtimeSchemas, tenantIds) + || status.controlAvailable !== true + || status.runtimeSafety?.passed !== true + || status.runtimeSafety?.rolesDistinct !== true + || status.liveIdentityScope !== 'process-local-keyed-hmac-v1' + || !/^graphile-configuration:ctf:v1:[a-f0-9]{64}$/.test( + status.configurationIdentity ?? '' + ) + || status.contractEvidence?.version !== 1 + || status.contractEvidence?.credentialFree !== true + || status.contractEvidence?.configurationIdentity + !== status.configurationIdentity + || !exactKeys(status.runtimePoolIdentities, tenantIds) + || !exactKeys(status.buildContracts, tenantIds) + || !exactKeys(status.builds?.byTenant, tenantIds) + ) { + throw new Error(`PDCF_HOSTILE_CHILD_STATUS_MISMATCH:${customer.id}`); + } + requireSha256( + status.runtimeArtifactFingerprint, + `PDCF_HOSTILE_RUNTIME_FINGERPRINT_REQUIRED:${customer.id}`, + ); + for (const tenant of TENANTS) { + if (status.realtimeSchemas[tenant.id] !== `${tenant.schema}_realtime`) { + throw new Error(`PDCF_HOSTILE_CHILD_CONTRACT_MISMATCH:${customer.id}:${tenant.id}`); + } + } + for (const tenantId of tenantIds) { + const poolEvidence = status.contractEvidence?.runtimePools?.[tenantId]; + const buildEvidence = status.contractEvidence?.graphileBuilds?.[tenantId]; + const binding = status.runtimeBindings?.[tenantId]; + if ( + !/^pg:v1:[a-f0-9]{64}$/i.test(status.runtimePoolIdentities[tenantId]) + || !String(status.buildContracts[tenantId]).startsWith('graphile:v1:') + || !/^pg-contract-evidence:v1:[a-f0-9]{64}$/.test( + poolEvidence?.fingerprint ?? '' + ) + || !/^graphile-contract-evidence:v1:[a-f0-9]{64}$/.test( + buildEvidence?.fingerprint ?? '' + ) + || poolEvidence?.input?.databaseName !== customer.database + || poolEvidence?.input?.role !== customer.roles?.[tenantId] + || binding?.databaseName !== customer.database + || binding?.role !== customer.roles?.[tenantId] + || JSON.stringify(binding?.schemas) !== JSON.stringify([`ctf_${tenantId}`]) + || !Number.isSafeInteger(status.builds.byTenant[tenantId]) + || status.builds.byTenant[tenantId] < 0 + ) { + throw new Error(`PDCF_HOSTILE_CHILD_CONTRACT_MISMATCH:${customer.id}:${tenantId}`); + } + } + return status; +}; + +const collectSourceProvenance = () => { + const commit = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: REPO_ROOT, + encoding: 'utf8', + }).trim(); + const worktreeState = execFileSync( + 'git', + ['status', '--porcelain=v1', '--untracked-files=all'], + { cwd: REPO_ROOT, encoding: 'utf8' }, + ); + const validatorEntries = VALIDATOR_ENTRY_FILES.map((relativePath) => ({ + path: relativePath, + sha256: fileSha256(path.join(REPO_ROOT, relativePath)), + })); + const provenance = { + git: { + commit, + worktreeDirty: worktreeState.length > 0, + worktreeStateSha256: sha256Buffer(worktreeState), + }, + lockfileSha256: fileSha256(path.join(REPO_ROOT, 'pnpm-lock.yaml')), + runtime: { + node: process.version, + v8: process.versions.v8, + platform: process.platform, + architecture: process.arch, + }, + validatorEntries, + }; + return { + ...provenance, + sourceStateSha256: canonicalSha256(provenance), + }; +}; + +const validateSourceProvenance = (provenance) => { + const { + sourceStateSha256, + ...sourceState + } = provenance ?? {}; + if ( + !/^[a-f0-9]{40,64}$/.test(provenance?.git?.commit ?? '') + || typeof provenance.git.worktreeDirty !== 'boolean' + || !SHA256_PATTERN.test(provenance.git.worktreeStateSha256 ?? '') + || !SHA256_PATTERN.test(provenance.lockfileSha256 ?? '') + || !SHA256_PATTERN.test(sourceStateSha256 ?? '') + || sourceStateSha256 !== canonicalSha256(sourceState) + || typeof provenance.runtime?.node !== 'string' + || typeof provenance.runtime?.v8 !== 'string' + || typeof provenance.runtime?.platform !== 'string' + || typeof provenance.runtime?.architecture !== 'string' + || !Array.isArray(provenance.validatorEntries) + || provenance.validatorEntries.length === 0 + || new Set(provenance.validatorEntries.map((entry) => entry?.path)).size + !== provenance.validatorEntries.length + || provenance.validatorEntries.some((entry) => + typeof entry?.path !== 'string' + || !entry.path + || !SHA256_PATTERN.test(entry.sha256 ?? '') + ) + ) { + throw new Error('PDCF_HOSTILE_SOURCE_PROVENANCE_INVALID'); + } + return provenance; +}; + +const childStatusUrl = (baseUrl, customer) => + `${baseUrl}${assertCustomerPathPrefix( + `/customer/${customer.id}`, + customer.id, + )}/__ctf/status`; + +const readChildStatus = async (baseUrl, customer, fetchImpl) => + requestJson(childStatusUrl(baseUrl, customer), { fetchImpl }); + +const assertBadRoleCoverage = (report, customer) => { + if (report?.passed !== true) { + throw new Error(`PDCF_HOSTILE_CUSTOMER_VALIDATION_FAILED:${customer.id}`); + } + const checks = new Set((report.checks ?? []).map((check) => check.name)); + for (const tenant of TENANTS) { + if (!checks.has(`bad-role-expected-failure:${tenant.id}`)) { + throw new Error(`PDCF_HOSTILE_BAD_ROLE_CHECK_MISSING:${customer.id}:${tenant.id}`); + } + } +}; + +const validateUnsafeRuntimeStartupAdmission = ( + report, + manifest, + expectedRuntimeArtifactFingerprint, +) => { + const expectedSurfaces = TENANTS.map((tenant) => tenant.id); + const expectedAttempts = PROBE_CAPABILITIES.length * expectedSurfaces.length; + const customer = manifest.customers[0]; + const expectedPairs = new Set(PROBE_CAPABILITIES.flatMap((capability) => + expectedSurfaces.map((tenantId) => `${capability}:${tenantId}`) + )); + const attempts = Array.isArray(report?.attempts) ? report.attempts : []; + const observedPairs = new Set(attempts.map((attempt) => + `${attempt.capability}:${attempt.tenantId}` + )); + if ( + !exactKeys(report, [ + 'version', + 'kind', + 'admissionScope', + 'provisionClone', + 'representativeCustomerId', + 'representativePhysicalDatabase', + 'representativeProvisionAttestationSha256', + 'canonicalDatabaseContractFingerprint', + 'runtimeArtifactFingerprint', + 'liveProvisionAttestation', + 'safeStartupControl', + 'roleProfileAudit', + 'cleanupAudit', + 'capabilities', + 'surfaces', + 'attempts', + 'expectedAttempts', + 'rejectedAttempts', + 'acceptedAttempts', + 'graphileBuildsStarted', + 'residentGraphileEntries', + 'passed', + ]) + || report.version !== 2 + || report.kind !== PROBE_KIND + || report.admissionScope !== ADMISSION_SCOPE + || !exactKeys( + report.provisionClone, + ['version', 'id', 'purpose', 'attestationSetSha256'], + ) + || !canonicalEqual(report.provisionClone, manifest.provisionClone) + || report.representativeCustomerId !== customer?.id + || report.representativePhysicalDatabase !== customer?.physicalIdentity + || report.representativeProvisionAttestationSha256 + !== customer?.provisionAttestation?.sha256 + || report.canonicalDatabaseContractFingerprint + !== manifest.canonicalDatabaseContractFingerprint + || report.runtimeArtifactFingerprint !== expectedRuntimeArtifactFingerprint + || !canonicalEqual( + report.liveProvisionAttestation, + expectedLiveProvisionAttestation(manifest, customer), + ) + || !exactKeys(report.safeStartupControl, [ + 'tenantId', + 'accepted', + 'physicalDatabaseVerifiedBeforeRoleAudit', + 'controlCredentialEnvironmentAbsent', + 'graphileBuildsStarted', + 'residentGraphileEntries', + 'passed', + ]) + || report.safeStartupControl.tenantId !== expectedSurfaces[0] + || report.safeStartupControl.accepted !== true + || report.safeStartupControl.physicalDatabaseVerifiedBeforeRoleAudit !== true + || report.safeStartupControl.controlCredentialEnvironmentAbsent !== true + || report.safeStartupControl.graphileBuildsStarted !== 0 + || report.safeStartupControl.residentGraphileEntries !== 0 + || report.safeStartupControl.passed !== true + || !exactKeys( + report.roleProfileAudit, + ['version', 'kind', 'database', 'profiles', 'passed'], + ) + || report.roleProfileAudit.version !== 1 + || report.roleProfileAudit.kind !== 'unsafe-runtime-role-profile-audit-v1' + || report.roleProfileAudit.database !== customer?.database + || !canonicalEqual( + report.roleProfileAudit.profiles, + expectedAuditedProfiles(), + ) + || report.roleProfileAudit.passed !== true + || !exactKeys(report.cleanupAudit, [ + 'version', + 'kind', + 'database', + 'remainingRoles', + 'remainingSchemas', + 'passed', + ]) + || report.cleanupAudit.version !== 1 + || report.cleanupAudit.kind !== CLEANUP_AUDIT_KIND + || report.cleanupAudit.database !== customer?.database + || report.cleanupAudit.remainingRoles !== 0 + || report.cleanupAudit.remainingSchemas !== 0 + || report.cleanupAudit.passed !== true + || JSON.stringify(report.capabilities) !== JSON.stringify(PROBE_CAPABILITIES) + || JSON.stringify(report.surfaces) !== JSON.stringify(expectedSurfaces) + || report.expectedAttempts !== expectedAttempts + || report.rejectedAttempts !== expectedAttempts + || report.acceptedAttempts !== 0 + || report.graphileBuildsStarted !== 0 + || report.residentGraphileEntries !== 0 + || report.passed !== true + || !Array.isArray(report.attempts) + || report.attempts.length !== expectedAttempts + || observedPairs.size !== expectedPairs.size + || [...expectedPairs].some((pair) => !observedPairs.has(pair)) + || report.attempts.some((attempt) => + !exactKeys(attempt, [ + 'capability', + 'tenantId', + 'rejectedCode', + 'controlCredentialEnvironmentAbsent', + 'graphileBuildsStarted', + 'residentGraphileEntries', + ]) + || attempt.rejectedCode !== 'GRAPHILE_UNSAFE_RUNTIME_ROLE' + || attempt.controlCredentialEnvironmentAbsent !== true + || attempt.graphileBuildsStarted !== 0 + || attempt.residentGraphileEntries !== 0 + ) + ) { + throw new Error('PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID'); + } + return report; +}; + +const runFleetInvalidateAndRebuild = async ({ + baseUrl, + controlToken, + manifest, + arm, + mode, + fetchImpl, +}) => { + const before = {}; + for (const customer of manifest.customers) { + before[customer.id] = validateChildStatus( + await readChildStatus(baseUrl, customer, fetchImpl), + { customer, manifest, arm, mode }, + ); + } + + for (const customer of manifest.customers) { + const pathPrefix = `/customer/${customer.id}`; + await control( + baseUrl, + pathPrefix, + controlToken, + 'invalidate-all', + null, + customer.physicalIdentity, + fetchImpl, + ); + } + + for (const customer of manifest.customers) { + const pathPrefix = `/customer/${customer.id}`; + for (const tenant of TENANTS) { + const response = await postGraphql( + baseUrl, + pathPrefix, + tenant.id, + identityOperation, + fetchImpl, + ); + assertIdentity(tenant, response, customer.physicalIdentity); + } + } + + const after = {}; + for (const customer of manifest.customers) { + after[customer.id] = validateChildStatus( + await readChildStatus(baseUrl, customer, fetchImpl), + { customer, manifest, arm, mode }, + ); + for (const tenant of TENANTS) { + const beforeCount = before[customer.id].builds.byTenant[tenant.id]; + const afterCount = after[customer.id].builds.byTenant[tenant.id]; + if (afterCount !== beforeCount + 1) { + throw new Error( + `PDCF_HOSTILE_FLEET_REBUILD_COUNT_MISMATCH:${customer.id}:${tenant.id}`, + ); + } + } + } + return { + invalidatedCustomers: manifest.customers.length, + rebuiltSurfaces: manifest.customers.length * TENANTS.length, + physicalIdentityMismatches: 0, + }; +}; + +const failureCode = (error) => String( + error instanceof Error ? error.message : error, +).split(':', 1)[0].replace(/[^A-Z0-9_-]/gi, '_').slice(0, 96); + +const runPhysicalHostilePreflight = async ({ + manifestFile, + secretsFile, + baseUrl, + controlToken, + arm, + mode, + preflightCloneId, + outputFile, + fetchImpl = fetch, + runCustomerValidation = runHostileValidation, + runUnsafeRoleMatrix = runUnsafeRuntimeStartupMatrix, + provenanceProvider = collectSourceProvenance, +} = {}) => { + if ( + process.env.GRAPHQL_CPERF_MEASURED_RUN === 'true' + || process.env.GRAPHQL_CPERF_RETAINED_HEAP_ENABLED === 'true' + ) { + throw new Error('PDCF_HOSTILE_MEASURED_PROCESS_FORBIDDEN'); + } + if (typeof controlToken !== 'string' || Buffer.byteLength(controlToken) < 32) { + throw new Error('PDCF_HOSTILE_CONTROL_TOKEN_REQUIRED'); + } + preflightCloneId = requireArtifactLabel( + preflightCloneId, + 'PDCF_HOSTILE_PREFLIGHT_CLONE_ID_REQUIRED', + ); + arm = requireArtifactLabel(arm, 'PDCF_HOSTILE_ARM_REQUIRED'); + if (mode !== 'stock' && mode !== 'scoped-required') { + throw new Error('PDCF_HOSTILE_MODE_INVALID'); + } + if (preflightCloneId === controlToken) { + throw new Error('PDCF_HOSTILE_PREFLIGHT_CLONE_ID_INVALID'); + } + const localBaseUrl = assertLoopbackBaseUrl(baseUrl); + const absoluteManifestFile = path.resolve(manifestFile); + const absoluteSecretsFile = path.resolve(requireString( + { secrets: secretsFile }, + 'secrets', + )); + const absoluteOutputFile = assertOutputDoesNotAliasInputs(outputFile, [ + absoluteManifestFile, + absoluteSecretsFile, + ]); + const manifest = validatePreflightManifest( + loadPrivateProvision(absoluteManifestFile, absoluteSecretsFile).manifest, + preflightCloneId, + ); + const startedAt = new Date().toISOString(); + const sourceProvenance = validateSourceProvenance(provenanceProvider()); + const report = { + version: 1, + kind: 'physical-hostile-preflight-v1', + fixture: FIXTURE_ID, + startedAt, + endedAt: null, + passed: false, + customerQualified: false, + performanceEvidence: false, + arm, + mode, + preflightCloneId, + provisionClone: { + ...manifest.provisionClone, + verifiedByServer: false, + }, + measuredCloneRequirement: { + mustBeFresh: true, + mustBeDistinctFromPreflight: true, + mustMatchCanonicalDatabaseContract: true, + }, + manifest: { + sha256: fileSha256(absoluteManifestFile), + canonicalStructuralFingerprint: + manifest.canonicalStructuralFingerprint?.combined?.sha256 ?? null, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint ?? null, + customerIds: manifest.customers.map((customer) => customer.id), + }, + sourceProvenance, + observedRuntimeArtifactFingerprint: null, + observedBlueprintCompatibilityFingerprint: null, + unsafeRuntimeStartupAdmission: null, + customerValidations: [], + fleetInvalidateAndRebuild: null, + unsupportedChecks: [], + }; + + try { + const physicalStatus = validatePhysicalStatus( + await requestJson(`${localBaseUrl}/__physical/status`, { fetchImpl }), + { manifest, arm, mode, preflightCloneId }, + ); + report.provisionClone.verifiedByServer = true; + report.observedBlueprintCompatibilityFingerprint = + physicalStatus.blueprintCompatibilityFingerprint; + const representativeCustomer = manifest.customers[0]; + const representativeChildStatus = validateChildStatus( + await readChildStatus(localBaseUrl, representativeCustomer, fetchImpl), + { customer: representativeCustomer, manifest, arm, mode }, + ); + report.unsafeRuntimeStartupAdmission = validateUnsafeRuntimeStartupAdmission( + await runUnsafeRoleMatrix({ + manifestFile: absoluteManifestFile, + secretsFile: absoluteSecretsFile, + expectedRuntimeArtifactFingerprint: + representativeChildStatus.runtimeArtifactFingerprint, + mode, + }), + manifest, + representativeChildStatus.runtimeArtifactFingerprint, + ); + + const runtimeFingerprints = new Set(); + for (const customer of manifest.customers) { + const childStatus = validateChildStatus( + await readChildStatus(localBaseUrl, customer, fetchImpl), + { customer, manifest, arm, mode }, + ); + runtimeFingerprints.add(childStatus.runtimeArtifactFingerprint); + const customerReport = await runCustomerValidation({ + baseUrl: localBaseUrl, + pathPrefix: `/customer/${customer.id}`, + expectedCustomerId: customer.id, + expectedPhysicalDatabaseIdentity: customer.physicalIdentity, + controlToken, + arm, + mode, + fetchImpl, + }); + assertBadRoleCoverage(customerReport, customer); + report.customerValidations.push({ + customerId: customer.id, + physicalDatabaseIdentity: customer.physicalIdentity, + provisionAttestationSha256: customer.provisionAttestation.sha256, + checks: customerReport.checks.length, + reportSha256: canonicalSha256(customerReport), + passed: true, + }); + } + if ( + runtimeFingerprints.size !== 1 + || !runtimeFingerprints.has( + report.unsafeRuntimeStartupAdmission.runtimeArtifactFingerprint, + ) + ) { + throw new Error('PDCF_HOSTILE_RUNTIME_FINGERPRINT_MISMATCH'); + } + report.observedRuntimeArtifactFingerprint = [...runtimeFingerprints][0]; + report.fleetInvalidateAndRebuild = await runFleetInvalidateAndRebuild({ + baseUrl: localBaseUrl, + controlToken, + manifest, + arm, + mode, + fetchImpl, + }); + report.passed = true; + report.endedAt = new Date().toISOString(); + report.artifactSha256 = canonicalSha256({ + ...report, + artifactSha256: undefined, + }); + assertCredentialFree(report); + if (absoluteOutputFile) atomicWriteJson(absoluteOutputFile, report, 0o600); + return report; + } catch (error) { + report.endedAt = new Date().toISOString(); + report.failureCode = failureCode(error); + report.artifactSha256 = canonicalSha256({ + ...report, + artifactSha256: undefined, + }); + assertCredentialFree(report); + if (absoluteOutputFile) atomicWriteJson(absoluteOutputFile, report, 0o600); + throw error; + } +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const report = await runPhysicalHostilePreflight({ + manifestFile: path.resolve(requireString(args, 'manifest')), + secretsFile: path.resolve(requireString(args, 'secrets')), + baseUrl: requireString(args, 'base-url'), + controlToken: process.env.CTF_CONTROL_TOKEN, + arm: requireString(args, 'arm'), + mode: requireString(args, 'mode', 'scoped-required'), + preflightCloneId: requireString(args, 'preflight-clone-id'), + outputFile: path.resolve(requireString( + args, + 'output', + path.join(FIXTURE_DIR, '.local', `hostile-preflight-${timestamp}.json`), + )), + }); + process.stdout.write(`${JSON.stringify({ + passed: report.passed, + customers: report.customerValidations.length, + artifactSha256: report.artifactSha256, + })}\n`); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${failureCode(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + VALIDATOR_ENTRY_FILES, + assertBadRoleCoverage, + assertOutputDoesNotAliasInputs, + canonicalSha256, + collectSourceProvenance, + runFleetInvalidateAndRebuild, + runPhysicalHostilePreflight, + validateUnsafeRuntimeStartupAdmission, + validateSourceProvenance, + validateChildStatus, + validatePreflightManifest, + validatePhysicalStatus, +}; diff --git a/research/graphile-density/physical-database-density/physical-hostile-preflight.test.cjs b/research/graphile-density/physical-database-density/physical-hostile-preflight.test.cjs new file mode 100644 index 0000000000..1d8fa62d8e --- /dev/null +++ b/research/graphile-density/physical-database-density/physical-hostile-preflight.test.cjs @@ -0,0 +1,658 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { TENANTS, assertCredentialFree } = require('../complete-tenant-fixture/lib.cjs'); +const { makeCustomers } = require('./lib.cjs'); +const { + provisionAttestationSetSha256, + provisionAttestationSha256, +} = require('./provision.cjs'); +const { + assertProvisionCloneManifest, + parseServerOptions, +} = require('./server.cjs'); +const { + assertOutputDoesNotAliasInputs, + canonicalSha256, + runPhysicalHostilePreflight, + validateUnsafeRuntimeStartupAdmission, + validatePhysicalStatus, +} = require('./physical-hostile-preflight.cjs'); +const { + ADMISSION_SCOPE, + CLEANUP_AUDIT_KIND, + PROBE_CAPABILITIES, + PROBE_KIND, + expectedAuditedProfiles, +} = require('./unsafe-runtime-startup-probe.cjs'); + +const digest = (character) => `sha256:${character.repeat(64)}`; +const arm = 'physical-db-idle-1s'; +const mode = 'scoped-required'; +const preflightCloneId = 'fresh-preflight-clone-20260802-a'; + +const makeManifest = () => { + const canonicalDatabaseContractFingerprint = digest('b'); + const canonicalStructuralFingerprint = { + combined: { sha256: digest('c') }, + }; + const customers = makeCustomers('pdc_preflight', 2).map((customer, index) => ({ + ...customer, + provisionAttestation: { + version: 1, + cloneId: preflightCloneId, + purpose: 'hostile-preflight', + sha256: digest(String(index + 3)), + }, + structuralFingerprints: canonicalStructuralFingerprint, + databaseContractFingerprint: canonicalDatabaseContractFingerprint, + })); + return { + version: 1, + fixture: 'physical-database-density-v1', + prefix: 'pdc_preflight', + createdAt: '2026-08-02T00:00:00.000Z', + provisionClone: { + version: 1, + id: preflightCloneId, + purpose: 'hostile-preflight', + attestationSetSha256: provisionAttestationSetSha256(customers), + }, + canonicalStructuralFingerprint, + canonicalDatabaseContractFingerprint, + customers, + }; +}; + +const physicalStatusFor = (manifest) => ({ + version: 1, + fixture: 'physical-database-density-v1', + arm, + runPurpose: 'hostile-preflight', + cloneId: preflightCloneId, + provisionClone: { + ...manifest.provisionClone, + verified: true, + }, + introspectionMode: mode, + introspectionClientReleaseMode: 'destroy', + runtimePoolMax: 2, + runtimePoolMaxUses: null, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint, + blueprintCompatibilityFingerprint: digest('d'), + canonicalStructuralFingerprint: manifest.canonicalStructuralFingerprint, + realtime: { + managersExpected: manifest.customers.length * TENANTS.length, + connectionsExpected: manifest.customers.length * TENANTS.length, + transportsExpected: manifest.customers.length * TENANTS.length, + notificationMode: 'dedicated', + }, + customers: manifest.customers.map((customer) => ({ + id: customer.id, + physicalDatabase: customer.database, + provisionAttestation: { + ...customer.provisionAttestation, + customerId: customer.id, + database: customer.database, + verified: true, + }, + structuralFingerprints: customer.structuralFingerprints, + canonicalStructuralFingerprint: + customer.structuralFingerprints.combined.sha256, + databaseContractFingerprint: customer.databaseContractFingerprint, + contractVerification: 'live-recomputed', + })), +}); + +const childStatusFor = (manifest, customer, buildCounts) => ({ + version: 1, + fixture: 'complete-tenant-abc-v1', + arm, + introspectionMode: mode, + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: true, + runtimeArtifactFingerprint: digest('e'), + physicalIsolation: 'dedicated-login-and-pool-per-tenant', + sharedRuntimePool: false, + runtimePoolMax: 2, + runtimePoolMaxUses: null, + enableRealtime: true, + realtimeNotificationMode: 'dedicated', + realtimeCursorPollIntervalMs: 5_000, + realtimeCursorHeartbeatIntervalMs: 30_000, + realtimeSchemas: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `${tenant.schema}_realtime`, + ])), + physicalDatabase: customer.physicalIdentity, + runPurpose: 'hostile-preflight', + provisionAttestation: { + ...customer.provisionAttestation, + customerId: customer.id, + database: customer.database, + verified: true, + }, + controlAvailable: true, + runtimePoolIdentities: Object.fromEntries(TENANTS.map((tenant, index) => [ + tenant.id, + `pg:v1:${String(index + 1).repeat(64)}`, + ])), + buildContracts: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `graphile:v1:${customer.id}:${tenant.id}`, + ])), + configurationIdentity: `graphile-configuration:ctf:v1:${'e'.repeat(64)}`, + liveIdentityScope: 'process-local-keyed-hmac-v1', + runtimeBindings: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + databaseName: customer.database, + role: customer.roles[tenant.id], + schemas: [tenant.schema], + }, + ])), + contractEvidence: { + version: 1, + credentialFree: true, + configurationIdentity: `graphile-configuration:ctf:v1:${'e'.repeat(64)}`, + runtimePools: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + version: 1, + fingerprint: `pg-contract-evidence:v1:${tenant.id.repeat(64)}`, + input: { + databaseName: customer.database, + role: customer.roles[tenant.id], + }, + }, + ])), + graphileBuilds: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + version: 1, + fingerprint: `graphile-contract-evidence:v1:${tenant.id.repeat(64)}`, + input: {}, + }, + ])), + }, + builds: { byTenant: { ...buildCounts } }, + runtimeSafety: { passed: true, rolesDistinct: true }, +}); + +const response = (body, status = 200) => ({ + ok: status >= 200 && status < 300, + status, + json: async () => body, +}); + +const makeProvenance = () => { + const provenance = { + git: { + commit: 'a'.repeat(40), + worktreeDirty: false, + worktreeStateSha256: digest('f'), + }, + lockfileSha256: digest('1'), + runtime: { + node: 'v24.0.0', + v8: '13.6', + platform: 'linux', + architecture: 'x64', + }, + validatorEntries: [{ path: 'validator.cjs', sha256: digest('2') }], + }; + return { + ...provenance, + sourceStateSha256: canonicalSha256(provenance), + }; +}; + +const writeSecrets = (directory, manifest) => { + const secretsFile = path.join(directory, 'runtime-secrets.json'); + fs.writeFileSync(secretsFile, JSON.stringify({ + version: 1, + fixture: 'physical-database-density-v1', + runtimePasswords: Object.fromEntries(manifest.customers.flatMap((customer) => + Object.values(customer.roles).map((role) => [ + role, + `test-runtime-password-${role}`, + ]) + )), + notificationPasswords: Object.fromEntries(manifest.customers.map((customer) => [ + customer.notificationRole, + `test-notification-password-${customer.notificationRole}`, + ])), + }), { mode: 0o600 }); + return secretsFile; +}; + +const unsafeStartupAdmissionFor = (manifest) => { + const attempts = PROBE_CAPABILITIES.flatMap((capability) => TENANTS.map((tenant) => ({ + capability, + tenantId: tenant.id, + rejectedCode: 'GRAPHILE_UNSAFE_RUNTIME_ROLE', + controlCredentialEnvironmentAbsent: true, + graphileBuildsStarted: 0, + residentGraphileEntries: 0, + }))); + return { + version: 2, + kind: PROBE_KIND, + admissionScope: ADMISSION_SCOPE, + provisionClone: { ...manifest.provisionClone }, + representativeCustomerId: manifest.customers[0].id, + representativePhysicalDatabase: manifest.customers[0].physicalIdentity, + representativeProvisionAttestationSha256: + manifest.customers[0].provisionAttestation.sha256, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint, + runtimeArtifactFingerprint: digest('e'), + liveProvisionAttestation: { + ...manifest.customers[0].provisionAttestation, + customerId: manifest.customers[0].id, + database: manifest.customers[0].database, + verified: true, + }, + safeStartupControl: { + tenantId: TENANTS[0].id, + accepted: true, + physicalDatabaseVerifiedBeforeRoleAudit: true, + controlCredentialEnvironmentAbsent: true, + graphileBuildsStarted: 0, + residentGraphileEntries: 0, + passed: true, + }, + roleProfileAudit: { + version: 1, + kind: 'unsafe-runtime-role-profile-audit-v1', + database: manifest.customers[0].database, + profiles: expectedAuditedProfiles(), + passed: true, + }, + cleanupAudit: { + version: 1, + kind: CLEANUP_AUDIT_KIND, + database: manifest.customers[0].database, + remainingRoles: 0, + remainingSchemas: 0, + passed: true, + }, + capabilities: [...PROBE_CAPABILITIES], + surfaces: TENANTS.map((tenant) => tenant.id), + attempts, + expectedAttempts: attempts.length, + rejectedAttempts: attempts.length, + acceptedAttempts: 0, + graphileBuildsStarted: 0, + residentGraphileEntries: 0, + passed: true, + }; +}; + +describe('aggregate physical hostile preflight', () => { + it('binds clone purpose and identity to opaque per-database nonce digests', () => { + const base = { + cloneId: preflightCloneId, + runPurpose: 'hostile-preflight', + customerId: 'physical-customer-0001', + database: 'pdc_preflight_db_0001', + nonce: '1'.repeat(64), + }; + const sha256 = provisionAttestationSha256(base); + assert.match(sha256, /^sha256:[a-f0-9]{64}$/); + assert.equal(sha256, provisionAttestationSha256(base)); + assert.notEqual(sha256, provisionAttestationSha256({ + ...base, + nonce: '2'.repeat(64), + })); + assert.notEqual(sha256, provisionAttestationSha256({ + ...base, + runPurpose: 'measurement', + })); + + const sql = fs.readFileSync(path.join(__dirname, 'provision-attestation.sql'), 'utf8'); + assert.match(sql, /CREATE SCHEMA ctf_provision_private/); + assert.match(sql, /REVOKE ALL ON SCHEMA ctf_provision_private FROM PUBLIC/); + assert.match(sql, /REVOKE ALL ON TABLE ctf_provision_private\.clone_attestation FROM PUBLIC/); + assert.doesNotMatch(sql, /GRANT .*runtime_role/i); + }); + + it('requires explicit purpose and clone identity on every physical server', () => { + const options = parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--run-purpose', 'hostile-preflight', + '--clone-id', preflightCloneId, + ]); + assert.equal(options.runPurpose, 'hostile-preflight'); + assert.equal(options.cloneId, preflightCloneId); + assert.throws(() => parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--run-purpose', 'diagnostic', + '--clone-id', preflightCloneId, + ]), /PDCF_RUN_PURPOSE_INVALID/); + assert.throws(() => parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--run-purpose', 'measurement', + ]), /CTF_ARGUMENT_REQUIRED:clone-id/); + + const manifest = makeManifest(); + assert.equal(assertProvisionCloneManifest(manifest, options), manifest.provisionClone); + assert.throws(() => assertProvisionCloneManifest(manifest, { + ...options, + runPurpose: 'measurement', + }), /PDCF_PROVISION_CLONE_MISMATCH/); + }); + + it('runs every mounted customer sequentially and rebuilds only after fleet invalidation', async (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-hostile-preflight-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const manifest = makeManifest(); + const manifestFile = path.join(temporary, 'provision.json'); + const outputFile = path.join(temporary, 'hostile-preflight.json'); + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + const secretsFile = writeSecrets(temporary, manifest); + + const counts = Object.fromEntries(manifest.customers.map((customer) => [ + customer.id, + { a: 3, b: 4, c: 5 }, + ])); + const pendingRebuilds = new Map(); + const customerOrder = []; + let activeCustomerValidations = 0; + let maxActiveCustomerValidations = 0; + const fetchImpl = async (url, options = {}) => { + const parsed = new URL(url); + if (parsed.pathname === '/__physical/status') { + return response(physicalStatusFor(manifest)); + } + const route = /^\/customer\/([^/]+)(.*)$/.exec(parsed.pathname); + assert.ok(route, `unexpected route ${parsed.pathname}`); + const customer = manifest.customers.find((candidate) => candidate.id === route[1]); + assert.ok(customer); + if (route[2] === '/__ctf/status') { + return response(childStatusFor(manifest, customer, counts[customer.id])); + } + if (route[2] === '/__ctf/control') { + const body = JSON.parse(options.body); + assert.equal(body.action, 'invalidate-all'); + pendingRebuilds.set(customer.id, new Set(TENANTS.map((tenant) => tenant.id))); + return response({ + ok: true, + action: body.action, + physicalDatabaseIdentity: customer.physicalIdentity, + }); + } + const graphqlRoute = /^\/tenant\/([abc])\/graphql$/.exec(route[2]); + assert.ok(graphqlRoute, `unexpected child route ${route[2]}`); + const tenant = TENANTS.find((candidate) => candidate.id === graphqlRoute[1]); + const pending = pendingRebuilds.get(customer.id); + if (pending?.delete(tenant.id)) counts[customer.id][tenant.id] += 1; + return response({ + data: { + tenantIdentity: tenant.token, + requestIdentity: `${tenant.token}:${tenant.databaseId}`, + physicalDatabaseIdentity: customer.physicalIdentity, + }, + }); + }; + const controlToken = 'preflight-control-value-that-is-never-persisted'; + const report = await runPhysicalHostilePreflight({ + manifestFile, + secretsFile, + baseUrl: 'http://127.0.0.1:3410', + controlToken, + arm, + mode, + preflightCloneId, + outputFile, + fetchImpl, + provenanceProvider: makeProvenance, + runUnsafeRoleMatrix: () => unsafeStartupAdmissionFor(manifest), + runCustomerValidation: async (options) => { + activeCustomerValidations += 1; + maxActiveCustomerValidations = Math.max( + maxActiveCustomerValidations, + activeCustomerValidations, + ); + customerOrder.push(options.expectedCustomerId); + assert.equal( + options.pathPrefix, + `/customer/${options.expectedCustomerId}`, + ); + const customer = manifest.customers.find( + (candidate) => candidate.id === options.expectedCustomerId, + ); + assert.equal(options.expectedPhysicalDatabaseIdentity, customer.physicalIdentity); + await Promise.resolve(); + activeCustomerValidations -= 1; + return { + passed: true, + checks: TENANTS.map((tenant) => ({ + name: `bad-role-expected-failure:${tenant.id}`, + passed: true, + })), + }; + }, + }); + + assert.equal(maxActiveCustomerValidations, 1); + assert.deepEqual(customerOrder, manifest.customers.map((customer) => customer.id)); + assert.equal(report.passed, true); + assert.equal(report.customerQualified, false); + assert.equal(report.performanceEvidence, false); + assert.equal(report.fleetInvalidateAndRebuild.invalidatedCustomers, 2); + assert.equal(report.fleetInvalidateAndRebuild.rebuiltSurfaces, 6); + assert.equal(report.observedRuntimeArtifactFingerprint, digest('e')); + assert.equal(report.observedBlueprintCompatibilityFingerprint, digest('d')); + assert.equal(report.unsafeRuntimeStartupAdmission.rejectedAttempts, 15); + assert.equal(report.unsafeRuntimeStartupAdmission.graphileBuildsStarted, 0); + assert.equal( + report.unsafeRuntimeStartupAdmission.runtimeArtifactFingerprint, + digest('e'), + ); + assert.equal(report.unsafeRuntimeStartupAdmission.cleanupAudit.passed, true); + assert.equal(report.provisionClone.id, preflightCloneId); + assert.equal(report.provisionClone.verifiedByServer, true); + assert.match(report.manifest.sha256, /^sha256:[a-f0-9]{64}$/); + assert.match(report.artifactSha256, /^sha256:[a-f0-9]{64}$/); + + const artifact = fs.readFileSync(outputFile, 'utf8'); + assert.doesNotMatch(artifact, new RegExp(controlToken)); + assert.doesNotThrow(() => assertCredentialFree(artifact)); + }); + + it('fails closed when the mounted server customer set or mode differs', () => { + const manifest = makeManifest(); + const status = physicalStatusFor(manifest); + assert.equal(validatePhysicalStatus(status, { + manifest, + arm, + mode, + preflightCloneId, + }), status); + assert.throws(() => validatePhysicalStatus({ + ...status, + introspectionMode: 'stock', + }, { manifest, arm, mode, preflightCloneId }), /PDCF_HOSTILE_PHYSICAL_STATUS_MISMATCH/); + assert.throws(() => validatePhysicalStatus({ + ...status, + customers: status.customers.slice(0, 1), + }, { manifest, arm, mode, preflightCloneId }), /PDCF_HOSTILE_CUSTOMER_SET_MISMATCH/); + assert.throws(() => validatePhysicalStatus({ + ...status, + customers: [...status.customers].reverse(), + }, { manifest, arm, mode, preflightCloneId }), /PDCF_HOSTILE_CUSTOMER_SET_MISMATCH/); + assert.throws(() => validatePhysicalStatus({ + ...status, + realtime: { + ...status.realtime, + managersExpected: 0, + }, + }, { + manifest, + arm, + mode, + preflightCloneId, + }), /PDCF_HOSTILE_PHYSICAL_STATUS_MISMATCH/); + + assert.throws(() => validatePhysicalStatus({ + ...status, + provisionClone: { + ...status.provisionClone, + verified: false, + }, + }, { + manifest, + arm, + mode, + preflightCloneId, + }), /PDCF_HOSTILE_PHYSICAL_STATUS_MISMATCH/); + }); + + it('persists a credential-free failure skeleton before any customer mutation', async (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-hostile-failure-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const manifest = makeManifest(); + const manifestFile = path.join(temporary, 'provision.json'); + const outputFile = path.join(temporary, 'hostile-preflight.json'); + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + const secretsFile = writeSecrets(temporary, manifest); + const controlToken = 'failure-control-value-that-is-never-persisted'; + let customerValidationStarted = false; + + await assert.rejects(() => runPhysicalHostilePreflight({ + manifestFile, + secretsFile, + baseUrl: 'http://127.0.0.1:3410', + controlToken, + arm, + mode, + preflightCloneId, + outputFile, + fetchImpl: async () => response({ + ...physicalStatusFor(manifest), + introspectionMode: 'stock', + }), + provenanceProvider: makeProvenance, + runUnsafeRoleMatrix: () => unsafeStartupAdmissionFor(manifest), + runCustomerValidation: async () => { + customerValidationStarted = true; + }, + }), /PDCF_HOSTILE_PHYSICAL_STATUS_MISMATCH/); + + assert.equal(customerValidationStarted, false); + const artifactText = fs.readFileSync(outputFile, 'utf8'); + const artifact = JSON.parse(artifactText); + assert.equal(artifact.passed, false); + assert.equal(artifact.customerQualified, false); + assert.equal(artifact.performanceEvidence, false); + assert.equal(artifact.failureCode, 'PDCF_HOSTILE_PHYSICAL_STATUS_MISMATCH'); + assert.deepEqual(artifact.customerValidations, []); + assert.doesNotMatch(artifactText, new RegExp(controlToken)); + assert.doesNotThrow(() => assertCredentialFree(artifact)); + }); + + it('rejects partial or post-publication unsafe-role evidence', () => { + const manifest = makeManifest(); + const valid = unsafeStartupAdmissionFor(manifest); + assert.equal(validateUnsafeRuntimeStartupAdmission(valid, manifest, digest('e')), valid); + assert.throws(() => validateUnsafeRuntimeStartupAdmission({ + ...valid, + attempts: valid.attempts.slice(1), + rejectedAttempts: valid.rejectedAttempts - 1, + }, manifest, digest('e')), /PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID/); + assert.throws(() => validateUnsafeRuntimeStartupAdmission({ + ...valid, + graphileBuildsStarted: 1, + }, manifest, digest('e')), /PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID/); + assert.throws(() => validateUnsafeRuntimeStartupAdmission({ + ...valid, + runtimeArtifactFingerprint: digest('f'), + }, manifest, digest('e')), /PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID/); + assert.throws(() => validateUnsafeRuntimeStartupAdmission({ + ...valid, + cleanupAudit: { + ...valid.cleanupAudit, + remainingRoles: 1, + passed: false, + }, + }, manifest, digest('e')), /PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID/); + assert.throws(() => validateUnsafeRuntimeStartupAdmission({ + ...valid, + safeStartupControl: { + ...valid.safeStartupControl, + accepted: false, + }, + }, manifest, digest('e')), /PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID/); + assert.throws(() => validateUnsafeRuntimeStartupAdmission({ + ...valid, + roleProfileAudit: { + ...valid.roleProfileAudit, + profiles: valid.roleProfileAudit.profiles.map((profile) => + profile.capability === 'bypassrls' + ? { ...profile, bypassRls: false } + : profile + ), + }, + }, manifest, digest('e')), /PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID/); + }); + + it('rejects a non-private runtime secrets input before network access', async (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-hostile-secrets-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const manifest = makeManifest(); + const manifestFile = path.join(temporary, 'provision.json'); + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + const secretsFile = writeSecrets(temporary, manifest); + fs.chmodSync(secretsFile, 0o644); + let fetched = false; + await assert.rejects(() => runPhysicalHostilePreflight({ + manifestFile, + secretsFile, + baseUrl: 'http://127.0.0.1:3410', + controlToken: 'private-secret-test-control-token-value', + arm, + mode, + preflightCloneId, + fetchImpl: async () => { + fetched = true; + return response({}); + }, + provenanceProvider: makeProvenance, + }), /PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE/); + assert.equal(fetched, false); + }); + + it('rejects output paths that alias manifest or secrets inputs', (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-hostile-output-alias-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const manifestFile = path.join(temporary, 'provision.json'); + const secretsFile = path.join(temporary, 'runtime-secrets.json'); + fs.writeFileSync(manifestFile, '{}'); + fs.writeFileSync(secretsFile, '{}', { mode: 0o600 }); + assert.throws( + () => assertOutputDoesNotAliasInputs(manifestFile, [manifestFile, secretsFile]), + /PDCF_HOSTILE_OUTPUT_ALIASES_INPUT/, + ); + assert.throws( + () => assertOutputDoesNotAliasInputs(secretsFile, [manifestFile, secretsFile]), + /PDCF_HOSTILE_OUTPUT_ALIASES_INPUT/, + ); + assert.equal( + assertOutputDoesNotAliasInputs( + path.join(temporary, 'hostile-preflight.json'), + [manifestFile, secretsFile], + ), + path.join(temporary, 'hostile-preflight.json'), + ); + }); +}); diff --git a/research/graphile-density/physical-database-density/physical-identity.sql b/research/graphile-density/physical-database-density/physical-identity.sql new file mode 100644 index 0000000000..c5326b3ea9 --- /dev/null +++ b/research/graphile-density/physical-database-density/physical-identity.sql @@ -0,0 +1,270 @@ +\set ON_ERROR_STOP on + +-- The function body is identical in every customer database. The returned +-- value differs because current_database() is connection-bound, which makes a +-- wrong-database route conclusive without making the canonical schema drift. +CREATE OR REPLACE FUNCTION ctf_a.physical_database_identity() +RETURNS text +LANGUAGE sql +STABLE +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ + SELECT pg_catalog.current_database()::text +$function$; + +CREATE OR REPLACE FUNCTION ctf_b.physical_database_identity() +RETURNS text +LANGUAGE sql +STABLE +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ + SELECT pg_catalog.current_database()::text +$function$; + +CREATE OR REPLACE FUNCTION ctf_c.physical_database_identity() +RETURNS text +LANGUAGE sql +STABLE +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ + SELECT pg_catalog.current_database()::text +$function$; + +-- Graphile exposes VOLATILE functions on the mutation root. Selecting this +-- fixture-only sibling in the same GraphQL mutation as uploadAppFile proves +-- which physical database executed every upload invocation, including timed +-- workload calls whose upload payload cannot carry a table-stamped column. +CREATE OR REPLACE FUNCTION ctf_a.physical_database_mutation_identity() +RETURNS text +LANGUAGE sql +VOLATILE +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ + SELECT pg_catalog.current_database()::text +$function$; + +CREATE OR REPLACE FUNCTION ctf_b.physical_database_mutation_identity() +RETURNS text +LANGUAGE sql +VOLATILE +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ + SELECT pg_catalog.current_database()::text +$function$; + +CREATE OR REPLACE FUNCTION ctf_c.physical_database_mutation_identity() +RETURNS text +LANGUAGE sql +VOLATILE +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ + SELECT pg_catalog.current_database()::text +$function$; + +-- Realtime verification must derive its database oracle inside PostgreSQL. +-- A caller-provided payload can be identical on the wrong physical database, +-- so each row carries an immutable value stamped from current_database(). The +-- BEFORE trigger overwrites both inserts and updates even if raw SQL or a +-- generated GraphQL mutation attempts to supply another value. +ALTER TABLE ctf_a.realtime_items + ADD COLUMN physical_database_identity text; +UPDATE ctf_a.realtime_items +SET physical_database_identity = pg_catalog.current_database()::text; +ALTER TABLE ctf_a.realtime_items + ALTER COLUMN physical_database_identity SET NOT NULL; + +CREATE FUNCTION ctf_a.stamp_realtime_physical_database_identity() +RETURNS trigger +LANGUAGE plpgsql +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ +BEGIN + NEW.physical_database_identity := pg_catalog.current_database()::text; + RETURN NEW; +END +$function$; + +CREATE TRIGGER realtime_items_physical_database_identity +BEFORE INSERT OR UPDATE ON ctf_a.realtime_items +FOR EACH ROW EXECUTE FUNCTION ctf_a.stamp_realtime_physical_database_identity(); + +ALTER TABLE ctf_b.realtime_items + ADD COLUMN physical_database_identity text; +UPDATE ctf_b.realtime_items +SET physical_database_identity = pg_catalog.current_database()::text; +ALTER TABLE ctf_b.realtime_items + ALTER COLUMN physical_database_identity SET NOT NULL; + +CREATE FUNCTION ctf_b.stamp_realtime_physical_database_identity() +RETURNS trigger +LANGUAGE plpgsql +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ +BEGIN + NEW.physical_database_identity := pg_catalog.current_database()::text; + RETURN NEW; +END +$function$; + +CREATE TRIGGER realtime_items_physical_database_identity +BEFORE INSERT OR UPDATE ON ctf_b.realtime_items +FOR EACH ROW EXECUTE FUNCTION ctf_b.stamp_realtime_physical_database_identity(); + +ALTER TABLE ctf_c.realtime_items + ADD COLUMN physical_database_identity text; +UPDATE ctf_c.realtime_items +SET physical_database_identity = pg_catalog.current_database()::text; +ALTER TABLE ctf_c.realtime_items + ALTER COLUMN physical_database_identity SET NOT NULL; + +CREATE FUNCTION ctf_c.stamp_realtime_physical_database_identity() +RETURNS trigger +LANGUAGE plpgsql +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ +BEGIN + NEW.physical_database_identity := pg_catalog.current_database()::text; + RETURN NEW; +END +$function$; + +CREATE TRIGGER realtime_items_physical_database_identity +BEFORE INSERT OR UPDATE ON ctf_c.realtime_items +FOR EACH ROW EXECUTE FUNCTION ctf_c.stamp_realtime_physical_database_identity(); + +-- Every capability response must carry evidence derived by the physical +-- database that executed its SQL. These columns are fixture-only and are +-- stamped in PostgreSQL, so a request payload cannot forge the oracle. The +-- same trigger also covers the tables written by the upload, bulk-mutation, +-- and function-binding plugins. +CREATE PROCEDURE pg_temp.add_physical_response_oracles(schema_name text) +LANGUAGE plpgsql +AS $procedure$ +DECLARE + table_name text; +BEGIN + IF schema_name NOT IN ('ctf_a', 'ctf_b', 'ctf_c') THEN + RAISE EXCEPTION 'PDCF_UNKNOWN_TENANT_SCHEMA:%', schema_name; + END IF; + + FOREACH table_name IN ARRAY ARRAY[ + 'documents', + 'posts', + 'posts_translations', + 'articles', + 'articles_chunks', + 'bulk_items', + 'app_files', + 'function_invocations' + ] + LOOP + EXECUTE format( + 'ALTER TABLE %I.%I ADD COLUMN physical_database_identity text', + schema_name, + table_name + ); + EXECUTE format( + 'UPDATE %I.%I SET physical_database_identity = pg_catalog.current_database()::text', + schema_name, + table_name + ); + EXECUTE format( + 'ALTER TABLE %I.%I ALTER COLUMN physical_database_identity SET NOT NULL', + schema_name, + table_name + ); + EXECUTE format( + 'ALTER TABLE %I.%I ALTER COLUMN physical_database_identity SET DEFAULT pg_catalog.current_database()::text', + schema_name, + table_name + ); + END LOOP; + + -- The i18n and RAG plugins return derived/custom shapes rather than every + -- source-table column. Stamp their returned text as an operation-specific + -- oracle in addition to the root/database field selected by the probe. + EXECUTE format( + 'UPDATE %I.posts_translations SET title = title || %L || pg_catalog.current_database()::text', + schema_name, + ' @' + ); + EXECUTE format( + 'UPDATE %I.articles_chunks SET content = content || %L || pg_catalog.current_database()::text', + schema_name, + ' @' + ); + + EXECUTE format($ddl$ + CREATE FUNCTION %I.stamp_physical_database_identity() + RETURNS trigger + LANGUAGE plpgsql + SECURITY INVOKER + SET search_path = pg_catalog + AS $function$ + BEGIN + NEW.physical_database_identity := pg_catalog.current_database()::text; + RETURN NEW; + END + $function$ + $ddl$, schema_name); + + FOREACH table_name IN ARRAY ARRAY[ + 'documents', + 'posts', + 'posts_translations', + 'articles', + 'articles_chunks', + 'bulk_items', + 'app_files', + 'function_invocations' + ] + LOOP + EXECUTE format( + 'CREATE TRIGGER %I BEFORE INSERT OR UPDATE ON %I.%I FOR EACH ROW EXECUTE FUNCTION %I.stamp_physical_database_identity()', + table_name || '_physical_database_identity', + schema_name, + table_name, + schema_name + ); + END LOOP; +END +$procedure$; + +CALL pg_temp.add_physical_response_oracles('ctf_a'); +CALL pg_temp.add_physical_response_oracles('ctf_b'); +CALL pg_temp.add_physical_response_oracles('ctf_c'); + +REVOKE ALL ON FUNCTION ctf_a.physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_b.physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_c.physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_a.physical_database_mutation_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_b.physical_database_mutation_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_c.physical_database_mutation_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_a.stamp_realtime_physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_b.stamp_realtime_physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_c.stamp_realtime_physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_a.stamp_physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_b.stamp_physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_c.stamp_physical_database_identity() FROM PUBLIC; + +GRANT EXECUTE ON FUNCTION ctf_a.physical_database_identity() TO :"runtime_role_a"; +GRANT EXECUTE ON FUNCTION ctf_b.physical_database_identity() TO :"runtime_role_b"; +GRANT EXECUTE ON FUNCTION ctf_c.physical_database_identity() TO :"runtime_role_c"; +GRANT EXECUTE ON FUNCTION ctf_a.physical_database_mutation_identity() TO :"runtime_role_a"; +GRANT EXECUTE ON FUNCTION ctf_b.physical_database_mutation_identity() TO :"runtime_role_b"; +GRANT EXECUTE ON FUNCTION ctf_c.physical_database_mutation_identity() TO :"runtime_role_c"; +GRANT EXECUTE ON FUNCTION ctf_a.stamp_realtime_physical_database_identity() TO :"runtime_role_a"; +GRANT EXECUTE ON FUNCTION ctf_b.stamp_realtime_physical_database_identity() TO :"runtime_role_b"; +GRANT EXECUTE ON FUNCTION ctf_c.stamp_realtime_physical_database_identity() TO :"runtime_role_c"; +GRANT EXECUTE ON FUNCTION ctf_a.stamp_physical_database_identity() TO :"runtime_role_a"; +GRANT EXECUTE ON FUNCTION ctf_b.stamp_physical_database_identity() TO :"runtime_role_b"; +GRANT EXECUTE ON FUNCTION ctf_c.stamp_physical_database_identity() TO :"runtime_role_c"; diff --git a/research/graphile-density/physical-database-density/prepare-measurement-run.cjs b/research/graphile-density/physical-database-density/prepare-measurement-run.cjs new file mode 100644 index 0000000000..ac762b393f --- /dev/null +++ b/research/graphile-density/physical-database-density/prepare-measurement-run.cjs @@ -0,0 +1,672 @@ +'use strict'; + +const { execFileSync, spawnSync } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + FIXTURE_ID, + loadProvision, +} = require('./lib.cjs'); +const { + parseArgs, + parsePositiveInteger, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); +const { provision } = require('./provision.cjs'); + +const TEMPLATE_KIND = 'physical-density-postgres-container-template-v1'; +const PREPARE_KIND = 'physical-density-measurement-prepare-v1'; +const CONTAINER_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/; +const CONTAINER_ID = /^[a-f0-9]{64}$/; +const IMAGE_ID = /^sha256:[a-f0-9]{64}$/; +const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost']); +const LABEL_FIXTURE = 'io.constructive.graphile-density.fixture'; +const LABEL_PREFIX = 'io.constructive.graphile-density.prefix'; +const LABEL_PURPOSE = 'io.constructive.graphile-density.purpose'; +const POSTGRES_SETTINGS = new Set([ + 'effective_cache_size', + 'maintenance_work_mem', + 'max_connections', + 'max_locks_per_transaction', + 'shared_buffers', + 'shared_preload_libraries', + 'track_io_timing', + 'work_mem', +]); +const POSTGRES_DATA_DIRECTORY = '/var/lib/postgresql/data'; + +const readRegularFile = (file) => { + const absolute = path.resolve(file); + const before = fs.lstatSync(absolute); + if (before.isSymbolicLink() || !before.isFile()) { + throw new Error('PDCF_MEASUREMENT_TEMPLATE_FILE_INVALID'); + } + const descriptor = fs.openSync( + absolute, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + try { + const opened = fs.fstatSync(descriptor); + if ( + !opened.isFile() + || opened.dev !== before.dev + || opened.ino !== before.ino + ) { + throw new Error('PDCF_MEASUREMENT_TEMPLATE_FILE_INVALID'); + } + return fs.readFileSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +}; + +const fileSha256 = (file) => `sha256:${crypto.createHash('sha256') + .update(readRegularFile(file)) + .digest('hex')}`; +const bufferSha256 = (value) => `sha256:${crypto.createHash('sha256') + .update(value) + .digest('hex')}`; + +const inspectDockerContainer = (container) => { + try { + const output = execFileSync('docker', ['inspect', container], { + encoding: 'utf8', + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }); + const records = JSON.parse(output); + return Array.isArray(records) && records.length === 1 ? records[0] : null; + } catch { + return null; + } +}; + +const requireNonnegativeInteger = (value, label) => { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`PDCF_CONTAINER_TEMPLATE_${label}_INVALID`); + } + return value; +}; + +const normalizePostgresCommand = (command, minimumMaxConnections) => { + const source = Array.isArray(command) && command.length > 0 + ? [...command] + : ['postgres']; + if ( + source[0] !== 'postgres' + || (source.length - 1) % 2 !== 0 + || !Number.isSafeInteger(minimumMaxConnections) + || minimumMaxConnections <= 0 + ) { + throw new Error('PDCF_CONTAINER_POSTGRES_COMMAND_INVALID'); + } + const settings = new Map(); + for (let index = 1; index < source.length; index += 2) { + const flag = source[index]; + const assignment = source[index + 1]; + const match = /^([a-z_]+)=([a-zA-Z0-9_.,:/+-]+)$/.exec(assignment ?? ''); + if ( + flag !== '-c' + || !match + || !POSTGRES_SETTINGS.has(match[1]) + || settings.has(match[1]) + ) { + throw new Error('PDCF_CONTAINER_POSTGRES_COMMAND_INVALID'); + } + settings.set(match[1], match[2]); + } + const configuredMax = Number(settings.get('max_connections')); + if (settings.has('max_connections') && ( + !Number.isSafeInteger(configuredMax) + || configuredMax < minimumMaxConnections + )) { + throw new Error('PDCF_CONTAINER_MAX_CONNECTIONS_INSUFFICIENT'); + } + if (!settings.has('max_connections')) { + settings.set('max_connections', String(minimumMaxConnections)); + } + return [ + 'postgres', + ...[...settings].sort(([left], [right]) => left.localeCompare(right)) + .flatMap(([name, value]) => ['-c', `${name}=${value}`]), + ]; +}; + +const postgresSettingsFromCommand = (command) => Object.fromEntries( + Array.from({ length: (command.length - 1) / 2 }, (_, index) => { + const [name, value] = command[index * 2 + 2].split('=', 2); + return [name, value]; + }), +); + +const captureContainerTemplate = ({ + inspection, + container, + prefix, + pgHost, + pgPort, + minimumMaxConnections, +}) => { + const name = String(inspection?.Name ?? '').replace(/^\//, ''); + const bindings = inspection?.NetworkSettings?.Ports?.['5432/tcp']; + const entrypoint = inspection?.Config?.Entrypoint ?? null; + const mounts = inspection?.Mounts ?? []; + if ( + !CONTAINER_NAME.test(container ?? '') + || !/^[a-z][a-z0-9_]{0,47}$/.test(prefix ?? '') + || !LOOPBACK_HOSTS.has(pgHost) + || !Number.isSafeInteger(pgPort) + || pgPort <= 0 + || pgPort > 65535 + || name !== container + || !CONTAINER_ID.test(inspection?.Id ?? '') + || !IMAGE_ID.test(inspection?.Image ?? '') + || inspection?.State?.Running !== true + || !Array.isArray(bindings) + || !bindings.some((binding) => + String(binding?.HostPort) === String(pgPort) + && LOOPBACK_HOSTS.has(binding?.HostIp ?? '') + ) + || !Array.isArray(mounts) + || mounts.some((mount) => mount?.Destination !== POSTGRES_DATA_DIRECTORY) + || ( + entrypoint != null + && ( + !Array.isArray(entrypoint) + || entrypoint.length !== 1 + || !/^[a-zA-Z0-9_./-]+$/.test(entrypoint[0] ?? '') + ) + ) + ) { + throw new Error('PDCF_CONTAINER_TEMPLATE_SOURCE_INVALID'); + } + const template = { + version: 1, + fixture: FIXTURE_ID, + kind: TEMPLATE_KIND, + containerName: container, + sourceContainerId: inspection.Id, + imageId: inspection.Image, + prefix, + pgHost, + pgPort, + entrypoint, + postgresCommand: normalizePostgresCommand( + inspection.Config?.Cmd, + minimumMaxConnections, + ), + resourceLimits: { + memoryBytes: requireNonnegativeInteger( + inspection.HostConfig?.Memory ?? 0, + 'MEMORY', + ), + memorySwapBytes: requireNonnegativeInteger( + inspection.HostConfig?.MemorySwap ?? 0, + 'MEMORY_SWAP', + ), + nanoCpus: requireNonnegativeInteger( + inspection.HostConfig?.NanoCpus ?? 0, + 'NANO_CPUS', + ), + shmSizeBytes: requireNonnegativeInteger( + inspection.HostConfig?.ShmSize ?? 0, + 'SHM_SIZE', + ), + }, + }; + return validateContainerTemplate(template); +}; + +const validateContainerTemplate = (template) => { + if ( + JSON.stringify(Object.keys(template ?? {}).sort()) !== JSON.stringify([ + 'containerName', + 'entrypoint', + 'fixture', + 'imageId', + 'kind', + 'pgHost', + 'pgPort', + 'postgresCommand', + 'prefix', + 'resourceLimits', + 'sourceContainerId', + 'version', + ]) + || JSON.stringify(Object.keys(template?.resourceLimits ?? {}).sort()) + !== JSON.stringify([ + 'memoryBytes', + 'memorySwapBytes', + 'nanoCpus', + 'shmSizeBytes', + ]) + || + template?.version !== 1 + || template.fixture !== FIXTURE_ID + || template.kind !== TEMPLATE_KIND + || !CONTAINER_NAME.test(template.containerName ?? '') + || !CONTAINER_ID.test(template.sourceContainerId ?? '') + || !IMAGE_ID.test(template.imageId ?? '') + || !/^[a-z][a-z0-9_]{0,47}$/.test(template.prefix ?? '') + || !LOOPBACK_HOSTS.has(template.pgHost) + || !Number.isSafeInteger(template.pgPort) + || template.pgPort <= 0 + || template.pgPort > 65535 + || !template.resourceLimits + || !Array.isArray(template.postgresCommand) + || ( + template.entrypoint != null + && ( + !Array.isArray(template.entrypoint) + || template.entrypoint.length !== 1 + || !/^[a-zA-Z0-9_./-]+$/.test(template.entrypoint[0] ?? '') + ) + ) + ) { + throw new Error('PDCF_CONTAINER_TEMPLATE_INVALID'); + } + for (const [key, value] of Object.entries(template.resourceLimits)) { + requireNonnegativeInteger(value, key.toUpperCase()); + } + normalizePostgresCommand( + template.postgresCommand, + Number(postgresSettingsFromCommand(template.postgresCommand).max_connections), + ); + return template; +}; + +const validateExistingTarget = (inspection, template) => { + if (!inspection) return null; + const name = String(inspection.Name ?? '').replace(/^\//, ''); + const labels = inspection.Config?.Labels ?? {}; + const ownedReplacement = labels[LABEL_FIXTURE] === FIXTURE_ID + && labels[LABEL_PREFIX] === template.prefix + && labels[LABEL_PURPOSE] === 'measurement' + && inspection.Image === template.imageId; + if ( + name !== template.containerName + || !CONTAINER_ID.test(inspection.Id ?? '') + || (inspection.Id !== template.sourceContainerId && !ownedReplacement) + ) { + throw new Error('PDCF_CONTAINER_RECREATE_TARGET_NOT_OWNED'); + } + return inspection.Id; +}; + +const dockerRunArgs = (template, environment) => { + const pgUser = environment.PGUSER; + const pgPassword = environment.PGPASSWORD; + const maintenanceDatabase = environment.PGDATABASE ?? 'postgres'; + if ( + typeof pgUser !== 'string' + || !pgUser + || typeof pgPassword !== 'string' + || !pgPassword + || typeof maintenanceDatabase !== 'string' + || !maintenanceDatabase + ) { + throw new Error('PDCF_CONTAINER_ADMIN_CREDENTIALS_REQUIRED'); + } + const limits = template.resourceLimits; + const args = [ + 'run', '--detach', + '--name', template.containerName, + '--label', `${LABEL_FIXTURE}=${FIXTURE_ID}`, + '--label', `${LABEL_PREFIX}=${template.prefix}`, + '--label', `${LABEL_PURPOSE}=measurement`, + '--publish', `127.0.0.1:${template.pgPort}:5432`, + '--env', 'POSTGRES_USER', + '--env', 'POSTGRES_PASSWORD', + '--env', 'POSTGRES_DB', + ]; + if (template.entrypoint) { + args.push('--entrypoint', template.entrypoint[0]); + } + if (limits.memoryBytes > 0) args.push('--memory', String(limits.memoryBytes)); + if (limits.memorySwapBytes > 0) { + args.push('--memory-swap', String(limits.memorySwapBytes)); + } + if (limits.nanoCpus > 0) args.push('--cpus', String(limits.nanoCpus / 1e9)); + if (limits.shmSizeBytes > 0) args.push('--shm-size', String(limits.shmSizeBytes)); + args.push(template.imageId, ...template.postgresCommand); + return args; +}; + +const validateRunningContainerAgainstTemplate = (inspection, template) => { + const labels = inspection?.Config?.Labels ?? {}; + const bindings = inspection?.NetworkSettings?.Ports?.['5432/tcp']; + if ( + !CONTAINER_ID.test(inspection?.Id ?? '') + || inspection.Image !== template.imageId + || inspection.State?.Running !== true + || JSON.stringify(inspection.Config?.Cmd ?? []) + !== JSON.stringify(template.postgresCommand) + || JSON.stringify(inspection.Config?.Entrypoint ?? null) + !== JSON.stringify(template.entrypoint) + || inspection.HostConfig?.Memory !== template.resourceLimits.memoryBytes + || inspection.HostConfig?.MemorySwap !== template.resourceLimits.memorySwapBytes + || inspection.HostConfig?.NanoCpus !== template.resourceLimits.nanoCpus + || inspection.HostConfig?.ShmSize !== template.resourceLimits.shmSizeBytes + || !Array.isArray(bindings) + || !bindings.some((binding) => + String(binding?.HostPort) === String(template.pgPort) + && LOOPBACK_HOSTS.has(binding?.HostIp ?? '') + ) + || labels[LABEL_FIXTURE] !== FIXTURE_ID + || labels[LABEL_PREFIX] !== template.prefix + || labels[LABEL_PURPOSE] !== 'measurement' + ) { + throw new Error('PDCF_FRESH_POSTGRES_CONTAINER_IDENTITY_INVALID'); + } + return inspection; +}; + +const dockerExec = (args, environment = process.env) => { + try { + return execFileSync('docker', args, { + encoding: 'utf8', + timeout: 120_000, + maxBuffer: 4 * 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + env: environment, + }); + } catch { + // Keep Docker diagnostics generic. The disposable admin password is passed + // only through the child environment and must never enter cperf logs. + throw new Error('PDCF_DOCKER_LIFECYCLE_COMMAND_FAILED'); + } +}; + +const waitForPostgres = ({ environment, timeoutMs = 120_000 }) => { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const result = spawnSync('psql', [ + '--no-psqlrc', + '--no-align', + '--tuples-only', + '--quiet', + '--set=ON_ERROR_STOP=1', + '--command', 'SELECT 1', + ], { + cwd: __dirname, + env: environment, + encoding: 'utf8', + timeout: 5_000, + }); + if (result.status === 0 && String(result.stdout).trim() === '1') return; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 250); + } + throw new Error('PDCF_FRESH_POSTGRES_READINESS_TIMEOUT'); +}; + +const runBindingCloneId = (run, randomBytes = crypto.randomBytes) => { + const coordinate = [ + run.arm, + run.heapMiB, + run.customerCount, + run.repetition, + run.runOrderIndex, + ].join('\0'); + const coordinateHash = crypto.createHash('sha256').update(coordinate).digest('hex'); + return `measurement-${coordinateHash.slice(0, 20)}-${randomBytes(8).toString('hex')}`; +}; + +const publishExclusive = (source, destination, mode) => { + fs.linkSync(source, destination); + if (mode != null) fs.chmodSync(destination, mode); + fs.unlinkSync(source); +}; + +const writeImmutableJson = (file, value, mode = 0o644) => { + const temporary = `${file}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`; + try { + fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { + flag: 'wx', + mode, + }); + publishExclusive(temporary, file, mode); + } finally { + try { fs.unlinkSync(temporary); } catch { /* Preserve the primary error. */ } + } +}; + +const prepareMeasurementRun = ({ + containerTemplateFile, + expectedContainerTemplateSha256, + manifestTemplateFile, + secretsTemplateFile, + expectedManifestTemplateSha256, + artifactDir, + run, + environment = process.env, +}, dependencies = {}) => { + const containerTemplateBytes = readRegularFile(containerTemplateFile); + if ( + !/^sha256:[a-f0-9]{64}$/.test(expectedContainerTemplateSha256 ?? '') + || bufferSha256(containerTemplateBytes) !== expectedContainerTemplateSha256 + ) { + throw new Error('PDCF_CONTAINER_TEMPLATE_SHA256_MISMATCH'); + } + const template = validateContainerTemplate(JSON.parse( + containerTemplateBytes.toString('utf8'), + )); + const manifestTemplateBytes = readRegularFile(manifestTemplateFile); + if ( + !/^sha256:[a-f0-9]{64}$/.test(expectedManifestTemplateSha256 ?? '') + || bufferSha256(manifestTemplateBytes) !== expectedManifestTemplateSha256 + ) { + throw new Error('PDCF_MEASUREMENT_MANIFEST_TEMPLATE_SHA256_MISMATCH'); + } + const absoluteArtifactDir = path.resolve(artifactDir); + fs.mkdirSync(absoluteArtifactDir, { recursive: true, mode: 0o700 }); + const manifestOut = path.join(absoluteArtifactDir, 'provision.json'); + const secretsOut = path.join(absoluteArtifactDir, 'runtime-secrets.json'); + const prepareOut = path.join(absoluteArtifactDir, 'prepare-attestation.json'); + if ([manifestOut, secretsOut, prepareOut].some((file) => fs.existsSync(file))) { + throw new Error('PDCF_FRESH_POSTGRES_ARTIFACT_ALREADY_EXISTS'); + } + if ( + !run + || typeof run.arm !== 'string' + || !run.arm + || !Number.isSafeInteger(run.heapMiB) + || run.heapMiB <= 0 + || !Number.isSafeInteger(run.customerCount) + || run.customerCount <= 0 + || !Number.isSafeInteger(run.repetition) + || run.repetition <= 0 + || !Number.isSafeInteger(run.runOrderIndex) + || run.runOrderIndex <= 0 + ) { + throw new Error('PDCF_MEASUREMENT_PREPARE_RUN_BINDING_INVALID'); + } + const { manifest: sourceManifest, secretResolver } = ( + dependencies.loadProvision ?? loadProvision + )(manifestTemplateFile, secretsTemplateFile); + if (fileSha256(manifestTemplateFile) !== expectedManifestTemplateSha256) { + throw new Error('PDCF_MEASUREMENT_MANIFEST_TEMPLATE_SHA256_MISMATCH'); + } + if ( + sourceManifest.prefix !== template.prefix + || sourceManifest.customers.length < run.customerCount + || sourceManifest.provisionClone?.purpose !== 'measurement' + ) { + throw new Error('PDCF_MEASUREMENT_PREPARE_TEMPLATE_MISMATCH'); + } + const selectedCustomers = sourceManifest.customers.slice(0, run.customerCount); + const credentialTemplate = { + runtimePasswords: Object.fromEntries(selectedCustomers.flatMap((customer) => + Object.values(customer.roles).map((role) => [ + role, + secretResolver.runtimePasswordFor(role), + ]) + )), + notificationPasswords: Object.fromEntries(selectedCustomers.map((customer) => [ + customer.notificationRole, + secretResolver.notificationPasswordFor(customer.notificationRole), + ])), + }; + const inspect = dependencies.inspectDockerContainer ?? inspectDockerContainer; + const removeContainer = dependencies.removeContainer + ?? ((container) => dockerExec(['container', 'rm', '--force', '--volumes', container])); + const startContainer = dependencies.startContainer + ?? ((args) => dockerExec(args, { + ...process.env, + POSTGRES_USER: environment.PGUSER, + POSTGRES_PASSWORD: environment.PGPASSWORD, + POSTGRES_DB: environment.PGDATABASE ?? 'postgres', + })); + const existing = inspect(template.containerName); + const existingTargetId = validateExistingTarget(existing, template); + if (existingTargetId) { + // Remove the immutable ID we just attested, not the mutable container name; + // this closes the inspect/remove name-swap window around a destructive call. + removeContainer(existingTargetId); + } + startContainer(dockerRunArgs(template, environment)); + const postgresEnvironment = { + ...environment, + PGHOST: template.pgHost, + PGPORT: String(template.pgPort), + }; + (dependencies.waitForPostgres ?? waitForPostgres)({ + environment: postgresEnvironment, + }); + + const cloneId = runBindingCloneId(run, dependencies.randomBytes); + const stagingDir = fs.mkdtempSync(path.join(absoluteArtifactDir, '.prepare-')); + let provisioned; + try { + provisioned = (dependencies.provision ?? provision)({ + prefix: template.prefix, + customerCount: run.customerCount, + outDir: stagingDir, + maintenanceDatabase: environment.PGDATABASE ?? 'postgres', + schemaFile: path.join(__dirname, '../complete-tenant-fixture/schema.sql'), + identityFile: path.join(__dirname, 'physical-identity.sql'), + attestationFile: path.join(__dirname, 'provision-attestation.sql'), + cloneId, + runPurpose: 'measurement', + recreate: false, + environment: postgresEnvironment, + canonicalSchemas: sourceManifest.canonicalSchemas, + credentialTemplate, + }); + if ( + provisioned.manifest.canonicalDatabaseContractFingerprint + !== sourceManifest.canonicalDatabaseContractFingerprint + || JSON.stringify(provisioned.manifest.canonicalStructuralFingerprint) + !== JSON.stringify(sourceManifest.canonicalStructuralFingerprint) + || JSON.stringify(provisioned.manifest.canonicalSchemas) + !== JSON.stringify(sourceManifest.canonicalSchemas) + ) { + throw new Error('PDCF_FRESH_POSTGRES_CONTRACT_DRIFT'); + } + publishExclusive(provisioned.secretsFile, secretsOut, 0o600); + publishExclusive(provisioned.manifestFile, manifestOut, 0o644); + fs.rmdirSync(stagingDir); + } catch (error) { + try { + if (fs.existsSync(stagingDir)) fs.rmSync(stagingDir, { recursive: true }); + } catch { + // Preserve the primary preparation error. + } + throw error; + } + const current = validateRunningContainerAgainstTemplate( + inspect(template.containerName), + template, + ); + if ( + current.Id === existing?.Id + ) { + throw new Error('PDCF_FRESH_POSTGRES_CONTAINER_IDENTITY_INVALID'); + } + const result = { + version: 1, + fixture: FIXTURE_ID, + kind: PREPARE_KIND, + preparedAt: new Date().toISOString(), + run, + container: { + name: template.containerName, + id: current.Id, + imageId: current.Image, + }, + cloneId, + manifestFile: manifestOut, + manifestSha256: fileSha256(manifestOut), + secretsFile: secretsOut, + credentialTemplate: 'private-0600-reused-without-serialization', + }; + writeImmutableJson(prepareOut, result); + return result; +}; + +const main = () => { + const args = parseArgs(process.argv.slice(2)); + const result = prepareMeasurementRun({ + containerTemplateFile: path.resolve(requireString(args, 'container-template')), + expectedContainerTemplateSha256: requireString( + args, + 'expected-container-template-sha256', + ), + manifestTemplateFile: path.resolve(requireString(args, 'manifest-template')), + secretsTemplateFile: path.resolve(requireString(args, 'secrets-template')), + expectedManifestTemplateSha256: requireString( + args, + 'expected-manifest-template-sha256', + ), + artifactDir: path.resolve(requireString(args, 'artifact-dir')), + run: { + arm: requireString(args, 'arm'), + heapMiB: parsePositiveInteger(requireString(args, 'heap-mib'), 'heap-mib'), + customerCount: parsePositiveInteger( + requireString(args, 'customers'), + 'customers', + ), + repetition: parsePositiveInteger( + requireString(args, 'repetition'), + 'repetition', + ), + runOrderIndex: parsePositiveInteger( + requireString(args, 'run-order-index'), + 'run-order-index', + ), + }, + }); + process.stdout.write(`${JSON.stringify({ + status: 'prepared', + containerId: result.container.id, + cloneId: result.cloneId, + manifestSha256: result.manifestSha256, + })}\n`); +}; + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} + +module.exports = { + PREPARE_KIND, + TEMPLATE_KIND, + captureContainerTemplate, + dockerRunArgs, + inspectDockerContainer, + normalizePostgresCommand, + postgresSettingsFromCommand, + prepareMeasurementRun, + publishExclusive, + runBindingCloneId, + validateContainerTemplate, + validateExistingTarget, + validateRunningContainerAgainstTemplate, + writeImmutableJson, +}; diff --git a/research/graphile-density/physical-database-density/prepare-measurement-run.test.cjs b/research/graphile-density/physical-database-density/prepare-measurement-run.test.cjs new file mode 100644 index 0000000000..2760498e7a --- /dev/null +++ b/research/graphile-density/physical-database-density/prepare-measurement-run.test.cjs @@ -0,0 +1,240 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { + captureContainerTemplate, + dockerRunArgs, + prepareMeasurementRun, + validateExistingTarget, +} = require('./prepare-measurement-run.cjs'); + +const sha256 = (file) => `sha256:${crypto.createHash('sha256') + .update(fs.readFileSync(file)) + .digest('hex')}`; + +const sourceInspection = ({ id = '1'.repeat(64), labels = {} } = {}) => ({ + Id: id, + Name: '/postgres-density-exact', + Image: `sha256:${'2'.repeat(64)}`, + State: { Running: true }, + Config: { + Cmd: [ + 'postgres', + '-c', 'shared_buffers=256MB', + '-c', 'max_connections=240', + ], + Labels: labels, + }, + HostConfig: { + Memory: 2 * 1024 ** 3, + MemorySwap: 2 * 1024 ** 3, + NanoCpus: 2_000_000_000, + ShmSize: 256 * 1024 ** 2, + }, + NetworkSettings: { + Ports: { '5432/tcp': [{ HostIp: '127.0.0.1', HostPort: '55432' }] }, + }, +}); + +const templateFrom = (inspection = sourceInspection()) => captureContainerTemplate({ + inspection, + container: 'postgres-density-exact', + prefix: 'pdc_test', + pgHost: '127.0.0.1', + pgPort: 55432, + minimumMaxConnections: 120, +}); + +describe('fresh PostgreSQL measurement preparation', () => { + it('preserves validated PostgreSQL settings and never falls back to default max_connections', () => { + const template = templateFrom(); + assert.deepEqual(template.postgresCommand, [ + 'postgres', + '-c', 'max_connections=240', + '-c', 'shared_buffers=256MB', + ]); + const args = dockerRunArgs(template, { + PGUSER: 'fixture_admin', + PGPASSWORD: 'private-admin-password', + PGDATABASE: 'postgres', + }); + assert.deepEqual(args.slice(args.indexOf(template.imageId)), [ + template.imageId, + ...template.postgresCommand, + ]); + assert.ok(args.includes('max_connections=240')); + assert.ok(args.includes('shared_buffers=256MB')); + assert.equal(args.some((argument) => argument.includes('private-admin-password')), false); + + const defaultTemplate = templateFrom({ + ...sourceInspection(), + Config: { Cmd: ['postgres'], Labels: {} }, + }); + assert.ok(defaultTemplate.postgresCommand.includes('max_connections=120')); + assert.throws(() => templateFrom({ + ...sourceInspection(), + Config: { + Cmd: ['postgres', '-c', 'max_connections=80'], + Labels: {}, + }, + }), /PDCF_CONTAINER_MAX_CONNECTIONS_INSUFFICIENT/); + }); + + it('will remove only the captured container or its exact owned replacement', () => { + const template = templateFrom(); + assert.equal(validateExistingTarget(sourceInspection(), template), '1'.repeat(64)); + assert.throws(() => validateExistingTarget(sourceInspection({ + id: '3'.repeat(64), + }), template), /PDCF_CONTAINER_RECREATE_TARGET_NOT_OWNED/); + assert.equal(validateExistingTarget(sourceInspection({ + id: '3'.repeat(64), + labels: { + 'io.constructive.graphile-density.fixture': + 'physical-database-density-v1', + 'io.constructive.graphile-density.prefix': 'pdc_test', + 'io.constructive.graphile-density.purpose': 'measurement', + }, + }), template), '3'.repeat(64)); + }); + + it('publishes run-local inputs with stable private credentials and a unique clone', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-prepare-')); + const artifactDir = path.join(directory, 'artifact'); + const containerTemplateFile = path.join(directory, 'container-template.json'); + const manifestTemplateFile = path.join(directory, 'provision.json'); + const secretsTemplateFile = path.join(directory, 'runtime-secrets.json'); + const template = templateFrom(); + fs.writeFileSync(containerTemplateFile, JSON.stringify(template)); + fs.writeFileSync(manifestTemplateFile, JSON.stringify({ template: true })); + fs.writeFileSync(secretsTemplateFile, '{}', { mode: 0o600 }); + const customer = { + id: 'physical-customer-0001', + roles: { a: 'role_a', b: 'role_b', c: 'role_c' }, + notificationRole: 'role_notify', + }; + const unselectedCustomer = { + id: 'physical-customer-0002', + roles: { a: 'role_2a', b: 'role_2b', c: 'role_2c' }, + notificationRole: 'role_2notify', + }; + const sourceManifest = { + prefix: 'pdc_test', + canonicalSchemas: ['ctf_a'], + canonicalStructuralFingerprint: { combined: { sha256: `sha256:${'4'.repeat(64)}` } }, + canonicalDatabaseContractFingerprint: `sha256:${'5'.repeat(64)}`, + provisionClone: { purpose: 'measurement' }, + customers: [customer, unselectedCustomer], + }; + const passwords = { + role_a: 'stable-password-role-a-123456', + role_b: 'stable-password-role-b-123456', + role_c: 'stable-password-role-c-123456', + role_notify: 'stable-password-notify-12345', + role_2a: 'unselected-password-role-2a', + role_2b: 'unselected-password-role-2b', + role_2c: 'unselected-password-role-2c', + role_2notify: 'unselected-password-notify-2', + }; + let inspectionCount = 0; + const replacementSource = sourceInspection({ + id: '6'.repeat(64), + labels: { + 'io.constructive.graphile-density.fixture': + 'physical-database-density-v1', + 'io.constructive.graphile-density.prefix': 'pdc_test', + 'io.constructive.graphile-density.purpose': 'measurement', + }, + }); + const replacement = { + ...replacementSource, + Config: { + ...replacementSource.Config, + Cmd: template.postgresCommand, + }, + }; + const removed = []; + const result = prepareMeasurementRun({ + containerTemplateFile, + expectedContainerTemplateSha256: sha256(containerTemplateFile), + manifestTemplateFile, + secretsTemplateFile, + expectedManifestTemplateSha256: sha256(manifestTemplateFile), + artifactDir, + run: { + arm: 'candidate', + heapMiB: 2048, + customerCount: 1, + repetition: 1, + runOrderIndex: 3, + }, + environment: { + PGHOST: '127.0.0.1', + PGPORT: '55432', + PGUSER: 'fixture_admin', + PGPASSWORD: 'admin-password', + PGDATABASE: 'postgres', + }, + }, { + inspectDockerContainer: () => inspectionCount++ === 0 + ? sourceInspection() + : replacement, + removeContainer: (container) => removed.push(container), + startContainer: () => undefined, + waitForPostgres: () => undefined, + randomBytes: () => Buffer.alloc(8, 7), + loadProvision: () => ({ + manifest: sourceManifest, + secretResolver: { + runtimePasswordFor: (role) => passwords[role], + notificationPasswordFor: (role) => passwords[role], + }, + }), + provision: (options) => { + assert.equal(options.customerCount, 1); + assert.deepEqual(options.credentialTemplate, { + runtimePasswords: { + role_a: passwords.role_a, + role_b: passwords.role_b, + role_c: passwords.role_c, + }, + notificationPasswords: { role_notify: passwords.role_notify }, + }); + const manifestFile = path.join(options.outDir, 'provision.json'); + const secretsFile = path.join(options.outDir, 'runtime-secrets.json'); + fs.writeFileSync(manifestFile, JSON.stringify({ cloneId: options.cloneId })); + fs.writeFileSync(secretsFile, JSON.stringify(passwords), { mode: 0o600 }); + return { + manifest: { + canonicalSchemas: sourceManifest.canonicalSchemas, + canonicalStructuralFingerprint: + sourceManifest.canonicalStructuralFingerprint, + canonicalDatabaseContractFingerprint: + sourceManifest.canonicalDatabaseContractFingerprint, + }, + manifestFile, + secretsFile, + }; + }, + }); + assert.deepEqual(removed, ['1'.repeat(64)]); + assert.equal(result.container.id, '6'.repeat(64)); + assert.match(result.cloneId, /^measurement-[a-f0-9]{20}-0707070707070707$/); + assert.equal(fs.statSync(result.secretsFile).mode & 0o777, 0o600); + const serialized = JSON.stringify({ + result, + prepare: JSON.parse(fs.readFileSync( + path.join(artifactDir, 'prepare-attestation.json'), + 'utf8', + )), + }); + for (const password of Object.values(passwords)) { + assert.doesNotMatch(serialized, new RegExp(password)); + } + }); +}); diff --git a/research/graphile-density/physical-database-density/provision-attestation.sql b/research/graphile-density/physical-database-density/provision-attestation.sql new file mode 100644 index 0000000000..6f977601de --- /dev/null +++ b/research/graphile-density/physical-database-density/provision-attestation.sql @@ -0,0 +1,35 @@ +\set ON_ERROR_STOP on + +BEGIN; + +CREATE SCHEMA ctf_provision_private; +REVOKE ALL ON SCHEMA ctf_provision_private FROM PUBLIC; + +CREATE TABLE ctf_provision_private.clone_attestation ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + clone_id text NOT NULL CHECK (clone_id ~ '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$'), + run_purpose text NOT NULL CHECK (run_purpose IN ('hostile-preflight', 'measurement')), + customer_id text NOT NULL CHECK (customer_id ~ '^[a-z0-9-]+$'), + attestation_nonce text NOT NULL CHECK (attestation_nonce ~ '^[a-f0-9]{64}$'), + attestation_sha256 text NOT NULL CHECK (attestation_sha256 ~ '^sha256:[a-f0-9]{64}$') +); + +REVOKE ALL ON TABLE ctf_provision_private.clone_attestation FROM PUBLIC; + +INSERT INTO ctf_provision_private.clone_attestation ( + singleton, + clone_id, + run_purpose, + customer_id, + attestation_nonce, + attestation_sha256 +) VALUES ( + true, + :'clone_id', + :'run_purpose', + :'customer_id', + :'attestation_nonce', + :'attestation_sha256' +); + +COMMIT; diff --git a/research/graphile-density/physical-database-density/provision.cjs b/research/graphile-density/physical-database-density/provision.cjs new file mode 100644 index 0000000000..b6240f5c32 --- /dev/null +++ b/research/graphile-density/physical-database-density/provision.cjs @@ -0,0 +1,837 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + FIXTURE_DIR, + FIXTURE_ID, + atomicWriteJson, + makeCustomers, +} = require('./lib.cjs'); +const { + TENANTS, + parseArgs, + parsePositiveInteger, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); + +const DEFAULT_CANONICAL_SCHEMAS = Object.freeze([ + // These shared schemas are visible to every build through type/function + // dependencies, so a tenant-only fingerprint would be insufficient proof + // that two physical databases are blueprint-compatible. + 'ctf_extensions', + 'ctf_a', + 'ctf_a_realtime', + 'ctf_b', + 'ctf_b_realtime', + 'ctf_c', + 'ctf_c_realtime', + 'jwt_private', +]); +const REQUIRED_EXTENSIONS = Object.freeze([ + 'ltree', + 'pg_textsearch', + 'pg_trgm', + 'postgis', + 'vector', +]); +const RUN_PURPOSES = Object.freeze(['hostile-preflight', 'measurement']); +const CLONE_ID_PATTERN = /^[a-z0-9][a-z0-9._-]{0,127}$/i; +const ATTESTATION_NONCE_PATTERN = /^[a-f0-9]{64}$/; + +const quoteIdentifier = (value) => `"${String(value).replace(/"/g, '""')}"`; +const quoteLiteral = (value) => `'${String(value).replace(/'/g, "''")}'`; + +const requireCloneId = (value) => { + if (typeof value !== 'string' || !CLONE_ID_PATTERN.test(value)) { + throw new Error('PDCF_CLONE_ID_INVALID'); + } + return value; +}; + +const requireRunPurpose = (value) => { + if (!RUN_PURPOSES.includes(value)) throw new Error('PDCF_RUN_PURPOSE_INVALID'); + return value; +}; + +const provisionAttestationSha256 = ({ + cloneId, + runPurpose, + customerId, + database, + nonce, +}) => { + requireCloneId(cloneId); + requireRunPurpose(runPurpose); + if (typeof customerId !== 'string' || !customerId) { + throw new Error('PDCF_ATTESTATION_CUSTOMER_ID_INVALID'); + } + if (typeof database !== 'string' || !database) { + throw new Error('PDCF_ATTESTATION_DATABASE_INVALID'); + } + if (typeof nonce !== 'string' || !ATTESTATION_NONCE_PATTERN.test(nonce)) { + throw new Error('PDCF_ATTESTATION_NONCE_INVALID'); + } + const digest = crypto.createHash('sha256'); + for (const value of [ + 'physical-database-density-provision-attestation-v1', + cloneId, + runPurpose, + customerId, + database, + nonce, + ]) { + digest.update(value); + digest.update('\0'); + } + return `sha256:${digest.digest('hex')}`; +}; + +const provisionAttestationSetSha256 = (customers) => sha256Json( + [...customers].map((customer) => ({ + customerId: customer.id, + database: customer.database, + sha256: customer.provisionAttestation.sha256, + })).sort((left, right) => left.customerId.localeCompare(right.customerId)), +); + +const run = (command, args, options = {}) => { + const result = spawnSync(command, args, { + cwd: options.cwd ?? FIXTURE_DIR, + env: options.env ?? process.env, + encoding: 'utf8', + input: options.input, + maxBuffer: 64 * 1024 * 1024, + }); + if (result.status !== 0) { + const detail = result.stderr?.trim() || result.stdout?.trim() || result.error?.message; + throw new Error(`PDCF_COMMAND_FAILED:${command}:${detail || `exit=${result.status}`}`); + } + return result.stdout ?? ''; +}; + +const psql = (database, sql, environment = process.env) => run( + 'psql', + ['--no-psqlrc', '--set=ON_ERROR_STOP=1', '--dbname', database], + { input: `${sql}\n`, env: environment }, +); + +const applySqlFile = (database, file, roles, environment = process.env) => run( + 'psql', + [ + '--no-psqlrc', + '--set=ON_ERROR_STOP=1', + '--dbname', database, + '--set', `runtime_role_a=${roles.a}`, + '--set', `runtime_role_b=${roles.b}`, + '--set', `runtime_role_c=${roles.c}`, + '--file', file, + ], + { env: environment }, +); + +const applyProvisionAttestation = ( + database, + file, + { cloneId, runPurpose, customerId, nonce, sha256 }, + environment = process.env, +) => run( + 'psql', + [ + '--no-psqlrc', + '--set=ON_ERROR_STOP=1', + '--dbname', database, + '--set', `clone_id=${cloneId}`, + '--set', `run_purpose=${runPurpose}`, + '--set', `customer_id=${customerId}`, + '--set', `attestation_nonce=${nonce}`, + '--set', `attestation_sha256=${sha256}`, + '--file', file, + ], + { env: environment }, +); + +const normalizeSchemaDump = (dump, roleAliases = {}) => { + const normalizedRoles = Object.entries(roleAliases) + .sort(([left], [right]) => right.length - left.length); + const normalizeRoles = (line) => normalizedRoles.reduce((value, [role, alias]) => { + const escaped = role.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return value.replace( + new RegExp(`(^|[^a-zA-Z0-9_])${escaped}(?=$|[^a-zA-Z0-9_])`, 'g'), + `$1${alias}`, + ); + }, line); + return `${dump + .split(/\r?\n/) + .filter((line) => !( + line.startsWith('\\restrict ') + || line.startsWith('\\unrestrict ') + || line.startsWith('-- Dumped from database version') + || line.startsWith('-- Dumped by pg_dump version') + || line.startsWith('-- Started on ') + || line.startsWith('-- Completed on ') + )) + .map(normalizeRoles) + .join('\n') + .trim()}\n`; +}; + +const fingerprintDump = ( + database, + schemas, + environment = process.env, + roleAliases = {}, +) => { + const args = [ + '--schema-only', + '--no-owner', + '--dbname', database, + ...schemas.flatMap((schema) => ['--schema', schema]), + ]; + // ACLs are part of Graphile's effective catalog. Preserve them in the dump, + // but replace per-customer login names with stable tenant slots so equivalent + // least-privilege grants compare byte-for-byte across physical databases. + const normalized = normalizeSchemaDump( + run('pg_dump', args, { env: environment }), + roleAliases, + ); + return { + sha256: `sha256:${crypto.createHash('sha256').update(normalized).digest('hex')}`, + bytes: Buffer.byteLength(normalized), + }; +}; + +const structuralFingerprints = (database, schemas, roles, environment) => { + const roleAliases = Object.fromEntries(TENANTS.map((tenant) => [ + roles[tenant.id], + `__runtime_${tenant.id}__`, + ])); + return { + combined: fingerprintDump(database, schemas, environment, roleAliases), + schemas: Object.fromEntries(schemas.map((schema) => [ + schema, + fingerprintDump(database, [schema], environment, roleAliases), + ])), + }; +}; + +const parseJsonRows = (stdout, label) => { + const value = stdout.trim(); + try { + return value ? JSON.parse(value) : []; + } catch { + throw new Error(`PDCF_${label}_JSON_INVALID`); + } +}; + +const sha256Json = (value) => `sha256:${crypto.createHash('sha256') + .update(JSON.stringify(value)) + .digest('hex')}`; + +const normalizedRoleSafetyProfile = (roles, roleAudit) => TENANTS.map((tenant) => { + const roleName = roles[tenant.id]; + const row = roleAudit.find((candidate) => candidate.role_name === roleName); + if (!row) throw new Error(`PDCF_ROLE_AUDIT_PROFILE_INCOMPLETE:${tenant.id}`); + const { role_name: _roleName, ...flags } = row; + return { slot: tenant.id, ...flags }; +}); + +const inspectCustomerContract = ({ + customer, + canonicalSchemas, + environment = process.env, +}) => { + const roleAudit = auditRuntimeRoles(customer.database, customer.roles, environment); + const roleSafetyProfile = normalizedRoleSafetyProfile(customer.roles, roleAudit); + const notificationRoleAudit = auditNotificationRole( + customer.database, + customer.notificationRole, + environment, + ); + const { + role_name: _notificationRoleName, + ...notificationRoleSafetyProfile + } = notificationRoleAudit; + const extensionVersions = auditExtensionVersions(customer.database, environment); + const fingerprints = structuralFingerprints( + customer.database, + canonicalSchemas, + customer.roles, + environment, + ); + const databaseContractFingerprint = sha256Json({ + structuralFingerprints: fingerprints, + extensionVersions, + roleSafetyProfile, + notificationRoleSafetyProfile, + }); + return { + roleAudit, + roleSafetyProfile, + notificationRoleAudit, + notificationRoleSafetyProfile, + extensionVersions, + structuralFingerprints: fingerprints, + databaseContractFingerprint, + }; +}; + +const auditNotificationRole = (database, role, environment) => { + const sql = ` +COPY ( + SELECT row_to_json(audit) + FROM ( + SELECT r.rolname AS role_name, + r.rolcanlogin AS can_login, + NOT r.rolinherit AS noinherit, + NOT r.rolsuper AS not_superuser, + NOT r.rolbypassrls AS no_bypassrls, + NOT r.rolcreaterole AS no_createrole, + NOT r.rolcreatedb AS no_createdb, + NOT r.rolreplication AS no_replication, + NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_auth_members membership + WHERE membership.member = r.oid OR membership.roleid = r.oid + ) AS no_membership, + pg_catalog.has_database_privilege(r.rolname, ${quoteLiteral(database)}, 'CONNECT') + AS target_connect, + NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_database d + WHERE d.datname <> ${quoteLiteral(database)} + AND pg_catalog.has_database_privilege(r.rolname, d.oid, 'CONNECT') + ) AS no_cross_database_connect, + NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_database d + WHERE d.datname = ${quoteLiteral(database)} + AND d.datdba = r.oid + ) AS not_database_owner, + NOT pg_catalog.has_database_privilege( + r.rolname, ${quoteLiteral(database)}, 'CREATE' + ) AS no_database_create, + NOT pg_catalog.has_database_privilege( + r.rolname, ${quoteLiteral(database)}, 'TEMP' + ) AS no_database_temp, + NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_namespace n + WHERE n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_' + AND ( + n.nspowner = r.oid + OR pg_catalog.has_schema_privilege(r.rolname, n.oid, 'CREATE') + OR pg_catalog.has_schema_privilege(r.rolname, n.oid, 'USAGE') + ) + ) AS no_application_schema_access, + NOT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_' + AND ( + CASE WHEN c.relkind IN ('r', 'p', 'v', 'm', 'f') THEN + pg_catalog.has_table_privilege( + r.rolname, c.oid, + 'SELECT,INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER' + ) + OR pg_catalog.has_any_column_privilege( + r.rolname, c.oid, 'SELECT,INSERT,UPDATE,REFERENCES' + ) + WHEN c.relkind = 'S' THEN + pg_catalog.has_sequence_privilege( + r.rolname, c.oid, 'USAGE,SELECT,UPDATE' + ) + ELSE false END + ) + ) AS no_application_relation_access, + NOT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_' + AND pg_catalog.has_function_privilege(r.rolname, p.oid, 'EXECUTE') + ) AS no_application_function_access + FROM pg_catalog.pg_roles r + WHERE r.rolname = ${quoteLiteral(role)} + ) audit +) TO STDOUT; +`; + const row = parseJsonRows(psql(database, sql, environment), 'NOTIFICATION_ROLE_AUDIT'); + const required = [ + 'can_login', + 'noinherit', + 'not_superuser', + 'no_bypassrls', + 'no_createrole', + 'no_createdb', + 'no_replication', + 'no_membership', + 'target_connect', + 'no_cross_database_connect', + 'not_database_owner', + 'no_database_create', + 'no_database_temp', + 'no_application_schema_access', + 'no_application_relation_access', + 'no_application_function_access', + ]; + if ( + !row + || Array.isArray(row) + || row.role_name !== role + || required.some((field) => row[field] !== true) + ) { + throw new Error(`PDCF_NOTIFICATION_ROLE_AUDIT_FAILED:${database}:${role}`); + } + return row; +}; + +const auditExtensionVersions = (database, environment) => { + const sql = ` +COPY ( + SELECT COALESCE( + pg_catalog.json_agg(row_to_json(extension_row) ORDER BY extension_row.name), + '[]'::json + ) + FROM ( + SELECT e.extname AS name, + e.extversion AS version, + n.nspname AS schema + FROM pg_catalog.pg_extension e + JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace + WHERE e.extname = ANY(ARRAY[${REQUIRED_EXTENSIONS.map(quoteLiteral).join(', ')}]::text[]) + ) extension_row +) TO STDOUT; +`; + const rows = parseJsonRows(psql(database, sql, environment), 'EXTENSION_AUDIT'); + if ( + !Array.isArray(rows) + || rows.length !== REQUIRED_EXTENSIONS.length + || rows.some((row, index) => + row.name !== REQUIRED_EXTENSIONS[index] + || typeof row.version !== 'string' + || row.version.length === 0 + || row.schema !== 'ctf_extensions' + ) + ) { + throw new Error(`PDCF_EXTENSION_AUDIT_FAILED:${database}`); + } + return rows; +}; + +const auditRuntimeRoles = (database, roles, environment) => { + const names = Object.values(roles); + const sql = ` +COPY ( + SELECT pg_catalog.json_agg(row_to_json(audit) ORDER BY audit.role_name) + FROM ( + SELECT r.rolname AS role_name, + r.rolcanlogin AS can_login, + NOT r.rolinherit AS noinherit, + NOT r.rolsuper AS not_superuser, + NOT r.rolbypassrls AS no_bypassrls, + NOT r.rolcreaterole AS no_createrole, + NOT r.rolcreatedb AS no_createdb, + NOT r.rolreplication AS no_replication, + NOT pg_catalog.has_database_privilege(r.rolname, current_database(), 'CREATE') AS no_database_create, + NOT pg_catalog.has_schema_privilege( + r.rolname, + 'ctf_provision_private', + 'USAGE' + ) AS no_provision_attestation_schema_usage, + NOT ( + pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'SELECT' + ) + OR pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'INSERT' + ) + OR pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'UPDATE' + ) + OR pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'DELETE' + ) + OR pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'TRUNCATE' + ) + OR pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'REFERENCES' + ) + OR pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'TRIGGER' + ) + ) AS no_provision_attestation_table_privileges, + NOT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace n + WHERE n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_' + AND ( + n.nspowner = r.oid + OR pg_catalog.has_schema_privilege(r.rolname, n.oid, 'CREATE') + ) + ) AS no_schema_owner_or_create + FROM pg_catalog.pg_roles r + WHERE r.rolname = ANY(ARRAY[${names.map(quoteLiteral).join(', ')}]::text[]) + ) audit +) TO STDOUT; +`; + const rows = parseJsonRows(psql(database, sql, environment), 'ROLE_AUDIT'); + if (!Array.isArray(rows) || rows.length !== names.length) { + throw new Error(`PDCF_ROLE_AUDIT_INCOMPLETE:${database}`); + } + const required = [ + 'can_login', + 'noinherit', + 'not_superuser', + 'no_bypassrls', + 'no_createrole', + 'no_createdb', + 'no_replication', + 'no_database_create', + 'no_provision_attestation_schema_usage', + 'no_provision_attestation_table_privileges', + 'no_schema_owner_or_create', + ]; + for (const row of rows) { + if (required.some((field) => row[field] !== true)) { + throw new Error(`PDCF_ROLE_AUDIT_FAILED:${database}:${row.role_name}`); + } + } + return rows; +}; + +const provisionCustomer = ({ + customer, + passwords, + maintenanceDatabase, + schemaFile, + identityFile, + attestationFile, + provisionAttestation, + recreate, + environment, + canonicalSchemas, +}) => { + if (recreate) { + psql(maintenanceDatabase, ` +SELECT pg_catalog.pg_terminate_backend(pid) +FROM pg_catalog.pg_stat_activity +WHERE datname = ${quoteLiteral(customer.database)} + AND pid <> pg_catalog.pg_backend_pid(); +DROP DATABASE IF EXISTS ${quoteIdentifier(customer.database)}; +${Object.values(customer.roles).map((role) => + `DROP ROLE IF EXISTS ${quoteIdentifier(role)};` + ).join('\n')} +DROP ROLE IF EXISTS ${quoteIdentifier(customer.notificationRole)}; +`, environment); + } + + psql(maintenanceDatabase, Object.entries(customer.roles).map(([tenantId, role]) => ` +CREATE ROLE ${quoteIdentifier(role)} + LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD ${quoteLiteral(passwords[role])}; +COMMENT ON ROLE ${quoteIdentifier(role)} IS ${quoteLiteral(`${FIXTURE_ID}:${customer.id}:${tenantId}`)}; +`).join('\n') + ` +CREATE ROLE ${quoteIdentifier(customer.notificationRole)} + LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD ${quoteLiteral(passwords[customer.notificationRole])}; +COMMENT ON ROLE ${quoteIdentifier(customer.notificationRole)} IS ${quoteLiteral( + `${FIXTURE_ID}:${customer.id}:notification-only` + )}; +`, environment); + + psql(maintenanceDatabase, ` +CREATE DATABASE ${quoteIdentifier(customer.database)}; +REVOKE ALL ON DATABASE ${quoteIdentifier(customer.database)} FROM PUBLIC; +GRANT CONNECT ON DATABASE ${quoteIdentifier(customer.database)} TO ${[ + ...Object.values(customer.roles), + customer.notificationRole, + ] + .map(quoteIdentifier) + .join(', ')}; +COMMENT ON DATABASE ${quoteIdentifier(customer.database)} IS ${quoteLiteral(`${FIXTURE_ID}:${customer.id}`)}; +`, environment); + + applySqlFile(customer.database, schemaFile, customer.roles, environment); + applySqlFile(customer.database, identityFile, customer.roles, environment); + applyProvisionAttestation( + customer.database, + attestationFile, + provisionAttestation, + environment, + ); + + const contract = inspectCustomerContract({ + customer, + canonicalSchemas, + environment, + }); + return { + ...customer, + provisionAttestation: { + version: 1, + cloneId: provisionAttestation.cloneId, + purpose: provisionAttestation.runPurpose, + sha256: provisionAttestation.sha256, + }, + ...contract, + }; +}; + +const provision = ({ + prefix, + customerCount, + outDir, + maintenanceDatabase, + schemaFile, + identityFile, + attestationFile, + cloneId, + runPurpose, + recreate, + environment = process.env, + canonicalSchemas = DEFAULT_CANONICAL_SCHEMAS, + credentialTemplate = null, +}) => { + cloneId = requireCloneId(cloneId); + runPurpose = requireRunPurpose(runPurpose); + const customers = makeCustomers(prefix, customerCount); + const requiredRuntimeRoles = customers.flatMap((customer) => + Object.values(customer.roles) + ).sort(); + const requiredNotificationRoles = customers.map((customer) => + customer.notificationRole + ).sort(); + const assertExactPasswords = (value, roles, label) => { + if ( + !value + || typeof value !== 'object' + || Array.isArray(value) + || JSON.stringify(Object.keys(value).sort()) !== JSON.stringify(roles) + || roles.some((role) => + typeof value[role] !== 'string' || Buffer.byteLength(value[role]) < 24 + ) + ) { + throw new Error(`PDCF_${label}_CREDENTIAL_TEMPLATE_INVALID`); + } + return Object.fromEntries(roles.map((role) => [role, value[role]])); + }; + const runtimePasswords = credentialTemplate + ? assertExactPasswords( + credentialTemplate.runtimePasswords, + requiredRuntimeRoles, + 'RUNTIME', + ) + : Object.fromEntries(requiredRuntimeRoles.map((role) => [ + role, + crypto.randomBytes(32).toString('base64url'), + ])); + const notificationPasswords = credentialTemplate + ? assertExactPasswords( + credentialTemplate.notificationPasswords, + requiredNotificationRoles, + 'NOTIFICATION', + ) + : Object.fromEntries(requiredNotificationRoles.map((role) => [ + role, + crypto.randomBytes(32).toString('base64url'), + ])); + const allPasswords = { ...runtimePasswords, ...notificationPasswords }; + + // The notification-role contract rejects CONNECT to every other database. + // This fixture owns a disposable PostgreSQL cluster, so remove PostgreSQL's + // default PUBLIC grants before any per-customer role is audited. + psql(maintenanceDatabase, ` +DO $revoke_public_connect$ +DECLARE + database_record record; +BEGIN + FOR database_record IN SELECT datname FROM pg_catalog.pg_database + LOOP + EXECUTE pg_catalog.format( + 'REVOKE CONNECT ON DATABASE %I FROM PUBLIC', + database_record.datname + ); + END LOOP; +END +$revoke_public_connect$; +`, environment); + const provisioned = customers.map((customer) => { + const nonce = crypto.randomBytes(32).toString('hex'); + const provisionAttestation = { + cloneId, + runPurpose, + customerId: customer.id, + nonce, + sha256: provisionAttestationSha256({ + cloneId, + runPurpose, + customerId: customer.id, + database: customer.database, + nonce, + }), + }; + return provisionCustomer({ + customer, + passwords: allPasswords, + maintenanceDatabase, + schemaFile, + identityFile, + attestationFile, + provisionAttestation, + recreate, + environment, + canonicalSchemas, + }); + }); + const expected = provisioned[0].structuralFingerprints; + const expectedDatabaseContractFingerprint = provisioned[0].databaseContractFingerprint; + for (const customer of provisioned.slice(1)) { + if (customer.databaseContractFingerprint !== expectedDatabaseContractFingerprint) { + throw new Error( + `PDCF_DATABASE_CONTRACT_MISMATCH:${provisioned[0].database}:${customer.database}` + ); + } + if (customer.structuralFingerprints.combined.sha256 !== expected.combined.sha256) { + throw new Error( + `PDCF_CANONICAL_SCHEMA_MISMATCH:${provisioned[0].database}:${customer.database}` + ); + } + for (const schema of canonicalSchemas) { + if ( + customer.structuralFingerprints.schemas[schema].sha256 + !== expected.schemas[schema].sha256 + ) { + throw new Error(`PDCF_CANONICAL_SCHEMA_MISMATCH:${schema}:${customer.database}`); + } + } + } + const manifest = { + version: 1, + fixture: FIXTURE_ID, + prefix, + createdAt: new Date().toISOString(), + provisionClone: { + version: 1, + id: cloneId, + purpose: runPurpose, + attestationSetSha256: provisionAttestationSetSha256(provisioned), + }, + canonicalSchemas, + canonicalStructuralFingerprint: expected, + canonicalDatabaseContractFingerprint: expectedDatabaseContractFingerprint, + pgDumpVersion: run('pg_dump', ['--version'], { env: environment }).trim(), + customers: provisioned, + }; + const secrets = { + version: 1, + fixture: FIXTURE_ID, + runtimePasswords, + notificationPasswords, + }; + const manifestFile = path.join(outDir, 'provision.json'); + const secretsFile = path.join(outDir, 'runtime-secrets.json'); + atomicWriteJson(manifestFile, manifest); + atomicWriteJson(secretsFile, secrets, 0o600); + return { manifest, manifestFile, secretsFile }; +}; + +const main = () => { + const args = parseArgs(process.argv.slice(2)); + const recreate = args.recreate === true; + if (recreate && args.yes !== true) throw new Error('PDCF_RECREATE_REQUIRES_YES'); + const prefix = requireString(args, 'prefix', 'pdc_density'); + const customerCount = parsePositiveInteger(args.customers ?? '3', 'customers'); + const outDir = path.resolve(requireString( + args, + 'out-dir', + path.join(FIXTURE_DIR, '.local'), + )); + const maintenanceDatabase = requireString( + args, + 'maintenance-database', + process.env.PGDATABASE ?? 'postgres', + ); + const schemaFile = path.resolve(requireString( + args, + 'schema-file', + path.join(FIXTURE_DIR, '../complete-tenant-fixture/schema.sql'), + )); + const identityFile = path.resolve(requireString( + args, + 'identity-file', + path.join(FIXTURE_DIR, 'physical-identity.sql'), + )); + const attestationFile = path.resolve(requireString( + args, + 'attestation-file', + path.join(FIXTURE_DIR, 'provision-attestation.sql'), + )); + const result = provision({ + prefix, + customerCount, + outDir, + maintenanceDatabase, + schemaFile, + identityFile, + attestationFile, + cloneId: requireString(args, 'clone-id'), + runPurpose: requireString(args, 'run-purpose'), + recreate, + }); + process.stdout.write(`${JSON.stringify({ + status: 'provisioned', + customers: result.manifest.customers.length, + canonicalStructuralFingerprint: + result.manifest.canonicalStructuralFingerprint.combined.sha256, + provisionClone: result.manifest.provisionClone, + manifestFile: result.manifestFile, + secretsFile: result.secretsFile, + })}\n`); +}; + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} + +module.exports = { + DEFAULT_CANONICAL_SCHEMAS, + REQUIRED_EXTENSIONS, + auditExtensionVersions, + auditNotificationRole, + auditRuntimeRoles, + fingerprintDump, + inspectCustomerContract, + normalizeSchemaDump, + normalizedRoleSafetyProfile, + provision, + provisionAttestationSetSha256, + provisionAttestationSha256, + requireCloneId, + requireRunPurpose, + structuralFingerprints, +}; diff --git a/research/graphile-density/physical-database-density/server-realtime.test.cjs b/research/graphile-density/physical-database-density/server-realtime.test.cjs new file mode 100644 index 0000000000..89ed9f3d36 --- /dev/null +++ b/research/graphile-density/physical-database-density/server-realtime.test.cjs @@ -0,0 +1,85 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { EventEmitter } = require('node:events'); +const { describe, it } = require('node:test'); + +const { + createRealtimeConnectionRegistry, + matchPhysicalUpgradeRoute, +} = require('./server.cjs'); + +const socket = () => Object.assign(new EventEmitter(), { + destroyed: false, + destroy() { this.destroyed = true; }, +}); + +describe('physical density server-side realtime accounting', () => { + it('routes only one exact customer and tenant upgrade path', () => { + assert.deepEqual( + matchPhysicalUpgradeRoute( + '/customer/physical-customer-0007/tenant/b/graphql' + ), + { customerId: 'physical-customer-0007', tenantId: 'b' } + ); + assert.equal(matchPhysicalUpgradeRoute('/customer/c1/tenant/b/graphql?token=x'), null); + assert.equal(matchPhysicalUpgradeRoute('/customer/c1/graphql'), null); + assert.equal(matchPhysicalUpgradeRoute('/customer/c1/tenant/b/other'), null); + }); + + it('counts one accepted live socket per surface without retaining clients', () => { + const registry = createRealtimeConnectionRegistry(['customer-1:a', 'customer-1:b']); + const a = socket(); + const b = socket(); + assert.equal(registry.trackAccepted('customer-1:a', a), true); + assert.equal(registry.trackAccepted('customer-1:b', b), true); + assert.deepEqual(registry.assertResident(), { + connectionsExpected: 2, + connectionsAccepted: 2, + connectionsActive: 2, + connectionDrops: 0, + connectionErrors: 0, + connectionsPerSurface: [ + { + key: 'customer-1:a', + accepted: 1, + active: 1, + peakActive: 1, + drops: 0, + errors: 0, + }, + { + key: 'customer-1:b', + accepted: 1, + active: 1, + peakActive: 1, + drops: 0, + errors: 0, + }, + ], + }); + }); + + it('fails residency after a drop, error, duplicate, or unknown route', () => { + const registry = createRealtimeConnectionRegistry(['customer-1:a']); + const accepted = socket(); + registry.trackAccepted('customer-1:a', accepted); + accepted.emit('error', new Error('reset')); + assert.throws( + () => registry.assertResident(), + /PDCF_REALTIME_CONNECTIONS_NOT_RESIDENT:0:1:1:1/ + ); + + const duplicateRegistry = createRealtimeConnectionRegistry(['customer-1:a']); + duplicateRegistry.trackAccepted('customer-1:a', socket()); + duplicateRegistry.trackAccepted('customer-1:a', socket()); + assert.throws( + () => duplicateRegistry.assertResident(), + /PDCF_REALTIME_CONNECTIONS_NOT_RESIDENT/ + ); + + const unknown = socket(); + assert.equal(duplicateRegistry.trackAccepted('customer-2:a', unknown), false); + assert.equal(unknown.destroyed, true); + }); +}); diff --git a/research/graphile-density/physical-database-density/server-retained-memory.test.cjs b/research/graphile-density/physical-database-density/server-retained-memory.test.cjs new file mode 100644 index 0000000000..b36f327bc2 --- /dev/null +++ b/research/graphile-density/physical-database-density/server-retained-memory.test.cjs @@ -0,0 +1,320 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { describe, it } = require('node:test'); + +const { + authorizeRetainedMemoryCheckpoint, + collectRetainedMemoryCheckpoint, + makeGraphileActivityVector, + makeRetainedMemoryGuard, + parseServerOptions, +} = require('./server.cjs'); + +const MIB = 1024 ** 2; + +const guard = (overrides = {}) => makeRetainedMemoryGuard({ + pid: 42, + graphileInFlight: 0, + residentBuildContracts: ['contract-a'], + graphileActivityByBuildContract: [{ + buildContract: 'contract-a', + inflight: 0, + websocketSockets: 0, + transientHttpInFlight: 0, + }], + cacheCounters: { + httpRequestsStarted: 0, + httpRequestsCompleted: 0, + websocketUpgradesStarted: 0, + websocketUpgradesCompleted: 0, + evictions: 0, + buildRefusals: 0, + }, + realtime: { + managersExpected: 0, + managersActive: 0, + connectionsExpected: 0, + connectionsAccepted: 0, + connectionsActive: 0, + connectionDrops: 0, + connectionErrors: 0, + connectionsPerSurface: [], + }, + buildCounters: { started: 1, succeeded: 1, failed: 0 }, + ...overrides, +}); + +const liveRealtimeGuard = (overrides = {}) => guard({ + graphileActivityByBuildContract: [{ + buildContract: 'contract-a', + inflight: 1, + websocketSockets: 1, + transientHttpInFlight: 0, + }], + cacheCounters: { + httpRequestsStarted: 4, + httpRequestsCompleted: 4, + websocketUpgradesStarted: 1, + websocketUpgradesCompleted: 0, + }, + realtime: { + managersExpected: 1, + managersActive: 1, + connectionsExpected: 1, + connectionsAccepted: 1, + connectionsActive: 1, + connectionDrops: 0, + connectionErrors: 0, + connectionsPerSurface: [{ + key: 'customer-1:a', + accepted: 1, + active: 1, + peakActive: 1, + drops: 0, + errors: 0, + }], + }, + ...overrides, +}); + +describe('physical density retained-memory checkpoint', () => { + it('requires the explicit benchmark flag and matching loopback bearer token', () => { + const request = (address, token) => ({ + socket: { remoteAddress: address }, + get: (name) => name === 'authorization' ? `Bearer ${token}` : undefined, + }); + const options = { + benchmarkRetainedHeapEnabled: true, + observabilityToken: 'checkpoint-secret', + }; + assert.equal(authorizeRetainedMemoryCheckpoint( + request('127.0.0.1', 'checkpoint-secret'), options + ), true); + assert.equal(authorizeRetainedMemoryCheckpoint( + request('10.0.0.2', 'checkpoint-secret'), options + ), false); + assert.equal(authorizeRetainedMemoryCheckpoint( + request('127.0.0.1', 'wrong-secret'), options + ), false); + assert.equal(authorizeRetainedMemoryCheckpoint( + request('127.0.0.1', 'checkpoint-secret'), { + ...options, + benchmarkRetainedHeapEnabled: false, + } + ), false); + }); + + it('parses the benchmark flag from the environment only', () => { + const options = parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/secrets.json', + '--run-purpose', 'measurement', + '--clone-id', 'measurement-clone-test', + ], { + GRAPHQL_CPERF_RETAINED_HEAP_ENABLED: 'true', + GRAPHQL_OBSERVABILITY_TOKEN: 'checkpoint-secret', + }); + assert.equal(options.benchmarkRetainedHeapEnabled, true); + assert.equal(options.observabilityToken, 'checkpoint-secret'); + }); + + it('runs eight full-GC turns and accepts only converged last-three samples', async () => { + let gcCalls = 0; + let reads = 0; + let monotonic = 0n; + const heapMiB = [120, 112, 106, 103, 101, 100.2, 100.1, 100]; + const checkpoint = await collectRetainedMemoryCheckpoint({ + forceGc: () => { gcCalls += 1; }, + readMemory: () => ({ + heapUsed: Math.round(heapMiB[reads++] * MIB), + external: 10 * MIB, + arrayBuffers: 2 * MIB, + rss: 180 * MIB, + }), + readGuard: () => guard(), + monotonicNow: () => ++monotonic, + yieldTurn: async () => undefined, + }); + assert.equal(gcCalls, 8); + assert.equal(checkpoint.samples.length, 8); + assert.equal(checkpoint.stableSampleCount, 3); + assert.equal(checkpoint.stable, true); + assert.deepEqual(checkpoint.errors, []); + assert.equal(checkpoint.guardBefore.stateSha256, checkpoint.guardAfter.stateSha256); + }); + + it('subtracts expected long-lived sockets from exact per-contract activity', () => { + const vector = makeGraphileActivityVector([ + { + cacheKey: 'contract-b', + inflight: 3, + websocketSockets: new Set([{}, {}]), + }, + { + cacheKey: 'contract-a', + inflight: 1, + websocketSockets: new Set([{}]), + }, + ]); + assert.deepEqual(vector, [ + { + buildContract: 'contract-a', + inflight: 1, + websocketSockets: 1, + transientHttpInFlight: 0, + }, + { + buildContract: 'contract-b', + inflight: 3, + websocketSockets: 2, + transientHttpInFlight: 1, + }, + ]); + }); + + it('permits stable expected realtime sockets during full GC', async () => { + let gcCalls = 0; + let monotonic = 0n; + const checkpoint = await collectRetainedMemoryCheckpoint({ + forceGc: () => { gcCalls += 1; }, + readMemory: () => ({ + heapUsed: 100 * MIB, + external: 10 * MIB, + arrayBuffers: 2 * MIB, + rss: 180 * MIB, + }), + readGuard: () => liveRealtimeGuard(), + monotonicNow: () => ++monotonic, + yieldTurn: async () => undefined, + }); + assert.equal(gcCalls, 8); + assert.equal(checkpoint.stable, true); + assert.equal(checkpoint.guardBefore.graphileInFlight, 0); + assert.equal(checkpoint.guardBefore.graphileWebsocketSockets, 1); + assert.equal(checkpoint.guardBefore.realtimeResident, true); + }); + + it('hashes balanced handler lifecycles that begin and end between reads', () => { + const baseline = guard({ + cacheCounters: { + httpRequestsStarted: 4, + httpRequestsCompleted: 4, + websocketUpgradesStarted: 0, + websocketUpgradesCompleted: 0, + }, + }); + const shortHttpRequest = guard({ + cacheCounters: { + httpRequestsStarted: 5, + httpRequestsCompleted: 5, + websocketUpgradesStarted: 0, + websocketUpgradesCompleted: 0, + }, + }); + const shortWebsocket = guard({ + cacheCounters: { + httpRequestsStarted: 5, + httpRequestsCompleted: 5, + websocketUpgradesStarted: 1, + websocketUpgradesCompleted: 1, + }, + }); + assert.notEqual(baseline.stateSha256, shortHttpRequest.stateSha256); + assert.notEqual(shortHttpRequest.stateSha256, shortWebsocket.stateSha256); + }); + + it('binds pg-cache capacity and failure counters into the stable guard', () => { + const baseline = guard({ + pgCacheMonotonicCounters: { + capacityEvictions: 0, + capacityRefusals: 0, + disposalFailures: 0, + }, + }); + const changed = guard({ + pgCacheMonotonicCounters: { + capacityEvictions: 0, + capacityRefusals: 1, + disposalFailures: 0, + }, + }); + assert.notEqual(baseline.stateSha256, changed.stateSha256); + }); + + it('fails closed when heap convergence or process state changes', async () => { + let reads = 0; + let guards = 0; + const heapMiB = [100, 100, 100, 100, 100, 100, 104, 100]; + const checkpoint = await collectRetainedMemoryCheckpoint({ + forceGc: () => undefined, + readMemory: () => ({ + heapUsed: heapMiB[reads++] * MIB, + external: 10 * MIB, + arrayBuffers: 2 * MIB, + rss: 180 * MIB, + }), + readGuard: () => guard({ counter: guards++ }), + monotonicNow: (() => { + let value = 0n; + return () => ++value; + })(), + yieldTurn: async () => undefined, + }); + assert.equal(checkpoint.stable, false); + assert.ok(checkpoint.errors.some((error) => + error.startsWith('PDCF_RETAINED_HEAP_NOT_CONVERGED:') + )); + assert.ok(checkpoint.errors.includes( + 'PDCF_RETAINED_MEMORY_RESIDENCY_OR_COUNTERS_CHANGED' + )); + }); + + it('does not force GC while Graphile work is in flight', async () => { + let gcCalls = 0; + await assert.rejects(collectRetainedMemoryCheckpoint({ + forceGc: () => { gcCalls += 1; }, + readGuard: () => guard({ graphileInFlight: 1 }), + }), /PDCF_RETAINED_MEMORY_IN_FLIGHT:1/); + assert.equal(gcCalls, 0); + }); + + it('does not force GC when expected realtime sockets are missing', async () => { + let gcCalls = 0; + await assert.rejects(collectRetainedMemoryCheckpoint({ + forceGc: () => { gcCalls += 1; }, + readGuard: () => liveRealtimeGuard({ + graphileActivityByBuildContract: [{ + buildContract: 'contract-a', + inflight: 0, + websocketSockets: 0, + transientHttpInFlight: 0, + }], + cacheCounters: { + httpRequestsStarted: 4, + httpRequestsCompleted: 4, + websocketUpgradesStarted: 1, + websocketUpgradesCompleted: 1, + }, + realtime: { + managersExpected: 1, + managersActive: 1, + connectionsExpected: 1, + connectionsAccepted: 1, + connectionsActive: 0, + connectionDrops: 1, + connectionErrors: 0, + connectionsPerSurface: [{ + key: 'customer-1:a', + accepted: 1, + active: 0, + peakActive: 1, + drops: 1, + errors: 0, + }], + }, + }), + }), /PDCF_RETAINED_MEMORY_REALTIME_NOT_RESIDENT:0:0:1/); + assert.equal(gcCalls, 0); + }); +}); diff --git a/research/graphile-density/physical-database-density/server.cjs b/research/graphile-density/physical-database-density/server.cjs new file mode 100644 index 0000000000..7d5b9889f0 --- /dev/null +++ b/research/graphile-density/physical-database-density/server.cjs @@ -0,0 +1,1299 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + FIXTURE_DIR, + REPO_ROOT, + loadProvision, +} = require('./lib.cjs'); +const { + TENANTS, + parseArgs, + parsePositiveInteger, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); +const completeServer = require('../complete-tenant-fixture/server.cjs'); +const { + inspectCustomerContract, + provisionAttestationSetSha256, + requireCloneId, + requireRunPurpose, +} = require('./provision.cjs'); + +const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost']); +const RETAINED_MEMORY_GC_ROUNDS = 8; +const RETAINED_MEMORY_STABLE_SAMPLES = 3; +const MIB = 1024 ** 2; +const SECURITY_ENVIRONMENT_KEYS = Object.freeze([ + 'PGDATABASE', + 'PG_POOL_MAX_USES', + 'CTF_RUNTIME_A_PGUSER', + 'CTF_RUNTIME_B_PGUSER', + 'CTF_RUNTIME_C_PGUSER', + 'CTF_RUNTIME_A_PGPASSWORD', + 'CTF_RUNTIME_B_PGPASSWORD', + 'CTF_RUNTIME_C_PGPASSWORD', + 'CTF_NOTIFICATION_PGUSER', + 'CTF_NOTIFICATION_PGPASSWORD', +]); + +const requireBuilt = (relativePath) => require(path.join(REPO_ROOT, relativePath)); + +const matchPhysicalUpgradeRoute = (rawUrl) => { + if (typeof rawUrl !== 'string' || rawUrl.includes('?')) return null; + const match = /^\/customer\/([a-z0-9-]+)\/tenant\/([a-z0-9-]+)\/graphql$/.exec(rawUrl); + return match ? { customerId: match[1], tenantId: match[2] } : null; +}; + +const matchPhysicalUpgradeCustomer = (rawUrl) => + matchPhysicalUpgradeRoute(rawUrl)?.customerId ?? null; + +const createRealtimeConnectionRegistry = (surfaceKeys) => { + const states = new Map(surfaceKeys.map((key) => [key, { + key, + accepted: 0, + active: 0, + peakActive: 0, + drops: 0, + errors: 0, + }])); + const snapshot = () => { + const surfaces = [...states.values()].map((state) => ({ ...state })); + return { + connectionsExpected: states.size, + connectionsAccepted: surfaces.reduce((sum, state) => sum + state.accepted, 0), + connectionsActive: surfaces.reduce((sum, state) => sum + state.active, 0), + connectionDrops: surfaces.reduce((sum, state) => sum + state.drops, 0), + connectionErrors: surfaces.reduce((sum, state) => sum + state.errors, 0), + connectionsPerSurface: surfaces, + }; + }; + const trackAccepted = (key, socket) => { + const state = states.get(key); + if (!state) { + socket.destroy(); + return false; + } + state.accepted += 1; + state.active += 1; + state.peakActive = Math.max(state.peakActive, state.active); + let released = false; + const release = (errored) => { + if (released) return; + released = true; + state.active = Math.max(0, state.active - 1); + state.drops += 1; + if (errored) state.errors += 1; + }; + socket.once('error', () => release(true)); + socket.once('close', () => release(false)); + if (socket.destroyed) release(false); + return true; + }; + const assertResident = () => { + const current = snapshot(); + const exact = current.connectionsPerSurface.every((state) => + state.accepted === 1 + && state.active === 1 + && state.peakActive === 1 + && state.drops === 0 + && state.errors === 0 + ); + if (!exact || current.connectionsActive !== current.connectionsExpected) { + throw new Error( + `PDCF_REALTIME_CONNECTIONS_NOT_RESIDENT:${current.connectionsActive}:${current.connectionsExpected}:${current.connectionDrops}:${current.connectionErrors}` + ); + } + return current; + }; + return { assertResident, snapshot, trackAccepted }; +}; + +const parseBoolean = (value, label) => { + if (value === true || value === 'true' || value === '1') return true; + if (value === false || value === 'false' || value === '0' || value == null) return false; + throw new Error(`PDCF_INVALID_BOOLEAN:${label}`); +}; + +const parseRuntimePoolMaxUses = (value) => { + if (value === 'unlimited') return null; + if (typeof value !== 'string' || !/^[1-9]\d*$/.test(value)) { + throw new Error('PDCF_INVALID_MAX_USES:runtime-pool-max-uses'); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error('PDCF_INVALID_MAX_USES:runtime-pool-max-uses'); + } + return parsed; +}; + +const optionalSha256 = (value, label) => { + if (value == null) return null; + if (typeof value !== 'string' || !/^sha256:[a-f0-9]{64}$/.test(value)) { + throw new Error(`PDCF_INVALID_SHA256:${label}`); + } + return value; +}; + +const assertProvisionCloneManifest = (manifest, { cloneId, runPurpose }) => { + const provisionClone = manifest?.provisionClone; + if ( + JSON.stringify(Object.keys(provisionClone ?? {}).sort()) + !== JSON.stringify(['attestationSetSha256', 'id', 'purpose', 'version']) + || + provisionClone?.version !== 1 + || provisionClone.id !== cloneId + || provisionClone.purpose !== runPurpose + || !/^sha256:[a-f0-9]{64}$/.test(provisionClone.attestationSetSha256 ?? '') + ) { + throw new Error('PDCF_PROVISION_CLONE_MISMATCH'); + } + for (const customer of manifest.customers ?? []) { + const attestation = customer.provisionAttestation; + if ( + JSON.stringify(Object.keys(attestation ?? {}).sort()) + !== JSON.stringify(['cloneId', 'purpose', 'sha256', 'version']) + || + attestation?.version !== 1 + || attestation.cloneId !== cloneId + || attestation.purpose !== runPurpose + || !/^sha256:[a-f0-9]{64}$/.test(attestation.sha256 ?? '') + ) { + throw new Error(`PDCF_PROVISION_ATTESTATION_INVALID:${customer.id ?? 'unknown'}`); + } + } + if (provisionAttestationSetSha256(manifest.customers) !== provisionClone.attestationSetSha256) { + throw new Error('PDCF_PROVISION_ATTESTATION_SET_MISMATCH'); + } + return provisionClone; +}; + +const assertCustomerContract = (customer, contract) => { + if ( + contract.databaseContractFingerprint !== customer.databaseContractFingerprint + || JSON.stringify(contract.structuralFingerprints) + !== JSON.stringify(customer.structuralFingerprints) + ) { + throw new Error(`PDCF_LIVE_DATABASE_CONTRACT_MISMATCH:${customer.id}`); + } + return contract; +}; + +const parseServerOptions = (argv, environment = process.env) => { + const args = parseArgs(argv); + const host = requireString(args, 'host', '127.0.0.1'); + if (!LOOPBACK_HOSTS.has(host)) throw new Error('PDCF_SERVER_LOOPBACK_REQUIRED'); + const mode = requireString(args, 'mode', 'scoped-required'); + if (!['stock', 'scoped-required'].includes(mode)) { + throw new Error(`PDCF_INTROSPECTION_MODE_INVALID:${mode}`); + } + const introspectionClientReleaseMode = requireString( + args, + 'introspection-client-release-mode', + 'destroy', + ); + if (!['reuse', 'destroy'].includes(introspectionClientReleaseMode)) { + throw new Error( + `PDCF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:${introspectionClientReleaseMode}` + ); + } + const realtimeNotificationMode = requireString( + args, + 'realtime-notification-mode', + 'dedicated', + ); + if (!['dedicated', 'shared-exact'].includes(realtimeNotificationMode)) { + throw new Error( + `PDCF_REALTIME_NOTIFICATION_MODE_INVALID:${realtimeNotificationMode}` + ); + } + const enableRealtime = parseBoolean(args['enable-realtime'], 'enable-realtime'); + if (!enableRealtime && realtimeNotificationMode !== 'dedicated') { + throw new Error('PDCF_SHARED_REALTIME_REQUIRES_REALTIME'); + } + return { + host, + port: parsePositiveInteger(args.port ?? '3410', 'port'), + arm: requireString(args, 'arm', 'physical-db-idle-30s'), + runPurpose: requireRunPurpose(requireString(args, 'run-purpose')), + cloneId: requireCloneId(requireString(args, 'clone-id')), + mode, + introspectionClientReleaseMode, + manifestFile: path.resolve(requireString(args, 'manifest')), + secretsFile: path.resolve(requireString(args, 'secrets')), + customerCount: parsePositiveInteger(args.customers ?? '1', 'customers'), + runtimePoolMax: parsePositiveInteger(args['runtime-pool-max'] ?? '2', 'runtime-pool-max'), + runtimePoolMaxUses: parseRuntimePoolMaxUses( + args['runtime-pool-max-uses'] ?? 'unlimited', + ), + enableRealtime, + realtimeNotificationMode, + realtimeCursorPollIntervalMs: parsePositiveInteger( + args['realtime-cursor-poll-ms'] ?? '5000', + 'realtime-cursor-poll-ms', + ), + realtimeCursorHeartbeatIntervalMs: parsePositiveInteger( + args['realtime-cursor-heartbeat-ms'] ?? '30000', + 'realtime-cursor-heartbeat-ms', + ), + expectedDatabaseContractFingerprint: optionalSha256( + args['expected-database-contract'], + 'expected-database-contract', + ), + blueprintCompatibilityFingerprint: optionalSha256( + args['blueprint-compatibility'], + 'blueprint-compatibility', + ), + expectedManifestSha256: optionalSha256( + args['expected-manifest-sha256'], + 'expected-manifest-sha256', + ), + observabilityToken: environment.GRAPHQL_OBSERVABILITY_TOKEN ?? '', + benchmarkRetainedHeapEnabled: parseBoolean( + environment.GRAPHQL_CPERF_RETAINED_HEAP_ENABLED, + 'GRAPHQL_CPERF_RETAINED_HEAP_ENABLED', + ), + }; +}; + +const withProcessEnvironment = async (overrides, callback) => { + const previous = Object.fromEntries(SECURITY_ENVIRONMENT_KEYS.map((key) => [ + key, + Object.prototype.hasOwnProperty.call(process.env, key) ? process.env[key] : undefined, + ])); + try { + for (const [key, value] of Object.entries(overrides)) { + if (value == null) delete process.env[key]; + else process.env[key] = value; + } + return await callback(); + } finally { + for (const [key, value] of Object.entries(previous)) { + if (value == null) delete process.env[key]; + else process.env[key] = value; + } + } +}; + +const isLoopbackRequest = (request) => { + const address = request.socket?.remoteAddress ?? ''; + return address === '127.0.0.1' || address === '::1' || address === '::ffff:127.0.0.1'; +}; + +const bearerToken = (request) => { + const value = request.get('authorization'); + return value?.startsWith('Bearer ') ? value.slice('Bearer '.length) : ''; +}; + +const tokenEqual = (left, right) => { + if (!left || !right) return false; + const leftBytes = Buffer.from(left); + const rightBytes = Buffer.from(right); + return leftBytes.length === rightBytes.length + && crypto.timingSafeEqual(leftBytes, rightBytes); +}; + +const canonicalJson = (value) => { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value && typeof value === 'object') { + return `{${Object.keys(value).sort().map((key) => + `${JSON.stringify(key)}:${canonicalJson(value[key])}` + ).join(',')}}`; + } + return JSON.stringify(value); +}; + +const makeGraphileActivityVector = (entries) => entries.map((entry) => { + const inflight = entry.inflight ?? 0; + const websocketSockets = entry.websocketSockets?.size ?? 0; + const transientHttpInFlight = inflight - websocketSockets; + if ( + !Number.isSafeInteger(inflight) + || inflight < 0 + || !Number.isSafeInteger(websocketSockets) + || websocketSockets < 0 + || transientHttpInFlight < 0 + ) { + throw new Error(`PDCF_GRAPHILE_ACTIVITY_ACCOUNTING_INVALID:${entry.cacheKey}`); + } + return { + buildContract: entry.cacheKey, + inflight, + websocketSockets, + transientHttpInFlight, + }; +}).sort((left, right) => left.buildContract.localeCompare(right.buildContract)); + +const counterValue = (value, label) => { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`PDCF_RETAINED_MEMORY_COUNTER_INVALID:${label}`); + } + return value; +}; + +const makeRetainedMemoryGuard = (state) => { + const residentBuildContracts = [...state.residentBuildContracts].sort(); + const graphileActivityByBuildContract = [ + ...(state.graphileActivityByBuildContract ?? []), + ].sort((left, right) => left.buildContract.localeCompare(right.buildContract)); + const realtime = state.realtime ?? {}; + const cacheCounters = state.cacheCounters ?? {}; + const graphileWebsocketSockets = graphileActivityByBuildContract.reduce( + (sum, entry) => sum + entry.websocketSockets, + 0, + ); + const graphileTransientHttpInFlight = graphileActivityByBuildContract.reduce( + (sum, entry) => sum + entry.transientHttpInFlight, + 0, + ); + const httpRequestsOutstanding = + counterValue(cacheCounters.httpRequestsStarted, 'httpRequestsStarted') + - counterValue(cacheCounters.httpRequestsCompleted, 'httpRequestsCompleted'); + const websocketUpgradesOutstanding = + counterValue(cacheCounters.websocketUpgradesStarted, 'websocketUpgradesStarted') + - counterValue(cacheCounters.websocketUpgradesCompleted, 'websocketUpgradesCompleted'); + const realtimeConnectionsExpected = counterValue( + realtime.connectionsExpected, + 'realtime.connectionsExpected', + ); + const realtimeConnectionsActive = counterValue( + realtime.connectionsActive, + 'realtime.connectionsActive', + ); + const realtimeConnectionsAccepted = counterValue( + realtime.connectionsAccepted, + 'realtime.connectionsAccepted', + ); + const realtimeConnectionDrops = counterValue( + realtime.connectionDrops, + 'realtime.connectionDrops', + ); + const realtimeConnectionErrors = counterValue( + realtime.connectionErrors, + 'realtime.connectionErrors', + ); + const realtimeManagersExpected = counterValue( + realtime.managersExpected, + 'realtime.managersExpected', + ); + const realtimeManagersActive = counterValue( + realtime.managersActive, + 'realtime.managersActive', + ); + const realtimePerSurface = realtime.connectionsPerSurface ?? []; + const activityContractsExact = + graphileActivityByBuildContract.length === residentBuildContracts.length + && graphileActivityByBuildContract.every( + (entry, index) => entry.buildContract === residentBuildContracts[index], + ); + const realtimeSocketsExactPerContract = realtimeConnectionsExpected === 0 + ? graphileActivityByBuildContract.every((entry) => entry.websocketSockets === 0) + : realtimeConnectionsExpected === graphileActivityByBuildContract.length + && graphileActivityByBuildContract.every((entry) => entry.websocketSockets === 1); + const realtimePerSurfaceExact = realtimePerSurface.length === realtimeConnectionsExpected + && realtimePerSurface.every((surface) => + surface.accepted === 1 + && surface.active === 1 + && surface.peakActive === 1 + && surface.drops === 0 + && surface.errors === 0 + ); + const realtimeResident = + realtimeManagersExpected === realtimeConnectionsExpected + && realtimeManagersActive === realtimeManagersExpected + && realtimeConnectionsAccepted === realtimeConnectionsExpected + && realtimeConnectionsActive === realtimeConnectionsExpected + && realtimeConnectionDrops === 0 + && realtimeConnectionErrors === 0 + && graphileWebsocketSockets === realtimeConnectionsExpected + && realtimeSocketsExactPerContract + && websocketUpgradesOutstanding === graphileWebsocketSockets + && realtimePerSurfaceExact; + const handlerAccountingExact = + activityContractsExact + && httpRequestsOutstanding === graphileTransientHttpInFlight + && websocketUpgradesOutstanding === graphileWebsocketSockets; + const normalizedState = { + ...state, + residentBuildContracts, + graphileActivityByBuildContract, + }; + return { + pid: state.pid, + graphileInFlight: state.graphileInFlight, + residentBuildContracts, + graphileActivityByBuildContract, + graphileTransientHttpInFlight, + graphileWebsocketSockets, + activityContractsExact, + httpRequestsOutstanding, + websocketUpgradesOutstanding, + handlerAccountingExact, + realtimeConnectionsExpected, + realtimeConnectionsActive, + realtimeResident, + stateSha256: `sha256:${crypto.createHash('sha256') + .update(canonicalJson(normalizedState)) + .digest('hex')}`, + state: normalizedState, + }; +}; + +const retainedMemoryGuardErrors = (guard) => { + const errors = []; + if (guard.graphileInFlight !== 0) { + errors.push(`PDCF_RETAINED_MEMORY_IN_FLIGHT:${guard.graphileInFlight}`); + } + if (!guard.handlerAccountingExact) { + errors.push( + `PDCF_RETAINED_MEMORY_HANDLER_ACCOUNTING_MISMATCH:` + + `${guard.httpRequestsOutstanding}:${guard.graphileTransientHttpInFlight}:` + + `${guard.websocketUpgradesOutstanding}:${guard.graphileWebsocketSockets}` + ); + } + if (!guard.realtimeResident) { + errors.push( + `PDCF_RETAINED_MEMORY_REALTIME_NOT_RESIDENT:` + + `${guard.graphileWebsocketSockets}:${guard.realtimeConnectionsActive}:` + + `${guard.realtimeConnectionsExpected}` + ); + } + return errors; +}; + +const memoryRange = (samples, field) => { + const values = samples.map((sample) => sample[field]); + return Math.max(...values) - Math.min(...values); +}; + +const memoryConvergenceThreshold = (samples, field) => Math.max( + MIB, + Math.ceil(Math.max(...samples.map((sample) => sample[field])) * 0.0025), +); + +const collectRetainedMemoryCheckpoint = async ({ + forceGc, + readMemory = () => process.memoryUsage(), + readGuard, + monotonicNow = () => process.hrtime.bigint(), + yieldTurn = () => new Promise((resolve) => setImmediate(resolve)), + rounds = RETAINED_MEMORY_GC_ROUNDS, +}) => { + if (typeof forceGc !== 'function') { + throw new Error('PDCF_RETAINED_MEMORY_GC_UNAVAILABLE'); + } + if (!Number.isSafeInteger(rounds) || rounds < 5 || rounds > 8) { + throw new Error('PDCF_RETAINED_MEMORY_GC_ROUNDS_INVALID'); + } + const guardBefore = readGuard(); + const beforeErrors = retainedMemoryGuardErrors(guardBefore); + if (beforeErrors.length > 0) throw new Error(beforeErrors[0]); + const samples = []; + for (let index = 0; index < rounds; index++) { + forceGc(); + await yieldTurn(); + const memory = readMemory(); + samples.push({ + timestamp: new Date().toISOString(), + monotonicNs: String(monotonicNow()), + heapUsedBytes: memory.heapUsed, + externalBytes: memory.external, + arrayBuffersBytes: memory.arrayBuffers, + rssBytes: memory.rss, + }); + } + const guardAfter = readGuard(); + const stableSamples = samples.slice(-RETAINED_MEMORY_STABLE_SAMPLES); + const heapSpreadBytes = memoryRange(stableSamples, 'heapUsedBytes'); + const externalSpreadBytes = memoryRange(stableSamples, 'externalBytes'); + const heapThresholdBytes = memoryConvergenceThreshold( + stableSamples, + 'heapUsedBytes', + ); + const externalThresholdBytes = memoryConvergenceThreshold( + stableSamples, + 'externalBytes', + ); + const errors = []; + errors.push(...retainedMemoryGuardErrors(guardAfter)); + if (guardBefore.pid !== guardAfter.pid) { + errors.push('PDCF_RETAINED_MEMORY_PID_CHANGED'); + } + if (guardBefore.stateSha256 !== guardAfter.stateSha256) { + errors.push('PDCF_RETAINED_MEMORY_RESIDENCY_OR_COUNTERS_CHANGED'); + } + if (heapSpreadBytes > heapThresholdBytes) { + errors.push( + `PDCF_RETAINED_HEAP_NOT_CONVERGED:${heapSpreadBytes}:${heapThresholdBytes}` + ); + } + if (externalSpreadBytes > externalThresholdBytes) { + errors.push( + `PDCF_RETAINED_EXTERNAL_NOT_CONVERGED:${externalSpreadBytes}:${externalThresholdBytes}` + ); + } + return { + version: 1, + fixture: 'physical-database-density-v1', + pid: guardBefore.pid, + gcRounds: rounds, + stableSampleCount: RETAINED_MEMORY_STABLE_SAMPLES, + stable: errors.length === 0, + samples, + guardBefore, + guardAfter, + errors, + }; +}; + +const authorizeRetainedMemoryCheckpoint = (request, options) => + options.benchmarkRetainedHeapEnabled === true + && isLoopbackRequest(request) + && tokenEqual(bearerToken(request), options.observabilityToken); + +const classifyDatabaseScope = (present, controlDatabase, fixtureDatabases) => { + const expected = new Set([controlDatabase, ...fixtureDatabases]); + const unexpected = present.filter((database) => !expected.has(database)); + const presentSet = new Set(present); + const missingFixture = fixtureDatabases.filter((database) => !presentSet.has(database)); + return { + dedicated: unexpected.length === 0 && missingFixture.length === 0, + databasesPresent: present.length, + fixtureDatabasesExpected: fixtureDatabases.length, + fixtureDatabasesPresent: fixtureDatabases.length - missingFixture.length, + unexpectedDatabases: unexpected.length, + missingFixtureDatabases: missingFixture.length, + unexpectedDatabaseSetSha256: unexpected.length === 0 + ? null + : `sha256:${crypto.createHash('sha256').update(unexpected.join('\0')).digest('hex')}`, + }; +}; + +const aggregateRuntimePoolStats = (children, requestedMaxUses) => { + const childStats = children.map(({ child }) => child.runtimePoolStats()); + const identitiesUnique = childStats.every((stats) => stats?.identitiesUnique === true); + const runtimePoolObjects = children.flatMap(({ child }) => + typeof child.runtimePoolObjects === 'function' + ? child.runtimePoolObjects() + : [] + ); + const effectiveKnown = childStats.every((stats) => + stats?.effectiveMaxUsesKnown === true + ); + const effectiveValues = effectiveKnown + ? [...new Set(childStats.map((stats) => stats.effectiveMaxUses))] + : []; + const effectiveMaxUsesKnown = effectiveValues.length === 1; + const effectiveMaxUses = effectiveMaxUsesKnown ? effectiveValues[0] : null; + const expectedPools = childStats.reduce( + (sum, stats) => sum + (Number.isSafeInteger(stats?.expectedPools) ? stats.expectedPools : 0), + 0, + ); + const observedPools = childStats.reduce( + (sum, stats) => sum + (Number.isSafeInteger(stats?.observedPools) ? stats.observedPools : 0), + 0, + ); + const poolObjectsUnique = childStats.every((stats) => stats?.poolObjectsUnique === true) + && runtimePoolObjects.length === expectedPools + && runtimePoolObjects.every(Boolean) + && new Set(runtimePoolObjects).size === runtimePoolObjects.length; + const available = childStats.length > 0 + && childStats.every((stats) => + stats?.scope === 'runtime-only-exact-identities' + && stats.available === true + && stats.identitiesUnique === true + && stats.poolObjectsUnique === true + && stats.requestedMaxUses === requestedMaxUses + && stats.maxUsesExact === true + ) + && observedPools === expectedPools + && poolObjectsUnique + && effectiveMaxUsesKnown; + const sum = (key) => available + ? childStats.reduce((total, stats) => total + stats[key], 0) + : null; + return { + scope: 'runtime-only-exact-identities', + available, + requestedMaxUses, + effectiveMaxUses, + effectiveMaxUsesKnown, + maxUsesExact: available && effectiveMaxUses === requestedMaxUses, + identitiesUnique, + poolObjectsUnique, + expectedPools, + observedPools, + totalClients: sum('totalClients'), + idleClients: sum('idleClients'), + waitingClients: sum('waitingClients'), + }; +}; + +const runtimeEnvironmentFor = ( + environment, + customer, + secretResolver, + includeNotification = false, +) => ({ + ...environment, + PGDATABASE: customer.database, + // Runtime maxUses is supplied through the explicit per-pool config. Keep + // every ambient/control/notification pool on unlimited reuse. + PG_POOL_MAX_USES: '0', + ...Object.fromEntries(TENANTS.flatMap((tenant) => [ + [`CTF_RUNTIME_${tenant.id.toUpperCase()}_PGUSER`, customer.roles[tenant.id]], + [ + `CTF_RUNTIME_${tenant.id.toUpperCase()}_PGPASSWORD`, + secretResolver.runtimePasswordFor(customer.roles[tenant.id]), + ], + ])), + ...(includeNotification ? { + CTF_NOTIFICATION_PGUSER: customer.notificationRole, + CTF_NOTIFICATION_PGPASSWORD: + secretResolver.notificationPasswordFor(customer.notificationRole), + } : {}), +}); + +const completeServerOptionsFor = (options, customer, environment) => ({ + ...completeServer.parseServerOptions([ + '--host', options.host, + '--port', String(options.port), + '--arm', options.arm, + '--mode', options.mode, + '--introspection-client-release-mode', options.introspectionClientReleaseMode, + '--runtime-pool-max', String(options.runtimePoolMax), + '--runtime-pool-max-uses', options.runtimePoolMaxUses == null + ? 'unlimited' + : String(options.runtimePoolMaxUses), + '--enable-realtime', String(options.enableRealtime), + '--realtime-notification-mode', options.realtimeNotificationMode, + '--realtime-cursor-poll-ms', String(options.realtimeCursorPollIntervalMs), + '--realtime-cursor-heartbeat-ms', + String(options.realtimeCursorHeartbeatIntervalMs), + ...(options.realtimeNotificationMode === 'shared-exact' ? [ + '--notification-role', customer.notificationRole, + ] : []), + ...TENANTS.flatMap((tenant) => [ + `--${tenant.runtimeRoleArgument}`, + customer.roles[tenant.id], + ]), + ], environment), + runPurpose: options.runPurpose, + cloneId: options.cloneId, + provisionCustomerId: customer.id, + provisionAttestation: customer.provisionAttestation, +}); + +const createPhysicalDatabaseServer = async (options, environment = process.env) => { + if ( + options.benchmarkRetainedHeapEnabled + && typeof global.gc !== 'function' + ) { + throw new Error('PDCF_RETAINED_MEMORY_REQUIRES_EXPOSE_GC'); + } + if (options.expectedManifestSha256) { + const actual = `sha256:${crypto.createHash('sha256') + .update(fs.readFileSync(options.manifestFile)) + .digest('hex')}`; + if (actual !== options.expectedManifestSha256) { + throw new Error('PDCF_EXPECTED_MANIFEST_SHA256_MISMATCH'); + } + } + const { manifest, secretResolver } = loadProvision( + options.manifestFile, + options.secretsFile, + ); + const provisionClone = assertProvisionCloneManifest(manifest, options); + if ( + options.expectedDatabaseContractFingerprint + && manifest.canonicalDatabaseContractFingerprint + !== options.expectedDatabaseContractFingerprint + ) { + throw new Error('PDCF_EXPECTED_DATABASE_CONTRACT_MISMATCH'); + } + if (options.customerCount !== manifest.customers.length) { + throw new Error( + `PDCF_CUSTOMER_COUNT_MUST_EQUAL_PROVISIONED:${options.customerCount}:${manifest.customers.length}` + ); + } + if ( + options.enableRealtime + && options.realtimeNotificationMode === 'dedicated' + && options.runtimePoolMax < 2 + ) { + throw new Error('PDCF_REALTIME_REQUIRES_RUNTIME_POOL_MAX_2'); + } + const customers = manifest.customers.slice(0, options.customerCount); + const verifiedContracts = new Map(customers.map((customer) => { + const contract = options.runPurpose === 'hostile-preflight' + ? assertCustomerContract(customer, inspectCustomerContract({ + customer, + canonicalSchemas: manifest.canonicalSchemas, + environment, + })) + : { + structuralFingerprints: customer.structuralFingerprints, + databaseContractFingerprint: customer.databaseContractFingerprint, + }; + return [customer.id, { + ...contract, + verification: options.runPurpose === 'hostile-preflight' + ? 'live-recomputed' + : 'provision-manifest', + }]; + })); + const children = []; + for (const customer of customers) { + const childEnvironment = runtimeEnvironmentFor( + environment, + customer, + secretResolver, + options.realtimeNotificationMode === 'shared-exact', + ); + const childOptions = completeServerOptionsFor(options, customer, childEnvironment); + const processOverrides = Object.fromEntries(SECURITY_ENVIRONMENT_KEYS.map((key) => [ + key, + childEnvironment[key], + ])); + const child = await withProcessEnvironment(processOverrides, () => + completeServer.createFixtureServer(childOptions, childEnvironment) + ); + children.push({ customer, child }); + } + + const express = require(path.join(REPO_ROOT, 'graphql/server/node_modules/express')); + const { Pool } = require(path.join(REPO_ROOT, 'graphql/server/node_modules/pg')); + const { getDebugMemorySnapshot } = requireBuilt( + 'graphql/server/dist/diagnostics/debug-memory-snapshot.js' + ); + const { + getGraphileRealtimeRoleAuditStats, + deleteGraphileCacheEntry, + getCacheCounters, + graphileCache, + } = requireBuilt('graphile/graphile-cache/dist/index.js'); + const { getInFlightCount } = requireBuilt( + 'graphql/server/dist/middleware/graphile.js' + ); + const { getGraphileGovernorCounters } = requireBuilt( + 'graphql/server/dist/middleware/graphile-build-governor.js' + ); + const { getGraphileBuildStats } = requireBuilt( + 'graphql/server/dist/middleware/observability/graphile-build-stats.js' + ); + const { + getPgCacheStats, + getPgNotificationBrokerStats, + } = requireBuilt('postgres/pg-cache/dist/index.js'); + const pgEnv = require(path.join(REPO_ROOT, 'graphql/server/node_modules/pg-env')); + const controlConfig = pgEnv.getPgEnvOptions({ + database: environment.PGDATABASE ?? 'postgres', + }); + const observerPool = new Pool({ + host: controlConfig.host, + port: Number(controlConfig.port), + database: controlConfig.database, + user: controlConfig.user, + password: controlConfig.password, + application_name: 'cperf-physical-database-observer', + max: 1, + idleTimeoutMillis: 0, + connectionTimeoutMillis: 5_000, + }); + observerPool.on('error', () => undefined); + + const realtimeConnections = createRealtimeConnectionRegistry( + options.enableRealtime + ? customers.flatMap((customer) => TENANTS.map((tenant) => + `${customer.id}:${tenant.id}` + )) + : [] + ); + let httpServer = null; + let closing = false; + let retainedMemoryCheckpointRunning = false; + + const cacheEntries = () => [...graphileCache.values()]; + const realtimeStats = () => { + const entries = cacheEntries(); + const connections = realtimeConnections.snapshot(); + const notificationBrokers = getPgNotificationBrokerStats(); + const notificationRoleAudits = getGraphileRealtimeRoleAuditStats(); + return { + managersExpected: connections.connectionsExpected, + managersActive: entries.filter((entry) => entry.realtimeManager?.isRunning).length, + ...connections, + // Compatibility names consumed by the v3 scorer. These now describe + // server-side accepted connections, never client objects in this process. + transportsExpected: connections.connectionsExpected, + transportsActive: connections.connectionsActive, + transportErrors: connections.connectionErrors > 0 + ? ['PDCF_REALTIME_SERVER_CONNECTION_ERROR'] + : [], + notificationMode: options.realtimeNotificationMode, + notificationBrokers, + notificationRoleAudits, + }; + }; + + const poolStats = () => aggregateRuntimePoolStats( + children, + options.runtimePoolMaxUses, + ); + + const buildContractFingerprintForLiveIdentity = (cacheKey) => { + const matches = children + .map(({ child }) => child.buildContractFingerprintForLiveIdentity(cacheKey)) + .filter(Boolean); + if (matches.length !== 1) { + throw new Error(`PDCF_BUILD_CONTRACT_EVIDENCE_MAPPING_INVALID:${matches.length}`); + } + return matches[0]; + }; + + const contractEvidence = () => ({ + version: 1, + credentialFree: true, + liveIdentityScope: 'process-local-keyed-hmac-v1', + customers: Object.fromEntries(children.map(({ customer, child }) => [ + customer.id, + child.contractEvidence(), + ])), + residentGraphileBuildFingerprints: [...graphileCache.keys()] + .map(buildContractFingerprintForLiveIdentity) + .sort(), + }); + + const retainedMemoryGuard = () => { + const residentBuildContracts = [...graphileCache.keys()].sort(); + const residentBuildContractFingerprints = residentBuildContracts + .map(buildContractFingerprintForLiveIdentity) + .sort(); + const graphileActivityByBuildContract = makeGraphileActivityVector(cacheEntries()); + const graphileTransientHttpInFlight = graphileActivityByBuildContract.reduce( + (sum, entry) => sum + entry.transientHttpInFlight, + 0, + ); + const graphileBuildsInFlight = getInFlightCount(); + const builds = getGraphileBuildStats(); + const pgCacheStats = getPgCacheStats(); + const realtime = realtimeStats(); + return makeRetainedMemoryGuard({ + pid: process.pid, + // Long-lived GraphQL WebSocket sockets are the expected resident state, + // so only build work and transient HTTP handlers block a full-GC sample. + graphileInFlight: graphileBuildsInFlight + graphileTransientHttpInFlight, + graphileBuildsInFlight, + graphileTransientHttpInFlight, + graphileActivityByBuildContract, + residentBuildContracts, + residentBuildContractFingerprints, + cacheCounters: getCacheCounters(), + governorCounters: getGraphileGovernorCounters(), + buildCounters: { + started: builds.started, + succeeded: builds.succeeded, + failed: builds.failed, + }, + pgCacheMonotonicCounters: { + capacityEvictions: pgCacheStats.capacityEvictions, + capacityRefusals: pgCacheStats.capacityRefusals, + disposalFailures: pgCacheStats.disposalFailures, + }, + realtime: { + managersExpected: realtime.managersExpected, + managersActive: realtime.managersActive, + connectionsExpected: realtime.connectionsExpected, + connectionsAccepted: realtime.connectionsAccepted, + connectionsActive: realtime.connectionsActive, + connectionDrops: realtime.connectionDrops, + connectionErrors: realtime.connectionErrors, + connectionsPerSurface: realtime.connectionsPerSurface, + }, + }); + }; + + const backendStats = async () => { + const result = await observerPool.query(` + SELECT datname, + COALESCE(state, 'unknown') AS state, + count(*)::integer AS count + FROM pg_catalog.pg_stat_activity + WHERE backend_type = 'client backend' + AND datname = ANY($1::text[]) + GROUP BY datname, COALESCE(state, 'unknown') + ORDER BY datname, state + `, [customers.map((customer) => customer.database)]); + const byDatabase = Object.fromEntries(customers.map((customer) => [ + customer.database, + { total: 0, active: 0, idle: 0, idleInTransaction: 0, other: 0 }, + ])); + for (const row of result.rows) { + const state = byDatabase[row.datname]; + if (!state) continue; + state.total += row.count; + if (row.state === 'active') state.active += row.count; + else if (row.state === 'idle') state.idle += row.count; + else if (row.state === 'idle in transaction') state.idleInTransaction += row.count; + else state.other += row.count; + } + return { + total: Object.values(byDatabase).reduce((sum, value) => sum + value.total, 0), + active: Object.values(byDatabase).reduce((sum, value) => sum + value.active, 0), + idle: Object.values(byDatabase).reduce((sum, value) => sum + value.idle, 0), + idleInTransaction: Object.values(byDatabase) + .reduce((sum, value) => sum + value.idleInTransaction, 0), + other: Object.values(byDatabase).reduce((sum, value) => sum + value.other, 0), + byDatabase, + observerExcluded: true, + }; + }; + + const databaseScope = async () => { + const result = await observerPool.query(` + SELECT datname + FROM pg_catalog.pg_database + WHERE NOT datistemplate + ORDER BY datname + `); + const present = result.rows.map((row) => row.datname); + const fixtureDatabases = customers.map((customer) => customer.database); + return classifyDatabaseScope(present, controlConfig.database, fixtureDatabases); + }; + + const assertRealtimeResident = () => { + const realtime = realtimeStats(); + if (realtime.managersActive !== realtime.managersExpected) { + throw new Error( + `PDCF_REALTIME_MANAGERS_NOT_READY:${realtime.managersActive}:${realtime.managersExpected}` + ); + } + if (options.realtimeNotificationMode === 'shared-exact') { + const expectedBrokers = customers.length; + const expectedLeases = customers.length * TENANTS.length; + const brokers = realtime.notificationBrokers; + const audits = realtime.notificationRoleAudits; + if ( + brokers.brokers !== expectedBrokers + || brokers.listenerConnections !== expectedBrokers + || brokers.leases !== expectedLeases + || brokers.topics !== expectedLeases + || brokers.subscribers !== expectedLeases + || brokers.queueOverflows !== 0 + || brokers.fatalFailures !== 0 + || audits.identities !== expectedBrokers + || audits.healthy !== expectedBrokers + || audits.failed !== 0 + || audits.stale !== 0 + || audits.catalogAuditAttempts < expectedLeases + || audits.catalogAuditFailures !== 0 + || audits.activeDatabaseTargets !== expectedBrokers + || audits.databaseConfigurationConflicts !== 0 + ) { + throw new Error( + `PDCF_SHARED_REALTIME_NOT_EXACT:${JSON.stringify({ brokers, audits })}` + ); + } + } + realtimeConnections.assertResident(); + return realtime; + }; + + const app = express(); + app.disable('x-powered-by'); + + app.get('/healthz', (_request, response) => { + response.json({ + status: 'ok', + fixture: 'physical-database-density-v1', + customers: customers.length, + physicalDatabases: customers.length, + }); + }); + + app.get('/debug/memory', async (request, response) => { + if (!isLoopbackRequest(request)) { + response.status(404).send('Not found'); + return; + } + if ( + environment.NODE_ENV !== 'development' + && !tokenEqual(bearerToken(request), options.observabilityToken) + ) { + response.status(401).json({ error: { code: 'PDCF_OBSERVABILITY_UNAUTHORIZED' } }); + return; + } + try { + const [backends, containerScope] = await Promise.all([ + backendStats(), + databaseScope(), + ]); + response.json({ + ...getDebugMemorySnapshot(), + physicalDatabaseFixture: { + fixture: 'physical-database-density-v1', + customers: customers.length, + physicalDatabases: customers.length, + canonicalStructuralFingerprint: + manifest.canonicalStructuralFingerprint?.combined?.sha256 ?? null, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint ?? null, + blueprintCompatibilityFingerprint: + options.blueprintCompatibilityFingerprint, + pools: poolStats(), + contractEvidence: contractEvidence(), + backends, + containerScope, + realtime: realtimeStats(), + }, + }); + } catch (error) { + response.status(503).json({ + error: { + code: 'PDCF_TELEMETRY_UNAVAILABLE', + message: error instanceof Error ? error.message : String(error), + }, + }); + } + }); + + app.post('/__cperf/post-warmup', async (request, response) => { + if ( + !isLoopbackRequest(request) + || !tokenEqual(bearerToken(request), options.observabilityToken) + ) { + response.status(404).send('Not found'); + return; + } + try { + // The perf-harness process owns and verifies graphql-ws clients. This + // measured process only proves its managers and accepted inbound sockets + // are resident at the exact warm boundary. + response.json({ ok: true, realtime: assertRealtimeResident() }); + } catch (error) { + response.status(503).json({ + error: { + code: 'PDCF_REALTIME_NOT_RESIDENT', + message: error instanceof Error ? error.message : String(error), + }, + }); + } + }); + + app.post('/__cperf/retained-memory-checkpoint', async (request, response) => { + if (!authorizeRetainedMemoryCheckpoint(request, options)) { + response.status(404).send('Not found'); + return; + } + if (retainedMemoryCheckpointRunning) { + response.status(409).json({ + error: { code: 'PDCF_RETAINED_MEMORY_CHECKPOINT_RUNNING' }, + }); + return; + } + retainedMemoryCheckpointRunning = true; + try { + const checkpoint = await collectRetainedMemoryCheckpoint({ + forceGc: global.gc, + readGuard: retainedMemoryGuard, + }); + response.status(checkpoint.stable ? 200 : 503).json(checkpoint); + } catch (error) { + response.status(503).json({ + error: { + code: error instanceof Error + ? error.message.split(':', 1)[0] + : 'PDCF_RETAINED_MEMORY_CHECKPOINT_FAILED', + message: error instanceof Error ? error.message : String(error), + }, + }); + } finally { + retainedMemoryCheckpointRunning = false; + } + }); + + app.get('/__physical/status', async (request, response) => { + if (!isLoopbackRequest(request)) { + response.status(404).send('Not found'); + return; + } + try { + const liveAttestations = await Promise.all(children.map(({ child }) => + child.readProvisionAttestation() + )); + const attestedCustomers = customers.map((customer, index) => ({ + ...customer, + provisionAttestation: liveAttestations[index], + })); + if ( + provisionAttestationSetSha256(attestedCustomers) + !== provisionClone.attestationSetSha256 + ) { + throw new Error('PDCF_LIVE_PROVISION_ATTESTATION_SET_MISMATCH'); + } + response.json({ + version: 1, + fixture: 'physical-database-density-v1', + arm: options.arm, + runPurpose: options.runPurpose, + cloneId: options.cloneId, + provisionClone: { + ...provisionClone, + verified: true, + }, + introspectionMode: options.mode, + introspectionClientReleaseMode: options.introspectionClientReleaseMode, + runtimePoolMax: options.runtimePoolMax, + runtimePoolMaxUses: options.runtimePoolMaxUses, + runtimePools: poolStats(), + contractEvidence: contractEvidence(), + customers: customers.map((customer, index) => { + const contract = verifiedContracts.get(customer.id); + return { + id: customer.id, + physicalDatabase: customer.database, + provisionAttestation: liveAttestations[index], + structuralFingerprints: contract.structuralFingerprints, + canonicalStructuralFingerprint: + contract.structuralFingerprints?.combined?.sha256 ?? null, + databaseContractFingerprint: contract.databaseContractFingerprint ?? null, + contractVerification: contract.verification, + }; + }), + canonicalStructuralFingerprint: + manifest.canonicalStructuralFingerprint ?? null, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint ?? null, + blueprintCompatibilityFingerprint: options.blueprintCompatibilityFingerprint, + retainedMemoryCheckpoint: { + enabled: options.benchmarkRetainedHeapEnabled, + gcExposed: typeof global.gc === 'function', + }, + realtime: realtimeStats(), + }); + } catch (error) { + response.status(503).json({ + error: { + code: error instanceof Error + ? error.message.split(':', 1)[0] + : 'PDCF_STATUS_ATTESTATION_FAILED', + }, + }); + } + }); + + for (const { customer, child } of children) { + app.use(`/customer/${customer.id}`, child.app); + } + + let upgradeListener = null; + const listen = () => new Promise((resolve, reject) => { + httpServer = app.listen(options.port, options.host, () => resolve(httpServer)); + httpServer.once('error', reject); + if (options.enableRealtime) { + upgradeListener = (request, socket, head) => { + const rawUrl = request.url ?? ''; + const route = matchPhysicalUpgradeRoute(rawUrl); + const child = route + ? children.find(({ customer }) => customer.id === route.customerId) + : null; + if (!child) { + socket.destroy(); + return; + } + void child.child.handleUpgrade(request, socket, head, { + pathPrefix: `/customer/${child.customer.id}`, + }).then((handled) => { + if (handled) { + realtimeConnections.trackAccepted( + `${route.customerId}:${route.tenantId}`, + socket + ); + } else if (!socket.destroyed) socket.destroy(); + }).catch(() => socket.destroy()); + }; + httpServer.on('upgrade', upgradeListener); + } + }); + + const close = async () => { + if (closing) return; + closing = true; + if (httpServer && upgradeListener) httpServer.off('upgrade', upgradeListener); + if (httpServer?.listening) { + await new Promise((resolve) => httpServer.close(resolve)); + } + // Dispose every customer's realtime manager while its pool is still live. + // Each child owns the process-global pool registry, so allowing the first + // child close to tear it down would strand later managers on ended pools. + await Promise.all([...graphileCache.keys()].map((key) => + deleteGraphileCacheEntry(key) + )); + for (const { child } of children) await child.close(); + await observerPool.end(); + }; + + return { + app, + children, + close, + customers, + listen, + options, + realtimeStats, + assertRealtimeResident, + }; +}; + +const main = async () => { + const options = parseServerOptions(process.argv.slice(2)); + const server = await createPhysicalDatabaseServer(options); + await server.listen(); + process.stdout.write(`${JSON.stringify({ + status: 'ready', + fixture: 'physical-database-density-v1', + host: options.host, + port: options.port, + arm: options.arm, + customers: server.customers.length, + })}\n`); + let stopping = false; + const shutdown = async (code) => { + if (stopping) return; + stopping = true; + await server.close(); + process.exitCode = code; + }; + process.once('SIGTERM', () => void shutdown(0)); + process.once('SIGINT', () => void shutdown(130)); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + authorizeRetainedMemoryCheckpoint, + assertCustomerContract, + assertProvisionCloneManifest, + aggregateRuntimePoolStats, + collectRetainedMemoryCheckpoint, + completeServerOptionsFor, + classifyDatabaseScope, + createRealtimeConnectionRegistry, + createPhysicalDatabaseServer, + makeGraphileActivityVector, + makeRetainedMemoryGuard, + matchPhysicalUpgradeCustomer, + matchPhysicalUpgradeRoute, + parseServerOptions, + parseRuntimePoolMaxUses, + runtimeEnvironmentFor, + tokenEqual, +}; diff --git a/research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.cjs b/research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.cjs new file mode 100644 index 0000000000..492a743854 --- /dev/null +++ b/research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.cjs @@ -0,0 +1,1130 @@ +'use strict'; + +const { spawnSync } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + REPO_ROOT, + TENANTS, + assertCredentialFree, + parseArgs, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); +const completeServer = require('../complete-tenant-fixture/server.cjs'); +const { + FIXTURE_ID, + validateProvisionManifest, + validateSecrets, +} = require('./lib.cjs'); + +const PROBE_KIND = 'unsafe-runtime-fixture-startup-admission-v2'; +const PROBE_VERSION = 2; +const ADMISSION_SCOPE = 'complete-tenant-fixture:createFixtureServer-pre-build-role-audit-v1'; +const CLONE_AUDIT_KIND = 'unsafe-runtime-live-clone-audit-v1'; +const ROLE_AUDIT_KIND = 'unsafe-runtime-role-profile-audit-v1'; +const CLEANUP_AUDIT_KIND = 'unsafe-runtime-role-cleanup-audit-v1'; +const SAFE_CONTROL_CAPABILITY = 'safe-control'; +const PROBE_CAPABILITIES = Object.freeze([ + 'superuser', + 'bypassrls', + 'createrole', + 'schema-owner', + 'schema-create', +]); +const PROBE_ROLE_PATTERNS = Object.freeze({ + superuser: /^ctf_unsafe_super_[a-f0-9]{12}$/, + bypassrls: /^ctf_unsafe_bypass_[a-f0-9]{12}$/, + createrole: /^ctf_unsafe_create_role_[a-f0-9]{12}$/, + 'schema-owner': /^ctf_unsafe_schema_owner_[a-f0-9]{12}$/, + 'schema-create': /^ctf_unsafe_schema_create_[a-f0-9]{12}$/, +}); +const SAFE_LABEL_PATTERN = /^[a-z0-9][a-z0-9-]{0,127}$/; +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/; + +const exactKeys = (value, expected) => + value !== null + && typeof value === 'object' + && !Array.isArray(value) + && JSON.stringify(Object.keys(value).sort()) + === JSON.stringify([...expected].sort()); + +const readJson = (file) => JSON.parse(fs.readFileSync(path.resolve(file), 'utf8')); + +const readPrivateJson = (file) => { + const absoluteFile = path.resolve(file); + let descriptor; + let contents; + try { + const before = fs.lstatSync(absoluteFile); + if ( + before.isSymbolicLink() + || !before.isFile() + || (before.mode & 0o777) !== 0o600 + || (typeof process.getuid === 'function' && before.uid !== process.getuid()) + ) { + throw new Error('PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE'); + } + descriptor = fs.openSync( + absoluteFile, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + const stat = fs.fstatSync(descriptor); + if ( + !stat.isFile() + || stat.dev !== before.dev + || stat.ino !== before.ino + || (stat.mode & 0o777) !== 0o600 + || (typeof process.getuid === 'function' && stat.uid !== process.getuid()) + ) { + throw new Error('PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE'); + } + contents = fs.readFileSync(descriptor, 'utf8'); + } catch (error) { + if (error?.message === 'PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE') throw error; + throw new Error('PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE'); + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + } + try { + return JSON.parse(contents); + } catch { + throw new Error('PDCF_UNSAFE_ROLE_SECRETS_INVALID'); + } +}; + +const loadPrivateProvision = (manifestFile, secretsFile) => { + const absoluteManifestFile = path.resolve(manifestFile); + const absoluteSecretsFile = path.resolve(secretsFile); + let manifestStat; + let secretsStat; + try { + manifestStat = fs.statSync(absoluteManifestFile); + secretsStat = fs.lstatSync(absoluteSecretsFile); + } catch { + throw new Error('PDCF_UNSAFE_ROLE_PROVISION_INPUT_INVALID'); + } + if ( + manifestStat.dev === secretsStat.dev + && manifestStat.ino === secretsStat.ino + ) { + throw new Error('PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE'); + } + const manifest = validateProvisionManifest(readJson(absoluteManifestFile)); + assertCredentialFree(manifest); + const secrets = validateSecrets(readPrivateJson(absoluteSecretsFile), manifest); + return { manifest, secrets }; +}; + +const quoteIdentifier = (value) => `"${String(value).replace(/"/g, '""')}"`; +const quoteLiteral = (value) => `'${String(value).replace(/'/g, "''")}'`; + +const requireSafeLabel = (value, code) => { + if (typeof value !== 'string' || !SAFE_LABEL_PATTERN.test(value)) { + throw new Error(code); + } + return value; +}; + +const requireProbeCapability = (value) => { + if (!PROBE_CAPABILITIES.includes(value)) { + throw new Error('PDCF_UNSAFE_ROLE_CAPABILITY_INVALID'); + } + return value; +}; + +const requireProbeCase = (value) => value === SAFE_CONTROL_CAPABILITY + ? value + : requireProbeCapability(value); + +const probeNames = (nonce) => { + if (!/^[a-f0-9]{12}$/.test(nonce)) { + throw new Error('PDCF_UNSAFE_ROLE_NONCE_INVALID'); + } + return { + roles: { + superuser: `ctf_unsafe_super_${nonce}`, + bypassrls: `ctf_unsafe_bypass_${nonce}`, + createrole: `ctf_unsafe_create_role_${nonce}`, + 'schema-owner': `ctf_unsafe_schema_owner_${nonce}`, + 'schema-create': `ctf_unsafe_schema_create_${nonce}`, + }, + ownerSchema: `ctf_unsafe_owner_${nonce}`, + createSchema: `ctf_unsafe_create_${nonce}`, + }; +}; + +const buildUnsafeRoleSetupSql = ({ database, names, passwords }) => { + const role = Object.fromEntries(Object.entries(names.roles).map(([capability, value]) => [ + capability, + quoteIdentifier(value), + ])); + const password = Object.fromEntries(Object.entries(passwords).map(([capability, value]) => [ + capability, + quoteLiteral(value), + ])); + const databaseIdentifier = quoteIdentifier(database); + return ` +BEGIN; +CREATE ROLE ${role.superuser} + LOGIN NOINHERIT SUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD ${password.superuser}; +CREATE ROLE ${role.bypassrls} + LOGIN NOINHERIT NOSUPERUSER BYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD ${password.bypassrls}; +CREATE ROLE ${role.createrole} + LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS NOCREATEDB CREATEROLE NOREPLICATION + PASSWORD ${password.createrole}; +CREATE ROLE ${role['schema-owner']} + LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD ${password['schema-owner']}; +CREATE ROLE ${role['schema-create']} + LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD ${password['schema-create']}; +GRANT CONNECT ON DATABASE ${databaseIdentifier} TO + ${Object.values(role).join(', ')}; +CREATE SCHEMA ${quoteIdentifier(names.ownerSchema)} + AUTHORIZATION ${role['schema-owner']}; +CREATE SCHEMA ${quoteIdentifier(names.createSchema)}; +REVOKE ALL ON SCHEMA ${quoteIdentifier(names.createSchema)} FROM PUBLIC; +GRANT CREATE ON SCHEMA ${quoteIdentifier(names.createSchema)} + TO ${role['schema-create']}; +COMMIT; +`; +}; + +const buildUnsafeRoleAuditSql = ({ names }) => { + const probes = PROBE_CAPABILITIES.map((capability, index) => + `(${index + 1}, ${quoteLiteral(capability)}, ${quoteLiteral(names.roles[capability])})` + ).join(',\n '); + const schemas = [ + ['owner', names.ownerSchema], + ['create', names.createSchema], + ].map(([label, schema]) => + `(${quoteLiteral(label)}, ${quoteLiteral(schema)})` + ).join(',\n '); + return ` +WITH probes(ordinal, capability, role_name) AS ( + VALUES + ${probes} +), probe_schemas(label, schema_name) AS ( + VALUES + ${schemas} +), profiles AS ( + SELECT p.ordinal, + p.capability, + p.role_name, + r.rolcanlogin AS can_login, + r.rolinherit AS inherits, + r.rolsuper AS superuser, + r.rolbypassrls AS bypass_rls, + r.rolcreaterole AS create_role, + r.rolcreatedb AS create_database, + r.rolreplication AS replication, + CASE WHEN r.oid IS NULL THEN false ELSE pg_catalog.has_database_privilege( + r.oid, + (SELECT oid FROM pg_catalog.pg_database WHERE datname = pg_catalog.current_database()), + 'CONNECT' + ) END AS database_connect, + COALESCE(( + SELECT pg_catalog.jsonb_agg(s.label ORDER BY s.label) + FROM probe_schemas s + JOIN pg_catalog.pg_namespace n ON n.nspname = s.schema_name + WHERE n.nspowner = r.oid + ), '[]'::jsonb) AS owned_probe_schemas, + COALESCE(( + SELECT pg_catalog.jsonb_agg(s.label ORDER BY s.label) + FROM probe_schemas s + JOIN pg_catalog.pg_namespace n ON n.nspname = s.schema_name + WHERE r.oid IS NOT NULL + AND pg_catalog.has_schema_privilege(r.oid, n.oid, 'CREATE') + ), '[]'::jsonb) AS create_on_probe_schemas, + CASE WHEN r.oid IS NULL THEN -1 ELSE ( + SELECT pg_catalog.count(*)::integer + FROM pg_catalog.pg_auth_members m + WHERE m.member = r.oid + ) END AS inherited_memberships + FROM probes p + LEFT JOIN pg_catalog.pg_roles r ON r.rolname = p.role_name +) +SELECT pg_catalog.jsonb_build_object( + 'version', 1, + 'kind', ${quoteLiteral(ROLE_AUDIT_KIND)}, + 'database', pg_catalog.current_database(), + 'profiles', pg_catalog.jsonb_agg(pg_catalog.jsonb_build_object( + 'capability', capability, + 'roleName', role_name, + 'canLogin', can_login, + 'inherits', inherits, + 'superuser', superuser, + 'bypassRls', bypass_rls, + 'createRole', create_role, + 'createDatabase', create_database, + 'replication', replication, + 'databaseConnect', database_connect, + 'ownedProbeSchemas', owned_probe_schemas, + 'createOnProbeSchemas', create_on_probe_schemas, + 'inheritedMemberships', inherited_memberships + ) ORDER BY ordinal) +)::text +FROM profiles; +`; +}; + +const buildLiveCloneAuditSql = () => ` +SELECT pg_catalog.jsonb_build_object( + 'version', 1, + 'kind', ${quoteLiteral(CLONE_AUDIT_KIND)}, + 'cloneId', clone_id, + 'purpose', run_purpose, + 'customerId', customer_id, + 'database', pg_catalog.current_database(), + 'nonce', attestation_nonce, + 'sha256', attestation_sha256 +)::text +FROM ctf_provision_private.clone_attestation +WHERE singleton = true; +`; + +const buildUnsafeRoleCleanupSql = ({ names }) => ` +BEGIN; +DROP SCHEMA IF EXISTS ${quoteIdentifier(names.ownerSchema)} CASCADE; +DROP SCHEMA IF EXISTS ${quoteIdentifier(names.createSchema)} CASCADE; +${Object.values(names.roles).map((role, index) => ` +DO $cleanup_${index}$ +BEGIN + IF EXISTS ( + SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = ${quoteLiteral(role)} + ) THEN + EXECUTE ${quoteLiteral(`DROP OWNED BY ${quoteIdentifier(role)}`)}; + END IF; +END +$cleanup_${index}$; +`).join('\n')} +${Object.values(names.roles).reverse().map((role) => + `DROP ROLE IF EXISTS ${quoteIdentifier(role)};` + ).join('\n')} +COMMIT; +`; + +const buildUnsafeRoleCleanupAuditSql = ({ names }) => ` +SELECT pg_catalog.jsonb_build_object( + 'version', 1, + 'kind', ${quoteLiteral(CLEANUP_AUDIT_KIND)}, + 'database', pg_catalog.current_database(), + 'remainingRoles', ( + SELECT pg_catalog.count(*)::integer + FROM pg_catalog.pg_roles + WHERE rolname = ANY(ARRAY[${Object.values(names.roles).map(quoteLiteral).join(', ')}]::text[]) + ), + 'remainingSchemas', ( + SELECT pg_catalog.count(*)::integer + FROM pg_catalog.pg_namespace + WHERE nspname = ANY(ARRAY[${[ + names.ownerSchema, + names.createSchema, + ].map(quoteLiteral).join(', ')}]::text[]) + ) +)::text; +`; + +const validateUnsafeRoleCleanupAudit = (audit, { database }) => { + if ( + !exactKeys(audit, [ + 'version', + 'kind', + 'database', + 'remainingRoles', + 'remainingSchemas', + ]) + || audit.version !== 1 + || audit.kind !== CLEANUP_AUDIT_KIND + || audit.database !== database + || audit.remainingRoles !== 0 + || audit.remainingSchemas !== 0 + ) { + throw new Error('PDCF_UNSAFE_ROLE_CLEANUP_AUDIT_FAILED'); + } + return { + ...audit, + passed: true, + }; +}; + +const runPsql = ({ database, sql, environment = process.env }) => { + const result = spawnSync('psql', [ + '--no-psqlrc', + '--no-align', + '--tuples-only', + '--quiet', + '--set=ON_ERROR_STOP=1', + '--dbname', database, + ], { + cwd: __dirname, + env: environment, + encoding: 'utf8', + input: sql, + maxBuffer: 16 * 1024 * 1024, + timeout: 120_000, + }); + if (result.status !== 0) { + throw new Error('PDCF_UNSAFE_ROLE_SQL_FAILED'); + } + return result.stdout; +}; + +const runPsqlJson = (input) => { + const output = String(runPsql(input) ?? '').trim(); + if (!output || output.includes('\n')) { + throw new Error('PDCF_UNSAFE_ROLE_AUDIT_RESULT_INVALID'); + } + try { + return JSON.parse(output); + } catch { + throw new Error('PDCF_UNSAFE_ROLE_AUDIT_RESULT_INVALID'); + } +}; + +const validateLiveCloneAudit = (audit, { manifest, customer }) => { + if ( + !exactKeys(audit, [ + 'version', + 'kind', + 'cloneId', + 'purpose', + 'customerId', + 'database', + 'nonce', + 'sha256', + ]) + || audit.version !== 1 + || audit.kind !== CLONE_AUDIT_KIND + || audit.cloneId !== manifest.provisionClone.id + || audit.purpose !== manifest.provisionClone.purpose + || audit.customerId !== customer.id + || audit.database !== customer.database + || !/^[a-f0-9]{64}$/.test(audit.nonce ?? '') + || audit.sha256 !== customer.provisionAttestation.sha256 + || completeServer.provisionAttestationSha256({ + cloneId: audit.cloneId, + purpose: audit.purpose, + customerId: audit.customerId, + database: audit.database, + nonce: audit.nonce, + }) !== audit.sha256 + ) { + throw new Error('PDCF_UNSAFE_ROLE_LIVE_CLONE_AUDIT_INVALID'); + } + return { + version: 1, + cloneId: audit.cloneId, + purpose: audit.purpose, + customerId: audit.customerId, + database: audit.database, + sha256: audit.sha256, + verified: true, + }; +}; + +const expectedAuditedProfiles = () => PROBE_CAPABILITIES.map((capability) => ({ + capability, + canLogin: true, + inherits: false, + superuser: capability === 'superuser', + bypassRls: capability === 'bypassrls', + createRole: capability === 'createrole', + createDatabase: false, + replication: false, + databaseConnect: true, + ownedProbeSchemas: capability === 'schema-owner' ? ['owner'] : [], + createOnProbeSchemas: capability === 'superuser' + ? ['create', 'owner'] + : capability === 'schema-owner' + ? ['owner'] + : capability === 'schema-create' + ? ['create'] + : [], + inheritedMemberships: 0, +})); + +const validateUnsafeRoleAudit = (audit, { database, names }) => { + if ( + !exactKeys(audit, ['version', 'kind', 'database', 'profiles']) + || audit.version !== 1 + || audit.kind !== ROLE_AUDIT_KIND + || audit.database !== database + || !Array.isArray(audit.profiles) + || audit.profiles.length !== PROBE_CAPABILITIES.length + ) { + throw new Error('PDCF_UNSAFE_ROLE_AUDIT_RESULT_INVALID'); + } + const expectedProfiles = expectedAuditedProfiles(); + const normalized = audit.profiles.map((profile, index) => { + const expected = expectedProfiles[index]; + if ( + !exactKeys(profile, [ + 'capability', + 'roleName', + 'canLogin', + 'inherits', + 'superuser', + 'bypassRls', + 'createRole', + 'createDatabase', + 'replication', + 'databaseConnect', + 'ownedProbeSchemas', + 'createOnProbeSchemas', + 'inheritedMemberships', + ]) + || profile.roleName !== names.roles[expected.capability] + ) { + throw new Error('PDCF_UNSAFE_ROLE_AUDIT_RESULT_INVALID'); + } + const { roleName: _roleName, ...credentialFreeProfile } = profile; + if (Object.entries(expected).some(([key, value]) => + JSON.stringify(credentialFreeProfile[key]) !== JSON.stringify(value) + )) { + throw new Error(`PDCF_UNSAFE_ROLE_PROFILE_MISMATCH:${expected.capability}`); + } + return expected; + }); + return { + version: 1, + kind: ROLE_AUDIT_KIND, + database, + profiles: normalized, + passed: true, + }; +}; + +const WORKER_RESULT_KEYS = Object.freeze([ + 'version', + 'kind', + 'admissionScope', + 'customerId', + 'tenantId', + 'capability', + 'cloneId', + 'provisionAttestationSha256', + 'runtimeArtifactFingerprint', + 'physicalDatabaseVerifiedBeforeRoleAudit', + 'controlCredentialEnvironmentAbsent', + 'accepted', + 'rejectedCode', + 'graphileBuildsStarted', + 'residentGraphileEntries', +]); + +const WORKER_ENVIRONMENT_ALLOWLIST = new Set([ + 'PATH', + 'NODE_ENV', + 'TMPDIR', + 'TMP', + 'TEMP', + 'LANG', + 'LC_ALL', + 'LC_CTYPE', + '__CF_USER_TEXT_ENCODING', + 'TZ', + 'PGHOST', + 'PGPORT', + 'PGSSLMODE', + 'PGSSLROOTCERT', + 'PGCHANNELBINDING', +]); +const WORKER_RUNTIME_PASSWORD_KEYS = Object.freeze(TENANTS.map( + (tenant) => tenant.runtimePasswordEnvironment, +)); +const WORKER_EXACT_ENVIRONMENT_KEYS = new Set([ + ...WORKER_ENVIRONMENT_ALLOWLIST, + 'PGDATABASE', + ...WORKER_RUNTIME_PASSWORD_KEYS, +]); + +const parseWorkerResult = (stdout) => { + const lines = String(stdout ?? '').split(/\r?\n/).map((line) => line.trim()).filter(Boolean); + const results = []; + for (const line of lines) { + try { + const value = JSON.parse(line); + if (value?.version === PROBE_VERSION && value.kind === PROBE_KIND) results.push(value); + } catch { + // Runtime logging may precede the final one-line result. Only the exact + // credential-free worker envelope is accepted. + } + } + if (results.length !== 1 || !exactKeys(results[0], WORKER_RESULT_KEYS)) { + throw new Error('PDCF_UNSAFE_ROLE_WORKER_RESULT_INVALID'); + } + return results[0]; +}; + +const makeWorkerEnvironment = (environment) => Object.fromEntries( + Object.entries(environment ?? {}).filter(([key]) => + WORKER_ENVIRONMENT_ALLOWLIST.has(key) + ), +); + +const assertExactWorkerEnvironment = (environment) => { + const keys = Object.keys(environment ?? {}); + if ( + keys.some((key) => !WORKER_EXACT_ENVIRONMENT_KEYS.has(key)) + || typeof environment?.PGDATABASE !== 'string' + || environment.PGDATABASE.length === 0 + || WORKER_RUNTIME_PASSWORD_KEYS.some((key) => + typeof environment[key] !== 'string' + || Buffer.byteLength(environment[key]) < 24 + ) + ) { + throw new Error('PDCF_UNSAFE_ROLE_WORKER_ENVIRONMENT_INVALID'); + } + return true; +}; + +const makeProbeWorkerEnvironment = ({ + environment, + database, + tenantId, + password, + runtimePasswords, +}) => { + const passwordEnvironment = Object.fromEntries(TENANTS.map((tenant) => { + const value = tenant.id === tenantId ? password : runtimePasswords?.[tenant.id]; + if (typeof value !== 'string' || Buffer.byteLength(value) < 24) { + throw new Error(`PDCF_UNSAFE_ROLE_PASSWORD_REQUIRED:${tenant.id}`); + } + return [tenant.runtimePasswordEnvironment, value]; + })); + const workerEnvironment = { + ...makeWorkerEnvironment(environment), + PGDATABASE: database, + ...passwordEnvironment, + }; + assertExactWorkerEnvironment(workerEnvironment); + return workerEnvironment; +}; + +const defaultRunWorker = ({ + manifestFile, + customerId, + database, + tenantId, + capability, + role, + password, + runtimePasswords, + mode, + environment, +}) => { + const workerEnvironment = makeProbeWorkerEnvironment({ + environment, + database, + tenantId, + password, + runtimePasswords, + }); + const result = spawnSync(process.execPath, [ + __filename, + '--worker', + '--manifest', manifestFile, + '--customer-id', customerId, + '--tenant', tenantId, + '--capability', capability, + '--probe-role', role, + '--mode', mode, + ], { + cwd: REPO_ROOT, + env: workerEnvironment, + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + timeout: 180_000, + }); + if (result.status !== 0 || result.signal) { + const workerCode = String(result.stderr ?? '').split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => /^[A-Z][A-Z0-9_]{2,95}$/.test(line)) + .at(-1) ?? 'PDCF_UNSAFE_ROLE_WORKER_UNKNOWN'; + throw new Error( + `PDCF_UNSAFE_ROLE_WORKER_FAILED:${capability}:${tenantId}:${workerCode}`, + ); + } + return parseWorkerResult(result.stdout); +}; + +const validateWorkerRejection = (result, { + customerId, + tenantId, + capability, + cloneId, + provisionAttestationSha256, + runtimeArtifactFingerprint, +}) => { + if ( + !exactKeys(result, WORKER_RESULT_KEYS) + || result.version !== PROBE_VERSION + || result.kind !== PROBE_KIND + || result.admissionScope !== ADMISSION_SCOPE + || result.cloneId !== cloneId + || result.provisionAttestationSha256 !== provisionAttestationSha256 + || result.runtimeArtifactFingerprint !== runtimeArtifactFingerprint + || result.physicalDatabaseVerifiedBeforeRoleAudit !== true + || result.controlCredentialEnvironmentAbsent !== true + || result.customerId !== customerId + || result.tenantId !== tenantId + || result.capability !== capability + || result.accepted !== false + || result.rejectedCode !== 'GRAPHILE_UNSAFE_RUNTIME_ROLE' + || result.graphileBuildsStarted !== 0 + || result.residentGraphileEntries !== 0 + ) { + throw new Error(`PDCF_UNSAFE_ROLE_NOT_REJECTED:${capability}:${tenantId}`); + } + return result; +}; + +const validateWorkerAcceptance = (result, { + customerId, + tenantId, + cloneId, + provisionAttestationSha256, + runtimeArtifactFingerprint, +}) => { + if ( + !exactKeys(result, WORKER_RESULT_KEYS) + || result.version !== PROBE_VERSION + || result.kind !== PROBE_KIND + || result.admissionScope !== ADMISSION_SCOPE + || result.customerId !== customerId + || result.tenantId !== tenantId + || result.capability !== SAFE_CONTROL_CAPABILITY + || result.cloneId !== cloneId + || result.provisionAttestationSha256 !== provisionAttestationSha256 + || result.runtimeArtifactFingerprint !== runtimeArtifactFingerprint + || result.physicalDatabaseVerifiedBeforeRoleAudit !== true + || result.controlCredentialEnvironmentAbsent !== true + || result.accepted !== true + || result.rejectedCode !== null + || result.graphileBuildsStarted !== 0 + || result.residentGraphileEntries !== 0 + ) { + throw new Error('PDCF_SAFE_RUNTIME_ROLE_CONTROL_REJECTED'); + } + return result; +}; + +const runUnsafeRuntimeStartupMatrix = ({ + manifestFile, + secretsFile, + expectedRuntimeArtifactFingerprint, + mode = 'scoped-required', + environment = process.env, + nonce = crypto.randomBytes(6).toString('hex'), + runSql = runPsql, + runCloneAudit = runPsqlJson, + runRoleAudit = runPsqlJson, + runCleanupAudit = runPsqlJson, + runWorker = defaultRunWorker, +} = {}) => { + const absoluteManifestFile = path.resolve(manifestFile); + const absoluteSecretsFile = path.resolve(secretsFile); + const { manifest, secrets } = loadPrivateProvision( + absoluteManifestFile, + absoluteSecretsFile, + ); + if (manifest.provisionClone?.purpose !== 'hostile-preflight') { + throw new Error('PDCF_UNSAFE_ROLE_HOSTILE_CLONE_REQUIRED'); + } + if (mode !== 'stock' && mode !== 'scoped-required') { + throw new Error('PDCF_UNSAFE_ROLE_MODE_INVALID'); + } + if (!SHA256_PATTERN.test(expectedRuntimeArtifactFingerprint ?? '')) { + throw new Error('PDCF_UNSAFE_ROLE_RUNTIME_FINGERPRINT_REQUIRED'); + } + const customer = manifest.customers[0]; + if (!customer) throw new Error('PDCF_UNSAFE_ROLE_CUSTOMER_REQUIRED'); + const liveProvisionAttestation = validateLiveCloneAudit(runCloneAudit({ + database: customer.database, + sql: buildLiveCloneAuditSql(), + environment, + }), { manifest, customer }); + const names = probeNames(nonce); + const passwords = Object.fromEntries(PROBE_CAPABILITIES.map((capability) => [ + capability, + crypto.randomBytes(32).toString('base64url'), + ])); + const safeTenant = TENANTS[0]; + const safeRole = customer.roles[safeTenant.id]; + const runtimePasswords = Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + secrets.runtimePasswords[customer.roles[tenant.id]], + ])); + const safeControlResult = validateWorkerAcceptance(runWorker({ + manifestFile: absoluteManifestFile, + customerId: customer.id, + database: customer.database, + tenantId: safeTenant.id, + capability: SAFE_CONTROL_CAPABILITY, + role: safeRole, + password: secrets.runtimePasswords[safeRole], + runtimePasswords, + mode, + environment, + }), { + customerId: customer.id, + tenantId: safeTenant.id, + cloneId: manifest.provisionClone.id, + provisionAttestationSha256: customer.provisionAttestation.sha256, + runtimeArtifactFingerprint: expectedRuntimeArtifactFingerprint, + }); + let setupAttempted = false; + let roleAudit; + let cleanupAudit; + let matrixError = null; + const attempts = []; + try { + setupAttempted = true; + runSql({ + database: customer.database, + sql: buildUnsafeRoleSetupSql({ + database: customer.database, + names, + passwords, + }), + environment, + }); + roleAudit = validateUnsafeRoleAudit(runRoleAudit({ + database: customer.database, + sql: buildUnsafeRoleAuditSql({ names }), + environment, + }), { + database: customer.database, + names, + }); + for (const capability of PROBE_CAPABILITIES) { + for (const tenant of TENANTS) { + const result = validateWorkerRejection(runWorker({ + manifestFile: absoluteManifestFile, + customerId: customer.id, + database: customer.database, + tenantId: tenant.id, + capability, + role: names.roles[capability], + password: passwords[capability], + runtimePasswords, + mode, + environment, + }), { + customerId: customer.id, + tenantId: tenant.id, + capability, + cloneId: manifest.provisionClone.id, + provisionAttestationSha256: customer.provisionAttestation.sha256, + runtimeArtifactFingerprint: expectedRuntimeArtifactFingerprint, + }); + attempts.push({ + capability, + tenantId: tenant.id, + rejectedCode: result.rejectedCode, + controlCredentialEnvironmentAbsent: + result.controlCredentialEnvironmentAbsent, + graphileBuildsStarted: result.graphileBuildsStarted, + residentGraphileEntries: result.residentGraphileEntries, + }); + } + } + } catch (error) { + matrixError = error; + } + let cleanupError = null; + if (setupAttempted) { + try { + runSql({ + database: customer.database, + sql: buildUnsafeRoleCleanupSql({ names }), + environment, + }); + cleanupAudit = validateUnsafeRoleCleanupAudit(runCleanupAudit({ + database: customer.database, + sql: buildUnsafeRoleCleanupAuditSql({ names }), + environment, + }), { database: customer.database }); + } catch (error) { + cleanupError = error; + } + } + if (cleanupError) { + throw new Error('PDCF_UNSAFE_ROLE_CLEANUP_FAILED', { cause: cleanupError }); + } + if (matrixError) throw matrixError; + const report = { + version: PROBE_VERSION, + kind: PROBE_KIND, + admissionScope: ADMISSION_SCOPE, + provisionClone: { + version: manifest.provisionClone.version, + id: manifest.provisionClone.id, + purpose: manifest.provisionClone.purpose, + attestationSetSha256: manifest.provisionClone.attestationSetSha256, + }, + representativeCustomerId: customer.id, + representativePhysicalDatabase: customer.physicalIdentity, + representativeProvisionAttestationSha256: + customer.provisionAttestation.sha256, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint, + runtimeArtifactFingerprint: expectedRuntimeArtifactFingerprint, + liveProvisionAttestation, + safeStartupControl: { + tenantId: safeControlResult.tenantId, + accepted: safeControlResult.accepted, + physicalDatabaseVerifiedBeforeRoleAudit: + safeControlResult.physicalDatabaseVerifiedBeforeRoleAudit, + controlCredentialEnvironmentAbsent: + safeControlResult.controlCredentialEnvironmentAbsent, + graphileBuildsStarted: safeControlResult.graphileBuildsStarted, + residentGraphileEntries: safeControlResult.residentGraphileEntries, + passed: true, + }, + roleProfileAudit: roleAudit, + cleanupAudit, + capabilities: [...PROBE_CAPABILITIES], + surfaces: TENANTS.map((tenant) => tenant.id), + attempts, + expectedAttempts: PROBE_CAPABILITIES.length * TENANTS.length, + rejectedAttempts: attempts.length, + acceptedAttempts: 0, + graphileBuildsStarted: attempts.reduce( + (sum, attempt) => sum + attempt.graphileBuildsStarted, + 0, + ), + residentGraphileEntries: attempts.reduce( + (sum, attempt) => sum + attempt.residentGraphileEntries, + 0, + ), + passed: attempts.length === PROBE_CAPABILITIES.length * TENANTS.length, + }; + assertCredentialFree(report); + return report; +}; + +const verifyPhysicalDatabaseWithRuntimeCredential = async ({ + database, + role, + password, +}) => { + const { Pool } = require(path.join(REPO_ROOT, 'graphql/server/node_modules/pg')); + const pool = new Pool({ + database, + user: role, + password, + max: 1, + connectionTimeoutMillis: 5_000, + application_name: 'unsafe-runtime-startup-probe', + }); + try { + const result = await pool.query(` + SELECT pg_catalog.current_database()::text AS database, + current_user::text AS role + `); + if ( + result.rowCount !== 1 + || result.rows[0]?.database !== database + || result.rows[0]?.role !== role + ) { + throw new Error('PDCF_UNSAFE_ROLE_PHYSICAL_DATABASE_MISMATCH'); + } + return true; + } finally { + await pool.end(); + } +}; + +const workerProbe = async ({ + manifestFile, + customerId, + tenantId, + capability, + probeRole, + mode, + environment = process.env, +}) => { + assertExactWorkerEnvironment(environment); + const controlCredentialEnvironmentAbsent = true; + requireSafeLabel(customerId, 'PDCF_UNSAFE_ROLE_CUSTOMER_INVALID'); + requireSafeLabel(tenantId, 'PDCF_UNSAFE_ROLE_TENANT_INVALID'); + requireProbeCase(capability); + if (!TENANTS.some((tenant) => tenant.id === tenantId)) { + throw new Error('PDCF_UNSAFE_ROLE_TENANT_INVALID'); + } + if (typeof probeRole !== 'string' || !/^[a-z_][a-z0-9_]{0,62}$/.test(probeRole)) { + throw new Error('PDCF_UNSAFE_ROLE_NAME_INVALID'); + } + const selectedTenant = TENANTS.find((tenant) => tenant.id === tenantId); + const password = environment[selectedTenant.runtimePasswordEnvironment]; + if (typeof password !== 'string' || Buffer.byteLength(password) < 24) { + throw new Error('PDCF_UNSAFE_ROLE_PASSWORD_REQUIRED'); + } + const manifest = validateProvisionManifest(readJson(manifestFile)); + assertCredentialFree(manifest); + if ( + manifest.fixture !== FIXTURE_ID + || manifest.provisionClone?.purpose !== 'hostile-preflight' + ) { + throw new Error('PDCF_UNSAFE_ROLE_HOSTILE_CLONE_REQUIRED'); + } + const customer = manifest.customers.find((candidate) => candidate.id === customerId); + if (!customer) throw new Error('PDCF_UNSAFE_ROLE_CUSTOMER_INVALID'); + if (environment.PGDATABASE !== customer.database) { + throw new Error('PDCF_UNSAFE_ROLE_PHYSICAL_DATABASE_MISMATCH'); + } + if ( + capability === SAFE_CONTROL_CAPABILITY + ? probeRole !== customer.roles[tenantId] + : !PROBE_ROLE_PATTERNS[capability].test(probeRole) + ) { + throw new Error('PDCF_UNSAFE_ROLE_NAME_INVALID'); + } + const runtimeRoles = capability === SAFE_CONTROL_CAPABILITY + ? { ...customer.roles } + : { ...customer.roles, [tenantId]: probeRole }; + const childEnvironment = { + ...makeWorkerEnvironment(environment), + PGDATABASE: customer.database, + ...Object.fromEntries(TENANTS.map((tenant) => [ + tenant.runtimePasswordEnvironment, + environment[tenant.runtimePasswordEnvironment], + ])), + }; + if (environment === process.env) { + Object.assign(process.env, childEnvironment); + } + const physicalDatabaseVerifiedBeforeRoleAudit = + await verifyPhysicalDatabaseWithRuntimeCredential({ + database: customer.database, + role: probeRole, + password, + }); + const options = { + ...completeServer.parseServerOptions([ + '--host', '127.0.0.1', + '--port', '3391', + '--arm', 'unsafe-runtime-startup-probe', + '--mode', mode, + '--introspection-client-release-mode', 'destroy', + '--runtime-pool-max', '2', + '--enable-realtime', 'true', + ...TENANTS.flatMap((tenant) => [ + `--${tenant.runtimeRoleArgument}`, + runtimeRoles[tenant.id], + ]), + ], childEnvironment), + runPurpose: 'hostile-preflight', + cloneId: manifest.provisionClone.id, + }; + let accepted = false; + let rejectedCode = null; + let server = null; + try { + server = await completeServer.createFixtureServer(options, childEnvironment); + accepted = true; + } catch (error) { + rejectedCode = typeof error?.code === 'string' ? error.code : null; + if (rejectedCode !== 'GRAPHILE_UNSAFE_RUNTIME_ROLE') throw error; + } finally { + if (server) { + await server.close(); + } else { + // createFixtureServer has not returned its close handle when startup is + // rejected, but its pre-publication role audit may already have leased + // pools. End them explicitly so each isolated worker exits immediately + // instead of waiting for node-postgres idle timeouts. + await require(path.join( + REPO_ROOT, + 'postgres/pg-cache/dist/index.js', + )).teardownPgPools(); + } + } + const graphileCache = require(path.join( + REPO_ROOT, + 'graphile/graphile-cache/dist/index.js', + )).graphileCache; + const buildStats = require(path.join( + REPO_ROOT, + 'graphql/server/dist/middleware/observability/graphile-build-stats.js', + )).getGraphileBuildStats(); + const result = { + version: PROBE_VERSION, + kind: PROBE_KIND, + admissionScope: ADMISSION_SCOPE, + customerId, + tenantId, + capability, + cloneId: manifest.provisionClone.id, + provisionAttestationSha256: customer.provisionAttestation.sha256, + runtimeArtifactFingerprint: completeServer.runtimeArtifactFingerprint(), + physicalDatabaseVerifiedBeforeRoleAudit, + controlCredentialEnvironmentAbsent, + accepted, + rejectedCode, + graphileBuildsStarted: buildStats.started, + residentGraphileEntries: graphileCache.size, + }; + assertCredentialFree(result); + return result; +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + if (args.worker !== true) throw new Error('PDCF_UNSAFE_ROLE_WORKER_REQUIRED'); + const result = await workerProbe({ + manifestFile: path.resolve(requireString(args, 'manifest')), + customerId: requireString(args, 'customer-id'), + tenantId: requireString(args, 'tenant'), + capability: requireString(args, 'capability'), + probeRole: requireString(args, 'probe-role'), + mode: requireString(args, 'mode', 'scoped-required'), + }); + process.stdout.write(`${JSON.stringify(result)}\n`); +}; + +if (require.main === module) { + main().catch((error) => { + const code = typeof error?.code === 'string' + ? error.code + : String(error instanceof Error ? error.message : error).split(':', 1)[0]; + process.stderr.write(`${code}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + ADMISSION_SCOPE, + CLEANUP_AUDIT_KIND, + CLONE_AUDIT_KIND, + PROBE_CAPABILITIES, + PROBE_KIND, + ROLE_AUDIT_KIND, + SAFE_CONTROL_CAPABILITY, + assertExactWorkerEnvironment, + buildLiveCloneAuditSql, + buildUnsafeRoleAuditSql, + buildUnsafeRoleCleanupAuditSql, + buildUnsafeRoleCleanupSql, + buildUnsafeRoleSetupSql, + expectedAuditedProfiles, + loadPrivateProvision, + makeProbeWorkerEnvironment, + makeWorkerEnvironment, + parseWorkerResult, + probeNames, + runUnsafeRuntimeStartupMatrix, + validateLiveCloneAudit, + validateUnsafeRoleCleanupAudit, + validateUnsafeRoleAudit, + validateWorkerAcceptance, + validateWorkerRejection, + verifyPhysicalDatabaseWithRuntimeCredential, + workerProbe, +}; diff --git a/research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.test.cjs b/research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.test.cjs new file mode 100644 index 0000000000..4d6c780db4 --- /dev/null +++ b/research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.test.cjs @@ -0,0 +1,435 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { + TENANTS, + assertCredentialFree, +} = require('../complete-tenant-fixture/lib.cjs'); +const { makeCustomers } = require('./lib.cjs'); +const { provisionAttestationSha256 } = require('./provision.cjs'); +const { + ADMISSION_SCOPE, + CLEANUP_AUDIT_KIND, + PROBE_CAPABILITIES, + PROBE_KIND, + SAFE_CONTROL_CAPABILITY, + assertExactWorkerEnvironment, + buildLiveCloneAuditSql, + buildUnsafeRoleAuditSql, + buildUnsafeRoleCleanupAuditSql, + buildUnsafeRoleCleanupSql, + buildUnsafeRoleSetupSql, + expectedAuditedProfiles, + makeProbeWorkerEnvironment, + makeWorkerEnvironment, + parseWorkerResult, + probeNames, + runUnsafeRuntimeStartupMatrix, + workerProbe, +} = require('./unsafe-runtime-startup-probe.cjs'); + +const digest = (character) => `sha256:${character.repeat(64)}`; +const runtimeArtifactFingerprint = digest('e'); + +const writeFixture = (directory) => { + const nonce = '1'.repeat(64); + const customer = { + ...makeCustomers('unsafe_probe', 1)[0], + provisionAttestation: { + version: 1, + cloneId: 'unsafe-probe-clone', + purpose: 'hostile-preflight', + sha256: provisionAttestationSha256({ + cloneId: 'unsafe-probe-clone', + runPurpose: 'hostile-preflight', + customerId: 'physical-customer-0001', + database: 'unsafe_probe_db_0001', + nonce, + }), + }, + structuralFingerprints: { combined: { sha256: digest('b') } }, + databaseContractFingerprint: digest('c'), + }; + const manifest = { + version: 1, + fixture: 'physical-database-density-v1', + prefix: 'unsafe_probe', + provisionClone: { + version: 1, + id: 'unsafe-probe-clone', + purpose: 'hostile-preflight', + attestationSetSha256: digest('d'), + }, + canonicalStructuralFingerprint: customer.structuralFingerprints, + canonicalDatabaseContractFingerprint: customer.databaseContractFingerprint, + customers: [customer], + }; + const secrets = { + version: 1, + fixture: 'physical-database-density-v1', + runtimePasswords: Object.fromEntries(Object.values(customer.roles).map((role) => [ + role, + `safe-runtime-password-${role}`, + ])), + notificationPasswords: { + [customer.notificationRole]: + `safe-notification-password-${customer.notificationRole}`, + }, + }; + const manifestFile = path.join(directory, 'provision.json'); + const secretsFile = path.join(directory, 'runtime-secrets.json'); + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + fs.writeFileSync(secretsFile, JSON.stringify(secrets), { mode: 0o600 }); + return { customer, manifestFile, nonce, secretsFile }; +}; + +const liveCloneAuditFor = (customer, nonce) => ({ + version: 1, + kind: 'unsafe-runtime-live-clone-audit-v1', + cloneId: customer.provisionAttestation.cloneId, + purpose: customer.provisionAttestation.purpose, + customerId: customer.id, + database: customer.database, + nonce, + sha256: customer.provisionAttestation.sha256, +}); + +const roleAuditFor = (customer, names) => ({ + version: 1, + kind: 'unsafe-runtime-role-profile-audit-v1', + database: customer.database, + profiles: expectedAuditedProfiles().map((profile) => ({ + ...profile, + roleName: names.roles[profile.capability], + })), +}); + +const workerResultFor = (customer, input, accepted) => ({ + version: 2, + kind: PROBE_KIND, + admissionScope: ADMISSION_SCOPE, + customerId: customer.id, + tenantId: input.tenantId, + capability: input.capability, + cloneId: customer.provisionAttestation.cloneId, + provisionAttestationSha256: customer.provisionAttestation.sha256, + runtimeArtifactFingerprint, + physicalDatabaseVerifiedBeforeRoleAudit: true, + controlCredentialEnvironmentAbsent: true, + accepted, + rejectedCode: accepted ? null : 'GRAPHILE_UNSAFE_RUNTIME_ROLE', + graphileBuildsStarted: 0, + residentGraphileEntries: 0, +}); + +const cleanupAuditFor = (customer) => ({ + version: 1, + kind: CLEANUP_AUDIT_KIND, + database: customer.database, + remainingRoles: 0, + remainingSchemas: 0, +}); + +describe('unsafe runtime startup admission matrix', () => { + it('builds five materially distinct PostgreSQL privilege profiles', () => { + const names = probeNames('012345abcdef'); + const passwords = Object.fromEntries(PROBE_CAPABILITIES.map((capability) => [ + capability, + `password-${capability}`, + ])); + const setup = buildUnsafeRoleSetupSql({ + database: 'unsafe_probe_db_0001', + names, + passwords, + }); + assert.match(setup, /SUPERUSER NOBYPASSRLS/); + assert.match(setup, /NOSUPERUSER BYPASSRLS/); + assert.match(setup, /NOCREATEDB CREATEROLE/); + assert.match(setup, /CREATE SCHEMA "ctf_unsafe_owner_012345abcdef"\s+AUTHORIZATION/); + assert.match(setup, /GRANT CREATE ON SCHEMA "ctf_unsafe_create_012345abcdef"/); + assert.equal((setup.match(/NOINHERIT/g) ?? []).length, 5); + + const audit = buildUnsafeRoleAuditSql({ names }); + assert.match(audit, /pg_catalog\.pg_roles/); + assert.match(audit, /pg_catalog\.pg_auth_members/); + assert.match(audit, /pg_catalog\.has_schema_privilege/); + assert.match(audit, /pg_catalog\.has_database_privilege/); + assert.match(buildLiveCloneAuditSql(), /ctf_provision_private\.clone_attestation/); + + const cleanup = buildUnsafeRoleCleanupSql({ names }); + assert.match(cleanup, /DROP SCHEMA IF EXISTS/); + assert.equal((cleanup.match(/DROP OWNED BY/g) ?? []).length, 5); + assert.equal((cleanup.match(/DROP ROLE IF EXISTS/g) ?? []).length, 5); + assert.match(buildUnsafeRoleCleanupAuditSql({ names }), /remainingRoles/); + assert.match(buildUnsafeRoleCleanupAuditSql({ names }), /remainingSchemas/); + }); + + it('requires all five capabilities to fail before publication on A, B, and C', (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-unsafe-probe-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const { customer, manifestFile, nonce, secretsFile } = writeFixture(temporary); + const sqlCalls = []; + const cloneAuditCalls = []; + const auditCalls = []; + const workerCalls = []; + const names = probeNames('012345abcdef'); + const report = runUnsafeRuntimeStartupMatrix({ + manifestFile, + secretsFile, + expectedRuntimeArtifactFingerprint: runtimeArtifactFingerprint, + nonce: '012345abcdef', + runSql: (input) => sqlCalls.push(input), + runCloneAudit: (input) => { + cloneAuditCalls.push(input); + return liveCloneAuditFor(customer, nonce); + }, + runRoleAudit: (input) => { + auditCalls.push(input); + return roleAuditFor(customer, names); + }, + runCleanupAudit: () => cleanupAuditFor(customer), + runWorker: (input) => { + workerCalls.push(input); + return workerResultFor( + customer, + input, + input.capability === SAFE_CONTROL_CAPABILITY, + ); + }, + }); + + assert.equal(sqlCalls.length, 2); + assert.equal(cloneAuditCalls.length, 1); + assert.match(cloneAuditCalls[0].sql, /clone_attestation/); + assert.match(sqlCalls[0].sql, /CREATE ROLE/); + assert.match(sqlCalls[1].sql, /DROP ROLE IF EXISTS/); + assert.equal(auditCalls.length, 1); + assert.match(auditCalls[0].sql, /unsafe-runtime-role-profile-audit-v1/); + assert.equal(workerCalls.length, 16); + assert.equal(workerCalls[0].capability, SAFE_CONTROL_CAPABILITY); + assert.deepEqual( + new Set(workerCalls.slice(1).map((call) => call.capability)), + new Set(PROBE_CAPABILITIES), + ); + assert.deepEqual( + new Set(workerCalls.slice(1).map((call) => call.tenantId)), + new Set(TENANTS.map((tenant) => tenant.id)), + ); + assert.equal(report.passed, true); + assert.equal(report.expectedAttempts, 15); + assert.equal(report.rejectedAttempts, 15); + assert.equal(report.acceptedAttempts, 0); + assert.equal(report.graphileBuildsStarted, 0); + assert.equal(report.residentGraphileEntries, 0); + assert.equal(report.safeStartupControl.passed, true); + assert.equal(report.safeStartupControl.physicalDatabaseVerifiedBeforeRoleAudit, true); + assert.equal(report.safeStartupControl.controlCredentialEnvironmentAbsent, true); + assert.equal(report.liveProvisionAttestation.verified, true); + assert.equal(report.roleProfileAudit.passed, true); + assert.equal(report.cleanupAudit.passed, true); + assert.equal(report.runtimeArtifactFingerprint, runtimeArtifactFingerprint); + assert.equal(report.admissionScope, ADMISSION_SCOPE); + assert.equal(report.provisionClone.id, 'unsafe-probe-clone'); + assert.equal( + report.representativeProvisionAttestationSha256, + customer.provisionAttestation.sha256, + ); + assert.doesNotThrow(() => assertCredentialFree(JSON.stringify(report))); + for (const call of workerCalls) { + assert.equal(Object.hasOwn(call, 'secretsFile'), false); + assert.deepEqual(Object.keys(call.runtimePasswords).sort(), ['a', 'b', 'c']); + assert.doesNotMatch(JSON.stringify(report), new RegExp(call.password)); + } + }); + + it('cleans up and fails closed if any startup reaches publication admission', (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-unsafe-accepted-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const { customer, manifestFile, nonce, secretsFile } = writeFixture(temporary); + const sqlCalls = []; + const names = probeNames('fedcba987654'); + assert.throws(() => runUnsafeRuntimeStartupMatrix({ + manifestFile, + secretsFile, + expectedRuntimeArtifactFingerprint: runtimeArtifactFingerprint, + nonce: 'fedcba987654', + runSql: (input) => sqlCalls.push(input), + runCloneAudit: () => liveCloneAuditFor(customer, nonce), + runRoleAudit: () => roleAuditFor(customer, names), + runCleanupAudit: () => cleanupAuditFor(customer), + runWorker: (input) => workerResultFor(customer, input, true), + }), /PDCF_UNSAFE_ROLE_NOT_REJECTED/); + assert.equal(sqlCalls.length, 2); + assert.match(sqlCalls[1].sql, /DROP ROLE IF EXISTS/); + }); + + it('attempts idempotent cleanup and audits zero leftovers after ambiguous setup failure', (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-unsafe-ambiguous-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const { customer, manifestFile, nonce, secretsFile } = writeFixture(temporary); + const sqlCalls = []; + assert.throws(() => runUnsafeRuntimeStartupMatrix({ + manifestFile, + secretsFile, + expectedRuntimeArtifactFingerprint: runtimeArtifactFingerprint, + nonce: 'abcdef012345', + runSql: (input) => { + sqlCalls.push(input); + if (sqlCalls.length === 1) throw new Error('PDCF_TEST_SETUP_RESULT_AMBIGUOUS'); + }, + runCloneAudit: () => liveCloneAuditFor(customer, nonce), + runRoleAudit: () => assert.fail('role audit must not run'), + runCleanupAudit: () => cleanupAuditFor(customer), + runWorker: (input) => workerResultFor(customer, input, true), + }), /PDCF_TEST_SETUP_RESULT_AMBIGUOUS/); + assert.equal(sqlCalls.length, 2); + assert.match(sqlCalls[0].sql, /CREATE ROLE/); + assert.match(sqlCalls[1].sql, /DROP ROLE IF EXISTS/); + }); + + it('parses only the final exact worker envelope', () => { + const customer = { + id: 'physical-customer-0001', + provisionAttestation: { + cloneId: 'unsafe-probe-clone', + sha256: digest('a'), + }, + }; + const expected = workerResultFor(customer, { + tenantId: 'a', + capability: 'superuser', + }, false); + assert.deepEqual(parseWorkerResult( + `runtime log\n${JSON.stringify(expected)}\n`, + ), expected); + assert.throws( + () => parseWorkerResult('{"kind":"wrong"}\n'), + /PDCF_UNSAFE_ROLE_WORKER_RESULT_INVALID/, + ); + assert.throws( + () => parseWorkerResult(`${JSON.stringify(expected)}\n${JSON.stringify(expected)}\n`), + /PDCF_UNSAFE_ROLE_WORKER_RESULT_INVALID/, + ); + }); + + it('requires a private regular secrets file before any startup probe', (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-unsafe-private-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const { manifestFile, secretsFile } = writeFixture(temporary); + fs.chmodSync(secretsFile, 0o640); + assert.throws(() => runUnsafeRuntimeStartupMatrix({ + manifestFile, + secretsFile, + runWorker: () => assert.fail('worker must not run'), + }), /PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE/); + }); + + it('rejects a manifest containing credential-shaped fields before any worker starts', (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-unsafe-manifest-secret-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const { manifestFile, secretsFile } = writeFixture(temporary); + const manifest = JSON.parse(fs.readFileSync(manifestFile, 'utf8')); + manifest.password = 'credential-that-must-not-reach-the-worker'; + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + assert.throws(() => runUnsafeRuntimeStartupMatrix({ + manifestFile, + secretsFile, + runWorker: () => assert.fail('worker must not run'), + }), /CTF_ARTIFACT_CONTAINS_CREDENTIAL_MARKER/); + }); + + it('passes only PostgreSQL transport, locale, and inert process settings to workers', () => { + assert.deepEqual(makeWorkerEnvironment({ + PATH: '/bin', + NODE_OPTIONS: '--require=/tmp/worker-injection.cjs', + NODE_PATH: '/tmp/untrusted-modules', + PGHOST: '127.0.0.1', + PGPORT: '5432', + PGSSLMODE: 'verify-full', + PGPASSWORD: 'control-secret', + PGUSER: 'control-user', + PGSERVICE: 'control-service', + PGSERVICEFILE: '/private/control-service-file', + PGPASSFILE: '/private/control-passfile', + PGSSLKEY: '/private/client-key', + PGDATABASE: 'wrong-database', + DATABASE_URL: 'postgresql://control:secret@database/control', + GRAPHILE_CACHE_MAX: '5', + GRAPHQL_OBSERVABILITY_TOKEN: 'must-not-cross-boundary', + GRAPHQL_RUNTIME_PGUSER: 'must-not-cross-boundary', + GRAPHQL_RUNTIME_PGPASSWORD: 'must-not-cross-boundary', + CTF_CONTROL_TOKEN: 'must-not-cross-boundary', + GITHUB_TOKEN: 'must-not-cross-boundary', + }), { + PATH: '/bin', + PGHOST: '127.0.0.1', + PGPORT: '5432', + PGSSLMODE: 'verify-full', + }); + }); + + it('accepts only the exact worker environment key contract', () => { + const valid = { + PGDATABASE: 'unsafe_probe_db_0001', + CTF_RUNTIME_A_PGPASSWORD: 'a'.repeat(24), + CTF_RUNTIME_B_PGPASSWORD: 'b'.repeat(24), + CTF_RUNTIME_C_PGPASSWORD: 'c'.repeat(24), + }; + assert.equal(assertExactWorkerEnvironment(valid), true); + assert.throws(() => assertExactWorkerEnvironment({ + ...valid, + NODE_OPTIONS: '--require=/tmp/worker-injection.cjs', + }), /PDCF_UNSAFE_ROLE_WORKER_ENVIRONMENT_INVALID/); + assert.throws(() => assertExactWorkerEnvironment({ + ...valid, + PGPASSWORD: 'control-secret', + }), /PDCF_UNSAFE_ROLE_WORKER_ENVIRONMENT_INVALID/); + assert.throws(() => assertExactWorkerEnvironment({ + ...valid, + GRAPHILE_CACHE_MAX: '5', + }), /PDCF_UNSAFE_ROLE_WORKER_ENVIRONMENT_INVALID/); + }); + + it('constructs exactly three customer-surface credentials and replaces the probe surface', () => { + const environment = makeProbeWorkerEnvironment({ + environment: { + PGHOST: '127.0.0.1', + NODE_OPTIONS: '--require=/tmp/worker-injection.cjs', + GRAPHILE_CACHE_MAX: '5', + }, + database: 'unsafe_probe_db_0001', + tenantId: 'b', + password: 'probe-b-password'.repeat(2), + runtimePasswords: { + a: 'safe-a-password'.repeat(2), + b: 'safe-b-password'.repeat(2), + c: 'safe-c-password'.repeat(2), + }, + }); + assert.deepEqual(Object.keys(environment).sort(), [ + 'CTF_RUNTIME_A_PGPASSWORD', + 'CTF_RUNTIME_B_PGPASSWORD', + 'CTF_RUNTIME_C_PGPASSWORD', + 'PGDATABASE', + 'PGHOST', + ]); + assert.equal(environment.CTF_RUNTIME_A_PGPASSWORD, 'safe-a-password'.repeat(2)); + assert.equal(environment.CTF_RUNTIME_B_PGPASSWORD, 'probe-b-password'.repeat(2)); + assert.equal(environment.CTF_RUNTIME_C_PGPASSWORD, 'safe-c-password'.repeat(2)); + assert.equal(Object.hasOwn(environment, 'NODE_OPTIONS'), false); + assert.equal(Object.hasOwn(environment, 'GRAPHILE_CACHE_MAX'), false); + }); + + it('fails before file or database access if an unexpected credential reaches a worker', async () => { + await assert.rejects(() => workerProbe({ + environment: { + PGPASSWORD: 'control-secret-that-must-not-reach-the-worker', + }, + }), /PDCF_UNSAFE_ROLE_WORKER_ENVIRONMENT_INVALID/); + }); +}); diff --git a/research/graphile-density/production-shaped-canary.sql b/research/graphile-density/production-shaped-canary.sql new file mode 100644 index 0000000000..477c03a263 --- /dev/null +++ b/research/graphile-density/production-shaped-canary.sql @@ -0,0 +1,22 @@ +\set ON_ERROR_STOP on + +-- Performance-only canary for the local ~62k-catalog fixture. Keeping the +-- field in a real application schema avoids exposing `public`, where PostGIS +-- installs extension-owned catalog views that are not part of this API. +CREATE OR REPLACE FUNCTION "simple-pets-public".tenant_token() +RETURNS text +LANGUAGE sql +STABLE +PARALLEL SAFE +SET search_path = pg_catalog +AS $function$ + SELECT 'production-shaped-token'::text +$function$; + +REVOKE ALL +ON FUNCTION "simple-pets-public".tenant_token() +FROM PUBLIC; + +GRANT EXECUTE +ON FUNCTION "simple-pets-public".tenant_token() +TO gdp_runtime_20260801_a; diff --git a/research/graphile-density/validate-uniform-density-fixture.sql b/research/graphile-density/validate-uniform-density-fixture.sql new file mode 100644 index 0000000000..254a3eba2d --- /dev/null +++ b/research/graphile-density/validate-uniform-density-fixture.sql @@ -0,0 +1,342 @@ +\set ON_ERROR_STOP on +\pset pager off + +-- Standalone, repeatable validation for the local uniform density fixture. +-- It does not mutate persistent objects. The temporary validation procedure +-- runs with invoker rights and is dropped before the session ends. + +\connect graphile_density_uniform_20260801_a + +SET statement_timeout = 0; +SET lock_timeout = '30s'; + +\echo 'PERFORMANCE_ONLY_ROUTING_CANARY: gd_runtime_20260801_a can read every tenant schema' +\echo 'This fixture measures Graphile memory density; it does not prove database-enforced tenant isolation or complete customer qualification.' + +DO $catalog_validation$ +DECLARE + class_count integer; + tenant_schema_count integer; +BEGIN + IF current_database() <> 'graphile_density_uniform_20260801_a' THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_WRONG_DATABASE: %', current_database(); + END IF; + + SELECT count(*) INTO class_count FROM pg_catalog.pg_class; + IF class_count <> 61239 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_FINAL_CLASS_COUNT_MISMATCH: expected 61239, got %', + class_count; + END IF; + + SELECT count(*) INTO tenant_schema_count + FROM pg_catalog.pg_namespace + WHERE nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$'; + IF tenant_schema_count <> 4000 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_SCHEMA_COUNT_MISMATCH: expected 4000, got %', + tenant_schema_count; + END IF; + + IF EXISTS ( + WITH expected AS ( + SELECT 'gd_t' || + CASE WHEN tenant_number < 1000 + THEN pg_catalog.lpad(tenant_number::text, 3, '0') + ELSE tenant_number::text + END || '_api' AS nspname + FROM pg_catalog.generate_series(1, 4000) AS tenant(tenant_number) + ), actual AS ( + SELECT nspname + FROM pg_catalog.pg_namespace + WHERE nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + ) + (SELECT nspname FROM expected EXCEPT SELECT nspname FROM actual) + UNION ALL + (SELECT nspname FROM actual EXCEPT SELECT nspname FROM expected) + ) THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_SCHEMA_SET_MISMATCH'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND ( + (SELECT pg_catalog.array_agg( + class.relname || ':' || class.relkind::text ORDER BY class.relname) + FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid) + IS DISTINCT FROM ARRAY[ + 'tenant_canary:r', + 'tenant_canary_id_seq:S', + 'tenant_canary_pkey:i', + 'tenant_canary_tenant_token_key:i', + 'widget:r', + 'widget_id_seq:S', + 'widget_pkey:i' + ]::text[] + OR + (SELECT pg_catalog.array_agg( + constraint_row.conname || ':' || constraint_row.contype::text + ORDER BY constraint_row.conname) + FROM pg_catalog.pg_constraint AS constraint_row + WHERE constraint_row.connamespace = namespace.oid) + IS DISTINCT FROM ARRAY[ + 'tenant_canary_id_not_null:n', + 'tenant_canary_pkey:p', + 'tenant_canary_tenant_token_key:u', + 'tenant_canary_tenant_token_not_null:n', + 'widget_canary_id_fkey:f', + 'widget_canary_id_not_null:n', + 'widget_id_not_null:n', + 'widget_label_not_null:n', + 'widget_pkey:p' + ]::text[] + OR + (SELECT pg_catalog.array_agg( + pg_catalog.concat_ws(':', class.relname, attribute.attnum, + attribute.attname, + pg_catalog.format_type(attribute.atttypid, attribute.atttypmod), + attribute.attnotnull, attribute.attidentity) + ORDER BY class.relname, attribute.attnum) + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_attribute AS attribute + ON attribute.attrelid = class.oid + WHERE class.relnamespace = namespace.oid + AND class.relkind = 'r' + AND attribute.attnum > 0 + AND NOT attribute.attisdropped) + IS DISTINCT FROM ARRAY[ + 'tenant_canary:1:id:bigint:t:a', + 'tenant_canary:2:tenant_token:text:t:', + 'widget:1:id:bigint:t:a', + 'widget:2:canary_id:bigint:t:', + 'widget:3:label:text:t:' + ]::text[] + OR + (SELECT count(*) + FROM pg_catalog.pg_proc AS procedure + WHERE procedure.pronamespace = namespace.oid + AND procedure.proname = 'tenant_token' + AND pg_catalog.pg_get_function_identity_arguments(procedure.oid) = '' + AND pg_catalog.pg_get_function_result(procedure.oid) = 'text' + AND procedure.provolatile = 's' + AND NOT procedure.prosecdef) <> 1 + OR + (SELECT count(*) + FROM pg_catalog.pg_class AS table_class + WHERE table_class.relnamespace = namespace.oid + AND table_class.relkind = 'r' + AND table_class.reltoastrelid <> 0) <> 2 + OR + (SELECT sum(1 + ( + SELECT count(*) + FROM pg_catalog.pg_index AS toast_index + WHERE toast_index.indrelid = table_class.reltoastrelid + )) + FROM pg_catalog.pg_class AS table_class + WHERE table_class.relnamespace = namespace.oid + AND table_class.relkind = 'r') <> 4 + ) + ) THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_NON_UNIFORM_SHAPE: expected identical 7/1/9 direct shape plus four TOAST classes'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND ( + namespace.nspowner <> 'postgres'::regrole + OR NOT pg_catalog.has_schema_privilege( + 'gd_runtime_20260801_a', namespace.oid, 'USAGE' + ) + OR pg_catalog.has_schema_privilege( + 'gd_runtime_20260801_a', namespace.oid, 'CREATE' + ) + ) + ) OR EXISTS ( + SELECT 1 + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = class.relnamespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND class.relowner <> 'postgres'::regrole + ) OR EXISTS ( + SELECT 1 + FROM pg_catalog.pg_proc AS procedure + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = procedure.pronamespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND procedure.proowner <> 'postgres'::regrole + ) THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_OWNER_OR_SCHEMA_PRIVILEGE_MISMATCH'; + END IF; + + IF pg_catalog.has_database_privilege( + 'gd_runtime_20260801_a', current_database(), 'CREATE' + ) OR NOT pg_catalog.has_database_privilege( + 'gd_runtime_20260801_a', current_database(), 'CONNECT' + ) THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_RUNTIME_DATABASE_PRIVILEGE_MISMATCH'; + END IF; +END +$catalog_validation$; + +CREATE PROCEDURE pg_temp.validate_runtime_batch( + batch_start integer, + batch_end integer +) +LANGUAGE plpgsql +AS $runtime_validation$ +DECLARE + tenant_number integer; + tenant_suffix text; + tenant_schema text; + expected_token text; + expected_label text; + function_token text; + table_token text; + widget_label text; + role_row record; +BEGIN + SELECT * INTO role_row + FROM pg_catalog.pg_roles + WHERE rolname = current_user; + + IF session_user <> 'gd_runtime_20260801_a' + OR current_user <> 'gd_runtime_20260801_a' + OR role_row.rolsuper + OR role_row.rolcreaterole + OR role_row.rolcreatedb + OR role_row.rolbypassrls THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_RUNTIME_ROLE_UNSAFE: %', current_user; + END IF; + + FOR tenant_number IN batch_start..batch_end LOOP + tenant_suffix := CASE + WHEN tenant_number < 1000 + THEN pg_catalog.lpad(tenant_number::text, 3, '0') + ELSE tenant_number::text + END; + tenant_schema := 'gd_t' || tenant_suffix || '_api'; + expected_token := 'tenant-' || tenant_suffix || '-token'; + expected_label := 'tenant-' || tenant_suffix || '-widget'; + + IF NOT pg_catalog.has_schema_privilege( + current_user, tenant_schema, 'USAGE' + ) OR pg_catalog.has_schema_privilege( + current_user, tenant_schema, 'CREATE' + ) OR NOT pg_catalog.has_table_privilege( + current_user, + pg_catalog.format('%I.tenant_canary', tenant_schema), + 'SELECT' + ) OR pg_catalog.has_table_privilege( + current_user, + pg_catalog.format('%I.tenant_canary', tenant_schema), + 'INSERT,UPDATE,DELETE' + ) OR NOT pg_catalog.has_function_privilege( + current_user, + pg_catalog.format('%I.tenant_token()', tenant_schema), + 'EXECUTE' + ) THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_RUNTIME_PRIVILEGE_MISMATCH: %', tenant_schema; + END IF; + + EXECUTE pg_catalog.format( + 'SELECT %I.tenant_token()', tenant_schema + ) INTO function_token; + EXECUTE pg_catalog.format( + 'SELECT tenant_token FROM %I.tenant_canary', tenant_schema + ) INTO table_token; + EXECUTE pg_catalog.format( + 'SELECT label FROM %I.widget', tenant_schema + ) INTO widget_label; + + IF function_token <> expected_token + OR table_token <> expected_token + OR widget_label <> expected_label THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_RUNTIME_CANARY_MISMATCH: schema %, function %, table %, widget %', + tenant_schema, function_token, table_token, widget_label; + END IF; + END LOOP; +END +$runtime_validation$; + +REVOKE ALL PRIVILEGES ON PROCEDURE + pg_temp.validate_runtime_batch(integer, integer) FROM PUBLIC; +GRANT EXECUTE ON PROCEDURE + pg_temp.validate_runtime_batch(integer, integer) + TO gd_runtime_20260801_a; + +SET SESSION AUTHORIZATION gd_runtime_20260801_a; + +SELECT pg_catalog.format( + 'CALL pg_temp.validate_runtime_batch(%s, %s)', + batch_start, + least(batch_start + 99, 4000) +) +FROM pg_catalog.generate_series(1, 4000, 100) AS batch(batch_start) +\gexec + +RESET SESSION AUTHORIZATION; + +DROP PROCEDURE pg_temp.validate_runtime_batch(integer, integer); + +WITH normalized_classes AS ( + SELECT pg_catalog.concat_ws('|', + CASE WHEN namespace.nspname = 'pg_toast' + THEN 'pg_toast.' + ELSE namespace.nspname || '.' || class.relname + END, + class.relkind, + class.relpersistence, + class.relowner::regrole::text, + coalesce(access_method.amname, ''), + class.relnatts, + class.relchecks, + class.relhasindex, + class.reltoastrelid <> 0, + class.relispartition, + coalesce(class.relacl::text, '') + ) AS logical_class + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = class.relnamespace + LEFT JOIN pg_catalog.pg_am AS access_method + ON access_method.oid = class.relam +), tenant_shapes AS ( + SELECT namespace.nspname, + pg_catalog.concat_ws('|', + (SELECT count(*) FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid), + (SELECT count(*) FROM pg_catalog.pg_proc AS procedure + WHERE procedure.pronamespace = namespace.oid), + (SELECT count(*) FROM pg_catalog.pg_constraint AS constraint_row + WHERE constraint_row.connamespace = namespace.oid), + (SELECT pg_catalog.string_agg( + class.relname || ':' || class.relkind::text, + ',' ORDER BY class.relname) + FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid) + ) AS logical_shape + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' +) +SELECT current_database() AS database_name, + (SELECT count(*) FROM pg_catalog.pg_class) AS pg_class_count, + (SELECT pg_catalog.md5(pg_catalog.string_agg( + logical_class, E'\n' ORDER BY logical_class)) + FROM normalized_classes) AS logical_pg_class_fingerprint, + (SELECT count(*) FROM tenant_shapes) AS tenant_schema_count, + (SELECT count(DISTINCT logical_shape) FROM tenant_shapes) + AS distinct_tenant_shapes, + (SELECT pg_catalog.md5(pg_catalog.string_agg( + nspname || '|' || logical_shape, E'\n' ORDER BY nspname)) + FROM tenant_shapes) AS tenant_shape_fingerprint; + +\echo 'Uniform density fixture validation completed successfully'