diff --git a/graphql/env/README.md b/graphql/env/README.md index e5084a59d8..c4c56978ce 100644 --- a/graphql/env/README.md +++ b/graphql/env/README.md @@ -43,6 +43,13 @@ In addition to all environment variables supported by `@pgpmjs/env`, this packag ### GraphQL Schema - `GRAPHILE_SCHEMA` - Comma-separated list of PostgreSQL schemas to expose +- `GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE` - `reuse` preserves the introspection backend; `destroy` retires that exact client after gather and reconnects lazily for runtime traffic; defaults to `reuse` +- `GRAPHILE_REALTIME_SCHEMA` - Exact physical schema containing realtime cursor functions; omission preserves `realtime_public` +- `GRAPHILE_REALTIME_NOTIFICATION_MODE` - `dedicated` keeps one PostGraphile subscriber per instance; `shared-exact` opts into the per-database exact-topic broker and requires an application `notificationPgResolver`; defaults to `dedicated` +- `GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS` - Maximum age of a successful shared-listener role audit; defaults to `60000` +- `GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS` - Realtime cursor recovery poll interval; defaults to `5000` +- `GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS` - Realtime cursor heartbeat interval; defaults to `30000` +- `GRAPHILE_RELEASE_BUILD_STATE_AFTER_VALIDATION` - Opt in to releasing schema-construction-only Graphile state after successful validation; defaults to `false` ### Feature Flags - `FEATURES_SIMPLE_INFLECTION` - Enable simple inflection plugin @@ -54,8 +61,18 @@ In addition to all environment variables supported by `@pgpmjs/env`, this packag - `API_IS_PUBLIC` - Whether API is public - `API_EXPOSED_SCHEMAS` - Comma-separated list of exposed schemas - `API_META_SCHEMAS` - Comma-separated list of meta schemas +- `API_ALLOW_META_SCHEMA_HEADER` - Explicitly enable the privileged `X-Meta-Schema` control-plane surface. Defaults to false and must only be used on a separate private admin ingress. - `API_ANON_ROLE` - Anonymous role name - `API_ROLE_NAME` - Default role name +- `GRAPHQL_INTERNAL_REQUEST_SECRET` - Minimum-32-byte token required before private routing/actor headers or the HTTP cache flush endpoint are trusted. `X-Schemata` remains prohibited; use an authoritative API name. + +### Routing Metadata Cache +- `GRAPHQL_ROUTING_CACHE_MAX_ENTRIES` - Capacity reserved for routing metadata diagnostics. Security-sensitive request routing is resolved authoritatively and never served from this cache. + +### Runtime PostgreSQL credentials + +- `GRAPHQL_RUNTIME_PGUSER` and `GRAPHQL_RUNTIME_PGPASSWORD` populate the legacy static `runtimePg` login. +- Production and `GRAPHILE_INTROSPECTION_MODE=scoped-required` do not accept those two values as a dynamic multi-tenant credential source. Use a programmatic `runtimePgResolver`; for a dedicated one-route server, pair an explicit static database with `runtimePgStaticIdentity` in trusted configuration. ## Defaults @@ -75,8 +92,10 @@ GraphQL defaults are provided by `@constructive-io/graphql-types`: roleName: 'administrator', isPublic: true, metaSchemas: ['routing_public', 'metaschema_public', 'metaschema_modules_public'], + allowMetaSchemaHeader: false, routingSchema: 'routing_public' - } + }, + routingCache: {} } ``` diff --git a/graphql/env/__tests__/__snapshots__/merge.test.ts.snap b/graphql/env/__tests__/__snapshots__/merge.test.ts.snap index 6383de2044..d16e94d4dc 100644 --- a/graphql/env/__tests__/__snapshots__/merge.test.ts.snap +++ b/graphql/env/__tests__/__snapshots__/merge.test.ts.snap @@ -3,6 +3,7 @@ exports[`getEnvOptions merges pgpm defaults, graphql defaults, config, env, and overrides 1`] = ` { "api": { + "allowMetaSchemaHeader": false, "anonRole": "env_anon", "exposedSchemas": [ "public", @@ -70,10 +71,19 @@ exports[`getEnvOptions merges pgpm defaults, graphql defaults, config, env, and }, "graphile": { "extends": [], + "introspectionClientReleaseMode": "reuse", + "introspectionDependencySchemas": [], + "introspectionMode": "stock", "preset": {}, + "realtimeCursorHeartbeatIntervalMs": 30000, + "realtimeCursorPollIntervalMs": 5000, + "realtimeNotificationMode": "dedicated", + "realtimeNotificationRoleRevalidationMs": 60000, + "releaseBuildStateAfterValidation": false, "schema": [ "override_schema", ], + "trustCallerPresetsInProduction": false, }, "migrations": { "codegen": { @@ -87,6 +97,7 @@ exports[`getEnvOptions merges pgpm defaults, graphql defaults, config, env, and "port": 5432, "user": "env-user", }, + "routingCache": {}, "server": { "host": "localhost", "port": 5000, diff --git a/graphql/env/__tests__/merge.test.ts b/graphql/env/__tests__/merge.test.ts index fa7dd645e8..d548504bb8 100644 --- a/graphql/env/__tests__/merge.test.ts +++ b/graphql/env/__tests__/merge.test.ts @@ -138,6 +138,46 @@ describe('getEnvOptions', () => { expect(result.api?.metaSchemas).toEqual(['env_meta', 'override_meta']); }); + it('parses the internal request secret without exposing a default', () => { + const secret = '0123456789abcdef0123456789abcdef'; + + expect(getGraphQLEnvVars({ GRAPHQL_INTERNAL_REQUEST_SECRET: secret }).api) + .toMatchObject({ internalRequestSecret: secret }); + expect(getGraphQLEnvVars({}).api?.internalRequestSecret).toBeUndefined(); + }); + + it('keeps the privileged metadata header disabled unless explicitly configured', () => { + expect(getGraphQLEnvVars({ API_ALLOW_META_SCHEMA_HEADER: 'true' }).api) + .toMatchObject({ allowMetaSchemaHeader: true }); + expect(getGraphQLEnvVars({ API_ALLOW_META_SCHEMA_HEADER: 'false' }).api) + .toMatchObject({ allowMetaSchemaHeader: false }); + expect(getGraphQLEnvVars({}).api?.allowMetaSchemaHeader).toBeUndefined(); + }); + + it('preserves the exact static runtime route contract from trusted config', () => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'graphql-env-runtime-pg-')); + const identity = { + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['tenant_a_public', 'tenant_a_auth'], + roles: ['tenant_a_anon', 'tenant_a_user'] + }; + writeConfig(tempDir, { + runtimePg: { + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + }, + runtimePgStaticIdentity: identity + }); + + const result = getEnvOptions({}, tempDir, {}); + + expect(result.runtimePgStaticIdentity).toEqual(identity); + expect(result.runtimePg?.database).toBe('tenant_a'); + }); + it('parses SMS environment variables into typed options', () => { const result = getGraphQLEnvVars({ SMS_PROVIDER: 'devsms', diff --git a/graphql/env/src/__tests__/runtime-pg.test.ts b/graphql/env/src/__tests__/runtime-pg.test.ts new file mode 100644 index 0000000000..5d00400674 --- /dev/null +++ b/graphql/env/src/__tests__/runtime-pg.test.ts @@ -0,0 +1,172 @@ +import { getGraphQLEnvVars } from '../env'; + +describe('GraphQL runtime PostgreSQL environment', () => { + it('maps the dedicated runtime credentials without changing control-plane pg', () => { + const result = getGraphQLEnvVars({ + GRAPHQL_RUNTIME_PGUSER: 'graphql_runtime', + GRAPHQL_RUNTIME_PGPASSWORD: 'runtime-secret' + }); + + expect(result.runtimePg).toEqual({ + user: 'graphql_runtime', + password: 'runtime-secret' + }); + expect(result.pg).toBeUndefined(); + }); + + it('does not create a runtime override when both variables are absent', () => { + expect(getGraphQLEnvVars({}).runtimePg).toBeUndefined(); + }); +}); + +describe('Graphile introspection environment', () => { + it.each(['stock', 'scoped-required'] as const)( + 'accepts the explicit %s mode', + (introspectionMode) => { + expect( + getGraphQLEnvVars({ GRAPHILE_INTROSPECTION_MODE: introspectionMode }).graphile + ).toEqual({ introspectionMode }); + } + ); + + it('rejects unknown modes instead of falling back to stock', () => { + expect(() => + getGraphQLEnvVars({ GRAPHILE_INTROSPECTION_MODE: 'scoped-if-possible' }) + ).toThrow("GRAPHILE_INTROSPECTION_MODE must be 'stock' or 'scoped-required'"); + }); + + it.each(['reuse', 'destroy'] as const)( + 'accepts the explicit %s introspection-client release mode', + (introspectionClientReleaseMode) => { + expect(getGraphQLEnvVars({ + GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE: introspectionClientReleaseMode + }).graphile).toEqual({ introspectionClientReleaseMode }); + } + ); + + it('rejects an unknown introspection-client release mode', () => { + expect(() => getGraphQLEnvVars({ + GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE: 'best-effort' + })).toThrow( + "GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE must be 'reuse' or 'destroy'" + ); + }); + + it('parses the ordered dependency-schema allowlist without duplicates', () => { + expect(getGraphQLEnvVars({ + GRAPHILE_INTROSPECTION_DEPENDENCY_SCHEMAS: 'extensions, shared_api,extensions' + }).graphile).toEqual({ + introspectionDependencySchemas: ['extensions', 'shared_api'] + }); + }); + + it('rejects an empty dependency-schema entry', () => { + expect(() => getGraphQLEnvVars({ + GRAPHILE_INTROSPECTION_DEPENDENCY_SCHEMAS: 'extensions, ,shared_api' + })).toThrow('must be a comma-separated list of non-empty schema names'); + }); +}); + +describe('Graphile realtime environment', () => { + it.each(['dedicated', 'shared-exact'] as const)( + 'maps the explicit %s notification mode', + (realtimeNotificationMode) => { + expect(getGraphQLEnvVars({ + GRAPHILE_REALTIME_NOTIFICATION_MODE: realtimeNotificationMode + }).graphile).toEqual({ realtimeNotificationMode }); + } + ); + + it('rejects unknown notification modes', () => { + expect(() => getGraphQLEnvVars({ + GRAPHILE_REALTIME_NOTIFICATION_MODE: 'shared-prefix' + })).toThrow("must be 'dedicated' or 'shared-exact'"); + }); + + it('maps role revalidation and cursor timing intervals', () => { + expect(getGraphQLEnvVars({ + GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS: '60000', + GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS: '30000', + GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS: '90000' + }).graphile).toEqual({ + realtimeNotificationRoleRevalidationMs: 60_000, + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 90_000 + }); + }); + + it('maps one exact cursor-function schema', () => { + expect(getGraphQLEnvVars({ + GRAPHILE_REALTIME_SCHEMA: ' tenant_a_realtime ' + }).graphile).toEqual({ + realtimeSchema: 'tenant_a_realtime' + }); + }); + + it('rejects a whitespace-only cursor schema', () => { + expect(() => getGraphQLEnvVars({ + GRAPHILE_REALTIME_SCHEMA: ' ' + })).toThrow('GRAPHILE_REALTIME_SCHEMA must be one non-empty exact schema name'); + }); + + it('preserves the compatibility default by omitting absent configuration', () => { + expect(getGraphQLEnvVars({}).graphile?.realtimeSchema).toBeUndefined(); + }); +}); + +describe('Grafast cache-limit environment', () => { + it('maps all three schema-local cache bounds', () => { + expect(getGraphQLEnvVars({ + GRAPHILE_QUERY_CACHE_MAX_LENGTH: '64', + GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH: '32', + GRAPHILE_OPERATION_PLANS_CACHE_MAX_LENGTH: '8' + }).graphile?.grafastCache).toEqual({ + queryCacheMaxLength: 64, + operationsCacheMaxLength: 32, + operationOperationPlansCacheMaxLength: 8 + }); + }); + + it.each(['0', '-1', '1.5', '12entries'])( + 'rejects an invalid cache bound %s', + (value) => { + expect(() => getGraphQLEnvVars({ + GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH: value + })).toThrow('must be a positive safe integer'); + } + ); +}); + +describe('Graphile build-state retirement environment', () => { + it.each([ + ['true', true], + ['false', false] + ])('maps the explicit %s value', (value, expected) => { + expect(getGraphQLEnvVars({ + GRAPHILE_RELEASE_BUILD_STATE_AFTER_VALIDATION: value + }).graphile?.releaseBuildStateAfterValidation).toBe(expected); + }); + + it('keeps retirement absent unless explicitly configured', () => { + expect( + getGraphQLEnvVars({}).graphile?.releaseBuildStateAfterValidation + ).toBeUndefined(); + }); +}); + +describe('Routing metadata cache environment', () => { + it('maps the explicit process capacity', () => { + expect(getGraphQLEnvVars({ + GRAPHQL_ROUTING_CACHE_MAX_ENTRIES: '4096' + }).routingCache).toEqual({ maxEntries: 4096 }); + }); + + it.each(['0', '-1', '1.5', '12entries'])( + 'rejects an invalid routing cache capacity %s', + (value) => { + expect(() => getGraphQLEnvVars({ + GRAPHQL_ROUTING_CACHE_MAX_ENTRIES: value + })).toThrow('must be a positive safe integer'); + } + ); +}); diff --git a/graphql/env/src/env.ts b/graphql/env/src/env.ts index 014924ef24..3345d7cfcb 100644 --- a/graphql/env/src/env.ts +++ b/graphql/env/src/env.ts @@ -7,6 +7,20 @@ import { parseEnvBoolean, parseEnvNumber } from '12factor-env'; export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial => { const { GRAPHILE_SCHEMA, + GRAPHILE_INTROSPECTION_MODE, + GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE, + GRAPHILE_INTROSPECTION_DEPENDENCY_SCHEMAS, + GRAPHILE_QUERY_CACHE_MAX_LENGTH, + GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH, + GRAPHILE_OPERATION_PLANS_CACHE_MAX_LENGTH, + GRAPHILE_REALTIME_SCHEMA, + GRAPHILE_REALTIME_NOTIFICATION_MODE, + GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS, + GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS, + GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS, + GRAPHILE_RELEASE_BUILD_STATE_AFTER_VALIDATION, + + GRAPHQL_ROUTING_CACHE_MAX_ENTRIES, FEATURES_SIMPLE_INFLECTION, FEATURES_OPPOSITE_BASE_NAMES, @@ -16,9 +30,15 @@ export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial API_IS_PUBLIC, API_EXPOSED_SCHEMAS, API_META_SCHEMAS, + API_ALLOW_META_SCHEMA_HEADER, API_ANON_ROLE, API_ROLE_NAME, + GRAPHQL_INTERNAL_REQUEST_SECRET, + + GRAPHQL_RUNTIME_PGUSER, + GRAPHQL_RUNTIME_PGPASSWORD, + EMBEDDER_PROVIDER, EMBEDDER_MODEL, EMBEDDER_BASE_URL, @@ -47,7 +67,93 @@ export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial ); return { + ...((GRAPHQL_RUNTIME_PGUSER || GRAPHQL_RUNTIME_PGPASSWORD) && { + runtimePg: { + ...(GRAPHQL_RUNTIME_PGUSER && { user: GRAPHQL_RUNTIME_PGUSER }), + ...(GRAPHQL_RUNTIME_PGPASSWORD && { password: GRAPHQL_RUNTIME_PGPASSWORD }) + } + }), + ...(GRAPHQL_ROUTING_CACHE_MAX_ENTRIES && { + routingCache: { + maxEntries: parsePositiveSafeInteger( + GRAPHQL_ROUTING_CACHE_MAX_ENTRIES, + 'GRAPHQL_ROUTING_CACHE_MAX_ENTRIES' + ) + } + }), graphile: { + ...(GRAPHILE_INTROSPECTION_MODE && { + introspectionMode: parseGraphileIntrospectionMode(GRAPHILE_INTROSPECTION_MODE) + }), + ...(GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE && { + introspectionClientReleaseMode: parseGraphileIntrospectionClientReleaseMode( + GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE + ) + }), + ...(GRAPHILE_INTROSPECTION_DEPENDENCY_SCHEMAS && { + introspectionDependencySchemas: parseSchemaList( + GRAPHILE_INTROSPECTION_DEPENDENCY_SCHEMAS, + 'GRAPHILE_INTROSPECTION_DEPENDENCY_SCHEMAS' + ) + }), + ...((GRAPHILE_QUERY_CACHE_MAX_LENGTH + || GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH + || GRAPHILE_OPERATION_PLANS_CACHE_MAX_LENGTH) && { + grafastCache: { + ...(GRAPHILE_QUERY_CACHE_MAX_LENGTH && { + queryCacheMaxLength: parsePositiveSafeInteger( + GRAPHILE_QUERY_CACHE_MAX_LENGTH, + 'GRAPHILE_QUERY_CACHE_MAX_LENGTH' + ) + }), + ...(GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH && { + operationsCacheMaxLength: parsePositiveSafeInteger( + GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH, + 'GRAPHILE_OPERATIONS_CACHE_MAX_LENGTH' + ) + }), + ...(GRAPHILE_OPERATION_PLANS_CACHE_MAX_LENGTH && { + operationOperationPlansCacheMaxLength: parsePositiveSafeInteger( + GRAPHILE_OPERATION_PLANS_CACHE_MAX_LENGTH, + 'GRAPHILE_OPERATION_PLANS_CACHE_MAX_LENGTH' + ) + }) + } + }), + ...(GRAPHILE_REALTIME_SCHEMA && { + realtimeSchema: parseExactSchemaName( + GRAPHILE_REALTIME_SCHEMA, + 'GRAPHILE_REALTIME_SCHEMA' + ) + }), + ...(GRAPHILE_REALTIME_NOTIFICATION_MODE && { + realtimeNotificationMode: parseGraphileRealtimeNotificationMode( + GRAPHILE_REALTIME_NOTIFICATION_MODE + ) + }), + ...(GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS && { + realtimeNotificationRoleRevalidationMs: parsePositiveSafeInteger( + GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS, + 'GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS' + ) + }), + ...(GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS && { + realtimeCursorPollIntervalMs: parsePositiveSafeInteger( + GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS, + 'GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS' + ) + }), + ...(GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS && { + realtimeCursorHeartbeatIntervalMs: parsePositiveSafeInteger( + GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS, + 'GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS' + ) + }), + ...(GRAPHILE_RELEASE_BUILD_STATE_AFTER_VALIDATION && { + releaseBuildStateAfterValidation: parseEnvBoolean( + GRAPHILE_RELEASE_BUILD_STATE_AFTER_VALIDATION + ) + }), ...(GRAPHILE_SCHEMA && { schema: GRAPHILE_SCHEMA.includes(',') ? GRAPHILE_SCHEMA.split(',').map(s => s.trim()) @@ -64,8 +170,14 @@ export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial ...(API_IS_PUBLIC && { isPublic: parseEnvBoolean(API_IS_PUBLIC) }), ...(API_EXPOSED_SCHEMAS && { exposedSchemas: API_EXPOSED_SCHEMAS.split(',').map(s => s.trim()) }), ...(API_META_SCHEMAS && { metaSchemas: API_META_SCHEMAS.split(',').map(s => s.trim()) }), + ...(API_ALLOW_META_SCHEMA_HEADER && { + allowMetaSchemaHeader: parseEnvBoolean(API_ALLOW_META_SCHEMA_HEADER) + }), ...(API_ANON_ROLE && { anonRole: API_ANON_ROLE }), - ...(API_ROLE_NAME && { roleName: API_ROLE_NAME }) + ...(API_ROLE_NAME && { roleName: API_ROLE_NAME }), + ...(GRAPHQL_INTERNAL_REQUEST_SECRET && { + internalRequestSecret: GRAPHQL_INTERNAL_REQUEST_SECRET + }) }, ...((EMBEDDER_PROVIDER || CHAT_PROVIDER) && { llm: { @@ -102,3 +214,56 @@ export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial }) }; }; + +const parseGraphileIntrospectionMode = ( + value: string +): 'stock' | 'scoped-required' => { + if (value === 'stock' || value === 'scoped-required') return value; + throw new Error( + `GRAPHILE_INTROSPECTION_MODE must be 'stock' or 'scoped-required'; received '${value}'` + ); +}; + +const parseGraphileIntrospectionClientReleaseMode = ( + value: string +): 'reuse' | 'destroy' => { + if (value === 'reuse' || value === 'destroy') return value; + throw new Error( + "GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE must be 'reuse' or 'destroy'; " + + `received '${value}'` + ); +}; + +const parseGraphileRealtimeNotificationMode = ( + value: string +): 'dedicated' | 'shared-exact' => { + if (value === 'dedicated' || value === 'shared-exact') return value; + throw new Error( + "GRAPHILE_REALTIME_NOTIFICATION_MODE must be 'dedicated' or 'shared-exact'; " + + `received '${value}'` + ); +}; + +const parseSchemaList = (value: string, variable: string): string[] => { + const schemas = value.split(',').map((schema) => schema.trim()); + if (schemas.some((schema) => schema.length === 0)) { + throw new Error(`${variable} must be a comma-separated list of non-empty schema names`); + } + return [...new Set(schemas)]; +}; + +const parseExactSchemaName = (value: string, variable: string): string => { + const schema = value.trim(); + if (schema.length === 0) { + throw new Error(`${variable} must be one non-empty exact schema name`); + } + return schema; +}; + +const parsePositiveSafeInteger = (value: string, variable: string): number => { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${variable} must be a positive safe integer; received '${value}'`); + } + return parsed; +}; diff --git a/graphql/env/src/merge.ts b/graphql/env/src/merge.ts index 15f1402c53..58ca0146fb 100644 --- a/graphql/env/src/merge.ts +++ b/graphql/env/src/merge.ts @@ -44,6 +44,11 @@ export const getEnvOptions = ( ...(configOptions.graphile && { graphile: configOptions.graphile }), ...(configOptions.features && { features: configOptions.features }), ...(configOptions.api && { api: configOptions.api }), + ...(configOptions.routingCache && { routingCache: configOptions.routingCache }), + ...(configOptions.runtimePg && { runtimePg: configOptions.runtimePg }), + ...(configOptions.runtimePgStaticIdentity && { + runtimePgStaticIdentity: configOptions.runtimePgStaticIdentity + }), ...(configOptions.sms && { sms: configOptions.sms }), }, graphqlEnvOptions, diff --git a/graphql/types/README.md b/graphql/types/README.md index e071cedb40..01a6ec9f16 100644 --- a/graphql/types/README.md +++ b/graphql/types/README.md @@ -43,6 +43,14 @@ const config: ConstructiveOptions = { routingSchema: 'routing_public', exposedSchemas: ['public'], }, + routingCache: { + maxEntries: 4096, + }, + runtimePgResolver: async (route) => ({ + database: route.databaseName, + user: await runtimeUsers.forRoute(route), + password: await runtimePasswords.forRoute(route), + }), features: { simpleInflection: true, postgis: true, @@ -64,6 +72,27 @@ PostGraphile/Graphile configuration including schema, plugins, and build options Configuration for the Constructive API including meta API settings, exposed schemas, and role configuration. +### RoutingCacheOptions + +Configuration for the process-wide routing/service-label metadata cache. This +cache is independent from Graphile build identity and its `maxEntries` value +must be at least the effective resident Graphile capacity. + +### RuntimePgResolver + +Production multi-tenant servers resolve a least-privilege login from the exact +credential-free `RuntimePgResolverInput`: database id/name, API id, ordered +schemas, and `[anonymous, authenticated]` roles. The resolver must return an +explicit user, password, and matching database. A static `runtimePg` is accepted +in production or scoped introspection only with `runtimePgStaticIdentity`, which +binds it to one byte-exact route contract. + +`runtimePgResolver` is trusted infrastructure and should look up the login by +immutable `databaseId`. Its normalized host, port, database, and TLS policy must +match the control-plane tenant connection. Multi-cluster routing requires a +future per-route resolver shared by both lanes; runtime-only endpoint divergence +fails closed. + ### GraphileFeatureOptions Feature flags for GraphQL/Graphile including inflection settings and PostGIS support. diff --git a/graphql/types/src/constructive.ts b/graphql/types/src/constructive.ts index 485a4f4a59..35243cd45c 100644 --- a/graphql/types/src/constructive.ts +++ b/graphql/types/src/constructive.ts @@ -7,7 +7,7 @@ import { PgTestConnectionOptions, ServerOptions} from '@pgpmjs/types'; import deepmerge from 'deepmerge'; -import { PgConfig } from 'pg-env'; +import type { PgConfig, PgPoolConfig } from 'pg-env'; import { apiDefaults, @@ -19,6 +19,53 @@ import { import { LlmOptions } from './llm'; import { SmsOptions } from './sms'; +/** Process-wide routing-label metadata cache configuration. */ +export interface RoutingCacheOptions { + /** Maximum resolved service labels retained by one GraphQL server process. */ + maxEntries?: number; +} + +/** Credential-free routing input for resolving one physical listener login. */ +export interface NotificationPgResolverInput { + databaseId: string; + databaseName: string; + apiId: string; + schemas: readonly string[]; +} + +export type NotificationPgConfig = Partial & { pool?: PgPoolConfig }; + +/** + * Resolve a dedicated notification login for one physical database. The + * result must explicitly contain its user and password; server code never + * falls back to runtime or control-plane credentials. + */ +export type NotificationPgResolver = ( + input: Readonly +) => NotificationPgConfig | Promise; + +/** Credential-free exact route contract for one tenant execution identity. */ +export interface RuntimePgResolverInput { + databaseId: string; + databaseName: string; + apiId: string; + /** Physical schemas in Graphile exposure order. */ + schemas: readonly string[]; + /** Request roles in `[anonymous, authenticated]` order. */ + roles: readonly [anonymous: string, authenticated: string]; +} + +export type RuntimePgConfig = Partial & { pool?: PgPoolConfig }; + +/** + * Resolve one least-privilege tenant execution login from the exact routed + * contract. Results must contain explicit user, password, and database fields; + * control-plane credentials are never inherited. + */ +export type RuntimePgResolver = ( + input: Readonly +) => RuntimePgConfig | Promise; + /** * GraphQL-specific options for Constructive */ @@ -29,6 +76,8 @@ export interface ConstructiveGraphQLOptions { features?: GraphileFeatureOptions; /** API configuration options */ api?: ApiOptions; + /** Routing-label metadata cache configuration */ + routingCache?: RoutingCacheOptions; } /** @@ -40,6 +89,19 @@ export interface ConstructiveOptions extends PgpmOptions, ConstructiveGraphQLOpt db?: Partial; /** PostgreSQL connection configuration */ pg?: Partial; + /** + * Static least-privilege PostgreSQL login used for tenant GraphQL execution. + * Production and scoped introspection require `runtimePgStaticIdentity` and + * accept this login for that one exact route only. Multi-tenant servers must + * use `runtimePgResolver` instead. + */ + runtimePg?: RuntimePgConfig; + /** Exact credential-free route authorized to use the static `runtimePg`. */ + runtimePgStaticIdentity?: RuntimePgResolverInput; + /** Per-route least-privilege tenant execution login resolver. */ + runtimePgResolver?: RuntimePgResolver; + /** Per-physical-database login resolver used only by shared realtime LISTEN. */ + notificationPgResolver?: NotificationPgResolver; /** PostGraphile/Graphile configuration */ graphile?: GraphileOptions; /** HTTP server configuration */ @@ -48,6 +110,8 @@ export interface ConstructiveOptions extends PgpmOptions, ConstructiveGraphQLOpt features?: GraphileFeatureOptions; /** API configuration options */ api?: ApiOptions; + /** Routing-label metadata cache configuration */ + routingCache?: RoutingCacheOptions; /** CDN and file storage configuration */ cdn?: CDNOptions; /** Module deployment configuration */ @@ -66,7 +130,8 @@ export interface ConstructiveOptions extends PgpmOptions, ConstructiveGraphQLOpt export const constructiveGraphqlDefaults: ConstructiveGraphQLOptions = { graphile: graphileDefaults, features: graphileFeatureDefaults, - api: apiDefaults + api: apiDefaults, + routingCache: {} }; /** diff --git a/graphql/types/src/graphile.ts b/graphql/types/src/graphile.ts index 72fff4c739..f0edc3b61b 100644 --- a/graphql/types/src/graphile.ts +++ b/graphql/types/src/graphile.ts @@ -1,15 +1,88 @@ import type { GraphileConfig } from 'graphile-config'; +export type GraphileIntrospectionMode = 'stock' | 'scoped-required'; +export type GraphileIntrospectionClientReleaseMode = 'reuse' | 'destroy'; +export type GraphileRealtimeNotificationMode = 'dedicated' | 'shared-exact'; + +/** Per-schema Grafast parse, operation, and plan cache bounds. */ +export interface GrafastCacheLimits { + /** Maximum parsed and validated GraphQL documents retained by one schema. */ + queryCacheMaxLength?: number; + /** Maximum GraphQL operations with retained plan lookup state per schema. */ + operationsCacheMaxLength?: number; + /** Maximum context/variable-specific plans retained for one operation. */ + operationOperationPlansCacheMaxLength?: number; +} + /** * PostGraphile/Graphile v5 configuration */ export interface GraphileOptions { /** Database schema(s) to expose through GraphQL */ schema?: string | string[]; - /** Additional presets to extend */ + /** + * Additional trusted startup presets, applied after Constructive's feature + * preset. The server rejects nested attempts to replace its pgServices, + * tenant request context, transport/error policy, or fixed runtime plugins. + */ extends?: GraphileConfig.Preset[]; - /** Preset overrides */ + /** + * Trusted startup preset overrides. Safe schema and runtime settings plus + * caller plugins are applied; Constructive-owned tenant boundaries remain + * authoritative and fail closed on explicit override attempts. + */ preset?: Partial; + /** + * Admit `extends` and `preset` as fully trusted in-process code in production. + * + * Graphile plugins are not sandboxed: an admitted plugin can execute raw SQL + * through the configured PostgreSQL service and can access the Node.js + * process. Production therefore rejects every non-empty caller preset unless + * the deployment explicitly opts it into the server trust boundary. + */ + trustCallerPresetsInProduction?: boolean; + /** PostgreSQL catalog introspection strategy; scoped mode fails if any requested schema is absent */ + introspectionMode?: GraphileIntrospectionMode; + /** + * Whether the exact PostgreSQL client used for catalog introspection is + * returned to the runtime pool or destroyed after the gather query. Destroy + * avoids carrying catalog-query backend memory into request traffic and + * costs one lazy reconnect after each schema build. + */ + introspectionClientReleaseMode?: GraphileIntrospectionClientReleaseMode; + /** + * Ordered, non-writable schemas that exposed objects may depend on (for + * example the schema containing PostGIS or pgvector). Scoped mode fails if + * catalog closure reaches any other non-system schema. + */ + introspectionDependencySchemas?: string[]; + /** Explicit per-schema Grafast cache bounds used for tenant-density control. */ + grafastCache?: GrafastCacheLimits; + /** + * Release schema-construction-only Graphile state after successful schema + * validation. This is an opt-in density optimization; materialized schemas + * and runtime execution state remain tenant-dedicated. + */ + releaseBuildStateAfterValidation?: boolean; + /** + * Exact physical schema containing realtime cursor functions. Omit for the + * compatibility default `realtime_public`. + */ + realtimeSchema?: string; + /** + * PostgreSQL notification transport. `dedicated` preserves the current + * per-Graphile PgSubscriber; `shared-exact` is an experimental, default-off, + * role-attested broker whose leases are restricted to compiled physical + * topics. The GraphQL server routes WebSocket upgrades independently through + * the exact tenant build contract and admission boundary. + */ + realtimeNotificationMode?: GraphileRealtimeNotificationMode; + /** Maximum age of a successful shared-listener role attestation. */ + realtimeNotificationRoleRevalidationMs?: number; + /** Cursor recovery poll interval; lower values trade database QPS for latency. */ + realtimeCursorPollIntervalMs?: number; + /** Cursor listener heartbeat interval. */ + realtimeCursorHeartbeatIntervalMs?: number; } /** @@ -38,12 +111,24 @@ export interface ApiOptions { isPublic?: boolean; /** Schemas containing metadata tables */ metaSchemas?: string[]; + /** + * Allow the authenticated X-Meta-Schema private-header surface. This is a + * privileged, potentially cross-tenant control-plane API and is disabled by + * default; it must never share a tenant-facing ingress. + */ + allowMetaSchemaHeader?: boolean; /** * Schema containing the compiled resolve_route() resolver. Requests are * always resolved through the scoped-routing plane via * .resolve_route() (host → tenant/api/db/role). */ routingSchema?: string; + /** + * Process secret that authenticates reserved internal routing, identity, and + * cache-administration headers. It must contain at least 32 bytes. When it + * is absent, those headers are rejected rather than trusted from the network. + */ + internalRequestSecret?: string; } /** @@ -52,7 +137,16 @@ export interface ApiOptions { export const graphileDefaults: GraphileOptions = { schema: [], extends: [], - preset: {} + preset: {}, + trustCallerPresetsInProduction: false, + introspectionMode: 'stock', + introspectionClientReleaseMode: 'reuse', + introspectionDependencySchemas: [], + releaseBuildStateAfterValidation: false, + realtimeNotificationMode: 'dedicated', + realtimeNotificationRoleRevalidationMs: 60_000, + realtimeCursorPollIntervalMs: 5_000, + realtimeCursorHeartbeatIntervalMs: 30_000 }; /** @@ -77,5 +171,6 @@ export const apiDefaults: ApiOptions = { 'metaschema_public', 'metaschema_modules_public' ], + allowMetaSchemaHeader: false, routingSchema: 'routing_public' }; diff --git a/graphql/types/src/index.ts b/graphql/types/src/index.ts index 895604e137..aea05ed4ab 100644 --- a/graphql/types/src/index.ts +++ b/graphql/types/src/index.ts @@ -2,17 +2,30 @@ export { apiDefaults, ApiOptions, + GrafastCacheLimits, graphileDefaults, graphileFeatureDefaults, GraphileFeatureOptions, - GraphileOptions} from './graphile'; + GraphileIntrospectionClientReleaseMode, + GraphileIntrospectionMode, + GraphileOptions, + GraphileRealtimeNotificationMode, +} from './graphile'; // Export Constructive combined types export { constructiveDefaults, constructiveGraphqlDefaults, ConstructiveGraphQLOptions, - ConstructiveOptions} from './constructive'; + ConstructiveOptions, + NotificationPgConfig, + NotificationPgResolver, + NotificationPgResolverInput, + RoutingCacheOptions, + RuntimePgConfig, + RuntimePgResolver, + RuntimePgResolverInput +} from './constructive'; // Export GraphQL adapter types export { diff --git a/packages/express-context/__tests__/context-pool-leases.test.ts b/packages/express-context/__tests__/context-pool-leases.test.ts new file mode 100644 index 0000000000..1d7b0b85f9 --- /dev/null +++ b/packages/express-context/__tests__/context-pool-leases.test.ts @@ -0,0 +1,279 @@ +import { EventEmitter } from 'node:events'; + +import type { NextFunction, Request, Response } from 'express'; +import type { Pool } from 'pg'; +import { acquirePgPool, getPgPool, getPgPoolIdentity } from 'pg-cache'; + +import { buildContext, createContextMiddleware } from '../src/context'; +import type { ApiStructure } from '../src/types'; + +jest.mock('pg-cache', () => ({ + acquirePgPool: jest.fn(), + getPgPool: jest.fn(), + getPgPoolIdentity: jest.fn(() => 'pg:v1:test') +})); + +const mockedAcquire = acquirePgPool as jest.MockedFunction; +const mockedGet = getPgPool as jest.MockedFunction; +const mockedIdentity = getPgPoolIdentity as jest.MockedFunction; + +const api: ApiStructure = { + apiId: 'api-a', + databaseId: 'database-a', + dbname: 'tenant_a', + anonRole: 'anonymous', + roleName: 'authenticated', + schema: ['tenant_a_public'], + domains: [], + isPublic: false +}; + +const makeRequest = (): Request => Object.assign(new EventEmitter(), { + api, + requestId: 'request-a', + get: jest.fn((): undefined => undefined), + aborted: false, + destroyed: false, + socket: { destroyed: false } +}) as unknown as Request; + +const makePool = (): Pool => ({ query: jest.fn() } as unknown as Pool); + +const makeResponse = (): Response => { + const response = new EventEmitter() as EventEmitter & { + destroyed: boolean; + writableEnded: boolean; + }; + response.destroyed = false; + response.writableEnded = false; + return response as unknown as Response; +}; + +const leaseFor = (pool: Pool) => ({ + pool, + identity: `pool-${Math.random()}`, + release: jest.fn() +}); + +describe('context PostgreSQL pool lifetimes', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('pins runtime and loader pools until the response finishes', () => { + const leases = [leaseFor(makePool()), leaseFor(makePool()), leaseFor(makePool())]; + mockedAcquire + .mockReturnValueOnce(leases[0]) + .mockReturnValueOnce(leases[1]) + .mockReturnValueOnce(leases[2]); + const response = makeResponse(); + const next = jest.fn() as unknown as NextFunction; + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + runtimePg: { user: 'runtime', password: 'secret' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware(makeRequest(), response, next); + + expect(next).toHaveBeenCalledWith(); + expect(mockedAcquire).toHaveBeenCalledTimes(3); + for (const lease of leases) expect(lease.release).not.toHaveBeenCalled(); + + response.emit('finish'); + response.emit('close'); + for (const lease of leases) expect(lease.release).toHaveBeenCalledTimes(1); + }); + + it('uses the exact server-owned runtime resolution and exposes only its opaque identity', () => { + const runtime = leaseFor(makePool()); + runtime.identity = 'pg:v1:exact-runtime'; + const otherLeases = [leaseFor(makePool()), leaseFor(makePool())]; + mockedAcquire + .mockReturnValueOnce(runtime) + .mockReturnValueOnce(otherLeases[0]) + .mockReturnValueOnce(otherLeases[1]); + const request = makeRequest(); + const getRuntimePgResolution = jest.fn(() => ({ + pgConfig: { + host: 'db.internal', + port: 5432, + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + }, + poolIdentity: runtime.identity + })); + + const context = buildContext(request, { + pg: { database: 'routing' }, + getRuntimePgResolution, + loaders: { resolve: jest.fn() } as any + }, []); + + expect(getRuntimePgResolution).toHaveBeenCalledWith(request, api); + expect(mockedAcquire.mock.calls[0][0]).toEqual({ + host: 'db.internal', + port: 5432, + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + }); + expect(context?.runtimePoolIdentity).toBe('pg:v1:exact-runtime'); + }); + + it('fails closed when the supplied runtime identity changes before acquisition', () => { + const runtime = leaseFor(makePool()); + runtime.identity = 'pg:v1:different-runtime'; + mockedAcquire.mockReturnValueOnce(runtime); + + expect(() => buildContext(makeRequest(), { + getRuntimePgResolution: () => ({ + pgConfig: { + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + }, + poolIdentity: 'pg:v1:expected-runtime' + }) + }, [])).toThrow('pool identity changed before context acquisition'); + }); + + it('does not acquire leases for a request that already ended', () => { + const request = makeRequest(); + Object.assign(request, { aborted: true }); + const next = jest.fn() as unknown as NextFunction; + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware(request, makeResponse(), next); + + expect(mockedAcquire).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + }); + + it('does not acquire leases after the request transport socket is destroyed', () => { + const request = makeRequest(); + Object.assign(request.socket, { destroyed: true }); + const next = jest.fn() as unknown as NextFunction; + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware(request, makeResponse(), next); + + expect(mockedAcquire).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + }); + + it('continues after a parser consumed and auto-destroyed the request stream', () => { + const request = makeRequest(); + Object.assign(request, { + destroyed: true, + readableEnded: true, + complete: true + }); + const leases = [leaseFor(makePool()), leaseFor(makePool()), leaseFor(makePool())]; + mockedAcquire + .mockReturnValueOnce(leases[0]) + .mockReturnValueOnce(leases[1]) + .mockReturnValueOnce(leases[2]); + const response = makeResponse(); + const next = jest.fn() as unknown as NextFunction; + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + runtimePg: { user: 'runtime', password: 'secret' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware(request, response, next); + + expect(next).toHaveBeenCalledWith(); + expect(mockedAcquire).toHaveBeenCalledTimes(3); + response.emit('finish'); + for (const lease of leases) expect(lease.release).toHaveBeenCalledTimes(1); + }); + + it('releases leases when the response ends during context construction', () => { + const response = makeResponse(); + const leases = [leaseFor(makePool()), leaseFor(makePool()), leaseFor(makePool())]; + mockedAcquire + .mockReturnValueOnce(leases[0]) + .mockImplementationOnce(() => { + Object.assign(response, { destroyed: true }); + return leases[1]; + }) + .mockReturnValueOnce(leases[2]); + const next = jest.fn() as unknown as NextFunction; + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + runtimePg: { user: 'runtime', password: 'secret' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware(makeRequest(), response, next); + + expect(mockedAcquire).toHaveBeenCalledTimes(3); + for (const lease of leases) expect(lease.release).toHaveBeenCalledTimes(1); + expect(next).not.toHaveBeenCalled(); + }); + + it('releases leases when the request aborts', () => { + const request = makeRequest(); + const response = makeResponse(); + const leases = [leaseFor(makePool()), leaseFor(makePool()), leaseFor(makePool())]; + mockedAcquire + .mockReturnValueOnce(leases[0]) + .mockReturnValueOnce(leases[1]) + .mockReturnValueOnce(leases[2]); + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + runtimePg: { user: 'runtime', password: 'secret' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware(request, response, jest.fn()); + request.emit('aborted'); + + for (const lease of leases) expect(lease.release).toHaveBeenCalledTimes(1); + }); + + it('releases earlier leases when a later acquisition fails', () => { + const first = leaseFor(makePool()); + const error = new Error('capacity'); + mockedAcquire.mockReturnValueOnce(first).mockImplementationOnce(() => { + throw error; + }); + const next = jest.fn() as unknown as NextFunction; + const middleware = createContextMiddleware({ + pg: { database: 'routing' }, + loaders: { resolve: jest.fn() } as any + }); + + middleware( + makeRequest(), + new EventEmitter() as unknown as Response, + next + ); + + expect(first.release).toHaveBeenCalledTimes(1); + expect(next).toHaveBeenCalledWith(error); + }); + + it('preserves unleased getPgPool behavior for direct buildContext callers', () => { + mockedGet.mockReturnValue(makePool()); + + const context = buildContext(makeRequest(), { + pg: { database: 'routing' }, + loaders: { resolve: jest.fn() } as any + }); + + expect(context).not.toBeNull(); + expect(mockedGet).toHaveBeenCalledTimes(3); + expect(mockedIdentity).toHaveBeenCalledTimes(3); + expect(mockedAcquire).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/express-context/__tests__/pg-settings.test.ts b/packages/express-context/__tests__/pg-settings.test.ts index 6b87b83b15..fafc90264f 100644 --- a/packages/express-context/__tests__/pg-settings.test.ts +++ b/packages/express-context/__tests__/pg-settings.test.ts @@ -29,14 +29,14 @@ describe('buildPgSettings — jwt.claims.api_id provenance', () => { expect(settings['jwt.claims.user_id']).toBe('u1'); }); - it('omits jwt.claims.api_id when the api has no apiId (non-API surface)', () => { + it('clears jwt.claims.api_id when the api has no apiId (non-API surface)', () => { const settings = buildPgSettings({ api: { ...api, apiId: undefined }, token: null, requestId: 'r1' }); - expect(settings['jwt.claims.api_id']).toBeUndefined(); + expect(settings['jwt.claims.api_id']).toBe(''); }); it('is derived only from the resolved api, never from the token', () => { diff --git a/packages/express-context/package.json b/packages/express-context/package.json index 4b5e4d78d2..32a60a7fde 100644 --- a/packages/express-context/package.json +++ b/packages/express-context/package.json @@ -34,6 +34,7 @@ "@pgpmjs/logger": "workspace:^", "@pgpmjs/server-utils": "workspace:^", "@pgpmjs/types": "workspace:^", + "@pgsql/quotes": "^18.2.0", "lru-cache": "^11.2.7", "pg": "^8.21.0", "pg-cache": "workspace:^", diff --git a/packages/express-context/src/__tests__/compute-loader.test.ts b/packages/express-context/src/__tests__/compute-loader.test.ts new file mode 100644 index 0000000000..5b7118648f --- /dev/null +++ b/packages/express-context/src/__tests__/compute-loader.test.ts @@ -0,0 +1,55 @@ +import type { Pool } from 'pg'; + +import { computeLoader } from '../loaders/compute'; + +describe('compute control-plane loader', () => { + afterEach(() => computeLoader.invalidate()); + + it('loads API bindings through the control-plane tenant pool', async () => { + const query = jest.fn() + .mockResolvedValueOnce({ + rows: [{ + functions_schema_name: 'compute"schema', + definitions_table_name: 'definitions', + bindings_table_name: 'bindings', + invocations_schema_name: 'invocations', + invocations_table_name: 'jobs', + invocations_entity_field: 'database_id' + }] + }) + .mockResolvedValueOnce({ + rows: [{ + id: 'binding-a', + alias: 'summarize', + config: { graphql: true }, + function_definition_id: 'definition-a', + task_identifier: 'summarize-task', + description: 'Summarize content', + payload_args: [{ name: 'body', type: 'string' }] + }] + }); + const ctx = { + routingPool: {} as Pool, + tenantPool: { query } as unknown as Pool, + databaseId: 'database-compute-loader-test', + apiId: 'api-a', + dbname: 'tenant_db' + }; + + const result = await computeLoader.resolve(ctx); + + expect(query).toHaveBeenCalledTimes(2); + expect(query.mock.calls[1][0]).toContain('FROM "compute""schema"."bindings" b'); + expect(query.mock.calls[1][1]).toEqual(['api-a']); + expect(result?.bindings).toEqual([{ + bindingId: 'binding-a', + alias: 'summarize', + config: { graphql: true }, + functionDefinitionId: 'definition-a', + taskIdentifier: 'summarize-task', + description: 'Summarize content', + payloadArgs: [{ name: 'body', type: 'string' }], + module: result?.modules[0] + }]); + }); +}); diff --git a/packages/express-context/src/__tests__/loader-cache-isolation.test.ts b/packages/express-context/src/__tests__/loader-cache-isolation.test.ts new file mode 100644 index 0000000000..c94c9b64fa --- /dev/null +++ b/packages/express-context/src/__tests__/loader-cache-isolation.test.ts @@ -0,0 +1,175 @@ +import type { Pool } from 'pg'; + +import { createModuleLoader } from '../loaders/create-loader'; +import type { LoaderContext } from '../loaders/types'; + +const context = ( + routingPool: Pool, + tenantPool: Pool, + suffix: string +): LoaderContext => ({ + routingPool, + routingPoolIdentity: `routing:${suffix}`, + tenantPool, + tenantPoolIdentity: `tenant:${suffix}`, + databaseId: 'cloned-database-id', + apiId: 'cloned-api-id', + dbname: 'cloned_database' +}); + +describe('module loader physical cache isolation', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('does not share one logical database/API entry across physical pool contracts', async () => { + const routingA = {} as Pool; + const routingB = {} as Pool; + const tenantA = {} as Pool; + const tenantB = {} as Pool; + const ctxA = context(routingA, tenantA, 'a'); + const ctxB = context(routingB, tenantB, 'b'); + const resolve = jest.fn(async (ctx: LoaderContext) => + ctx.tenantPoolIdentity === 'tenant:a' ? 'config-a' : 'config-b' + ); + const loader = createModuleLoader({ name: 'physical-isolation', resolve }); + + await expect(loader.resolve(ctxA)).resolves.toBe('config-a'); + await expect(loader.resolve(ctxB)).resolves.toBe('config-b'); + await expect(loader.resolve(ctxA)).resolves.toBe('config-a'); + await expect(loader.resolve(ctxB)).resolves.toBe('config-b'); + + expect(resolve).toHaveBeenCalledTimes(2); + }); + + it('can invalidate one physical contract without evicting its logical twin', async () => { + const ctxA = context({} as Pool, {} as Pool, 'a'); + const ctxB = context({} as Pool, {} as Pool, 'b'); + let generation = 0; + const resolve = jest.fn(async (ctx: LoaderContext) => + `${ctx.tenantPoolIdentity}:${++generation}` + ); + const loader = createModuleLoader({ name: 'physical-invalidation', resolve }); + + const firstA = await loader.resolve(ctxA); + const firstB = await loader.resolve(ctxB); + loader.invalidate(ctxA.databaseId, ctxA); + + await expect(loader.resolve(ctxB)).resolves.toBe(firstB); + await expect(loader.resolve(ctxA)).resolves.not.toBe(firstA); + expect(resolve).toHaveBeenCalledTimes(3); + }); + + it('falls back to pool object identity for generic callers without opaque identities', async () => { + const routingPool = {} as Pool; + const tenantA = {} as Pool; + const tenantB = {} as Pool; + const base = { + routingPool, + databaseId: 'cloned-database-id', + apiId: 'cloned-api-id', + dbname: 'cloned_database' + }; + const resolve = jest.fn(async (ctx: LoaderContext) => + ctx.tenantPool === tenantA ? 'config-a' : 'config-b' + ); + const loader = createModuleLoader({ name: 'object-isolation', resolve }); + + await expect(loader.resolve({ ...base, tenantPool: tenantA })).resolves.toBe('config-a'); + await expect(loader.resolve({ ...base, tenantPool: tenantB })).resolves.toBe('config-b'); + await expect(loader.resolve({ ...base, tenantPool: tenantA })).resolves.toBe('config-a'); + + expect(resolve).toHaveBeenCalledTimes(2); + }); + + it('isolates routing schemas even when the physical pools and logical IDs match', async () => { + const base = context({} as Pool, {} as Pool, 'shared'); + const resolve = jest.fn(async (ctx: LoaderContext) => ctx.routingSchema); + const loader = createModuleLoader({ name: 'routing-schema-isolation', resolve }); + + await expect(loader.resolve({ ...base, routingSchema: 'routing_a' })) + .resolves.toBe('routing_a'); + await expect(loader.resolve({ ...base, routingSchema: 'routing_b' })) + .resolves.toBe('routing_b'); + await expect(loader.resolve({ ...base, routingSchema: 'routing_a' })) + .resolves.toBe('routing_a'); + + expect(resolve).toHaveBeenCalledTimes(2); + }); + + it('coalesces concurrent misses for one exact build contract', async () => { + const ctx = context({} as Pool, {} as Pool, 'shared'); + const resolve = jest.fn(async () => 'shared-config'); + const loader = createModuleLoader({ name: 'concurrent-coalescing', resolve }); + + await expect(Promise.all([ + loader.resolve(ctx), + loader.resolve(ctx), + loader.resolve(ctx) + ])).resolves.toEqual(['shared-config', 'shared-config', 'shared-config']); + expect(resolve).toHaveBeenCalledTimes(1); + }); + + it('does not publish a resolution invalidated while its query is in flight', async () => { + const ctx = context({} as Pool, {} as Pool, 'shared'); + let complete!: (value: string) => void; + const first = new Promise((resolve) => { + complete = resolve; + }); + const resolve = jest.fn() + .mockImplementationOnce(() => first) + .mockResolvedValueOnce('fresh-config'); + const loader = createModuleLoader({ + name: 'inflight-invalidation', + resolve + }); + + const stale = loader.resolve(ctx); + loader.invalidate(ctx.databaseId, ctx); + const fresh = loader.resolve(ctx); + await expect(fresh).resolves.toBe('fresh-config'); + complete('stale-config'); + await expect(stale).resolves.toBe('stale-config'); + await expect(loader.resolve(ctx)).resolves.toBe('fresh-config'); + expect(resolve).toHaveBeenCalledTimes(2); + }); + + it('uses a hard TTL that cache hits cannot extend indefinitely', async () => { + let now = 1; + jest.spyOn(performance, 'now').mockImplementation(() => now); + const ctx = context({} as Pool, {} as Pool, 'shared'); + let generation = 0; + const resolve = jest.fn(async () => `config-${++generation}`); + const loader = createModuleLoader({ + name: 'hard-expiry', + ttlMs: 100, + resolve + }); + + await expect(loader.resolve(ctx)).resolves.toBe('config-1'); + now = 76; + await expect(loader.resolve(ctx)).resolves.toBe('config-1'); + now = 106; + await expect(loader.resolve(ctx)).resolves.toBe('config-2'); + expect(resolve).toHaveBeenCalledTimes(2); + }); + + it('does not cache or coalesce authoritative loader reads', async () => { + const ctx = context({} as Pool, {} as Pool, 'shared'); + let generation = 0; + const resolve = jest.fn(async () => `config-${++generation}`); + const loader = createModuleLoader({ + name: 'authoritative', + cache: false, + resolve + }); + + await expect(Promise.all([ + loader.resolve(ctx), + loader.resolve(ctx) + ])).resolves.toEqual(['config-1', 'config-2']); + await expect(loader.resolve(ctx)).resolves.toBe('config-3'); + expect(resolve).toHaveBeenCalledTimes(3); + expect(loader.cacheSize).toBe(0); + }); +}); diff --git a/packages/express-context/src/__tests__/pg-settings.test.ts b/packages/express-context/src/__tests__/pg-settings.test.ts new file mode 100644 index 0000000000..0bfaf36ec3 --- /dev/null +++ b/packages/express-context/src/__tests__/pg-settings.test.ts @@ -0,0 +1,104 @@ +import { buildPgSettings, SECURITY_GUC_KEYS } from '../pg-settings'; +import type { ApiStructure } from '../types'; + +const api: ApiStructure = { + apiId: 'api-a', + databaseId: 'database-a', + dbname: 'tenant_a', + schema: ['tenant_a_public'], + roleName: 'tenant_user', + anonRole: 'tenant_anon' +}; + +describe('buildPgSettings', () => { + it('initializes every security GUC and explicit transaction state for anonymous requests', () => { + const settings = buildPgSettings({ api, token: null, requestId: 'request-a' }); + + expect(settings.role).toBe('tenant_anon'); + expect(settings['request.id']).toBe('request-a'); + expect(settings['transaction_read_only']).toBe('off'); + expect(settings['search_path']).toBe('pg_catalog, "tenant_a_public"'); + expect(settings['row_security']).toBe('on'); + for (const key of SECURITY_GUC_KEYS) { + expect(Object.prototype.hasOwnProperty.call(settings, key)).toBe(true); + } + expect(settings['jwt.claims.user_id']).toBe(''); + }); + + it('quotes every physical schema in the pinned search path', () => { + const settings = buildPgSettings({ + api: { ...api, schema: ['tenant-a', 'quoted"schema'] }, + token: null, + requestId: 'request-search-path', + dependencySchemas: ['postgis-ext', 'shared"api'] + }); + + expect(settings['search_path']).toBe( + 'pg_catalog, "postgis-ext", "shared""api", "tenant-a", "quoted""schema"' + ); + }); + + it('sets known claims while keeping every absent claim empty', () => { + const settings = buildPgSettings({ + api, + token: { + id: 'token-a', + user_id: 'user-a', + access_level: 'read_only', + kind: 'api_key' + }, + requestId: 'request-b', + clientIp: '127.0.0.1', + origin: 'https://example.test', + userAgent: 'test-agent', + deviceToken: 'device-a' + }); + + expect(settings).toMatchObject({ + role: 'tenant_user', + 'jwt.claims.token_id': 'token-a', + 'jwt.claims.user_id': 'user-a', + 'jwt.claims.principal_id': 'user-a', + 'jwt.claims.session_id': '', + 'jwt.claims.access_level': 'read_only', + 'jwt.claims.device_token': 'device-a', + 'transaction_read_only': 'on' + }); + }); + + it('does not retain claims or read-only state across requests', () => { + buildPgSettings({ + api, + token: { user_id: 'user-a', access_level: 'read_only' }, + requestId: 'request-a' + }); + const next = buildPgSettings({ api, token: null, requestId: 'request-b' }); + + expect(next['jwt.claims.user_id']).toBe(''); + expect(next['jwt.claims.access_level']).toBe(''); + expect(next['transaction_read_only']).toBe('off'); + }); + + it('rejects runtime-shaped trusted claims that could override session state', () => { + expect(() => buildPgSettings({ + api, + token: null, + requestId: 'request-extra-claim', + trustedClaims: { + role: 'cross_tenant_owner' + } as unknown as Record<'jwt.claims.user_id', string> + })).toThrow("trustedClaims contains unsupported security GUC 'role'"); + + const claims = Object.create(null) as Record; + Object.defineProperty(claims, 'jwt.claims.user_id', { + enumerable: true, + get: () => 'getter-value' + }); + expect(() => buildPgSettings({ + api, + token: null, + requestId: 'request-accessor-claim', + trustedClaims: claims as Record<'jwt.claims.user_id', string> + })).toThrow('trustedClaims.jwt.claims.user_id must be a string data property'); + }); +}); diff --git a/packages/express-context/src/__tests__/security-loader-freshness.test.ts b/packages/express-context/src/__tests__/security-loader-freshness.test.ts new file mode 100644 index 0000000000..a8e9cc7f8d --- /dev/null +++ b/packages/express-context/src/__tests__/security-loader-freshness.test.ts @@ -0,0 +1,269 @@ +import type { Pool } from 'pg'; + +import { authSettingsLoader } from '../loaders/auth-settings'; +import { corsLoader } from '../loaders/cors'; +import { databaseSettingsLoader } from '../loaders/database-settings'; +import { pubkeyLoader } from '../loaders/pubkey'; +import { rlsLoader } from '../loaders/rls'; +import type { LoaderContext } from '../loaders/types'; +import { webauthnLoader } from '../loaders/webauthn'; + +const loaderContext = ( + routingPool: Pool, + tenantPool: Pool +): LoaderContext => ({ + routingPool, + routingPoolIdentity: 'routing:security-test', + routingSchema: 'routing_public', + tenantPool, + tenantPoolIdentity: 'tenant:security-test', + databaseId: 'database-123', + apiId: 'api-123', + dbname: 'tenant_database' +}); + +describe('security-sensitive module freshness', () => { + afterEach(() => { + rlsLoader.invalidate(); + authSettingsLoader.invalidate(); + corsLoader.invalidate(); + databaseSettingsLoader.invalidate(); + pubkeyLoader.invalidate(); + webauthnLoader.invalidate(); + }); + + it('reads RLS authentication routing authoritatively on every request', async () => { + const query = jest.fn() + .mockResolvedValueOnce({ rows: [{ + authenticate: 'authenticate_v1', + authenticate_strict: 'authenticate_strict_v1', + authenticate_schema: 'auth_private', + role_schema: 'auth_public', + current_role: 'current_role', + current_role_id: 'current_role_id', + current_ip_address: 'current_ip_address', + current_user_agent: 'current_user_agent' + }] }) + .mockResolvedValueOnce({ rows: [{ + authenticate: 'authenticate_v2', + authenticate_strict: 'authenticate_strict_v2', + authenticate_schema: 'auth_private', + role_schema: 'auth_public', + current_role: 'current_role', + current_role_id: 'current_role_id', + current_ip_address: 'current_ip_address', + current_user_agent: 'current_user_agent' + }] }); + const routingPool = { query } as unknown as Pool; + const ctx = loaderContext(routingPool, {} as Pool); + + await expect(rlsLoader.resolve(ctx)).resolves.toMatchObject({ + authenticate: 'authenticate_v1' + }); + await expect(rlsLoader.resolve(ctx)).resolves.toMatchObject({ + authenticate: 'authenticate_v2' + }); + + expect(query).toHaveBeenCalledTimes(2); + expect(rlsLoader.cacheSize).toBe(0); + }); + + it('reads cookie and CAPTCHA policy authoritatively on every request', async () => { + const query = jest.fn() + .mockResolvedValueOnce({ + rows: [{ schema_name: 'sessions_private', table_name: 'auth_settings' }] + }) + .mockResolvedValueOnce({ rows: [{ + cookie_secure: true, + cookie_samesite: 'lax', + cookie_domain: null, + cookie_httponly: true, + cookie_max_age: '3600', + cookie_path: '/', + remember_me_duration: '86400', + enable_captcha: false, + captcha_site_key: null + }] }) + .mockResolvedValueOnce({ + rows: [{ schema_name: 'sessions_private', table_name: 'auth_settings' }] + }) + .mockResolvedValueOnce({ rows: [{ + cookie_secure: true, + cookie_samesite: 'strict', + cookie_domain: null, + cookie_httponly: true, + cookie_max_age: '1800', + cookie_path: '/', + remember_me_duration: '43200', + enable_captcha: true, + captcha_site_key: 'site-key-v2' + }] }); + const tenantPool = { query } as unknown as Pool; + const ctx = loaderContext({} as Pool, tenantPool); + + await expect(authSettingsLoader.resolve(ctx)).resolves.toMatchObject({ + cookieSamesite: 'lax', + enableCaptcha: false + }); + await expect(authSettingsLoader.resolve(ctx)).resolves.toMatchObject({ + cookieSamesite: 'strict', + enableCaptcha: true + }); + + expect(query).toHaveBeenCalledTimes(4); + expect(authSettingsLoader.cacheSize).toBe(0); + }); + + it('does not retain revoked CORS policy', async () => { + const query = jest.fn() + .mockResolvedValueOnce({ rows: [{ allowed_origins: ['https://old.example'] }] }) + .mockResolvedValueOnce({ rows: [{ allowed_origins: [] }] }); + const ctx = loaderContext({ query } as unknown as Pool, {} as Pool); + + await expect(corsLoader.resolve(ctx)).resolves.toEqual(['https://old.example']); + await expect(corsLoader.resolve(ctx)).resolves.toEqual([]); + + expect(query).toHaveBeenCalledTimes(2); + expect(corsLoader.cacheSize).toBe(0); + }); + + it('does not retain a revoked Graphile/realtime feature surface', async () => { + const settings = (enabled: boolean) => ({ + resolved_enable_aggregates: enabled, + resolved_enable_postgis: enabled, + resolved_enable_search: enabled, + resolved_enable_direct_uploads: enabled, + resolved_enable_presigned_uploads: enabled, + resolved_enable_many_to_many: enabled, + resolved_enable_connection_filter: enabled, + resolved_enable_ltree: enabled, + resolved_enable_llm: enabled, + resolved_enable_realtime: enabled, + resolved_enable_bulk: enabled, + resolved_enable_i18n: enabled + }); + const query = jest.fn() + .mockResolvedValueOnce({ rows: [settings(true)] }) + .mockResolvedValueOnce({ rows: [settings(false)] }); + const ctx = loaderContext({ query } as unknown as Pool, {} as Pool); + + await expect(databaseSettingsLoader.resolve(ctx)).resolves.toMatchObject({ + enableRealtime: true, + enableSearch: true + }); + await expect(databaseSettingsLoader.resolve(ctx)).resolves.toMatchObject({ + enableRealtime: false, + enableSearch: false + }); + + expect(query).toHaveBeenCalledTimes(2); + expect(databaseSettingsLoader.cacheSize).toBe(0); + }); + + it('rejects ambiguous or incomplete Graphile feature contracts', async () => { + const settings = { + resolved_enable_aggregates: false, + resolved_enable_postgis: false, + resolved_enable_search: false, + resolved_enable_direct_uploads: false, + resolved_enable_presigned_uploads: false, + resolved_enable_many_to_many: false, + resolved_enable_connection_filter: false, + resolved_enable_ltree: false, + resolved_enable_llm: false, + resolved_enable_realtime: false, + resolved_enable_bulk: false, + resolved_enable_i18n: false + }; + const ambiguous = loaderContext({ + query: jest.fn().mockResolvedValue({ rows: [settings, settings] }) + } as unknown as Pool, {} as Pool); + const incomplete = loaderContext({ + query: jest.fn().mockResolvedValue({ + rows: [{ ...settings, resolved_enable_search: null }] + }) + } as unknown as Pool, {} as Pool); + + await expect(databaseSettingsLoader.resolve(ambiguous)) + .rejects.toThrow('Ambiguous database feature configuration'); + await expect(databaseSettingsLoader.resolve(incomplete)) + .rejects.toThrow('Incomplete database feature configuration'); + }); + + it('does not retain changed public-key or WebAuthn policy', async () => { + const pubkeyQuery = jest.fn() + .mockResolvedValueOnce({ rows: [{ + schema: 'auth_public', + crypto_network: 'mainnet', + sign_up_with_key: 'sign_up_v1', + sign_in_request_challenge: 'request_v1', + sign_in_record_failure: 'failure_v1', + sign_in_with_challenge: 'sign_in_v1' + }] }) + .mockResolvedValueOnce({ rows: [{ + schema: 'auth_public', + crypto_network: 'mainnet', + sign_up_with_key: 'sign_up_v2', + sign_in_request_challenge: 'request_v2', + sign_in_record_failure: 'failure_v2', + sign_in_with_challenge: 'sign_in_v2' + }] }); + const pubkeyContext = loaderContext( + { query: pubkeyQuery } as unknown as Pool, + {} as Pool + ); + + await expect(pubkeyLoader.resolve(pubkeyContext)).resolves.toMatchObject({ + signUpWithKey: 'sign_up_v1' + }); + await expect(pubkeyLoader.resolve(pubkeyContext)).resolves.toMatchObject({ + signUpWithKey: 'sign_up_v2' + }); + + const webauthnQuery = jest.fn() + .mockResolvedValueOnce({ rows: [{ + schema: 'auth_public', + credentials_schema: 'auth_private', + sessions_schema: 'sessions_private', + session_secrets_schema: 'sessions_private', + rp_id: 'old.example', + rp_name: 'Old', + origin_allowlist: ['https://old.example'], + attestation_type: 'none', + require_user_verification: false, + resident_key: 'preferred', + challenge_expiry_seconds: 300 + }] }) + .mockResolvedValueOnce({ rows: [{ + schema: 'auth_public', + credentials_schema: 'auth_private', + sessions_schema: 'sessions_private', + session_secrets_schema: 'sessions_private', + rp_id: 'new.example', + rp_name: 'New', + origin_allowlist: ['https://new.example'], + attestation_type: 'direct', + require_user_verification: true, + resident_key: 'required', + challenge_expiry_seconds: 60 + }] }); + const webauthnContext = loaderContext( + { query: webauthnQuery } as unknown as Pool, + {} as Pool + ); + + await expect(webauthnLoader.resolve(webauthnContext)).resolves.toMatchObject({ + rpId: 'old.example', + requireUserVerification: false + }); + await expect(webauthnLoader.resolve(webauthnContext)).resolves.toMatchObject({ + rpId: 'new.example', + requireUserVerification: true + }); + + expect(pubkeyQuery).toHaveBeenCalledTimes(2); + expect(webauthnQuery).toHaveBeenCalledTimes(2); + expect(pubkeyLoader.cacheSize).toBe(0); + expect(webauthnLoader.cacheSize).toBe(0); + }); +}); diff --git a/packages/express-context/src/__tests__/security-metadata.test.ts b/packages/express-context/src/__tests__/security-metadata.test.ts new file mode 100644 index 0000000000..935d6f4f5d --- /dev/null +++ b/packages/express-context/src/__tests__/security-metadata.test.ts @@ -0,0 +1,134 @@ +import type { Pool } from 'pg'; + +import { createBillingClient } from '../billing-client'; +import { quoteQualifiedSqlIdentifier, quoteSqlIdentifier } from '../sql-identifiers'; +import { agentChatLoader } from '../loaders/agent-chat'; +import { authSettingsLoader } from '../loaders/auth-settings'; +import { rlsLoader } from '../loaders/rls'; +import type { LoaderContext } from '../loaders/types'; + +const context = ( + routingQuery: jest.Mock, + tenantQuery: jest.Mock +): LoaderContext => ({ + routingPool: { query: routingQuery } as unknown as Pool, + routingPoolIdentity: 'routing-a', + tenantPool: { query: tenantQuery } as unknown as Pool, + tenantPoolIdentity: 'tenant-a', + databaseId: '11111111-1111-4111-8111-111111111111', + apiId: '22222222-2222-4222-8222-222222222222', + dbname: 'tenant_a' +}); + +describe('security-sensitive metadata SQL', () => { + afterEach(() => { + agentChatLoader.invalidate(); + authSettingsLoader.invalidate(); + rlsLoader.invalidate(); + }); + + it('quotes arbitrary PostgreSQL identifiers and rejects truncation/NUL cases', () => { + expect(quoteQualifiedSqlIdentifier('tenant-a', 'table"name')) + .toBe('"tenant-a"."table""name"'); + expect(() => quoteSqlIdentifier('')).toThrow('Invalid SQL identifier'); + expect(() => quoteSqlIdentifier('bad\0name')).toThrow('Invalid SQL identifier'); + expect(() => quoteSqlIdentifier('a'.repeat(64))).toThrow('Invalid SQL identifier'); + }); + + it('constrains RLS schemas and functions to the requested database and schema', async () => { + const routingQuery = jest.fn().mockResolvedValue({ rows: [{ + authenticate_schema: 'auth_private', + role_schema: 'auth_public', + authenticate: 'authenticate', + authenticate_strict: 'authenticate_strict', + current_role: 'current_role', + current_role_id: 'current_role_id', + current_user_agent: 'current_user_agent', + current_ip_address: 'current_ip_address' + }] }); + await rlsLoader.resolve(context(routingQuery, jest.fn())); + + const [sql, values] = routingQuery.mock.calls[0]; + expect(values).toEqual(['11111111-1111-4111-8111-111111111111']); + expect(sql).toContain('auth_fn.database_id = rs.database_id'); + expect(sql).toContain('auth_fn.schema_id = rs.authenticate_schema_id'); + expect(sql).toContain('role_fn.schema_id = rs.role_schema_id'); + }); + + it('rejects an RLS row whose referenced metadata did not resolve exactly', async () => { + const routingQuery = jest.fn().mockResolvedValue({ rows: [{ + authenticate_schema: null, + role_schema: 'auth_public', + authenticate: null, + authenticate_strict: null, + current_role: 'current_role', + current_role_id: 'current_role_id', + current_user_agent: 'current_user_agent', + current_ip_address: 'current_ip_address' + }] }); + + await expect(rlsLoader.resolve(context(routingQuery, jest.fn()))) + .rejects.toThrow('Incomplete or cross-database RLS module configuration'); + }); + + it('scopes tenant module discovery and safely quotes discovered identifiers', async () => { + const tenantQuery = jest.fn() + .mockResolvedValueOnce({ + rows: [{ schema_name: 'session-private', table_name: 'auth"settings' }] + }) + .mockResolvedValueOnce({ rows: [{ + cookie_secure: true, + cookie_samesite: 'lax', + cookie_domain: null, + cookie_httponly: true, + cookie_max_age: null, + cookie_path: '/', + remember_me_duration: null, + enable_captcha: false, + captcha_site_key: null + }] }); + await authSettingsLoader.resolve(context(jest.fn(), tenantQuery)); + + expect(tenantQuery.mock.calls[0][0]).toContain('WHERE sm.database_id = $1'); + expect(tenantQuery.mock.calls[0][1]).toEqual([ + '11111111-1111-4111-8111-111111111111' + ]); + expect(tenantQuery.mock.calls[1][0]) + .toContain('FROM "session-private"."auth""settings"'); + }); + + it('scopes agent chat discovery to the exact logical database', async () => { + const tenantQuery = jest.fn().mockResolvedValue({ rows: [{ + schema_name: 'agent_public', + thread_table_name: 'threads', + message_table_name: 'messages', + task_table_name: 'tasks' + }] }); + await agentChatLoader.resolve(context(jest.fn(), tenantQuery)); + + expect(tenantQuery.mock.calls[0][0]).toContain('WHERE acm.database_id = $1'); + expect(tenantQuery.mock.calls[0][1]).toEqual([ + '11111111-1111-4111-8111-111111111111' + ]); + }); + + it('fails a configured billing quota check closed and quotes its function', async () => { + const query = jest.fn().mockRejectedValue(new Error('billing unavailable')); + const withPgClient = jest.fn(async (callback) => callback({ query })); + const billing = createBillingClient( + withPgClient as never, + '33333333-3333-4333-8333-333333333333', + { + publicSchema: 'billing-public', + privateSchema: 'billing"private', + recordUsageFunction: 'record_usage', + checkBillingQuotaFunction: 'check"quota' + }, + null + ); + + await expect(billing.checkQuota('tokens')).resolves.toBe(false); + expect(query.mock.calls[0][0]) + .toContain('SELECT "billing""private"."check""quota"('); + }); +}); diff --git a/packages/express-context/src/__tests__/storage-loader.test.ts b/packages/express-context/src/__tests__/storage-loader.test.ts new file mode 100644 index 0000000000..416d27791c --- /dev/null +++ b/packages/express-context/src/__tests__/storage-loader.test.ts @@ -0,0 +1,55 @@ +import type { Pool } from 'pg'; + +import { storageLoader, STORAGE_MODULE_SQL } from '../loaders/storage'; + +describe('storage control-plane loader', () => { + afterEach(() => storageLoader.invalidate()); + + it('normalizes immutable module metadata and caches it by database contract', async () => { + const query = jest.fn().mockResolvedValue({ + rows: [{ + id: 'storage-a', + scope: 'app', + entity_table_id: null, + buckets_schema: 'tenant-a', + buckets_table: 'buckets"table', + files_schema: 'tenant-a', + files_table: 'files', + endpoint: null, + public_url_prefix: null, + provider: null, + allowed_origins: null, + upload_url_expiry_seconds: null, + download_url_expiry_seconds: null, + default_max_file_size: null, + max_filename_length: null, + cache_ttl_seconds: null, + max_bulk_files: null, + max_bulk_total_size: '1073741824', + has_path_shares: null, + entity_schema: null, + entity_table: null + }] + }); + const ctx = { + routingPool: {} as Pool, + tenantPool: { query } as unknown as Pool, + databaseId: 'database-storage-loader-test', + dbname: 'tenant_db' + }; + + const first = await storageLoader.resolve(ctx); + const cached = await storageLoader.resolve(ctx); + + expect(query).toHaveBeenCalledTimes(1); + expect(query).toHaveBeenCalledWith(STORAGE_MODULE_SQL, [ctx.databaseId]); + expect(cached).toBe(first); + expect(first?.modules[0]).toMatchObject({ + bucketsQualifiedName: '"tenant-a"."buckets""table"', + filesQualifiedName: '"tenant-a"."files"', + uploadUrlExpirySeconds: 900, + maxBulkTotalSize: 1073741824, + hasPathShares: false + }); + }); +}); diff --git a/packages/express-context/src/billing-client.ts b/packages/express-context/src/billing-client.ts index ae13066707..27e00d5313 100644 --- a/packages/express-context/src/billing-client.ts +++ b/packages/express-context/src/billing-client.ts @@ -15,6 +15,7 @@ import { Logger } from '@pgpmjs/logger'; +import { quoteQualifiedSqlIdentifier } from './sql-identifiers'; import type { BillingConfig, InferenceLogConfig, WithPgClient } from './types'; const log = new Logger('billing-client'); @@ -48,7 +49,8 @@ export interface BillingClient { * Check if the entity has sufficient quota for the requested amount. * Returns true if allowed, false if quota is exceeded. * - * Gracefully returns true if billing is not provisioned or errors. + * Returns true when billing is not provisioned. Once billing is configured, + * lookup failures deny the request so quota enforcement cannot fail open. */ checkQuota(meterSlug: string, amount?: number): Promise; @@ -79,14 +81,19 @@ export function createBillingClient( try { return await withPgClient(async (client) => { - const sql = `SELECT "${billing.privateSchema}"."${billing.checkBillingQuotaFunction}"($1, $2::uuid, $3) AS allowed`; + const fn = quoteQualifiedSqlIdentifier( + billing.privateSchema, + billing.checkBillingQuotaFunction, + 'billing quota function' + ); + const sql = `SELECT ${fn}($1, $2::uuid, $3) AS allowed`; const result = await client.query(sql, [meterSlug, entityId, amount]); return result.rows[0]?.allowed !== false; }); } catch (e: unknown) { const message = e instanceof Error ? e.message : String(e); - log.warn(`check_billing_quota failed (allowing): ${message}`); - return true; + log.warn(`check_billing_quota failed (denying): ${message}`); + return false; } }, @@ -95,7 +102,12 @@ export function createBillingClient( try { await withPgClient(async (client) => { - const sql = `SELECT "${billing.privateSchema}"."${billing.recordUsageFunction}"($1, $2::uuid, $3, $4::jsonb)`; + const fn = quoteQualifiedSqlIdentifier( + billing.privateSchema, + billing.recordUsageFunction, + 'billing usage function' + ); + const sql = `SELECT ${fn}($1, $2::uuid, $3, $4::jsonb)`; await client.query(sql, [meterSlug, entityId, amount, JSON.stringify(metadata ?? {})]); }); } catch (e: unknown) { @@ -109,8 +121,13 @@ export function createBillingClient( try { await withPgClient(async (client) => { + const table = quoteQualifiedSqlIdentifier( + inferenceLog.schema, + inferenceLog.tableName, + 'inference log table' + ); await client.query( - `INSERT INTO "${inferenceLog.schema}"."${inferenceLog.tableName}" + `INSERT INTO ${table} (entity_id, actor_id, model, provider, service, operation, input_tokens, output_tokens, total_tokens, latency_ms, status, cache_read_tokens, cache_write_tokens, diff --git a/packages/express-context/src/context.ts b/packages/express-context/src/context.ts index 82d87de9d3..31ad836510 100644 --- a/packages/express-context/src/context.ts +++ b/packages/express-context/src/context.ts @@ -17,7 +17,13 @@ import type { PgpmOptions } from '@pgpmjs/types'; import type { NextFunction, Request, RequestHandler, Response } from 'express'; import type { Pool } from 'pg'; -import { getPgPool } from 'pg-cache'; +import { + acquirePgPool, + getPgPool, + getPgPoolIdentity, + type GetPgPoolOptions, + type PgPoolLease +} from 'pg-cache'; import type { BillingClient } from './billing-client'; import { createBillingClient } from './billing-client'; @@ -25,23 +31,69 @@ import type { LoaderRegistry } from './loaders/registry'; import type { LoaderContext } from './loaders/types'; import { withPgClient as withPgClientFn } from './pg-client'; import { buildPgSettings } from './pg-settings'; -import type { BillingConfig, BuiltinModuleMap, ConstructiveContext, InferenceLogConfig, LlmConfig } from './types'; +import type { ApiStructure, BillingConfig, BuiltinModuleMap, ConstructiveContext, InferenceLogConfig, LlmConfig } from './types'; + +type PoolConfig = Parameters[0]; + +/** + * Secret-bearing connection config resolved by the owning server, paired with + * the opaque identity that both request context and Graphile must consume. + */ +export interface RuntimePgPoolResolution { + pgConfig: PoolConfig; + poolIdentity: string; +} export interface ContextMiddlewareOptions { /** Base PG options for pool creation (host, port, user, password) */ pg?: PgpmOptions['pg']; + /** Least-privilege tenant execution login; inherits unspecified pg fields. */ + runtimePg?: PgpmOptions['pg']; + /** + * Read the server-owned request resolution. Implementations should keep raw + * credentials outside the Express request object (for example in a WeakMap). + */ + getRuntimePgResolution?: ( + req: Request, + api: ApiStructure + ) => Readonly; + /** Optional fail-closed admission check for the tenant execution pool. */ + validateRuntimePool?: (pool: Pool, api: ApiStructure) => Promise; + /** Ordered, audited extension/shared schemas used by request SQL. */ + dependencySchemas?: readonly string[]; /** Module loader registry for per-database cached lookups */ loaders?: LoaderRegistry; /** Routing-plane schema loaders query (defaults to routing_public) */ routingSchema?: string; } +interface ResolvedPool { + pool: Pool; + identity: string; +} + +const resolvePool = ( + config: PoolConfig, + options: GetPgPoolOptions, + leases?: PgPoolLease[] +): ResolvedPool => { + if (!leases) { + return { + pool: getPgPool(config, options), + identity: getPgPoolIdentity(config, options) + }; + } + const lease = acquirePgPool(config, options); + leases.push(lease); + return { pool: lease.pool, identity: lease.identity }; +}; + /** * Create a `useModule` function bound to the given loader context. * - * Calling `useModule('rlsModule')` lazily resolves the RLS loader, - * hitting the DB only on cache miss. The function is a no-op (returns - * undefined) when no registry is configured. + * Calling `useModule('rlsModule')` lazily resolves the RLS loader according to + * that loader's freshness policy. The function is a no-op (returns undefined) + * when no registry is configured. */ function createUseModule( registry: LoaderRegistry | undefined, @@ -65,7 +117,9 @@ function createUseModule( */ export function buildContext( req: Request, - opts: ContextMiddlewareOptions = {} + opts: ContextMiddlewareOptions = {}, + /** Internal request lifetime. Omit for backwards-compatible direct use. */ + poolLeases?: PgPoolLease[] ): ConstructiveContext | null { const api = req.api; if (!api) return null; @@ -77,30 +131,75 @@ export function buildContext( api, token, requestId, - clientIp: req.clientIp + clientIp: req.clientIp, + origin: req.get('origin'), + userAgent: req.get('User-Agent'), + deviceToken: req.deviceToken, + dependencySchemas: opts.dependencySchemas }); - const tenantPool: Pool = getPgPool({ + const suppliedRuntimeResolution = opts.getRuntimePgResolution + ? opts.getRuntimePgResolution(req, api) + : undefined; + if (opts.getRuntimePgResolution && !suppliedRuntimeResolution) { + throw new Error( + 'Runtime PostgreSQL resolution provider returned no exact identity' + ); + } + const runtimeConfig = suppliedRuntimeResolution?.pgConfig ?? { ...opts.pg, + ...opts.runtimePg, database: api.dbname - }); + }; + const runtimePool = resolvePool( + runtimeConfig, + { purpose: 'runtime', sanitizeOnCheckout: true }, + poolLeases + ); + if ( + suppliedRuntimeResolution + && runtimePool.identity !== suppliedRuntimeResolution.poolIdentity + ) { + throw new Error( + 'Resolved runtime PostgreSQL pool identity changed before context acquisition' + ); + } + const tenantPool = runtimePool.pool; // Build loader context (if registry provided and databaseId known) let loaderCtx: LoaderContext | null = null; if (opts.loaders && api.databaseId) { - const routingPool: Pool = getPgPool(opts.pg); + const routingPool = resolvePool(opts.pg ?? {}, { + purpose: 'routing-request-control', + sanitizeOnCheckout: true + }, poolLeases); + const controlTenantPool = resolvePool({ + ...opts.pg, + database: api.dbname + }, { + purpose: 'tenant-request-control', + sanitizeOnCheckout: true + }, poolLeases); loaderCtx = { - routingPool, + routingPool: routingPool.pool, + routingPoolIdentity: routingPool.identity, routingSchema: opts.routingSchema, - tenantPool, + tenantPool: controlTenantPool.pool, + tenantPoolIdentity: controlTenantPool.identity, databaseId: api.databaseId, apiId: api.apiId, dbname: api.dbname }; } + let runtimeSafetyPromise: Promise | null = null; + const ensureRuntimePoolIsSafe = (): Promise => { + if (!opts.validateRuntimePool) return Promise.resolve(); + runtimeSafetyPromise ??= opts.validateRuntimePool(tenantPool, api); + return runtimeSafetyPromise; + }; const withPgClient = (fn: (client: any) => Promise) => - withPgClientFn(tenantPool, pgSettings, fn); + ensureRuntimePoolIsSafe().then(() => withPgClientFn(tenantPool, pgSettings, fn)); const useModule = createUseModule(opts.loaders, loaderCtx); // Lazy-initialized billing client (cached per request) @@ -116,6 +215,7 @@ export function buildContext( userId: token?.user_id ?? null, requestId, pool: tenantPool, + runtimePoolIdentity: runtimePool.identity, withPgClient, useModule, async useBilling() { @@ -172,8 +272,8 @@ export function buildContext( * // Downstream middleware/routes call useModule on demand: * app.post('/v1/chat', async (req, res) => { * const ctx = req.constructive; - * const rls = await ctx.useModule('rlsModule'); // only fires if not cached - * const auth = await ctx.useModule('authSettings'); // only fires if not cached + * const rls = await ctx.useModule('rlsModule'); // authoritative read + * const auth = await ctx.useModule('authSettings'); // authoritative read * // webauthnSettings loader never fires if nobody asks for it * }); * ``` @@ -181,11 +281,48 @@ export function buildContext( export function createContextMiddleware( opts: ContextMiddlewareOptions = {} ): RequestHandler { - return (req: Request, _res: Response, next: NextFunction): void => { - const ctx = buildContext(req, opts); - if (ctx) { + return (req: Request, res: Response, next: NextFunction): void => { + const requestEnded = (): boolean => + Boolean( + req.aborted + || req.socket?.destroyed + || res.destroyed + || res.writableEnded + ); + if (requestEnded()) return; + + const leases: PgPoolLease[] = []; + let released = false; + const releaseLeases = (): void => { + if (released) return; + released = true; + req.removeListener('aborted', releaseLeases); + res.removeListener('finish', releaseLeases); + res.removeListener('close', releaseLeases); + for (const lease of leases.reverse()) lease.release(); + }; + + try { + const ctx = buildContext(req, opts, leases); + if (!ctx) { + releaseLeases(); + next(); + return; + } req.constructive = ctx; + req.once('aborted', releaseLeases); + res.once('finish', releaseLeases); + res.once('close', releaseLeases); + // The response may have ended while the synchronous context builder was + // acquiring its pool leases, before these listeners could be attached. + if (requestEnded()) { + releaseLeases(); + return; + } + next(); + } catch (error) { + releaseLeases(); + next(error); } - next(); }; } diff --git a/packages/express-context/src/index.ts b/packages/express-context/src/index.ts index 013e195f5d..baa4a5a5fb 100644 --- a/packages/express-context/src/index.ts +++ b/packages/express-context/src/index.ts @@ -9,7 +9,7 @@ * - withPgClient (tenant-scoped RLS transaction helper) * - requestId middleware (UUID correlation ID) * - Context middleware (composes all of the above into req.constructive) - * - Module loaders (pluggable per-database cached lookups) + * - Module loaders (pluggable authoritative or hard-TTL lookups) * * @example * ```typescript @@ -28,8 +28,8 @@ * * app.post('/v1/chat', async (req, res) => { * const ctx = req.constructive; - * const rls = await ctx.useModule('rlsModule'); // only fires if not cached - * const auth = await ctx.useModule('authSettings'); // only fires if not cached + * const rls = await ctx.useModule('rlsModule'); // authoritative read + * const auth = await ctx.useModule('authSettings'); // authoritative read * // webauthnSettings loader never fires if nobody asks for it * }); * ``` @@ -45,6 +45,7 @@ export type { AuthSurface, BillingConfig, BuiltinModuleMap, + ComputeBindingConfig, ComputeConfig, ComputeModuleConfig, ConstructiveAPIToken, @@ -56,6 +57,8 @@ export type { LlmConfig, PubkeyChallengeSettings, RlsModule, + StorageConfig, + StorageModuleConfig, WebauthnSettings, WithPgClient, } from './types'; @@ -65,8 +68,15 @@ export type { BillingClient, InferenceLogEntry } from './billing-client'; export { createBillingClient } from './billing-client'; // pgSettings builder -export type { PgSettingsInput } from './pg-settings'; -export { buildPgSettings } from './pg-settings'; +export type { PgSettingsInput, SecurityGucKey } from './pg-settings'; +export { buildPgSettings, SECURITY_GUC_KEYS } from './pg-settings'; + +// Safe interpolation for trusted metadata identifiers. Request values still +// belong in query parameters. +export { + quoteQualifiedSqlIdentifier, + quoteSqlIdentifier +} from './sql-identifiers'; // withPgClient helper export { withPgClient } from './pg-client'; @@ -75,7 +85,10 @@ export { withPgClient } from './pg-client'; export { requestIdMiddleware } from './request-id'; // Context middleware -export type { ContextMiddlewareOptions } from './context'; +export type { + ContextMiddlewareOptions, + RuntimePgPoolResolution +} from './context'; export { buildContext, createContextMiddleware } from './context'; // Module loaders @@ -103,6 +116,7 @@ export { requireDatabaseId, requireIdentityProvider, rlsLoader, + storageLoader, webauthnLoader, } from './loaders'; diff --git a/packages/express-context/src/loaders/agent-chat.ts b/packages/express-context/src/loaders/agent-chat.ts index 7138dee58f..b41618b869 100644 --- a/packages/express-context/src/loaders/agent-chat.ts +++ b/packages/express-context/src/loaders/agent-chat.ts @@ -23,9 +23,10 @@ const AGENT_CHAT_MODULE_SQL = ` acm.message_table_name, acm.task_table_name FROM metaschema_modules_public.agent_chat_module acm - JOIN metaschema_public.schema s ON s.id = acm.schema_id + JOIN metaschema_public.schema s + ON s.id = acm.schema_id + AND s.database_id = acm.database_id WHERE acm.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -50,6 +51,9 @@ export const agentChatLoader: ModuleLoader = createModuleLoader AGENT_CHAT_MODULE_SQL, [databaseId], ); + if (result.rows.length > 1) { + throw new Error('Ambiguous agent chat module configuration'); + } const row = result.rows[0]; if (!row) return undefined; diff --git a/packages/express-context/src/loaders/auth-settings.ts b/packages/express-context/src/loaders/auth-settings.ts index e5734cc603..f03f7b8424 100644 --- a/packages/express-context/src/loaders/auth-settings.ts +++ b/packages/express-context/src/loaders/auth-settings.ts @@ -16,6 +16,7 @@ * makes that resolution sticky for its TTL. */ +import { quoteQualifiedSqlIdentifier } from '../sql-identifiers'; import type { AuthSettings } from '../types'; import { createModuleLoader } from './create-loader'; import type { LoaderContext, ModuleLoader } from './types'; @@ -26,9 +27,10 @@ import { requireDatabaseId } from './types'; const AUTH_SETTINGS_DISCOVERY_SQL = ` SELECT s.schema_name, sm.auth_settings_table_name AS table_name FROM metaschema_modules_public.sessions_module sm - JOIN metaschema_public.schema s ON s.id = sm.schema_id + JOIN metaschema_public.schema s + ON s.id = sm.schema_id + AND s.database_id = sm.database_id WHERE sm.database_id = $1 - LIMIT 1 `; const buildAuthSettingsQuery = (schemaName: string, tableName: string) => ` @@ -42,7 +44,7 @@ const buildAuthSettingsQuery = (schemaName: string, tableName: string) => ` remember_me_duration, enable_captcha, captcha_site_key - FROM "${schemaName}"."${tableName}" + FROM ${quoteQualifiedSqlIdentifier(schemaName, tableName, 'auth settings table')} LIMIT 1 `; @@ -64,7 +66,9 @@ interface AuthSettingsRow { export const authSettingsLoader: ModuleLoader = createModuleLoader({ name: 'authSettings', - ttlMs: 5 * 60_000, + // Cookie and CAPTCHA policy changes must take effect on the next request, + // independently of lossy LISTEN delivery. + cache: false, async resolve(ctx: LoaderContext) { const { tenantPool, databaseId } = ctx; requireDatabaseId(databaseId, 'authSettings'); @@ -74,6 +78,9 @@ export const authSettingsLoader: ModuleLoader = createModuleLoader AUTH_SETTINGS_DISCOVERY_SQL, [databaseId] ); + if (discovery.rows.length > 1) { + throw new Error('Ambiguous sessions module configuration'); + } const resolved = discovery.rows[0]; if (!resolved) return undefined; diff --git a/packages/express-context/src/loaders/billing.ts b/packages/express-context/src/loaders/billing.ts index 3e743499fb..02054bc092 100644 --- a/packages/express-context/src/loaders/billing.ts +++ b/packages/express-context/src/loaders/billing.ts @@ -17,10 +17,13 @@ const BILLING_MODULE_SQL = ` ps.schema_name AS private_schema, bm.record_usage_function FROM metaschema_modules_public.billing_module bm - JOIN metaschema_public.schema s ON bm.schema_id = s.id - JOIN metaschema_public.schema ps ON bm.private_schema_id = ps.id + JOIN metaschema_public.schema s + ON bm.schema_id = s.id + AND s.database_id = bm.database_id + JOIN metaschema_public.schema ps + ON bm.private_schema_id = ps.id + AND ps.database_id = bm.database_id WHERE bm.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -43,8 +46,14 @@ export const billingLoader: ModuleLoader = createModuleLoader 1) { + throw new Error('Ambiguous billing module configuration'); + } const row = result.rows[0]; - if (!row?.record_usage_function) return undefined; + if (!row) return undefined; + if (!row.public_schema || !row.private_schema || !row.record_usage_function) { + throw new Error('Incomplete or cross-database billing module configuration'); + } return { publicSchema: row.public_schema, diff --git a/packages/express-context/src/loaders/compute.ts b/packages/express-context/src/loaders/compute.ts index b9b52ca013..4c65dc6ace 100644 --- a/packages/express-context/src/loaders/compute.ts +++ b/packages/express-context/src/loaders/compute.ts @@ -12,9 +12,10 @@ * the underlying tables governs access. */ -import type { ComputeConfig } from '../types'; -import { createModuleLoader } from './create-loader'; +import type { ComputeBindingConfig, ComputeConfig, ComputeModuleConfig } from '../types'; +import { quoteQualifiedSqlIdentifier } from '../sql-identifiers'; import type { LoaderContext, ModuleLoader } from './types'; +import { createModuleLoader } from './create-loader'; // ─── SQL ──────────────────────────────────────────────────────────────────── @@ -27,10 +28,14 @@ const COMPUTE_MODULE_SQL = ` ivm.invocations_table_name, ivm.entity_field AS invocations_entity_field FROM metaschema_modules_public.function_module fm - JOIN metaschema_public.schema fs ON fs.id = fm.schema_id + JOIN metaschema_public.schema fs + ON fs.id = fm.schema_id + AND fs.database_id = fm.database_id JOIN metaschema_modules_public.function_invocation_module ivm ON ivm.database_id = fm.database_id AND ivm.scope = fm.scope - JOIN metaschema_public.schema ivs ON ivs.id = ivm.schema_id + JOIN metaschema_public.schema ivs + ON ivs.id = ivm.schema_id + AND ivs.database_id = ivm.database_id WHERE fm.database_id = $1 ORDER BY fs.schema_name `; @@ -46,6 +51,32 @@ interface ComputeModuleRow { invocations_entity_field: string | null; } +interface ComputeBindingRow { + id: string; + alias: string; + config: Record | null; + function_definition_id: string; + task_identifier: string; + description: string | null; + payload_args: ComputeBindingConfig['payloadArgs']; +} + +const bindingSql = (module: ComputeModuleConfig): string => ` + SELECT + b.id, + b.alias, + b.config, + b.function_definition_id, + d.task_identifier, + d.description, + d.payload_args + FROM ${quoteQualifiedSqlIdentifier(module.schemaName, module.bindingsTableName, 'compute bindings table')} b + JOIN ${quoteQualifiedSqlIdentifier(module.schemaName, module.definitionsTableName, 'compute definitions table')} d + ON d.id = b.function_definition_id + WHERE b.api_id = $1 + ORDER BY b.alias +`; + // ─── Loader ───────────────────────────────────────────────────────────────── export const computeLoader: ModuleLoader = createModuleLoader({ @@ -60,8 +91,7 @@ export const computeLoader: ModuleLoader = createModuleLoader ({ + const modules: ComputeModuleConfig[] = result.rows.map((row) => ({ schemaName: row.functions_schema_name, definitionsTableName: row.definitions_table_name, // Physical bindings table name, recorded by the metaschema generator @@ -72,8 +102,27 @@ export const computeLoader: ModuleLoader = createModuleLoader { + const bindingResult = await tenantPool.query( + bindingSql(module), + [ctx.apiId] + ); + return bindingResult.rows.map((row): ComputeBindingConfig => ({ + bindingId: row.id, + alias: row.alias, + config: row.config, + functionDefinitionId: row.function_definition_id, + taskIdentifier: row.task_identifier, + description: row.description, + payloadArgs: row.payload_args, + module + })); + }))).flat() + : []; + + return { modules, bindings }; }, }); diff --git a/packages/express-context/src/loaders/cors.ts b/packages/express-context/src/loaders/cors.ts index 2caa417bc4..219fb163ca 100644 --- a/packages/express-context/src/loaders/cors.ts +++ b/packages/express-context/src/loaders/cors.ts @@ -36,7 +36,8 @@ interface CorsSettingsRow { export const corsLoader: ModuleLoader = createModuleLoader({ name: 'corsOrigins', - ttlMs: 5 * 60_000, + // Revoking an allowed browser/WebSocket origin is a security policy change. + cache: false, async resolve(ctx: LoaderContext) { const { routingPool, databaseId, apiId } = ctx; const schema = routingSchemaOf(ctx); diff --git a/packages/express-context/src/loaders/create-loader.ts b/packages/express-context/src/loaders/create-loader.ts index 25aabd333e..2f0e191ea7 100644 --- a/packages/express-context/src/loaders/create-loader.ts +++ b/packages/express-context/src/loaders/create-loader.ts @@ -1,85 +1,205 @@ /** - * create-loader — Factory for building cached ModuleLoader instances. + * create-loader — Factory for building ModuleLoader instances. * - * Wraps a raw resolve function with an LRU cache keyed by databaseId:apiId. - * Each loader gets its own independent cache with configurable TTL and - * max entries. + * Optionally wraps a raw resolve function with an LRU cache keyed by the exact + * routing and tenant pool identities plus databaseId:apiId. Each cached loader + * gets its own independent hard TTL and max entries. */ import { Logger } from '@pgpmjs/logger'; import { LRUCache } from 'lru-cache'; -import type { LoaderContext, ModuleLoader } from './types'; +import { + type LoaderContext, + type ModuleLoader, + routingSchemaOf +} from './types'; export interface CreateLoaderOptions { /** Unique loader name (used in log prefix and modules map key) */ name: string; + /** + * Whether successful/absent results may be shared across requests. + * Security-boundary configuration should set this to false so every + * request observes an authoritative database read. + */ + cache?: boolean; /** TTL in milliseconds (default: 60_000 — 1 minute) */ ttlMs?: number; - /** Max cache entries before LRU eviction (default: 100) */ + /** Max cache entries before LRU eviction (default: 1024) */ max?: number; /** The actual resolution function. Called on cache miss. */ resolve: (ctx: LoaderContext) => Promise; } const DEFAULT_TTL_MS = 60_000; -const DEFAULT_MAX = 100; +// Match the Graphile instance governor's hard ceiling. A smaller hidden +// metadata cache would thrash control-plane queries long before heap pressure +// requires evicting the corresponding resident tenant handlers. +const DEFAULT_MAX = 1024; + +let nextPoolObjectIdentity = 0; +const poolObjectIdentities = new WeakMap(); + +const poolIdentity = (pool: object, explicitIdentity?: string): string => { + if (explicitIdentity?.trim()) return explicitIdentity; + let identity = poolObjectIdentities.get(pool); + if (!identity) { + identity = `pool-object:${++nextPoolObjectIdentity}`; + poolObjectIdentities.set(pool, identity); + } + return identity; +}; + +interface LoaderCacheContract { + databaseId: string; + routingSchema: string; + routingPoolIdentity: string; + tenantPoolIdentity: string; +} + +const cacheContract = (ctx: LoaderContext): LoaderCacheContract => ({ + databaseId: ctx.databaseId, + routingSchema: routingSchemaOf(ctx), + routingPoolIdentity: poolIdentity(ctx.routingPool, ctx.routingPoolIdentity), + tenantPoolIdentity: poolIdentity(ctx.tenantPool, ctx.tenantPoolIdentity) +}); + +const cacheKey = (ctx: LoaderContext, contract: LoaderCacheContract): string => + JSON.stringify([ + contract.routingPoolIdentity, + contract.tenantPoolIdentity, + contract.routingSchema, + contract.databaseId, + ctx.apiId ?? null + ]); + +interface LoaderCacheEntry { + contract: LoaderCacheContract; + value: T | undefined; +} + +interface PendingResolution { + contract: LoaderCacheContract; + invalidated: boolean; + promise: Promise; +} + +const samePhysicalContract = ( + left: LoaderCacheContract, + right: LoaderCacheContract +): boolean => + left.routingPoolIdentity === right.routingPoolIdentity + && left.tenantPoolIdentity === right.tenantPoolIdentity + && left.routingSchema === right.routingSchema; export function createModuleLoader(opts: CreateLoaderOptions): ModuleLoader { const log = new Logger(`loader:${opts.name}`); - const cache = new LRUCache({ + const cacheEnabled = opts.cache !== false; + const cache = new LRUCache>({ max: opts.max ?? DEFAULT_MAX, ttl: opts.ttlMs ?? DEFAULT_TTL_MS, - updateAgeOnGet: true, - allowStale: false, + ttlResolution: 0, + // A hit must never extend configuration lifetime indefinitely. This is a + // hard maximum staleness bound for non-security-sensitive module data. + updateAgeOnGet: false, + allowStale: false }); + const pending = new Map>(); return { name: opts.name, async resolve(ctx: LoaderContext): Promise { - const key = ctx.apiId ? `${ctx.databaseId}:${ctx.apiId}` : ctx.databaseId; + const logicalKey = ctx.apiId + ? `${ctx.databaseId}:${ctx.apiId}` + : ctx.databaseId; + + if (!cacheEnabled) { + log.debug(`Authoritative resolve databaseId=${logicalKey}`); + try { + return await opts.resolve(ctx); + } catch (e: any) { + if (e.code === '42P01') { + log.debug(`Module tables absent for databaseId=${logicalKey}: ${e.message}`); + return undefined; + } + log.warn(`Failed to resolve databaseId=${logicalKey}: ${e.message}`); + throw e; + } + } - if (cache.has(key)) { - log.debug(`Cache HIT databaseId=${key}`); - return cache.get(key); + const contract = cacheContract(ctx); + const key = cacheKey(ctx, contract); + const cached = cache.get(key); + if (cached !== undefined) { + log.debug(`Cache HIT databaseId=${logicalKey}`); + return cached.value; } - log.debug(`Cache MISS databaseId=${key}, resolving`); + const existing = pending.get(key); + if (existing && !existing.invalidated) { + log.debug(`Cache COALESCE databaseId=${logicalKey}`); + return existing.promise; + } + + log.debug(`Cache MISS databaseId=${logicalKey}, resolving`); // "Not provisioned" is expressed by the loader returning undefined, or // by the module's tables not existing at all (42P01 undefined_table). // Any other resolution error (bad query, ambiguous config) propagates — // never silently coerced into "module absent". - try { - const value = await opts.resolve(ctx); - cache.set(key, value); - return value; - } catch (e: any) { - if (e.code === '42P01') { - log.debug(`Module tables absent for databaseId=${key}: ${e.message}`); - cache.set(key, undefined); - return undefined; + const resolution: PendingResolution = { + contract, + invalidated: false, + promise: undefined as unknown as Promise + }; + resolution.promise = Promise.resolve().then(async () => { + try { + const value = await opts.resolve(ctx); + if (!resolution.invalidated) cache.set(key, { contract, value }); + return value; + } catch (e: any) { + if (e.code === '42P01') { + log.debug(`Module tables absent for databaseId=${logicalKey}: ${e.message}`); + if (!resolution.invalidated) { + cache.set(key, { contract, value: undefined }); + } + return undefined; + } + log.warn(`Failed to resolve databaseId=${logicalKey}: ${e.message}`); + throw e; + } finally { + if (pending.get(key) === resolution) pending.delete(key); } - log.warn(`Failed to resolve databaseId=${key}: ${e.message}`); - throw e; - } + }); + pending.set(key, resolution); + return resolution.promise; }, - invalidate(databaseId?: string): void { - if (databaseId) { - // Clear the plain databaseId key and any composite databaseId:apiId keys - let cleared = 0; - for (const k of cache.keys()) { - if (k === databaseId || k.startsWith(`${databaseId}:`)) { - cache.delete(k); - cleared++; - } - } - log.debug(`Invalidated ${cleared} entries for databaseId=${databaseId}`); - } else { + invalidate(databaseId?: string, context?: LoaderContext): void { + if (!databaseId && !context) { + const previousSize = cache.size; cache.clear(); - log.debug(`Invalidated all entries (was size=${cache.size})`); + for (const resolution of pending.values()) resolution.invalidated = true; + log.debug(`Invalidated all entries (was size=${previousSize})`); + return; + } + + const exact = context ? cacheContract(context) : null; + const matches = (contract: LoaderCacheContract): boolean => + (!databaseId || contract.databaseId === databaseId) + && (!exact || samePhysicalContract(contract, exact)); + let cleared = 0; + for (const [key, entry] of cache.entries()) { + if (!matches(entry.contract)) continue; + if (cache.delete(key)) cleared++; + } + for (const resolution of pending.values()) { + if (matches(resolution.contract)) resolution.invalidated = true; } + log.debug( + `Invalidated ${cleared} entries${databaseId ? ` for databaseId=${databaseId}` : ''}` + ); }, get cacheSize(): number { diff --git a/packages/express-context/src/loaders/database-settings.ts b/packages/express-context/src/loaders/database-settings.ts index aa7232f4fb..b10e318342 100644 --- a/packages/express-context/src/loaders/database-settings.ts +++ b/packages/express-context/src/loaders/database-settings.ts @@ -30,7 +30,6 @@ const databaseSettingsSql = (schema: string): string => ` FROM "${schema}".database_settings ds LEFT JOIN "${schema}".api_settings aps ON ds.database_id = aps.database_id AND aps.api_id = $2 WHERE ds.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -50,11 +49,28 @@ interface DatabaseSettingsRow { resolved_enable_i18n: boolean; } +const BOOLEAN_COLUMNS: readonly (keyof DatabaseSettingsRow)[] = [ + 'resolved_enable_aggregates', + 'resolved_enable_postgis', + 'resolved_enable_search', + 'resolved_enable_direct_uploads', + 'resolved_enable_presigned_uploads', + 'resolved_enable_many_to_many', + 'resolved_enable_connection_filter', + 'resolved_enable_ltree', + 'resolved_enable_llm', + 'resolved_enable_realtime', + 'resolved_enable_bulk', + 'resolved_enable_i18n' +]; + // ─── Loader ───────────────────────────────────────────────────────────────── export const databaseSettingsLoader: ModuleLoader = createModuleLoader({ name: 'databaseSettings', - ttlMs: 5 * 60_000, + // These flags select the executable Graphile/plugin surface and realtime + // admission, so a disable/revocation must alter the next build contract. + cache: false, async resolve(ctx: LoaderContext) { const { routingPool, databaseId, apiId } = ctx; @@ -62,8 +78,18 @@ export const databaseSettingsLoader: ModuleLoader = createModu databaseSettingsSql(routingSchemaOf(ctx)), [databaseId, apiId ?? null] ); + if (result.rows.length > 1) { + throw new Error( + `Ambiguous database feature configuration for database ${databaseId}` + ); + } const row = result.rows[0]; if (!row) return undefined; + if (BOOLEAN_COLUMNS.some((column) => typeof row[column] !== 'boolean')) { + throw new Error( + `Incomplete database feature configuration for database ${databaseId}` + ); + } return { enableAggregates: row.resolved_enable_aggregates, diff --git a/packages/express-context/src/loaders/index.ts b/packages/express-context/src/loaders/index.ts index a8a3203428..dfe045501d 100644 --- a/packages/express-context/src/loaders/index.ts +++ b/packages/express-context/src/loaders/index.ts @@ -55,6 +55,7 @@ export { inferenceLogLoader } from './inference-log'; export { llmLoader } from './llm'; export { pubkeyLoader } from './pubkey'; export { rlsLoader } from './rls'; +export { storageLoader } from './storage'; export { webauthnLoader } from './webauthn'; /** @@ -72,6 +73,7 @@ import { llmLoader } from './llm'; import { pubkeyLoader } from './pubkey'; import { createLoaderRegistry } from './registry'; import { rlsLoader } from './rls'; +import { storageLoader } from './storage'; import { webauthnLoader } from './webauthn'; export function createDefaultRegistry() { @@ -88,5 +90,6 @@ export function createDefaultRegistry() { registry.register(agentChatLoader); registry.register(llmLoader); registry.register(computeLoader); + registry.register(storageLoader); return registry; } diff --git a/packages/express-context/src/loaders/inference-log.ts b/packages/express-context/src/loaders/inference-log.ts index 2488792460..8d419c5754 100644 --- a/packages/express-context/src/loaders/inference-log.ts +++ b/packages/express-context/src/loaders/inference-log.ts @@ -16,9 +16,10 @@ const INFERENCE_LOG_MODULE_SQL = ` s.schema_name AS schema, ilm.inference_log_table_name AS table_name FROM metaschema_modules_public.inference_log_module ilm - JOIN metaschema_public.schema s ON ilm.schema_id = s.id + JOIN metaschema_public.schema s + ON ilm.schema_id = s.id + AND s.database_id = ilm.database_id WHERE ilm.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -40,8 +41,14 @@ export const inferenceLogLoader: ModuleLoader = createModule INFERENCE_LOG_MODULE_SQL, [databaseId], ); + if (result.rows.length > 1) { + throw new Error('Ambiguous inference-log module configuration'); + } const row = result.rows[0]; - if (!row?.schema || !row?.table_name) return undefined; + if (!row) return undefined; + if (!row.schema || !row.table_name) { + throw new Error('Incomplete or cross-database inference-log configuration'); + } return { schema: row.schema, diff --git a/packages/express-context/src/loaders/llm.ts b/packages/express-context/src/loaders/llm.ts index 33cb28f51d..8ded558f4c 100644 --- a/packages/express-context/src/loaders/llm.ts +++ b/packages/express-context/src/loaders/llm.ts @@ -28,7 +28,6 @@ const LLM_MODULE_SQL = ` lm.rag_context_limit FROM metaschema_modules_public.llm_module lm WHERE lm.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -58,6 +57,9 @@ export const llmLoader: ModuleLoader = createModuleLoader( LLM_MODULE_SQL, [databaseId], ); + if (result.rows.length > 1) { + throw new Error('Ambiguous LLM module configuration'); + } const row = result.rows[0]; if (!row) return undefined; diff --git a/packages/express-context/src/loaders/pubkey.ts b/packages/express-context/src/loaders/pubkey.ts index 9ff560d103..de1db8661c 100644 --- a/packages/express-context/src/loaders/pubkey.ts +++ b/packages/express-context/src/loaders/pubkey.ts @@ -22,13 +22,26 @@ const pubkeySettingsSql = (schema: string): string => ` sign_in_fail_fn.name AS sign_in_record_failure, sign_in_fn.name AS sign_in_with_challenge FROM "${schema}".pubkey_settings ps - LEFT JOIN metaschema_public.schema s ON ps.schema_id = s.id - LEFT JOIN metaschema_public.function sign_up_fn ON ps.sign_up_with_key_function_id = sign_up_fn.id - LEFT JOIN metaschema_public.function sign_in_req_fn ON ps.sign_in_request_challenge_function_id = sign_in_req_fn.id - LEFT JOIN metaschema_public.function sign_in_fail_fn ON ps.sign_in_record_failure_function_id = sign_in_fail_fn.id - LEFT JOIN metaschema_public.function sign_in_fn ON ps.sign_in_with_challenge_function_id = sign_in_fn.id + LEFT JOIN metaschema_public.schema s + ON ps.schema_id = s.id + AND s.database_id = ps.database_id + LEFT JOIN metaschema_public.function sign_up_fn + ON ps.sign_up_with_key_function_id = sign_up_fn.id + AND sign_up_fn.database_id = ps.database_id + AND sign_up_fn.schema_id = ps.schema_id + LEFT JOIN metaschema_public.function sign_in_req_fn + ON ps.sign_in_request_challenge_function_id = sign_in_req_fn.id + AND sign_in_req_fn.database_id = ps.database_id + AND sign_in_req_fn.schema_id = ps.schema_id + LEFT JOIN metaschema_public.function sign_in_fail_fn + ON ps.sign_in_record_failure_function_id = sign_in_fail_fn.id + AND sign_in_fail_fn.database_id = ps.database_id + AND sign_in_fail_fn.schema_id = ps.schema_id + LEFT JOIN metaschema_public.function sign_in_fn + ON ps.sign_in_with_challenge_function_id = sign_in_fn.id + AND sign_in_fn.database_id = ps.database_id + AND sign_in_fn.schema_id = ps.schema_id WHERE ps.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -45,7 +58,18 @@ interface PubkeySettingsRow { // ─── Transforms ───────────────────────────────────────────────────────────── function fromRow(row: PubkeySettingsRow | null): PubkeyChallengeSettings | undefined { - if (!row?.schema || !row?.sign_up_with_key) return undefined; + if (!row) return undefined; + const required = [ + row.schema, + row.crypto_network, + row.sign_up_with_key, + row.sign_in_request_challenge, + row.sign_in_record_failure, + row.sign_in_with_challenge + ]; + if (required.some((value) => typeof value !== 'string' || value.length === 0)) { + throw new Error('Incomplete or cross-database public-key authentication configuration'); + } return { schema: row.schema, cryptoNetwork: row.crypto_network, @@ -60,10 +84,14 @@ function fromRow(row: PubkeySettingsRow | null): PubkeyChallengeSettings | undef export const pubkeyLoader: ModuleLoader = createModuleLoader({ name: 'pubkeyChallengeSettings', - ttlMs: 5 * 60_000, + // Public-key authentication policy must be authoritative per request. + cache: false, async resolve(ctx: LoaderContext) { const { routingPool, databaseId } = ctx; const result = await routingPool.query(pubkeySettingsSql(routingSchemaOf(ctx)), [databaseId]); + if (result.rows.length > 1) { + throw new Error('Ambiguous public-key authentication configuration'); + } return fromRow(result.rows[0] ?? null); } }); diff --git a/packages/express-context/src/loaders/registry.ts b/packages/express-context/src/loaders/registry.ts index d4d8701eff..c632cc6c81 100644 --- a/packages/express-context/src/loaders/registry.ts +++ b/packages/express-context/src/loaders/registry.ts @@ -40,8 +40,8 @@ export interface LoaderRegistry { /** Check whether a loader is registered. */ has(name: string): boolean; - /** Invalidate caches for one database (or all databases if omitted). */ - invalidate(databaseId?: string): void; + /** Invalidate caches for one database, optionally limited to an exact pool pair. */ + invalidate(databaseId?: string, context?: LoaderContext): void; /** List all registered loader names. */ readonly names: string[]; @@ -96,9 +96,9 @@ export function createLoaderRegistry(): LoaderRegistry { return loaders.has(name); }, - invalidate(databaseId?: string): void { + invalidate(databaseId?: string, context?: LoaderContext): void { for (const loader of loaders.values()) { - loader.invalidate(databaseId); + loader.invalidate(databaseId, context); } log.debug( databaseId diff --git a/packages/express-context/src/loaders/rls.ts b/packages/express-context/src/loaders/rls.ts index 6fa7aa4425..c4a1e08c2a 100644 --- a/packages/express-context/src/loaders/rls.ts +++ b/packages/express-context/src/loaders/rls.ts @@ -24,16 +24,37 @@ const rlsSettingsSql = (schema: string): string => ` ua_fn.name AS current_user_agent, ip_fn.name AS current_ip_address FROM "${schema}".rls_settings rs - LEFT JOIN metaschema_public.schema auth_schema ON rs.authenticate_schema_id = auth_schema.id - LEFT JOIN metaschema_public.schema role_schema ON rs.role_schema_id = role_schema.id - LEFT JOIN metaschema_public.function auth_fn ON rs.authenticate_function_id = auth_fn.id - LEFT JOIN metaschema_public.function auth_strict_fn ON rs.authenticate_strict_function_id = auth_strict_fn.id - LEFT JOIN metaschema_public.function role_fn ON rs.current_role_function_id = role_fn.id - LEFT JOIN metaschema_public.function role_id_fn ON rs.current_role_id_function_id = role_id_fn.id - LEFT JOIN metaschema_public.function ua_fn ON rs.current_user_agent_function_id = ua_fn.id - LEFT JOIN metaschema_public.function ip_fn ON rs.current_ip_address_function_id = ip_fn.id + LEFT JOIN metaschema_public.schema auth_schema + ON rs.authenticate_schema_id = auth_schema.id + AND auth_schema.database_id = rs.database_id + LEFT JOIN metaschema_public.schema role_schema + ON rs.role_schema_id = role_schema.id + AND role_schema.database_id = rs.database_id + LEFT JOIN metaschema_public.function auth_fn + ON rs.authenticate_function_id = auth_fn.id + AND auth_fn.database_id = rs.database_id + AND auth_fn.schema_id = rs.authenticate_schema_id + LEFT JOIN metaschema_public.function auth_strict_fn + ON rs.authenticate_strict_function_id = auth_strict_fn.id + AND auth_strict_fn.database_id = rs.database_id + AND auth_strict_fn.schema_id = rs.authenticate_schema_id + LEFT JOIN metaschema_public.function role_fn + ON rs.current_role_function_id = role_fn.id + AND role_fn.database_id = rs.database_id + AND role_fn.schema_id = rs.role_schema_id + LEFT JOIN metaschema_public.function role_id_fn + ON rs.current_role_id_function_id = role_id_fn.id + AND role_id_fn.database_id = rs.database_id + AND role_id_fn.schema_id = rs.role_schema_id + LEFT JOIN metaschema_public.function ua_fn + ON rs.current_user_agent_function_id = ua_fn.id + AND ua_fn.database_id = rs.database_id + AND ua_fn.schema_id = rs.role_schema_id + LEFT JOIN metaschema_public.function ip_fn + ON rs.current_ip_address_function_id = ip_fn.id + AND ip_fn.database_id = rs.database_id + AND ip_fn.schema_id = rs.role_schema_id WHERE rs.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -53,7 +74,18 @@ interface RlsSettingsRow { function fromSettings(row: RlsSettingsRow | null): RlsModule | undefined { if (!row) return undefined; - if (!row.authenticate || !row.authenticate_schema) return undefined; + const required = [ + row.authenticate, + row.authenticate_schema, + row.role_schema, + row.current_role, + row.current_role_id, + row.current_ip_address, + row.current_user_agent + ]; + if (required.some((value) => typeof value !== 'string' || value.length === 0)) { + throw new Error('Incomplete or cross-database RLS module configuration'); + } return { authenticate: row.authenticate, authenticateStrict: row.authenticate_strict, @@ -70,10 +102,16 @@ function fromSettings(row: RlsSettingsRow | null): RlsModule | undefined { export const rlsLoader: ModuleLoader = createModuleLoader({ name: 'rlsModule', - ttlMs: 5 * 60_000, + // Authentication routing is an authorization boundary. Resolve it from the + // routing plane on every request; LISTEN notifications and TTLs are not an + // acceptable revocation mechanism because notifications can be missed. + cache: false, async resolve(ctx: LoaderContext) { const { routingPool, databaseId } = ctx; const result = await routingPool.query(rlsSettingsSql(routingSchemaOf(ctx)), [databaseId]); + if (result.rows.length > 1) { + throw new Error('Ambiguous RLS module configuration'); + } return fromSettings(result.rows[0] ?? null); } }); diff --git a/packages/express-context/src/loaders/storage.ts b/packages/express-context/src/loaders/storage.ts new file mode 100644 index 0000000000..f20859ab8f --- /dev/null +++ b/packages/express-context/src/loaders/storage.ts @@ -0,0 +1,157 @@ +/** + * Storage Module Loader + * + * Resolves immutable storage-module routing metadata through the privileged + * control-plane tenant pool. Graphile receives the normalized descriptors at + * build time, so its least-privilege runtime pool never reads metaschema + * configuration with `withPgClient(null)`. + */ + +import { quoteQualifiedSqlIdentifier } from '../sql-identifiers'; +import type { StorageConfig, StorageModuleConfig } from '../types'; +import { createModuleLoader } from './create-loader'; +import type { LoaderContext, ModuleLoader } from './types'; + +const DEFAULT_UPLOAD_URL_EXPIRY_SECONDS = 900; +const DEFAULT_DOWNLOAD_URL_EXPIRY_SECONDS = 3600; +const DEFAULT_MAX_FILE_SIZE = 200 * 1024 * 1024; +const DEFAULT_MAX_FILENAME_LENGTH = 1024; +const DEFAULT_CACHE_TTL_SECONDS = process.env.NODE_ENV === 'development' ? 300 : 3600; +const DEFAULT_MAX_BULK_FILES = 100; +const DEFAULT_MAX_BULK_TOTAL_SIZE = 1024 * 1024 * 1024; + +export const STORAGE_MODULE_SQL = ` + SELECT + sm.id, + sm.scope, + sm.entity_table_id, + bs.schema_name AS buckets_schema, + bt.name AS buckets_table, + fs.schema_name AS files_schema, + ft.name AS files_table, + sm.endpoint, + sm.public_url_prefix, + sm.provider, + sm.allowed_origins, + sm.upload_url_expiry_seconds, + sm.download_url_expiry_seconds, + sm.default_max_file_size, + sm.max_filename_length, + sm.cache_ttl_seconds, + sm.max_bulk_files, + sm.max_bulk_total_size, + sm.has_path_shares, + es.schema_name AS entity_schema, + et.name AS entity_table + FROM metaschema_modules_public.storage_module sm + JOIN metaschema_public.table bt + ON bt.id = sm.buckets_table_id + AND bt.database_id = sm.database_id + JOIN metaschema_public.schema bs + ON bs.id = bt.schema_id + AND bs.database_id = sm.database_id + JOIN metaschema_public.table ft + ON ft.id = sm.files_table_id + AND ft.database_id = sm.database_id + JOIN metaschema_public.schema fs + ON fs.id = ft.schema_id + AND fs.database_id = sm.database_id + LEFT JOIN metaschema_public.table et + ON et.id = sm.entity_table_id + AND et.database_id = sm.database_id + LEFT JOIN metaschema_public.schema es + ON es.id = et.schema_id + AND es.database_id = sm.database_id + WHERE sm.database_id = $1 + ORDER BY sm.scope, sm.id +`; + +interface StorageModuleRow { + id: string; + scope: string; + entity_table_id: string | null; + buckets_schema: string; + buckets_table: string; + files_schema: string; + files_table: string; + endpoint: string | null; + public_url_prefix: string | null; + provider: string | null; + allowed_origins: string[] | null; + upload_url_expiry_seconds: number | string | null; + download_url_expiry_seconds: number | string | null; + default_max_file_size: number | string | null; + max_filename_length: number | string | null; + cache_ttl_seconds: number | string | null; + max_bulk_files: number | string | null; + max_bulk_total_size: number | string | null; + has_path_shares: boolean | null; + entity_schema: string | null; + entity_table: string | null; +} + +const numberOr = (value: number | string | null, fallback: number): number => { + if (value == null) return fallback; + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 0) { + throw new Error(`Invalid storage module numeric setting '${value}'`); + } + return parsed; +}; + +export const normalizeStorageModule = (row: StorageModuleRow): StorageModuleConfig => ({ + id: row.id, + bucketsQualifiedName: quoteQualifiedSqlIdentifier( + row.buckets_schema, + row.buckets_table, + 'storage buckets table' + ), + filesQualifiedName: quoteQualifiedSqlIdentifier( + row.files_schema, + row.files_table, + 'storage files table' + ), + schemaName: row.buckets_schema, + bucketsTableName: row.buckets_table, + filesTableName: row.files_table, + scope: row.scope, + entityTableId: row.entity_table_id, + entityQualifiedName: row.entity_schema && row.entity_table + ? quoteQualifiedSqlIdentifier( + row.entity_schema, + row.entity_table, + 'storage entity table' + ) + : null, + endpoint: row.endpoint, + publicUrlPrefix: row.public_url_prefix, + provider: row.provider, + allowedOrigins: row.allowed_origins, + uploadUrlExpirySeconds: numberOr( + row.upload_url_expiry_seconds, + DEFAULT_UPLOAD_URL_EXPIRY_SECONDS + ), + downloadUrlExpirySeconds: numberOr( + row.download_url_expiry_seconds, + DEFAULT_DOWNLOAD_URL_EXPIRY_SECONDS + ), + defaultMaxFileSize: numberOr(row.default_max_file_size, DEFAULT_MAX_FILE_SIZE), + maxFilenameLength: numberOr(row.max_filename_length, DEFAULT_MAX_FILENAME_LENGTH), + cacheTtlSeconds: numberOr(row.cache_ttl_seconds, DEFAULT_CACHE_TTL_SECONDS), + hasPathShares: row.has_path_shares ?? false, + maxBulkFiles: numberOr(row.max_bulk_files, DEFAULT_MAX_BULK_FILES), + maxBulkTotalSize: numberOr(row.max_bulk_total_size, DEFAULT_MAX_BULK_TOTAL_SIZE) +}); + +export const storageLoader: ModuleLoader = createModuleLoader({ + name: 'storage', + ttlMs: 60_000, + async resolve(ctx: LoaderContext) { + const result = await ctx.tenantPool.query( + STORAGE_MODULE_SQL, + [ctx.databaseId] + ); + if (result.rows.length === 0) return undefined; + return { modules: result.rows.map(normalizeStorageModule) }; + } +}); diff --git a/packages/express-context/src/loaders/types.ts b/packages/express-context/src/loaders/types.ts index cec903d7b3..b41d9d39aa 100644 --- a/packages/express-context/src/loaders/types.ts +++ b/packages/express-context/src/loaders/types.ts @@ -1,9 +1,8 @@ /** * Module Loader Types * - * A ModuleLoader is a per-database cached lookup that resolves config - * from the routing DB or tenant DB. Each loader owns its own LRU cache - * keyed by databaseId, with independent TTL and eviction. + * A ModuleLoader resolves per-database config from the routing DB or tenant + * DB. Each loader chooses authoritative reads or an independent hard-TTL LRU. * * Loaders are registered in a LoaderRegistry and resolved in parallel * during context building. The result is a typed modules map on @@ -56,10 +55,14 @@ export function requireDatabaseId( export interface LoaderContext { /** Routing/configuration database pool (for routing-plane lookups) */ routingPool: Pool; + /** Opaque identity of the exact routing-pool connection contract. */ + routingPoolIdentity?: string; /** Routing-plane schema to query (defaults to the published routing_public) */ routingSchema?: string; /** Tenant database pool (for metaschema_modules_public.* lookups) */ tenantPool: Pool; + /** Opaque identity of the exact tenant control-pool connection contract. */ + tenantPoolIdentity?: string; /** UUID of the database being resolved */ databaseId: string; /** UUID of the API (if resolved from domain/api-name lookup) */ @@ -69,16 +72,19 @@ export interface LoaderContext { } /** - * A single module loader. Encapsulates the SQL query, type transform, - * and per-databaseId LRU cache for one piece of per-database config. + * A single module loader. Encapsulates the SQL query, type transform, and + * freshness policy for one piece of per-database config. */ export interface ModuleLoader { /** Unique name (used in log prefix and as the key in the modules map) */ readonly name: string; /** Resolve the module config for a given database. Returns undefined if not provisioned. */ resolve(ctx: LoaderContext): Promise; - /** Invalidate the cache for one database (or all databases if omitted) */ - invalidate(databaseId?: string): void; + /** + * Invalidate one logical database across all physical pools, or only the + * exact pool pair represented by `context`. Omitting both clears everything. + */ + invalidate(databaseId?: string, context?: LoaderContext): void; /** Current number of cached entries */ readonly cacheSize: number; } diff --git a/packages/express-context/src/loaders/webauthn.ts b/packages/express-context/src/loaders/webauthn.ts index 2d1642756f..0921235674 100644 --- a/packages/express-context/src/loaders/webauthn.ts +++ b/packages/express-context/src/loaders/webauthn.ts @@ -26,12 +26,19 @@ const webauthnSettingsSql = (schema: string): string => ` ws.resident_key, ws.challenge_expiry_seconds FROM "${schema}".webauthn_settings ws - LEFT JOIN metaschema_public.schema s ON ws.schema_id = s.id - LEFT JOIN metaschema_public.schema cred_s ON ws.credentials_schema_id = cred_s.id - LEFT JOIN metaschema_public.schema sess_s ON ws.sessions_schema_id = sess_s.id - LEFT JOIN metaschema_public.schema sec_s ON ws.session_secrets_schema_id = sec_s.id + LEFT JOIN metaschema_public.schema s + ON ws.schema_id = s.id + AND s.database_id = ws.database_id + LEFT JOIN metaschema_public.schema cred_s + ON ws.credentials_schema_id = cred_s.id + AND cred_s.database_id = ws.database_id + LEFT JOIN metaschema_public.schema sess_s + ON ws.sessions_schema_id = sess_s.id + AND sess_s.database_id = ws.database_id + LEFT JOIN metaschema_public.schema sec_s + ON ws.session_secrets_schema_id = sec_s.id + AND sec_s.database_id = ws.database_id WHERE ws.database_id = $1 - LIMIT 1 `; // ─── Row Types ────────────────────────────────────────────────────────────── @@ -54,13 +61,36 @@ interface WebauthnSettingsRow { export const webauthnLoader: ModuleLoader = createModuleLoader({ name: 'webauthnSettings', - ttlMs: 5 * 60_000, + // RP/origin/verification policy revocation must take effect immediately. + cache: false, async resolve(ctx: LoaderContext) { const { routingPool, databaseId } = ctx; const result = await routingPool.query(webauthnSettingsSql(routingSchemaOf(ctx)), [databaseId]); + if (result.rows.length > 1) { + throw new Error('Ambiguous WebAuthn configuration'); + } const row = result.rows[0]; - if (!row?.schema) return undefined; + if (!row) return undefined; + const required = [ + row.schema, + row.credentials_schema, + row.sessions_schema, + row.session_secrets_schema, + row.rp_id, + row.rp_name, + row.attestation_type, + row.resident_key + ]; + if ( + required.some((value) => typeof value !== 'string' || value.length === 0) + || !Array.isArray(row.origin_allowlist) + || typeof row.require_user_verification !== 'boolean' + || !Number.isSafeInteger(row.challenge_expiry_seconds) + || row.challenge_expiry_seconds <= 0 + ) { + throw new Error('Incomplete or cross-database WebAuthn configuration'); + } return { schema: row.schema, diff --git a/packages/express-context/src/pg-settings.ts b/packages/express-context/src/pg-settings.ts index cb86336456..82daddc8f5 100644 --- a/packages/express-context/src/pg-settings.ts +++ b/packages/express-context/src/pg-settings.ts @@ -12,6 +12,56 @@ import type { ApiStructure, ConstructiveAPIToken } from './types'; +export const SECURITY_GUC_KEYS = [ + 'jwt.claims.access_level', + 'jwt.claims.api_id', + 'jwt.claims.database_id', + 'jwt.claims.device_token', + 'jwt.claims.email', + 'jwt.claims.entity_id', + 'jwt.claims.ip_address', + 'jwt.claims.kind', + 'jwt.claims.organization_id', + 'jwt.claims.origin', + 'jwt.claims.principal_id', + 'jwt.claims.role_type', + 'jwt.claims.session_id', + 'jwt.claims.tenant_id', + 'jwt.claims.token_id', + 'jwt.claims.user_agent', + 'jwt.claims.user_email', + 'jwt.claims.user_id' +] as const; + +export type SecurityGucKey = typeof SECURITY_GUC_KEYS[number]; + +const SECURITY_GUC_KEY_SET: ReadonlySet = new Set(SECURITY_GUC_KEYS); + +const applyTrustedClaims = ( + settings: Record, + trustedClaims: PgSettingsInput['trustedClaims'] +): void => { + if (trustedClaims === undefined) return; + if ( + typeof trustedClaims !== 'object' + || trustedClaims === null + || Array.isArray(trustedClaims) + ) { + throw new TypeError('trustedClaims must be an object of security GUC strings'); + } + + for (const key of Reflect.ownKeys(trustedClaims)) { + if (typeof key !== 'string' || !SECURITY_GUC_KEY_SET.has(key)) { + throw new TypeError(`trustedClaims contains unsupported security GUC '${String(key)}'`); + } + const descriptor = Object.getOwnPropertyDescriptor(trustedClaims, key); + if (!descriptor || !('value' in descriptor) || typeof descriptor.value !== 'string') { + throw new TypeError(`trustedClaims.${key} must be a string data property`); + } + settings[key] = descriptor.value; + } +}; + export interface PgSettingsInput { /** Resolved API config (provides role names, database_id) */ api: ApiStructure; @@ -21,8 +71,21 @@ export interface PgSettingsInput { requestId: string; /** Client IP address (from request-ip middleware) */ clientIp?: string; + /** Origin header captured by the server */ + origin?: string; + /** User-Agent header captured by the server */ + userAgent?: string; + /** Trusted device cookie resolved by authentication middleware */ + deviceToken?: string; + /** Server-derived claims for trusted private surfaces */ + trustedClaims?: Partial>; + /** Ordered, audited extension/shared schemas needed for runtime operators and functions. */ + dependencySchemas?: readonly string[]; } +const quoteIdentifier = (identifier: string): string => + `"${identifier.replace(/"/g, '""')}"`; + /** * Build pgSettings from the resolved API + auth token. * @@ -30,8 +93,10 @@ export interface PgSettingsInput { * making them available to RLS policies and SQL functions. */ export function buildPgSettings(input: PgSettingsInput): Record { - const { api, token, requestId, clientIp } = input; - const settings: Record = {}; + const { api, token, requestId, clientIp, origin, userAgent, deviceToken } = input; + const settings: Record = Object.fromEntries( + SECURITY_GUC_KEYS.map((key) => [key, '']) + ); // Role: from token (authenticated) or api (anonymous fallback) if (token?.user_id) { @@ -41,14 +106,24 @@ export function buildPgSettings(input: PgSettingsInput): Record settings['role'] = api.anonRole || 'anonymous'; } + if (token?.id) settings['jwt.claims.token_id'] = token.id; + if (token?.access_level) settings['jwt.claims.access_level'] = token.access_level; + if (token?.kind) settings['jwt.claims.kind'] = token.kind; + if (typeof token?.email === 'string') settings['jwt.claims.email'] = token.email; + if (typeof token?.user_email === 'string') settings['jwt.claims.user_email'] = token.user_email; + if (typeof token?.entity_id === 'string') settings['jwt.claims.entity_id'] = token.entity_id; + if (typeof token?.organization_id === 'string') settings['jwt.claims.organization_id'] = token.organization_id; + if (typeof token?.tenant_id === 'string') settings['jwt.claims.tenant_id'] = token.tenant_id; + if (typeof token?.role_type === 'string') settings['jwt.claims.role_type'] = token.role_type; + // Session claims if (token?.session_id) { settings['jwt.claims.session_id'] = token.session_id; } // Principal identity (service accounts / bots) - if (token?.principal_id) { - settings['jwt.claims.principal_id'] = token.principal_id; + if (token?.principal_id || token?.user_id) { + settings['jwt.claims.principal_id'] = token.principal_id || token.user_id || ''; } // Database context @@ -72,5 +147,27 @@ export function buildPgSettings(input: PgSettingsInput): Record settings['jwt.claims.ip_address'] = clientIp; } + if (origin) settings['jwt.claims.origin'] = origin; + if (userAgent) settings['jwt.claims.user_agent'] = userAgent; + if (deviceToken) settings['jwt.claims.device_token'] = deviceToken; + // This is an exported boundary and TypeScript types do not constrain runtime + // objects. Reject extra keys/accessors so a future caller cannot smuggle + // role, search_path, or other session state through this trusted seam. + applyTrustedClaims(settings, input.trustedClaims); + + // Explicitly undo read-only state inherited from a previous request. + settings['transaction_read_only'] = token?.access_level === 'read_only' ? 'on' : 'off'; + // Pin name resolution after SET ROLE. DISCARD ALL resets to role/database + // defaults, which are mutable control-plane state and must not route a + // request into an unapproved schema. + settings['search_path'] = [ + 'pg_catalog', + ...[...new Set(input.dependencySchemas ?? [])].map(quoteIdentifier), + ...api.schema.map(quoteIdentifier) + ].join(', '); + // Owners and BYPASSRLS logins are rejected separately, but this makes the + // intended RLS state explicit for every transaction and clears prior state. + settings['row_security'] = 'on'; + return settings; } diff --git a/packages/express-context/src/sql-identifiers.ts b/packages/express-context/src/sql-identifiers.ts new file mode 100644 index 0000000000..1850cdcee3 --- /dev/null +++ b/packages/express-context/src/sql-identifiers.ts @@ -0,0 +1,31 @@ +import { QuoteUtils } from '@pgsql/quotes'; + +const POSTGRES_IDENTIFIER_MAX_BYTES = 63; + +/** + * Quote a metadata-derived PostgreSQL identifier without accepting values that + * PostgreSQL would truncate or that cannot be identifiers at all. Request data + * must still be passed as query parameters. + */ +export const quoteSqlIdentifier = ( + identifier: string, + label = 'SQL identifier' +): string => { + if ( + typeof identifier !== 'string' + || identifier.length === 0 + || identifier.includes('\0') + || Buffer.byteLength(identifier, 'utf8') > POSTGRES_IDENTIFIER_MAX_BYTES + ) { + throw new Error(`Invalid ${label}`); + } + const quoted = QuoteUtils.quoteIdentifier(identifier); + return quoted.startsWith('"') ? quoted : `"${quoted}"`; +}; + +export const quoteQualifiedSqlIdentifier = ( + schema: string, + object: string, + label = 'qualified SQL identifier' +): string => + `${quoteSqlIdentifier(schema, `${label} schema`)}.${quoteSqlIdentifier(object, `${label} object`)}`; diff --git a/packages/express-context/src/types.ts b/packages/express-context/src/types.ts index 4316018209..9f479626b4 100644 --- a/packages/express-context/src/types.ts +++ b/packages/express-context/src/types.ts @@ -204,12 +204,54 @@ export interface ComputeModuleConfig { invocationsEntityField: string | null; } +export interface ComputeBindingConfig { + bindingId: string; + alias: string; + config: Record | null; + functionDefinitionId: string; + taskIdentifier: string; + description: string | null; + payloadArgs: Array<{ name: string; type: string }> | null; + module: ComputeModuleConfig; +} + /** * All function modules provisioned on the database. A database may have one * per scope; every module is exposed and RLS governs access to each. */ export interface ComputeConfig { modules: ComputeModuleConfig[]; + /** API-scoped binding metadata loaded with the control-plane tenant pool. */ + bindings: ComputeBindingConfig[]; +} + +/** Immutable storage-module routing metadata loaded through the control plane. */ +export interface StorageModuleConfig { + id: string; + bucketsQualifiedName: string; + filesQualifiedName: string; + schemaName: string; + bucketsTableName: string; + filesTableName: string; + scope: string; + entityTableId: string | null; + entityQualifiedName: string | null; + endpoint: string | null; + publicUrlPrefix: string | null; + provider: string | null; + allowedOrigins: string[] | null; + uploadUrlExpirySeconds: number; + downloadUrlExpirySeconds: number; + defaultMaxFileSize: number; + maxFilenameLength: number; + cacheTtlSeconds: number; + hasPathShares: boolean; + maxBulkFiles: number; + maxBulkTotalSize: number; +} + +export interface StorageConfig { + modules: StorageModuleConfig[]; } export interface LlmConfig { @@ -249,6 +291,7 @@ export interface BuiltinModuleMap { agentChat: AgentChatConfig; llm: LlmConfig; compute: ComputeConfig; + storage: StorageConfig; } // ─── Constructive Context ─────────────────────────────────────────────────── @@ -280,14 +323,16 @@ export interface ConstructiveContext { requestId: string; /** Tenant database connection pool */ pool: Pool; + /** Opaque exact identity of the tenant execution pool. */ + runtimePoolIdentity: string; /** Execute a function within a tenant-scoped RLS transaction */ withPgClient: WithPgClient; /** - * Resolve a per-database module on demand (lazy, cached). + * Resolve a per-database module on demand. * - * Only fires the SQL query on the first call per databaseId per TTL window. - * Subsequent calls return the cached result instantly. + * Each loader owns its freshness policy. Security-sensitive built-ins read + * authoritatively on every call; nonsecurity loaders may use a hard TTL. * * Built-in modules are typed: * const rls = await ctx.useModule('rlsModule'); // RlsModule | undefined @@ -340,6 +385,9 @@ declare global { clientIp?: string; requestId?: string; token?: ConstructiveAPIToken; + deviceToken?: string; + /** Set by the GraphQL ingress after authenticating reserved internal headers. */ + internalTrusted?: boolean; constructive?: ConstructiveContext; } } diff --git a/packages/server-utils/src/__tests__/lru.test.ts b/packages/server-utils/src/__tests__/lru.test.ts new file mode 100644 index 0000000000..ba391ef3a2 --- /dev/null +++ b/packages/server-utils/src/__tests__/lru.test.ts @@ -0,0 +1,97 @@ +import { + configureSvcCache, + DEFAULT_SVC_CACHE_MAX_ENTRIES, + getSvcCacheStats, + resetSvcCacheCounters, + resolveSvcCacheMaxEntries, + svcCache +} from '../lru'; + +describe('routing service cache', () => { + beforeEach(() => { + svcCache.clear(); + configureSvcCache({ maxEntries: DEFAULT_SVC_CACHE_MAX_ENTRIES }); + resetSvcCacheCounters(); + }); + + afterEach(() => { + svcCache.clear(); + configureSvcCache({ maxEntries: DEFAULT_SVC_CACHE_MAX_ENTRIES }); + resetSvcCacheCounters(); + }); + + it('uses at least the required resident capacity by default', () => { + expect(resolveSvcCacheMaxEntries({ minimumEntries: 8 })).toBe( + DEFAULT_SVC_CACHE_MAX_ENTRIES + ); + expect(resolveSvcCacheMaxEntries({ minimumEntries: 2048 })).toBe(2048); + }); + + it('rejects an explicit capacity below the required resident floor', () => { + expect(() => resolveSvcCacheMaxEntries({ + maxEntries: 63, + minimumEntries: 64 + })).toThrow('must be at least the required minimum (64)'); + }); + + it.each([0, -1, 1.5, Number.NaN])( + 'rejects invalid capacity %s', + (maxEntries) => { + expect(() => resolveSvcCacheMaxEntries({ maxEntries })).toThrow( + 'must be a positive safe integer' + ); + } + ); + + it('reports lookups, capacity eviction, and current residency', () => { + configureSvcCache({ maxEntries: 2 }); + svcCache.set('label-a', { apiId: 'api-a' }); + svcCache.set('label-b', { apiId: 'api-b' }); + + expect(svcCache.get('label-a')).toEqual({ apiId: 'api-a' }); + expect(svcCache.get('missing')).toBeUndefined(); + svcCache.set('label-c', { apiId: 'api-c' }); + + expect(svcCache.has('label-a')).toBe(true); + expect(svcCache.has('label-b')).toBe(false); + expect(svcCache.has('label-c')).toBe(true); + expect(getSvcCacheStats()).toMatchObject({ + size: 2, + max: 2, + hits: 1, + misses: 1, + evictions: 1, + evictionsByReason: { + capacity: 1, + ttl: 0 + } + }); + }); + + it('does not classify explicit invalidation as cache pressure eviction', () => { + configureSvcCache({ maxEntries: 2 }); + svcCache.set('label-a', { apiId: 'api-a' }); + svcCache.delete('label-a'); + svcCache.set('label-b', { apiId: 'api-b' }); + svcCache.clear(); + + expect(getSvcCacheStats()).toMatchObject({ + size: 0, + evictions: 0, + evictionsByReason: { + capacity: 0, + ttl: 0 + } + }); + }); + + it('refuses to resize a live process cache instead of silently evicting metadata', () => { + svcCache.set('label-a', { apiId: 'api-a' }); + + expect(() => configureSvcCache({ maxEntries: 2048 })).toThrow( + 'cannot be reconfigured while routing metadata is resident' + ); + expect(svcCache.peek('label-a')).toEqual({ apiId: 'api-a' }); + expect(svcCache.max).toBe(DEFAULT_SVC_CACHE_MAX_ENTRIES); + }); +}); diff --git a/packages/server-utils/src/lru.ts b/packages/server-utils/src/lru.ts index 7f07b2ff53..e7c3b8281c 100644 --- a/packages/server-utils/src/lru.ts +++ b/packages/server-utils/src/lru.ts @@ -1,21 +1,204 @@ import { Logger } from '@pgpmjs/logger'; import { LRUCache } from 'lru-cache'; -const log = new Logger('pg-cache'); +const log = new Logger('routing-service-cache'); const ONE_HOUR_IN_MS = 1000 * 60 * 60; const ONE_DAY = ONE_HOUR_IN_MS * 24; const ONE_YEAR = ONE_DAY * 366; export const SVC_CACHE_TTL_MS = ONE_YEAR; +export const DEFAULT_SVC_CACHE_MAX_ENTRIES = 1024; -// --- Service Cache --- -// Keep max aligned with PG_CACHE_MAX and GRAPHILE_CACHE_MAX (default: 50) -export const svcCache = new LRUCache({ - max: 50, - ttl: SVC_CACHE_TTL_MS, - updateAgeOnGet: true, - dispose: (_, key) => { - log.debug(`Disposing service[${key}]`); +export type SvcCacheEvictionReason = 'capacity' | 'ttl'; + +export interface SvcCacheStats { + size: number; + max: number; + ttlMs: number; + hits: number; + misses: number; + evictions: number; + evictionsByReason: Record; + oldestKeyAgeMs: number | null; + keys: string[]; +} + +export interface ConfigureSvcCacheOptions { + /** Exact operator ceiling. Omit to use the safe process default. */ + maxEntries?: number; + /** Capacity floor imposed by the caller, such as resident Graphile capacity. */ + minimumEntries?: number; +} + +const assertPositiveSafeInteger = (value: number, label: string): void => { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${label} must be a positive safe integer`); + } +}; + +export const resolveSvcCacheMaxEntries = ({ + maxEntries, + minimumEntries = 1 +}: ConfigureSvcCacheOptions = {}): number => { + assertPositiveSafeInteger(minimumEntries, 'svcCache minimumEntries'); + if (maxEntries === undefined) { + return Math.max(DEFAULT_SVC_CACHE_MAX_ENTRIES, minimumEntries); + } + + assertPositiveSafeInteger(maxEntries, 'svcCache maxEntries'); + if (maxEntries < minimumEntries) { + throw new Error( + `svcCache maxEntries (${maxEntries}) must be at least the required minimum (${minimumEntries})` + ); + } + return maxEntries; +}; + +/** + * Process-wide routing-label metadata cache. + * + * This cache deliberately owns only resolved routing metadata. Capacity or TTL + * eviction never disposes a PostGraphile instance; a later request simply + * resolves the label again and reuses the independently keyed Graphile build. + */ +class RoutingServiceCache { + private cache: LRUCache; + private hits = 0; + private misses = 0; + private readonly evictionsByReason: Record = { + capacity: 0, + ttl: 0 + }; + + constructor(maxEntries: number) { + this.cache = this.createCache(maxEntries); + } + + private createCache(maxEntries: number): LRUCache { + return new LRUCache({ + max: maxEntries, + ttl: SVC_CACHE_TTL_MS, + updateAgeOnGet: true, + dispose: (_, key, reason) => { + if (reason === 'evict') { + this.evictionsByReason.capacity++; + log.debug(`Evicting routing metadata[${key}] (capacity)`); + } else if (reason === 'expire') { + this.evictionsByReason.ttl++; + log.debug(`Evicting routing metadata[${key}] (ttl)`); + } + } + }); } -}); \ No newline at end of file + + configure(maxEntries: number): void { + assertPositiveSafeInteger(maxEntries, 'svcCache maxEntries'); + if (maxEntries === this.cache.max) return; + if (this.cache.size > 0) { + throw new Error( + 'svcCache cannot be reconfigured while routing metadata is resident; clear it first' + ); + } + this.cache = this.createCache(maxEntries); + } + + get size(): number { + return this.cache.size; + } + + get max(): number { + return this.cache.max; + } + + get(key: string): T | undefined { + const value = this.cache.get(key); + if (value === undefined) this.misses++; + else this.hits++; + return value; + } + + /** Existence inspection is intentionally not counted as a request lookup. */ + has(key: string): boolean { + return this.cache.has(key); + } + + /** Non-mutating inspection that does not affect LRU age or lookup counters. */ + peek(key: string): T | undefined { + return this.cache.peek(key); + } + + set(key: string, value: T): this { + this.cache.set(key, value); + return this; + } + + delete(key: string): boolean { + return this.cache.delete(key); + } + + clear(): void { + this.cache.clear(); + } + + keys(): IterableIterator { + return this.cache.keys(); + } + + entries(): IterableIterator<[string, T]> { + return this.cache.entries(); + } + + getRemainingTTL(key: string): number { + return this.cache.getRemainingTTL(key); + } + + resetCounters(): void { + this.hits = 0; + this.misses = 0; + this.evictionsByReason.capacity = 0; + this.evictionsByReason.ttl = 0; + } + + getStats(maxKeys = 200): SvcCacheStats { + assertPositiveSafeInteger(maxKeys, 'svcCache stats maxKeys'); + let minRemaining = Infinity; + for (const key of this.cache.keys()) { + const remaining = this.cache.getRemainingTTL(key); + if (remaining < minRemaining) minRemaining = remaining; + } + const evictionsByReason = { ...this.evictionsByReason }; + + return { + size: this.cache.size, + max: this.cache.max, + ttlMs: SVC_CACHE_TTL_MS, + hits: this.hits, + misses: this.misses, + evictions: evictionsByReason.capacity + evictionsByReason.ttl, + evictionsByReason, + oldestKeyAgeMs: Number.isFinite(minRemaining) + ? Math.max(0, SVC_CACHE_TTL_MS - minRemaining) + : null, + keys: [...this.cache.keys()].slice(0, maxKeys) + }; + } +} + +export const svcCache = new RoutingServiceCache( + DEFAULT_SVC_CACHE_MAX_ENTRIES +); + +export const configureSvcCache = ( + options: ConfigureSvcCacheOptions = {} +): SvcCacheStats => { + svcCache.configure(resolveSvcCacheMaxEntries(options)); + return svcCache.getStats(); +}; + +export const getSvcCacheStats = (maxKeys = 200): SvcCacheStats => + svcCache.getStats(maxKeys); + +export const resetSvcCacheCounters = (): void => { + svcCache.resetCounters(); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 19d7b0a115..dc6de2bdd4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2531,6 +2531,9 @@ importers: '@pgpmjs/types': specifier: workspace:^ version: link:../../pgpm/types/dist + '@pgsql/quotes': + specifier: ^18.2.0 + version: 18.2.1 lru-cache: specifier: ^11.2.7 version: 11.2.7 diff --git a/postgres/pg-cache/README.md b/postgres/pg-cache/README.md index 8888375971..a2a0be4619 100644 --- a/postgres/pg-cache/README.md +++ b/postgres/pg-cache/README.md @@ -22,8 +22,11 @@ npm install pg-cache ## Features -- LRU cache for PostgreSQL connection pools +- Lease-aware LRU registry for PostgreSQL connection pools +- Fail-closed capacity admission for long-lived production consumers - Automatic pool cleanup and disposal +- Pool identity, lease, and disposal observability +- Checkout queue/sanitation timing and fast-path counters - Extensible cleanup callback system - Service cache for general use - Graceful shutdown handling @@ -52,6 +55,105 @@ const result = await pool.query('SELECT NOW()'); const samePool = getPgPool({ database: 'mydb' }); // Returns cached pool ``` +`getPgPool()` remains the synchronous compatibility API. A component that keeps +a pool across requests or asynchronous lifecycle boundaries should hold a lease: + +```typescript +import { acquirePgPool, PgPoolCapacityError } from 'pg-cache'; + +try { + const lease = acquirePgPool( + { database: 'tenant_a', user: 'graphql_runtime' }, + { purpose: 'runtime', sanitizeOnCheckout: true } + ); + + // Retain lease.pool for the owning handler or request lifecycle. + // release() is idempotent and must run only after the owner has drained. + lease.release(); +} catch (error) { + if (error instanceof PgPoolCapacityError) { + // Map error.code === 'PG_POOL_CAPACITY' to HTTP 503 and Retry-After. + } +} +``` + +Acquisition is synchronous and atomic. Reusing an existing exact identity costs +no new slot; a new identity evicts only an unleased pool. If every slot is +leased, admission throws before constructing a pool or ending an existing one. + +Sanitized default-driver pools issue `DISCARD ALL` before every reused checkout +and clear node-postgres/Graphile prepared-statement bookkeeping. The first +checkout of a brand-new factory-owned connection skips that redundant round +trip only when its trusted startup baseline is pinned and no other `connect` +listener could have changed the session. `getPgCheckoutSanitizerStats()` exposes +checkout wait, queue, sanitation, failure, and virgin-fast-path counters. +For alternate pool factories, pg-cache replaces direct `pool.query()` calls +with a sanitized checkout/query/release cycle; a custom sanitized pool must +therefore provide a replaceable `query()` method and Promise-based client +queries. Query failures destroy the checked-out client before the error is +returned through either the Promise or callback API. + +Pool sizing accepts `max`, `idleTimeoutMillis`, `connectionTimeoutMillis`, +`allowExitOnIdle`, and native pg-pool `maxUses`. Set `maxUses: 1` to retire a +client after every checkout, or leave it unset/use `0` for unlimited reuse. +The equivalent environment setting is `PG_POOL_MAX_USES`; it accepts only a +canonical decimal integer, so alternate numeric spellings, negative, +fractional, and unsafe-integer values fail before a pool identity is published. +Because `maxUses` changes connection lifecycle and latency, it participates in +the opaque exact-pool identity and should be benchmarked under the real request +rate before production use. + +Exact-pool and physical-target identities use a process-random keyed HMAC. They +are stable for the lifetime of one pool registry, but intentionally differ in +another process; this prevents an emitted identity from becoming an offline +password verifier. Connection and pool identity inputs must be primitive, +canonical data, and node-postgres password callbacks are rejected because a +callback's captured credential cannot be represented without risking an alias. + +### Shared Notification Broker + +`acquirePgNotificationBroker(listenerPgConfig, { topics })` leases one +process-local LISTEN connection per opaque, versioned pool identity. The +identity includes the canonical connection, credentials, pool, driver, +sanitation, and data-only TLS settings. The broker LISTENs only to each lease's +exact channel allowlist, rejects identifiers PostgreSQL would truncate past 63 +UTF-8 bytes, and awaits UNLISTEN plus connection and pool-lease release. +The final lease destroys its listener client after UNLISTEN, so an inactive +database does not retain an idle PostgreSQL backend until the pool timeout. + +Production broker acquisition performs a fresh, read-only catalog audit on the +same pinned client before each listener lease is admitted, so a pool with +`max: 1` cannot deadlock waiting for a second checkout. The attested lease's +narrow `revalidateRole()` capability serializes TTL audits on that client and +never exposes general SQL access. A conforming login may +CONNECT only to that exact database and has no role memberships, privileged +role attributes, database CREATE/TEMP, or effective privileges on non-system +schemas, relations, routines, and sequences. The returned audit contains only +the role, database, stable violation codes, and audit version; callers must keep +the separate `PgConfig` credentials in their secret store. Use +`normalizePgNotificationRoleContracts()` to reject one login spanning physical +databases or multiple listener logins targeting one physical database. +The live contract test is opt-in with +`PG_CACHE_RUN_NOTIFICATION_ROLE_INTEGRATION=1` and uses the standard `PG*` +connection settings for a pre-provisioned conforming notification login. + +Every role-audit, `LISTEN`, and `UNLISTEN` command has the same bounded deadline +as the listener pool's `connectionTimeoutMillis`. Configure it through the +listener's `pool.connectionTimeoutMillis`, or through +`PG_POOL_CONNECTION_TIMEOUT_MS` when the pool setting is omitted; the default is +5 seconds. The broker rejects zero, fractional, negative, or setTimeout-unsafe +values before publishing an identity. A command that exceeds the deadline +fatally closes every lease and destroys the pinned client, so TTL refresh and +shutdown cannot wait forever on an abandoned driver query. + +Each GraphQL subscriber has a fixed 256-message queue. A slow subscriber that +overflows is failed independently; a listener connection error fails every +lease and is never reconnected while any failed owner remains. The caller must +provide a dedicated least-privilege listener login and remains responsible for +the deployment's certificate and role policy. `lease.terminated` and +`getPgNotificationBrokerStats()` expose failure and lifecycle state without +revealing connection credentials. + ### Direct Cache Access ```typescript @@ -103,7 +205,7 @@ const service = svcCache.get('my-service'); ```typescript import { close, teardownPgPools } from 'pg-cache'; -// In your shutdown handler +// The executable owns process signals; importing pg-cache never installs one. process.on('SIGTERM', async () => { await close(); // or teardownPgPools() process.exit(0); @@ -119,14 +221,28 @@ The main PostgreSQL pool cache instance. - `get(key: string): Pool | undefined` - Get a cached pool - `set(key: string, pool: Pool): void` - Cache a pool - `has(key: string): boolean` - Check if a pool is cached -- `delete(key: string): void` - Remove and dispose a pool -- `clear(): void` - Remove and dispose all pools +- `delete(key: string): void` - Remove an unleased pool +- `clear(): void` - Remove all currently unleased pools +- `acquire(key: string, factory: () => Pool): PgPoolLease` - Atomically acquire a lease +- `getStats(): PgPoolCacheStats` - Read capacity and lifecycle counters - `registerCleanupCallback(callback: (key: string) => void): () => void` - Register a cleanup callback ### getPgPool(config: Partial): Pool Get or create a cached PostgreSQL pool using the provided configuration. +### acquirePgPool(config, options): PgPoolLease + +Get or create the exact pool identity and protect it from TTL/LRU disposal until +the returned idempotent `release()` is called. + +### Capacity + +`PG_CACHE_MAX` limits lazy pool identities, not eagerly allocated connections. +The default is 2064: two identities for each of 1024 database-per-tenant Graphile +contracts plus a 16-identity operational reserve. `PG_CACHE_TTL_MS` applies only +while an identity has zero leases. + ### svcCache A general-purpose LRU cache for services and objects. diff --git a/postgres/pg-cache/src/__tests__/driver.test.ts b/postgres/pg-cache/src/__tests__/driver.test.ts index dd155d538f..eacdd9e116 100644 --- a/postgres/pg-cache/src/__tests__/driver.test.ts +++ b/postgres/pg-cache/src/__tests__/driver.test.ts @@ -3,13 +3,17 @@ // lets an alternate backend (e.g. PGlite) plug in without any change to pgpm / // pgsql-* — and guarantees the default path is untouched when nothing registers. -import { randomUUID } from 'crypto'; +import { createHash, randomUUID } from 'crypto'; import pg from 'pg'; import { + acquirePgPool, defaultPgPoolFactory, getActivePgPoolFactory, getPgPool, + getPgPoolConfig, + getPgPoolDriverIdentity, + getPgPoolIdentity, hasPgPoolFactory, PgPoolFactory, registerPgPoolFactory @@ -55,10 +59,10 @@ describe('pg-cache pool-factory seam', () => { expect(factory).toHaveBeenCalledTimes(1); expect(pool).toBe(mock); - pgCache.delete(cfg.database); + pgCache.delete(getPgPoolIdentity(cfg)); }); - it('caches by database: a second call reuses the pool and does not re-invoke the factory', () => { + it('caches by exact pool identity: an identical call reuses the pool', () => { const cfg = freshConfig(); const factory = jest.fn(() => createMockPool()); registerPgPoolFactory(factory); @@ -69,7 +73,324 @@ describe('pg-cache pool-factory seam', () => { expect(first).toBe(second); expect(factory).toHaveBeenCalledTimes(1); - pgCache.delete(cfg.database); + pgCache.delete(getPgPoolIdentity(cfg)); + }); + + it('acquires an idempotently releasable lease for the exact pool identity', () => { + const cfg = freshConfig(); + const mock = createMockPool(); + const factory = jest.fn(() => mock); + registerPgPoolFactory(factory); + + const first = acquirePgPool(cfg, { purpose: 'runtime' }); + const second = acquirePgPool(cfg, { purpose: 'runtime' }); + + expect(first.identity).toBe(getPgPoolIdentity(cfg, { purpose: 'runtime' })); + expect(first.pool).toBe(mock); + expect(second.pool).toBe(mock); + expect(factory).toHaveBeenCalledTimes(1); + + first.release(); + first.release(); + second.release(); + pgCache.delete(first.identity); + }); + + it('separates credentials, purpose, and sanitation mode for the same database', () => { + const cfg = freshConfig(); + const factory = jest.fn(() => createMockPool()); + registerPgPoolFactory(factory); + + const control = getPgPool(cfg, { purpose: 'control' }); + const runtime = getPgPool({ ...cfg, user: 'runtime' }, { + purpose: 'runtime', + sanitizeOnCheckout: true + }); + const unsanitizedRuntime = getPgPool({ ...cfg, user: 'runtime' }, { + purpose: 'runtime' + }); + + expect(control).not.toBe(runtime); + expect(runtime).not.toBe(unsanitizedRuntime); + expect(factory).toHaveBeenCalledTimes(3); + + pgCache.delete(getPgPoolIdentity(cfg, { purpose: 'control' })); + pgCache.delete(getPgPoolIdentity({ ...cfg, user: 'runtime' }, { + purpose: 'runtime', + sanitizeOnCheckout: true + })); + pgCache.delete(getPgPoolIdentity({ ...cfg, user: 'runtime' }, { purpose: 'runtime' })); + }); + + it('normalizes maxUses into the exact pool identity', () => { + const cfg = freshConfig(); + const unlimited = getPgPoolIdentity(cfg); + const explicitUnlimited = getPgPoolIdentity({ ...cfg, pool: { maxUses: 0 } }); + const singleUse = getPgPoolIdentity({ ...cfg, pool: { maxUses: 1 } }); + const doubleUse = getPgPoolIdentity({ ...cfg, pool: { maxUses: 2 } }); + + expect(explicitUnlimited).toBe(unlimited); + expect(singleUse).not.toBe(unlimited); + expect(doubleUse).not.toBe(singleUse); + }); + + it('uses a process-keyed identity instead of an offline password verifier', () => { + const cfg = { + ...freshConfig(), + pool: { + max: 3, + idleTimeoutMillis: 1234, + connectionTimeoutMillis: 5678, + allowExitOnIdle: true + } + }; + const identity = getPgPoolIdentity(cfg, { + purpose: 'runtime', + sanitizeOnCheckout: true + }); + const unkeyedInput = JSON.stringify({ + version: 1, + driver: getPgPoolDriverIdentity(), + host: cfg.host, + port: cfg.port, + database: cfg.database, + user: cfg.user, + password: cfg.password, + ssl: null, + pool: { + max: 3, + maxUses: null, + idleTimeoutMillis: 1234, + connectionTimeoutMillis: 5678, + allowExitOnIdle: true + }, + purpose: 'runtime', + sanitizeOnCheckout: true + }); + const offlineDigest = `pg:v1:${createHash('sha256') + .update(unkeyedInput) + .digest('hex')}`; + + expect(getPgPoolIdentity(cfg, { + purpose: 'runtime', + sanitizeOnCheckout: true + })).toBe(identity); + expect(identity).toMatch(/^pg:v1:[a-f0-9]{64}$/); + expect(identity).not.toBe(offlineDigest); + }); + + it('rejects password callbacks before they can alias a pool identity', () => { + const cfg = freshConfig(); + const first = async () => 'first-secret'; + const second = async () => 'second-secret'; + const factory = jest.fn(() => createMockPool()); + registerPgPoolFactory(factory); + + for (const password of [first, second]) { + const invalid = { + ...cfg, + password: password as unknown as string + }; + expect(() => getPgPoolIdentity(invalid)).toThrow( + 'pg.password must be a string' + ); + expect(() => getPgPool(invalid)).toThrow( + 'pg.password must be a string' + ); + } + expect(factory).not.toHaveBeenCalled(); + }); + + it('rejects noncanonical exact-identity inputs', () => { + const cfg = freshConfig(); + const accessorSsl = {} as Record; + Object.defineProperty(accessorSsl, 'ca', { get: () => 'dynamic-ca' }); + const symbolSsl = { ca: 'tenant-ca' } as Record; + symbolSsl[Symbol('hidden')] = 'untracked'; + const sparseCa = new Array(2); + sparseCa[1] = 'tenant-ca'; + const undefinedSsl: { ca: undefined } = { ca: undefined }; + + expect(() => getPgPoolIdentity({ + ...cfg, + port: '5432' as unknown as number + })).toThrow('pg.port must be a safe integer'); + expect(() => getPgPoolIdentity({ + ...cfg, + pool: { max: '2' as unknown as number } + })).toThrow('pool.max must be a safe integer'); + expect(() => getPgPoolIdentity(cfg, { + purpose: {} as unknown as string + })).toThrow('pg pool purpose must be a non-empty string'); + expect(() => getPgPoolIdentity(cfg, { + sanitizeOnCheckout: 'false' as unknown as boolean + })).toThrow('pg pool sanitizeOnCheckout must be a boolean'); + expect(() => getPgPoolIdentity({ + ...cfg, + ssl: accessorSsl as never + })).toThrow('pg.ssl.ca must be a data property'); + expect(() => getPgPoolIdentity({ + ...cfg, + ssl: symbolSsl as never + })).toThrow('pg.ssl must not contain symbol properties'); + expect(() => getPgPoolIdentity({ + ...cfg, + ssl: { ca: sparseCa } as never + })).toThrow('pg.ssl.ca must be a dense array without custom properties'); + expect(() => getPgPoolIdentity({ + ...cfg, + ssl: undefinedSsl as never + })).toThrow('pg.ssl.ca must not be undefined'); + }); + + it('parses PG_POOL_MAX_USES as an unlimited sentinel or positive safe integer', () => { + const previous = process.env.PG_POOL_MAX_USES; + try { + process.env.PG_POOL_MAX_USES = '0'; + expect(getPgPoolConfig().maxUses).toBeUndefined(); + + process.env.PG_POOL_MAX_USES = '17'; + expect(getPgPoolConfig().maxUses).toBe(17); + + for (const invalid of [ + '-1', + '1.5', + '01', + '1e2', + '0x10', + ' 1', + '1 ', + ' ', + 'not-a-number', + '9007199254740992' + ]) { + process.env.PG_POOL_MAX_USES = invalid; + expect(() => getPgPoolConfig()).toThrow( + 'PG_POOL_MAX_USES must be 0 or a positive safe integer' + ); + } + } finally { + if (previous === undefined) delete process.env.PG_POOL_MAX_USES; + else process.env.PG_POOL_MAX_USES = previous; + } + }); + + it('validates explicit maxUses overrides before constructing an identity or pool', () => { + expect(getPgPoolConfig({ maxUses: 0 }).maxUses).toBeUndefined(); + expect(getPgPoolConfig({ maxUses: 23 }).maxUses).toBe(23); + + for (const invalid of [-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, 9007199254740992]) { + expect(() => getPgPoolConfig({ maxUses: invalid })).toThrow( + 'pool.maxUses must be 0 or a positive safe integer' + ); + } + for (const invalid of [true, null, {}]) { + expect(() => getPgPoolConfig({ + maxUses: invalid as unknown as number + })).toThrow('pool.maxUses must be 0 or a positive safe integer'); + } + }); + + it('replaces a sanitized custom factory query that bypasses connect', async () => { + const cfg = freshConfig(); + const queryResult = { rows: [{ value: 42 }] }; + const client = { + query: jest.fn(async (text: string) => text === 'SELECT $1::int AS value' + ? queryResult + : { rows: [] }), + release: jest.fn() + }; + const bypassingQuery = jest.fn(async () => ({ rows: [{ value: -1 }] })); + const connect = jest.fn(async () => client); + const pool = { + query: bypassingQuery, + connect, + end: jest.fn(async (): Promise => undefined) + } as unknown as pg.Pool; + const factory = jest.fn(() => pool); + registerPgPoolFactory(factory); + const options = { purpose: 'runtime', sanitizeOnCheckout: true } as const; + const identity = getPgPoolIdentity(cfg, options); + + try { + const sanitizedPool = getPgPool(cfg, options); + + await expect(sanitizedPool.query( + 'SELECT $1::int AS value', + [42] + )).resolves.toBe(queryResult); + expect(bypassingQuery).not.toHaveBeenCalled(); + expect(connect).toHaveBeenCalledTimes(1); + expect(client.query).toHaveBeenNthCalledWith(1, 'DISCARD ALL'); + expect(client.query).toHaveBeenNthCalledWith( + 2, + 'SET search_path TO pg_catalog; SET row_security TO on; SET jit_optimize_above_cost TO -1' + ); + expect(client.query).toHaveBeenNthCalledWith(3, 'SELECT $1::int AS value', [42]); + expect(client.release).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith(); + } finally { + pgCache.delete(identity); + await pgCache.waitForDisposals(); + } + }); + + it('separates TLS trust contracts for the same database and role', () => { + const cfg = freshConfig(); + + const verified = getPgPoolIdentity({ + ...cfg, + ssl: { ca: 'tenant-ca', rejectUnauthorized: true, servername: 'db.internal' } + }); + const insecure = getPgPoolIdentity({ + ...cfg, + ssl: { ca: 'tenant-ca', rejectUnauthorized: false, servername: 'db.internal' } + }); + const plaintext = getPgPoolIdentity(cfg); + + expect(verified).not.toBe(insecure); + expect(verified).not.toBe(plaintext); + expect(insecure).not.toBe(plaintext); + }); + + it('canonicalizes TLS data and rejects identity inputs JSON would omit', () => { + const cfg = freshConfig(); + const first = getPgPoolIdentity({ + ...cfg, + ssl: { ca: 'tenant-ca', rejectUnauthorized: true } + }); + const second = getPgPoolIdentity({ + ...cfg, + ssl: { rejectUnauthorized: true, ca: 'tenant-ca' } + }); + + expect(first).toBe(second); + expect(() => getPgPoolIdentity({ + ...cfg, + ssl: { checkServerIdentity: (): undefined => undefined } as any + })).toThrow('pg.ssl.checkServerIdentity must contain only deterministic data values'); + + const bufferIdentity = getPgPoolIdentity({ + ...cfg, + ssl: { ca: Buffer.from('tenant-ca') } + }); + const mimickedBufferIdentity = getPgPoolIdentity({ + ...cfg, + ssl: { + ca: { + bufferSha256: 'b60c1883ea3c4bf71a5959468ac16f36e2aa4f5c8702ca157fbbae61415f2f10' + } + } as any + }); + expect(bufferIdentity).not.toBe(mimickedBufferIdentity); + }); + + it('uses opaque identities that never disclose credentials', () => { + const cfg = { ...freshConfig(), password: 'top-secret-password' }; + const identity = getPgPoolIdentity(cfg, { purpose: 'runtime' }); + expect(identity).toMatch(/^pg:v1:[a-f0-9]{64}$/); + expect(identity).not.toContain(cfg.user); + expect(identity).not.toContain(cfg.password); }); it('falls back to defaultPgPoolFactory when nothing is registered', () => { @@ -78,7 +399,7 @@ describe('pg-cache pool-factory seam', () => { // query runs, so this is safe without a live server. const pool = getPgPool(cfg); expect(pool).toBeInstanceOf(pg.Pool); - pgCache.delete(cfg.database); + pgCache.delete(getPgPoolIdentity(cfg)); }); it('defaultPgPoolFactory returns a pg.Pool', () => { @@ -86,4 +407,60 @@ describe('pg-cache pool-factory seam', () => { expect(pool).toBeInstanceOf(pg.Pool); return pool.end(); }); + + it('passes credentials as discrete fields instead of reparsing them as a URI', async () => { + const cfg = { + ...freshConfig(), + user: 'runtime@tenant', + password: 'x@evil.example/other?sslmode=require', + database: 'tenant/database' + }; + const pool = defaultPgPoolFactory(cfg); + const options = (pool as pg.Pool & { options: pg.PoolConfig }).options; + + expect(options.host).toBe(cfg.host); + expect(options.port).toBe(cfg.port); + expect(options.database).toBe(cfg.database); + expect(options.user).toBe(cfg.user); + expect(options.password).toBe(cfg.password); + await pool.end(); + }); + + it('passes maxUses to the native pg.Pool driver', async () => { + const pool = defaultPgPoolFactory({ ...freshConfig(), pool: { maxUses: 1 } }); + const options = (pool as pg.Pool & { options: pg.PoolConfig }).options; + + expect(options.maxUses).toBe(1); + await pool.end(); + }); + + it('passes the exact TLS contract to node-postgres', async () => { + const ssl = { + ca: 'tenant-ca', + cert: 'runtime-cert', + key: 'runtime-key', + rejectUnauthorized: true, + servername: 'db.internal', + minVersion: 'TLSv1.2' as const + }; + const pool = defaultPgPoolFactory({ ...freshConfig(), ssl }); + const options = (pool as pg.Pool & { options: pg.PoolConfig }).options; + + expect(options.ssl).toEqual(ssl); + await pool.end(); + }); + + it('pins the trusted baseline in sanitized node-postgres startup options', async () => { + const pool = defaultPgPoolFactory(freshConfig(), { + purpose: 'runtime', + sanitizeOnCheckout: true + }); + const options = (pool as pg.Pool & { options: pg.PoolConfig }).options; + + expect(options.options).toContain('-c search_path=pg_catalog'); + expect(options.options).toContain('-c row_security=on'); + expect(options.options).toContain('-c jit_optimize_above_cost=-1'); + expect(pool.query).toBe(pg.Pool.prototype.query); + await pool.end(); + }); }); diff --git a/postgres/pg-cache/src/__tests__/lru.test.ts b/postgres/pg-cache/src/__tests__/lru.test.ts index a68afa616f..1a10500d88 100644 --- a/postgres/pg-cache/src/__tests__/lru.test.ts +++ b/postgres/pg-cache/src/__tests__/lru.test.ts @@ -1,15 +1,26 @@ -// Guards against the pg-cache close() resource leak fixed in feat/observability. -// -// Previously, close() reset this.closed = false after shutdown, allowing -// set() to silently accept new pools that were never cleaned up. The module- -// level closePromise also reset to null, enabling double-shutdown. -// -// These tests lock the fix: close() is final, set() rejects, and repeated -// close() calls are idempotent. See pg-cache-close-leak.md for full details. - import pg from 'pg'; -import { PgPoolCacheManager } from '../lru'; +import { + DEFAULT_PG_CACHE_MAX, + PG_CACHE_GRAPHILE_CONTRACT_CAPACITY, + PG_CACHE_OPERATIONAL_RESERVE, + PgPoolCacheManager, + PgPoolCapacityError +} from '../lru'; + +describe('process lifecycle ownership', () => { + it('does not install process signal handlers from a library import', () => { + const beforeSigterm = process.listenerCount('SIGTERM'); + const beforeSigint = process.listenerCount('SIGINT'); + + jest.isolateModules(() => { + jest.requireActual('../lru'); + }); + + expect(process.listenerCount('SIGTERM')).toBe(beforeSigterm); + expect(process.listenerCount('SIGINT')).toBe(beforeSigint); + }); +}); // Minimal mock — we only need pool.end() and pool.ended const createMockPool = (): pg.Pool => { @@ -45,8 +56,11 @@ describe('PgPoolCacheManager', () => { }); describe('configuration', () => { - it('uses env-var defaults (max=50) when no overrides given', () => { - expect(cache.config.max).toBe(50); + it('reserves two identities per supported Graphile contract plus operations', () => { + expect(DEFAULT_PG_CACHE_MAX).toBe( + PG_CACHE_GRAPHILE_CONTRACT_CAPACITY * 2 + PG_CACHE_OPERATIONAL_RESERVE + ); + expect(cache.config.max).toBe(2064); }); it('accepts constructor overrides', () => { @@ -90,6 +104,157 @@ describe('PgPoolCacheManager', () => { }); }); + describe('leases and fail-closed admission', () => { + it('counts an existing exact identity as zero new slots', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const pool = createMockPool(); + const factory = jest.fn(() => pool); + + const first = small.acquire('runtime-a', factory); + const second = small.acquire('runtime-a', factory); + + expect(first.pool).toBe(pool); + expect(second.pool).toBe(pool); + expect(factory).toHaveBeenCalledTimes(1); + expect(small.getStats()).toMatchObject({ + size: 1, + leasedPools: 1, + activeLeases: 2, + leasesAcquired: 2 + }); + + first.release(); + first.release(); + expect(small.getStats().activeLeases).toBe(1); + second.release(); + await small.close(); + }); + + it('refuses before constructing or ending when every slot is leased', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const firstPool = createMockPool(); + const first = small.acquire('runtime-a', () => firstPool); + const rejectedFactory = jest.fn(() => createMockPool()); + + let capacityError: PgPoolCapacityError | undefined; + try { + small.acquire('runtime-b', rejectedFactory); + } catch (error) { + capacityError = error as PgPoolCapacityError; + } + + expect(capacityError).toBeInstanceOf(PgPoolCapacityError); + expect(capacityError).toMatchObject({ + code: 'PG_POOL_CAPACITY', + retryAfterSeconds: 15, + max: 1, + size: 1, + leased: 1 + }); + expect(rejectedFactory).not.toHaveBeenCalled(); + expect(firstPool.end).not.toHaveBeenCalled(); + expect(small.getStats().capacityRefusals).toBe(1); + + first.release(); + await small.close(); + }); + + it('evicts only the least-recent zero-lease identity', async () => { + const small = new PgPoolCacheManager({ max: 2 }); + const leasedPool = createMockPool(); + const idlePool = createMockPool(); + const replacementPool = createMockPool(); + const lease = small.acquire('leased', () => leasedPool); + small.set('idle', idlePool); + + small.set('replacement', replacementPool); + await small.waitForDisposals(); + + expect(small.has('leased')).toBe(true); + expect(leasedPool.end).not.toHaveBeenCalled(); + expect(small.has('idle')).toBe(false); + expect(idlePool.end).toHaveBeenCalledTimes(1); + expect(small.has('replacement')).toBe(true); + + lease.release(); + await small.close(); + }); + + it('keeps an expired leased identity until release', async () => { + jest.useFakeTimers(); + const small = new PgPoolCacheManager({ max: 1, ttl: 50 }); + const pool = createMockPool(); + const lease = small.acquire('runtime', () => pool); + try { + jest.advanceTimersByTime(51); + expect(small.has('runtime')).toBe(true); + expect(pool.end).not.toHaveBeenCalled(); + + lease.release(); + await small.waitForDisposals(); + + expect(small.has('runtime')).toBe(false); + expect(pool.end).toHaveBeenCalledTimes(1); + expect(small.getStats().ttlExpirations).toBe(1); + } finally { + jest.useRealTimers(); + await small.close(); + } + }); + + it('deterministically gives the final slot to the first synchronous acquisition', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const firstFactory = jest.fn(() => createMockPool()); + const secondFactory = jest.fn(() => createMockPool()); + + const outcomes = await Promise.allSettled([ + Promise.resolve().then(() => small.acquire('first', firstFactory)), + Promise.resolve().then(() => small.acquire('second', secondFactory)) + ]); + + expect(outcomes[0].status).toBe('fulfilled'); + expect(outcomes[1].status).toBe('rejected'); + expect((outcomes[1] as PromiseRejectedResult).reason).toBeInstanceOf( + PgPoolCapacityError + ); + expect(firstFactory).toHaveBeenCalledTimes(1); + expect(secondFactory).not.toHaveBeenCalled(); + + if (outcomes[0].status === 'fulfilled') outcomes[0].value.release(); + await small.close(); + }); + + it('rolls back its reservation if pool construction fails', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const retained = createMockPool(); + small.set('retained', retained); + + expect(() => small.acquire('broken', () => { + throw new Error('factory failed'); + })).toThrow('factory failed'); + + expect(small.has('retained')).toBe(true); + expect(retained.end).not.toHaveBeenCalled(); + expect(small.getStats()).toMatchObject({ size: 1, reservations: 0 }); + await small.close(); + }); + + it('does not end a physical pool retained under another exact identity', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const sharedPool = createMockPool(); + + small.set('identity-a', sharedPool); + small.set('identity-b', sharedPool); + await small.waitForDisposals(); + expect(sharedPool.end).not.toHaveBeenCalled(); + + small.delete('identity-b'); + await small.waitForDisposals(); + expect(sharedPool.end).toHaveBeenCalledTimes(1); + await small.close(); + }); + }); + describe('close() lifecycle', () => { it('set() after close() succeeds (cache re-opens for restart)', async () => { const pool1 = createMockPool(); diff --git a/postgres/pg-cache/src/__tests__/notification-broker.integration.test.ts b/postgres/pg-cache/src/__tests__/notification-broker.integration.test.ts new file mode 100644 index 0000000000..a7a8ef585a --- /dev/null +++ b/postgres/pg-cache/src/__tests__/notification-broker.integration.test.ts @@ -0,0 +1,154 @@ +import type pg from 'pg'; +import { getPgEnvOptions, type PgConfig } from 'pg-env'; + +import { teardownPgPools } from '../lru'; +import { + acquirePgNotificationBroker, + getPgNotificationBrokerStats, + PgNotificationTopicError, + teardownPgNotificationBrokers +} from '../notification-broker'; +import { defaultPgPoolFactory, getPgPool } from '../pg'; + +// Production acquisition always audits the login on its pinned listener, so +// this test requires the dedicated least-privilege notification fixture. +const describeWithPostgres = + process.env.PG_CACHE_RUN_NOTIFICATION_ROLE_INTEGRATION === '1' + ? describe + : describe.skip; + +describeWithPostgres('notification broker against PostgreSQL', () => { + let observerPool: pg.Pool; + let listenerPgConfig: PgConfig & { pool: { max: number } }; + + beforeAll(() => { + listenerPgConfig = { + ...getPgEnvOptions(), + pool: { max: 1 } + }; + observerPool = defaultPgPoolFactory( + { ...listenerPgConfig, pool: { max: 1 } }, + { + purpose: 'notification-broker-integration-observer', + sanitizeOnCheckout: false + } + ) as pg.Pool; + }); + + afterAll(async () => { + await teardownPgNotificationBrokers(); + await teardownPgPools(); + await observerPool?.end(); + }); + + it('shares one LISTEN backend across three isolated generation leases and releases it', async () => { + const nonce = `${process.pid.toString(36)}_${Date.now().toString(36)}`; + const topics = [ + `pg_cache_it_${nonce}_a`, + `pg_cache_it_${nonce}_b`, + `pg_cache_it_${nonce}_c` + ]; + const listenQueries = topics.map((topic) => `LISTEN "${topic}"`); + + const first = await acquirePgNotificationBroker(listenerPgConfig, { + topics: [topics[0]] + }); + const second = await acquirePgNotificationBroker(listenerPgConfig, { + topics: [topics[1]] + }); + const third = await acquirePgNotificationBroker(listenerPgConfig, { + topics: [topics[2]] + }); + const brokerPool = getPgPool(listenerPgConfig, { + purpose: 'notification-broker', + sanitizeOnCheckout: true + }); + + expect(new Set([first.identity, second.identity, third.identity]).size).toBe(1); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 1, + listenerConnections: 1, + leases: 3, + topics: 3 + }); + expect(brokerPool.totalCount).toBe(1); + expect(brokerPool.idleCount).toBe(0); + + const activeListeners = await observerPool.query<{ + pid: number; + query: string; + }>(` + SELECT pid, query + FROM pg_stat_activity + WHERE datname = current_database() + AND usename = current_user + AND pid <> pg_backend_pid() + AND query = ANY($1::text[]) + `, [listenQueries]); + expect(activeListeners.rows).toEqual([ + { pid: expect.any(Number), query: listenQueries[2] } + ]); + const listenerPid = activeListeners.rows[0].pid; + + expect(() => first.subscribe(topics[1])).toThrow(PgNotificationTopicError); + const firstStream = first.subscribe(topics[0]); + const secondStream = second.subscribe(topics[1]); + const thirdStream = third.subscribe(topics[2]); + let firstResolved = false; + let thirdResolved = false; + const firstNext = firstStream.next().then((result) => { + firstResolved = true; + return result; + }); + const secondNext = secondStream.next(); + const thirdNext = thirdStream.next().then((result) => { + thirdResolved = true; + return result; + }); + + await observerPool.query('SELECT pg_notify($1, $2)', [topics[1], 'for-second']); + await expect(secondNext).resolves.toEqual({ done: false, value: 'for-second' }); + // Delivery to every lease happens synchronously inside one notification + // callback, so these flags prove the second topic did not reach its peers. + expect(firstResolved).toBe(false); + expect(thirdResolved).toBe(false); + + await observerPool.query('SELECT pg_notify($1, $2)', [topics[0], 'for-first']); + await observerPool.query('SELECT pg_notify($1, $2)', [topics[2], 'for-third']); + await expect(firstNext).resolves.toEqual({ done: false, value: 'for-first' }); + await expect(thirdNext).resolves.toEqual({ done: false, value: 'for-third' }); + + await second.release(); + await first.release(); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 1, + listenerConnections: 1, + leases: 1, + topics: 1 + }); + expect(brokerPool.idleCount).toBe(0); + + await third.release(); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + topics: 0 + }); + expect(brokerPool.totalCount).toBe(0); + expect(brokerPool.idleCount).toBe(0); + + let releasedListenerRows: Array<{ pid: number }> = []; + for (let attempt = 0; attempt < 50; attempt++) { + const releasedListener = await observerPool.query<{ pid: number }>(` + SELECT pid + FROM pg_stat_activity + WHERE pid = $1 + `, [listenerPid]); + releasedListenerRows = releasedListener.rows; + if (releasedListenerRows.length === 0) break; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + expect(releasedListenerRows).toEqual([]); + }); +}); diff --git a/postgres/pg-cache/src/__tests__/notification-broker.test.ts b/postgres/pg-cache/src/__tests__/notification-broker.test.ts new file mode 100644 index 0000000000..b179908f25 --- /dev/null +++ b/postgres/pg-cache/src/__tests__/notification-broker.test.ts @@ -0,0 +1,855 @@ +import { EventEmitter } from 'node:events'; + +import { + DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS, + getPgNotificationBrokerIdentity, + getPgNotificationDatabaseIdentity, + PgNotificationBrokerFailedError, + PgNotificationBrokerRegistry, + PgNotificationConnectionSource, + PgNotificationOperationTimeoutError, + PgNotificationQueueOverflowError, + PgNotificationTopicError +} from '../notification-broker'; +import { + PG_NOTIFICATION_ROLE_AUDIT_SQL, + UnsafePgNotificationRoleError +} from '../notification-role'; + +const roleContract = { + role: 'tenant_a_notify', + database: 'tenant_a' +}; + +const safeRoleAuditRow = { + expected_role: roleContract.role, + session_role: roleContract.role, + active_role: roleContract.role, + active_database: roleContract.database, + rolcanlogin: true, + rolinherit: false, + rolsuper: false, + rolbypassrls: false, + rolcreaterole: false, + rolcreatedb: false, + rolreplication: false, + membership_count: 0, + target_database_exists: true, + target_connect: true, + other_database_connect_count: 0, + target_database_owner: false, + target_database_create: false, + target_database_temp: false, + schema_owner_count: 0, + schema_create_count: 0, + schema_usage_count: 0, + relation_privilege_count: 0, + function_privilege_count: 0, + sequence_privilege_count: 0 +}; + +class MockNotificationClient extends EventEmitter { + readonly queries: string[] = []; + roleAuditRow: Record | undefined = safeRoleAuditRow; + readonly query = jest.fn(async ( + text: string, + _values?: readonly unknown[] + ): Promise => { + this.queries.push(text); + if (text === PG_NOTIFICATION_ROLE_AUDIT_SQL) { + return { rows: this.roleAuditRow ? [this.roleAuditRow] : [] }; + } + return { rows: [] }; + }); + readonly release = jest.fn(async (_error?: Error | boolean): Promise => {}); + + notification(channel: string, payload?: string): void { + this.emit('notification', { channel, payload }); + } +} + +const createSource = (client = new MockNotificationClient()) => { + const source: PgNotificationConnectionSource = { + connect: jest.fn(async () => client), + release: jest.fn(async () => {}) + }; + return { client, source }; +}; + +const deferred = () => { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +const flushMicrotasks = async (): Promise => { + for (let index = 0; index < 20; index++) await Promise.resolve(); +}; + +describe('PgNotificationBrokerRegistry', () => { + it('shares one dedicated listener and reference-counts exact topics', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const sourceFactory = jest.fn(() => source); + + const [first, second] = await Promise.all([ + registry.acquireForTests('opaque-a', sourceFactory, ['tenant.a', 'shared']), + registry.acquireForTests('opaque-a', sourceFactory, ['shared', 'tenant.b']) + ]); + + expect(sourceFactory).toHaveBeenCalledTimes(1); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(client.queries).toEqual([ + 'LISTEN "tenant.a"', + 'LISTEN "shared"', + 'LISTEN "tenant.b"' + ]); + expect(registry.stats()).toMatchObject({ + brokers: 1, + listenerConnections: 1, + leases: 2, + topics: 3 + }); + + await first.release(); + expect(client.queries).toContain('UNLISTEN "tenant.a"'); + expect(client.queries).not.toContain('UNLISTEN "shared"'); + expect(client.release).not.toHaveBeenCalled(); + + await second.release(); + expect(client.queries.slice(-2)).toEqual([ + 'UNLISTEN "shared"', + 'UNLISTEN "tenant.b"' + ]); + expect(client.release).toHaveBeenCalledWith(true); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ brokers: 0, leases: 0, topics: 0 }); + }); + + it('audits three generations on the one pinned listener before admission', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const sourceFactory = jest.fn(() => source); + + const first = await registry.acquireAttestedForTests( + 'opaque-attested', sourceFactory, ['tenant.a'], roleContract + ); + const second = await registry.acquireAttestedForTests( + 'opaque-attested', sourceFactory, ['tenant.b'], roleContract + ); + const third = await registry.acquireAttestedForTests( + 'opaque-attested', sourceFactory, ['tenant.c'], roleContract + ); + + expect(sourceFactory).toHaveBeenCalledTimes(1); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(client.queries.filter( + (query) => query === PG_NOTIFICATION_ROLE_AUDIT_SQL + )).toHaveLength(3); + expect(client.queries).toEqual([ + 'BEGIN READ ONLY', + 'SET LOCAL jit TO off', + PG_NOTIFICATION_ROLE_AUDIT_SQL, + 'COMMIT', + 'LISTEN "tenant.a"', + 'BEGIN READ ONLY', + 'SET LOCAL jit TO off', + PG_NOTIFICATION_ROLE_AUDIT_SQL, + 'COMMIT', + 'LISTEN "tenant.b"', + 'BEGIN READ ONLY', + 'SET LOCAL jit TO off', + PG_NOTIFICATION_ROLE_AUDIT_SQL, + 'COMMIT', + 'LISTEN "tenant.c"' + ]); + expect(first.roleAudit).toMatchObject({ ...roleContract, safe: true }); + expect(second.roleAudit).toMatchObject({ ...roleContract, safe: true }); + expect(third.roleAudit).toMatchObject({ ...roleContract, safe: true }); + expect(registry.stats()).toMatchObject({ + listenerConnections: 1, + leases: 3, + roleAuditAttempts: 3, + roleAuditFailures: 0 + }); + + await Promise.all([first.release(), second.release(), third.release()]); + }); + + it('serializes concurrent admission audits without another connection', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const firstCatalogAudit = deferred(); + let catalogAuditsStarted = 0; + let activeCatalogAudits = 0; + let peakCatalogAudits = 0; + client.query.mockImplementation(async (text: string) => { + client.queries.push(text); + if (text === PG_NOTIFICATION_ROLE_AUDIT_SQL) { + catalogAuditsStarted++; + activeCatalogAudits++; + peakCatalogAudits = Math.max(peakCatalogAudits, activeCatalogAudits); + if (catalogAuditsStarted === 1) await firstCatalogAudit.promise; + activeCatalogAudits--; + return { rows: [safeRoleAuditRow] }; + } + return { rows: [] }; + }); + + const acquisitions = [ + registry.acquireAttestedForTests( + 'opaque-attested', () => source, ['a'], roleContract + ), + registry.acquireAttestedForTests( + 'opaque-attested', () => source, ['b'], roleContract + ), + registry.acquireAttestedForTests( + 'opaque-attested', () => source, ['c'], roleContract + ) + ]; + await flushMicrotasks(); + expect(catalogAuditsStarted).toBe(1); + expect(source.connect).toHaveBeenCalledTimes(1); + + firstCatalogAudit.resolve(); + const leases = await Promise.all(acquisitions); + expect(catalogAuditsStarted).toBe(3); + expect(peakCatalogAudits).toBe(1); + expect(source.connect).toHaveBeenCalledTimes(1); + await Promise.all(leases.map((lease) => lease.release())); + }); + + it('bounds a never-resolving admission audit and destroys its client', async () => { + jest.useFakeTimers(); + try { + const registry = new PgNotificationBrokerRegistry(4, 25); + const { client, source } = createSource(); + client.query.mockImplementation((text: string) => { + client.queries.push(text); + if (text === 'BEGIN READ ONLY') return new Promise(() => undefined); + return Promise.resolve({ rows: [] }); + }); + + const acquiring = registry.acquireAttestedForTests( + 'opaque-timeout', + () => source, + ['a'], + roleContract + ); + const rejected = expect(acquiring).rejects.toBeInstanceOf( + PgNotificationOperationTimeoutError + ); + await flushMicrotasks(); + await jest.advanceTimersByTimeAsync(25); + await rejected; + + expect(client.queries).toEqual(['BEGIN READ ONLY']); + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + fatalFailures: 1, + roleAuditAttempts: 1, + roleAuditFailures: 1 + }); + } finally { + jest.useRealTimers(); + } + }); + + it('revalidates on the pinned listener and fails every lease closed on drift', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const first = await registry.acquireAttestedForTests( + 'opaque-attested', () => source, ['a'], roleContract + ); + const second = await registry.acquireAttestedForTests( + 'opaque-attested', () => source, ['b'], roleContract + ); + + await expect(first.revalidateRole()).resolves.toMatchObject({ safe: true }); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + roleAuditAttempts: 3, + roleAuditFailures: 0 + }); + + const firstNext = first.subscribe('a').next(); + const secondNext = second.subscribe('b').next(); + client.roleAuditRow = { ...safeRoleAuditRow, rolsuper: true }; + await expect(second.revalidateRole()).rejects.toBeInstanceOf( + UnsafePgNotificationRoleError + ); + await expect(firstNext).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + await expect(secondNext).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + await expect(first.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await expect(second.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(registry.stats()).toMatchObject({ + listenerConnections: 0, + leases: 2, + fatalFailures: 1, + roleAuditAttempts: 4, + roleAuditFailures: 1 + }); + + await Promise.all([first.release(), second.release()]); + }); + + it('bounds a never-resolving TTL role refresh on the pinned listener', async () => { + jest.useFakeTimers(); + try { + const registry = new PgNotificationBrokerRegistry(4, 25); + const { client, source } = createSource(); + const lease = await registry.acquireAttestedForTests( + 'opaque-refresh-timeout', + () => source, + ['a'], + roleContract + ); + client.query.mockImplementation((text: string) => { + client.queries.push(text); + if (text === 'BEGIN READ ONLY') return new Promise(() => undefined); + return Promise.resolve({ rows: [] }); + }); + + const refreshing = lease.revalidateRole(); + const rejected = expect(refreshing).rejects.toBeInstanceOf( + PgNotificationOperationTimeoutError + ); + await flushMicrotasks(); + await jest.advanceTimersByTimeAsync(25); + await rejected; + await expect(lease.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(registry.stats()).toMatchObject({ + listenerConnections: 0, + leases: 1, + fatalFailures: 1, + roleAuditAttempts: 2, + roleAuditFailures: 1 + }); + await lease.release(); + } finally { + jest.useRealTimers(); + } + }); + + it('uses exact topic equality for prefix and quoted-identifier channels', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const hostileButValid = 'tenant"; UNLISTEN *;--'; + const lease = await registry.acquireForTests( + 'opaque-a', + () => source, + ['tenant', 'tenant.longer', hostileButValid] + ); + const exact = lease.subscribe('tenant'); + const longer = lease.subscribe('tenant.longer'); + const hostile = lease.subscribe(hostileButValid); + + expect(() => lease.subscribe('ten')).toThrow(PgNotificationTopicError); + expect(client.queries).toContain( + 'LISTEN "tenant""; UNLISTEN *;--"' + ); + + client.notification('ten', 'wrong-prefix'); + client.notification('tenant.longer', 'longer'); + client.notification(hostileButValid, 'quoted'); + client.notification('tenant', 'exact'); + + await expect(exact.next()).resolves.toEqual({ done: false, value: 'exact' }); + await expect(longer.next()).resolves.toEqual({ done: false, value: 'longer' }); + await expect(hostile.next()).resolves.toEqual({ done: false, value: 'quoted' }); + expect(registry.stats().ignoredNotifications).toBe(1); + await lease.release(); + }); + + it('rejects channels PostgreSQL would truncate, including multi-byte Unicode', async () => { + const registry = new PgNotificationBrokerRegistry(); + const { source } = createSource(); + + const ascii63 = 'a'.repeat(63); + const unicode63 = '界'.repeat(21); + const lease = await registry.acquireForTests( + 'opaque-a', + () => source, + [ascii63, unicode63] + ); + expect(lease.topics).toEqual([ascii63, unicode63]); + + await expect( + registry.acquireForTests('opaque-b', () => source, ['a'.repeat(64)]) + ).rejects.toBeInstanceOf(PgNotificationTopicError); + await expect( + registry.acquireForTests('opaque-b', () => source, ['界'.repeat(22)]) + ).rejects.toBeInstanceOf(PgNotificationTopicError); + await expect( + registry.acquireForTests( + 'opaque-b', + () => source, + [`bad${String.fromCharCode(0xd800)}`] + ) + ).rejects.toBeInstanceOf(PgNotificationTopicError); + await lease.release(); + }); + + it('does not normalize canonically equivalent Unicode topics', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const composed = 'réaltime'; + const decomposed = 're\u0301altime'; + const lease = await registry.acquireForTests( + 'opaque-a', + () => source, + [composed, decomposed] + ); + const composedStream = lease.subscribe(composed); + const decomposedStream = lease.subscribe(decomposed); + + client.notification(composed, 'composed-only'); + client.notification(decomposed, 'decomposed-only'); + + await expect(composedStream.next()).resolves.toMatchObject({ value: 'composed-only' }); + await expect(decomposedStream.next()).resolves.toMatchObject({ value: 'decomposed-only' }); + await lease.release(); + }); + + it('fans out only to subscribers for the exact allowed topic', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const first = await registry.acquireForTests('opaque-a', () => source, ['a']); + const second = await registry.acquireForTests('opaque-a', () => source, ['a', 'b']); + const firstA = first.subscribe('a'); + const secondA = second.subscribe('a'); + const secondB = second.subscribe('b'); + + client.notification('a', 'for-a'); + client.notification('b', 'for-b'); + + await expect(firstA.next()).resolves.toMatchObject({ value: 'for-a' }); + await expect(secondA.next()).resolves.toMatchObject({ value: 'for-a' }); + await expect(secondB.next()).resolves.toMatchObject({ value: 'for-b' }); + await Promise.all([first.release(), second.release()]); + }); + + it('fails only the slow subscriber when its bounded queue overflows', async () => { + const registry = new PgNotificationBrokerRegistry(1); + const { client, source } = createSource(); + const lease = await registry.acquireForTests('opaque-a', () => source, ['events']); + const slow = lease.subscribe('events'); + const fast = lease.subscribe('events'); + + const fastFirst = fast.next(); + client.notification('events', 'one'); + const fastSecond = fast.next(); + client.notification('events', 'two'); + + await expect(fastFirst).resolves.toMatchObject({ value: 'one' }); + await expect(fastSecond).resolves.toMatchObject({ value: 'two' }); + await expect(slow.next()).rejects.toBeInstanceOf(PgNotificationQueueOverflowError); + expect(registry.stats()).toMatchObject({ subscribers: 1, queueOverflows: 1 }); + await lease.release(); + }); + + it('fails every active subscriber and never silently reconnects', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const sourceFactory = jest.fn(() => source); + const first = await registry.acquireForTests('opaque-a', sourceFactory, ['a']); + const second = await registry.acquireForTests('opaque-a', sourceFactory, ['b']); + const firstNext = first.subscribe('a').next(); + const secondNext = second.subscribe('b').next(); + + client.emit('error', new Error('socket lost')); + + await expect(firstNext).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + await expect(secondNext).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + await expect(first.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await expect( + registry.acquireForTests('opaque-a', sourceFactory, ['a']) + ).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith(expect.any(PgNotificationBrokerFailedError)); + + await Promise.all([first.release(), second.release()]); + const replacement = createSource(); + const explicitReplacement = await registry.acquireForTests( + 'opaque-a', + () => replacement.source, + ['a'] + ); + expect(replacement.source.connect).toHaveBeenCalledTimes(1); + await explicitReplacement.release(); + }); + + it('bounds a never-resolving LISTEN and fails admission closed', async () => { + jest.useFakeTimers(); + try { + const registry = new PgNotificationBrokerRegistry(4, 25); + const { client, source } = createSource(); + client.query.mockImplementation((text: string) => { + client.queries.push(text); + if (text.startsWith('LISTEN')) return new Promise(() => undefined); + return Promise.resolve({ rows: [] }); + }); + + const acquiring = registry.acquireForTests('opaque-listen-timeout', () => source, ['a']); + const rejected = expect(acquiring).rejects.toMatchObject({ + code: 'PG_NOTIFICATION_BROKER_FAILED', + cause: { code: 'PG_NOTIFICATION_OPERATION_TIMEOUT' } + }); + await flushMicrotasks(); + await jest.advanceTimersByTimeAsync(25); + await rejected; + + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + fatalFailures: 1 + }); + } finally { + jest.useRealTimers(); + } + }); + + it('fails closed when a listener emits a malformed notification', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const lease = await registry.acquireForTests('opaque-a', () => source, ['a']); + const next = lease.subscribe('a').next(); + + client.emit('notification', { channel: 'a', payload: { hostile: true } }); + + await expect(next).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + await expect(lease.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await lease.release(); + }); + + it('makes double release idempotent and awaits UNLISTEN plus both releases', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const unlisten = deferred(); + const clientReleased = deferred(); + const sourceReleased = deferred(); + client.query.mockImplementation(async (text: string) => { + client.queries.push(text); + if (text.startsWith('UNLISTEN')) await unlisten.promise; + return { rows: [] }; + }); + client.release.mockImplementation(async () => clientReleased.promise); + (source.release as jest.Mock).mockImplementation(async () => sourceReleased.promise); + const lease = await registry.acquireForTests('opaque-a', () => source, ['a']); + + const firstRelease = lease.release(); + const secondRelease = lease.release(); + expect(firstRelease).toBe(secondRelease); + await flushMicrotasks(); + expect(client.queries).toContain('UNLISTEN "a"'); + + let settled = false; + void firstRelease.then(() => { + settled = true; + }); + unlisten.resolve(); + await flushMicrotasks(); + expect(settled).toBe(false); + clientReleased.resolve(); + await flushMicrotasks(); + expect(settled).toBe(false); + sourceReleased.resolve(); + await firstRelease; + await expect(lease.terminated).resolves.toBeNull(); + expect(settled).toBe(true); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + }); + + it('serializes a final release against a concurrent new acquisition', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const firstSource = createSource(); + const unlisten = deferred(); + firstSource.client.query.mockImplementation(async (text: string) => { + firstSource.client.queries.push(text); + if (text.startsWith('UNLISTEN')) await unlisten.promise; + return { rows: [] }; + }); + const first = await registry.acquireForTests( + 'opaque-a', + () => firstSource.source, + ['a'] + ); + const releasing = first.release(); + await flushMicrotasks(); + + const secondSource = createSource(); + const acquiring = registry.acquireForTests( + 'opaque-a', + () => secondSource.source, + ['a'] + ); + await flushMicrotasks(); + expect(secondSource.source.connect).not.toHaveBeenCalled(); + + unlisten.resolve(); + await releasing; + const second = await acquiring; + expect(secondSource.source.connect).toHaveBeenCalledTimes(1); + await second.release(); + }); + + it('makes concurrent registry close calls await the same teardown', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { source } = createSource(); + const sourceReleased = deferred(); + (source.release as jest.Mock).mockImplementation(async () => sourceReleased.promise); + await registry.acquireForTests('opaque-a', () => source, ['a']); + + const firstClose = registry.close(); + const secondClose = registry.close(); + let firstSettled = false; + let secondSettled = false; + void firstClose.then(() => { + firstSettled = true; + }); + void secondClose.then(() => { + secondSettled = true; + }); + await flushMicrotasks(); + + expect(firstSettled).toBe(false); + expect(secondSettled).toBe(false); + sourceReleased.resolve(); + await Promise.all([firstClose, secondClose]); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ brokers: 0, leases: 0 }); + }); + + it('bounds a never-resolving UNLISTEN so teardown cannot hang', async () => { + jest.useFakeTimers(); + try { + const registry = new PgNotificationBrokerRegistry(4, 25); + const { client, source } = createSource(); + await registry.acquireForTests('opaque-unlisten-timeout', () => source, ['a']); + client.query.mockImplementation((text: string) => { + client.queries.push(text); + if (text.startsWith('UNLISTEN')) return new Promise(() => undefined); + return Promise.resolve({ rows: [] }); + }); + + const closing = registry.close(); + const rejected = expect(closing).rejects.toMatchObject({ + code: 'PG_NOTIFICATION_BROKER_FAILED', + cause: { code: 'PG_NOTIFICATION_OPERATION_TIMEOUT' } + }); + await flushMicrotasks(); + await jest.advanceTimersByTimeAsync(25); + await rejected; + + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + fatalFailures: 1 + }); + } finally { + jest.useRealTimers(); + } + }); + + it('drains an in-flight acquisition before registry close resolves', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const connected = deferred(); + const sourceReleased = deferred(); + (source.connect as jest.Mock).mockImplementation(async () => connected.promise); + (source.release as jest.Mock).mockImplementation(async () => sourceReleased.promise); + const acquiring = registry.acquireForTests('opaque-a', () => source, ['a']); + await flushMicrotasks(); + expect(source.connect).toHaveBeenCalledTimes(1); + + const closing = registry.close(); + let closeSettled = false; + void closing.then(() => { + closeSettled = true; + }); + connected.resolve(client); + await flushMicrotasks(); + const closeSettledBeforeSourceRelease = closeSettled; + const issuedListenDuringClose = client.queries.includes('LISTEN "a"'); + sourceReleased.resolve(); + await expect(acquiring).rejects.toThrow( + 'PostgreSQL notification broker registry is closed' + ); + await closing; + expect(closeSettledBeforeSourceRelease).toBe(false); + expect(issuedListenDuringClose).toBe(false); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ brokers: 0, leases: 0 }); + }); + + it('UNLISTENs a provisional topic when close races an in-flight LISTEN', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const listened = deferred(); + client.query.mockImplementation(async (text: string) => { + client.queries.push(text); + if (text === 'LISTEN "a"') await listened.promise; + return { rows: [] }; + }); + const acquiring = registry.acquireForTests('opaque-a', () => source, ['a']); + await flushMicrotasks(); + expect(client.queries).toEqual(['LISTEN "a"']); + + const closing = registry.close(); + listened.resolve(); + await expect(acquiring).rejects.toThrow( + 'PostgreSQL notification broker registry is closed' + ); + await closing; + + expect(client.queries).toEqual(['LISTEN "a"', 'UNLISTEN *']); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + }); + + it('reports a failed UNLISTEN only after finishing registry teardown', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + client.query.mockImplementation(async (text: string) => { + client.queries.push(text); + if (text.startsWith('UNLISTEN')) throw new Error('unlisten failed'); + return { rows: [] }; + }); + await registry.acquireForTests('opaque-a', () => source, ['a']); + + await expect(registry.close()).rejects.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ brokers: 0, leases: 0 }); + }); +}); + +describe('getPgNotificationBrokerIdentity', () => { + const baseConfig = { + host: 'db.internal', + port: 5432, + database: 'customer', + user: 'listener', + password: 'secret' + }; + + it('is versioned, opaque, stable, and includes the canonical SSL contract', () => { + const first = getPgNotificationBrokerIdentity({ + ...baseConfig, + ssl: { rejectUnauthorized: true, ca: 'ca-one' }, + pool: { connectionTimeoutMillis: DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS } + }); + const reordered = getPgNotificationBrokerIdentity({ + ...baseConfig, + ssl: { ca: 'ca-one', rejectUnauthorized: true }, + pool: { connectionTimeoutMillis: DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS } + }); + const changedTls = getPgNotificationBrokerIdentity({ + ...baseConfig, + ssl: { rejectUnauthorized: false, ca: 'ca-one' }, + pool: { connectionTimeoutMillis: DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS } + }); + const changedDeadline = getPgNotificationBrokerIdentity({ + ...baseConfig, + ssl: { rejectUnauthorized: true, ca: 'ca-one' }, + pool: { + connectionTimeoutMillis: + DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS + 1 + } + }); + + expect(first).toBe(reordered); + expect(first).not.toBe(changedTls); + expect(first).not.toBe(changedDeadline); + expect(first).toMatch(/^pg-notification-broker:v1:pg:v1:[a-f0-9]{64}$/); + expect(first).not.toContain('listener'); + expect(first).not.toContain('secret'); + }); + + it.each([0, -1, 1.5, 2_147_483_648])( + 'rejects invalid notification operation timeout %p before identity publication', + (connectionTimeoutMillis) => { + expect(() => getPgNotificationBrokerIdentity({ + ...baseConfig, + pool: { connectionTimeoutMillis } + })).toThrow('notification operation timeout'); + } + ); + + it('uses one credential-free identity for the same physical database target', () => { + const first = getPgNotificationDatabaseIdentity({ + ...baseConfig, + ssl: { rejectUnauthorized: true, ca: 'ca-one' }, + pool: { max: 2 } + }); + const rotated = getPgNotificationDatabaseIdentity({ + ...baseConfig, + user: 'rotated-listener', + password: 'rotated-secret', + ssl: { ca: 'different-ca', rejectUnauthorized: false }, + pool: { max: 20, idleTimeoutMillis: 99_000 } + }); + const otherHost = getPgNotificationDatabaseIdentity({ + ...baseConfig, + host: 'other-db.internal' + }); + const otherPort = getPgNotificationDatabaseIdentity({ + ...baseConfig, + port: 5433 + }); + const otherDatabase = getPgNotificationDatabaseIdentity({ + ...baseConfig, + database: 'other-customer', + ssl: { rejectUnauthorized: true, ca: 'ca-one' } + }); + + expect(first).toBe(rotated); + expect(first).not.toBe(otherHost); + expect(first).not.toBe(otherPort); + expect(first).not.toBe(otherDatabase); + expect(first).toMatch(/^pg-notification-database:v1:pg-target:v1:[a-f0-9]{64}$/); + expect(first).not.toContain('listener'); + expect(first).not.toContain('secret'); + }); +}); diff --git a/postgres/pg-cache/src/__tests__/notification-role.integration.test.ts b/postgres/pg-cache/src/__tests__/notification-role.integration.test.ts new file mode 100644 index 0000000000..587e07c28d --- /dev/null +++ b/postgres/pg-cache/src/__tests__/notification-role.integration.test.ts @@ -0,0 +1,120 @@ +import type pg from 'pg'; +import { getPgEnvOptions } from 'pg-env'; + +import { teardownPgPools } from '../lru'; +import { + acquirePgNotificationBroker, + getPgNotificationBrokerStats, + teardownPgNotificationBrokers +} from '../notification-broker'; +import { + assertPgNotificationRole, + auditPgNotificationRole +} from '../notification-role'; +import { defaultPgPoolFactory, getPgPool } from '../pg'; + +const describeWithNotificationRole = + process.env.PG_CACHE_RUN_NOTIFICATION_ROLE_INTEGRATION === '1' + ? describe + : describe.skip; + +describeWithNotificationRole('dedicated notification role against PostgreSQL', () => { + const pgConfig = getPgEnvOptions(); + let pool: pg.Pool; + + beforeAll(() => { + pool = defaultPgPoolFactory( + { ...pgConfig, pool: { max: 1 } }, + { purpose: 'notification-role-integration', sanitizeOnCheckout: true } + ) as pg.Pool; + }); + + afterAll(async () => { + await teardownPgNotificationBrokers(); + await teardownPgPools(); + await pool?.end(); + }); + + it('accepts only the exact credential-free role/database contract', async () => { + const audit = await assertPgNotificationRole(pool, { + role: pgConfig.user, + database: pgConfig.database + }); + + expect(audit).toMatchObject({ + role: pgConfig.user, + database: pgConfig.database, + safe: true, + violations: [] + }); + expect(Object.keys(audit).sort()).toEqual([ + 'database', + 'role', + 'safe', + 'version', + 'violations' + ]); + expect(audit).not.toHaveProperty('password'); + expect(audit).not.toHaveProperty('host'); + + const wrongRole = await auditPgNotificationRole(pool, { + role: `wrong_${process.pid}`, + database: pgConfig.database + }); + expect(wrongRole).toMatchObject({ + safe: false, + violations: expect.arrayContaining(['LOGIN_ROLE_MISMATCH']) + }); + + const wrongDatabase = await auditPgNotificationRole(pool, { + role: pgConfig.user, + database: `wrong_${process.pid}` + }); + expect(wrongDatabase).toMatchObject({ + safe: false, + violations: expect.arrayContaining([ + 'DATABASE_MISMATCH', + 'TARGET_DATABASE_MISSING', + 'TARGET_CONNECT_REQUIRED', + 'CROSS_DATABASE_CONNECT' + ]) + }); + }); + + it('retains enough privilege for isolated LISTEN and NOTIFY delivery', async () => { + const nonce = `${process.pid}_${Date.now().toString(36)}`; + const topics = [0, 1, 2].map((index) => `notify_role_it_${nonce}_${index}`); + const listenerConfig = { ...pgConfig, pool: { max: 1 } }; + const statsBefore = getPgNotificationBrokerStats(); + const [first, second, third] = await Promise.all(topics.map((topic) => + acquirePgNotificationBroker(listenerConfig, { topics: [topic] }) + )); + const brokerPool = getPgPool(listenerConfig, { + purpose: 'notification-broker', + sanitizeOnCheckout: true + }); + await first.revalidateRole(); + const next = second.subscribe(topics[1]).next(); + + await pool.query('SELECT pg_notify($1, $2)', [topics[1], 'safe-listener']); + await expect(next).resolves.toEqual({ done: false, value: 'safe-listener' }); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 1, + listenerConnections: 1, + leases: 3, + topics: 3, + roleAuditAttempts: statsBefore.roleAuditAttempts + 4, + roleAuditFailures: statsBefore.roleAuditFailures + }); + expect(brokerPool.totalCount).toBe(1); + expect(brokerPool.idleCount).toBe(0); + + await Promise.all([first.release(), second.release(), third.release()]); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + topics: 0 + }); + }); +}); diff --git a/postgres/pg-cache/src/__tests__/notification-role.test.ts b/postgres/pg-cache/src/__tests__/notification-role.test.ts new file mode 100644 index 0000000000..365cd220d7 --- /dev/null +++ b/postgres/pg-cache/src/__tests__/notification-role.test.ts @@ -0,0 +1,262 @@ +import type { Pool } from 'pg'; + +import { + assertPgNotificationRole, + assertPgNotificationRoleClient, + auditPgNotificationRole, + auditPgNotificationRoleClient, + normalizePgNotificationRoleContracts, + PG_NOTIFICATION_ROLE_AUDIT_SQL, + PG_NOTIFICATION_ROLE_AUDIT_VERSION, + type PgNotificationRoleClient, + PgNotificationRoleContractError, + type PgNotificationRoleViolationCode, + UnsafePgNotificationRoleError +} from '../notification-role'; + +const contract = { + role: 'tenant_001_notification', + database: 'tenant_001' +}; + +const safeRow = { + expected_role: contract.role, + session_role: contract.role, + active_role: contract.role, + active_database: contract.database, + rolcanlogin: true, + rolinherit: false, + rolsuper: false, + rolbypassrls: false, + rolcreaterole: false, + rolcreatedb: false, + rolreplication: false, + membership_count: 0, + target_database_exists: true, + target_connect: true, + other_database_connect_count: 0, + target_database_owner: false, + target_database_create: false, + target_database_temp: false, + schema_owner_count: 0, + schema_create_count: 0, + schema_usage_count: 0, + relation_privilege_count: 0, + function_privilege_count: 0, + sequence_privilege_count: 0 +}; + +const poolWithRow = (row: Record | undefined) => { + const client = { + query: jest.fn(async (query: string) => query === PG_NOTIFICATION_ROLE_AUDIT_SQL + ? { rows: row ? [row] : [] } + : { rows: [] }), + release: jest.fn() + }; + return { + pool: { connect: jest.fn(async () => client) } as unknown as Pool, + client + }; +}; + +describe('PostgreSQL notification-role audit', () => { + it('returns a frozen credential-free attestation for an exact safe login', async () => { + const { pool, client } = poolWithRow(safeRow); + const audit = await assertPgNotificationRole( + pool, + { ...contract, password: 'must-not-escape' } as typeof contract + ); + + expect(audit).toEqual({ + version: PG_NOTIFICATION_ROLE_AUDIT_VERSION, + ...contract, + safe: true, + violations: [] + }); + expect(Object.isFrozen(audit)).toBe(true); + expect(Object.isFrozen(audit.violations)).toBe(true); + expect(JSON.stringify(audit)).not.toContain('must-not-escape'); + expect(client.query).toHaveBeenNthCalledWith( + 1, + 'BEGIN READ ONLY' + ); + expect(client.query).toHaveBeenNthCalledWith(2, 'SET LOCAL jit TO off'); + expect(client.query).toHaveBeenNthCalledWith(3, PG_NOTIFICATION_ROLE_AUDIT_SQL, [ + contract.role, + contract.database + ]); + expect(client.query).toHaveBeenNthCalledWith(4, 'COMMIT'); + expect(client.release).toHaveBeenCalledWith(false); + }); + + it('audits an already-owned listener client without releasing it', async () => { + const { client } = poolWithRow(safeRow); + + await expect(assertPgNotificationRoleClient( + client as unknown as PgNotificationRoleClient, + contract + )).resolves + .toMatchObject({ ...contract, safe: true }); + expect(client.release).not.toHaveBeenCalled(); + }); + + it('rolls back a failed pinned-client audit without taking ownership of release', async () => { + const failure = new Error('catalog unavailable'); + const client = { + query: jest.fn() + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce({ rows: [] }), + release: jest.fn() + }; + + await expect(auditPgNotificationRoleClient(client, contract)).rejects.toBe(failure); + expect(client.query).toHaveBeenNthCalledWith(4, 'ROLLBACK'); + expect(client.release).not.toHaveBeenCalled(); + }); + + it.each<[ + keyof typeof safeRow, + unknown, + PgNotificationRoleViolationCode + ]>([ + ['session_role', 'different_login', 'LOGIN_ROLE_MISMATCH'], + ['active_role', 'set_role_target', 'CURRENT_ROLE_MISMATCH'], + ['active_database', 'different_database', 'DATABASE_MISMATCH'], + ['rolcanlogin', false, 'LOGIN_REQUIRED'], + ['rolinherit', true, 'NOINHERIT_REQUIRED'], + ['rolsuper', true, 'SUPERUSER'], + ['rolbypassrls', true, 'BYPASSRLS'], + ['rolcreaterole', true, 'CREATEROLE'], + ['rolcreatedb', true, 'CREATEDB'], + ['rolreplication', true, 'REPLICATION'], + ['membership_count', 1, 'ROLE_MEMBERSHIP'], + ['target_database_exists', false, 'TARGET_DATABASE_MISSING'], + ['target_connect', false, 'TARGET_CONNECT_REQUIRED'], + ['other_database_connect_count', 1, 'CROSS_DATABASE_CONNECT'], + ['target_database_owner', true, 'DATABASE_OWNER'], + ['target_database_create', true, 'DATABASE_CREATE'], + ['target_database_temp', true, 'DATABASE_TEMP'], + ['schema_owner_count', 1, 'SCHEMA_OWNER'], + ['schema_create_count', 1, 'SCHEMA_CREATE'], + ['schema_usage_count', 1, 'SCHEMA_USAGE'], + ['relation_privilege_count', 1, 'RELATION_PRIVILEGE'], + ['function_privilege_count', 1, 'FUNCTION_PRIVILEGE'], + ['sequence_privilege_count', 1, 'SEQUENCE_PRIVILEGE'] + ])('maps %s to its stable violation code', async (field, unsafeValue, code) => { + const { pool } = poolWithRow({ ...safeRow, [field]: unsafeValue }); + const audit = await auditPgNotificationRole(pool, contract); + + expect(audit.safe).toBe(false); + expect(audit.violations).toContain(code); + await expect(assertPgNotificationRole( + poolWithRow({ ...safeRow, [field]: unsafeValue }).pool, + contract + )).rejects.toMatchObject({ + code: 'PG_NOTIFICATION_ROLE_UNSAFE', + audit: expect.objectContaining({ violations: expect.arrayContaining([code]) }) + }); + }); + + it('fails closed when the catalog audit returns no role row', async () => { + const { pool } = poolWithRow(undefined); + const audit = await auditPgNotificationRole(pool, contract); + + expect(audit).toMatchObject({ safe: false, violations: ['AUDIT_NO_RESULT'] }); + await expect(assertPgNotificationRole(poolWithRow(undefined).pool, contract)) + .rejects.toBeInstanceOf(UnsafePgNotificationRoleError); + }); + + it('rolls back and destroys the client when the catalog query fails', async () => { + const failure = new Error('catalog unavailable'); + const client = { + query: jest.fn() + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce({ rows: [] }), + release: jest.fn() + }; + const pool = { connect: jest.fn(async () => client) } as unknown as Pool; + + await expect(auditPgNotificationRole(pool, contract)).rejects.toBe(failure); + expect(client.query).toHaveBeenNthCalledWith(4, 'ROLLBACK'); + expect(client.release).toHaveBeenCalledWith(true); + }); + + it('audits exact database scope, membership edges, and every prohibited ACL class', () => { + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'membership.member = r.oid OR membership.roleid = r.oid' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'database_record.datname <> $2::text' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("'CONNECT'"); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("'CREATE'"); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("'TEMP'"); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain('schema_record.nspowner = r.oid'); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("'USAGE'"); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'pg_catalog.has_table_privilege' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'pg_catalog.has_any_column_privilege' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'pg_catalog.has_function_privilege' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'pg_catalog.has_sequence_privilege' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("n.nspname !~ '^pg_'"); + }); +}); + +describe('notification-role fleet contract', () => { + it('collapses exact generation duplicates and returns a deterministic frozen mapping', () => { + const normalized = normalizePgNotificationRoleContracts([ + { role: 'notify_b', database: 'tenant_b' }, + { ...contract }, + { ...contract } + ]); + + expect(normalized).toEqual([ + contract, + { role: 'notify_b', database: 'tenant_b' } + ]); + expect(Object.isFrozen(normalized)).toBe(true); + expect(normalized.every(Object.isFrozen)).toBe(true); + }); + + it('rejects multiple logins for one database and one login spanning databases', () => { + expect(() => normalizePgNotificationRoleContracts([ + contract, + { role: 'another_notification', database: contract.database } + ])).toThrow('maps to multiple login roles'); + expect(() => normalizePgNotificationRoleContracts([ + contract, + { role: contract.role, database: 'tenant_002' } + ])).toThrow('maps to multiple databases'); + }); + + const malformedContracts: Array<{ + contracts: readonly { role: string; database: string }[]; + }> = [ + { contracts: [] }, + { contracts: [{ role: '', database: 'tenant_001' }] }, + { contracts: [{ role: 'notify', database: '' }] }, + { contracts: [{ role: 'n'.repeat(64), database: 'tenant_001' }] }, + { + contracts: [{ + role: 'notify', + database: `bad${String.fromCharCode(0xd800)}` + }] + } + ]; + + it.each(malformedContracts)('rejects malformed contract input', ({ contracts }) => { + expect(() => normalizePgNotificationRoleContracts(contracts)) + .toThrow(PgNotificationRoleContractError); + }); +}); diff --git a/postgres/pg-cache/src/__tests__/sanitizer.integration.test.ts b/postgres/pg-cache/src/__tests__/sanitizer.integration.test.ts new file mode 100644 index 0000000000..30272f5d74 --- /dev/null +++ b/postgres/pg-cache/src/__tests__/sanitizer.integration.test.ts @@ -0,0 +1,103 @@ +import type pg from 'pg'; + +import { defaultPgPoolFactory, getPgCheckoutSanitizerStats } from '../pg'; + +const describeWithPostgres = process.env.PG_CACHE_RUN_PG_INTEGRATION === '1' + ? describe + : describe.skip; + +describeWithPostgres('runtime checkout sanitation against PostgreSQL', () => { + let pool: pg.Pool; + + beforeAll(() => { + pool = defaultPgPoolFactory( + { pool: { max: 1 } }, + { purpose: 'runtime', sanitizeOnCheckout: true } + ) as pg.Pool; + }); + + afterAll(async () => { + await pool?.end(); + }); + + it('restores the startup baseline and clears poisoned prepared state', async () => { + const poisoned = await pool.connect(); + const poisonedPid = await poisoned.query<{ pid: number }>( + 'SELECT pg_catalog.pg_backend_pid()::integer AS pid' + ); + await poisoned.query('SET search_path TO public'); + await poisoned.query('SET row_security TO off'); + await poisoned.query('SET jit_optimize_above_cost TO 123'); + await poisoned.query("SET application_name TO 'poisoned-tenant-session'"); + await poisoned.query({ name: 'tenant_cache_canary', text: 'SELECT 1 AS value' }); + poisoned.release(); + expect(getPgCheckoutSanitizerStats(pool)).toMatchObject({ + virginFastPathCheckouts: 1, + sanitizedReuseCheckouts: 0 + }); + + const clean = await pool.connect(); + try { + const cleanPid = await clean.query<{ pid: number }>( + 'SELECT pg_catalog.pg_backend_pid()::integer AS pid' + ); + expect(cleanPid.rows[0]?.pid).toBe(poisonedPid.rows[0]?.pid); + const settings = await clean.query<{ + search_path: string; + row_security: string; + jit_optimize_above_cost: string; + application_name: string; + }>(` + SELECT + current_setting('search_path') AS search_path, + current_setting('row_security') AS row_security, + current_setting('jit_optimize_above_cost') AS jit_optimize_above_cost, + current_setting('application_name') AS application_name + `); + expect(settings.rows[0]).toMatchObject({ + search_path: 'pg_catalog', + row_security: 'on', + jit_optimize_above_cost: '-1' + }); + expect(settings.rows[0].application_name).not.toBe('poisoned-tenant-session'); + + await expect(clean.query({ + name: 'tenant_cache_canary', + text: 'SELECT 2 AS value' + })).resolves.toMatchObject({ rows: [{ value: 2 }] }); + expect(getPgCheckoutSanitizerStats(pool)).toMatchObject({ + virginFastPathCheckouts: 1, + sanitizedReuseCheckouts: 1, + sanitationFailures: 0 + }); + } finally { + clean.release(); + } + }); + + it('proves maxUses=1 rotates the PostgreSQL backend instead of reusing it', async () => { + const rotatingPool = defaultPgPoolFactory( + { pool: { max: 1, maxUses: 1 } }, + { purpose: 'runtime-max-uses-one', sanitizeOnCheckout: true } + ) as pg.Pool; + try { + const first = await rotatingPool.connect(); + const firstPid = await first.query<{ pid: number }>( + 'SELECT pg_catalog.pg_backend_pid()::integer AS pid' + ); + first.release(); + + const second = await rotatingPool.connect(); + try { + const secondPid = await second.query<{ pid: number }>( + 'SELECT pg_catalog.pg_backend_pid()::integer AS pid' + ); + expect(secondPid.rows[0]?.pid).not.toBe(firstPid.rows[0]?.pid); + } finally { + second.release(); + } + } finally { + await rotatingPool.end(); + } + }); +}); diff --git a/postgres/pg-cache/src/__tests__/sanitizer.test.ts b/postgres/pg-cache/src/__tests__/sanitizer.test.ts new file mode 100644 index 0000000000..78a4890722 --- /dev/null +++ b/postgres/pg-cache/src/__tests__/sanitizer.test.ts @@ -0,0 +1,285 @@ +import { EventEmitter } from 'node:events'; + +import type pg from 'pg'; + +import { + getPgCheckoutSanitizerStats, + installCheckoutSanitizer, + sanitizePgClient +} from '../pg'; + +const mockClient = () => ({ + query: jest.fn(async () => ({ rows: [] as unknown[] })), + release: jest.fn(), + connection: { + parsedStatements: { tenant_query: 'select 1' }, + _graphilePreparedStatementCache: { reset: jest.fn() } + } +}) as unknown as pg.PoolClient & { + connection: { + parsedStatements: Record; + _graphilePreparedStatementCache?: { reset: jest.Mock }; + }; +}; + +describe('runtime checkout sanitation', () => { + it('discards server state and clears both prepared-statement caches', async () => { + const client = mockClient(); + + await expect(sanitizePgClient(client)).resolves.toBe(client); + + expect(client.query).toHaveBeenNthCalledWith(1, 'DISCARD ALL'); + expect(client.query).toHaveBeenNthCalledWith( + 2, + 'SET search_path TO pg_catalog; SET row_security TO on; SET jit_optimize_above_cost TO -1' + ); + expect(client.connection.parsedStatements).toEqual({}); + expect(client.connection).not.toHaveProperty('_graphilePreparedStatementCache'); + expect(client.release).not.toHaveBeenCalled(); + }); + + it('does not run Dataplan LRU disposers after DISCARD ALL', async () => { + const client = mockClient(); + const reset = client.connection._graphilePreparedStatementCache!.reset; + + await sanitizePgClient(client, true); + + expect(reset).not.toHaveBeenCalled(); + expect(client.query).toHaveBeenCalledTimes(1); + expect(client.query).toHaveBeenCalledWith('DISCARD ALL'); + }); + + it('uses one checkout query when DISCARD restores a pinned startup baseline', async () => { + const client = mockClient(); + + await expect(sanitizePgClient(client, true)).resolves.toBe(client); + + expect(client.query).toHaveBeenCalledTimes(1); + expect(client.query).toHaveBeenCalledWith('DISCARD ALL'); + expect(client.connection.parsedStatements).toEqual({}); + }); + + it('destroys a connection when DISCARD ALL fails', async () => { + const client = mockClient(); + (client.query as jest.Mock).mockRejectedValueOnce(new Error('idle in transaction')); + + await expect(sanitizePgClient(client)).rejects.toThrow('idle in transaction'); + expect(client.release).toHaveBeenCalledWith(true); + }); + + it('destroys a connection when restoring the trusted baseline fails', async () => { + const client = mockClient(); + (client.query as jest.Mock) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(new Error('baseline rejected')); + + await expect(sanitizePgClient(client)).rejects.toThrow('baseline rejected'); + expect(client.release).toHaveBeenCalledWith(true); + }); + + it('skips DISCARD only for a factory-marked virgin with no competing connect hook', async () => { + const client = mockClient(); + let connected = false; + const pool = Object.assign(new EventEmitter(), { + waitingCount: 0, + query: jest.fn(), + connect: jest.fn(async () => { + if (!connected) { + connected = true; + pool.emit('connect', client); + } + return client; + }) + }) as unknown as pg.Pool; + + installCheckoutSanitizer(pool, true, true); + + await expect(pool.connect()).resolves.toBe(client); + expect(client.query).not.toHaveBeenCalled(); + expect(getPgCheckoutSanitizerStats(pool)).toMatchObject({ + checkoutAttempts: 1, + virginFastPathCheckouts: 1, + sanitizedReuseCheckouts: 0 + }); + + await expect(pool.connect()).resolves.toBe(client); + expect(client.query).toHaveBeenCalledTimes(1); + expect(client.query).toHaveBeenCalledWith('DISCARD ALL'); + expect(getPgCheckoutSanitizerStats(pool)).toMatchObject({ + checkoutAttempts: 2, + virginFastPathCheckouts: 1, + sanitizedReuseCheckouts: 1, + sanitationFailures: 0 + }); + }); + + it('fully sanitizes a virgin after a self-removing connect hook can touch it', async () => { + const client = mockClient(); + let connected = false; + const pool = Object.assign(new EventEmitter(), { + waitingCount: 0, + query: jest.fn(), + connect: jest.fn(async () => { + if (!connected) { + connected = true; + pool.emit('connect', client); + } + return client; + }) + }) as unknown as pg.Pool; + + installCheckoutSanitizer(pool, true, true); + pool.prependOnceListener('connect', () => undefined); + + await expect(pool.connect()).resolves.toBe(client); + expect(client.query).toHaveBeenCalledWith('DISCARD ALL'); + expect(getPgCheckoutSanitizerStats(pool)).toMatchObject({ + virginFastPathCheckouts: 0, + sanitizedReuseCheckouts: 1 + }); + }); + + it('fully sanitizes a minimal custom pool without EventEmitter methods', async () => { + const client = mockClient(); + const pool = { + waitingCount: 0, + query: jest.fn(), + connect: jest.fn(async () => client) + } as unknown as pg.Pool; + + installCheckoutSanitizer(pool); + + await expect(pool.connect()).resolves.toBe(client); + expect(client.query).toHaveBeenNthCalledWith(1, 'DISCARD ALL'); + expect(client.query).toHaveBeenNthCalledWith( + 2, + 'SET search_path TO pg_catalog; SET row_security TO on; SET jit_optimize_above_cost TO -1' + ); + expect(getPgCheckoutSanitizerStats(pool)).toMatchObject({ + checkoutAttempts: 1, + virginFastPathCheckouts: 0, + sanitizedReuseCheckouts: 1 + }); + }); + + it('routes a custom pool query through one sanitized checkout', async () => { + const client = mockClient(); + const queryResult = { rows: [{ value: 7 }] }; + (client.query as jest.Mock) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce(queryResult); + const bypassingQuery = jest.fn(async () => ({ rows: [{ value: -1 }] })); + const connect = jest.fn(async () => client); + const pool = { + waitingCount: 0, + query: bypassingQuery, + connect + } as unknown as pg.Pool; + + installCheckoutSanitizer(pool); + + await expect(pool.query('SELECT $1::int AS value', [7])).resolves.toBe(queryResult); + expect(bypassingQuery).not.toHaveBeenCalled(); + expect(connect).toHaveBeenCalledTimes(1); + expect(client.query).toHaveBeenNthCalledWith(1, 'DISCARD ALL'); + expect(client.query).toHaveBeenNthCalledWith( + 2, + 'SET search_path TO pg_catalog; SET row_security TO on; SET jit_optimize_above_cost TO -1' + ); + expect(client.query).toHaveBeenNthCalledWith(3, 'SELECT $1::int AS value', [7]); + expect(client.release).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith(); + }); + + it('preserves callback queries without executing the custom pool bypass', async () => { + const client = mockClient(); + const queryResult = { rows: [{ value: 9 }] }; + (client.query as jest.Mock) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce(queryResult); + const bypassingQuery = jest.fn(); + const pool = { + waitingCount: 0, + query: bypassingQuery, + connect: jest.fn(async () => client) + } as unknown as pg.Pool; + + installCheckoutSanitizer(pool); + + const callbackResult = await new Promise((resolve, reject) => { + const returned = pool.query( + 'SELECT $1::int AS value', + [9], + (error, result) => error ? reject(error) : resolve(result) + ); + expect(returned).toBeUndefined(); + }); + + expect(callbackResult).toBe(queryResult); + expect(bypassingQuery).not.toHaveBeenCalled(); + expect(client.query).toHaveBeenCalledTimes(3); + expect(client.query).toHaveBeenNthCalledWith(3, 'SELECT $1::int AS value', [9]); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it('destroys the checked-out custom client when a direct query fails', async () => { + const client = mockClient(); + const queryError = new Error('query rejected'); + (client.query as jest.Mock) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(queryError); + const bypassingQuery = jest.fn(); + const pool = { + waitingCount: 0, + query: bypassingQuery, + connect: jest.fn(async () => client) + } as unknown as pg.Pool; + + installCheckoutSanitizer(pool); + + await expect(pool.query('SELECT broken')).rejects.toBe(queryError); + expect(bypassingQuery).not.toHaveBeenCalled(); + expect(client.query).toHaveBeenCalledTimes(3); + expect(client.query).toHaveBeenNthCalledWith(3, 'SELECT broken'); + expect(client.release).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith(queryError); + }); + + it('does not release twice when direct-query sanitation fails', async () => { + const client = mockClient(); + const sanitationError = new Error('discard rejected'); + (client.query as jest.Mock).mockRejectedValueOnce(sanitationError); + const bypassingQuery = jest.fn(); + const pool = { + waitingCount: 0, + query: bypassingQuery, + connect: jest.fn(async () => client) + } as unknown as pg.Pool; + + installCheckoutSanitizer(pool); + + await expect(pool.query('SELECT unsafe')).rejects.toBe(sanitationError); + expect(bypassingQuery).not.toHaveBeenCalled(); + expect(client.query).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith(true); + }); + + it('rejects a custom sanitized pool whose query method cannot be replaced', () => { + const pool = { + waitingCount: 0, + connect: jest.fn() + } as unknown as pg.Pool; + Object.defineProperty(pool, 'query', { + value: jest.fn(), + writable: false + }); + + expect(() => installCheckoutSanitizer(pool)).toThrow( + 'A sanitized custom PostgreSQL pool must expose a replaceable query() method' + ); + }); +}); diff --git a/postgres/pg-cache/src/driver.ts b/postgres/pg-cache/src/driver.ts index 9a6c22ffb7..16b715ae40 100644 --- a/postgres/pg-cache/src/driver.ts +++ b/postgres/pg-cache/src/driver.ts @@ -14,10 +14,15 @@ import type { PgConfig, PgPoolConfig } from 'pg-env'; * `end()` (plus an `ended` flag for disposal), so a factory may return anything * implementing that subset — `QueryablePool`. A real `pg.Pool` structurally * satisfies it, so the default path is unchanged and fully backward-compatible. + * When checkout sanitation is requested, pg-cache replaces a custom pool's + * `query()` method so direct queries also use a sanitized `connect()`/`release()` + * cycle. Custom pools must therefore expose a replaceable `query()` property, + * and connected clients must implement the Promise-based contract below. */ export interface QueryableClient { query(text: string, values?: any[]): Promise; - release(...args: any[]): void; + /** A truthy error argument must permanently discard this client. */ + release(error?: Error | boolean): void; } export interface QueryablePool { @@ -27,10 +32,17 @@ export interface QueryablePool { } export type PgPoolFactory = ( - config: Partial & { pool?: PgPoolConfig } + config: Partial & { pool?: PgPoolConfig }, + options?: PgPoolFactoryOptions ) => pg.Pool | QueryablePool; +export interface PgPoolFactoryOptions { + purpose: string; + sanitizeOnCheckout: boolean; +} + let activeFactory: PgPoolFactory | undefined; +let driverGeneration = 0; /** * Register the factory `getPgPool` uses to build new pools. Pass `undefined` @@ -42,6 +54,7 @@ let activeFactory: PgPoolFactory | undefined; */ export const registerPgPoolFactory = (factory: PgPoolFactory | undefined): void => { activeFactory = factory; + driverGeneration++; }; /** The currently-registered factory, or `undefined` when using the default. */ @@ -49,3 +62,7 @@ export const getActivePgPoolFactory = (): PgPoolFactory | undefined => activeFac /** Whether a non-default pool factory is currently registered. */ export const hasPgPoolFactory = (): boolean => activeFactory !== undefined; + +/** Stable until the active factory registration changes. */ +export const getPgPoolDriverIdentity = (): string => + activeFactory ? `registered:${driverGeneration}` : 'node-postgres'; diff --git a/postgres/pg-cache/src/index.ts b/postgres/pg-cache/src/index.ts index 286748297a..21f071edf1 100644 --- a/postgres/pg-cache/src/index.ts +++ b/postgres/pg-cache/src/index.ts @@ -1,23 +1,91 @@ // Main exports from pg-cache package export { getActivePgPoolFactory, + getPgPoolDriverIdentity, hasPgPoolFactory, registerPgPoolFactory } from './driver'; -export { +export { close, + DEFAULT_PG_CACHE_MAX, getPgCacheConfig, - pgCache, - PgPoolCacheManager, + getPgCacheStats, + PG_CACHE_GRAPHILE_CONTRACT_CAPACITY, + PG_CACHE_OPERATIONAL_RESERVE, + PG_POOL_CAPACITY_ERROR_CODE, + pgCache, + PgPoolCacheManager, + PgPoolCapacityError, teardownPgPools } from './lru'; export { + acquirePgNotificationBroker, + assertValidPgNotificationTopic, + DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS, + getPgNotificationBrokerIdentity, + getPgNotificationBrokerStats, + getPgNotificationDatabaseIdentity, + PG_NOTIFICATION_BROKER_FAILED_ERROR_CODE, + PG_NOTIFICATION_BROKER_IDENTITY_VERSION, + PG_NOTIFICATION_DATABASE_IDENTITY_VERSION, + PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE, + PG_NOTIFICATION_OPERATION_TIMEOUT_ERROR_CODE, + PG_NOTIFICATION_QUEUE_CAPACITY, + PG_NOTIFICATION_QUEUE_OVERFLOW_ERROR_CODE, + PG_NOTIFICATION_TOPIC_ERROR_CODE, + PgNotificationBrokerFailedError, + PgNotificationLeaseReleasedError, + PgNotificationOperationTimeoutError, + PgNotificationQueueOverflowError, + PgNotificationTopicError, + teardownPgNotificationBrokers +} from './notification-broker'; +export { + assertPgNotificationRole, + assertPgNotificationRoleClient, + auditPgNotificationRole, + auditPgNotificationRoleClient, + normalizePgNotificationRoleContracts, + PG_NOTIFICATION_ROLE_AUDIT_SQL, + PG_NOTIFICATION_ROLE_AUDIT_VERSION, + PG_NOTIFICATION_ROLE_CONTRACT_ERROR_CODE, + PG_NOTIFICATION_ROLE_UNSAFE_ERROR_CODE, + PgNotificationRoleContractError, + UnsafePgNotificationRoleError +} from './notification-role'; +export { + acquirePgPool, buildConnectionString, + clearPreparedStatementBookkeeping, defaultPgPoolFactory, + getPgCheckoutSanitizerStats, getPgPool, - getPgPoolConfig + getPgPoolConfig, + getPgPoolIdentity, + installCheckoutSanitizer, + sanitizePgClient } from './pg'; // Re-export types -export type { PgPoolFactory, QueryableClient, QueryablePool } from './driver'; -export type { PgCacheConfig, PoolCleanupCallback } from './lru'; \ No newline at end of file +export type { PgPoolFactory, PgPoolFactoryOptions, QueryableClient, QueryablePool } from './driver'; +export type { + PgCacheConfig, + PgPoolCacheStats, + PgPoolDisposalReason, + PgPoolLease, + PoolCleanupCallback +} from './lru'; +export type { + AcquirePgNotificationBrokerOptions, + PgAttestedNotificationBrokerLease, + PgNotificationBrokerLease, + PgNotificationBrokerStats, + PgNotificationListenerConfig +} from './notification-broker'; +export type { + PgNotificationRoleAudit, + PgNotificationRoleClient, + PgNotificationRoleContract, + PgNotificationRoleViolationCode +} from './notification-role'; +export type { GetPgPoolOptions, PgCheckoutSanitizerStats } from './pg'; diff --git a/postgres/pg-cache/src/lru.ts b/postgres/pg-cache/src/lru.ts index 633dc6388e..de328f5a95 100644 --- a/postgres/pg-cache/src/lru.ts +++ b/postgres/pg-cache/src/lru.ts @@ -1,6 +1,5 @@ import { Logger } from '@pgpmjs/logger'; import { parseEnvNumber } from '12factor-env'; -import { LRUCache } from 'lru-cache'; import pg from 'pg'; const log = new Logger('pg-cache'); @@ -9,58 +8,141 @@ const ONE_HOUR_IN_MS = 1000 * 60 * 60; const ONE_DAY = ONE_HOUR_IN_MS * 24; const ONE_YEAR = ONE_DAY * 366; -// Kubernetes sends only SIGTERM on pod shutdown -const SYS_EVENTS = ['SIGTERM']; +// One runtime and one control identity per database-per-tenant Graphile +// contract, plus room for routing, diagnostics, listeners, and build overlap. +export const PG_CACHE_GRAPHILE_CONTRACT_CAPACITY = 1024; +export const PG_CACHE_OPERATIONAL_RESERVE = 16; +export const DEFAULT_PG_CACHE_MAX = + PG_CACHE_GRAPHILE_CONTRACT_CAPACITY * 2 + PG_CACHE_OPERATIONAL_RESERVE; type PgPoolKey = string; +type PoolFactory = () => pg.Pool; -// Cleanup callback type - called when a pg pool is disposed +export type PgPoolDisposalReason = + | 'capacity' + | 'ttl' + | 'delete' + | 'clear' + | 'close' + | 'replace'; + +// Called only when an identity is actually removed from the registry. export type PoolCleanupCallback = (pgPoolKey: string) => void; -// --- Cache Configuration --- +export interface PgPoolLease { + pool: pg.Pool; + identity: string; + /** Idempotently release this exact ownership claim. */ + release(): void; +} export interface PgCacheConfig { - /** Maximum number of pools in the LRU cache (env: PG_CACHE_MAX, default: 50) */ + /** Maximum number of lazy pool identities retained by this process. */ max: number; - /** TTL for cached pools in ms (default: ONE_YEAR) */ + /** Idle identity TTL in milliseconds. Leased identities never expire. */ ttl: number; } -/** - * Read cache configuration from environment variables. - * - * Supports: - * - PG_CACHE_MAX: Maximum number of pools (default: 50) - * - PG_CACHE_TTL_MS: TTL in milliseconds (default: ONE_YEAR) - */ +export interface PgPoolCacheStats { + size: number; + max: number; + ttl: number; + leasedPools: number; + idlePools: number; + activeLeases: number; + reservations: number; + pendingDisposals: number; + hits: number; + misses: number; + poolsCreated: number; + leasesAcquired: number; + leasesReleased: number; + capacityEvictions: number; + ttlExpirations: number; + capacityRefusals: number; + disposalsStarted: number; + disposalsCompleted: number; + disposalFailures: number; +} + +interface PgPoolCacheCounters { + hits: number; + misses: number; + poolsCreated: number; + leasesAcquired: number; + leasesReleased: number; + capacityEvictions: number; + ttlExpirations: number; + capacityRefusals: number; + disposalsStarted: number; + disposalsCompleted: number; + disposalFailures: number; +} + +interface SlotReservation { + key: PgPoolKey; + victims: ManagedPgPool[]; +} + +export const PG_POOL_CAPACITY_ERROR_CODE = 'PG_POOL_CAPACITY'; + +/** Fail-closed pool admission error suitable for a stable HTTP 503 mapping. */ +export class PgPoolCapacityError extends Error { + readonly code = PG_POOL_CAPACITY_ERROR_CODE; + readonly retryAfterSeconds = 15; + + constructor( + readonly max: number, + readonly size: number, + readonly leased: number + ) { + super( + `PostgreSQL pool capacity exhausted: ${size}/${max} identities are retained ` + + `and ${leased} are leased` + ); + this.name = 'PgPoolCapacityError'; + } +} + +/** Read cache configuration without allocating any pools or connections. */ export function getPgCacheConfig(): PgCacheConfig { return { - max: parseEnvNumber(process.env.PG_CACHE_MAX) ?? 50, + max: parseEnvNumber(process.env.PG_CACHE_MAX) ?? DEFAULT_PG_CACHE_MAX, ttl: parseEnvNumber(process.env.PG_CACHE_TTL_MS) ?? ONE_YEAR, }; } class ManagedPgPool { public isDisposed = false; + public leaseCount = 0; + public lastAccessOrder = 0; + public expiresAt = 0; private disposePromise: Promise | null = null; - constructor(public readonly pool: pg.Pool, public readonly key: string) {} + constructor( + public readonly pool: pg.Pool, + public readonly key: string + ) {} + + touch(order: number, now: number, ttl: number): void { + this.lastAccessOrder = order; + this.expiresAt = now + ttl; + } + + isExpired(now: number): boolean { + return now >= this.expiresAt; + } async dispose(): Promise { if (this.isDisposed) return this.disposePromise; this.isDisposed = true; this.disposePromise = (async () => { - try { - if (!this.pool.ended) { - await this.pool.end(); - log.success(`pg.Pool ${this.key} ended.`); - } else { - log.info(`pg.Pool ${this.key} already ended.`); - } - } catch (err) { - log.error(`Error ending pg.Pool ${this.key}: ${(err as Error).message}`); - throw err; + if (!this.pool.ended) { + await this.pool.end(); + log.success(`pg.Pool ${this.key} ended.`); + } else { + log.info(`pg.Pool ${this.key} already ended.`); } })(); @@ -68,37 +150,55 @@ class ManagedPgPool { } } +/** + * A lease-aware, lazy pool registry. + * + * JavaScript executes acquisition synchronously, including slot reservation and + * factory invocation. Two callers therefore cannot both claim the final slot. + * Pools may finish ending asynchronously after a zero-lease identity is removed. + */ export class PgPoolCacheManager { - private cleanupTasks: Promise[] = []; + private readonly records = new Map(); + private readonly cleanupTasks = new Set>(); + private readonly cleanupCallbacks = new Set(); + private readonly reservedKeys = new Set(); + private reservations = 0; + private accessOrder = 0; private closed = false; - private cleanupCallbacks: Set = new Set(); readonly config: PgCacheConfig; - private readonly pgCache: LRUCache; + private readonly counters: PgPoolCacheCounters = { + hits: 0, + misses: 0, + poolsCreated: 0, + leasesAcquired: 0, + leasesReleased: 0, + capacityEvictions: 0, + ttlExpirations: 0, + capacityRefusals: 0, + disposalsStarted: 0, + disposalsCompleted: 0, + disposalFailures: 0 + }; constructor(config?: Partial) { const defaults = getPgCacheConfig(); this.config = { ...defaults, ...config }; + if (!Number.isSafeInteger(this.config.max) || this.config.max <= 0) { + throw new Error('pg-cache max must be a positive safe integer'); + } + if (!Number.isFinite(this.config.ttl) || this.config.ttl <= 0) { + throw new Error('pg-cache ttl must be a positive number'); + } + } - this.pgCache = new LRUCache({ - max: this.config.max, - ttl: this.config.ttl, - updateAgeOnGet: true, - dispose: (managedPool, key, reason) => { - log.debug(`Disposing pg pool [${key}] (${reason})`); - this.notifyCleanup(key); - this.disposePool(managedPool); - } - }); + get size(): number { + return this.records.size; } - // Register a cleanup callback to be called when pools are disposed registerCleanupCallback(callback: PoolCleanupCallback): () => void { this.cleanupCallbacks.add(callback); - // Return unregister function - return () => { - this.cleanupCallbacks.delete(callback); - }; + return () => this.cleanupCallbacks.delete(callback); } get(key: PgPoolKey): pg.Pool | undefined { @@ -106,73 +206,315 @@ export class PgPoolCacheManager { log.warn(`Cache is closed, ignoring get(${key})`); return undefined; } - return this.pgCache.get(key)?.pool; + const managedPool = this.getLiveRecord(key, true); + if (!managedPool) { + this.counters.misses++; + return undefined; + } + this.counters.hits++; + return managedPool.pool; } has(key: PgPoolKey): boolean { - return this.pgCache.has(key); + if (this.closed) return false; + return Boolean(this.getLiveRecord(key, false)); } + /** + * Legacy direct insertion. Prefer getOrCreate/acquire so capacity is checked + * before the caller constructs a pool. + */ set(key: PgPoolKey, pool: pg.Pool): void { - if (this.closed) throw new Error(`Cannot add to cache after it has been closed (key: ${key})`); - this.pgCache.set(key, new ManagedPgPool(pool, key)); + this.assertOpen(key); + const existing = this.records.get(key); + if (existing?.pool === pool) { + this.touch(existing); + return; + } + if (existing?.leaseCount) { + throw new Error(`Cannot replace leased pg pool identity ${key}`); + } + if (existing) this.removeRecord(existing, 'replace'); + + const reservation = this.reserveSlot(key); + this.commitReservation(reservation, pool, 0); } - delete(key: PgPoolKey): void { - const managedPool = this.pgCache.get(key); - const existed = this.pgCache.delete(key); - if (!existed && managedPool) { - this.notifyCleanup(key); - this.disposePool(managedPool); + /** Atomically capacity-check, synchronously construct, and cache an idle pool. */ + getOrCreate(key: PgPoolKey, factory: PoolFactory): pg.Pool { + this.assertOpen(key); + const existing = this.getLiveRecord(key, true); + if (existing) { + this.counters.hits++; + return existing.pool; } + + this.counters.misses++; + return this.createWithReservation(key, factory, 0).pool; } + /** + * Atomically get/create and lease an exact identity. A leased identity cannot + * be selected by capacity or TTL eviction until every lease is released. + */ + acquire(key: PgPoolKey, factory: PoolFactory): PgPoolLease { + this.assertOpen(key); + let managedPool = this.getLiveRecord(key, true); + if (managedPool) { + this.counters.hits++; + managedPool.leaseCount++; + } else { + this.counters.misses++; + managedPool = this.createWithReservation(key, factory, 1); + } + this.counters.leasesAcquired++; + return this.makeLease(managedPool); + } + + /** Explicit deletion never interrupts a lease; callers may retry after release. */ + delete(key: PgPoolKey): void { + const managedPool = this.records.get(key); + if (!managedPool || managedPool.leaseCount > 0) return; + this.removeRecord(managedPool, 'delete'); + } + + /** Clear every currently unleased identity. */ clear(): void { - const entries = [...this.pgCache.entries()]; - this.pgCache.clear(); - for (const [key, managedPool] of entries) { - this.notifyCleanup(key); - this.disposePool(managedPool); + for (const managedPool of [...this.records.values()]) { + if (managedPool.leaseCount === 0) this.removeRecord(managedPool, 'clear'); } } async close(): Promise { if (this.closed) return; this.closed = true; - this.clear(); + // Explicit process teardown is the only operation that may override leases. + for (const managedPool of [...this.records.values()]) { + this.removeRecord(managedPool, 'close'); + } await this.waitForDisposals(); - // Re-open the cache so it can accept new entries if the process - // survives the shutdown signal (e.g. during provisioning or restart). + // Preserve the established restart/provisioning behavior. this.closed = false; } async waitForDisposals(): Promise { - if (this.cleanupTasks.length === 0) return; - const tasks = [...this.cleanupTasks]; - this.cleanupTasks = []; - await Promise.allSettled(tasks); + while (this.cleanupTasks.size > 0) { + await Promise.allSettled([...this.cleanupTasks]); + } + } + + getStats(): PgPoolCacheStats { + let leasedPools = 0; + let activeLeases = 0; + for (const managedPool of this.records.values()) { + if (managedPool.leaseCount > 0) leasedPools++; + activeLeases += managedPool.leaseCount; + } + return { + size: this.records.size, + max: this.config.max, + ttl: this.config.ttl, + leasedPools, + idlePools: this.records.size - leasedPools, + activeLeases, + reservations: this.reservations, + pendingDisposals: this.cleanupTasks.size, + ...this.counters + }; + } + + private assertOpen(key: PgPoolKey): void { + if (this.closed) { + throw new Error(`Cannot access pg cache while it is closed (key: ${key})`); + } + } + + private touch(managedPool: ManagedPgPool): void { + managedPool.touch(++this.accessOrder, Date.now(), this.config.ttl); + } + + private getLiveRecord(key: PgPoolKey, updateAge: boolean): ManagedPgPool | undefined { + const managedPool = this.records.get(key); + if (!managedPool) return undefined; + if (managedPool.leaseCount === 0 && managedPool.isExpired(Date.now())) { + this.removeRecord(managedPool, 'ttl'); + return undefined; + } + if (updateAge) this.touch(managedPool); + return managedPool; + } + + private idleRecordsByAge(): ManagedPgPool[] { + return [...this.records.values()] + .filter((managedPool) => managedPool.leaseCount === 0) + .sort((a, b) => a.lastAccessOrder - b.lastAccessOrder); + } + + private reserveSlot(key: PgPoolKey): SlotReservation { + if (this.reservedKeys.has(key)) { + throw new Error(`Re-entrant pg pool acquisition for identity ${key}`); + } + + const overflow = Math.max( + 0, + this.records.size + this.reservations + 1 - this.config.max + ); + const candidates = this.idleRecordsByAge(); + if (candidates.length < overflow) { + this.counters.capacityRefusals++; + throw new PgPoolCapacityError( + this.config.max, + this.records.size + this.reservations, + this.countLeasedPools() + ); + } + + const victims = candidates.slice(0, overflow); + for (const victim of victims) this.records.delete(victim.key); + this.reservations++; + this.reservedKeys.add(key); + return { key, victims }; + } + + private rollbackReservation(reservation: SlotReservation): void { + this.reservations = Math.max(0, this.reservations - 1); + this.reservedKeys.delete(reservation.key); + for (const victim of reservation.victims) { + this.records.set(victim.key, victim); + } + } + + private commitReservation( + reservation: SlotReservation, + pool: pg.Pool, + leaseCount: number + ): ManagedPgPool { + const managedPool = new ManagedPgPool(pool, reservation.key); + managedPool.leaseCount = leaseCount; + this.touch(managedPool); + this.records.set(reservation.key, managedPool); + this.reservations = Math.max(0, this.reservations - 1); + this.reservedKeys.delete(reservation.key); + this.counters.poolsCreated++; + + for (const victim of reservation.victims) { + this.counters.capacityEvictions++; + this.disposeRemovedRecord(victim); + } + return managedPool; + } + + private createWithReservation( + key: PgPoolKey, + factory: PoolFactory, + leaseCount: number + ): ManagedPgPool { + const reservation = this.reserveSlot(key); + let pool: pg.Pool; + try { + pool = factory(); + } catch (error) { + this.rollbackReservation(reservation); + throw error; + } + return this.commitReservation(reservation, pool, leaseCount); + } + + private makeLease(managedPool: ManagedPgPool): PgPoolLease { + let released = false; + return { + pool: managedPool.pool, + identity: managedPool.key, + release: () => { + if (released) return; + released = true; + this.counters.leasesReleased++; + managedPool.leaseCount = Math.max(0, managedPool.leaseCount - 1); + + // close() may already have detached this record. + if (this.records.get(managedPool.key) !== managedPool) return; + if (managedPool.leaseCount > 0) return; + if (managedPool.isExpired(Date.now())) { + this.removeRecord(managedPool, 'ttl'); + return; + } + this.enforceCapacity(); + } + }; + } + + private enforceCapacity(): void { + while (this.records.size > this.config.max) { + const victim = this.idleRecordsByAge()[0]; + if (!victim) return; + this.removeRecord(victim, 'capacity'); + } + } + + private countLeasedPools(): number { + let leased = 0; + for (const managedPool of this.records.values()) { + if (managedPool.leaseCount > 0) leased++; + } + return leased; + } + + private removeRecord( + managedPool: ManagedPgPool, + reason: PgPoolDisposalReason + ): void { + if (this.records.get(managedPool.key) !== managedPool) return; + this.records.delete(managedPool.key); + if (reason === 'capacity') this.counters.capacityEvictions++; + if (reason === 'ttl') this.counters.ttlExpirations++; + this.disposeRemovedRecord(managedPool); + } + + private disposeRemovedRecord(managedPool: ManagedPgPool): void { + this.notifyCleanup(managedPool.key); + + // Alternate drivers may intentionally return one physical pool for multiple + // exact identities. Never end it while another retained identity owns it. + if ([...this.records.values()].some((entry) => entry.pool === managedPool.pool)) { + return; + } + if (managedPool.isDisposed) return; + + this.counters.disposalsStarted++; + let task: Promise; + task = managedPool.dispose() + .then(() => { + this.counters.disposalsCompleted++; + }) + .catch((error) => { + this.counters.disposalFailures++; + log.error( + `Error ending pg.Pool ${managedPool.key}: ${(error as Error).message}` + ); + }) + .finally(() => this.cleanupTasks.delete(task)); + this.cleanupTasks.add(task); } private notifyCleanup(pgPoolKey: string): void { this.cleanupCallbacks.forEach(callback => { try { callback(pgPoolKey); - } catch (err) { - log.error(`Error in cleanup callback for pool ${pgPoolKey}: ${(err as Error).message}`); + } catch (error) { + log.error( + `Error in cleanup callback for pool ${pgPoolKey}: ${(error as Error).message}` + ); } }); } - - private disposePool(managedPool: ManagedPgPool): void { - if (managedPool.isDisposed) return; - const task = managedPool.dispose(); - this.cleanupTasks.push(task); - } } -// Create the singleton instance +// Process-wide registry. Its large capacity is only a key limit; pools and +// PostgreSQL connections remain lazily allocated on first use. export const pgCache = new PgPoolCacheManager(); +export const getPgCacheStats = (): PgPoolCacheStats => pgCache.getStats(); + // --- Graceful Shutdown --- const closePromise: { promise: Promise | null } = { promise: null }; @@ -185,7 +527,6 @@ export const close = async (verbose = false): Promise => { await pgCache.close(); if (verbose) log.success('PG cache disposed.'); } finally { - // Reset so close() can be called again if the process survives. closePromise.promise = null; } })(); @@ -193,13 +534,6 @@ export const close = async (verbose = false): Promise => { return closePromise.promise; }; -SYS_EVENTS.forEach(event => { - process.on(event, () => { - log.info(`Received ${event}`); - close(); - }); -}); - export const teardownPgPools = async (verbose = false): Promise => { return close(verbose); }; diff --git a/postgres/pg-cache/src/notification-broker.ts b/postgres/pg-cache/src/notification-broker.ts new file mode 100644 index 0000000000..cbb2e3afed --- /dev/null +++ b/postgres/pg-cache/src/notification-broker.ts @@ -0,0 +1,1087 @@ +import type { PgConfig, PgPoolConfig } from 'pg-env'; + +import { + assertPgNotificationRoleClient, + type PgNotificationRoleAudit, + type PgNotificationRoleClient, + type PgNotificationRoleContract +} from './notification-role'; +import { + acquirePgPool, + getPgDatabaseTargetIdentity, + getPgPoolConfig, + getPgPoolIdentity +} from './pg'; + +export const PG_NOTIFICATION_BROKER_IDENTITY_VERSION = 'pg-notification-broker:v1'; +export const PG_NOTIFICATION_DATABASE_IDENTITY_VERSION = 'pg-notification-database:v1'; +export const PG_NOTIFICATION_QUEUE_CAPACITY = 256; +export const DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS = 5_000; + +export const PG_NOTIFICATION_TOPIC_ERROR_CODE = 'PG_NOTIFICATION_TOPIC_INVALID'; +export const PG_NOTIFICATION_BROKER_FAILED_ERROR_CODE = 'PG_NOTIFICATION_BROKER_FAILED'; +export const PG_NOTIFICATION_QUEUE_OVERFLOW_ERROR_CODE = 'PG_NOTIFICATION_QUEUE_OVERFLOW'; +export const PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE = 'PG_NOTIFICATION_LEASE_RELEASED'; +export const PG_NOTIFICATION_OPERATION_TIMEOUT_ERROR_CODE = + 'PG_NOTIFICATION_OPERATION_TIMEOUT'; + +type PromiseOrDirect = T | Promise; + +export interface PgNotification { + channel: string; + payload?: string; +} + +export interface PgNotificationClient { + query(text: string, values?: readonly unknown[]): Promise; + on(event: string, listener: (...args: any[]) => void): unknown; + off(event: string, listener: (...args: any[]) => void): unknown; + release(error?: Error | boolean): PromiseOrDirect; +} + +export interface PgNotificationConnectionSource { + connect(): Promise; + release(): PromiseOrDirect; +} + +export interface PgNotificationBrokerLease { + /** Versioned digest of the complete listener connection contract. */ + readonly identity: string; + /** Frozen, exact PostgreSQL channels this lease may subscribe to. */ + readonly topics: readonly string[]; + /** Resolves on fatal broker failure or with null after graceful release. */ + readonly terminated: Promise; + subscribe(topic: string): AsyncIterableIterator; + /** Idempotent and awaited through UNLISTEN and connection release. */ + release(): Promise; +} + +/** + * A production lease whose login was audited on the same pinned PostgreSQL + * client before admission. Arbitrary SQL and the client itself stay private. + */ +export interface PgAttestedNotificationBrokerLease +extends PgNotificationBrokerLease { + readonly roleAudit: PgNotificationRoleAudit; + revalidateRole(): Promise; +} + +export interface AcquirePgNotificationBrokerOptions { + /** Every channel this generation may observe. Prefix matching is never used. */ + topics: readonly string[]; +} + +export type PgNotificationListenerConfig = PgConfig & { pool?: PgPoolConfig }; + +export interface PgNotificationBrokerStats { + brokers: number; + listenerConnections: number; + leases: number; + topics: number; + subscribers: number; + acquisitions: number; + releases: number; + notifications: number; + ignoredNotifications: number; + queueOverflows: number; + fatalFailures: number; + roleAuditAttempts: number; + roleAuditFailures: number; +} + +interface MutableBrokerCounters { + acquisitions: number; + releases: number; + notifications: number; + ignoredNotifications: number; + queueOverflows: number; + fatalFailures: number; + roleAuditAttempts: number; + roleAuditFailures: number; +} + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(error: unknown): void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +export class PgNotificationTopicError extends Error { + readonly code = PG_NOTIFICATION_TOPIC_ERROR_CODE; + + constructor(readonly topic: unknown, reason: string) { + super(`Invalid PostgreSQL notification topic: ${reason}`); + this.name = 'PgNotificationTopicError'; + } +} + +export class PgNotificationBrokerFailedError extends Error { + readonly code = PG_NOTIFICATION_BROKER_FAILED_ERROR_CODE; + + constructor(reason: unknown) { + const cause = reason instanceof Error ? reason : new Error(String(reason)); + super('PostgreSQL notification broker failed; all subscribers were terminated', { + cause + }); + this.name = 'PgNotificationBrokerFailedError'; + } +} + +export class PgNotificationQueueOverflowError extends Error { + readonly code = PG_NOTIFICATION_QUEUE_OVERFLOW_ERROR_CODE; + + constructor( + readonly topic: string, + readonly capacity: number + ) { + super( + `PostgreSQL notification subscriber queue for ${JSON.stringify(topic)} ` + + `exceeded its fixed capacity of ${capacity}` + ); + this.name = 'PgNotificationQueueOverflowError'; + } +} + +export class PgNotificationLeaseReleasedError extends Error { + readonly code = PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE; + + constructor() { + super('PostgreSQL notification broker lease has been released'); + this.name = 'PgNotificationLeaseReleasedError'; + } +} + +export class PgNotificationOperationTimeoutError extends Error { + readonly code = PG_NOTIFICATION_OPERATION_TIMEOUT_ERROR_CODE; + + constructor( + readonly operation: 'role-audit' | 'listen' | 'unlisten', + readonly timeoutMs: number + ) { + super( + `PostgreSQL notification ${operation} exceeded its fixed ${timeoutMs}ms deadline` + ); + this.name = 'PgNotificationOperationTimeoutError'; + } +} + +class BrokerClosedError extends Error {} + +const containsUnpairedSurrogate = (value: string): boolean => { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return true; + index++; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +}; + +/** + * PostgreSQL identifiers are limited to 63 UTF-8 bytes. PostgreSQL truncates + * longer identifiers, so accepting them here could collapse distinct tenant + * topics onto one physical LISTEN channel. + */ +export function assertValidPgNotificationTopic(topic: unknown): asserts topic is string { + if (typeof topic !== 'string') { + throw new PgNotificationTopicError(topic, 'the topic must be a string'); + } + if (topic.length === 0) { + throw new PgNotificationTopicError(topic, 'the topic must not be empty'); + } + if (topic.includes('\0')) { + throw new PgNotificationTopicError(topic, 'NUL bytes are not allowed'); + } + if (containsUnpairedSurrogate(topic)) { + throw new PgNotificationTopicError(topic, 'unpaired UTF-16 surrogates are not allowed'); + } + const bytes = Buffer.byteLength(topic, 'utf8'); + if (bytes > 63) { + throw new PgNotificationTopicError( + topic, + `the UTF-8 encoding is ${bytes} bytes; PostgreSQL allows at most 63` + ); + } +} + +const normalizeTopics = (topics: readonly string[]): readonly string[] => { + if (!Array.isArray(topics) || topics.length === 0) { + throw new PgNotificationTopicError(topics, 'at least one exact topic is required'); + } + for (const topic of topics) assertValidPgNotificationTopic(topic); + return Object.freeze([...new Set(topics)]); +}; + +const quoteIdentifier = (identifier: string): string => + `"${identifier.replace(/"/g, '""')}"`; + +class BoundedNotificationQueue implements AsyncIterableIterator { + private readonly buffered: string[] = []; + private readonly waiting: Deferred>[] = []; + private terminal: 'open' | 'complete' | 'failed' = 'open'; + private failure: Error | null = null; + + constructor( + private readonly topic: string, + private readonly capacity: number, + private readonly onClose: () => void, + private readonly onOverflow: () => void + ) {} + + [Symbol.asyncIterator](): AsyncIterableIterator { + return this; + } + + next(): Promise> { + const buffered = this.buffered.shift(); + if (buffered !== undefined) { + return Promise.resolve({ done: false, value: buffered }); + } + if (this.terminal === 'failed') return Promise.reject(this.failure); + if (this.terminal === 'complete') { + return Promise.resolve({ done: true, value: undefined }); + } + + const result = deferred>(); + this.waiting.push(result); + return result.promise; + } + + return(value?: unknown): Promise> { + this.complete(); + return Promise.resolve({ done: true, value: value as string }); + } + + throw(error?: unknown): Promise> { + const failure = error instanceof Error ? error : new Error(String(error)); + this.fail(failure); + return Promise.reject(failure); + } + + push(payload: string): void { + if (this.terminal !== 'open') return; + const waiter = this.waiting.shift(); + if (waiter) { + waiter.resolve({ done: false, value: payload }); + return; + } + if (this.buffered.length >= this.capacity) { + this.onOverflow(); + this.fail(new PgNotificationQueueOverflowError(this.topic, this.capacity)); + return; + } + this.buffered.push(payload); + } + + complete(): void { + if (this.terminal !== 'open') return; + this.terminal = 'complete'; + this.buffered.length = 0; + for (const waiter of this.waiting.splice(0)) { + waiter.resolve({ done: true, value: undefined }); + } + this.onClose(); + } + + fail(error: Error): void { + if (this.terminal !== 'open') return; + this.terminal = 'failed'; + this.failure = error; + this.buffered.length = 0; + for (const waiter of this.waiting.splice(0)) waiter.reject(error); + this.onClose(); + } +} + +type BrokerState = 'new' | 'active' | 'failed' | 'closing' | 'closed'; +type ConnectionSourceFactory = () => PromiseOrDirect; +type NotificationOperation = PgNotificationOperationTimeoutError['operation']; + +const MAX_TIMER_DELAY_MS = 2_147_483_647; + +const assertNotificationOperationTimeoutMs = (timeoutMs: number): number => { + if ( + !Number.isSafeInteger(timeoutMs) + || timeoutMs <= 0 + || timeoutMs > MAX_TIMER_DELAY_MS + ) { + throw new TypeError( + 'PostgreSQL notification operation timeout must be an integer ' + + `between 1 and ${MAX_TIMER_DELAY_MS}` + ); + } + return timeoutMs; +}; + +const getNotificationOperationTimeoutMs = ( + listenerPgConfig: PgNotificationListenerConfig +): number => { + // Preserve this API's narrower deadline contract and stable error before the + // generic pool validator runs as part of identity construction. + const configured = listenerPgConfig.pool?.connectionTimeoutMillis; + return assertNotificationOperationTimeoutMs( + configured ?? getPgPoolConfig(listenerPgConfig.pool).connectionTimeoutMillis + ?? DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS + ); +}; + +class NotificationBrokerLease implements PgAttestedNotificationBrokerLease { + readonly topics: readonly string[]; + readonly terminated: Promise; + private readonly allowedTopics: ReadonlySet; + private readonly termination = deferred(); + private readonly queues = new Map>(); + private audit: PgNotificationRoleAudit | null = null; + private released = false; + private releasePromise: Promise | null = null; + + constructor( + readonly identity: string, + topics: readonly string[], + private readonly broker: NotificationBrokerRecord, + private readonly queueCapacity: number, + private readonly counters: MutableBrokerCounters, + readonly roleContract: Readonly | null + ) { + this.topics = topics; + this.terminated = this.termination.promise; + this.allowedTopics = new Set(topics); + } + + get subscriberCount(): number { + let count = 0; + for (const topicQueues of this.queues.values()) count += topicQueues.size; + return count; + } + + get isReleased(): boolean { + return this.released; + } + + get roleAudit(): PgNotificationRoleAudit { + if (!this.audit) { + throw new Error('PostgreSQL notification broker lease is not role-attested'); + } + return this.audit; + } + + setRoleAudit(audit: PgNotificationRoleAudit): void { + this.audit = audit; + } + + revalidateRole(): Promise { + if (this.released) return Promise.reject(new PgNotificationLeaseReleasedError()); + if (!this.roleContract) { + return Promise.reject( + new Error('PostgreSQL notification broker lease is not role-attested') + ); + } + return this.broker.revalidateLeaseRole(this); + } + + subscribe(topic: string): AsyncIterableIterator { + if (this.released) throw new PgNotificationLeaseReleasedError(); + this.broker.assertAvailable(); + if (!this.allowedTopics.has(topic)) { + throw new PgNotificationTopicError( + topic, + 'the topic is not in this lease\'s exact allowlist' + ); + } + + let topicQueues = this.queues.get(topic); + if (!topicQueues) { + topicQueues = new Set(); + this.queues.set(topic, topicQueues); + } + let queue!: BoundedNotificationQueue; + queue = new BoundedNotificationQueue( + topic, + this.queueCapacity, + () => { + topicQueues!.delete(queue); + if (topicQueues!.size === 0) this.queues.delete(topic); + }, + () => { + this.counters.queueOverflows++; + } + ); + topicQueues.add(queue); + return queue; + } + + dispatch(topic: string, payload: string): void { + const queues = this.queues.get(topic); + if (!queues) return; + for (const queue of [...queues]) queue.push(payload); + } + + fail(error: PgNotificationBrokerFailedError): void { + this.termination.resolve(error); + for (const queues of [...this.queues.values()]) { + for (const queue of [...queues]) queue.fail(error); + } + } + + release(): Promise { + if (this.releasePromise) return this.releasePromise; + this.released = true; + for (const queues of [...this.queues.values()]) { + for (const queue of [...queues]) queue.complete(); + } + this.releasePromise = this.broker.releaseLease(this); + void this.releasePromise.then( + () => this.termination.resolve(null), + (error) => this.termination.resolve( + error instanceof PgNotificationBrokerFailedError + ? error + : new PgNotificationBrokerFailedError(error) + ) + ); + return this.releasePromise; + } +} + +class NotificationBrokerRecord { + private state: BrokerState = 'new'; + private acceptingLeases = true; + private operation: Promise = Promise.resolve(); + private source: PgNotificationConnectionSource | null = null; + private client: PgNotificationClient | null = null; + private clientCleanup: Promise | null = null; + private sourceCleanup: Promise | null = null; + private fatalError: PgNotificationBrokerFailedError | null = null; + private readonly leases = new Set(); + private readonly topicReferences = new Map(); + /** Includes provisional LISTENs whose lease admission has not committed yet. */ + private readonly listenedTopics = new Set(); + + private readonly onNotification = (notification: PgNotification): void => { + if (this.state !== 'active') return; + if ( + !notification + || typeof notification.channel !== 'string' + || ( + notification.payload !== undefined + && typeof notification.payload !== 'string' + ) + ) { + this.markFailed(new Error('PostgreSQL listener emitted a malformed notification')); + return; + } + if (!this.topicReferences.has(notification.channel)) { + this.counters.ignoredNotifications++; + return; + } + this.counters.notifications++; + const payload = notification.payload ?? ''; + for (const lease of [...this.leases]) { + lease.dispatch(notification.channel, payload); + } + }; + + private readonly onClientError = (error: unknown): void => { + this.markFailed(error); + }; + + private readonly onClientEnd = (): void => { + this.markFailed(new Error('PostgreSQL notification listener connection ended')); + }; + + constructor( + readonly identity: string, + private readonly sourcePromise: Promise, + private readonly queueCapacity: number, + private readonly operationTimeoutMs: number, + private readonly counters: MutableBrokerCounters, + private readonly onTerminal: (record: NotificationBrokerRecord) => void + ) {} + + get snapshot(): Pick< + PgNotificationBrokerStats, + 'listenerConnections' | 'leases' | 'topics' | 'subscribers' + > { + let subscribers = 0; + for (const lease of this.leases) subscribers += lease.subscriberCount; + return { + listenerConnections: this.client ? 1 : 0, + leases: this.leases.size, + topics: this.topicReferences.size, + subscribers + }; + } + + assertAvailable(): void { + if (this.state === 'failed') throw this.fatalError!; + if (this.state !== 'active') throw new PgNotificationLeaseReleasedError(); + } + + async acquire( + topics: readonly string[], + roleContract: Readonly | null = null + ): Promise { + const lease = new NotificationBrokerLease( + this.identity, + topics, + this, + this.queueCapacity, + this.counters, + roleContract + ); + await this.enqueue(async () => { + if (!this.acceptingLeases) throw new BrokerClosedError(); + if (this.state === 'failed') throw this.fatalError!; + if (this.state === 'closing' || this.state === 'closed') { + throw new BrokerClosedError(); + } + const client = await this.ensureClient(); + if (!this.acceptingLeases) throw new BrokerClosedError(); + if (roleContract) { + lease.setRoleAudit(await this.auditRole(client, roleContract)); + } + if (!this.acceptingLeases) throw new BrokerClosedError(); + for (const topic of topics) { + if ((this.topicReferences.get(topic) ?? 0) === 0) { + await this.executeListenerQuery(client, `LISTEN ${quoteIdentifier(topic)}`); + this.listenedTopics.add(topic); + } + } + if (!this.acceptingLeases || this.state !== 'active') { + if (this.fatalError) throw this.fatalError; + throw new BrokerClosedError(); + } + for (const topic of topics) { + this.topicReferences.set(topic, (this.topicReferences.get(topic) ?? 0) + 1); + } + this.leases.add(lease); + this.counters.acquisitions++; + }); + return lease; + } + + async revalidateLeaseRole( + lease: NotificationBrokerLease + ): Promise { + return this.enqueue(async () => { + if (lease.isReleased || !this.leases.has(lease)) { + throw new PgNotificationLeaseReleasedError(); + } + if (this.state === 'failed') throw this.fatalError!; + if (this.state !== 'active' || !this.client || !lease.roleContract) { + throw new PgNotificationLeaseReleasedError(); + } + const audit = await this.auditRole(this.client, lease.roleContract); + lease.setRoleAudit(audit); + return audit; + }); + } + + async releaseLease(lease: NotificationBrokerLease): Promise { + return this.enqueue(async () => { + if (!this.leases.delete(lease)) return; + this.counters.releases++; + + const topicsToUnlisten: string[] = []; + for (const topic of lease.topics) { + const next = (this.topicReferences.get(topic) ?? 0) - 1; + if (next <= 0) { + this.topicReferences.delete(topic); + topicsToUnlisten.push(topic); + } else { + this.topicReferences.set(topic, next); + } + } + + let releaseError: Error | null = null; + if (this.state === 'active' && this.client) { + for (const topic of topicsToUnlisten) { + try { + await this.executeListenerQuery( + this.client, + `UNLISTEN ${quoteIdentifier(topic)}` + ); + this.listenedTopics.delete(topic); + } catch (error) { + releaseError = this.fatalError + ?? new PgNotificationBrokerFailedError(error); + break; + } + } + } + + if (this.leases.size === 0) await this.closeUnused(); + if (releaseError) throw releaseError; + }); + } + + async closeAll(): Promise { + this.acceptingLeases = false; + // Cross the serialized-operation barrier before snapshotting leases. This + // either rejects an acquisition already waiting on connect/LISTEN or makes + // its completed lease visible to the release snapshot below. + await this.enqueue((): void => undefined); + const releases = [...this.leases].map((lease) => lease.release()); + const releaseResults = await Promise.allSettled(releases); + let finalCleanupError: unknown; + let finalCleanupFailed = false; + try { + await this.enqueue(() => this.closeUnused()); + } catch (error) { + finalCleanupError = error; + finalCleanupFailed = true; + } + const failedRelease = releaseResults.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ); + if (failedRelease) throw failedRelease.reason; + if (finalCleanupFailed) throw finalCleanupError; + } + + async closeIfUnused(): Promise { + await this.enqueue(() => this.closeUnused()); + } + + private enqueue(operation: () => PromiseOrDirect): Promise { + const pending = this.operation.then(operation, operation); + this.operation = pending.then( + (): void => undefined, + (): void => undefined + ); + return pending; + } + + private async ensureClient(): Promise { + if (this.client) return this.client; + try { + this.source = await this.sourcePromise; + if (this.fatalError) throw this.fatalError; + const client = await this.source.connect(); + this.client = client; + client.on('notification', this.onNotification); + client.on('error', this.onClientError); + client.on('end', this.onClientEnd); + this.state = 'active'; + return client; + } catch (error) { + this.markFailed(error); + await this.clientCleanup; + throw this.fatalError!; + } + } + + private async executeListenerQuery( + client: PgNotificationClient, + text: string + ): Promise { + try { + await this.runWithOperationDeadline( + text.startsWith('UNLISTEN') ? 'unlisten' : 'listen', + () => client.query(text) + ); + if (this.state === 'failed') throw this.fatalError!; + } catch (error) { + this.markFailed(error); + await this.clientCleanup; + throw this.fatalError!; + } + } + + private async auditRole( + client: PgNotificationClient, + contract: Readonly + ): Promise { + this.counters.roleAuditAttempts++; + try { + const audit = await this.runWithOperationDeadline( + 'role-audit', + () => assertPgNotificationRoleClient( + client as unknown as PgNotificationRoleClient, + contract + ) + ); + if (this.state === 'failed') throw this.fatalError!; + return audit; + } catch (error) { + this.counters.roleAuditFailures++; + this.markFailed(error); + await this.clientCleanup; + // Preserve the stable unsafe-role error for startup and attestation + // diagnostics. Active leases separately observe the broker-failed latch. + throw error; + } + } + + private async runWithOperationDeadline( + operation: NotificationOperation, + task: () => PromiseOrDirect + ): Promise { + let timer: ReturnType | null = null; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + const error = new PgNotificationOperationTimeoutError( + operation, + this.operationTimeoutMs + ); + // Latch failure and start client destruction at the exact deadline. The + // driver promise remains observed below, so a later rejection is safe. + this.markFailed(error); + reject(error); + }, this.operationTimeoutMs); + timer.unref?.(); + }); + // Promise.race installs a rejection handler on the driver query. If the + // deadline wins, destroying the client may settle that abandoned query + // later without producing an unhandled rejection. + const operationPromise = Promise.resolve().then(task); + try { + return await Promise.race([operationPromise, timeout]); + } finally { + if (timer) clearTimeout(timer); + } + } + + private markFailed(reason: unknown): void { + if ( + this.state === 'failed' + || this.state === 'closing' + || this.state === 'closed' + ) return; + this.state = 'failed'; + this.fatalError = reason instanceof PgNotificationBrokerFailedError + ? reason + : new PgNotificationBrokerFailedError(reason); + this.counters.fatalFailures++; + for (const lease of [...this.leases]) lease.fail(this.fatalError); + + const client = this.client; + this.client = null; + if (client) this.clientCleanup = this.releaseClient(client, this.fatalError); + } + + private async releaseClient( + client: PgNotificationClient, + error?: Error, + destroy = false + ): Promise { + client.off('notification', this.onNotification); + client.off('end', this.onClientEnd); + try { + await client.release(error ?? (destroy ? true : undefined)); + } catch (releaseError) { + // The broker is already failed or closing. Source cleanup below remains + // mandatory, and the original delivery failure is the useful error. + if (!error) throw releaseError; + } finally { + client.off('error', this.onClientError); + } + } + + private async closeUnused(): Promise { + if (this.leases.size > 0 || this.state === 'closed') return; + + let cleanupError: unknown; + let cleanupFailed = false; + if (this.client && this.listenedTopics.size > 0) { + try { + // This also covers a shutdown racing between a successful LISTEN and + // lease admission, where no committed topic reference exists yet. + await this.executeListenerQuery( + this.client, + 'UNLISTEN *' + ); + this.listenedTopics.clear(); + } catch (error) { + cleanupError = error; + cleanupFailed = true; + } + } + if (this.state !== 'failed') this.state = 'closing'; + + const client = this.client; + this.client = null; + if (client) { + const releaseError = cleanupFailed + ? new PgNotificationBrokerFailedError(cleanupError) + : undefined; + // Once the last exact-generation lease is gone, retaining an idle + // listener backend only delays PostgreSQL memory reclamation. Destroy it + // after UNLISTEN; the identity-only pool can create a fresh client later. + this.clientCleanup = this.releaseClient(client, releaseError, true); + } + try { + if (this.clientCleanup) await this.clientCleanup; + } catch (error) { + cleanupError = error; + cleanupFailed = true; + } + + if (this.source && !this.sourceCleanup) { + const source = this.source; + this.source = null; + this.sourceCleanup = Promise.resolve(source.release()); + } + try { + if (this.sourceCleanup) await this.sourceCleanup; + } catch (error) { + if (!cleanupFailed) cleanupError = error; + cleanupFailed = true; + } + + this.state = 'closed'; + this.onTerminal(this); + if (cleanupFailed) throw cleanupError; + } +} + +/** + * Registry implementation exposed for deterministic unit tests. Production + * callers must use acquirePgNotificationBroker so identity and pool ownership + * always come from the canonical PgConfig path. + * + * @internal + */ +export class PgNotificationBrokerRegistry { + private readonly records = new Map(); + private closed = false; + private closePromise: Promise | null = null; + private readonly counters: MutableBrokerCounters = { + acquisitions: 0, + releases: 0, + notifications: 0, + ignoredNotifications: 0, + queueOverflows: 0, + fatalFailures: 0, + roleAuditAttempts: 0, + roleAuditFailures: 0 + }; + + constructor( + private readonly queueCapacity = PG_NOTIFICATION_QUEUE_CAPACITY, + private readonly defaultOperationTimeoutMs = + DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS + ) { + if (!Number.isSafeInteger(queueCapacity) || queueCapacity <= 0) { + throw new Error('PostgreSQL notification queue capacity must be a positive safe integer'); + } + assertNotificationOperationTimeoutMs(defaultOperationTimeoutMs); + } + + async acquireForTests( + identity: string, + sourceFactory: ConnectionSourceFactory, + topics: readonly string[], + operationTimeoutMs = this.defaultOperationTimeoutMs + ): Promise { + return this.acquireInternal( + identity, + sourceFactory, + topics, + null, + operationTimeoutMs + ); + } + + /** @internal Exercise production attestation without constructing PgConfig. */ + async acquireAttestedForTests( + identity: string, + sourceFactory: ConnectionSourceFactory, + topics: readonly string[], + roleContract: PgNotificationRoleContract, + operationTimeoutMs = this.defaultOperationTimeoutMs + ): Promise { + return this.acquireInternal( + identity, + sourceFactory, + topics, + roleContract, + operationTimeoutMs + ); + } + + private async acquireInternal( + identity: string, + sourceFactory: ConnectionSourceFactory, + topics: readonly string[], + roleContract: PgNotificationRoleContract | null, + operationTimeoutMs: number + ): Promise { + if (this.closed) throw new Error('PostgreSQL notification broker registry is closed'); + if (typeof identity !== 'string' || identity.length === 0) { + throw new Error('PostgreSQL notification broker identity must be a non-empty string'); + } + const normalizedTopics = normalizeTopics(topics); + const normalizedOperationTimeoutMs = assertNotificationOperationTimeoutMs( + operationTimeoutMs + ); + + for (;;) { + let record = this.records.get(identity); + if (!record) { + const sourcePromise = Promise.resolve(sourceFactory()); + // Acquisition consumes this immediately, but guard the small interval + // before its serialized operation attaches a rejection handler. + void sourcePromise.catch(() => {}); + record = new NotificationBrokerRecord( + identity, + sourcePromise, + this.queueCapacity, + normalizedOperationTimeoutMs, + this.counters, + (terminal) => { + if (this.records.get(identity) === terminal) this.records.delete(identity); + } + ); + this.records.set(identity, record); + } + try { + const lease = await record.acquire(normalizedTopics, roleContract); + if (this.closed) { + await lease.release(); + throw new Error('PostgreSQL notification broker registry is closed'); + } + return lease; + } catch (error) { + if (error instanceof BrokerClosedError && !this.closed) continue; + // A failed broker remains pinned until every existing owner explicitly + // releases it. This prevents an acquisition attempt from silently + // replacing a listener after a possible notification gap. + await record.closeIfUnused(); + if (error instanceof BrokerClosedError && this.closed) { + throw new Error('PostgreSQL notification broker registry is closed'); + } + throw error; + } + } + } + + stats(): PgNotificationBrokerStats { + let listenerConnections = 0; + let leases = 0; + let topics = 0; + let subscribers = 0; + for (const record of this.records.values()) { + const snapshot = record.snapshot; + listenerConnections += snapshot.listenerConnections; + leases += snapshot.leases; + topics += snapshot.topics; + subscribers += snapshot.subscribers; + } + return { + brokers: this.records.size, + listenerConnections, + leases, + topics, + subscribers, + ...this.counters + }; + } + + close(): Promise { + if (this.closePromise) return this.closePromise; + this.closed = true; + this.closePromise = (async () => { + const closeResults = await Promise.allSettled( + [...this.records.values()].map((record) => record.closeAll()) + ); + this.records.clear(); + const failedClose = closeResults.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ); + if (failedClose) throw failedClose.reason; + })(); + return this.closePromise; + } +} + +let brokerRegistry = new PgNotificationBrokerRegistry(); +let brokerTeardownTail: Promise = Promise.resolve(); + +/** Opaque identity over the complete canonical listener pool contract. */ +export const getPgNotificationBrokerIdentity = ( + listenerPgConfig: PgNotificationListenerConfig +): string => { + // The operation deadline is represented by the pool connection timeout in + // the identity below. Validate it before publishing an apparently usable key. + getNotificationOperationTimeoutMs(listenerPgConfig); + const poolIdentity = getPgPoolIdentity(listenerPgConfig, { + purpose: 'notification-broker', + sanitizeOnCheckout: true + }); + return `${PG_NOTIFICATION_BROKER_IDENTITY_VERSION}:${poolIdentity}`; +}; + +/** + * Opaque identity for one physical database target, deliberately excluding + * credentials, TLS policy, pool sizing, and checkout behavior. Those inputs + * split listener pools, but must not let two active listener contracts silently + * fragment one database's broker. + */ +export const getPgNotificationDatabaseIdentity = ( + listenerPgConfig: PgNotificationListenerConfig +): string => { + const targetIdentity = getPgDatabaseTargetIdentity(listenerPgConfig); + return `${PG_NOTIFICATION_DATABASE_IDENTITY_VERSION}:${targetIdentity}`; +}; + +/** + * Acquire a generation lease over one process-local listener. The supplied + * config must name the dedicated least-privilege notification login; this API + * never falls back to a request runtime or control-plane credential. + */ +export const acquirePgNotificationBroker = async ( + listenerPgConfig: PgNotificationListenerConfig, + options: AcquirePgNotificationBrokerOptions +): Promise => { + const operationTimeoutMs = getNotificationOperationTimeoutMs(listenerPgConfig); + const identity = getPgNotificationBrokerIdentity(listenerPgConfig); + return brokerRegistry.acquireAttestedForTests( + identity, + () => { + const poolLease = acquirePgPool(listenerPgConfig, { + purpose: 'notification-broker', + sanitizeOnCheckout: true + }); + return { + connect: () => poolLease.pool.connect() as Promise, + release: () => poolLease.release() + }; + }, + options.topics, + { + role: listenerPgConfig.user, + database: listenerPgConfig.database + }, + operationTimeoutMs + ); +}; + +export const getPgNotificationBrokerStats = (): PgNotificationBrokerStats => + brokerRegistry.stats(); + +/** Await every UNLISTEN and checked-out connection release, then reset. */ +export const teardownPgNotificationBrokers = (): Promise => { + const closing = brokerRegistry; + brokerRegistry = new PgNotificationBrokerRegistry(); + const teardown = brokerTeardownTail.then(() => closing.close()); + // A later teardown must wait until this registry has fully drained even when + // this caller observes a cleanup failure. + brokerTeardownTail = teardown.then( + (): void => undefined, + (): void => undefined + ); + return teardown; +}; diff --git a/postgres/pg-cache/src/notification-role.ts b/postgres/pg-cache/src/notification-role.ts new file mode 100644 index 0000000000..cd6398c917 --- /dev/null +++ b/postgres/pg-cache/src/notification-role.ts @@ -0,0 +1,419 @@ +import type { Pool, PoolClient, QueryResult } from 'pg'; + +export const PG_NOTIFICATION_ROLE_AUDIT_VERSION = 'pg-notification-role:v1'; +export const PG_NOTIFICATION_ROLE_UNSAFE_ERROR_CODE = 'PG_NOTIFICATION_ROLE_UNSAFE'; +export const PG_NOTIFICATION_ROLE_CONTRACT_ERROR_CODE = + 'PG_NOTIFICATION_ROLE_CONTRACT_INVALID'; + +export type PgNotificationRoleViolationCode = + | 'LOGIN_ROLE_MISMATCH' + | 'CURRENT_ROLE_MISMATCH' + | 'DATABASE_MISMATCH' + | 'LOGIN_REQUIRED' + | 'NOINHERIT_REQUIRED' + | 'SUPERUSER' + | 'BYPASSRLS' + | 'CREATEROLE' + | 'CREATEDB' + | 'REPLICATION' + | 'ROLE_MEMBERSHIP' + | 'TARGET_DATABASE_MISSING' + | 'TARGET_CONNECT_REQUIRED' + | 'CROSS_DATABASE_CONNECT' + | 'DATABASE_OWNER' + | 'DATABASE_CREATE' + | 'DATABASE_TEMP' + | 'SCHEMA_OWNER' + | 'SCHEMA_CREATE' + | 'SCHEMA_USAGE' + | 'RELATION_PRIVILEGE' + | 'FUNCTION_PRIVILEGE' + | 'SEQUENCE_PRIVILEGE' + | 'AUDIT_NO_RESULT'; + +/** Credential-free identity expected from one dedicated listener login. */ +export interface PgNotificationRoleContract { + role: string; + database: string; +} + +/** Safe to persist in diagnostics: connection secrets/config are never copied. */ +export interface PgNotificationRoleAudit { + version: typeof PG_NOTIFICATION_ROLE_AUDIT_VERSION; + role: string; + database: string; + safe: boolean; + violations: readonly PgNotificationRoleViolationCode[]; +} + +/** Catalog-query capability used by the broker's pinned LISTEN client. */ +export type PgNotificationRoleClient = Pick; + +interface PgNotificationRoleAuditRow { + expected_role: string; + session_role: string; + active_role: string; + active_database: string; + rolcanlogin: boolean; + rolinherit: boolean; + rolsuper: boolean; + rolbypassrls: boolean; + rolcreaterole: boolean; + rolcreatedb: boolean; + rolreplication: boolean; + membership_count: number; + target_database_exists: boolean; + target_connect: boolean; + other_database_connect_count: number; + target_database_owner: boolean; + target_database_create: boolean; + target_database_temp: boolean; + schema_owner_count: number; + schema_create_count: number; + schema_usage_count: number; + relation_privilege_count: number; + function_privilege_count: number; + sequence_privilege_count: number; +} + +/** + * Audit only the session login's effective privileges. PostgreSQL system + * schemas/objects are excluded because ordinary logins necessarily use the + * catalog; every non-system schema and object remains in scope. + */ +export const PG_NOTIFICATION_ROLE_AUDIT_SQL = ` +WITH login_role AS MATERIALIZED ( + SELECT r.oid, r.rolname, r.rolcanlogin, r.rolinherit, r.rolsuper, + r.rolbypassrls, r.rolcreaterole, r.rolcreatedb, r.rolreplication + FROM pg_catalog.pg_roles r + WHERE r.rolname = session_user +), target_database AS MATERIALIZED ( + SELECT d.oid, d.datname, d.datdba + FROM pg_catalog.pg_database d + WHERE d.datname = $2::text +), application_schemas AS MATERIALIZED ( + SELECT n.oid, n.nspname, n.nspowner + FROM pg_catalog.pg_namespace n + WHERE n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_' +) +SELECT $1::text AS expected_role, + session_user AS session_role, + current_user AS active_role, + pg_catalog.current_database() AS active_database, + r.rolcanlogin, + r.rolinherit, + r.rolsuper, + r.rolbypassrls, + r.rolcreaterole, + r.rolcreatedb, + r.rolreplication, + ( + SELECT count(*)::int + FROM pg_catalog.pg_auth_members membership + WHERE membership.member = r.oid OR membership.roleid = r.oid + ) AS membership_count, + (target.oid IS NOT NULL) AS target_database_exists, + COALESCE( + pg_catalog.has_database_privilege(r.rolname, target.oid, 'CONNECT'), + false + ) AS target_connect, + ( + SELECT count(*)::int + FROM pg_catalog.pg_database database_record + WHERE database_record.datname <> $2::text + AND pg_catalog.has_database_privilege( + r.rolname, + database_record.oid, + 'CONNECT' + ) + ) AS other_database_connect_count, + COALESCE(target.datdba = r.oid, false) AS target_database_owner, + COALESCE( + pg_catalog.has_database_privilege(r.rolname, target.oid, 'CREATE'), + false + ) AS target_database_create, + COALESCE( + pg_catalog.has_database_privilege(r.rolname, target.oid, 'TEMP'), + false + ) AS target_database_temp, + ( + SELECT count(*)::int + FROM application_schemas schema_record + WHERE schema_record.nspowner = r.oid + ) AS schema_owner_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + WHERE pg_catalog.has_schema_privilege( + r.rolname, + schema_record.oid, + 'CREATE' + ) + ) AS schema_create_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + WHERE pg_catalog.has_schema_privilege( + r.rolname, + schema_record.oid, + 'USAGE' + ) + ) AS schema_usage_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + INNER JOIN pg_catalog.pg_class relation + ON relation.relnamespace = schema_record.oid + WHERE CASE WHEN relation.relkind IN ('r', 'p', 'v', 'm', 'f') + THEN pg_catalog.has_table_privilege( + r.rolname, + relation.oid, + 'SELECT,INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER' + ) + OR pg_catalog.has_any_column_privilege( + r.rolname, + relation.oid, + 'SELECT,INSERT,UPDATE,REFERENCES' + ) + ELSE false + END + ) AS relation_privilege_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + INNER JOIN pg_catalog.pg_proc routine + ON routine.pronamespace = schema_record.oid + WHERE pg_catalog.has_function_privilege( + r.rolname, + routine.oid, + 'EXECUTE' + ) + ) AS function_privilege_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + INNER JOIN pg_catalog.pg_class sequence_record + ON sequence_record.relnamespace = schema_record.oid + WHERE CASE WHEN sequence_record.relkind = 'S' + THEN pg_catalog.has_sequence_privilege( + r.rolname, + sequence_record.oid, + 'USAGE,SELECT,UPDATE' + ) + ELSE false + END + ) AS sequence_privilege_count +FROM login_role r +LEFT JOIN target_database target ON true +`; + +const containsUnpairedSurrogate = (value: string): boolean => { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return true; + index++; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +}; + +const assertIdentifier = (kind: 'role' | 'database', value: unknown): string => { + if (typeof value !== 'string' || value.length === 0) { + throw new PgNotificationRoleContractError(`${kind} must be a non-empty string`); + } + if (value.includes('\0') || containsUnpairedSurrogate(value)) { + throw new PgNotificationRoleContractError(`${kind} is not a valid PostgreSQL name`); + } + const bytes = Buffer.byteLength(value, 'utf8'); + if (bytes > 63) { + throw new PgNotificationRoleContractError( + `${kind} is ${bytes} UTF-8 bytes; PostgreSQL allows at most 63` + ); + } + return value; +}; + +const normalizeContract = ( + contract: PgNotificationRoleContract +): Readonly => Object.freeze({ + role: assertIdentifier('role', contract?.role), + database: assertIdentifier('database', contract?.database) +}); + +export class PgNotificationRoleContractError extends Error { + readonly code = PG_NOTIFICATION_ROLE_CONTRACT_ERROR_CODE; + + constructor(reason: string) { + super(`Invalid PostgreSQL notification-role contract: ${reason}`); + this.name = 'PgNotificationRoleContractError'; + } +} + +export class UnsafePgNotificationRoleError extends Error { + readonly code = PG_NOTIFICATION_ROLE_UNSAFE_ERROR_CODE; + + constructor( + readonly audit: PgNotificationRoleAudit + ) { + super( + `PostgreSQL notification role ${JSON.stringify(audit.role)} for database ` + + `${JSON.stringify(audit.database)} is unsafe: ${audit.violations.join(',')}` + ); + this.name = 'UnsafePgNotificationRoleError'; + } +} + +/** + * Enforce a one-to-one role/database mapping without accepting connection + * config. Exact duplicate pairs are collapsed for multi-generation reuse. + */ +export const normalizePgNotificationRoleContracts = ( + contracts: readonly PgNotificationRoleContract[] +): readonly Readonly[] => { + if (!Array.isArray(contracts) || contracts.length === 0) { + throw new PgNotificationRoleContractError('at least one role/database pair is required'); + } + const byDatabase = new Map(); + const byRole = new Map(); + const unique = new Map>(); + for (const candidate of contracts) { + const contract = normalizeContract(candidate); + const databaseRole = byDatabase.get(contract.database); + if (databaseRole && databaseRole !== contract.role) { + throw new PgNotificationRoleContractError( + `database ${JSON.stringify(contract.database)} maps to multiple login roles` + ); + } + const roleDatabase = byRole.get(contract.role); + if (roleDatabase && roleDatabase !== contract.database) { + throw new PgNotificationRoleContractError( + `login role ${JSON.stringify(contract.role)} maps to multiple databases` + ); + } + byDatabase.set(contract.database, contract.role); + byRole.set(contract.role, contract.database); + unique.set(`${contract.database}\0${contract.role}`, contract); + } + return Object.freeze( + [...unique.values()].sort((left, right) => { + if (left.database !== right.database) { + return left.database < right.database ? -1 : 1; + } + if (left.role === right.role) return 0; + return left.role < right.role ? -1 : 1; + }) + ); +}; + +const violationCodes = ( + row: PgNotificationRoleAuditRow | undefined, + contract: Readonly +): PgNotificationRoleViolationCode[] => { + if (!row) return ['AUDIT_NO_RESULT']; + const violations: PgNotificationRoleViolationCode[] = []; + if (row.session_role !== contract.role) violations.push('LOGIN_ROLE_MISMATCH'); + if (row.active_role !== row.session_role) violations.push('CURRENT_ROLE_MISMATCH'); + if (row.active_database !== contract.database) violations.push('DATABASE_MISMATCH'); + if (!row.rolcanlogin) violations.push('LOGIN_REQUIRED'); + if (row.rolinherit) violations.push('NOINHERIT_REQUIRED'); + if (row.rolsuper) violations.push('SUPERUSER'); + if (row.rolbypassrls) violations.push('BYPASSRLS'); + if (row.rolcreaterole) violations.push('CREATEROLE'); + if (row.rolcreatedb) violations.push('CREATEDB'); + if (row.rolreplication) violations.push('REPLICATION'); + if (row.membership_count > 0) violations.push('ROLE_MEMBERSHIP'); + if (!row.target_database_exists) violations.push('TARGET_DATABASE_MISSING'); + if (!row.target_connect) violations.push('TARGET_CONNECT_REQUIRED'); + if (row.other_database_connect_count > 0) violations.push('CROSS_DATABASE_CONNECT'); + if (row.target_database_owner) violations.push('DATABASE_OWNER'); + if (row.target_database_create) violations.push('DATABASE_CREATE'); + if (row.target_database_temp) violations.push('DATABASE_TEMP'); + if (row.schema_owner_count > 0) violations.push('SCHEMA_OWNER'); + if (row.schema_create_count > 0) violations.push('SCHEMA_CREATE'); + if (row.schema_usage_count > 0) violations.push('SCHEMA_USAGE'); + if (row.relation_privilege_count > 0) violations.push('RELATION_PRIVILEGE'); + if (row.function_privilege_count > 0) violations.push('FUNCTION_PRIVILEGE'); + if (row.sequence_privilege_count > 0) violations.push('SEQUENCE_PRIVILEGE'); + return violations; +}; + +/** Execute one fresh audit on an already-owned client without releasing it. */ +export const auditPgNotificationRoleClient = async ( + client: PgNotificationRoleClient, + candidate: PgNotificationRoleContract +): Promise => { + const contract = normalizeContract(candidate); + let inTransaction = false; + let result: QueryResult; + try { + await client.query('BEGIN READ ONLY'); + inTransaction = true; + await client.query('SET LOCAL jit TO off'); + result = await client.query( + PG_NOTIFICATION_ROLE_AUDIT_SQL, + [contract.role, contract.database] + ); + await client.query('COMMIT'); + inTransaction = false; + } catch (error) { + if (inTransaction) { + try { + await client.query('ROLLBACK'); + } catch { + // Preserve the catalog-audit failure; the owning broker destroys the client. + } + } + throw error; + } + + const violations = Object.freeze(violationCodes(result.rows[0], contract)); + return Object.freeze({ + version: PG_NOTIFICATION_ROLE_AUDIT_VERSION, + role: contract.role, + database: contract.database, + safe: violations.length === 0, + violations + }); +}; + +/** Execute one fresh, read-only catalog audit. Successful results are not cached. */ +export const auditPgNotificationRole = async ( + pool: Pool, + candidate: PgNotificationRoleContract +): Promise => { + const client: PoolClient = await pool.connect(); + let destroyClient = false; + try { + return await auditPgNotificationRoleClient(client, candidate); + } catch (error) { + destroyClient = true; + throw error; + } finally { + client.release(destroyClient); + } +}; + +/** Fail closed on a pinned client without exposing general query access. */ +export const assertPgNotificationRoleClient = async ( + client: PgNotificationRoleClient, + contract: PgNotificationRoleContract +): Promise => { + const audit = await auditPgNotificationRoleClient(client, contract); + if (!audit.safe) throw new UnsafePgNotificationRoleError(audit); + return audit; +}; + +/** Fail closed with a stable code while retaining a credential-free audit. */ +export const assertPgNotificationRole = async ( + pool: Pool, + contract: PgNotificationRoleContract +): Promise => { + const audit = await auditPgNotificationRole(pool, contract); + if (!audit.safe) throw new UnsafePgNotificationRoleError(audit); + return audit; +}; diff --git a/postgres/pg-cache/src/pg.ts b/postgres/pg-cache/src/pg.ts index 08920e924d..888c6b1dd5 100644 --- a/postgres/pg-cache/src/pg.ts +++ b/postgres/pg-cache/src/pg.ts @@ -1,49 +1,719 @@ +import { createHash, createHmac, randomBytes } from 'node:crypto'; +import type { EventEmitter } from 'node:events'; +import { performance } from 'node:perf_hooks'; + import { Logger } from '@pgpmjs/logger'; import { parseEnvNumber } from '12factor-env'; import pg from 'pg'; import { getPgEnvOptions, PgConfig, PgPoolConfig } from 'pg-env'; -import { getActivePgPoolFactory, PgPoolFactory } from './driver'; -import { pgCache } from './lru'; +import { getActivePgPoolFactory, getPgPoolDriverIdentity, PgPoolFactory } from './driver'; +import { pgCache, type PgPoolLease } from './lru'; const log = new Logger('pg-cache'); +export interface GetPgPoolOptions { + /** Separates pools used by different trust boundaries. */ + purpose?: string; + /** Reset all server and driver session state before every checkout. */ + sanitizeOnCheckout?: boolean; +} + +interface NodePostgresConnection { + parsedStatements?: Record; + _graphilePreparedStatementCache?: unknown; +} + +interface NodePostgresClient extends pg.PoolClient { + connection?: NodePostgresConnection; +} + +type PoolQueryCallback = (error: Error | undefined, result?: unknown) => void; + +const requireSanitizableClient = (value: unknown): NodePostgresClient => { + const client = value as Partial | null | undefined; + if ( + client + && typeof client.query === 'function' + && typeof client.release === 'function' + ) { + return client as NodePostgresClient; + } + if (client && typeof client.release === 'function') { + try { + client.release(true); + } catch { + // The factory contract is already invalid; never hand this client out. + } + } + throw new TypeError( + 'A sanitized PostgreSQL pool must return clients with callable query() and release() methods' + ); +}; + +export interface PgCheckoutSanitizerStats { + checkoutAttempts: number; + checkoutFailures: number; + queuedCheckouts: number; + virginFastPathCheckouts: number; + sanitizedReuseCheckouts: number; + sanitationFailures: number; + checkoutWaitMsTotal: number; + checkoutWaitMsMax: number; + sanitationMsTotal: number; + sanitationMsMax: number; +} + +const makeCheckoutSanitizerStats = (): PgCheckoutSanitizerStats => ({ + checkoutAttempts: 0, + checkoutFailures: 0, + queuedCheckouts: 0, + virginFastPathCheckouts: 0, + sanitizedReuseCheckouts: 0, + sanitationFailures: 0, + checkoutWaitMsTotal: 0, + checkoutWaitMsMax: 0, + sanitationMsTotal: 0, + sanitationMsMax: 0 +}); + +const aggregateCheckoutSanitizerStats = makeCheckoutSanitizerStats(); +const poolCheckoutSanitizerStats = new WeakMap(); + +const recordCount = ( + stats: PgCheckoutSanitizerStats, + key: 'checkoutAttempts' + | 'checkoutFailures' + | 'queuedCheckouts' + | 'virginFastPathCheckouts' + | 'sanitizedReuseCheckouts' + | 'sanitationFailures' +): void => { + stats[key]++; + aggregateCheckoutSanitizerStats[key]++; +}; + +const recordDuration = ( + stats: PgCheckoutSanitizerStats, + totalKey: 'checkoutWaitMsTotal' | 'sanitationMsTotal', + maxKey: 'checkoutWaitMsMax' | 'sanitationMsMax', + durationMs: number +): void => { + stats[totalKey] += durationMs; + stats[maxKey] = Math.max(stats[maxKey], durationMs); + aggregateCheckoutSanitizerStats[totalKey] += durationMs; + aggregateCheckoutSanitizerStats[maxKey] = Math.max( + aggregateCheckoutSanitizerStats[maxKey], + durationMs + ); +}; + +/** Aggregate checkout/sanitation telemetry, or telemetry for one exact pool. */ +export const getPgCheckoutSanitizerStats = ( + pool?: pg.Pool +): Readonly => ({ + ...(pool ? poolCheckoutSanitizerStats.get(pool) : aggregateCheckoutSanitizerStats) + ?? makeCheckoutSanitizerStats() +}); + +const normalizePoolOptions = (options: GetPgPoolOptions = {}) => ({ + purpose: options.purpose ?? 'default', + sanitizeOnCheckout: options.sanitizeOnCheckout ?? false +}); + +// Pool identities may be emitted through diagnostics and cache lifecycle logs. +// A plain digest over a known connection shape would let that digest act as an +// offline password verifier. Keep the key private and process-local: identities +// remain deterministic for this registry's lifetime but are intentionally not +// portable evidence across processes. +const pgIdentityHmacKey = randomBytes(32); + +const hmacIdentity = (prefix: string, identity: string): string => + `${prefix}:${createHmac('sha256', pgIdentityHmacKey) + .update(identity) + .digest('hex')}`; + +const requireIdentityString = (value: unknown, path: string): string => { + if (typeof value !== 'string') { + throw new TypeError(`${path} must be a string`); + } + return value; +}; + +const requireIdentityInteger = ( + value: unknown, + path: string, + minimum: number, + maximum = Number.MAX_SAFE_INTEGER +): number => { + if ( + typeof value !== 'number' + || !Number.isSafeInteger(value) + || value < minimum + || value > maximum + ) { + throw new TypeError( + `${path} must be a safe integer between ${minimum} and ${maximum}` + ); + } + return value; +}; + +const normalizeIdentityOptions = ( + options: GetPgPoolOptions = {} +): Required => { + const purpose = options.purpose ?? 'default'; + const sanitizeOnCheckout = options.sanitizeOnCheckout ?? false; + if (typeof purpose !== 'string' || purpose.length === 0) { + throw new TypeError('pg pool purpose must be a non-empty string'); + } + if (typeof sanitizeOnCheckout !== 'boolean') { + throw new TypeError('pg pool sanitizeOnCheckout must be a boolean'); + } + return { purpose, sanitizeOnCheckout }; +}; + +const canonicalizeIdentityValue = ( + value: unknown, + path: string, + ancestors = new Set() +): unknown => { + if ( + value === null + || typeof value === 'string' + || typeof value === 'boolean' + ) { + return value; + } + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + throw new TypeError(`${path} must not contain a non-finite number`); + } + return Object.is(value, -0) ? ['number', '-0'] : value; + } + if (Buffer.isBuffer(value)) { + return ['buffer-sha256', createHash('sha256').update(value).digest('hex')]; + } + if (Array.isArray(value)) { + if (ancestors.has(value)) throw new TypeError(`${path} must not be cyclic`); + const ownKeys = Reflect.ownKeys(value); + if ( + ownKeys.some((key) => typeof key !== 'string') + || ownKeys.some((key) => key !== 'length' && !/^(?:0|[1-9]\d*)$/.test(key as string)) + || value.some((_entry, index) => !Object.prototype.hasOwnProperty.call(value, index)) + || Object.keys(value).length !== value.length + ) { + throw new TypeError(`${path} must be a dense array without custom properties`); + } + ancestors.add(value); + const result = value.map((entry, index) => { + if (entry === undefined) { + throw new TypeError(`${path}[${index}] must not be undefined`); + } + return canonicalizeIdentityValue(entry, `${path}[${index}]`, ancestors); + }); + ancestors.delete(value); + return ['array', result]; + } + if (typeof value === 'object') { + const record = value as Record; + const prototype = Object.getPrototypeOf(record); + if (prototype !== Object.prototype && prototype !== null) { + throw new TypeError(`${path} must contain only data values`); + } + if (ancestors.has(record)) throw new TypeError(`${path} must not be cyclic`); + ancestors.add(record); + const result: Array<[string, unknown]> = []; + const ownKeys = Reflect.ownKeys(record); + if (ownKeys.some((key) => typeof key !== 'string')) { + throw new TypeError(`${path} must not contain symbol properties`); + } + for (const key of (ownKeys as string[]).sort()) { + const descriptor = Object.getOwnPropertyDescriptor(record, key); + if (!descriptor || !('value' in descriptor)) { + throw new TypeError(`${path}.${key} must be a data property`); + } + const entry = descriptor.value; + if (entry === undefined) { + throw new TypeError(`${path}.${key} must not be undefined`); + } + result.push([ + key, + canonicalizeIdentityValue(entry, `${path}.${key}`, ancestors) + ]); + } + ancestors.delete(record); + return ['object', result]; + } + throw new TypeError(`${path} must contain only deterministic data values`); +}; + export const buildConnectionString = ( user: string, password: string, host: string, port: string | number, database: string -): string => - `postgres://${user}:${password}@${host}:${port}/${database}`; +): string => { + const encodedHost = host.includes(':') && !host.startsWith('[') + ? `[${host}]` + : encodeURIComponent(host); + return `postgres://${encodeURIComponent(user)}:${encodeURIComponent(password)}` + + `@${encodedHost}:${port}/${encodeURIComponent(database)}`; +}; /** * Read per-pool configuration from environment variables. * * Supports: * - PG_POOL_MAX: Maximum clients per pool (default: 5) + * - PG_POOL_MAX_USES: Retire a client after this many checkouts (0/unset: unlimited) * - PG_POOL_IDLE_TIMEOUT_MS: Close idle clients after ms (default: 30000) * - PG_POOL_CONNECTION_TIMEOUT_MS: Fail connect() after ms (default: 5000) */ +const normalizeMaxUses = ( + value: number | string | undefined, + source: 'pool.maxUses' | 'PG_POOL_MAX_USES' +): number | undefined => { + if (value === undefined || value === '') { + return undefined; + } + if (typeof value !== 'number' && typeof value !== 'string') { + throw new TypeError(`${source} must be 0 or a positive safe integer`); + } + if (typeof value === 'string' && !/^(?:0|[1-9]\d*)$/.test(value)) { + throw new TypeError(`${source} must be 0 or a positive safe integer`); + } + const parsed = typeof value === 'number' ? value : Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 0) { + throw new TypeError(`${source} must be 0 or a positive safe integer`); + } + return parsed === 0 ? undefined : parsed; +}; + export function getPgPoolConfig(overrides?: PgPoolConfig): pg.PoolConfig { - return { + const maxUses = overrides?.maxUses !== undefined + ? normalizeMaxUses(overrides.maxUses, 'pool.maxUses') + : normalizeMaxUses(process.env.PG_POOL_MAX_USES, 'PG_POOL_MAX_USES'); + const pool = { max: overrides?.max ?? parseEnvNumber(process.env.PG_POOL_MAX) ?? 5, + ...(maxUses !== undefined && { maxUses }), idleTimeoutMillis: overrides?.idleTimeoutMillis ?? parseEnvNumber(process.env.PG_POOL_IDLE_TIMEOUT_MS) ?? 30000, connectionTimeoutMillis: overrides?.connectionTimeoutMillis ?? parseEnvNumber(process.env.PG_POOL_CONNECTION_TIMEOUT_MS) ?? 5000, ...(overrides?.allowExitOnIdle !== undefined && { allowExitOnIdle: overrides.allowExitOnIdle }), }; + requireIdentityInteger(pool.max, 'pool.max', 1); + if (pool.maxUses !== undefined) { + requireIdentityInteger(pool.maxUses, 'pool.maxUses', 1); + } + requireIdentityInteger(pool.idleTimeoutMillis, 'pool.idleTimeoutMillis', 0); + requireIdentityInteger( + pool.connectionTimeoutMillis, + 'pool.connectionTimeoutMillis', + 0 + ); + if ( + pool.allowExitOnIdle !== undefined + && typeof pool.allowExitOnIdle !== 'boolean' + ) { + throw new TypeError('pool.allowExitOnIdle must be a boolean'); + } + return pool; +} + +const normalizeIdentityConfig = ( + pgConfig: Partial & { pool?: PgPoolConfig } +): { + config: PgConfig; + ssl: unknown; +} => { + const config = getPgEnvOptions(pgConfig); + requireIdentityString(config.host, 'pg.host'); + requireIdentityInteger(config.port, 'pg.port', 1, 65_535); + requireIdentityString(config.database, 'pg.database'); + requireIdentityString(config.user, 'pg.user'); + // node-postgres also accepts password callbacks at runtime, despite the + // narrower public PgConfig type. A callback's captured secret cannot be + // represented exactly, so accepting it could alias two security principals. + requireIdentityString(config.password, 'pg.password'); + return { + config, + ssl: canonicalizeIdentityValue(config.ssl ?? null, 'pg.ssl') + }; +}; + +/** + * Opaque identity for the exact connection and checkout contract. + * + * The digest deliberately includes credentials: two roles that happen to + * connect to the same database must never share a pool. Only the digest is + * exposed, so passwords cannot leak through cache keys or logs. + */ +export function getPgPoolIdentity( + pgConfig: Partial & { pool?: PgPoolConfig }, + options: GetPgPoolOptions = {} +): string { + const { config, ssl } = normalizeIdentityConfig(pgConfig); + const pool = getPgPoolConfig(pgConfig.pool); + const normalizedOptions = normalizeIdentityOptions(options); + const driver = requireIdentityString( + getPgPoolDriverIdentity(), + 'pg driver identity' + ); + const identity = JSON.stringify({ + version: 1, + driver, + host: config.host, + port: config.port, + database: config.database, + user: config.user, + password: config.password, + ssl, + pool: { + max: pool.max, + maxUses: pool.maxUses ?? null, + idleTimeoutMillis: pool.idleTimeoutMillis, + connectionTimeoutMillis: pool.connectionTimeoutMillis, + allowExitOnIdle: pool.allowExitOnIdle ?? false + }, + ...normalizedOptions + }); + return hmacIdentity('pg:v1', identity); +} + +/** + * Opaque identity for one configured physical PostgreSQL database target. + * + * Credentials, TLS policy, pool sizing, and checkout behavior deliberately do + * not participate: those inputs must split connection pools, but they must not + * let two active listener contracts evade a one-target reservation. + */ +export function getPgDatabaseTargetIdentity( + pgConfig: Partial +): string { + const { config } = normalizeIdentityConfig(pgConfig); + const driver = requireIdentityString( + getPgPoolDriverIdentity(), + 'pg driver identity' + ); + const identity = JSON.stringify({ + version: 1, + driver, + host: config.host, + port: config.port, + database: config.database + }); + return hmacIdentity('pg-target:v1', identity); +} + +/** Clear client-side state whose server-side counterpart DISCARD ALL removed. */ +export function clearPreparedStatementBookkeeping(client: NodePostgresClient): void { + const connection = client.connection; + if (!connection) return; + + if (connection.parsedStatements) { + for (const statementName of Object.keys(connection.parsedStatements)) { + delete connection.parsedStatements[statementName]; + } + } + // Dataplan's LRU disposer issues asynchronous DEALLOCATE queries. DISCARD + // ALL has already removed every server-side prepared statement, so invoking + // that disposer here would race those cleanup queries with the next tenant + // transaction on this client. Drop the now-invalid client-side LRU instead; + // Dataplan will create a fresh one on demand. + delete connection._graphilePreparedStatementCache; +} + +const SANITIZED_SESSION_BASELINE = [ + 'SET search_path TO pg_catalog', + 'SET row_security TO on' +] as string[]; + +const sanitizedStartupOptions = (): string => { + const settings = [ + '-c search_path=pg_catalog', + '-c row_security=on' + ]; + if (process.env.DATAPLAN_PG_DONT_DISABLE_JIT !== '1') { + settings.push('-c jit_optimize_above_cost=-1'); + } + return settings.join(' '); +}; + +const sanitizedSessionBaseline = (): string => { + const statements = [...SANITIZED_SESSION_BASELINE]; + if (process.env.DATAPLAN_PG_DONT_DISABLE_JIT !== '1') { + statements.push('SET jit_optimize_above_cost TO -1'); + } + return statements.join('; '); +}; + +/** + * Reset a checked-out connection before it crosses a request boundary. + * A failed reset destroys the connection; it is never returned to a caller. + */ +export async function sanitizePgClient( + client: NodePostgresClient, + baselineRestoredByDiscard = false +): Promise { + try { + await client.query('DISCARD ALL'); + clearPreparedStatementBookkeeping(client); + if (!baselineRestoredByDiscard) { + // Custom drivers may not support PostgreSQL startup options. Restore all + // trusted defaults in one simple-query round trip after DISCARD ALL. + await client.query(sanitizedSessionBaseline()); + } + return client; + } catch (error) { + client.release(true); + throw error; + } +} + +/** Sanitize Promise/callback checkouts and every custom-pool direct query. */ +export function installCheckoutSanitizer( + pool: pg.Pool, + baselineRestoredByDiscard = false, + /** @internal Only the default factory may assert this startup contract. */ + factoryOwnedVirginFastPath = false +): pg.Pool { + if (typeof pool.connect !== 'function' || typeof pool.query !== 'function') { + throw new TypeError( + 'A sanitized PostgreSQL pool must expose callable connect() and query() methods' + ); + } + const stats = makeCheckoutSanitizerStats(); + poolCheckoutSanitizerStats.set(pool, stats); + const virginClients = new WeakSet(); + const eventPool = pool as unknown as Partial; + const canProveFactoryListenerContract = + typeof eventPool.rawListeners === 'function' + && typeof eventPool.on === 'function' + && typeof eventPool.prependListener === 'function'; + const connectListeners = (): Function[] => canProveFactoryListenerContract + ? eventPool.rawListeners!('connect') + : []; + // A non-EventEmitter custom QueryablePool can still use full sanitation, but + // it can never qualify for the default node-postgres virgin fast path. + let factoryContractContaminated = factoryOwnedVirginFastPath + && (!canProveFactoryListenerContract || connectListeners().length > 0); + const markFactoryOwnedVirgin = (client: pg.PoolClient): void => { + const listeners = connectListeners(); + if ( + !factoryContractContaminated + && listeners.length === 1 + && listeners[0] === markFactoryOwnedVirgin + ) { + virginClients.add(client); + } + }; + if (factoryOwnedVirginFastPath && canProveFactoryListenerContract) { + // Remember that an untrusted connect hook has existed even if it is a + // self-removing once/prependOnce listener and disappears during emission. + eventPool.on!('newListener', (eventName, listener) => { + if (eventName === 'connect' && listener !== markFactoryOwnedVirgin) { + factoryContractContaminated = true; + } + }); + // This listener is installed before the lazy pool can open a connection. + // A second connect listener disables the fast path because that listener + // could mutate session state before the checkout reaches this wrapper. + eventPool.prependListener!('connect', markFactoryOwnedVirgin); + } + const originalConnect = pool.connect.bind(pool); + const sanitizedConnect = async (): Promise => { + recordCount(stats, 'checkoutAttempts'); + const checkoutStartedAt = performance.now(); + const waitingBefore = pool.waitingCount; + const pending = originalConnect(); + if (pool.waitingCount > waitingBefore) recordCount(stats, 'queuedCheckouts'); + + let client: pg.PoolClient; + try { + client = requireSanitizableClient(await pending); + } catch (error) { + recordDuration( + stats, + 'checkoutWaitMsTotal', + 'checkoutWaitMsMax', + performance.now() - checkoutStartedAt + ); + recordCount(stats, 'checkoutFailures'); + throw error; + } + recordDuration( + stats, + 'checkoutWaitMsTotal', + 'checkoutWaitMsMax', + performance.now() - checkoutStartedAt + ); + + const virgin = virginClients.delete(client); + const markerIsExclusive = factoryOwnedVirginFastPath + && !factoryContractContaminated + && connectListeners().length === 1 + && connectListeners()[0] === markFactoryOwnedVirgin; + if (virgin && markerIsExclusive) { + // The default factory supplies the trusted baseline in the startup + // packet. With no other connect listener and no prior checkout, neither + // server nor driver state exists to discard. + clearPreparedStatementBookkeeping(client as NodePostgresClient); + recordCount(stats, 'virginFastPathCheckouts'); + return client; + } + + const sanitationStartedAt = performance.now(); + try { + const sanitized = await sanitizePgClient( + client as NodePostgresClient, + baselineRestoredByDiscard + ); + recordCount(stats, 'sanitizedReuseCheckouts'); + return sanitized; + } catch (error) { + recordCount(stats, 'sanitationFailures'); + throw error; + } finally { + recordDuration( + stats, + 'sanitationMsTotal', + 'sanitationMsMax', + performance.now() - sanitationStartedAt + ); + } + }; + + pool.connect = ((callback?: ( + error: Error | undefined, + client: pg.PoolClient | undefined, + done: ((release?: boolean | Error) => void) | undefined + ) => void) => { + const pending = sanitizedConnect(); + if (!callback) return pending; + pending.then( + (client) => callback(undefined, client, client.release.bind(client)), + (error) => callback(error as Error, undefined, undefined) + ); + }) as typeof pool.connect; + + if (!factoryOwnedVirginFastPath) { + // node-postgres' own pool.query dynamically calls this.connect(), so the + // default factory already reaches the sanitizer while retaining the full + // native query contract. An arbitrary QueryablePool may bypass connect() + // in its query implementation, so never trust that method in sanitized + // mode: acquire the sanitized client here and execute the query once. + const sanitizedQuery = ((...args: unknown[]) => { + if (typeof args[0] === 'function') { + const callback = args[0] as PoolQueryCallback; + queueMicrotask(() => callback( + new TypeError('Passing a function as the first parameter to pool.query is not supported') + )); + return undefined; + } + + const callback = typeof args[args.length - 1] === 'function' + ? args.pop() as PoolQueryCallback + : undefined; + const execute = async (): Promise => { + const client = await sanitizedConnect() as NodePostgresClient; + let settled = false; + const canObserveErrors = + typeof client.once === 'function' + && typeof client.removeListener === 'function'; + + return new Promise((resolve, reject) => { + const removeErrorListener = (): void => { + if (!canObserveErrors) return; + try { + client.removeListener('error', onClientError); + } catch { + // A broken optional EventEmitter surface must not prevent release. + } + }; + const releaseAfterError = (error: unknown): void => { + client.release(error instanceof Error ? error : true); + }; + const fail = (error: unknown): void => { + if (settled) return; + settled = true; + removeErrorListener(); + try { + releaseAfterError(error); + } catch { + // Preserve the query/connection error; release errors cannot make + // an already unsafe client eligible for reuse. + } + reject(error); + }; + const succeed = (result: unknown): void => { + if (settled) return; + settled = true; + removeErrorListener(); + try { + client.release(); + resolve(result); + } catch (error) { + reject(error); + } + }; + const onClientError = (error: Error): void => fail(error); + + try { + if (canObserveErrors) client.once('error', onClientError); + Promise.resolve((client.query as (...queryArgs: unknown[]) => unknown)(...args)) + .then(succeed, fail); + } catch (error) { + fail(error); + } + }); + }; + + const pending = execute(); + if (!callback) return pending; + pending.then( + (result) => callback(undefined, result), + (error) => callback(error as Error) + ); + return undefined; + }) as typeof pool.query; + + try { + pool.query = sanitizedQuery; + } catch { + throw new TypeError( + 'A sanitized custom PostgreSQL pool must expose a replaceable query() method' + ); + } + if (pool.query !== sanitizedQuery) { + throw new TypeError( + 'A sanitized custom PostgreSQL pool must expose a replaceable query() method' + ); + } + } + + return pool; } /** * Default pool factory: builds a real `pg.Pool` over TCP. This is the behavior * used whenever no alternate driver is registered (see `./driver`). */ -export const defaultPgPoolFactory: PgPoolFactory = (pgConfig): pg.Pool => { - const config = getPgEnvOptions(pgConfig); - const { user, password, host, port, database } = config; - const connectionString = buildConnectionString(user, password, host, port, database); +export const defaultPgPoolFactory: PgPoolFactory = (pgConfig, options): pg.Pool => { + const { config } = normalizeIdentityConfig(pgConfig); + normalizeIdentityOptions(options); + const { user, password, host, port, database, ssl } = config; const poolConfig = getPgPoolConfig(pgConfig.pool); - const pgPool = new pg.Pool({ connectionString, ...poolConfig }); + const pgPool = new pg.Pool({ + host, + port: Number(port), + database, + user, + password, + ...(ssl !== undefined && { ssl }), + ...(options?.sanitizeOnCheckout && { options: sanitizedStartupOptions() }), + ...poolConfig + }); /** * IMPORTANT: Pool-level error handler for idle connection errors. @@ -97,23 +767,57 @@ export const defaultPgPoolFactory: PgPoolFactory = (pgConfig): pg.Pool => { } }); - return pgPool; + // DISCARD ALL restores startup parameters. Pinning the security baseline in + // the startup packet makes the default driver a one-round-trip checkout; + // custom drivers use the explicit post-DISCARD fallback above. + return options?.sanitizeOnCheckout + ? installCheckoutSanitizer(pgPool, true, true) + : pgPool; }; -export const getPgPool = (pgConfig: Partial & { pool?: PgPoolConfig }): pg.Pool => { - const config = getPgEnvOptions(pgConfig); - const { database } = config; - if (pgCache.has(database)) { - const cached = pgCache.get(database); - if (cached) return cached; - } - +const createPgPool = ( + pgConfig: Partial & { pool?: PgPoolConfig }, + normalizedOptions: ReturnType +): pg.Pool => { // Route through the registered driver (default = pg.Pool over TCP). A custom // factory may return any QueryablePool (e.g. an in-process PGlite pool); it is // treated as a pg.Pool since that is the only surface consumers use. const factory = getActivePgPoolFactory() ?? defaultPgPoolFactory; - const pgPool = factory(pgConfig) as pg.Pool; - - pgCache.set(database, pgPool); + const pgPool = factory(pgConfig, normalizedOptions) as pg.Pool; + if (normalizedOptions.sanitizeOnCheckout && factory !== defaultPgPoolFactory) { + installCheckoutSanitizer(pgPool); + } return pgPool; }; + +/** Synchronously get or create an unleased pool for backwards compatibility. */ +export const getPgPool = ( + pgConfig: Partial & { pool?: PgPoolConfig }, + options: GetPgPoolOptions = {} +): pg.Pool => { + const normalizedOptions = normalizePoolOptions(options); + const poolIdentity = getPgPoolIdentity(pgConfig, normalizedOptions); + return pgCache.getOrCreate( + poolIdentity, + () => createPgPool(pgConfig, normalizedOptions) + ); +}; + +/** + * Atomically get/create and lease the exact connection identity. + * + * Callers that retain a pool beyond the current stack frame should use this + * production API and release only after their final request or long-lived + * resource has drained. + */ +export const acquirePgPool = ( + pgConfig: Partial & { pool?: PgPoolConfig }, + options: GetPgPoolOptions = {} +): PgPoolLease => { + const normalizedOptions = normalizePoolOptions(options); + const poolIdentity = getPgPoolIdentity(pgConfig, normalizedOptions); + return pgCache.acquire( + poolIdentity, + () => createPgPool(pgConfig, normalizedOptions) + ); +}; diff --git a/postgres/pg-env/README.md b/postgres/pg-env/README.md index dcf748b4c1..db6168a16e 100644 --- a/postgres/pg-env/README.md +++ b/postgres/pg-env/README.md @@ -97,9 +97,31 @@ interface PgConfig { user: string; password: string; database: string; + ssl?: boolean | PgSslOptions; } ``` +`ssl` is a data-only subset of Node TLS options (`ca`, `cert`, `key`, +`passphrase`, hostname verification, protocol bounds, and ciphers). Callback +and pre-opened-socket TLS options are deliberately excluded so connection pool +identities can account for the complete trust contract deterministically. + +#### `PgPoolConfig` + +```typescript +interface PgPoolConfig { + max?: number; + maxUses?: number; + idleTimeoutMillis?: number; + connectionTimeoutMillis?: number; + allowExitOnIdle?: boolean; +} +``` + +`maxUses` is passed to native pg-pool by `pg-cache`. Its +`PG_POOL_MAX_USES` parser treats `0` or an unset value as unlimited reuse and +accepts only canonical positive decimal safe integers otherwise. + ### Functions - `getPgEnvOptions(overrides?: Partial): PgConfig` - Get config from environment with overrides diff --git a/postgres/pg-env/src/index.ts b/postgres/pg-env/src/index.ts index 118de7b4d5..e4622cf002 100644 --- a/postgres/pg-env/src/index.ts +++ b/postgres/pg-env/src/index.ts @@ -4,4 +4,10 @@ export { getSpawnEnvWithPg, toPgEnvVars} from './env'; export { getPgClientCommand,PgClientTool } from './pg-client'; -export { defaultPgConfig,PgConfig,PgPoolConfig } from './pg-config'; \ No newline at end of file +export { + defaultPgConfig, + PgConfig, + PgPoolConfig, + PgSslConfig, + PgSslOptions +} from './pg-config'; diff --git a/postgres/pg-env/src/pg-config.ts b/postgres/pg-env/src/pg-config.ts index 7ed78ce5cd..ec7b05f430 100644 --- a/postgres/pg-env/src/pg-config.ts +++ b/postgres/pg-env/src/pg-config.ts @@ -1,9 +1,33 @@ +import type { SecureVersion } from 'node:tls'; + +/** + * Serializable TLS options supported by the shared PostgreSQL connection + * contract. Keeping this surface data-only is intentional: pool identities + * must account for every TLS input, which callback and socket objects cannot + * do deterministically. + */ +export interface PgSslOptions { + ca?: string | Buffer | Array; + cert?: string | Buffer | Array; + key?: string | Buffer | Array; + passphrase?: string; + rejectUnauthorized?: boolean; + servername?: string; + minVersion?: SecureVersion; + maxVersion?: SecureVersion; + ciphers?: string; +} + +export type PgSslConfig = boolean | PgSslOptions; + export interface PgConfig { host: string; port: number; user: string; password: string; database: string; + /** TLS settings passed directly to node-postgres. */ + ssl?: PgSslConfig; } /** @@ -15,6 +39,8 @@ export interface PgConfig { export interface PgPoolConfig { /** Maximum number of clients in the pool (env: PG_POOL_MAX, default: 5) */ max?: number; + /** Retire a client after this many checkouts (env: PG_POOL_MAX_USES, 0/unset: unlimited) */ + maxUses?: number; /** Close idle clients after this many ms (env: PG_POOL_IDLE_TIMEOUT_MS, default: 30000) */ idleTimeoutMillis?: number; /** Reject pool.connect() after this many ms (env: PG_POOL_CONNECTION_TIMEOUT_MS, default: 5000) */ @@ -29,4 +55,4 @@ export const defaultPgConfig: PgConfig = { user: 'postgres', password: 'password', database: 'postgres' -}; \ No newline at end of file +}; diff --git a/postgres/pg-query-context/README.md b/postgres/pg-query-context/README.md index 7bed4c693b..3235016f51 100644 --- a/postgres/pg-query-context/README.md +++ b/postgres/pg-query-context/README.md @@ -24,7 +24,8 @@ npm install pg-query-context ## Features -* Sets session-level context (e.g., role, user ID) using `set_config`. +* Sets the complete transaction-local context (e.g., role, user ID) with one + parameterized `set_config` batch. * Automatically wraps execution in a transaction (`BEGIN`/`COMMIT`). * Automatically rolls back on error. * Supports both `Pool` and `Client` from `pg`. @@ -91,7 +92,12 @@ const user = await withPgClient( | `pool` | `Pool` | ✅ | The PostgreSQL pool to acquire a client from | | `context` | `Record` | ✅ | Session variables set via `set_config` | | `fn` | `(client: PoolClient) => T` | ✅ | Callback receiving the connected client | -| `opts` | `{ skipTransaction?: boolean }` | ❌ | Skip BEGIN/COMMIT wrapping (e.g., inside existing txn) | +| `opts` | `{ skipTransaction?: boolean }` | ❌ | Skip BEGIN/COMMIT only when no context is supplied; pooled transaction-local context fails closed | + +`set_config(..., true)` is transaction-local. A checked-out PostgreSQL client may +use the one-query API with `skipTransaction` inside a transaction managed by its +caller, but a pool cannot prove that transaction ownership. Both pooled APIs +therefore reject `skipTransaction` when the context is non-empty. ## Example with `express` diff --git a/postgres/pg-query-context/src/__tests__/index.test.ts b/postgres/pg-query-context/src/__tests__/index.test.ts new file mode 100644 index 0000000000..294b6a555c --- /dev/null +++ b/postgres/pg-query-context/src/__tests__/index.test.ts @@ -0,0 +1,151 @@ +import type { Pool, PoolClient } from 'pg'; + +import pgQueryContext, { + UNSAFE_POOLED_CONTEXT_ERROR_CODE, + UnsafePooledContextError, + withPgClient +} from '../index'; + +const SETTINGS_SQL = + 'SELECT pg_catalog.set_config(setting->>0, setting->>1, true) ' + + 'FROM pg_catalog.json_array_elements($1::json) AS setting'; + +const makePool = () => { + const client = { + query: jest.fn(async () => ({ rows: [] as unknown[] })), + release: jest.fn() + } as unknown as PoolClient; + const pool = { + connect: jest.fn(async () => client) + } as unknown as Pool; + return { client, pool }; +}; + +describe('pg query context', () => { + it('applies the complete ordered context in one parameterized round trip', async () => { + const { client, pool } = makePool(); + const context = { + 'jwt.claims.user_id': '', + role: 'tenant_runtime', + transaction_read_only: 'off', + search_path: 'pg_catalog, "tenant_api"', + row_security: 'on' + }; + const callback = jest.fn(async () => 'ok'); + + await expect(withPgClient(pool, context, callback)).resolves.toBe('ok'); + + expect(client.query).toHaveBeenNthCalledWith(1, 'BEGIN'); + expect(client.query).toHaveBeenNthCalledWith(2, SETTINGS_SQL, [ + JSON.stringify(Object.entries(context)) + ]); + expect(client.query).toHaveBeenNthCalledWith(3, 'COMMIT'); + expect(callback).toHaveBeenCalledWith(client); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it('does not issue a context query for an empty context', async () => { + const { client, pool } = makePool(); + + await withPgClient(pool, {}, async (): Promise => undefined); + + expect(client.query).toHaveBeenCalledTimes(2); + expect(client.query).toHaveBeenNthCalledWith(1, 'BEGIN'); + expect(client.query).toHaveBeenNthCalledWith(2, 'COMMIT'); + }); + + it('fails closed instead of coercing non-string security settings', async () => { + const { client, pool } = makePool(); + + await expect(withPgClient( + pool, + { 'jwt.claims.user_id': null } as unknown as Record, + async (): Promise => undefined + )).rejects.toThrow( + "PostgreSQL context setting 'jwt.claims.user_id' must be a string" + ); + + expect(client.query).toHaveBeenNthCalledWith(1, 'BEGIN'); + expect(client.query).toHaveBeenNthCalledWith(2, 'ROLLBACK'); + expect(client.query).toHaveBeenCalledTimes(2); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it('rolls back and releases when the batched context is rejected', async () => { + const { client, pool } = makePool(); + (client.query as jest.Mock) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(new Error('invalid role')) + .mockResolvedValueOnce({ rows: [] }); + + await expect(withPgClient( + pool, + { role: 'missing_role' }, + async (): Promise => undefined + )).rejects.toThrow('invalid role'); + + expect(client.query).toHaveBeenNthCalledWith(3, 'ROLLBACK'); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it('uses the same single context batch for the one-query API', async () => { + const { client, pool } = makePool(); + (client.query as jest.Mock).mockImplementation(async (query: unknown) => ({ + rows: [query === 'SELECT tenant_id FROM documents' ? { tenant_id: 'a' } : undefined] + .filter(Boolean) + })); + + await pgQueryContext({ + client: pool, + context: { role: 'tenant_runtime', 'jwt.claims.tenant_id': 'a' }, + query: 'SELECT tenant_id FROM documents' + }); + + expect(client.query).toHaveBeenNthCalledWith(2, SETTINGS_SQL, [ + JSON.stringify([ + ['role', 'tenant_runtime'], + ['jwt.claims.tenant_id', 'a'] + ]) + ]); + expect(client.query).toHaveBeenCalledTimes(4); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it('rejects transaction-local context through a pool without a transaction', async () => { + const { pool } = makePool(); + + await expect(withPgClient( + pool, + { role: 'tenant_runtime' }, + async (): Promise => undefined, + { skipTransaction: true } + )).rejects.toMatchObject({ + name: UnsafePooledContextError.name, + code: UNSAFE_POOLED_CONTEXT_ERROR_CODE + }); + + expect(pool.connect).not.toHaveBeenCalled(); + + await expect(pgQueryContext({ + client: pool, + context: { 'jwt.claims.tenant_id': 'tenant-a' }, + query: 'SELECT 1', + skipTransaction: true + })).rejects.toBeInstanceOf(UnsafePooledContextError); + expect(pool.connect).not.toHaveBeenCalled(); + }); + + it('allows transaction-free pooled execution only when no context is requested', async () => { + const { client, pool } = makePool(); + + await expect(withPgClient( + pool, + {}, + async () => 'ok', + { skipTransaction: true } + )).resolves.toBe('ok'); + + expect(client.query).not.toHaveBeenCalled(); + expect(client.release).toHaveBeenCalledTimes(1); + }); +}); diff --git a/postgres/pg-query-context/src/__tests__/integration.test.ts b/postgres/pg-query-context/src/__tests__/integration.test.ts new file mode 100644 index 0000000000..3019d4b329 --- /dev/null +++ b/postgres/pg-query-context/src/__tests__/integration.test.ts @@ -0,0 +1,131 @@ +import { Pool, PoolClient } from 'pg'; + +import { withPgClient } from '../index'; + +const describeWithPostgres = process.env.PG_QUERY_CONTEXT_RUN_PG_INTEGRATION === '1' + ? describe + : describe.skip; + +interface SessionState { + role: string; + transaction_read_only: string; + search_path: string; + row_security: string; + user_id: string | null; +} + +async function readSessionState(client: PoolClient): Promise { + const result = await client.query(` + SELECT + current_setting('role') AS role, + current_setting('transaction_read_only') AS transaction_read_only, + current_setting('search_path') AS search_path, + current_setting('row_security') AS row_security, + current_setting('jwt.claims.user_id', true) AS user_id + `); + return result.rows[0]; +} + +describeWithPostgres('transaction-local PostgreSQL context', () => { + let pool: Pool; + let runtimeRole: string; + + beforeAll(async () => { + pool = new Pool({ max: 1 }); + const client = await pool.connect(); + try { + const identity = await client.query<{ current_user: string }>( + 'SELECT current_user' + ); + runtimeRole = identity.rows[0].current_user; + + // Deliberately establish a visibly different session baseline. With a + // one-client pool, every assertion below observes the same backend. + await client.query('RESET ROLE'); + await client.query('SET transaction_read_only TO off'); + await client.query('SET search_path TO public'); + await client.query('SET row_security TO off'); + await client.query( + "SELECT pg_catalog.set_config('jwt.claims.user_id', 'baseline-user', false)" + ); + } finally { + client.release(); + } + }); + + afterAll(async () => { + if (!pool) return; + const client = await pool.connect(); + try { + await client.query('RESET ROLE'); + await client.query('RESET ALL'); + } finally { + client.release(); + await pool.end(); + } + }); + + it('applies every security setting locally and restores the session after commit and rollback', async () => { + const committedInside = await withPgClient(pool, { + role: runtimeRole, + transaction_read_only: 'on', + search_path: 'pg_catalog', + row_security: 'on', + 'jwt.claims.user_id': '' + }, readSessionState); + + expect(committedInside).toEqual({ + role: runtimeRole, + transaction_read_only: 'on', + search_path: 'pg_catalog', + row_security: 'on', + user_id: '' + }); + + const afterCommitClient = await pool.connect(); + try { + await expect(readSessionState(afterCommitClient)).resolves.toEqual({ + role: 'none', + transaction_read_only: 'off', + search_path: 'public', + row_security: 'off', + user_id: 'baseline-user' + }); + } finally { + afterCommitClient.release(); + } + + let rolledBackInside: SessionState | undefined; + await expect(withPgClient(pool, { + role: runtimeRole, + transaction_read_only: 'on', + search_path: 'pg_catalog', + row_security: 'on', + 'jwt.claims.user_id': 'rollback-canary' + }, async (client) => { + rolledBackInside = await readSessionState(client); + throw new Error('force rollback'); + })).rejects.toThrow('force rollback'); + + expect(rolledBackInside).toEqual({ + role: runtimeRole, + transaction_read_only: 'on', + search_path: 'pg_catalog', + row_security: 'on', + user_id: 'rollback-canary' + }); + + const afterRollbackClient = await pool.connect(); + try { + await expect(readSessionState(afterRollbackClient)).resolves.toEqual({ + role: 'none', + transaction_read_only: 'off', + search_path: 'public', + row_security: 'off', + user_id: 'baseline-user' + }); + } finally { + afterRollbackClient.release(); + } + }); +}); diff --git a/postgres/pg-query-context/src/index.ts b/postgres/pg-query-context/src/index.ts index 188d8670f2..7a228c6c15 100644 --- a/postgres/pg-query-context/src/index.ts +++ b/postgres/pg-query-context/src/index.ts @@ -2,18 +2,54 @@ import { ClientBase, Pool, PoolClient, QueryResult } from 'pg'; // --- Internal helpers --- -function setContext(ctx: Record): { query: string; values: string[] }[] { - return Object.keys(ctx || {}).reduce<{ query: string; values: string[] }[]>((m, el) => { - m.push({ query: 'SELECT set_config($1, $2, true)', values: [el, ctx[el]] }); - return m; - }, []); +export const UNSAFE_POOLED_CONTEXT_ERROR_CODE = + 'PG_QUERY_CONTEXT_UNSAFE_POOLED_CONTEXT'; + +export class UnsafePooledContextError extends Error { + readonly code = UNSAFE_POOLED_CONTEXT_ERROR_CODE; + + constructor() { + super( + 'Transaction-local PostgreSQL context cannot be applied through a pool ' + + 'when skipTransaction is enabled' + ); + this.name = 'UnsafePooledContextError'; + } +} + +function assertContextHasTransaction( + usesPool: boolean, + skipTransaction: boolean, + context: Record +): void { + if (usesPool && skipTransaction && Object.keys(context).length > 0) { + throw new UnsafePooledContextError(); + } } async function execContext(client: ClientBase, ctx: Record): Promise { - const local = setContext(ctx); - for (const { query, values } of local) { - await client.query(query, values); + const entries = Object.entries(ctx || {}); + if (entries.length === 0) return; + + for (const [key, value] of entries) { + // This API establishes the request's security boundary. Runtime callers + // can still bypass TypeScript, so reject ambiguous null/object values + // instead of silently installing literal "null" or "[object Object]" + // session settings. + if (typeof value !== 'string') { + throw new TypeError(`PostgreSQL context setting '${key}' must be a string`); + } } + + // Apply the complete request context in one parameterized round trip. The + // array preserves insertion order (including role, read-only, search_path, + // and every explicitly-empty security claim), while set_config(..., true) + // keeps every value transaction-local exactly as the former per-key loop did. + await client.query( + 'SELECT pg_catalog.set_config(setting->>0, setting->>1, true) ' + + 'FROM pg_catalog.json_array_elements($1::json) AS setting', + [JSON.stringify(entries)] + ); } // --- Single-query API (original) --- @@ -31,6 +67,8 @@ async function pgQueryContext({ client, context = {}, query = '', variables = [] const shouldRelease = isPool; let pgClient: ClientBase | PoolClient | null = null; + assertContextHasTransaction(isPool, skipTransaction, context); + try { pgClient = isPool ? await (client as Pool).connect() : client as ClientBase; @@ -80,6 +118,7 @@ export async function withPgClient( fn: (client: PoolClient) => Promise, opts: WithPgClientOptions = {}, ): Promise { + assertContextHasTransaction(true, opts.skipTransaction === true, context); const client = await pool.connect(); try { if (!opts.skipTransaction) {