From ec24b46b0bbc1bb87c5f1a58024e16c620953ba1 Mon Sep 17 00:00:00 2001 From: yyyyaaa Date: Wed, 5 Aug 2026 09:27:12 +0700 Subject: [PATCH] Fail closed at the tenant request boundary --- .../__tests__/server.integration.test.ts | 62 +- .../__tests__/upload.integration.test.ts | 35 +- graphql/server-test/src/get-connections.ts | 12 +- graphql/server-test/src/index.ts | 2 +- graphql/server-test/src/server.ts | 4 + graphql/server/README.md | 118 +- .../__tests__/server-cache-lifecycle.test.ts | 98 ++ .../server-pool-listener-lease.test.ts | 230 ++++ .../__tests__/server-process-shutdown.test.ts | 71 ++ .../src/__tests__/websocket-upgrade.test.ts | 324 +++++ graphql/server/src/agentic/router.ts | 52 +- .../__tests__/observability.test.ts | 18 +- .../src/diagnostics/debug-db-snapshot.ts | 2 +- .../src/diagnostics/debug-memory-snapshot.ts | 43 +- .../server/src/diagnostics/observability.ts | 30 +- graphql/server/src/index.ts | 4 + .../src/middleware/__tests__/api.test.ts | 375 +++++- .../__tests__/auth-pool-lease.test.ts | 268 ++++ .../src/middleware/__tests__/captcha.test.ts | 294 +++++ .../__tests__/error-handler.test.ts | 43 + .../middleware/__tests__/flush-auth.test.ts | 61 + .../__tests__/flush-pool-lease.test.ts | 160 +++ .../graphile-build-admission-response.test.ts | 74 ++ .../graphile-internal-claims.test.ts | 77 ++ .../graphile-pool-lease-publication.test.ts | 222 ++++ .../graphile-request-terminal.test.ts | 142 +++ .../__tests__/internal-request.test.ts | 167 +++ .../src/middleware/__tests__/routing.test.ts | 141 +- .../__tests__/runtime-pg-config.test.ts | 203 +++ .../__tests__/runtime-pg-requirements.test.ts | 111 ++ .../runtime-role-safety.integration.test.ts | 199 +++ .../__tests__/runtime-role-safety.test.ts | 514 ++++++++ graphql/server/src/middleware/api.ts | 407 ++++-- graphql/server/src/middleware/auth.ts | 104 +- graphql/server/src/middleware/captcha.ts | 239 +++- graphql/server/src/middleware/cors.ts | 81 +- .../server/src/middleware/error-handler.ts | 21 +- graphql/server/src/middleware/flush.ts | 115 +- graphql/server/src/middleware/graphile.ts | 1129 ++++++++++++++--- .../server/src/middleware/internal-request.ts | 156 +++ .../observability/__tests__/guard.test.ts | 53 +- .../src/middleware/observability/guard.ts | 24 +- graphql/server/src/middleware/routing.ts | 56 +- .../src/middleware/runtime-pg-config.ts | 548 ++++++++ .../src/middleware/runtime-pg-requirements.ts | 113 ++ .../src/middleware/runtime-role-safety.ts | 883 +++++++++++++ graphql/server/src/middleware/types.ts | 4 + ...bsocket-operation-admission-plugin.test.ts | 234 ++++ .../websocket-operation-admission-plugin.ts | 212 ++++ graphql/server/src/server.ts | 592 +++++++-- graphql/server/src/websocket-upgrade.ts | 447 +++++++ packages/cli/src/commands/explorer.ts | 7 +- packages/cli/src/commands/server.ts | 12 +- 53 files changed, 8981 insertions(+), 612 deletions(-) create mode 100644 graphql/server/src/__tests__/server-cache-lifecycle.test.ts create mode 100644 graphql/server/src/__tests__/server-pool-listener-lease.test.ts create mode 100644 graphql/server/src/__tests__/server-process-shutdown.test.ts create mode 100644 graphql/server/src/__tests__/websocket-upgrade.test.ts create mode 100644 graphql/server/src/middleware/__tests__/auth-pool-lease.test.ts create mode 100644 graphql/server/src/middleware/__tests__/captcha.test.ts create mode 100644 graphql/server/src/middleware/__tests__/error-handler.test.ts create mode 100644 graphql/server/src/middleware/__tests__/flush-auth.test.ts create mode 100644 graphql/server/src/middleware/__tests__/flush-pool-lease.test.ts create mode 100644 graphql/server/src/middleware/__tests__/graphile-build-admission-response.test.ts create mode 100644 graphql/server/src/middleware/__tests__/graphile-internal-claims.test.ts create mode 100644 graphql/server/src/middleware/__tests__/graphile-pool-lease-publication.test.ts create mode 100644 graphql/server/src/middleware/__tests__/graphile-request-terminal.test.ts create mode 100644 graphql/server/src/middleware/__tests__/internal-request.test.ts create mode 100644 graphql/server/src/middleware/__tests__/runtime-pg-config.test.ts create mode 100644 graphql/server/src/middleware/__tests__/runtime-pg-requirements.test.ts create mode 100644 graphql/server/src/middleware/__tests__/runtime-role-safety.integration.test.ts create mode 100644 graphql/server/src/middleware/__tests__/runtime-role-safety.test.ts create mode 100644 graphql/server/src/middleware/internal-request.ts create mode 100644 graphql/server/src/middleware/runtime-pg-config.ts create mode 100644 graphql/server/src/middleware/runtime-pg-requirements.ts create mode 100644 graphql/server/src/middleware/runtime-role-safety.ts create mode 100644 graphql/server/src/plugins/__tests__/websocket-operation-admission-plugin.test.ts create mode 100644 graphql/server/src/plugins/websocket-operation-admission-plugin.ts create mode 100644 graphql/server/src/websocket-upgrade.ts diff --git a/graphql/server-test/__tests__/server.integration.test.ts b/graphql/server-test/__tests__/server.integration.test.ts index cec379129a..23dfef27f3 100644 --- a/graphql/server-test/__tests__/server.integration.test.ts +++ b/graphql/server-test/__tests__/server.integration.test.ts @@ -14,7 +14,11 @@ import path from 'path'; import type supertest from 'supertest'; -import { getConnections, seed } from '../src'; +import { + getConnections, + seed, + TEST_INTERNAL_REQUEST_SECRET +} from '../src'; import type { ServerInfo } from '../src/types'; jest.setTimeout(60000); @@ -51,6 +55,7 @@ type Scenario = { isPublic: boolean; metaSchemas?: string[]; routingSchema?: string; + allowMetaSchemaHeader?: boolean; }; headers?: Record; }; @@ -87,17 +92,8 @@ const scenarios: Scenario[] = [ api: { isPublic: false, metaSchemas: scopedMetaSchemas }, headers: { 'X-Database-Id': scopedDatabaseId, - 'X-Api-Name': 'private' - } - }, - { - name: 'scoped private via X-Schemata', - seedDir: 'simple-seed-scoped', - useRouting: true, - api: { isPublic: false, metaSchemas: scopedMetaSchemas }, - headers: { - 'X-Database-Id': scopedDatabaseId, - 'X-Schemata': schemas.join(',') + 'X-Api-Name': 'private', + 'X-Constructive-Internal-Token': TEST_INTERNAL_REQUEST_SECRET } } ]; @@ -267,6 +263,7 @@ describe('scoped private via X-Meta-Schema', () => { const headers: Record = { 'X-Database-Id': scopedDatabaseId, 'X-Meta-Schema': 'true', + 'X-Constructive-Internal-Token': TEST_INTERNAL_REQUEST_SECRET, ...extraHeaders }; for (const [header, value] of Object.entries(headers)) { @@ -284,7 +281,8 @@ describe('scoped private via X-Meta-Schema', () => { useRouting: true, api: { isPublic: false, - metaSchemas: metaApiSchemas + metaSchemas: metaApiSchemas, + allowMetaSchemaHeader: true } } }, @@ -342,7 +340,7 @@ describe('scoped private via X-Meta-Schema', () => { * Error path tests * * Exercise the api middleware error conditions under scoped routing: - * - Invalid X-Schemata (ApiError with errorHtml → 404) + * - Raw X-Schemata is rejected before database routing (→ 403) * - Host that resolves to no route (→ 404, no legacy fallback) * - NO_VALID_SCHEMAS (configured metaSchemas absent → 404) */ @@ -368,16 +366,40 @@ describe('Error paths', () => { teardowns.push(teardown); }); - describe('Invalid X-Schemata (returns 404)', () => { - it('should return 404 when X-Schemata contains schemas not in the DB', async () => { + describe('Raw X-Schemata (returns 403)', () => { + it('rejects physical schema selection even from an authenticated internal caller', async () => { const res = await request .post('/graphql') .set('X-Database-Id', scopedDatabaseId) .set('X-Schemata', 'nonexistent_schema_abc,another_fake_schema') + .set('X-Constructive-Internal-Token', TEST_INTERNAL_REQUEST_SECRET) .send({ query: '{ __typename }' }); - expect(res.status).toBe(404); - expect(res.text).toContain('No valid schemas found for the supplied X-Schemata header'); + expect(res.status).toBe(403); + expect(res.text).toBe('Forbidden'); + }); + }); + + describe('Unauthenticated internal selectors (returns 403)', () => { + it('rejects API/database selectors before any routing query', async () => { + const res = await request + .post('/graphql') + .set('X-Database-Id', scopedDatabaseId) + .set('X-Api-Name', 'private') + .send({ query: '{ __typename }' }); + + expect(res.status).toBe(403); + expect(res.text).toBe('Forbidden'); + }); + + it('rejects actor claims before any routing query', async () => { + const res = await request + .post('/graphql') + .set('X-Actor-Id', 'attacker-controlled-actor') + .send({ query: '{ __typename }' }); + + expect(res.status).toBe(403); + expect(res.text).toBe('Forbidden'); }); }); @@ -408,7 +430,8 @@ describe('Error paths', () => { useRouting: true, api: { isPublic: false, - metaSchemas: scopedMetaSchemas + metaSchemas: scopedMetaSchemas, + allowMetaSchemaHeader: true } } }, @@ -422,6 +445,7 @@ describe('Error paths', () => { .post('/graphql') .set('X-Database-Id', 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee') .set('X-Meta-Schema', 'true') + .set('X-Constructive-Internal-Token', TEST_INTERNAL_REQUEST_SECRET) .send({ query: '{ __typename }' }); expect(res.status).toBe(404); diff --git a/graphql/server-test/__tests__/upload.integration.test.ts b/graphql/server-test/__tests__/upload.integration.test.ts index a95abbd053..b6b3137bdb 100644 --- a/graphql/server-test/__tests__/upload.integration.test.ts +++ b/graphql/server-test/__tests__/upload.integration.test.ts @@ -33,7 +33,11 @@ import path from 'path'; import type { PgTestClient } from 'pgsql-test'; import type supertest from 'supertest'; -import { getConnections, seed } from '../src'; +import { + getConnections, + seed, + TEST_INTERNAL_REQUEST_SECRET +} from '../src'; jest.setTimeout(120000); @@ -288,7 +292,8 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { return request .post('/graphql') .set('X-Database-Id', aliceDatabaseId) - .set('X-Schemata', aliceSchemas.join(',')) + .set('X-Api-Name', 'app') + .set('X-Constructive-Internal-Token', TEST_INTERNAL_REQUEST_SECRET) .send(payload); }; @@ -304,6 +309,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { .post('/graphql') .set('X-Database-Id', databaseId) .set('X-Api-Name', apiName) + .set('X-Constructive-Internal-Token', TEST_INTERNAL_REQUEST_SECRET) .send(payload); }; @@ -319,6 +325,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { .post('/graphql') .set('X-Database-Id', databaseId) .set('X-Schemata', schemas.join(',')) + .set('X-Constructive-Internal-Token', TEST_INTERNAL_REQUEST_SECRET) .send(payload); }; @@ -1050,27 +1057,16 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { expect(res.status).toBe(404); }); - it('X-Schemata with Bob schema + Alice database_id does NOT leak Alice data', async () => { + it('rejects Bob physical schemas paired with Alice database_id', async () => { const res = await postGraphQLViaSchemata(aliceDatabaseId, bobSchemas, { query: APP_FILES }); - if (res.status === 200 && res.body.data) { - const names = (res.body.data.appFiles?.nodes ?? []).map( - (f: { filename: string }) => f.filename - ); - expect(names).not.toContain('hello-public.txt'); - expect(names).not.toContain('hello-private.txt'); - } + expect(res.status).toBe(403); + expect(res.text).toBe('Forbidden'); }); - it('X-Schemata with Mallory schema + Bob database_id does NOT leak Bob data', async () => { + it('rejects Mallory physical schemas paired with Bob database_id', async () => { const res = await postGraphQLViaSchemata(bobDatabaseId, mallorySchemas, { query: APP_FILES }); - if (res.status === 200 && res.body.data) { - const names = (res.body.data.appFiles?.nodes ?? []).map( - (f: { filename: string }) => f.filename - ); - expect(names).not.toContain('bob-file.txt'); - expect(names).not.toContain('bob-seeded-public.txt'); - expect(names).not.toContain('bob-seeded-private.txt'); - } + expect(res.status).toBe(403); + expect(res.text).toBe('Forbidden'); }); }); @@ -1106,4 +1102,3 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { }); }); }); - diff --git a/graphql/server-test/src/get-connections.ts b/graphql/server-test/src/get-connections.ts index 875c30b9cd..0e20088b01 100644 --- a/graphql/server-test/src/get-connections.ts +++ b/graphql/server-test/src/get-connections.ts @@ -3,7 +3,11 @@ import type { GetConnectionOpts, GetConnectionResult } from 'pgsql-test'; import { getConnections as getPgConnections } from 'pgsql-test'; import type { SeedAdapter } from 'pgsql-test/seed/types'; -import { createDevTestServer, createTestServer } from './server'; +import { + createDevTestServer, + createTestServer, + TEST_INTERNAL_REQUEST_SECRET +} from './server'; import { createQueryFn,createSuperTestAgent } from './supertest'; import type { GetConnectionsInput, GetConnectionsResult } from './types'; @@ -51,6 +55,12 @@ export const getConnections = async ( api: { // Start with user-provided api options from server.api ...input.server?.api, + // Production routing/identity headers fail closed unless the ingress is + // authenticated. Tests use one fixture-only credential and send it only + // on cases that intentionally exercise the reserved header boundary. + internalRequestSecret: + input.server?.api?.internalRequestSecret + ?? TEST_INTERNAL_REQUEST_SECRET, // Apply convenience properties (these take precedence) exposedSchemas: input.schemas, ...(input.authRole && { anonRole: input.authRole, roleName: input.authRole }) diff --git a/graphql/server-test/src/index.ts b/graphql/server-test/src/index.ts index 5b4b7e7f82..bf9bc942a0 100644 --- a/graphql/server-test/src/index.ts +++ b/graphql/server-test/src/index.ts @@ -2,7 +2,7 @@ export * from './types'; // Export server utilities -export { createTestServer } from './server'; +export { createTestServer, TEST_INTERNAL_REQUEST_SECRET } from './server'; // Export SuperTest utilities export { createSuperTestAgent } from './supertest'; diff --git a/graphql/server-test/src/server.ts b/graphql/server-test/src/server.ts index c8fbfc1e2d..87c0ef2ae5 100644 --- a/graphql/server-test/src/server.ts +++ b/graphql/server-test/src/server.ts @@ -5,6 +5,10 @@ import { Server as HttpServer } from 'http'; import type { ServerInfo, ServerOptions } from './types'; +/** Credential used only by in-process integration fixtures. */ +export const TEST_INTERNAL_REQUEST_SECRET = + 'graphql-server-test-internal-secret-32-bytes'; + /** * Create a single-tenant dev test server (no scoped routing, no database id). * diff --git a/graphql/server/README.md b/graphql/server/README.md index f874d18e0f..80d9f6a84e 100644 --- a/graphql/server/README.md +++ b/graphql/server/README.md @@ -65,7 +65,7 @@ Runs an Express server that wires CORS, uploads, domain parsing, auth, and PostG - Meta-schema routing by domain + subdomain - File uploads via `graphql-upload` - GraphiQL and health check endpoints -- Schema cache flush via `/flush` or database notifications +- Schema cache flush via authenticated `/flush` or database notifications - Opt-in observability for memory, DB activity, and Graphile build debugging ## Observability @@ -95,7 +95,7 @@ For the operational workflow, sampler output, and heap snapshot usage, see [docs - `GET /graphiql` -> GraphiQL UI - `GET /graphql` / `POST /graphql` -> GraphQL endpoint - `POST /graphql` (multipart) -> file uploads -- `POST /flush` -> clears cached Graphile schema for the current API +- `POST /flush` -> clears the current API's cached Graphile schema; requires `X-Constructive-Internal-Token` - `GET /debug/memory` -> memory/process/Graphile debug snapshot when observability is enabled - `GET /debug/db` -> PostgreSQL activity/locks/pool debug snapshot when observability is enabled @@ -103,14 +103,61 @@ For the operational workflow, sampler output, and heap snapshot usage, see [docs This is a production-only server: every request is resolved through the scoped-routing plane. There is no static single-tenant mode and no flag to disable routing. For single-database local development without route resolution or a database id, use [`@constructive-io/graphql-dev-server`](../dev-server/README.md). -- The server resolves the request host with a single `resolve_route()` call against the compiled route bindings in the scoped routing schema (`API_ROUTING_SCHEMA`, default `routing_public`), mapping host → tenant/api/database/role. +- The server resolves every request host with a fresh `resolve_route()` call against the compiled route bindings in the scoped routing schema (`API_ROUTING_SCHEMA`, default `routing_public`), mapping host → tenant/api/database/role. Routing metadata is not served from the process cache because a missed notification must never retain an old hostname-to-tenant assignment. - Only APIs where `api.is_public` matches `API_IS_PUBLIC` are served. -- In private mode (`API_IS_PUBLIC=false`), you can override with headers: +- In private mode (`API_IS_PUBLIC=false`), an internal caller can select an authoritative surface with these headers only when it also supplies the exact `X-Constructive-Internal-Token` configured by `GRAPHQL_INTERNAL_REQUEST_SECRET`: - `X-Api-Name` + `X-Database-Id` - - `X-Schemata` + `X-Database-Id` - - `X-Meta-Schema` + `X-Database-Id` +- `X-Meta-Schema` is a privileged, potentially cross-tenant control-plane API. It is rejected by default and can only be enabled with `API_ALLOW_META_SCHEMA_HEADER=true` on a separate private admin ingress; it is never a tenant-routing mechanism. +- `X-Schemata` is rejected even from an authenticated internal caller because an unchecked physical schema list is not a tenant-safe routing contract. Provision an API record and select it by name instead. +- The ingress must remove any caller-supplied reserved headers before injecting its own token and selectors, and the hop to this server must use an authenticated encrypted channel. - A resolved database id is always required. There is no default database, so a request that resolves without a database id is rejected (`NO_DATABASE_ID` → HTTP 500). +Production multi-tenant execution requires `runtimePgResolver`. The server calls +it once per request with the credential-free exact route contract: database id, +physical database name, API id, ordered physical schemas, and roles in +`[anonymous, authenticated]` order. The result must contain an explicit user, +password, and matching database; `connectionString` and control-plane credential +fallbacks are rejected. The secret-bearing result remains in a server-owned +`WeakMap`, while Express context and Graphile consume the same frozen resolution +and independently verify its opaque pool identity. + +```typescript +GraphQLServer({ + pg: controlPlanePg, + graphile: { introspectionMode: 'scoped-required' }, + runtimePgResolver: async ({ databaseId, databaseName, apiId, schemas, roles }) => { + const login = await credentialStore.get({ + databaseId, + databaseName, + apiId, + schemas, + roles + }); + return { + database: databaseName, + user: login.user, + password: login.password + }; + } +}); +``` + +`runtimePg` remains a compatibility path for one statically configured route. +In production or `scoped-required` mode it must include an explicit database and +be paired with an exact credential-free `runtimePgStaticIdentity`; any request +whose database/API/schema/role contract differs fails closed. A dynamic server +must use the resolver even when several databases happen to share a login. + +The resolver is part of the trusted routing boundary and must key its lookup by +immutable `databaseId`. The server requires its normalized host, port, database, +and TLS policy to match the control-plane tenant connection exactly, then binds +the complete target/login/pool contract into an opaque identity and rechecks the +route before every consumer reads it. A deployment where tenant databases live +on different network endpoints needs one future per-route resolver shared by +both control and runtime lanes; this implementation rejects that topology +rather than authenticating/configuring against one server and executing against +another. + ## Configuration Configuration is merged from defaults, config files, and env vars via `@constructive-io/graphql-env`. See `graphql/env/README.md` for the full list and examples. @@ -123,6 +170,13 @@ Configuration is merged from defaults, config files, and env vars via `@construc | `PGPASSWORD` | Postgres password | `password` | | `PGDATABASE` | Postgres database | `postgres` | | `GRAPHILE_SCHEMA` | Comma-separated schemas to expose | empty | +| `GRAPHILE_INTROSPECTION_CLIENT_RELEASE_MODE` | Reuse or destroy the exact catalog-introspection client after gather | `reuse` | +| `GRAPHILE_REALTIME_SCHEMA` | Exact schema containing realtime cursor functions | `realtime_public` | +| `GRAPHILE_REALTIME_NOTIFICATION_MODE` | Dedicated subscriber or opt-in exact-topic broker | `dedicated` | +| `GRAPHILE_REALTIME_NOTIFICATION_ROLE_REVALIDATION_MS` | Maximum age of shared-listener role audit | `60000` | +| `GRAPHILE_REALTIME_CURSOR_POLL_INTERVAL_MS` | Cursor recovery poll interval | `5000` | +| `GRAPHILE_REALTIME_CURSOR_HEARTBEAT_INTERVAL_MS` | Cursor listener heartbeat interval | `30000` | +| `GRAPHILE_RELEASE_BUILD_STATE_AFTER_VALIDATION` | Release schema-construction-only state after successful validation | `false` | | `FEATURES_SIMPLE_INFLECTION` | Enable simple inflection | `true` | | `FEATURES_OPPOSITE_BASE_NAMES` | Enable opposite base names | `true` | | `FEATURES_POSTGIS` | Enable PostGIS support | `true` | @@ -130,12 +184,64 @@ Configuration is merged from defaults, config files, and env vars via `@construc | `API_IS_PUBLIC` | Serve public APIs only | `true` | | `API_EXPOSED_SCHEMAS` | Additional schemas to expose | empty | | `API_META_SCHEMAS` | Meta schemas to query | `routing_public,metaschema_public,metaschema_modules_public` | +| `API_ALLOW_META_SCHEMA_HEADER` | Enable the privileged metadata admin surface on an isolated private ingress | `false` | | `API_ANON_ROLE` | Anonymous role name | `administrator` | | `API_ROLE_NAME` | Authenticated role name | `administrator` | +| `GRAPHQL_INTERNAL_REQUEST_SECRET` | Minimum-32-byte token for reserved routing, actor-identity, and cache-administration headers | empty; reserved headers fail closed | +| `GRAPHQL_ROUTING_CACHE_MAX_ENTRIES` | Resolved routing/service labels retained per process; must be at least the effective Graphile resident capacity | `max(1024, effective Graphile capacity)` | | `GRAPHQL_OBSERVABILITY_ENABLED` | Master switch for debug routes and sampler | `false` | +| `GRAPHQL_OBSERVABILITY_TOKEN` | Bearer token (minimum 32 bytes) required for loopback-only production observability | empty | | `GRAPHQL_DEBUG_SAMPLER_ENABLED` | Enables periodic NDJSON sampling when observability is on | `true` | | `GRAPHQL_DEBUG_SAMPLER_INTERVAL_MS` | Sampler interval in milliseconds | `10000` | | `GRAPHQL_DEBUG_SAMPLER_DIR` | Override output directory for sampler logs | `graphql/server/logs` | +| `GRAPHILE_BUILD_WATCHDOG_MS` | Latch schema-build admission unhealthy after one admitted build exceeds this duration; recovery requires a process restart | `300000` | + +The build watchdog never cancels or releases an overdue build, because JavaScript +and plugin work cannot be canceled safely. It rejects queued and subsequent +builds with `GRAPHILE_BUILD_STUCK_RESTART_REQUIRED`, prevents late publication, +and leaves resident handlers available while the process is restarted. + +Programmatic `graphile.extends` and `graphile.preset` values are applied after +Constructive's feature preset, so trusted caller plugins and ordinary Graphile +schema/runtime settings take effect. They cannot replace the exact tenant +`pgServices`, security-GUC context, GraphQL/WebSocket transport policy, error +masking, or server-owned auth/admission plugins; explicit attempts fail startup +with `GRAPHILE_PROTECTED_PRESET_OVERRIDE`. Graphile plugins execute trusted +server-side code, so this boundary prevents structural misconfiguration rather +than sandboxing a hostile plugin implementation. + +The routing cache stores host/header labels and their resolved API metadata. Its +capacity is independent from Graphile build identity: evicting a routing label +causes the next request to resolve that label again, but it never disposes a +valid resident Graphile instance. `/debug/memory` reports its size, capacity, +hits, misses, and capacity/TTL evictions. + +`GRAPHILE_REALTIME_SCHEMA` changes only the exact cursor-function schema for an +API whose database settings enable realtime. Cursor events are accepted only +from that API's exposed physical schemas. A foreign cursor row or lost +subscriber emitter latches that exact generation unavailable, and the next HTTP +request receives `503 GRAPHILE_REALTIME_UNAVAILABLE` instead of entering its +Graphile handler. The failed generation is identity-checked and retired so the +following request can build a fresh one without a stale callback evicting a +healthy replacement. Realtime-enabled cached instances expose a no-server +Grafserv upgrade handler. The shared server routes `/graphql` upgrades through +the same API resolution, origin, authentication, request-context, build +contract, runtime-role, listener-attestation, and cache-admission path as HTTP; +other paths and failed admission close with stable metadata-free errors. +Accepted sockets retain their exact cache generation until close and are +destroyed before that generation is disposed. + +`GRAPHILE_REALTIME_NOTIFICATION_MODE=shared-exact` is an experimental, +default-off transport seam and additionally requires a +`notificationPgResolver` in `ConstructiveOptions`. It must return explicit +credentials for a dedicated listener login and the exact routed physical +database; runtime or control-plane credentials are never a fallback. The +listener identity in a Graphile build contract is an opaque digest, and raw +connection configuration is neither serialized into the contract nor exposed +through cache statistics. The transport remains experimental until the hostile +cross-tenant subscription suite and loaded churn qualification pass on the +production-shaped fixture; the upgrade router itself is now production-wired +and fail-closed. ## Testing diff --git a/graphql/server/src/__tests__/server-cache-lifecycle.test.ts b/graphql/server/src/__tests__/server-cache-lifecycle.test.ts new file mode 100644 index 0000000000..c95982e383 --- /dev/null +++ b/graphql/server/src/__tests__/server-cache-lifecycle.test.ts @@ -0,0 +1,98 @@ +import { + getGraphileGovernorCounters, + reopenGraphileBuildCoordinator, + runGraphileBuild +} from '../middleware/graphile-build-governor'; +import { + GRAPHILE_CACHE_SHUTDOWN_DRAIN_TIMEOUT_CODE, + GraphileCacheShutdownError, + Server +} from '../server'; + +const deferred = () => { + let resolve!: (value: T) => void; + const promise = new Promise((accept) => { + resolve = accept; + }); + return { promise, resolve }; +}; + +const settle = async (): Promise => { + await Promise.resolve(); + await Promise.resolve(); +}; + +describe('process-wide Graphile cache lifecycle', () => { + const previousShutdownTimeout = process.env.GRAPHILE_BUILD_SHUTDOWN_TIMEOUT_MS; + + afterEach(() => { + if (previousShutdownTimeout === undefined) { + delete process.env.GRAPHILE_BUILD_SHUTDOWN_TIMEOUT_MS; + } else { + process.env.GRAPHILE_BUILD_SHUTDOWN_TIMEOUT_MS = previousShutdownTimeout; + } + jest.useRealTimers(); + }); + + it('drains an admitted build before a direct process-wide cache clear', async () => { + const gate = deferred(); + const build = runGraphileBuild(() => gate.promise); + await settle(); + + let closeSettled = false; + const close = Server.closeCaches().then(() => { + closeSettled = true; + }); + await settle(); + + expect(closeSettled).toBe(false); + expect(getGraphileGovernorCounters().activeBuilds).toBe(1); + + gate.resolve('built'); + await expect(build).resolves.toBe('built'); + await expect(close).resolves.toBeUndefined(); + expect(closeSettled).toBe(true); + await expect(runGraphileBuild(async () => 'reopened')).resolves.toBe('reopened'); + }); + + it('also drains when caches are requested after an ordinary Server close', async () => { + const gate = deferred(); + const build = runGraphileBuild(() => gate.promise); + await settle(); + const server = Object.create(Server.prototype) as Server; + Object.assign(server, { closed: true }); + + let closeSettled = false; + const close = server.close({ closeCaches: true }).then(() => { + closeSettled = true; + }); + await settle(); + expect(closeSettled).toBe(false); + + gate.resolve('built'); + await build; + await close; + expect(closeSettled).toBe(true); + await expect(runGraphileBuild(async () => 'reopened')).resolves.toBe('reopened'); + }); + + it('leaves caches intact and admission closed when a build cannot drain', async () => { + jest.useFakeTimers(); + process.env.GRAPHILE_BUILD_SHUTDOWN_TIMEOUT_MS = '10'; + const gate = deferred(); + const build = runGraphileBuild(() => gate.promise); + await settle(); + + const closeFailure = expect(Server.closeCaches()).rejects.toMatchObject({ + code: GRAPHILE_CACHE_SHUTDOWN_DRAIN_TIMEOUT_CODE + } satisfies Partial); + await settle(); + await jest.advanceTimersByTimeAsync(10); + await closeFailure; + + gate.resolve('late completion'); + await expect(build).resolves.toBe('late completion'); + expect(reopenGraphileBuildCoordinator()).toBe(true); + await expect(runGraphileBuild(async () => 'recovered')).resolves.toBe('recovered'); + }); +}); diff --git a/graphql/server/src/__tests__/server-pool-listener-lease.test.ts b/graphql/server/src/__tests__/server-pool-listener-lease.test.ts new file mode 100644 index 0000000000..a661fba7e1 --- /dev/null +++ b/graphql/server/src/__tests__/server-pool-listener-lease.test.ts @@ -0,0 +1,230 @@ +import { EventEmitter } from 'node:events'; + +jest.mock('pg-cache', () => { + class MockPgPoolCapacityError extends Error { + readonly code = 'PG_POOL_CAPACITY'; + readonly retryAfterSeconds = 15; + } + return { + acquirePgPool: jest.fn(), + getPgPool: jest.fn(), + pgCache: { + registerCleanupCallback: jest.fn(() => jest.fn()) + }, + PgPoolCapacityError: MockPgPoolCapacityError + }; +}); + +import type { PoolClient } from 'pg'; +import { acquirePgPool, PgPoolCapacityError } from 'pg-cache'; + +import { Server } from '../server'; + +const mockAcquirePgPool = acquirePgPool as jest.MockedFunction; + +class FakeClient extends EventEmitter { + query = jest.fn().mockResolvedValue({}); +} + +const settle = async (): Promise => { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +}; + +const serverWithoutConstructor = (): Server => { + const server = Object.create(Server.prototype) as Server & Record; + Object.assign(server, { + opts: { pg: { database: 'routing' } }, + listenAttempt: null, + listenRetryTimer: null, + listenCleanupTasks: new Set>(), + shuttingDown: false, + closed: false, + moduleRegistry: { invalidate: jest.fn() } + }); + server.error = jest.fn(); + server.log = jest.fn(); + server.flush = jest.fn().mockResolvedValue(undefined); + return server; +}; + +const poolLease = (connect: jest.Mock) => { + const release = jest.fn(); + return { + value: { + pool: { connect } as never, + identity: 'pg:control', + release + }, + release + }; +}; + +describe('server LISTEN PostgreSQL pool-lease lifecycle', () => { + beforeEach(() => { + jest.useFakeTimers(); + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('uses a dedicated identity so a one-client routing pool is not starved', async () => { + const retained = poolLease(jest.fn()); + mockAcquirePgPool.mockReturnValue(retained.value); + const server = serverWithoutConstructor(); + + server.addEventListener(); + + expect(mockAcquirePgPool).toHaveBeenCalledWith( + { database: 'routing' }, + { purpose: 'notifications' } + ); + (server as unknown as { shuttingDown: boolean }).shuttingDown = true; + await server.removeEventListener(); + }); + + it('releases the pool lease and schedules one retry when checkout fails', async () => { + const retained = poolLease(jest.fn((callback) => { + callback(new Error('connect failed')); + })); + mockAcquirePgPool.mockReturnValue(retained.value); + const server = serverWithoutConstructor(); + + server.addEventListener(); + await settle(); + + expect(retained.release).toHaveBeenCalledTimes(1); + expect(jest.getTimerCount()).toBe(1); + await server.removeEventListener(); + expect(jest.getTimerCount()).toBe(0); + }); + + it('releases both ownership layers and cancels retry when LISTEN rejects', async () => { + const client = new FakeClient(); + const listenFailure = new Error('LISTEN failed'); + client.query.mockRejectedValueOnce(listenFailure); + const releaseClient = jest.fn(); + const retained = poolLease(jest.fn((callback) => { + callback(null, client as unknown as PoolClient, releaseClient); + })); + mockAcquirePgPool.mockReturnValue(retained.value); + const server = serverWithoutConstructor(); + + server.addEventListener(); + await settle(); + + expect(releaseClient).toHaveBeenCalledTimes(1); + expect(releaseClient).toHaveBeenCalledWith(listenFailure); + expect(retained.release).toHaveBeenCalledTimes(1); + expect(jest.getTimerCount()).toBe(1); + await server.removeEventListener(); + expect(jest.getTimerCount()).toBe(0); + }); + + it('coalesces repeated connection errors into one cleanup and one reconnect', async () => { + const firstClient = new FakeClient(); + const firstClientRelease = jest.fn(); + const first = poolLease(jest.fn((callback) => { + callback(null, firstClient as unknown as PoolClient, firstClientRelease); + })); + const secondConnect = jest.fn(); + const second = poolLease(secondConnect); + mockAcquirePgPool + .mockReturnValueOnce(first.value) + .mockReturnValueOnce(second.value); + const server = serverWithoutConstructor(); + + server.addEventListener(); + await settle(); + const registry = (server as unknown as { + moduleRegistry: { invalidate: jest.Mock }; + }).moduleRegistry; + expect(registry.invalidate).toHaveBeenCalledTimes(1); + const errorHandler = firstClient.listeners('error')[0] as (error: Error) => void; + const socketFailure = new Error('socket failed'); + errorHandler(socketFailure); + errorHandler(new Error('duplicate socket failure')); + await settle(); + + expect(firstClientRelease).toHaveBeenCalledTimes(1); + expect(firstClientRelease).toHaveBeenCalledWith(socketFailure); + expect(first.release).toHaveBeenCalledTimes(1); + expect(registry.invalidate).toHaveBeenCalledTimes(2); + expect(jest.getTimerCount()).toBe(1); + + await jest.advanceTimersByTimeAsync(5000); + expect(mockAcquirePgPool).toHaveBeenCalledTimes(2); + expect(secondConnect).toHaveBeenCalledTimes(1); + + (server as unknown as { shuttingDown: boolean }).shuttingDown = true; + await server.removeEventListener(); + expect(second.release).toHaveBeenCalledTimes(1); + expect(jest.getTimerCount()).toBe(0); + }); + + it('releases a late successful checkout after shutdown without double-releasing its pool', async () => { + let connectCallback!: ( + error: Error | null, + client?: PoolClient, + release?: () => void + ) => void; + const retained = poolLease(jest.fn((callback) => { + connectCallback = callback; + })); + mockAcquirePgPool.mockReturnValue(retained.value); + const server = serverWithoutConstructor(); + const client = new FakeClient(); + const releaseClient = jest.fn(); + + server.addEventListener(); + (server as unknown as { shuttingDown: boolean }).shuttingDown = true; + await server.removeEventListener(); + connectCallback(null, client as unknown as PoolClient, releaseClient); + await settle(); + + expect(client.query).not.toHaveBeenCalled(); + expect(releaseClient).toHaveBeenCalledTimes(1); + expect(retained.release).toHaveBeenCalledTimes(1); + expect(jest.getTimerCount()).toBe(0); + }); + + it('UNLISTENs and releases exactly once during active shutdown', async () => { + const client = new FakeClient(); + const releaseClient = jest.fn(); + const retained = poolLease(jest.fn((callback) => { + callback(null, client as unknown as PoolClient, releaseClient); + })); + mockAcquirePgPool.mockReturnValue(retained.value); + const server = serverWithoutConstructor(); + + server.addEventListener(); + await settle(); + (server as unknown as { shuttingDown: boolean }).shuttingDown = true; + await server.removeEventListener(); + + expect(client.query).toHaveBeenNthCalledWith(1, 'LISTEN "schema:update"'); + expect(client.query).toHaveBeenNthCalledWith(2, 'UNLISTEN "schema:update"'); + expect(releaseClient).toHaveBeenCalledTimes(1); + expect(retained.release).toHaveBeenCalledTimes(1); + expect(jest.getTimerCount()).toBe(0); + }); + + it('cancels a capacity retry timer during shutdown', async () => { + mockAcquirePgPool.mockImplementation(() => { + throw new PgPoolCapacityError(1, 1, 1); + }); + const server = serverWithoutConstructor(); + + server.addEventListener(); + expect(jest.getTimerCount()).toBe(1); + (server as unknown as { shuttingDown: boolean }).shuttingDown = true; + await server.removeEventListener(); + + expect(jest.getTimerCount()).toBe(0); + await jest.advanceTimersByTimeAsync(15_000); + expect(mockAcquirePgPool).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphql/server/src/__tests__/server-process-shutdown.test.ts b/graphql/server/src/__tests__/server-process-shutdown.test.ts new file mode 100644 index 0000000000..ac1397742c --- /dev/null +++ b/graphql/server/src/__tests__/server-process-shutdown.test.ts @@ -0,0 +1,71 @@ +import { EventEmitter } from 'node:events'; + +import { + installProcessShutdownHandlers, + type ProcessShutdownTarget +} from '../server'; + +class FakeProcess extends EventEmitter implements ProcessShutdownTarget { + readonly exit = jest.fn((_code?: number): void => undefined); +} + +const flushPromises = async (): Promise => { + await Promise.resolve(); + await Promise.resolve(); +}; + +describe('GraphQL server process shutdown', () => { + afterEach(() => { + jest.useRealTimers(); + }); + + it('drains once and exits cleanly on the first signal', async () => { + const processTarget = new FakeProcess(); + const shutdown = jest.fn(async (): Promise => undefined); + + installProcessShutdownHandlers(shutdown, { processTarget, timeoutMs: 1000 }); + processTarget.emit('SIGTERM'); + await flushPromises(); + + expect(shutdown).toHaveBeenCalledTimes(1); + expect(processTarget.exit).toHaveBeenCalledTimes(1); + expect(processTarget.exit).toHaveBeenCalledWith(0); + expect(processTarget.listenerCount('SIGINT')).toBe(0); + expect(processTarget.listenerCount('SIGTERM')).toBe(0); + }); + + it('forces exit on a repeated signal without starting a second drain', async () => { + let resolveShutdown!: () => void; + const processTarget = new FakeProcess(); + const shutdown = jest.fn(() => new Promise((resolve) => { + resolveShutdown = resolve; + })); + + installProcessShutdownHandlers(shutdown, { processTarget, timeoutMs: 1000 }); + processTarget.emit('SIGTERM'); + processTarget.emit('SIGINT'); + + expect(shutdown).toHaveBeenCalledTimes(1); + expect(processTarget.exit).toHaveBeenCalledTimes(1); + expect(processTarget.exit).toHaveBeenCalledWith(1); + + resolveShutdown(); + await flushPromises(); + expect(processTarget.exit).toHaveBeenCalledTimes(1); + }); + + it('forces exit when graceful shutdown exceeds its deadline', () => { + jest.useFakeTimers(); + const processTarget = new FakeProcess(); + const shutdown = jest.fn(() => new Promise(() => undefined)); + + installProcessShutdownHandlers(shutdown, { processTarget, timeoutMs: 1000 }); + processTarget.emit('SIGTERM'); + jest.advanceTimersByTime(1000); + + expect(shutdown).toHaveBeenCalledTimes(1); + expect(processTarget.exit).toHaveBeenCalledTimes(1); + expect(processTarget.exit).toHaveBeenCalledWith(1); + expect(jest.getTimerCount()).toBe(0); + }); +}); diff --git a/graphql/server/src/__tests__/websocket-upgrade.test.ts b/graphql/server/src/__tests__/websocket-upgrade.test.ts new file mode 100644 index 0000000000..739fa5d81e --- /dev/null +++ b/graphql/server/src/__tests__/websocket-upgrade.test.ts @@ -0,0 +1,324 @@ +import { EventEmitter } from 'node:events'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { PassThrough } from 'node:stream'; + +import express, { + type Express, + type NextFunction, + type Request, + type Response +} from 'express'; + +import { + createGraphileWebSocketOriginGuard, + createGraphileWebSocketUpgradeGateway, + getGraphileWebSocketUpgradeTransport, + GRAPHILE_WEBSOCKET_ADMISSION_FAILED_CODE, + GRAPHILE_WEBSOCKET_ADMISSION_TIMEOUT_CODE, + GRAPHILE_WEBSOCKET_BAD_UPGRADE_CODE, + GRAPHILE_WEBSOCKET_ROUTE_NOT_FOUND_CODE, + handoffGraphileWebSocketUpgrade +} from '../websocket-upgrade'; + +const makeRequest = ( + overrides: Partial = {} +): IncomingMessage => Object.assign(new EventEmitter(), { + method: 'GET', + url: '/graphql', + headers: { + connection: 'keep-alive, Upgrade', + upgrade: 'websocket', + host: 'a.example.test' + }, + aborted: false, + httpVersion: '1.1', + httpVersionMajor: 1, + httpVersionMinor: 1, + socket: { destroyed: false } +}, overrides) as unknown as IncomingMessage; + +const makeSocket = (): PassThrough => new PassThrough(); + +const outputFrom = (socket: PassThrough): { read(): string } => { + let value = ''; + socket.on('data', (chunk) => { + value += chunk.toString(); + }); + return { read: () => value }; +}; + +const settle = async (): Promise => { + await new Promise((resolve) => setImmediate(resolve)); +}; + +describe('production Graphile WebSocket upgrade gateway', () => { + afterEach(() => { + jest.useRealTimers(); + }); + + it('rejects the wrong path before tenant routing runs', async () => { + const app = jest.fn() as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app); + const socket = makeSocket(); + const output = outputFrom(socket); + + gateway.handle(makeRequest({ url: '/graphiql' }), socket, Buffer.alloc(0)); + await settle(); + + expect(app).not.toHaveBeenCalled(); + expect(output.read()).toContain('HTTP/1.1 404'); + expect(output.read()).toContain(GRAPHILE_WEBSOCKET_ROUTE_NOT_FOUND_CODE); + }); + + it('rejects malformed upgrades before tenant routing runs', async () => { + const app = jest.fn() as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app); + const socket = makeSocket(); + const output = outputFrom(socket); + + gateway.handle(makeRequest({ + headers: { host: 'a.example.test', upgrade: 'h2c' } + }), socket, Buffer.alloc(0)); + await settle(); + + expect(app).not.toHaveBeenCalled(); + expect(output.read()).toContain('HTTP/1.1 400'); + expect(output.read()).toContain(GRAPHILE_WEBSOCKET_BAD_UPGRADE_CODE); + }); + + it('preserves A/B routing and auth state on the exact handed-off request', () => { + const observed: Array> = []; + const app = express(); + app.use((request, _response, next) => { + const host = request.headers.host; + request.api = { + dbname: host === 'a.example.test' ? 'tenant_a' : 'tenant_b', + databaseId: host === 'a.example.test' ? 'database-a' : 'database-b', + apiId: host === 'a.example.test' ? 'api-a' : 'api-b', + schema: [host === 'a.example.test' ? 'a_public' : 'b_public'], + anonRole: 'tenant_anon', + roleName: 'tenant_user', + domains: [], + isPublic: true + }; + next(); + }); + app.use((request, _response, next) => { + request.token = { + user_id: request.headers.authorization?.slice('Bearer '.length) + }; + next(); + }); + app.use((request, response) => { + const transport = getGraphileWebSocketUpgradeTransport(request); + const accepted = handoffGraphileWebSocketUpgrade(request, response); + observed.push({ + request, + databaseId: request.api.databaseId, + userId: request.token.user_id, + socket: accepted.socket, + head: accepted.head, + transport + }); + }); + const gateway = createGraphileWebSocketUpgradeGateway(app); + const firstSocket = makeSocket(); + const secondSocket = makeSocket(); + const firstHead = Buffer.from('first-head'); + const secondHead = Buffer.from('second-head'); + const first = makeRequest({ + headers: { + connection: 'upgrade', + upgrade: 'websocket', + host: 'a.example.test', + authorization: 'Bearer actor-a' + } + }); + const second = makeRequest({ + headers: { + connection: 'upgrade', + upgrade: 'websocket', + host: 'b.example.test', + authorization: 'Bearer actor-b' + } + }); + + gateway.handle(first, firstSocket, firstHead); + gateway.handle(second, secondSocket, secondHead); + + expect(observed).toHaveLength(2); + expect(observed[0]).toMatchObject({ + request: first, + databaseId: 'database-a', + userId: 'actor-a', + socket: firstSocket, + head: firstHead, + transport: { socket: firstSocket, head: firstHead } + }); + expect(observed[1]).toMatchObject({ + request: second, + databaseId: 'database-b', + userId: 'actor-b', + socket: secondSocket, + head: secondHead, + transport: { socket: secondSocket, head: secondHead } + }); + expect(gateway.pendingCount).toBe(0); + }); + + it('retires and detaches the synthetic response before handoff', () => { + const socket = makeSocket(); + const head = Buffer.from('preserved-head'); + let responseAtHandoff: ServerResponse | undefined; + let releaseCount = 0; + const app = ((request: Request, response: Response): void => { + responseAtHandoff = response as unknown as ServerResponse; + response.once('close', () => { + releaseCount++; + }); + handoffGraphileWebSocketUpgrade(request, response); + }) as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app); + + gateway.handle(makeRequest(), socket, head); + + expect(responseAtHandoff?.socket).toBeNull(); + expect(releaseCount).toBe(1); + expect(socket.destroyed).toBe(false); + expect(gateway.pendingCount).toBe(0); + }); + + it('does not leak middleware response bodies or tenant metadata', async () => { + const app = ((_request: Request, response: Response): void => { + response.statusCode = 500; + response.end('tenant_a secret-password cache-key'); + }) as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app); + const socket = makeSocket(); + const output = outputFrom(socket); + + gateway.handle(makeRequest(), socket, Buffer.alloc(0)); + await settle(); + + expect(output.read()).toContain('HTTP/1.1 500'); + expect(output.read()).toContain(GRAPHILE_WEBSOCKET_ADMISSION_FAILED_CODE); + expect(output.read()).not.toContain('tenant_a'); + expect(output.read()).not.toContain('secret-password'); + expect(output.read()).not.toContain('cache-key'); + }); + + it('rejects an untrusted browser origin before authentication work', async () => { + const authenticate = jest.fn(( + _request: Request, + _response: Response, + next: NextFunction + ) => next()); + const app = express(); + app.use((request, _response, next) => { + request.api = { + databaseId: 'database-a', + dbname: 'tenant_a', + schema: ['a_public'], + anonRole: 'tenant_anon', + roleName: 'tenant_user', + domains: [], + corsOrigins: ['https://console.example.test'] + }; + next(); + }); + app.use(createGraphileWebSocketOriginGuard()); + app.use(authenticate); + const gateway = createGraphileWebSocketUpgradeGateway(app); + const socket = makeSocket(); + const output = outputFrom(socket); + + gateway.handle(makeRequest({ + headers: { + connection: 'upgrade', + upgrade: 'websocket', + host: 'a.example.test', + origin: 'https://attacker.example.test', + cookie: 'constructive_session=session-a' + } + }), socket, Buffer.alloc(0)); + await settle(); + + expect(authenticate).not.toHaveBeenCalled(); + expect(output.read()).toContain('HTTP/1.1 403'); + expect(output.read()).toContain('GRAPHILE_WEBSOCKET_AUTH_REJECTED'); + }); + + it('aborts bounded pre-upgrade work when the peer disconnects', async () => { + let aborted = 0; + const app = ((request: Request): void => { + request.once('aborted', () => { + aborted++; + }); + }) as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app); + const socket = makeSocket(); + + gateway.handle(makeRequest(), socket, Buffer.alloc(0)); + socket.destroy(); + await settle(); + + expect(aborted).toBe(1); + expect(gateway.pendingCount).toBe(0); + }); + + it('times out pre-upgrade work with a stable fail-closed response', async () => { + jest.useFakeTimers(); + let aborted = 0; + let responseClosed = 0; + const app = ((request: Request, response: Response): void => { + request.once('aborted', () => { + aborted++; + }); + response.once('close', () => { + responseClosed++; + }); + }) as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app, { + admissionTimeoutMs: 25 + }); + const socket = makeSocket(); + const output = outputFrom(socket); + + gateway.handle(makeRequest(), socket, Buffer.alloc(0)); + await jest.advanceTimersByTimeAsync(25); + + expect(aborted).toBe(1); + expect(responseClosed).toBe(1); + expect(output.read()).toContain('HTTP/1.1 503'); + expect(output.read()).toContain(GRAPHILE_WEBSOCKET_ADMISSION_TIMEOUT_CODE); + expect(output.read()).toContain('Retry-After: 1'); + expect(gateway.pendingCount).toBe(0); + }); + + it('aborts waiters and closes synthetic responses before shutdown rejection', async () => { + let aborted = 0; + let responseClosed = 0; + const app = ((request: Request, response: Response): void => { + request.once('aborted', () => { + aborted++; + }); + response.once('close', () => { + responseClosed++; + }); + }) as unknown as Express; + const gateway = createGraphileWebSocketUpgradeGateway(app); + const socket = makeSocket(); + const output = outputFrom(socket); + + gateway.handle(makeRequest(), socket, Buffer.alloc(0)); + expect(gateway.pendingCount).toBe(1); + gateway.close(); + await settle(); + + expect(aborted).toBe(1); + expect(responseClosed).toBe(1); + expect(gateway.pendingCount).toBe(0); + expect(output.read()).toContain('HTTP/1.1 503'); + expect(output.read()).toContain('GRAPHILE_WEBSOCKET_SERVER_CLOSING'); + }); +}); diff --git a/graphql/server/src/agentic/router.ts b/graphql/server/src/agentic/router.ts index d1faf65320..151836ce27 100644 --- a/graphql/server/src/agentic/router.ts +++ b/graphql/server/src/agentic/router.ts @@ -19,7 +19,11 @@ */ import { OllamaAdapter } from '@agentic-kit/ollama'; -import type { BillingClient, LlmConfig } from '@constructive-io/express-context'; +import { + quoteQualifiedSqlIdentifier, + type BillingClient, + type LlmConfig +} from '@constructive-io/express-context'; import { getEnvOptions as getLlmEnvOptions } from '@constructive-io/llm-env'; import { Logger } from '@pgpmjs/logger'; import express, { Request, Response,Router } from 'express'; @@ -129,10 +133,15 @@ async function handleCreateThread( const body: CreateThreadBody = req.body || {}; const { schemaName, threadTableName } = agentChat; + const threadTableSql = quoteQualifiedSqlIdentifier( + schemaName, + threadTableName, + 'agent thread table' + ); const result = await ctx.withPgClient(async (client) => { const { rows } = await client.query( - `INSERT INTO "${schemaName}"."${threadTableName}" + `INSERT INTO ${threadTableSql} (entity_id, owner_id, mode, model, system_prompt, title) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, mode, model, system_prompt, status, created_at`, @@ -182,6 +191,16 @@ async function handleSendMessage( } const { schemaName, threadTableName, messageTableName } = agentChat; + const threadTableSql = quoteQualifiedSqlIdentifier( + schemaName, + threadTableName, + 'agent thread table' + ); + const messageTableSql = quoteQualifiedSqlIdentifier( + schemaName, + messageTableName, + 'agent message table' + ); const threadId = req.params.thread_id; const userId = ctx.userId; @@ -189,7 +208,7 @@ async function handleSendMessage( const threadRow = await ctx.withPgClient(async (client) => { const { rows } = await client.query( `SELECT id, mode, model, system_prompt, status - FROM "${schemaName}"."${threadTableName}" + FROM ${threadTableSql} WHERE id = $1`, [threadId] ); @@ -231,9 +250,9 @@ async function handleSendMessage( for (const msg of body.messages) { if (msg.role === 'user') { await client.query( - `INSERT INTO "${schemaName}"."${messageTableName}" + `INSERT INTO ${messageTableSql} (thread_id, owner_id, entity_id, author_role, parts) - VALUES ($1, $2, (SELECT entity_id FROM "${schemaName}"."${threadTableName}" WHERE id = $1), $3, $4)`, + VALUES ($1, $2, (SELECT entity_id FROM ${threadTableSql} WHERE id = $1), $3, $4)`, [threadId, userId, 'user', JSON.stringify([{ type: 'text', text: msg.content }])] ); } @@ -244,7 +263,7 @@ async function handleSendMessage( const history = await ctx.withPgClient(async (client) => { const { rows } = await client.query( `SELECT author_role, parts, created_at - FROM "${schemaName}"."${messageTableName}" + FROM ${messageTableSql} WHERE thread_id = $1 ORDER BY created_at ASC`, [threadId] @@ -277,14 +296,14 @@ async function handleSendMessage( await handleStreamingResponse(req, res, { ctx, chatAdapter, model, llmMessages, body, entityId, userId, threadId, - schemaName, threadTableName, messageTableName, + threadTableSql, messageTableSql, billing, startTime, meterSlug }); } else { await handleBatchResponse(req, res, { ctx, chatAdapter, model, llmMessages, body, entityId, userId, threadId, - schemaName, threadTableName, messageTableName, + threadTableSql, messageTableSql, billing, startTime, meterSlug }); } @@ -299,9 +318,8 @@ interface MessageContext { entityId: string; userId: string; threadId: string; - schemaName: string; - threadTableName: string; - messageTableName: string; + threadTableSql: string; + messageTableSql: string; billing: BillingClient | null; startTime: number; meterSlug: string; @@ -312,7 +330,7 @@ async function handleStreamingResponse( res: Response, mc: MessageContext ): Promise { - const { ctx, chatAdapter, model, llmMessages, body, entityId, userId, threadId, schemaName, threadTableName, messageTableName, billing, startTime, meterSlug } = mc; + const { ctx, chatAdapter, model, llmMessages, body, entityId, userId, threadId, threadTableSql, messageTableSql, billing, startTime, meterSlug } = mc; res.writeHead(200, { 'Content-Type': 'text/event-stream', @@ -375,9 +393,9 @@ async function handleStreamingResponse( if (content) { ctx.withPgClient(async (client) => { await client.query( - `INSERT INTO "${schemaName}"."${messageTableName}" + `INSERT INTO ${messageTableSql} (thread_id, owner_id, entity_id, author_role, parts, model) - VALUES ($1, $2, (SELECT entity_id FROM "${schemaName}"."${threadTableName}" WHERE id = $1), $3, $4, $5)`, + VALUES ($1, $2, (SELECT entity_id FROM ${threadTableSql} WHERE id = $1), $3, $4, $5)`, [threadId, userId, 'assistant', JSON.stringify([{ type: 'text', text: content }]), model] ); }).catch((err) => log.error('Failed to persist assistant message:', err)); @@ -416,7 +434,7 @@ async function handleBatchResponse( res: Response, mc: MessageContext ): Promise { - const { ctx, chatAdapter, model, llmMessages, body, entityId, userId, threadId, schemaName, threadTableName, messageTableName, billing, startTime, meterSlug } = mc; + const { ctx, chatAdapter, model, llmMessages, body, entityId, userId, threadId, threadTableSql, messageTableSql, billing, startTime, meterSlug } = mc; const systemMsg = llmMessages.find(m => m.role === 'system'); const nonSystem = llmMessages.filter(m => m.role !== 'system'); @@ -451,9 +469,9 @@ async function handleBatchResponse( // Persist assistant message await ctx.withPgClient(async (client) => { await client.query( - `INSERT INTO "${schemaName}"."${messageTableName}" + `INSERT INTO ${messageTableSql} (thread_id, owner_id, entity_id, author_role, parts, model) - VALUES ($1, $2, (SELECT entity_id FROM "${schemaName}"."${threadTableName}" WHERE id = $1), $3, $4, $5)`, + VALUES ($1, $2, (SELECT entity_id FROM ${threadTableSql} WHERE id = $1), $3, $4, $5)`, [threadId, userId, 'assistant', JSON.stringify([{ type: 'text', text: content }]), model] ); }); diff --git a/graphql/server/src/diagnostics/__tests__/observability.test.ts b/graphql/server/src/diagnostics/__tests__/observability.test.ts index 507584a5ce..799dde74cd 100644 --- a/graphql/server/src/diagnostics/__tests__/observability.test.ts +++ b/graphql/server/src/diagnostics/__tests__/observability.test.ts @@ -2,6 +2,7 @@ import { isDevelopmentObservabilityMode, isGraphqlDebugSamplerEnabled, isGraphqlObservabilityEnabled, + isGraphqlObservabilityTokenValid, isLoopbackAddress, isLoopbackHost, } from '../observability'; @@ -36,12 +37,27 @@ describe('observability helpers', () => { expect(isGraphqlDebugSamplerEnabled('0.0.0.0')).toBe(false); }); - it('disables observability outside development even when requested', () => { + it('disables production observability without a strong token', () => { process.env.NODE_ENV = 'production'; process.env.GRAPHQL_OBSERVABILITY_ENABLED = 'true'; expect(isDevelopmentObservabilityMode()).toBe(false); expect(isGraphqlObservabilityEnabled('localhost')).toBe(false); expect(isGraphqlDebugSamplerEnabled('localhost')).toBe(false); + + process.env.GRAPHQL_OBSERVABILITY_TOKEN = 'too-short'; + expect(isGraphqlObservabilityEnabled('localhost')).toBe(false); + }); + + it('allows token-authenticated production observability only on loopback', () => { + process.env.NODE_ENV = 'production'; + process.env.GRAPHQL_OBSERVABILITY_ENABLED = 'true'; + process.env.GRAPHQL_OBSERVABILITY_TOKEN = 'a'.repeat(64); + + expect(isGraphqlObservabilityEnabled('localhost')).toBe(true); + expect(isGraphqlDebugSamplerEnabled('127.0.0.1')).toBe(true); + expect(isGraphqlObservabilityEnabled('0.0.0.0')).toBe(false); + expect(isGraphqlObservabilityTokenValid('a'.repeat(64))).toBe(true); + expect(isGraphqlObservabilityTokenValid('b'.repeat(64))).toBe(false); }); }); diff --git a/graphql/server/src/diagnostics/debug-db-snapshot.ts b/graphql/server/src/diagnostics/debug-db-snapshot.ts index 666ad9619b..75932986d8 100644 --- a/graphql/server/src/diagnostics/debug-db-snapshot.ts +++ b/graphql/server/src/diagnostics/debug-db-snapshot.ts @@ -203,7 +203,7 @@ export interface DebugDatabaseSnapshot { export const getDebugDatabaseSnapshot = async ( opts: ConstructiveOptions, ): Promise => { - const appPool = getPgPool(opts.pg); + const appPool = getPgPool(opts.pg, { purpose: 'diagnostics' }); const { activity, blocked, diff --git a/graphql/server/src/diagnostics/debug-memory-snapshot.ts b/graphql/server/src/diagnostics/debug-memory-snapshot.ts index b35f5e8779..f85e42d765 100644 --- a/graphql/server/src/diagnostics/debug-memory-snapshot.ts +++ b/graphql/server/src/diagnostics/debug-memory-snapshot.ts @@ -1,11 +1,14 @@ import os from 'node:os'; import v8 from 'node:v8'; -import { SVC_CACHE_TTL_MS,svcCache } from '@pgpmjs/server-utils'; -import { getCacheStats } from 'graphile-cache'; +import { getSvcCacheStats } from '@pgpmjs/server-utils'; +import { getCacheCounters, getCacheStats } from 'graphile-cache'; +import { getPgCacheStats, getPgCheckoutSanitizerStats } from 'pg-cache'; import { getInFlightCount, getInFlightKeys } from '../middleware/graphile'; +import { getGraphileGovernorCounters } from '../middleware/graphile-build-governor'; import { getGraphileBuildStats } from '../middleware/observability/graphile-build-stats'; +import { getRuntimeRoleSafetyStats } from '../middleware/runtime-role-safety'; const toMB = (bytes: number): string => `${(bytes / 1024 / 1024).toFixed(1)} MB`; @@ -43,13 +46,12 @@ export interface DebugMemorySnapshot { }>; }; graphileCache: ReturnType; - svcCache: { - size: number; - max: number; - ttlMs: number; - oldestKeyAgeMs: number | null; - keys: string[]; - }; + graphileCacheCounters: ReturnType; + graphileGovernor: ReturnType; + pgCache: ReturnType; + pgCheckoutSanitizer: ReturnType; + runtimeRoleSafety: ReturnType; + svcCache: ReturnType; inFlight: { count: number; keys: string[]; @@ -97,23 +99,12 @@ export const getDebugMemorySnapshot = (): DebugMemorySnapshot => { heapSpaces, }, graphileCache: getCacheStats(), - svcCache: { - size: svcCache.size, - max: svcCache.max, - ttlMs: SVC_CACHE_TTL_MS, - // Note: with updateAgeOnGet: true, this is "time since last access" not "time since creation" - oldestKeyAgeMs: (() => { - let minRemaining = Infinity; - for (const key of svcCache.keys()) { - const remaining = svcCache.getRemainingTTL(key); - if (remaining < minRemaining) { - minRemaining = remaining; - } - } - return Number.isFinite(minRemaining) ? SVC_CACHE_TTL_MS - minRemaining : null; - })(), - keys: [...svcCache.keys()].slice(0, 200), - }, + graphileCacheCounters: getCacheCounters(), + graphileGovernor: getGraphileGovernorCounters(), + pgCache: getPgCacheStats(), + pgCheckoutSanitizer: getPgCheckoutSanitizerStats(), + runtimeRoleSafety: getRuntimeRoleSafetyStats(), + svcCache: getSvcCacheStats(), inFlight: { count: getInFlightCount(), keys: getInFlightKeys(), diff --git a/graphql/server/src/diagnostics/observability.ts b/graphql/server/src/diagnostics/observability.ts index bf0e8d466a..09d1e9c642 100644 --- a/graphql/server/src/diagnostics/observability.ts +++ b/graphql/server/src/diagnostics/observability.ts @@ -1,5 +1,8 @@ +import { timingSafeEqual } from 'node:crypto'; + const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1', '[::1]']); const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1']); +const MIN_OBSERVABILITY_TOKEN_BYTES = 32; const parseBooleanEnv = (value: string | undefined, fallback: boolean): boolean => { if (value == null) { @@ -46,6 +49,29 @@ const normalizeAddress = (value: string | null | undefined): string | null => { export const isDevelopmentObservabilityMode = (): boolean => process.env.NODE_ENV === 'development'; +/** + * Production observability is reserved for an explicitly authenticated local + * process such as cperf. Reject short secrets so an accidental boolean-like + * value cannot turn a production debug route on. + */ +export const getGraphqlObservabilityToken = (): string | null => { + const token = process.env.GRAPHQL_OBSERVABILITY_TOKEN?.trim(); + if (!token || Buffer.byteLength(token) < MIN_OBSERVABILITY_TOKEN_BYTES) { + return null; + } + return token; +}; + +export const isGraphqlObservabilityTokenValid = ( + candidate: string | null | undefined +): boolean => { + const token = getGraphqlObservabilityToken(); + if (!token || !candidate) return false; + const expected = Buffer.from(token); + const actual = Buffer.from(candidate); + return expected.length === actual.length && timingSafeEqual(expected, actual); +}; + export const isLoopbackHost = (value: string | null | undefined): boolean => { const normalized = normalizeHost(value); return normalized != null && LOOPBACK_HOSTS.has(normalized); @@ -60,9 +86,9 @@ export const isGraphqlObservabilityRequested = (): boolean => parseBooleanEnv(process.env.GRAPHQL_OBSERVABILITY_ENABLED, false); export const isGraphqlObservabilityEnabled = (serverHost?: string | null): boolean => - isDevelopmentObservabilityMode() && isGraphqlObservabilityRequested() && - isLoopbackHost(serverHost); + isLoopbackHost(serverHost) && + (isDevelopmentObservabilityMode() || getGraphqlObservabilityToken() !== null); export const isGraphqlDebugSamplerEnabled = (serverHost?: string | null): boolean => isGraphqlObservabilityEnabled(serverHost) && diff --git a/graphql/server/src/index.ts b/graphql/server/src/index.ts index edd35483ad..75cb2de241 100644 --- a/graphql/server/src/index.ts +++ b/graphql/server/src/index.ts @@ -6,3 +6,7 @@ export { createAuthenticateMiddleware } from './middleware/auth'; export { cors } from './middleware/cors'; export { flush, flushService } from './middleware/flush'; export { graphile } from './middleware/graphile'; +export { + GRAPHILE_PROTECTED_PRESET_OVERRIDE_CODE, + GraphileProtectedPresetOverrideError +} from './middleware/graphile-preset-composition'; diff --git a/graphql/server/src/middleware/__tests__/api.test.ts b/graphql/server/src/middleware/__tests__/api.test.ts index 965d01094c..4f4c02b0d4 100644 --- a/graphql/server/src/middleware/__tests__/api.test.ts +++ b/graphql/server/src/middleware/__tests__/api.test.ts @@ -1,22 +1,68 @@ jest.mock('pg-cache', () => ({ - getPgPool: jest.fn() + acquirePgPool: jest.fn(), + getPgPoolIdentity: jest.fn(), + PG_POOL_CAPACITY_ERROR_CODE: 'PG_POOL_CAPACITY' })); jest.mock('@constructive-io/express-context', () => ({ createDefaultRegistry: jest.fn(() => ({ - resolve: jest.fn().mockResolvedValue(undefined) + resolve: jest.fn(async (name: string) => name === 'databaseSettings' ? { + enableAggregates: false, + enablePostgis: false, + enableSearch: false, + enableDirectUploads: false, + enablePresignedUploads: false, + enableManyToMany: false, + enableConnectionFilter: false, + enableLtree: false, + enableLlm: false, + enableRealtime: false, + enableBulk: false, + enableI18n: false + } : undefined) })) })); +import { createDefaultRegistry } from '@constructive-io/express-context'; import { svcCache } from '@pgpmjs/server-utils'; -import type { Request } from 'express'; +import type { NextFunction, Request, Response } from 'express'; import type { Pool } from 'pg'; -import { getPgPool } from 'pg-cache'; +import { acquirePgPool, getPgPoolIdentity } from 'pg-cache'; import type { ApiOptions } from '../../types'; -import { getApiConfig, getSvcKey } from '../api'; +import { + createApiMiddleware, + getApiConfig, + getSvcCacheKey, + getSvcKey +} from '../api'; +import { + authorizeInternalRequest, + INTERNAL_REQUEST_TOKEN_HEADER +} from '../internal-request'; -const mockGetPgPool = getPgPool as jest.MockedFunction; +const INTERNAL_SECRET = 'test-internal-secret-with-at-least-32-bytes'; + +const withInternalAuth = ( + headers: Record +): Record => ({ + ...headers, + [INTERNAL_REQUEST_TOKEN_HEADER]: INTERNAL_SECRET +}); + +const mockAcquirePgPool = acquirePgPool as jest.MockedFunction; +const mockGetPgPoolIdentity = getPgPoolIdentity as jest.MockedFunction< + typeof getPgPoolIdentity +>; +const mockRegistryResolve = ( + createDefaultRegistry as jest.MockedFunction +).mock.results[0].value.resolve as jest.Mock; + +const leasePool = (pool: Pool, release = jest.fn()) => ({ + pool, + identity: 'pg:test', + release +}); const createRequest = (headers: Record): Request => { const normalized = new Map( @@ -36,7 +82,8 @@ const createPrivateOptions = (): ApiOptions => ({ }, api: { isPublic: false, - metaSchemas: ['metaschema_public'] + metaSchemas: ['metaschema_public'], + internalRequestSecret: INTERNAL_SECRET } } as unknown as ApiOptions); @@ -44,24 +91,28 @@ describe('api middleware routing priority', () => { beforeEach(() => { svcCache.clear(); jest.clearAllMocks(); + mockGetPgPoolIdentity.mockImplementation((config) => + `pg:${config.host ?? 'test'}` + ); }); afterEach(() => { svcCache.clear(); }); - it('uses X-Api-Name before X-Schemata when building private service keys', () => { - const req = createRequest({ + it('uses an authenticated X-Api-Name when building private service keys', () => { + const opts = createPrivateOptions(); + const req = createRequest(withInternalAuth({ host: 'admin.localhost', 'X-Database-Id': 'db-123', - 'X-Api-Name': 'customer-api', - 'X-Schemata': 'app_public' - }); + 'X-Api-Name': 'customer-api' + })); + authorizeInternalRequest(opts, req); - expect(getSvcKey(createPrivateOptions(), req)).toBe('api:db-123:customer-api'); + expect(getSvcKey(opts, req)).toBe('api:db-123:customer-api'); }); - it('uses the same X-Api-Name priority when resolving and caching API config', async () => { + it('resolves an authenticated X-Api-Name without caching routing authority', async () => { const query = jest.fn(async (_sql: string, params: unknown[]) => { if (Array.isArray(params[0])) { return { @@ -88,18 +139,27 @@ describe('api middleware routing priority', () => { return { rows: [] }; }); - mockGetPgPool.mockReturnValue({ query } as unknown as Pool); + const pool = { query } as unknown as Pool; + const releases: jest.Mock[] = []; + mockAcquirePgPool.mockImplementation(() => { + const release = jest.fn(); + releases.push(release); + return leasePool(pool, release); + }); - const req = createRequest({ + const req = createRequest(withInternalAuth({ host: 'admin.localhost', 'X-Database-Id': 'db-123', - 'X-Api-Name': 'customer-api', - 'X-Schemata': 'app_public' - }); + 'X-Api-Name': 'customer-api' + })); const result = await getApiConfig(createPrivateOptions(), req); expect(req.svc_key).toBe('api:db-123:customer-api'); + expect(req.svc_cache_key).toBe(getSvcCacheKey( + createPrivateOptions(), + 'api:db-123:customer-api' + )); expect(result).toMatchObject({ apiId: 'api-123', dbname: 'tenant_db', @@ -109,9 +169,284 @@ describe('api middleware routing priority', () => { databaseId: 'db-123', isPublic: false }); - expect(svcCache.get('api:db-123:customer-api')).toBe(result); + expect(svcCache.has(getSvcCacheKey( + createPrivateOptions(), + 'api:db-123:customer-api' + ))).toBe(false); expect(query.mock.calls).toEqual(expect.arrayContaining([ [expect.stringContaining('FROM "routing_public".apis'), ['db-123', 'customer-api']] ])); + expect(query.mock.calls.some(([sql]) => + String(sql).includes('aps.database_id = a.database_id') + )).toBe(true); + expect(mockAcquirePgPool).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ database: 'constructive' }), + { purpose: 'routing-request-control', sanitizeOnCheckout: true } + ); + expect(mockAcquirePgPool).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ database: 'tenant_db' }), + { purpose: 'tenant-request-control', sanitizeOnCheckout: true } + ); + expect(releases).toHaveLength(2); + expect(releases.every((release) => release.mock.calls.length === 1)).toBe(true); + }); + + it('fails closed when an authenticated API selector resolves an incomplete contract', async () => { + const query = jest.fn(async (_sql: string, params: unknown[]) => { + if (Array.isArray(params[0])) { + return { + rows: (params[0] as string[]).map((schemaName) => ({ schema_name: schemaName })) + }; + } + return { + rows: [{ + api_id: 'api-123', + database_id: 'db-123', + dbname: 'tenant_db', + role_name: '', + anon_role: 'api_anon', + is_public: false, + schemas: ['api_public'] + }] + }; + }); + mockAcquirePgPool.mockReturnValue(leasePool({ query } as unknown as Pool)); + + const result = await getApiConfig(createPrivateOptions(), createRequest(withInternalAuth({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Api-Name': 'customer-api' + }))); + + expect(result).toBeNull(); + }); + + it('fails closed when an exact tenant API has no feature contract', async () => { + const query = jest.fn(async (_sql: string, params: unknown[]) => { + if (Array.isArray(params[0])) { + return { + rows: (params[0] as string[]).map((schemaName) => ({ schema_name: schemaName })) + }; + } + return { rows: [{ + api_id: 'api-123', + database_id: 'db-123', + dbname: 'tenant_db', + role_name: 'api_user', + anon_role: 'api_anon', + is_public: false, + schemas: ['api_public'] + }] }; + }); + mockAcquirePgPool.mockReturnValue(leasePool({ query } as unknown as Pool)); + mockRegistryResolve.mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined); + const req = createRequest(withInternalAuth({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Api-Name': 'customer-api' + })); + + await expect(getApiConfig(createPrivateOptions(), req)).rejects.toMatchObject({ + code: 'GRAPHILE_DATABASE_FEATURE_CONTRACT_MISSING' + }); + }); + + it('ignores a stale routing-cache entry and resolves the authoritative API', async () => { + const opts = createPrivateOptions(); + const req = createRequest(withInternalAuth({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Api-Name': 'customer-api' + })); + svcCache.set(getSvcCacheKey( + opts, + 'api:db-123:customer-api' + ), { databaseId: 'db-old', dbname: 'wrong_tenant' }); + + const query = jest.fn(async (_sql: string, params: unknown[]) => { + if (Array.isArray(params[0])) { + return { + rows: (params[0] as string[]).map((schemaName) => ({ + schema_name: schemaName + })) + }; + } + return { + rows: [{ + api_id: 'api-123', + database_id: 'db-123', + dbname: 'tenant_fresh', + role_name: 'api_role', + anon_role: 'api_anon', + is_public: false, + schemas: ['api_public'] + }] + }; + }); + mockAcquirePgPool.mockImplementation(() => + leasePool({ query } as unknown as Pool) + ); + + await expect(getApiConfig(opts, req)).resolves.toMatchObject({ + databaseId: 'db-123', + dbname: 'tenant_fresh' + }); + expect(mockAcquirePgPool).toHaveBeenCalled(); + }); + + it('isolates one routing label across exact control-pool contracts', () => { + const optsA = createPrivateOptions(); + const optsB = createPrivateOptions(); + optsA.pg = { ...optsA.pg, host: 'routing-a.internal' }; + optsB.pg = { ...optsB.pg, host: 'routing-b.internal' }; + const label = 'api:db-123:customer-api'; + + expect(getSvcCacheKey(optsA, label)).not.toBe(getSvcCacheKey(optsB, label)); + }); + + it('does not publish authoritative meta-schema routing results', async () => { + const query = jest.fn(async (_sql: string, params: unknown[]) => ({ + rows: (params[0] as string[]).map((schemaName) => ({ schema_name: schemaName })) + })); + mockAcquirePgPool.mockReturnValue(leasePool({ query } as unknown as Pool)); + const opts = createPrivateOptions(); + opts.api!.allowMetaSchemaHeader = true; + const req = createRequest(withInternalAuth({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Meta-Schema': 'metaschema_public' + })); + + await expect(getApiConfig(opts, req)).resolves.toMatchObject({ databaseId: 'db-123' }); + expect(svcCache.has(req.svc_cache_key!)).toBe(false); + }); + + it('rejects raw physical schema routing even with a valid internal token', async () => { + const opts = createPrivateOptions(); + const req = createRequest(withInternalAuth({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Schemata': 'app_public' + })); + + await expect(getApiConfig(opts, req)).rejects.toMatchObject({ + code: 'INTERNAL_REQUEST_FORBIDDEN' + }); + expect(mockAcquirePgPool).not.toHaveBeenCalled(); + }); + + it('rejects unauthenticated private routing headers before touching PostgreSQL', async () => { + const req = createRequest({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Api-Name': 'customer-api' + }); + + await expect(getApiConfig(createPrivateOptions(), req)).rejects.toMatchObject({ + code: 'INTERNAL_REQUEST_FORBIDDEN' + }); + expect(mockAcquirePgPool).not.toHaveBeenCalled(); + }); + + it('returns 403 for an invalid internal token without leaking configuration', async () => { + const req = createRequest({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Api-Name': 'customer-api', + [INTERNAL_REQUEST_TOKEN_HEADER]: 'wrong-secret-with-at-least-32-bytes' + }); + const res = { + status: jest.fn().mockReturnThis(), + send: jest.fn() + } as unknown as Response; + const next = jest.fn() as NextFunction; + + await createApiMiddleware(createPrivateOptions())(req, res, next); + + expect(res.status).toHaveBeenCalledWith(403); + expect(res.send).toHaveBeenCalledWith('Forbidden'); + expect(next).not.toHaveBeenCalled(); + expect(mockAcquirePgPool).not.toHaveBeenCalled(); + }); + + it('releases the routing lease when schema validation fails', async () => { + const release = jest.fn(); + const query = jest.fn().mockRejectedValue(new Error('validation failed')); + mockAcquirePgPool.mockReturnValue( + leasePool({ query } as unknown as Pool, release) + ); + + const req = createRequest({ host: 'admin.localhost' }); + await expect(getApiConfig(createPrivateOptions(), req)).rejects.toThrow('validation failed'); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('releases tenant and routing leases when module resolution fails', async () => { + const query = jest.fn(async (_sql: string, params: unknown[]) => { + if (Array.isArray(params[0])) { + return { + rows: (params[0] as string[]).map((schemaName) => ({ + schema_name: schemaName + })) + }; + } + return { + rows: [{ + api_id: 'api-123', + database_id: 'db-123', + dbname: 'tenant_db', + role_name: 'api_role', + anon_role: 'api_anon', + is_public: false, + schemas: ['api_public'] + }] + }; + }); + const releaseOrder: string[] = []; + mockAcquirePgPool + .mockReturnValueOnce(leasePool( + { query } as unknown as Pool, + jest.fn(() => releaseOrder.push('routing')) + )) + .mockReturnValueOnce(leasePool( + { query } as unknown as Pool, + jest.fn(() => releaseOrder.push('tenant')) + )); + mockRegistryResolve.mockRejectedValueOnce(new Error('loader failed')); + const req = createRequest(withInternalAuth({ + host: 'admin.localhost', + 'X-Database-Id': 'db-123', + 'X-Api-Name': 'customer-api' + })); + + await expect(getApiConfig(createPrivateOptions(), req)).rejects.toThrow('loader failed'); + expect(releaseOrder).toEqual(['tenant', 'routing']); + }); + + it('forwards pool-capacity refusal to the shared HTTP error handler', async () => { + const capacityError = Object.assign(new Error('sensitive capacity details'), { + code: 'PG_POOL_CAPACITY' + }); + mockAcquirePgPool.mockImplementation(() => { + throw capacityError; + }); + const req = createRequest({ host: 'admin.localhost' }); + const res = { + status: jest.fn().mockReturnThis(), + send: jest.fn() + } as unknown as Response; + const next = jest.fn() as NextFunction; + + await createApiMiddleware(createPrivateOptions())(req, res, next); + + expect(next).toHaveBeenCalledWith(capacityError); + expect((res.status as jest.Mock)).not.toHaveBeenCalled(); }); }); diff --git a/graphql/server/src/middleware/__tests__/auth-pool-lease.test.ts b/graphql/server/src/middleware/__tests__/auth-pool-lease.test.ts new file mode 100644 index 0000000000..2ae5daa1c9 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/auth-pool-lease.test.ts @@ -0,0 +1,268 @@ +jest.mock('pg-cache', () => ({ + acquirePgPool: jest.fn(), + PG_POOL_CAPACITY_ERROR_CODE: 'PG_POOL_CAPACITY' +})); + +jest.mock('pg-query-context', () => ({ + __esModule: true, + default: jest.fn() +})); + +import type { PgpmOptions } from '@pgpmjs/types'; +import type { NextFunction, Request, Response } from 'express'; +import { acquirePgPool } from 'pg-cache'; +import pgQueryContext from 'pg-query-context'; + +import type { ApiStructure, RlsModule } from '../../types'; +import { createAuthenticateMiddleware } from '../auth'; + +const mockAcquirePgPool = acquirePgPool as jest.MockedFunction; +const mockPgQueryContext = pgQueryContext as jest.MockedFunction; + +const rlsModule: RlsModule = { + authenticate: 'authenticate', + authenticateStrict: 'authenticate_strict', + privateSchema: { schemaName: 'auth_private' }, + publicSchema: { schemaName: 'auth_public' }, + currentRole: 'current_role', + currentRoleId: 'current_role_id', + currentIpAddress: 'current_ip_address', + currentUserAgent: 'current_user_agent' +}; + +const api = (overrides: Partial = {}): ApiStructure => ({ + dbname: 'tenant_db', + anonRole: 'anonymous', + roleName: 'authenticated', + schema: ['app_public'], + databaseId: 'db-1', + rlsModule, + ...overrides +}); + +const request = ( + apiConfig: ApiStructure, + headers: Record = {} +): Request => { + const normalized = Object.fromEntries( + Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]) + ); + return { + api: apiConfig, + clientIp: '127.0.0.1', + headers: normalized, + get: jest.fn((name: string) => normalized[name.toLowerCase()]) + } as unknown as Request; +}; + +const response = (): Response => { + const res = { + status: jest.fn(), + json: jest.fn(), + send: jest.fn() + }; + res.status.mockReturnValue(res); + return res as unknown as Response; +}; + +const opts = { + pg: { + database: 'routing_db', + user: 'control_user' + }, + server: { + strictAuth: false + } +} as unknown as PgpmOptions; + +describe('authenticate middleware PostgreSQL pool leases', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it.each([ + ['an API without RLS', api({ rlsModule: undefined }), {}], + ['an anonymous request', api(), {}] + ])('does not allocate a tenant control pool for %s', async (_label, apiConfig, headers) => { + const req = request(apiConfig as ApiStructure, headers as Record); + const res = response(); + const next = jest.fn() as NextFunction; + + await createAuthenticateMiddleware(opts)(req, res, next); + + expect(mockAcquirePgPool).not.toHaveBeenCalled(); + expect(mockPgQueryContext).not.toHaveBeenCalled(); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('fails closed when the selected authentication function is absent', async () => { + const req = request(api({ + rlsModule: { ...rlsModule, authenticate: '' } + }), { authorization: 'Bearer credential' }); + const res = response(); + const next = jest.fn() as NextFunction; + + await createAuthenticateMiddleware(opts)(req, res, next); + + expect(mockAcquirePgPool).not.toHaveBeenCalled(); + expect(mockPgQueryContext).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(200); + expect(next).not.toHaveBeenCalled(); + }); + + it('fails closed when strict authentication cannot resolve an RLS module', async () => { + const strictOpts = { + ...opts, + server: { ...opts.server, strictAuth: true } + } as unknown as PgpmOptions; + const res = response(); + const next = jest.fn() as NextFunction; + + await createAuthenticateMiddleware(strictOpts)( + request(api({ rlsModule: undefined })), + res, + next + ); + + expect(mockAcquirePgPool).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(200); + expect(next).not.toHaveBeenCalled(); + }); + + it('quotes metadata-derived authentication identifiers', async () => { + const release = jest.fn(); + mockAcquirePgPool.mockReturnValue({ + pool: {} as never, + identity: 'pg:tenant-control', + release + }); + mockPgQueryContext.mockResolvedValue({ + rowCount: 1, + rows: [{ role: 'authenticated', user_id: 'user-1' }] + } as never); + const req = request(api({ + rlsModule: { + ...rlsModule, + privateSchema: { schemaName: 'auth";select pg_sleep(9);--' }, + authenticate: 'authenticate";drop schema public;--' + } + }), { authorization: 'Bearer credential' }); + + await createAuthenticateMiddleware(opts)( + req, + response(), + jest.fn() as NextFunction + ); + + expect(mockPgQueryContext).toHaveBeenCalledWith(expect.objectContaining({ + query: 'SELECT * FROM "auth"";select pg_sleep(9);--"."authenticate"";drop schema public;--"($1)', + variables: ['credential'] + })); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('leases the exact tenant control pool only for the credential query', async () => { + const release = jest.fn(); + const pool = { query: jest.fn() }; + mockAcquirePgPool.mockReturnValue({ + pool: pool as never, + identity: 'pg:tenant-control', + release + }); + mockPgQueryContext.mockResolvedValue({ + rowCount: 1, + rows: [{ role: 'authenticated', user_id: 'user-1' }] + } as never); + const req = request(api(), { authorization: 'Bearer credential' }); + const res = response(); + const next = jest.fn() as NextFunction; + + await createAuthenticateMiddleware(opts)(req, res, next); + + expect(mockAcquirePgPool).toHaveBeenCalledTimes(1); + expect(mockAcquirePgPool).toHaveBeenCalledWith( + expect.objectContaining({ + database: 'tenant_db', + user: 'control_user' + }), + { purpose: 'tenant-request-control', sanitizeOnCheckout: true } + ); + expect(mockPgQueryContext).toHaveBeenCalledWith(expect.objectContaining({ + client: pool, + query: 'SELECT * FROM "auth_private"."authenticate"($1)', + variables: ['credential'] + })); + expect(release).toHaveBeenCalledTimes(1); + expect(req.token).toEqual(expect.objectContaining({ user_id: 'user-1' })); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('omits an unavailable client IP from the credential-query context', async () => { + const release = jest.fn(); + mockAcquirePgPool.mockReturnValue({ + pool: {} as never, + identity: 'pg:tenant-control', + release + }); + mockPgQueryContext.mockResolvedValue({ + rowCount: 1, + rows: [{ role: 'authenticated', user_id: 'user-1' }] + } as never); + const req = request(api(), { authorization: 'Bearer credential' }); + req.clientIp = undefined; + + await createAuthenticateMiddleware( + opts + )(req, response(), jest.fn() as NextFunction); + + expect(mockPgQueryContext).toHaveBeenCalledWith(expect.objectContaining({ + context: expect.objectContaining({ + 'jwt.claims.ip_address': '', + 'jwt.claims.origin': '', + 'jwt.claims.user_agent': '', + 'jwt.claims.database_id': 'db-1', + 'row_security': 'on', + 'search_path': 'pg_catalog', + 'transaction_read_only': 'on' + }) + })); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('releases the tenant control pool when the credential query fails', async () => { + const release = jest.fn(); + mockAcquirePgPool.mockReturnValue({ + pool: {} as never, + identity: 'pg:tenant-control', + release + }); + mockPgQueryContext.mockRejectedValue(new Error('query failed')); + const req = request(api(), { authorization: 'Bearer credential' }); + const res = response(); + const next = jest.fn() as NextFunction; + + await createAuthenticateMiddleware(opts)(req, res, next); + + expect(release).toHaveBeenCalledTimes(1); + expect(res.status).toHaveBeenCalledWith(200); + expect(next).not.toHaveBeenCalled(); + }); + + it('forwards pool-capacity refusal to the shared HTTP error handler', async () => { + const capacityError = Object.assign(new Error('sensitive capacity details'), { + code: 'PG_POOL_CAPACITY' + }); + mockAcquirePgPool.mockImplementation(() => { + throw capacityError; + }); + const req = request(api(), { authorization: 'Bearer credential' }); + const res = response(); + const next = jest.fn() as NextFunction; + + await createAuthenticateMiddleware(opts)(req, res, next); + + expect(next).toHaveBeenCalledWith(capacityError); + expect(mockPgQueryContext).not.toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/captcha.test.ts b/graphql/server/src/middleware/__tests__/captcha.test.ts new file mode 100644 index 0000000000..f25ab7ddc8 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/captcha.test.ts @@ -0,0 +1,294 @@ +import express, { type NextFunction, type Request, type Response } from 'express'; +import supertest from 'supertest'; + +import type { ApiStructure } from '../../types'; +import { + createCaptchaGraphqlBodyParsers, + createCaptchaMiddleware, + inspectCaptchaOperation +} from '../captcha'; + +const api = (enableCaptcha: boolean): ApiStructure => ({ + dbname: 'tenant_db', + anonRole: 'anonymous', + roleName: 'authenticated', + schema: ['app_public'], + databaseId: 'database-a', + apiId: 'api-a', + authSettings: { + cookieSecure: true, + cookieSamesite: 'lax', + cookieDomain: null, + cookieHttponly: true, + cookieMaxAge: null, + cookiePath: '/', + rememberMeDuration: null, + enableCaptcha, + captchaSiteKey: null + } +}); + +interface RequestOptions { + operationName?: unknown; + enableCaptcha?: boolean; + headers?: Record; + method?: string; + path?: string; + body?: unknown; +} + +const request = ( + query: string, + options: RequestOptions = {} +): Request => { + const normalized = Object.fromEntries( + Object.entries(options.headers ?? {}) + .map(([name, value]) => [name.toLowerCase(), value]) + ); + return { + api: api(options.enableCaptcha ?? true), + body: options.body ?? { query, operationName: options.operationName }, + method: options.method ?? 'POST', + path: options.path ?? '/graphql', + query: {}, + get: jest.fn((name: string) => normalized[name.toLowerCase()]) + } as unknown as Request; +}; + +const response = (): Response => { + const res = { + status: jest.fn(), + json: jest.fn() + }; + res.status.mockReturnValue(res); + return res as unknown as Response; +}; + +describe('CAPTCHA GraphQL operation inspection', () => { + it.each([ + ['an arbitrary operation label', 'mutation Harmless { signUp }'], + ['a root field alias', 'mutation Harmless { allowed: resetPassword }'], + [ + 'a fragment spread', + 'mutation Harmless { ...Protected } fragment Protected on Mutation { signUpWithSms }' + ], + [ + 'an inline fragment', + 'mutation Harmless { ... on Mutation { requestPasswordReset } }' + ] + ])('finds a protected root mutation through %s', (_label, query) => { + expect(inspectCaptchaOperation(query, 'Harmless')).toEqual({ + kind: 'protected', + fields: expect.any(Array) + }); + }); + + it('uses operationName only to select one operation from a multi-operation document', () => { + const query = ` + query Safe { viewer { id } } + mutation Protected { signUp } + `; + + expect(inspectCaptchaOperation(query, 'Safe')).toEqual({ + kind: 'not-protected' + }); + expect(inspectCaptchaOperation(query, 'Protected')).toEqual({ + kind: 'protected', + fields: ['signUp'] + }); + expect(inspectCaptchaOperation(query, undefined)).toEqual({ + kind: 'invalid', + reason: 'ambiguous or missing GraphQL operation' + }); + }); + + it.each([ + ['malformed syntax', 'mutation {', undefined], + [ + 'a missing fragment', + 'mutation Protected { ...Missing }', + 'Protected' + ], + [ + 'a cyclic fragment', + `mutation Protected { ...A } + fragment A on Mutation { ...B } + fragment B on Mutation { ...A }`, + 'Protected' + ], + ['a missing selected operation', 'query Safe { viewer { id } }', 'Other'] + ])('fails closed for %s', (_label, query, operationName) => { + expect(inspectCaptchaOperation(query, operationName)).toEqual( + expect.objectContaining({ kind: 'invalid' }) + ); + }); +}); + +describe('captcha middleware admission', () => { + const originalSecret = process.env.RECAPTCHA_SECRET_KEY; + + beforeEach(() => { + delete process.env.RECAPTCHA_SECRET_KEY; + }); + + afterAll(() => { + if (originalSecret === undefined) { + delete process.env.RECAPTCHA_SECRET_KEY; + } else { + process.env.RECAPTCHA_SECRET_KEY = originalSecret; + } + }); + + it('fails closed in production when tenant policy enables CAPTCHA', async () => { + const req = request('mutation AnyName { signUp }'); + const res = response(); + const next = jest.fn() as NextFunction; + + await createCaptchaMiddleware({ nodeEnv: 'production' })(req, res, next); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith({ + errors: [{ + message: 'Something went wrong: authentication failed', + extensions: expect.objectContaining({ + code: 'INTERNAL_FAILURE', + http: 500 + }) + }] + }); + expect(next).not.toHaveBeenCalled(); + }); + + it('fails closed under strict authentication outside production', async () => { + const res = response(); + const next = jest.fn() as NextFunction; + + await createCaptchaMiddleware({ + strictAuth: true, + nodeEnv: 'development' + })(request('mutation Reset { requestPasswordReset }'), res, next); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ + errors: [expect.objectContaining({ + extensions: expect.objectContaining({ code: 'INTERNAL_FAILURE' }) + })] + })); + expect(next).not.toHaveBeenCalled(); + }); + + it('preserves the local non-strict compatibility behavior', async () => { + const res = response(); + const next = jest.fn() as NextFunction; + + await createCaptchaMiddleware({ + strictAuth: false, + nodeEnv: 'development' + })(request('mutation Register { signUp }'), res, next); + + expect(next).toHaveBeenCalledTimes(1); + expect(res.status).not.toHaveBeenCalled(); + }); + + it.each([ + [ + 'disabled tenant policy', + request('mutation Register { signUp }', { enableCaptcha: false }) + ], + ['an unprotected mutation', request('mutation Login { signIn }')], + ['a query', request('query Viewer { viewer { id } }')], + [ + 'a non-GraphQL route', + request('mutation Register { signUp }', { path: '/fn/register' }) + ], + [ + 'a WebSocket handshake', + request('', { + method: 'GET', + headers: { upgrade: 'websocket' } + }) + ] + ])('does not require a secret for %s', async (_label, req) => { + const res = response(); + const next = jest.fn() as NextFunction; + + await createCaptchaMiddleware({ + strictAuth: true, + nodeEnv: 'production' + })(req, res, next); + + expect(next).toHaveBeenCalledTimes(1); + expect(res.status).not.toHaveBeenCalled(); + }); + + it.each([ + ['a malformed document', request('mutation {')], + [ + 'an ambiguous document', + request('query A { viewer { id } } query B { viewer { id } }') + ], + ['a missing body', request('', { body: null })], + ['a batched body', request('', { body: [] })] + ])('fails closed before GraphQL for %s', async (_label, req) => { + const res = response(); + const next = jest.fn() as NextFunction; + + await createCaptchaMiddleware({ + strictAuth: false, + nodeEnv: 'development' + })(req, res, next); + + expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ + errors: [expect.objectContaining({ + extensions: expect.objectContaining({ code: 'INTERNAL_FAILURE' }) + })] + })); + expect(next).not.toHaveBeenCalled(); + }); + + it('still requires a CAPTCHA token when the secret is configured', async () => { + process.env.RECAPTCHA_SECRET_KEY = 'server-side-secret'; + const res = response(); + const next = jest.fn() as NextFunction; + + await createCaptchaMiddleware({ nodeEnv: 'production' })( + request('mutation Reset { resetPassword }'), + res, + next + ); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ + errors: [expect.objectContaining({ + extensions: expect.objectContaining({ code: 'CAPTCHA_REQUIRED' }) + })] + })); + expect(next).not.toHaveBeenCalled(); + }); + + it.each([ + ['application/json', { query: 'mutation Register { signUp }' }], + ['application/graphql', 'mutation Register { signUp }'], + [ + 'application/x-www-form-urlencoded', + 'query=mutation%20Register%20%7B%20signUp%20%7D' + ] + ])('parses and gates a real %s request before Grafserv', async (contentType, body) => { + const app = express(); + app.use((req, _res, next) => { + req.api = api(true); + next(); + }); + app.use('/graphql', ...createCaptchaGraphqlBodyParsers()); + app.use(createCaptchaMiddleware({ nodeEnv: 'production' })); + app.use((_req, res) => res.status(204).end()); + + const result = await supertest(app) + .post('/graphql') + .set('content-type', contentType) + .send(body); + + expect(result.status).toBe(200); + expect(result.body.errors[0].extensions.code).toBe('INTERNAL_FAILURE'); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/error-handler.test.ts b/graphql/server/src/middleware/__tests__/error-handler.test.ts new file mode 100644 index 0000000000..1b54a1421e --- /dev/null +++ b/graphql/server/src/middleware/__tests__/error-handler.test.ts @@ -0,0 +1,43 @@ +import type { NextFunction, Request, Response } from 'express'; + +import { errorHandler } from '../error-handler'; + +describe('shared error handler pool-capacity response', () => { + it('returns a stable retryable 503 without exposing capacity details', () => { + const req = { + requestId: 'request-1', + path: '/graphql', + method: 'POST', + get: jest.fn((name: string) => { + if (name === 'Accept') return 'text/html'; + if (name === 'host') return 'api.example.com'; + return undefined; + }) + } as unknown as Request; + const res = { + headersSent: false, + set: jest.fn(), + status: jest.fn(), + json: jest.fn(), + send: jest.fn() + }; + res.status.mockReturnValue(res); + const error = Object.assign( + new Error('PostgreSQL pool capacity exhausted: 2050/2064 and 2050 leased'), + { code: 'PG_POOL_CAPACITY' } + ); + + errorHandler(error, req, res as unknown as Response, jest.fn() as NextFunction); + + expect(res.set).toHaveBeenCalledWith('Retry-After', '15'); + expect(res.status).toHaveBeenCalledWith(503); + expect(res.json).toHaveBeenCalledWith({ + error: { + code: 'PG_POOL_CAPACITY', + message: 'Service temporarily unavailable', + requestId: 'request-1' + } + }); + expect(JSON.stringify(res.json.mock.calls)).not.toContain('2050'); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/flush-auth.test.ts b/graphql/server/src/middleware/__tests__/flush-auth.test.ts new file mode 100644 index 0000000000..cdb9d69cc6 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/flush-auth.test.ts @@ -0,0 +1,61 @@ +jest.mock('../graphile', () => ({ + invalidateInFlightBuilds: jest.fn() +})); + +import type { LoaderRegistry } from '@constructive-io/express-context'; +import type { NextFunction, Request, Response } from 'express'; + +import { createFlushMiddleware, flush } from '../flush'; + +const response = (): Response => ({ + status: jest.fn().mockReturnThis(), + send: jest.fn() +} as unknown as Response); + +describe('HTTP cache flush authorization', () => { + it('rejects an unauthenticated cache flush', async () => { + const req = { url: '/flush', internalTrusted: false } as Request; + const res = response(); + const next = jest.fn() as NextFunction; + + await flush(req, res, next); + + expect(res.status).toHaveBeenCalledWith(403); + expect(res.send).toHaveBeenCalledWith('Forbidden'); + expect(next).not.toHaveBeenCalled(); + }); + + it('allows a request already authenticated at the internal ingress boundary', async () => { + const req = { + url: '/flush', + internalTrusted: true, + svc_key: 'api.example.test' + } as Request; + const res = response(); + const next = jest.fn() as NextFunction; + + await flush(req, res, next); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.send).toHaveBeenCalledWith('OK'); + expect(next).not.toHaveBeenCalled(); + }); + + it('invalidates module metadata before acknowledging an authenticated flush', async () => { + const invalidate = jest.fn(); + const registry = { invalidate } as unknown as LoaderRegistry; + const req = { + url: '/flush', + internalTrusted: true, + svc_key: 'api.example.test', + databaseId: 'database-123' + } as Request; + const res = response(); + const next = jest.fn() as NextFunction; + + await createFlushMiddleware(registry)(req, res, next); + + expect(invalidate).toHaveBeenCalledWith('database-123'); + expect(res.status).toHaveBeenCalledWith(200); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/flush-pool-lease.test.ts b/graphql/server/src/middleware/__tests__/flush-pool-lease.test.ts new file mode 100644 index 0000000000..49731183b8 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/flush-pool-lease.test.ts @@ -0,0 +1,160 @@ +jest.mock('pg-cache', () => ({ + acquirePgPool: jest.fn(), + getPgPoolIdentity: jest.fn((config: { host?: string }) => + `pg:${config.host ?? 'control'}` + ) +})); + +jest.mock('graphile-cache', () => ({ + deleteGraphileCacheEntry: jest.fn().mockResolvedValue(true), + graphileCache: new Map() +})); + +jest.mock('../graphile', () => ({ + invalidateInFlightBuilds: jest.fn() +})); + +import type { ConstructiveOptions } from '@constructive-io/graphql-types'; +import { svcCache } from '@pgpmjs/server-utils'; +import { deleteGraphileCacheEntry, graphileCache } from 'graphile-cache'; +import { acquirePgPool } from 'pg-cache'; + +import { getSvcCacheKey } from '../api'; +import { flushService } from '../flush'; + +const mockAcquirePgPool = acquirePgPool as jest.MockedFunction; +const mockDeleteGraphileCacheEntry = deleteGraphileCacheEntry as jest.MockedFunction< + typeof deleteGraphileCacheEntry +>; + +describe('flushService PostgreSQL pool ownership', () => { + beforeEach(() => { + jest.clearAllMocks(); + graphileCache.clear(); + svcCache.clear(); + mockDeleteGraphileCacheEntry.mockImplementation(async (key) => graphileCache.delete(key)); + }); + + it.each([true, false])( + 'evicts database entries before a routing failure for isPublic=%s', + async (isPublic) => { + const queryFailure = new Error('routing query failed'); + const release = jest.fn(); + const order: string[] = []; + graphileCache.set('database-a-public', { + databaseId: 'database-a', + serviceKey: 'api:database-a:public' + } as never); + graphileCache.set('database-a-private', { + databaseId: 'database-a', + serviceKey: 'api:database-a:private' + } as never); + graphileCache.set('database-b', { + databaseId: 'database-b', + serviceKey: 'api:database-b:public' + } as never); + mockDeleteGraphileCacheEntry.mockImplementation(async (key) => { + order.push(`delete:${key}`); + return graphileCache.delete(key); + }); + mockAcquirePgPool.mockReturnValue({ + identity: 'pg:control', + pool: { + query: jest.fn().mockImplementation(async () => { + order.push('query'); + throw queryFailure; + }) + } as never, + release + }); + const options = { + pg: { database: 'routing' }, + api: { isPublic } + } as ConstructiveOptions; + + await expect(flushService(options, 'database-a')).rejects.toBe(queryFailure); + + expect(mockAcquirePgPool).toHaveBeenCalledWith( + { database: 'routing' }, + { purpose: 'routing-request-control', sanitizeOnCheckout: true } + ); + expect(mockDeleteGraphileCacheEntry).toHaveBeenCalledTimes(2); + expect(mockDeleteGraphileCacheEntry).toHaveBeenCalledWith('database-a-public'); + expect(mockDeleteGraphileCacheEntry).toHaveBeenCalledWith('database-a-private'); + expect(graphileCache.has('database-b')).toBe(true); + expect(order.indexOf('delete:database-a-public')).toBeLessThan(order.indexOf('query')); + expect(order.indexOf('delete:database-a-private')).toBeLessThan(order.indexOf('query')); + expect(release).toHaveBeenCalledTimes(1); + } + ); + + it.each([true, false])( + 'evicts database entries when routing has no domains for isPublic=%s', + async (isPublic) => { + const release = jest.fn(); + const order: string[] = []; + graphileCache.set('database-a', { + databaseId: 'database-a', + serviceKey: 'api:database-a:public' + } as never); + graphileCache.set('database-b', { + databaseId: 'database-b', + serviceKey: 'api:database-b:public' + } as never); + mockDeleteGraphileCacheEntry.mockImplementation(async (key) => { + order.push(`delete:${key}`); + return graphileCache.delete(key); + }); + mockAcquirePgPool.mockReturnValue({ + identity: 'pg:control', + pool: { + query: jest.fn().mockImplementation(async () => { + order.push('query'); + return { rows: [], rowCount: 0 }; + }) + } as never, + release + }); + const options = { + pg: { database: 'routing' }, + api: { isPublic } + } as ConstructiveOptions; + + await flushService(options, 'database-a'); + + expect(mockDeleteGraphileCacheEntry).toHaveBeenCalledTimes(1); + expect(mockDeleteGraphileCacheEntry).toHaveBeenCalledWith('database-a'); + expect(graphileCache.has('database-b')).toBe(true); + expect(order).toEqual(['delete:database-a', 'query']); + expect(release).toHaveBeenCalledTimes(1); + } + ); + + it('invalidates routing metadata only inside the exact control-pool contract', async () => { + const optsA = { + pg: { database: 'routing', host: 'routing-a.internal' }, + api: { isPublic: true } + } as ConstructiveOptions; + const optsB = { + pg: { database: 'routing', host: 'routing-b.internal' }, + api: { isPublic: true } + } as ConstructiveOptions; + const serviceKey = 'api.example.com'; + const keyA = getSvcCacheKey(optsA, serviceKey); + const keyB = getSvcCacheKey(optsB, serviceKey); + svcCache.set(keyA, { databaseId: 'database-a' }); + svcCache.set(keyB, { databaseId: 'database-a' }); + mockAcquirePgPool.mockReturnValue({ + identity: 'pg:routing-a.internal', + pool: { + query: jest.fn().mockResolvedValue({ rows: [], rowCount: 0 }) + } as never, + release: jest.fn() + }); + + await flushService(optsA, 'database-a'); + + expect(svcCache.has(keyA)).toBe(false); + expect(svcCache.peek(keyB)).toEqual({ databaseId: 'database-a' }); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-build-admission-response.test.ts b/graphql/server/src/middleware/__tests__/graphile-build-admission-response.test.ts new file mode 100644 index 0000000000..d1b34d6233 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-build-admission-response.test.ts @@ -0,0 +1,74 @@ +import type { Response } from 'express'; +import { + CacheBuildAdmissionError, + GraphileRealtimeStartupError +} from 'graphile-cache'; + +import { + GRAPHILE_BUILD_RESIDENT_CAPACITY_CODE, + handleBuildAvailabilityError +} from '../graphile'; +import { GraphileRealtimeNotificationConfigError } from '../realtime-notification-config'; + +describe('Graphile build admission responses', () => { + it('maps preserved resident capacity to a stable retryable 503', () => { + const response = { + destroyed: false, + writableEnded: false, + setHeader: jest.fn(), + status: jest.fn(), + json: jest.fn() + }; + response.status.mockReturnValue(response); + + expect(handleBuildAvailabilityError( + response as unknown as Response, + new CacheBuildAdmissionError('resident_capacity') + )).toBe(true); + expect(response.setHeader).toHaveBeenCalledWith('Retry-After', '15'); + expect(response.status).toHaveBeenCalledWith(503); + expect(response.json).toHaveBeenCalledWith({ + error: { + code: GRAPHILE_BUILD_RESIDENT_CAPACITY_CODE, + message: 'GraphQL schema capacity is temporarily unavailable' + } + }); + }); + + it.each([ + [ + new GraphileRealtimeNotificationConfigError('secret resolver detail'), + 'GRAPHILE_REALTIME_NOTIFICATION_CONFIG_INVALID', + 'Shared realtime notification configuration is unavailable' + ], + [ + new GraphileRealtimeStartupError('opaque-cache-key', new Error('secret startup detail')), + 'GRAPHILE_REALTIME_STARTUP_FAILED', + 'Realtime delivery could not be activated for this GraphQL instance' + ] + ])('maps realtime activation failures to credential-free stable 503s', ( + error, + code, + message + ) => { + const response = { + destroyed: false, + writableEnded: false, + setHeader: jest.fn(), + status: jest.fn(), + json: jest.fn() + }; + response.status.mockReturnValue(response); + + expect(handleBuildAvailabilityError( + response as unknown as Response, + error + )).toBe(true); + expect(response.setHeader).toHaveBeenCalledWith('Retry-After', '15'); + expect(response.status).toHaveBeenCalledWith(503); + expect(response.json).toHaveBeenCalledWith({ + error: { code, message } + }); + expect(JSON.stringify(response.json.mock.calls)).not.toContain('secret'); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-internal-claims.test.ts b/graphql/server/src/middleware/__tests__/graphile-internal-claims.test.ts new file mode 100644 index 0000000000..3110724de9 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-internal-claims.test.ts @@ -0,0 +1,77 @@ +import type { Request } from 'express'; + +import { getTrustedInternalClaims } from '../internal-request'; + +const request = ({ + isPublic, + internalTrusted, + userId, + headers = {} +}: { + isPublic: boolean; + internalTrusted: boolean; + userId?: string; + headers?: Record; +}): Request => { + const normalized = new Map( + Object.entries(headers).map(([name, value]) => [name.toLowerCase(), value]) + ); + return { + api: { + dbname: 'tenant', + schema: ['app_public'], + anonRole: 'api_anon', + roleName: 'api_role', + isPublic + }, + internalTrusted, + token: userId ? { user_id: userId } : undefined, + get: jest.fn((name: string) => normalized.get(name.toLowerCase())) + } as unknown as Request; +}; + +describe('private ingress actor claims', () => { + const actorHeaders = { + 'X-Actor-Id': 'actor-a', + 'X-Entity-Id': 'entity-a', + 'X-Organization-Id': 'organization-a' + }; + + it('does not trust actor headers from the public ingress', () => { + expect(getTrustedInternalClaims(request({ + isPublic: true, + internalTrusted: true, + headers: actorHeaders + }))).toEqual({}); + }); + + it('does not trust actor headers without internal request authentication', () => { + expect(getTrustedInternalClaims(request({ + isPublic: false, + internalTrusted: false, + headers: actorHeaders + }))).toEqual({}); + }); + + it('lets an authenticated user token outrank internal actor headers', () => { + expect(getTrustedInternalClaims(request({ + isPublic: false, + internalTrusted: true, + userId: 'token-user', + headers: actorHeaders + }))).toEqual({}); + }); + + it('maps authenticated private actor headers onto the exact claim allowlist', () => { + expect(getTrustedInternalClaims(request({ + isPublic: false, + internalTrusted: true, + headers: actorHeaders + }))).toEqual({ + 'jwt.claims.user_id': 'actor-a', + 'jwt.claims.principal_id': 'actor-a', + 'jwt.claims.entity_id': 'entity-a', + 'jwt.claims.organization_id': 'organization-a' + }); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-pool-lease-publication.test.ts b/graphql/server/src/middleware/__tests__/graphile-pool-lease-publication.test.ts new file mode 100644 index 0000000000..a5a3fec66d --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-pool-lease-publication.test.ts @@ -0,0 +1,222 @@ +import type { GraphileCacheEntry } from 'graphile-cache'; +import type { PgPoolLease } from 'pg-cache'; + +import { + GraphileBuildPoolLeaseOwner, + GraphileBuildPublicationError, + publishGraphileBuild +} from '../graphile'; + +const lease = (identity = 'pg:runtime') => { + const release = jest.fn(); + return { + value: { + identity, + pool: {} as PgPoolLease['pool'], + release + } as PgPoolLease, + release + }; +}; + +const entry = ( + cacheKey: string, + overrides: Partial = {} +): GraphileCacheEntry => ({ + cacheKey, + poolIdentity: overrides.poolLease?.identity, + createdAt: Date.now(), + ...overrides +} as GraphileCacheEntry); + +describe('Graphile build PostgreSQL pool-lease publication', () => { + it('keeps the lease with the build until a matching entry accepts ownership', () => { + const retained = lease(); + const owner = new GraphileBuildPoolLeaseOwner(retained.value); + const candidate = entry('build-a', { + poolLease: retained.value, + poolIdentity: retained.value.identity + }); + + owner.transferTo(candidate); + owner.release(); + owner.release(); + + expect(retained.release).not.toHaveBeenCalled(); + candidate.poolLease?.release(); + expect(retained.release).toHaveBeenCalledTimes(1); + }); + + it('releases an untransferred build lease exactly once', () => { + const retained = lease(); + const owner = new GraphileBuildPoolLeaseOwner(retained.value); + + expect(() => owner.transferTo(entry('build-a'))).toThrow( + 'did not retain the build pool lease' + ); + owner.release(); + owner.release(); + + expect(retained.release).toHaveBeenCalledTimes(1); + }); + + it('leaves identity-mismatch cleanup to the entry that received the lease', () => { + const retained = lease(); + const owner = new GraphileBuildPoolLeaseOwner(retained.value); + const candidate = entry('build-a', { + poolLease: retained.value, + poolIdentity: 'pg:wrong' + }); + + expect(() => owner.transferTo(candidate)).toThrow('unexpected pool identity'); + owner.release(); + expect(retained.release).not.toHaveBeenCalled(); + candidate.poolLease?.release(); + expect(retained.release).toHaveBeenCalledTimes(1); + }); + + it('publishes one candidate without disposing its retained lease', async () => { + const values = new Map(); + const cache = { + get: jest.fn((key: string) => values.get(key)), + set: jest.fn((key: string, value: GraphileCacheEntry) => values.set(key, value)), + delete: jest.fn((key: string) => values.delete(key)) + }; + const dispose = jest.fn(async (): Promise => undefined); + const candidate = entry('build-a'); + + await expect(publishGraphileBuild('build-a', candidate, false, { + cache, + dispose + })).resolves.toBe(candidate); + expect(cache.set).toHaveBeenCalledTimes(1); + expect(dispose).not.toHaveBeenCalled(); + }); + + it('disposes a candidate when cache publication throws', async () => { + const candidate = entry('build-a'); + const dispose = jest.fn(async (): Promise => undefined); + const cache = { + get: jest.fn((): GraphileCacheEntry | undefined => undefined), + set: jest.fn(() => { + throw new Error('set failed'); + }), + delete: jest.fn(() => false) + }; + + await expect(publishGraphileBuild('build-a', candidate, false, { + cache, + dispose + })).rejects.toMatchObject({ + code: 'GRAPHILE_BUILD_PUBLICATION_FAILED' + } satisfies Partial); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it('disposes an unexpected duplicate and returns the authoritative resident', async () => { + const candidate = entry('build-a'); + const resident = entry('build-a'); + const dispose = jest.fn(async (): Promise => undefined); + const cache = { + get: jest.fn(() => resident), + set: jest.fn(), + delete: jest.fn(() => false) + }; + + await expect(publishGraphileBuild('build-a', candidate, false, { + cache, + dispose + })).resolves.toBe(resident); + expect(cache.set).not.toHaveBeenCalled(); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it('disposes a candidate replaced during publication and returns the stable replacement', async () => { + const candidate = entry('build-a'); + const resident = entry('build-a'); + const dispose = jest.fn(async (): Promise => undefined); + let reads = 0; + const cache = { + get: jest.fn((): GraphileCacheEntry | undefined => { + reads++; + return reads === 1 ? undefined : resident; + }), + set: jest.fn(), + delete: jest.fn(() => false) + }; + + await expect(publishGraphileBuild('build-a', candidate, false, { + cache, + dispose + })).resolves.toBe(resident); + expect(cache.set).toHaveBeenCalledTimes(1); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it('disposes an invalidated candidate before rejecting the build', async () => { + const candidate = entry('build-a'); + const dispose = jest.fn(async (): Promise => undefined); + + await expect(publishGraphileBuild('build-a', candidate, true, { + cache: { get: jest.fn(), set: jest.fn(), delete: jest.fn() }, + dispose + })).rejects.toMatchObject({ code: 'GRAPHILE_BUILD_INVALIDATED' }); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it('rejects a candidate that failed before publication', async () => { + const candidate = entry('build-a', { + disposing: true, + realtimeHealth: { + status: 'failed', + failureCode: 'INSUFFICIENT_PRIVILEGE', + failedAt: Date.now() + } + }); + const dispose = jest.fn(async (): Promise => undefined); + const cache = { + get: jest.fn((): GraphileCacheEntry | undefined => undefined), + set: jest.fn(), + delete: jest.fn(() => false) + }; + + await expect(publishGraphileBuild('build-a', candidate, false, { + cache, + dispose + })).rejects.toMatchObject({ + code: 'GRAPHILE_BUILD_PUBLICATION_FAILED' + }); + expect(cache.set).not.toHaveBeenCalled(); + expect(cache.delete).not.toHaveBeenCalled(); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it('removes and disposes a candidate that fails during publication', async () => { + const candidate = entry('build-a'); + const values = new Map(); + const dispose = jest.fn(async (): Promise => undefined); + const cache = { + get: jest.fn((key: string) => values.get(key)), + set: jest.fn((key: string, value: GraphileCacheEntry) => { + values.set(key, value); + value.realtimeHealth = { + status: 'failed', + failureCode: 'INSUFFICIENT_PRIVILEGE', + failedAt: Date.now() + }; + }), + delete: jest.fn((key: string) => values.delete(key)) + }; + + await expect(publishGraphileBuild('build-a', candidate, false, { + cache, + dispose + })).rejects.toMatchObject({ + code: 'GRAPHILE_BUILD_PUBLICATION_FAILED' + }); + expect(cache.set).toHaveBeenCalledTimes(1); + expect(cache.delete).toHaveBeenCalledWith('build-a'); + expect(values.has('build-a')).toBe(false); + expect(dispose).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/graphile-request-terminal.test.ts b/graphql/server/src/middleware/__tests__/graphile-request-terminal.test.ts new file mode 100644 index 0000000000..e7d998d347 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-request-terminal.test.ts @@ -0,0 +1,142 @@ +import { EventEmitter } from 'node:events'; + +import type { Request, Response } from 'express'; + +import { isGraphileWebSocketOriginAllowed } from '../../websocket-upgrade'; +import { + getGraphileTransportRequest, + isGraphileRequestTerminal +} from '../graphile'; + +const makeRequest = (overrides: Record = {}): Request => + Object.assign(new EventEmitter(), { + aborted: false, + destroyed: false, + readableEnded: false, + complete: false, + socket: { destroyed: false }, + ...overrides + }) as unknown as Request; + +const makeResponse = (overrides: Record = {}): Response => + Object.assign(new EventEmitter(), { + destroyed: false, + writableEnded: false, + ...overrides + }) as unknown as Response; + +describe('Graphile request terminal detection', () => { + it('keeps serving a parsed POST whose consumed request stream was auto-destroyed', () => { + const request = makeRequest({ + destroyed: true, + readableEnded: true, + complete: true + }); + + expect(isGraphileRequestTerminal(request, makeResponse())).toBe(false); + }); + + it.each([ + ['request aborted', { aborted: true }, {}], + ['socket destroyed', { socket: { destroyed: true } }, {}], + ['response destroyed', {}, { destroyed: true }], + ['response ended', {}, { writableEnded: true }] + ])('detects a terminal %s', (_label, request, response) => { + expect(isGraphileRequestTerminal( + makeRequest(request), + makeResponse(response) + )).toBe(true); + }); +}); + +describe('Graphile transport request identity', () => { + it('uses the already-routed and authenticated request for WebSocket execution', () => { + const request = makeRequest({ + api: { databaseId: 'database-a', schema: ['a_public'] }, + token: { user_id: 'actor-a' } + }); + + expect(getGraphileTransportRequest({ + ws: { request } + } as unknown as Partial)).toBe(request); + }); + + it('keeps the existing Express request path for HTTP execution', () => { + const request = makeRequest({ + api: { databaseId: 'database-b', schema: ['b_public'] }, + token: { user_id: 'actor-b' } + }); + + expect(getGraphileTransportRequest({ + expressv4: { req: request } + } as unknown as Partial)).toBe(request); + }); +}); + +describe('Graphile WebSocket origin policy', () => { + const originRequest = ( + headers: Record, + corsOrigins: string[] = [] + ): Request => makeRequest({ + headers, + api: { + databaseId: 'database-a', + dbname: 'tenant_a', + schema: ['a_public'], + anonRole: 'tenant_anon', + roleName: 'tenant_user', + domains: [], + corsOrigins + }, + get(name: string) { + return headers[name.toLowerCase()]; + } + }); + + it('allows same-host and configured browser origins', () => { + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test', + origin: 'https://a.example.test', + cookie: 'constructive_session=session-a' + }))).toBe(true); + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test', + origin: 'https://console.example.test', + cookie: 'constructive_session=session-a' + }, ['https://console.example.test']))).toBe(true); + }); + + it('does not authorize cookie WebSockets through wildcard or localhost shortcuts', () => { + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test', + origin: 'https://attacker.example.test', + cookie: 'constructive_session=session-a' + }), '*')).toBe(false); + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'api.localhost:3000', + origin: 'http://attacker.localhost:3001', + cookie: 'constructive_session=session-a' + }))).toBe(false); + }); + + it('rejects a cross-origin browser and originless cookie authentication', () => { + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test', + origin: 'https://attacker.example.test' + }))).toBe(false); + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test', + cookie: 'constructive_session=session-a' + }))).toBe(false); + }); + + it('allows originless bearer and anonymous non-browser clients', () => { + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test', + authorization: 'Bearer token-a' + }))).toBe(true); + expect(isGraphileWebSocketOriginAllowed(originRequest({ + host: 'a.example.test' + }))).toBe(true); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/internal-request.test.ts b/graphql/server/src/middleware/__tests__/internal-request.test.ts new file mode 100644 index 0000000000..8665a07a6c --- /dev/null +++ b/graphql/server/src/middleware/__tests__/internal-request.test.ts @@ -0,0 +1,167 @@ +import type { Request } from 'express'; + +import type { ApiOptions } from '../../types'; +import { + assertInternalRequestSecret, + authorizeInternalRequest, + INTERNAL_REQUEST_TOKEN_HEADER +} from '../internal-request'; + +const SECRET = '0123456789abcdef0123456789abcdef'; + +const request = (headers: Record): Request => { + const normalized = new Map( + Object.entries(headers).map(([name, value]) => [name.toLowerCase(), value]) + ); + return { + get: jest.fn((name: string) => normalized.get(name.toLowerCase())) + } as unknown as Request; +}; + +const options = (isPublic: boolean, secret: string | undefined = SECRET): ApiOptions => ({ + api: { + isPublic, + ...(secret === undefined ? {} : { internalRequestSecret: secret }) + } +} as ApiOptions); + +describe('internal request boundary', () => { + it('allows ordinary requests without granting internal trust', () => { + const req = request({ host: 'api.example.com' }); + + authorizeInternalRequest(options(true), req); + + expect(req.internalTrusted).toBe(false); + }); + + it('authenticates a token-only administrative request in constant-time path', () => { + const req = request({ [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET }); + + authorizeInternalRequest(options(true), req); + + expect(req.internalTrusted).toBe(true); + }); + + it('rejects private actor claims without the internal token', () => { + const req = request({ 'X-Actor-Id': 'actor-a' }); + + expect(() => authorizeInternalRequest(options(false), req)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + expect(req.internalTrusted).toBe(false); + }); + + it('accepts private API and actor selectors only with the exact token', () => { + const req = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Api-Name': 'api-a', + 'X-Actor-Id': 'actor-a' + }); + + authorizeInternalRequest(options(false), req); + + expect(req.internalTrusted).toBe(true); + }); + + it('rejects private selectors on a public ingress even with the exact token', () => { + const req = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Api-Name': 'api-a' + }); + + expect(() => authorizeInternalRequest(options(true), req)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + expect(req.internalTrusted).toBe(false); + }); + + it('always rejects caller-supplied physical schemas', () => { + const req = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Schemata': 'tenant_b_public' + }); + + expect(() => authorizeInternalRequest(options(false), req)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + expect(req.internalTrusted).toBe(false); + }); + + it('rejects the privileged metadata surface unless explicitly enabled', () => { + const req = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Meta-Schema': 'true' + }); + + expect(() => authorizeInternalRequest(options(false), req)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + expect(req.internalTrusted).toBe(false); + }); + + it('allows the privileged metadata surface only under an explicit private-ingress gate', () => { + const opts = options(false); + opts.api!.allowMetaSchemaHeader = true; + const req = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Meta-Schema': 'true' + }); + + authorizeInternalRequest(opts, req); + + expect(req.internalTrusted).toBe(true); + }); + + it('rejects missing and conflicting private-selector identities', () => { + const opts = options(false); + opts.api!.allowMetaSchemaHeader = true; + const missingDatabase = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Api-Name': 'api-a' + }); + const conflicting = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Api-Name': 'api-a', + 'X-Meta-Schema': 'true' + }); + + expect(() => authorizeInternalRequest(opts, missingDatabase)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + expect(() => authorizeInternalRequest(opts, conflicting)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + }); + + it('rejects empty reserved routing and identity values', () => { + const blankApi = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Database-Id': 'database-a', + 'X-Api-Name': ' ' + }); + const blankActor = request({ + [INTERNAL_REQUEST_TOKEN_HEADER]: SECRET, + 'X-Actor-Id': '' + }); + + expect(() => authorizeInternalRequest(options(false), blankApi)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + expect(() => authorizeInternalRequest(options(false), blankActor)).toThrow( + expect.objectContaining({ code: 'INTERNAL_REQUEST_FORBIDDEN' }) + ); + }); + + it('rejects short configured secrets at startup', () => { + expect(() => assertInternalRequestSecret(options(false, 'too-short'))).toThrow( + 'at least 32 bytes' + ); + expect(() => assertInternalRequestSecret(options(false, undefined))).not.toThrow(); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/routing.test.ts b/graphql/server/src/middleware/__tests__/routing.test.ts index bcd013a888..9176e2bd5f 100644 --- a/graphql/server/src/middleware/__tests__/routing.test.ts +++ b/graphql/server/src/middleware/__tests__/routing.test.ts @@ -1,23 +1,38 @@ jest.mock('pg-cache', () => ({ - getPgPool: jest.fn() + acquirePgPool: jest.fn(), + getPgPoolIdentity: jest.fn().mockReturnValue('pg:test'), + PG_POOL_CAPACITY_ERROR_CODE: 'PG_POOL_CAPACITY' })); jest.mock('@constructive-io/express-context', () => ({ createDefaultRegistry: jest.fn(() => ({ - resolve: jest.fn().mockResolvedValue(undefined) + resolve: jest.fn(async (name: string) => name === 'databaseSettings' ? { + enableAggregates: false, + enablePostgis: false, + enableSearch: false, + enableDirectUploads: false, + enablePresignedUploads: false, + enableManyToMany: false, + enableConnectionFilter: false, + enableLtree: false, + enableLlm: false, + enableRealtime: false, + enableBulk: false, + enableI18n: false + } : undefined) })) })); import { svcCache } from '@pgpmjs/server-utils'; import type { Request } from 'express'; import type { Pool } from 'pg'; -import { getPgPool } from 'pg-cache'; +import { acquirePgPool } from 'pg-cache'; import type { ApiOptions } from '../../types'; import { getApiConfig } from '../api'; import { ResolvedRoute, resolveRoute, routeToApiStructure } from '../routing'; -const mockGetPgPool = getPgPool as jest.MockedFunction; +const mockAcquirePgPool = acquirePgPool as jest.MockedFunction; const matchedRoute = (overrides: Partial = {}): ResolvedRoute => ({ route_binding_id: 'rb-1', @@ -29,7 +44,7 @@ const matchedRoute = (overrides: Partial = {}): ResolvedRoute => domain_id: 'dom-1', target_catalog_id: 'cat-1', target_module: 'apis', - target_source_id: 'api-src-1', + target_source_id: 'api-1', target_owner_scope: 'database', target_owner_key: 'db-1', resolved_config: { @@ -51,6 +66,11 @@ const noMatchRoute = (): ResolvedRoute => matchedRoute({ route_binding_id: null, target_module: null, resolved_config: null }); const createPool = (query: jest.Mock): Pool => ({ query } as unknown as Pool); +const leasePool = (pool: Pool) => ({ + pool, + identity: 'pg:test', + release: jest.fn() +}); describe('resolveRoute', () => { it('returns the row when a route matches', async () => { @@ -69,6 +89,22 @@ describe('resolveRoute', () => { expect(row).toBeNull(); }); + it('fails closed when the resolver violates its exactly-one-row contract', async () => { + const zeroRows = await resolveRoute( + createPool(jest.fn().mockResolvedValue({ rows: [] })), + 'constructive_routing_public', + 'api.example.com' + ); + const duplicateRows = await resolveRoute( + createPool(jest.fn().mockResolvedValue({ rows: [matchedRoute(), matchedRoute()] })), + 'constructive_routing_public', + 'api.example.com' + ); + + expect(zeroRows).toBeNull(); + expect(duplicateRows).toBeNull(); + }); + it('returns null when the resolver function is not installed', async () => { const query = jest.fn().mockRejectedValue(Object.assign(new Error('undefined function'), { code: '42883' })); const row = await resolveRoute(createPool(query), 'constructive_routing_public', 'api.example.com'); @@ -121,6 +157,55 @@ describe('routeToApiStructure', () => { it('returns null when resolved_config lacks api essentials', () => { expect(routeToApiStructure(matchedRoute({ resolved_config: {} }), opts)).toBeNull(); }); + + it('fails closed when route visibility does not match the server ingress', () => { + const privateRoute = matchedRoute({ + resolved_config: { + ...(matchedRoute().resolved_config as Record), + is_public: false + } + }); + + expect(routeToApiStructure(privateRoute, opts)).toBeNull(); + }); + + it('fails closed when exact roles or physical schemas are absent', () => { + expect(routeToApiStructure(matchedRoute({ + resolved_config: { + ...(matchedRoute().resolved_config as Record), + anon_role: undefined + } + }), opts)).toBeNull(); + expect(routeToApiStructure(matchedRoute({ + resolved_config: { + ...(matchedRoute().resolved_config as Record), + schemas: ['app_public', 'app_public'] + } + }), opts)).toBeNull(); + }); + + it('accepts Constructive dash-prefixed physical schemas', () => { + expect(routeToApiStructure(matchedRoute({ + resolved_config: { + ...(matchedRoute().resolved_config as Record), + schemas: ['customer-db-a1b2c3d4-app-public'] + } + }), opts)).toMatchObject({ + schema: ['customer-db-a1b2c3d4-app-public'] + }); + }); + + it('fails closed when route and resolved-config identities disagree', () => { + expect(routeToApiStructure(matchedRoute({ + target_source_id: 'another-api' + }), opts)).toBeNull(); + expect(routeToApiStructure(matchedRoute({ + target_owner_key: 'another-database' + }), opts)).toBeNull(); + expect(routeToApiStructure(matchedRoute({ + target_owner_scope: 'organization' + }), opts)).toBeNull(); + }); }); describe('getApiConfig with scoped routing enabled', () => { @@ -164,7 +249,7 @@ describe('getApiConfig with scoped routing enabled', () => { if (sql.includes('resolve_route')) return { rows: [matchedRoute()] }; throw new Error(`unexpected query: ${sql}`); }); - mockGetPgPool.mockReturnValue(createPool(query) as never); + mockAcquirePgPool.mockImplementation(() => leasePool(createPool(query))); const result = await getApiConfig(createOptions(), createRequest({ host: 'api.example.com' })); @@ -181,7 +266,7 @@ describe('getApiConfig with scoped routing enabled', () => { if (sql.includes('resolve_route')) return { rows: [noMatchRoute()] }; throw new Error(`unexpected query (no legacy fallback): ${sql}`); }); - mockGetPgPool.mockReturnValue(createPool(query) as never); + mockAcquirePgPool.mockImplementation(() => leasePool(createPool(query))); const result = await getApiConfig(createOptions(), createRequest({ host: 'nomatch.example.com' })); @@ -189,7 +274,43 @@ describe('getApiConfig with scoped routing enabled', () => { expect(query.mock.calls.some(([sql]) => String(sql).includes('services_public'))).toBe(false); }); - it('throws NO_DATABASE_ID when a route resolves without a database id (no default database)', async () => { + it('re-resolves a hot hostname so a reassignment cannot use stale tenant metadata', async () => { + let route = matchedRoute(); + const query = jest.fn(async (sql: string, params: unknown[]) => { + if (sql.includes('information_schema.schemata')) return schemaValidationRows(params); + if (sql.includes('resolve_route')) return { rows: [route] }; + throw new Error(`unexpected query: ${sql}`); + }); + mockAcquirePgPool.mockImplementation(() => leasePool(createPool(query))); + + const first = await getApiConfig( + createOptions(), + createRequest({ host: 'api.example.com' }) + ); + route = matchedRoute({ + target_source_id: 'api-2', + target_owner_key: 'db-2', + resolved_config: { + api_id: 'api-2', + database_id: 'db-2', + dbname: 'tenant_db_2', + role_name: 'api_role_2', + anon_role: 'api_anon_2', + is_public: true, + schemas: ['app_two_public'] + } + }); + const second = await getApiConfig( + createOptions(), + createRequest({ host: 'api.example.com' }) + ); + + expect(first).toMatchObject({ databaseId: 'db-1' }); + expect(second).toMatchObject({ databaseId: 'db-2', dbname: 'tenant_db_2' }); + expect(query.mock.calls.filter(([sql]) => String(sql).includes('resolve_route'))).toHaveLength(2); + }); + + it('fails closed when a route resolves without a database id', async () => { const routeWithoutDbId = matchedRoute({ resolved_config: { api_id: 'api-1', @@ -205,10 +326,10 @@ describe('getApiConfig with scoped routing enabled', () => { if (sql.includes('resolve_route')) return { rows: [routeWithoutDbId] }; throw new Error(`unexpected query: ${sql}`); }); - mockGetPgPool.mockReturnValue(createPool(query) as never); + mockAcquirePgPool.mockImplementation(() => leasePool(createPool(query))); await expect( getApiConfig(createOptions(), createRequest({ host: 'api.example.com' })) - ).rejects.toMatchObject({ code: 'NO_DATABASE_ID' }); + ).resolves.toBeNull(); }); }); diff --git a/graphql/server/src/middleware/__tests__/runtime-pg-config.test.ts b/graphql/server/src/middleware/__tests__/runtime-pg-config.test.ts new file mode 100644 index 0000000000..d396ad985d --- /dev/null +++ b/graphql/server/src/middleware/__tests__/runtime-pg-config.test.ts @@ -0,0 +1,203 @@ +import { EventEmitter } from 'node:events'; + +import type { + ConstructiveOptions, + RuntimePgResolverInput +} from '@constructive-io/graphql-types'; +import type { NextFunction, Request, Response } from 'express'; + +import { + createRuntimePgResolutionStore, + resolveRuntimePgConfig +} from '../runtime-pg-config'; +import { InvalidRuntimePgConfigurationError } from '../runtime-pg-requirements'; + +const route: RuntimePgResolverInput = { + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['tenant_a_public', 'tenant_a_auth'], + roles: ['tenant_a_anonymous', 'tenant_a_authenticated'] +}; + +const resolverOptions = ( + resolver: ConstructiveOptions['runtimePgResolver'] +): ConstructiveOptions => ({ + pg: { + host: 'db.internal', + port: 6432, + database: 'control', + user: 'control_owner', + password: 'control-secret', + ssl: true + }, + graphile: { introspectionMode: 'scoped-required' }, + runtimePgResolver: resolver +}); + +describe('exact runtime PostgreSQL resolution', () => { + it('resolves one frozen credential-free route and normalizes an opaque pool identity', async () => { + const resolver = jest.fn((_input: Readonly) => ({ + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret', + pool: { max: 2, maxUses: 1 } + })); + + const resolution = await resolveRuntimePgConfig( + resolverOptions(resolver), + route, + 'production' + ); + + expect(resolution.pgConfig).toEqual({ + host: 'db.internal', + port: 6432, + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret', + ssl: true, + pool: { max: 2, maxUses: 1 } + }); + expect(resolution.poolIdentity).toMatch(/^pg:v1:/); + expect(Object.isFrozen(resolution)).toBe(true); + expect(Object.isFrozen(resolution.pgConfig)).toBe(true); + expect(resolver).toHaveBeenCalledTimes(1); + const input = resolver.mock.calls[0][0]; + expect(input).toEqual(route); + expect(Object.isFrozen(input)).toBe(true); + expect(Object.isFrozen(input.schemas)).toBe(true); + expect(Object.isFrozen(input.roles)).toBe(true); + }); + + it('rejects ambiguous connection strings and physical database mismatches', async () => { + await expect(resolveRuntimePgConfig(resolverOptions(() => ({ + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret', + connectionString: 'postgres://other:secret@foreign/tenant_b' + } as never)), route, 'production')).rejects.toThrow( + 'must not return a connectionString' + ); + + await expect(resolveRuntimePgConfig(resolverOptions(() => ({ + database: 'tenant_b', + user: 'tenant_b_runtime', + password: 'runtime-secret' + })), route, 'production')).rejects.toThrow( + 'does not match the routed physical database' + ); + }); + + it('binds login and pool policy into the opaque identity on one attested target', async () => { + const base = await resolveRuntimePgConfig(resolverOptions(() => ({ + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret', + pool: { max: 2 } + })), route, 'production'); + const otherTarget = await resolveRuntimePgConfig(resolverOptions(() => ({ + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'rotated-runtime-secret', + pool: { max: 3 } + })), route, 'production'); + + expect(otherTarget.poolIdentity).not.toBe(base.poolIdentity); + }); + + it.each([ + { host: 'other.internal' }, + { port: 5433 }, + { ssl: false } + ])('rejects runtime/control endpoint divergence: %p', async (networkOverride) => { + await expect(resolveRuntimePgConfig(resolverOptions(() => ({ + ...networkOverride, + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + })), route, 'production')).rejects.toThrow( + 'network/TLS endpoint does not match the routed control-plane database' + ); + }); + + it('authorizes static credentials for one exact ordered route only', async () => { + const options: ConstructiveOptions = { + pg: { host: 'db.internal', port: 5432, ssl: true }, + graphile: { introspectionMode: 'scoped-required' }, + runtimePg: { + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + }, + runtimePgStaticIdentity: route + }; + + await expect(resolveRuntimePgConfig(options, route, 'production')) + .resolves.toMatchObject({ + pgConfig: { + database: 'tenant_a', + user: 'tenant_a_runtime' + } + }); + await expect(resolveRuntimePgConfig(options, { + ...route, + schemas: [...route.schemas].reverse() + }, 'production')).rejects.toThrow( + 'not authorized for the requested exact route' + ); + await expect(resolveRuntimePgConfig(options, { + ...route, + roles: [route.roles[1], route.roles[0]] + }, 'production')).rejects.toThrow( + 'not authorized for the requested exact route' + ); + }); + + it('keeps the secret-bearing resolution outside req and resolves only once', async () => { + const resolver = jest.fn(() => ({ + database: 'tenant_a', + user: 'tenant_a_runtime', + password: 'runtime-secret' + })); + const store = createRuntimePgResolutionStore(resolverOptions(resolver)); + const req = Object.assign(new EventEmitter(), { + api: { + apiId: route.apiId, + databaseId: route.databaseId, + dbname: route.databaseName, + schema: [...route.schemas], + anonRole: route.roles[0], + roleName: route.roles[1] + } + }) as unknown as Request; + const res = new EventEmitter() as unknown as Response; + const next = jest.fn() as unknown as NextFunction; + + await store.middleware(req, res, next); + + expect(next).toHaveBeenCalledWith(); + expect(resolver).toHaveBeenCalledTimes(1); + const first = store.getRuntimePgResolution(req); + const second = store.getRuntimePgResolution(req); + expect(second).toBe(first); + expect(Reflect.ownKeys(req)).not.toContain('runtimePg'); + expect(JSON.stringify(req)).not.toContain('runtime-secret'); + + req.api = { + ...req.api!, + databaseId: 'database-b' + }; + expect(() => store.getRuntimePgResolution(req)).toThrow( + 'Authoritative API route changed after runtime PostgreSQL resolution' + ); + req.api = { + ...req.api, + databaseId: route.databaseId + }; + + (res as unknown as EventEmitter).emit('finish'); + expect(() => store.getRuntimePgResolution(req)) + .toThrow(InvalidRuntimePgConfigurationError); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/runtime-pg-requirements.test.ts b/graphql/server/src/middleware/__tests__/runtime-pg-requirements.test.ts new file mode 100644 index 0000000000..7a4c7576e6 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/runtime-pg-requirements.test.ts @@ -0,0 +1,111 @@ +import { + assertRuntimePgCredentials, + InvalidRuntimePgConfigurationError, + MissingRuntimePgCredentialsError, + shouldValidateRuntimeRoleSafety, + usesUnsafeDevelopmentRuntimePgFallback +} from '../runtime-pg-requirements'; + +describe('GraphQL runtime PostgreSQL requirements', () => { + it('preserves an explicitly named stock-mode fallback only outside production', () => { + const options = { + graphile: { introspectionMode: 'stock' } + } as const; + expect(usesUnsafeDevelopmentRuntimePgFallback(options, 'development')).toBe(true); + expect(usesUnsafeDevelopmentRuntimePgFallback(options, 'test')).toBe(true); + expect(usesUnsafeDevelopmentRuntimePgFallback(options, 'production')).toBe(false); + expect(() => assertRuntimePgCredentials(options, 'development')).not.toThrow(); + }); + + it('rejects production stock mode without an explicit runtime login', () => { + expect(() => assertRuntimePgCredentials({ + graphile: { introspectionMode: 'stock' } + }, 'production')).toThrow(MissingRuntimePgCredentialsError); + }); + + it.each([ + undefined, + {}, + { user: 'runtime' }, + { password: 'secret' }, + { user: ' ', password: 'secret' }, + { user: 'runtime', password: '' }, + { user: 42, password: 'secret' }, + { user: 'runtime', password: async () => 'secret' } + ])('rejects scoped mode without a complete explicit runtime login: %p', (runtimePg) => { + expect(() => assertRuntimePgCredentials({ + graphile: { introspectionMode: 'scoped-required' }, + runtimePg: runtimePg as never + }, 'test')).toThrow(MissingRuntimePgCredentialsError); + }); + + it('rejects an incomplete explicitly supplied login even in stock development', () => { + expect(() => assertRuntimePgCredentials({ + graphile: { introspectionMode: 'stock' }, + runtimePg: { user: 'runtime' } + }, 'development')).toThrow(MissingRuntimePgCredentialsError); + }); + + it('accepts a static login in stock development compatibility mode', () => { + const stock = { + graphile: { introspectionMode: 'stock' as const }, + runtimePg: { user: 'runtime', password: 'secret' } + }; + expect(() => assertRuntimePgCredentials(stock, 'development')).not.toThrow(); + expect(shouldValidateRuntimeRoleSafety(stock, 'development')).toBe(true); + }); + + it('requires a resolver or one exact static route in scoped/production modes', () => { + expect(() => assertRuntimePgCredentials({ + graphile: { introspectionMode: 'scoped-required' }, + runtimePg: { user: 'runtime', password: 'secret' } + }, 'test')).toThrow(InvalidRuntimePgConfigurationError); + + expect(() => assertRuntimePgCredentials({ + graphile: { introspectionMode: 'scoped-required' }, + runtimePg: { + database: 'tenant_a', + user: 'runtime', + password: 'secret' + }, + runtimePgStaticIdentity: { + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['tenant_a_public'], + roles: ['anonymous', 'authenticated'] + } + }, 'test')).not.toThrow(); + + expect(() => assertRuntimePgCredentials({ + graphile: { introspectionMode: 'scoped-required' }, + runtimePgResolver: async () => ({ + database: 'tenant_a', + user: 'runtime', + password: 'secret' + }) + }, 'test')).not.toThrow(); + }); + + it('always enables role safety in production, even for stock mode', () => { + expect(shouldValidateRuntimeRoleSafety({ + graphile: { introspectionMode: 'stock' }, + runtimePgResolver: () => ({ + database: 'tenant_a', + user: 'runtime', + password: 'secret' + }) + }, 'production')).toBe(true); + }); + + it('rejects ambiguous static and resolver credentials', () => { + expect(() => assertRuntimePgCredentials({ + runtimePg: { user: 'runtime', password: 'secret' }, + runtimePgResolver: () => ({ + database: 'tenant_a', + user: 'other', + password: 'other-secret' + }) + }, 'development')).toThrow(InvalidRuntimePgConfigurationError); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/runtime-role-safety.integration.test.ts b/graphql/server/src/middleware/__tests__/runtime-role-safety.integration.test.ts new file mode 100644 index 0000000000..4e55e34679 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/runtime-role-safety.integration.test.ts @@ -0,0 +1,199 @@ +import { randomUUID } from 'node:crypto'; + +import pg from 'pg'; +import { getPgEnvOptions } from 'pg-env'; + +import { + assertRuntimeRoleSafety, + UnsafeRuntimeRoleError +} from '../runtime-role-safety'; + +const describeWithPostgres = + process.env.GRAPHQL_SERVER_RUN_ROLE_SAFETY_INTEGRATION === '1' + ? describe + : describe.skip; + +describeWithPostgres('runtime role safety PostgreSQL integration', () => { + jest.setTimeout(30_000); + + it('rejects ownership inherited only after SET ROLE to a configured request role', async () => { + const suffix = randomUUID().replace(/-/g, '').slice(0, 12); + const parentRole = `rrs_parent_${suffix}`; + const requestRole = `rrs_request_${suffix}`; + const runtimeRole = `rrs_runtime_${suffix}`; + const schema = `rrs_schema_${suffix}`; + const password = `rrs-${randomUUID()}`; + const quoteIdentifier = pg.escapeIdentifier; + const adminConfig = getPgEnvOptions({}); + const adminPool = new pg.Pool({ ...adminConfig, max: 1 }); + let runtimePool: pg.Pool | null = null; + + try { + await adminPool.query(` + CREATE ROLE ${quoteIdentifier(parentRole)} NOLOGIN; + CREATE ROLE ${quoteIdentifier(requestRole)} NOLOGIN INHERIT; + CREATE ROLE ${quoteIdentifier(runtimeRole)} LOGIN NOINHERIT + PASSWORD ${pg.escapeLiteral(password)}; + CREATE SCHEMA ${quoteIdentifier(schema)} AUTHORIZATION CURRENT_USER; + CREATE TABLE ${quoteIdentifier(schema)}.owned_table (id integer); + ALTER TABLE ${quoteIdentifier(schema)}.owned_table + OWNER TO ${quoteIdentifier(parentRole)}; + GRANT USAGE ON SCHEMA ${quoteIdentifier(schema)} + TO ${quoteIdentifier(requestRole)}; + GRANT ${quoteIdentifier(parentRole)} TO ${quoteIdentifier(requestRole)} + WITH INHERIT TRUE, SET FALSE; + GRANT ${quoteIdentifier(requestRole)} TO ${quoteIdentifier(runtimeRole)} + WITH INHERIT FALSE, SET TRUE; + `); + + runtimePool = new pg.Pool({ + ...adminConfig, + user: runtimeRole, + password, + max: 1 + }); + const client = await runtimePool.connect(); + try { + const before = await client.query<{ + parent_usage: boolean; + parent_set: boolean; + request_set: boolean; + }>(` + SELECT pg_catalog.pg_has_role(current_user, $1, 'USAGE') AS parent_usage, + pg_catalog.pg_has_role(current_user, $1, 'SET') AS parent_set, + pg_catalog.pg_has_role(current_user, $2, 'SET') AS request_set + `, [parentRole, requestRole]); + expect(before.rows[0]).toEqual({ + parent_usage: false, + parent_set: false, + request_set: true + }); + + await client.query('BEGIN'); + await client.query(`SET ROLE ${quoteIdentifier(requestRole)}`); + const after = await client.query<{ parent_usage: boolean }>( + 'SELECT pg_catalog.pg_has_role(current_user, $1, \'USAGE\') AS parent_usage', + [parentRole] + ); + expect(after.rows[0]?.parent_usage).toBe(true); + await expect(client.query( + `ALTER TABLE ${quoteIdentifier(schema)}.owned_table ADD COLUMN escaped integer` + )).resolves.toBeDefined(); + await client.query('ROLLBACK'); + } finally { + client.release(true); + } + + let rejected: unknown; + try { + await assertRuntimeRoleSafety( + runtimePool, + [requestRole], + [schema] + ); + } catch (error) { + rejected = error; + } + expect(rejected).toBeInstanceOf(UnsafeRuntimeRoleError); + expect((rejected as UnsafeRuntimeRoleError).violations).toContain( + `${requestRole} can reach role ${parentRole}` + + ' after SET ROLE (USAGE=true,SET=false)' + ); + } finally { + await runtimePool?.end(); + await adminPool.query(` + DROP SCHEMA IF EXISTS ${quoteIdentifier(schema)} CASCADE; + DROP ROLE IF EXISTS ${quoteIdentifier(runtimeRole)}; + DROP ROLE IF EXISTS ${quoteIdentifier(requestRole)}; + DROP ROLE IF EXISTS ${quoteIdentifier(parentRole)}; + `); + await adminPool.end(); + } + }); + + it('rejects BYPASSRLS, object ownership, SECURITY DEFINER, and cross-schema privileges', async () => { + const suffix = randomUUID().replace(/-/g, '').slice(0, 12); + const requestRole = `rrs_request_${suffix}`; + const runtimeRole = `rrs_runtime_${suffix}`; + const approvedSchema = `rrs_approved_${suffix}`; + const externalSchema = `rrs_external_${suffix}`; + const password = `rrs-${randomUUID()}`; + const quoteIdentifier = pg.escapeIdentifier; + const adminConfig = getPgEnvOptions({}); + const adminPool = new pg.Pool({ ...adminConfig, max: 1 }); + let runtimePool: pg.Pool | null = null; + + try { + await adminPool.query(` + CREATE ROLE ${quoteIdentifier(requestRole)} NOLOGIN NOINHERIT BYPASSRLS; + CREATE ROLE ${quoteIdentifier(runtimeRole)} LOGIN NOINHERIT + PASSWORD ${pg.escapeLiteral(password)}; + CREATE SCHEMA ${quoteIdentifier(approvedSchema)} AUTHORIZATION CURRENT_USER; + CREATE SCHEMA ${quoteIdentifier(externalSchema)} AUTHORIZATION CURRENT_USER; + CREATE TABLE ${quoteIdentifier(approvedSchema)}.owned_table (id integer); + ALTER TABLE ${quoteIdentifier(approvedSchema)}.owned_table + OWNER TO ${quoteIdentifier(requestRole)}; + CREATE FUNCTION ${quoteIdentifier(approvedSchema)}.privileged_function() + RETURNS integer LANGUAGE sql SECURITY DEFINER AS 'SELECT 1'; + CREATE TABLE ${quoteIdentifier(externalSchema)}.external_table (id integer); + CREATE SEQUENCE ${quoteIdentifier(externalSchema)}.external_sequence; + CREATE FUNCTION ${quoteIdentifier(externalSchema)}.external_function() + RETURNS integer LANGUAGE sql AS 'SELECT 1'; + CREATE TYPE ${quoteIdentifier(externalSchema)}.external_type AS ENUM ('one'); + GRANT USAGE ON SCHEMA ${quoteIdentifier(approvedSchema)}, + ${quoteIdentifier(externalSchema)} TO ${quoteIdentifier(requestRole)}; + GRANT SELECT ON ${quoteIdentifier(externalSchema)}.external_table + TO ${quoteIdentifier(requestRole)}; + GRANT USAGE ON SEQUENCE ${quoteIdentifier(externalSchema)}.external_sequence + TO ${quoteIdentifier(requestRole)}; + GRANT EXECUTE ON FUNCTION ${quoteIdentifier(externalSchema)}.external_function() + TO ${quoteIdentifier(requestRole)}; + GRANT USAGE ON TYPE ${quoteIdentifier(externalSchema)}.external_type + TO ${quoteIdentifier(requestRole)}; + GRANT ${quoteIdentifier(requestRole)} TO ${quoteIdentifier(runtimeRole)} + WITH INHERIT FALSE, SET TRUE; + `); + + runtimePool = new pg.Pool({ + ...adminConfig, + user: runtimeRole, + password, + max: 1 + }); + + let rejected: unknown; + try { + await assertRuntimeRoleSafety( + runtimePool, + [requestRole], + [approvedSchema] + ); + } catch (error) { + rejected = error; + } + + expect(rejected).toBeInstanceOf(UnsafeRuntimeRoleError); + const violations = (rejected as UnsafeRuntimeRoleError).violations; + expect(violations).toContain(`${requestRole} has BYPASSRLS`); + expect(violations).toContain( + `${requestRole} owns RELATION ${approvedSchema}.owned_table` + ); + expect(violations).toContain( + `SECURITY DEFINER FUNCTION ${approvedSchema}.privileged_function ` + + 'is not allowed in the approved GraphQL schema scope' + ); + expect(violations).toContain( + `${requestRole} has RELATION,SEQUENCE,FUNCTION,TYPE on unapproved schema ${externalSchema}` + ); + } finally { + await runtimePool?.end(); + await adminPool.query(` + DROP SCHEMA IF EXISTS ${quoteIdentifier(approvedSchema)} CASCADE; + DROP SCHEMA IF EXISTS ${quoteIdentifier(externalSchema)} CASCADE; + DROP ROLE IF EXISTS ${quoteIdentifier(runtimeRole)}; + DROP ROLE IF EXISTS ${quoteIdentifier(requestRole)}; + `); + await adminPool.end(); + } + }); +}); diff --git a/graphql/server/src/middleware/__tests__/runtime-role-safety.test.ts b/graphql/server/src/middleware/__tests__/runtime-role-safety.test.ts new file mode 100644 index 0000000000..a09dafa84d --- /dev/null +++ b/graphql/server/src/middleware/__tests__/runtime-role-safety.test.ts @@ -0,0 +1,514 @@ +import type { Pool } from 'pg'; + +import { + assertRuntimeRoleSafety, + DEFAULT_RUNTIME_ROLE_SAFETY_MAX_AGE_MS, + ensureRuntimeRoleSafety, + getRuntimeRoleSafetyStats, + invalidateRuntimeRoleSafety, + MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS, + refreshRuntimeRoleSafety, + RUNTIME_ROLE_SAFETY_SQL, + UnsafeRuntimeRoleError +} from '../runtime-role-safety'; + +const poolWithRow = (row: Record) => { + const client = { + query: jest.fn(async (query: string) => query === RUNTIME_ROLE_SAFETY_SQL + ? { rows: [row] } + : { rows: [] }), + release: jest.fn() + }; + return { + pool: { connect: jest.fn(async () => client) } as unknown as Pool, + client + }; +}; + +const safeRow = { + login_role: 'graphql_runtime', + login_role_violations: [] as Array<{ capabilities: string[] }>, + inherited_role_violations: [] as Array<{ rolname: string }>, + unexpected_set_role_violations: [] as Array<{ rolname: string }>, + request_role_reachability_violations: [] as Array<{ + request_role: string; + reachable_role: string; + via_usage: boolean; + via_set: boolean; + }>, + role_violations: [] as Array<{ rolname: string; capabilities: string[] }>, + database_violations: [] as Array<{ + rolname: string; + datname: string; + capability: string; + }>, + cross_database_violations: [] as Array<{ + rolname: string; + datname: string; + }>, + schema_violations: [] as Array<{ rolname: string; nspname: string; capability: string }>, + cross_schema_violations: [] as Array<{ + rolname: string; + nspname: string; + capabilities: string[]; + }>, + object_owner_violations: [] as Array<{ + rolname: string; + nspname: string; + object_name: string; + object_kind: string; + }>, + privileged_object_violations: [] as Array<{ + nspname: string; + object_name: string; + reason: string; + }>, + stored_dependency_violations: [] as Array<{ + nspname: string; + object_name: string; + reason: string; + dependency: string; + }>, + missing_roles: [] as string[], + inaccessible_roles: [] as string[], + missing_schemas: [] as string[] +}; + +describe('runtime role safety', () => { + it('accepts a least-privilege login and parameterizes roles and schemas', async () => { + const { pool, client } = poolWithRow(safeRow); + await expect(assertRuntimeRoleSafety( + pool, + ['tenant_anon', 'tenant_user'], + ['tenant_public'], + ['extensions'] + )).resolves.toBeUndefined(); + + expect(client.query).toHaveBeenNthCalledWith(2, RUNTIME_ROLE_SAFETY_SQL, [ + ['tenant_anon', 'tenant_user'], + ['tenant_public'], + ['extensions'] + ]); + expect(client.query).toHaveBeenNthCalledWith( + 1, + 'BEGIN READ ONLY; SET LOCAL jit TO off' + ); + expect(client.query).toHaveBeenNthCalledWith(3, 'COMMIT'); + expect(client.release).toHaveBeenCalledTimes(1); + }); + + it.each(['SUPERUSER', 'BYPASSRLS', 'CREATEROLE', 'CREATEDB', 'REPLICATION'])( + 'rejects %s', + async (capability) => { + const { pool } = poolWithRow({ + ...safeRow, + role_violations: [{ rolname: 'graphql_runtime', capabilities: [capability] }] + }); + await expect(assertRuntimeRoleSafety(pool, [], [])).rejects.toBeInstanceOf( + UnsafeRuntimeRoleError + ); + } + ); + + it('requires the runtime login to be NOINHERIT', async () => { + const { pool } = poolWithRow({ + ...safeRow, + login_role_violations: [{ capabilities: ['INHERIT'] }] + }); + await expect(assertRuntimeRoleSafety(pool, [], [])).rejects.toThrow( + 'graphql_runtime has INHERIT' + ); + }); + + it('rejects privileges inherited through a membership-level INHERIT grant', async () => { + const { pool } = poolWithRow({ + ...safeRow, + inherited_role_violations: [{ rolname: 'cross_tenant_reader' }] + }); + await expect(assertRuntimeRoleSafety(pool, [], [])).rejects.toThrow( + 'graphql_runtime inherits privileges from role cross_tenant_reader' + ); + }); + + it('rejects SET-able roles outside the exact configured request-role set', async () => { + const { pool } = poolWithRow({ + ...safeRow, + unexpected_set_role_violations: [{ rolname: 'tenant_admin' }] + }); + await expect(assertRuntimeRoleSafety( + pool, + ['tenant_anon', 'tenant_user'], + ['tenant_public'] + )).rejects.toThrow( + 'graphql_runtime can SET ROLE to unconfigured role tenant_admin' + ); + }); + + it('rejects roles reachable only after SET ROLE to a configured request role', async () => { + const { pool } = poolWithRow({ + ...safeRow, + request_role_reachability_violations: [{ + request_role: 'tenant_user', + reachable_role: 'tenant_owner', + via_usage: true, + via_set: false + }] + }); + await expect(assertRuntimeRoleSafety( + pool, + ['tenant_user'], + ['tenant_public'] + )).rejects.toThrow( + 'tenant_user can reach role tenant_owner after SET ROLE (USAGE=true,SET=false)' + ); + }); + + it.each(['OWNER', 'CREATE'])('rejects schema %s capability', async (capability) => { + const { pool } = poolWithRow({ + ...safeRow, + schema_violations: [{ + rolname: 'graphql_runtime', + nspname: 'tenant_public', + capability + }] + }); + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + `has ${capability} on schema tenant_public` + ); + }); + + it.each(['OWNER', 'CREATE', 'TEMP'])('rejects database %s capability', async (capability) => { + const { pool } = poolWithRow({ + ...safeRow, + database_violations: [{ + rolname: 'graphql_runtime', + datname: 'tenant_database', + capability + }] + }); + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + `graphql_runtime has ${capability} on database tenant_database` + ); + }); + + it('rejects CONNECT to a non-target database', async () => { + const { pool } = poolWithRow({ + ...safeRow, + cross_database_violations: [{ + rolname: 'tenant_user', + datname: 'tenant_b' + }] + }); + await expect(assertRuntimeRoleSafety( + pool, + ['tenant_user'], + ['tenant_public'] + )).rejects.toThrow( + 'tenant_user has CONNECT on non-target database tenant_b' + ); + }); + + it.each([ + 'login_role_violations', + 'inherited_role_violations', + 'database_violations', + 'cross_database_violations', + 'unexpected_set_role_violations', + 'request_role_reachability_violations', + 'role_violations', + 'schema_violations', + 'cross_schema_violations', + 'object_owner_violations', + 'privileged_object_violations', + 'stored_dependency_violations' + ])( + 'fails closed when the safety query omits %s', + async (column) => { + const row = { ...safeRow } as Record; + delete row[column]; + const { pool } = poolWithRow(row); + + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + `safety query did not return ${column} as a JSON array` + ); + } + ); + + it.each(['missing_roles', 'inaccessible_roles', 'missing_schemas'])( + 'fails closed when the safety query omits %s', + async (column) => { + const row = { ...safeRow } as Record; + delete row[column]; + const { pool } = poolWithRow(row); + + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + `safety query did not return ${column} as a text array` + ); + } + ); + + it('fails closed when the safety query omits the login role', async () => { + const { pool } = poolWithRow({ ...safeRow, login_role: null }); + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + 'safety query did not return a non-empty login_role' + ); + }); + + it('rejects effective object access to an unapproved tenant schema', async () => { + const { pool } = poolWithRow({ + ...safeRow, + cross_schema_violations: [{ + rolname: 'graphql_runtime', + nspname: 'tenant_b', + capabilities: ['RELATION', 'FUNCTION'] + }] + }); + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_a'])).rejects.toThrow( + 'graphql_runtime has RELATION,FUNCTION on unapproved schema tenant_b' + ); + }); + + it.each([ + 'SECURITY DEFINER FUNCTION', + 'OWNER-RIGHTS VIEW', + 'FOREIGN TABLE', + 'MATERIALIZED VIEW' + ])('rejects approved-scope %s paths that can escape invoker privileges', async (reason) => { + const { pool } = poolWithRow({ + ...safeRow, + privileged_object_violations: [{ + nspname: 'tenant_public', + object_name: 'unsafe_path', + reason + }] + }); + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + `${reason} tenant_public.unsafe_path is not allowed in the approved GraphQL schema scope` + ); + }); + + it('walks tracked dependencies transitively from every stored-expression class', () => { + expect(RUNTIME_ROLE_SAFETY_SQL).toContain('WITH RECURSIVE'); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain( + "pg_catalog.pg_has_role(current_user, r.oid, 'SET')" + ); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain( + "pg_catalog.pg_has_role(current_user, r.oid, 'USAGE')" + ); + expect(RUNTIME_ROLE_SAFETY_SQL).not.toContain( + "pg_catalog.pg_has_role(current_user, r.oid, 'MEMBER')" + ); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain('pg_catalog.current_database()'); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain( + "pg_catalog.has_database_privilege(r.rolname, d.oid, 'CREATE')" + ); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain( + "pg_catalog.has_database_privilege(r.rolname, d.oid, 'TEMP')" + ); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain( + 'd.oid <> current_database.oid' + ); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain( + "pg_catalog.has_database_privilege(r.rolname, d.oid, 'CONNECT')" + ); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain("'pg_catalog.pg_constraint'::regclass::oid"); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain("index_class.relkind IN ('i', 'I')"); + expect(RUNTIME_ROLE_SAFETY_SQL).toContain('FROM stored_dependency_closure closure'); + }); + + it.each(['RELATION', 'SEQUENCE', 'FUNCTION', 'TYPE'])('rejects %s ownership', async (objectKind) => { + const { pool } = poolWithRow({ + ...safeRow, + object_owner_violations: [{ + rolname: 'tenant_user', + nspname: 'tenant_public', + object_name: 'owned_object', + object_kind: objectKind + }] + }); + await expect(assertRuntimeRoleSafety(pool, ['tenant_user'], ['tenant_public'])).rejects.toThrow( + `tenant_user owns ${objectKind} tenant_public.owned_object` + ); + }); + + it('rejects stored expressions that reach a privileged helper', async () => { + const { pool } = poolWithRow({ + ...safeRow, + stored_dependency_violations: [{ + nspname: 'tenant_public', + object_name: 'documents:stamp_owner', + reason: 'STORED EXPRESSION CALLS SECURITY DEFINER', + dependency: 'hidden_private.lookup_owner' + }] + }); + await expect(assertRuntimeRoleSafety(pool, [], ['tenant_public'])).rejects.toThrow( + 'STORED EXPRESSION CALLS SECURITY DEFINER from tenant_public.documents:stamp_owner to hidden_private.lookup_owner' + ); + }); + + it('rejects missing roles and schemas instead of silently weakening the check', async () => { + const { pool } = poolWithRow({ + ...safeRow, + missing_roles: ['tenant_user'], + missing_schemas: ['tenant_public'] + }); + await expect(assertRuntimeRoleSafety(pool, ['tenant_user'], ['tenant_public'])).rejects.toThrow( + 'request role tenant_user does not exist' + ); + }); + + it('coalesces concurrent checks and bounds successful-result reuse from completion', async () => { + const now = jest.spyOn(Date, 'now').mockReturnValue(1_000); + const reuseOptions = { maxSuccessAgeMs: MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS }; + let releaseFirst!: () => void; + const first = new Promise((resolve) => { + releaseFirst = resolve; + }); + const query = jest.fn() + .mockImplementationOnce(async () => { + await first; + return { rows: [safeRow] }; + }) + .mockResolvedValue({ rows: [safeRow] }); + const client = { + query: jest.fn(async (sql: string) => sql === RUNTIME_ROLE_SAFETY_SQL + ? query() + : { rows: [] }), + release: jest.fn() + }; + const pool = { connect: jest.fn(async () => client) } as unknown as Pool; + + const one = ensureRuntimeRoleSafety(pool, [], ['tenant_public'], [], reuseOptions); + const concurrent = ensureRuntimeRoleSafety( + pool, + [], + ['tenant_public'], + [], + reuseOptions + ); + expect(pool.connect).toHaveBeenCalledTimes(1); + now.mockReturnValue(1_200); + releaseFirst(); + await Promise.all([one, concurrent]); + expect(query).toHaveBeenCalledTimes(1); + + now.mockReturnValue(1_200 + MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS - 1); + await ensureRuntimeRoleSafety(pool, [], ['tenant_public'], [], reuseOptions); + expect(query).toHaveBeenCalledTimes(1); + + now.mockReturnValue(1_200 + MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS); + await ensureRuntimeRoleSafety(pool, [], ['tenant_public'], [], reuseOptions); + expect(query).toHaveBeenCalledTimes(2); + now.mockRestore(); + }); + + it('defaults to a zero-age policy and supports explicit invalidation', async () => { + const client = { + query: jest.fn(async (sql: string) => sql === RUNTIME_ROLE_SAFETY_SQL + ? { rows: [safeRow] } + : { rows: [] }), + release: jest.fn() + }; + const pool = { connect: jest.fn(async () => client) } as unknown as Pool; + + expect(DEFAULT_RUNTIME_ROLE_SAFETY_MAX_AGE_MS).toBe(0); + await ensureRuntimeRoleSafety(pool, [], ['tenant_public']); + await ensureRuntimeRoleSafety(pool, [], ['tenant_public']); + expect(pool.connect).toHaveBeenCalledTimes(2); + + await ensureRuntimeRoleSafety( + pool, + [], + ['tenant_public'], + [], + { maxSuccessAgeMs: MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS } + ); + expect(pool.connect).toHaveBeenCalledTimes(2); + invalidateRuntimeRoleSafety(pool); + await ensureRuntimeRoleSafety(pool, [], ['tenant_public']); + expect(pool.connect).toHaveBeenCalledTimes(3); + + await refreshRuntimeRoleSafety(pool, [], ['tenant_public']); + expect(pool.connect).toHaveBeenCalledTimes(4); + }); + + it('never caches a failed catalog audit', async () => { + let auditAttempts = 0; + const client = { + query: jest.fn(async (sql: string) => { + if (sql !== RUNTIME_ROLE_SAFETY_SQL) return { rows: [] }; + auditAttempts += 1; + if (auditAttempts === 1) throw new Error('catalog unavailable'); + return { rows: [safeRow] }; + }), + release: jest.fn() + }; + const pool = { connect: jest.fn(async () => client) } as unknown as Pool; + + await expect( + ensureRuntimeRoleSafety(pool, [], ['tenant_public']) + ).rejects.toThrow('catalog unavailable'); + await expect( + ensureRuntimeRoleSafety(pool, [], ['tenant_public']) + ).resolves.toBeUndefined(); + + expect(auditAttempts).toBe(2); + expect(pool.connect).toHaveBeenCalledTimes(2); + expect(client.release).toHaveBeenNthCalledWith(1, true); + }); + + it('reports actual checks separately from coalesced and reused callers', async () => { + const before = getRuntimeRoleSafetyStats(); + const reuseOptions = { maxSuccessAgeMs: MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS }; + let resolveAudit!: () => void; + const auditBlocked = new Promise((resolve) => { + resolveAudit = resolve; + }); + const client = { + query: jest.fn(async (sql: string) => { + if (sql !== RUNTIME_ROLE_SAFETY_SQL) return { rows: [] }; + await auditBlocked; + return { rows: [safeRow] }; + }), + release: jest.fn() + }; + const pool = { connect: jest.fn(async () => client) } as unknown as Pool; + + const first = ensureRuntimeRoleSafety( + pool, + [], + ['tenant_public'], + [], + reuseOptions + ); + const coalesced = ensureRuntimeRoleSafety( + pool, + [], + ['tenant_public'], + [], + reuseOptions + ); + resolveAudit(); + await Promise.all([first, coalesced]); + await ensureRuntimeRoleSafety(pool, [], ['tenant_public'], [], reuseOptions); + + const after = getRuntimeRoleSafetyStats(); + expect(after.checksStarted - before.checksStarted).toBe(1); + expect(after.checksSucceeded - before.checksSucceeded).toBe(1); + expect(after.checksFailed - before.checksFailed).toBe(0); + expect(after.inFlightCoalesces - before.inFlightCoalesces).toBe(1); + expect(after.successfulResultReuses - before.successfulResultReuses).toBe(1); + expect(after.durationMsTotal).toBeGreaterThanOrEqual(before.durationMsTotal); + }); + + it('rejects attempts to extend the successful-audit freshness bound', () => { + const { pool } = poolWithRow(safeRow); + expect(() => ensureRuntimeRoleSafety( + pool, + [], + ['tenant_public'], + [], + { maxSuccessAgeMs: MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS + 1 } + )).toThrow('maxSuccessAgeMs must be an integer between 0 and'); + }); +}); diff --git a/graphql/server/src/middleware/api.ts b/graphql/server/src/middleware/api.ts index 24f28e9228..51a2a4f08e 100644 --- a/graphql/server/src/middleware/api.ts +++ b/graphql/server/src/middleware/api.ts @@ -10,12 +10,24 @@ import { Logger } from '@pgpmjs/logger'; import { svcCache } from '@pgpmjs/server-utils'; import { NextFunction, Request, Response } from 'express'; import { Pool } from 'pg'; -import { getPgPool } from 'pg-cache'; +import { + acquirePgPool, + getPgPoolIdentity, + PG_POOL_CAPACITY_ERROR_CODE, + type PgPoolLease +} from 'pg-cache'; import errorPage50x from '../errors/50x'; import errorPage404Message from '../errors/404-message'; import { ApiConfigResult, ApiError, ApiOptions, ApiStructure, AuthSettings, DatabaseSettings, PubkeyChallengeSettings, RlsModule, WebauthnSettings } from '../types'; -import { getRoutingSchema, isValidSchemaName, resolveRoute, routeToApiStructure } from './routing'; +import { authorizeInternalRequest } from './internal-request'; +import { + getRoutingSchema, + isValidPhysicalSchemaName, + isValidSchemaName, + resolveRoute, + routeToApiStructure +} from './routing'; const log = new Logger('api'); @@ -25,6 +37,118 @@ const log = new Logger('api'); const defaultRegistry: LoaderRegistry = createDefaultRegistry(); +const SVC_CACHE_CONTRACT_VERSION = 'constructive-routing-cache:v1'; + +interface SvcCacheContract { + routingPoolIdentity: string; + routingSchema: string; + serviceKey: string; +} + +const getSvcCacheContract = ( + opts: ApiOptions, + serviceKey: string +): SvcCacheContract => ({ + routingPoolIdentity: getPgPoolIdentity(opts.pg, { + purpose: 'routing-request-control', + sanitizeOnCheckout: true + }), + routingSchema: getRoutingSchema(opts), + serviceKey +}); + +export const getSvcCacheKey = ( + opts: ApiOptions, + serviceKey: string +): string => { + const contract = getSvcCacheContract(opts, serviceKey); + return JSON.stringify([ + SVC_CACHE_CONTRACT_VERSION, + contract.routingPoolIdentity, + contract.routingSchema, + contract.serviceKey + ]); +}; + +const parseSvcCacheKey = (key: string): SvcCacheContract | null => { + try { + const parsed = JSON.parse(key); + if ( + !Array.isArray(parsed) + || parsed.length !== 4 + || parsed[0] !== SVC_CACHE_CONTRACT_VERSION + || parsed.slice(1).some((value) => typeof value !== 'string') + ) { + return null; + } + return { + routingPoolIdentity: parsed[1], + routingSchema: parsed[2], + serviceKey: parsed[3] + }; + } catch { + return null; + } +}; + +/** Invalidate one exact physical routing entry left by an older caller. */ +export const invalidateSvcCacheKey = (cacheKey: string): boolean => { + return svcCache.delete(cacheKey); +}; + +const invalidateSvcCacheWhere = ( + predicate: (contract: SvcCacheContract, value: unknown) => boolean +): number => { + const keys: string[] = []; + for (const [key, value] of svcCache.entries()) { + const contract = parseSvcCacheKey(key); + if (contract && predicate(contract, value)) keys.push(key); + } + for (const key of keys) svcCache.delete(key); + return keys.length; +}; + +const sameSvcCacheScope = ( + left: SvcCacheContract, + right: SvcCacheContract +): boolean => + left.routingPoolIdentity === right.routingPoolIdentity + && left.routingSchema === right.routingSchema; + +export const invalidateSvcCacheForService = ( + opts: ApiOptions, + serviceKey: string +): number => { + const expected = getSvcCacheContract(opts, serviceKey); + return invalidateSvcCacheWhere((contract) => + sameSvcCacheScope(contract, expected) + && contract.serviceKey === serviceKey + ); +}; + +export const invalidateSvcCacheForDatabase = ( + opts: ApiOptions, + databaseId: string +): number => { + const expected = getSvcCacheContract(opts, ''); + const apiPrefix = `api:${databaseId}:`; + const schemataPrefix = `schemata:${databaseId}:`; + const metaKey = `metaschema:api:${databaseId}`; + return invalidateSvcCacheWhere((contract, value) => { + if (!sameSvcCacheScope(contract, expected)) return false; + const cachedDatabaseId = (value as { databaseId?: unknown })?.databaseId; + return cachedDatabaseId === databaseId + || contract.serviceKey.startsWith(apiPrefix) + || contract.serviceKey.startsWith(schemataPrefix) + || contract.serviceKey === metaKey; + }); +}; + +/** Clear process-wide routing metadata and retire every in-flight publication. */ +export const clearSvcCache = (): void => { + svcCache.clear(); +}; + // ============================================================================= // SQL Queries (API resolution only — module queries now live in loaders) // ============================================================================= @@ -44,12 +168,15 @@ const scopedApiNameLookupSql = (routingSchema: string): string => ` a.is_published as is_public, COALESCE(array_agg(s.schema_name) FILTER (WHERE s.schema_name IS NOT NULL), '{}') as schemas FROM "${routingSchema}".apis a - LEFT JOIN "${routingSchema}".api_schemas aps ON a.id = aps.api_id - LEFT JOIN metaschema_public.schema s ON aps.schema_id = s.id + LEFT JOIN "${routingSchema}".api_schemas aps + ON a.id = aps.api_id + AND aps.database_id = a.database_id + LEFT JOIN metaschema_public.schema s + ON aps.schema_id = s.id + AND s.database_id = a.database_id WHERE a.database_id = $1 AND a.name = $2 GROUP BY a.id, a.database_id, a.dbname, a.role_name, a.anon_role, a.is_published - LIMIT 1 `; // ============================================================================= @@ -68,7 +195,10 @@ interface ApiRow { interface ResolveContext { opts: ApiOptions; + registry: LoaderRegistry; pool: Pool; + routingPoolIdentity: string; + leases: PgPoolLease[]; domain: string; subdomain: string | null; cacheKey: string; @@ -77,7 +207,6 @@ interface ResolveContext { } type ResolutionMode = - | 'schemata-header' | 'api-name-header' | 'meta-schema-header' | 'scoped-route'; @@ -85,7 +214,6 @@ type ResolutionMode = type PrivateHeaderMode = Exclude; interface RoutingHeaders { - schemata?: string; apiName?: string; metaSchema?: string; databaseId?: string; @@ -104,22 +232,42 @@ interface ResolvedModuleSettings { webauthnSettings?: WebauthnSettings; } +export class MissingDatabaseFeatureContractError extends Error { + readonly code = 'GRAPHILE_DATABASE_FEATURE_CONTRACT_MISSING'; + + constructor(databaseId: string, apiId: string) { + super( + `No exact database feature contract resolved for database ${databaseId} and API ${apiId}` + ); + this.name = 'MissingDatabaseFeatureContractError'; + } +} + /** * Build a LoaderContext from the API row and options. * This is used to resolve per-database module settings via the loader registry. */ const buildLoaderContext = ( routingPool: Pool, + routingPoolIdentity: string, opts: ApiOptions, - row: ApiRow + row: ApiRow, + leases: PgPoolLease[] ): LoaderContext => { // Scoped APIs leave dbname NULL when their schemas live in the serving // database (pooled tenants); fall back to the server's own database. const dbname = row.dbname || opts.pg?.database || ''; + const tenantLease = acquirePgPool( + { ...opts.pg, database: dbname }, + { purpose: 'tenant-request-control', sanitizeOnCheckout: true } + ); + leases.push(tenantLease); return { routingPool, + routingPoolIdentity, routingSchema: getRoutingSchema(opts), - tenantPool: getPgPool({ ...opts.pg, database: dbname }), + tenantPool: tenantLease.pool, + tenantPoolIdentity: tenantLease.identity, databaseId: row.database_id, apiId: row.api_id, dbname @@ -150,6 +298,13 @@ const resolveModuleSettings = async ( registry.resolve('webauthnSettings', ctx) ]); + // These flags select executable plugins, realtime, uploads, and search. A + // missing metadata row must not silently expand the surface through preset + // defaults on an otherwise authoritative tenant route. + if (!databaseSettings) { + throw new MissingDatabaseFeatureContractError(ctx.databaseId, ctx.apiId ?? ''); + } + return { rlsModule, authSettings, @@ -182,18 +337,13 @@ const assertDatabaseId = (result: ApiStructure): void => { } }; -const parseCommaSeparatedHeader = (value: string): string[] => - value.split(',').map((s) => s.trim()).filter(Boolean); - const getPrivateHeaderMode = (headers: RoutingHeaders): PrivateHeaderMode | null => { if (headers.apiName) return 'api-name-header'; - if (headers.schemata) return 'schemata-header'; if (headers.metaSchema) return 'meta-schema-header'; return null; }; const getRoutingHeaders = (req: Request): RoutingHeaders => ({ - schemata: req.get('X-Schemata'), apiName: req.get('X-Api-Name'), metaSchema: req.get('X-Meta-Schema'), databaseId: req.get('X-Database-Id') @@ -217,15 +367,12 @@ export const getSvcKey = (opts: ApiOptions, req: Request): string => { const { domain, subdomains } = getUrlDomains(req); const baseKey = subdomains.filter((n) => n !== 'www').concat(domain).join('.'); - if (opts.api?.isPublic === false) { + if (opts.api?.isPublic === false && req.internalTrusted === true) { const headers = getRoutingHeaders(req); const mode = getPrivateHeaderMode(headers); if (mode === 'api-name-header') { return `api:${headers.databaseId}:${headers.apiName}`; } - if (mode === 'schemata-header') { - return `schemata:${headers.databaseId}:${headers.schemata}`; - } if (mode === 'meta-schema-header') { return `metaschema:api:${headers.databaseId}`; } @@ -236,9 +383,9 @@ export const getSvcKey = (opts: ApiOptions, req: Request): string => { const toApiStructure = (row: ApiRow, opts: ApiOptions, settings: ResolvedModuleSettings = {}): ApiStructure => ({ apiId: row.api_id, dbname: row.dbname || opts.pg?.database || '', - anonRole: row.anon_role || 'anon', - roleName: row.role_name || 'authenticated', - schema: row.schemas || [], + anonRole: row.anon_role, + roleName: row.role_name, + schema: row.schemas, rlsModule: settings.rlsModule, domains: [], databaseId: row.database_id, @@ -250,14 +397,31 @@ const toApiStructure = (row: ApiRow, opts: ApiOptions, settings: ResolvedModuleS webauthnSettings: settings.webauthnSettings }); +const isExactApiRow = (row: ApiRow, requestedDatabaseId: string): boolean => + typeof row.api_id === 'string' + && row.api_id.length > 0 + && row.database_id === requestedDatabaseId + && typeof row.role_name === 'string' + && row.role_name.length > 0 + && typeof row.anon_role === 'string' + && row.anon_role.length > 0 + && typeof row.is_public === 'boolean' + && Array.isArray(row.schemas) + && row.schemas.length > 0 + && row.schemas.every(isValidPhysicalSchemaName) + && new Set(row.schemas).size === row.schemas.length; + const createAdminStructure = ( opts: ApiOptions, schemas: string[], databaseId?: string ): ApiStructure => ({ dbname: opts.pg?.database ?? '', - anonRole: 'administrator', - roleName: 'administrator', + // Private header/meta-schema surfaces must be able to use a dedicated + // non-BYPASSRLS execution role. Keep the legacy default for compatibility; + // production admission will reject it unless operators configure safe roles. + anonRole: opts.api?.anonRole ?? 'administrator', + roleName: opts.api?.roleName ?? 'administrator', schema: schemas, domains: [], databaseId, @@ -288,7 +452,14 @@ const queryByApiName = async ( return null; } const result = await pool.query(scopedApiNameLookupSql(routingSchema), [databaseId, name]); - return result.rows[0] ?? null; + if (result.rows.length !== 1) { + log.warn( + `[api-name-lookup] expected one exact API row for databaseId=${databaseId}; ` + + `received ${result.rows.length}` + ); + return null; + } + return result.rows[0]; }; // ============================================================================= @@ -304,35 +475,25 @@ const determineMode = (ctx: ResolveContext): ResolutionMode => { return 'scoped-route'; }; -const resolveSchemataHeader = async ( - ctx: ResolveContext, - validatedSchemas: string[] -): Promise => { - const { opts, headers } = ctx; - const headerSchemas = parseCommaSeparatedHeader(headers.schemata!); - const validSet = new Set(validatedSchemas); - const validHeaderSchemas = headerSchemas.filter((s) => validSet.has(s)); - - if (validHeaderSchemas.length === 0) { - return { errorHtml: 'No valid schemas found for the supplied X-Schemata header.' }; - } - - return createAdminStructure(opts, validHeaderSchemas, headers.databaseId); -}; - const resolveApiNameHeader = async (ctx: ResolveContext): Promise => { const { opts, pool, headers } = ctx; if (!headers.databaseId) return null; const row = await queryByApiName(pool, opts, headers.databaseId, headers.apiName!); - if (!row) { + if (!row || !isExactApiRow(row, headers.databaseId)) { log.debug(`[api-name-lookup] No API found for databaseId=${headers.databaseId} name=${headers.apiName}`); return null; } - const loaderCtx = buildLoaderContext(pool, opts, row); - const settings = await resolveModuleSettings(defaultRegistry, loaderCtx); + const loaderCtx = buildLoaderContext( + pool, + ctx.routingPoolIdentity, + opts, + row, + ctx.leases + ); + const settings = await resolveModuleSettings(ctx.registry, loaderCtx); log.debug(`[api-name-lookup] resolved schemas: [${row.schemas?.join(', ')}], rlsModule: ${settings.rlsModule ? 'found' : 'none'}, authSettings: ${settings.authSettings ? 'found' : 'none'}`); return toApiStructure(row, opts, settings); }; @@ -366,7 +527,7 @@ const resolveScopedRoute = async (ctx: ResolveContext): Promise => { - const pool = getPgPool(opts.pg); + authorizeInternalRequest(opts, req); const { domain, subdomains } = getUrlDomains(req); const subdomain = getSubdomain(subdomains); - const cacheKey = getSvcKey(opts, req); - - req.svc_key = cacheKey; - - // Check cache first - if (svcCache.has(cacheKey)) { - log.debug(`Cache HIT for key=${cacheKey}`); - return svcCache.get(cacheKey) as ApiStructure; - } - - log.debug(`Cache MISS for key=${cacheKey}, resolving API`); - - const ctx: ResolveContext = { - opts, - pool, - domain, - subdomain, - cacheKey, - headers: getRoutingHeaders(req), - host: req.get('host') || '' - }; - - // Validate schemas upfront for modes that need them - const apiOpts = opts.api || {}; - const headerSchemas = ctx.headers.schemata ? parseCommaSeparatedHeader(ctx.headers.schemata) : []; - const candidateSchemas = - apiOpts.isPublic === false && headerSchemas.length - ? [...new Set([...(apiOpts.metaSchemas || []), ...headerSchemas])] - : apiOpts.metaSchemas || []; - - const validatedSchemas = await validateSchemata(pool, candidateSchemas); - - if (validatedSchemas.length === 0) { - const source = headerSchemas.length ? headerSchemas : apiOpts.metaSchemas || []; - const label = headerSchemas.length ? 'X-Schemata' : 'metaSchemas'; - const error = new Error(`No valid schemas found. Configured ${label}: [${source.join(', ')}]`) as Error & { code?: string }; - error.code = 'NO_VALID_SCHEMAS'; - throw error; - } + const serviceKey = getSvcKey(opts, req); + const cacheKey = getSvcCacheKey(opts, serviceKey); + + req.svc_key = serviceKey; + req.svc_cache_key = cacheKey; + + // Hostname and private-selector routing is an authorization boundary. LISTEN + // notifications are lossy across disconnects, so cached metadata cannot be + // authoritative after a domain or API is reassigned. Resolve every request; + // the independently keyed PostGraphile build cache still provides the large + // memory and build-latency win once this exact contract is known. + log.debug(`Authoritatively resolving API for key=${cacheKey}`); + const leases: PgPoolLease[] = []; + + try { + const routingLease = acquirePgPool(opts.pg, { + purpose: 'routing-request-control', + sanitizeOnCheckout: true + }); + leases.push(routingLease); + const pool = routingLease.pool; + const ctx: ResolveContext = { + opts, + pool, + routingPoolIdentity: routingLease.identity, + leases, + domain, + subdomain, + cacheKey, + headers: getRoutingHeaders(req), + host: req.get('host') || '', + registry + }; + + // Validate schemas upfront for modes that need them + const apiOpts = opts.api || {}; + const candidateSchemas = apiOpts.metaSchemas || []; + + const validatedSchemas = await validateSchemata(pool, candidateSchemas); + + if (validatedSchemas.length === 0) { + const source = apiOpts.metaSchemas || []; + const error = new Error(`No valid schemas found. Configured metaSchemas: [${source.join(', ')}]`) as Error & { code?: string }; + error.code = 'NO_VALID_SCHEMAS'; + throw error; + } - // Route to appropriate resolver based on mode - const mode = determineMode(ctx); - let result: ApiConfigResult; + // Route to appropriate resolver based on mode + const mode = determineMode(ctx); + let result: ApiConfigResult; - switch (mode) { - case 'schemata-header': - result = await resolveSchemataHeader(ctx, validatedSchemas); - break; + switch (mode) { + case 'api-name-header': + result = await resolveApiNameHeader(ctx); + break; - case 'api-name-header': - result = await resolveApiNameHeader(ctx); - break; + case 'meta-schema-header': + result = resolveMetaSchemaHeader(ctx, validatedSchemas); + break; - case 'meta-schema-header': - result = resolveMetaSchemaHeader(ctx, validatedSchemas); - break; + case 'scoped-route': + result = await resolveScopedRoute(ctx); + break; + } - case 'scoped-route': - result = await resolveScopedRoute(ctx); - break; - } + // Assert the complete routing identity before any downstream middleware. + // Deliberately do not publish this result to svcCache; see above. + if (result && !isApiError(result)) { + assertDatabaseId(result); + } - // Cache successful results - if (result && !isApiError(result)) { - assertDatabaseId(result); - svcCache.set(cacheKey, result); + return result; + } finally { + for (let i = leases.length - 1; i >= 0; i--) { + leases[i].release(); + } } - - return result; }; // ============================================================================= // Express Middleware // ============================================================================= -export const createApiMiddleware = (opts: ApiOptions) => { +export const createApiMiddleware = ( + opts: ApiOptions, + registry: LoaderRegistry = defaultRegistry +) => { return async (req: Request, res: Response, next: NextFunction): Promise => { log.debug(`[api-middleware] ${req.method} ${req.path}`); try { - const apiConfig = await getApiConfig(opts, req); + const apiConfig = await getApiConfig(opts, req, registry); if (isApiError(apiConfig)) { res.status(404).send(errorPage404Message('API not found', apiConfig.errorHtml)); @@ -497,6 +670,11 @@ export const createApiMiddleware = (opts: ApiOptions) => { } catch (error: unknown) { const err = error as Error & { code?: string }; + if (err.code === 'INTERNAL_REQUEST_FORBIDDEN') { + res.status(403).send('Forbidden'); + return; + } + if (err.code === 'NO_VALID_SCHEMAS') { res.status(404).send(errorPage404Message(err.message)); return; @@ -508,6 +686,11 @@ export const createApiMiddleware = (opts: ApiOptions) => { return; } + if (err.code === PG_POOL_CAPACITY_ERROR_CODE) { + next(err); + return; + } + if (err.message?.includes('does not exist')) { res.status(404).send(errorPage404Message("The resource you're looking for does not exist.")); return; diff --git a/graphql/server/src/middleware/auth.ts b/graphql/server/src/middleware/auth.ts index ef6da3f3a2..71c8e8d9ea 100644 --- a/graphql/server/src/middleware/auth.ts +++ b/graphql/server/src/middleware/auth.ts @@ -1,11 +1,19 @@ import './types'; // for Request type import { errors } from '@constructive-io/errors'; +import { + quoteQualifiedSqlIdentifier, + SECURITY_GUC_KEYS +} from '@constructive-io/express-context'; import { getNodeEnv } from '@pgpmjs/env'; import { Logger } from '@pgpmjs/logger'; import { PgpmOptions } from '@pgpmjs/types'; import { NextFunction, Request, RequestHandler, Response } from 'express'; -import { getPgPool } from 'pg-cache'; +import { + acquirePgPool, + PG_POOL_CAPACITY_ERROR_CODE, + type PgPoolLease +} from 'pg-cache'; import pgQueryContext from 'pg-query-context'; import { respondWithGraphQLError } from '../errors/graphql-response'; @@ -19,6 +27,23 @@ const SESSION_COOKIE_NAME = 'constructive_session'; /** Cookie name for trusted device tracking. */ const DEVICE_TOKEN_COOKIE_NAME = 'constructive_device_token'; +/** Complete transaction-local context for the sanitized authentication lane. */ +export const buildAuthenticationContext = ( + req: Request, + api: NonNullable +): Record => ({ + ...Object.fromEntries(SECURITY_GUC_KEYS.map((key) => [key, ''])), + 'jwt.claims.api_id': api.apiId ?? '', + 'jwt.claims.database_id': api.databaseId ?? '', + 'jwt.claims.ip_address': req.clientIp ?? '', + 'jwt.claims.origin': req.get('origin') ?? '', + 'jwt.claims.user_agent': req.get('User-Agent') ?? '', + 'request.id': req.requestId ?? '', + 'row_security': 'on', + 'search_path': 'pg_catalog', + 'transaction_read_only': 'on' +}); + /** * Extract a named cookie value from the raw Cookie header. * Avoids pulling in cookie-parser as a dependency. @@ -45,10 +70,6 @@ export const createAuthenticateMiddleware = ( return; } - const pool = getPgPool({ - ...opts.pg, - database: api.dbname, - }); const rlsModule = api.rlsModule; log.info( @@ -59,6 +80,18 @@ export const createAuthenticateMiddleware = ( ); if (!rlsModule) { + if (opts.server?.strictAuth) { + log.error('[auth] Strict authentication requires an RLS module'); + respondWithGraphQLError( + res, + errors.INTERNAL_FAILURE({ + details: isDev() + ? 'Strict authentication requires an RLS module' + : 'authentication failed' + }) + ); + return; + } log.info('[auth] No RLS module configured, skipping auth'); return next(); } @@ -71,6 +104,19 @@ export const createAuthenticateMiddleware = ( `[auth] strictAuth=${opts.server?.strictAuth ?? false}, authFn=${authFn ?? 'none'}` ); + if (!authFn || !rlsModule.privateSchema.schemaName) { + log.error('[auth] RLS authentication configuration is incomplete'); + respondWithGraphQLError( + res, + errors.INTERNAL_FAILURE({ + details: isDev() + ? 'RLS authentication configuration is incomplete' + : 'authentication failed' + }) + ); + return; + } + if (authFn && rlsModule.privateSchema.schemaName) { const { authorization = '' } = req.headers; const [authType, authToken] = authorization.split(' '); @@ -90,23 +136,38 @@ export const createAuthenticateMiddleware = ( if (effectiveToken) { log.info(`[auth] Processing ${tokenSource} authentication`); - const context: Record = { - 'jwt.claims.ip_address': req.clientIp, - }; + const context = buildAuthenticationContext(req, api); - if (req.get('origin')) { - context['jwt.claims.origin'] = req.get('origin'); - } - if (req.get('User-Agent')) { - context['jwt.claims.user_agent'] = req.get('User-Agent'); + let authQuery: string; + try { + authQuery = `SELECT * FROM ${quoteQualifiedSqlIdentifier( + rlsModule.privateSchema.schemaName, + authFn, + 'authentication function' + )}($1)`; + } catch (e: unknown) { + const message = e instanceof Error + ? e.message + : 'invalid authentication function'; + log.error('[auth] Invalid authentication function metadata:', message); + respondWithGraphQLError( + res, + errors.INTERNAL_FAILURE({ + details: isDev() ? message : 'authentication failed' + }) + ); + return; } - - const authQuery = `SELECT * FROM "${rlsModule.privateSchema.schemaName}"."${authFn}"($1)`; log.info(`[auth] Executing auth query: ${authQuery}`); + let poolLease: PgPoolLease | undefined; try { + poolLease = acquirePgPool({ + ...opts.pg, + database: api.dbname, + }, { purpose: 'tenant-request-control', sanitizeOnCheckout: true }); const result = await pgQueryContext({ - client: pool, + client: poolLease.pool, context, query: authQuery, variables: [effectiveToken], @@ -123,6 +184,10 @@ export const createAuthenticateMiddleware = ( token = result.rows[0]; log.info(`[auth] Auth success: role=${token.role}, user_id=${token.user_id}`); } catch (e: any) { + if (e?.code === PG_POOL_CAPACITY_ERROR_CODE) { + next(e); + return; + } log.error('[auth] Auth error:', e.message); respondWithGraphQLError( res, @@ -131,17 +196,14 @@ export const createAuthenticateMiddleware = ( }) ); return; + } finally { + poolLease?.release(); } } else { log.info('[auth] No credential provided (no bearer token or session cookie), using anonymous auth'); } req.token = token; - } else { - log.info( - `[auth] Skipping auth: authFn=${authFn ?? 'none'}, ` + - `privateSchema=${rlsModule.privateSchema?.schemaName ?? 'none'}` - ); } // Read device token cookie for trusted device tracking diff --git a/graphql/server/src/middleware/captcha.ts b/graphql/server/src/middleware/captcha.ts index 7a4da18955..8ca26d0ddd 100644 --- a/graphql/server/src/middleware/captcha.ts +++ b/graphql/server/src/middleware/captcha.ts @@ -1,8 +1,22 @@ import './types'; // for Request type import { errors } from '@constructive-io/errors'; +import { getNodeEnv } from '@pgpmjs/env'; import { Logger } from '@pgpmjs/logger'; -import type { NextFunction, Request, RequestHandler, Response } from 'express'; +import express, { + type NextFunction, + type Request, + type RequestHandler, + type Response +} from 'express'; +import { + Kind, + parse, + type DocumentNode, + type FragmentDefinitionNode, + type OperationDefinitionNode, + type SelectionSetNode +} from 'graphql'; import { respondWithGraphQLError } from '../errors/graphql-response'; @@ -17,6 +31,9 @@ const RECAPTCHA_VERIFY_URL = 'https://www.google.com/recaptcha/api/siteverify'; */ const CAPTCHA_HEADER = 'x-captcha-token'; +/** Match Grafserv's default maximum GraphQL request length. */ +export const CAPTCHA_GRAPHQL_BODY_LIMIT_BYTES = 100_000; + /** * GraphQL mutation names that require CAPTCHA verification when enabled. * Only sign-up and password-reset are gated; normal sign-in is not. @@ -29,25 +46,181 @@ const CAPTCHA_PROTECTED_OPERATIONS = new Set([ 'requestPasswordReset', ]); +export type CaptchaOperationInspection = + | { kind: 'protected'; fields: readonly string[] } + | { kind: 'not-protected' } + | { kind: 'invalid'; reason: string }; + interface RecaptchaResponse { success: boolean; 'error-codes'?: string[]; } +export interface CaptchaMiddlewareOptions { + /** Authentication-required deployments must never disable CAPTCHA implicitly. */ + strictAuth?: boolean; + /** @internal Deterministic environment seam for focused tests. */ + nodeEnv?: ReturnType; +} + /** - * Attempt to extract the GraphQL operation name from the request body. - * Works for both JSON and already-parsed bodies. + * Parse the GraphQL request formats Grafserv accepts before CAPTCHA admission. + * Multipart requests are deliberately left to graphql-upload, which supplies + * the same object-shaped body before the CAPTCHA middleware runs. */ -const getOperationName = (req: Request): string | undefined => { - const body = (req as any).body; - if (!body) return undefined; - // Already parsed (express.json ran first) - if (typeof body === 'object' && body.operationName) { - return body.operationName; +export const createCaptchaGraphqlBodyParsers = (): RequestHandler[] => [ + express.json({ limit: CAPTCHA_GRAPHQL_BODY_LIMIT_BYTES }), + express.text({ + type: 'application/graphql', + limit: CAPTCHA_GRAPHQL_BODY_LIMIT_BYTES + }), + express.urlencoded({ + extended: false, + limit: CAPTCHA_GRAPHQL_BODY_LIMIT_BYTES + }) +]; + +const selectOperation = ( + document: DocumentNode, + operationName: string | undefined +): OperationDefinitionNode | undefined => { + const operations = document.definitions.filter( + (definition): definition is OperationDefinitionNode => + definition.kind === Kind.OPERATION_DEFINITION + ); + if (operationName === undefined) { + return operations.length === 1 ? operations[0] : undefined; + } + const matches = operations.filter( + (operation) => operation.name?.value === operationName + ); + return matches.length === 1 ? matches[0] : undefined; +}; + +const collectRootFields = ( + selectionSet: SelectionSetNode, + fragments: ReadonlyMap, + activeFragments: Set, + fields: Set +): string | undefined => { + for (const selection of selectionSet.selections) { + if (selection.kind === Kind.FIELD) { + fields.add(selection.name.value); + continue; + } + if (selection.kind === Kind.INLINE_FRAGMENT) { + const invalid = collectRootFields( + selection.selectionSet, + fragments, + activeFragments, + fields + ); + if (invalid) return invalid; + continue; + } + + const fragmentName = selection.name.value; + const fragment = fragments.get(fragmentName); + if (!fragment) return `missing fragment ${fragmentName}`; + if (activeFragments.has(fragmentName)) { + return `cyclic fragment ${fragmentName}`; + } + activeFragments.add(fragmentName); + const invalid = collectRootFields( + fragment.selectionSet, + fragments, + activeFragments, + fields + ); + activeFragments.delete(fragmentName); + if (invalid) return invalid; } return undefined; }; +/** + * Classify the selected operation from the GraphQL document itself. Operation + * labels are client-controlled and therefore never stand in for root fields. + */ +export const inspectCaptchaOperation = ( + query: unknown, + operationName: unknown +): CaptchaOperationInspection => { + if (typeof query !== 'string' || query.trim().length === 0) { + return { kind: 'invalid', reason: 'missing GraphQL query' }; + } + if ( + operationName !== undefined + && operationName !== null + && (typeof operationName !== 'string' || operationName.length === 0) + ) { + return { kind: 'invalid', reason: 'invalid GraphQL operation name' }; + } + + let document: DocumentNode; + try { + document = parse(query); + } catch { + return { kind: 'invalid', reason: 'malformed GraphQL document' }; + } + + const selected = selectOperation( + document, + typeof operationName === 'string' ? operationName : undefined + ); + if (!selected) { + return { kind: 'invalid', reason: 'ambiguous or missing GraphQL operation' }; + } + if (selected.operation !== 'mutation') return { kind: 'not-protected' }; + + const fragments = new Map(); + for (const definition of document.definitions) { + if (definition.kind !== Kind.FRAGMENT_DEFINITION) continue; + if (fragments.has(definition.name.value)) { + return { kind: 'invalid', reason: `duplicate fragment ${definition.name.value}` }; + } + fragments.set(definition.name.value, definition); + } + + const fields = new Set(); + const invalid = collectRootFields( + selected.selectionSet, + fragments, + new Set(), + fields + ); + if (invalid) return { kind: 'invalid', reason: invalid }; + + const protectedFields = [...fields] + .filter((field) => CAPTCHA_PROTECTED_OPERATIONS.has(field)) + .sort(); + return protectedFields.length > 0 + ? { kind: 'protected', fields: protectedFields } + : { kind: 'not-protected' }; +}; + +const isGraphqlPath = (req: Request): boolean => req.path === '/graphql'; + +const isWebSocketUpgrade = (req: Request): boolean => + req.method === 'GET' + && req.get('upgrade')?.trim().toLowerCase() === 'websocket'; + +const inspectHttpRequest = (req: Request): CaptchaOperationInspection => { + if (req.method === 'GET' || req.method === 'HEAD') { + return inspectCaptchaOperation(req.query?.query, req.query?.operationName); + } + + const body = (req as Request & { body?: unknown }).body; + if (typeof body === 'string') { + return inspectCaptchaOperation(body, undefined); + } + if (!body || Array.isArray(body) || typeof body !== 'object') { + return { kind: 'invalid', reason: 'invalid GraphQL request body' }; + } + const graphqlBody = body as Record; + return inspectCaptchaOperation(graphqlBody.query, graphqlBody.operationName); +}; + /** * Verify a reCAPTCHA token with Google's API. */ @@ -80,9 +253,17 @@ const verifyToken = async (token: string, secretKey: string): Promise = * Skips verification when: * - CAPTCHA is not enabled in auth settings * - The request is not a protected mutation - * - No secret key is configured server-side + * - No secret key is configured in a non-production, non-strict local server + * + * Production and strict-auth servers fail closed when tenant policy enables + * CAPTCHA but the server-side secret is missing. */ -export const createCaptchaMiddleware = (): RequestHandler => { +export const createCaptchaMiddleware = ( + options: CaptchaMiddlewareOptions = {} +): RequestHandler => { + const failClosedWithoutSecret = options.strictAuth === true + || (options.nodeEnv ?? getNodeEnv()) === 'production'; + return async (req: Request, res: Response, next: NextFunction): Promise => { const authSettings = req.api?.authSettings; @@ -91,16 +272,40 @@ export const createCaptchaMiddleware = (): RequestHandler => { return next(); } - // Only gate protected operations - const opName = getOperationName(req); - if (!opName || !CAPTCHA_PROTECTED_OPERATIONS.has(opName)) { + // WebSocket handshakes have no operation document. The generation-scoped + // onSubscribe admission hook rejects protected mutations per operation. + if (!isGraphqlPath(req) || isWebSocketUpgrade(req) || req.method === 'OPTIONS') { return next(); } + const inspection = inspectHttpRequest(req); + if (inspection.kind === 'not-protected') return next(); + if (inspection.kind === 'invalid') { + log.warn(`[captcha] Rejecting GraphQL request: ${inspection.reason}`); + respondWithGraphQLError( + res, + errors.INTERNAL_FAILURE({ details: 'authentication failed' }) + ); + return; + } + // Secret key must be set server-side (env var, not stored in DB for security) const secretKey = process.env.RECAPTCHA_SECRET_KEY; - if (!secretKey) { - log.warn('[captcha] enable_captcha is true but RECAPTCHA_SECRET_KEY env var is not set; skipping verification'); + if (!secretKey?.trim()) { + if (failClosedWithoutSecret) { + log.error( + '[captcha] enable_captcha is true but RECAPTCHA_SECRET_KEY is not configured; rejecting protected operation' + ); + respondWithGraphQLError( + res, + errors.INTERNAL_FAILURE({ details: 'authentication failed' }) + ); + return; + } + log.warn( + '[captcha] enable_captcha is true but RECAPTCHA_SECRET_KEY is not configured; ' + + 'skipping verification only for non-production, non-strict local mode' + ); return next(); } @@ -116,7 +321,7 @@ export const createCaptchaMiddleware = (): RequestHandler => { return; } - log.info(`[captcha] Verified for operation=${opName}`); + log.info(`[captcha] Verified for fields=${inspection.fields.join(',')}`); next(); }; }; diff --git a/graphql/server/src/middleware/cors.ts b/graphql/server/src/middleware/cors.ts index 8bb7cefebb..b77e1d6d30 100644 --- a/graphql/server/src/middleware/cors.ts +++ b/graphql/server/src/middleware/cors.ts @@ -6,6 +6,41 @@ import type { Request, RequestHandler } from 'express'; import type { ApiStructure } from '../types'; +export interface CorsOriginInput { + origin?: string; + fallbackOrigin?: string; + api?: ApiStructure; + requestHost?: string; +} + +/** Shared HTTP/WebSocket origin policy. Missing origins are handled by the caller. */ +export const isCorsOriginAllowed = ({ + origin, + fallbackOrigin, + api, + requestHost +}: CorsOriginInput): boolean => { + if (!origin) return false; + const fallback = fallbackOrigin?.trim(); + if (fallback === '*') return true; + if (fallback && origin.trim() === fallback) return true; + + if ([...(api?.corsOrigins ?? []), ...(api?.domains ?? [])].includes(origin)) { + return true; + } + + try { + const parsedOrigin = new URL(origin); + if (requestHost && parsedOrigin.host.toLowerCase() === requestHost.toLowerCase()) { + return true; + } + const parsed = parseUrl(parsedOrigin); + return parsed.domain === 'localhost'; + } catch { + return false; + } +}; + /** * Unified CORS middleware for Constructive API * @@ -20,47 +55,13 @@ import type { ApiStructure } from '../types'; export const cors = (fallbackOrigin?: string): RequestHandler => { // Use the cors library's dynamic origin function to decide per request const dynamicOrigin = (origin: string | undefined, callback: (err: Error | null, allow?: boolean | string) => void, req: Request) => { - // 1) Global fallback (fast path) - if (fallbackOrigin && fallbackOrigin.trim().length) { - if (fallbackOrigin.trim() === '*') { - // Reflect whatever Origin the caller sent - return callback(null, true); - } - if (origin && origin.trim() === fallbackOrigin.trim()) { - return callback(null, true); - } - // If a strict fallback origin is provided and does not match, - // continue to per-API checks below (do not immediately deny). - } - - // 2) Per-API allowlist sourced from req.api (if available) - // createApiMiddleware runs before this in server.ts, so req.api should be set const api = (req as any).api as ApiStructure | undefined; - if (api) { - // Typed cors_settings origins - const typedOrigins = api.corsOrigins || []; - const siteUrls = api.domains || []; - const listOfDomains = [...typedOrigins, ...siteUrls]; - - if (origin && listOfDomains.includes(origin)) { - return callback(null, true); - } - } - - // 3) Localhost is always allowed - if (origin) { - try { - const parsed = parseUrl(new URL(origin)); - if (parsed.domain === 'localhost') { - return callback(null, true); - } - } catch { - // ignore invalid origin - } - } - - // Default: not allowed - return callback(null, false); + return callback(null, isCorsOriginAllowed({ + origin, + fallbackOrigin, + api, + requestHost: req.get('host') + })); }; // Wrap in the cors plugin with our dynamic origin resolver diff --git a/graphql/server/src/middleware/error-handler.ts b/graphql/server/src/middleware/error-handler.ts index bbf63de194..a4a17f4853 100644 --- a/graphql/server/src/middleware/error-handler.ts +++ b/graphql/server/src/middleware/error-handler.ts @@ -3,6 +3,7 @@ import './types'; import { getNodeEnv } from '@pgpmjs/env'; import { Logger } from '@pgpmjs/logger'; import type { ErrorRequestHandler, NextFunction, Request, Response } from 'express'; +import { PG_POOL_CAPACITY_ERROR_CODE } from 'pg-cache'; import errorPage50x from '../errors/50x'; import errorPage404Message from '../errors/404-message'; @@ -38,7 +39,18 @@ const isCsrfError = (err: Error): boolean => { return typeof code === 'string' && code.startsWith('CSRF_'); }; +const isPgPoolCapacityError = (err: Error): boolean => + (err as Error & { code?: string }).code === PG_POOL_CAPACITY_ERROR_CODE; + const categorizeError = (err: Error): ErrorResponse => { + if (isPgPoolCapacityError(err)) { + return { + statusCode: 503, + code: PG_POOL_CAPACITY_ERROR_CODE, + message: 'Service temporarily unavailable', + logLevel: 'warn' + }; + } if (isApiError(err)) { return { statusCode: err.statusCode, @@ -61,6 +73,11 @@ const categorizeError = (err: Error): ErrorResponse => { }; const sendResponse = (req: Request, res: Response, { statusCode, code, message }: ErrorResponse): void => { + if (code === PG_POOL_CAPACITY_ERROR_CODE) { + res.set('Retry-After', '15'); + res.status(statusCode).json({ error: { code, message, requestId: req.requestId } }); + return; + } if (wantsJson(req)) { res.status(statusCode).json({ error: { code, message, requestId: req.requestId } }); } else { @@ -79,7 +96,9 @@ const logError = (err: Error, req: Request, level: 'warn' | 'error'): void => { clientIp: req.clientIp, }; - if (isApiError(err)) { + if (isPgPoolCapacityError(err)) { + log.warn({ event: 'pg_pool_capacity', code: PG_POOL_CAPACITY_ERROR_CODE, ...context }); + } else if (isApiError(err)) { log[level]({ event: 'api_error', code: err.code, statusCode: err.statusCode, message: err.message, ...context }); } else { log[level]({ event: 'unexpected_error', name: err.name, message: err.message, stack: isDevelopment() ? err.stack : undefined, ...context }); diff --git a/graphql/server/src/middleware/flush.ts b/graphql/server/src/middleware/flush.ts index 1ff9ee6d3f..20a0530bd3 100644 --- a/graphql/server/src/middleware/flush.ts +++ b/graphql/server/src/middleware/flush.ts @@ -1,70 +1,127 @@ import './types'; // for Request type +import type { LoaderRegistry } from '@constructive-io/express-context'; import { ConstructiveOptions } from '@constructive-io/graphql-types'; import { Logger } from '@pgpmjs/logger'; -import { svcCache } from '@pgpmjs/server-utils'; import { NextFunction, Request, Response } from 'express'; -import { graphileCache } from 'graphile-cache'; -import { getPgPool } from 'pg-cache'; +import { deleteGraphileCacheEntry, graphileCache } from 'graphile-cache'; +import { acquirePgPool } from 'pg-cache'; +import { + invalidateSvcCacheForDatabase, + invalidateSvcCacheForService, + invalidateSvcCacheKey +} from './api'; +import { invalidateInFlightBuilds } from './graphile'; import { getRoutingSchema, isValidSchemaName } from './routing'; const log = new Logger('flush'); -export const flush = async ( +const flushRequest = async ( req: Request, res: Response, - next: NextFunction + next: NextFunction, + registry?: LoaderRegistry ): Promise => { if (req.url === '/flush') { - // TODO: check bearer for a flush / special key - graphileCache.delete((req as any).svc_key); - svcCache.delete((req as any).svc_key); + if (req.internalTrusted !== true) { + res.status(403).send('Forbidden'); + return; + } + const serviceKey = req.svc_key; + // Module metadata and Graphile residents are one publication boundary. + // Retire both before acknowledging the flush; otherwise a revoked module + // configuration can outlive the schema instance it helped configure. + registry?.invalidate(req.databaseId); + if (serviceKey) invalidateInFlightBuilds({ serviceKey }); + if (req.svc_cache_key) invalidateSvcCacheKey(req.svc_cache_key); + const cacheKeys = [...graphileCache.entries()] + .filter(([, entry]) => entry.serviceKey === serviceKey) + .map(([key]) => key); + await Promise.all(cacheKeys.map((key) => deleteGraphileCacheEntry(key))); res.status(200).send('OK'); return; } return next(); }; +export const flush = ( + req: Request, + res: Response, + next: NextFunction +): Promise => flushRequest(req, res, next); + +export const createFlushMiddleware = (registry: LoaderRegistry) => ( + req: Request, + res: Response, + next: NextFunction +): Promise => flushRequest(req, res, next, registry); + export const flushService = async ( opts: ConstructiveOptions, - databaseId: string + databaseId: string, + registry?: LoaderRegistry ): Promise => { - const pgPool = getPgPool(opts.pg); log.info('flushing db ' + databaseId); + registry?.invalidate(databaseId); + invalidateInFlightBuilds({ databaseId }); + invalidateSvcCacheForDatabase(opts, databaseId); const api = new RegExp(`^api:${databaseId}:.*`); const schemata = new RegExp(`^schemata:${databaseId}:.*`); const meta = new RegExp(`^metaschema:api:${databaseId}`); - if (!opts.api.isPublic) { - graphileCache.forEach((_, k: string) => { - if (api.test(k) || schemata.test(k) || meta.test(k)) { - graphileCache.delete(k); - svcCache.delete(k); + // Evict by the authoritative database identity before consulting routing. + // Routing is fallible and may legitimately return no domains; neither case + // may leave a resident instance for the database being flushed. + const databaseCacheKeys = new Set(); + graphileCache.forEach((entry, key: string) => { + if (entry.databaseId === databaseId) { + databaseCacheKeys.add(key); + } + + if (!opts.api.isPublic) { + const serviceKey = entry.serviceKey; + if (serviceKey && (api.test(serviceKey) || schemata.test(serviceKey) || meta.test(serviceKey))) { + invalidateSvcCacheForService(opts, serviceKey); } - }); - } + } + }); + await Promise.all([...databaseCacheKeys].map((key) => deleteGraphileCacheEntry(key))); const routingSchema = getRoutingSchema(opts); if (!isValidSchemaName(routingSchema)) { log.warn(`[flush] invalid routing schema name: ${routingSchema}`); return; } - const svc = await pgPool.query( - `SELECT hostname - FROM "${routingSchema}".domains - WHERE database_id = $1`, - [databaseId] - ); + const poolLease = acquirePgPool(opts.pg, { + purpose: 'routing-request-control', + sanitizeOnCheckout: true + }); + try { + const svc = await poolLease.pool.query( + `SELECT hostname + FROM "${routingSchema}".domains + WHERE database_id = $1`, + [databaseId] + ); - if (svc.rowCount === 0) return; + if (svc.rowCount === 0) return; - for (const row of svc.rows) { - const key: string | undefined = row.hostname || undefined; - if (key) { - graphileCache.delete(key); - svcCache.delete(key); + for (const row of svc.rows) { + const key: string | undefined = row.hostname || undefined; + if (key) { + const graphileKeys = new Set(); + graphileCache.forEach((entry, cacheKey) => { + if (entry.serviceKey === key || entry.databaseId === databaseId) { + graphileKeys.add(cacheKey); + } + }); + await Promise.all([...graphileKeys].map((cacheKey) => deleteGraphileCacheEntry(cacheKey))); + invalidateSvcCacheForService(opts, key); + } } + } finally { + poolLease.release(); } }; diff --git a/graphql/server/src/middleware/graphile.ts b/graphql/server/src/middleware/graphile.ts index e6de98f7ad..c530b41208 100644 --- a/graphql/server/src/middleware/graphile.ts +++ b/graphql/server/src/middleware/graphile.ts @@ -3,29 +3,134 @@ import './types'; // for Request type import crypto from 'node:crypto'; import { classify, type ErrorContext, errors, parse } from '@constructive-io/errors'; -import type { ComputeConfig } from '@constructive-io/express-context'; +import { + buildPgSettings, + type ComputeConfig, + type RuntimePgPoolResolution, + type StorageConfig +} from '@constructive-io/express-context'; import type { ConstructiveOptions } from '@constructive-io/graphql-types'; import { getNodeEnv } from '@pgpmjs/env'; import { Logger } from '@pgpmjs/logger'; import type { NextFunction, Request, RequestHandler, Response } from 'express'; -import type { GraphQLError, GraphQLFormattedError } from 'grafast/graphql'; -import { createGraphileInstance, graphileCache,type GraphileCacheEntry } from 'graphile-cache'; +import { + type BuildRefusalReason, + CacheBuildAdmissionError, + createGraphileInstance, + disposeUncachedEntry, + evaluateBuildAdmission, + graphileCache, + type GraphileCacheEntry, + GraphileRealtimeStartupError, + invokeEntryHandler, + invokeEntryUpgradeHandler, + isEntryRealtimeUnavailable, + prepareCacheForBuild, + recordBuildRefusal, + revalidateEntryRealtimeRole +} from 'graphile-cache'; import type { GraphileConfig } from 'graphile-config'; import { createFunctionBindingsPlugin } from 'graphile-function-bindings'; -import { createConstructivePreset, makePgService } from 'graphile-settings'; -import { getPgPool } from 'pg-cache'; -import { getPgEnvOptions } from 'pg-env'; +import { + ActivatableGenerationScopedRealtimeSubscriber, + RealtimeTopicCollector +} from 'graphile-realtime-subscriptions'; +import { + createConstructivePreset, + createGrafastCacheLimitsPreset, + makePgService, + normalizeIntrospectionDependencySchemas, + resolveConstructiveIntrospectionCapabilityExtensions +} from 'graphile-settings'; +import type { GraphQLError, GraphQLFormattedError } from 'graphql'; +import { + acquirePgPool, + getPgNotificationBrokerIdentity, + getPgPoolIdentity, + PgPoolCapacityError, + type PgPoolLease +} from 'pg-cache'; import { isGraphqlObservabilityEnabled } from '../diagnostics/observability'; import { HandlerCreationError } from '../errors/api-errors'; import { respondWithGraphQLError } from '../errors/graphql-response'; import { AuthCookiePlugin } from '../plugins/auth-cookie-plugin'; +import { + createGraphileWebSocketOperationAdmission, + type GraphileWebSocketOperationAdmission +} from '../plugins/websocket-operation-admission-plugin'; import type { DatabaseSettings } from '../types'; +import { + getGraphileWebSocketUpgradeTransport, + GRAPHILE_WEBSOCKET_AUTH_REJECTED_CODE, + handoffGraphileWebSocketUpgrade, + isGraphileWebSocketOriginAllowed +} from '../websocket-upgrade'; +import { + createGraphileBuildContract, + hashGraphileBuildContract +} from './graphile-build-contract'; +import { + captureGraphileBuildGeneration, + GRAPHILE_BUILD_QUEUE_FULL_CODE, + GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE, + GraphileBuildCoordinatorError, + GraphileBuildWaitAbortedError, + isGraphileBuildGenerationCurrent, + recordCoalescedRequest, + runGraphileBuild, + waitForGraphileBuild +} from './graphile-build-governor'; +import { + assertGraphileCallerPresetsSafe, + composeGraphilePreset +} from './graphile-preset-composition'; +import { getTrustedInternalClaims } from './internal-request'; import { observeGraphileBuild } from './observability/graphile-build-stats'; +import { + addRealtimeRuntimeDependencySchema, + resolveGraphileRealtimeSchema +} from './realtime-config'; +import { + GraphileRealtimeNotificationConfigError, + resolveRealtimeCursorIntervals, + resolveRealtimeNotificationMode, + resolveRealtimeNotificationPgConfig, + resolveRealtimeNotificationRoleRevalidationMs +} from './realtime-notification-config'; +import { + createRuntimePgResolverInput, + resolveRuntimePgConfig +} from './runtime-pg-config'; +import { + assertRuntimePgCredentials, + shouldValidateRuntimeRoleSafety +} from './runtime-pg-requirements'; +import { + ensureRuntimeRoleSafety, + refreshRuntimeRoleSafety +} from './runtime-role-safety'; const maskErrorLog = new Logger('graphile:maskError'); const isDev = (): boolean => getNodeEnv() === 'development'; +const GRAPHILE_SURFACE_FLAGS = Object.freeze({ + graphiql: true, + graphiqlOnGraphQLGET: false +}); + +let nextGraphileConfigurationIdentity = 0; + +/** @internal Resolve the same routed/authenticated request for HTTP and WS. */ +export const getGraphileTransportRequest = ( + requestContext: Partial +): Request | undefined => { + const typedContext = requestContext as { + expressv4?: { req?: Request }; + ws?: { request?: Request }; + }; + return typedContext.expressv4?.req ?? typedContext.ws?.request; +}; /** * GraphQL framework protocol codes. These originate in the GraphQL/grafast @@ -124,7 +229,17 @@ const maskError = (error: GraphQLError): GraphQLError | GraphQLFormattedError => * When multiple concurrent requests arrive for the same cache key, only the * first request creates the handler while others wait on the same promise. */ -const creating = new Map>(); +interface InFlightGraphileBuild { + promise: Promise; + serviceKey: string; + databaseId: string | null; + invalidated: boolean; + admitted: boolean; + waiterCount: number; + abortController: AbortController; +} + +const creating = new Map(); /** * Returns the number of currently in-flight handler creation operations. @@ -146,12 +261,80 @@ export function getInFlightKeys(): string[] { * Clears the in-flight map. Used for testing purposes. */ export function clearInFlightMap(): void { + for (const build of creating.values()) { + if (!build.admitted) build.abortController.abort(); + } creating.clear(); } +export const invalidateInFlightBuilds = (selector: { + serviceKey?: string; + databaseId?: string; +}): number => { + let invalidated = 0; + for (const build of creating.values()) { + if ( + (selector.serviceKey && build.serviceKey === selector.serviceKey) || + (selector.databaseId && build.databaseId === selector.databaseId) + ) { + build.invalidated = true; + invalidated++; + } + } + return invalidated; +}; + const log = new Logger('graphile'); const reqLabel = (req: Request): string => (req.requestId ? `[${req.requestId}]` : '[req]'); +/** + * A consumed IncomingMessage may be auto-destroyed while its keep-alive socket + * remains healthy. Only the transport socket, an explicit abort, or the + * response state tells us that the request can no longer receive a response. + */ +export const isGraphileRequestTerminal = (req: Request, res: Response): boolean => + Boolean( + req.aborted + || req.socket?.destroyed + || res.destroyed + || res.writableEnded + ); + +const createRequestAbortHandle = ( + req: Request, + res: Response +): { signal: AbortSignal; cleanup(): void } => { + const controller = new AbortController(); + const abort = (): void => controller.abort(); + req.once('aborted', abort); + res.once('close', abort); + if (isGraphileRequestTerminal(req, res)) abort(); + return { + signal: controller.signal, + cleanup: () => { + req.removeListener('aborted', abort); + res.removeListener('close', abort); + } + }; +}; + +const waitForInFlightGraphileBuild = async ( + build: InFlightGraphileBuild, + signal: AbortSignal +): Promise => { + build.waiterCount++; + try { + return await waitForGraphileBuild(build.promise, undefined, signal); + } finally { + build.waiterCount = Math.max(0, build.waiterCount - 1); + // Active builds are allowed to finish and become useful residents. Queued + // builds retain no pool lease and are canceled once nobody can consume them. + if (build.waiterCount === 0 && !build.admitted) { + build.abortController.abort(); + } + } +}; + /** * Build a PostGraphile v5 preset for a tenant. * @@ -167,12 +350,53 @@ const buildPreset = ( roleName: string, databaseSettings?: DatabaseSettings, apiId?: string, - compute?: ComputeConfig + compute?: ComputeConfig, + storage?: StorageConfig, + introspectionMode: 'stock' | 'scoped-required' = 'stock', + introspectionClientReleaseMode: 'reuse' | 'destroy' = 'reuse', + introspectionDependencySchemas: readonly string[] = [], + grafastCache: NonNullable['grafastCache'] = {}, + releaseBuildStateAfterValidation = false, + enableWebsockets = false, + sharedRealtime?: { + subscriber: ActivatableGenerationScopedRealtimeSubscriber; + topicCollector: RealtimeTopicCollector; + }, + websocketOperationAdmissionPlugin?: GraphileConfig.Plugin, + callerExtends: readonly GraphileConfig.Preset[] = [], + callerPreset?: Partial, + callerPresetsTrusted = false ): GraphileConfig.Preset => { - return { - extends: [createConstructivePreset(databaseSettings)], - plugins: [ + if (enableWebsockets && !websocketOperationAdmissionPlugin) { + throw new Error( + 'Graphile WebSockets require exact per-operation safety admission' + ); + } + return composeGraphilePreset({ + basePresets: [ + createConstructivePreset({ + ...databaseSettings, + // The server always supplies an authoritative control-plane snapshot. + // Undefined means "module not provisioned", not "query as runtime". + preloadedStorageModules: storage?.modules ?? [], + ...(sharedRealtime ? { + realtimeSubscriptions: { + onTopicsDiscovered: sharedRealtime.topicCollector.collect + } + } : {}) + }) + ], + callerExtends, + callerPreset, + callerPresetsTrusted, + protectedPresets: [ + createGrafastCacheLimitsPreset(grafastCache) + ], + protectedPlugins: [ AuthCookiePlugin, + ...(websocketOperationAdmissionPlugin + ? [websocketOperationAdmissionPlugin] + : []), // Only registered when the compute module is provisioned for this // database — all schema/table names come from the constructive // metaschema (express-context compute module loader); the plugin has @@ -188,6 +412,17 @@ const buildPreset = ( invocationsSchema: m.invocationsSchemaName, invocationsTable: m.invocationsTableName, invocationsEntityField: m.invocationsEntityField + })), + preloadedBindings: compute.bindings.map((binding) => ({ + ...binding, + module: { + computeSchema: binding.module.schemaName, + bindingsTable: binding.module.bindingsTableName, + definitionsTable: binding.module.definitionsTableName, + invocationsSchema: binding.module.invocationsSchemaName, + invocationsTable: binding.module.invocationsTableName, + invocationsEntityField: binding.module.invocationsEntityField + } })) }) ] @@ -196,134 +431,394 @@ const buildPreset = ( pgServices: [ makePgService({ pool, - schemas + schemas, + introspectionMode, + introspectionClientReleaseMode, + introspectionAllowedDependencySchemas: introspectionDependencySchemas, + ...(introspectionMode === 'scoped-required' ? { + introspectionCapabilityExtensions: + resolveConstructiveIntrospectionCapabilityExtensions(databaseSettings) + } : {}), + ...(sharedRealtime ? { + pubsub: false, + pgSubscriber: sharedRealtime.subscriber + } : {}) }) ], + schema: { + releaseBuildStateAfterValidation + }, grafserv: { graphqlPath: '/graphql', graphiqlPath: '/graphiql', - graphiql: true, - graphiqlOnGraphQLGET: false, + ...GRAPHILE_SURFACE_FLAGS, + websockets: enableWebsockets, maskError }, grafast: { - explain: process.env.NODE_ENV === 'development', + explain: isDev(), context: (requestContext: Partial) => { - // In grafserv/express/v4, the request is available at requestContext.expressv4.req - const req = (requestContext as { expressv4?: { req?: Request } })?.expressv4?.req; - const context: Record = {}; + // HTTP carries the Express request directly. WebSocket execution keeps + // the same already-routed/authenticated IncomingMessage under `ws`, so + // both transports derive identical roles, claims, and security GUCs. + const req = getGraphileTransportRequest(requestContext); + const api = req?.api ?? { + dbname: '', + schema: schemas, + anonRole, + roleName + }; + const trustedClaims = getTrustedInternalClaims(req); - if (req) { - if (req.databaseId) { - context['jwt.claims.database_id'] = req.databaseId; - } - // API provenance — which API surface this request arrived through. - // Derived server-side by resolving the hostname through the scoped - // routing plane (resolve_route -> api_id); never taken from - // client-supplied headers, body, or token payload. - if (req.api?.apiId) { - context['jwt.claims.api_id'] = req.api.apiId; - } - if (req.clientIp) { - context['jwt.claims.ip_address'] = req.clientIp; - } - if (req.get('origin')) { - context['jwt.claims.origin'] = req.get('origin') as string; - } - if (req.get('User-Agent')) { - context['jwt.claims.user_agent'] = req.get('User-Agent') as string; - } - if (req.deviceToken) { - context['jwt.claims.device_token'] = req.deviceToken; - } + return { + pgSettings: buildPgSettings({ + api, + token: req?.token ?? null, + requestId: req?.requestId ?? '', + clientIp: req?.clientIp, + origin: req?.get('origin'), + userAgent: req?.get('User-Agent'), + deviceToken: req?.deviceToken, + trustedClaims, + dependencySchemas: introspectionDependencySchemas + }) + }; + } + } + }); +}; - if (req.token?.user_id) { - const pgSettings: Record = { - role: roleName, - 'jwt.claims.token_id': req.token.id, - 'jwt.claims.user_id': req.token.user_id, - ...context - }; +export class GraphileBuildInvalidatedError extends Error { + readonly code = 'GRAPHILE_BUILD_INVALIDATED'; - if (req.token.session_id) { - pgSettings['jwt.claims.session_id'] = req.token.session_id; - } + constructor() { + super('Graphile build was invalidated before it could become resident'); + this.name = 'GraphileBuildInvalidatedError'; + } +} - // Propagate credential metadata as JWT claims so PG functions - // can read them via current_setting('jwt.claims.access_level') etc. - if (req.token.access_level) { - pgSettings['jwt.claims.access_level'] = req.token.access_level; - } - if (req.token.kind) { - pgSettings['jwt.claims.kind'] = req.token.kind; - } +export class GraphileBuildPublicationError extends Error { + readonly code = 'GRAPHILE_BUILD_PUBLICATION_FAILED'; - // Principal identity — always set; equals user_id for human sessions - pgSettings['jwt.claims.principal_id'] = req.token.principal_id || req.token.user_id; + constructor(message: string, readonly cause?: unknown) { + super(message); + this.name = 'GraphileBuildPublicationError'; + } +} - // Enforce read-only transactions for read_only credentials - if (req.token.access_level === 'read_only') { - pgSettings['default_transaction_read_only'] = 'on'; - } +/** @internal Explicit ownership state for the build-to-entry pool lease handoff. */ +export class GraphileBuildPoolLeaseOwner { + private pending: PgPoolLease | undefined; - if (req.requestId) { - pgSettings['request.id'] = req.requestId; - } + constructor(lease: PgPoolLease | undefined) { + this.pending = lease; + } - return { pgSettings }; - } + get lease(): PgPoolLease | undefined { + return this.pending; + } - // Private (in-cluster) surface: there is no token — identity - // arrives on the trusted internal X-* headers stamped by the - // dispatching worker/sync gateway (the same vocabulary as - // X-Database-Id above). Map it into per-request claims so writes - // made through this surface carry actor attribution. Never applied - // on the public surface, where client-supplied identity headers - // must not assert identity. - const headerActorId = req.get('X-Actor-Id'); - if (req.api?.isPublic === false && headerActorId) { - const pgSettings: Record = { - role: roleName, - 'jwt.claims.user_id': headerActorId, - 'jwt.claims.principal_id': headerActorId, - ...context - }; - const headerEntityId = req.get('X-Entity-Id'); - if (headerEntityId) { - pgSettings['jwt.claims.entity_id'] = headerEntityId; - } - const headerOrganizationId = req.get('X-Organization-Id'); - if (headerOrganizationId) { - pgSettings['jwt.claims.organization_id'] = headerOrganizationId; - } - if (req.requestId) { - pgSettings['request.id'] = req.requestId; - } - return { pgSettings }; - } - } + transferTo(entry: GraphileCacheEntry): void { + const expected = this.pending; + if (!expected) { + throw new GraphileBuildPublicationError( + `PostGraphile[${entry.cacheKey}] has no pending pool lease to transfer` + ); + } + if (entry.poolLease !== expected) { + throw new GraphileBuildPublicationError( + `PostGraphile[${entry.cacheKey}] did not retain the build pool lease` + ); + } - const anonSettings: Record = { - role: anonRole, - ...context - }; - if (req?.requestId) { - anonSettings['request.id'] = req.requestId; - } + // The entry now owns the lease even when a later identity assertion fails; + // its disposal path, rather than the build finally block, must release it. + this.pending = undefined; + if (entry.poolIdentity !== expected.identity) { + throw new GraphileBuildPublicationError( + `PostGraphile[${entry.cacheKey}] retained an unexpected pool identity` + ); + } + } - return { - pgSettings: anonSettings - }; - } + release(): void { + const pending = this.pending; + this.pending = undefined; + pending?.release(); + } +} + +interface GraphileBuildPublicationCache { + get(key: string): GraphileCacheEntry | undefined; + set(key: string, entry: GraphileCacheEntry): unknown; + delete(key: string): boolean; +} + +interface GraphileBuildPublicationDependencies { + cache?: GraphileBuildPublicationCache; + dispose?: (entry: GraphileCacheEntry, key: string) => Promise; +} + +/** @internal Publish exactly one authoritative entry or dispose the candidate. */ +export const publishGraphileBuild = async ( + key: string, + candidate: GraphileCacheEntry, + invalidated: boolean, + dependencies: GraphileBuildPublicationDependencies = {} +): Promise => { + const cache = dependencies.cache ?? graphileCache; + const dispose = dependencies.dispose ?? disposeUncachedEntry; + const cleanupCandidate = async (message: string): Promise => { + try { + await dispose(candidate, key); + } catch (cleanupError) { + throw new GraphileBuildPublicationError( + `${message}; candidate disposal also failed`, + cleanupError + ); } }; + const disposeCandidate = async (message: string, cause?: unknown): Promise => { + await cleanupCandidate(message); + throw new GraphileBuildPublicationError(message, cause); + }; + const candidateUnavailable = (): boolean => + candidate.disposing === true || isEntryRealtimeUnavailable(candidate); + const rejectPublishedCandidate = async (message: string): Promise => { + if (cache.get(key) === candidate) cache.delete(key); + return disposeCandidate(message); + }; + + if (invalidated) { + await cleanupCandidate(`PostGraphile[${key}] invalidation disposal failed`); + throw new GraphileBuildInvalidatedError(); + } + if (candidateUnavailable()) { + return disposeCandidate( + `PostGraphile[${key}] became unavailable before publication` + ); + } + + const resident = cache.get(key); + if (resident && resident !== candidate) { + if (resident.disposing) { + return disposeCandidate( + `PostGraphile[${key}] collided with a disposing resident entry` + ); + } + await cleanupCandidate(`PostGraphile[${key}] duplicate disposal failed`); + const authoritative = cache.get(key); + if (authoritative !== resident || resident.disposing) { + throw new GraphileBuildPublicationError( + `PostGraphile[${key}] resident changed while discarding a duplicate build` + ); + } + log.warn(`Discarded duplicate PostGraphile[${key}] build; using the resident entry`); + return resident; + } + if (resident === candidate) { + if (candidateUnavailable()) { + return rejectPublishedCandidate( + `PostGraphile[${key}] resident candidate became unavailable` + ); + } + return candidate; + } + + try { + cache.set(key, candidate); + } catch (error) { + return disposeCandidate(`Failed to publish PostGraphile[${key}]`, error); + } + + const published = cache.get(key); + if (published === candidate) { + if (candidateUnavailable()) { + return rejectPublishedCandidate( + `PostGraphile[${key}] became unavailable during publication` + ); + } + return candidate; + } + if (published && !published.disposing) { + await cleanupCandidate(`PostGraphile[${key}] replaced-candidate disposal failed`); + const authoritative = cache.get(key); + if (authoritative !== published || published.disposing) { + throw new GraphileBuildPublicationError( + `PostGraphile[${key}] resident changed after publication replacement` + ); + } + log.warn(`PostGraphile[${key}] publication was replaced; using the resident entry`); + return published; + } + return disposeCandidate(`PostGraphile[${key}] was not resident after publication`); }; -export const graphile = (opts: ConstructiveOptions): RequestHandler => { +export const GRAPHILE_BUILD_RESIDENT_CAPACITY_CODE = + 'GRAPHILE_BUILD_RESIDENT_CAPACITY'; + +export const BUILD_REFUSAL_CODES = { + critical_pressure: 'GRAPHILE_BUILD_MEMORY_PRESSURE', + insufficient_budget: 'GRAPHILE_BUILD_BUDGET_EXCEEDED', + rss_budget_exceeded: 'GRAPHILE_BUILD_RSS_BUDGET_EXCEEDED', + disposal_timeout: 'GRAPHILE_BUILD_DISPOSAL_TIMEOUT', + resident_busy: 'GRAPHILE_BUILD_CAPACITY_BUSY', + resident_capacity: GRAPHILE_BUILD_RESIDENT_CAPACITY_CODE, + disposal_failed: 'GRAPHILE_BUILD_DISPOSAL_FAILED' +} as const satisfies Record; + +const respondBuildUnavailable = ( + res: Response, + code: string, + message: string, + retryAfterSeconds = 15 +): void => { + if (res.destroyed || res.writableEnded) return; + res.setHeader('Retry-After', String(retryAfterSeconds)); + res.status(503).json({ error: { code, message } }); +}; + +export const handleBuildAvailabilityError = ( + res: Response, + error: unknown +): boolean => { + if (error instanceof PgPoolCapacityError) { + respondBuildUnavailable( + res, + error.code, + 'PostgreSQL connection capacity is temporarily unavailable', + error.retryAfterSeconds + ); + return true; + } + if (error instanceof CacheBuildAdmissionError) { + respondBuildUnavailable( + res, + BUILD_REFUSAL_CODES[error.reason], + 'GraphQL schema capacity is temporarily unavailable', + error.retryAfterSeconds + ); + return true; + } + if (error instanceof GraphileBuildCoordinatorError) { + respondBuildUnavailable( + res, + error.code, + error.code === GRAPHILE_BUILD_QUEUE_FULL_CODE + ? 'GraphQL schema build queue is full; retry shortly' + : error.code === GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE + ? 'GraphQL schema build admission is unhealthy; process restart is required' + : 'GraphQL schema build admission is closed for shutdown', + error.retryAfterSeconds + ); + return true; + } + if (error instanceof GraphileBuildInvalidatedError) { + respondBuildUnavailable( + res, + error.code, + 'GraphQL schema changed while it was building; retry shortly', + 1 + ); + return true; + } + if (error instanceof GraphileBuildPublicationError) { + respondBuildUnavailable( + res, + error.code, + 'GraphQL schema publication failed; retry shortly', + 1 + ); + return true; + } + if (error instanceof GraphileRealtimeNotificationConfigError) { + respondBuildUnavailable( + res, + error.code, + 'Shared realtime notification configuration is unavailable' + ); + return true; + } + if (error instanceof GraphileRealtimeStartupError) { + respondBuildUnavailable( + res, + error.code, + 'Realtime delivery could not be activated for this GraphQL instance' + ); + return true; + } + return false; +}; + +const handleBuildWaitAbort = ( + res: Response, + error: unknown, + requestSignal: AbortSignal +): boolean => { + if (!(error instanceof GraphileBuildWaitAbortedError)) return false; + if (!requestSignal.aborted) { + respondBuildUnavailable( + res, + 'GRAPHILE_BUILD_CANCELED', + 'GraphQL schema build was canceled before admission; retry shortly', + 1 + ); + } + return true; +}; + +export const graphile = ( + opts: ConstructiveOptions, + getRuntimePgResolution?: ( + req: Request + ) => Readonly +): RequestHandler => { + // The resident cache is process-wide, but caller presets may contain hooks + // whose captured state cannot be serialized. Never share a generation + // across two independently constructed server configurations, even when all + // visible data fields happen to compare equal. + const configurationIdentity = + `graphile-configuration:v1:${++nextGraphileConfigurationIdentity}`; + const callerPresetsTrusted = + getNodeEnv() !== 'production' + || opts.graphile?.trustCallerPresetsInProduction === true; + assertRuntimePgCredentials(opts, getNodeEnv()); + assertGraphileCallerPresetsSafe({ + callerExtends: opts.graphile?.extends, + callerPreset: opts.graphile?.preset, + callerPresetsTrusted + }); + const introspectionDependencySchemas = normalizeIntrospectionDependencySchemas( + opts.graphile?.introspectionDependencySchemas + ); const observabilityEnabled = isGraphqlObservabilityEnabled(opts.server?.host); + const runtimeSafetyRequired = shouldValidateRuntimeRoleSafety(opts, getNodeEnv()); + const realtimeNotificationMode = resolveRealtimeNotificationMode(opts); + const realtimeNotificationRoleRevalidationMs = + resolveRealtimeNotificationRoleRevalidationMs(opts); + const realtimeCursorIntervals = resolveRealtimeCursorIntervals(opts); return async (req: Request, res: Response, next: NextFunction) => { const label = reqLabel(req); + const requestAbort = createRequestAbortHandle(req, res); + const websocketUpgrade = getGraphileWebSocketUpgradeTransport(req); + const invokeEntry = (entry: GraphileCacheEntry): boolean => { + if (!websocketUpgrade) return invokeEntryHandler(entry, req, res, next); + return invokeEntryUpgradeHandler( + entry, + req, + websocketUpgrade.socket, + websocketUpgrade.head, + { + onAccepted: () => { + handoffGraphileWebSocketUpgrade(req, res); + }, + onRejected: () => { + handoffGraphileWebSocketUpgrade(req, res); + } + } + ); + }; try { const api = req.api; if (!api) { @@ -331,8 +826,20 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { respondWithGraphQLError(res, errors.INTERNAL_FAILURE({ details: 'Missing API info' })); return; } - const key = req.svc_key; - if (!key) { + if ( + websocketUpgrade + && !isGraphileWebSocketOriginAllowed(req, opts.server?.origin) + ) { + res.status(403).json({ + error: { + code: GRAPHILE_WEBSOCKET_AUTH_REJECTED_CODE, + message: 'WebSocket origin is not allowed' + } + }); + return; + } + const serviceKey = req.svc_key; + if (!serviceKey) { log.error(`${label} Missing service cache key`); respondWithGraphQLError( res, @@ -342,89 +849,361 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { } const { dbname, anonRole, roleName, schema } = api; const schemaLabel = schema?.join(',') || 'unknown'; + const poolOptions = { purpose: 'runtime', sanitizeOnCheckout: true } as const; + const runtimePgResolution = getRuntimePgResolution + ? getRuntimePgResolution(req) + : await resolveRuntimePgConfig( + opts, + createRuntimePgResolverInput(api) + ); + const pgConfig = runtimePgResolution.pgConfig; + const poolIdentity = runtimePgResolution.poolIdentity; + if (getPgPoolIdentity(pgConfig, poolOptions) !== poolIdentity) { + throw new Error( + 'Resolved runtime PostgreSQL pool identity changed before Graphile acquisition' + ); + } + if ( + req.constructive + && req.constructive.runtimePoolIdentity !== poolIdentity + ) { + throw new Error( + 'Request context and Graphile resolved different runtime PostgreSQL pools' + ); + } + const [compute, storage] = await Promise.all([ + api.apiId ? req.constructive?.useModule('compute') : undefined, + (api.databaseSettings?.enablePresignedUploads ?? true) + ? req.constructive?.useModule('storage') + : undefined + ]); + const introspectionMode = opts.graphile?.introspectionMode ?? 'stock'; + const introspectionClientReleaseMode = + opts.graphile?.introspectionClientReleaseMode ?? 'reuse'; + const realtimeEnabled = api.databaseSettings?.enableRealtime ?? false; + const realtimeSchema = resolveGraphileRealtimeSchema(opts, realtimeEnabled); + const notificationPgConfig = realtimeEnabled + && realtimeNotificationMode === 'shared-exact' + ? await resolveRealtimeNotificationPgConfig(opts, { + databaseId: api.databaseId ?? '', + databaseName: dbname, + apiId: api.apiId ?? '', + schemas: schema ?? [] + }) + : null; + const realtimeListenerPoolIdentity = notificationPgConfig + ? getPgNotificationBrokerIdentity(notificationPgConfig) + : null; + const runtimeDependencySchemas = addRealtimeRuntimeDependencySchema( + introspectionDependencySchemas, + realtimeSchema + ); + const buildContract = createGraphileBuildContract({ + configurationIdentity, + poolIdentity, + databaseId: api.databaseId ?? '', + databaseName: dbname, + apiId: api.apiId ?? '', + schemas: schema ?? [], + authenticatedRole: roleName, + anonymousRole: anonRole, + pluginSettings: api.databaseSettings, + graphileSettings: opts.graphile, + compute, + storage, + isPublic: api.isPublic, + enableRealtime: realtimeEnabled, + realtimeSchema: realtimeSchema ?? undefined, + realtimeNotificationMode, + realtimeListenerPoolIdentity: realtimeListenerPoolIdentity ?? undefined, + realtimeNotificationRoleRevalidationMs, + realtimeCursorPollIntervalMs: realtimeCursorIntervals.pollIntervalMs, + realtimeCursorHeartbeatIntervalMs: realtimeCursorIntervals.heartbeatIntervalMs, + ...GRAPHILE_SURFACE_FLAGS, + explain: isDev(), + introspectionMode, + introspectionClientReleaseMode + }); + const key = hashGraphileBuildContract(buildContract); + const ensureRuntimeSafety = async (): Promise => { + // Hold an operation-scoped lease even when a resident entry already + // owns this pool. The entry may be evicted while the async audit runs; + // this lease prevents pool teardown until the audit has settled. + const auditLease = acquirePgPool(pgConfig, poolOptions); + try { + await ensureRuntimeRoleSafety( + auditLease.pool, + [anonRole, roleName], + schema ?? [], + runtimeDependencySchemas + ); + } finally { + auditLease.release(); + } + }; - // ========================================================================= - // Phase A: Cache Check (fast path) - // ========================================================================= const cached = graphileCache.get(key); if (cached) { - log.debug(`${label} PostGraphile cache hit key=${key} db=${dbname} schemas=${schemaLabel}`); - return cached.handler(req, res, next); + // A role or schema can drift after the instance was built. Re-enter + // the fail-closed audit on every resident path; the audit itself + // coalesces requests and reuses only recent successful results. + if (runtimeSafetyRequired) await ensureRuntimeSafety(); + await revalidateEntryRealtimeRole(cached); + if (invokeEntry(cached)) { + log.debug(`${label} PostGraphile cache hit key=${key} route=${serviceKey} db=${dbname} schemas=${schemaLabel}`); + return; + } + if (isGraphileRequestTerminal(req, res)) return; } - log.debug(`${label} PostGraphile cache miss key=${key} db=${dbname} schemas=${schemaLabel}`); + log.debug(`${label} PostGraphile cache miss key=${key} route=${serviceKey} db=${dbname} schemas=${schemaLabel}`); + if (isGraphileRequestTerminal(req, res)) return; - // ========================================================================= - // Phase B: In-Flight Check (single-flight coalescing) - // ========================================================================= const inFlight = creating.get(key); if (inFlight) { + recordCoalescedRequest(); log.debug(`${label} Coalescing request for PostGraphile[${key}] - waiting for in-flight creation`); try { - const instance = await inFlight; - return instance.handler(req, res, next); + const instance = await waitForInFlightGraphileBuild(inFlight, requestAbort.signal); + if (!instance) { + respondBuildUnavailable( + res, + 'GRAPHILE_BUILD_WAIT_TIMEOUT', + 'GraphQL schema build is still in progress' + ); + return; + } + if (runtimeSafetyRequired) await ensureRuntimeSafety(); + await revalidateEntryRealtimeRole(instance); + if (invokeEntry(instance)) return; + respondBuildUnavailable( + res, + 'GRAPHILE_INSTANCE_ROTATING', + 'GraphQL schema instance is rotating', + 1 + ); + return; } catch (error) { - log.warn(`${label} Coalesced request failed for PostGraphile[${key}], retrying`); - // Fall through to Phase C to retry creation + if (handleBuildWaitAbort(res, error, requestAbort.signal)) return; + if (handleBuildAvailabilityError(res, error)) return; + throw error; } } - // ========================================================================= - // Phase C: Create New Handler (first request for this key) - // ========================================================================= - - // Re-check cache after coalesced request failure (another retry may have succeeded) - const recheckedCache = graphileCache.get(key); - if (recheckedCache) { - log.debug(`${label} PostGraphile cache hit on re-check key=${key}`); - return recheckedCache.handler(req, res, next); + const earlyDecision = evaluateBuildAdmission(); + if (!earlyDecision.admit && earlyDecision.reason === 'critical_pressure') { + recordBuildRefusal(earlyDecision.reason); + respondBuildUnavailable( + res, + 'GRAPHILE_BUILD_MEMORY_PRESSURE', + 'Server memory pressure is too high to start a new GraphQL schema build' + ); + return; } - - // Re-check in-flight map (another retry may have started creation) - const retryInFlight = creating.get(key); - if (retryInFlight) { - log.debug(`${label} Re-coalescing request for PostGraphile[${key}]`); - const retryInstance = await retryInFlight; - return retryInstance.handler(req, res, next); + if (!earlyDecision.admit && earlyDecision.reason === 'resident_capacity') { + recordBuildRefusal(earlyDecision.reason); + handleBuildAvailabilityError(res, new CacheBuildAdmissionError( + earlyDecision.reason + )); + return; } log.info( - `${label} Building PostGraphile v5 handler key=${key} db=${dbname} schemas=${schemaLabel} role=${roleName} anon=${anonRole}` + `${label} Building PostGraphile v5 handler key=${key} route=${serviceKey} db=${dbname} schemas=${schemaLabel} role=${roleName} anon=${anonRole}` ); - const pgConfig = getPgEnvOptions({ - ...opts.pg, - database: dbname - }); + const buildGeneration = captureGraphileBuildGeneration(); + const buildState: InFlightGraphileBuild = { + promise: null as unknown as Promise, + serviceKey, + databaseId: api.databaseId ?? null, + invalidated: false, + admitted: false, + waiterCount: 0, + abortController: new AbortController() + }; + const creationPromise = runGraphileBuild(async () => { + let poolLeaseOwner: GraphileBuildPoolLeaseOwner | undefined; + let sharedRealtimeBuild: { + subscriber: ActivatableGenerationScopedRealtimeSubscriber; + topicCollector: RealtimeTopicCollector; + } | undefined; + let sharedRealtimeOwnershipTransferred = false; + let websocketOperationAdmission: + GraphileWebSocketOperationAdmission | undefined; + try { + const builtWhileQueued = graphileCache.get(key); + if (builtWhileQueued && !builtWhileQueued.disposing) return builtWhileQueued; - // Route through pg-cache so the pool is tracked and can be cleaned up - // properly, preventing leaked connections during database teardown. - const pool = getPgPool(pgConfig); + await prepareCacheForBuild(); + if ( + buildState.invalidated + || !isGraphileBuildGenerationCurrent(buildGeneration) + ) { + throw new GraphileBuildInvalidatedError(); + } - // Create promise and store in in-flight map BEFORE try block - const compute = api.apiId ? await req.constructive?.useModule('compute') : undefined; - const preset = buildPreset(pool, schema || [], anonRole, roleName, api.databaseSettings, api.apiId, compute); - const creationPromise = observeGraphileBuild( - { - cacheKey: key, - serviceKey: key, - databaseId: api.databaseId ?? null - }, - () => createGraphileInstance({ - preset, - cacheKey: key, - enableRealtime: api.databaseSettings?.enableRealtime - }), - { enabled: observabilityEnabled } - ); - creating.set(key, creationPromise); + // A queued build retains only immutable contract inputs. The large + // preset and runtime-pool lease are acquired after the serialized + // heap slot is granted and are owned until publication or failure. + const buildPoolLease = acquirePgPool(pgConfig, poolOptions); + poolLeaseOwner = new GraphileBuildPoolLeaseOwner(buildPoolLease); + const pool = buildPoolLease.pool; + if (notificationPgConfig) { + sharedRealtimeBuild = { + subscriber: new ActivatableGenerationScopedRealtimeSubscriber(), + topicCollector: new RealtimeTopicCollector() + }; + } + if (realtimeEnabled) { + websocketOperationAdmission = createGraphileWebSocketOperationAdmission({ + cacheKey: key, + databaseId: api.databaseId ?? '', + databaseName: dbname, + apiId: api.apiId ?? '', + schemas: schema ?? [], + authenticatedRole: roleName, + anonymousRole: anonRole, + dependencySchemas: runtimeDependencySchemas, + runtimeSafetyRequired + }); + } + const preset = buildPreset( + pool, + schema || [], + anonRole, + roleName, + api.databaseSettings, + api.apiId, + compute, + storage, + introspectionMode, + introspectionClientReleaseMode, + introspectionDependencySchemas, + opts.graphile?.grafastCache, + opts.graphile?.releaseBuildStateAfterValidation ?? false, + realtimeEnabled, + sharedRealtimeBuild, + websocketOperationAdmission?.plugin, + opts.graphile?.extends, + opts.graphile?.preset, + callerPresetsTrusted + ); + + const instance = await observeGraphileBuild( + { + cacheKey: key, + serviceKey, + databaseId: api.databaseId ?? null + }, + async () => { + if (runtimeSafetyRequired) { + await refreshRuntimeRoleSafety( + pool, + [anonRole, roleName], + schema ?? [], + runtimeDependencySchemas + ); + } + const built = await createGraphileInstance({ + preset, + cacheKey: key, + poolIdentity, + poolLease: poolLeaseOwner!.lease, + serviceKey, + databaseId: api.databaseId ?? null, + enableRealtime: realtimeEnabled, + enableWebsockets: realtimeEnabled, + realtimeSchema: realtimeSchema ?? undefined, + realtimeSourceSchemas: schema ?? [], + realtimeCursorPollIntervalMs: realtimeCursorIntervals.pollIntervalMs, + realtimeCursorHeartbeatIntervalMs: + realtimeCursorIntervals.heartbeatIntervalMs, + ...(notificationPgConfig && sharedRealtimeBuild + && realtimeListenerPoolIdentity ? { + sharedRealtime: { + ...sharedRealtimeBuild, + listenerPgConfig: notificationPgConfig, + listenerIdentity: realtimeListenerPoolIdentity, + roleRevalidationMs: + realtimeNotificationRoleRevalidationMs + } + } : {}) + }); + sharedRealtimeOwnershipTransferred = Boolean(sharedRealtimeBuild); + try { + websocketOperationAdmission?.bind(built); + poolLeaseOwner!.transferTo(built); + } catch (transferError) { + try { + await disposeUncachedEntry(built, key); + } catch (cleanupError) { + throw new GraphileBuildPublicationError( + `PostGraphile[${key}] lease-transfer cleanup failed`, + cleanupError + ); + } + throw transferError; + } + return built; + }, + { enabled: observabilityEnabled } + ); + return publishGraphileBuild( + key, + instance, + buildState.invalidated || !isGraphileBuildGenerationCurrent(buildGeneration) + ); + } finally { + // Covers queued-cache hits, admission/safety failures, and rejected + // instance creation. Entry-owned leases were cleared above. + poolLeaseOwner?.release(); + if (sharedRealtimeBuild && !sharedRealtimeOwnershipTransferred) { + await sharedRealtimeBuild.subscriber.release(); + } + } + }, { + signal: buildState.abortController.signal, + onAdmitted: () => { + buildState.admitted = true; + } + }); + buildState.promise = creationPromise; + creating.set(key, buildState); + + void creationPromise + .then(() => log.info(`${label} PostGraphile v5 handler ready key=${key} db=${dbname}`)) + .catch(() => { + // The request path records the concrete failure. Detached builds may + // finish after a waiter timed out; their rejection is intentionally consumed. + }) + .finally(() => { + if (creating.get(key) === buildState) creating.delete(key); + }); try { - const instance = await creationPromise; - graphileCache.set(key, instance); - log.info(`${label} Cached PostGraphile v5 handler key=${key} db=${dbname}`); - return instance.handler(req, res, next); + const instance = await waitForInFlightGraphileBuild(buildState, requestAbort.signal); + if (!instance) { + respondBuildUnavailable( + res, + 'GRAPHILE_BUILD_WAIT_TIMEOUT', + 'GraphQL schema build is still in progress' + ); + return; + } + if (runtimeSafetyRequired) await ensureRuntimeSafety(); + await revalidateEntryRealtimeRole(instance); + if (invokeEntry(instance)) return; + respondBuildUnavailable( + res, + 'GRAPHILE_INSTANCE_ROTATING', + 'GraphQL schema instance is rotating', + 1 + ); + return; } catch (error) { + if (handleBuildWaitAbort(res, error, requestAbort.signal)) return; + if (handleBuildAvailabilityError(res, error)) return; log.error(`${label} Failed to create PostGraphile[${key}]:`, error); throw new HandlerCreationError( `Failed to create handler for ${key}: ${error instanceof Error ? error.message : String(error)}`, @@ -433,11 +1212,11 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { cause: error instanceof Error ? error.message : String(error) } ); - } finally { - // Always clean up in-flight tracker - creating.delete(key); } } catch (e: any) { + if (isGraphileRequestTerminal(req, res)) return; + if (handleBuildWaitAbort(res, e, requestAbort.signal)) return; + if (!res.headersSent && handleBuildAvailabilityError(res, e)) return; log.error(`${label} PostGraphile middleware error`, e); if (!res.headersSent) { respondWithGraphQLError( @@ -449,6 +1228,8 @@ export const graphile = (opts: ConstructiveOptions): RequestHandler => { return; } next(e); + } finally { + requestAbort.cleanup(); } }; }; diff --git a/graphql/server/src/middleware/internal-request.ts b/graphql/server/src/middleware/internal-request.ts new file mode 100644 index 0000000000..56ba932882 --- /dev/null +++ b/graphql/server/src/middleware/internal-request.ts @@ -0,0 +1,156 @@ +import { timingSafeEqual } from 'node:crypto'; + +import type { SecurityGucKey } from '@constructive-io/express-context'; +import type { Request } from 'express'; + +import type { ApiOptions } from '../types'; + +export const INTERNAL_REQUEST_TOKEN_HEADER = 'X-Constructive-Internal-Token'; +export const MIN_INTERNAL_REQUEST_SECRET_BYTES = 32; + +const PRIVATE_ROUTING_HEADERS = [ + 'X-Api-Name', + 'X-Schemata', + 'X-Meta-Schema', + 'X-Database-Id' +] as const; + +const PRIVATE_IDENTITY_HEADERS = [ + 'X-Actor-Id', + 'X-Entity-Id', + 'X-Organization-Id' +] as const; + +const hasHeader = (req: Request, name: string): boolean => + req.get(name) !== undefined; + +const hasAnyHeader = (req: Request, names: readonly string[]): boolean => + names.some((name) => hasHeader(req, name)); + +const hasBlankHeader = (req: Request, names: readonly string[]): boolean => + names.some((name) => { + const value = req.get(name); + return value !== undefined && value.trim().length === 0; + }); + +const secretIsWellFormed = (secret: string | undefined): secret is string => + typeof secret === 'string' + && Buffer.byteLength(secret) >= MIN_INTERNAL_REQUEST_SECRET_BYTES; + +const secretsEqual = (expected: string, actual: string): boolean => { + const expectedBytes = Buffer.from(expected); + const actualBytes = Buffer.from(actual); + return expectedBytes.length === actualBytes.length + && timingSafeEqual(expectedBytes, actualBytes); +}; + +const forbidden = (message: string): Error & { code: string } => + Object.assign(new Error(message), { code: 'INTERNAL_REQUEST_FORBIDDEN' }); + +/** + * Reject a configured internal secret at startup when it cannot provide a + * meaningful bearer-token boundary. Omitting the secret is allowed, but then + * every reserved internal header and the HTTP cache flush endpoint fail closed. + */ +export const assertInternalRequestSecret = (opts: ApiOptions): void => { + const secret = opts.api?.internalRequestSecret; + if (secret !== undefined && !secretIsWellFormed(secret)) { + throw new Error( + `api.internalRequestSecret must contain at least ${MIN_INTERNAL_REQUEST_SECRET_BYTES} bytes` + ); + } +}; + +/** + * Authenticate reserved ingress headers before they can influence routing or + * database claims. The raw X-Schemata selector is deliberately prohibited: an + * authenticated proxy must select an authoritative API record by name instead + * of supplying an unchecked physical schema list. + */ +export const authorizeInternalRequest = ( + opts: ApiOptions, + req: Request +): void => { + req.internalTrusted = false; + + const hasRoutingHeaders = hasAnyHeader(req, PRIVATE_ROUTING_HEADERS); + const hasIdentityHeaders = hasAnyHeader(req, PRIVATE_IDENTITY_HEADERS); + const presentedSecret = req.get(INTERNAL_REQUEST_TOKEN_HEADER); + const hasInternalCredential = presentedSecret !== undefined; + + if (!hasRoutingHeaders && !hasIdentityHeaders && !hasInternalCredential) { + return; + } + + const configuredSecret = opts.api?.internalRequestSecret; + if ( + !secretIsWellFormed(configuredSecret) + || !presentedSecret + || !secretsEqual(configuredSecret, presentedSecret) + ) { + throw forbidden('Reserved internal request headers require authentication.'); + } + + // Internal route and actor selectors have no meaning on the public ingress. + // The token by itself remains valid there so operators can authenticate the + // cache-administration endpoint for an already-authoritatively-routed host. + if ((hasRoutingHeaders || hasIdentityHeaders) && opts.api?.isPublic !== false) { + throw forbidden('Private routing and identity headers are disabled on the public ingress.'); + } + + if (hasHeader(req, 'X-Schemata')) { + throw forbidden( + 'X-Schemata is not a production-safe routing contract; use X-Api-Name with X-Database-Id.' + ); + } + + if (hasBlankHeader(req, [...PRIVATE_ROUTING_HEADERS, ...PRIVATE_IDENTITY_HEADERS])) { + throw forbidden('Reserved internal request headers must not be empty.'); + } + + const hasApiName = hasHeader(req, 'X-Api-Name'); + const hasMetaSchema = hasHeader(req, 'X-Meta-Schema'); + const hasDatabaseId = hasHeader(req, 'X-Database-Id'); + if (hasApiName && hasMetaSchema) { + throw forbidden('Private requests must select exactly one API surface.'); + } + if (hasDatabaseId !== (hasApiName || hasMetaSchema)) { + throw forbidden( + 'X-Database-Id must be paired with exactly one private API selector.' + ); + } + if (hasMetaSchema && opts.api?.allowMetaSchemaHeader !== true) { + throw forbidden( + 'The privileged metadata API is disabled on this ingress.' + ); + } + + req.internalTrusted = true; +}; + +/** Translate authenticated private-ingress identity headers only. */ +export const getTrustedInternalClaims = ( + req: Request | undefined +): Partial> => { + if ( + !req + || req.api?.isPublic !== false + || req.internalTrusted !== true + || req.token?.user_id + ) { + return {}; + } + + const actorId = req.get('X-Actor-Id'); + if (!actorId) return {}; + + const claims: Partial> = { + 'jwt.claims.user_id': actorId, + 'jwt.claims.principal_id': actorId + }; + const entityId = req.get('X-Entity-Id'); + const organizationId = req.get('X-Organization-Id'); + if (entityId) claims['jwt.claims.entity_id'] = entityId; + if (organizationId) claims['jwt.claims.organization_id'] = organizationId; + return claims; +}; diff --git a/graphql/server/src/middleware/observability/__tests__/guard.test.ts b/graphql/server/src/middleware/observability/__tests__/guard.test.ts index 6473519968..e09d887bf7 100644 --- a/graphql/server/src/middleware/observability/__tests__/guard.test.ts +++ b/graphql/server/src/middleware/observability/__tests__/guard.test.ts @@ -2,12 +2,19 @@ import type { NextFunction, Request, Response } from 'express'; import { localObservabilityOnly } from '../guard'; -function makeReq(input: { remoteAddress?: string | null; host?: string } = {}): Request { +function makeReq(input: { + remoteAddress?: string | null; + host?: string; + authorization?: string; +} = {}): Request { return { socket: { remoteAddress: input.remoteAddress ?? '::1', }, - headers: input.host ? { host: input.host } : {}, + headers: { + ...(input.host ? { host: input.host } : {}), + ...(input.authorization ? { authorization: input.authorization } : {}) + }, } as unknown as Request; } @@ -23,6 +30,17 @@ function makeNext(): NextFunction { } describe('localObservabilityOnly', () => { + const originalEnv = { ...process.env }; + + beforeEach(() => { + process.env.NODE_ENV = 'development'; + delete process.env.GRAPHQL_OBSERVABILITY_TOKEN; + }); + + afterEach(() => { + process.env = { ...originalEnv }; + }); + it('allows loopback requests', () => { const req = makeReq({ remoteAddress: '::ffff:127.0.0.1', host: 'localhost:3000' }); const res = makeRes(); @@ -45,4 +63,35 @@ describe('localObservabilityOnly', () => { expect(res.status).toHaveBeenCalledWith(404); expect(res.send).toHaveBeenCalledWith('Not found'); }); + + it('requires the configured bearer token for production loopback requests', () => { + process.env.NODE_ENV = 'production'; + process.env.GRAPHQL_OBSERVABILITY_TOKEN = 'c'.repeat(64); + const missing = makeReq({ remoteAddress: '127.0.0.1' }); + const wrong = makeReq({ + remoteAddress: '127.0.0.1', + authorization: `Bearer ${'d'.repeat(64)}` + }); + const valid = makeReq({ + remoteAddress: '127.0.0.1', + authorization: `Bearer ${'c'.repeat(64)}` + }); + const missingRes = makeRes(); + const wrongRes = makeRes(); + const validRes = makeRes(); + const missingNext = makeNext(); + const wrongNext = makeNext(); + const validNext = makeNext(); + + localObservabilityOnly(missing, missingRes, missingNext); + localObservabilityOnly(wrong, wrongRes, wrongNext); + localObservabilityOnly(valid, validRes, validNext); + + expect(missingNext).not.toHaveBeenCalled(); + expect(wrongNext).not.toHaveBeenCalled(); + expect(missingRes.status).toHaveBeenCalledWith(404); + expect(wrongRes.status).toHaveBeenCalledWith(404); + expect(validNext).toHaveBeenCalledTimes(1); + expect(validRes.status).not.toHaveBeenCalled(); + }); }); diff --git a/graphql/server/src/middleware/observability/guard.ts b/graphql/server/src/middleware/observability/guard.ts index 6790e15e15..f68c6c5c0a 100644 --- a/graphql/server/src/middleware/observability/guard.ts +++ b/graphql/server/src/middleware/observability/guard.ts @@ -1,16 +1,26 @@ import type { RequestHandler } from 'express'; -import { isLoopbackAddress, isLoopbackHost } from '../../diagnostics/observability'; +import { + isDevelopmentObservabilityMode, + isGraphqlObservabilityTokenValid, + isLoopbackAddress, + isLoopbackHost +} from '../../diagnostics/observability'; + +const bearerToken = (authorization: string | undefined): string | null => { + const match = /^Bearer\s+(.+)$/i.exec(authorization?.trim() ?? ''); + return match?.[1] ?? null; +}; export const localObservabilityOnly: RequestHandler = (req, res, next) => { const remoteAddress = req.socket.remoteAddress; - if (isLoopbackAddress(remoteAddress)) { - next(); - return; - } - const hostHeader = req.headers.host; - if (!remoteAddress && isLoopbackHost(hostHeader)) { + const isLocal = isLoopbackAddress(remoteAddress) + || (!remoteAddress && isLoopbackHost(hostHeader)); + const isAuthorized = isDevelopmentObservabilityMode() + || isGraphqlObservabilityTokenValid(bearerToken(req.headers.authorization)); + + if (isLocal && isAuthorized) { next(); return; } diff --git a/graphql/server/src/middleware/routing.ts b/graphql/server/src/middleware/routing.ts index 752b32bdff..6dc2db7474 100644 --- a/graphql/server/src/middleware/routing.ts +++ b/graphql/server/src/middleware/routing.ts @@ -53,6 +53,17 @@ export const getRoutingSchema = (opts: { export const isValidSchemaName = (name: string): boolean => /^[a-z_][a-z0-9_]*$/.test(name); +/** + * Constructive physical schemas may contain the generated dash separators used + * by tenant prefixes. They are always passed as data or quoted identifiers, but + * keep the accepted alphabet deliberately narrow and reject system namespaces. + */ +export const isValidPhysicalSchemaName = (name: string): boolean => + /^[a-z_][a-z0-9_-]*$/.test(name) + && name.length <= 63 + && name !== 'information_schema' + && !name.startsWith('pg_'); + /** * Resolve a hostname through the compiled scoped-routing plane (host-only: * path/method routing belongs to Traefik/Ingress, not the server). @@ -75,6 +86,13 @@ export const resolveRoute = async ( `SELECT * FROM "${schema}".${RESOLVER_FUNCTION}($1, '/', NULL)`, [host] ); + if (result.rows.length !== 1) { + log.warn( + `[resolve-route] expected exactly one resolver row for host=${host}; ` + + `received ${result.rows.length}` + ); + return null; + } const row = result.rows[0]; if (!row || row.route_binding_id === null) { log.debug(`[resolve-route] no match for host=${host}`); @@ -121,21 +139,47 @@ export const routeToApiStructure = ( } const config = (route.resolved_config ?? {}) as ApiSurfaceConfig; - if (!config.schemas?.length) { - log.debug('[resolve-route] api target missing schemas in resolved_config; no match'); + const expectedPublic = opts.api?.isPublic ?? false; + if (typeof config.is_public !== 'boolean' || config.is_public !== expectedPublic) { + log.warn('[resolve-route] api visibility does not match this server ingress; no match'); + return null; + } + + if ( + !config.api_id + || !config.database_id + || route.target_source_id !== config.api_id + || route.target_owner_scope !== 'database' + || route.target_owner_key !== config.database_id + ) { + log.warn('[resolve-route] api target missing exact api/database identity; no match'); + return null; + } + + if ( + !config.schemas?.length + || config.schemas.some((schema) => !isValidPhysicalSchemaName(schema)) + || new Set(config.schemas).size !== config.schemas.length + ) { + log.warn('[resolve-route] api target has an invalid physical schema contract; no match'); + return null; + } + + if (!config.role_name || !config.anon_role) { + log.warn('[resolve-route] api target missing exact request roles; no match'); return null; } return { - apiId: config.api_id ?? route.target_source_id ?? undefined, + apiId: config.api_id, // Scoped APIs leave dbname NULL when their schemas live in the serving // database; fall back to the server's own database in that case. dbname: config.dbname || opts.pg?.database || '', - anonRole: config.anon_role || 'anon', - roleName: config.role_name || 'authenticated', + anonRole: config.anon_role, + roleName: config.role_name, schema: config.schemas, domains: [], databaseId: config.database_id, - isPublic: config.is_public ?? (opts.api?.isPublic ?? false) + isPublic: config.is_public }; }; diff --git a/graphql/server/src/middleware/runtime-pg-config.ts b/graphql/server/src/middleware/runtime-pg-config.ts new file mode 100644 index 0000000000..a2da4c251e --- /dev/null +++ b/graphql/server/src/middleware/runtime-pg-config.ts @@ -0,0 +1,548 @@ +import type { + RuntimePgPoolResolution +} from '@constructive-io/express-context'; +import type { + ConstructiveOptions, + RuntimePgConfig, + RuntimePgResolverInput +} from '@constructive-io/graphql-types'; +import { getNodeEnv } from '@pgpmjs/env'; +import type { NextFunction, Request, RequestHandler, Response } from 'express'; +import { getPgPoolIdentity } from 'pg-cache'; +import { getPgEnvOptions } from 'pg-env'; + +import type { ApiStructure } from '../types'; +import { + assertRuntimePgCredentials, + InvalidRuntimePgConfigurationError, + requiresExactRuntimePgResolution +} from './runtime-pg-requirements'; + +const RUNTIME_POOL_OPTIONS = { + purpose: 'runtime', + sanitizeOnCheckout: true +} as const; + +const TARGET_ATTESTATION_POOL = Object.freeze({ + max: 1, + maxUses: 1, + idleTimeoutMillis: 0, + connectionTimeoutMillis: 0, + allowExitOnIdle: true +}); + +const TARGET_ATTESTATION_OPTIONS = { + purpose: 'runtime-target-attestation', + sanitizeOnCheckout: true +} as const; + +const CONFIG_KEYS = new Set([ + 'host', + 'port', + 'user', + 'password', + 'database', + 'ssl', + 'pool' +]); + +const POOL_KEYS = new Set([ + 'max', + 'maxUses', + 'idleTimeoutMillis', + 'connectionTimeoutMillis', + 'allowExitOnIdle' +]); + +const STATIC_IDENTITY_KEYS = new Set([ + 'databaseId', + 'databaseName', + 'apiId', + 'schemas', + 'roles' +]); + +const ownDataRecord = ( + value: unknown, + label: string, + allowedKeys: ReadonlySet +): Record => { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new InvalidRuntimePgConfigurationError( + `${label} must be a PostgreSQL configuration object` + ); + } + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new InvalidRuntimePgConfigurationError( + `${label} must contain only plain data` + ); + } + const keys = Reflect.ownKeys(value); + if (keys.some((key) => typeof key !== 'string')) { + throw new InvalidRuntimePgConfigurationError( + `${label} must not contain symbol properties` + ); + } + for (const key of keys as string[]) { + if (key === 'connectionString') { + throw new InvalidRuntimePgConfigurationError( + `${label} must not return a connectionString; use explicit fields` + ); + } + if (!allowedKeys.has(key)) { + throw new InvalidRuntimePgConfigurationError( + `${label} contains unsupported field '${key}'` + ); + } + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (!descriptor || !('value' in descriptor) || descriptor.value === undefined) { + throw new InvalidRuntimePgConfigurationError( + `${label}.${key} must be an explicit data value` + ); + } + } + return value as Record; +}; + +const exactArray = ( + value: unknown, + label: string, + length?: number +): readonly unknown[] => { + if (!Array.isArray(value) || (length !== undefined && value.length !== length)) { + throw new InvalidRuntimePgConfigurationError( + `${label} must be ${length === undefined ? 'an array' : `an array of length ${length}`}` + ); + } + const keys = Reflect.ownKeys(value); + if ( + keys.some((key) => typeof key !== 'string') + || keys.some((key) => key !== 'length' && !/^(?:0|[1-9]\d*)$/.test(key as string)) + || Object.keys(value).length !== value.length + ) { + throw new InvalidRuntimePgConfigurationError( + `${label} must be a dense array without custom properties` + ); + } + return value; +}; + +const exactString = ( + value: unknown, + label: string, + allowEmpty = false +): string => { + if ( + typeof value !== 'string' + || (!allowEmpty && value.trim().length === 0) + ) { + throw new InvalidRuntimePgConfigurationError( + `${label} must be ${allowEmpty ? 'a string' : 'a non-empty string'}` + ); + } + return value; +}; + +const normalizeResolverInput = ( + value: RuntimePgResolverInput, + label = 'runtime PostgreSQL route identity' +): Readonly => { + const record = ownDataRecord(value, label, STATIC_IDENTITY_KEYS); + if (Reflect.ownKeys(record).length !== STATIC_IDENTITY_KEYS.size) { + throw new InvalidRuntimePgConfigurationError( + `${label} must contain databaseId, databaseName, apiId, schemas, and roles` + ); + } + const schemas = exactArray(record.schemas, `${label}.schemas`).map( + (schema, index) => exactString(schema, `${label}.schemas[${index}]`) + ); + if (schemas.length === 0 || new Set(schemas).size !== schemas.length) { + throw new InvalidRuntimePgConfigurationError( + `${label}.schemas must contain at least one unique physical schema` + ); + } + const roles = exactArray(record.roles, `${label}.roles`, 2).map( + (role, index) => exactString(role, `${label}.roles[${index}]`) + ) as [string, string]; + return Object.freeze({ + databaseId: exactString(record.databaseId, `${label}.databaseId`), + databaseName: exactString(record.databaseName, `${label}.databaseName`), + apiId: exactString(record.apiId, `${label}.apiId`, true), + schemas: Object.freeze(schemas), + roles: Object.freeze(roles) as readonly [string, string] + }); +}; + +const sameResolverInput = ( + left: Readonly, + right: Readonly +): boolean => + left.databaseId === right.databaseId + && left.databaseName === right.databaseName + && left.apiId === right.apiId + && left.schemas.length === right.schemas.length + && left.schemas.every((schema, index) => schema === right.schemas[index]) + && left.roles[0] === right.roles[0] + && left.roles[1] === right.roles[1]; + +const cloneIdentityData = ( + value: unknown, + path: string, + ancestors = new Set() +): unknown => { + if ( + value === null + || typeof value === 'string' + || typeof value === 'number' + || typeof value === 'boolean' + ) { + return value; + } + if (Buffer.isBuffer(value)) return Buffer.from(value); + if (Array.isArray(value)) { + exactArray(value, path); + if (ancestors.has(value)) { + throw new InvalidRuntimePgConfigurationError(`${path} must not be cyclic`); + } + ancestors.add(value); + const cloned = value.map((entry, index) => + cloneIdentityData(entry, `${path}[${index}]`, ancestors) + ); + ancestors.delete(value); + return Object.freeze(cloned); + } + if (typeof value === 'object') { + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new InvalidRuntimePgConfigurationError( + `${path} must contain only plain data` + ); + } + if (ancestors.has(value)) { + throw new InvalidRuntimePgConfigurationError(`${path} must not be cyclic`); + } + ancestors.add(value); + const cloned: Record = {}; + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== 'string') { + throw new InvalidRuntimePgConfigurationError( + `${path} must not contain symbol properties` + ); + } + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (!descriptor || !('value' in descriptor) || descriptor.value === undefined) { + throw new InvalidRuntimePgConfigurationError( + `${path}.${key} must be an explicit data value` + ); + } + cloned[key] = cloneIdentityData( + descriptor.value, + `${path}.${key}`, + ancestors + ); + } + ancestors.delete(value); + return Object.freeze(cloned); + } + throw new InvalidRuntimePgConfigurationError( + `${path} must contain only deterministic data values` + ); +}; + +const networkDefaults = (options: ConstructiveOptions): RuntimePgConfig => ({ + ...(options.pg?.host === undefined ? {} : { host: options.pg.host }), + ...(options.pg?.port === undefined ? {} : { port: options.pg.port }), + ...(options.pg?.ssl === undefined + ? {} + : { ssl: cloneIdentityData(options.pg.ssl, 'pg.ssl') as RuntimePgConfig['ssl'] }) +}); + +const networkTargetIdentity = (config: RuntimePgConfig): string => + getPgPoolIdentity({ + host: config.host, + port: config.port, + database: config.database, + // Fixed non-connection sentinels make the existing exact/HMAC pool + // identity machinery attest only this physical network/TLS target. + user: 'constructive_target_attestation', + password: 'constructive_target_attestation', + ...(config.ssl === undefined ? {} : { ssl: config.ssl }), + pool: TARGET_ATTESTATION_POOL + }, TARGET_ATTESTATION_OPTIONS); + +const normalizeRuntimePgConfig = ( + options: ConstructiveOptions, + input: Readonly, + rawValue: unknown, + label: string +): Readonly => { + const raw = ownDataRecord(rawValue, label, CONFIG_KEYS); + const user = exactString(raw.user, `${label}.user`); + const password = exactString(raw.password, `${label}.password`); + const database = exactString(raw.database, `${label}.database`); + if (database !== input.databaseName) { + throw new InvalidRuntimePgConfigurationError( + `${label} database does not match the routed physical database` + ); + } + + let pool: RuntimePgConfig['pool']; + if (raw.pool !== undefined) { + const poolRecord = ownDataRecord(raw.pool, `${label}.pool`, POOL_KEYS); + pool = Object.freeze({ ...poolRecord }) as RuntimePgConfig['pool']; + } + const normalized = getPgEnvOptions({ + ...networkDefaults(options), + ...raw, + user, + password, + database, + ...(raw.ssl === undefined + ? {} + : { ssl: cloneIdentityData(raw.ssl, `${label}.ssl`) as RuntimePgConfig['ssl'] }) + }); + if ( + normalized.user !== user + || normalized.password !== password + || normalized.database !== input.databaseName + ) { + throw new InvalidRuntimePgConfigurationError( + `${label} identity changed during normalization` + ); + } + + try { + const controlTarget = getPgEnvOptions({ + ...networkDefaults(options), + database: input.databaseName + }); + if ( + networkTargetIdentity(normalized) + !== networkTargetIdentity(controlTarget) + ) { + throw new InvalidRuntimePgConfigurationError( + `${label} network/TLS endpoint does not match the routed control-plane database` + ); + } + } catch (error) { + if (error instanceof InvalidRuntimePgConfigurationError) throw error; + throw new InvalidRuntimePgConfigurationError( + `${label} could not attest the routed control-plane network/TLS endpoint` + ); + } + + const pgConfig = Object.freeze({ + host: normalized.host, + port: normalized.port, + user: normalized.user, + password: normalized.password, + database: normalized.database, + ...(normalized.ssl === undefined + ? {} + : { ssl: cloneIdentityData(normalized.ssl, `${label}.ssl`) as RuntimePgConfig['ssl'] }), + ...(pool ? { pool } : {}) + }) as RuntimePgConfig; + let poolIdentity: string; + try { + poolIdentity = getPgPoolIdentity(pgConfig, RUNTIME_POOL_OPTIONS); + } catch { + throw new InvalidRuntimePgConfigurationError( + `${label} could not form an exact normalized pool identity` + ); + } + return Object.freeze({ pgConfig, poolIdentity }); +}; + +/** Build the credential-free exact resolver key from authoritative routing. */ +export const createRuntimePgResolverInput = ( + api: ApiStructure +): Readonly => normalizeResolverInput({ + databaseId: api.databaseId ?? '', + databaseName: api.dbname, + apiId: api.apiId ?? '', + schemas: api.schema, + roles: [api.anonRole, api.roleName] +}); + +/** Resolve and normalize one request's tenant execution identity exactly once. */ +export const resolveRuntimePgConfig = async ( + options: ConstructiveOptions, + inputValue: RuntimePgResolverInput, + nodeEnv = getNodeEnv() +): Promise> => { + assertRuntimePgCredentials(options, nodeEnv); + const input = normalizeResolverInput(inputValue); + const resolver = options.runtimePgResolver; + if (resolver) { + let resolved: Awaited>; + try { + resolved = await resolver(input); + } catch { + throw new InvalidRuntimePgConfigurationError( + 'runtimePgResolver failed for the requested exact route' + ); + } + return normalizeRuntimePgConfig( + options, + input, + resolved, + 'runtimePgResolver result' + ); + } + + if (options.runtimePg) { + if (options.runtimePgStaticIdentity) { + const staticIdentity = normalizeResolverInput( + options.runtimePgStaticIdentity, + 'runtimePgStaticIdentity' + ); + if (!sameResolverInput(staticIdentity, input)) { + throw new InvalidRuntimePgConfigurationError( + 'Static runtimePg is not authorized for the requested exact route' + ); + } + } else if (requiresExactRuntimePgResolution(options, nodeEnv)) { + throw new InvalidRuntimePgConfigurationError( + 'Static runtimePg requires one exact runtimePgStaticIdentity' + ); + } + const configuredDatabase = options.runtimePg.database; + if ( + configuredDatabase !== undefined + && configuredDatabase !== input.databaseName + ) { + throw new InvalidRuntimePgConfigurationError( + 'runtimePg database does not match the routed physical database' + ); + } + return normalizeRuntimePgConfig( + options, + input, + { + ...options.runtimePg, + database: configuredDatabase ?? input.databaseName + }, + 'runtimePg' + ); + } + + // Explicitly unsafe compatibility path for stock local development/tests. + // The startup assertion above prevents this path in production or scoped mode. + const fallback = getPgEnvOptions({ + ...options.pg, + database: input.databaseName + }); + return normalizeRuntimePgConfig( + options, + input, + fallback, + 'development control-plane runtime fallback' + ); +}; + +export interface RuntimePgResolutionStore { + middleware: RequestHandler; + getRuntimePgResolution: ( + req: Request, + api?: ApiStructure + ) => Readonly; +} + +/** + * Keep raw credentials in a server-owned WeakMap, never on `req`. Context and + * Graphile receive the same frozen resolution and verify its opaque identity. + */ +export const createRuntimePgResolutionStore = ( + options: ConstructiveOptions +): RuntimePgResolutionStore => { + assertRuntimePgCredentials(options); + let staticResolution: StoredResolution | null = null; + if (options.runtimePg) { + ownDataRecord(options.runtimePg, 'runtimePg', CONFIG_KEYS); + if (options.runtimePgStaticIdentity) { + const input = normalizeResolverInput( + options.runtimePgStaticIdentity, + 'runtimePgStaticIdentity' + ); + const resolution = normalizeRuntimePgConfig( + options, + input, + options.runtimePg, + 'runtimePg' + ); + staticResolution = { input, resolution }; + } + } + interface StoredResolution { + input: Readonly; + resolution: Readonly; + } + const resolutions = new WeakMap(); + const getRuntimePgResolution = ( + req: Request, + api = req.api + ): Readonly => { + const stored = resolutions.get(req); + if (!stored || !api) { + throw new InvalidRuntimePgConfigurationError( + 'Runtime PostgreSQL resolution is unavailable for this request' + ); + } + const currentInput = createRuntimePgResolverInput(api); + if (!sameResolverInput(stored.input, currentInput)) { + throw new InvalidRuntimePgConfigurationError( + 'Authoritative API route changed after runtime PostgreSQL resolution' + ); + } + return stored.resolution; + }; + const middleware: RequestHandler = async ( + req: Request, + res: Response, + next: NextFunction + ): Promise => { + const requestEnded = (): boolean => Boolean( + req.aborted + || req.socket?.destroyed + || res.destroyed + || res.writableEnded + ); + const cleanup = (): void => { + resolutions.delete(req); + req.removeListener('aborted', cleanup); + res.removeListener('finish', cleanup); + res.removeListener('close', cleanup); + }; + try { + if (requestEnded()) return; + if (!req.api) { + throw new InvalidRuntimePgConfigurationError( + 'Runtime PostgreSQL resolution requires an authoritative API route' + ); + } + const input = createRuntimePgResolverInput(req.api); + let resolution: Readonly; + if (staticResolution) { + if (!sameResolverInput(staticResolution.input, input)) { + throw new InvalidRuntimePgConfigurationError( + 'Static runtimePg is not authorized for the requested exact route' + ); + } + resolution = staticResolution.resolution; + } else { + resolution = await resolveRuntimePgConfig(options, input); + } + if (requestEnded()) return; + resolutions.set(req, { input, resolution }); + req.once('aborted', cleanup); + res.once('finish', cleanup); + res.once('close', cleanup); + next(); + } catch (error) { + cleanup(); + next(error); + } + }; + return { middleware, getRuntimePgResolution }; +}; diff --git a/graphql/server/src/middleware/runtime-pg-requirements.ts b/graphql/server/src/middleware/runtime-pg-requirements.ts new file mode 100644 index 0000000000..b4e43aa05d --- /dev/null +++ b/graphql/server/src/middleware/runtime-pg-requirements.ts @@ -0,0 +1,113 @@ +import type { ConstructiveOptions } from '@constructive-io/graphql-types'; +import { getNodeEnv } from '@pgpmjs/env'; + +export class MissingRuntimePgCredentialsError extends Error { + readonly code = 'GRAPHILE_RUNTIME_PG_REQUIRED'; + + constructor() { + super( + 'GraphQL runtime execution requires an explicit PostgreSQL user and password' + ); + this.name = 'MissingRuntimePgCredentialsError'; + } +} + +export class InvalidRuntimePgConfigurationError extends Error { + readonly code = 'GRAPHILE_RUNTIME_PG_CONFIG_INVALID'; + + constructor(message: string) { + super(message); + this.name = 'InvalidRuntimePgConfigurationError'; + } +} + +export const requiresExactRuntimePgResolution = ( + options: ConstructiveOptions, + nodeEnv = getNodeEnv() +): boolean => + nodeEnv === 'production' + || options.graphile?.introspectionMode === 'scoped-required'; + +/** + * The control-plane login fallback exists only for backwards-compatible local + * stock-mode development and tests. Production and scoped introspection always + * use an explicit, independently audited runtime login. + */ +export const usesUnsafeDevelopmentRuntimePgFallback = ( + options: ConstructiveOptions, + nodeEnv = getNodeEnv() +): boolean => + !requiresExactRuntimePgResolution(options, nodeEnv) + && options.runtimePg === undefined + && options.runtimePgResolver === undefined; + +export const shouldValidateRuntimeRoleSafety = ( + options: ConstructiveOptions, + nodeEnv = getNodeEnv() +): boolean => + nodeEnv === 'production' + || options.runtimePg !== undefined + || options.runtimePgResolver !== undefined + || options.graphile?.introspectionMode === 'scoped-required'; + +export const assertRuntimePgCredentials = ( + options: ConstructiveOptions, + nodeEnv = getNodeEnv() +): void => { + const hasResolver = options.runtimePgResolver !== undefined; + const hasStatic = options.runtimePg !== undefined; + const hasStaticIdentity = options.runtimePgStaticIdentity !== undefined; + if (hasResolver && (hasStatic || hasStaticIdentity)) { + throw new InvalidRuntimePgConfigurationError( + 'runtimePgResolver is mutually exclusive with runtimePg and runtimePgStaticIdentity' + ); + } + if (hasResolver) { + if (typeof options.runtimePgResolver !== 'function') { + throw new InvalidRuntimePgConfigurationError( + 'runtimePgResolver must be a function' + ); + } + return; + } + if (hasStaticIdentity && !hasStatic) { + throw new InvalidRuntimePgConfigurationError( + 'runtimePgStaticIdentity requires runtimePg' + ); + } + if (usesUnsafeDevelopmentRuntimePgFallback(options, nodeEnv)) return; + if (!hasStatic) throw new MissingRuntimePgCredentialsError(); + if (Object.prototype.hasOwnProperty.call(options.runtimePg, 'connectionString')) { + throw new InvalidRuntimePgConfigurationError( + 'runtimePg must not contain a connectionString; use explicit fields' + ); + } + const user = options.runtimePg?.user; + const password = options.runtimePg?.password; + if ( + typeof user !== 'string' + || user.trim().length === 0 + || typeof password !== 'string' + || password.length === 0 + ) { + throw new MissingRuntimePgCredentialsError(); + } + if (requiresExactRuntimePgResolution(options, nodeEnv)) { + if (!hasStaticIdentity) { + throw new InvalidRuntimePgConfigurationError( + 'Production and scoped introspection require runtimePgResolver, or runtimePgStaticIdentity for one exact route' + ); + } + if ( + typeof options.runtimePg?.database !== 'string' + || options.runtimePg.database.length === 0 + ) { + throw new InvalidRuntimePgConfigurationError( + 'Static production runtimePg requires an explicit database' + ); + } + } +}; + +/** @deprecated Use `assertRuntimePgCredentials`; retained for package compatibility. */ +export const assertScopedRuntimePgCredentials = assertRuntimePgCredentials; diff --git a/graphql/server/src/middleware/runtime-role-safety.ts b/graphql/server/src/middleware/runtime-role-safety.ts new file mode 100644 index 0000000000..1cbff507d1 --- /dev/null +++ b/graphql/server/src/middleware/runtime-role-safety.ts @@ -0,0 +1,883 @@ +import { performance } from 'node:perf_hooks'; + +import type { Pool, PoolClient, QueryResult } from 'pg'; + +export const RUNTIME_ROLE_SAFETY_SQL = ` +WITH RECURSIVE execution_roles AS ( + SELECT r.oid, r.rolname + FROM pg_catalog.pg_roles r + WHERE r.rolname = current_user + OR r.rolname = ANY($1::text[]) +), execution_role_reachability AS MATERIALIZED ( + SELECT execution_role.oid AS execution_role_oid, + execution_role.rolname AS execution_role_name, + candidate.oid AS reachable_role_oid, + candidate.rolname AS reachable_role_name, + pg_catalog.pg_has_role(execution_role.oid, candidate.oid, 'USAGE') + AS via_usage, + pg_catalog.pg_has_role(execution_role.oid, candidate.oid, 'SET') + AS via_set + FROM execution_roles execution_role + INNER JOIN pg_catalog.pg_roles candidate + ON candidate.oid = execution_role.oid + OR pg_catalog.pg_has_role(execution_role.oid, candidate.oid, 'USAGE') + OR pg_catalog.pg_has_role(execution_role.oid, candidate.oid, 'SET') +), accessible_roles AS ( + SELECT r.oid, r.rolname, r.rolsuper, r.rolbypassrls, r.rolcreaterole, + r.rolcreatedb, r.rolreplication, r.rolinherit + FROM pg_catalog.pg_roles r + WHERE r.rolname = current_user + OR r.rolname = ANY($1::text[]) + OR pg_catalog.pg_has_role(current_user, r.oid, 'USAGE') + OR pg_catalog.pg_has_role(current_user, r.oid, 'SET') + OR EXISTS ( + SELECT 1 + FROM execution_role_reachability reachable + WHERE reachable.reachable_role_oid = r.oid + ) +), exposed_schemas AS ( + SELECT n.oid, n.nspname, n.nspowner + FROM pg_catalog.pg_namespace n + WHERE n.nspname = ANY($2::text[]) +), approved_schemas AS ( + SELECT n.oid, n.nspname, n.nspowner + FROM pg_catalog.pg_namespace n + WHERE n.nspname = ANY($2::text[] || $3::text[]) +), current_database_record AS ( + SELECT d.oid, d.datname, d.datdba + FROM pg_catalog.pg_database d + WHERE d.datname = pg_catalog.current_database() +), unapproved_schema_access AS MATERIALIZED ( + SELECT r.oid AS role_oid, r.rolname, n.oid AS namespace_oid, n.nspname, + n.nspowner = r.oid AS is_owner, + pg_catalog.has_schema_privilege(r.rolname, n.oid, 'CREATE') AS can_create, + pg_catalog.has_schema_privilege(r.rolname, n.oid, 'USAGE') AS can_use + FROM accessible_roles r + INNER JOIN pg_catalog.pg_namespace n ON true + WHERE n.nspname <> 'information_schema' + AND n.nspname NOT LIKE 'pg\\_%' + AND NOT EXISTS (SELECT 1 FROM approved_schemas a WHERE a.oid = n.oid) + AND ( + n.nspowner = r.oid + OR pg_catalog.has_schema_privilege(r.rolname, n.oid, 'CREATE') + OR pg_catalog.has_schema_privilege(r.rolname, n.oid, 'USAGE') + ) +), login_role_violations AS ( + SELECT array_remove(ARRAY[ + CASE WHEN rolinherit THEN 'INHERIT' END + ], NULL) AS capabilities + FROM accessible_roles + WHERE rolname = current_user AND rolinherit +), inherited_role_violations AS ( + SELECT r.rolname + FROM pg_catalog.pg_roles r + WHERE r.rolname <> current_user + AND pg_catalog.pg_has_role(current_user, r.oid, 'USAGE') +), unexpected_set_role_violations AS ( + SELECT r.rolname + FROM pg_catalog.pg_roles r + WHERE r.rolname <> current_user + AND NOT (r.rolname = ANY($1::text[])) + AND pg_catalog.pg_has_role(current_user, r.oid, 'SET') +), request_role_reachability_violations AS ( + SELECT reachable.execution_role_name AS request_role, + reachable.reachable_role_name AS reachable_role, + reachable.via_usage, + reachable.via_set + FROM execution_role_reachability reachable + WHERE reachable.execution_role_name = ANY($1::text[]) + AND reachable.execution_role_oid <> reachable.reachable_role_oid +), role_violations AS ( + SELECT rolname, + array_remove(ARRAY[ + CASE WHEN rolsuper THEN 'SUPERUSER' END, + CASE WHEN rolbypassrls THEN 'BYPASSRLS' END, + CASE WHEN rolcreaterole THEN 'CREATEROLE' END, + CASE WHEN rolcreatedb THEN 'CREATEDB' END, + CASE WHEN rolreplication THEN 'REPLICATION' END + ], NULL) AS capabilities + FROM accessible_roles + WHERE rolsuper OR rolbypassrls OR rolcreaterole OR rolcreatedb OR rolreplication +), database_violations AS ( + SELECT r.rolname, d.datname, violation.capability + FROM accessible_roles r + INNER JOIN current_database_record d ON true + CROSS JOIN LATERAL ( + VALUES + ('OWNER'::text, d.datdba = r.oid), + ('CREATE'::text, pg_catalog.has_database_privilege(r.rolname, d.oid, 'CREATE')), + ('TEMP'::text, pg_catalog.has_database_privilege(r.rolname, d.oid, 'TEMP')) + ) AS violation(capability, present) + WHERE violation.present +), cross_database_violations AS ( + SELECT r.rolname, d.datname + FROM accessible_roles r + INNER JOIN current_database_record current_database ON true + INNER JOIN pg_catalog.pg_database d ON d.oid <> current_database.oid + WHERE pg_catalog.has_database_privilege(r.rolname, d.oid, 'CONNECT') +), schema_violations AS ( + SELECT r.rolname, n.nspname, + CASE WHEN n.nspowner = r.oid THEN 'OWNER' ELSE 'CREATE' END AS capability + FROM accessible_roles r + INNER JOIN approved_schemas n ON true + WHERE n.nspowner = r.oid + OR pg_catalog.has_schema_privilege(r.rolname, n.nspname, 'CREATE') +), cross_schema_violations AS ( + SELECT access.rolname, access.nspname, + array_remove(ARRAY[ + CASE WHEN access.can_create OR access.is_owner THEN 'CREATE/OWNER' END, + CASE WHEN EXISTS ( + SELECT 1 + FROM pg_catalog.pg_class c + WHERE c.relnamespace = access.namespace_oid + AND c.relkind IN ('r', 'p', 'v', 'm', 'f') + AND pg_catalog.has_table_privilege( + access.rolname, + c.oid, + 'SELECT,INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER' + ) + ) AND access.can_use + THEN 'RELATION' END, + CASE WHEN EXISTS ( + SELECT 1 + FROM pg_catalog.pg_class c + WHERE c.relnamespace = access.namespace_oid + AND CASE WHEN c.relkind = 'S' + THEN pg_catalog.has_sequence_privilege( + access.rolname, + c.oid, + 'USAGE,SELECT,UPDATE' + ) + ELSE false + END + ) AND access.can_use + THEN 'SEQUENCE' END, + CASE WHEN EXISTS ( + SELECT 1 + FROM pg_catalog.pg_proc p + WHERE p.pronamespace = access.namespace_oid + AND pg_catalog.has_function_privilege(access.rolname, p.oid, 'EXECUTE') + ) AND access.can_use + THEN 'FUNCTION' END, + CASE WHEN EXISTS ( + SELECT 1 + FROM pg_catalog.pg_type t + WHERE t.typnamespace = access.namespace_oid + AND pg_catalog.has_type_privilege(access.rolname, t.oid, 'USAGE') + ) AND access.can_use + THEN 'TYPE' END + ], NULL) AS capabilities + FROM unapproved_schema_access access + WHERE access.can_create + OR access.is_owner + OR ( + access.can_use + AND ( + EXISTS ( + SELECT 1 + FROM pg_catalog.pg_class c + WHERE c.relnamespace = access.namespace_oid + AND ( + (c.relkind IN ('r', 'p', 'v', 'm', 'f') AND pg_catalog.has_table_privilege( + access.rolname, + c.oid, + 'SELECT,INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER' + )) + OR CASE WHEN c.relkind = 'S' + THEN pg_catalog.has_sequence_privilege( + access.rolname, + c.oid, + 'USAGE,SELECT,UPDATE' + ) + ELSE false + END + ) + ) + OR EXISTS ( + SELECT 1 + FROM pg_catalog.pg_proc p + WHERE p.pronamespace = access.namespace_oid + AND pg_catalog.has_function_privilege(access.rolname, p.oid, 'EXECUTE') + ) + OR EXISTS ( + SELECT 1 + FROM pg_catalog.pg_type t + WHERE t.typnamespace = access.namespace_oid + AND pg_catalog.has_type_privilege(access.rolname, t.oid, 'USAGE') + ) + ) + ) +), object_owner_violations AS ( + SELECT r.rolname, n.nspname, c.relname AS object_name, + CASE c.relkind + WHEN 'S' THEN 'SEQUENCE' + WHEN 'v' THEN 'VIEW' + WHEN 'm' THEN 'MATERIALIZED VIEW' + WHEN 'f' THEN 'FOREIGN TABLE' + ELSE 'RELATION' + END AS object_kind + FROM accessible_roles r + INNER JOIN approved_schemas n ON true + INNER JOIN pg_catalog.pg_class c + ON c.relnamespace = n.oid AND c.relowner = r.oid + + UNION ALL + + SELECT r.rolname, n.nspname, p.proname, 'FUNCTION' + FROM accessible_roles r + INNER JOIN approved_schemas n ON true + INNER JOIN pg_catalog.pg_proc p + ON p.pronamespace = n.oid AND p.proowner = r.oid + + UNION ALL + + SELECT r.rolname, n.nspname, t.typname, 'TYPE' + FROM accessible_roles r + INNER JOIN approved_schemas n ON true + INNER JOIN pg_catalog.pg_type t + ON t.typnamespace = n.oid AND t.typowner = r.oid +), stored_expression_roots AS ( + SELECT 'pg_catalog.pg_trigger'::regclass::oid AS root_class, + trigger.oid AS root_id, + namespace.nspname, + class.relname || ':' || trigger.tgname AS object_name + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_class class ON class.relnamespace = namespace.oid + INNER JOIN pg_catalog.pg_trigger trigger ON trigger.tgrelid = class.oid + WHERE NOT trigger.tgisinternal + + UNION ALL + + SELECT 'pg_catalog.pg_attrdef'::regclass::oid, + attribute_default.oid, + namespace.nspname, + class.relname || '.' || attribute.attname + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_class class ON class.relnamespace = namespace.oid + INNER JOIN pg_catalog.pg_attrdef attribute_default ON attribute_default.adrelid = class.oid + INNER JOIN pg_catalog.pg_attribute attribute + ON attribute.attrelid = class.oid AND attribute.attnum = attribute_default.adnum + + UNION ALL + + SELECT 'pg_catalog.pg_policy'::regclass::oid, + policy.oid, + namespace.nspname, + class.relname || ':' || policy.polname + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_class class ON class.relnamespace = namespace.oid + INNER JOIN pg_catalog.pg_policy policy ON policy.polrelid = class.oid + + UNION ALL + + SELECT 'pg_catalog.pg_rewrite'::regclass::oid, + rewrite.oid, + namespace.nspname, + class.relname || ':' || rewrite.rulename + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_class class ON class.relnamespace = namespace.oid + INNER JOIN pg_catalog.pg_rewrite rewrite ON rewrite.ev_class = class.oid + + UNION ALL + + SELECT 'pg_catalog.pg_constraint'::regclass::oid, + constraint_record.oid, + namespace.nspname, + COALESCE(class.relname || ':', '') || constraint_record.conname + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_constraint constraint_record + ON constraint_record.connamespace = namespace.oid + LEFT JOIN pg_catalog.pg_class class ON class.oid = constraint_record.conrelid + + UNION ALL + + SELECT 'pg_catalog.pg_class'::regclass::oid, + index_class.oid, + namespace.nspname, + index_class.relname + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_class index_class + ON index_class.relnamespace = namespace.oid + AND index_class.relkind IN ('i', 'I') + + UNION ALL + + SELECT 'pg_catalog.pg_proc'::regclass::oid, + procedure.oid, + namespace.nspname, + procedure.proname + FROM exposed_schemas namespace + INNER JOIN pg_catalog.pg_proc procedure ON procedure.pronamespace = namespace.oid +), stored_dependency_closure( + root_class, + root_id, + nspname, + object_name, + dependency_class, + dependency_id +) AS ( + SELECT root.root_class, + root.root_id, + root.nspname, + root.object_name, + dependency.refclassid, + dependency.refobjid + FROM stored_expression_roots root + INNER JOIN pg_catalog.pg_depend dependency + ON dependency.classid = root.root_class + AND dependency.objid = root.root_id + WHERE dependency.refobjid <> 0 + + UNION + + SELECT closure.root_class, + closure.root_id, + closure.nspname, + closure.object_name, + dependency.refclassid, + dependency.refobjid + FROM stored_dependency_closure closure + INNER JOIN pg_catalog.pg_depend dependency + ON dependency.classid = closure.dependency_class + AND dependency.objid = closure.dependency_id + WHERE dependency.refobjid <> 0 +), stored_dependency_violations AS ( + SELECT DISTINCT root.nspname, root.object_name, + CASE + WHEN dependency_proc.prosecdef + THEN 'STORED EXPRESSION CALLS SECURITY DEFINER' + ELSE 'STORED EXPRESSION CROSSES SCHEMA' + END AS reason, + dependency_namespace.nspname || '.' || dependency_proc.proname AS dependency + FROM stored_dependency_closure root + INNER JOIN pg_catalog.pg_proc dependency_proc + ON root.dependency_class = 'pg_catalog.pg_proc'::regclass + AND dependency_proc.oid = root.dependency_id + INNER JOIN pg_catalog.pg_namespace dependency_namespace + ON dependency_namespace.oid = dependency_proc.pronamespace + WHERE dependency_proc.prosecdef + OR ( + dependency_namespace.nspname <> 'pg_catalog' + AND NOT EXISTS ( + SELECT 1 FROM approved_schemas approved + WHERE approved.oid = dependency_namespace.oid + ) + ) + + UNION + + SELECT DISTINCT root.nspname, root.object_name, + 'STORED EXPRESSION CROSSES SCHEMA', + dependency_namespace.nspname || '.' || dependency_class.relname + FROM stored_dependency_closure root + INNER JOIN pg_catalog.pg_class dependency_class + ON root.dependency_class = 'pg_catalog.pg_class'::regclass + AND dependency_class.oid = root.dependency_id + INNER JOIN pg_catalog.pg_namespace dependency_namespace + ON dependency_namespace.oid = dependency_class.relnamespace + WHERE dependency_namespace.nspname <> 'pg_catalog' + AND NOT EXISTS ( + SELECT 1 FROM approved_schemas approved + WHERE approved.oid = dependency_namespace.oid + ) +), privileged_object_violations AS ( + SELECT n.nspname, p.proname AS object_name, 'SECURITY DEFINER FUNCTION' AS reason + FROM approved_schemas n + INNER JOIN pg_catalog.pg_proc p ON p.pronamespace = n.oid + WHERE p.prosecdef + + UNION ALL + + SELECT n.nspname, c.relname, 'OWNER-RIGHTS VIEW' + FROM approved_schemas n + INNER JOIN pg_catalog.pg_class c ON c.relnamespace = n.oid + WHERE c.relkind = 'v' + AND NOT COALESCE(c.reloptions @> ARRAY['security_invoker=true'], false) + + UNION ALL + + SELECT n.nspname, c.relname, 'FOREIGN TABLE' + FROM approved_schemas n + INNER JOIN pg_catalog.pg_class c ON c.relnamespace = n.oid + WHERE c.relkind = 'f' + + UNION ALL + + SELECT n.nspname, c.relname, 'MATERIALIZED VIEW' + FROM approved_schemas n + INNER JOIN pg_catalog.pg_class c ON c.relnamespace = n.oid + WHERE c.relkind = 'm' +) +SELECT current_user AS login_role, + COALESCE((SELECT json_agg(login_role_violations) FROM login_role_violations), '[]'::json) AS login_role_violations, + COALESCE((SELECT json_agg(inherited_role_violations) FROM inherited_role_violations), '[]'::json) AS inherited_role_violations, + COALESCE((SELECT json_agg(unexpected_set_role_violations) FROM unexpected_set_role_violations), '[]'::json) AS unexpected_set_role_violations, + COALESCE((SELECT json_agg(request_role_reachability_violations) FROM request_role_reachability_violations), '[]'::json) AS request_role_reachability_violations, + COALESCE((SELECT json_agg(role_violations) FROM role_violations), '[]'::json) AS role_violations, + COALESCE((SELECT json_agg(database_violations) FROM database_violations), '[]'::json) AS database_violations, + COALESCE((SELECT json_agg(cross_database_violations) FROM cross_database_violations), '[]'::json) AS cross_database_violations, + COALESCE((SELECT json_agg(schema_violations) FROM schema_violations), '[]'::json) AS schema_violations, + COALESCE((SELECT json_agg(cross_schema_violations) FROM cross_schema_violations), '[]'::json) AS cross_schema_violations, + COALESCE((SELECT json_agg(object_owner_violations) FROM object_owner_violations), '[]'::json) AS object_owner_violations, + COALESCE((SELECT json_agg(privileged_object_violations) FROM privileged_object_violations), '[]'::json) AS privileged_object_violations, + COALESCE((SELECT json_agg(stored_dependency_violations) FROM stored_dependency_violations), '[]'::json) AS stored_dependency_violations, + ARRAY( + SELECT requested + FROM unnest($1::text[]) requested + WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.rolname = requested) + ) AS missing_roles, + ARRAY( + SELECT r.rolname::text + FROM pg_catalog.pg_roles r + WHERE r.rolname = ANY($1::text[]) + AND r.rolname <> current_user + AND NOT pg_catalog.pg_has_role(current_user, r.oid, 'SET') + ) AS inaccessible_roles, + ARRAY( + SELECT requested + FROM unnest($2::text[] || $3::text[]) requested + WHERE NOT EXISTS (SELECT 1 FROM approved_schemas n WHERE n.nspname = requested) + ) AS missing_schemas +`; + +interface RuntimeRoleSafetyRow { + login_role: string; + login_role_violations: Array<{ capabilities: string[] }> | string; + inherited_role_violations: Array<{ rolname: string }> | string; + unexpected_set_role_violations: Array<{ rolname: string }> | string; + request_role_reachability_violations: Array<{ + request_role: string; + reachable_role: string; + via_usage: boolean; + via_set: boolean; + }> | string; + role_violations: Array<{ rolname: string; capabilities: string[] }> | string; + database_violations: Array<{ + rolname: string; + datname: string; + capability: string; + }> | string; + cross_database_violations: Array<{ + rolname: string; + datname: string; + }> | string; + schema_violations: Array<{ rolname: string; nspname: string; capability: string }> | string; + cross_schema_violations: Array<{ + rolname: string; + nspname: string; + capabilities: string[]; + }> | string; + object_owner_violations: Array<{ + rolname: string; + nspname: string; + object_name: string; + object_kind: string; + }> | string; + privileged_object_violations: Array<{ + nspname: string; + object_name: string; + reason: string; + }> | string; + stored_dependency_violations: Array<{ + nspname: string; + object_name: string; + reason: string; + dependency: string; + }> | string; + missing_roles: string[]; + inaccessible_roles: string[]; + missing_schemas: string[]; +} + +const parseRequiredJsonColumn = ( + value: T[] | string | null | undefined, + column: string +): T[] => { + let parsed: unknown = value; + if (typeof value === 'string') { + try { + parsed = JSON.parse(value); + } catch { + throw new UnsafeRuntimeRoleError([ + `safety query returned invalid JSON for ${column}` + ]); + } + } + if (!Array.isArray(parsed)) { + throw new UnsafeRuntimeRoleError([ + `safety query did not return ${column} as a JSON array` + ]); + } + return parsed as T[]; +}; + +const parseRequiredTextArrayColumn = ( + value: string[] | null | undefined, + column: string +): string[] => { + if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) { + throw new UnsafeRuntimeRoleError([ + `safety query did not return ${column} as a text array` + ]); + } + return value; +}; + +export class UnsafeRuntimeRoleError extends Error { + readonly code = 'GRAPHILE_UNSAFE_RUNTIME_ROLE'; + + constructor(readonly violations: string[]) { + super(`GraphQL runtime role safety check failed: ${violations.join('; ')}`); + this.name = 'UnsafeRuntimeRoleError'; + } +} + +export const assertRuntimeRoleSafety = async ( + pool: Pool, + requestRoles: string[], + exposedSchemas: string[], + dependencySchemas: string[] = [] +): Promise => { + const uniqueRoles = [...new Set(requestRoles.filter(Boolean))]; + const uniqueExposedSchemas = [...new Set(exposedSchemas.filter(Boolean))]; + const uniqueDependencySchemas = [...new Set( + dependencySchemas.filter((schema) => schema && !uniqueExposedSchemas.includes(schema)) + )]; + const client: PoolClient = await pool.connect(); + let inTransaction = false; + let destroyClient = false; + let result: QueryResult; + try { + // Both commands use the simple protocol, so send them together and avoid a + // second network round trip without changing the read-only transaction or + // the per-audit JIT policy. + await client.query('BEGIN READ ONLY; SET LOCAL jit TO off'); + inTransaction = true; + // The catalog ACL audit has a deliberately broad static plan. On large + // catalogs PostgreSQL can spend over a second compiling hundreds of JIT + // functions for a query that executes in milliseconds once compiled. + result = await client.query(RUNTIME_ROLE_SAFETY_SQL, [ + uniqueRoles, + uniqueExposedSchemas, + uniqueDependencySchemas + ]); + await client.query('COMMIT'); + inTransaction = false; + } catch (error) { + destroyClient = true; + if (inTransaction) { + try { + await client.query('ROLLBACK'); + } catch { + // Preserve the safety-check failure; pg-pool will discard a broken + // connection through its normal error path. + } + } + throw error; + } finally { + client.release(destroyClient); + } + const row = result.rows[0]; + if (!row) { + throw new UnsafeRuntimeRoleError(['safety query returned no result']); + } + + if (typeof row.login_role !== 'string' || row.login_role.length === 0) { + throw new UnsafeRuntimeRoleError([ + 'safety query did not return a non-empty login_role' + ]); + } + + // Every result column participates in the tenant boundary. Treat query/result + // drift as an unsafe audit instead of interpreting an absent check as an + // empty violation set. + const loginRoleViolations = parseRequiredJsonColumn<{ + capabilities: string[]; + }>(row.login_role_violations, 'login_role_violations'); + const inheritedRoleViolations = parseRequiredJsonColumn<{ + rolname: string; + }>(row.inherited_role_violations, 'inherited_role_violations'); + const databaseViolations = parseRequiredJsonColumn<{ + rolname: string; + datname: string; + capability: string; + }>(row.database_violations, 'database_violations'); + const crossDatabaseViolations = parseRequiredJsonColumn<{ + rolname: string; + datname: string; + }>(row.cross_database_violations, 'cross_database_violations'); + const unexpectedSetRoleViolations = parseRequiredJsonColumn<{ + rolname: string; + }>(row.unexpected_set_role_violations, 'unexpected_set_role_violations'); + const requestRoleReachabilityViolations = parseRequiredJsonColumn<{ + request_role: string; + reachable_role: string; + via_usage: boolean; + via_set: boolean; + }>( + row.request_role_reachability_violations, + 'request_role_reachability_violations' + ); + const roleViolations = parseRequiredJsonColumn<{ + rolname: string; + capabilities: string[]; + }>(row.role_violations, 'role_violations'); + const schemaViolations = parseRequiredJsonColumn<{ + rolname: string; + nspname: string; + capability: string; + }>(row.schema_violations, 'schema_violations'); + const crossSchemaViolations = parseRequiredJsonColumn<{ + rolname: string; + nspname: string; + capabilities: string[]; + }>(row.cross_schema_violations, 'cross_schema_violations'); + const objectOwnerViolations = parseRequiredJsonColumn<{ + rolname: string; + nspname: string; + object_name: string; + object_kind: string; + }>(row.object_owner_violations, 'object_owner_violations'); + const privilegedObjectViolations = parseRequiredJsonColumn<{ + nspname: string; + object_name: string; + reason: string; + }>(row.privileged_object_violations, 'privileged_object_violations'); + const storedDependencyViolations = parseRequiredJsonColumn<{ + nspname: string; + object_name: string; + reason: string; + dependency: string; + }>(row.stored_dependency_violations, 'stored_dependency_violations'); + const missingRoles = parseRequiredTextArrayColumn( + row.missing_roles, + 'missing_roles' + ); + const inaccessibleRoles = parseRequiredTextArrayColumn( + row.inaccessible_roles, + 'inaccessible_roles' + ); + const missingSchemas = parseRequiredTextArrayColumn( + row.missing_schemas, + 'missing_schemas' + ); + + const violations = [ + ...loginRoleViolations.map( + (role) => `${row.login_role} has ${role.capabilities.join(',')}` + ), + ...inheritedRoleViolations.map( + (role) => `${row.login_role} inherits privileges from role ${role.rolname}` + ), + ...unexpectedSetRoleViolations.map( + (role) => `${row.login_role} can SET ROLE to unconfigured role ${role.rolname}` + ), + ...requestRoleReachabilityViolations.map( + (role) => `${role.request_role} can reach role ${role.reachable_role}` + + ` after SET ROLE (USAGE=${role.via_usage},SET=${role.via_set})` + ), + ...roleViolations.map( + (role) => `${role.rolname} has ${role.capabilities.join(',')}` + ), + ...databaseViolations.map( + (database) => `${database.rolname} has ${database.capability} on database ${database.datname}` + ), + ...crossDatabaseViolations.map( + (database) => `${database.rolname} has CONNECT on non-target database ${database.datname}` + ), + ...schemaViolations.map( + (schema) => `${schema.rolname} has ${schema.capability} on schema ${schema.nspname}` + ), + ...crossSchemaViolations.map( + (schema) => `${schema.rolname} has ${schema.capabilities.join(',')} on unapproved schema ${schema.nspname}` + ), + ...objectOwnerViolations.map( + (object) => `${object.rolname} owns ${object.object_kind} ${object.nspname}.${object.object_name}` + ), + ...privilegedObjectViolations.map( + (object) => `${object.reason} ${object.nspname}.${object.object_name} is not allowed in the approved GraphQL schema scope` + ), + ...storedDependencyViolations.map( + (object) => `${object.reason} from ${object.nspname}.${object.object_name} to ${object.dependency}` + ), + ...missingRoles.map((role) => `request role ${role} does not exist`), + ...inaccessibleRoles.map( + (role) => `runtime login ${row.login_role} cannot SET ROLE ${role}` + ), + ...missingSchemas.map((schema) => `exposed schema ${schema} does not exist`) + ]; + + if (violations.length > 0) throw new UnsafeRuntimeRoleError(violations); +}; + +interface CachedSafetyCheck { + promise: Promise; + /** Wall-clock time when the successful catalog audit completed. */ + validatedAt: number | null; +} + +export interface RuntimeRoleSafetyStats { + checksStarted: number; + checksSucceeded: number; + checksFailed: number; + inFlightCoalesces: number; + successfulResultReuses: number; + durationMsTotal: number; + durationMsMax: number; +} + +const runtimeRoleSafetyStats: RuntimeRoleSafetyStats = { + checksStarted: 0, + checksSucceeded: 0, + checksFailed: 0, + inFlightCoalesces: 0, + successfulResultReuses: 0, + durationMsTotal: 0, + durationMsMax: 0 +}; + +/** Process-level audit timing and coalescing telemetry for local diagnostics. */ +export const getRuntimeRoleSafetyStats = (): Readonly => ({ + ...runtimeRoleSafetyStats +}); + +const recordRuntimeRoleSafetyDuration = (startedAt: number): void => { + const durationMs = performance.now() - startedAt; + runtimeRoleSafetyStats.durationMsTotal += durationMs; + runtimeRoleSafetyStats.durationMsMax = Math.max( + runtimeRoleSafetyStats.durationMsMax, + durationMs + ); +}; + +/** + * Successful catalog audits may only be reused for a narrowly bounded window. + * Callers may choose a fresher policy, including zero for in-flight coalescing + * without any completed-result reuse, but may not extend this safety bound. + */ +// Without an authoritative control-plane epoch, a completed catalog result is +// stale immediately. Keep the opt-in cap small for deployments that wire the +// invalidation seam to every DDL/ACL commit, but make fresh checks the default. +export const DEFAULT_RUNTIME_ROLE_SAFETY_MAX_AGE_MS = 0; +export const MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS = 1_000; + +export interface RuntimeRoleSafetyCacheOptions { + maxSuccessAgeMs?: number; +} + +const safetyChecks = new WeakMap>(); + +const normalizeMaxSuccessAgeMs = (value: number | undefined): number => { + const maxSuccessAgeMs = value ?? DEFAULT_RUNTIME_ROLE_SAFETY_MAX_AGE_MS; + if ( + !Number.isSafeInteger(maxSuccessAgeMs) + || maxSuccessAgeMs < 0 + || maxSuccessAgeMs > MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS + ) { + throw new RangeError( + `runtime role safety maxSuccessAgeMs must be an integer between 0 and ${MAX_RUNTIME_ROLE_SAFETY_MAX_AGE_MS}` + ); + } + return maxSuccessAgeMs; +}; + +/** Coalesce identical safety checks for every consumer of a runtime pool. */ +const safetyCheckKey = ( + requestRoles: string[], + exposedSchemas: string[], + dependencySchemas: string[] +): string => JSON.stringify([ + [...new Set(requestRoles.filter(Boolean))].sort(), + [...new Set(exposedSchemas.filter(Boolean))].sort(), + [...new Set(dependencySchemas.filter(Boolean))].sort() +]); + +/** Reuse a recent successful audit while coalescing concurrent callers. */ +export const ensureRuntimeRoleSafety = ( + pool: Pool, + requestRoles: string[], + exposedSchemas: string[], + dependencySchemas: string[] = [], + options: RuntimeRoleSafetyCacheOptions = {} +): Promise => { + const maxSuccessAgeMs = normalizeMaxSuccessAgeMs(options.maxSuccessAgeMs); + const key = safetyCheckKey(requestRoles, exposedSchemas, dependencySchemas); + let checksForPool = safetyChecks.get(pool); + if (!checksForPool) { + checksForPool = new Map(); + safetyChecks.set(pool, checksForPool); + } + const existing = checksForPool.get(key); + if (existing) { + if (existing.validatedAt == null) { + runtimeRoleSafetyStats.inFlightCoalesces++; + return existing.promise; + } + const now = Date.now(); + const successAgeMs = now - existing.validatedAt; + if ( + maxSuccessAgeMs > 0 + && successAgeMs >= 0 + && successAgeMs < maxSuccessAgeMs + ) { + runtimeRoleSafetyStats.successfulResultReuses++; + return existing.promise; + } + } + + let check!: CachedSafetyCheck; + runtimeRoleSafetyStats.checksStarted++; + const startedAt = performance.now(); + const pending = assertRuntimeRoleSafety( + pool, + requestRoles, + exposedSchemas, + dependencySchemas + ).then(() => { + runtimeRoleSafetyStats.checksSucceeded++; + recordRuntimeRoleSafetyDuration(startedAt); + check.validatedAt = Date.now(); + }).catch((error) => { + runtimeRoleSafetyStats.checksFailed++; + recordRuntimeRoleSafetyDuration(startedAt); + if (checksForPool?.get(key) === check) checksForPool.delete(key); + throw error; + }); + check = { promise: pending, validatedAt: null }; + checksForPool.set(key, check); + return pending; +}; + +/** + * Invalidate one audit contract, or every cached audit for the pool when the + * contract arguments are omitted. Control-plane DDL/GRANT/REVOKE paths should + * call this immediately after committing catalog changes. + */ +export const invalidateRuntimeRoleSafety = ( + pool: Pool, + requestRoles?: string[], + exposedSchemas?: string[], + dependencySchemas: string[] = [] +): void => { + const checksForPool = safetyChecks.get(pool); + if (!checksForPool) return; + if (requestRoles == null || exposedSchemas == null) { + safetyChecks.delete(pool); + return; + } + checksForPool.delete( + safetyCheckKey(requestRoles, exposedSchemas, dependencySchemas) + ); + if (checksForPool.size === 0) safetyChecks.delete(pool); +}; + +/** Force a new audit for schema build admission, even after a cached success. */ +export const refreshRuntimeRoleSafety = ( + pool: Pool, + requestRoles: string[], + exposedSchemas: string[], + dependencySchemas: string[] = [] +): Promise => { + invalidateRuntimeRoleSafety(pool, requestRoles, exposedSchemas, dependencySchemas); + return ensureRuntimeRoleSafety(pool, requestRoles, exposedSchemas, dependencySchemas); +}; diff --git a/graphql/server/src/middleware/types.ts b/graphql/server/src/middleware/types.ts index 5b0868f764..911939baf6 100644 --- a/graphql/server/src/middleware/types.ts +++ b/graphql/server/src/middleware/types.ts @@ -15,6 +15,10 @@ declare global { interface Request { api?: ApiStructure; svc_key?: string; + /** Opaque physical routing-cache identity; never used as a service label. */ + svc_cache_key?: string; + /** True only after constant-time authentication of the internal request token. */ + internalTrusted?: boolean; clientIp?: string; databaseId?: string; requestId?: string; diff --git a/graphql/server/src/plugins/__tests__/websocket-operation-admission-plugin.test.ts b/graphql/server/src/plugins/__tests__/websocket-operation-admission-plugin.test.ts new file mode 100644 index 0000000000..eee6885278 --- /dev/null +++ b/graphql/server/src/plugins/__tests__/websocket-operation-admission-plugin.test.ts @@ -0,0 +1,234 @@ +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; + +import type { Request } from 'express'; +import type { GraphileCacheEntry } from 'graphile-cache'; + +import { + createGraphileWebSocketOperationAdmission, + GRAPHILE_WEBSOCKET_CAPTCHA_REQUIRED_CODE, + GRAPHILE_WEBSOCKET_OPERATION_SAFETY_CODE +} from '../websocket-operation-admission-plugin'; + +const contract = { + cacheKey: 'contract-a', + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['a_public'], + authenticatedRole: 'a_user', + anonymousRole: 'a_anon', + dependencySchemas: ['realtime_a'], + runtimeSafetyRequired: true +} as const; + +const makeFixture = (options: { + enableCaptcha?: boolean; + query?: string; + operationName?: string; +} = {}) => { + const socket = new PassThrough(); + const request = Object.assign(new EventEmitter(), { + aborted: false, + socket, + api: { + databaseId: contract.databaseId, + dbname: contract.databaseName, + apiId: contract.apiId, + schema: [...contract.schemas], + roleName: contract.authenticatedRole, + anonRole: contract.anonymousRole, + authSettings: options.enableCaptcha + ? { enableCaptcha: true } + : undefined + } + }) as unknown as Request; + const entry = { + cacheKey: contract.cacheKey, + websocketSockets: new Set([socket]), + disposing: false + } as unknown as GraphileCacheEntry; + const ensureRuntimeSafety = jest.fn(async (): Promise => undefined); + const revalidateRealtimeRole = jest.fn(async (): Promise => true); + const retire = jest.fn((): boolean => true); + const admission = createGraphileWebSocketOperationAdmission(contract, { + ensureRuntimeSafety, + revalidateRealtimeRole, + retire + }); + admission.bind(entry); + const callback = ( + admission.plugin.grafserv?.middleware?.onSubscribe as { + callback: (next: () => unknown, event: unknown) => Promise; + } + ).callback; + const event = { + ctx: { extra: { request } }, + message: { + payload: { + query: options.query ?? 'subscription Events { events { id } }', + operationName: options.operationName + } + } + }; + return { + admission, + callback, + ensureRuntimeSafety, + entry, + event, + request, + retire, + revalidateRealtimeRole, + socket + }; +}; + +describe('Graphile WebSocket per-operation safety admission', () => { + it('revalidates both exact safety boundaries before every operation', async () => { + const fixture = makeFixture(); + const next = jest.fn(() => ({ accepted: true })); + + await expect(fixture.callback(next, fixture.event)).resolves.toEqual({ + accepted: true + }); + await expect(fixture.callback(next, fixture.event)).resolves.toEqual({ + accepted: true + }); + + expect(fixture.ensureRuntimeSafety).toHaveBeenCalledTimes(2); + expect(fixture.revalidateRealtimeRole).toHaveBeenCalledTimes(2); + expect(next).toHaveBeenCalledTimes(2); + expect(fixture.retire).not.toHaveBeenCalled(); + }); + + it('retires the exact generation when runtime-role safety cannot be proved', async () => { + const fixture = makeFixture(); + const failure = new Error('unsafe runtime role'); + fixture.ensureRuntimeSafety.mockRejectedValueOnce(failure); + const next = jest.fn(); + + const result = await fixture.callback(next, fixture.event) as Array<{ + extensions?: Record; + }>; + + expect(result[0]?.extensions?.code).toBe( + GRAPHILE_WEBSOCKET_OPERATION_SAFETY_CODE + ); + expect(fixture.retire).toHaveBeenCalledWith(fixture.entry, failure); + expect(fixture.revalidateRealtimeRole).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + }); + + it('retires the generation when listener-role refresh fails', async () => { + const fixture = makeFixture(); + fixture.revalidateRealtimeRole.mockResolvedValueOnce(false); + const next = jest.fn(); + + const result = await fixture.callback(next, fixture.event) as unknown[]; + + expect(result).toHaveLength(1); + expect(fixture.retire).toHaveBeenCalledTimes(1); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects a request whose routed tenant differs from the bound generation', async () => { + const fixture = makeFixture(); + fixture.request.api.databaseId = 'database-b'; + const next = jest.fn(); + + const result = await fixture.callback(next, fixture.event) as unknown[]; + + expect(result).toHaveLength(1); + expect(fixture.retire).toHaveBeenCalledTimes(1); + expect(fixture.ensureRuntimeSafety).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + }); + + it('does not enter GraphQL when invalidation wins an asynchronous audit race', async () => { + const fixture = makeFixture(); + fixture.ensureRuntimeSafety.mockImplementationOnce(async () => { + fixture.entry.disposing = true; + }); + const next = jest.fn(); + + const result = await fixture.callback(next, fixture.event) as unknown[]; + + expect(result).toHaveLength(1); + expect(next).not.toHaveBeenCalled(); + }); + + it.each([ + [ + 'an aliased root field', + 'mutation Harmless { allowed: signUp }', + 'Harmless' + ], + [ + 'a fragment root field', + `mutation Harmless { ...Protected } + fragment Protected on Mutation { requestPasswordReset }`, + 'Harmless' + ], + [ + 'the selected operation in a multi-operation document', + 'query Safe { viewer { id } } mutation Protected { resetPassword }', + 'Protected' + ] + ])('rejects CAPTCHA-protected WebSocket mutations through %s', async ( + _label, + query, + operationName + ) => { + const fixture = makeFixture({ enableCaptcha: true, query, operationName }); + const next = jest.fn(); + + const result = await fixture.callback(next, fixture.event) as Array<{ + extensions?: Record; + }>; + + expect(result[0]?.extensions?.code).toBe( + GRAPHILE_WEBSOCKET_CAPTCHA_REQUIRED_CODE + ); + expect(fixture.ensureRuntimeSafety).not.toHaveBeenCalled(); + expect(fixture.revalidateRealtimeRole).not.toHaveBeenCalled(); + expect(fixture.retire).not.toHaveBeenCalled(); + expect(next).not.toHaveBeenCalled(); + }); + + it.each([ + ['an ambiguous document', 'query A { viewer { id } } query B { viewer { id } }'], + ['a malformed document', 'mutation {'] + ])('fails closed when a CAPTCHA-enabled WebSocket sends %s', async ( + _label, + query + ) => { + const fixture = makeFixture({ enableCaptcha: true, query }); + const next = jest.fn(); + + const result = await fixture.callback(next, fixture.event) as Array<{ + extensions?: Record; + }>; + + expect(result[0]?.extensions?.code).toBe( + GRAPHILE_WEBSOCKET_CAPTCHA_REQUIRED_CODE + ); + expect(next).not.toHaveBeenCalled(); + }); + + it('keeps ordinary subscriptions available when CAPTCHA is enabled', async () => { + const fixture = makeFixture({ + enableCaptcha: true, + query: 'subscription Events { events { id } }', + operationName: 'Events' + }); + const next = jest.fn(() => ({ accepted: true })); + + await expect(fixture.callback(next, fixture.event)).resolves.toEqual({ + accepted: true + }); + expect(fixture.ensureRuntimeSafety).toHaveBeenCalledTimes(1); + expect(fixture.revalidateRealtimeRole).toHaveBeenCalledTimes(1); + expect(next).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphql/server/src/plugins/websocket-operation-admission-plugin.ts b/graphql/server/src/plugins/websocket-operation-admission-plugin.ts new file mode 100644 index 0000000000..6241151193 --- /dev/null +++ b/graphql/server/src/plugins/websocket-operation-admission-plugin.ts @@ -0,0 +1,212 @@ +import type { Request } from 'express'; +import { + type GraphileCacheEntry, + isEntryRealtimeUnavailable, + retireGraphileCacheEntry, + revalidateEntryRealtimeRole +} from 'graphile-cache'; +import type { GraphileConfig } from 'graphile-config'; +import { GraphQLError } from 'graphql'; + +import { inspectCaptchaOperation } from '../middleware/captcha'; +import { ensureRuntimeRoleSafety } from '../middleware/runtime-role-safety'; + +export const GRAPHILE_WEBSOCKET_OPERATION_SAFETY_CODE = + 'GRAPHILE_WEBSOCKET_OPERATION_SAFETY_FAILED'; +export const GRAPHILE_WEBSOCKET_CAPTCHA_REQUIRED_CODE = 'CAPTCHA_REQUIRED'; + +export interface GraphileWebSocketOperationContract { + cacheKey: string; + databaseId: string; + databaseName: string; + apiId: string; + schemas: readonly string[]; + authenticatedRole: string; + anonymousRole: string; + dependencySchemas: readonly string[]; + runtimeSafetyRequired: boolean; +} + +interface OperationAdmissionDependencies { + ensureRuntimeSafety(entry: GraphileCacheEntry): Promise; + revalidateRealtimeRole(entry: GraphileCacheEntry): Promise; + retire(entry: GraphileCacheEntry, error: unknown): boolean; +} + +export interface GraphileWebSocketOperationAdmission { + readonly plugin: GraphileConfig.Plugin; + bind(entry: GraphileCacheEntry): void; +} + +const sameStrings = ( + left: readonly string[] | undefined, + right: readonly string[] +): boolean => Boolean( + left + && left.length === right.length + && left.every((value, index) => value === right[index]) +); + +const operationRequest = (event: { + ctx?: { extra?: unknown }; +}): Request | undefined => { + const extra = event.ctx?.extra as { request?: Request } | undefined; + return extra?.request; +}; + +const requestMatchesContract = ( + request: Request, + entry: GraphileCacheEntry, + contract: Readonly +): boolean => { + const api = request.api; + return Boolean( + api + && (api.databaseId ?? '') === contract.databaseId + && api.dbname === contract.databaseName + && (api.apiId ?? '') === contract.apiId + && api.roleName === contract.authenticatedRole + && api.anonRole === contract.anonymousRole + && sameStrings(api.schema, contract.schemas) + && entry.cacheKey === contract.cacheKey + && entry.websocketSockets?.has(request.socket) + ); +}; + +const unavailable = (): readonly GraphQLError[] => [ + new GraphQLError('WebSocket operation safety could not be verified', { + extensions: { code: GRAPHILE_WEBSOCKET_OPERATION_SAFETY_CODE } + }) +]; + +const captchaRequired = (): readonly GraphQLError[] => [ + new GraphQLError('CAPTCHA-protected mutations must use the HTTP endpoint', { + extensions: { code: GRAPHILE_WEBSOCKET_CAPTCHA_REQUIRED_CODE } + }) +]; + +const defaultDependencies = ( + contract: Readonly +): OperationAdmissionDependencies => ({ + ensureRuntimeSafety: async (entry) => { + if (!contract.runtimeSafetyRequired) return; + const pool = entry.poolLease?.pool; + if (!pool) { + throw new Error('Resident Graphile generation has no retained runtime pool'); + } + await ensureRuntimeRoleSafety( + pool, + [contract.anonymousRole, contract.authenticatedRole], + [...contract.schemas], + [...contract.dependencySchemas] + ); + }, + revalidateRealtimeRole: revalidateEntryRealtimeRole, + retire: retireGraphileCacheEntry +}); + +/** + * Bind Grafserv's per-operation WebSocket hook to one exact cache generation. + * The initial HTTP upgrade admission remains authoritative for routing and + * authentication; this hook prevents a long-lived socket from bypassing later + * role or listener-attestation checks when it starts another operation. + */ +export const createGraphileWebSocketOperationAdmission = ( + contract: Readonly, + dependencies: OperationAdmissionDependencies = defaultDependencies(contract) +): GraphileWebSocketOperationAdmission => { + const expected = Object.freeze({ + ...contract, + schemas: Object.freeze([...contract.schemas]), + dependencySchemas: Object.freeze([...contract.dependencySchemas]) + }); + let entry: GraphileCacheEntry | null = null; + + const reject = (error: unknown): readonly GraphQLError[] => { + if (entry) dependencies.retire(entry, error); + return unavailable(); + }; + + const plugin: GraphileConfig.Plugin = { + name: 'ConstructiveWebSocketOperationAdmissionPlugin', + version: '1.0.0', + grafserv: { + middleware: { + onSubscribe: { + callback: async (next, event) => { + const current = entry; + const request = operationRequest(event); + if (!current || !request) { + return reject(new Error('WebSocket operation has no bound generation')); + } + if ( + request.aborted + || request.socket.destroyed + || current.disposing + ) { + return unavailable(); + } + if (!requestMatchesContract(request, current, expected)) { + return reject(new Error( + 'WebSocket operation request does not match its bound generation' + )); + } + + if (request.api?.authSettings?.enableCaptcha) { + const message = (event as { + message?: { + payload?: { query?: unknown; operationName?: unknown }; + }; + }).message; + const inspection = inspectCaptchaOperation( + message?.payload?.query, + message?.payload?.operationName + ); + // CAPTCHA tokens are verified by the HTTP middleware. Protected + // mutations and documents we cannot classify never reach GraphQL + // over WebSocket, so a transport switch cannot bypass the gate. + if (inspection.kind !== 'not-protected') return captchaRequired(); + } + + try { + await dependencies.ensureRuntimeSafety(current); + const attested = await dependencies.revalidateRealtimeRole(current); + if (!attested || isEntryRealtimeUnavailable(current)) { + throw new Error( + 'WebSocket operation listener-role attestation is unavailable' + ); + } + } catch (error) { + return reject(error); + } + + // Schema invalidation or broker failure may retire the generation + // while either asynchronous audit is running. + if ( + request.aborted + || request.socket.destroyed + || current.disposing + || isEntryRealtimeUnavailable(current) + ) { + return unavailable(); + } + return next(); + } + } + } + } + }; + + return Object.freeze({ + plugin, + bind(candidate: GraphileCacheEntry): void { + if (candidate.cacheKey !== expected.cacheKey) { + throw new Error('WebSocket operation admission cache key mismatch'); + } + if (entry && entry !== candidate) { + throw new Error('WebSocket operation admission is already bound'); + } + entry = candidate; + } + }); +}; diff --git a/graphql/server/src/server.ts b/graphql/server/src/server.ts index 8ddd11c483..4e3b3fcfc2 100644 --- a/graphql/server/src/server.ts +++ b/graphql/server/src/server.ts @@ -1,18 +1,39 @@ import { createCsrfMiddleware } from '@constructive-io/csrf'; -import { createContextMiddleware, createDefaultRegistry, requestIdMiddleware } from '@constructive-io/express-context'; +import { + createContextMiddleware, + createDefaultRegistry, + type LoaderRegistry, + requestIdMiddleware +} from '@constructive-io/express-context'; import { getEnvOptions } from '@constructive-io/graphql-env'; import type { ConstructiveOptions } from '@constructive-io/graphql-types'; import { middleware as parseDomains } from '@constructive-io/url-domains'; +import { getNodeEnv } from '@pgpmjs/env'; import { Logger } from '@pgpmjs/logger'; -import { healthz, poweredBy, svcCache, trustProxy } from '@pgpmjs/server-utils'; +import { + configureSvcCache, + healthz, + poweredBy, + trustProxy +} from '@pgpmjs/server-utils'; import { PgpmOptions } from '@pgpmjs/types'; import cookieParser from 'cookie-parser'; import express, { Express, NextFunction, Request, RequestHandler, Response } from 'express'; -import { closeAllCaches,graphileCache } from 'graphile-cache'; +import { + clearGraphileCache, + closeAllCaches, + getCacheConfig, + startMemoryGovernor +} from 'graphile-cache'; import graphqlUpload from 'graphql-upload'; import type { Server as HttpServer } from 'http'; -import { Pool, PoolClient } from 'pg'; -import { getPgPool } from 'pg-cache'; +import { type Notification, Pool, type PoolClient } from 'pg'; +import { + acquirePgPool, + getPgPool, + PgPoolCapacityError, + type PgPoolLease +} from 'pg-cache'; import requestIp from 'request-ip'; import { createAgenticRouter } from './agentic'; @@ -20,31 +41,184 @@ import { closeDebugDatabasePools } from './diagnostics/debug-db-snapshot'; import type { DebugSamplerHandle } from './diagnostics/debug-sampler'; import { startDebugSampler } from './diagnostics/debug-sampler'; import { + getGraphqlObservabilityToken, isDevelopmentObservabilityMode, isGraphqlObservabilityEnabled, isGraphqlObservabilityRequested, isLoopbackHost } from './diagnostics/observability'; -import { createApiMiddleware } from './middleware/api'; +import { clearSvcCache, createApiMiddleware } from './middleware/api'; import { createAuthenticateMiddleware } from './middleware/auth'; // Auth cookie handling is done via AuthCookiePlugin in grafserv -import { createCaptchaMiddleware } from './middleware/captcha'; +import { + createCaptchaGraphqlBodyParsers, + createCaptchaMiddleware +} from './middleware/captcha'; import { parseCookieValue, SESSION_COOKIE_NAME } from './middleware/cookie'; import { cors } from './middleware/cors'; import { errorHandler, notFoundHandler } from './middleware/error-handler'; import { favicon } from './middleware/favicon'; -import { flush, flushService } from './middleware/flush'; +import { createFlushMiddleware, flushService } from './middleware/flush'; import { createFnRouter } from './middleware/fn'; import { graphile } from './middleware/graphile'; +import { + closeGraphileBuildCoordinator, + getGraphileGovernorCounters, + GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE, + reopenGraphileBuildCoordinator +} from './middleware/graphile-build-governor'; +import { assertInternalRequestSecret } from './middleware/internal-request'; import { multipartBridge } from './middleware/multipart-bridge'; import { createDebugDatabaseMiddleware } from './middleware/observability/debug-db'; import { debugMemory } from './middleware/observability/debug-memory'; import { localObservabilityOnly } from './middleware/observability/guard'; import { createRequestLogger } from './middleware/observability/request-logger'; +import { + addRealtimeRuntimeDependencySchema, + resolveGraphileRealtimeSchema +} from './middleware/realtime-config'; import { getRoutingSchema } from './middleware/routing'; +import { createRuntimePgResolutionStore } from './middleware/runtime-pg-config'; +import { + assertRuntimePgCredentials, + shouldValidateRuntimeRoleSafety +} from './middleware/runtime-pg-requirements'; +import { ensureRuntimeRoleSafety } from './middleware/runtime-role-safety'; +import { + createGraphileWebSocketOriginGuard, + createGraphileWebSocketUpgradeGateway, + type GraphileWebSocketUpgradeGateway +} from './websocket-upgrade'; const log = new Logger('server'); +export const GRAPHILE_CACHE_SHUTDOWN_DRAIN_TIMEOUT_CODE = + 'GRAPHILE_CACHE_SHUTDOWN_DRAIN_TIMEOUT'; +export const GRAPHILE_CACHE_SHUTDOWN_RESTART_REQUIRED_CODE = + 'GRAPHILE_CACHE_SHUTDOWN_RESTART_REQUIRED'; + +export class GraphileCacheShutdownError extends Error { + constructor( + readonly code: + | typeof GRAPHILE_CACHE_SHUTDOWN_DRAIN_TIMEOUT_CODE + | typeof GRAPHILE_CACHE_SHUTDOWN_RESTART_REQUIRED_CODE, + message: string + ) { + super(message); + this.name = 'GraphileCacheShutdownError'; + } +} + +// A process-wide cache clear owns the process-wide build coordinator. Coalesce +// concurrent callers so no invocation can reopen admission while another is +// still disposing residents or closing their pools. +let processCacheClose: Promise | null = null; +let processCacheClosePoolsRequested = false; + +const once = ( + callback: (...args: Args) => void +): ((...args: Args) => void) => { + let called = false; + return (...args: Args) => { + if (called) return; + called = true; + callback(...args); + }; +}; + +interface ListenAttempt { + releasePoolLease: () => void; + client: PoolClient | null; + releaseClient: ((error?: Error | boolean) => void) | null; + notificationHandler: ((message: Notification) => void) | null; + errorHandler: ((error: Error) => void) | null; + closed: boolean; + cleanupPromise: Promise | null; +} + +const PROCESS_SHUTDOWN_SIGNALS = ['SIGINT', 'SIGTERM'] as const; + +/** @internal Process seam used by the executable shutdown boundary and its tests. */ +export interface ProcessShutdownTarget { + on(signal: NodeJS.Signals, listener: () => void): unknown; + removeListener(signal: NodeJS.Signals, listener: () => void): unknown; + exit(code?: number): void; +} + +export interface ProcessShutdownOptions { + timeoutMs?: number; + processTarget?: ProcessShutdownTarget; +} + +/** + * Install process-level shutdown ownership at the executable boundary. + * A second signal forces exit, while the first gets a bounded graceful drain. + */ +export const installProcessShutdownHandlers = ( + shutdown: () => Promise, + options: ProcessShutdownOptions = {} +): (() => void) => { + const { timeoutMs = 30_000, processTarget = process } = options; + let started = false; + let finished = false; + let timeout: ReturnType | null = null; + const listeners = new Map void>(); + + const uninstall = (): void => { + for (const [signal, listener] of listeners) { + processTarget.removeListener(signal, listener); + } + listeners.clear(); + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + }; + + const finish = (exitCode: number): void => { + if (finished) return; + finished = true; + uninstall(); + processTarget.exit(exitCode); + }; + + const beginShutdown = (signal: NodeJS.Signals): void => { + if (started) { + log.warn(`Received ${signal} while shutdown is in progress; forcing exit`); + finish(1); + return; + } + started = true; + log.info(`Received ${signal}; draining GraphQL server resources`); + timeout = setTimeout(() => { + log.error(`GraphQL server shutdown exceeded ${timeoutMs}ms; forcing exit`); + finish(1); + }, Math.max(1, timeoutMs)); + timeout.unref?.(); + + let shutdownPromise: Promise; + try { + shutdownPromise = shutdown(); + } catch (error) { + shutdownPromise = Promise.reject(error); + } + void shutdownPromise.then( + () => finish(0), + (error) => { + log.error('GraphQL server shutdown failed', error); + finish(1); + } + ); + }; + + for (const signal of PROCESS_SHUTDOWN_SIGNALS) { + const listener = (): void => beginShutdown(signal); + listeners.set(signal, listener); + processTarget.on(signal, listener); + } + return uninstall; +}; + /** * Creates and starts a GraphQL server instance * @@ -71,26 +245,50 @@ export const GraphQLServer = (rawOpts: ConstructiveOptions | PgpmOptions = {}) = const app = new Server(opts); app.addEventListener(); app.listen(); + installProcessShutdownHandlers(() => app.close({ closeCaches: true })); }; class Server { private app: Express; private opts: ConstructiveOptions; - private listenClient: PoolClient | null = null; - private listenRelease: (() => void) | null = null; + private listenAttempt: ListenAttempt | null = null; + private listenRetryTimer: ReturnType | null = null; + private readonly listenCleanupTasks = new Set>(); private shuttingDown = false; private closed = false; private httpServer: HttpServer | null = null; private debugSampler: DebugSamplerHandle | null = null; + private stopMemoryGovernor: (() => void) | null = null; + private websocketUpgradeGateway: GraphileWebSocketUpgradeGateway | null = null; + private readonly moduleRegistry: LoaderRegistry; constructor(opts: ConstructiveOptions) { + if (!reopenGraphileBuildCoordinator()) { + log.warn( + 'GraphQL schema build admission remains closed because a previous generation is still draining' + ); + } this.opts = getEnvOptions(opts); + this.moduleRegistry = createDefaultRegistry(); const effectiveOpts = this.opts; + assertInternalRequestSecret(effectiveOpts); + const residentGraphileCapacity = getCacheConfig().max; + const routingCache = configureSvcCache({ + maxEntries: effectiveOpts.routingCache?.maxEntries, + minimumEntries: residentGraphileCapacity + }); + assertRuntimePgCredentials(effectiveOpts, getNodeEnv()); + const validateRuntimeRole = shouldValidateRuntimeRoleSafety( + effectiveOpts, + getNodeEnv() + ); const observabilityRequested = isGraphqlObservabilityRequested(); const observabilityEnabled = isGraphqlObservabilityEnabled(effectiveOpts.server?.host); + const runtimePgResolutions = createRuntimePgResolutionStore(effectiveOpts); const app = express(); - const api = createApiMiddleware(effectiveOpts); + this.stopMemoryGovernor = startMemoryGovernor(); + const api = createApiMiddleware(effectiveOpts, this.moduleRegistry); const authenticate = createAuthenticateMiddleware(effectiveOpts); const requestLogger = createRequestLogger({ observabilityEnabled }); @@ -105,13 +303,20 @@ class Server { apiIsPublic: apiOpts.isPublic, routingSchema: apiOpts.routingSchema, metaSchemas: apiOpts.metaSchemas?.join(',') || 'default', + routingCacheMaxEntries: routingCache.max, + residentGraphileCapacity, observabilityEnabled }); if (observabilityRequested && !observabilityEnabled) { const reasons = []; - if (!isDevelopmentObservabilityMode()) { - reasons.push('NODE_ENV must be development'); + if ( + !isDevelopmentObservabilityMode() + && !getGraphqlObservabilityToken() + ) { + reasons.push( + 'NODE_ENV must be development or GRAPHQL_OBSERVABILITY_TOKEN must contain at least 32 bytes' + ); } if (!isLoopbackHost(effectiveOpts.server?.host)) { reasons.push('server host must be localhost, 127.0.0.1, or ::1'); @@ -124,6 +329,21 @@ class Server { ); } + // Keep the generic health endpoint reusable, but fail this server's probe + // once the build watchdog has latched. Orchestrators can then replace the + // process; admitting a second build in-process would overlap an unknown + // amount of retained work from the stuck generation. + app.get('/healthz', (_req, res, next) => { + const governor = getGraphileGovernorCounters(); + if (!governor.restartRequired) { + next(); + return; + } + res.status(503).json({ + status: 'unhealthy', + code: GRAPHILE_BUILD_STUCK_RESTART_REQUIRED_CODE + }); + }); healthz(app); if (observabilityEnabled) { app.get('/debug/memory', localObservabilityOnly, debugMemory); @@ -150,6 +370,7 @@ class Server { app.use(poweredBy('constructive')); app.use(cookieParser()); app.use(cors(fallbackOrigin)); + app.use('/graphql', ...createCaptchaGraphqlBodyParsers()); app.use('/graphql', graphqlUpload.graphqlUploadExpress({ maxFileSize: 10 * 1024 * 1024, // 10 MB maxFiles: 10 @@ -162,13 +383,38 @@ class Server { app.use(requestIdMiddleware()); app.use(requestLogger); app.use(api); + // Browser WebSockets do not enforce CORS. Reject an untrusted Origin after + // exact tenant routing but before auth or any tenant-specific module I/O. + app.use(createGraphileWebSocketOriginGuard(fallbackOrigin)); app.use(authenticate); + app.use(runtimePgResolutions.middleware); app.use(createContextMiddleware({ pg: effectiveOpts.pg, - loaders: createDefaultRegistry(), + getRuntimePgResolution: runtimePgResolutions.getRuntimePgResolution, + dependencySchemas: effectiveOpts.graphile?.introspectionDependencySchemas, + validateRuntimePool: validateRuntimeRole + ? (pool, resolvedApi) => { + const realtimeSchema = resolveGraphileRealtimeSchema( + effectiveOpts, + resolvedApi.databaseSettings?.enableRealtime ?? false + ); + return ensureRuntimeRoleSafety( + pool, + [resolvedApi.anonRole, resolvedApi.roleName], + resolvedApi.schema, + addRealtimeRuntimeDependencySchema( + effectiveOpts.graphile?.introspectionDependencySchemas ?? [], + realtimeSchema + ) + ); + } + : undefined, + loaders: this.moduleRegistry, routingSchema: getRoutingSchema(effectiveOpts) })); - app.use(createCaptchaMiddleware()); + app.use(createCaptchaMiddleware({ + strictAuth: effectiveOpts.server?.strictAuth + })); // CSRF protection for cookie-authenticated requests // Skip CSRF for Bearer token auth (not vulnerable to CSRF) and anonymous requests @@ -206,14 +452,18 @@ class Server { // REST function invocation routes (POST /fn/:alias, GET /fn/invocations/:id) app.use(createFnRouter()); - app.use(graphile(effectiveOpts)); - app.use(flush); + app.use(graphile( + effectiveOpts, + runtimePgResolutions.getRuntimePgResolution + )); + app.use(createFlushMiddleware(this.moduleRegistry)); // Error handling - MUST be LAST app.use(notFoundHandler); // Catches unmatched routes (404) app.use(errorHandler); // Catches all thrown errors this.app = app; + this.websocketUpgradeGateway = createGraphileWebSocketUpgradeGateway(app); this.debugSampler = observabilityEnabled ? startDebugSampler(effectiveOpts) : null; } @@ -231,84 +481,227 @@ class Server { } throw err; }); + if (!this.websocketUpgradeGateway) { + throw new Error('Graphile WebSocket upgrade gateway is unavailable'); + } + httpServer.on('upgrade', this.websocketUpgradeGateway.handle); this.httpServer = httpServer; return httpServer; } async flush(databaseId: string): Promise { - await flushService(this.opts, databaseId); + await flushService(this.opts, databaseId, this.moduleRegistry); + } + + /** + * LISTEN delivery has no replay. Clear every local metadata publication when + * the listener is lost and again after LISTEN succeeds, so a missed change + * cannot extend a cached module value past reconnection. Security-sensitive + * auth/RLS loaders are additionally uncached and do not depend on this path. + */ + private invalidateConfigurationCaches(reason: string): void { + clearSvcCache(); + this.moduleRegistry.invalidate(); + log.info(`Invalidated configuration caches after notification ${reason}`); } getPool(): Pool { - return getPgPool(this.opts.pg); + return getPgPool(this.opts.pg, { purpose: 'control' }); } - addEventListener(): void { - if (this.shuttingDown) return; - const pgPool = this.getPool(); - pgPool.connect(this.listenForChanges.bind(this)); + private clearListenRetry(): void { + if (!this.listenRetryTimer) return; + clearTimeout(this.listenRetryTimer); + this.listenRetryTimer = null; } - listenForChanges(err: Error | null, client: PoolClient, release: () => void): void { - if (err) { - this.error('Error connecting with notify listener', err); + private scheduleListenRetry(delayMs: number): void { + if (this.shuttingDown || this.listenRetryTimer || this.listenAttempt) return; + this.listenRetryTimer = setTimeout(() => { + this.listenRetryTimer = null; + this.addEventListener(); + }, delayMs); + this.listenRetryTimer.unref?.(); + } + + private cleanupListenAttempt( + attempt: ListenAttempt, + unlisten: boolean, + connectionError?: Error + ): Promise { + if (attempt.cleanupPromise) return attempt.cleanupPromise; + attempt.closed = true; + if (this.listenAttempt === attempt) this.listenAttempt = null; + + const pending = (async () => { + const client = attempt.client; + if (client && attempt.notificationHandler) { + client.removeListener('notification', attempt.notificationHandler); + } + if (client && attempt.errorHandler) { + client.removeListener('error', attempt.errorHandler); + } + let clientReleaseError = connectionError; + if (client && unlisten) { + try { + // node-postgres serializes queries on one client. This also safely + // queues behind an in-progress LISTEN during a shutdown race. + await client.query('UNLISTEN "schema:update"'); + } catch (error) { + // The connection may already be unusable; release still must run. + clientReleaseError ??= error instanceof Error + ? error + : new Error(String(error)); + } + } + let releaseError: unknown; + try { + attempt.releaseClient?.(clientReleaseError); + } catch (error) { + releaseError = error; + } + attempt.releaseClient = null; + try { + attempt.releasePoolLease(); + } catch (error) { + releaseError ??= error; + } + if (releaseError) this.error('Error releasing database notify listener', releaseError); + })(); + attempt.cleanupPromise = pending; + this.listenCleanupTasks.add(pending); + void pending.then( + () => this.listenCleanupTasks.delete(pending), + () => this.listenCleanupTasks.delete(pending) + ); + return pending; + } + + addEventListener(): void { + if (this.shuttingDown || this.listenAttempt) return; + this.clearListenRetry(); + let lease: PgPoolLease; + try { + // LISTEN owns a client for the process lifetime. Give it a distinct + // identity so a one-client routing pool remains available to ordinary + // control-plane requests instead of being permanently starved. + lease = acquirePgPool(this.opts.pg, { purpose: 'notifications' }); + } catch (error) { + this.error('Error acquiring pool for notify listener', error); if (!this.shuttingDown) { - setTimeout(() => this.addEventListener(), 5000); + const retryMs = error instanceof PgPoolCapacityError + ? error.retryAfterSeconds * 1000 + : 5000; + this.scheduleListenRetry(retryMs); } return; } + const attempt: ListenAttempt = { + releasePoolLease: once(() => lease.release()), + client: null, + releaseClient: null, + notificationHandler: null, + errorHandler: null, + closed: false, + cleanupPromise: null + }; + this.listenAttempt = attempt; + lease.pool.connect((err, client, release) => { + void this.listenForChanges( + err ?? null, + client as PoolClient | undefined, + release as ((error?: Error | boolean) => void) | undefined, + attempt + ).catch(async (error) => { + this.error('Unexpected notify listener setup failure', error); + await this.cleanupListenAttempt( + attempt, + false, + error instanceof Error ? error : new Error(String(error)) + ); + this.scheduleListenRetry(5000); + }); + }); + } - if (this.shuttingDown) { - release(); + private async listenForChanges( + err: Error | null, + client: PoolClient | undefined, + release: ((error?: Error | boolean) => void) | undefined, + attempt: ListenAttempt + ): Promise { + if (attempt.closed || this.listenAttempt !== attempt || this.shuttingDown) { + release?.(); + attempt.releasePoolLease(); return; } - this.listenClient = client; - this.listenRelease = release; + if (err) { + this.error('Error connecting with notify listener', err); + this.invalidateConfigurationCaches('connection failure'); + await this.cleanupListenAttempt(attempt, false); + this.scheduleListenRetry(5000); + return; + } - client.on('notification', ({ channel, payload }) => { + if (!client || !release) { + this.error('Notify listener connected without a client release handle'); + this.invalidateConfigurationCaches('invalid checkout'); + await this.cleanupListenAttempt(attempt, false); + this.scheduleListenRetry(5000); + return; + } + + attempt.client = client; + attempt.releaseClient = once(release); + attempt.notificationHandler = ({ channel, payload }) => { if (channel === 'schema:update' && payload) { log.info('schema:update', payload); - this.flush(payload); + void this.flush(payload).catch((error) => { + this.error('Error flushing schema:update notification', error); + }); } - }); - - client.query('LISTEN "schema:update"'); - - client.on('error', (e) => { - if (this.shuttingDown) { - release(); - return; - } - this.error('Error with database notify listener', e); - release(); - this.addEventListener(); - }); + }; + attempt.errorHandler = (error) => { + if (attempt.closed) return; + if (!this.shuttingDown) this.error('Error with database notify listener', error); + this.invalidateConfigurationCaches('connection loss'); + void this.cleanupListenAttempt(attempt, false, error).then(() => { + this.scheduleListenRetry(5000); + }); + }; + client.on('notification', attempt.notificationHandler); + client.on('error', attempt.errorHandler); + try { + await client.query('LISTEN "schema:update"'); + } catch (error) { + this.error('Error starting database notify listener', error); + this.invalidateConfigurationCaches('LISTEN failure'); + await this.cleanupListenAttempt( + attempt, + false, + error instanceof Error ? error : new Error(String(error)) + ); + this.scheduleListenRetry(5000); + return; + } + if (attempt.closed || this.listenAttempt !== attempt || this.shuttingDown) { + await this.cleanupListenAttempt(attempt, true); + return; + } + this.invalidateConfigurationCaches('reconnect'); this.log('connected and listening for changes...'); } async removeEventListener(): Promise { - if (!this.listenClient || !this.listenRelease) { - return; + this.clearListenRetry(); + const attempt = this.listenAttempt; + if (attempt) await this.cleanupListenAttempt(attempt, true); + if (this.listenCleanupTasks.size > 0) { + await Promise.allSettled([...this.listenCleanupTasks]); } - - const client = this.listenClient; - const release = this.listenRelease; - this.listenClient = null; - this.listenRelease = null; - - client.removeAllListeners('notification'); - client.removeAllListeners('error'); - - try { - await client.query('UNLISTEN "schema:update"'); - } catch { - // Ignore listener cleanup errors during shutdown. - } - - release(); } async close(opts: { closeCaches?: boolean } = {}): Promise { @@ -321,30 +714,85 @@ class Server { } this.closed = true; this.shuttingDown = true; + // Only process-wide cache shutdown owns the process-global build + // coordinator. Closing one exported Server must not disable cold builds in + // another Server instance in the same process. + const buildDrain = closeCaches + ? closeGraphileBuildCoordinator() + : Promise.resolve(true); await this.removeEventListener(); + this.moduleRegistry.invalidate(); if (this.debugSampler) { await this.debugSampler.stop(); this.debugSampler = null; } + if (this.stopMemoryGovernor) { + this.stopMemoryGovernor(); + this.stopMemoryGovernor = null; + } + if (this.httpServer && this.websocketUpgradeGateway) { + this.httpServer.off('upgrade', this.websocketUpgradeGateway.handle); + } + this.websocketUpgradeGateway?.close(); if (this.httpServer?.listening) { await new Promise((resolve) => this.httpServer!.close(() => resolve())); } + const buildsDrained = await buildDrain; + if (!buildsDrained) { + log.warn( + 'GraphQL schema builds exceeded the shutdown drain deadline; late publication is disabled' + ); + } await closeDebugDatabasePools(); if (closeCaches) { await Server.closeCaches({ closePools: true }); + if (buildsDrained) reopenGraphileBuildCoordinator(); } } static async closeCaches(opts: { closePools?: boolean } = {}): Promise { - const { closePools = false } = opts; - svcCache.clear(); - // Use closeAllCaches to properly await async disposal of PostGraphile instances - // before closing pg pools - this ensures all connections are released - if (closePools) { - await closeAllCaches(); - } else { - graphileCache.clear(); + processCacheClosePoolsRequested ||= opts.closePools === true; + if (!processCacheClose) { + const closeTask = (async (): Promise => { + const buildsDrained = await closeGraphileBuildCoordinator(); + if (!buildsDrained) { + throw new GraphileCacheShutdownError( + GRAPHILE_CACHE_SHUTDOWN_DRAIN_TIMEOUT_CODE, + 'GraphQL schema builds did not drain; caches and pools were left intact' + ); + } + + clearSvcCache(); + let poolsClosed = false; + if (processCacheClosePoolsRequested) { + await closeAllCaches(); + poolsClosed = true; + } else { + await clearGraphileCache(); + } + // A concurrent closeCaches({ closePools: true }) may have joined while + // the resident-only clear was awaiting disposal. Honor that escalation + // before build admission can reopen. + if (processCacheClosePoolsRequested && !poolsClosed) { + await closeAllCaches(); + } + + if (!reopenGraphileBuildCoordinator()) { + throw new GraphileCacheShutdownError( + GRAPHILE_CACHE_SHUTDOWN_RESTART_REQUIRED_CODE, + 'GraphQL build admission cannot reopen safely; process restart is required' + ); + } + })(); + const tracked = closeTask.finally(() => { + if (processCacheClose === tracked) { + processCacheClose = null; + processCacheClosePoolsRequested = false; + } + }); + processCacheClose = tracked; } + return processCacheClose!; } log(text: string): void { diff --git a/graphql/server/src/websocket-upgrade.ts b/graphql/server/src/websocket-upgrade.ts new file mode 100644 index 0000000000..6bd3f5adcb --- /dev/null +++ b/graphql/server/src/websocket-upgrade.ts @@ -0,0 +1,447 @@ +import { type IncomingMessage,ServerResponse, STATUS_CODES } from 'node:http'; +import type { Socket } from 'node:net'; +import { type Duplex,PassThrough } from 'node:stream'; + +import type { + Express, + NextFunction, + Request, + RequestHandler, + Response +} from 'express'; + +import { parseCookieValue, SESSION_COOKIE_NAME } from './middleware/cookie'; +import { isCorsOriginAllowed } from './middleware/cors'; + +export const GRAPHILE_WEBSOCKET_PATH = '/graphql'; +export const GRAPHILE_WEBSOCKET_ROUTE_NOT_FOUND_CODE = + 'GRAPHILE_WEBSOCKET_ROUTE_NOT_FOUND'; +export const GRAPHILE_WEBSOCKET_BAD_UPGRADE_CODE = + 'GRAPHILE_WEBSOCKET_BAD_UPGRADE'; +export const GRAPHILE_WEBSOCKET_ADMISSION_TIMEOUT_CODE = + 'GRAPHILE_WEBSOCKET_ADMISSION_TIMEOUT'; +export const GRAPHILE_WEBSOCKET_ADMISSION_FAILED_CODE = + 'GRAPHILE_WEBSOCKET_ADMISSION_FAILED'; +export const GRAPHILE_WEBSOCKET_AUTH_REJECTED_CODE = + 'GRAPHILE_WEBSOCKET_AUTH_REJECTED'; +export const GRAPHILE_WEBSOCKET_SERVER_CLOSING_CODE = + 'GRAPHILE_WEBSOCKET_SERVER_CLOSING'; + +const DEFAULT_ADMISSION_TIMEOUT_MS = 180_000; + +interface UpgradeResponse { + status: number; + code: string; + retryAfterSeconds?: number; +} + +interface PendingUpgrade { + readonly request: IncomingMessage; + readonly socket: Duplex; + readonly head: Buffer; + readonly response: ServerResponse; + readonly responseSocket: Socket; + readonly timer: ReturnType; + readonly onSocketClose: () => void; + readonly onSocketError: () => void; + readonly onResponseFinish: () => void; + readonly onResponseClose: () => void; + readonly removePending: () => void; + handedOff: boolean; + terminal: boolean; +} + +const pendingByRequest = new WeakMap(); + +const safeStatus = (status: number): number => + Number.isSafeInteger(status) && status >= 400 && status <= 599 ? status : 500; + +const reasonPhrase = (status: number): string => + STATUS_CODES[status] ?? 'Error'; + +const writeUpgradeResponse = ( + socket: Duplex, + response: UpgradeResponse +): void => { + if (socket.destroyed || !socket.writable) return; + const status = safeStatus(response.status); + const body = JSON.stringify({ error: { code: response.code } }); + const headers = [ + `HTTP/1.1 ${status} ${reasonPhrase(status)}`, + 'Connection: close', + 'Content-Type: application/json; charset=utf-8', + `Content-Length: ${Buffer.byteLength(body)}`, + ...(response.retryAfterSeconds == null + ? [] + : [`Retry-After: ${response.retryAfterSeconds}`]), + '', + body + ].join('\r\n'); + try { + socket.end(headers); + } catch { + socket.destroy(); + } +}; + +const websocketPath = (request: IncomingMessage): string => { + const raw = request.url ?? ''; + const queryStart = raw.indexOf('?'); + return queryStart < 0 ? raw : raw.slice(0, queryStart); +}; + +const headerContainsToken = ( + value: string | string[] | undefined, + expected: string +): boolean => { + const values = Array.isArray(value) ? value : value == null ? [] : [value]; + return values.some((item) => + item.split(',').some((token) => token.trim().toLowerCase() === expected) + ); +}; + +const isGraphileWebSocketRequest = (request: IncomingMessage): boolean => + request.method === 'GET' + && websocketPath(request) === GRAPHILE_WEBSOCKET_PATH + && headerContainsToken(request.headers.connection, 'upgrade') + && headerContainsToken(request.headers.upgrade, 'websocket'); + +const responseFailure = (response: ServerResponse): UpgradeResponse => { + const status = safeStatus(response.statusCode); + const retryAfterValue = response.getHeader('Retry-After'); + const parsedRetryAfter = typeof retryAfterValue === 'string' + ? Number.parseInt(retryAfterValue, 10) + : typeof retryAfterValue === 'number' + ? retryAfterValue + : undefined; + const retryAfterSeconds = Number.isSafeInteger(parsedRetryAfter) + && (parsedRetryAfter as number) >= 0 + ? parsedRetryAfter + : undefined; + return { + status, + code: status === 401 || status === 403 + ? GRAPHILE_WEBSOCKET_AUTH_REJECTED_CODE + : status === 404 + ? GRAPHILE_WEBSOCKET_ROUTE_NOT_FOUND_CODE + : GRAPHILE_WEBSOCKET_ADMISSION_FAILED_CODE, + retryAfterSeconds + }; +}; + +export interface GraphileWebSocketUpgradeGatewayOptions { + /** Total time allowed for routing, auth, safety checks, and a cold build. */ + admissionTimeoutMs?: number; +} + +export interface GraphileWebSocketUpgradeGateway { + handle(request: IncomingMessage, socket: Duplex, head: Buffer): void; + close(): void; + readonly pendingCount: number; +} + +/** + * Feed upgrade requests through the same Express application as HTTP without + * exposing middleware-generated bodies on the wire. Express writes to a + * private sink; only stable, metadata-free admission errors reach the client. + */ +export const createGraphileWebSocketUpgradeGateway = ( + app: Express, + options: GraphileWebSocketUpgradeGatewayOptions = {} +): GraphileWebSocketUpgradeGateway => { + const admissionTimeoutMs = options.admissionTimeoutMs + ?? DEFAULT_ADMISSION_TIMEOUT_MS; + if (!Number.isSafeInteger(admissionTimeoutMs) || admissionTimeoutMs <= 0) { + throw new Error('WebSocket admission timeout must be a positive safe integer'); + } + + const pending = new Set(); + let closed = false; + + const cleanup = (context: PendingUpgrade): void => { + if (context.terminal) return; + context.terminal = true; + clearTimeout(context.timer); + pending.delete(context); + pendingByRequest.delete(context.request); + context.socket.removeListener('close', context.onSocketClose); + context.socket.removeListener('error', context.onSocketError); + context.response.removeListener('finish', context.onResponseFinish); + context.response.removeListener('close', context.onResponseClose); + if (context.response.socket === context.responseSocket) { + context.response.detachSocket(context.responseSocket); + } + context.responseSocket.destroy(); + }; + + const signalAdmissionAbort = (context: PendingUpgrade): void => { + // Remove only the gateway's terminal listeners before emitting the ordinary + // Express lifecycle signals. Request-scoped middleware must still observe + // them, but the gateway must retain the caller-selected stable response. + context.response.removeListener('finish', context.onResponseFinish); + context.response.removeListener('close', context.onResponseClose); + try { + context.request.emit('aborted'); + } catch { + // Cleanup and transport rejection remain mandatory even if an observer + // violates EventEmitter's no-throw expectation. + } + if (!context.response.destroyed && !context.response.writableEnded) { + try { + context.response.emit('close'); + } catch { + // See above: lifecycle observers are advisory to gateway cleanup. + } + } + }; + + const abortRequest = (context: PendingUpgrade): void => { + if (context.handedOff || context.terminal) return; + // Upgrade IncomingMessage instances are no longer owned by Node's HTTP + // parser, so a peer disconnect does not reliably emit `aborted`. Re-emit + // the ordinary request signal so queued Graphile builds release the waiter. + signalAdmissionAbort(context); + cleanup(context); + }; + + const rejectPending = ( + context: PendingUpgrade, + response: UpgradeResponse + ): void => { + if (context.handedOff || context.terminal) return; + cleanup(context); + writeUpgradeResponse(context.socket, response); + }; + + const abortAndRejectPending = ( + context: PendingUpgrade, + response: UpgradeResponse + ): void => { + if (context.handedOff || context.terminal) return; + signalAdmissionAbort(context); + rejectPending(context, response); + }; + + const handle = ( + request: IncomingMessage, + socket: Duplex, + head: Buffer + ): void => { + if (closed) { + writeUpgradeResponse(socket, { + status: 503, + code: GRAPHILE_WEBSOCKET_SERVER_CLOSING_CODE, + retryAfterSeconds: 1 + }); + return; + } + if (websocketPath(request) !== GRAPHILE_WEBSOCKET_PATH) { + writeUpgradeResponse(socket, { + status: 404, + code: GRAPHILE_WEBSOCKET_ROUTE_NOT_FOUND_CODE + }); + return; + } + if (!isGraphileWebSocketRequest(request)) { + writeUpgradeResponse(socket, { + status: 400, + code: GRAPHILE_WEBSOCKET_BAD_UPGRADE_CODE + }); + return; + } + + // Never attach the real upgrade socket to ServerResponse: an API/auth/build + // error may contain development detail. The response sink lets the normal + // Express lifecycle run while the gateway emits only stable error codes. + const response = new ServerResponse(request); + const responseSocket = new PassThrough() as unknown as Socket; + response.assignSocket(responseSocket); + + let context!: PendingUpgrade; + const onSocketClose = (): void => abortRequest(context); + const onSocketError = (): void => abortRequest(context); + const onResponseFinish = (): void => { + if (context.handedOff || context.terminal) return; + rejectPending(context, responseFailure(response)); + }; + const onResponseClose = (): void => { + if (context.handedOff || context.terminal) return; + rejectPending(context, responseFailure(response)); + }; + const timer = setTimeout(() => { + if (context.handedOff || context.terminal) return; + // Abort the build waiter before closing the transport, then surface a + // stable response that does not disclose the routed tenant or cache key. + abortAndRejectPending(context, { + status: 503, + code: GRAPHILE_WEBSOCKET_ADMISSION_TIMEOUT_CODE, + retryAfterSeconds: 1 + }); + }, admissionTimeoutMs); + timer.unref?.(); + + context = { + request, + socket, + head, + response, + responseSocket, + timer, + onSocketClose, + onSocketError, + onResponseFinish, + onResponseClose, + removePending: () => pending.delete(context), + handedOff: false, + terminal: false + }; + pending.add(context); + pendingByRequest.set(request, context); + socket.once('close', onSocketClose); + socket.once('error', onSocketError); + response.once('finish', onResponseFinish); + response.once('close', onResponseClose); + + try { + app(request, response); + } catch { + abortAndRejectPending(context, { + status: 500, + code: GRAPHILE_WEBSOCKET_ADMISSION_FAILED_CODE + }); + } + }; + + return { + handle, + close: () => { + if (closed) return; + closed = true; + for (const context of [...pending]) { + abortAndRejectPending(context, { + status: 503, + code: GRAPHILE_WEBSOCKET_SERVER_CLOSING_CODE, + retryAfterSeconds: 1 + }); + } + }, + get pendingCount(): number { + return pending.size; + } + }; +}; + +export const isGraphileWebSocketUpgrade = (request: Request): boolean => + pendingByRequest.has(request as unknown as IncomingMessage); + +/** Cookie-authenticated WebSockets require an origin a browser can prove. */ +export const isGraphileWebSocketOriginAllowed = ( + request: Request, + fallbackOrigin?: string +): boolean => { + const origin = request.get('origin'); + const bearer = request.headers.authorization + ?.toLowerCase().startsWith('bearer ') === true; + const sessionCookie = parseCookieValue(request, SESSION_COOKIE_NAME); + if (!origin) return bearer || !sessionCookie; + if (!sessionCookie) { + return isCorsOriginAllowed({ + origin, + fallbackOrigin, + api: request.api, + requestHost: request.get('host') + }); + } + + // A wildcard HTTP CORS policy and the localhost development convenience are + // not sufficient for a credentialed WebSocket: browsers attach cookies to + // the handshake but do not enforce CORS on the upgraded connection. Require + // an exact configured origin or exact same-host origin for session auth. + const normalizedOrigin = origin.trim(); + const fallback = fallbackOrigin?.trim(); + if (fallback && fallback !== '*' && normalizedOrigin === fallback) return true; + if ( + [...(request.api?.corsOrigins ?? []), ...(request.api?.domains ?? [])] + .includes(normalizedOrigin) + ) { + return true; + } + try { + return new URL(normalizedOrigin).host.toLowerCase() + === request.get('host')?.toLowerCase(); + } catch { + return false; + } +}; + +/** Mount immediately after API routing and before authentication/database I/O. */ +export const createGraphileWebSocketOriginGuard = ( + fallbackOrigin?: string +): RequestHandler => ( + request: Request, + response: Response, + next: NextFunction +): void => { + if ( + !isGraphileWebSocketUpgrade(request) + || isGraphileWebSocketOriginAllowed(request, fallbackOrigin) + ) { + next(); + return; + } + response.status(403).json({ + error: { + code: GRAPHILE_WEBSOCKET_AUTH_REJECTED_CODE, + message: 'WebSocket origin is not allowed' + } + }); +}; + +export interface AcceptedGraphileWebSocketUpgrade { + readonly socket: Duplex; + readonly head: Buffer; +} + +export const getGraphileWebSocketUpgradeTransport = ( + request: Request +): AcceptedGraphileWebSocketUpgrade | undefined => { + const context = pendingByRequest.get(request as unknown as IncomingMessage); + return context ? { socket: context.socket, head: context.head } : undefined; +}; + +/** + * Complete the synthetic response lifecycle before Grafserv owns the socket. + * This releases request-scoped pool leases while retaining the routed API and + * authenticated token on the IncomingMessage used by GraphQL over WebSocket. + */ +export const handoffGraphileWebSocketUpgrade = ( + request: Request, + response: Response +): AcceptedGraphileWebSocketUpgrade => { + const context = pendingByRequest.get(request as unknown as IncomingMessage); + if (!context || context.response !== (response as unknown as ServerResponse)) { + throw new Error('WebSocket upgrade context is unavailable'); + } + if (context.terminal || context.handedOff || context.socket.destroyed) { + throw new Error('WebSocket upgrade request is no longer active'); + } + if (context.response.socket !== context.responseSocket) { + throw new Error('Synthetic WebSocket admission response lost socket ownership'); + } + + context.handedOff = true; + clearTimeout(context.timer); + context.removePending(); + pendingByRequest.delete(context.request); + context.socket.removeListener('close', context.onSocketClose); + context.socket.removeListener('error', context.onSocketError); + context.response.removeListener('finish', context.onResponseFinish); + context.response.removeListener('close', context.onResponseClose); + context.response.detachSocket(context.responseSocket); + context.responseSocket.destroy(); + context.terminal = true; + // `finish` would mean an HTTP body was completed. `close` accurately tells + // request-scoped middleware that the synthetic response has been retired. + context.response.emit('close'); + return { socket: context.socket, head: context.head }; +}; diff --git a/packages/cli/src/commands/explorer.ts b/packages/cli/src/commands/explorer.ts index 5d24a50f8b..e622204c51 100644 --- a/packages/cli/src/commands/explorer.ts +++ b/packages/cli/src/commands/explorer.ts @@ -104,9 +104,10 @@ export default async ( }); log.success('✅ Selected Configuration:'); - for (const [key, value] of Object.entries(options)) { - log.debug(`${key}: ${JSON.stringify(value)}`); - } + // The merged options object contains database and provider credentials. + // Keep startup diagnostics explicitly credential-free. + log.debug(`database: ${options.pg?.database ?? 'default'}`); + log.debug(`server: ${options.server?.host ?? 'localhost'}:${options.server?.port ?? port}`); log.success('🚀 Launching Explorer...\n'); explorer(options); diff --git a/packages/cli/src/commands/server.ts b/packages/cli/src/commands/server.ts index 0004362a17..8217be05a2 100644 --- a/packages/cli/src/commands/server.ts +++ b/packages/cli/src/commands/server.ts @@ -145,15 +145,19 @@ export default async ( } as ConstructiveOptions); log.success('✅ Selected Configuration:'); - for (const [key, value] of Object.entries(options)) { - log.debug(`${key}: ${JSON.stringify(value)}`); - } + // Never serialize the merged options object: it contains PostgreSQL + // passwords, provider credentials, and the internal-request secret. + log.debug(`database: ${options.pg?.database ?? selectedDb}`); + log.debug(`server: ${options.server?.host ?? 'localhost'}:${options.server?.port ?? port}`); // Debug: Log API routing configuration const apiOpts = (options as any).api || {}; log.debug(`📡 API Routing: isPublic=${apiOpts.isPublic}, routingSchema=${apiOpts.routingSchema}`); if (apiOpts.isPublic === false) { - log.debug(` Header-based routing enabled (X-Api-Name, X-Database-Id, X-Meta-Schema)`); + log.debug(` Authenticated header routing available (X-Api-Name, X-Database-Id)`); + if (apiOpts.allowMetaSchemaHeader === true) { + log.warn(' Privileged X-Meta-Schema admin routing is enabled; isolate this listener from tenant ingress'); + } } if (apiOpts.metaSchemas?.length) { log.debug(` Meta schemas: ${apiOpts.metaSchemas.join(', ')}`);