diff --git a/graphile/graphile-cache/README.md b/graphile/graphile-cache/README.md index 5eb5c674b..7582dd6db 100644 --- a/graphile/graphile-cache/README.md +++ b/graphile/graphile-cache/README.md @@ -13,7 +13,8 @@

-PostGraphile instance LRU cache with automatic cleanup when PostgreSQL pools are disposed. +Heap-budgeted PostGraphile v5 instance cache with request draining, serialized +build admission, and explicit PostgreSQL pool ownership. ## Installation @@ -21,119 +22,134 @@ PostGraphile instance LRU cache with automatic cleanup when PostgreSQL pools are npm install graphile-cache pg-cache ``` -Note: This package depends on `pg-cache` for the PostgreSQL pool management. +`graphile-cache` uses `pg-cache` leases to keep each resident instance's exact +runtime pool alive until the instance has fully drained and shut down. ## Features -- LRU cache for PostGraphile instances -- Automatic cleanup when associated PostgreSQL pools are disposed -- Integrates seamlessly with `pg-cache` -- Service cache re-exported for convenience -- TypeScript support +- Heap-derived residency limits plus an optional process-RSS admission ceiling +- Request-aware eviction that never tears down an instance in use +- Awaited HTTP, realtime, PostGraphile, and pool-lease teardown +- Memory-pressure refusal and eviction counters +- Exact pool identities protected by reference-counted `pg-cache` leases ## How It Works -When you import this package, it automatically registers a cleanup callback with `pg-cache`. When a PostgreSQL pool is disposed, any PostGraphile instances using that pool are automatically removed from the cache. +Long-lived callers acquire a `PgPoolLease`, configure PostGraphile with the +lease's pool, and pass the same lease to `createGraphileInstance()`. Ownership +transfers to the returned entry only when that promise resolves. If creation +rejects, the caller still owns the lease and must release it. + +Eviction marks the entry as disposing, waits for its requests to drain, closes +the HTTP server, stops realtime delivery, attempts `pgl.release()`, and finally +releases the pool lease. Every teardown stage is attempted even when an earlier +stage fails, and duplicate disposal calls share one promise. ## Usage -### Basic Usage +### Creating a leased instance ```typescript -import { graphileCache, GraphileCache } from 'graphile-cache'; -import { getPgPool } from 'pg-cache'; -import { postgraphile } from 'postgraphile'; - -// Create a PostGraphile instance -const pgPool = getPgPool({ database: 'mydb' }); -const handler = postgraphile(pgPool, 'public', { - // PostGraphile options -}); - -// Cache it -const cacheEntry: GraphileCache = { - pgPool, - pgPoolKey: 'mydb', - handler -}; - -graphileCache.set('mydb.public', cacheEntry); +import { + createGraphileInstance, + disposeUncachedEntry, + graphileCache +} from 'graphile-cache'; +import { acquirePgPool } from 'pg-cache'; + +const cacheKey = 'tenant-id:api-id:build-contract-hash'; +const lease = acquirePgPool( + { database: 'tenant_database' }, + { purpose: 'runtime', sanitizeOnCheckout: true } +); + +// Application code builds this preset with makePgService({ pool: lease.pool, +// schemas: ['tenant_api'] }) and its exact plugin/settings contract. +const preset = makePreset(lease.pool); +let entry; +try { + entry = await createGraphileInstance({ + preset, + cacheKey, + poolLease: lease, + poolIdentity: lease.identity, + enableRealtime: true, + realtimeSchema: 'tenant_a_realtime', + realtimeSourceSchemas: ['tenant_api'] + }); +} catch (error) { + // Creation rejected before ownership transfer. + lease.release(); + throw error; +} -// Retrieve it later -const cached = graphileCache.get('mydb.public'); -if (cached) { - // Use cached.handler +// Creation resolved, so disposal must now release the entry-owned lease if +// admission or publication fails. +try { + graphileCache.set(cacheKey, entry); +} catch (error) { + await disposeUncachedEntry(entry, cacheKey); + throw error; } ``` -### Automatic Cleanup +`poolIdentity` is optional when `poolLease` is present because the lease identity +becomes the entry's authoritative identity. Supplying both with different values +fails before ownership transfers. -The cleanup happens automatically: +### Serving and eviction ```typescript -import { pgCache } from 'pg-cache'; -import { graphileCache } from 'graphile-cache'; - -// Add entries -graphileCache.set('mydb.public', { pgPoolKey: 'mydb', ... }); -graphileCache.set('mydb.private', { pgPoolKey: 'mydb', ... }); - -// When the pool is removed... -pgCache.delete('mydb'); - -// Both graphile entries are automatically cleaned up! -console.log(graphileCache.has('mydb.public')); // false -console.log(graphileCache.has('mydb.private')); // false -``` - -### Complete Example - -```typescript -import { graphileCache, GraphileCache } from 'graphile-cache'; -import { getPgPool } from 'pg-cache'; -import { postgraphile } from 'postgraphile'; - -function getGraphileInstance(database: string, schema: string): GraphileCache { - const key = `${database}.${schema}`; - - // Check cache first - const cached = graphileCache.get(key); - if (cached) { - return cached; - } - - // Create new instance - const pgPool = getPgPool({ database }); - const handler = postgraphile(pgPool, schema, { - graphqlRoute: '/graphql', - graphiqlRoute: '/graphiql', - // other options... - }); - - const entry: GraphileCache = { - pgPool, - pgPoolKey: database, - handler - }; - - // Cache it - graphileCache.set(key, entry); - return entry; +import { + deleteGraphileCacheEntry, + graphileCache, + invokeEntryHandler +} from 'graphile-cache'; + +const entry = graphileCache.get(cacheKey); +if (entry && invokeEntryHandler(entry, req, res, next)) { + return; } -// Use in Express -app.use((req, res, next) => { - const { handler } = getGraphileInstance('mydb', 'public'); - handler(req, res, next); -}); +// Resolves only after teardown and pool-lease release complete. +await deleteGraphileCacheEntry(cacheKey, 'manual'); ``` +Use `invokeEntryHandler()` for resident traffic so disposal can observe in-flight +requests. A false return means the entry has started draining; route the request +through normal cache-miss/build admission instead. + +### Shared exact-topic realtime + +`sharedRealtime` is an opt-in build-time seam. The caller installs one +`ActivatableGenerationScopedRealtimeSubscriber` in the PostGraphile service, +collects the exact physical `@realtime` topics during schema construction, and +supplies a dedicated least-privilege listener login. Instance creation audits +that login on the broker's pinned client, acquires only those topics, and +activates the subscriber before the entry can be published. Audit and LISTEN +therefore remain safe when the notification pool has `max: 1`. + +One canonical host/port/database target may have only one active opaque listener +identity and role. TLS remains part of that listener identity, so a TLS, +credential, or pool-contract change fails closed while the old generation is +resident instead of opening a second listener and silently reducing density; +rotate by invalidating and draining the old generations first. Resolver output +must use stable canonical connection target values, because two DNS aliases for +the same server cannot be proven to name one physical database in-process. + +Successful role audits have an explicit TTL. One unref'ed timer per exact +listener identity proactively re-audits idle subscriptions, while HTTP and +WebSocket operation boundaries use the same coalesced refresh as an immediate +gate. Broker termination and privilege drift latch every affected generation +unavailable. The timer is cancelled after the last generation releases. The +default realtime mode remains the dedicated PostGraphile subscriber. + ### Graceful Shutdown ```typescript import { closeAllCaches } from 'graphile-cache'; -// This closes all caches including pg pools +// Drains Graphile entries first, then closes the remaining pg-cache pools. process.on('SIGTERM', async () => { await closeAllCaches(); process.exit(0); @@ -142,39 +158,64 @@ process.on('SIGTERM', async () => { ## API Reference -### graphileCache - -The main PostGraphile instance cache. - -- `get(key: string): GraphileCache | undefined` - Get a cached instance -- `set(key: string, value: GraphileCache): void` - Cache an instance -- `has(key: string): boolean` - Check if an instance is cached -- `delete(key: string): void` - Remove an instance -- `clear(): void` - Remove all instances - -### GraphileCache Interface - -```typescript -interface GraphileCache { - pgPool: pg.Pool; - pgPoolKey: string; - handler: HttpRequestHandler; -} -``` - -### closeAllCaches() - -Closes all caches including the service cache, graphile cache, and all PostgreSQL pools. - -### svcCache - -Re-exported from `pg-cache` for convenience. - -## Integration Details - -The integration with `pg-cache` happens automatically when this module is imported. The cleanup callback is registered immediately, ensuring that PostGraphile instances are cleaned up whenever their associated PostgreSQL pools are disposed. - -This design ensures: -- No memory leaks from orphaned PostGraphile instances -- Automatic cleanup without manual intervention -- Loose coupling between packages +### Main lifecycle APIs + +- `createGraphileInstance(options)` creates a ready PostGraphile entry and + accepts an optional retained `PgPoolLease`. Realtime callers may provide the + exact cursor-function schema through `realtimeSchema`; omission preserves the + `realtime_public` compatibility default. Realtime also requires exact + `realtimeSourceSchemas`; a foreign cursor row stops delivery before any row + in that batch is emitted. Cursor node IDs combine a process-unique replica + identity with the exact cache contract so replicas cannot share cursor state. + A fatal delivery-integrity failure latches that exact generation unhealthy; + the next request receives `503 GRAPHILE_REALTIME_UNAVAILABLE`, never enters + its Graphile handler, and identity-checks the generation before retiring it + so a later request can rebuild without risking a healthy replacement. +- `invokeEntryHandler(entry, req, res, next)` tracks a request against an exact + resident entry. +- `deleteGraphileCacheEntry(key, reason)` evicts and awaits teardown. +- `clearGraphileCache()` evicts and awaits every resident entry. +- `closeAllCaches()` drains Graphile entries, then closes `pg-cache`. + +### Capacity and observability + +- `prepareCacheForBuild()` serializes admission with awaited eviction. +- `getCacheConfig()` reports the heap-derived capacity and calibration sources. +- `getCacheStats()` reports residency, realtime-unhealthy generations, + aggregate credential-free listener-role attestation health, unique active + broker identities, and monotonic catalog-audit attempts/failures. Generation + references are reported separately, so three API surfaces sharing one role + audit don't triple-count its database QPS. +- `getCacheCounters()` reports monotonic admitted/completed HTTP and WebSocket + lifecycles alongside evictions, disposal failures, and build refusals. The + lifecycle counters make short-lived work observable even when both ends fall + between two state snapshots. +- `startMemoryGovernor()` starts pressure-driven idle eviction and returns an + idempotent stop callback. + +`GRAPHILE_CACHE_MAX` caps Graphile build contracts by heap budget. +`GRAPHILE_CACHE_ADMISSION_MODE=preserve-resident` makes that ceiling a strict +admission boundary: a new contract receives `resident_capacity` without +evicting an existing resident. The default, `evict-idle`, retains the ordinary +LRU replacement behavior. +`GRAPHILE_CACHE_RSS_LIMIT_BYTES` adds a fail-closed process-RSS ceiling, and +admission reserves `GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES` (768 MiB by +default) above current RSS before starting a build. When the RSS ceiling is not +set, RSS remains present in cache pressure telemetry but does not constrain +admission. `PG_CACHE_MAX` +caps PostgreSQL connection identities, which may include runtime, control-plane, +listener, and diagnostic pools. They are independent limits: a resident entry's +lease prevents ordinary pool LRU or TTL eviction, and acquiring a new identity +fails closed when every registry slot is leased. + +The LRU's internal ceiling scales with the configured V8 heap (one sparse slot +per 256 KiB, bounded from 1,024 to 65,536). It is only a backing-structure +limit; measured instance cost, server/build reserves, and live pressure still +decide how many entries may become resident. + +## Pool disposal integration + +The package still registers a `pg-cache` cleanup callback as a fail-safe for +legacy unleased entries and explicit process-wide shutdown. Normal resident +lifetime is lease-driven: Graphile disposal releases the lease, after which +`pg-cache` may evict or expire the now-idle pool identity. diff --git a/graphile/graphile-cache/package.json b/graphile/graphile-cache/package.json index 28bcab596..4b54df31c 100644 --- a/graphile/graphile-cache/package.json +++ b/graphile/graphile-cache/package.json @@ -2,7 +2,7 @@ "name": "graphile-cache", "version": "4.10.2", "author": "Constructive ", - "description": "PostGraphile v5 LRU cache with automatic pool cleanup integration", + "description": "Heap-aware PostGraphile v5 cache with leased PostgreSQL pool lifecycle", "main": "index.js", "module": "esm/index.js", "types": "index.d.ts", diff --git a/graphile/graphile-cache/src/__tests__/build-readiness.test.ts b/graphile/graphile-cache/src/__tests__/build-readiness.test.ts new file mode 100644 index 000000000..c054217bd --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/build-readiness.test.ts @@ -0,0 +1,73 @@ +import { awaitGraphileBuildReadiness } from '../build-readiness'; + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(error: Error): void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +const flushPromises = (): Promise => new Promise((resolve) => setImmediate(resolve)); + +describe('awaitGraphileBuildReadiness', () => { + it('does not resolve before the schema build completes', async () => { + const schemaResult = deferred(); + const release = jest.fn().mockResolvedValue(undefined); + let resolved = false; + const buildPromise = awaitGraphileBuildReadiness({ + schemaResult: schemaResult.promise, + addTo: jest.fn().mockResolvedValue(undefined), + ready: jest.fn().mockResolvedValue(undefined), + release + }).then(() => { + resolved = true; + }); + + await flushPromises(); + expect(resolved).toBe(false); + + schemaResult.resolve({}); + await buildPromise; + expect(release).not.toHaveBeenCalled(); + }); + + it('releases the failed generation before rejecting', async () => { + const schemaResult = deferred(); + const release = jest.fn().mockResolvedValue(undefined); + const buildPromise = awaitGraphileBuildReadiness({ + schemaResult: schemaResult.promise, + addTo: jest.fn().mockResolvedValue(undefined), + ready: jest.fn().mockResolvedValue(undefined), + release + }); + const failure = new Error('schema build failed'); + schemaResult.reject(failure); + + await expect(buildPromise).rejects.toBe(failure); + expect(release).toHaveBeenCalledTimes(1); + }); + + it('preserves the build failure if cleanup also fails', async () => { + const failure = new Error('schema build failed'); + const cleanupFailure = new Error('release failed'); + const onReleaseError = jest.fn(); + + await expect(awaitGraphileBuildReadiness({ + schemaResult: Promise.reject(failure), + addTo: jest.fn().mockResolvedValue(undefined), + ready: jest.fn().mockResolvedValue(undefined), + release: jest.fn().mockRejectedValue(cleanupFailure), + onReleaseError + })).rejects.toBe(failure); + expect(onReleaseError).toHaveBeenCalledWith(cleanupFailure); + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/governor.test.ts b/graphile/graphile-cache/src/__tests__/governor.test.ts new file mode 100644 index 000000000..54d40b784 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/governor.test.ts @@ -0,0 +1,634 @@ +import { EventEmitter } from 'node:events'; + +import type { NextFunction, Request, Response } from 'express'; +import type { PgPoolLease } from 'pg-cache'; + +import { + computeBackingCacheMax, + computeCapacityFromBudget, + disposeUncachedEntry, + evaluateBuildAdmission, + getCacheConfig, + getCacheCounters, + getCacheStats, + getInstanceHeapEstimate, + getMemoryPressure, + graphileCache, + type GraphileCacheEntry, + invokeEntryHandler, + prepareCacheForBuild, + raceWithClearedTimeout, + recordInstanceHeapSample, + resetInstanceHeapSamples, + waitForEntryDisposal +} from '../graphile-cache'; +import { GRAPHILE_REALTIME_UNAVAILABLE_CODE } from '../realtime-readiness'; + +const MB = 1024 * 1024; + +const makeEntry = (releaseDelayMs = 0): GraphileCacheEntry => ({ + pgl: { + release: jest.fn(() => new Promise((resolve) => setTimeout(resolve, releaseDelayMs))) + } as unknown as GraphileCacheEntry['pgl'], + serv: {} as GraphileCacheEntry['serv'], + handler: jest.fn() as unknown as GraphileCacheEntry['handler'], + httpServer: null, + cacheKey: 'test', + createdAt: Date.now() +}); + +const makePoolLease = (onRelease?: () => void): PgPoolLease => ({ + pool: {} as PgPoolLease['pool'], + identity: 'pg:v1:test-runtime', + release: jest.fn(() => onRelease?.()) +}); + +describe('heap budget capacity', () => { + const calibrationEnv = [ + 'GRAPHILE_CACHE_MAX', + 'GRAPHILE_CACHE_ADMISSION_MODE', + 'GRAPHILE_CACHE_INSTANCE_HEAP_BYTES', + 'GRAPHILE_CACHE_SERVER_RESERVE_BYTES', + 'GRAPHILE_CACHE_BUILD_RESERVE_BYTES', + 'GRAPHILE_CACHE_RSS_LIMIT_BYTES', + 'GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES', + 'GRAPHILE_CACHE_CALIBRATION_ID' + ] as const; + let previousEnv: Record; + + beforeEach(() => { + previousEnv = Object.fromEntries( + calibrationEnv.map((name) => [name, process.env[name]]) + ); + for (const name of calibrationEnv) delete process.env[name]; + resetInstanceHeapSamples(); + }); + + afterEach(() => { + resetInstanceHeapSamples(); + for (const name of calibrationEnv) { + const value = previousEnv[name]; + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + it('fits residency and one serialized build transient', () => { + expect(computeCapacityFromBudget(3584 * MB, 1450 * MB, 256 * MB, 768 * MB)).toBe(2); + expect(computeCapacityFromBudget(2048 * MB, 1450 * MB, 256 * MB, 768 * MB)).toBe(1); + }); + + it('returns zero when even the server and build reserves cannot fit', () => { + expect(computeCapacityFromBudget(512 * MB, 64 * MB, 256 * MB, 768 * MB)).toBe(0); + }); + + it('does not hide validated density behind a fixed backing-cache ceiling', () => { + expect(computeCapacityFromBudget(238 * MB, MB, MB, MB)).toBe(237); + expect(computeBackingCacheMax(1024 * MB)).toBe(4096); + expect(computeBackingCacheMax(4096 * MB)).toBe(16_384); + expect(graphileCache.max).toBeGreaterThanOrEqual(4096); + }); + + it('derives the backing ceiling from the modeled heap rather than this process', () => { + expect(computeCapacityFromBudget(1024 * MB, 1, 1, 1)).toBe(4096); + }); + + it('treats runtime samples as a safety floor rather than an unsafe downsize', () => { + recordInstanceHeapSample(30 * MB); + recordInstanceHeapSample(32 * MB); + recordInstanceHeapSample(34 * MB); + expect(getInstanceHeapEstimate()).toBe(512 * MB); + expect(getCacheConfig().calibration).toMatchObject({ + instanceHeapSource: 'default', + instanceHeapSampleCount: 3 + }); + }); + + it('lets runtime samples raise an explicit calibrated floor', () => { + process.env.GRAPHILE_CACHE_INSTANCE_HEAP_BYTES = String(32 * MB); + recordInstanceHeapSample(40 * MB); + recordInstanceHeapSample(50 * MB); + recordInstanceHeapSample(60 * MB); + expect(getInstanceHeapEstimate()).toBe(60 * MB); + expect(getCacheConfig().calibration.instanceHeapSource).toBe( + 'runtime-safety-floor' + ); + }); + + it('reports explicit calibration provenance and respects the operator ceiling', () => { + process.env.GRAPHILE_CACHE_INSTANCE_HEAP_BYTES = String(MB); + process.env.GRAPHILE_CACHE_SERVER_RESERVE_BYTES = String(MB); + process.env.GRAPHILE_CACHE_BUILD_RESERVE_BYTES = String(MB); + process.env.GRAPHILE_CACHE_MAX = '128'; + process.env.GRAPHILE_CACHE_CALIBRATION_ID = 'cperf:fixture:sha256'; + const config = getCacheConfig(); + expect(config.max).toBe(128); + expect(config.calibration).toEqual({ + id: 'cperf:fixture:sha256', + instanceHeapSource: 'environment', + instanceHeapSampleCount: 0, + serverReserveSource: 'environment', + buildReserveSource: 'environment' + }); + }); + + it('defaults to idle eviction and strictly validates preserve-resident admission', () => { + expect(getCacheConfig().admissionMode).toBe('evict-idle'); + process.env.GRAPHILE_CACHE_ADMISSION_MODE = 'preserve-resident'; + expect(getCacheConfig().admissionMode).toBe('preserve-resident'); + process.env.GRAPHILE_CACHE_ADMISSION_MODE = 'preserve'; + expect(() => getCacheConfig()).toThrow( + 'GRAPHILE_CACHE_ADMISSION_MODE must be evict-idle or preserve-resident' + ); + }); + + it('reports an explicit RSS ceiling and transient reservation', () => { + process.env.GRAPHILE_CACHE_RSS_LIMIT_BYTES = String(3 * 1024 * MB); + process.env.GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES = String(96 * MB); + + const config = getCacheConfig(); + const pressure = getMemoryPressure(); + const stats = getCacheStats(); + + expect(config).toMatchObject({ + rssLimitBytes: 3 * 1024 * MB, + rssBuildReserveBytes: 96 * MB + }); + expect(pressure).toMatchObject({ + rssLimitBytes: 3 * 1024 * MB, + rssBytes: expect.any(Number), + rssRatio: expect.any(Number) + }); + expect(stats).toMatchObject({ + rssLimitBytes: 3 * 1024 * MB, + rssBuildReserveBytes: 96 * MB + }); + }); + + it('keeps RSS observable but unbounded unless an operator sets a ceiling', () => { + expect(getMemoryPressure()).toMatchObject({ + rssLimitBytes: null, + rssRatio: null, + rssLevel: 'unbounded', + rssBytes: expect.any(Number) + }); + }); + + it('refuses a build whose live RSS plus transient reserve crosses the ceiling', () => { + const rssBytes = process.memoryUsage().rss; + process.env.GRAPHILE_CACHE_RSS_LIMIT_BYTES = String(rssBytes * 4); + process.env.GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES = String(rssBytes * 5); + + expect(evaluateBuildAdmission(0)).toMatchObject({ + admit: false, + reason: 'rss_budget_exceeded', + rssLimitBytes: rssBytes * 4 + }); + }); + + it.each([ + ['GRAPHILE_CACHE_INSTANCE_HEAP_BYTES', '0'], + ['GRAPHILE_CACHE_SERVER_RESERVE_BYTES', '-1'], + ['GRAPHILE_CACHE_BUILD_RESERVE_BYTES', '1.5'], + ['GRAPHILE_CACHE_RSS_LIMIT_BYTES', '0'], + ['GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES', '-10'], + ['GRAPHILE_CACHE_MAX', '12entries'], + ['GRAPHILE_CACHE_MAX', String(Number.MAX_SAFE_INTEGER + 1)] + ])('rejects invalid explicit calibration %s=%s', (name, value) => { + process.env[name] = value; + expect(() => getCacheConfig()).toThrow('must be a positive safe integer'); + }); + + it('rejects an operator ceiling above the heap-scaled backing cache', () => { + process.env.GRAPHILE_CACHE_MAX = String(graphileCache.max + 1); + expect(() => getCacheConfig()).toThrow('exceeds heap-scaled backing ceiling'); + }); +}); + +describe('entry-scoped awaited disposal', () => { + afterEach(async () => { + graphileCache.clear(); + await new Promise((resolve) => setTimeout(resolve, 5)); + }); + + it('disposes distinct rebuilt entries with the same key exactly once each', async () => { + const first = makeEntry(); + const second = makeEntry(); + await Promise.all([ + disposeUncachedEntry(first, 'same-key'), + disposeUncachedEntry(second, 'same-key') + ]); + expect(first.pgl.release).toHaveBeenCalledTimes(1); + expect(second.pgl.release).toHaveBeenCalledTimes(1); + }); + + it('waits for a resident request before releasing the instance', async () => { + const entry = makeEntry(); + const response = new EventEmitter() as unknown as Response; + invokeEntryHandler( + entry, + {} as Request, + response, + (() => undefined) as NextFunction + ); + graphileCache.set('drain', entry); + graphileCache.delete('drain'); + + await new Promise((resolve) => setTimeout(resolve, 5)); + expect(entry.pgl.release).not.toHaveBeenCalled(); + (response as unknown as EventEmitter).emit('finish'); + await expect(waitForEntryDisposal(entry, 100)).resolves.toBe(true); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + }); + + it('does not enter an instance after the request has already closed', () => { + const entry = makeEntry(); + const request = Object.assign(new EventEmitter(), { + aborted: true, + destroyed: true, + socket: { destroyed: true } + }) as unknown as Request; + const response = Object.assign(new EventEmitter(), { + destroyed: true, + writableEnded: true + }) as unknown as Response; + + expect(invokeEntryHandler( + entry, + request, + response, + (() => undefined) as NextFunction + )).toBe(false); + expect(entry.handler).not.toHaveBeenCalled(); + expect(entry.inflight ?? 0).toBe(0); + }); + + it('does enter after a JSON body parser consumed the request stream', () => { + const entry = makeEntry(); + const countersBefore = getCacheCounters(); + const request = Object.assign(new EventEmitter(), { + aborted: false, + // Express/raw-body may destroy the readable request stream after fully + // consuming it while the underlying keep-alive socket remains healthy. + destroyed: true, + socket: { destroyed: false } + }) as unknown as Request; + const response = Object.assign(new EventEmitter(), { + destroyed: false, + writableEnded: false + }) as unknown as Response; + + expect(invokeEntryHandler( + entry, + request, + response, + (() => undefined) as NextFunction + )).toBe(true); + expect(entry.handler).toHaveBeenCalledTimes(1); + expect(entry.inflight).toBe(1); + expect(getCacheCounters().httpRequestsStarted).toBe( + countersBefore.httpRequestsStarted + 1 + ); + expect(getCacheCounters().httpRequestsCompleted).toBe( + countersBefore.httpRequestsCompleted + ); + (response as unknown as EventEmitter).emit('finish'); + // Express can emit close after finish; the completion counter remains + // monotonic and records this request exactly once. + (response as unknown as EventEmitter).emit('close'); + expect(entry.inflight).toBe(0); + expect(getCacheCounters().httpRequestsCompleted).toBe( + countersBefore.httpRequestsCompleted + 1 + ); + }); + + it('returns a stable 503 and retires the exact realtime-unhealthy generation', async () => { + const entry = makeEntry(); + entry.cacheKey = 'realtime-unhealthy'; + entry.realtimeHealth = { + status: 'failed', + failureCode: 'REALTIME_SOURCE_SCHEMA_VIOLATION', + failedAt: 1_000 + }; + graphileCache.set('realtime-unhealthy', entry); + expect(getCacheStats().realtimeUnhealthy).toBe(1); + const response = Object.assign(new EventEmitter(), { + destroyed: false, + writableEnded: false, + headersSent: false, + setHeader: jest.fn(), + status: jest.fn(), + json: jest.fn() + }); + response.status.mockReturnValue(response); + + expect(invokeEntryHandler( + entry, + {} as Request, + response as unknown as Response, + (() => undefined) as NextFunction + )).toBe(true); + + expect(entry.handler).not.toHaveBeenCalled(); + expect(entry.inflight ?? 0).toBe(0); + expect(response.setHeader).toHaveBeenCalledWith('Retry-After', '15'); + expect(response.status).toHaveBeenCalledWith(503); + expect(response.json).toHaveBeenCalledWith({ + error: { + code: GRAPHILE_REALTIME_UNAVAILABLE_CODE, + message: 'Realtime delivery is unavailable for this GraphQL instance' + } + }); + expect(graphileCache.has('realtime-unhealthy')).toBe(false); + await expect(waitForEntryDisposal(entry, 100)).resolves.toBe(true); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + }); + + it('fails closed when a listener-role attestation expires before invocation', () => { + const entry = makeEntry(); + entry.realtimeRoleAttestation = { + snapshot: jest.fn(() => ({ + version: 1, + mode: 'shared-exact', + listenerIdentity: 'opaque-listener-identity', + auditVersion: 'pg-notification-role:v1', + role: 'listener', + database: 'tenant_a', + lastAttestedAt: 1, + validUntil: 2, + checks: 1, + status: 'healthy', + failureCode: null as string | null, + failedAt: null as number | null + })), + revalidateIfDue: jest.fn(async () => true), + release: jest.fn() + }; + const response = Object.assign(new EventEmitter(), { + destroyed: false, + writableEnded: false, + headersSent: false, + setHeader: jest.fn(), + status: jest.fn(), + json: jest.fn() + }); + response.status.mockReturnValue(response); + + expect(invokeEntryHandler( + entry, + {} as Request, + response as unknown as Response, + (() => undefined) as NextFunction + )).toBe(true); + + expect(entry.handler).not.toHaveBeenCalled(); + expect(response.status).toHaveBeenCalledWith(503); + expect(response.json).toHaveBeenCalledWith({ + error: { + code: GRAPHILE_REALTIME_UNAVAILABLE_CODE, + message: 'Realtime delivery is unavailable for this GraphQL instance' + } + }); + }); + + it('never lets a stale realtime generation evict a healthy replacement', () => { + const stale = makeEntry(); + stale.cacheKey = 'shared-contract'; + stale.realtimeHealth = { + status: 'failed', + failureCode: 'REALTIME_SOURCE_SCHEMA_VIOLATION', + failedAt: 1_000 + }; + const replacement = makeEntry(); + replacement.cacheKey = 'shared-contract'; + graphileCache.set('shared-contract', replacement); + const response = Object.assign(new EventEmitter(), { + destroyed: false, + writableEnded: false, + headersSent: false, + setHeader: jest.fn(), + status: jest.fn(), + json: jest.fn() + }); + response.status.mockReturnValue(response); + + expect(invokeEntryHandler( + stale, + {} as Request, + response as unknown as Response, + (() => undefined) as NextFunction + )).toBe(true); + + expect(graphileCache.peek('shared-contract')).toBe(replacement); + expect(replacement.disposing).not.toBe(true); + expect(stale.disposing).not.toBe(true); + expect(stale.handler).not.toHaveBeenCalled(); + expect(response.status).toHaveBeenCalledWith(503); + }); + + it('releases if the response closes while terminal listeners are attached', () => { + const entry = makeEntry(); + const request = new EventEmitter() as unknown as Request; + const response = new EventEmitter() as unknown as Response; + let terminalChecks = 0; + Object.defineProperty(response, 'writableEnded', { + get: () => ++terminalChecks >= 2 + }); + + expect(invokeEntryHandler( + entry, + request, + response, + (() => undefined) as NextFunction + )).toBe(false); + expect(entry.handler).not.toHaveBeenCalled(); + expect(entry.inflight).toBe(0); + expect((response as unknown as EventEmitter).listenerCount('finish')).toBe(0); + expect((response as unknown as EventEmitter).listenerCount('close')).toBe(0); + }); + + it('releases the pool lease after the complete teardown sequence', async () => { + const events: string[] = []; + const entry = makeEntry(); + entry.httpServer = { + close: (callback: () => void) => { + events.push('http-close'); + callback(); + } + } as unknown as GraphileCacheEntry['httpServer']; + entry.realtimeManager = { + stop: jest.fn(async () => { + events.push('realtime-stop'); + }) + }; + entry.pgl = { + release: jest.fn(async () => { + events.push('postgraphile-release'); + }) + } as unknown as GraphileCacheEntry['pgl']; + entry.releasePresetServices = jest.fn(async () => { + events.push('preset-services-release'); + }); + entry.poolLease = makePoolLease(() => events.push('pool-lease-release')); + + const response = new EventEmitter() as unknown as Response; + invokeEntryHandler(entry, {} as Request, response, (() => undefined) as NextFunction); + const disposal = disposeUncachedEntry(entry, 'ordered'); + + await new Promise((resolve) => setImmediate(resolve)); + expect(events).toEqual([]); + + (response as unknown as EventEmitter).emit('finish'); + await disposal; + expect(events).toEqual([ + 'http-close', + 'postgraphile-release', + 'preset-services-release', + 'realtime-stop', + 'pool-lease-release' + ]); + }); + + it('releases the pool lease exactly once under duplicate disposal', async () => { + const entry = makeEntry(); + entry.poolLease = makePoolLease(); + + await Promise.all([ + disposeUncachedEntry(entry, 'duplicate'), + disposeUncachedEntry(entry, 'duplicate'), + disposeUncachedEntry(entry, 'duplicate') + ]); + + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + expect(entry.poolLease.release).toHaveBeenCalledTimes(1); + }); + + it('awaits released memory before admitting the next build', async () => { + const previousMax = process.env.GRAPHILE_CACHE_MAX; + const previousMode = process.env.GRAPHILE_CACHE_ADMISSION_MODE; + process.env.GRAPHILE_CACHE_MAX = '1'; + process.env.GRAPHILE_CACHE_ADMISSION_MODE = 'evict-idle'; + const entry = makeEntry(20); + graphileCache.set('resident', entry); + const startedAt = Date.now(); + try { + const result = await prepareCacheForBuild(200); + expect(result.evicted).toBe(1); + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(15); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + } finally { + if (previousMax === undefined) delete process.env.GRAPHILE_CACHE_MAX; + else process.env.GRAPHILE_CACHE_MAX = previousMax; + if (previousMode === undefined) delete process.env.GRAPHILE_CACHE_ADMISSION_MODE; + else process.env.GRAPHILE_CACHE_ADMISSION_MODE = previousMode; + } + }); + + it('refuses at preserve-resident capacity before evicting an idle resident', async () => { + const previousMax = process.env.GRAPHILE_CACHE_MAX; + const previousMode = process.env.GRAPHILE_CACHE_ADMISSION_MODE; + process.env.GRAPHILE_CACHE_MAX = '1'; + process.env.GRAPHILE_CACHE_ADMISSION_MODE = 'preserve-resident'; + const entry = makeEntry(); + graphileCache.set('preserved', entry); + try { + expect(evaluateBuildAdmission()).toMatchObject({ + admit: false, + reason: 'resident_capacity' + }); + await expect(prepareCacheForBuild(100)).rejects.toMatchObject({ + reason: 'resident_capacity' + }); + expect(graphileCache.peek('preserved')).toBe(entry); + expect(entry.pgl.release).not.toHaveBeenCalled(); + } finally { + graphileCache.delete('preserved'); + await waitForEntryDisposal(entry, 100); + if (previousMax === undefined) delete process.env.GRAPHILE_CACHE_MAX; + else process.env.GRAPHILE_CACHE_MAX = previousMax; + if (previousMode === undefined) delete process.env.GRAPHILE_CACHE_ADMISSION_MODE; + else process.env.GRAPHILE_CACHE_ADMISSION_MODE = previousMode; + } + }); + + it('refuses admission without evicting the only busy resident', async () => { + const previousMax = process.env.GRAPHILE_CACHE_MAX; + const previousMode = process.env.GRAPHILE_CACHE_ADMISSION_MODE; + process.env.GRAPHILE_CACHE_MAX = '1'; + process.env.GRAPHILE_CACHE_ADMISSION_MODE = 'evict-idle'; + const entry = makeEntry(); + const response = new EventEmitter() as unknown as Response; + invokeEntryHandler(entry, {} as Request, response, (() => undefined) as NextFunction); + graphileCache.set('busy', entry); + try { + await expect(prepareCacheForBuild(10)).rejects.toMatchObject({ + reason: 'resident_busy' + }); + expect(graphileCache.has('busy')).toBe(true); + } finally { + (response as unknown as EventEmitter).emit('finish'); + await waitForEntryDisposal(entry, 100); + if (previousMax === undefined) delete process.env.GRAPHILE_CACHE_MAX; + else process.env.GRAPHILE_CACHE_MAX = previousMax; + if (previousMode === undefined) delete process.env.GRAPHILE_CACHE_ADMISSION_MODE; + else process.env.GRAPHILE_CACHE_ADMISSION_MODE = previousMode; + } + }); + + it('releases the pool lease when PostGraphile release fails', async () => { + const releaseFailure = new Error('PostGraphile release failed'); + const events: string[] = []; + const entry = makeEntry(); + entry.pgl = { + release: jest.fn(async () => { + events.push('postgraphile-release'); + throw releaseFailure; + }) + } as unknown as GraphileCacheEntry['pgl']; + entry.poolLease = makePoolLease(() => events.push('pool-lease-release')); + + await expect(disposeUncachedEntry(entry, 'release-failure')).rejects.toBe( + releaseFailure + ); + expect(events).toEqual(['postgraphile-release', 'pool-lease-release']); + expect(entry.poolLease.release).toHaveBeenCalledTimes(1); + }); + + it('continues realtime and pool cleanup after a PostGraphile release failure', async () => { + const releaseFailure = new Error('PostGraphile release failed'); + const realtimeFailure = new Error('Realtime stop failed'); + const entry = makeEntry(); + entry.pgl = { + release: jest.fn(async () => { + throw releaseFailure; + }) + } as unknown as GraphileCacheEntry['pgl']; + entry.realtimeManager = { + stop: jest.fn(async () => { + throw realtimeFailure; + }) + }; + entry.poolLease = makePoolLease(); + + await expect(Promise.all([ + disposeUncachedEntry(entry, 'aggregate-release-failure'), + disposeUncachedEntry(entry, 'aggregate-release-failure') + ])).rejects.toBe(releaseFailure); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + expect(entry.realtimeManager.stop).toHaveBeenCalledTimes(1); + expect(entry.poolLease.release).toHaveBeenCalledTimes(1); + }); +}); + +describe('timer cleanup', () => { + it('clears the timeout when work settles first', async () => { + jest.useFakeTimers(); + try { + const result = await raceWithClearedTimeout(Promise.resolve('done'), 60_000); + expect(result).toEqual({ timedOut: false, value: 'done' }); + expect(jest.getTimerCount()).toBe(0); + } finally { + jest.useRealTimers(); + } + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/http-adapter.test.ts b/graphile/graphile-cache/src/__tests__/http-adapter.test.ts new file mode 100644 index 000000000..9212215a6 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/http-adapter.test.ts @@ -0,0 +1,123 @@ +import type { Server } from 'node:http'; + +import express, { type Express } from 'express'; + +import { + disposeUncachedEntry, + type GraphileCacheEntry +} from '../graphile-cache'; +import { + attachGraphileHttpHandler, + createGraphileHttpHandler +} from '../http-adapter'; + +const closeServer = (server: Server): Promise => + new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + }); + +describe('lean Graphile HTTP adapter', () => { + it('matches the pinned Grafserv HTTP-only runtime contract', async () => { + // Use require so ts-jest's legacy resolver does not reject Grafserv's + // conditional `./express/v4` export, which the package build resolves. + const { grafserv } = require('grafserv/express/v4'); + const serv = grafserv({ + preset: { grafserv: { graphqlPath: '/graphql' } }, + schema: null + }); + const handler = createGraphileHttpHandler(); + + try { + await attachGraphileHttpHandler(serv, handler, serv.getPreset()); + expect((handler as any).stack).toHaveLength(1); + expect((handler as any).listen).toBeUndefined(); + } finally { + await serv.release(); + } + }); + + it('mounts Grafserv on a router with websocket/server allocation disabled', async () => { + const handler = createGraphileHttpHandler(); + const serv = { + addTo: jest.fn(async (app: Express) => { + app.use('/graphql', (_req, res) => { + res.status(200).json({ data: { adapter: 'router' } }); + }); + }) + }; + + await attachGraphileHttpHandler(serv, handler, { grafserv: {} }); + expect(serv.addTo).toHaveBeenCalledWith(handler, null, false); + expect((handler as any).listen).toBeUndefined(); + + const outerApp = express(); + outerApp.use(handler); + const outerServer = await new Promise((resolve, reject) => { + const server = outerApp.listen(0, '127.0.0.1', () => resolve(server)); + server.once('error', reject); + }); + try { + const address = outerServer.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected an address for the test HTTP server'); + } + const response = await fetch(`http://127.0.0.1:${address.port}/graphql`); + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ + data: { adapter: 'router' } + }); + } finally { + await closeServer(outerServer); + } + }); + + it('fails closed instead of silently disabling configured WebSockets', () => { + const handler = createGraphileHttpHandler(); + const serv = { addTo: jest.fn() }; + + expect(() => attachGraphileHttpHandler(serv, handler, { + grafserv: { websockets: true } + })).toThrow(/tenant-aware upgrade handler on the shared server/); + expect(serv.addTo).not.toHaveBeenCalled(); + }); + + it('mounts HTTP without an exclusive listener when shared routing is explicit', async () => { + const handler = createGraphileHttpHandler(); + const serv = { addTo: jest.fn() }; + + await attachGraphileHttpHandler(serv, handler, { + grafserv: { websockets: true } + }, { + sharedWebsocketRouting: true + }); + + expect(serv.addTo).toHaveBeenCalledWith(handler, null, false); + }); + + it('disposes a serverless adapter and its realtime manager in order', async () => { + const events: string[] = []; + const entry: GraphileCacheEntry = { + pgl: { + release: jest.fn(async () => { + events.push('postgraphile-release'); + }) + } as unknown as GraphileCacheEntry['pgl'], + serv: {} as GraphileCacheEntry['serv'], + handler: createGraphileHttpHandler(), + httpServer: null, + cacheKey: 'lean-adapter', + createdAt: Date.now(), + realtimeManager: { + stop: jest.fn(async () => { + events.push('realtime-stop'); + }) + } + }; + + await disposeUncachedEntry(entry); + + expect(events).toEqual(['postgraphile-release', 'realtime-stop']); + expect(entry.realtimeManager?.stop).toHaveBeenCalledTimes(1); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/preset-services.test.ts b/graphile/graphile-cache/src/__tests__/preset-services.test.ts new file mode 100644 index 000000000..5807f04f7 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/preset-services.test.ts @@ -0,0 +1,29 @@ +import { createPresetServicesReleaser } from '../preset-services'; + +describe('preset service ownership', () => { + it('releases services in reverse order exactly once under concurrent teardown', async () => { + const events: string[] = []; + const first = { release: jest.fn(async () => { events.push('first'); }) }; + const second = { release: jest.fn(async () => { events.push('second'); }) }; + const release = createPresetServicesReleaser({ + pgServices: [first, second, first] + }); + + await Promise.all([release(), release(), release()]); + + expect(events).toEqual(['second', 'first']); + expect(first.release).toHaveBeenCalledTimes(1); + expect(second.release).toHaveBeenCalledTimes(1); + }); + + it('continues releasing services and preserves the first cleanup error', async () => { + const firstFailure = new Error('second failed'); + const first = { release: jest.fn(async (): Promise => undefined) }; + const second = { release: jest.fn(async () => { throw firstFailure; }) }; + const release = createPresetServicesReleaser({ pgServices: [first, second] }); + + await expect(release()).rejects.toBe(firstFailure); + expect(first.release).toHaveBeenCalledTimes(1); + expect(second.release).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/realtime-readiness.test.ts b/graphile/graphile-cache/src/__tests__/realtime-readiness.test.ts new file mode 100644 index 000000000..897289a18 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/realtime-readiness.test.ts @@ -0,0 +1,204 @@ +import { + createGraphileRealtimeHealth, + createGraphileRealtimeNodeId, + DEFAULT_GRAPHILE_REALTIME_SCHEMA, + GraphileRealtimeStartupError, + startConfiguredRealtime, + withGraphileRealtimeFailure +} from '../realtime-readiness'; + +const makeManager = () => { + const start = jest.fn().mockResolvedValue(undefined); + const stop = jest.fn().mockResolvedValue(undefined); + const constructor = jest.fn().mockImplementation(() => ({ start, stop })); + return { constructor, start, stop }; +}; + +describe('configured realtime instance readiness', () => { + it('fails closed and releases PostGraphile when the subscriber is missing', async () => { + const manager = makeManager(); + const releasePostGraphile = jest.fn().mockResolvedValue(undefined); + + await expect(startConfiguredRealtime({ + cacheKey: 'missing-subscriber', + resolvedPreset: { + pgServices: [{ adaptorSettings: { pool: {} } }] + }, + allowedSourceSchemas: ['tenant_a'], + releasePostGraphile, + loadManager: async () => manager.constructor + })).rejects.toBeInstanceOf(GraphileRealtimeStartupError); + + expect(manager.constructor).not.toHaveBeenCalled(); + expect(releasePostGraphile).toHaveBeenCalledTimes(1); + }); + + it('stops a partially started manager and releases PostGraphile', async () => { + const manager = makeManager(); + const startupFailure = new Error('realtime startup failed'); + manager.start.mockRejectedValue(startupFailure); + const releasePostGraphile = jest.fn().mockResolvedValue(undefined); + + await expect(startConfiguredRealtime({ + cacheKey: 'start-failure', + resolvedPreset: { + pgServices: [{ + pgSubscriber: {}, + adaptorSettings: { pool: {} } + }] + }, + allowedSourceSchemas: ['tenant_a'], + releasePostGraphile, + loadManager: async () => manager.constructor + })).rejects.toMatchObject({ + code: 'GRAPHILE_REALTIME_STARTUP_FAILED', + cause: startupFailure + }); + + expect(manager.stop).toHaveBeenCalledTimes(1); + expect(releasePostGraphile).toHaveBeenCalledTimes(1); + }); + + it('returns a started manager without releasing a healthy generation', async () => { + const manager = makeManager(); + const releasePostGraphile = jest.fn().mockResolvedValue(undefined); + + const result = await startConfiguredRealtime({ + cacheKey: 'ready', + resolvedPreset: { + pgServices: [{ + pgSubscriber: { eventEmitter: { emit: jest.fn() } }, + adaptorSettings: { pool: {} } + }] + }, + allowedSourceSchemas: ['tenant_a'], + releasePostGraphile, + loadManager: async () => manager.constructor, + replicaIdentity: 'replica-a' + }); + + expect(result).toEqual({ start: manager.start, stop: manager.stop }); + expect(manager.constructor).toHaveBeenCalledWith(expect.objectContaining({ + schema: DEFAULT_GRAPHILE_REALTIME_SCHEMA, + allowedSourceSchemas: ['tenant_a'], + nodeId: 'graphile-cache:replica-a:ready' + })); + expect(manager.start).toHaveBeenCalledTimes(1); + expect(releasePostGraphile).not.toHaveBeenCalled(); + }); + + it('passes an exact custom cursor schema to the manager', async () => { + const manager = makeManager(); + const releasePostGraphile = jest.fn().mockResolvedValue(undefined); + const onFatalError = jest.fn(); + + await startConfiguredRealtime({ + cacheKey: 'tenant-a', + resolvedPreset: { + pgServices: [{ + pgSubscriber: { eventEmitter: { emit: jest.fn() } }, + adaptorSettings: { pool: {} } + }] + }, + realtimeSchema: 'ctf_a_realtime', + allowedSourceSchemas: ['ctf_a'], + onFatalError, + releasePostGraphile, + loadManager: async () => manager.constructor, + replicaIdentity: 'replica-a' + }); + + expect(manager.constructor).toHaveBeenCalledWith({ + pgSubscriber: { eventEmitter: { emit: expect.any(Function) } }, + pool: {}, + nodeId: 'graphile-cache:replica-a:tenant-a', + schema: 'ctf_a_realtime', + allowedSourceSchemas: ['ctf_a'], + onFatalError + }); + expect(releasePostGraphile).not.toHaveBeenCalled(); + }); + + it('uses an explicit generation publisher and configured cursor intervals', async () => { + const manager = makeManager(); + const releasePostGraphile = jest.fn().mockResolvedValue(undefined); + const publisher = { + assertTopics: jest.fn(), + publish: jest.fn() + }; + + await startConfiguredRealtime({ + cacheKey: 'shared-exact', + resolvedPreset: { + pgServices: [{ adaptorSettings: { pool: {} } }] + }, + publisher, + pollIntervalMs: 30_000, + heartbeatIntervalMs: 90_000, + allowedSourceSchemas: ['tenant_a'], + releasePostGraphile, + loadManager: async () => manager.constructor + }); + + expect(manager.constructor).toHaveBeenCalledWith(expect.objectContaining({ + publisher, + pollIntervalMs: 30_000, + heartbeatIntervalMs: 90_000 + })); + expect(manager.constructor.mock.calls[0][0]).not.toHaveProperty('pgSubscriber'); + }); + + it('fails closed before loading a manager when no source schema is allowed', async () => { + const manager = makeManager(); + const releasePostGraphile = jest.fn().mockResolvedValue(undefined); + + await expect(startConfiguredRealtime({ + cacheKey: 'no-sources', + resolvedPreset: { + pgServices: [{ + pgSubscriber: { eventEmitter: { emit: jest.fn() } }, + adaptorSettings: { pool: {} } + }] + }, + allowedSourceSchemas: [], + releasePostGraphile, + loadManager: async () => manager.constructor + })).rejects.toMatchObject({ + code: 'GRAPHILE_REALTIME_STARTUP_FAILED' + }); + + expect(manager.constructor).not.toHaveBeenCalled(); + expect(releasePostGraphile).toHaveBeenCalledTimes(1); + }); + + it('separates replica cursor identities while retaining the exact contract key', () => { + const cacheKey = 'graphile:v1:contract-a'; + const first = createGraphileRealtimeNodeId(cacheKey, 'replica-a'); + const second = createGraphileRealtimeNodeId(cacheKey, 'replica-b'); + + expect(first).not.toBe(second); + expect(first).toBe(`graphile-cache:replica-a:${cacheKey}`); + expect(second).toBe(`graphile-cache:replica-b:${cacheKey}`); + }); + + it('latches the first fatal delivery failure for one exact generation', () => { + const health = createGraphileRealtimeHealth(); + const first = Object.assign(new Error('foreign source'), { + code: 'REALTIME_SOURCE_SCHEMA_VIOLATION' + }); + const second = Object.assign(new Error('emitter missing'), { + code: 'REALTIME_SUBSCRIBER_UNAVAILABLE' + }); + + const failed = withGraphileRealtimeFailure(health, first, 1_000); + const stillFailed = withGraphileRealtimeFailure(failed, second, 2_000); + + expect(health).toEqual({ status: 'healthy' }); + expect(failed).toEqual({ + status: 'failed', + failureCode: 'REALTIME_SOURCE_SCHEMA_VIOLATION', + failedAt: 1_000 + }); + expect(stillFailed).toBe(failed); + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/shared-realtime.test.ts b/graphile/graphile-cache/src/__tests__/shared-realtime.test.ts new file mode 100644 index 000000000..1d360a387 --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/shared-realtime.test.ts @@ -0,0 +1,479 @@ +const acquirePgNotificationBroker = jest.fn(); +const getPgNotificationBrokerStats = jest.fn(); +const getPgNotificationBrokerIdentity = jest.fn((config: { password?: string }) => + config.password === 'rotated-secret' + ? 'broker:v1:rotated' + : 'broker:v1:expected'); +const getPgNotificationDatabaseIdentity = jest.fn(() => 'database-target:v1:tenant-a'); + +jest.mock('pg-cache', () => ({ + acquirePgNotificationBroker, + getPgNotificationBrokerStats, + getPgNotificationBrokerIdentity, + getPgNotificationDatabaseIdentity, + PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE: 'PG_NOTIFICATION_LEASE_RELEASED' +})); + +import { + ActivatableGenerationScopedRealtimeSubscriber, + RealtimeTopicCollector +} from 'graphile-realtime-subscriptions'; + +import { + activateGraphileSharedRealtime, + getGraphileRealtimeRoleAuditStats, + GraphileSharedRealtimeDatabaseConflictError, + GraphileSharedRealtimeIdentityError +} from '../shared-realtime'; + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(reason: unknown): void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T) => void; + let reject!: (reason: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +const listenerConfig = { + host: 'db.internal', + port: 5432, + database: 'tenant_a', + user: 'tenant_a_notify', + password: 'never-log-this' +}; + +const successfulAudit = { + version: 'pg-notification-role:v1' as const, + role: 'tenant_a_notify', + database: 'tenant_a', + safe: true, + violations: [] as const +}; + +let brokerAuditAttempts = 0; +let brokerAuditFailures = 0; + +const makeCollector = (): RealtimeTopicCollector => { + const collector = new RealtimeTopicCollector(); + collector.collect([{ + topic: 'realtime:tenant_a.contacts', + schema: 'tenant_a', + table: 'contacts' + }]); + return collector; +}; + +const makeBrokerLease = ( + revalidate = async () => successfulAudit +) => { + const termination = deferred(); + const release = jest.fn(async (): Promise => { + termination.resolve(null); + }); + const revalidateRole = jest.fn(async () => { + brokerAuditAttempts++; + try { + return await revalidate(); + } catch (error) { + brokerAuditFailures++; + throw error; + } + }); + return { + identity: 'broker:v1:expected', + topics: ['realtime:tenant_a.contacts'], + terminated: termination.promise, + roleAudit: successfulAudit, + revalidateRole, + subscribe: jest.fn(() => { + const iterator: AsyncIterableIterator = { + [Symbol.asyncIterator]: () => iterator, + next: () => new Promise(() => undefined), + return: async (): Promise> => ({ + done: true, + value: undefined + }) + }; + return iterator; + }), + release, + termination + }; +}; + +const useBrokerLeases = (...leases: ReturnType[]): void => { + const pending = [...leases]; + acquirePgNotificationBroker.mockImplementation(async () => { + brokerAuditAttempts++; + const lease = pending.shift(); + if (!lease) throw new Error('No mocked notification broker lease remains'); + return lease; + }); +}; + +describe('shared exact realtime activation', () => { + beforeEach(() => { + jest.clearAllMocks(); + brokerAuditAttempts = 0; + brokerAuditFailures = 0; + getPgNotificationBrokerStats.mockImplementation(() => ({ + roleAuditAttempts: brokerAuditAttempts, + roleAuditFailures: brokerAuditFailures + })); + acquirePgNotificationBroker.mockImplementation(async () => { + brokerAuditAttempts++; + return makeBrokerLease(); + }); + }); + + it('installs exact topics only after the broker returns its pinned-client audit', async () => { + const order: string[] = []; + const broker = makeBrokerLease(); + acquirePgNotificationBroker.mockImplementation(async () => { + brokerAuditAttempts++; + order.push('broker'); + return broker; + }); + const subscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const onFatalError = jest.fn(); + + const attestation = await activateGraphileSharedRealtime({ + subscriber, + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError + }); + + expect(order).toEqual(['broker']); + expect(acquirePgNotificationBroker).toHaveBeenCalledWith(listenerConfig, { + topics: ['realtime:tenant_a.contacts'] + }); + expect(attestation.snapshot()).toMatchObject({ + mode: 'shared-exact', + listenerIdentity: 'broker:v1:expected', + auditVersion: 'pg-notification-role:v1', + role: 'tenant_a_notify', + database: 'tenant_a', + status: 'healthy', + checks: 1 + }); + + attestation.release(); + await subscriber.release(); + expect(broker.release).toHaveBeenCalledTimes(1); + }); + + it('latches broker termination into the exact generation health callback', async () => { + const broker = makeBrokerLease(); + useBrokerLeases(broker); + const subscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const onFatalError = jest.fn(); + const attestation = await activateGraphileSharedRealtime({ + subscriber, + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError + }); + const failure = Object.assign(new Error('listener ended'), { + code: 'PG_NOTIFICATION_BROKER_FAILED' + }); + + broker.termination.resolve(failure); + await Promise.resolve(); + await Promise.resolve(); + expect(onFatalError).toHaveBeenCalledWith(failure); + + attestation.release(); + await subscriber.release(); + }); + + it('proactively revalidates once per identity and cancels its unref timer', async () => { + jest.useFakeTimers(); + try { + jest.setSystemTime(1_000); + const firstBroker = makeBrokerLease(); + const secondBroker = makeBrokerLease(); + useBrokerLeases(firstBroker, secondBroker); + const firstSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const secondSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const common = { + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 100, + onFatalError: jest.fn() + }; + const first = await activateGraphileSharedRealtime({ + ...common, + subscriber: firstSubscriber + }); + const second = await activateGraphileSharedRealtime({ + ...common, + subscriber: secondSubscriber + }); + + expect(jest.getTimerCount()).toBe(1); + await jest.advanceTimersByTimeAsync(99); + expect(firstBroker.revalidateRole).not.toHaveBeenCalled(); + expect(secondBroker.revalidateRole).not.toHaveBeenCalled(); + await jest.advanceTimersByTimeAsync(1); + expect(firstBroker.revalidateRole).toHaveBeenCalledTimes(1); + expect(secondBroker.revalidateRole).not.toHaveBeenCalled(); + expect(first.snapshot()).toMatchObject({ + lastAttestedAt: 1_100, + checks: 3, + status: 'healthy' + }); + expect(second.snapshot()).toMatchObject({ checks: 3, status: 'healthy' }); + expect(jest.getTimerCount()).toBe(1); + + first.release(); + expect(jest.getTimerCount()).toBe(1); + second.release(); + expect(jest.getTimerCount()).toBe(0); + await Promise.all([firstSubscriber.release(), secondSubscriber.release()]); + await jest.advanceTimersByTimeAsync(100); + expect(firstBroker.revalidateRole).toHaveBeenCalledTimes(1); + expect(secondBroker.revalidateRole).not.toHaveBeenCalled(); + } finally { + jest.useRealTimers(); + } + }); + + it('retries another generation when the selected revalidator is released', async () => { + const now = jest.spyOn(Date, 'now').mockReturnValue(1_000); + const selectedAudit = deferred(); + const firstBroker = makeBrokerLease(() => selectedAudit.promise); + const secondBroker = makeBrokerLease(); + const thirdBroker = makeBrokerLease(); + useBrokerLeases(firstBroker, secondBroker, thirdBroker); + const firstSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const secondSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const thirdSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const firstFailure = jest.fn(); + const secondFailure = jest.fn(); + const thirdFailure = jest.fn(); + const common = { + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000 + }; + const first = await activateGraphileSharedRealtime({ + ...common, + subscriber: firstSubscriber, + onFatalError: firstFailure + }); + const second = await activateGraphileSharedRealtime({ + ...common, + subscriber: secondSubscriber, + onFatalError: secondFailure + }); + const third = await activateGraphileSharedRealtime({ + ...common, + subscriber: thirdSubscriber, + onFatalError: thirdFailure + }); + + now.mockReturnValue(61_001); + const refreshing = second.revalidateIfDue(); + await Promise.resolve(); + expect(firstBroker.revalidateRole).toHaveBeenCalledTimes(1); + + first.release(); + await firstSubscriber.release(); + selectedAudit.reject(Object.assign(new Error('lease released'), { + code: 'PG_NOTIFICATION_LEASE_RELEASED' + })); + + await expect(refreshing).resolves.toBe(true); + expect(secondBroker.revalidateRole).toHaveBeenCalledTimes(1); + expect(thirdBroker.revalidateRole).not.toHaveBeenCalled(); + expect(second.snapshot()).toMatchObject({ status: 'healthy', checks: 4 }); + expect(third.snapshot()).toMatchObject({ status: 'healthy', checks: 4 }); + expect(firstFailure).not.toHaveBeenCalled(); + expect(secondFailure).not.toHaveBeenCalled(); + expect(thirdFailure).not.toHaveBeenCalled(); + + second.release(); + third.release(); + await Promise.all([secondSubscriber.release(), thirdSubscriber.release()]); + now.mockRestore(); + }); + + it('coalesces TTL refresh and fails every sharing generation closed on drift', async () => { + const statsBefore = getGraphileRealtimeRoleAuditStats(); + const now = jest.spyOn(Date, 'now').mockReturnValue(1_000); + const drift = Object.assign(new Error('role drift'), { + code: 'PG_NOTIFICATION_ROLE_UNSAFE' + }); + const firstBroker = makeBrokerLease(async () => { + throw drift; + }); + const secondBroker = makeBrokerLease(); + useBrokerLeases(firstBroker, secondBroker); + const firstSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const secondSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const firstFailure = jest.fn(); + const secondFailure = jest.fn(); + const common = { + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000 + }; + const first = await activateGraphileSharedRealtime({ + ...common, + subscriber: firstSubscriber, + onFatalError: firstFailure + }); + const second = await activateGraphileSharedRealtime({ + ...common, + subscriber: secondSubscriber, + onFatalError: secondFailure + }); + expect(acquirePgNotificationBroker).toHaveBeenCalledTimes(2); + + now.mockReturnValue(61_001); + await expect(Promise.all([ + first.revalidateIfDue(), + second.revalidateIfDue() + ])).resolves.toEqual([false, false]); + + expect(firstBroker.revalidateRole).toHaveBeenCalledTimes(1); + expect(secondBroker.revalidateRole).not.toHaveBeenCalled(); + expect(firstFailure).toHaveBeenCalledWith(drift); + expect(secondFailure).toHaveBeenCalledWith(drift); + expect(first.snapshot()).toMatchObject({ + status: 'failed', + failureCode: 'PG_NOTIFICATION_ROLE_UNSAFE', + failedAt: 61_001 + }); + expect(getGraphileRealtimeRoleAuditStats()).toMatchObject({ + identities: 1, + failed: 1, + activeIdentityAuditAttempts: 3, + catalogAuditAttempts: statsBefore.catalogAuditAttempts + 3, + catalogAuditFailures: statsBefore.catalogAuditFailures + 1, + activeDatabaseTargets: 1 + }); + + first.release(); + second.release(); + await Promise.all([firstSubscriber.release(), secondSubscriber.release()]); + now.mockRestore(); + }); + + it('rejects a second active listener identity for one physical database', async () => { + const firstBroker = makeBrokerLease(); + const rotatedBroker = makeBrokerLease(); + useBrokerLeases(firstBroker, rotatedBroker); + const firstSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const first = await activateGraphileSharedRealtime({ + subscriber: firstSubscriber, + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError: jest.fn() + }); + const rotatedConfig = { + ...listenerConfig, + password: 'rotated-secret' + }; + const rotatedSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + + await expect(activateGraphileSharedRealtime({ + subscriber: rotatedSubscriber, + topicCollector: makeCollector(), + listenerPgConfig: rotatedConfig, + listenerIdentity: 'broker:v1:rotated', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError: jest.fn() + })).rejects.toBeInstanceOf(GraphileSharedRealtimeDatabaseConflictError); + expect(acquirePgNotificationBroker).toHaveBeenCalledTimes(1); + + first.release(); + await firstSubscriber.release(); + const rotated = await activateGraphileSharedRealtime({ + subscriber: rotatedSubscriber, + topicCollector: makeCollector(), + listenerPgConfig: rotatedConfig, + listenerIdentity: 'broker:v1:rotated', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError: jest.fn() + }); + expect(acquirePgNotificationBroker).toHaveBeenCalledTimes(2); + + rotated.release(); + await rotatedSubscriber.release(); + }); + + it('releases the physical-database reservation when the initial audit fails', async () => { + const auditFailure = new Error('catalog unavailable'); + acquirePgNotificationBroker.mockRejectedValueOnce(auditFailure); + const failedSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + await expect(activateGraphileSharedRealtime({ + subscriber: failedSubscriber, + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:expected', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError: jest.fn() + })).rejects.toBe(auditFailure); + await failedSubscriber.release(); + + const rotatedBroker = makeBrokerLease(); + useBrokerLeases(rotatedBroker); + const rotatedSubscriber = new ActivatableGenerationScopedRealtimeSubscriber(); + const rotated = await activateGraphileSharedRealtime({ + subscriber: rotatedSubscriber, + topicCollector: makeCollector(), + listenerPgConfig: { + ...listenerConfig, + password: 'rotated-secret' + }, + listenerIdentity: 'broker:v1:rotated', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError: jest.fn() + }); + + rotated.release(); + await rotatedSubscriber.release(); + }); + + it('rejects a caller-supplied listener identity mismatch before audit', async () => { + await expect(activateGraphileSharedRealtime({ + subscriber: new ActivatableGenerationScopedRealtimeSubscriber(), + topicCollector: makeCollector(), + listenerPgConfig: listenerConfig, + listenerIdentity: 'broker:v1:wrong', + allowedSourceSchemas: ['tenant_a'], + roleRevalidationMs: 60_000, + onFatalError: jest.fn() + })).rejects.toBeInstanceOf(GraphileSharedRealtimeIdentityError); + expect(acquirePgNotificationBroker).not.toHaveBeenCalled(); + }); +}); diff --git a/graphile/graphile-cache/src/__tests__/websocket-lifecycle.test.ts b/graphile/graphile-cache/src/__tests__/websocket-lifecycle.test.ts new file mode 100644 index 000000000..abdf2734d --- /dev/null +++ b/graphile/graphile-cache/src/__tests__/websocket-lifecycle.test.ts @@ -0,0 +1,263 @@ +import { once } from 'node:events'; +import { PassThrough } from 'node:stream'; + +import type { IncomingMessage } from 'http'; + +import { + disposeUncachedEntry, + getCacheCounters, + GRAPHILE_WEBSOCKET_UNAVAILABLE_CODE, + graphileCache, + type GraphileCacheEntry, + invokeEntryUpgradeHandler, + retireGraphileCacheEntry, + waitForEntryDisposal +} from '../graphile-cache'; +import { createGraphileHttpHandler } from '../http-adapter'; +import { GRAPHILE_REALTIME_UNAVAILABLE_CODE } from '../realtime-readiness'; + +const makeEntry = ( + overrides: Partial = {} +): GraphileCacheEntry => ({ + pgl: { + release: jest.fn(async (): Promise => undefined) + } as unknown as GraphileCacheEntry['pgl'], + serv: {} as GraphileCacheEntry['serv'], + handler: createGraphileHttpHandler(), + httpServer: null, + cacheKey: 'websocket-lifecycle', + createdAt: Date.now(), + ...overrides +}); + +const request = (): IncomingMessage => ({ + aborted: false +}) as IncomingMessage; + +describe('cached Graphile WebSocket lifecycle', () => { + it('retains an exact entry until its accepted socket closes', async () => { + const socket = new PassThrough(); + const upgradeHandler = jest.fn(); + const entry = makeEntry({ upgradeHandler }); + const countersBefore = getCacheCounters(); + + expect(invokeEntryUpgradeHandler(entry, request(), socket, Buffer.alloc(0))).toBe(true); + expect(upgradeHandler).toHaveBeenCalledWith( + expect.anything(), + socket, + expect.any(Buffer) + ); + expect(entry.inflight).toBe(1); + expect(entry.websocketSockets?.has(socket)).toBe(true); + expect(getCacheCounters().websocketUpgradesStarted).toBe( + countersBefore.websocketUpgradesStarted + 1 + ); + expect(getCacheCounters().websocketUpgradesCompleted).toBe( + countersBefore.websocketUpgradesCompleted + ); + + socket.destroy(); + await once(socket, 'close'); + + expect(entry.inflight).toBe(0); + expect(entry.websocketSockets?.size).toBe(0); + expect(getCacheCounters().websocketUpgradesCompleted).toBe( + countersBefore.websocketUpgradesCompleted + 1 + ); + }); + + it('transfers the outer transport only after the exact generation is retained', () => { + const socket = new PassThrough(); + const events: string[] = []; + const entry = makeEntry({ + upgradeHandler: jest.fn(() => events.push('grafserv')) + }); + + expect(invokeEntryUpgradeHandler( + entry, + request(), + socket, + Buffer.from('head'), + { onAccepted: () => events.push('accepted') } + )).toBe(true); + + expect(events).toEqual(['accepted', 'grafserv']); + expect(entry.inflight).toBe(1); + socket.destroy(); + }); + + it('terminates long-lived sockets before disposing their generation', async () => { + const socket = new PassThrough(); + const entry = makeEntry({ upgradeHandler: jest.fn() }); + expect(invokeEntryUpgradeHandler(entry, request(), socket, Buffer.alloc(0))).toBe(true); + + await disposeUncachedEntry(entry); + + expect(socket.destroyed).toBe(true); + expect(entry.inflight).toBe(0); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + }); + + it('retires the exact resident generation and its sockets on a fatal audit', async () => { + const socket = new PassThrough(); + const entry = makeEntry({ + cacheKey: 'websocket-fatal-audit', + upgradeHandler: jest.fn() + }); + graphileCache.set(entry.cacheKey, entry); + expect(invokeEntryUpgradeHandler( + entry, + request(), + socket, + Buffer.alloc(0) + )).toBe(true); + + expect(retireGraphileCacheEntry( + entry, + Object.assign(new Error('listener role changed'), { + code: 'INSUFFICIENT_PRIVILEGE' + }) + )).toBe(true); + + await once(socket, 'close'); + await expect(waitForEntryDisposal(entry, 100)).resolves.toBe(true); + expect(graphileCache.peek(entry.cacheKey)).toBeUndefined(); + expect(entry.realtimeHealth).toMatchObject({ + status: 'failed', + failureCode: 'INSUFFICIENT_PRIVILEGE' + }); + expect(entry.inflight).toBe(0); + expect(entry.pgl.release).toHaveBeenCalledTimes(1); + }); + + it('releases PgSubscriber before cursor cleanup in a saturated max=2 pool', async () => { + const socket = new PassThrough(); + const events: string[] = []; + // Model the two production runtime slots while a subscription is live: + // one PgSubscriber LISTEN checkout and one cursor-tracker checkout. + let occupiedSlots = 2; + const entry = makeEntry({ + upgradeHandler: jest.fn(), + pgl: { + release: jest.fn(async () => { + events.push('postgraphile-release'); + }) + } as unknown as GraphileCacheEntry['pgl'], + releasePresetServices: jest.fn(async () => { + events.push('preset-services-release'); + occupiedSlots -= 1; + }), + realtimeManager: { + stop: jest.fn(async () => { + events.push('realtime-stop'); + if (occupiedSlots >= 2) { + throw new Error('timeout exceeded when trying to connect'); + } + occupiedSlots -= 1; + }) + } + }); + expect(invokeEntryUpgradeHandler(entry, request(), socket, Buffer.alloc(0))).toBe(true); + + await expect(disposeUncachedEntry(entry, 'max-2-live-subscription')).resolves.toBeUndefined(); + + expect(socket.destroyed).toBe(true); + expect(occupiedSlots).toBe(0); + expect(events).toEqual([ + 'postgraphile-release', + 'preset-services-release', + 'realtime-stop' + ]); + }); + + it('releases a caller-owned shared subscriber and attestation exactly once', async () => { + const realtimeSubscriber = { + release: jest.fn(async (): Promise => undefined) + }; + const realtimeRoleAttestation = { + snapshot: jest.fn(), + revalidateIfDue: jest.fn(async () => true), + release: jest.fn() + }; + const entry = makeEntry({ + realtimeSubscriber, + realtimeRoleAttestation + }); + + const first = disposeUncachedEntry(entry, 'shared-owner'); + const second = disposeUncachedEntry(entry, 'shared-owner'); + expect(first).toBe(second); + await first; + + expect(realtimeRoleAttestation.release).toHaveBeenCalledTimes(1); + expect(realtimeSubscriber.release).toHaveBeenCalledTimes(1); + }); + + it('fails closed with a stable response when no upgrade handler exists', async () => { + const socket = new PassThrough(); + let response = ''; + socket.on('data', (chunk) => { + response += chunk.toString(); + }); + const ended = once(socket, 'end'); + + const rejected = jest.fn(); + expect(invokeEntryUpgradeHandler( + makeEntry(), + request(), + socket, + Buffer.alloc(0), + { onRejected: rejected } + )).toBe(true); + await ended; + + expect(response).toContain('HTTP/1.1 503'); + expect(response).toContain(GRAPHILE_WEBSOCKET_UNAVAILABLE_CODE); + expect(rejected).toHaveBeenCalledTimes(1); + }); + + it('rejects a WebSocket upgrade when its listener-role attestation is stale', async () => { + const socket = new PassThrough(); + let response = ''; + socket.on('data', (chunk) => { + response += chunk.toString(); + }); + const ended = once(socket, 'end'); + const entry = makeEntry({ + upgradeHandler: jest.fn(), + realtimeRoleAttestation: { + snapshot: jest.fn(() => ({ + version: 1, + mode: 'shared-exact', + listenerIdentity: 'opaque-listener-identity', + auditVersion: 'pg-notification-role:v1', + role: 'listener', + database: 'tenant_a', + lastAttestedAt: 1, + validUntil: 2, + checks: 1, + status: 'healthy', + failureCode: null as string | null, + failedAt: null as number | null + })), + revalidateIfDue: jest.fn(async () => true), + release: jest.fn() + } + }); + + const rejected = jest.fn(); + expect(invokeEntryUpgradeHandler( + entry, + request(), + socket, + Buffer.alloc(0), + { onRejected: rejected } + )).toBe(true); + await ended; + + expect(entry.upgradeHandler).not.toHaveBeenCalled(); + expect(rejected).toHaveBeenCalledTimes(1); + expect(response).toContain('HTTP/1.1 503'); + expect(response).toContain(GRAPHILE_REALTIME_UNAVAILABLE_CODE); + }); +}); diff --git a/graphile/graphile-cache/src/build-readiness.ts b/graphile/graphile-cache/src/build-readiness.ts new file mode 100644 index 000000000..4b3d1a54c --- /dev/null +++ b/graphile/graphile-cache/src/build-readiness.ts @@ -0,0 +1,27 @@ +export interface GraphileBuildReadiness { + schemaResult: PromiseLike | unknown; + addTo(): PromiseLike | unknown; + ready(): PromiseLike | unknown; + release(): PromiseLike | unknown; + onReleaseError?(error: unknown): void; +} + +/** + * Keep the build coordinator occupied until both schema gathering and the + * HTTP adapter are ready. Failed generations are released before returning. + */ +export const awaitGraphileBuildReadiness = async ( + build: GraphileBuildReadiness +): Promise => { + try { + await build.addTo(); + await Promise.all([build.schemaResult, build.ready()]); + } catch (error) { + try { + await build.release(); + } catch (releaseError) { + build.onReleaseError?.(releaseError); + } + throw error; + } +}; diff --git a/graphile/graphile-cache/src/create-instance.ts b/graphile/graphile-cache/src/create-instance.ts index 575b76758..df973a4d4 100644 --- a/graphile/graphile-cache/src/create-instance.ts +++ b/graphile/graphile-cache/src/create-instance.ts @@ -1,23 +1,80 @@ -import { createServer } from 'node:http'; - import { Logger } from '@pgpmjs/logger'; -import express from 'express'; import { grafserv } from 'grafserv/express/v4'; +import { + ActivatableGenerationScopedRealtimeSubscriber, + type RealtimeTopicCollector +} from 'graphile-realtime-subscriptions'; +import type { PgNotificationListenerConfig, PgPoolLease } from 'pg-cache'; import { postgraphile } from 'postgraphile'; -import type { GraphileCacheEntry } from './graphile-cache'; +import { awaitGraphileBuildReadiness } from './build-readiness'; +import type { + GraphileCacheEntry, + GraphileUpgradeHandler +} from './graphile-cache'; +import { retireGraphileCacheEntry } from './graphile-cache'; +import { + attachGraphileHttpHandler, + createGraphileHttpHandler +} from './http-adapter'; +import { createPresetServicesReleaser } from './preset-services'; +import { + createGraphileRealtimeHealth, + GraphileRealtimeStartupError, + startConfiguredRealtime +} from './realtime-readiness'; +import { + activateGraphileSharedRealtime, + type GraphileRealtimeRoleAttestation +} from './shared-realtime'; const log = new Logger('graphile-cache:create'); -interface GraphileInstanceOptions { +export interface GraphileInstanceOptions { preset: any; cacheKey: string; + poolIdentity?: string; + /** + * Lease protecting the runtime pool for the lifetime of this instance. + * + * The caller owns the lease until `createGraphileInstance()` resolves. Once + * it resolves, ownership transfers to the returned cache entry and its + * disposal lifecycle releases the lease after PostGraphile teardown. + */ + poolLease?: PgPoolLease; + serviceKey?: string; + databaseId?: string | null; /** * When true, a RealtimeManager is created and started alongside the * PostGraphile instance. The pool is extracted from the preset's * pgServices (managed by pg-cache) rather than passed separately. */ enableRealtime?: boolean; + /** + * Build a no-server Grafserv upgrade handler for an outer tenant-aware + * router. The preset must explicitly enable `grafserv.websockets`; the + * cached instance still never attaches its own upgrade listener. + */ + enableWebsockets?: boolean; + /** + * Physical schema containing this instance's realtime cursor functions. + * Omit to use the compatibility default `realtime_public`. + */ + realtimeSchema?: string; + /** Exact physical source schemas allowed to produce realtime events. */ + realtimeSourceSchemas?: readonly string[]; + /** Cursor recovery polling interval; defaults to RealtimeManager's 5s. */ + realtimeCursorPollIntervalMs?: number; + /** Cursor listener heartbeat interval; defaults to RealtimeManager's 30s. */ + realtimeCursorHeartbeatIntervalMs?: number; + /** Opt-in exact-topic shared notification transport. */ + sharedRealtime?: { + subscriber: ActivatableGenerationScopedRealtimeSubscriber; + topicCollector: RealtimeTopicCollector; + listenerPgConfig: PgNotificationListenerConfig; + listenerIdentity: string; + roleRevalidationMs: number; + }; } /** @@ -29,6 +86,8 @@ interface GraphileInstanceOptions { * * Callers are responsible for building the `GraphileConfig.Preset` (including * pgServices, grafserv options, grafast context, etc.) before passing it here. + * When `poolLease` is supplied, ownership transfers only when this promise + * resolves. If instance creation rejects, the caller must release the lease. * * When `enableRealtime` is true, a RealtimeManager is created that bridges * cursor-tracked events from `drain_changes()` into the PostGraphile @@ -39,56 +98,182 @@ interface GraphileInstanceOptions { export const createGraphileInstance = async ( opts: GraphileInstanceOptions ): Promise => { - const { preset, cacheKey, enableRealtime = false } = opts; + const { + preset, + cacheKey, + poolIdentity, + poolLease, + serviceKey, + databaseId, + enableRealtime = false, + enableWebsockets = false, + realtimeSchema, + realtimeSourceSchemas, + realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs, + sharedRealtime + } = opts; + + if (poolLease && poolIdentity && poolLease.identity !== poolIdentity) { + throw new Error( + `PostGraphile[${cacheKey}] pool identity does not match its retained lease` + ); + } const pgl = postgraphile(preset); + const resolvedPreset = pgl.getResolvedPreset(); + const releasePresetServices = createPresetServicesReleaser(resolvedPreset); const serv = pgl.createServ(grafserv); + const handler = createGraphileHttpHandler(); + let upgradeHandler: GraphileUpgradeHandler | null = null; + let startupAttestation: GraphileRealtimeRoleAttestation | undefined; + let startupReleasePromise: Promise | null = null; + const releaseFailedGeneration = (): Promise => { + if (startupReleasePromise) return startupReleasePromise; + startupReleasePromise = (async () => { + let firstError: unknown; + try { + await pgl.release(); + } catch (error) { + firstError = error; + } + try { + await releasePresetServices(); + } catch (error) { + firstError ??= error; + } + try { + startupAttestation?.release(); + } catch (error) { + firstError ??= error; + } + try { + await sharedRealtime?.subscriber.release(); + } catch (error) { + firstError ??= error; + } + if (firstError) throw firstError; + })(); + return startupReleasePromise; + }; - const handler = express(); - const httpServer = createServer(handler); - await serv.addTo(handler, httpServer); - await serv.ready(); + // Start the schema build before wiring grafserv, but do not let this + // factory resolve until both are ready. `serv.ready()` alone does not + // guarantee that PostGraphile's gather/build phase has completed. + await awaitGraphileBuildReadiness({ + schemaResult: pgl.getSchemaResult(), + addTo: async () => { + const presetWebsockets = resolvedPreset.grafserv?.websockets === true; + if (presetWebsockets !== enableWebsockets) { + throw new Error( + `PostGraphile[${cacheKey}] websocket preset and shared routing must agree` + ); + } + await attachGraphileHttpHandler(serv, handler, resolvedPreset, { + sharedWebsocketRouting: enableWebsockets + }); + if (enableWebsockets) { + upgradeHandler = await serv.getUpgradeHandler(); + if (!upgradeHandler) { + throw new Error( + `PostGraphile[${cacheKey}] websocket upgrade handler is unavailable` + ); + } + } + }, + ready: () => serv.ready(), + release: releaseFailedGeneration, + onReleaseError: (releaseError) => { + log.error(`Failed to release PostGraphile[${cacheKey}] after build failure:`, releaseError); + } + }); const entry: GraphileCacheEntry = { pgl, serv, handler, - httpServer, + upgradeHandler, + httpServer: null, cacheKey, + poolIdentity: poolLease?.identity ?? poolIdentity, + poolLease, + releasePresetServices, + serviceKey, + databaseId, createdAt: Date.now(), + ...(sharedRealtime ? { realtimeSubscriber: sharedRealtime.subscriber } : {}) }; if (enableRealtime) { - try { - const { RealtimeManager } = await import('graphile-realtime-subscriptions'); - - // Extract PgSubscriber and pool from the resolved preset's pgServices. - // The pool is the same instance managed by pg-cache (via getPgPool) - // and threaded into the preset by makePgService({ pool, schemas }). - const resolvedPreset = pgl.getResolvedPreset(); - const pgService = (resolvedPreset as any).pgServices?.[0]; - const pgSubscriber = pgService?.pgSubscriber ?? null; - const pool = pgService?.adaptorSettings?.pool ?? null; - - if (!pgSubscriber) { - log.warn(`PostGraphile[${cacheKey}] has no pgSubscriber — RealtimeManager will not be started`); - } else if (!pool) { - log.warn(`PostGraphile[${cacheKey}] has no pool in pgService — RealtimeManager will not be started`); - } else { - const manager = new RealtimeManager({ - pgSubscriber, - pool, - nodeId: `graphile-cache:${cacheKey}`, - schema: 'realtime_public', + const realtimeHealth = createGraphileRealtimeHealth(); + entry.realtimeHealth = realtimeHealth; + const onFatalError = (error: Error): void => { + const alreadyFailed = entry.realtimeHealth?.status === 'failed'; + retireGraphileCacheEntry(entry, error); + if (!alreadyFailed) { + log.error( + `PostGraphile[${cacheKey}] realtime delivery became unavailable:`, + error + ); + } + }; + if (sharedRealtime) { + const pgService = (resolvedPreset as any)?.pgServices?.[0]; + if (pgService?.pgSubscriber !== sharedRealtime.subscriber) { + await releaseFailedGeneration(); + throw new GraphileRealtimeStartupError( + cacheKey, + new Error('Resolved pgService did not retain the provided generation subscriber') + ); + } + try { + startupAttestation = await activateGraphileSharedRealtime({ + ...sharedRealtime, + allowedSourceSchemas: realtimeSourceSchemas ?? [], + onFatalError }); - - await manager.start(); - entry.realtimeManager = manager; - log.info(`RealtimeManager started for PostGraphile[${cacheKey}]`); + entry.realtimeRoleAttestation = startupAttestation; + } catch (error) { + try { + await releaseFailedGeneration(); + } catch (releaseError) { + log.error( + `Failed to release PostGraphile[${cacheKey}] after shared realtime activation failure:`, + releaseError + ); + } + throw error instanceof GraphileRealtimeStartupError + ? error + : new GraphileRealtimeStartupError(cacheKey, error); + } + } + entry.realtimeManager = await startConfiguredRealtime({ + cacheKey, + resolvedPreset, + realtimeSchema, + allowedSourceSchemas: realtimeSourceSchemas ?? [], + ...(sharedRealtime ? { publisher: sharedRealtime.subscriber } : {}), + ...(realtimeCursorPollIntervalMs === undefined + ? {} + : { pollIntervalMs: realtimeCursorPollIntervalMs }), + ...(realtimeCursorHeartbeatIntervalMs === undefined + ? {} + : { heartbeatIntervalMs: realtimeCursorHeartbeatIntervalMs }), + onFatalError, + releasePostGraphile: releaseFailedGeneration + }); + if (entry.realtimeHealth.status === 'failed') { + try { + await entry.realtimeManager.stop(); + } finally { + await releaseFailedGeneration(); } - } catch (err) { - log.error(`Failed to start RealtimeManager for PostGraphile[${cacheKey}]:`, err); + throw new GraphileRealtimeStartupError( + cacheKey, + new Error('Realtime delivery failed during generation activation') + ); } + log.info(`RealtimeManager started for PostGraphile[${cacheKey}]`); } return entry; diff --git a/graphile/graphile-cache/src/graphile-cache.ts b/graphile/graphile-cache/src/graphile-cache.ts index 83782c6a2..bcd7a2a00 100644 --- a/graphile/graphile-cache/src/graphile-cache.ts +++ b/graphile/graphile-cache/src/graphile-cache.ts @@ -1,23 +1,111 @@ +import type { Duplex } from 'node:stream'; +import { getHeapStatistics } from 'node:v8'; + import { Logger } from '@pgpmjs/logger'; -import { parseEnvNumber } from '12factor-env'; import { EventEmitter } from 'events'; -import type { Express } from 'express'; +import type { NextFunction, Request, Response, Router } from 'express'; import type { GrafservBase } from 'grafserv'; -import type { Server as HttpServer } from 'http'; +import type { IncomingMessage, Server as HttpServer } from 'http'; import { LRUCache } from 'lru-cache'; -import { pgCache } from 'pg-cache'; +import { pgCache, type PgPoolLease } from 'pg-cache'; import type { PostGraphileInstance } from 'postgraphile'; +import { + GRAPHILE_REALTIME_UNAVAILABLE_CODE, + type GraphileRealtimeHealth, + withGraphileRealtimeFailure +} from './realtime-readiness'; +import { + getGraphileRealtimeRoleAuditStats, + type GraphileRealtimeRoleAttestation +} from './shared-realtime'; + const log = new Logger('graphile-cache'); +export const GRAPHILE_WEBSOCKET_UNAVAILABLE_CODE = + 'GRAPHILE_WEBSOCKET_UNAVAILABLE'; + +export type GraphileUpgradeHandler = ( + request: IncomingMessage, + socket: Duplex, + head: Buffer +) => void; + // --- Time Constants --- export const ONE_HOUR_MS = 1000 * 60 * 60; export const FIVE_MINUTES_MS = 1000 * 60 * 5; const ONE_DAY = ONE_HOUR_MS * 24; -const ONE_YEAR = ONE_DAY * 366; +const SIX_HOURS_MS = ONE_DAY / 4; // --- Eviction Types --- -export type EvictionReason = 'lru' | 'ttl' | 'manual'; +export type EvictionReason = + | 'lru' + | 'ttl' + | 'manual' + | 'governor' + | 'admission' + | 'realtime'; + +export interface CacheCounters { + /** Transient HTTP requests admitted to an exact resident handler. */ + httpRequestsStarted: number; + /** Admitted HTTP requests that reached a terminal response state. */ + httpRequestsCompleted: number; + /** WebSocket upgrades admitted to an exact resident upgrade handler. */ + websocketUpgradesStarted: number; + /** Admitted WebSocket lifecycles that closed or errored. */ + websocketUpgradesCompleted: number; + evictions: Record; + disposalsStarted: number; + disposalsCompleted: number; + disposalFailures: number; + drainTimeouts: number; + disposalTimeouts: number; + buildRefusals: Record; +} + +const cacheCounters: CacheCounters = { + httpRequestsStarted: 0, + httpRequestsCompleted: 0, + websocketUpgradesStarted: 0, + websocketUpgradesCompleted: 0, + evictions: { + lru: 0, + ttl: 0, + manual: 0, + governor: 0, + admission: 0, + realtime: 0 + }, + disposalsStarted: 0, + disposalsCompleted: 0, + disposalFailures: 0, + drainTimeouts: 0, + disposalTimeouts: 0, + buildRefusals: { + critical_pressure: 0, + insufficient_budget: 0, + rss_budget_exceeded: 0, + disposal_timeout: 0, + resident_busy: 0, + resident_capacity: 0, + disposal_failed: 0 + } +}; + +export const getCacheCounters = (): CacheCounters => ({ + httpRequestsStarted: cacheCounters.httpRequestsStarted, + httpRequestsCompleted: cacheCounters.httpRequestsCompleted, + websocketUpgradesStarted: cacheCounters.websocketUpgradesStarted, + websocketUpgradesCompleted: cacheCounters.websocketUpgradesCompleted, + evictions: { ...cacheCounters.evictions }, + disposalsStarted: cacheCounters.disposalsStarted, + disposalsCompleted: cacheCounters.disposalsCompleted, + disposalFailures: cacheCounters.disposalFailures, + drainTimeouts: cacheCounters.drainTimeouts, + disposalTimeouts: cacheCounters.disposalTimeouts, + buildRefusals: { ...cacheCounters.buildRefusals } +}); // --- Cache Event Emitter --- export interface CacheEvictionEvent { @@ -42,30 +130,243 @@ export const cacheEvents = new CacheEventEmitter(); export interface CacheConfig { max: number; ttl: number; + admissionMode: CacheAdmissionMode; + heapLimitBytes: number; + /** Explicit process-RSS ceiling. Null leaves RSS observable but unbounded. */ + rssLimitBytes: number | null; + instanceHeapBytes: number; + serverReserveBytes: number; + buildReserveBytes: number; + /** Transient RSS reserved before admitting one serialized build. */ + rssBuildReserveBytes: number; + budgetCapacity: number; + calibration: CacheCalibrationProvenance; +} + +export type CacheAdmissionMode = 'evict-idle' | 'preserve-resident'; + +export type CacheCalibrationSource = + | 'default' + | 'environment' + | 'runtime-safety-floor'; + +export interface CacheCalibrationProvenance { + id: string | null; + instanceHeapSource: CacheCalibrationSource; + instanceHeapSampleCount: number; + serverReserveSource: Exclude; + buildReserveSource: Exclude; } +const DEFAULT_INSTANCE_HEAP_BYTES = 512 * 1024 * 1024; +const DEFAULT_SERVER_RESERVE_BYTES = 256 * 1024 * 1024; +const DEFAULT_BUILD_RESERVE_BYTES = 768 * 1024 * 1024; +const DEFAULT_RSS_BUILD_RESERVE_BYTES = DEFAULT_BUILD_RESERVE_BYTES; +const MIN_BACKING_CACHE_ENTRIES = 1024; +const MAX_BACKING_CACHE_ENTRIES = 65_536; +// This is only a sparse-LRU allocation budget, never an estimate of a real +// Graphile instance. Keep it comfortably below every measured instance cost so +// the backing data structure cannot become the density limit before heap +// admission does. +const BACKING_CACHE_BYTES_PER_ENTRY = 256 * 1024; + +export const computeBackingCacheMax = (heapLimitBytes: number): number => { + if (!Number.isFinite(heapLimitBytes) || heapLimitBytes <= 0) { + return MIN_BACKING_CACHE_ENTRIES; + } + return Math.max( + MIN_BACKING_CACHE_ENTRIES, + Math.min( + MAX_BACKING_CACHE_ENTRIES, + Math.floor(heapLimitBytes / BACKING_CACHE_BYTES_PER_ENTRY) + ) + ); +}; + +const BACKING_CACHE_MAX = computeBackingCacheMax( + getHeapStatistics().heap_size_limit +); + +const parsePositiveInt = (value: string | undefined, fallback: number): number => { + const parsed = value ? Number.parseInt(value, 10) : Number.NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +}; + +const parseExplicitPositiveInt = ( + name: string, + value: string | undefined +): number | undefined => { + if (value === undefined) return undefined; + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive safe integer`); + } + return parsed; +}; + +const parseAdmissionMode = (value: string | undefined): CacheAdmissionMode => { + if (value === undefined || value === 'evict-idle') return 'evict-idle'; + if (value === 'preserve-resident') return 'preserve-resident'; + throw new Error( + 'GRAPHILE_CACHE_ADMISSION_MODE must be evict-idle or preserve-resident' + ); +}; + +const resolveCalibrationValue = ( + name: string, + fallback: number +): { bytes: number; source: 'default' | 'environment' } => { + const configured = parseExplicitPositiveInt(name, process.env[name]); + return configured === undefined + ? { bytes: fallback, source: 'default' } + : { bytes: configured, source: 'environment' }; +}; + +const measuredInstanceSamples: number[] = []; + +/** Record a retained-heap sample from a validated warm instance. */ +export const recordInstanceHeapSample = (bytes: number): void => { + if (!Number.isFinite(bytes) || bytes <= 0) return; + measuredInstanceSamples.push(Math.round(bytes)); + if (measuredInstanceSamples.length > 31) measuredInstanceSamples.shift(); +}; + +export const resetInstanceHeapSamples = (): void => { + measuredInstanceSamples.length = 0; +}; + +const median = (values: number[]): number => { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]; +}; + +const resolveInstanceHeapEstimate = (): { + bytes: number; + source: CacheCalibrationSource; +} => { + const configured = resolveCalibrationValue( + 'GRAPHILE_CACHE_INSTANCE_HEAP_BYTES', + DEFAULT_INSTANCE_HEAP_BYTES + ); + if (measuredInstanceSamples.length === 0) return configured; + const measuredWithReserve = Math.ceil(median(measuredInstanceSamples) * 1.2); + if (measuredWithReserve <= configured.bytes) return configured; + return { bytes: measuredWithReserve, source: 'runtime-safety-floor' }; +}; + +export const getInstanceHeapEstimate = (): number => + resolveInstanceHeapEstimate().bytes; + +/** + * Return the number of resident instances for which both steady-state and + * one-build-transient budgets fit. Zero means a build cannot be admitted. + */ +export const computeCapacityFromBudget = ( + heapLimitBytes: number, + instanceHeapBytes: number, + serverReserveBytes = DEFAULT_SERVER_RESERVE_BYTES, + buildReserveBytes = DEFAULT_BUILD_RESERVE_BYTES +): number => { + if ( + heapLimitBytes <= 0 || + instanceHeapBytes <= 0 || + serverReserveBytes + buildReserveBytes > heapLimitBytes + ) { + return 0; + } + const byResidency = Math.floor( + (heapLimitBytes - serverReserveBytes) / instanceHeapBytes + ); + const byRebuild = Math.floor( + (heapLimitBytes - serverReserveBytes - buildReserveBytes) / instanceHeapBytes + ) + 1; + return Math.max( + 0, + Math.min(computeBackingCacheMax(heapLimitBytes), byResidency, byRebuild) + ); +}; + /** * Get cache configuration from environment variables * * Supports: - * - GRAPHILE_CACHE_MAX: Maximum number of entries (default: 50) + * - GRAPHILE_CACHE_MAX: Operator ceiling (default: heap-budget-derived) + * - GRAPHILE_CACHE_ADMISSION_MODE: evict-idle (default) or preserve-resident + * - GRAPHILE_CACHE_RSS_LIMIT_BYTES: Optional absolute process-RSS ceiling + * - GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES: RSS reserved for one build * - GRAPHILE_CACHE_TTL_MS: TTL in milliseconds * - Production default: ONE_YEAR * - Development default: FIVE_MINUTES_MS * - * NOTE: This value should be <= PG_CACHE_MAX (also default: 50) so that - * every cached PostGraphile instance has a live pool backing it. + * Resident instances protect their exact runtime pools with `PgPoolLease`, so + * pool capacity and Graphile heap capacity are independent limits. Pool + * exhaustion fails closed when every registry identity is leased. */ export function getCacheConfig(): CacheConfig { const isDevelopment = process.env.NODE_ENV === 'development'; + const heapLimitBytes = getHeapStatistics().heap_size_limit; + const instanceHeap = resolveInstanceHeapEstimate(); + const serverReserve = resolveCalibrationValue( + 'GRAPHILE_CACHE_SERVER_RESERVE_BYTES', + DEFAULT_SERVER_RESERVE_BYTES + ); + const buildReserve = resolveCalibrationValue( + 'GRAPHILE_CACHE_BUILD_RESERVE_BYTES', + DEFAULT_BUILD_RESERVE_BYTES + ); + const rssLimitBytes = parseExplicitPositiveInt( + 'GRAPHILE_CACHE_RSS_LIMIT_BYTES', + process.env.GRAPHILE_CACHE_RSS_LIMIT_BYTES + ) ?? null; + const rssBuildReserveBytes = parseExplicitPositiveInt( + 'GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES', + process.env.GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES + ) ?? DEFAULT_RSS_BUILD_RESERVE_BYTES; + const instanceHeapBytes = instanceHeap.bytes; + const serverReserveBytes = serverReserve.bytes; + const buildReserveBytes = buildReserve.bytes; + const budgetCapacity = computeCapacityFromBudget( + heapLimitBytes, + instanceHeapBytes, + serverReserveBytes, + buildReserveBytes + ); + const requestedMax = parseExplicitPositiveInt( + 'GRAPHILE_CACHE_MAX', + process.env.GRAPHILE_CACHE_MAX + ) ?? (budgetCapacity || 1); + if (requestedMax > BACKING_CACHE_MAX) { + throw new Error( + `GRAPHILE_CACHE_MAX exceeds heap-scaled backing ceiling ${BACKING_CACHE_MAX}` + ); + } + // The backing LRU requires at least one slot. Admission still fails closed + // when budgetCapacity is zero, so the synthetic slot is never built into. + const max = Math.max(1, Math.min(requestedMax, budgetCapacity || 1)); + const ttl = parsePositiveInt( + process.env.GRAPHILE_CACHE_TTL_MS, + isDevelopment ? FIVE_MINUTES_MS : SIX_HOURS_MS + ); - const max = parseEnvNumber(process.env.GRAPHILE_CACHE_MAX) ?? 50; - - const ttl = - parseEnvNumber(process.env.GRAPHILE_CACHE_TTL_MS) ?? - (isDevelopment ? FIVE_MINUTES_MS : ONE_YEAR); - - return { max, ttl }; + return { + max, + ttl, + admissionMode: parseAdmissionMode(process.env.GRAPHILE_CACHE_ADMISSION_MODE), + heapLimitBytes, + rssLimitBytes, + instanceHeapBytes, + serverReserveBytes, + buildReserveBytes, + rssBuildReserveBytes, + budgetCapacity, + calibration: { + id: process.env.GRAPHILE_CACHE_CALIBRATION_ID?.trim() || null, + instanceHeapSource: instanceHeap.source, + instanceHeapSampleCount: measuredInstanceSamples.length, + serverReserveSource: serverReserve.source, + buildReserveSource: buildReserve.source + } + }; } /** @@ -74,79 +375,230 @@ export function getCacheConfig(): CacheConfig { * Each entry contains: * - pgl: The PostGraphile instance (manages schema, plugins, etc.) * - serv: The Grafserv server instance (handles HTTP/WS) - * - handler: Express app for routing requests - * - httpServer: Node HTTP server (required by grafserv) + * - handler: Lean Express router for routing requests + * - httpServer: Optional legacy/custom server; cached instances use the shared + * outer server and leave this null * - cacheKey: Unique identifier for this entry * - createdAt: Timestamp when this entry was created */ export interface GraphileCacheEntry { pgl: PostGraphileInstance; serv: GrafservBase; - handler: Express; - httpServer: HttpServer; + handler: Router; + /** No-server Grafserv handler selected only after exact tenant routing. */ + upgradeHandler?: GraphileUpgradeHandler | null; + /** Raw sockets retained so disposal can terminate long-lived subscriptions. */ + websocketSockets?: Set; + httpServer: HttpServer | null; cacheKey: string; + /** Opaque pg-cache identity used by this instance. */ + poolIdentity?: string; + /** + * Runtime pool ownership transferred from `createGraphileInstance()`. + * Disposal releases it only after requests and long-lived resources drain. + */ + poolLease?: PgPoolLease; + /** Idempotent release for pgServices owned by this exact preset generation. */ + releasePresetServices?: () => Promise; + /** Routing label for diagnostics and targeted invalidation only. */ + serviceKey?: string; + /** Tenant database id for targeted invalidation. */ + databaseId?: string | null; createdAt: number; /** Optional RealtimeManager for cursor-tracked subscription delivery */ realtimeManager?: { stop(): Promise } | null; + /** Caller-provided shared subscriber; preset services do not own it. */ + realtimeSubscriber?: { release(): Promise } | null; + /** Credential-free role-audit provenance plus coalesced TTL refresh. */ + realtimeRoleAttestation?: GraphileRealtimeRoleAttestation; + /** Fatal delivery failures latch this generation unavailable until rebuilt. */ + realtimeHealth?: GraphileRealtimeHealth; + /** Requests currently executing through this exact instance. */ + inflight?: number; + /** Once true, no new request may enter this instance. */ + disposing?: boolean; + /** Optional retained-heap measurement supplied by the validation harness. */ + retainedHeapBytes?: number; } -// Track disposed entries to prevent double-disposal -const disposedKeys = new Set(); +const disposalPromises = new WeakMap>(); +const activeDisposals = new Set>(); +let failedDisposalCount = 0; +const drainWaiters = new WeakMap void>>(); +const pendingEvictionReasons = new Map(); -// Track keys that are being manually evicted for accurate eviction reason -const manualEvictionKeys = new Set(); +export const getDrainingCount = (): number => activeDisposals.size; + +const notifyDrained = (entry: GraphileCacheEntry): void => { + if ((entry.inflight ?? 0) > 0) return; + const waiters = drainWaiters.get(entry); + if (!waiters) return; + drainWaiters.delete(entry); + for (const resolve of waiters) resolve(); +}; + +const waitForEntryDrain = (entry: GraphileCacheEntry): Promise => { + if ((entry.inflight ?? 0) === 0) return Promise.resolve(); + return new Promise((resolve) => { + const waiters = drainWaiters.get(entry) ?? new Set<() => void>(); + waiters.add(resolve); + drainWaiters.set(entry, waiters); + }); +}; + +export const raceWithClearedTimeout = async ( + promise: Promise, + timeoutMs: number +): Promise<{ timedOut: false; value: T } | { timedOut: true }> => { + let timer: ReturnType | undefined; + const timeout = new Promise<{ timedOut: true }>((resolve) => { + timer = setTimeout(() => resolve({ timedOut: true }), timeoutMs); + timer.unref?.(); + }); + try { + return await Promise.race([ + promise.then((value) => ({ timedOut: false as const, value })), + timeout + ]); + } finally { + if (timer) clearTimeout(timer); + } +}; /** * Dispose a PostGraphile v5 cache entry * * Properly releases resources by: - * 1. Closing the HTTP server if listening - * 2. Releasing the PostGraphile instance (which internally releases grafserv) + * 1. Waiting for resident requests to drain + * 2. Closing the HTTP server and releasing PostGraphile/Grafserv + * 3. Releasing the generation's preset services and PgSubscriber checkout + * 4. Stopping cursor-tracked realtime delivery + * 4. Releasing the retained runtime-pool lease * - * Uses disposedKeys set to prevent double-disposal when closeAllCaches() - * explicitly disposes entries and then clear() triggers the dispose callback. + * The promise is keyed by entry identity, so two generations with the same + * cache key both release exactly once and duplicate teardown is coalesced. */ -const disposeEntry = async (entry: GraphileCacheEntry, key: string): Promise => { - // Prevent double-disposal - if (disposedKeys.has(key)) { - return; - } - disposedKeys.add(key); +const scheduleDisposal = (entry: GraphileCacheEntry, key: string): Promise => { + const existing = disposalPromises.get(entry); + if (existing) return existing; - log.debug(`Disposing PostGraphile[${key}]`); - try { - // Close HTTP server if it's listening - if (entry.httpServer?.listening) { - await new Promise((resolve) => { - entry.httpServer.close(() => resolve()); - }); + entry.disposing = true; + // WebSocket subscriptions are deliberately long-lived. Waiting for clients + // to leave voluntarily would make LRU eviction and shutdown unbounded, so a + // retiring generation terminates only its own exact sockets before draining. + for (const socket of entry.websocketSockets ?? []) socket.destroy(); + cacheCounters.disposalsStarted++; + const pending = (async () => { + const drainTimeoutMs = parsePositiveInt( + process.env.GRAPHILE_CACHE_DRAIN_TIMEOUT_MS, + 30_000 + ); + const initialDrain = await raceWithClearedTimeout(waitForEntryDrain(entry), drainTimeoutMs); + if (initialDrain.timedOut) { + cacheCounters.drainTimeouts++; + log.warn( + `PostGraphile[${key}] still has ${entry.inflight ?? 0} request(s) after ` + + `${drainTimeoutMs}ms; teardown remains deferred until they finish` + ); + // Correctness wins over reclaim speed: never release an instance while a + // resident request is still executing through it. + await waitForEntryDrain(entry); } - // Stop RealtimeManager if present (before releasing PostGraphile) - if (entry.realtimeManager) { - try { - await entry.realtimeManager.stop(); - } catch (err) { - log.error(`Error stopping RealtimeManager for PostGraphile[${key}]:`, err); + + log.debug(`Disposing PostGraphile[${key}]`); + let firstError: unknown; + try { + if (entry.httpServer) { + await new Promise((resolve) => entry.httpServer.close(() => resolve())); } + } catch (error) { + firstError = error; } - // Release PostGraphile instance (this also releases grafserv internally) - if (entry.pgl) { + try { await entry.pgl.release(); + } catch (error) { + firstError ??= error; } - } catch (err) { - log.error(`Error disposing PostGraphile[${key}]:`, err); - } finally { - disposedKeys.delete(key); - } + try { + await entry.releasePresetServices?.(); + } catch (error) { + firstError ??= error; + } + try { + // A live GraphQL subscription may hold the PgSubscriber checkout while + // cursor tracking uses the other slot in the minimum max=2 runtime pool. + // Release Grafserv and the preset services first so cursor cleanup cannot + // deadlock waiting for a checkout that only PgSubscriber teardown returns. + if (entry.realtimeManager) await entry.realtimeManager.stop(); + } catch (error) { + firstError ??= error; + } + try { + entry.realtimeRoleAttestation?.release(); + } catch (error) { + firstError ??= error; + } + try { + await entry.realtimeSubscriber?.release(); + } catch (error) { + firstError ??= error; + } + try { + entry.poolLease?.release(); + } catch (error) { + firstError ??= error; + } + if (firstError) throw firstError; + cacheCounters.disposalsCompleted++; + })(); + + disposalPromises.set(entry, pending); + activeDisposals.add(pending); + void pending + .catch((error) => { + failedDisposalCount++; + cacheCounters.disposalFailures++; + log.error(`Failed to dispose PostGraphile[${key}]:`, error); + }) + .finally(() => activeDisposals.delete(pending)); + return pending; +}; + +/** Dispose an instance that finished building after its contract was invalidated. */ +export const disposeUncachedEntry = ( + entry: GraphileCacheEntry, + key = entry.cacheKey +): Promise => scheduleDisposal(entry, key); + +export const waitForEntryDisposal = async ( + entry: GraphileCacheEntry, + timeoutMs = 20_000 +): Promise => { + const pending = disposalPromises.get(entry); + if (!pending) return true; + const result = await raceWithClearedTimeout(pending, timeoutMs); + if (result.timedOut) cacheCounters.disposalTimeouts++; + return !result.timedOut; +}; + +export const waitForActiveDisposals = async (timeoutMs = 20_000): Promise => { + if (activeDisposals.size === 0) return true; + const result = await raceWithClearedTimeout( + Promise.allSettled([...activeDisposals]), + timeoutMs + ); + if (result.timedOut) cacheCounters.disposalTimeouts++; + return !result.timedOut; }; /** * Determine the eviction reason for a cache entry */ const getEvictionReason = (key: string, entry: GraphileCacheEntry): EvictionReason => { - if (manualEvictionKeys.has(key)) { - manualEvictionKeys.delete(key); - return 'manual'; + const explicit = pendingEvictionReasons.get(key); + if (explicit) { + pendingEvictionReasons.delete(key); + return explicit; } // Check if TTL expired @@ -164,32 +616,597 @@ const initialConfig = getCacheConfig(); // --- Graphile Cache --- export const graphileCache = new LRUCache({ - max: initialConfig.max, + // Admission enforces the dynamic heap-derived maximum. Keep the backing LRU + // at the hard ceiling so validated lower per-instance measurements can raise + // density without reconstructing the cache object. + max: BACKING_CACHE_MAX, ttl: initialConfig.ttl, updateAgeOnGet: true, dispose: (entry, key) => { - // Determine eviction reason before disposal const reason = getEvictionReason(key, entry); + cacheCounters.evictions[reason]++; // Emit eviction event cacheEvents.emitEviction({ key, reason, entry }); log.debug(`Evicting PostGraphile[${key}] (reason: ${reason})`); - // LRU dispose is synchronous, but v5 disposal is async - // Fire and forget the async cleanup - disposeEntry(entry, key).catch((err) => { - log.error(`Failed to dispose PostGraphile[${key}]:`, err); - }); + scheduleDisposal(entry, key); } }); +/** + * The server normally refreshes an expired role attestation before invoking a + * resident entry. Keep the cache boundary fail-closed too: direct consumers + * and synchronous WebSocket upgrades must not serve through an expired or + * failed listener-role proof. + */ +export const isEntryRealtimeUnavailable = (entry: GraphileCacheEntry): boolean => { + if (entry.realtimeHealth?.status === 'failed') return true; + const attestation = entry.realtimeRoleAttestation?.snapshot(); + return Boolean( + attestation + && (attestation.status === 'failed' || Date.now() >= attestation.validUntil) + ); +}; + +/** + * Permanently retire one exact generation after a fail-closed safety check. + * Marking the entry unavailable happens before cache removal so no concurrent + * HTTP request or WebSocket operation can enter between the failure and the + * disposal callback. Only the same resident object may be evicted; a healthy + * replacement with the same deterministic contract key is never touched. + */ +export const retireGraphileCacheEntry = ( + entry: GraphileCacheEntry, + error: unknown, + reason: EvictionReason = 'realtime' +): boolean => { + entry.realtimeHealth = withGraphileRealtimeFailure( + entry.realtimeHealth ?? { status: 'healthy' }, + error + ); + const resident = graphileCache.peek(entry.cacheKey, { allowStale: true }); + if (resident === entry) { + entry.disposing = true; + pendingEvictionReasons.set(entry.cacheKey, reason); + graphileCache.delete(entry.cacheKey); + return true; + } + + // An unpublished failed candidate must be rejected by publication. A stale + // object racing a healthy replacement is already detached and must not alter + // that replacement's lifecycle or masquerade as its disposal. + if (!resident) entry.disposing = true; + + // Cache removal normally destroys these through scheduleDisposal(). Keep the + // boundary fail-closed for an entry racing publication/removal as well. + for (const socket of entry.websocketSockets ?? []) socket.destroy(); + return false; +}; + +/** Enter an instance only while it is resident and not being torn down. */ +export const invokeEntryHandler = ( + entry: GraphileCacheEntry, + req: Request, + res: Response, + next: NextFunction +): boolean => { + const requestEnded = (): boolean => + Boolean( + req.aborted + || req.socket?.destroyed + || res.destroyed + || res.writableEnded + ); + if (requestEnded()) return false; + if (isEntryRealtimeUnavailable(entry)) { + // Retire only this exact resident generation. A delayed fatal callback or + // stale in-flight waiter must never evict a healthy replacement that uses + // the same deterministic build-contract key. + retireGraphileCacheEntry( + entry, + new Error('Graphile realtime generation is unavailable') + ); + if (!res.headersSent) { + res.setHeader('Retry-After', '15'); + res.status(503).json({ + error: { + code: GRAPHILE_REALTIME_UNAVAILABLE_CODE, + message: 'Realtime delivery is unavailable for this GraphQL instance' + } + }); + } + return true; + } + if (entry.disposing) return false; + cacheCounters.httpRequestsStarted++; + entry.inflight = (entry.inflight ?? 0) + 1; + let released = false; + const release = (): void => { + if (released) return; + released = true; + cacheCounters.httpRequestsCompleted++; + entry.inflight = Math.max(0, (entry.inflight ?? 1) - 1); + notifyDrained(entry); + }; + res.once('finish', release); + res.once('close', release); + // The response can close between the initial check and listener attachment. + // Rechecking after attachment turns that race into an ordinary release. + if (requestEnded()) { + res.removeListener('finish', release); + res.removeListener('close', release); + release(); + return false; + } + try { + entry.handler(req, res, next); + } catch (error) { + release(); + throw error; + } + return true; +}; + +/** + * Refresh an expired shared-listener role audit before serving through a + * resident generation. A failed refresh latches realtimeHealth via the + * activation observer, so the normal invocation boundary returns 503. + */ +export const revalidateEntryRealtimeRole = async ( + entry: GraphileCacheEntry +): Promise => { + if (!entry.realtimeRoleAttestation) return true; + return entry.realtimeRoleAttestation.revalidateIfDue(); +}; + +export interface GraphileUpgradeInvocationOptions { + /** Transfer the outer transport after this exact generation is retained. */ + onAccepted?: () => void; + /** Retire outer admission state before a stable cache-level rejection. */ + onRejected?: () => void; +} + +const writeUpgradeError = ( + socket: Duplex, + status: number, + code: string, + retryAfter?: number +): void => { + if (socket.destroyed) return; + const body = JSON.stringify({ error: { code } }); + const headers = [ + `HTTP/1.1 ${status} Service Unavailable`, + 'Connection: close', + 'Content-Type: application/json; charset=utf-8', + `Content-Length: ${Buffer.byteLength(body)}`, + ...(retryAfter == null ? [] : [`Retry-After: ${retryAfter}`]), + '', + body + ].join('\r\n'); + try { + socket.end(headers); + } catch { + socket.destroy(); + } +}; + +/** + * Route one already-authorized WebSocket upgrade into an exact cache entry. + * The outer server owns host/path/API selection; this function owns generation + * health, drain accounting, and bounded teardown of the accepted socket. + */ +export const invokeEntryUpgradeHandler = ( + entry: GraphileCacheEntry, + request: IncomingMessage, + socket: Duplex, + head: Buffer, + options: GraphileUpgradeInvocationOptions = {} +): boolean => { + if (request.aborted || socket.destroyed) return false; + if (isEntryRealtimeUnavailable(entry)) { + retireGraphileCacheEntry( + entry, + new Error('Graphile realtime generation is unavailable') + ); + try { + options.onRejected?.(); + writeUpgradeError(socket, 503, GRAPHILE_REALTIME_UNAVAILABLE_CODE, 15); + } catch (error) { + socket.destroy(); + throw error; + } + return true; + } + if (entry.disposing) return false; + if (!entry.upgradeHandler) { + try { + options.onRejected?.(); + writeUpgradeError(socket, 503, GRAPHILE_WEBSOCKET_UNAVAILABLE_CODE, 15); + } catch (error) { + socket.destroy(); + throw error; + } + return true; + } + + cacheCounters.websocketUpgradesStarted++; + entry.inflight = (entry.inflight ?? 0) + 1; + const sockets = entry.websocketSockets ?? new Set(); + entry.websocketSockets = sockets; + sockets.add(socket); + let released = false; + const release = (): void => { + if (released) return; + released = true; + cacheCounters.websocketUpgradesCompleted++; + socket.removeListener('close', release); + socket.removeListener('error', release); + sockets.delete(socket); + entry.inflight = Math.max(0, (entry.inflight ?? 1) - 1); + notifyDrained(entry); + }; + socket.once('close', release); + socket.once('error', release); + if (request.aborted || socket.destroyed || entry.disposing) { + release(); + return false; + } + try { + // The outer router may own a synthetic HTTP response while it runs tenant + // routing, authentication, and build admission. Transfer that transport + // only after this exact generation has passed every fail-closed check and + // is already accounted as in-flight. + options.onAccepted?.(); + entry.upgradeHandler(request, socket, head); + } catch (error) { + release(); + socket.destroy(); + throw error; + } + return true; +}; + +export type MemoryPressureLevel = 'ok' | 'elevated' | 'critical'; + +export interface MemoryPressure { + level: MemoryPressureLevel; + heapLevel: MemoryPressureLevel; + rssLevel: MemoryPressureLevel | 'unbounded'; + heapUsed: number; + heapLimit: number; + available: number; + ratio: number; + rssBytes: number; + rssLimitBytes: number | null; + rssRatio: number | null; +} + +const parseFraction = (value: string | undefined, fallback: number): number => { + const parsed = value ? Number.parseFloat(value) : Number.NaN; + return Number.isFinite(parsed) && parsed > 0 && parsed < 1 ? parsed : fallback; +}; + +const pressureLevel = ( + ratio: number, + elevatedAt: number, + criticalAt: number +): MemoryPressureLevel => ratio >= criticalAt + ? 'critical' + : ratio >= elevatedAt + ? 'elevated' + : 'ok'; + +export const getMemoryPressure = (): MemoryPressure => { + const stats = getHeapStatistics(); + const memory = process.memoryUsage(); + const heapUsed = memory.heapUsed; + const available = stats.total_available_size ?? Math.max(0, stats.heap_size_limit - heapUsed); + const exhaustible = heapUsed + available; + const ratio = exhaustible > 0 ? heapUsed / exhaustible : 0; + const elevatedAt = parseFraction( + process.env.GRAPHILE_MEMORY_GOVERNOR_ELEVATED, + 0.85 + ); + const criticalAt = parseFraction( + process.env.GRAPHILE_MEMORY_GOVERNOR_CRITICAL, + 0.92 + ); + const heapLevel = pressureLevel(ratio, elevatedAt, criticalAt); + const rssLimitBytes = getCacheConfig().rssLimitBytes; + const rssRatio = rssLimitBytes == null ? null : memory.rss / rssLimitBytes; + const rssLevel = rssRatio == null + ? 'unbounded' as const + : pressureLevel(rssRatio, elevatedAt, criticalAt); + const level: MemoryPressureLevel = heapLevel === 'critical' || rssLevel === 'critical' + ? 'critical' + : heapLevel === 'elevated' || rssLevel === 'elevated' + ? 'elevated' + : 'ok'; + return { + level, + heapLevel, + rssLevel, + heapUsed, + heapLimit: stats.heap_size_limit, + available, + ratio, + rssBytes: memory.rss, + rssLimitBytes, + rssRatio + }; +}; + +export type BuildRefusalReason = + | 'critical_pressure' + | 'insufficient_budget' + | 'rss_budget_exceeded' + | 'disposal_timeout' + | 'resident_busy' + | 'resident_capacity' + | 'disposal_failed'; + +export interface BuildAdmissionDecision { + admit: boolean; + reason?: BuildRefusalReason; + pressure: MemoryPressure; + projectedBytes: number; + heapLimitBytes: number; + projectedRssBytes: number; + rssLimitBytes: number | null; +} + +export const evaluateBuildAdmission = ( + residentCount = graphileCache.size +): BuildAdmissionDecision => { + const config = getCacheConfig(); + const pressure = getMemoryPressure(); + const projectedBytes = + config.serverReserveBytes + + residentCount * config.instanceHeapBytes + + config.buildReserveBytes; + const projectedRssBytes = pressure.rssBytes + config.rssBuildReserveBytes; + if (pressure.level === 'critical') { + return { + admit: false, + reason: 'critical_pressure', + pressure, + projectedBytes, + heapLimitBytes: config.heapLimitBytes, + projectedRssBytes, + rssLimitBytes: config.rssLimitBytes + }; + } + if (failedDisposalCount > 0) { + return { + admit: false, + reason: 'disposal_failed', + pressure, + projectedBytes, + heapLimitBytes: config.heapLimitBytes, + projectedRssBytes, + rssLimitBytes: config.rssLimitBytes + }; + } + // The preserve-resident mode turns the calibrated ceiling into a hard + // admission boundary. Check it before the transient-build calculation: the + // default mode deliberately evaluates a full cache, evicts one idle entry, + // and then evaluates the transient budget again. + if (config.admissionMode === 'preserve-resident' && residentCount >= config.max) { + return { + admit: false, + reason: 'resident_capacity', + pressure, + projectedBytes, + heapLimitBytes: config.heapLimitBytes, + projectedRssBytes, + rssLimitBytes: config.rssLimitBytes + }; + } + if (config.budgetCapacity === 0 || projectedBytes > config.heapLimitBytes) { + return { + admit: false, + reason: 'insufficient_budget', + pressure, + projectedBytes, + heapLimitBytes: config.heapLimitBytes, + projectedRssBytes, + rssLimitBytes: config.rssLimitBytes + }; + } + if ( + config.rssLimitBytes != null + && projectedRssBytes > config.rssLimitBytes + ) { + return { + admit: false, + reason: 'rss_budget_exceeded', + pressure, + projectedBytes, + heapLimitBytes: config.heapLimitBytes, + projectedRssBytes, + rssLimitBytes: config.rssLimitBytes + }; + } + return { + admit: true, + pressure, + projectedBytes, + heapLimitBytes: config.heapLimitBytes, + projectedRssBytes, + rssLimitBytes: config.rssLimitBytes + }; +}; + +export const recordBuildRefusal = (reason: BuildRefusalReason): void => { + cacheCounters.buildRefusals[reason]++; +}; + +export class CacheBuildAdmissionError extends Error { + readonly retryAfterSeconds = 15; + + constructor(readonly reason: BuildRefusalReason) { + super(`Graphile build admission refused: ${reason}`); + this.name = 'CacheBuildAdmissionError'; + } +} + +const evictEntry = ( + key: string, + reason: EvictionReason +): GraphileCacheEntry | undefined => { + const entry = graphileCache.peek(key); + if (!entry) return undefined; + pendingEvictionReasons.set(key, reason); + graphileCache.delete(key); + return entry; +}; + +export const deleteGraphileCacheEntry = async ( + key: string, + reason: EvictionReason = 'manual' +): Promise => { + const entry = evictEntry(key, reason); + if (!entry) return false; + await (disposalPromises.get(entry) ?? Promise.resolve()); + return true; +}; + +/** + * Make one build slot and wait until every evicted instance has truly released. + * This runs inside the global build coordinator, so the size check and eviction + * cannot race another large build. + */ +export const prepareCacheForBuild = async ( + timeoutMs = 20_000 +): Promise<{ evicted: number; decision: BuildAdmissionDecision }> => { + const initial = evaluateBuildAdmission(); + if ( + !initial.admit && + (initial.reason === 'critical_pressure' + || initial.reason === 'disposal_failed' + || initial.reason === 'resident_capacity') + ) { + recordBuildRefusal(initial.reason); + throw new CacheBuildAdmissionError(initial.reason); + } + + const startedAt = Date.now(); + if (!await waitForActiveDisposals(timeoutMs)) { + recordBuildRefusal('disposal_timeout'); + throw new CacheBuildAdmissionError('disposal_timeout'); + } + const targetSize = Math.max(0, getCacheConfig().max - 1); + let evicted = 0; + while (graphileCache.size > targetSize) { + const keys = [...graphileCache.rkeys()]; + const idleKey = keys.find((key) => { + const entry = graphileCache.peek(key); + return entry && !entry.disposing && (entry.inflight ?? 0) === 0; + }); + if (!idleKey) { + recordBuildRefusal('resident_busy'); + throw new CacheBuildAdmissionError('resident_busy'); + } + const victimKey = idleKey; + const entry = evictEntry(victimKey, 'admission'); + if (!entry) continue; + evicted++; + + const remainingMs = Math.max(1, timeoutMs - (Date.now() - startedAt)); + let disposed = false; + try { + disposed = await waitForEntryDisposal(entry, remainingMs); + } catch (error) { + log.error(`PostGraphile[${victimKey}] disposal failed during build admission`, error); + } + if (!disposed) { + recordBuildRefusal('disposal_timeout'); + throw new CacheBuildAdmissionError('disposal_timeout'); + } + } + + const decision = evaluateBuildAdmission(graphileCache.size); + if (!decision.admit && decision.reason) { + recordBuildRefusal(decision.reason); + throw new CacheBuildAdmissionError(decision.reason); + } + return { evicted, decision }; +}; + +let governorTimer: ReturnType | null = null; +let governorUsers = 0; + +export const startMemoryGovernor = (intervalMs = 10_000): (() => void) => { + if (process.env.GRAPHILE_MEMORY_GOVERNOR === '0') return () => {}; + governorUsers++; + if (!governorTimer) { + governorTimer = setInterval(() => { + const pressure = getMemoryPressure(); + if (pressure.level === 'ok') return; + for (const key of graphileCache.rkeys()) { + const entry = graphileCache.peek(key); + // A pressure governor must not interrupt a resident request. + if (entry && !entry.disposing && (entry.inflight ?? 0) === 0) { + log.warn( + `Memory governor evicting PostGraphile[${key}] at ${pressure.level} pressure` + ); + evictEntry(key, 'governor'); + break; + } + } + }, intervalMs); + governorTimer.unref?.(); + } + let released = false; + return () => { + if (released) return; + released = true; + governorUsers = Math.max(0, governorUsers - 1); + if (governorUsers === 0 && governorTimer) { + clearInterval(governorTimer); + governorTimer = null; + } + }; +}; + +export const stopMemoryGovernor = (): void => { + governorUsers = 0; + if (!governorTimer) return; + clearInterval(governorTimer); + governorTimer = null; +}; + // --- Cache Stats --- export interface CacheStats { size: number; max: number; ttl: number; + admissionMode: CacheAdmissionMode; keys: string[]; + realtimeUnhealthy: number; + realtimeRoleAttestations: { + generations: number; + identities: number; + healthy: number; + failed: number; + stale: number; + activeIdentityAuditAttempts: number; + catalogAuditAttempts: number; + catalogAuditFailures: number; + activeDatabaseTargets: number; + databaseConfigurationConflicts: number; + oldestLastAttestedAt: number | null; + }; + draining: number; + budgetCapacity: number; + instanceHeapBytes: number; + heapLimitBytes: number; + rssLimitBytes: number | null; + rssBuildReserveBytes: number; + calibration: CacheCalibrationProvenance; + pressure: MemoryPressure; } /** @@ -197,11 +1214,30 @@ export interface CacheStats { */ export function getCacheStats(): CacheStats { const config = getCacheConfig(); + const realtimeRoleAttestationGenerations = [...graphileCache.values()] + .filter((entry) => Boolean(entry.realtimeRoleAttestation)).length; + const realtimeRoleAuditStats = getGraphileRealtimeRoleAuditStats(); return { size: graphileCache.size, max: config.max, ttl: config.ttl, - keys: [...graphileCache.keys()] + admissionMode: config.admissionMode, + keys: [...graphileCache.keys()], + realtimeUnhealthy: [...graphileCache.values()].filter( + (entry) => entry.realtimeHealth?.status === 'failed' + ).length, + realtimeRoleAttestations: { + generations: realtimeRoleAttestationGenerations, + ...realtimeRoleAuditStats + }, + draining: getDrainingCount(), + budgetCapacity: config.budgetCapacity, + instanceHeapBytes: config.instanceHeapBytes, + heapLimitBytes: config.heapLimitBytes, + rssLimitBytes: config.rssLimitBytes, + rssBuildReserveBytes: config.rssBuildReserveBytes, + calibration: config.calibration, + pressure: getMemoryPressure() }; } @@ -217,8 +1253,7 @@ export function clearMatchingEntries(pattern: RegExp): number { for (const key of graphileCache.keys()) { if (pattern.test(key)) { - // Mark as manual eviction before deleting - manualEvictionKeys.add(key); + pendingEvictionReasons.set(key, 'manual'); graphileCache.delete(key); cleared++; } @@ -227,16 +1262,17 @@ export function clearMatchingEntries(pattern: RegExp): number { return cleared; } -// Register cleanup callback with pgCache -// When a pg pool is disposed, clean up any graphile instances using it -const unregister = pgCache.registerCleanupCallback((pgPoolKey: string) => { +// A retained lease prevents ordinary pg-cache eviction while an entry is +// resident. This callback remains a fail-safe for legacy unleased entries and +// explicit process-wide pg-cache shutdown, which is allowed to override leases. +pgCache.registerCleanupCallback((pgPoolKey: string) => { log.debug(`pgPool[${pgPoolKey}] disposed - checking graphile entries`); // Remove graphile entries that reference this pool key graphileCache.forEach((entry, k) => { - if (entry.cacheKey.includes(pgPoolKey)) { + if (entry.poolIdentity === pgPoolKey) { log.debug(`Removing graphileCache[${k}] due to pgPool[${pgPoolKey}] disposal`); - manualEvictionKeys.add(k); + pendingEvictionReasons.set(k, 'manual'); graphileCache.delete(k); } }); @@ -245,6 +1281,18 @@ const unregister = pgCache.registerCleanupCallback((pgPoolKey: string) => { // Enhanced close function that handles all caches const closePromise: { promise: Promise | null } = { promise: null }; +export const clearGraphileCache = async (): Promise => { + const entries = [...graphileCache.entries()]; + for (const [key] of entries) pendingEvictionReasons.set(key, 'manual'); + graphileCache.clear(); + const disposePromises = entries.map(([, entry]) => disposalPromises.get(entry)); + await Promise.allSettled([ + ...disposePromises.filter((promise): promise is Promise => Boolean(promise)), + ...activeDisposals + ]); + pendingEvictionReasons.clear(); +}; + /** * Close all caches and release resources * @@ -262,28 +1310,9 @@ export const closeAllCaches = async (verbose = false): Promise => { closePromise.promise = (async () => { try { if (verbose) log.info('Closing all server caches...'); + stopMemoryGovernor(); - // Collect all entries and dispose them properly - const entries = [...graphileCache.entries()]; - - // Mark all as manual evictions - for (const [key] of entries) { - manualEvictionKeys.add(key); - } - - const disposePromises = entries.map(([key, entry]) => - disposeEntry(entry, key) - ); - - // Wait for all disposals to complete - await Promise.allSettled(disposePromises); - - // Clear the cache after disposal (dispose callback will no-op due to disposedKeys) - graphileCache.clear(); - - // Clear disposed keys tracking after full cleanup - disposedKeys.clear(); - manualEvictionKeys.clear(); + await clearGraphileCache(); // Close pg pools await pgCache.close(); diff --git a/graphile/graphile-cache/src/http-adapter.ts b/graphile/graphile-cache/src/http-adapter.ts new file mode 100644 index 000000000..7b9e53ddc --- /dev/null +++ b/graphile/graphile-cache/src/http-adapter.ts @@ -0,0 +1,52 @@ +import type { Server as HttpServer } from 'node:http'; +import type { Server as HttpsServer } from 'node:https'; + +import express, { type Express, type Router } from 'express'; + +/** The narrow part of ExpressGrafserv used by a cached HTTP-only instance. */ +export interface GrafservExpressAttachment { + addTo( + app: Express, + server: HttpServer | HttpsServer | null, + addExclusiveWebsocketHandler?: boolean + ): PromiseLike | void; +} + +export interface GraphileHttpAttachmentOptions { + /** + * The caller will route upgrades to this exact cached instance from the + * shared outer HTTP server. Grafserv must never install an exclusive + * listener for a tenant instance because that listener would reject every + * other tenant's path. + */ + sharedWebsocketRouting?: boolean; +} + +/** Allocate only the middleware router that the shared outer server invokes. */ +export const createGraphileHttpHandler = (): Router => express.Router(); + +/** + * Attach Grafserv's HTTP middleware without a private Node server. + * + * With exclusive websocket handling disabled, Grafserv's Express adapter only + * calls `app.use(...)`; Router implements that exact runtime contract. A cached + * per-tenant server never listens, so websocket upgrades must be owned by the + * shared outer server rather than retained on an unreachable dummy server. + */ +export const attachGraphileHttpHandler = ( + serv: GrafservExpressAttachment, + handler: Router, + resolvedPreset: unknown, + options: GraphileHttpAttachmentOptions = {} +): PromiseLike | void => { + if ( + (resolvedPreset as any)?.grafserv?.websockets === true + && options.sharedWebsocketRouting !== true + ) { + throw new Error( + '[graphile-cache] Cached Grafserv instances cannot own WebSocket ' + + 'upgrades; configure a tenant-aware upgrade handler on the shared server' + ); + } + return serv.addTo(handler as unknown as Express, null, false); +}; diff --git a/graphile/graphile-cache/src/index.ts b/graphile/graphile-cache/src/index.ts index 9a845fafe..2f38bf4ab 100644 --- a/graphile/graphile-cache/src/index.ts +++ b/graphile/graphile-cache/src/index.ts @@ -1,29 +1,96 @@ // Main exports from graphile-cache package export { + BuildAdmissionDecision, + BuildRefusalReason, + CacheAdmissionMode, + CacheBuildAdmissionError, + CacheCalibrationProvenance, + CacheCalibrationSource, // Cache configuration CacheConfig, + // Process counters + CacheCounters, // Event emitter for cache events CacheEventEmitter, cacheEvents, CacheEvictionEvent, // Cache stats CacheStats, + clearGraphileCache, // Clear matching entries clearMatchingEntries, closeAllCaches, + // Capacity model and measured instance cost + computeBackingCacheMax, + computeCapacityFromBudget, + deleteGraphileCacheEntry, + disposeUncachedEntry, + evaluateBuildAdmission, // Eviction tracking EvictionReason, FIVE_MINUTES_MS, getCacheConfig, + getCacheCounters, getCacheStats, + getDrainingCount, + getInstanceHeapEstimate, + // Memory pressure governor + getMemoryPressure, + GRAPHILE_WEBSOCKET_UNAVAILABLE_CODE, // Cache instance and entry type graphileCache, GraphileCacheEntry, + GraphileUpgradeHandler, + // Request draining and build admission + invokeEntryHandler, + invokeEntryUpgradeHandler, + isEntryRealtimeUnavailable, + MemoryPressure, + MemoryPressureLevel, // Time constants - ONE_HOUR_MS} from './graphile-cache'; + ONE_HOUR_MS, + prepareCacheForBuild, + raceWithClearedTimeout, + recordBuildRefusal, + recordInstanceHeapSample, + resetInstanceHeapSamples, + retireGraphileCacheEntry, + revalidateEntryRealtimeRole, + startMemoryGovernor, + stopMemoryGovernor, + waitForActiveDisposals, + waitForEntryDisposal +} from './graphile-cache'; // Factory for creating PostGraphile v5 instances +export type { GraphileInstanceOptions } from './create-instance'; export { createGraphileInstance } from './create-instance'; +export type { + GraphileRealtimeHealth, + GraphileRealtimeManager +} from './realtime-readiness'; +export { + createGraphileRealtimeHealth, + createGraphileRealtimeNodeId, + DEFAULT_GRAPHILE_REALTIME_SCHEMA, + GRAPHILE_REALTIME_UNAVAILABLE_CODE, + GraphileRealtimeStartupError, + startConfiguredRealtime, + withGraphileRealtimeFailure +} from './realtime-readiness'; +export type { + ActivateGraphileSharedRealtimeOptions, + GraphileRealtimeRoleAttestation, + GraphileRealtimeRoleAttestationSnapshot, + GraphileRealtimeRoleAuditStats} from './shared-realtime'; +export { + activateGraphileSharedRealtime, + getGraphileRealtimeRoleAuditStats, + GRAPHILE_SHARED_REALTIME_DATABASE_CONFLICT_ERROR_CODE, + GRAPHILE_SHARED_REALTIME_IDENTITY_ERROR_CODE, + GraphileSharedRealtimeDatabaseConflictError, + GraphileSharedRealtimeIdentityError +} from './shared-realtime'; // Generic module config cache for plugin lookups export { ModuleConfigCache, ModuleConfigCacheOptions } from './module-config-cache'; diff --git a/graphile/graphile-cache/src/preset-services.ts b/graphile/graphile-cache/src/preset-services.ts new file mode 100644 index 000000000..c2210828b --- /dev/null +++ b/graphile/graphile-cache/src/preset-services.ts @@ -0,0 +1,33 @@ +interface ReleasablePresetService { + release?: () => void | Promise; +} + +/** + * Own the pgServices created for one resolved PostGraphile preset. + * + * PostGraphile 5.0.3 releases Grafserv but does not release pgServices. Cached + * generations therefore have to do this explicitly or an evicted + * PgSubscriber can retain its LISTEN checkout in the next generation's pool. + */ +export const createPresetServicesReleaser = ( + resolvedPreset: { pgServices?: readonly ReleasablePresetService[] } +): (() => Promise) => { + const services = [...new Set(resolvedPreset.pgServices ?? [])]; + let releasePromise: Promise | null = null; + + return (): Promise => { + if (releasePromise) return releasePromise; + releasePromise = (async () => { + let firstError: unknown; + for (const service of [...services].reverse()) { + try { + await service.release?.(); + } catch (error) { + firstError ??= error; + } + } + if (firstError) throw firstError; + })(); + return releasePromise; + }; +}; diff --git a/graphile/graphile-cache/src/realtime-readiness.ts b/graphile/graphile-cache/src/realtime-readiness.ts new file mode 100644 index 000000000..81294d371 --- /dev/null +++ b/graphile/graphile-cache/src/realtime-readiness.ts @@ -0,0 +1,186 @@ +import { randomUUID } from 'node:crypto'; + +import { Logger } from '@pgpmjs/logger'; +import type { RealtimePublisher } from 'graphile-realtime-subscriptions'; + +const log = new Logger('graphile-cache:realtime'); + +export const DEFAULT_GRAPHILE_REALTIME_SCHEMA = 'realtime_public'; +export const GRAPHILE_REALTIME_UNAVAILABLE_CODE = 'GRAPHILE_REALTIME_UNAVAILABLE'; + +// One module instance represents one Node.js process/worker runtime. A random +// component prevents two replicas serving the same exact build contract from +// sharing a database cursor identity and cleaning up each other's state. +const GRAPHILE_REALTIME_PROCESS_ID = `${process.pid}-${randomUUID()}`; + +export const createGraphileRealtimeNodeId = ( + cacheKey: string, + replicaIdentity = GRAPHILE_REALTIME_PROCESS_ID +): string => `graphile-cache:${replicaIdentity}:${cacheKey}`; + +export type GraphileRealtimeHealth = + | { readonly status: 'healthy' } + | { + readonly status: 'failed'; + readonly failureCode: string | null; + readonly failedAt: number; + }; + +export const createGraphileRealtimeHealth = (): GraphileRealtimeHealth => ({ + status: 'healthy' +}); + +const errorCode = (error: unknown): string | null => { + if (!error || typeof error !== 'object') return null; + const code = (error as { code?: unknown }).code; + return typeof code === 'string' && code.length > 0 ? code : null; +}; + +/** Return the first fatal delivery state; a failed generation stays failed. */ +export const withGraphileRealtimeFailure = ( + health: GraphileRealtimeHealth, + error: unknown, + failedAt = Date.now() +): GraphileRealtimeHealth => health.status === 'failed' + ? health + : { + status: 'failed', + failureCode: errorCode(error), + failedAt + }; + +export class GraphileRealtimeStartupError extends Error { + readonly code = 'GRAPHILE_REALTIME_STARTUP_FAILED'; + + constructor(cacheKey: string, readonly cause?: unknown) { + super(`PostGraphile[${cacheKey}] realtime was configured but could not start`); + this.name = 'GraphileRealtimeStartupError'; + } +} + +export interface GraphileRealtimeManager { + start(): Promise; + stop(): Promise; +} + +export interface GraphileRealtimeManagerConstructor { + new(options: { + pgSubscriber?: any; + publisher?: RealtimePublisher; + pool: any; + nodeId: string; + schema: string; + allowedSourceSchemas: readonly string[]; + pollIntervalMs?: number; + heartbeatIntervalMs?: number; + onFatalError?: (error: Error) => void; + }): GraphileRealtimeManager; +} + +export interface StartConfiguredRealtimeOptions { + cacheKey: string; + resolvedPreset: unknown; + /** + * Physical schema containing the cursor functions for this exact runtime + * identity. Omit to preserve the historical `realtime_public` behavior. + */ + realtimeSchema?: string; + /** Exact physical schemas exposed by this Graphile instance. */ + allowedSourceSchemas: readonly string[]; + /** Explicit generation-local publisher used by shared-exact mode. */ + publisher?: RealtimePublisher; + /** Cursor recovery polling interval. */ + pollIntervalMs?: number; + /** Cursor listener heartbeat interval. */ + heartbeatIntervalMs?: number; + /** Synchronous fatal-delivery callback used to remove the owner from service. */ + onFatalError?: (error: Error) => void; + releasePostGraphile(): PromiseLike | void; + loadManager?: () => Promise; + /** @internal Deterministic injection for replica-identity tests. */ + replicaIdentity?: string; +} + +const defaultLoadManager = async (): Promise => { + const { RealtimeManager } = await import('graphile-realtime-subscriptions'); + return RealtimeManager; +}; + +/** + * Realtime is part of readiness when configured. Any missing dependency or + * startup failure releases the PostGraphile generation before rejecting. + */ +export const startConfiguredRealtime = async ( + options: StartConfiguredRealtimeOptions +): Promise => { + const { + cacheKey, + resolvedPreset, + realtimeSchema = DEFAULT_GRAPHILE_REALTIME_SCHEMA, + allowedSourceSchemas, + publisher, + pollIntervalMs, + heartbeatIntervalMs, + onFatalError, + releasePostGraphile, + loadManager = defaultLoadManager, + replicaIdentity + } = options; + let manager: GraphileRealtimeManager | undefined; + try { + const pgService = (resolvedPreset as any)?.pgServices?.[0]; + const pgSubscriber = pgService?.pgSubscriber ?? null; + const pool = pgService?.adaptorSettings?.pool ?? null; + if (!publisher && !pgSubscriber) { + throw new Error(`PostGraphile[${cacheKey}] resolved without a pgSubscriber`); + } + if (!pool) { + throw new Error(`PostGraphile[${cacheKey}] resolved without a runtime pool`); + } + const exactSourceSchemas = [...new Set(allowedSourceSchemas ?? [])]; + if ( + exactSourceSchemas.length === 0 + || exactSourceSchemas.some( + (schema) => typeof schema !== 'string' || schema.length === 0 + ) + ) { + throw new Error( + `PostGraphile[${cacheKey}] realtime requires at least one allowed source schema` + ); + } + + const RealtimeManager = await loadManager(); + manager = new RealtimeManager({ + ...(publisher ? { publisher } : { pgSubscriber }), + pool, + nodeId: createGraphileRealtimeNodeId(cacheKey, replicaIdentity), + schema: realtimeSchema, + allowedSourceSchemas: exactSourceSchemas, + ...(pollIntervalMs === undefined ? {} : { pollIntervalMs }), + ...(heartbeatIntervalMs === undefined ? {} : { heartbeatIntervalMs }), + ...(onFatalError ? { onFatalError } : {}) + }); + await manager.start(); + return manager; + } catch (error) { + if (manager) { + try { + await manager.stop(); + } catch (stopError) { + log.error( + `Failed to stop partially started RealtimeManager for PostGraphile[${cacheKey}]:`, + stopError + ); + } + } + try { + await releasePostGraphile(); + } catch (releaseError) { + log.error( + `Failed to release PostGraphile[${cacheKey}] after realtime startup failure:`, + releaseError + ); + } + throw new GraphileRealtimeStartupError(cacheKey, error); + } +}; diff --git a/graphile/graphile-cache/src/shared-realtime.ts b/graphile/graphile-cache/src/shared-realtime.ts new file mode 100644 index 000000000..51b3bdd78 --- /dev/null +++ b/graphile/graphile-cache/src/shared-realtime.ts @@ -0,0 +1,490 @@ +import { + ActivatableGenerationScopedRealtimeSubscriber, + type RealtimeTopicCollector +} from 'graphile-realtime-subscriptions'; +import { + acquirePgNotificationBroker, + getPgNotificationBrokerIdentity, + getPgNotificationBrokerStats, + getPgNotificationDatabaseIdentity, + PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE, + type PgAttestedNotificationBrokerLease, + type PgNotificationListenerConfig, + type PgNotificationRoleAudit +} from 'pg-cache'; + +export const GRAPHILE_SHARED_REALTIME_IDENTITY_ERROR_CODE = + 'GRAPHILE_SHARED_REALTIME_IDENTITY_MISMATCH'; +export const GRAPHILE_SHARED_REALTIME_DATABASE_CONFLICT_ERROR_CODE = + 'GRAPHILE_SHARED_REALTIME_DATABASE_CONFLICT'; + +export class GraphileSharedRealtimeIdentityError extends Error { + readonly code = GRAPHILE_SHARED_REALTIME_IDENTITY_ERROR_CODE; + + constructor() { + super('Shared realtime listener identity does not match its connection contract'); + this.name = 'GraphileSharedRealtimeIdentityError'; + } +} + +export class GraphileSharedRealtimeDatabaseConflictError extends Error { + readonly code = GRAPHILE_SHARED_REALTIME_DATABASE_CONFLICT_ERROR_CODE; + + constructor(database: string) { + super( + `Physical database ${JSON.stringify(database)} already has a different active ` + + 'shared realtime listener contract' + ); + this.name = 'GraphileSharedRealtimeDatabaseConflictError'; + } +} + +export interface GraphileRealtimeRoleAttestationSnapshot { + readonly version: 1; + readonly mode: 'shared-exact'; + readonly listenerIdentity: string; + readonly auditVersion: string; + readonly role: string; + readonly database: string; + readonly lastAttestedAt: number; + readonly validUntil: number; + readonly checks: number; + readonly status: 'healthy' | 'failed'; + readonly failureCode: string | null; + readonly failedAt: number | null; +} + +export interface GraphileRealtimeRoleAttestation { + snapshot(): Readonly; + /** Re-audit once this generation's explicit validity window has elapsed. */ + revalidateIfDue(): Promise; + release(): void; +} + +interface SharedAttestationRecord { + readonly identity: string; + readonly role: string; + readonly database: string; + audit: PgNotificationRoleAudit; + lastAttestedAt: number; + revalidationMs: number; + checks: number; + refreshPromise: Promise | null; + refreshTimer: ReturnType | null; + failure: { code: string | null; failedAt: number } | null; + bindings: Set; +} + +interface SharedAttestationBinding { + readonly revalidationMs: number; + readonly onFailure: (error: Error) => void; + readonly revalidateRole: () => Promise; +} + +interface ActiveDatabaseListenerContract { + readonly listenerIdentity: string; + readonly role: string; + references: number; +} + +const attestationRecords = new Map(); +const activeDatabaseListenerContracts = new Map< +string, +ActiveDatabaseListenerContract +>(); +let databaseConfigurationConflicts = 0; + +export interface GraphileRealtimeRoleAuditStats { + readonly identities: number; + readonly healthy: number; + readonly failed: number; + readonly stale: number; + readonly activeIdentityAuditAttempts: number; + readonly catalogAuditAttempts: number; + readonly catalogAuditFailures: number; + readonly activeDatabaseTargets: number; + readonly databaseConfigurationConflicts: number; + readonly oldestLastAttestedAt: number | null; +} + +/** Process-level unique identity counts plus monotonic catalog-audit counters. */ +export const getGraphileRealtimeRoleAuditStats = ( + now = Date.now() +): Readonly => { + const records = [...attestationRecords.values()]; + const brokerStats = getPgNotificationBrokerStats(); + return Object.freeze({ + identities: records.length, + healthy: records.filter(({ failure }) => !failure).length, + failed: records.filter(({ failure }) => Boolean(failure)).length, + stale: records.filter( + ({ lastAttestedAt, revalidationMs }) => now >= lastAttestedAt + revalidationMs + ).length, + activeIdentityAuditAttempts: records.reduce( + (sum, { checks }) => sum + checks, + 0 + ), + catalogAuditAttempts: brokerStats.roleAuditAttempts, + catalogAuditFailures: brokerStats.roleAuditFailures, + activeDatabaseTargets: activeDatabaseListenerContracts.size, + databaseConfigurationConflicts, + oldestLastAttestedAt: records.length === 0 + ? null + : Math.min(...records.map(({ lastAttestedAt }) => lastAttestedAt)) + }); +}; + +const errorCode = (error: unknown): string | null => { + if (!error || typeof error !== 'object') return null; + const code = (error as { code?: unknown }).code; + return typeof code === 'string' && code.length > 0 ? code : null; +}; + +const reserveDatabaseListenerContract = (options: { + databaseIdentity: string; + listenerIdentity: string; + role: string; + database: string; +}): (() => void) => { + const { databaseIdentity, listenerIdentity, role, database } = options; + let record = activeDatabaseListenerContracts.get(databaseIdentity); + if ( + record + && (record.listenerIdentity !== listenerIdentity || record.role !== role) + ) { + databaseConfigurationConflicts++; + throw new GraphileSharedRealtimeDatabaseConflictError(database); + } + if (record) { + record.references++; + } else { + record = { listenerIdentity, role, references: 1 }; + activeDatabaseListenerContracts.set(databaseIdentity, record); + } + let released = false; + return (): void => { + if (released) return; + released = true; + record!.references--; + if ( + record!.references === 0 + && activeDatabaseListenerContracts.get(databaseIdentity) === record + ) { + activeDatabaseListenerContracts.delete(databaseIdentity); + } + }; +}; + +const withDatabaseContractReservation = ( + source: PgAttestedNotificationBrokerLease, + releaseReservation: () => void +): PgAttestedNotificationBrokerLease => { + let releasePromise: Promise | null = null; + return Object.freeze({ + identity: source.identity, + topics: source.topics, + terminated: source.terminated, + get roleAudit(): PgNotificationRoleAudit { + return source.roleAudit; + }, + revalidateRole(): Promise { + return source.revalidateRole(); + }, + subscribe(topic: string): AsyncIterableIterator { + return source.subscribe(topic); + }, + release(): Promise { + if (releasePromise) return releasePromise; + releasePromise = (async () => { + try { + await source.release(); + } finally { + releaseReservation(); + } + })(); + return releasePromise; + } + }); +}; + +const MAX_TIMER_DELAY_MS = 2_147_483_647; + +const clearRefreshTimer = (record: SharedAttestationRecord): void => { + if (!record.refreshTimer) return; + clearTimeout(record.refreshTimer); + record.refreshTimer = null; +}; + +function scheduleRefresh(record: SharedAttestationRecord): void { + clearRefreshTimer(record); + if (record.failure || record.bindings.size === 0) return; + const dueAt = record.lastAttestedAt + record.revalidationMs; + const delay = Math.max( + 0, + Math.min(MAX_TIMER_DELAY_MS, dueAt - Date.now()) + ); + record.refreshTimer = setTimeout(() => { + record.refreshTimer = null; + if (record.failure || record.bindings.size === 0) return; + // Very large TTLs are scheduled in safe setTimeout-sized chunks. + if (Date.now() < record.lastAttestedAt + record.revalidationMs) { + scheduleRefresh(record); + return; + } + void refreshRecord(record); + }, delay); + record.refreshTimer.unref?.(); +} + +const revalidateWithActiveBinding = async ( + record: SharedAttestationRecord +): Promise => { + const attempted = new Set(); + for (;;) { + const binding = [...record.bindings].find((candidate) => !attempted.has(candidate)); + if (!binding) { + throw new Error('Shared realtime role attestation has no active broker lease'); + } + attempted.add(binding); + try { + return await binding.revalidateRole(); + } catch (error) { + if ( + errorCode(error) === PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE + && !record.bindings.has(binding) + ) { + continue; + } + throw error; + } + } +}; + +function refreshRecord(record: SharedAttestationRecord): Promise { + if (record.failure) return Promise.resolve(false); + if (record.refreshPromise) return record.refreshPromise; + record.checks++; + const pending = (async (): Promise => { + try { + const audit = await revalidateWithActiveBinding(record); + record.audit = audit; + record.lastAttestedAt = Date.now(); + return true; + } catch (reason) { + const error = reason instanceof Error ? reason : new Error(String(reason)); + record.failure = { + code: errorCode(error), + failedAt: Date.now() + }; + clearRefreshTimer(record); + for (const binding of [...record.bindings]) { + try { + binding.onFailure(error); + } catch { + // Every observer is advisory; the failed record remains latched. + } + } + return false; + } + })(); + record.refreshPromise = pending; + void pending.then(() => { + if (record.refreshPromise === pending) record.refreshPromise = null; + if (record.bindings.size === 0) { + clearRefreshTimer(record); + if (attestationRecords.get(record.identity) === record) { + attestationRecords.delete(record.identity); + } + } else if (!record.failure) { + scheduleRefresh(record); + } + }); + return pending; +} + +const registerAttestation = (options: { + identity: string; + audit: PgNotificationRoleAudit; + attestedAt: number; + revalidationMs: number; + onFailure(error: Error): void; + revalidateRole(): Promise; +}): GraphileRealtimeRoleAttestation => { + const { + identity, + audit, + attestedAt, + revalidationMs, + onFailure, + revalidateRole + } = options; + let record = attestationRecords.get(identity); + if (!record) { + record = { + identity, + role: audit.role, + database: audit.database, + audit, + lastAttestedAt: attestedAt, + revalidationMs, + checks: 1, + refreshPromise: null, + refreshTimer: null, + failure: null, + bindings: new Set() + }; + attestationRecords.set(identity, record); + } else { + // Broker identity covers credentials, database, pool, TLS, and driver. + // A freshly successful acquisition audit supersedes older provenance. + record.audit = audit; + record.lastAttestedAt = attestedAt; + record.checks++; + record.failure = null; + } + const binding: SharedAttestationBinding = { + revalidationMs, + onFailure, + revalidateRole + }; + record.bindings.add(binding); + record.revalidationMs = Math.min( + ...[...record.bindings].map((active) => active.revalidationMs) + ); + scheduleRefresh(record); + let released = false; + + return Object.freeze({ + snapshot(): Readonly { + const failure = record!.failure; + return Object.freeze({ + version: 1, + mode: 'shared-exact', + listenerIdentity: identity, + auditVersion: record!.audit.version, + role: record!.role, + database: record!.database, + lastAttestedAt: record!.lastAttestedAt, + validUntil: record!.lastAttestedAt + revalidationMs, + checks: record!.checks, + status: failure ? 'failed' : 'healthy', + failureCode: failure?.code ?? null, + failedAt: failure?.failedAt ?? null + }); + }, + async revalidateIfDue(): Promise { + if (released || record!.failure) return false; + if (Date.now() < record!.lastAttestedAt + revalidationMs) return true; + return refreshRecord(record!); + }, + release(): void { + if (released) return; + released = true; + record!.bindings.delete(binding); + if (record!.bindings.size === 0) { + clearRefreshTimer(record!); + if (!record!.refreshPromise) attestationRecords.delete(identity); + } else { + record!.revalidationMs = Math.min( + ...[...record!.bindings].map((active) => active.revalidationMs) + ); + scheduleRefresh(record!); + } + } + }); +}; + +export interface ActivateGraphileSharedRealtimeOptions { + subscriber: ActivatableGenerationScopedRealtimeSubscriber; + topicCollector: RealtimeTopicCollector; + listenerPgConfig: PgNotificationListenerConfig; + listenerIdentity: string; + allowedSourceSchemas: readonly string[]; + roleRevalidationMs: number; + onFatalError(error: Error): void; +} + +/** + * Cross the shared-listener publication boundary. Topic validation and a fresh + * role audit finish before the broker lease is installed into PostGraphile. + */ +export const activateGraphileSharedRealtime = async ( + options: ActivateGraphileSharedRealtimeOptions +): Promise => { + const { + subscriber, + topicCollector, + listenerPgConfig, + listenerIdentity, + allowedSourceSchemas, + roleRevalidationMs, + onFatalError + } = options; + const expectedIdentity = getPgNotificationBrokerIdentity(listenerPgConfig); + if (expectedIdentity !== listenerIdentity) { + throw new GraphileSharedRealtimeIdentityError(); + } + if (!Number.isSafeInteger(roleRevalidationMs) || roleRevalidationMs <= 0) { + throw new Error('Shared realtime role revalidation interval must be positive'); + } + const topics = topicCollector.exactTopics(allowedSourceSchemas); + const role = listenerPgConfig.user; + const database = listenerPgConfig.database; + const databaseIdentity = getPgNotificationDatabaseIdentity(listenerPgConfig); + const releaseDatabaseReservation = reserveDatabaseListenerContract({ + databaseIdentity, + listenerIdentity, + role, + database + }); + + // This audit is intentionally fresh for every generation acquisition. The + // role may have drifted since an older generation joined the same broker. + let brokerLease: Awaited>; + try { + // Broker admission serializes this generation's fresh role audit and LISTEN + // on the same pinned client, which remains safe with pool max=1. + brokerLease = await acquirePgNotificationBroker(listenerPgConfig, { topics }); + } catch (error) { + releaseDatabaseReservation(); + throw error; + } + + const reservedBrokerLease = withDatabaseContractReservation( + brokerLease, + releaseDatabaseReservation + ); + + const reportBrokerTermination = (failure: Error): void => { + try { + onFatalError(failure); + } catch { + // The subscriber still fails all streams even if an observer throws. + } + }; + void reservedBrokerLease.terminated.then((failure) => { + if (failure) reportBrokerTermination(failure); + }); + + try { + await subscriber.activate({ + source: reservedBrokerLease, + allowedTopics: topics + }); + } catch (error) { + try { + await reservedBrokerLease.release(); + } catch { + // Preserve the activation failure; reservation release runs in finally. + } + throw error; + } + return registerAttestation({ + identity: listenerIdentity, + audit: reservedBrokerLease.roleAudit, + attestedAt: Date.now(), + revalidationMs: roleRevalidationMs, + onFailure: reportBrokerTermination, + revalidateRole: () => reservedBrokerLease.revalidateRole() + }); +}; diff --git a/graphile/graphile-realtime-subscriptions/README.md b/graphile/graphile-realtime-subscriptions/README.md index b546a5067..3ef950d1e 100644 --- a/graphile/graphile-realtime-subscriptions/README.md +++ b/graphile/graphile-realtime-subscriptions/README.md @@ -30,6 +30,29 @@ const preset = { 4. The subscription re-queries the source table with RLS enforced 5. The client receives `{ event, row }` where `row` reflects the current state +## Generation-Scoped Delivery + +`GenerationScopedRealtimeSubscriber` wraps a shared Grafast notification +source with an exact topic allowlist. Database notifications still fan out to +every generation that leased that topic, while `publish()` sends cursor +catch-up events only to subscriptions owned by that one Graphile generation. +The facade uses fixed bounded queues, fails a slow subscription on overflow, +and awaits its source iterators and source lease during `release()`. + +`RealtimeManager` accepts this explicit publisher capability. A transitional +`createPgSubscriberPublisher()` adapter retains compatibility with the current +`@dataplan/pg` subscriber, keeping its private emitter access out of the +manager. New shared-listener integrations should use the generation-scoped +facade so cursor events cannot cross generation boundaries. + +`RealtimeTopicCollector` receives the plugin's physical schema/table +descriptors during build and rejects missing, empty, changed, malformed, or +foreign topic sets. `ActivatableGenerationScopedRealtimeSubscriber` gives +PostGraphile a stable subscriber identity before schema construction, but +fails every subscribe/publish call until the validated exact-topic source is +installed. This two-phase boundary prevents an instance from serving while its +shared listener is incomplete. + ## Subscription Modes ### Phase 3a (current) diff --git a/graphile/graphile-realtime-subscriptions/__tests__/cursor-tracker.test.ts b/graphile/graphile-realtime-subscriptions/__tests__/cursor-tracker.test.ts index 05a3c9b50..bd43487f7 100644 --- a/graphile/graphile-realtime-subscriptions/__tests__/cursor-tracker.test.ts +++ b/graphile/graphile-realtime-subscriptions/__tests__/cursor-tracker.test.ts @@ -21,6 +21,7 @@ jest.mock('@pgpmjs/logger', () => ({ import { CursorTracker, + CursorTrackerStartAbortedError, DEFAULT_BATCH_LIMIT, DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_POLL_INTERVAL_MS, @@ -51,6 +52,16 @@ function createChangeLogEntry(overrides: Partial = {}): ChangeLo }; } +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + // --- Tests --- describe('CursorTracker defaults', () => { @@ -152,6 +163,38 @@ describe('CursorTracker.start()', () => { await tracker.stop(); }); + + it('fails readiness and rolls back when listener registration fails', async () => { + const error = new Error('touch denied'); + const pool: Queryable = { query: jest.fn().mockRejectedValue(error) }; + const onError = jest.fn(); + const tracker = new CursorTracker({ pool, onError }); + + await expect(tracker.start()).rejects.toBe(error); + + expect(tracker.isRunning).toBe(false); + expect(onError).toHaveBeenCalledWith(error); + expect((pool.query as jest.Mock).mock.calls).toHaveLength(1); + }); + + it('fails readiness and cleans up when the initial drain fails', async () => { + const error = new Error('drain denied'); + const pool: Queryable = { + query: jest.fn().mockImplementation(async (sql: string) => { + if (sql.includes('drain_changes')) throw error; + return { rows: [] }; + }) + }; + const tracker = new CursorTracker({ nodeId: 'strict-node', pool }); + + await expect(tracker.start()).rejects.toBe(error); + + expect(tracker.isRunning).toBe(false); + expect(pool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['strict-node'] + ); + }); }); describe('CursorTracker.stop()', () => { @@ -223,6 +266,112 @@ describe('CursorTracker.stop()', () => { expect(clearSpy).toHaveBeenCalledTimes(2); clearSpy.mockRestore(); }); + + it('waits for an active poll and suppresses its dispatch after stop begins', async () => { + const pool = createMockPool(); + const onChanges = jest.fn(); + const tracker = new CursorTracker({ + nodeId: 'poll-stop-node', + pool, + onChanges, + }); + await tracker.start(); + + const poll = deferred<{ rows: { drain_changes: ChangeLogEntry }[] }>(); + pool.query.mockImplementation((sql: string) => { + if (sql.includes('drain_changes')) return poll.promise; + return Promise.resolve({ rows: [] }); + }); + pool.query.mockClear(); + + const activeDrain = tracker.drain(); + const stopping = tracker.stop(); + let stopped = false; + void stopping.then(() => { + stopped = true; + }); + await Promise.resolve(); + + expect(stopped).toBe(false); + expect(pool.query.mock.calls.some(([sql]) => sql.includes('cleanup_ephemeral'))).toBe(false); + + const entry = createChangeLogEntry(); + poll.resolve({ rows: [{ drain_changes: entry }] }); + await expect(activeDrain).resolves.toEqual([entry]); + await stopping; + + expect(onChanges).not.toHaveBeenCalled(); + expect(pool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['poll-stop-node'] + ); + }); + + it('waits for an active heartbeat before cleaning up the listener', async () => { + const pool = createMockPool(); + const tracker = new CursorTracker({ + nodeId: 'heartbeat-stop-node', + pool, + }); + await tracker.start(); + + const heartbeat = deferred<{ rows: never[] }>(); + pool.query.mockImplementation((sql: string) => { + if (sql.includes('touch_listener')) return heartbeat.promise; + return Promise.resolve({ rows: [] }); + }); + pool.query.mockClear(); + + const activeHeartbeat = tracker.touchListener(); + const stopping = tracker.stop(); + let stopped = false; + void stopping.then(() => { + stopped = true; + }); + await Promise.resolve(); + + expect(stopped).toBe(false); + expect(pool.query.mock.calls.some(([sql]) => sql.includes('cleanup_ephemeral'))).toBe(false); + + heartbeat.resolve({ rows: [] }); + await activeHeartbeat; + await stopping; + + expect(pool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['heartbeat-stop-node'] + ); + }); + + it('aborts startup deterministically when stop wins the registration race', async () => { + const registration = deferred<{ rows: never[] }>(); + const pool: jest.Mocked = { + query: jest.fn().mockImplementation((sql: string) => { + if (sql.includes('touch_listener')) return registration.promise; + return Promise.resolve({ rows: [] }); + }), + }; + const tracker = new CursorTracker({ + nodeId: 'start-stop-node', + pool, + }); + + const starting = tracker.start(); + const startResult = expect(starting).rejects.toBeInstanceOf(CursorTrackerStartAbortedError); + await Promise.resolve(); + await Promise.resolve(); + expect(pool.query.mock.calls.some(([sql]) => sql.includes('touch_listener'))).toBe(true); + + const stopping = tracker.stop(); + registration.resolve({ rows: [] }); + + await startResult; + await stopping; + + expect(tracker.isRunning).toBe(false); + expect(pool.query.mock.calls.some(([sql]) => sql.includes('drain_changes'))).toBe(false); + expect(pool.query.mock.calls.filter(([sql]) => sql.includes('cleanup_ephemeral'))).toHaveLength(1); + }); }); describe('CursorTracker.drain()', () => { diff --git a/graphile/graphile-realtime-subscriptions/__tests__/generation-subscriber.test.ts b/graphile/graphile-realtime-subscriptions/__tests__/generation-subscriber.test.ts new file mode 100644 index 000000000..8ee65a7e1 --- /dev/null +++ b/graphile/graphile-realtime-subscriptions/__tests__/generation-subscriber.test.ts @@ -0,0 +1,285 @@ +import type { GrafastSubscriber } from 'grafast'; + +import { + ActivatableGenerationScopedRealtimeSubscriber, + GENERATION_SUBSCRIBER_QUEUE_CAPACITY, + GenerationScopedRealtimeSubscriber, + RealtimeGenerationNotActiveError, + RealtimeGenerationOverflowError, + RealtimeGenerationSourceEndedError, + RealtimeGenerationTopicError +} from '../src/generation-subscriber'; + +interface Deferred { + promise: Promise; + resolve(value: T | PromiseLike): void; + reject(error: unknown): void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +class ManualIterator implements AsyncIterableIterator { + private readonly buffered: string[] = []; + private readonly waiting: Deferred>[] = []; + private failure: Error | null = null; + private done = false; + readonly returnMock = jest.fn(async (): Promise> => { + this.complete(); + return { done: true, value: undefined }; + }); + + [Symbol.asyncIterator](): AsyncIterableIterator { + return this; + } + + next(): Promise> { + const value = this.buffered.shift(); + if (value !== undefined) return Promise.resolve({ done: false, value }); + if (this.failure) return Promise.reject(this.failure); + if (this.done) return Promise.resolve({ done: true, value: undefined }); + const result = deferred>(); + this.waiting.push(result); + return result.promise; + } + + return(): Promise> { + return this.returnMock(); + } + + throw(error?: unknown): Promise> { + const failure = error instanceof Error ? error : new Error(String(error)); + this.fail(failure); + return Promise.reject(failure); + } + + push(value: string): void { + const waiter = this.waiting.shift(); + if (waiter) waiter.resolve({ done: false, value }); + else this.buffered.push(value); + } + + fail(error: Error): void { + this.failure = error; + for (const waiter of this.waiting.splice(0)) waiter.reject(error); + } + + complete(): void { + this.done = true; + for (const waiter of this.waiting.splice(0)) { + waiter.resolve({ done: true, value: undefined }); + } + } +} + +class ManualSource implements GrafastSubscriber> { + readonly streams = new Map>(); + readonly release = jest.fn(async (): Promise => {}); + + subscribe(topic: string): AsyncIterableIterator { + const stream = new ManualIterator(); + let streams = this.streams.get(topic); + if (!streams) { + streams = new Set(); + this.streams.set(topic, streams); + } + streams.add(stream); + return stream; + } + + publish(topic: string, payload: string): void { + for (const stream of this.streams.get(topic) ?? []) stream.push(payload); + } + + fail(topic: string, error: Error): void { + for (const stream of this.streams.get(topic) ?? []) stream.fail(error); + } + + complete(topic: string): void { + for (const stream of this.streams.get(topic) ?? []) stream.complete(); + } +} + +const flushMicrotasks = async (): Promise => { + for (let index = 0; index < 8; index++) await Promise.resolve(); +}; + +describe('GenerationScopedRealtimeSubscriber', () => { + it('merges database notifications with generation-local cursor publications', async () => { + const source = new ManualSource(); + const facade = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['realtime:tenant_a.contacts'] + }); + const stream = facade.subscribe('realtime:tenant_a.contacts'); + await flushMicrotasks(); + + source.publish('realtime:tenant_a.contacts', 'INSERT:db-row'); + await expect(stream.next()).resolves.toMatchObject({ value: 'INSERT:db-row' }); + + facade.publish('realtime:tenant_a.contacts', 'UPDATE:cursor-row'); + await expect(stream.next()).resolves.toMatchObject({ value: 'UPDATE:cursor-row' }); + await facade.release(); + expect(source.release).toHaveBeenCalledTimes(1); + }); + + it('enforces exact allowlists rather than prefixes', async () => { + const source = new ManualSource(); + const facade = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['realtime:tenant.contacts'] + }); + + expect(() => facade.subscribe('realtime:tenant.contacts.private')) + .toThrow(RealtimeGenerationTopicError); + expect(() => facade.publish('realtime:tenant', 'INSERT:wrong')) + .toThrow(RealtimeGenerationTopicError); + await facade.release(); + }); + + it('keeps cursor publications inside their Graphile generation', async () => { + const source = new ManualSource(); + const first = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['realtime:shared.contacts'], + releaseSourceOnRelease: false + }); + const second = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['realtime:shared.contacts'], + releaseSourceOnRelease: false + }); + const firstStream = first.subscribe('realtime:shared.contacts'); + const secondStream = second.subscribe('realtime:shared.contacts'); + await flushMicrotasks(); + + first.publish('realtime:shared.contacts', 'INSERT:first-cursor'); + await expect(firstStream.next()).resolves.toMatchObject({ + value: 'INSERT:first-cursor' + }); + + source.publish('realtime:shared.contacts', 'UPDATE:database'); + await expect(firstStream.next()).resolves.toMatchObject({ value: 'UPDATE:database' }); + await expect(secondStream.next()).resolves.toMatchObject({ value: 'UPDATE:database' }); + await Promise.all([first.release(), second.release()]); + }); + + it('fails an overflowing local subscriber without poisoning its peers', async () => { + const source = new ManualSource(); + const facade = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['realtime:events'] + }); + const slow = facade.subscribe('realtime:events'); + + for (let index = 0; index <= GENERATION_SUBSCRIBER_QUEUE_CAPACITY; index++) { + facade.publish('realtime:events', `INSERT:${index}`); + } + await expect(slow.next()).rejects.toBeInstanceOf(RealtimeGenerationOverflowError); + + const healthy = facade.subscribe('realtime:events'); + facade.publish('realtime:events', 'INSERT:healthy'); + await expect(healthy.next()).resolves.toMatchObject({ value: 'INSERT:healthy' }); + await facade.release(); + }); + + it('propagates source failure and unexpected completion', async () => { + const source = new ManualSource(); + const facade = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['a', 'b'] + }); + const failed = facade.subscribe('a'); + const ended = facade.subscribe('b'); + await flushMicrotasks(); + + source.fail('a', new Error('listener failed')); + source.complete('b'); + + await expect(failed.next()).rejects.toThrow('listener failed'); + await expect(ended.next()).rejects.toBeInstanceOf( + RealtimeGenerationSourceEndedError + ); + await facade.release(); + }); + + it('makes release idempotent and awaits stream and source teardown', async () => { + const source = new ManualSource(); + const streamReleased = deferred>(); + const sourceReleased = deferred(); + const facade = new GenerationScopedRealtimeSubscriber({ + source, + allowedTopics: ['a'] + }); + facade.subscribe('a'); + await flushMicrotasks(); + const sourceStream = [...source.streams.get('a')!][0]; + sourceStream.returnMock.mockImplementation(async () => streamReleased.promise); + source.release.mockImplementation(async () => sourceReleased.promise); + + const first = facade.release(); + const second = facade.release(); + expect(first).toBe(second); + await flushMicrotasks(); + expect(source.release).not.toHaveBeenCalled(); + + streamReleased.resolve({ done: true, value: undefined }); + await flushMicrotasks(); + expect(source.release).toHaveBeenCalledTimes(1); + + let settled = false; + void first.then(() => { + settled = true; + }); + await flushMicrotasks(); + expect(settled).toBe(false); + sourceReleased.resolve(); + await first; + expect(settled).toBe(true); + }); +}); + +describe('ActivatableGenerationScopedRealtimeSubscriber', () => { + it('fails closed before activation and owns an activated source exactly once', async () => { + const source = new ManualSource(); + const facade = new ActivatableGenerationScopedRealtimeSubscriber(); + + expect(() => facade.subscribe('realtime:tenant_a.contacts')) + .toThrow(RealtimeGenerationNotActiveError); + await facade.activate({ + source, + allowedTopics: ['realtime:tenant_a.contacts'] + }); + + const stream = facade.subscribe('realtime:tenant_a.contacts'); + await flushMicrotasks(); + source.publish('realtime:tenant_a.contacts', 'INSERT:row-a'); + await expect(stream.next()).resolves.toMatchObject({ value: 'INSERT:row-a' }); + + const first = facade.release(); + const second = facade.release(); + expect(first).toBe(second); + await first; + expect(source.release).toHaveBeenCalledTimes(1); + }); + + it('releases a rejected second activation source', async () => { + const firstSource = new ManualSource(); + const secondSource = new ManualSource(); + const facade = new ActivatableGenerationScopedRealtimeSubscriber(); + await facade.activate({ source: firstSource, allowedTopics: ['a'] }); + + await expect(facade.activate({ source: secondSource, allowedTopics: ['a'] })) + .rejects.toMatchObject({ code: 'REALTIME_GENERATION_ALREADY_ACTIVE' }); + expect(secondSource.release).toHaveBeenCalledTimes(1); + await facade.release(); + expect(firstSource.release).toHaveBeenCalledTimes(1); + }); +}); diff --git a/graphile/graphile-realtime-subscriptions/__tests__/plugin.test.ts b/graphile/graphile-realtime-subscriptions/__tests__/plugin.test.ts index 7f6669bbf..a4a9d7204 100644 --- a/graphile/graphile-realtime-subscriptions/__tests__/plugin.test.ts +++ b/graphile/graphile-realtime-subscriptions/__tests__/plugin.test.ts @@ -12,7 +12,7 @@ * - NOTIFY payload parsing (TG_OP:id1,id2,... and INVALIDATE) * - Per-subscriber event throttling with configurable limit * - Sparse set subscriptions (ids: [UUID!]) with row ID intersection filtering - * - RLS-aware rowId masking in payload resolvers + * - RLS-aware event suppression and rowId masking */ jest.mock('@pgpmjs/logger', () => ({ @@ -26,17 +26,20 @@ jest.mock('@pgpmjs/logger', () => ({ const mockListen = jest.fn(); const mockConstant = jest.fn((val: any) => `constant(${val})`); -const mockObject = jest.fn((obj: any) => obj); const mockLambda = jest.fn((input: any, fn: Function) => fn(input)); +const mockGet = jest.fn((parent: any, key: string) => + typeof parent?.get === 'function' ? parent.get(key) : parent?.[key] +); +let mockPgSubscriber: any = 'mock-pgSubscriber'; const mockContext = jest.fn(() => ({ - get: jest.fn((key: string) => `mock-${key}`), + get: jest.fn((key: string) => key === 'pgSubscriber' ? mockPgSubscriber : `mock-${key}`), })); jest.mock('grafast', () => ({ context: mockContext, listen: mockListen, - object: mockObject, constant: mockConstant, + get: mockGet, lambda: mockLambda, })); @@ -62,6 +65,7 @@ import { MalformedNotifyPayloadError, parseNotifyPayload, RealtimeSubscriptionsPlugin, + selectCandidateRowId, } from '../src/plugin'; // --- Test helpers --- @@ -85,8 +89,48 @@ function createMockCodec( }; } -function createMockResource(name: string, codec: any) { - return { codec, name }; +function createMockExecutorContext( + visibleIds: readonly string[] = [], + pgSettings: Record = { role: 'tenant_runtime' }, +) { + const visible = new Set(visibleIds); + const query = jest.fn(async ({ values }: { text: string; values?: unknown[] }) => { + const requestedIds = (values?.[0] ?? []) as string[]; + const rows = requestedIds + .filter((rowId) => visible.has(rowId)) + .map((id) => ({ id })); + return { rows }; + }); + const withPgClient = jest.fn(async (_settings: unknown, callback: Function) => + callback({ query }) + ); + + return { + executorContext: { pgSettings, withPgClient }, + query, + withPgClient, + }; +} + +function createMockResource(name: string, codec: any, executorContext?: any) { + const context = executorContext ?? createMockExecutorContext().executorContext; + return { + codec, + name, + executor: { + context: jest.fn(() => context), + }, + }; +} + +async function* notifications(payloads: readonly string[]) { + for (const payload of payloads) yield payload; +} + +async function collectNotifications(iterable: AsyncIterable) { + const result: unknown[] = []; + for await (const payload of iterable) result.push(payload); + return result; } function createMockBuild(resources: Record, inflectionOverrides: Record = {}) { @@ -219,6 +263,7 @@ describe('createRealtimeSubscriptionsPlugin', () => { beforeEach(() => { jest.clearAllMocks(); capturedFactory = null; + mockPgSubscriber = 'mock-pgSubscriber'; }); describe('plugin structure', () => { @@ -235,6 +280,38 @@ describe('createRealtimeSubscriptionsPlugin', () => { }); describe('table discovery', () => { + it('reports sorted credential-free physical topic descriptors during build', () => { + const onTopicsDiscovered = jest.fn(); + createRealtimeSubscriptionsPlugin({ onTopicsDiscovered }); + + const zeta = createMockCodec('zeta', { + realtime: true, + schemaName: 'tenant_a' + }); + const alpha = createMockCodec('alpha', { + realtime: true, + schemaName: 'tenant_a' + }); + capturedFactory!(createMockBuild({ + zeta: createMockResource('zeta', zeta), + alpha: createMockResource('alpha', alpha) + })); + + expect(onTopicsDiscovered).toHaveBeenCalledTimes(1); + expect(onTopicsDiscovered).toHaveBeenCalledWith([ + { topic: 'realtime:tenant_a.alpha', schema: 'tenant_a', table: 'alpha' }, + { topic: 'realtime:tenant_a.zeta', schema: 'tenant_a', table: 'zeta' } + ]); + }); + + it('reports an explicit empty topic set', () => { + const onTopicsDiscovered = jest.fn(); + createRealtimeSubscriptionsPlugin({ onTopicsDiscovered }); + capturedFactory!(createMockBuild({})); + + expect(onTopicsDiscovered).toHaveBeenCalledWith([]); + }); + it('discovers tables with @realtime tag', () => { createRealtimeSubscriptionsPlugin(); @@ -336,7 +413,7 @@ describe('createRealtimeSubscriptionsPlugin', () => { expect(result.typeDefs).toContain('documents: Documents'); expect(result.typeDefs).toContain('rowId: UUID'); expect(result.typeDefs).toContain('overflow: Boolean!'); - expect(result.typeDefs).toContain('masked when RLS denies access'); + expect(result.typeDefs).toContain('after RLS authorization'); }); it('extends Subscription type', () => { @@ -371,7 +448,7 @@ describe('createRealtimeSubscriptionsPlugin', () => { expect(result.plans['Subscription']).toBeDefined(); expect(result.plans['Subscription']['onProjectsChanged']).toBeDefined(); - const mockArgs = { getRaw: jest.fn(() => 'test-id') }; + const mockArgs = { getRaw: jest.fn(() => ['test-id']) }; result.plans['Subscription']['onProjectsChanged'].subscribePlan(null, mockArgs); expect(mockConstant).toHaveBeenCalledWith('realtime:app_public.projects'); @@ -390,7 +467,7 @@ describe('createRealtimeSubscriptionsPlugin', () => { const result = capturedFactory!(build); - const mockArgs = { getRaw: jest.fn(() => 'test-id') }; + const mockArgs = { getRaw: jest.fn(() => ['test-id']) }; result.plans['Subscription']['onItemsChanged'].subscribePlan(null, mockArgs); expect(mockConstant).toHaveBeenCalledWith('realtime:inventory_public.items'); @@ -422,7 +499,7 @@ describe('createRealtimeSubscriptionsPlugin', () => { }); const result = capturedFactory!(build); - const mockArgs = { getRaw: jest.fn(() => 'some-id') }; + const mockArgs = { getRaw: jest.fn(() => ['some-id']) }; result.plans['Subscription']['onTasksChanged'].subscribePlan(null, mockArgs); @@ -479,7 +556,7 @@ describe('createRealtimeSubscriptionsPlugin', () => { expect(mockParent.get).toHaveBeenCalledWith('parsed'); }); - it('payload row resolver uses parsed rowId for full collection mode', () => { + it('payload row resolver uses the authorized row for full collection mode', () => { createRealtimeSubscriptionsPlugin(); const codec = createMockCodec('tasks', { realtime: true }); @@ -492,14 +569,18 @@ describe('createRealtimeSubscriptionsPlugin', () => { }); const result = capturedFactory!(build); - const mockParent = { get: jest.fn(() => ({ event: 'INSERT', rowIds: ['row-uuid'], overflow: false })) }; + const mockParent = { get: jest.fn((key: string) => { + if (key === 'parsed') { + return { event: 'INSERT', rowIds: ['row-uuid'], overflow: false }; + } + if (key === 'subscribedIds') return null; + return null; + }) }; result.plans['TasksSubscriptionPayload'].tasks(mockParent); - // The gate already narrowed rowIds to the subscription's ids, so the - // resolver reads nothing but 'parsed'. expect(mockParent.get).toHaveBeenCalledWith('parsed'); - expect(mockParent.get).not.toHaveBeenCalledWith('subscribedIds'); - expect(mockResource.get).toHaveBeenCalled(); + expect(mockParent.get).toHaveBeenCalledWith('subscribedIds'); + expect(mockResource.get).toHaveBeenCalledWith({ id: 'row-uuid' }); }); }); @@ -627,12 +708,19 @@ describe('createRealtimeSubscriptionsPlugin', () => { await expect(collect(gated)).rejects.toThrow(MalformedNotifyPayloadError); }); - it('subscribePlan reads the ids argument and gates the subscriber', () => { + it('threads ids into the pre-delivery authorization filter', async () => { createRealtimeSubscriptionsPlugin(); const codec = createMockCodec('tasks', { realtime: true }); + const { executorContext, query } = createMockExecutorContext(['id-a']); + mockPgSubscriber = { + subscribe: jest.fn(() => notifications([ + 'INSERT:id-other', + 'UPDATE:id-a', + ])), + }; const build = createMockBuild({ - tasks: createMockResource('tasks', codec), + tasks: createMockResource('tasks', codec, executorContext), }); const result = capturedFactory!(build); @@ -641,17 +729,336 @@ describe('createRealtimeSubscriptionsPlugin', () => { result.plans['Subscription']['onTasksChanged'].subscribePlan(null, mockArgs); expect(mockArgs.getRaw).toHaveBeenCalledWith('ids'); + expect(mockListen).toHaveBeenCalled(); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + const events = await collectNotifications( + authorizedSubscriber.subscribe('realtime:app_public.tasks'), + ); + + expect(events).toEqual([{ + parsed: { event: 'UPDATE', rowIds: ['id-a'], overflow: false }, + subscribedIds: ['id-a'], + }]); + expect(query).toHaveBeenCalledTimes(1); + expect(query.mock.calls[0][0].values).toEqual([['id-a']]); + }); + + it('rowId resolver returns a sparse-set ID only when RLS exposes the row', () => { + createRealtimeSubscriptionsPlugin(); + + const codec = createMockCodec('tasks', { realtime: true }); + const getAuthorizedId = jest.fn(() => 'id-b'); + const get = jest.fn(() => ({ get: getAuthorizedId })); + const build = createMockBuild({ + tasks: { ...createMockResource('tasks', codec), get }, + }); - // 'parsed' is the whole payload now — nothing downstream needs the ids. - const listenCallback = mockListen.mock.calls[mockListen.mock.calls.length - 1][2]; - listenCallback({ event: 'INSERT', rowIds: ['id-a'], overflow: false }); - const objectArg = mockObject.mock.calls[mockObject.mock.calls.length - 1][0]; - expect(Object.keys(objectArg)).toEqual(['parsed']); + const result = capturedFactory!(build); + const payload = result.plans['TasksSubscriptionPayload']; + + const mockParent = { get: jest.fn((key: string) => { + if (key === 'parsed') return { event: 'UPDATE', rowIds: ['id-x', 'id-b', 'id-a'], overflow: false }; + if (key === 'subscribedIds') return ['id-a', 'id-b']; + return null; + }) }; + + expect(payload.rowId(mockParent)).toBe('id-b'); + expect(mockParent.get).toHaveBeenCalledWith('parsed'); + expect(mockParent.get).toHaveBeenCalledWith('subscribedIds'); + expect(get).toHaveBeenCalledWith({ id: 'id-b' }); + expect(getAuthorizedId).toHaveBeenCalledWith('id'); + }); + + it('rowId resolver returns null when RLS hides a sparse-set row', () => { + createRealtimeSubscriptionsPlugin(); + + const codec = createMockCodec('tasks', { realtime: true }); + const getAuthorizedId = jest.fn((): null => null); + const get = jest.fn(() => ({ get: getAuthorizedId })); + const build = createMockBuild({ + tasks: { ...createMockResource('tasks', codec), get }, + }); + + const result = capturedFactory!(build); + const payload = result.plans['TasksSubscriptionPayload']; + + const mockParent = { get: jest.fn((key: string) => { + if (key === 'parsed') return { event: 'INSERT', rowIds: ['id-a'], overflow: false }; + if (key === 'subscribedIds') return ['id-a', 'id-b']; + return null; + }) }; + + expect(payload.rowId(mockParent)).toBeNull(); + expect(mockParent.get).toHaveBeenCalledWith('subscribedIds'); + expect(get).toHaveBeenCalledWith({ id: 'id-a' }); + expect(getAuthorizedId).toHaveBeenCalledWith('id'); + }); + + it('rowId resolver never exposes IDs in collection mode', () => { + createRealtimeSubscriptionsPlugin(); + + const codec = createMockCodec('tasks', { realtime: true }); + const getAuthorizedId = jest.fn((): null => null); + const get = jest.fn(() => ({ get: getAuthorizedId })); + const build = createMockBuild({ + tasks: { ...createMockResource('tasks', codec), get }, + }); + + const result = capturedFactory!(build); + const payload = result.plans['TasksSubscriptionPayload']; + + const mockParent = { get: jest.fn((key: string) => { + if (key === 'parsed') return { event: 'INSERT', rowIds: ['id-first', 'id-second'], overflow: false }; + if (key === 'subscribedIds') return null; + return null; + }) }; + + expect(payload.rowId(mockParent)).toBeNull(); + expect(mockParent.get).toHaveBeenCalledWith('subscribedIds'); + expect(get).toHaveBeenCalledWith({ id: null }); + expect(getAuthorizedId).toHaveBeenCalledWith('id'); + }); + + it.each(['DELETE', 'INVALIDATE'])('rowId resolver never exposes IDs for %s', (event) => { + createRealtimeSubscriptionsPlugin(); + + const codec = createMockCodec('tasks', { realtime: true }); + const getAuthorizedId = jest.fn((): null => null); + const get = jest.fn(() => ({ get: getAuthorizedId })); + const build = createMockBuild({ + tasks: { ...createMockResource('tasks', codec), get }, + }); + + const result = capturedFactory!(build); + const payload = result.plans['TasksSubscriptionPayload']; + const mockParent = { get: jest.fn((key: string) => { + if (key === 'parsed') { + return { + event, + rowIds: event === 'INVALIDATE' ? [] : ['id-a'], + overflow: event === 'INVALIDATE', + }; + } + if (key === 'subscribedIds') return ['id-a']; + return null; + }) }; + + expect(payload.rowId(mockParent)).toBeNull(); + expect(get).toHaveBeenCalledWith({ id: null }); + expect(getAuthorizedId).toHaveBeenCalledWith('id'); }); }); describe('RLS-aware event delivery', () => { - it('rowId doc comment mentions RLS masking', () => { + it('suppresses an unauthorized collection event before Grafast can emit its timing or type', async () => { + createRealtimeSubscriptionsPlugin(); + + const codec = createMockCodec('items', { realtime: true }); + const { executorContext, query, withPgClient } = createMockExecutorContext( + ['visible-id'], + { role: 'tenant_a', 'jwt.claims.tenant_id': 'tenant-a' }, + ); + mockPgSubscriber = { + subscribe: jest.fn(() => notifications([ + 'INSERT:hidden-id', + 'UPDATE:hidden-id,visible-id', + ])), + }; + const build = createMockBuild({ + items: createMockResource('items', codec, executorContext), + }); + + const result = capturedFactory!(build); + result.plans['Subscription']['onItemsChanged'].subscribePlan( + null, + { getRaw: jest.fn((): null => null) }, + ); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + const events = await collectNotifications( + authorizedSubscriber.subscribe('realtime:app_public.items'), + ); + + expect(events).toEqual([{ + parsed: { event: 'UPDATE', rowIds: ['visible-id'], overflow: false }, + subscribedIds: null, + }]); + expect(query.mock.calls.map(([request]) => request.values)).toEqual([ + [['hidden-id']], + [['hidden-id', 'visible-id']], + ]); + expect(withPgClient).toHaveBeenCalledWith( + { role: 'tenant_a', 'jwt.claims.tenant_id': 'tenant-a' }, + expect.any(Function), + ); + }); + + it('suppresses DELETE and database INVALIDATE before querying', async () => { + createRealtimeSubscriptionsPlugin(); + + const codec = createMockCodec('items', { realtime: true }); + const { executorContext, query } = createMockExecutorContext(['visible-id']); + mockPgSubscriber = { + subscribe: jest.fn(() => notifications([ + 'DELETE:visible-id', + 'INVALIDATE', + 'INSERT:visible-id', + ])), + }; + const build = createMockBuild({ + items: createMockResource('items', codec, executorContext), + }); + + const result = capturedFactory!(build); + result.plans['Subscription']['onItemsChanged'].subscribePlan( + null, + { getRaw: jest.fn((): null => null) }, + ); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + const events = await collectNotifications( + authorizedSubscriber.subscribe('realtime:app_public.items'), + ); + + expect(events).toEqual([{ + parsed: { event: 'INSERT', rowIds: ['visible-id'], overflow: false }, + subscribedIds: null, + }]); + expect(query).toHaveBeenCalledTimes(1); + }); + + it.each(['UPDATE', 'TRUNCATE:visible-id'])( + 'fails closed when the NOTIFY payload is malformed: %s', + async (payload) => { + createRealtimeSubscriptionsPlugin(); + + const codec = createMockCodec('items', { realtime: true }); + const { executorContext, query } = createMockExecutorContext(['visible-id']); + mockPgSubscriber = { + subscribe: jest.fn(() => notifications([payload, 'INSERT:visible-id'])), + }; + const build = createMockBuild({ + items: createMockResource('items', codec, executorContext), + }); + + const result = capturedFactory!(build); + result.plans['Subscription']['onItemsChanged'].subscribePlan( + null, + { getRaw: jest.fn((): null => null) }, + ); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + + await expect(collectNotifications( + authorizedSubscriber.subscribe('realtime:app_public.items'), + )).rejects.toThrow(MalformedNotifyPayloadError); + expect(query).not.toHaveBeenCalled(); + }, + ); + + it('fails closed when the RLS visibility query errors', async () => { + createRealtimeSubscriptionsPlugin(); + + const codec = createMockCodec('items', { realtime: true }); + const withPgClient = jest.fn(async () => { + throw new Error('database unavailable'); + }); + const executorContext = { + pgSettings: { role: 'tenant_runtime' }, + withPgClient, + }; + mockPgSubscriber = { + subscribe: jest.fn(() => notifications(['INSERT:possibly-visible-id'])), + }; + const build = createMockBuild({ + items: createMockResource('items', codec, executorContext), + }); + + const result = capturedFactory!(build); + result.plans['Subscription']['onItemsChanged'].subscribePlan( + null, + { getRaw: jest.fn((): null => null) }, + ); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + + await expect(collectNotifications( + authorizedSubscriber.subscribe('realtime:app_public.items'), + )).resolves.toEqual([]); + expect(withPgClient).toHaveBeenCalledTimes(1); + }); + + it('quotes physical identifiers and binds hostile row IDs as values', async () => { + createRealtimeSubscriptionsPlugin(); + + const hostileId = "00000000-0000-0000-0000-000000000000' OR true --"; + const codec = createMockCodec('tasks', { + realtime: true, + schemaName: 'tenant"; set role postgres; --', + }); + codec.extensions.pg.name = 'tasks"; drop table audit; --'; + const { executorContext, query } = createMockExecutorContext([hostileId]); + mockPgSubscriber = { + subscribe: jest.fn(() => notifications([`INSERT:${hostileId}`])), + }; + const build = createMockBuild({ + tasks: createMockResource('tasks', codec, executorContext), + }); + + const result = capturedFactory!(build); + result.plans['Subscription']['onTasksChanged'].subscribePlan( + null, + { getRaw: jest.fn((): null => null) }, + ); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + await collectNotifications( + authorizedSubscriber.subscribe('realtime:hostile'), + ); + + const request = query.mock.calls[0][0]; + expect(request.text).toContain( + '"tenant""; set role postgres; --"."tasks""; drop table audit; --"', + ); + expect(request.text).toContain('any($1::text[])'); + expect(request.text).not.toContain(hostileId); + expect(request.values).toEqual([[hostileId]]); + }); + + it('counts only authorized events toward the subscriber throttle', async () => { + createRealtimeSubscriptionsPlugin({ overflowThreshold: 1 }); + + const codec = createMockCodec('items', { realtime: true }); + const { executorContext } = createMockExecutorContext(['visible-a', 'visible-b']); + mockPgSubscriber = { + subscribe: jest.fn(() => notifications([ + 'INSERT:hidden-id', + 'INSERT:visible-a', + 'UPDATE:visible-b', + ])), + }; + const build = createMockBuild({ + items: createMockResource('items', codec, executorContext), + }); + + const result = capturedFactory!(build); + result.plans['Subscription']['onItemsChanged'].subscribePlan( + null, + { getRaw: jest.fn((): null => null) }, + ); + const authorizedSubscriber = mockListen.mock.calls[mockListen.mock.calls.length - 1][0]; + const events = await collectNotifications( + authorizedSubscriber.subscribe('realtime:app_public.items'), + ); + + expect(events).toEqual([ + { + parsed: { event: 'INSERT', rowIds: ['visible-a'], overflow: false }, + subscribedIds: null, + }, + { + parsed: { event: 'INVALIDATE', rowIds: [], overflow: true }, + subscribedIds: null, + }, + ]); + }); + + it('rowId doc comment states the fail-closed visibility rules', () => { createRealtimeSubscriptionsPlugin(); const codec = createMockCodec('items', { realtime: true }); @@ -660,7 +1067,8 @@ describe('createRealtimeSubscriptionsPlugin', () => { }); const result = capturedFactory!(build); - expect(result.typeDefs).toContain('masked when RLS denies access'); + expect(result.typeDefs).toContain('after RLS authorization'); + expect(result.typeDefs).toContain('Null for collection, INVALIDATE, or denied rows'); }); it('type defs include sparse set ids argument', () => { @@ -689,3 +1097,25 @@ describe('createRealtimeSubscriptionsPlugin', () => { }); }); }); + +describe('selectCandidateRowId', () => { + const insert = { event: 'INSERT', rowIds: ['id-a', 'id-b'], overflow: false }; + + it('allows collection row fetching without allowing collection rowId exposure', () => { + expect(selectCandidateRowId(insert, null, true)).toBe('id-a'); + expect(selectCandidateRowId(insert, null, false)).toBeNull(); + }); + + it('selects only a caller-supplied sparse ID', () => { + expect(selectCandidateRowId(insert, ['id-b'], false)).toBe('id-b'); + expect(selectCandidateRowId(insert, ['id-x'], false)).toBeNull(); + }); + + it.each(['DELETE', 'INVALIDATE', 'UNKNOWN'])('rejects %s before any row lookup', (event) => { + expect(selectCandidateRowId({ + event, + rowIds: ['id-a'], + overflow: event === 'INVALIDATE', + }, ['id-a'], false)).toBeNull(); + }); +}); diff --git a/graphile/graphile-realtime-subscriptions/__tests__/realtime-manager.test.ts b/graphile/graphile-realtime-subscriptions/__tests__/realtime-manager.test.ts index c0d10650e..3d97d1209 100644 --- a/graphile/graphile-realtime-subscriptions/__tests__/realtime-manager.test.ts +++ b/graphile/graphile-realtime-subscriptions/__tests__/realtime-manager.test.ts @@ -1,7 +1,14 @@ import { EventEmitter } from 'events'; -import { RealtimeManager } from '../src/realtime-manager'; -import { entryToChannel,entryToNotifyPayload, extractRowId } from '../src/realtime-manager'; +import { + entryToChannel, + entryToNotifyPayload, + extractRowId, + RealtimeManager, + RealtimeSourceSchemaConfigurationError, + RealtimeSourceSchemaViolationError, + RealtimeSubscriberUnavailableError +} from '../src/realtime-manager'; import type { ChangeLogEntry, Queryable } from '../src/types'; // --------------------------------------------------------------------------- @@ -34,6 +41,20 @@ function createMockPgSubscriber() { return { eventEmitter, subscribe: jest.fn() }; } +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +async function flushMicrotasks(): Promise { + for (let i = 0; i < 6; i++) await Promise.resolve(); +} + // --------------------------------------------------------------------------- // Unit tests: helper functions // --------------------------------------------------------------------------- @@ -129,6 +150,7 @@ describe('RealtimeManager', () => { return new RealtimeManager({ pgSubscriber: mockSubscriber, pool: mockPool, + allowedSourceSchemas: ['public', 'billing'], nodeId: 'test-manager-node', pollIntervalMs: 1000, heartbeatIntervalMs: 5000, @@ -175,6 +197,108 @@ describe('RealtimeManager', () => { ); }); + it('fails startup before registration when the subscriber emitter is unavailable', async () => { + const manager = createManager({ pgSubscriber: {} }); + + await expect(manager.start()).rejects.toBeInstanceOf( + RealtimeSubscriberUnavailableError + ); + + expect(manager.isRunning).toBe(false); + expect(mockPool.query).not.toHaveBeenCalled(); + }); + + it('uses an explicit publisher without inspecting PgSubscriber internals', async () => { + const publish = jest.fn(); + const opaqueSubscriber = Object.defineProperty({}, 'eventEmitter', { + get() { + throw new Error('private field accessed'); + } + }); + mockPool.query.mockImplementation(async (sql: string) => { + if (sql.includes('drain_changes')) { + return { + rows: [{ + drain_changes: makeEntry({ payload_after: { id: 'cursor-row' } }) + }] + }; + } + return { rows: [] }; + }); + const manager = createManager({ + publisher: { publish }, + pgSubscriber: opaqueSubscriber + }); + + await manager.start(); + expect(publish).toHaveBeenCalledWith( + 'realtime:public.contact', + 'INSERT:cursor-row' + ); + await manager.stop(); + }); + + it('fails the generation when the explicit publisher rejects delivery', async () => { + const failure = new Error('generation released'); + const fatalErrors: Error[] = []; + mockPool.query.mockImplementation(async (sql: string) => { + if (sql.includes('drain_changes')) { + return { rows: [{ drain_changes: makeEntry() }] }; + } + return { rows: [] }; + }); + const manager = createManager({ + publisher: { + publish() { + throw failure; + } + }, + onFatalError: (error: Error) => fatalErrors.push(error) + }); + + await expect(manager.start()).rejects.toBe(failure); + expect(fatalErrors).toEqual([failure]); + expect(manager.isRunning).toBe(false); + }); + + it('preflights every cursor topic before publishing any row in the batch', async () => { + const publish = jest.fn(); + const topicFailure = new Error('topic outside generation'); + mockPool.query.mockImplementation(async (sql: string) => { + if (sql.includes('drain_changes')) { + return { + rows: [ + { drain_changes: makeEntry({ source_table: 'contact' }) }, + { drain_changes: makeEntry({ source_table: 'private_table' }) } + ] + }; + } + return { rows: [] }; + }); + const manager = createManager({ + publisher: { + assertTopics(topics: readonly string[]) { + if (topics.includes('realtime:public.private_table')) throw topicFailure; + }, + publish + } + }); + + await expect(manager.start()).rejects.toBe(topicFailure); + expect(publish).not.toHaveBeenCalled(); + }); + + it('fails startup before registration when no source schema is allowed', async () => { + const manager = createManager({ allowedSourceSchemas: [] }); + + await expect(manager.start()).rejects.toBeInstanceOf( + RealtimeSourceSchemaConfigurationError + ); + + expect(manager.isRunning).toBe(false); + expect(mockPool.query).not.toHaveBeenCalled(); + }); + it('is idempotent for start', async () => { const manager = createManager(); await manager.start(); @@ -190,7 +314,184 @@ describe('RealtimeManager', () => { await manager.stop(); // should be no-op }); + it('fails a running generation when periodic cursor polling fails', async () => { + const failure = new Error('periodic drain failed'); + const errors: Error[] = []; + const fatalErrors: Error[] = []; + let rejectDrain = false; + mockPool.query.mockImplementation(async (sql: string) => { + if (rejectDrain && sql.includes('drain_changes')) throw failure; + return { rows: [] }; + }); + const manager = createManager({ + onError: (error: Error) => errors.push(error), + onFatalError: (error: Error) => fatalErrors.push(error) + }); + + await manager.start(); + rejectDrain = true; + await jest.advanceTimersByTimeAsync(1000); + await flushMicrotasks(); + await manager.stop(); + + expect(errors).toEqual([failure]); + expect(fatalErrors).toEqual([failure]); + expect(manager.isRunning).toBe(false); + expect(mockPool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['test-manager-node'] + ); + }); + + it('fails a running generation when its periodic heartbeat fails', async () => { + const failure = new Error('periodic heartbeat failed'); + const errors: Error[] = []; + const fatalErrors: Error[] = []; + let rejectHeartbeat = false; + mockPool.query.mockImplementation(async (sql: string) => { + if (rejectHeartbeat && sql.includes('touch_listener')) throw failure; + return { rows: [] }; + }); + const manager = createManager({ + onError: (error: Error) => errors.push(error), + onFatalError: (error: Error) => fatalErrors.push(error) + }); + + await manager.start(); + rejectHeartbeat = true; + await jest.advanceTimersByTimeAsync(5000); + await flushMicrotasks(); + await manager.stop(); + + expect(errors).toEqual([failure]); + expect(fatalErrors).toEqual([failure]); + expect(manager.isRunning).toBe(false); + expect(mockPool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['test-manager-node'] + ); + }); + + it('does not dispatch a deferred startup drain after stop begins', async () => { + const entry = makeEntry({ payload_after: { id: 'late-row' } }); + const drain = deferred<{ rows: { drain_changes: ChangeLogEntry }[] }>(); + const emitted: string[] = []; + mockSubscriber.eventEmitter.on('realtime:public.contact', (payload: string) => { + emitted.push(payload); + }); + mockPool.query.mockImplementation((sql: string) => { + if (sql.includes('drain_changes')) return drain.promise; + return Promise.resolve({ rows: [] }); + }); + + const manager = createManager(); + const starting = manager.start(); + const startResult = expect(starting).rejects.toMatchObject({ + code: 'CURSOR_TRACKER_START_ABORTED', + }); + await flushMicrotasks(); + expect(mockPool.query.mock.calls.some(([sql]) => sql.includes('drain_changes'))).toBe(true); + + const stopping = manager.stop(); + drain.resolve({ rows: [{ drain_changes: entry }] }); + + await startResult; + await stopping; + + expect(emitted).toEqual([]); + expect(manager.isRunning).toBe(false); + expect(mockPool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['test-manager-node'] + ); + }); + describe('event dispatching', () => { + it('rejects a mixed batch atomically when it contains a foreign source schema', async () => { + const emitted: string[] = []; + const errors: Error[] = []; + const fatalErrors: Error[] = []; + mockSubscriber.eventEmitter.on('realtime:public.contact', (payload: string) => { + emitted.push(payload); + }); + const entries = [ + makeEntry({ payload_after: { id: 'allowed-row' } }), + makeEntry({ + source_schema: 'tenant_b', + payload_after: { id: 'foreign-row' } + }) + ]; + mockPool.query.mockImplementation(async (sql: string) => { + if (sql.includes('drain_changes')) { + return { rows: entries.map((entry) => ({ drain_changes: entry })) }; + } + return { rows: [] }; + }); + + const manager = createManager({ + allowedSourceSchemas: ['public'], + onError: (error: Error) => errors.push(error), + onFatalError: (error: Error) => fatalErrors.push(error) + }); + + await expect(manager.start()).rejects.toBeInstanceOf( + RealtimeSourceSchemaViolationError + ); + await manager.stop(); + + expect(emitted).toEqual([]); + expect(errors).toHaveLength(1); + expect(errors[0]).toMatchObject({ + code: 'REALTIME_SOURCE_SCHEMA_VIOLATION', + sourceSchema: 'tenant_b', + allowedSourceSchemas: ['public'] + }); + expect(fatalErrors).toEqual([errors[0]]); + expect(manager.isRunning).toBe(false); + }); + + it('stops a running manager before a foreign periodic batch can emit', async () => { + const errors: Error[] = []; + const fatalErrors: Error[] = []; + const emitted: string[] = []; + mockSubscriber.eventEmitter.on('realtime:public.contact', (payload: string) => { + emitted.push(payload); + }); + const manager = createManager({ + allowedSourceSchemas: ['public'], + onError: (error: Error) => errors.push(error), + onFatalError: (error: Error) => fatalErrors.push(error) + }); + await manager.start(); + mockPool.query.mockImplementation(async (sql: string) => { + if (sql.includes('drain_changes')) { + return { + rows: [{ + drain_changes: makeEntry({ + source_schema: 'tenant_b', + payload_after: { id: 'foreign-periodic-row' } + }) + }] + }; + } + return { rows: [] }; + }); + + await jest.advanceTimersByTimeAsync(1000); + await flushMicrotasks(); + await manager.stop(); + + expect(emitted).toEqual([]); + expect(errors).toHaveLength(1); + expect(errors[0]).toBeInstanceOf(RealtimeSourceSchemaViolationError); + expect(fatalErrors).toEqual([errors[0]]); + expect(manager.isRunning).toBe(false); + expect(mockPool.query).toHaveBeenCalledWith( + expect.stringContaining('cleanup_ephemeral'), + ['test-manager-node'] + ); + }); + it('emits cursor-tracked events on PgSubscriber eventEmitter', async () => { const emitted: { channel: string; payload: string }[] = []; mockSubscriber.eventEmitter.on('realtime:public.contact', (payload: string) => { @@ -290,7 +591,7 @@ describe('RealtimeManager', () => { }); describe('error handling', () => { - it('calls onError when drain fails', async () => { + it('fails startup and rolls back readiness when the initial drain fails', async () => { const errors: Error[] = []; mockPool.query.mockImplementation(async (sql: string) => { @@ -301,30 +602,12 @@ describe('RealtimeManager', () => { }); const manager = createManager({ onError: (err: Error) => errors.push(err) }); - await manager.start(); + await expect(manager.start()).rejects.toThrow('drain failed'); expect(errors).toHaveLength(1); expect(errors[0].message).toBe('drain failed'); - - await manager.stop(); + expect(manager.isRunning).toBe(false); }); - it('handles missing eventEmitter gracefully', async () => { - const entries: ChangeLogEntry[] = [ - makeEntry({ operation: 'INSERT', payload_after: { id: 'row-x' } }), - ]; - - mockPool.query.mockImplementation(async (sql: string) => { - if (typeof sql === 'string' && sql.includes('drain_changes')) { - return { rows: entries.map((e) => ({ drain_changes: e })) }; - } - return { rows: [] }; - }); - - // pgSubscriber without eventEmitter — should not crash - const manager = createManager({ pgSubscriber: {} }); - await manager.start(); - await manager.stop(); - }); }); }); diff --git a/graphile/graphile-realtime-subscriptions/__tests__/topic-collector.test.ts b/graphile/graphile-realtime-subscriptions/__tests__/topic-collector.test.ts new file mode 100644 index 000000000..f38af6551 --- /dev/null +++ b/graphile/graphile-realtime-subscriptions/__tests__/topic-collector.test.ts @@ -0,0 +1,66 @@ +import { + RealtimeTopicCollector, + RealtimeTopicDiscoveryError +} from '../src/topic-collector'; + +describe('RealtimeTopicCollector', () => { + it('returns sorted exact physical topics for allowed schemas', () => { + const collector = new RealtimeTopicCollector(); + collector.collect([ + { topic: 'realtime:tenant_a.z', schema: 'tenant_a', table: 'z' }, + { topic: 'realtime:tenant_a.a', schema: 'tenant_a', table: 'a' } + ]); + + expect(collector.exactTopics(['tenant_a'])).toEqual([ + 'realtime:tenant_a.a', + 'realtime:tenant_a.z' + ]); + }); + + it.each([ + { + descriptors: [], + schemas: ['tenant_a'], + code: 'REALTIME_TOPIC_DISCOVERY_EMPTY' + }, + { + descriptors: [ + { topic: 'realtime:tenant_b.items', schema: 'tenant_b', table: 'items' } + ], + schemas: ['tenant_a'], + code: 'REALTIME_TOPIC_DISCOVERY_FOREIGN' + }, + { + descriptors: [ + { topic: 'realtime:tenant.a.items', schema: 'tenant.a', table: 'items' } + ], + schemas: ['tenant.a'], + code: 'REALTIME_TOPIC_DISCOVERY_INVALID' + } + ])('fails closed for $code', ({ descriptors, schemas, code }) => { + const collector = new RealtimeTopicCollector(); + expect(() => { + collector.collect(descriptors); + collector.exactTopics(schemas); + }).toThrow(expect.objectContaining({ + code + }) as RealtimeTopicDiscoveryError); + }); + + it('rejects missing discovery and post-discovery topic drift', () => { + const missing = new RealtimeTopicCollector(); + expect(() => missing.exactTopics(['tenant_a'])).toThrow(expect.objectContaining({ + code: 'REALTIME_TOPIC_DISCOVERY_MISSING' + }) as RealtimeTopicDiscoveryError); + + const changed = new RealtimeTopicCollector(); + changed.collect([ + { topic: 'realtime:tenant_a.items', schema: 'tenant_a', table: 'items' } + ]); + expect(() => changed.collect([ + { topic: 'realtime:tenant_a.users', schema: 'tenant_a', table: 'users' } + ])).toThrow(expect.objectContaining({ + code: 'REALTIME_TOPIC_DISCOVERY_CHANGED' + }) as RealtimeTopicDiscoveryError); + }); +}); diff --git a/graphile/graphile-realtime-subscriptions/src/cursor-tracker.ts b/graphile/graphile-realtime-subscriptions/src/cursor-tracker.ts index ab1f1204b..b18849c1b 100644 --- a/graphile/graphile-realtime-subscriptions/src/cursor-tracker.ts +++ b/graphile/graphile-realtime-subscriptions/src/cursor-tracker.ts @@ -30,6 +30,17 @@ const DEFAULT_HEARTBEAT_INTERVAL_MS = 30000; const DEFAULT_BATCH_LIMIT = 500; const DEFAULT_SCHEMA = 'realtime_public'; +type CursorTrackerState = 'stopped' | 'starting' | 'running' | 'stopping'; + +export class CursorTrackerStartAbortedError extends Error { + readonly code = 'CURSOR_TRACKER_START_ABORTED'; + + constructor() { + super('CursorTracker was stopped before startup completed'); + this.name = 'CursorTrackerStartAbortedError'; + } +} + export class CursorTracker { readonly nodeId: string; @@ -43,8 +54,13 @@ export class CursorTracker { private pollTimer: ReturnType | null = null; private heartbeatTimer: ReturnType | null = null; - private running = false; - private draining = false; + private state: CursorTrackerState = 'stopped'; + private generation = 0; + private registered = false; + private startPromise: Promise | null = null; + private stopPromise: Promise | null = null; + private activeDrain: Promise | null = null; + private activeHeartbeat: Promise | null = null; constructor(options: CursorTrackerOptions) { this.nodeId = options.nodeId ?? randomUUID(); @@ -59,32 +75,107 @@ export class CursorTracker { }); } - async start(): Promise { - if (this.running) return; - this.running = true; + start(): Promise { + if (this.state === 'running') return Promise.resolve(); + if (this.state === 'starting') return this.startPromise!; + if (this.state === 'stopping') { + return (this.stopPromise ?? Promise.resolve()).then(() => this.start()); + } + const generation = ++this.generation; + this.state = 'starting'; + const pending = this.startInternal(generation); + this.startPromise = pending; + void pending.then( + () => { + if (this.startPromise === pending) this.startPromise = null; + }, + () => { + if (this.startPromise === pending) this.startPromise = null; + } + ); + return pending; + } + + private async startInternal(generation: number): Promise { log.info(`Starting cursor tracker: node=${this.nodeId}, schema=${this.schema}`); + try { + // A manual operation may have started while the tracker was stopped. + // Readiness must execute its own strict registration and drain rather + // than coalescing onto a non-strict operation. + await this.waitForActiveWork(); + this.assertStartCurrent(generation); - await this.touchListener(); + // Startup is a readiness boundary: the instance must not become resident + // when the runtime role cannot register or drain the configured schema. + await this.touchListenerInternal(true); + this.registered = true; + this.assertStartCurrent(generation); - // Initial drain immediately after registration - await this.drain(); + // A caller can request a manual drain while registration is in flight. + // Let it settle, then run the strict readiness drain ourselves so a + // best-effort call can never satisfy the startup boundary. + await this.waitForActiveWork(); + this.assertStartCurrent(generation); + await this.drainInternal(true, generation); + this.assertStartCurrent(generation); - this.pollTimer = setInterval(() => { - void this.drain(); - }, this.pollIntervalMs); + this.state = 'running'; - this.heartbeatTimer = setInterval(() => { - void this.touchListener(); - }, this.heartbeatIntervalMs); + this.pollTimer = setInterval(() => { + void this.drain(); + }, this.pollIntervalMs); + this.pollTimer.unref?.(); + + this.heartbeatTimer = setInterval(() => { + void this.touchListener(); + }, this.heartbeatIntervalMs); + this.heartbeatTimer.unref?.(); + } catch (error) { + this.clearTimers(); + if (this.registered) { + await this.cleanupEphemeralInternal(); + this.registered = false; + } + if (this.state === 'starting') this.state = 'stopped'; + throw error; + } } - async stop(): Promise { - if (!this.running) return; - this.running = false; + stop(): Promise { + if (this.state === 'stopped') return Promise.resolve(); + if (this.state === 'stopping') return this.stopPromise!; + + const startInFlight = this.startPromise; + ++this.generation; + this.state = 'stopping'; + this.clearTimers(); log.info(`Stopping cursor tracker: node=${this.nodeId}`); + const pending = this.stopInternal(startInFlight); + this.stopPromise = pending; + void pending.then( + () => { + if (this.stopPromise === pending) this.stopPromise = null; + }, + () => { + if (this.stopPromise === pending) this.stopPromise = null; + } + ); + return pending; + } + private async stopInternal(startInFlight: Promise | null): Promise { + if (startInFlight) await Promise.allSettled([startInFlight]); + await this.waitForActiveWork(); + if (this.registered) { + await this.cleanupEphemeralInternal(); + this.registered = false; + } + this.state = 'stopped'; + } + + private clearTimers(): void { if (this.pollTimer) { clearInterval(this.pollTimer); this.pollTimer = null; @@ -94,14 +185,39 @@ export class CursorTracker { clearInterval(this.heartbeatTimer); this.heartbeatTimer = null; } + } - await this.cleanupEphemeral(); + drain(): Promise { + if (this.state === 'stopping') return Promise.resolve([]); + const dispatchGeneration = this.state === 'starting' || this.state === 'running' + ? this.generation + : undefined; + return this.drainInternal(false, dispatchGeneration); } - async drain(): Promise { - if (this.draining) return []; - this.draining = true; + private drainInternal( + throwOnError: boolean, + dispatchGeneration?: number + ): Promise { + if (this.activeDrain) return Promise.resolve([]); + const pending = this.executeDrain(throwOnError, dispatchGeneration); + this.activeDrain = pending; + void pending.then( + () => { + if (this.activeDrain === pending) this.activeDrain = null; + }, + () => { + if (this.activeDrain === pending) this.activeDrain = null; + } + ); + return pending; + } + + private async executeDrain( + throwOnError: boolean, + dispatchGeneration?: number + ): Promise { try { const sql = `SELECT * FROM ${this.quoteIdent(this.schema)}.drain_changes($1, $2)`; const result = await this.pool.query<{ drain_changes: ChangeLogEntry }>( @@ -110,30 +226,56 @@ export class CursorTracker { ); const entries = result.rows.map((row) => row.drain_changes); - if (entries.length > 0) { + if (entries.length > 0 && this.mayDispatch(dispatchGeneration)) { log.info(`Drained ${entries.length} change(s) for node=${this.nodeId}`); this.onChanges(entries); } return entries; } catch (err) { - this.onError(err instanceof Error ? err : new Error(String(err))); + const error = err instanceof Error ? err : new Error(String(err)); + this.onError(error); + if (throwOnError) throw error; return []; - } finally { - this.draining = false; } } - async touchListener(): Promise { + touchListener(): Promise { + if (this.state === 'stopping') return Promise.resolve(); + return this.touchListenerInternal(false); + } + + private touchListenerInternal(throwOnError: boolean): Promise { + if (this.activeHeartbeat) return this.activeHeartbeat; + const pending = this.executeTouchListener(throwOnError); + this.activeHeartbeat = pending; + void pending.then( + () => { + if (this.activeHeartbeat === pending) this.activeHeartbeat = null; + }, + () => { + if (this.activeHeartbeat === pending) this.activeHeartbeat = null; + } + ); + return pending; + } + + private async executeTouchListener(throwOnError: boolean): Promise { try { const sql = `SELECT ${this.quoteIdent(this.schema)}.touch_listener($1)`; await this.pool.query(sql, [this.nodeId]); } catch (err) { - this.onError(err instanceof Error ? err : new Error(String(err))); + const error = err instanceof Error ? err : new Error(String(err)); + this.onError(error); + if (throwOnError) throw error; } } async cleanupEphemeral(): Promise { + await this.cleanupEphemeralInternal(); + } + + private async cleanupEphemeralInternal(): Promise { try { const sql = `SELECT ${this.quoteIdent(this.schema)}.cleanup_ephemeral($1)`; await this.pool.query(sql, [this.nodeId]); @@ -144,7 +286,26 @@ export class CursorTracker { } get isRunning(): boolean { - return this.running; + return this.state === 'running'; + } + + private assertStartCurrent(generation: number): void { + if (this.state !== 'starting' || this.generation !== generation) { + throw new CursorTrackerStartAbortedError(); + } + } + + private mayDispatch(generation: number | undefined): boolean { + if (generation === undefined) return this.state !== 'stopping'; + return this.generation === generation + && (this.state === 'starting' || this.state === 'running'); + } + + private async waitForActiveWork(): Promise { + const active: Promise[] = []; + if (this.activeDrain) active.push(this.activeDrain); + if (this.activeHeartbeat) active.push(this.activeHeartbeat); + if (active.length > 0) await Promise.allSettled(active); } private quoteIdent(identifier: string): string { diff --git a/graphile/graphile-realtime-subscriptions/src/generation-subscriber.ts b/graphile/graphile-realtime-subscriptions/src/generation-subscriber.ts new file mode 100644 index 000000000..4b94da150 --- /dev/null +++ b/graphile/graphile-realtime-subscriptions/src/generation-subscriber.ts @@ -0,0 +1,423 @@ +import type { GrafastSubscriber } from 'grafast'; + +import type { RealtimePublisher } from './types'; + +export const GENERATION_SUBSCRIBER_QUEUE_CAPACITY = 256; +export const REALTIME_GENERATION_TOPIC_ERROR_CODE = 'REALTIME_GENERATION_TOPIC_INVALID'; +export const REALTIME_GENERATION_RELEASED_ERROR_CODE = 'REALTIME_GENERATION_RELEASED'; +export const REALTIME_GENERATION_OVERFLOW_ERROR_CODE = 'REALTIME_GENERATION_OVERFLOW'; +export const REALTIME_GENERATION_SOURCE_ENDED_ERROR_CODE = 'REALTIME_GENERATION_SOURCE_ENDED'; +export const REALTIME_GENERATION_NOT_ACTIVE_ERROR_CODE = 'REALTIME_GENERATION_NOT_ACTIVE'; +export const REALTIME_GENERATION_ALREADY_ACTIVE_ERROR_CODE = 'REALTIME_GENERATION_ALREADY_ACTIVE'; + +type RealtimeTopicMap = Record; + +export interface GenerationScopedRealtimeSubscriberOptions< + TTopics extends RealtimeTopicMap +> { + /** Shared database notification source owned by this generation facade. */ + source: GrafastSubscriber; + /** Exact topics compiled into this Graphile generation. */ + allowedTopics: readonly (keyof TTopics & string)[]; + /** Defaults to true; set false only when lifecycle ownership lives elsewhere. */ + releaseSourceOnRelease?: boolean; +} + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(error: unknown): void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +export class RealtimeGenerationTopicError extends Error { + readonly code = REALTIME_GENERATION_TOPIC_ERROR_CODE; + + constructor(readonly topic: unknown) { + super(`Realtime topic ${JSON.stringify(topic)} is outside this generation's allowlist`); + this.name = 'RealtimeGenerationTopicError'; + } +} + +export class RealtimeGenerationReleasedError extends Error { + readonly code = REALTIME_GENERATION_RELEASED_ERROR_CODE; + + constructor() { + super('Realtime generation subscriber has been released'); + this.name = 'RealtimeGenerationReleasedError'; + } +} + +export class RealtimeGenerationOverflowError extends Error { + readonly code = REALTIME_GENERATION_OVERFLOW_ERROR_CODE; + + constructor( + readonly topic: string, + readonly capacity: number + ) { + super( + `Realtime generation queue for ${JSON.stringify(topic)} exceeded its ` + + `fixed capacity of ${capacity}` + ); + this.name = 'RealtimeGenerationOverflowError'; + } +} + +export class RealtimeGenerationSourceEndedError extends Error { + readonly code = REALTIME_GENERATION_SOURCE_ENDED_ERROR_CODE; + + constructor(readonly topic: string) { + super(`Realtime source for ${JSON.stringify(topic)} ended unexpectedly`); + this.name = 'RealtimeGenerationSourceEndedError'; + } +} + +export class RealtimeGenerationNotActiveError extends Error { + readonly code = REALTIME_GENERATION_NOT_ACTIVE_ERROR_CODE; + + constructor() { + super('Realtime generation subscriber has not been activated'); + this.name = 'RealtimeGenerationNotActiveError'; + } +} + +export class RealtimeGenerationAlreadyActiveError extends Error { + readonly code = REALTIME_GENERATION_ALREADY_ACTIVE_ERROR_CODE; + + constructor() { + super('Realtime generation subscriber has already been activated'); + this.name = 'RealtimeGenerationAlreadyActiveError'; + } +} + +class LocalQueue { + private readonly buffered: T[] = []; + private readonly waiting: Deferred>[] = []; + private terminal: 'open' | 'complete' | 'failed' = 'open'; + private failure: Error | null = null; + + constructor( + private readonly topic: string, + private readonly capacity: number + ) {} + + next(): Promise> { + if (this.buffered.length > 0) { + return Promise.resolve({ done: false, value: this.buffered.shift()! }); + } + if (this.terminal === 'failed') return Promise.reject(this.failure); + if (this.terminal === 'complete') { + return Promise.resolve({ done: true, value: undefined }); + } + const result = deferred>(); + this.waiting.push(result); + return result.promise; + } + + push(value: T): RealtimeGenerationOverflowError | null { + if (this.terminal !== 'open') return null; + const waiter = this.waiting.shift(); + if (waiter) { + waiter.resolve({ done: false, value }); + return null; + } + if (this.buffered.length >= this.capacity) { + const error = new RealtimeGenerationOverflowError(this.topic, this.capacity); + this.fail(error); + return error; + } + this.buffered.push(value); + return null; + } + + complete(): void { + if (this.terminal !== 'open') return; + this.terminal = 'complete'; + this.buffered.length = 0; + for (const waiter of this.waiting.splice(0)) { + waiter.resolve({ done: true, value: undefined }); + } + } + + fail(error: Error): void { + if (this.terminal !== 'open') return; + this.terminal = 'failed'; + this.failure = error; + this.buffered.length = 0; + for (const waiter of this.waiting.splice(0)) waiter.reject(error); + } +} + +class GenerationSubscription implements AsyncIterableIterator { + private readonly queue: LocalQueue; + private readonly sourceIteratorPromise: Promise>; + private sourceReturnPromise: Promise | null = null; + private stopped = false; + private stopPromise: Promise | null = null; + + constructor( + readonly topic: string, + source: GrafastSubscriber>, + private readonly onStop: (subscription: GenerationSubscription) => void + ) { + this.queue = new LocalQueue(topic, GENERATION_SUBSCRIBER_QUEUE_CAPACITY); + this.sourceIteratorPromise = Promise.resolve().then(() => source.subscribe(topic)); + void this.pump(); + } + + [Symbol.asyncIterator](): AsyncIterableIterator { + return this; + } + + next(): Promise> { + return this.queue.next(); + } + + async return(value?: unknown): Promise> { + await this.stop(); + return { done: true, value: value as T }; + } + + async throw(error?: unknown): Promise> { + const failure = error instanceof Error ? error : new Error(String(error)); + await this.stop(failure); + throw failure; + } + + publish(value: T): void { + if (this.stopped) return; + const overflow = this.queue.push(value); + if (overflow) void this.stop(overflow).catch(() => {}); + } + + fail(error: Error): void { + if (this.stopped) return; + void this.stop(error).catch(() => {}); + } + + stop(error?: Error): Promise { + if (this.stopPromise) return this.stopPromise; + this.stopped = true; + if (error) this.queue.fail(error); + else this.queue.complete(); + this.stopPromise = this.returnSource().finally(() => this.onStop(this)); + return this.stopPromise; + } + + private async pump(): Promise { + try { + const iterator = await this.sourceIteratorPromise; + if (this.stopped) { + await this.returnSource(); + return; + } + for (;;) { + const result = await iterator.next(); + if (this.stopped) return; + if (result.done) { + this.fail(new RealtimeGenerationSourceEndedError(this.topic)); + return; + } + this.publish(result.value); + } + } catch (error) { + if (!this.stopped) { + this.fail(error instanceof Error ? error : new Error(String(error))); + } + } + } + + private returnSource(): Promise { + if (this.sourceReturnPromise) return this.sourceReturnPromise; + this.sourceReturnPromise = this.sourceIteratorPromise.then(async (iterator) => { + await iterator.return?.(); + }, () => { + // Source acquisition failure is already delivered to the output queue. + }); + return this.sourceReturnPromise; + } +} + +/** + * A Graphile-generation-local GrafastSubscriber. Database notifications are + * forwarded from the shared source, while cursor catch-up events published + * through publish() remain inside this exact generation. + */ +export class GenerationScopedRealtimeSubscriber< + TTopics extends RealtimeTopicMap = RealtimeTopicMap +> implements GrafastSubscriber, RealtimePublisher { + readonly allowedTopics: readonly (keyof TTopics & string)[]; + private readonly allowedTopicSet: ReadonlySet; + private readonly subscriptions = new Map< + string, + Set> + >(); + private readonly releaseSourceOnRelease: boolean; + private readonly source: GrafastSubscriber; + private released = false; + private releasePromise: Promise | null = null; + + constructor(options: GenerationScopedRealtimeSubscriberOptions) { + if (!Array.isArray(options.allowedTopics) || options.allowedTopics.length === 0) { + throw new RealtimeGenerationTopicError(options.allowedTopics); + } + if (options.allowedTopics.some((topic) => typeof topic !== 'string')) { + throw new RealtimeGenerationTopicError(options.allowedTopics); + } + this.allowedTopics = Object.freeze([...new Set(options.allowedTopics)]); + this.allowedTopicSet = new Set(this.allowedTopics); + this.source = options.source; + this.releaseSourceOnRelease = options.releaseSourceOnRelease ?? true; + } + + subscribe( + topic: TTopic + ): AsyncIterableIterator { + if (this.released) throw new RealtimeGenerationReleasedError(); + if (typeof topic !== 'string' || !this.allowedTopicSet.has(topic)) { + throw new RealtimeGenerationTopicError(topic); + } + + let topicSubscriptions = this.subscriptions.get(topic); + if (!topicSubscriptions) { + topicSubscriptions = new Set(); + this.subscriptions.set(topic, topicSubscriptions); + } + const subscription = new GenerationSubscription( + topic, + this.source as GrafastSubscriber>, + (stopped) => { + topicSubscriptions!.delete(stopped); + if (topicSubscriptions!.size === 0) this.subscriptions.delete(topic); + } + ); + topicSubscriptions.add(subscription); + return subscription as AsyncIterableIterator; + } + + assertTopics(topics: readonly string[]): void { + if (this.released) throw new RealtimeGenerationReleasedError(); + const invalid = topics.find((topic) => !this.allowedTopicSet.has(topic)); + if (invalid !== undefined) throw new RealtimeGenerationTopicError(invalid); + } + + publish(topic: string, payload: string): void { + this.assertTopics([topic]); + const subscriptions = this.subscriptions.get(topic); + if (!subscriptions) return; + for (const subscription of [...subscriptions]) subscription.publish(payload); + } + + release(): Promise { + if (this.releasePromise) return this.releasePromise; + this.released = true; + const active = [...this.subscriptions.values()].flatMap((entries) => [...entries]); + this.releasePromise = (async () => { + const results = await Promise.allSettled(active.map((subscription) => subscription.stop())); + if (this.releaseSourceOnRelease) await this.source.release?.(); + const rejected = results.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ); + if (rejected) throw rejected.reason; + })(); + return this.releasePromise; + } +} + +/** + * Stable subscriber identity installed into a PostGraphile pgService before + * schema construction. Activation installs the exact generation facade only + * after the build has reported all physical @realtime topics. + */ +export class ActivatableGenerationScopedRealtimeSubscriber< + TTopics extends RealtimeTopicMap = RealtimeTopicMap +> implements GrafastSubscriber, RealtimePublisher { + private delegate: GenerationScopedRealtimeSubscriber | null = null; + private released = false; + private releasePromise: Promise | null = null; + + async activate( + options: GenerationScopedRealtimeSubscriberOptions + ): Promise { + if (this.released) { + await options.source.release?.(); + throw new RealtimeGenerationReleasedError(); + } + if (this.delegate) { + await options.source.release?.(); + throw new RealtimeGenerationAlreadyActiveError(); + } + + try { + this.delegate = new GenerationScopedRealtimeSubscriber(options); + } catch (error) { + await options.source.release?.(); + throw error; + } + } + + subscribe( + topic: TTopic + ): AsyncIterableIterator { + if (this.released) throw new RealtimeGenerationReleasedError(); + if (!this.delegate) throw new RealtimeGenerationNotActiveError(); + return this.delegate.subscribe(topic); + } + + assertTopics(topics: readonly string[]): void { + if (this.released) throw new RealtimeGenerationReleasedError(); + if (!this.delegate) throw new RealtimeGenerationNotActiveError(); + this.delegate.assertTopics(topics); + } + + publish(topic: string, payload: string): void { + if (this.released) throw new RealtimeGenerationReleasedError(); + if (!this.delegate) throw new RealtimeGenerationNotActiveError(); + this.delegate.publish(topic, payload); + } + + release(): Promise { + if (this.releasePromise) return this.releasePromise; + this.released = true; + this.releasePromise = this.delegate?.release() ?? Promise.resolve(); + return this.releasePromise; + } +} + +type LegacyEventEmitter = { + emit(topic: string, payload: string): boolean; +}; + +/** + * Transitional adapter for @dataplan/pg's current PgSubscriber. Private-field + * access is quarantined here; RealtimeManager and new integrations depend only + * on the explicit publisher capability. + */ +export const createPgSubscriberPublisher = ( + pgSubscriber: unknown +): RealtimePublisher | null => { + const candidate = pgSubscriber as { eventEmitter?: LegacyEventEmitter } | null; + const emitter = candidate && typeof candidate === 'object' + ? candidate.eventEmitter + : null; + if (!emitter || typeof emitter.emit !== 'function') return null; + const emit = emitter.emit.bind(emitter); + return Object.freeze({ + assertTopics(): void { + // The legacy PgSubscriber owns topic validation. New integrations use + // GenerationScopedRealtimeSubscriber's exact preflight instead. + }, + publish(topic: string, payload: string): void { + emit(topic, payload); + } + }); +}; diff --git a/graphile/graphile-realtime-subscriptions/src/index.ts b/graphile/graphile-realtime-subscriptions/src/index.ts index d0fdf741c..456531e86 100644 --- a/graphile/graphile-realtime-subscriptions/src/index.ts +++ b/graphile/graphile-realtime-subscriptions/src/index.ts @@ -17,14 +17,52 @@ * ``` */ -export { CursorTracker } from './cursor-tracker'; +export { CursorTracker, CursorTrackerStartAbortedError } from './cursor-tracker'; +export type { GenerationScopedRealtimeSubscriberOptions } from './generation-subscriber'; +export { + ActivatableGenerationScopedRealtimeSubscriber, + createPgSubscriberPublisher, + GENERATION_SUBSCRIBER_QUEUE_CAPACITY, + GenerationScopedRealtimeSubscriber, + REALTIME_GENERATION_ALREADY_ACTIVE_ERROR_CODE, + REALTIME_GENERATION_NOT_ACTIVE_ERROR_CODE, + REALTIME_GENERATION_OVERFLOW_ERROR_CODE, + REALTIME_GENERATION_RELEASED_ERROR_CODE, + REALTIME_GENERATION_SOURCE_ENDED_ERROR_CODE, + REALTIME_GENERATION_TOPIC_ERROR_CODE, + RealtimeGenerationAlreadyActiveError, + RealtimeGenerationNotActiveError, + RealtimeGenerationOverflowError, + RealtimeGenerationReleasedError, + RealtimeGenerationSourceEndedError, + RealtimeGenerationTopicError +} from './generation-subscriber'; export { createRealtimeSubscriptionsPlugin, RealtimeSubscriptionsPlugin } from './plugin'; export { RealtimeSubscriptionsPreset } from './preset'; -export { RealtimeManager } from './realtime-manager'; -export type { RealtimeSubscriptionsPluginOptions } from './types'; +export { + RealtimeManager, + RealtimeManagerStartAbortedError, + RealtimeSourceSchemaConfigurationError, + RealtimeSourceSchemaViolationError, + RealtimeSubscriberUnavailableError +} from './realtime-manager'; +export { + REALTIME_TOPIC_DISCOVERY_CHANGED_ERROR_CODE, + REALTIME_TOPIC_DISCOVERY_EMPTY_ERROR_CODE, + REALTIME_TOPIC_DISCOVERY_FOREIGN_ERROR_CODE, + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + REALTIME_TOPIC_DISCOVERY_MISSING_ERROR_CODE, + RealtimeTopicCollector, + RealtimeTopicDiscoveryError +} from './topic-collector'; +export type { + RealtimeSubscriptionsPluginOptions, + RealtimeTopicDescriptor +} from './types'; export type { ChangeLogEntry, CursorTrackerOptions, Queryable, RealtimeManagerOptions, + RealtimePublisher, } from './types'; diff --git a/graphile/graphile-realtime-subscriptions/src/plugin.ts b/graphile/graphile-realtime-subscriptions/src/plugin.ts index a013d778b..da1204f43 100644 --- a/graphile/graphile-realtime-subscriptions/src/plugin.ts +++ b/graphile/graphile-realtime-subscriptions/src/plugin.ts @@ -37,25 +37,34 @@ * drops individual events and sends a single INVALIDATE when exceeded * * Security / RLS enforcement: - * - Row data is always fetched via resource.get() which runs through the - * authenticated user's connection with their JWT role and pgSettings applied. - * - For INSERT/UPDATE events, if RLS denies access (resource.get returns null), - * the rowId is masked (set to null) to prevent metadata leaks. - * - For DELETE events, row is naturally null (the row no longer exists). - * - For INVALIDATE (overflow), the client should refetch via a normal query - * which is also RLS-gated. + * - INSERT/UPDATE notifications are filtered at the AsyncIterable boundary by + * a parameterized visibility query under the request role and pgSettings. + * Grafast never observes an event unless at least one changed row is visible. + * - Row data is fetched via resource.get() under the same request RLS context. + * - Collection subscriptions never expose row IDs because merely observing + * identifiers from rows hidden by RLS is a metadata leak. + * - Sparse INSERT/UPDATE subscriptions expose a requested row ID only after + * resource.get() confirms that the row remains visible under request RLS. + * - DELETE and database-originated INVALIDATE events are suppressed because + * neither carries a sound post-change audience proof. Plugin throttling may + * emit INVALIDATE only after an authorized INSERT/UPDATE event. * - When ids are provided, only events for those specific rows are delivered, * preventing cross-tenant event leaks. */ import { Logger } from '@pgpmjs/logger'; -import { constant, context as grafastContext, lambda,listen, object } from 'grafast'; +import { QuoteUtils } from '@pgsql/quotes'; +import type { Step } from 'grafast'; +import { constant, context as grafastContext, get, lambda, listen } from 'grafast'; import type { GraphileConfig } from 'graphile-config'; import { extendSchema } from 'graphile-utils'; import type { ParsedPayload } from './event-gate'; -import { createGatedSubscriber } from './event-gate'; -import type { RealtimeSubscriptionsPluginOptions } from './types'; +import { EventThrottle, parseNotifyPayload } from './event-gate'; +import type { + RealtimeSubscriptionsPluginOptions, + RealtimeTopicDescriptor, +} from './types'; const log = new Logger('graphile-realtime-subscriptions'); @@ -73,6 +82,158 @@ interface RealtimeTableInfo { pgTable: string; } +interface RealtimeEvent { + parsed: ParsedPayload; + subscribedIds: string[] | null | undefined; +} + +interface PgExecutorContextLike { + pgSettings: Record | null; + withPgClient( + pgSettings: Record | null, + callback: (client: { + query(query: { + text: string; + values?: unknown[]; + }): Promise<{ rows: readonly TData[] }>; + }) => Promise | T, + ): Promise; +} + +interface RealtimeSubscriberLike { + subscribe(topic: string | number): + | AsyncIterableIterator + | Promise>; +} + +/** + * Select the row that may be fetched under the request's RLS context. + * + * Collection subscriptions may fetch INSERT/UPDATE rows, but their public + * rowId field remains hidden. Sparse subscriptions only consider IDs the + * caller supplied. DELETE cannot be authorized after the row is gone. + */ +function selectCandidateRowId( + parsed: ParsedPayload | null, + subscribedIds: string[] | null | undefined, + allowCollection: boolean, +): string | null { + if ( + !parsed + || parsed.overflow + || (parsed.event !== 'INSERT' && parsed.event !== 'UPDATE') + || parsed.rowIds.length === 0 + ) { + return null; + } + + if (subscribedIds && subscribedIds.length > 0) { + return parsed.rowIds.find((rowId) => subscribedIds.includes(rowId)) ?? null; + } + + return allowCollection ? parsed.rowIds[0] : null; +} + +function selectCandidateRowIds( + parsed: ParsedPayload, + subscribedIds: string[] | null | undefined, +): string[] { + if ( + parsed.overflow + || (parsed.event !== 'INSERT' && parsed.event !== 'UPDATE') + ) { + return []; + } + + const candidates = subscribedIds && subscribedIds.length > 0 + ? parsed.rowIds.filter((rowId) => subscribedIds.includes(rowId)) + : parsed.rowIds; + + return [...new Set(candidates)]; +} + +/** + * Filter the notification stream before Grafast observes a subscription event. + * Returning a nullable payload from an item plan would still emit an observable + * GraphQL result, so authorization has to happen at the AsyncIterable boundary. + */ +async function* authorizeNotificationStream( + sourceOrPromise: + | AsyncIterableIterator + | Promise>, + executorContext: PgExecutorContextLike, + subscribedIds: string[] | null | undefined, + visibilitySql: string, + overflowThreshold: number, +): AsyncGenerator { + const source = await sourceOrPromise; + const throttle = new EventThrottle(overflowThreshold); + + for await (const raw of source) { + const parsed = parseNotifyPayload(String(raw)); + const candidateRowIds = selectCandidateRowIds(parsed, subscribedIds); + + // DELETE cannot be reauthorized after the row is gone. Database-originated + // INVALIDATE and malformed/unknown events carry no audience proof either. + if (candidateRowIds.length === 0) continue; + + let visibleRowIds: Set; + try { + visibleRowIds = await executorContext.withPgClient( + executorContext.pgSettings, + async (client) => { + const result = await client.query<{ id: string }>({ + text: visibilitySql, + values: [candidateRowIds], + }); + return new Set(result.rows.map((row) => String(row.id))); + }, + ); + } catch { + // Authorization errors must never turn into an event-existence oracle. + log.warn('Suppressing realtime event because RLS reauthorization failed'); + continue; + } + + const authorizedRowIds = candidateRowIds.filter((rowId) => visibleRowIds.has(rowId)); + if (authorizedRowIds.length === 0) continue; + + // Count only authorized events. Hidden-tenant traffic must not influence a + // subscriber's throttle state because that would be an observable side channel. + const action = throttle.check(); + if (action === 'drop') continue; + + const authorizedPayload = action === 'overflow' + ? { event: 'INVALIDATE', rowIds: [], overflow: true } + : { ...parsed, rowIds: authorizedRowIds }; + + yield { + parsed: authorizedPayload, + subscribedIds, + }; + } +} + +function createRlsAuthorizedSubscriber( + subscriber: RealtimeSubscriberLike, + executorContext: PgExecutorContextLike, + subscribedIds: string[] | null | undefined, + visibilitySql: string, + overflowThreshold: number, +): RealtimeSubscriberLike { + return { + subscribe(topic: string | number) { + return authorizeNotificationStream( + subscriber.subscribe(topic), + executorContext, + subscribedIds, + visibilitySql, + overflowThreshold, + ); + }, + }; +} + function discoverRealtimeTables(build: any): RealtimeTableInfo[] { const { pgRegistry } = build.input; const resources = pgRegistry.pgResources; @@ -123,11 +284,11 @@ function buildTypeDefs(tables: RealtimeTableInfo[]): string { .map(({ payloadTypeName, typeName, rowFieldName }) => `"""Payload delivered when a ${typeName} row changes."""\n` + `type ${payloadTypeName} {\n` + - ` """The DML operation: INSERT, UPDATE, DELETE, or INVALIDATE."""\n` + + ` """The authorized operation: INSERT, UPDATE, or plugin-generated INVALIDATE."""\n` + ` event: String!\n` + - ` """The current state of the row (null for DELETE, INVALIDATE, or if RLS denies access)."""\n` + + ` """The current state of the row (null for INVALIDATE or an RLS visibility race)."""\n` + ` ${rowFieldName}: ${typeName}\n` + - ` """The ID of the changed row (null for INVALIDATE, or masked when RLS denies access)."""\n` + + ` """The requested row ID for a sparse INSERT/UPDATE subscription after RLS authorization. Null for collection, INVALIDATE, or denied rows."""\n` + ` rowId: UUID\n` + ` """True when too many changes occurred and the client should refetch."""\n` + ` overflow: Boolean!\n` + @@ -145,23 +306,13 @@ function buildTypeDefs(tables: RealtimeTableInfo[]): string { function requirePayload(payload: unknown): ParsedPayload { if (payload === null || payload === undefined) { throw new Error( - 'Realtime subscription payload is missing: the gated subscriber only ever ' + - 'yields parsed payloads, so this event bypassed createGatedSubscriber.', + 'Realtime subscription payload is missing: the authorized subscriber only ever ' + + 'yields parsed payloads, so this event bypassed the pre-delivery authorization gate.', ); } return payload as ParsedPayload; } -/** - * The row this event reports. `rowIds` is already narrowed to the - * subscription's `ids`, so the first entry is the one to surface; INVALIDATE - * carries none, which is a genuine absence rather than a suppressed error. - */ -function reportedRowId(payload: ParsedPayload): string | null { - if (payload.overflow) return null; - return payload.rowIds[0] ?? null; -} - function buildPlans( tables: RealtimeTableInfo[], overflowThreshold: number, @@ -169,22 +320,50 @@ function buildPlans( const subscriptionPlans: Record = {}; const allPlans: Record = {}; - for (const { resource, fieldName, payloadTypeName, rowFieldName, notifyChannel } of tables) { + for (const { + resource, + fieldName, + payloadTypeName, + rowFieldName, + notifyChannel, + pgSchema, + pgTable, + } of tables) { + const qualifiedTable = QuoteUtils.quoteQualifiedIdentifier(pgSchema, pgTable); + const idColumn = QuoteUtils.quoteIdentifier('id'); + const visibilitySql = + `select ${idColumn}::text as id from ${qualifiedTable} ` + // Notification payloads are text, and @realtime tables may use UUID, + // integer, bigint, or text primary keys. Comparing their canonical text + // form keeps the query parameterized and avoids a UUID-only cast that + // silently suppresses otherwise authorized events. + + `where ${idColumn}::text = any($1::text[])`; + subscriptionPlans[fieldName] = { subscribePlan(_$root: any, args: any) { const $pgSubscriber = (grafastContext() as any).get('pgSubscriber'); + const $executorContext = resource.executor.context(); const $topic = constant(notifyChannel); const $ids = args.getRaw('ids'); + const $authorizedSubscriber = lambda( + [$pgSubscriber, $executorContext, $ids], + (values: unknown) => { + const [subscriber, executorContext, subscribedIds] = values as readonly [ + RealtimeSubscriberLike, + PgExecutorContextLike, + string[] | null | undefined, + ]; + return createRlsAuthorizedSubscriber( + subscriber, + executorContext, + subscribedIds, + visibilitySql, + overflowThreshold, + ); + }, + ); - // Parsing, throttling and sparse-set filtering all happen in the gate, - // built once per subscription, so the stream below yields only events - // that should reach this client — every step after it is total. - const $subscriber = lambda([$pgSubscriber, $ids], (pair: unknown) => { - const [pgSubscriber, ids] = pair as readonly [any, string[] | null | undefined]; - return createGatedSubscriber(pgSubscriber, { ids, threshold: overflowThreshold }); - }); - - return listen($subscriber, $topic, ($payload: any) => object({ parsed: $payload })); + return listen($authorizedSubscriber, $topic); }, plan($event: any) { return $event; @@ -192,18 +371,46 @@ function buildPlans( }; allPlans[payloadTypeName] = { - event($parent: any) { - return lambda($parent.get('parsed'), (p: unknown) => requirePayload(p).event); + event($parent: Step) { + const $parsed = get($parent, 'parsed'); + return lambda($parsed, (p: unknown) => requirePayload(p).event); }, - rowId($parent: any) { - return lambda($parent.get('parsed'), (p: unknown) => reportedRowId(requirePayload(p))); + rowId($parent: Step) { + const $parsed = get($parent, 'parsed'); + const $subscribedIds = get($parent, 'subscribedIds'); + const $candidateRowId = lambda( + [$parsed, $subscribedIds], + (pair: unknown) => { + const [parsed, subscribedIds] = pair as readonly [ + ParsedPayload | null, + string[] | null | undefined, + ]; + // Collection mode deliberately cannot surface a row identifier. + return selectCandidateRowId(requirePayload(parsed), subscribedIds, false); + }, + ); + const $authorizedRow = resource.get({ id: $candidateRowId }); + // Selecting through the PgSelectSingleStep makes the ID null whenever + // request RLS hides the row; the raw notification ID is never returned. + return $authorizedRow.get('id'); }, - overflow($parent: any) { - return lambda($parent.get('parsed'), (p: unknown) => requirePayload(p).overflow); + overflow($parent: Step) { + const $parsed = get($parent, 'parsed'); + return lambda($parsed, (p: unknown) => requirePayload(p).overflow); }, - [rowFieldName]($parent: any) { - const $rowId = lambda($parent.get('parsed'), (p: unknown) => - reportedRowId(requirePayload(p)), + [rowFieldName]($parent: Step) { + const $parsed = get($parent, 'parsed'); + const $subscribedIds = get($parent, 'subscribedIds'); + + const $rowId = lambda( + [$parsed, $subscribedIds], + (tuple: unknown) => { + const [parsed, subscribedIds] = tuple as readonly [ + ParsedPayload | null, + string[] | null | undefined, + ]; + return selectCandidateRowId(requirePayload(parsed), subscribedIds, true); + }, ); return resource.get({ id: $rowId }); @@ -223,6 +430,16 @@ export function createRealtimeSubscriptionsPlugin( return extendSchema( (build) => { const tables = discoverRealtimeTables(build); + const discoveredTopics: readonly RealtimeTopicDescriptor[] = Object.freeze( + tables + .map(({ notifyChannel, pgSchema, pgTable }) => Object.freeze({ + topic: notifyChannel, + schema: pgSchema, + table: pgTable, + })) + .sort((left, right) => left.topic.localeCompare(right.topic)), + ); + options.onTopicsDiscovered?.(discoveredTopics); if (tables.length === 0) { log.info('No tables with @realtime tag found — skipping subscription generation'); @@ -254,4 +471,8 @@ export { } from './event-gate'; export { RealtimeManager } from './realtime-manager'; export type { ChangeLogEntry, CursorTrackerOptions, Queryable, RealtimeManagerOptions } from './types'; -export { DEFAULT_OVERFLOW_THRESHOLD }; +// Exported for testing +export { + DEFAULT_OVERFLOW_THRESHOLD, + selectCandidateRowId, +}; diff --git a/graphile/graphile-realtime-subscriptions/src/realtime-manager.ts b/graphile/graphile-realtime-subscriptions/src/realtime-manager.ts index 58bcf1192..53c77362d 100644 --- a/graphile/graphile-realtime-subscriptions/src/realtime-manager.ts +++ b/graphile/graphile-realtime-subscriptions/src/realtime-manager.ts @@ -1,17 +1,13 @@ /** * RealtimeManager — bridges CursorTracker (polling drain_changes) into - * PostGraphile's PgSubscriber so cursor-tracked events flow through the - * same subscription plans as NOTIFY events. + * a generation-local publisher so cursor-tracked events flow through the same + * subscription plans as NOTIFY events. * * Architecture: - * PgSubscriber uses an internal EventEmitter. NOTIFY payloads arrive via - * pg's `notification` event and are emitted as `eventEmitter.emit(channel, payload)`. - * The `listen()` step in grafast subscribes to the same EventEmitter. - * * RealtimeManager converts ChangeLogEntry objects from drain_changes() into - * the same NOTIFY payload format ("OP:rowId1,rowId2,...") and emits them on - * the PgSubscriber's EventEmitter, so existing subscription plans handle - * them identically to real NOTIFY events. + * the same NOTIFY payload format ("OP:rowId1,rowId2,...") and publishes them + * through an explicit capability. The generation-scoped subscriber keeps + * these cursor events local even when PostgreSQL LISTEN is shared. * * This provides at-least-once delivery: NOTIFY is instant but best-effort; * cursor polling catches up on anything missed (disconnects, restarts). @@ -26,13 +22,59 @@ import { Logger } from '@pgpmjs/logger'; import { CursorTracker } from './cursor-tracker'; +import { createPgSubscriberPublisher } from './generation-subscriber'; import type { ChangeLogEntry, RealtimeManagerOptions, + RealtimePublisher, } from './types'; const log = new Logger('realtime-manager'); +type RealtimeManagerState = 'stopped' | 'starting' | 'running' | 'stopping'; + +export class RealtimeManagerStartAbortedError extends Error { + readonly code = 'REALTIME_MANAGER_START_ABORTED'; + + constructor() { + super('RealtimeManager was stopped before startup completed'); + this.name = 'RealtimeManagerStartAbortedError'; + } +} + +export class RealtimeSubscriberUnavailableError extends Error { + readonly code = 'REALTIME_SUBSCRIBER_UNAVAILABLE'; + + constructor() { + super('RealtimeManager requires a usable local publisher'); + this.name = 'RealtimeSubscriberUnavailableError'; + } +} + +export class RealtimeSourceSchemaViolationError extends Error { + readonly code = 'REALTIME_SOURCE_SCHEMA_VIOLATION'; + + constructor( + readonly sourceSchema: unknown, + readonly allowedSourceSchemas: readonly string[] + ) { + super( + `Realtime cursor returned source schema ${JSON.stringify(sourceSchema)} ` + + `outside the allowed Graphile schemas: ${allowedSourceSchemas.join(', ')}` + ); + this.name = 'RealtimeSourceSchemaViolationError'; + } +} + +export class RealtimeSourceSchemaConfigurationError extends Error { + readonly code = 'REALTIME_SOURCE_SCHEMAS_REQUIRED'; + + constructor() { + super('RealtimeManager requires at least one exact allowed source schema'); + this.name = 'RealtimeSourceSchemaConfigurationError'; + } +} + /** * Extract row IDs from a ChangeLogEntry. * @@ -69,12 +111,37 @@ function entryToChannel(entry: ChangeLogEntry): string { export class RealtimeManager { private readonly cursorTracker: CursorTracker; - private readonly subscriber: unknown; - private started = false; + private readonly publisher: RealtimePublisher | null; + private readonly allowedSourceSchemas: ReadonlySet; + private readonly allowedSourceSchemaList: readonly string[]; + private readonly sourceSchemaConfigurationValid: boolean; + private readonly onFatalError?: (error: Error) => void; + private state: RealtimeManagerState = 'stopped'; + private generation = 0; + private dispatchEnabled = false; + private fatalError: Error | null = null; + private startPromise: Promise | null = null; + private stopPromise: Promise | null = null; constructor(options: RealtimeManagerOptions) { - const { pgSubscriber, pool, ...cursorOpts } = options; - this.subscriber = pgSubscriber; + const { + publisher, + pgSubscriber, + pool, + allowedSourceSchemas, + onFatalError, + ...cursorOpts + } = options; + this.publisher = publisher ?? createPgSubscriberPublisher(pgSubscriber); + this.onFatalError = onFatalError; + this.sourceSchemaConfigurationValid = Array.isArray(allowedSourceSchemas) + && allowedSourceSchemas.every( + (schema) => typeof schema === 'string' && schema.length > 0 + ); + this.allowedSourceSchemaList = Object.freeze([ + ...new Set(allowedSourceSchemas ?? []) + ]); + this.allowedSourceSchemas = new Set(this.allowedSourceSchemaList); this.cursorTracker = new CursorTracker({ nodeId: cursorOpts.nodeId, @@ -84,9 +151,26 @@ export class RealtimeManager { batchLimit: cursorOpts.batchLimit, pool, onChanges: (entries) => this.dispatchEntries(entries), - onError: cursorOpts.onError ?? ((err) => { - log.error(`RealtimeManager error: ${err.message}`); - }), + onError: (error) => { + // Once readiness has completed, losing either cursor polling or the + // listener heartbeat means at-least-once delivery can no longer be + // claimed. Disable dispatch and begin shutdown before invoking the + // observational callback so a callback cannot leave a stale + // generation serving traffic by throwing or stopping it itself. + if (this.state === 'running') this.failDelivery(error); + + try { + if (cursorOpts.onError) { + cursorOpts.onError(error); + } else { + log.error(`RealtimeManager error: ${error.message}`); + } + } catch (callbackError) { + log.error( + `RealtimeManager error callback failed: ${String(callbackError)}` + ); + } + }, }); } @@ -95,62 +179,160 @@ export class RealtimeManager { } get isRunning(): boolean { - return this.started && this.cursorTracker.isRunning; + return this.state === 'running' && this.cursorTracker.isRunning; } - async start(): Promise { - if (this.started) return; - this.started = true; + start(): Promise { + if (this.state === 'running') return Promise.resolve(); + if (this.state === 'starting') return this.startPromise!; + if (this.state === 'stopping') { + return (this.stopPromise ?? Promise.resolve()).then(() => this.start()); + } + const generation = ++this.generation; + this.state = 'starting'; + this.dispatchEnabled = true; log.info(`Starting RealtimeManager: node=${this.nodeId}`); - await this.cursorTracker.start(); + const pending = this.startInternal(generation); + this.startPromise = pending; + void pending.then( + () => { + if (this.startPromise === pending) this.startPromise = null; + }, + () => { + if (this.startPromise === pending) this.startPromise = null; + } + ); + return pending; + } + + private async startInternal(generation: number): Promise { + try { + if ( + !this.sourceSchemaConfigurationValid + || this.allowedSourceSchemas.size === 0 + ) { + throw new RealtimeSourceSchemaConfigurationError(); + } + if (!this.publisher || typeof this.publisher.publish !== 'function') { + throw new RealtimeSubscriberUnavailableError(); + } + await this.cursorTracker.start(); + if (this.state !== 'starting' || this.generation !== generation) { + throw new RealtimeManagerStartAbortedError(); + } + this.state = 'running'; + } catch (error) { + this.dispatchEnabled = false; + if (this.state === 'starting') this.state = 'stopped'; + throw error; + } } - async stop(): Promise { - if (!this.started) return; - this.started = false; + stop(): Promise { + if (this.state === 'stopped') return Promise.resolve(); + if (this.state === 'stopping') return this.stopPromise!; + const startInFlight = this.startPromise; + ++this.generation; + this.state = 'stopping'; + this.dispatchEnabled = false; log.info(`Stopping RealtimeManager: node=${this.nodeId}`); - await this.cursorTracker.stop(); + // Start the tracker shutdown synchronously so an in-flight drain is + // invalidated before it can dispatch after this method is called. + const trackerStop = this.cursorTracker.stop(); + const pending = this.stopInternal(startInFlight, trackerStop); + this.stopPromise = pending; + void pending.then( + () => { + if (this.stopPromise === pending) this.stopPromise = null; + }, + () => { + if (this.stopPromise === pending) this.stopPromise = null; + } + ); + return pending; + } + + private async stopInternal( + startInFlight: Promise | null, + trackerStop: Promise + ): Promise { + try { + if (startInFlight) await Promise.allSettled([startInFlight]); + await trackerStop; + } finally { + this.state = 'stopped'; + this.dispatchEnabled = false; + } } /** - * Convert ChangeLogEntry objects to NOTIFY-format payloads and emit - * them on the PgSubscriber's internal EventEmitter. + * Convert ChangeLogEntry objects to NOTIFY-format payloads and publish them + * through the exact generation's explicit local capability. */ private dispatchEntries(entries: ChangeLogEntry[]): void { - const emitter = this.getEventEmitter(); - if (!emitter) { - log.warn('PgSubscriber has no eventEmitter; cursor events cannot be dispatched'); - return; + if (!this.dispatchEnabled) return; + + const publisher = this.publisher; + if (!publisher) { + const error = new RealtimeSubscriberUnavailableError(); + this.failDelivery(error); + throw error; } - for (const entry of entries) { - const channel = entryToChannel(entry); - const payload = entryToNotifyPayload(entry); - emitter.emit(channel, payload); + // Validate the complete batch before emitting the first event. This keeps + // a mixed valid/foreign batch atomic from the tenant-isolation boundary's + // perspective: no event is delivered when routing is inconclusive. + const foreignEntry = entries.find( + (entry) => !this.allowedSourceSchemas.has(entry.source_schema) + ); + if (foreignEntry) { + const error = new RealtimeSourceSchemaViolationError( + foreignEntry.source_schema, + this.allowedSourceSchemaList + ); + this.failDelivery(error); + throw error; } - log.info(`Dispatched ${entries.length} cursor-tracked event(s) to PgSubscriber`); + const notifications = entries.map((entry) => ({ + channel: entryToChannel(entry), + payload: entryToNotifyPayload(entry) + })); + try { + publisher.assertTopics?.(notifications.map(({ channel }) => channel)); + for (const { channel, payload } of notifications) { + publisher.publish(channel, payload); + } + } catch (reason) { + const error = reason instanceof Error ? reason : new Error(String(reason)); + this.failDelivery(error); + throw error; + } + + log.info(`Dispatched ${entries.length} cursor-tracked event(s)`); } - /** - * Access PgSubscriber's internal EventEmitter. - * - * PgSubscriber from @dataplan/pg stores an EventEmitter3 instance as - * `this.eventEmitter`. It is private but stable across v1.x releases. - * This is the same emitter that NOTIFY events are dispatched through. - */ - private getEventEmitter(): { emit(event: string, payload: string): boolean } | null { - const sub = this.subscriber as Record; - if (sub && typeof sub === 'object' && 'eventEmitter' in sub) { - const ee = sub.eventEmitter as { emit(event: string, payload: string): boolean }; - if (typeof ee?.emit === 'function') { - return ee; + private failDelivery(error: Error): void { + this.dispatchEnabled = false; + const stopping = this.stop(); + if (!this.fatalError) { + this.fatalError = error; + try { + this.onFatalError?.(error); + } catch (callbackError) { + log.error( + `RealtimeManager fatal-error callback failed: ${String(callbackError)}` + ); } } - return null; + void stopping.catch((stopError) => { + log.error( + `RealtimeManager failed to stop after a delivery violation: ${String(stopError)}` + ); + }); } } -export { entryToChannel,entryToNotifyPayload, extractRowId }; +export { entryToChannel, entryToNotifyPayload, extractRowId }; diff --git a/graphile/graphile-realtime-subscriptions/src/topic-collector.ts b/graphile/graphile-realtime-subscriptions/src/topic-collector.ts new file mode 100644 index 000000000..b65e88f0f --- /dev/null +++ b/graphile/graphile-realtime-subscriptions/src/topic-collector.ts @@ -0,0 +1,171 @@ +import type { RealtimeTopicDescriptor } from './types'; + +export const REALTIME_TOPIC_DISCOVERY_MISSING_ERROR_CODE = + 'REALTIME_TOPIC_DISCOVERY_MISSING'; +export const REALTIME_TOPIC_DISCOVERY_EMPTY_ERROR_CODE = + 'REALTIME_TOPIC_DISCOVERY_EMPTY'; +export const REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE = + 'REALTIME_TOPIC_DISCOVERY_INVALID'; +export const REALTIME_TOPIC_DISCOVERY_FOREIGN_ERROR_CODE = + 'REALTIME_TOPIC_DISCOVERY_FOREIGN'; +export const REALTIME_TOPIC_DISCOVERY_CHANGED_ERROR_CODE = + 'REALTIME_TOPIC_DISCOVERY_CHANGED'; + +type RealtimeTopicDiscoveryCode = + | typeof REALTIME_TOPIC_DISCOVERY_MISSING_ERROR_CODE + | typeof REALTIME_TOPIC_DISCOVERY_EMPTY_ERROR_CODE + | typeof REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE + | typeof REALTIME_TOPIC_DISCOVERY_FOREIGN_ERROR_CODE + | typeof REALTIME_TOPIC_DISCOVERY_CHANGED_ERROR_CODE; + +export class RealtimeTopicDiscoveryError extends Error { + constructor( + readonly code: RealtimeTopicDiscoveryCode, + message: string + ) { + super(message); + this.name = 'RealtimeTopicDiscoveryError'; + } +} + +const containsUnpairedSurrogate = (value: string): boolean => { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return true; + index++; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +}; + +const assertIdentifierPart = ( + part: 'schema' | 'table', + value: unknown +): string => { + if (typeof value !== 'string' || value.length === 0) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + `Realtime ${part} must be a non-empty string` + ); + } + if ( + value.includes('\0') + || value.includes('.') + || containsUnpairedSurrogate(value) + ) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + `Realtime ${part} cannot be represented unambiguously in a notification topic` + ); + } + return value; +}; + +const normalizeDescriptor = ( + descriptor: RealtimeTopicDescriptor +): Readonly => { + const schema = assertIdentifierPart('schema', descriptor?.schema); + const table = assertIdentifierPart('table', descriptor?.table); + const expectedTopic = `realtime:${schema}.${table}`; + if ( + descriptor?.topic !== expectedTopic + || expectedTopic.includes('\0') + || containsUnpairedSurrogate(expectedTopic) + || Buffer.byteLength(expectedTopic, 'utf8') > 63 + ) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + 'Realtime topic does not exactly match its physical schema/table or exceeds PostgreSQL limits' + ); + } + return Object.freeze({ topic: expectedTopic, schema, table }); +}; + +const descriptorKey = (descriptor: RealtimeTopicDescriptor): string => + `${descriptor.schema}\0${descriptor.table}\0${descriptor.topic}`; + +/** + * One schema-generation collector. It accepts repeated byte-equivalent build + * callbacks, but rejects topic drift so an already activated listener cannot + * silently become incomplete after a Graphile rebuild. + */ +export class RealtimeTopicCollector { + private descriptors: readonly Readonly[] | null = null; + + readonly collect = (input: readonly RealtimeTopicDescriptor[]): void => { + if (!Array.isArray(input)) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + 'Realtime topic discovery did not provide an array' + ); + } + const byTopic = new Map>(); + for (const candidate of input) { + const descriptor = normalizeDescriptor(candidate); + const previous = byTopic.get(descriptor.topic); + if (previous && descriptorKey(previous) !== descriptorKey(descriptor)) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + `Realtime notification topic ${JSON.stringify(descriptor.topic)} is ambiguous` + ); + } + byTopic.set(descriptor.topic, descriptor); + } + const next = Object.freeze( + [...byTopic.values()].sort((left, right) => left.topic.localeCompare(right.topic)) + ); + if (this.descriptors) { + const previousKeys = this.descriptors.map(descriptorKey); + const nextKeys = next.map(descriptorKey); + if ( + previousKeys.length !== nextKeys.length + || previousKeys.some((key, index) => key !== nextKeys[index]) + ) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_CHANGED_ERROR_CODE, + 'Realtime topics changed after the generation discovery boundary' + ); + } + return; + } + this.descriptors = next; + }; + + exactTopics(allowedSchemas: readonly string[]): readonly string[] { + if (!this.descriptors) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_MISSING_ERROR_CODE, + 'Realtime plugin did not report its compiled notification topics' + ); + } + if (this.descriptors.length === 0) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_EMPTY_ERROR_CODE, + 'Shared realtime requires at least one compiled @realtime topic' + ); + } + if ( + !Array.isArray(allowedSchemas) + || allowedSchemas.length === 0 + || allowedSchemas.some((schema) => typeof schema !== 'string' || schema.length === 0) + ) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_INVALID_ERROR_CODE, + 'Shared realtime requires at least one exact allowed physical schema' + ); + } + const allowed = new Set(allowedSchemas); + const foreign = this.descriptors.find(({ schema }) => !allowed.has(schema)); + if (foreign) { + throw new RealtimeTopicDiscoveryError( + REALTIME_TOPIC_DISCOVERY_FOREIGN_ERROR_CODE, + `Realtime topic ${JSON.stringify(foreign.topic)} is outside this Graphile generation` + ); + } + return Object.freeze(this.descriptors.map(({ topic }) => topic)); + } +} diff --git a/graphile/graphile-realtime-subscriptions/src/types.ts b/graphile/graphile-realtime-subscriptions/src/types.ts index bbf220ba4..37de46fb3 100644 --- a/graphile/graphile-realtime-subscriptions/src/types.ts +++ b/graphile/graphile-realtime-subscriptions/src/types.ts @@ -11,6 +11,25 @@ export interface RealtimeSubscriptionsPluginOptions { * Default: 50 */ overflowThreshold?: number; + + /** + * Receives the exact physical PostgreSQL notification topics compiled into + * this schema. The callback runs during schema construction, including with + * an empty list when no @realtime table was discovered. + * + * This is a build-time integration seam. It must not retain Graphile build + * objects or database resources; descriptors contain strings only. + */ + onTopicsDiscovered?: ( + topics: readonly RealtimeTopicDescriptor[] + ) => void; +} + +/** Credential-free description of one compiled @realtime channel. */ +export interface RealtimeTopicDescriptor { + readonly topic: string; + readonly schema: string; + readonly table: string; } /** @@ -28,6 +47,13 @@ export interface Queryable { ): Promise<{ rows: R[] }>; } +/** Explicit local delivery capability used by cursor catch-up. */ +export interface RealtimePublisher { + /** Optional batch preflight used to keep routing violations fail-closed. */ + assertTopics?(topics: readonly string[]): void; + publish(topic: string, payload: string): void; +} + /** * A single entry from drain_changes(), representing a change_log row * matched against subscriber tables. @@ -111,11 +137,31 @@ export interface CursorTrackerOptions { */ export interface RealtimeManagerOptions { /** - * The PgSubscriber instance from PostGraphile's context. - * RealtimeManager emits cursor-tracked events on its internal EventEmitter - * so they flow through existing subscription plans. + * Generation-local publisher used for cursor catch-up delivery. New callers + * should always provide this capability explicitly. + */ + publisher?: RealtimePublisher; + + /** + * Transitional compatibility input for the current @dataplan/pg + * PgSubscriber. Its private emitter is adapted outside RealtimeManager. + * @deprecated Provide publisher instead. + */ + pgSubscriber?: unknown; + + /** + * Exact physical schemas this Graphile instance exposes. Cursor rows naming + * any other source schema stop delivery and surface an error before any row + * in that batch is emitted. */ - pgSubscriber: unknown; + allowedSourceSchemas: readonly string[]; + + /** + * Called once when delivery can no longer be trusted, after new dispatch is + * disabled and manager shutdown has begun. Callers should synchronously + * remove the owning Graphile generation from service. + */ + onFatalError?: (error: Error) => void; /** * A query-capable object (typically a pg.Pool from pg-cache) used by @@ -160,8 +206,10 @@ export interface RealtimeManagerOptions { batchLimit?: number; /** - * Called when an error occurs during polling, heartbeat, or cleanup. - * If not provided, errors are logged via @pgpmjs/logger. + * Observes polling, heartbeat, or cleanup errors. A polling or heartbeat + * error after startup is independently treated as fatal and delivered to + * onFatalError because cursor recovery can no longer be guaranteed. + * If omitted, the error is logged via @pgpmjs/logger. */ onError?: (error: Error) => void; } diff --git a/graphql/server/package.json b/graphql/server/package.json index c63248828..b149a052e 100644 --- a/graphql/server/package.json +++ b/graphql/server/package.json @@ -67,6 +67,7 @@ "graphile-cache": "workspace:^", "graphile-config": "1.0.1", "graphile-function-bindings": "workspace:^", + "graphile-realtime-subscriptions": "workspace:^", "graphile-settings": "workspace:^", "graphile-utils": "5.0.1", "graphql": "16.13.0", diff --git a/graphql/server/src/middleware/__tests__/realtime-config.test.ts b/graphql/server/src/middleware/__tests__/realtime-config.test.ts new file mode 100644 index 000000000..26201a16c --- /dev/null +++ b/graphql/server/src/middleware/__tests__/realtime-config.test.ts @@ -0,0 +1,43 @@ +import type { ConstructiveOptions } from '@constructive-io/graphql-types'; + +import { + addRealtimeRuntimeDependencySchema, + resolveGraphileRealtimeSchema +} from '../realtime-config'; + +describe('Graphile realtime configuration', () => { + it('preserves the compatibility default only for enabled realtime surfaces', () => { + expect(resolveGraphileRealtimeSchema({} as ConstructiveOptions, true)).toBe( + 'realtime_public' + ); + expect(resolveGraphileRealtimeSchema({} as ConstructiveOptions, false)).toBeNull(); + }); + + it('preserves one exact configured cursor schema', () => { + const options = { + graphile: { realtimeSchema: 'tenant_a_realtime' } + } as ConstructiveOptions; + + expect(resolveGraphileRealtimeSchema(options, true)).toBe('tenant_a_realtime'); + }); + + it('rejects an empty configured schema when realtime is enabled', () => { + const options = { + graphile: { realtimeSchema: '' } + } as ConstructiveOptions; + + expect(() => resolveGraphileRealtimeSchema(options, true)).toThrow( + 'graphile.realtimeSchema must be one non-empty exact schema name' + ); + }); + + it('adds and deduplicates the cursor schema only in the runtime allowlist', () => { + expect(addRealtimeRuntimeDependencySchema( + ['extensions', 'tenant_a_realtime'], + 'tenant_a_realtime' + )).toEqual(['extensions', 'tenant_a_realtime']); + expect(addRealtimeRuntimeDependencySchema(['extensions'], null)).toEqual([ + 'extensions' + ]); + }); +}); diff --git a/graphql/server/src/middleware/__tests__/realtime-notification-config.test.ts b/graphql/server/src/middleware/__tests__/realtime-notification-config.test.ts new file mode 100644 index 000000000..c9bebec25 --- /dev/null +++ b/graphql/server/src/middleware/__tests__/realtime-notification-config.test.ts @@ -0,0 +1,129 @@ +import type { + ConstructiveOptions, + NotificationPgResolverInput +} from '@constructive-io/graphql-types'; + +import { + GraphileRealtimeNotificationConfigError, + resolveRealtimeCursorIntervals, + resolveRealtimeNotificationMode, + resolveRealtimeNotificationPgConfig, + resolveRealtimeNotificationRoleRevalidationMs +} from '../realtime-notification-config'; + +const route = { + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['tenant_a_public'] +}; + +describe('shared realtime notification configuration', () => { + it('defaults to the current dedicated subscriber and current cursor timings', () => { + const options = {} as ConstructiveOptions; + expect(resolveRealtimeNotificationMode(options)).toBe('dedicated'); + expect(resolveRealtimeNotificationRoleRevalidationMs(options)).toBe(60_000); + expect(resolveRealtimeCursorIntervals(options)).toEqual({ + pollIntervalMs: 5_000, + heartbeatIntervalMs: 30_000 + }); + }); + + it('accepts explicit shared mode and cursor timing contracts', () => { + const options = { + graphile: { + realtimeNotificationMode: 'shared-exact', + realtimeNotificationRoleRevalidationMs: 30_000, + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 90_000 + } + } as ConstructiveOptions; + + expect(resolveRealtimeNotificationMode(options)).toBe('shared-exact'); + expect(resolveRealtimeNotificationRoleRevalidationMs(options)).toBe(30_000); + expect(resolveRealtimeCursorIntervals(options)).toEqual({ + pollIntervalMs: 30_000, + heartbeatIntervalMs: 90_000 + }); + }); + + it('requires explicit listener credentials and never falls back to control credentials', async () => { + const options = { + pg: { + host: 'db.internal', + port: 5432, + database: 'control', + user: 'control_owner', + password: 'control-secret' + }, + notificationPgResolver: () => ({ + database: 'tenant_a', + user: 'tenant_a_notify' + }) + } as ConstructiveOptions; + + await expect(resolveRealtimeNotificationPgConfig(options, route)) + .rejects.toThrow('must return an explicit password'); + }); + + it('combines network defaults with one exact per-database listener identity', async () => { + const resolver = jest.fn((_input: Readonly) => ({ + database: 'tenant_a', + user: 'tenant_a_notify', + password: 'notification-secret', + pool: { max: 2 } + })); + const options = { + pg: { + host: 'db.internal', + port: 6432, + database: 'control', + user: 'control_owner', + password: 'control-secret', + ssl: true + }, + notificationPgResolver: resolver + } as ConstructiveOptions; + + await expect(resolveRealtimeNotificationPgConfig(options, route)).resolves.toEqual({ + host: 'db.internal', + port: 6432, + database: 'tenant_a', + user: 'tenant_a_notify', + password: 'notification-secret', + ssl: true, + pool: { max: 2 } + }); + const input = resolver.mock.calls[0][0]; + expect(input).toEqual(route); + expect(Object.isFrozen(input)).toBe(true); + expect(Object.isFrozen(input.schemas)).toBe(true); + }); + + it('rejects a resolver that routes to a different physical database', async () => { + const options = { + notificationPgResolver: () => ({ + database: 'tenant_b', + user: 'tenant_b_notify', + password: 'notification-secret' + }) + } as ConstructiveOptions; + + await expect(resolveRealtimeNotificationPgConfig(options, route)).rejects + .toBeInstanceOf(GraphileRealtimeNotificationConfigError); + }); + + it('rejects an ambiguous connection string even when explicit fields are present', async () => { + const options = { + notificationPgResolver: () => ({ + database: 'tenant_a', + user: 'tenant_a_notify', + password: 'explicit-secret', + connectionString: 'postgres://other:override@foreign/tenant_b' + }) + } as ConstructiveOptions; + + await expect(resolveRealtimeNotificationPgConfig(options, route)).rejects + .toThrow('must not return a connectionString'); + }); +}); diff --git a/graphql/server/src/middleware/realtime-config.ts b/graphql/server/src/middleware/realtime-config.ts new file mode 100644 index 000000000..eb5a24914 --- /dev/null +++ b/graphql/server/src/middleware/realtime-config.ts @@ -0,0 +1,27 @@ +import type { ConstructiveOptions } from '@constructive-io/graphql-types'; +import { DEFAULT_GRAPHILE_REALTIME_SCHEMA } from 'graphile-cache'; + +/** Resolve the exact cursor-function schema for one enabled Graphile surface. */ +export const resolveGraphileRealtimeSchema = ( + opts: ConstructiveOptions, + enableRealtime: boolean +): string | null => { + if (!enableRealtime) return null; + const configured = opts.graphile?.realtimeSchema; + if (configured === undefined) return DEFAULT_GRAPHILE_REALTIME_SCHEMA; + if (typeof configured !== 'string' || configured.length === 0) { + throw new Error('graphile.realtimeSchema must be one non-empty exact schema name'); + } + return configured; +}; + +/** Approve the cursor schema for runtime-role safety without exposing it. */ +export const addRealtimeRuntimeDependencySchema = ( + dependencySchemas: readonly string[], + realtimeSchema: string | null +): string[] => [ + ...new Set([ + ...dependencySchemas, + ...(realtimeSchema ? [realtimeSchema] : []) + ]) +]; diff --git a/graphql/server/src/middleware/realtime-notification-config.ts b/graphql/server/src/middleware/realtime-notification-config.ts new file mode 100644 index 000000000..531fcf345 --- /dev/null +++ b/graphql/server/src/middleware/realtime-notification-config.ts @@ -0,0 +1,144 @@ +import type { + ConstructiveOptions, + GraphileRealtimeNotificationMode, + NotificationPgResolverInput +} from '@constructive-io/graphql-types'; +import type { PgNotificationListenerConfig } from 'pg-cache'; +import { getPgEnvOptions } from 'pg-env'; + +export const GRAPHILE_REALTIME_NOTIFICATION_CONFIG_ERROR_CODE = + 'GRAPHILE_REALTIME_NOTIFICATION_CONFIG_INVALID'; + +export class GraphileRealtimeNotificationConfigError extends Error { + readonly code = GRAPHILE_REALTIME_NOTIFICATION_CONFIG_ERROR_CODE; + + constructor(message: string) { + super(message); + this.name = 'GraphileRealtimeNotificationConfigError'; + } +} + +export const resolveRealtimeNotificationMode = ( + options: ConstructiveOptions +): GraphileRealtimeNotificationMode => { + const mode = options.graphile?.realtimeNotificationMode ?? 'dedicated'; + if (mode !== 'dedicated' && mode !== 'shared-exact') { + throw new GraphileRealtimeNotificationConfigError( + 'graphile.realtimeNotificationMode must be dedicated or shared-exact' + ); + } + return mode; +}; + +export const resolveRealtimeNotificationRoleRevalidationMs = ( + options: ConstructiveOptions +): number => { + const value = options.graphile?.realtimeNotificationRoleRevalidationMs ?? 60_000; + if (!Number.isSafeInteger(value) || value <= 0) { + throw new GraphileRealtimeNotificationConfigError( + 'graphile.realtimeNotificationRoleRevalidationMs must be a positive safe integer' + ); + } + return value; +}; + +const positiveInterval = (value: number, setting: string): number => { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new GraphileRealtimeNotificationConfigError( + `${setting} must be a positive safe integer` + ); + } + return value; +}; + +export const resolveRealtimeCursorIntervals = ( + options: ConstructiveOptions +): { pollIntervalMs: number; heartbeatIntervalMs: number } => ({ + pollIntervalMs: positiveInterval( + options.graphile?.realtimeCursorPollIntervalMs ?? 5_000, + 'graphile.realtimeCursorPollIntervalMs' + ), + heartbeatIntervalMs: positiveInterval( + options.graphile?.realtimeCursorHeartbeatIntervalMs ?? 30_000, + 'graphile.realtimeCursorHeartbeatIntervalMs' + ) +}); + +/** + * Resolve one dedicated listener login without inheriting control-plane or + * runtime credentials. Network/TLS defaults may be shared, but user, password, + * and physical database must be explicit in every resolver result. + */ +export const resolveRealtimeNotificationPgConfig = async ( + options: ConstructiveOptions, + input: NotificationPgResolverInput +): Promise => { + const resolver = options.notificationPgResolver; + if (typeof resolver !== 'function') { + throw new GraphileRealtimeNotificationConfigError( + 'shared-exact realtime requires notificationPgResolver' + ); + } + + let resolved: Awaited>; + try { + resolved = await resolver(Object.freeze({ + databaseId: input.databaseId, + databaseName: input.databaseName, + apiId: input.apiId, + schemas: Object.freeze([...input.schemas]) + })); + } catch { + throw new GraphileRealtimeNotificationConfigError( + 'notificationPgResolver failed for the requested physical database' + ); + } + if (!resolved || typeof resolved !== 'object' || Array.isArray(resolved)) { + throw new GraphileRealtimeNotificationConfigError( + 'notificationPgResolver must return a PostgreSQL configuration object' + ); + } + if (Object.prototype.hasOwnProperty.call(resolved, 'connectionString')) { + throw new GraphileRealtimeNotificationConfigError( + 'notificationPgResolver must not return a connectionString; use explicit fields' + ); + } + if (typeof resolved.user !== 'string' || resolved.user.trim().length === 0) { + throw new GraphileRealtimeNotificationConfigError( + 'notificationPgResolver must return an explicit user' + ); + } + if (typeof resolved.password !== 'string' || resolved.password.length === 0) { + throw new GraphileRealtimeNotificationConfigError( + 'notificationPgResolver must return an explicit password' + ); + } + if (resolved.database !== input.databaseName) { + throw new GraphileRealtimeNotificationConfigError( + 'notificationPgResolver database does not match the routed physical database' + ); + } + + const networkDefaults = { + ...(options.pg?.host === undefined ? {} : { host: options.pg.host }), + ...(options.pg?.port === undefined ? {} : { port: options.pg.port }), + ...(options.pg?.ssl === undefined ? {} : { ssl: options.pg.ssl }) + }; + const normalized = getPgEnvOptions({ + ...networkDefaults, + ...resolved + }); + if ( + normalized.user !== resolved.user + || normalized.password !== resolved.password + || normalized.database !== input.databaseName + ) { + throw new GraphileRealtimeNotificationConfigError( + 'notification PostgreSQL identity changed during normalization' + ); + } + return { + ...normalized, + ...(resolved.pool ? { pool: { ...resolved.pool } } : {}) + }; +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 06e9c6501..2831e757f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2036,6 +2036,9 @@ importers: graphile-function-bindings: specifier: workspace:^ version: link:../../graphile/graphile-function-bindings/dist + graphile-realtime-subscriptions: + specifier: workspace:^ + version: link:../../graphile/graphile-realtime-subscriptions/dist graphile-settings: specifier: workspace:^ version: link:../../graphile/graphile-settings/dist