From 5fde64eb0b53524067835543e4457f7a5991232d Mon Sep 17 00:00:00 2001 From: yyyyaaa Date: Wed, 5 Aug 2026 09:27:33 +0700 Subject: [PATCH] Add tenant-density validation and production evidence --- .gitignore | 2 + packages/perf-harness/README.md | 210 + packages/perf-harness/jest.config.js | 11 + packages/perf-harness/package.json | 41 + .../src/__tests__/catalog-bench.test.ts | 1088 +++++ .../perf-harness/src/__tests__/config.test.ts | 875 ++++ .../src/__tests__/evidence.test.ts | 244 + .../perf-harness/src/__tests__/http.test.ts | 792 ++++ .../perf-harness/src/__tests__/memory.test.ts | 454 ++ .../src/__tests__/postgres.test.ts | 63 + .../src/__tests__/process.test.ts | 117 + .../src/__tests__/realtime-evidence.test.ts | 137 + .../src/__tests__/realtime.test.ts | 575 +++ .../perf-harness/src/__tests__/report.test.ts | 681 +++ .../src/__tests__/run-attestation.test.ts | 164 + .../perf-harness/src/__tests__/run.test.ts | 67 + .../perf-harness/src/__tests__/score.test.ts | 1754 ++++++++ packages/perf-harness/src/catalog-bench.ts | 3996 +++++++++++++++++ packages/perf-harness/src/config.ts | 1207 +++++ packages/perf-harness/src/evidence.ts | 1781 ++++++++ packages/perf-harness/src/http.ts | 888 ++++ packages/perf-harness/src/index.ts | 127 + packages/perf-harness/src/memory.ts | 650 +++ packages/perf-harness/src/postgres.ts | 466 ++ packages/perf-harness/src/process.ts | 434 ++ .../perf-harness/src/realtime-evidence.ts | 243 + packages/perf-harness/src/realtime.ts | 764 ++++ packages/perf-harness/src/report.ts | 1006 +++++ packages/perf-harness/src/run-attestation.ts | 364 ++ packages/perf-harness/src/run.ts | 856 ++++ packages/perf-harness/src/schedule.ts | 133 + packages/perf-harness/src/score.ts | 2634 +++++++++++ packages/perf-harness/src/types.ts | 962 ++++ packages/perf-harness/tsconfig.esm.json | 8 + packages/perf-harness/tsconfig.json | 8 + pnpm-lock.yaml | 64 + .../graphile-density/IMPLEMENTATION-NOTES.md | 44 + research/graphile-density/ORIGINAL-STACK.md | 16 + research/graphile-density/PLUGIN-SQL-AUDIT.md | 38 + research/graphile-density/RANGE-DIFF.md | 45 + research/graphile-density/REPORT.md | 163 + research/graphile-density/SECURITY-AUDIT.md | 154 + .../UNIFORM-DENSITY-FIXTURE.md | 63 + research/graphile-density/UPSTREAM-REVIEW.md | 51 + .../artifacts/scoped-introspection-smoke.json | 73 + .../complete-tenant-fixture/.gitignore | 3 + .../complete-tenant-fixture/README.md | 192 + .../coverage-manifest.json | 78 + .../generate-inputs.cjs | 260 ++ .../generate-inputs.test.cjs | 216 + .../hostile-validation.cjs | 589 +++ .../hostile-validation.test.cjs | 187 + .../complete-tenant-fixture/lib.cjs | 718 +++ .../complete-tenant-fixture/lib.test.cjs | 165 + .../qualification-runner.cjs | 350 ++ .../qualification-runner.test.cjs | 74 + .../complete-tenant-fixture/schema.sql | 709 +++ .../complete-tenant-fixture/schema.test.cjs | 95 + .../complete-tenant-fixture/server.cjs | 2007 +++++++++ .../complete-tenant-fixture/server.test.cjs | 543 +++ .../create-uniform-density-fixture.sql | 796 ++++ research/graphile-density/fleet.example.json | 82 + .../four-arm-plan.example.json | 121 + .../physical-database-density/.gitignore | 2 + .../HOSTILE-PREFLIGHT.md | 62 + .../physical-database-density/README.md | 100 + .../cache-calibration.cjs | 390 ++ .../cache-calibration.test.cjs | 146 + .../generate-inputs.cjs | 976 ++++ .../physical-database-density/inputs.test.cjs | 870 ++++ .../physical-database-density/lib.cjs | 1357 ++++++ .../physical-database-density/lib.test.cjs | 702 +++ .../measurement-attestation.cjs | 558 +++ .../measurement-attestation.test.cjs | 175 + .../physical-hostile-preflight.cjs | 831 ++++ .../physical-hostile-preflight.test.cjs | 658 +++ .../physical-identity.sql | 270 ++ .../prepare-measurement-run.cjs | 672 +++ .../prepare-measurement-run.test.cjs | 240 + .../provision-attestation.sql | 35 + .../physical-database-density/provision.cjs | 837 ++++ .../server-realtime.test.cjs | 85 + .../server-retained-memory.test.cjs | 320 ++ .../physical-database-density/server.cjs | 1299 ++++++ .../unsafe-runtime-startup-probe.cjs | 1130 +++++ .../unsafe-runtime-startup-probe.test.cjs | 435 ++ .../production-shaped-canary.sql | 22 + .../validate-uniform-density-fixture.sql | 342 ++ 88 files changed, 44212 insertions(+) create mode 100644 packages/perf-harness/README.md create mode 100644 packages/perf-harness/jest.config.js create mode 100644 packages/perf-harness/package.json create mode 100644 packages/perf-harness/src/__tests__/catalog-bench.test.ts create mode 100644 packages/perf-harness/src/__tests__/config.test.ts create mode 100644 packages/perf-harness/src/__tests__/evidence.test.ts create mode 100644 packages/perf-harness/src/__tests__/http.test.ts create mode 100644 packages/perf-harness/src/__tests__/memory.test.ts create mode 100644 packages/perf-harness/src/__tests__/postgres.test.ts create mode 100644 packages/perf-harness/src/__tests__/process.test.ts create mode 100644 packages/perf-harness/src/__tests__/realtime-evidence.test.ts create mode 100644 packages/perf-harness/src/__tests__/realtime.test.ts create mode 100644 packages/perf-harness/src/__tests__/report.test.ts create mode 100644 packages/perf-harness/src/__tests__/run-attestation.test.ts create mode 100644 packages/perf-harness/src/__tests__/run.test.ts create mode 100644 packages/perf-harness/src/__tests__/score.test.ts create mode 100644 packages/perf-harness/src/catalog-bench.ts create mode 100644 packages/perf-harness/src/config.ts create mode 100644 packages/perf-harness/src/evidence.ts create mode 100644 packages/perf-harness/src/http.ts create mode 100644 packages/perf-harness/src/index.ts create mode 100644 packages/perf-harness/src/memory.ts create mode 100644 packages/perf-harness/src/postgres.ts create mode 100644 packages/perf-harness/src/process.ts create mode 100644 packages/perf-harness/src/realtime-evidence.ts create mode 100644 packages/perf-harness/src/realtime.ts create mode 100644 packages/perf-harness/src/report.ts create mode 100644 packages/perf-harness/src/run-attestation.ts create mode 100644 packages/perf-harness/src/run.ts create mode 100644 packages/perf-harness/src/schedule.ts create mode 100644 packages/perf-harness/src/score.ts create mode 100644 packages/perf-harness/src/types.ts create mode 100644 packages/perf-harness/tsconfig.esm.json create mode 100644 packages/perf-harness/tsconfig.json create mode 100644 research/graphile-density/IMPLEMENTATION-NOTES.md create mode 100644 research/graphile-density/ORIGINAL-STACK.md create mode 100644 research/graphile-density/PLUGIN-SQL-AUDIT.md create mode 100644 research/graphile-density/RANGE-DIFF.md create mode 100644 research/graphile-density/REPORT.md create mode 100644 research/graphile-density/SECURITY-AUDIT.md create mode 100644 research/graphile-density/UNIFORM-DENSITY-FIXTURE.md create mode 100644 research/graphile-density/UPSTREAM-REVIEW.md create mode 100644 research/graphile-density/artifacts/scoped-introspection-smoke.json create mode 100644 research/graphile-density/complete-tenant-fixture/.gitignore create mode 100644 research/graphile-density/complete-tenant-fixture/README.md create mode 100644 research/graphile-density/complete-tenant-fixture/coverage-manifest.json create mode 100644 research/graphile-density/complete-tenant-fixture/generate-inputs.cjs create mode 100644 research/graphile-density/complete-tenant-fixture/generate-inputs.test.cjs create mode 100644 research/graphile-density/complete-tenant-fixture/hostile-validation.cjs create mode 100644 research/graphile-density/complete-tenant-fixture/hostile-validation.test.cjs create mode 100644 research/graphile-density/complete-tenant-fixture/lib.cjs create mode 100644 research/graphile-density/complete-tenant-fixture/lib.test.cjs create mode 100644 research/graphile-density/complete-tenant-fixture/qualification-runner.cjs create mode 100644 research/graphile-density/complete-tenant-fixture/qualification-runner.test.cjs create mode 100644 research/graphile-density/complete-tenant-fixture/schema.sql create mode 100644 research/graphile-density/complete-tenant-fixture/schema.test.cjs create mode 100644 research/graphile-density/complete-tenant-fixture/server.cjs create mode 100644 research/graphile-density/complete-tenant-fixture/server.test.cjs create mode 100644 research/graphile-density/create-uniform-density-fixture.sql create mode 100644 research/graphile-density/fleet.example.json create mode 100644 research/graphile-density/four-arm-plan.example.json create mode 100644 research/graphile-density/physical-database-density/.gitignore create mode 100644 research/graphile-density/physical-database-density/HOSTILE-PREFLIGHT.md create mode 100644 research/graphile-density/physical-database-density/README.md create mode 100644 research/graphile-density/physical-database-density/cache-calibration.cjs create mode 100644 research/graphile-density/physical-database-density/cache-calibration.test.cjs create mode 100644 research/graphile-density/physical-database-density/generate-inputs.cjs create mode 100644 research/graphile-density/physical-database-density/inputs.test.cjs create mode 100644 research/graphile-density/physical-database-density/lib.cjs create mode 100644 research/graphile-density/physical-database-density/lib.test.cjs create mode 100644 research/graphile-density/physical-database-density/measurement-attestation.cjs create mode 100644 research/graphile-density/physical-database-density/measurement-attestation.test.cjs create mode 100644 research/graphile-density/physical-database-density/physical-hostile-preflight.cjs create mode 100644 research/graphile-density/physical-database-density/physical-hostile-preflight.test.cjs create mode 100644 research/graphile-density/physical-database-density/physical-identity.sql create mode 100644 research/graphile-density/physical-database-density/prepare-measurement-run.cjs create mode 100644 research/graphile-density/physical-database-density/prepare-measurement-run.test.cjs create mode 100644 research/graphile-density/physical-database-density/provision-attestation.sql create mode 100644 research/graphile-density/physical-database-density/provision.cjs create mode 100644 research/graphile-density/physical-database-density/server-realtime.test.cjs create mode 100644 research/graphile-density/physical-database-density/server-retained-memory.test.cjs create mode 100644 research/graphile-density/physical-database-density/server.cjs create mode 100644 research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.cjs create mode 100644 research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.test.cjs create mode 100644 research/graphile-density/production-shaped-canary.sql create mode 100644 research/graphile-density/validate-uniform-density-fixture.sql diff --git a/.gitignore b/.gitignore index 60d96a6d7e..0f22ee4a51 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ postgres/pgsql-test/output/ .env.local graphql/server/logs/ graphql/server/*.heapsnapshot +graphile-density-artifacts/ +/research/graphile-density/artifacts/ # Ephemeral pgpm modules installed by `pnpm fixtures:install` (pgpm install) /extensions/ diff --git a/packages/perf-harness/README.md b/packages/perf-harness/README.md new file mode 100644 index 0000000000..c5fa156545 --- /dev/null +++ b/packages/perf-harness/README.md @@ -0,0 +1,210 @@ +# cperf Graphile density harness + +`cperf` is a local-only runner for the Graphile customer-density spike. It launches a fresh production-mode server process for each arm/heap/customer-count/repetition, warms every configured GraphQL surface with bounded concurrency and a fleet-size-scaled deadline, drives an open-loop workload, runs hostile isolation canaries, samples `/debug/memory` and the dedicated PostgreSQL container, and writes one timestamped JSON result per run plus an NDJSON ledger. + +The score is deliberately strict. A customer counts only when every declared surface warms, receives the configured minimum number of workload-phase requests, has an error rate of at most 0.5% and customer-workload p99 of at most 150 ms, runs all required capability operations, passes every required isolation canary conclusively with zero bleed, and sees no post-warmup Graphile or PostgreSQL pool eviction/refusal/build/disposal activity. Coverage probes prove operation support but do not contribute traffic, latency, or error samples. Results report customer workload RPS, periodic validation RPS, realtime validation RPS, and their combined HTTP RPS separately, so security probes cannot inflate customer throughput or pollute its latency percentiles. Runs shorter than 15 minutes always fail qualification, including `--smoke` runs. The JSON retains legacy `tenant*` aliases while the research interfaces migrate to customer terminology. + +## Capacity methodology + +The target is qualified complete customers per actual service memory unit, not fitting the fleet into a 1 GiB process. `tenantCountsByHeapMiB` retains its legacy name and supplies a different increasing customer ramp for each configured old-space size, while `tenantCounts` applies one ramp to every heap. A capacity result is complete only when all repetitions pass at one customer count and a greater count fails; an unbracketed last successful checkpoint is reported as observed capacity, not maximum capacity. + +Use exactly one load mode. `rps` holds total offered load fixed as tenants are added, which isolates memory capacity but reduces per-tenant traffic; `rpsPerTenant` holds per-tenant load fixed, so total offered load grows with the fleet. Every result records the resolved total and per-tenant load. `minWorkloadRequestsPerSurface` prevents a tenant from qualifying without representative traffic, and `warmupTimeoutPerSurfaceMs` scales the warmup allowance by the number of bounded-concurrency waves in addition to `warmupTimeoutMs`. + +Runs are deterministically interleaved across arms using `runOrderSeed`, so repeated experiments reproduce the order without always favoring the same arm. Each invocation also creates a random campaign ID and an immutable manifest for that exact ordered schedule. Per-campaign results form a forward SHA-256 chain, and report validation requires the manifest order, non-overlapping chronology, common runtime platform, and every chain pointer to agree; separately collected records cannot be spliced into qualification evidence. Results record the plan and fleet hashes, runtime versions, order, and resolved memory-governor policy. The runner also persists request, canary, memory, PostgreSQL, and workload-progress artifacts when a run fails partway through, so a failed capacity point remains diagnosable. + +Spawned Node arms may select only `v8Profile: stock`, `optimize-for-size`, `baseline-optimize-for-size`, or `jitless-optimize-for-size`. The baseline-size profile uses `--max-opt=1 --optimize-for-size`, retaining Sparkplug while excluding the higher optimization tiers. The runner injects the exact allowlisted flags, strips inherited copies, rejects managed flags hidden in the command or plan `NODE_OPTIONS`, and records the profile, sanitized `NODE_OPTIONS`, direct Node arguments, and their effective ordered combination in provenance. Every non-stock profile is an explicit candidate and must pass the full loaded density curve, p99, throughput, and isolation gates. + +`periodicCanarySchedule` defaults to the legacy `full-sweep` behavior. The `rotating-one` mode executes one deterministically staggered canary per tenant/surface in every timed round while retaining full initial and final sweeps. Timed rounds occupy only interval slots strictly before the workload deadline, so a 900-second run at a 60-second interval has exactly 14 rounds. Rounds are serialized and never dropped when one overlaps the next slot; each overlap, incomplete round, and deadline-late completion is recorded in `canary-schedule.json`. `canaryConcurrency` bounds parallelism across surfaces while every surface's probes stay sequential. A qualifying rotating plan should enable `requireCompletePeriodicCanaryCoverage`, which requires exact boundary sweeps, one exact result per selected target/round, complete configured-canary coverage, and every periodic round to finish by the workload deadline. + +Realtime GraphQL routes use the same strict timed-workload boundary. After one initial correlated mutation/subscription delivery, the driver schedules a fresh delivery in every 60-second slot strictly before the workload deadline, serializes rounds, and persists credential-free correlation receipts for every exact tenant/surface route. The report derives counts, globally unique ordered digests, prime-request volume, prime-response p99, and delivery p99 from those raw receipts instead of trusting the summary fields; append-only histories and the single timed-coverage completion transition are also verified. A late, missed, reused, or unverified recurring round sets qualified customers to zero even when the final post-workload probe succeeds, so a healthy connection at the two bookends cannot conceal a subscription that stopped delivering during the workload. + +Every persisted v6 result carries a SHA-256 binding over its complete result payload and the exact memory, PostgreSQL, request, canary, canary-schedule, retained-memory, workload-progress, realtime, and score-context evidence files. The credential-free score context binds the plan/fleet hashes and the few run facts that cannot be reconstructed from those raw files; workload load and warmup limits are re-derived from the plan and fleet. The report loader verifies every bound file, reconstructs the complete `scoreRun` input, reruns the scorer, and requires byte-equivalent result semantics, so recomputing public hashes cannot bless a hand-edited result. Soak records render in a separate section and never enter matrix medians, capacity boundaries, or candidate comparisons. + +An arm may declare `envByHeapMiB` to override its base environment for each configured heap. When present, it must contain exactly every plan heap and only string values. This is the intended path for measured Graphile governor calibration: each checkpoint can pin its cache ceiling, instance estimate, server/build reserves, RSS build reserve, and `GRAPHILE_CACHE_CALIBRATION_ID`, and qualifying physical-database runs verify that the live cache reports that identity and enough configured/budget capacity for every requested surface. Physical fixture cache keys are process-random keyed HMACs, so cross-process scoring compares the fleet against the fixture's credential-free Graphile contract fingerprints; the live keys remain in same-process guard state to prove that no resident entry changed during a run. + +The primary report metric is qualified customers divided by the maximum post-warmup time-aligned sum of current Node RSS and the dedicated PostgreSQL container's raw cgroup-v2 `memory.current` charge. On Linux, exact-process current RSS comes from `/proc` at 100 ms; on non-Linux diagnostic runs it comes from a bearer-authenticated loopback endpoint at 250 ms. Publication-quality qualification still requires Linux and cgroup v2. The runner pairs timestamped Node and PostgreSQL samples within one second and fails qualification when aligned service telemetry is unavailable. It also reports a conservative non-simultaneous upper bound—Node RSS high-water plus PostgreSQL peak—and retains Docker working set, configured old-space, and Node-only peak-RSS density as diagnostics. Candidate acceptance requires a complete paired matrix, the configured additional customers at every heap, the configured median improvement in both actual service-memory measures, and no per-heap regression in either of those measures; the heap sizes are measurement points rather than capacity targets. + +Smoke results also include `configuredCustomersPerAlignedServiceGiB` and `configuredCustomersPerServiceMemoryUpperBoundGiB`. These diagnostic fields make a short fully warmed mechanics run numerically useful, but they use configured rather than qualified customers and never participate in acceptance; `customersPerAlignedServiceGiB` remains zero until the full duration, traffic, correctness, isolation, and residency gates pass. + +Physical qualification arms must configure a prepare command that creates one fresh PostgreSQL fixture under the current run artifact directory before the measured Node process starts. The following audit binds the exact matrix coordinate, plan/fleet hashes, Docker image and resource/command configuration, cgroup-v2 identity, PostgreSQL system identifier and start time, exact database inventory, unique clone/nonce set, and recomputed live DDL/ACL/role/extension contracts. The sampler resolves the mutable Docker name once, pins the attested 64-character container ID for every `stats` and cgroup read, and revalidates the start time and cgroup identity after the final sample. The server then receives the attested manifest path, manifest hash, and clone ID as resolved command templates. Reusing or replacing any container, cluster, clone, attestation set, or nonce-set identity fails both the in-process schedule and cross-file report aggregation. + +Measured GraphQL canaries remain request-path correctness evidence; they are not a substitute for an induced hostile campaign. A qualifying plan must bind one immutable `exact-runtime-hostile-validation-v1` artifact per arm, including the exact runtime-artifact and configuration fingerprints. If those artifacts are absent or mismatched, the runner labels the entire campaign diagnostic and the report refuses to promote it, even when every passive canary passed. + +That full structural audit intentionally reads PostgreSQL catalogs before Graphile starts, so reported build latency is a post-attestation warm-catalog measurement. It is comparable across equally audited arms, but it is not evidence for a pristine-catalog cold start. + +## Commands + +```bash +pnpm --filter @constructive-io/perf-harness build + +node packages/perf-harness/dist/index.js validate --plan path/to/completed-plan.json + +node packages/perf-harness/dist/index.js run \ + --plan research/graphile-density/four-arm-plan.example.json \ + --smoke --arm scoped-introspection + +node packages/perf-harness/dist/index.js report \ + --plan research/graphile-density/four-arm-plan.example.json \ + --results graphile-density-artifacts/results.ndjson \ + --out graphile-density-artifacts/report.md +``` + +### Catalog cache-warmth benchmark + +Scoped runs accept `--scoped-catalog-types all|dependency-closure`. The default +is `all`, which preserves the current scoped-required query; the experimental +`dependency-closure` arm retains only catalog types reached by the requested +schemas' object closure. The flag is rejected for stock mode, and its value is +recorded in worker configs, results, summaries, provenance, and cache build +identities so the two scoped arms cannot share a Graphile instance. + +`--release-build-state-after-validation` enables the opt-in lifecycle candidate +for `catalog-bench`. Its boolean value is written to the worker config, progress, +result, summary, provenance, Graphile preset, and cache identity; omitting the +flag always measures the default retained-build-state behavior. + +`--introspection-client-release-mode reuse|destroy` selects how the PostgreSQL +checkout used for catalog introspection is released and defaults to `reuse`. +In `destroy` mode the worker proves the full PID plus SQL `backend_start` +identity has disappeared through a separate control connection before it +acquires a replacement; token canaries and cache-warm operations must then +leave that replacement identity unchanged. Snapshot RSS is the steady +replacement backend's RSS and its delta is relative to replacement acquisition. + +`--postgres-backend-sampler off|diagnostic-lower-bound` defaults to +`diagnostic-lower-bound` and gives paired sampler-on/off runs for quantifying +observer cost. Before each destroy-mode build, the fixed external sampler binds +the SQL `backend_start` to `/proc//stat` within an explicit 1.5-second +boot-time tolerance, then revalidates the immutable proc start token, PostgreSQL +process name, and PID namespace identity on every 10 ms sample. A Linux Docker +host prefers the container's procfs through host procfs; the fallback pins one +`docker exec` to the inspected 64-character container ID and revalidates the +name, ID, start time, and init PID after sampling. The fallback starts +`/usr/bin/env -i` with a path-only shell environment, but the initial Docker +exec process may briefly inherit the container's configured environment before +`env -i` clears it. Artifacts record only the allowlisted host variable names, +never their values. + +The worker traps shell exits, stops the sampler process group with bounded +graceful, TERM, and KILL phases, and awaits tree closure before backend +retirement. Even a cadence-complete `VmRSS`/`VmHWM` trace is a diagnostic lower +bound because Graphile has no pre-destroy acknowledgement guaranteeing a final +sample; artifacts never promote it to an exact peak or density authority. +Sampler launch and shutdown time are recorded without subtracting a correction. +Service-density authority remains the separately validated Linux cgroup-v2 +`memory.current` measurement, while Docker Desktop backend traces carry an +additional VM-boundary limitation. + +`--v8-profile stock|optimize-for-size|baseline-optimize-for-size|jitless-optimize-for-size` selects the +worker's named V8 configuration and defaults to `stock`. The parent sanitizes +inherited managed flags, launches the worker with the profile's exact direct +Node arguments, and pins `--heap-mib` through `NODE_OPTIONS`. Worker config, +progress, result, summary, and provenance artifacts record the selected profile, +the sanitized `NODE_OPTIONS`, its tokenization, `process.execArgv`, and the +effective ordered combination; a mismatch fails the run. Starting the parent +Node process with an optimization flag is not evidence that workers inherited +it, so benchmark comparisons must select the profile explicitly through this +flag. The baseline and jitless profiles remain opt-in candidates and must pass the same +loaded latency, throughput, and isolation gates as stock. + +`catalog-bench` can populate every resident schema's Grafast parse/query and +operation-plan caches with reproducible, distinct named operations. A nonzero +`--warm-operations-per-instance` requires one exact `--expected-tokens` value +per schema; every operation executes through `grafast({ source })`, and the +artifact records p50/p99 population latency plus conclusive token correctness. +`--warm-operation-replay-passes N` then replays that exact ordered source set +through Grafast `N` times for each instance. Replay execution counts, p50/p99 +latency, errors, exact-token correctness, mismatches, and cross-tenant results +are recorded separately from population, so cache-limit comparisons do not +mix cold source admission with cache-hit or cache-churn behavior. Replay is +disabled by default, and a positive pass count requires a nonempty population +set from `--warm-operations-per-instance`. +The three cache-limit flags are independently optional. Omitting all three uses +Grafast's defaults, while providing them installs the shared +`createGrafastCacheLimitsPreset` before schema construction. + +`--tenant-proxy-surfaces N` adds an explicitly synthetic density projection to +the parent `summary.json`; it does not change the worker or turn these fixtures +into measured complete tenants. The projection divides resident surface +instances into full groups of `N`, records any remainder, and reports group +density against both the configured `--max-old-space-size` GiB and the absolute +lifetime process peak-RSS GiB. It never uses baseline-relative RSS or the +per-instance slope as the peak-RSS denominator. Because the final checkpoint is +a scheduled stop rather than a discovered memory boundary, the summary records +`capacityBoundaryReached: false` and must be read as an observed synthetic +checkpoint, not maximum customer capacity. + +These four commands reproduce the default-versus-all8 comparison at 100 and +500 operations for the disposable density fixture, assuming the `PG*` +environment variables already select its least-privilege runtime login: + +```bash +node packages/perf-harness/dist/index.js catalog-bench --database graphile_density_20260801_a --mode scoped-required --schemas gd_t001_api --instances 1 --expected-tokens tenant-001-token --warm-operations-per-instance 100 --warm-operation-replay-passes 3 --heap-mib 2048 --repetitions 3 --postgres-container postgres --out /tmp/cperf-cache-default-100 + +node packages/perf-harness/dist/index.js catalog-bench --database graphile_density_20260801_a --mode scoped-required --schemas gd_t001_api --instances 1 --expected-tokens tenant-001-token --warm-operations-per-instance 500 --warm-operation-replay-passes 3 --heap-mib 2048 --repetitions 3 --postgres-container postgres --out /tmp/cperf-cache-default-500 + +node packages/perf-harness/dist/index.js catalog-bench --database graphile_density_20260801_a --mode scoped-required --schemas gd_t001_api --instances 1 --expected-tokens tenant-001-token --warm-operations-per-instance 100 --warm-operation-replay-passes 3 --grafast-query-cache-max 8 --grafast-operations-cache-max 8 --grafast-operation-plans-cache-max 8 --heap-mib 2048 --repetitions 3 --postgres-container postgres --out /tmp/cperf-cache-all8-100 + +node packages/perf-harness/dist/index.js catalog-bench --database graphile_density_20260801_a --mode scoped-required --schemas gd_t001_api --instances 1 --expected-tokens tenant-001-token --warm-operations-per-instance 500 --warm-operation-replay-passes 3 --grafast-query-cache-max 8 --grafast-operations-cache-max 8 --grafast-operation-plans-cache-max 8 --heap-mib 2048 --repetitions 3 --postgres-container postgres --out /tmp/cperf-cache-all8-500 +``` + +With no warmth or cache-limit flags, the command retains its prior behavior and +does not install a cache-limit preset. Each build now also records an +approximate transient heap/RSS peak sampled every 5 ms from an immediately +preceding forced-GC resident baseline. The process RSS high-water is recorded +as a backstop, but synchronous event-loop work can still hide a short heap peak, +so this number is a measured reserve input rather than an exact maximum. +`--heap-mib` configures Node's old-space flag; the worker records V8's effective +total heap limit separately because the two values are not interchangeable. + +The legacy `--schemas a,b --instances 1,2` form still means one schema per +resident instance. To measure one production-shaped surface that exposes an +ordered schema set, use `--surface-schemas` with exactly one instance and an +explicit `--allowed-dependency-schemas` list: + +```bash +node packages/perf-harness/dist/index.js catalog-bench \ + --database production_shape \ + --mode scoped-required \ + --scoped-catalog-types dependency-closure \ + --surface-schemas app_public,app_auth,app_users \ + --allowed-dependency-schemas app_extensions,jwt_private \ + --instances 1 \ + --heap-mib 2048 \ + --repetitions 3 \ + --out /tmp/cperf-production-shape-scoped +``` + +Both ordered lists are validated, included in the worker config and provenance, +and hashed into the Graphile build and fixture identities. The exposed list must +be nonempty; an explicitly supplied empty dependency value is retained as `[]` +and remains distinct from an omitted flag. Names must be unique within each +list and the lists must be disjoint, and a manually edited worker config fails +closed under the same checks. + +The checked-in example plan and fleet are deliberately incomplete placeholders, so `validate` rejects them until they are copied and filled with real tenant routes, queries, tokens, credentials, and separate worktree paths. Spawned arms must pin a commit. Each run verifies and records the actual Git HEAD and dirty state, command, working directory, entry and lockfile hashes, server PID, and effective V8 heap limit; optional `entrySha256` and `lockfileSha256` plan pins make mismatches fail before traffic starts. + +`postgresContainer` is sampled from raw cgroup-v2 `memory.current`, `memory.peak`, `memory.stat`, and `memory.events` at 250 ms when available; Docker working set is sampled separately at a lower frequency as a diagnostic. The recorded cold-build spike is the greatest sampled raw charge before the post-warmup boundary minus the first raw sample, and it is meaningful only when the container is dedicated to the arm. Backend process RSS is never summed because PostgreSQL processes share pages; backend and concrete pool-client counts are reported separately instead. The aligned service metric is meaningful only when that PostgreSQL container is dedicated to the measured Node process and no unrelated workload runs in either boundary. + +## Fleet contract + +A fleet contains customers (the legacy JSON key remains `tenants`), each with one or more named surfaces. A qualifying fleet also declares the exact customer → logical database → API topology: stable database/API IDs, a credential-free physical database label, ordered physical schemas, opaque credential-sensitive runtime-pool identities, and the surface names served by each API. Validation requires every surface to appear exactly once and rejects build-contract or runtime-pool identities reused across customers, so host labels and instance counts cannot be mistaken for customer isolation or density. + +Every surface defines a warmup query, weighted workload operations tagged with the capability they actually exercise, and isolation canaries. Operations may use typed `requiredMatches` and `forbiddenMatches` response oracles; a matching forbidden value produces `GRAPHQL_OPERATION_ORACLE_FORBIDDEN`, while a successful response missing required evidence produces `GRAPHQL_OPERATION_ORACLE_MISSING`. Wildcard-capable `invariants` add an exhaustive `everyEquals` assertion with positive `min` and optional `max`, so an empty collection or a later foreign row cannot pass after checking node zero. Transport and GraphQL failures remain inside the configured 0.5% error budget and are marked oracle-unavailable, while missing or unexpected evidence in a successful response still fails immediately and every operation must have exactly one conclusive coverage result. + +Mutations may declare an untimed `postCoverageVerification` query with the same oracle contract. `variablesFromResponse` binds each verification variable to exactly one JSON pointer from the primary response; a missing or ambiguous extraction fails closed before the verification query runs. This lets side-effect checks correlate by the ID returned by the current mutation instead of accepting a stale row selected by a reusable content hash. `requireConclusiveOperationOracles` rejects the plan unless every warmup and operation has direct or post-coverage evidence, and rejects the run on missing or foreign evidence without changing the production GraphQL API. + +Canaries use the same RFC 6901 JSON `path` plus exact JSON `value` model; point them at customer-specific result fields so unrelated strings elsewhere in a response cannot trigger or satisfy an isolation check. A realtime surface also declares its exact subscription and prime mutation with permanent required/forbidden identity invariants plus correlation paths. The driver replaces the declared prime variable with a fresh opaque nonce for every delivery round and requires that exact nonce in both the mutation response and subscription event, so a stale cursor replay cannot satisfy recurring coverage. Artifacts contain only ordered SHA-256 bindings of issued and verified nonces. Those clients remain in the driver rather than the measured server process, and sensitive HTTP/websocket headers are resolved from declared environment-variable names without entering the fleet or artifacts. + +If arms produce different cache identities, set `buildContracts` on each surface +with one exact hash per arm name. Validation rejects a partial mapping, and the +runner selects only the current arm's hash; a stock identity therefore cannot +silently satisfy a scoped run (or vice versa). + +Capabilities and canaries are plan-level allowlists. A run fails unless every tenant serves every configured operation and capability on its configured surface, every tenant covers every required capability, and every surface runs every required canary. The production plan should require generated Graphile plans plus i18n, LLM/RAG, BM25, tsvector, trigram, vector, PostGIS, ltree, uploads/storage, bulk mutations, realtime, and function bindings. It should also require cross-schema identifiers, metadata, functions, sequences, prepared-statement reuse, poisoned GUCs, rollback/savepoints, plugin raw SQL, owner/BYPASS-role probes, schema drift, cache invalidation, concurrent builds, and connection reuse. + +## Safety + +Ports 3000–3002, 5432, and 9000 are rejected unless `--allow-reserved-ports` is explicit. For each spawned process, cperf generates a fresh strong observability token and sends it only as an `Authorization` header to the loopback memory endpoint; the token is never put in a URL, log, provenance record, or artifact. Server credentials stay in the inherited environment and are never serialized into result files. An arm without a launch command is treated as an external reused server and can produce diagnostics, but it cannot qualify because the cache and process boundary are not fresh. Missing endpoint fields remain `null` and disqualify the run instead of becoming zero-valued measurements. + +This harness never provisions or modifies `constructive-db`. Fixture creation belongs in a disposable PostgreSQL database or an independently managed validation environment. diff --git a/packages/perf-harness/jest.config.js b/packages/perf-harness/jest.config.js new file mode 100644 index 0000000000..363ad24f79 --- /dev/null +++ b/packages/perf-harness/jest.config.js @@ -0,0 +1,11 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }] + }, + testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + modulePathIgnorePatterns: ['dist/*'] +}; diff --git a/packages/perf-harness/package.json b/packages/perf-harness/package.json new file mode 100644 index 0000000000..9e8e4d7e7d --- /dev/null +++ b/packages/perf-harness/package.json @@ -0,0 +1,41 @@ +{ + "name": "@constructive-io/perf-harness", + "version": "0.2.0", + "private": true, + "description": "Local Graphile tenant-density and isolation validation harness", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "bin": { + "cperf": "index.js" + }, + "scripts": { + "clean": "makage clean", + "build": "makage build", + "build:dev": "makage build --dev", + "lint": "eslint . --fix", + "test": "jest --passWithNoTests" + }, + "dependencies": { + "grafast": "1.0.2", + "graphile-build-pg": "5.0.2", + "graphile-cache": "workspace:^", + "graphile-settings": "workspace:^", + "graphql": "16.13.0", + "graphql-ws": "^6.0.8", + "pg": "^8.21.0", + "pg-env": "workspace:^", + "ws": "^8.20.0" + }, + "devDependencies": { + "@types/node": "^22.19.11", + "@types/pg": "^8.20.0", + "@types/ws": "^8.18.1", + "makage": "^0.3.0", + "ts-node": "^10.9.2" + }, + "engines": { + "node": ">=22" + }, + "license": "MIT" +} diff --git a/packages/perf-harness/src/__tests__/catalog-bench.test.ts b/packages/perf-harness/src/__tests__/catalog-bench.test.ts new file mode 100644 index 0000000000..6bf2c51839 --- /dev/null +++ b/packages/perf-harness/src/__tests__/catalog-bench.test.ts @@ -0,0 +1,1088 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { parse } from 'graphql'; + +import { + assertCatalogDockerContainerIdentity, + catalogBackendSamplerEnvironment, + catalogIntrospectionBuildIdentity, + type CatalogMemorySnapshot, + catalogPercentile, + catalogProgressPath, + catalogSchemaContractIdentity, + makeCatalogBackendSamplerLaunchSpec, + makeCatalogWarmOperationSource, + measureCatalogBuildWithBackendSampler, + parseCatalogBackendProcStatus, + parseCatalogBackendSamplerMode, + parseCatalogBuildStateRetirement, + parseCatalogDockerContainerIdentity, + parseCatalogIntrospectionClientReleaseMode, + parseCatalogSchemaLayout, + parseCatalogScopedCatalogTypes, + parseCatalogTenantProxySurfaces, + parseCatalogV8Profile, + parseCatalogWarmthCliOptions, + projectCatalogTenantDensity, + resolveCatalogBackendPidAfterBuild, + resolveCatalogSchemaLayout, + stopCatalogBackendSamplerProcessTree, + summarizeBuildTransientSamples, + summarizeCatalogBackendMemorySamples, + validateCatalogPostgresContainer, + validateCatalogRuntimeFlags, + validateCatalogWarmthConfig, + writeCatalogProgress +} from '../catalog-bench'; + +describe('catalog benchmark schema layout', () => { + it('preserves the legacy one-schema-per-instance CLI shape', () => { + expect(parseCatalogSchemaLayout([ + '--schemas', 'gd_t001_api,gd_t002_api' + ], 2)).toEqual({ + schemas: ['gd_t001_api', 'gd_t002_api'], + schemaSets: null, + allowedDependencySchemas: null + }); + }); + + it('parses one ordered multi-schema surface and explicit dependency closure', () => { + const layout = parseCatalogSchemaLayout([ + '--surface-schemas', 'app_public,app_auth,app_users', + '--allowed-dependency-schemas', 'app_extensions,jwt_private' + ], 1); + expect(layout).toEqual({ + schemas: ['app_public'], + schemaSets: [['app_public', 'app_auth', 'app_users']], + allowedDependencySchemas: ['app_extensions', 'jwt_private'] + }); + expect(resolveCatalogSchemaLayout({ + schemas: layout.schemas, + schemaSets: layout.schemaSets!, + allowedDependencySchemas: layout.allowedDependencySchemas!, + checkpoints: [1] + })).toEqual(layout); + expect(parseCatalogSchemaLayout([ + '--surface-schemas', 'app_public,app_auth', + '--allowed-dependency-schemas', '' + ], 1)).toEqual({ + schemas: ['app_public'], + schemaSets: [['app_public', 'app_auth']], + allowedDependencySchemas: [] + }); + }); + + it.each([ + [ + ['--surface-schemas', 'app_public,app_auth'], + 1, + 'requires --allowed-dependency-schemas' + ], + [ + [ + '--surface-schemas', 'app_public,,app_auth', + '--allowed-dependency-schemas', 'app_extensions' + ], + 1, + 'must not contain empty schema names' + ], + [ + [ + '--surface-schemas', 'app_public,app_public', + '--allowed-dependency-schemas', 'app_extensions' + ], + 1, + 'must contain unique schema names' + ], + [ + [ + '--surface-schemas', 'app_public,app_auth', + '--allowed-dependency-schemas', 'app_auth' + ], + 1, + 'must not overlap' + ], + [ + [ + '--surface-schemas', 'app_public,app_auth', + '--allowed-dependency-schemas', 'app_extensions' + ], + 2, + 'requires exactly one resident instance' + ], + [ + ['--schemas', 'app_public', '--surface-schemas', 'app_auth'], + 1, + 'mutually exclusive' + ], + [ + ['--schemas', 'app_public', '--allowed-dependency-schemas', 'app_extensions'], + 1, + 'requires --surface-schemas' + ] + ])('rejects ambiguous schema layout %#', (args, instances, message) => { + expect(() => parseCatalogSchemaLayout( + args as string[], + instances as number + )).toThrow(message as string); + }); + + it('rejects manually edited worker layouts and hashes both ordered lists', () => { + expect(() => resolveCatalogSchemaLayout({ + schemas: ['app_auth'], + schemaSets: [['app_public', 'app_auth']], + allowedDependencySchemas: ['app_extensions'], + checkpoints: [1] + })).toThrow('schemas[0] must equal the first ordered'); + expect(() => resolveCatalogSchemaLayout({ + schemas: ['app_public'], + schemaSets: [['app_public', 'app_auth']], + allowedDependencySchemas: ['app_extensions', 'app_extensions'], + checkpoints: [1] + })).toThrow('must contain unique schema names'); + + const identity = catalogSchemaContractIdentity( + ['app_public', 'app_auth'], + ['app_extensions', 'jwt_private'] + ); + expect(identity).not.toBe(catalogSchemaContractIdentity( + ['app_auth', 'app_public'], + ['app_extensions', 'jwt_private'] + )); + expect(identity).not.toBe(catalogSchemaContractIdentity( + ['app_public', 'app_auth'], + ['jwt_private', 'app_extensions'] + )); + expect(catalogSchemaContractIdentity(['app_public'], [])).not.toBe( + catalogSchemaContractIdentity(['app_public'], ['app_extensions']) + ); + }); +}); + +describe('catalog benchmark scoped catalog type policy', () => { + it('preserves all catalog types as the scoped default', () => { + expect(parseCatalogScopedCatalogTypes([], 'scoped-required')).toBe('all'); + expect(parseCatalogScopedCatalogTypes([], 'stock')).toBeNull(); + }); + + it('parses the dependency-closure experiment strictly', () => { + expect(parseCatalogScopedCatalogTypes([ + '--scoped-catalog-types', 'dependency-closure' + ], 'scoped-required')).toBe('dependency-closure'); + }); + + it.each([ + [['--scoped-catalog-types'], 'scoped-required', 'requires a value'], + [[ + '--scoped-catalog-types', 'all', + '--scoped-catalog-types', 'dependency-closure' + ], 'scoped-required', 'may only be specified once'], + [['--scoped-catalog-types', 'closure'], 'scoped-required', "must be 'all' or 'dependency-closure'"], + [['--scoped-catalog-types', 'all'], 'stock', 'requires --mode scoped-required'] + ])('rejects malformed catalog policy arguments %j', (args, mode, message) => { + expect(() => parseCatalogScopedCatalogTypes( + args as string[], + mode as 'stock' | 'scoped-required' + )).toThrow(message as string); + }); + + it('separates all-types and dependency-closure build identities', () => { + const all = catalogIntrospectionBuildIdentity('scoped-required', 'all'); + const closure = catalogIntrospectionBuildIdentity( + 'scoped-required', + 'dependency-closure' + ); + + expect(all).not.toBe(closure); + expect(catalogIntrospectionBuildIdentity('stock', null)).not.toBe(all); + expect(catalogIntrospectionBuildIdentity( + 'scoped-required', + 'dependency-closure', + true + )).not.toBe(closure); + expect(catalogIntrospectionBuildIdentity( + 'scoped-required', + 'dependency-closure', + false, + 'destroy' + )).not.toBe(closure); + }); + + it('keeps exact introspection client destruction explicitly opt-in', () => { + expect(parseCatalogIntrospectionClientReleaseMode([])).toBe('reuse'); + expect(parseCatalogIntrospectionClientReleaseMode([ + '--introspection-client-release-mode', 'destroy' + ])).toBe('destroy'); + }); + + it.each([ + [['--introspection-client-release-mode'], 'requires a value'], + [[ + '--introspection-client-release-mode', 'reuse', + '--introspection-client-release-mode', 'destroy' + ], 'may only be specified once'], + [[ + '--introspection-client-release-mode', 'discard' + ], "must be 'reuse' or 'destroy'"] + ])('rejects malformed introspection release arguments %j', (args, message) => { + expect(() => parseCatalogIntrospectionClientReleaseMode(args)).toThrow(message); + }); + + it('keeps build-state retirement explicitly opt-in', () => { + expect(parseCatalogBuildStateRetirement([])).toBe(false); + expect(parseCatalogBuildStateRetirement([ + '--release-build-state-after-validation' + ])).toBe(true); + expect(() => parseCatalogBuildStateRetirement([ + '--release-build-state-after-validation', + '--release-build-state-after-validation' + ])).toThrow('may only be specified once'); + }); +}); + +describe('catalog benchmark V8 runtime provenance', () => { + it('defaults to stock and parses only the named profiles', () => { + expect(parseCatalogV8Profile([])).toBe('stock'); + expect(parseCatalogV8Profile([ + '--v8-profile', 'optimize-for-size' + ])).toBe('optimize-for-size'); + expect(parseCatalogV8Profile([ + '--v8-profile', 'baseline-optimize-for-size' + ])).toBe('baseline-optimize-for-size'); + expect(parseCatalogV8Profile([ + '--v8-profile', 'jitless-optimize-for-size' + ])).toBe('jitless-optimize-for-size'); + }); + + it.each([ + [['--v8-profile'], 'requires a value'], + [[ + '--v8-profile', 'stock', + '--v8-profile', 'optimize-for-size' + ], 'may only be specified once'], + [['--v8-profile', 'jitless'], "v8Profile must be 'stock'"] + ])('rejects malformed V8 profile arguments %j', (args, message) => { + expect(() => parseCatalogV8Profile(args as string[])).toThrow(message as string); + }); + + it('proves the exact configured and observed worker flags', () => { + const runtime = { + heapMiB: 1024, + v8Profile: 'jitless-optimize-for-size' as const, + nodeOptions: '--max-old-space-size=1024', + nodeOptionsArgv: ['--max-old-space-size=1024'], + nodeExecArgv: ['--jitless', '--optimize-for-size', '--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--max-old-space-size=1024', + '--jitless', + '--optimize-for-size', + '--expose-gc' + ] + }; + + expect(() => validateCatalogRuntimeFlags(runtime, { + nodeOptions: runtime.nodeOptions, + nodeOptionsArgv: [...runtime.nodeOptionsArgv], + nodeExecArgv: [...runtime.nodeExecArgv], + effectiveNodeRuntimeFlags: [...runtime.effectiveNodeRuntimeFlags] + })).not.toThrow(); + expect(() => validateCatalogRuntimeFlags(runtime, { + nodeOptions: runtime.nodeOptions, + nodeOptionsArgv: [...runtime.nodeOptionsArgv], + nodeExecArgv: ['--optimize-for-size', '--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--max-old-space-size=1024', + '--optimize-for-size', + '--expose-gc' + ] + })).toThrow('process.execArgv does not match'); + }); + + it('proves the baseline-size worker flags exactly', () => { + const runtime = { + heapMiB: 1024, + v8Profile: 'baseline-optimize-for-size' as const, + nodeOptions: '--max-old-space-size=1024', + nodeOptionsArgv: ['--max-old-space-size=1024'], + nodeExecArgv: ['--max-opt=1', '--optimize-for-size', '--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--max-old-space-size=1024', + '--max-opt=1', + '--optimize-for-size', + '--expose-gc' + ] + }; + expect(() => validateCatalogRuntimeFlags(runtime, runtime)).not.toThrow(); + }); + + it('rejects a managed profile flag hidden in NODE_OPTIONS', () => { + expect(() => validateCatalogRuntimeFlags({ + heapMiB: 1024, + v8Profile: 'stock', + nodeOptions: '--jitless --max-old-space-size=1024', + nodeOptionsArgv: ['--jitless', '--max-old-space-size=1024'], + nodeExecArgv: ['--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--jitless', + '--max-old-space-size=1024', + '--expose-gc' + ] + }, { + nodeOptions: '--jitless --max-old-space-size=1024', + nodeOptionsArgv: ['--jitless', '--max-old-space-size=1024'], + nodeExecArgv: ['--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--jitless', + '--max-old-space-size=1024', + '--expose-gc' + ] + })).toThrow('configured Node runtime flags are inconsistent'); + }); +}); + +describe('catalog benchmark PostgreSQL backend lifecycle', () => { + const backendIdentity = { + pid: 101, + backendStartEpochMs: 1_700_000_000_500 + }; + const replacementIdentity = { + pid: 202, + backendStartEpochMs: 1_700_000_010_500 + }; + const backendStatus = (input: { + name?: string; + namespacePid?: number; + rssKiB?: number; + highWaterKiB?: number; + } = {}): string => [ + `Name:\t${input.name ?? 'postgres'}`, + `NSpid:\t70000\t${input.namespacePid ?? 101}`, + `VmHWM:\t${input.highWaterKiB ?? 2048} kB`, + `VmRSS:\t${input.rssKiB ?? 1024} kB` + ].join('\n'); + + const backendMeasurement = (input: { + source?: 'docker-container-procfs-diagnostic' | 'local-linux-procfs'; + } = {}) => summarizeCatalogBackendMemorySamples({ + backendIdentity, + samplerPid: 501, + source: input.source ?? 'docker-container-procfs-diagnostic', + postgresContainer: 'postgres-density', + samples: [ + { + monotonicMs: 1_000, + rssBytes: 100, + highWaterBytes: 150, + procStartTicks: 50_000, + procStartEpochMs: 1_700_000_000_000, + bootTimeEpochSeconds: 1_699_999_500, + clockTicksPerSecond: 100 + }, + { + monotonicMs: 1_010, + rssBytes: 180, + highWaterBytes: 220, + procStartTicks: 50_000, + procStartEpochMs: 1_700_000_000_000, + bootTimeEpochSeconds: 1_699_999_500, + clockTicksPerSecond: 100 + }, + { + monotonicMs: 1_020, + rssBytes: 140, + highWaterBytes: 240, + procStartTicks: 50_000, + procStartEpochMs: 1_700_000_000_000, + bootTimeEpochSeconds: 1_699_999_500, + clockTicksPerSecond: 100 + } + ], + targetExitedBeforeStop: true, + targetExitedAtMonotonicMs: 1_030, + samplerStartedAt: '2026-08-02T00:00:00.000Z', + samplerReadyAt: '2026-08-02T00:00:00.005Z', + buildStartedAt: '2026-08-02T00:00:00.010Z', + buildCompletedAt: '2026-08-02T00:00:00.040Z', + samplerStopRequestedAt: '2026-08-02T00:00:00.045Z', + samplerStoppedAt: '2026-08-02T00:00:00.050Z', + buildDurationMs: 30, + clientPlatform: 'linux', + clientArchitecture: 'x64' + }); + + it('binds proc status to the exact PostgreSQL namespace PID', () => { + expect(parseCatalogBackendProcStatus(backendStatus(), 101)).toEqual({ + rssBytes: 1024 * 1024, + highWaterBytes: 2048 * 1024 + }); + expect(() => parseCatalogBackendProcStatus( + backendStatus({ namespacePid: 202 }), + 101 + )).toThrow('identity did not match exact PID 101'); + expect(() => parseCatalogBackendProcStatus( + backendStatus({ name: 'node' }), + 101 + )).toThrow('identity did not match exact PID 101'); + expect(() => parseCatalogBackendProcStatus( + backendStatus().replace(/^VmHWM:.*$/m, ''), + 101 + )).toThrow('valid VmHWM'); + }); + + it('rejects container arguments that Docker could parse as options', () => { + expect(() => validateCatalogPostgresContainer('postgres-density.1')).not.toThrow(); + expect(() => validateCatalogPostgresContainer('--privileged')).toThrow( + "invalid PostgreSQL container name '--privileged'" + ); + expect(() => validateCatalogPostgresContainer('postgres/density')).toThrow( + 'invalid PostgreSQL container name' + ); + }); + + it('pins diagnostic Docker launches to an immutable ID and clears shell env', () => { + const secretVariableNames = [ + 'PGPASSWORD', + 'DATABASE_URL', + 'GRAPHQL_OBSERVABILITY_TOKEN', + 'AWS_SECRET_ACCESS_KEY' + ]; + const environment = { + PATH: '/usr/bin', + HOME: '/tmp/test-home', + DOCKER_HOST: 'unix:///tmp/docker.sock', + ...Object.fromEntries(secretVariableNames.map((name) => [name, `value-${name}`])) + }; + const containerIdentity = parseCatalogDockerContainerIdentity( + `${'a'.repeat(64)}\t2026-08-02T00:00:00.000Z\t70000`, + 'postgres-density' + ); + const launch = makeCatalogBackendSamplerLaunchSpec({ + backendIdentity, + containerIdentity, + clientPlatform: 'darwin', + environment + })!; + + expect(catalogBackendSamplerEnvironment(environment)).toEqual({ + PATH: '/usr/bin', + HOME: '/tmp/test-home', + DOCKER_HOST: 'unix:///tmp/docker.sock' + }); + expect(launch.command).toBe('docker'); + expect(launch.args.slice(0, 7)).toEqual([ + 'exec', + '-i', + 'a'.repeat(64), + '/usr/bin/env', + '-i', + 'PATH=/usr/bin:/bin', + '/bin/sh' + ]); + expect(launch.hostEnvironmentVariableNames).toEqual([ + 'DOCKER_HOST', + 'HOME', + 'PATH' + ]); + const serializedLaunch = JSON.stringify(launch); + for (const name of secretVariableNames) { + expect(launch.hostEnvironmentVariableNames).not.toContain(name); + expect(serializedLaunch).not.toContain(`value-${name}`); + } + const script = launch.args[launch.args.indexOf('-c') + 1]; + expect(script).toContain('trap cleanup_sampler EXIT'); + expect(script).toContain('wait "$sampler_pid"'); + }); + + it('fails immutable container revalidation even when a backend PID matches', () => { + const expected = parseCatalogDockerContainerIdentity( + `${'a'.repeat(64)}\t2026-08-02T00:00:00.000Z\t70000`, + 'postgres-density' + ); + const wrongContainer = parseCatalogDockerContainerIdentity( + `${'b'.repeat(64)}\t2026-08-02T00:00:00.000Z\t70001`, + 'postgres-density' + ); + expect(() => assertCatalogDockerContainerIdentity( + expected, + wrongContainer + )).toThrow('changed immutable identity'); + }); + + it('records identity-bound sampled peaks only as diagnostic lower bounds', () => { + expect(backendMeasurement()).toEqual(expect.objectContaining({ + backendPid: 101, + backendStartEpochMs: 1_700_000_000_500, + baselineRssBytes: 100, + baselineHighWaterBytes: 150, + sampledPeakRssLowerBoundBytes: 180, + sampledHighWaterLowerBoundBytes: 240, + sampledPeakRssDeltaLowerBoundBytes: 80, + sampledHighWaterDeltaLowerBoundBytes: 90, + sampleCount: 3, + targetExitedBeforeStop: true, + timing: expect.objectContaining({ + configuredIntervalMs: 10, + maximumConclusiveGapMs: 50, + maximumObservedGapMs: 10, + coveredBuildWindow: true, + cadenceConclusive: true, + samplerLaunchToReadyMs: 5, + samplerStopRequestToCloseMs: 5 + }), + observerEffect: expect.objectContaining({ + correctionApplied: false, + pairedComparisonSupported: true, + measuredLaunchToReadyMs: 5, + measuredStopRequestToCloseMs: 5 + }), + provenance: expect.objectContaining({ + samplerProcess: 'dedicated-external-procfs-loop', + samplerPid: 501, + source: 'docker-container-procfs-diagnostic', + backendSamplerAuthority: 'diagnostic-only', + serviceDensityMemoryAuthority: + 'separately-validated-linux-cgroup-v2-memory.current', + semantics: 'diagnostic-lower-bound-without-pre-destroy-acknowledgement', + dockerInitialExecEnvironment: + 'may-inherit-container-config-before-env-i', + samplerShellEnvironment: 'env-i-path-only', + backendIdentity: expect.objectContaining({ + sqlBackendStartEpochMs: 1_700_000_000_500, + procStartTicks: 50_000, + toleranceMs: 1_500 + }) + }) + })); + expect(backendMeasurement().provenance.limitation).toContain( + 'diagnostic lower bound' + ); + }); + + it('labels Docker transport separately without changing lower-bound semantics', () => { + const measurement = backendMeasurement(); + expect(measurement.timing.cadenceConclusive).toBe(true); + expect(measurement.provenance.backendSamplerAuthority).toBe('diagnostic-only'); + expect(measurement.provenance.limitation).toContain('Docker Desktop'); + expect(measurement.provenance.limitation).toContain( + 'separately validated Linux cgroup-v2 memory.current' + ); + }); + + it('rejects a changed proc start token and an out-of-tolerance SQL identity', () => { + const changedToken = backendMeasurement({ source: 'local-linux-procfs' }); + const mismatchedSamples = [ + { + monotonicMs: 1_000, + rssBytes: 100, + highWaterBytes: 150, + procStartTicks: 60_000, + procStartEpochMs: 1_700_000_010_000, + bootTimeEpochSeconds: 1_699_999_500, + clockTicksPerSecond: 100 + } + ]; + expect(changedToken.provenance.backendIdentity.procStartTicks).toBe(50_000); + expect(() => summarizeCatalogBackendMemorySamples({ + backendIdentity, + samplerPid: 501, + source: 'local-linux-procfs', + postgresContainer: null, + samples: mismatchedSamples, + targetExitedBeforeStop: false, + samplerStartedAt: '2026-08-02T00:00:00.000Z', + samplerReadyAt: '2026-08-02T00:00:00.005Z', + buildStartedAt: '2026-08-02T00:00:00.010Z', + buildCompletedAt: '2026-08-02T00:00:00.040Z', + samplerStopRequestedAt: '2026-08-02T00:00:00.045Z', + samplerStoppedAt: '2026-08-02T00:00:00.050Z', + buildDurationMs: 30 + })).toThrow('mismatched process start identity'); + }); + + it('marks sparse cadence inconclusive without promoting its lower bound', () => { + const measurement = summarizeCatalogBackendMemorySamples({ + backendIdentity, + samplerPid: 501, + source: 'local-linux-procfs', + postgresContainer: null, + samples: [ + { + monotonicMs: 1_000, + rssBytes: 100, + highWaterBytes: 150, + procStartTicks: 50_000, + procStartEpochMs: 1_700_000_000_000, + bootTimeEpochSeconds: 1_699_999_500, + clockTicksPerSecond: 100 + }, + { + monotonicMs: 1_075, + rssBytes: 180, + highWaterBytes: 220, + procStartTicks: 50_000, + procStartEpochMs: 1_700_000_000_000, + bootTimeEpochSeconds: 1_699_999_500, + clockTicksPerSecond: 100 + } + ], + targetExitedBeforeStop: false, + samplerStartedAt: '2026-08-02T00:00:00.000Z', + samplerReadyAt: '2026-08-02T00:00:00.005Z', + buildStartedAt: '2026-08-02T00:00:00.010Z', + buildCompletedAt: '2026-08-02T00:00:00.070Z', + samplerStopRequestedAt: '2026-08-02T00:00:00.075Z', + samplerStoppedAt: '2026-08-02T00:00:00.080Z', + buildDurationMs: 60 + }); + + expect(measurement.timing.maximumObservedGapMs).toBe(75); + expect(measurement.timing.cadenceConclusive).toBe(false); + expect(measurement.sampledHighWaterLowerBoundBytes).toBe(220); + expect(measurement.sampledHighWaterDeltaLowerBoundBytes).toBe(70); + expect(measurement.provenance.backendSamplerAuthority).toBe('diagnostic-only'); + expect(measurement.provenance.limitation).toContain('maximum observed gap'); + }); + + it('supports explicit sampler-on and sampler-off observer comparisons', () => { + expect(parseCatalogBackendSamplerMode([])).toBe('diagnostic-lower-bound'); + expect(parseCatalogBackendSamplerMode([ + '--postgres-backend-sampler', 'off' + ])).toBe('off'); + expect(() => parseCatalogBackendSamplerMode([ + '--postgres-backend-sampler', 'exact' + ])).toThrow("must be 'off' or 'diagnostic-lower-bound'"); + }); + + it('stops an already-exited worker without signaling a reused process group', async () => { + const requestGracefulStop = jest.fn(); + const signalProcessGroup = jest.fn(); + await expect(stopCatalogBackendSamplerProcessTree({ + requestGracefulStop, + waitForTreeExit: async () => true, + signalProcessGroup + })).resolves.toBe('already-exited'); + expect(requestGracefulStop).not.toHaveBeenCalled(); + expect(signalProcessGroup).not.toHaveBeenCalled(); + }); + + it('escalates bounded cleanup through TERM and KILL until no tree remains', async () => { + const waits = [false, false, false, true]; + const signalProcessGroup = jest.fn(); + await expect(stopCatalogBackendSamplerProcessTree({ + requestGracefulStop: jest.fn(), + waitForTreeExit: async () => waits.shift()!, + signalProcessGroup, + gracefulTimeoutMs: 1, + termTimeoutMs: 1, + killTimeoutMs: 1 + })).resolves.toBe('sigkill'); + expect(waits).toHaveLength(0); + expect(signalProcessGroup.mock.calls).toEqual([ + ['SIGTERM'], + ['SIGKILL'] + ]); + }); + + it('fails cleanup when the process tree survives KILL', async () => { + await expect(stopCatalogBackendSamplerProcessTree({ + requestGracefulStop: jest.fn(), + waitForTreeExit: async () => false, + signalProcessGroup: jest.fn(), + gracefulTimeoutMs: 1, + termTimeoutMs: 1, + killTimeoutMs: 1 + })).rejects.toThrow('survived SIGKILL'); + }); + + it('still reaps the tree when the graceful stop write fails', async () => { + const waits = [false, false, true]; + const signalProcessGroup = jest.fn(); + await expect(stopCatalogBackendSamplerProcessTree({ + requestGracefulStop: () => { + throw new Error('stdin failed'); + }, + waitForTreeExit: async () => waits.shift()!, + signalProcessGroup, + gracefulTimeoutMs: 1, + termTimeoutMs: 1, + killTimeoutMs: 1 + })).rejects.toThrow('stdin failed'); + expect(waits).toHaveLength(0); + expect(signalProcessGroup).toHaveBeenCalledWith('SIGTERM'); + }); + + it('starts before the build and stops before PID retirement and replacement', async () => { + const order: string[] = []; + const measurement = backendMeasurement(); + const sampled = await measureCatalogBuildWithBackendSampler({ + startSampler: async () => { + order.push('sampler:start'); + return { + stop: async () => { + order.push('sampler:stop'); + return measurement; + } + }; + }, + build: async () => { + order.push('build'); + return 'built'; + } + }); + const transition = await resolveCatalogBackendPidAfterBuild( + 'destroy', + backendIdentity, + { + waitForRetirement: async () => { + order.push('backend:retired'); + }, + acquireBackendIdentity: async () => { + order.push('replacement:acquired'); + return replacementIdentity; + } + } + ); + + expect(sampled.value).toBe('built'); + expect(sampled.backendMemoryLowerBound).toBe(measurement); + expect(transition.steadyBackendPid).toBe(202); + expect(order).toEqual([ + 'sampler:start', + 'build', + 'sampler:stop', + 'backend:retired', + 'replacement:acquired' + ]); + }); + + it('still stops the sampler when the Graphile build fails', async () => { + const stop = jest.fn(async () => backendMeasurement()); + await expect(measureCatalogBuildWithBackendSampler({ + startSampler: async () => ({ stop }), + build: async () => { + throw new Error('build failed'); + } + })).rejects.toThrow('build failed'); + expect(stop).toHaveBeenCalledTimes(1); + }); + + it('fails the build result when the configured sampler fails', async () => { + await expect(measureCatalogBuildWithBackendSampler({ + startSampler: async () => ({ + stop: async () => { + throw new Error('sampler failed'); + } + }), + build: async () => 'built' + })).rejects.toThrow('sampler failed'); + }); + + it('keeps the same backend in reuse mode without a retirement probe', async () => { + const waitForRetirement = jest.fn(async (): Promise => undefined); + const acquireBackendIdentity = jest.fn(async () => backendIdentity); + + await expect(resolveCatalogBackendPidAfterBuild('reuse', backendIdentity, { + waitForRetirement, + acquireBackendIdentity + })).resolves.toEqual({ + introspectionBackendPid: 101, + introspectionBackendStartEpochMs: 1_700_000_000_500, + steadyBackendPid: 101, + steadyBackendStartEpochMs: 1_700_000_000_500, + introspectionBackendRetired: false + }); + expect(waitForRetirement).not.toHaveBeenCalled(); + expect(acquireBackendIdentity).toHaveBeenCalledTimes(1); + }); + + it('proves retirement before acquiring and recording the replacement', async () => { + const order: string[] = []; + const waitForRetirement = jest.fn(async ( + identity: typeof backendIdentity + ): Promise => { + order.push(`retired:${identity.pid}`); + }); + const acquireBackendIdentity = jest.fn(async () => { + order.push('acquired:202'); + return replacementIdentity; + }); + + await expect(resolveCatalogBackendPidAfterBuild('destroy', backendIdentity, { + waitForRetirement, + acquireBackendIdentity + })).resolves.toEqual({ + introspectionBackendPid: 101, + introspectionBackendStartEpochMs: 1_700_000_000_500, + steadyBackendPid: 202, + steadyBackendStartEpochMs: 1_700_000_010_500, + introspectionBackendRetired: true + }); + expect(order).toEqual(['retired:101', 'acquired:202']); + }); + + it('fails closed on unexpected PID reuse or rotation', async () => { + await expect(resolveCatalogBackendPidAfterBuild('destroy', backendIdentity, { + waitForRetirement: async () => undefined, + acquireBackendIdentity: async () => ({ + pid: 101, + backendStartEpochMs: 1_700_000_020_500 + }) + })).rejects.toThrow('destroyed PostgreSQL introspection backend 101 was reused'); + await expect(resolveCatalogBackendPidAfterBuild('reuse', backendIdentity, { + waitForRetirement: async () => undefined, + acquireBackendIdentity: async () => replacementIdentity + })).rejects.toThrow('PostgreSQL benchmark backend identity changed'); + }); +}); + +describe('catalog benchmark cache warmth', () => { + it('preserves the disabled defaults', () => { + expect(parseCatalogWarmthCliOptions([])).toEqual({ + warmOperationsPerInstance: 0, + warmOperationReplayPasses: 0, + grafastCacheLimits: { + queryCacheMaxLength: null, + operationsCacheMaxLength: null, + operationOperationPlansCacheMaxLength: null + } + }); + }); + + it('parses independently configurable positive cache limits', () => { + expect(parseCatalogWarmthCliOptions([ + '--warm-operations-per-instance', '500', + '--warm-operation-replay-passes', '3', + '--grafast-query-cache-max', '8', + '--grafast-operations-cache-max', '16', + '--grafast-operation-plans-cache-max', '32' + ])).toEqual({ + warmOperationsPerInstance: 500, + warmOperationReplayPasses: 3, + grafastCacheLimits: { + queryCacheMaxLength: 8, + operationsCacheMaxLength: 16, + operationOperationPlansCacheMaxLength: 32 + } + }); + }); + + it.each([ + [['--warm-operations-per-instance'], 'requires a value'], + [['--warm-operations-per-instance', '-1'], 'non-negative integer'], + [['--warm-operations-per-instance', '1.5'], 'non-negative integer'], + [['--warm-operations-per-instance', '01'], 'non-negative integer'], + [['--warm-operation-replay-passes'], 'requires a value'], + [['--warm-operation-replay-passes', '-1'], 'non-negative integer'], + [['--warm-operation-replay-passes', '1.5'], 'non-negative integer'], + [['--warm-operation-replay-passes', '01'], 'non-negative integer'], + [['--grafast-query-cache-max', '0'], 'positive safe integer'], + [['--grafast-query-cache-max', '1'], 'safe integer of at least 2'], + [['--grafast-operations-cache-max', '-1'], 'positive integer'], + [['--grafast-operation-plans-cache-max', '1e2'], 'positive integer'], + [[ + '--grafast-query-cache-max', '8', + '--grafast-query-cache-max', '16' + ], 'may only be specified once'] + ])('rejects malformed warmth arguments %j', (args, message) => { + expect(() => parseCatalogWarmthCliOptions(args as string[])).toThrow(message as string); + }); + + it('rejects incomplete worker cache-limit configuration', () => { + expect(() => validateCatalogWarmthConfig({ + warmOperationsPerInstance: 1, + warmOperationReplayPasses: 0, + grafastCacheLimits: { + queryCacheMaxLength: null, + operationsCacheMaxLength: null + } as never + })).toThrow('must define all three cache limit fields'); + }); + + it('requires a populated source set when replay is enabled', () => { + expect(() => parseCatalogWarmthCliOptions([ + '--warm-operation-replay-passes', '1' + ])).toThrow( + 'warmOperationReplayPasses requires warmOperationsPerInstance' + ); + }); + + it('generates stable, distinct, valid source operations', () => { + const sources = Array.from( + { length: 500 }, + (_, index) => makeCatalogWarmOperationSource(index + 1) + ); + expect(new Set(sources).size).toBe(500); + expect(sources[0]).toBe( + 'query CatalogWarm1 { warmTenantToken: tenantToken }' + ); + expect(() => sources.forEach((source) => parse(source))).not.toThrow(); + expect(() => makeCatalogWarmOperationSource(0)).toThrow('positive safe integer'); + }); + + it('uses the nearest-rank percentile deterministically', () => { + expect(catalogPercentile([], 0.5)).toBeNull(); + expect(catalogPercentile([9, 1, 5, 3], 0.5)).toBe(3); + expect(catalogPercentile([9, 1, 5, 3], 0.99)).toBe(9); + expect(() => catalogPercentile([1], 0)).toThrow('percentile probability'); + }); + + it('summarizes sampled and process-high-water build transients', () => { + expect(summarizeBuildTransientSamples( + { heapUsedBytes: 100, rssBytes: 200, processPeakRssBytes: 250 }, + [ + { heapUsedBytes: 100, rssBytes: 200, processPeakRssBytes: 250 }, + { heapUsedBytes: 170, rssBytes: 260, processPeakRssBytes: 300 }, + { heapUsedBytes: 140, rssBytes: 240, processPeakRssBytes: 320 } + ] + )).toEqual({ + baselineHeapUsedBytes: 100, + baselineRssBytes: 200, + sampledPeakHeapUsedBytes: 170, + sampledPeakHeapDeltaBytes: 70, + sampledPeakRssBytes: 260, + sampledPeakRssDeltaBytes: 60, + processPeakRssBytes: 320, + processPeakRssDeltaBytes: 70, + sampleCount: 3 + }); + }); +}); + +describe('catalog benchmark tenant-density projection', () => { + it('is opt-in and parses one positive surface count', () => { + expect(parseCatalogTenantProxySurfaces([])).toBeNull(); + expect(parseCatalogTenantProxySurfaces([ + '--tenant-proxy-surfaces', '5' + ])).toBe(5); + }); + + it.each([ + [['--tenant-proxy-surfaces'], 'requires a value'], + [['--tenant-proxy-surfaces', '0'], 'positive safe integer'], + [['--tenant-proxy-surfaces', '1.5'], 'positive integer'], + [['--tenant-proxy-surfaces', '05'], 'positive integer'], + [[ + '--tenant-proxy-surfaces', '5', + '--tenant-proxy-surfaces', '6' + ], 'may only be specified once'] + ])('rejects malformed tenant proxy arguments %j', (args, message) => { + expect(() => parseCatalogTenantProxySurfaces(args as string[])).toThrow( + message as string + ); + }); + + it('projects the 350-instance result across configured old space and peak RSS', () => { + const density = projectCatalogTenantDensity({ + tenantProxySurfaces: 5, + configuredOldSpaceMiB: 1024, + snapshot: { + instances: 350, + processPeakRssBytes: 1_100_939_264, + processPeakRssDeltaBytes: 956_203_008 + } + }); + + expect(density).toMatchObject({ + residentSurfaceInstances: 350, + fullTenantProxyGroups: 70, + remainderSurfaceInstances: 0, + configuredOldSpaceMiB: 1024, + absolutePeakProcessRssBytes: 1_100_939_264, + groupsPerConfiguredOldSpaceGiB: 70 + }); + expect(density.groupsPerAbsolutePeakProcessRssGiB).toBeCloseTo( + 68.27073040061909, + 10 + ); + expect(density.groupsPerAbsolutePeakProcessRssGiB).not.toBeCloseTo( + 78.60457146773585, + 10 + ); + }); + + it('counts only full proxy groups and records leftover surface instances', () => { + expect(projectCatalogTenantDensity({ + tenantProxySurfaces: 5, + configuredOldSpaceMiB: 2048, + snapshot: { + instances: 24, + processPeakRssBytes: 2 ** 30, + processPeakRssDeltaBytes: 2 ** 29 + } + })).toEqual({ + residentSurfaceInstances: 24, + fullTenantProxyGroups: 4, + remainderSurfaceInstances: 4, + configuredOldSpaceMiB: 2048, + absolutePeakProcessRssBytes: 2 ** 30, + groupsPerConfiguredOldSpaceGiB: 2, + groupsPerAbsolutePeakProcessRssGiB: 4 + }); + }); +}); + +describe('catalog benchmark crash progress', () => { + it('atomically replaces one credential-free progress artifact', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-progress-')); + const resultFile = path.join(directory, 'result.json'); + const snapshot: CatalogMemorySnapshot = { + instances: 25, + heapUsedBytes: 100, + heapDeltaBytes: 50, + rssBytes: 200, + rssDeltaBytes: 75, + externalBytes: 10, + externalDeltaBytes: 1, + processPeakRssBytes: 220, + processPeakRssDeltaBytes: 80, + postgresBackendRssBytes: null, + postgresBackendRssDeltaBytes: null, + postgresBackendHighWaterBytes: null, + postgresBackendHighWaterDeltaBytes: null + }; + try { + writeCatalogProgress(resultFile, { + version: 1, + status: 'in-progress', + mode: 'scoped-required', + scopedCatalogTypes: 'dependency-closure', + introspectionClientReleaseMode: 'destroy', + postgresBackendSamplerMode: 'diagnostic-lower-bound', + releaseBuildStateAfterValidation: true, + repetition: 1, + heapMiB: 1024, + v8Profile: 'jitless-optimize-for-size', + nodeOptions: '--max-old-space-size=1024', + nodeOptionsArgv: ['--max-old-space-size=1024'], + nodeExecArgv: ['--jitless', '--optimize-for-size', '--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--max-old-space-size=1024', + '--jitless', + '--optimize-for-size', + '--expose-gc' + ], + targetInstances: 500, + completedInstances: 25, + configuredCheckpoints: [25, 500], + completedCheckpoints: [25], + buildsCompleted: 25, + canariesCompleted: 50, + mismatchViolations: 0, + crossTenantViolations: 0, + lastSnapshot: snapshot, + updatedAt: '2026-08-01T00:00:00.000Z' + }); + const progressFile = catalogProgressPath(resultFile); + expect(JSON.parse(fs.readFileSync(progressFile, 'utf8'))).toMatchObject({ + status: 'in-progress', + completedInstances: 25, + lastSnapshot: { instances: 25 } + }); + expect(fs.readdirSync(directory)).toEqual(['progress.json']); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/perf-harness/src/__tests__/config.test.ts b/packages/perf-harness/src/__tests__/config.test.ts new file mode 100644 index 0000000000..d8e07e6414 --- /dev/null +++ b/packages/perf-harness/src/__tests__/config.test.ts @@ -0,0 +1,875 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + armEnvironmentForHeap, + assertIsolatedPort, + assertLoopbackObservabilityUrl, + assertLoopbackRetainedHeapCheckpointUrl, + loadFleet, + loadPlan, + resolveTemplate, + tenantCountsForHeap, + validateAcceptanceGates, + validateCoverage, + validateWorkloadPlan} from '../config'; +import type { AcceptanceGates, DensityPlanV1, FleetV1 } from '../types'; + +const validGates: AcceptanceGates = { + maxErrorRate: 0.005, + maxP99Ms: 150, + maxPostWarmupHeapGrowthMiBPerHour: 5, + minMedianDensityImprovement: 0.15, + minAdditionalTenantsEveryRun: 1, + requireZeroBleed: true, + requireNoPostWarmupEvictions: true, + requireNoPostWarmupBuildRefusals: true, + requireNoPostWarmupBuilds: true, + requirePostgresMemoryTelemetry: false, + requireFreshPostgresRunAttestation: false, + requireRetainedMemoryCheckpoints: false, + requirePhysicalDatabaseTelemetry: false, + requireConclusiveCanaries: true, + requireCompletePeriodicCanaryCoverage: false, + requireConclusiveOperationOracles: false, + requireExplicitCustomerTopology: false, + requiredCacheAdmissionMode: null +}; + +describe('density harness configuration', () => { + it.each(Object.keys(validGates) as Array)( + 'fails closed when acceptance gate %s is omitted', + (key) => { + const malformed: Partial = { ...validGates }; + delete malformed[key]; + expect(() => validateAcceptanceGates(malformed as AcceptanceGates)).toThrow( + `plan.gates.${key}` + ); + } + ); + + it('validates optional aligned-memory cadence and workload-coverage gates', () => { + expect(() => validateAcceptanceGates({ + ...validGates, + maxAlignedMemorySampleGapMs: 1_000, + minAlignedMemoryCoverageRatio: 0.99 + })).not.toThrow(); + expect(() => validateAcceptanceGates({ + ...validGates, + maxAlignedMemorySampleGapMs: 0 + })).toThrow('plan.gates.maxAlignedMemorySampleGapMs must be positive'); + expect(() => validateAcceptanceGates({ + ...validGates, + minAlignedMemoryCoverageRatio: 1.01 + })).toThrow('plan.gates.minAlignedMemoryCoverageRatio must be at most 1'); + }); + + it('refuses shared workspace ports by default', () => { + expect(() => assertIsolatedPort(3000)).toThrow('reserved shared-workspace port'); + expect(() => assertIsolatedPort(5432)).toThrow('reserved shared-workspace port'); + expect(() => assertIsolatedPort(3345)).not.toThrow(); + expect(() => assertIsolatedPort(3000, true)).not.toThrow(); + }); + + it('resolves only known template variables', () => { + expect(resolveTemplate('http://127.0.0.1:{port}/{mode}', { + port: 3345, + mode: 'stock' + })).toBe('http://127.0.0.1:3345/stock'); + expect(() => resolveTemplate('{missing}', {})).toThrow("unknown template variable 'missing'"); + }); + + it('accepts exactly one offered-load mode and validates workload traffic budgets', () => { + const workload = { + durationSec: 900, + rps: 50, + minWorkloadRequestsPerSurface: 10, + requestTimeoutMs: 30_000, + maxInFlight: 128, + canaryIntervalSec: 60, + warmupTimeoutMs: 180_000, + warmupTimeoutPerSurfaceMs: 2_000 + }; + expect(() => validateWorkloadPlan(workload)).not.toThrow(); + expect(() => validateWorkloadPlan({ + ...workload, + rps: undefined, + rpsPerTenant: 0.2 + })).not.toThrow(); + expect(() => validateWorkloadPlan({ ...workload, rpsPerTenant: 1 })) + .toThrow('exactly one'); + expect(() => validateWorkloadPlan({ + ...workload, + rps: undefined, + rpsPerTenant: undefined + })).toThrow('exactly one'); + expect(() => validateWorkloadPlan({ + ...workload, + minWorkloadRequestsPerSurface: 0 + })).toThrow('minWorkloadRequestsPerSurface'); + expect(() => validateWorkloadPlan({ + ...workload, + periodicCanarySchedule: 'rotating-one', + canaryConcurrency: 16 + })).not.toThrow(); + expect(() => validateWorkloadPlan({ + ...workload, + periodicCanarySchedule: 'drop-overlap' as any + })).toThrow('periodicCanarySchedule'); + expect(() => validateWorkloadPlan({ + ...workload, + canaryConcurrency: 0 + })).toThrow('canaryConcurrency'); + }); + + it('resolves heap-specific ramps with a legacy fallback', () => { + const plan = { + tenantCounts: [1, 2], + tenantCountsByHeapMiB: { 2048: [4, 8] } + } as unknown as DensityPlanV1; + expect(tenantCountsForHeap(plan, 1024)).toEqual([1, 2]); + expect(tenantCountsForHeap(plan, 2048)).toEqual([4, 8]); + expect(() => tenantCountsForHeap({} as DensityPlanV1, 4096)) + .toThrow('no tenant-count ramp'); + }); + + it('overrides only the selected heap-specific environment', () => { + const arm = { + env: { SHARED: 'base', OVERRIDE: 'base' }, + envByHeapMiB: { + 1024: { OVERRIDE: 'one', CALIBRATION: 'cal-1' }, + 2048: { OVERRIDE: 'two', CALIBRATION: 'cal-2' } + } + }; + expect(armEnvironmentForHeap(arm, 1024)).toEqual({ + SHARED: 'base', + OVERRIDE: 'one', + CALIBRATION: 'cal-1' + }); + expect(armEnvironmentForHeap(arm, 2048).CALIBRATION).toBe('cal-2'); + }); + + it('requires a complete and exact heap-specific environment map', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-plan-env-')); + const file = path.join(directory, 'plan.json'); + const plan: any = { + version: 1, + fleetFile: 'fleet.json', + artifactDir: 'artifacts', + arms: [{ + name: 'calibrated', + port: 3345, + readinessUrl: 'http://127.0.0.1:3345/healthz', + memoryUrl: 'http://127.0.0.1:3345/debug/memory', + introspectionMode: 'stock', + envByHeapMiB: { 1024: { GRAPHILE_CACHE_CALIBRATION_ID: 'cal-1' } } + }], + heapMiB: [1024, 2048], + tenantCounts: [1], + repetitions: 1, + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + workload: { + durationSec: 900, + rpsPerTenant: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 30_000, + maxInFlight: 1, + canaryIntervalSec: 60, + warmupTimeoutMs: 30_000, + warmupTimeoutPerSurfaceMs: 30_000 + }, + gates: { ...validGates, requireExplicitCustomerTopology: false } + }; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow("envByHeapMiB is missing heap '2048'"); + plan.arms[0].envByHeapMiB['2048'] = { GRAPHILE_CACHE_CALIBRATION_ID: 'cal-2' }; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).not.toThrow(); + plan.arms[0].envByHeapMiB['4096'] = { GRAPHILE_CACHE_CALIBRATION_ID: 'cal-3' }; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow("contains unconfigured heap '4096'"); + }); + + it('validates an enabled soak against an exact configured arm and heap', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-plan-soak-')); + const file = path.join(directory, 'plan.json'); + const plan: any = { + version: 1, + fleetFile: 'fleet.json', + artifactDir: 'artifacts', + arms: [{ + name: 'candidate', + port: 3345, + readinessUrl: 'http://127.0.0.1:3345/healthz', + memoryUrl: 'http://127.0.0.1:3345/debug/memory', + introspectionMode: 'scoped-required' + }], + heapMiB: [1024], + tenantCounts: [1], + repetitions: 1, + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + workload: { + durationSec: 900, + rpsPerTenant: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 30_000, + maxInFlight: 1, + canaryIntervalSec: 60, + warmupTimeoutMs: 30_000, + warmupTimeoutPerSurfaceMs: 30_000 + }, + gates: validGates, + soak: { + enabled: true, + arm: 'candidate', + durationSec: 7_200, + tenantCount: 1, + heapMiB: 1024 + } + }; + const write = (): void => fs.writeFileSync(file, JSON.stringify(plan)); + + write(); + expect(() => loadPlan(file)).not.toThrow(); + plan.soak.heapMiB = 2048; + write(); + expect(() => loadPlan(file)).toThrow('plan.soak.heapMiB=2048 is not configured'); + plan.soak.heapMiB = 1024; + plan.soak.arm = 'missing'; + write(); + expect(() => loadPlan(file)).toThrow("plan.soak.arm 'missing' is not configured"); + plan.soak.arm = 'candidate'; + plan.soak.durationSec = 1.5; + write(); + expect(() => loadPlan(file)).toThrow('plan.soak.durationSec must be a safe integer'); + + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it('sends observability credentials only to the exact loopback memory route', () => { + expect(() => assertLoopbackObservabilityUrl( + 'http://127.0.0.1:3345/debug/memory', + 3345 + )).not.toThrow(); + expect(() => assertLoopbackObservabilityUrl( + 'http://[::1]:3345/debug/memory', + 3345 + )).not.toThrow(); + expect(() => assertLoopbackObservabilityUrl( + 'https://example.com:3345/debug/memory', + 3345 + )).toThrow('memoryUrl must be the credential-free URL'); + expect(() => assertLoopbackObservabilityUrl( + 'http://127.0.0.1:3345/debug/memory?token=secret', + 3345 + )).toThrow('memoryUrl must be the credential-free URL'); + }); + + it('accepts only the exact credential-free retained-memory checkpoint route', () => { + expect(() => assertLoopbackRetainedHeapCheckpointUrl( + 'http://127.0.0.1:3345/__cperf/retained-memory-checkpoint', + 3345 + )).not.toThrow(); + expect(() => assertLoopbackRetainedHeapCheckpointUrl( + 'http://127.0.0.1:3345/__cperf/retained-memory-checkpoint?token=secret', + 3345 + )).toThrow('retainedHeapCheckpointUrl must be the credential-free URL'); + expect(() => assertLoopbackRetainedHeapCheckpointUrl( + 'https://example.com:3345/__cperf/retained-memory-checkpoint', + 3345 + )).toThrow('retainedHeapCheckpointUrl must be the credential-free URL'); + }); + + it('requires spawned arms to expose GC and explicitly enable the checkpoint', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-plan-gc-')); + const file = path.join(directory, 'plan.json'); + const plan: any = { + version: 1, + fleetFile: 'fleet.json', + artifactDir: 'artifacts', + arms: [{ + name: 'candidate', + commit: 'a'.repeat(40), + command: [process.execPath, '/tmp/server.cjs'], + port: 3345, + readinessUrl: 'http://127.0.0.1:3345/healthz', + memoryUrl: 'http://127.0.0.1:3345/debug/memory', + retainedHeapCheckpointUrl: + 'http://127.0.0.1:{port}/__cperf/retained-memory-checkpoint', + introspectionMode: 'stock', + env: {} + }], + heapMiB: [1024], + tenantCounts: [1], + repetitions: 1, + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + workload: { + durationSec: 900, + rpsPerTenant: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 30_000, + maxInFlight: 1, + canaryIntervalSec: 60, + warmupTimeoutMs: 30_000, + warmupTimeoutPerSurfaceMs: 30_000 + }, + gates: { + ...validGates, + requireExplicitCustomerTopology: false, + requireRetainedMemoryCheckpoints: true + } + }; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('--expose-gc'); + plan.arms[0].command.splice(1, 0, '--expose-gc'); + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('GRAPHQL_CPERF_RETAINED_HEAP_ENABLED=true'); + plan.arms[0].env.GRAPHQL_CPERF_RETAINED_HEAP_ENABLED = 'true'; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).not.toThrow(); + plan.arms[0].v8Profile = 'jitless-optimize-for-size'; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).not.toThrow(); + plan.arms[0].v8Profile = 'baseline-optimize-for-size'; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).not.toThrow(); + plan.arms[0].v8Profile = 'jitless-optimize-for-size'; + plan.arms[0].command.splice(1, 0, '--jitless'); + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('managed V8 flags through v8Profile'); + plan.arms[0].command.splice(1, 1); + plan.arms[0].v8Profile = 'arbitrary-flags'; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('unknown v8Profile'); + plan.arms[0].v8Profile = 'stock'; + plan.gates.requiredCacheAdmissionMode = 'drop-resident'; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('requiredCacheAdmissionMode'); + }); + + it('requires a concrete fresh PostgreSQL prepare and server binding', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-plan-pg-run-')); + const file = path.join(directory, 'plan.json'); + const plan: any = { + version: 1, + fleetFile: 'fleet.json', + artifactDir: 'artifacts', + arms: [{ + name: 'candidate', + commit: 'a'.repeat(40), + command: [process.execPath, '/tmp/server.cjs'], + port: 3345, + readinessUrl: 'http://127.0.0.1:3345/healthz', + memoryUrl: 'http://127.0.0.1:3345/debug/memory', + introspectionMode: 'stock' + }], + heapMiB: [1024], + tenantCounts: [1], + repetitions: 1, + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + workload: { + durationSec: 900, + rpsPerTenant: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 30_000, + maxInFlight: 1, + canaryIntervalSec: 60, + warmupTimeoutMs: 30_000, + warmupTimeoutPerSurfaceMs: 30_000 + }, + gates: { + ...validGates, + requireExplicitCustomerTopology: false, + requireFreshPostgresRunAttestation: true + } + }; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('postgresRunAttestation.command'); + plan.arms[0].postgresRunAttestation = { + command: [process.execPath, '/tmp/audit.cjs'], + prepareCommand: [process.execPath, '/tmp/prepare.cjs'] + }; + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).toThrow('does not bind the fresh PostgreSQL fixture'); + plan.arms[0].command.push( + '{postgresManifestFile}', + '{postgresSecretsFile}', + '{postgresManifestSha256}', + '{postgresCloneId}' + ); + plan.arms[0].postgresRunAttestation.prepareCommand.push( + '{postgresFixtureDir}', + '{arm}', + '{heapMiB}', + '{tenantCount}', + '{repetition}', + '{runOrderIndex}' + ); + plan.arms[0].postgresRunAttestation.command.push( + '{postgresManifestFile}', + '{postgresSecretsFile}', + '{attestationFile}', + '{planSha256}', + '{fleetSha256}', + '{notBeforeEpochMs}' + ); + fs.writeFileSync(file, JSON.stringify(plan)); + expect(() => loadPlan(file)).not.toThrow(); + }); + + it('rejects a surface without isolation canaries', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-config-')); + const file = path.join(directory, 'fleet.json'); + fs.writeFileSync(file, JSON.stringify({ + version: 1, + tenants: [{ + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: 'tenant-a-api', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [] + }] + }] + })); + expect(() => loadFleet(file)).toThrow('has no isolation canaries'); + }); + + it('rejects a canary that can pass on an empty result', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-config-')); + const file = path.join(directory, 'fleet.json'); + fs.writeFileSync(file, JSON.stringify({ + version: 1, + tenants: [{ + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: 'tenant-a-api', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ __typename }', + forbiddenMatches: [{ path: '/data/token', value: 'tenant-b' }] + }] + }] + }] + })); + expect(() => loadFleet(file)).toThrow('requiredMatches'); + }); + + it('validates paired operation oracles and post-coverage verification queries', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-operation-oracle-')); + const file = path.join(directory, 'fleet.json'); + const operation: any = { + name: 'upload', + capability: 'uploads', + query: 'mutation { upload { id } }', + postCoverageVerification: { + query: 'query ($fileId: UUID!, $contentHash: String!) { uploadedFiles(where: { id: { equalTo: $fileId }, contentHash: { equalTo: $contentHash } }) { nodes { physicalDatabaseIdentity } } }', + variables: { contentHash: 'fixture-hash' }, + variablesFromResponse: { fileId: '/data/upload/id' }, + requiredMatches: [{ + path: '/data/uploadedFiles/nodes/0/physicalDatabaseIdentity', + value: 'physical-db-a' + }], + forbiddenMatches: [{ + path: '/data/uploadedFiles/nodes/0/physicalDatabaseIdentity', + value: 'physical-db-b' + }], + invariants: [{ + path: '/data/uploadedFiles/nodes/*/physicalDatabaseIdentity', + everyEquals: 'physical-db-a', + min: 1, + max: 1 + }] + } + }; + const fleet = { + version: 1, + tenants: [{ + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: 'tenant-a-api', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [operation], + canaries: [{ + name: 'cross-schema', + query: '{ __typename }', + requiredMatches: [{ path: '/data/token', value: 'tenant-a' }], + forbiddenMatches: [{ path: '/data/token', value: 'tenant-b' }] + }] + }] + }] + }; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).not.toThrow(); + + operation.postCoverageVerification.variablesFromResponse.contentHash = + '/data/upload/contentHash'; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow('collides with a static variable'); + delete operation.postCoverageVerification.variablesFromResponse.contentHash; + + operation.postCoverageVerification.invariants[0].min = 0; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow('min must be a positive safe integer'); + operation.postCoverageVerification.invariants[0].min = 1; + + delete operation.postCoverageVerification.forbiddenMatches; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow( + 'postCoverageVerification.forbiddenMatches' + ); + + delete operation.postCoverageVerification; + operation.requiredMatches = [{ path: '/data/token', value: 'tenant-a' }]; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow( + 'must configure requiredMatches and forbiddenMatches together' + ); + }); + + it('validates exact realtime probes and keeps sensitive headers environment-backed', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-realtime-fleet-')); + const file = path.join(directory, 'fleet.json'); + const surface: any = { + name: 'api', + buildContract: 'customer-a-api', + url: 'http://127.0.0.1:3345/customer/customer-a/tenant/a/graphql', + headers: { 'accept-language': 'es' }, + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ token }', + requiredMatches: [{ path: '/data/token', value: 'tenant-a' }], + forbiddenMatches: [{ path: '/data/token', value: 'tenant-b' }] + }], + realtime: { + headersFromEnvironment: { authorization: 'CPERF_RUNTIME_TOKEN' }, + subscription: { + query: 'subscription { event { token } }', + requiredMatches: [{ path: '/data/event/token', value: 'tenant-a' }], + forbiddenMatches: [{ path: '/data/event/token', value: 'tenant-b' }] + }, + prime: { + query: 'mutation Prime($payload: String!) { prime(payload: $payload) { token payload } }', + variables: { payload: 'configured-placeholder' }, + requiredMatches: [{ path: '/data/prime/token', value: 'tenant-a' }], + forbiddenMatches: [{ path: '/data/prime/token', value: 'tenant-b' }] + }, + correlation: { + primeVariable: 'payload', + primeResponsePath: '/data/prime/payload', + subscriptionEventPath: '/data/event/payload' + } + } + }; + const fleet: any = { + version: 1, + tenants: [{ + id: 'customer-a', + databases: [{ + id: 'database-a', + physicalDatabase: 'customer_a', + apis: [{ + id: 'api-a', + runtimePoolIdentity: 'pg:v1:customer-a', + physicalSchemas: ['tenant_a'], + routingLabels: ['customer-a'], + realtime: true, + surfaces: ['api'] + }] + }], + surfaces: [surface] + }] + }; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).not.toThrow(); + + surface.realtime.subscription.requiredMatches.push({ + path: '/data/event/payload', + value: 'configured-placeholder' + }); + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow( + 'correlation paths must not carry a static required match' + ); + surface.realtime.subscription.requiredMatches.pop(); + + surface.realtime.correlation.subscriptionEventPath = '/data/event/*/payload'; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow('realtime.correlation is invalid'); + surface.realtime.correlation.subscriptionEventPath = '/data/event/payload'; + + surface.realtime.correlation.subscriptionEventPath = '/data/event/~2payload'; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow('realtime.correlation is invalid'); + surface.realtime.correlation.subscriptionEventPath = '/data/event/payload'; + + surface.headers.authorization = 'persisted-secret'; + fs.writeFileSync(file, JSON.stringify(fleet)); + expect(() => loadFleet(file)).toThrow( + 'authorization must use realtime.headersFromEnvironment' + ); + }); + + it('fails coverage when a required canary is absent from any surface', () => { + const fleet = { + version: 1, + tenants: [{ + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: 'tenant-a-api', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ canary }', + forbiddenMatches: [{ path: '/data/canary', value: 'tenant-b' }], + requiredMatches: [{ path: '/data/canary', value: 'tenant-a' }] + }] + }] + }] + } as FleetV1; + const plan = { + tenantCounts: [1], + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema', 'prepared-reuse'] + } as DensityPlanV1; + expect(() => validateCoverage(plan, fleet)).toThrow('lacks canaries: prepared-reuse'); + }); + + it('requires every tenant to configure every required capability', () => { + const surface = (tenant: string, capability: string) => ({ + name: 'api', + buildContract: `${tenant}-api`, + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability, query: '{ __typename }' }, + operations: [{ name: 'read', capability, query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ token }', + forbiddenMatches: [{ path: '/data/token', value: 'other' }], + requiredMatches: [{ path: '/data/token', value: tenant }] + }] + }); + const fleet = { + version: 1, + tenants: [ + { id: 'tenant-a', surfaces: [surface('tenant-a', 'graphile')] }, + { id: 'tenant-b', surfaces: [surface('tenant-b', 'bm25')] } + ] + } as FleetV1; + const plan = { + tenantCounts: [2], + requiredCapabilities: ['graphile', 'bm25'], + requiredCanaries: ['cross-schema'] + } as DensityPlanV1; + + expect(() => validateCoverage(plan, fleet)).toThrow( + 'tenant-a has no operations for capabilities: bm25' + ); + }); + + it('requires an exact contract for every arm when arm-specific identities are used', () => { + const fleet = { + version: 1, + tenants: [{ + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: '', + buildContracts: { stock: 'stock-hash' }, + url: 'http://127.0.0.1:{port}/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ token }', + forbiddenMatches: [{ path: '/data/token', value: 'tenant-b' }], + requiredMatches: [{ path: '/data/token', value: 'tenant-a' }] + }] + }] + }] + } as FleetV1; + const plan = { + arms: [ + { name: 'stock' }, + { name: 'scoped' } + ], + tenantCounts: [1], + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'] + } as DensityPlanV1; + + expect(() => validateCoverage(plan, fleet)).toThrow( + 'lacks exact build contracts for arms: scoped' + ); + }); + + it('rejects one build contract reused across different tenants', () => { + const makeTenant = (id: string) => ({ + id, + surfaces: [{ + name: 'api', + buildContract: 'shared-contract', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ token }', + forbiddenMatches: [{ path: '/data/token', value: 'other' }], + requiredMatches: [{ path: '/data/token', value: id }] + }] + }] + }); + const fleet = { + version: 1, + tenants: [makeTenant('tenant-a'), makeTenant('tenant-b')] + } as FleetV1; + const plan = { + tenantCounts: [1, 2], + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'] + } as DensityPlanV1; + expect(() => validateCoverage(plan, fleet)).toThrow( + "build contract 'shared-contract' for arm 'default' is reused across tenants" + ); + }); + + it('requires an explicit customer/database/API map for qualifying fleets', () => { + const surface = { + name: 'api', + buildContract: 'tenant-a-build', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ token }', + forbiddenMatches: [{ path: '/data/token', value: 'tenant-b' }], + requiredMatches: [{ path: '/data/token', value: 'tenant-a' }] + }] + }; + const fleet = { + version: 1, + tenants: [{ id: 'customer-a', surfaces: [surface] }] + } as FleetV1; + const plan = { + tenantCounts: [1], + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + gates: { requireExplicitCustomerTopology: true } + } as DensityPlanV1; + expect(() => validateCoverage(plan, fleet)).toThrow( + 'customer-a has no explicit customer -> database -> API topology' + ); + }); + + it('rejects one runtime pool identity reused across customers', () => { + const customer = (id: string) => ({ + id, + databases: [{ + id: `${id}-database`, + physicalDatabase: 'fixture', + apis: [{ + id: `${id}-api`, + runtimePoolIdentity: 'pg:v1:shared', + physicalSchemas: [`${id}_api`], + routingLabels: [`${id}.api.localhost`], + realtime: false, + surfaces: ['api'] + }] + }], + surfaces: [{ + name: 'api', + buildContract: `${id}-build`, + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ token }', + forbiddenMatches: [{ path: '/data/token', value: 'other' }], + requiredMatches: [{ path: '/data/token', value: id }] + }] + }] + }); + const fleet = { + version: 1, + tenants: [customer('customer-a'), customer('customer-b')] + } as FleetV1; + const plan = { + tenantCounts: [1, 2], + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + gates: { requireExplicitCustomerTopology: true } + } as DensityPlanV1; + expect(() => validateCoverage(plan, fleet)).toThrow( + "runtime pool identity 'pg:v1:shared' for arm 'default' is reused across customers" + ); + }); + + it('rejects a strict rotating qualification with fewer rounds than canaries', () => { + const canaries = Array.from({ length: 4 }, (_, index) => ({ + name: `canary-${index}`, + query: '{ token }', + forbiddenMatches: [{ path: '/data/token', value: 'tenant-b' }], + requiredMatches: [{ path: '/data/token', value: 'tenant-a' }] + })); + const fleet = { + version: 1, + tenants: [{ + id: 'customer-a', + surfaces: [{ + name: 'api', + buildContract: 'customer-a-api', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries + }] + }] + } as FleetV1; + const plan = { + tenantCounts: [1], + requiredCapabilities: ['graphile'], + requiredCanaries: canaries.map((canary) => canary.name), + workload: { + durationSec: 120, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 60, + periodicCanarySchedule: 'rotating-one', + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100 + }, + gates: { + requireExplicitCustomerTopology: false, + requireCompletePeriodicCanaryCoverage: true + } + } as DensityPlanV1; + expect(() => validateCoverage(plan, fleet)).toThrow( + 'rotating periodic canary schedule has 1 timed rounds but a qualifying surface configures 4 canaries' + ); + plan.workload.durationSec = 300; + expect(() => validateCoverage(plan, fleet)).not.toThrow(); + }); +}); diff --git a/packages/perf-harness/src/__tests__/evidence.test.ts b/packages/perf-harness/src/__tests__/evidence.test.ts new file mode 100644 index 0000000000..ea73dbca94 --- /dev/null +++ b/packages/perf-harness/src/__tests__/evidence.test.ts @@ -0,0 +1,244 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + readRealtimeCoverageEvidence, + readScoreContextEvidence, + scoreContextFromInput, + writeScoreContext +} from '../evidence'; +import { summarizeRealtimeReceiptEvidence } from '../realtime-evidence'; +import type { ScoreInput } from '../score'; +import type { RealtimeCorrelationReceipt } from '../types'; + +const hash = (value: string): string => createHash('sha256') + .update(value) + .digest('hex'); + +const artifactDir = (): string => fs.mkdtempSync( + path.join(os.tmpdir(), 'cperf-evidence-test-') +); + +const contextInput = ( + command: string[] = ['node', 'server.cjs', '--secrets', '/tmp/runtime-secrets.json'], + executionErrors: string[] = [] +): ScoreInput => ({ + arm: 'stock', + evidenceMode: 'diagnostic', + runKind: 'matrix', + heapMiB: 2048, + tenants: [{ + id: 'customer-secret-id', + surfaces: [{ + name: 'api', + buildContract: 'customer-secret-contract', + url: 'http://127.0.0.1:3000/graphql', + headers: { authorization: 'Bearer customer-secret-token' } + }] + }], + repetition: 1, + runOrderIndex: 1, + startedAt: '2026-08-02T00:00:00.000Z', + endedAt: '2026-08-02T00:00:05.000Z', + configuredDurationSec: 5, + serverExit: null, + externalServer: false, + executionErrors, + provenance: { command }, + provenanceErrors: [], + postgresRunAttestation: null +} as unknown as ScoreInput); + +const metadata = (knownRuntimeSecretValues: readonly string[] = []) => ({ + planSha256: 'a'.repeat(64), + fleetSha256: 'b'.repeat(64), + campaignId: 'c'.repeat(64), + scheduleSha256: 'd'.repeat(64), + previousResultPayloadSha256: null as string | null, + notBeforeEpochMs: Date.parse('2026-08-02T00:00:00.000Z'), + knownRuntimeSecretValues +}); + +const receipt = ( + sequence: number, + nonce: string, + timed = true +): RealtimeCorrelationReceipt => { + const sha256 = hash(nonce); + return { + sequence, + timed, + deadlineAt: '2026-08-02T00:02:00.000Z', + issuedAt: '2026-08-02T00:01:00.000Z', + issuedSha256: sha256, + primeResponseAt: '2026-08-02T00:01:00.010Z', + primeResponseSha256: sha256, + eventAt: '2026-08-02T00:01:00.020Z', + eventSha256: sha256 + }; +}; + +const realtimeSurface = ( + tenantId: string, + nonce: string +) => ({ + tenantId, + surface: 'api', + route: `/customer/${tenantId}/graphql`, + active: true, + verified: true, + deliveryEvents: 1, + deliveryRoundsStarted: 1, + deliveryRoundsVerified: 1, + deliveryRoundPending: false, + timedRoundsExpected: 1, + timedRoundsStarted: 1, + timedRoundsVerified: 1, + timedRoundsDeadlineLate: 0, + correlationReceipts: [receipt(1, nonce)] +}); + +const realtimeSnapshot = (surfaces: ReturnType[]) => { + const coverage = summarizeRealtimeReceiptEvidence({ + deliveryIntervalMs: 60_000, + workloadStartedAt: '2026-08-02T00:00:00.000Z', + workloadDeadlineAt: '2026-08-02T00:02:00.000Z', + workloadEndedAt: '2026-08-02T00:02:00.000Z', + surfaces: surfaces.map((surface) => ({ + tenantId: surface.tenantId, + surface: surface.surface, + route: surface.route, + expectedRecurringRounds: surface.timedRoundsExpected, + startedRecurringRounds: surface.timedRoundsStarted, + verifiedRecurringRounds: surface.timedRoundsVerified, + deadlineLateRecurringRounds: surface.timedRoundsDeadlineLate, + receipts: surface.correlationReceipts + })) + }).coverage; + return { + expected: surfaces.length, + active: surfaces.length, + verified: surfaces.length, + deliveryIntervalMs: 60_000, + deliveryEvents: surfaces.length, + deliveryRoundsStarted: 1, + deliveryRoundsVerified: 1, + deliveryRoundsPending: 0, + timedCoverage: coverage, + errors: [] as string[], + surfaces + }; +}; + +describe('density score evidence', () => { + it('persists only credential-free scoring context', () => { + const dir = artifactDir(); + const knownSecret = 'known-runtime-secret-marker'; + writeScoreContext(dir, contextInput(), metadata([knownSecret])); + + const serialized = fs.readFileSync(path.join(dir, 'score-context.json'), 'utf8'); + expect(serialized).not.toContain(knownSecret); + expect(serialized).not.toContain('customer-secret-id'); + expect(serialized).not.toContain('customer-secret-contract'); + expect(serialized).not.toContain('customer-secret-token'); + expect(serialized).not.toContain('authorization'); + expect(serialized).toContain('/tmp/runtime-secrets.json'); + }); + + it.each([ + ['known runtime value', ['node', '--label=known-runtime-secret-marker'], ['known-runtime-secret-marker']], + ['separate password', ['node', '--password', 'literal-password'], []], + ['URL userinfo', ['node', 'postgres://runtime:literal-password@localhost/db'], []], + ['URL token parameter', ['node', 'https://localhost/start?token=literal-token'], []], + ['authorization header', ['node', 'Authorization: Bearer literal-token'], []] + ])('rejects credential-bearing provenance: %s', (_label, command, knownSecrets) => { + expect(() => scoreContextFromInput( + contextInput(command), + metadata(knownSecrets) + )).toThrow('provenance command contains credential material'); + }); + + it('requires execution failures to contain only a stable code and digest', () => { + expect(() => scoreContextFromInput( + contextInput(undefined, ['CAPACITY']), + metadata() + )).toThrow('code-and-SHA-256 evidence'); + + const safe = `CAPACITY:sha256:${'c'.repeat(64)}`; + expect(scoreContextFromInput( + contextInput(undefined, [safe]), + metadata() + ).executionErrors).toEqual([safe]); + }); + + it('rejects credential material nested anywhere in provenance', () => { + const input = contextInput(['node', 'server.cjs']); + input.provenance = { + ...input.provenance!, + memoryPolicy: { + nested: { + password: 'nested-secret-value' + } + } + } as unknown as ScoreInput['provenance']; + expect(() => scoreContextFromInput(input, metadata())).toThrow( + 'provenance contains credential material at provenance.memoryPolicy.nested.password' + ); + }); + + it('rejects unversioned additions to the persisted context shape', () => { + const dir = artifactDir(); + writeScoreContext(dir, contextInput(), metadata()); + const file = path.join(dir, 'score-context.json'); + const context = JSON.parse(fs.readFileSync(file, 'utf8')); + context.tenants = ['customer-secret-id']; + fs.writeFileSync(file, `${JSON.stringify(context)}\n`, 'utf8'); + + expect(() => readScoreContextEvidence(dir)).toThrow( + 'unexpected=tenants' + ); + }); + + it('rejects a correlation digest reused across tenant routes', () => { + const dir = artifactDir(); + const snapshot = realtimeSnapshot([ + realtimeSurface('customer-a', 'shared-nonce'), + realtimeSurface('customer-b', 'shared-nonce') + ]); + fs.writeFileSync(path.join(dir, 'realtime-driver.json'), `${JSON.stringify([{ + phase: 'timed-coverage-complete', + timestamp: '2026-08-02T00:02:00.000Z', + snapshot + }])}\n`, 'utf8'); + + expect(() => readRealtimeCoverageEvidence(dir)).toThrow( + 'reused realtime receipt digest: customer-b/api' + ); + }); + + it('requires receipt and error histories to be append-only', () => { + const dir = artifactDir(); + const surface = realtimeSurface('customer-a', 'nonce-a'); + const first = realtimeSnapshot([surface]); + first.errors = ['delivery failed']; + const second = realtimeSnapshot([surface]); + fs.writeFileSync(path.join(dir, 'realtime-driver.json'), `${JSON.stringify([ + { + phase: 'failed', + timestamp: '2026-08-02T00:02:00.000Z', + snapshot: first + }, + { + phase: 'disposed-after-failure', + timestamp: '2026-08-02T00:02:01.000Z', + snapshot: second + } + ])}\n`, 'utf8'); + + expect(() => readRealtimeCoverageEvidence(dir)).toThrow( + 'realtime error history is not append-only' + ); + }); +}); diff --git a/packages/perf-harness/src/__tests__/http.test.ts b/packages/perf-harness/src/__tests__/http.test.ts new file mode 100644 index 0000000000..905351f598 --- /dev/null +++ b/packages/perf-harness/src/__tests__/http.test.ts @@ -0,0 +1,792 @@ +import http from 'node:http'; + +import { + createWorkloadCapture, + deterministicCanaryOffset, + deterministicOperationOffset, + jsonPointerValues, + resolveOfferedLoad, + resolveWarmupTimeoutMs, + rotatingCanaryIndex, + runWorkload +} from '../http'; +import type { GraphqlSurface, TenantTarget } from '../types'; + +describe('open-loop workload', () => { + let server: http.Server; + let url: string; + let activeSlowRequests = 0; + let peakSlowRequests = 0; + let slowWarmRequests = 0; + const verificationVariables: Array> = []; + + beforeAll(async () => { + server = http.createServer(async (request, response) => { + let raw = ''; + for await (const chunk of request) raw += String(chunk); + const payload = JSON.parse(raw || '{}') as { + query?: string; + variables?: Record; + }; + const query = payload.query ?? ''; + if (query.includes('Slow')) { + if (query.includes('SlowWarm')) slowWarmRequests++; + activeSlowRequests++; + peakSlowRequests = Math.max(peakSlowRequests, activeSlowRequests); + await new Promise((resolve) => setTimeout(resolve, 40)); + activeSlowRequests--; + } + const physicalDatabaseIdentity = query.includes('ForeignPhysicalOracle') + ? 'physical-db-b' + : query.includes('MissingPhysicalOracle') + ? undefined + : 'physical-db-a'; + response.setHeader('content-type', 'application/json'); + if (query.includes('PartialForeignCanary')) { + response.end(JSON.stringify({ + data: { tenantToken: 'tenant-b-token' }, + errors: [{ message: 'partial resolver failure', extensions: { code: 'PARTIAL' } }] + })); + return; + } + if (query.includes('UniversalRows')) { + const nodes = query.includes('Empty') + ? [] + : query.includes('Foreign') + ? [ + { physicalDatabaseIdentity: 'physical-db-a' }, + { physicalDatabaseIdentity: 'physical-db-b' } + ] + : [{ physicalDatabaseIdentity: 'physical-db-a' }]; + response.end(JSON.stringify({ data: { documents: { nodes } } })); + return; + } + if (query.includes('CorrelatedUploadSubject')) { + response.end(JSON.stringify({ + data: { uploadAppFile: { fileId: 'file-current' } } + })); + return; + } + if (query.includes('AmbiguousCorrelationSubject')) { + response.end(JSON.stringify({ + data: { + uploads: [{ fileId: 'file-one' }, { fileId: 'file-two' }] + } + })); + return; + } + if (query.includes('MissingCorrelationSubject')) { + response.end(JSON.stringify({ data: { uploadAppFile: {} } })); + return; + } + if (query.includes('VerifyCorrelatedUpload')) { + verificationVariables.push(payload.variables ?? {}); + response.end(JSON.stringify({ + data: { + physicalDatabaseIdentity: payload.variables?.fileId === 'file-current' + ? 'physical-db-a' + : 'physical-db-b' + } + })); + return; + } + response.end(JSON.stringify({ + data: { + tenantToken: 'tenant-a-token', + ...(physicalDatabaseIdentity === undefined + ? {} + : { physicalDatabaseIdentity }) + }, + extensions: { note: 'tenant-b-token appears outside the asserted path' } + })); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('test server has no TCP address'); + url = `http://127.0.0.1:${address.port}/graphql`; + }); + + afterAll(async () => { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + }); + + const surface = ( + name: string, + warmupQuery = '{ tenantToken }', + operationQuery = '{ tenantToken }' + ): GraphqlSurface => ({ + name, + buildContract: `tenant-a-${name}`, + url, + warmup: { name: 'warm', capability: 'generated', query: warmupQuery }, + operations: [ + { name: 'generated', capability: 'generated', weight: 0.1, query: operationQuery }, + { name: 'search', capability: 'bm25', weight: 0.1, query: operationQuery } + ], + canaries: [{ + name: 'cross-schema', + query: '{ tenantToken }', + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b-token' }], + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a-token' }] + }] + }); + + it('stably staggers weighted operation cursors across tenant surfaces', () => { + const offsets = [ + deterministicOperationOffset('physical-customer-0001', 'a', 100), + deterministicOperationOffset('physical-customer-0001', 'b', 100), + deterministicOperationOffset('physical-customer-0002', 'a', 100) + ]; + expect(new Set(offsets).size).toBe(3); + expect(deterministicOperationOffset('physical-customer-0001', 'a', 100)) + .toBe(offsets[0]); + expect(offsets.every((offset) => offset >= 0 && offset < 100)).toBe(true); + expect(deterministicOperationOffset('tenant', 'api', 0)).toBe(0); + }); + + it('fails operation samples closed with stable missing and forbidden oracle codes', async () => { + const run = async (query: string) => { + const api = surface('api', '{ tenantToken }', query); + api.operations = [{ + name: 'physical-read', + capability: 'generated', + query, + requiredMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-a' + }], + forbiddenMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-b' + }] + }]; + return runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + }; + + const missing = await run('query MissingPhysicalOracle { physicalDatabaseIdentity }'); + expect(missing.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: false, + oracleConfigured: true, + oracleConclusive: false, + oracleViolation: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_MISSING' + }); + + const forbidden = await run('query ForeignPhysicalOracle { physicalDatabaseIdentity }'); + expect(forbidden.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: false, + oracleConfigured: true, + oracleViolation: true, + errorCode: 'GRAPHQL_OPERATION_ORACLE_FORBIDDEN' + }); + }); + + it('enforces nonempty cardinality and every-value invariants across collections', async () => { + const run = async (query: string, requiredMatches: any[]) => { + const api = surface('api', '{ tenantToken }', query); + api.operations = [{ + name: 'universal-read', + capability: 'generated', + query, + requiredMatches, + forbiddenMatches: [{ + path: '/data/documents/nodes/*/physicalDatabaseIdentity', + value: 'physical-db-c' + }], + invariants: [{ + path: '/data/documents/nodes/*/physicalDatabaseIdentity', + everyEquals: 'physical-db-a', + min: 1, + max: 1 + }] + }]; + return runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + }; + + const foreign = await run('query UniversalRowsForeign { documents { nodes { physicalDatabaseIdentity } } }', [{ + path: '/data/documents/nodes/0/physicalDatabaseIdentity', + value: 'physical-db-a' + }]); + expect(foreign.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: false, + oracleConclusive: true, + oracleViolation: true, + oracleUnavailable: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_INVARIANT_UNEXPECTED' + }); + + const empty = await run('query UniversalRowsEmpty { documents { nodes { physicalDatabaseIdentity } } }', [{ + path: '/data/documents/nodes', + value: [] + }]); + expect(empty.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: false, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_INVARIANT_MISSING' + }); + }); + + it('keeps forbidden canary evidence conclusive when GraphQL also returns errors', async () => { + const api = surface('api'); + api.canaries = [{ + name: 'partial-foreign', + query: 'query PartialForeignCanary { tenantToken }', + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a-token' }], + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b-token' }] + }]; + const result = await runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + expect(result.canaries).toHaveLength(2); + expect(result.canaries.every((canary) => + canary.conclusive && canary.violation + )).toBe(true); + expect(result.canaries[0].detail).toBe('GRAPHQL_OPERATION_ORACLE_FORBIDDEN'); + }); + + it('uses an untimed post-coverage query as mutation side-effect evidence', async () => { + const api = surface('api', '{ tenantToken }', 'mutation UploadSubject { __typename }'); + api.operations = [{ + name: 'upload-subject', + capability: 'uploads', + query: 'mutation UploadSubject { __typename }', + postCoverageVerification: { + query: 'query VerifySideEffect { physicalDatabaseIdentity }', + requiredMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-a' + }], + forbiddenMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-b' + }] + } + }]; + const result = await runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + expect(result.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + operation: 'upload-subject', + ok: true, + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: false, + postCoverageVerification: true + }); + }); + + it('extracts post-verification variables exactly from the primary response', async () => { + verificationVariables.length = 0; + const api = surface('api', '{ tenantToken }', 'mutation CorrelatedUploadSubject { uploadAppFile { fileId } }'); + api.operations = [{ + name: 'correlated-upload', + capability: 'uploads', + query: 'mutation CorrelatedUploadSubject { uploadAppFile { fileId } }', + postCoverageVerification: { + query: 'query VerifyCorrelatedUpload($fileId: ID!, $contentHash: String!) { physicalDatabaseIdentity }', + variables: { contentHash: 'current-hash' }, + variablesFromResponse: { fileId: '/data/uploadAppFile/fileId' }, + requiredMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-a' + }], + forbiddenMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-b' + }] + } + }]; + const result = await runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + expect(result.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: true, + oracleConclusive: true, + postCoverageVerification: true + }); + expect(verificationVariables).toEqual([{ + contentHash: 'current-hash', + fileId: 'file-current' + }]); + }); + + it('fails post-verification before I/O on missing or ambiguous correlation evidence', async () => { + const run = async (query: string, pointer: string) => { + const api = surface('api', '{ tenantToken }', query); + api.operations = [{ + name: 'correlation-failure', + capability: 'uploads', + query, + postCoverageVerification: { + query: 'query VerifyCorrelatedUpload($fileId: ID!) { physicalDatabaseIdentity }', + variablesFromResponse: { fileId: pointer }, + requiredMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-a' + }], + forbiddenMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-b' + }] + } + }]; + return runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + }; + + verificationVariables.length = 0; + const missing = await run( + 'mutation MissingCorrelationSubject { uploadAppFile { fileId } }', + '/data/uploadAppFile/fileId' + ); + expect(missing.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: false, + oracleConclusive: false, + oracleUnavailable: false, + errorCode: 'GRAPHQL_POST_COVERAGE_VARIABLE_MISSING' + }); + + const ambiguous = await run( + 'mutation AmbiguousCorrelationSubject { uploads { fileId } }', + '/data/uploads/*/fileId' + ); + expect(ambiguous.samples.find((sample) => sample.phase === 'coverage')) + .toMatchObject({ + ok: false, + oracleConclusive: false, + oracleUnavailable: false, + errorCode: 'GRAPHQL_POST_COVERAGE_VARIABLE_AMBIGUOUS' + }); + expect(verificationVariables).toEqual([]); + }); + + it('rotates through a stable, staggered 14-round canary permutation', () => { + const permutation = (tenantId: string, surfaceName: string) => + Array.from({ length: 14 }, (_unused, index) => + rotatingCanaryIndex(tenantId, surfaceName, 14, index + 1) + ); + const first = permutation('physical-customer-0001', 'api'); + expect(new Set(first)).toEqual(new Set(Array.from({ length: 14 }, (_, index) => index))); + expect(permutation('physical-customer-0001', 'api')).toEqual(first); + const offsets = [ + deterministicCanaryOffset('physical-customer-0001', 'api', 14), + deterministicCanaryOffset('physical-customer-0001', 'admin', 14), + deterministicCanaryOffset('physical-customer-0002', 'api', 14) + ]; + expect(new Set(offsets).size).toBeGreaterThan(1); + expect(permutation('physical-customer-0001', 'admin')).not.toEqual(first); + }); + + it('runs four rotating periodic canaries in four strict timed rounds', async () => { + const api = surface('api'); + api.canaries = Array.from({ length: 4 }, (_, index) => ({ + name: `canary-${index}`, + query: '{ tenantToken }', + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b-token' }], + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a-token' }] + })); + const result = await runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.25, + rps: 4, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 4, + canaryIntervalSec: 0.05, + periodicCanarySchedule: 'rotating-one', + canaryConcurrency: 2, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + + const periodic = result.canaries.filter((canary) => canary.phase === 'periodic'); + expect(periodic).toHaveLength(4); + expect(periodic.map((canary) => canary.periodicRound)).toEqual([1, 2, 3, 4]); + expect(new Set(periodic.map((canary) => canary.canary))).toEqual( + new Set(api.canaries.map((canary) => canary.name)) + ); + expect(result.canaries.filter((canary) => canary.phase === 'initial')).toHaveLength(4); + expect(result.canaries.filter((canary) => canary.phase === 'final')).toHaveLength(4); + expect(result.canarySchedule).toMatchObject({ + schedule: 'rotating-one', + planned: 4, + started: 4, + completed: 4, + missed: 0, + checksPlanned: 4, + checksStarted: 4, + checksCompleted: 4 + }); + expect(periodic.every((canary) => + Date.parse(canary.completedAt) >= Date.parse(canary.startedAt) + && canary.latencyMs >= 0 + )).toBe(true); + }); + + it('serializes overlapping rounds without dropping them and records deadline-late drain', async () => { + const api = surface('api'); + api.canaries = [{ + name: 'slow-canary', + query: '{ SlowCanary: tenantToken }', + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b-token' }], + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a-token' }] + }]; + const startedAt = performance.now(); + const result = await runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.1, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 2, + canaryIntervalSec: 0.02, + periodicCanarySchedule: 'rotating-one', + canaryConcurrency: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + + expect(performance.now() - startedAt).toBeLessThan(1_500); + expect(result.canaries.filter((canary) => canary.phase === 'periodic')).toHaveLength(4); + expect(result.canarySchedule).toMatchObject({ + planned: 4, + started: 4, + completed: 4, + missed: 0 + }); + expect(result.canarySchedule.overlapped).toBeGreaterThan(0); + expect(result.canarySchedule.deadlineLate).toBeGreaterThan(0); + expect(result.canarySchedule.rounds.every((round) => + round.targetsStarted === 1 + && round.targetsCompleted === 1 + && round.checksCompleted === 1 + )).toBe(true); + }); + + it('warms every surface and proves every configured operation received traffic', async () => { + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: [surface('api')] + }]; + const result = await runWorkload(tenants, { + durationSec: 0.2, + rps: 10, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 4, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + + expect(result.warmedSurfaces.get('tenant-a')).toEqual(new Set(['api'])); + expect(result.capabilities).toEqual(new Set(['generated', 'bm25'])); + expect(result.capabilitiesByTenantSurface.get('tenant-a/api')) + .toEqual(new Set(['generated', 'bm25'])); + expect(new Set(result.samples.filter((sample) => sample.ok).map((sample) => sample.operation))) + .toEqual(new Set(['generated', 'search'])); + expect(result.canaries.length).toBeGreaterThanOrEqual(2); + expect(result.canaries.every((canary) => canary.conclusive && !canary.violation)).toBe(true); + expect(result.missedArrivals).toBe(0); + }); + + it('signals the warm boundary after coverage and initial canaries but before timed load', async () => { + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: [surface('api'), surface('admin')] + }]; + const capture = createWorkloadCapture(); + let boundaryCalls = 0; + + const result = await runWorkload(tenants, { + durationSec: 0.1, + rps: 20, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 4, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 2 + }, async () => { + boundaryCalls++; + expect(capture.warmedSurfaces.get('tenant-a')).toEqual(new Set(['api', 'admin'])); + expect(capture.samples.filter((sample) => sample.phase === 'coverage')).toHaveLength(4); + expect(capture.samples.filter((sample) => sample.phase === 'workload')).toHaveLength(0); + expect(capture.capabilitiesByTenantSurface.get('tenant-a/api')) + .toEqual(new Set(['generated', 'bm25'])); + expect(capture.capabilitiesByTenantSurface.get('tenant-a/admin')) + .toEqual(new Set(['generated', 'bm25'])); + expect(capture.canaries).toHaveLength(2); + expect(capture.canaries.every((canary) => canary.conclusive && !canary.violation)) + .toBe(true); + }, capture); + + expect(boundaryCalls).toBe(1); + expect(result.samples.some((sample) => sample.phase === 'workload')).toBe(true); + expect(result.canaries.length).toBeGreaterThan(2); + }); + + it('fails closed before timed load when the warm-boundary callback rejects', async () => { + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: [surface('api')] + }]; + const capture = createWorkloadCapture(); + + await expect(runWorkload(tenants, { + durationSec: 0.1, + rps: 20, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 4, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }, async () => { + expect(capture.samples.filter((sample) => sample.phase === 'coverage')).toHaveLength(2); + expect(capture.canaries).toHaveLength(1); + throw new Error('warm-boundary setup failed'); + }, capture)).rejects.toThrow('warm-boundary setup failed'); + + expect(capture.samples.filter((sample) => sample.phase === 'workload')).toHaveLength(0); + }); + + it('submits each surface canary sequentially so validation cannot monopolize its pool', async () => { + peakSlowRequests = 0; + const api = surface('api'); + api.canaries = Array.from({ length: 4 }, (_, index) => ({ + name: `slow-canary-${index}`, + query: `{ SlowCanary${index}: tenantToken }`, + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b-token' }], + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a-token' }] + })); + + const result = await runWorkload([{ id: 'tenant-a', surfaces: [api] }], { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 8, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + + expect(result.canaries).toHaveLength(8); + expect(result.canaries.every((canary) => canary.conclusive && !canary.violation)) + .toBe(true); + expect(peakSlowRequests).toBe(1); + }); + + it('uses typed JSON pointers, including wildcards, instead of raw response substrings', () => { + const body = { + data: { rows: [{ token: 'tenant-a' }, { token: 'tenant-b' }] }, + extensions: { note: 'tenant-c' } + }; + expect(jsonPointerValues(body, '/data/rows/*/token')).toEqual(['tenant-a', 'tenant-b']); + expect(jsonPointerValues(body, '/data/missing')).toEqual([]); + expect(jsonPointerValues(body, '/extensions/note')).toEqual(['tenant-c']); + }); + + it('bounds concurrent warmups with the configured limit', async () => { + peakSlowRequests = 0; + slowWarmRequests = 0; + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: Array.from({ length: 5 }, (_, index) => + surface(`api-${index}`, `{ SlowWarm${index}: tenantToken }`) + ) + }]; + + const result = await runWorkload(tenants, { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 8, + canaryIntervalSec: 1, + warmupTimeoutMs: 2_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 2 + }); + + expect(result.warmedSurfaces.get('tenant-a')?.size).toBe(5); + expect(peakSlowRequests).toBeLessThanOrEqual(2); + }); + + it('uses one global warmup deadline instead of starting queued work after expiry', async () => { + slowWarmRequests = 0; + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: Array.from({ length: 5 }, (_, index) => + surface(`api-${index}`, `{ SlowWarm${index}: tenantToken }`) + ) + }]; + + const result = await runWorkload(tenants, { + durationSec: 0.02, + rps: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 10, + warmupTimeoutPerSurfaceMs: 1, + warmupConcurrency: 1 + }); + + expect(slowWarmRequests).toBeLessThanOrEqual(1); + expect(result.warmedSurfaces.get('tenant-a')).toBeUndefined(); + }); + + it('records saturated arrivals as failures without dispatching a catch-up burst', async () => { + peakSlowRequests = 0; + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: [surface('api', '{ tenantToken }', '{ SlowOperation: tenantToken }')] + }]; + + const result = await runWorkload(tenants, { + durationSec: 0.12, + rps: 100, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 1, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + + expect(result.missedArrivals).toBeGreaterThan(0); + expect(result.samples.filter((sample) => + sample.errorCode === 'LOAD_GENERATOR_MISSED_ARRIVAL' + )).toHaveLength(result.missedArrivals); + expect(peakSlowRequests).toBe(1); + }); + + it('measures workload latency from the scheduled open-loop arrival', async () => { + const tenants: TenantTarget[] = [{ + id: 'tenant-a', + surfaces: [surface('api', '{ tenantToken }', '{ SlowOperation: tenantToken }')] + }]; + + const result = await runWorkload(tenants, { + durationSec: 0.08, + rps: 20, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 2, + canaryIntervalSec: 1, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 100, + warmupConcurrency: 1 + }); + + const dispatched = result.samples.filter((sample) => + sample.phase === 'workload' + && sample.errorCode !== 'LOAD_GENERATOR_MISSED_ARRIVAL' + ); + expect(dispatched.length).toBeGreaterThan(0); + expect(dispatched.every((sample) => sample.scheduledAtMs != null)).toBe(true); + expect(dispatched.every((sample) => sample.latencyMs >= 35)).toBe(true); + }); + + it('resolves fixed-total and per-tenant offered load explicitly', () => { + expect(resolveOfferedLoad({ rps: 50 }, 10)).toEqual({ + mode: 'fixed-total', + configuredRps: 50, + tenantCount: 10, + totalRps: 50, + rpsPerTenant: 5 + }); + expect(resolveOfferedLoad({ rpsPerTenant: 2 }, 10)).toEqual({ + mode: 'per-tenant', + configuredRps: 2, + tenantCount: 10, + totalRps: 20, + rpsPerTenant: 2 + }); + expect(() => resolveOfferedLoad({ rps: 1, rpsPerTenant: 1 }, 1)) + .toThrow('exactly one'); + }); + + it('scales the global warmup deadline by concurrency waves', () => { + expect(resolveWarmupTimeoutMs({ + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 500, + warmupConcurrency: 2 + }, 10)).toBe(2_500); + expect(resolveWarmupTimeoutMs({ + warmupTimeoutMs: 10_000, + warmupTimeoutPerSurfaceMs: 500, + warmupConcurrency: 2 + }, 10)).toBe(10_000); + }); +}); diff --git a/packages/perf-harness/src/__tests__/memory.test.ts b/packages/perf-harness/src/__tests__/memory.test.ts new file mode 100644 index 0000000000..3378ac4c69 --- /dev/null +++ b/packages/perf-harness/src/__tests__/memory.test.ts @@ -0,0 +1,454 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + normalizeMemorySnapshot, + normalizeRetainedMemoryCheckpoint, + readLinuxProcessMemory, + startMemorySampler +} from '../memory'; + +describe('memory snapshot normalization', () => { + it('normalizes complete retained-memory checkpoints and rejects truncated samples', () => { + const guard = { + pid: 55, + graphileInFlight: 0, + residentBuildContracts: ['contract'], + stateSha256: `sha256:${'a'.repeat(64)}`, + state: { pid: 55, graphileInFlight: 0 } + }; + const raw = { + version: 1, + fixture: 'physical-database-density-v1', + pid: 55, + gcRounds: 8, + stableSampleCount: 3, + stable: true, + samples: Array.from({ length: 8 }, (_, index) => ({ + timestamp: `2026-08-01T00:00:0${index}.000Z`, + monotonicNs: String(index + 1), + heapUsedBytes: 100, + externalBytes: 20, + arrayBuffersBytes: 10, + rssBytes: 200 + })), + guardBefore: guard, + guardAfter: guard, + errors: [] as string[] + }; + expect(normalizeRetainedMemoryCheckpoint(raw)?.samples).toHaveLength(8); + expect(normalizeRetainedMemoryCheckpoint({ ...raw, samples: raw.samples.slice(1) })) + .toBeNull(); + }); + + it('sums stable cache/governor counters', () => { + const snapshot = normalizeMemorySnapshot({ + timestamp: '2026-07-31T00:00:00.000Z', + pid: 123, + nodeEnv: 'production', + memory: { heapUsedBytes: 10, rssBytes: 20 }, + resourceUsage: { maxRSS: 40 }, + v8: { heapStatistics: { heap_size_limit: 1024 } }, + graphileCache: { + size: 3, + max: 9, + admissionMode: 'preserve-resident', + budgetCapacity: 8, + instanceHeapBytes: 16 * 1024 ** 2, + calibration: { id: 'measured-cache-v1' }, + keys: ['build-a', 'build-b'] + }, + graphileCacheCounters: { + evictions: { lru: 1, ttl: 2 }, + buildRefusals: { critical_pressure: 4, resident_busy: 5 } + }, + graphileGovernor: { buildsStarted: 7 }, + graphileBuilds: { succeeded: 6, maxMs: 42 }, + pgCache: { + size: 12, + leasedPools: 4, + activeLeases: 6, + capacityEvictions: 1, + capacityRefusals: 2, + disposalFailures: 3 + }, + physicalDatabaseFixture: { + physicalDatabases: 3, + containerScope: { dedicated: true, unexpectedDatabases: 0 }, + pools: { + scope: 'runtime-only-exact-identities', + available: true, + requestedMaxUses: 1, + effectiveMaxUses: 1, + effectiveMaxUsesKnown: true, + maxUsesExact: true, + expectedPools: 9, + observedPools: 9, + totalClients: 8, + idleClients: 2, + waitingClients: 1 + }, + backends: { total: 9, active: 2, idle: 6, idleInTransaction: 1 }, + realtime: { + managersExpected: 6, + managersActive: 6, + transportsExpected: 6, + transportsActive: 6, + notificationMode: 'shared-exact', + notificationBrokers: { + brokers: 3, + listenerConnections: 3, + leases: 6, + topics: 6, + subscribers: 6, + queueOverflows: 0, + fatalFailures: 0 + }, + notificationRoleAudits: { + identities: 3, + healthy: 3, + failed: 0, + stale: 0, + catalogAuditAttempts: 6, + catalogAuditFailures: 0, + activeDatabaseTargets: 3, + databaseConfigurationConflicts: 0 + } + } + } + }); + expect(snapshot).toMatchObject({ + heapUsedBytes: 10, + rssBytes: 20, + pid: 123, + nodeEnv: 'production', + heapLimitBytes: 1024, + processPeakRssBytes: 40 * 1024, + cacheSize: 3, + cacheConfiguredMax: 9, + cacheBudgetCapacity: 8, + cacheInstanceHeapBytes: 16 * 1024 ** 2, + cacheCalibrationId: 'measured-cache-v1', + cacheAdmissionMode: 'preserve-resident', + residentBuildContracts: ['build-a', 'build-b'], + evictions: 3, + buildRefusals: 9, + buildsStarted: 7, + buildsSucceeded: 6, + buildMaxMs: 42, + pgPoolCacheSize: 12, + pgPoolLeasedPools: 4, + pgPoolActiveLeases: 6, + pgPoolCapacityEvictions: 1, + pgPoolCapacityRefusals: 2, + pgPoolDisposalFailures: 3, + pgPoolTotalClients: 8, + pgPoolIdleClients: 2, + pgPoolWaitingClients: 1, + runtimePoolTelemetryScope: 'runtime-only-exact-identities', + runtimePoolTelemetryAvailable: true, + runtimePoolRequestedMaxUses: 1, + runtimePoolEffectiveMaxUses: 1, + runtimePoolEffectiveMaxUsesKnown: true, + runtimePoolMaxUsesExact: true, + runtimePoolExpectedPools: 9, + runtimePoolObservedPools: 9, + runtimePoolTotalClients: 8, + runtimePoolIdleClients: 2, + runtimePoolWaitingClients: 1, + postgresBackendTotal: 9, + postgresBackendActive: 2, + postgresBackendIdle: 6, + postgresBackendIdleInTransaction: 1, + physicalDatabases: 3, + postgresContainerDedicated: true, + unexpectedPostgresDatabases: 0, + realtimeManagersExpected: 6, + realtimeManagersActive: 6, + realtimeTransportsExpected: 6, + realtimeTransportsActive: 6, + realtimeNotificationMode: 'shared-exact', + notificationBrokers: 3, + notificationListenerConnections: 3, + notificationBrokerLeases: 6, + notificationBrokerTopics: 6, + notificationBrokerSubscribers: 6, + notificationBrokerQueueOverflows: 0, + notificationBrokerFatalFailures: 0, + notificationAuditIdentities: 3, + notificationAuditsHealthy: 3, + notificationAuditsFailed: 0, + notificationAuditsStale: 0, + notificationAuditAttempts: 6, + notificationAuditFailures: 0, + notificationAuditActiveDatabaseTargets: 3, + notificationAuditDatabaseConflicts: 0, + cacheCountersAvailable: true, + buildCountersAvailable: true + }); + }); + + it('keeps missing measurements and counters null for an older endpoint', () => { + const snapshot = normalizeMemorySnapshot({ + graphileCache: {} + }); + expect(snapshot).toMatchObject({ + pid: null, + nodeEnv: null, + heapLimitBytes: null, + heapUsedBytes: null, + rssBytes: null, + processPeakRssBytes: null, + cacheSize: null, + cacheConfiguredMax: null, + cacheBudgetCapacity: null, + cacheInstanceHeapBytes: null, + cacheCalibrationId: null, + residentBuildContracts: null, + evictions: null, + buildRefusals: null, + buildsStarted: null, + buildsSucceeded: null, + buildMaxMs: null, + pgPoolCacheSize: null, + pgPoolLeasedPools: null, + pgPoolActiveLeases: null, + pgPoolCapacityEvictions: null, + pgPoolCapacityRefusals: null, + pgPoolDisposalFailures: null, + cacheCountersAvailable: false, + buildCountersAvailable: false + }); + }); + + it('reads Linux current and high-water RSS from the exact pid status file', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-proc-')); + const pid = 4321; + const pidDir = path.join(root, String(pid)); + fs.mkdirSync(pidDir); + fs.writeFileSync( + path.join(pidDir, 'status'), + 'Name:\tnode\nVmHWM:\t2048 kB\nVmRSS:\t1024 kB\n', + 'utf8' + ); + expect(readLinuxProcessMemory(pid, root)).toEqual({ + rssBytes: 1024 * 1024, + peakRssBytes: 2048 * 1024 + }); + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('reports proc read and malformed-status failures instead of silently dropping samples', () => { + const errors: string[] = []; + expect(readLinuxProcessMemory(9876, '/definitely-not-proc', (error) => { + errors.push(error); + })).toBeNull(); + expect(errors[0]).toContain('OS RSS proc read failed for pid 9876'); + + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-proc-invalid-')); + const pid = 9877; + const pidDir = path.join(root, String(pid)); + fs.mkdirSync(pidDir); + fs.writeFileSync(path.join(pidDir, 'status'), 'Name:\tnode\n', 'utf8'); + expect(readLinuxProcessMemory(pid, root, (error) => errors.push(error))).toBeNull(); + expect(errors.slice(-2)).toEqual([ + 'OS RSS proc status for pid 9877 omitted VmRSS', + 'OS RSS proc status for pid 9877 omitted VmHWM' + ]); + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('binds endpoint identity and heap limit while retaining resourceUsage peak RSS', async () => { + const originalFetch = global.fetch; + const authorization = `Bearer ${'test-observability-token-'.repeat(2)}`; + const fetchMock = jest.fn(async ( + _input: string | URL | Request, + _init?: RequestInit + ) => ({ + ok: true, + json: async () => ({ + timestamp: '2026-08-01T00:00:00.000Z', + pid: 55, + nodeEnv: 'production', + memory: { heapUsedBytes: 100, rssBytes: 200 }, + resourceUsage: { maxRSS: 300 }, + v8: { heapStatistics: { heap_size_limit: 400 } }, + graphileCache: { size: 1, keys: ['contract'] }, + graphileCacheCounters: { evictions: {}, buildRefusals: {} }, + graphileGovernor: { buildsStarted: 0 }, + graphileBuilds: { succeeded: 0, maxMs: 0 }, + pgCache: { + size: 2, + leasedPools: 1, + activeLeases: 1, + capacityEvictions: 0, + capacityRefusals: 0, + disposalFailures: 0 + } + }) + })); + global.fetch = fetchMock as unknown as typeof fetch; + try { + const sampler = startMemorySampler('http://127.0.0.1/debug/memory', { + intervalMs: 60_000, + osSampleIntervalMs: 60_000, + expectedPid: 55, + expectedHeapLimitBytes: 400, + procRoot: '/definitely-not-proc', + headers: { Authorization: authorization } + }); + await sampler.ready; + await sampler.markWarmupComplete(); + expect(sampler.warmupIndex).toBe(1); + expect(sampler.snapshots).toHaveLength(2); + await sampler.stop(); + expect(sampler.errors).toEqual([ + expect.stringContaining('OS RSS proc read failed for pid 55') + ]); + expect(sampler.snapshots[0]).toMatchObject({ + pid: 55, + heapLimitBytes: 400, + processPeakRssBytes: 300 * 1024 + }); + expect(fetchMock.mock.calls.every(([, init]) => + ((init as RequestInit).headers as Record)?.Authorization === authorization + )).toBe(true); + expect(JSON.stringify(sampler).includes(authorization)).toBe(false); + } finally { + global.fetch = originalFetch; + } + }); + + it('samples exact-pid current RSS through the authenticated endpoint', async () => { + const originalFetch = global.fetch; + const authorization = `Bearer ${'darwin-observability-token-'.repeat(2)}`; + const fetchMock = jest.fn(async ( + _input: string | URL | Request, + _init?: RequestInit + ) => ({ + ok: true, + json: async () => ({ + timestamp: '2000-01-01T00:00:00.000Z', + pid: 55, + nodeEnv: 'production', + memory: { heapUsedBytes: 100, rssBytes: 200 }, + resourceUsage: { maxRSS: 300 }, + v8: { heapStatistics: { heap_size_limit: 400 } }, + graphileCache: { size: 1, keys: ['contract'] }, + graphileCacheCounters: { evictions: {}, buildRefusals: {} }, + graphileGovernor: { buildsStarted: 0 }, + graphileBuilds: { succeeded: 0, maxMs: 0 }, + pgCache: { + size: 2, + leasedPools: 1, + activeLeases: 1, + capacityEvictions: 0, + capacityRefusals: 0, + disposalFailures: 0 + } + }) + })); + global.fetch = fetchMock as unknown as typeof fetch; + const startedAtMs = Date.now(); + try { + const sampler = startMemorySampler('http://127.0.0.1/debug/memory', { + intervalMs: 60_000, + osSampleIntervalMs: 60_000, + expectedPid: 55, + expectedHeapLimitBytes: 400, + currentRssSource: 'authenticated-endpoint', + headers: { Authorization: authorization } + }); + await sampler.ready; + await sampler.markWarmupComplete(); + await sampler.stop(); + + expect(sampler.errors).toEqual([]); + expect(sampler.osSnapshots.length).toBeGreaterThanOrEqual(3); + expect(sampler.osSnapshots.every(({ rssBytes, timestamp }) => + rssBytes === 200 + && Date.parse(timestamp) >= startedAtMs + && timestamp !== '2000-01-01T00:00:00.000Z' + )).toBe(true); + expect(sampler.osPeakRssBytes).toBe(300 * 1024); + expect(fetchMock.mock.calls.every(([, init]) => + ((init as RequestInit).headers as Record)?.Authorization + === authorization + )).toBe(true); + } finally { + global.fetch = originalFetch; + } + }); + + it('fails closed when endpoint RSS sampling lacks bearer authorization', async () => { + const originalFetch = global.fetch; + global.fetch = jest.fn(async () => ({ + ok: true, + json: async () => ({ + pid: 55, + nodeEnv: 'production', + memory: { heapUsedBytes: 100, rssBytes: 200 }, + resourceUsage: { maxRSS: 300 }, + v8: { heapStatistics: { heap_size_limit: 400 } }, + pgCache: { + size: 2, + leasedPools: 1, + activeLeases: 1, + capacityEvictions: 0, + capacityRefusals: 0, + disposalFailures: 0 + } + }) + })) as unknown as typeof fetch; + try { + const sampler = startMemorySampler('http://127.0.0.1/debug/memory', { + intervalMs: 60_000, + osSampleIntervalMs: 60_000, + expectedPid: 55, + expectedHeapLimitBytes: 400, + currentRssSource: 'authenticated-endpoint' + }); + await sampler.ready; + await sampler.stop(); + expect(sampler.osSnapshots).toHaveLength(0); + expect(sampler.errors).toContain( + 'authenticated memory-endpoint RSS sampling requires bearer authorization' + ); + } finally { + global.fetch = originalFetch; + } + }); + + it('records identity and heap mismatches instead of accepting the sample silently', async () => { + const originalFetch = global.fetch; + global.fetch = jest.fn(async () => ({ + ok: true, + json: async () => ({ + pid: 56, + nodeEnv: 'development', + memory: { heapUsedBytes: 100, rssBytes: 200 }, + resourceUsage: { maxRSS: 300 }, + v8: { heapStatistics: { heap_size_limit: 401 } } + }) + })) as unknown as typeof fetch; + try { + const sampler = startMemorySampler('http://127.0.0.1/debug/memory', { + intervalMs: 60_000, + osSampleIntervalMs: 60_000, + expectedPid: 55, + expectedHeapLimitBytes: 400, + procRoot: '/definitely-not-proc' + }); + await sampler.ready; + await sampler.stop(); + expect(sampler.errors).toEqual(expect.arrayContaining([ + 'memory endpoint pid mismatch: expected 55, observed 56', + 'memory endpoint NODE_ENV must be production, observed development', + 'V8 heap limit mismatch: expected 400, observed 401' + ])); + } finally { + global.fetch = originalFetch; + } + }); +}); diff --git a/packages/perf-harness/src/__tests__/postgres.test.ts b/packages/perf-harness/src/__tests__/postgres.test.ts new file mode 100644 index 0000000000..6aea1228b5 --- /dev/null +++ b/packages/perf-harness/src/__tests__/postgres.test.ts @@ -0,0 +1,63 @@ +import { + parseCgroupKeyValues, + parseCgroupV2Memory, + parseDockerBytes +} from '../postgres'; + +describe('PostgreSQL container telemetry', () => { + it('parses Docker memory units without decimal loss', () => { + expect(parseDockerBytes('1.5GiB')).toBe(1.5 * 1024 ** 3); + expect(parseDockerBytes('256MiB')).toBe(256 * 1024 ** 2); + expect(parseDockerBytes('unknown')).toBeNull(); + }); + + it('parses raw cgroup-v2 charge, peak, limits, stats, and events', () => { + const raw = [ + '__CPERF_CGROUP_FILE__ memory.current', + '104857600', + '__CPERF_CGROUP_FILE__ memory.peak', + '157286400', + '__CPERF_CGROUP_FILE__ memory.max', + '2147483648', + '__CPERF_CGROUP_FILE__ memory.stat', + 'anon 73400320', + 'file 20971520', + 'shmem 1048576', + '__CPERF_CGROUP_FILE__ memory.events', + 'low 0', + 'high 2', + 'oom 0', + 'oom_kill 0' + ].join('\n'); + expect(parseCgroupV2Memory(raw)).toEqual({ + currentBytes: 104857600, + peakBytes: 157286400, + maxBytes: 2147483648, + stat: { + anon: 73400320, + file: 20971520, + shmem: 1048576 + }, + events: { + low: 0, + high: 2, + oom: 0, + oom_kill: 0 + } + }); + }); + + it('treats an unlimited cgroup max as null and rejects missing current charge', () => { + expect(parseCgroupV2Memory([ + '__CPERF_CGROUP_FILE__ memory.current', + '4096', + '__CPERF_CGROUP_FILE__ memory.max', + 'max' + ].join('\n'))).toMatchObject({ currentBytes: 4096, maxBytes: null }); + expect(parseCgroupV2Memory('__CPERF_CGROUP_FILE__ memory.max\nmax')).toBeNull(); + expect(parseCgroupKeyValues('anon 10\ninvalid\nfile nope\nshmem 20')).toEqual({ + anon: 10, + shmem: 20 + }); + }); +}); diff --git a/packages/perf-harness/src/__tests__/process.test.ts b/packages/perf-harness/src/__tests__/process.test.ts new file mode 100644 index 0000000000..1bb781c5f1 --- /dev/null +++ b/packages/perf-harness/src/__tests__/process.test.ts @@ -0,0 +1,117 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { + collectArmProvenance, + createObservabilityHeaders, + expectedHeapLimitForNodeOptions, + nodeFlagsForV8Profile, + replaceMaxOldSpaceSize +} from '../process'; + +describe('arm process isolation and provenance', () => { + it('creates a fresh, strong bearer header without a separately exposed token', () => { + const first = createObservabilityHeaders(); + const second = createObservabilityHeaders(); + expect(Object.keys(first)).toEqual(['Authorization']); + expect(/^Bearer [A-Za-z0-9_-]+$/.test(first.Authorization)).toBe(true); + expect(Buffer.byteLength(first.Authorization.slice('Bearer '.length))).toBeGreaterThanOrEqual(32); + expect(first.Authorization === second.Authorization).toBe(false); + }); + + it('replaces all inherited max-old-space aliases without dropping other options', () => { + expect(replaceMaxOldSpaceSize( + '--trace-warnings --max-old-space-size=256 --max_old_space_size 512', + 1024 + )).toBe('--trace-warnings --max-old-space-size=1024'); + }); + + it('uses only the closed, named V8 profile flag combinations', () => { + expect(nodeFlagsForV8Profile('stock')).toEqual([]); + expect(nodeFlagsForV8Profile('optimize-for-size')).toEqual([ + '--optimize-for-size' + ]); + expect(nodeFlagsForV8Profile('baseline-optimize-for-size')).toEqual([ + '--max-opt=1', + '--optimize-for-size' + ]); + expect(nodeFlagsForV8Profile('jitless-optimize-for-size')).toEqual([ + '--jitless', + '--optimize-for-size' + ]); + expect(replaceMaxOldSpaceSize( + '--jitless --max-opt=1 --optimize-for-size --trace-warnings', + 1024 + )).toBe('--trace-warnings --max-old-space-size=1024'); + }); + + it('preserves quoted NODE_OPTIONS values while replacing the heap flag', () => { + expect(replaceMaxOldSpaceSize( + '--require "/tmp/a b.js" --max-old-space-size 128', + 2048 + )).toBe('--require "/tmp/a b.js" --max-old-space-size=2048'); + }); + + it('derives the actual V8 heap limit produced by the sanitized options', () => { + const nodeOptions = replaceMaxOldSpaceSize(undefined, 128); + expect(expectedHeapLimitForNodeOptions(nodeOptions)).toBeGreaterThanOrEqual(128 * 1024 ** 2); + }); + + it('records git, lockfile, entry, command, cwd, and server pid provenance', () => { + const cwd = path.resolve(__dirname, '../../../..'); + const entryPath = path.join(cwd, 'packages/perf-harness/src/index.ts'); + const result = collectArmProvenance(cwd, [process.execPath, entryPath], 9876); + expect(result.errors).toEqual([]); + expect(result.provenance).toMatchObject({ + cwd, + command: [process.execPath, entryPath], + serverPid: 9876, + worktreeDirty: expect.any(Boolean), + gitHead: expect.stringMatching(/^[0-9a-f]{40}$/), + gitStatusSha256: expect.stringMatching(/^[0-9a-f]{64}$/), + lockfilePath: path.join(cwd, 'pnpm-lock.yaml'), + entryPath, + v8Profile: 'stock', + nodeOptions: null, + nodeOptionsArgv: [], + nodeExecArgv: [], + effectiveNodeRuntimeFlags: [] + }); + expect(result.provenance.entrySha256).toBe( + createHash('sha256').update(fs.readFileSync(entryPath)).digest('hex') + ); + expect(result.provenance.lockfileSha256).toBe( + createHash('sha256').update(fs.readFileSync(path.join(cwd, 'pnpm-lock.yaml'))).digest('hex') + ); + }); + + it('binds the exact direct and NODE_OPTIONS runtime flags to provenance', () => { + const cwd = path.resolve(__dirname, '../../../..'); + const entryPath = path.join(cwd, 'packages/perf-harness/src/index.ts'); + const command = [ + process.execPath, + '--jitless', + '--optimize-for-size', + '--expose-gc', + entryPath + ]; + const result = collectArmProvenance(cwd, command, 9876, { + v8Profile: 'jitless-optimize-for-size', + nodeOptions: '--trace-warnings --max-old-space-size=1024', + nodeOptionsArgv: ['--trace-warnings', '--max-old-space-size=1024'], + nodeExecArgv: ['--jitless', '--optimize-for-size', '--expose-gc'] + }); + expect(result.provenance).toMatchObject({ + v8Profile: 'jitless-optimize-for-size', + nodeExecArgv: ['--jitless', '--optimize-for-size', '--expose-gc'], + effectiveNodeRuntimeFlags: [ + '--trace-warnings', + '--max-old-space-size=1024', + '--jitless', + '--optimize-for-size', + '--expose-gc' + ] + }); + }); +}); diff --git a/packages/perf-harness/src/__tests__/realtime-evidence.test.ts b/packages/perf-harness/src/__tests__/realtime-evidence.test.ts new file mode 100644 index 0000000000..9cfad48d4c --- /dev/null +++ b/packages/perf-harness/src/__tests__/realtime-evidence.test.ts @@ -0,0 +1,137 @@ +import { createHash } from 'node:crypto'; + +import { summarizeRealtimeReceiptEvidence } from '../realtime-evidence'; +import type { RealtimeCorrelationReceipt } from '../types'; + +const digest = (value: string): string => createHash('sha256') + .update(value) + .digest('hex'); + +const receipt = ( + sequence: number, + issuedAt: string, + primeResponseAt: string, + eventAt: string, + value = `nonce-${sequence}` +): RealtimeCorrelationReceipt => { + const sha256 = digest(value); + const deadlineAt = new Date(Date.parse(issuedAt) + 30_000).toISOString(); + return { + sequence, + timed: true, + deadlineAt, + issuedAt, + issuedSha256: sha256, + primeResponseAt, + primeResponseSha256: sha256, + eventAt, + eventSha256: sha256 + }; +}; + +const evidence = (receipts: RealtimeCorrelationReceipt[]) => ({ + deliveryIntervalMs: 60_000, + workloadStartedAt: '2026-08-02T00:00:00.000Z', + workloadDeadlineAt: '2026-08-02T00:03:00.000Z', + workloadEndedAt: '2026-08-02T00:03:00.000Z', + surfaces: [{ + tenantId: 'customer-a', + surface: 'api-a', + route: '/customer/customer-a/tenant/a/graphql', + expectedRecurringRounds: receipts.length, + startedRecurringRounds: receipts.length, + verifiedRecurringRounds: receipts.length, + deadlineLateRecurringRounds: 0, + receipts + }] +}); + +describe('realtime receipt evidence', () => { + it('derives exact counts, digests, and latency from ordered receipts', () => { + const summary = summarizeRealtimeReceiptEvidence(evidence([ + receipt( + 1, + '2026-08-02T00:01:00.000Z', + '2026-08-02T00:01:00.020Z', + '2026-08-02T00:01:00.040Z' + ), + receipt( + 2, + '2026-08-02T00:02:00.000Z', + '2026-08-02T00:02:00.030Z', + '2026-08-02T00:02:00.050Z' + ) + ])); + + expect(summary.failures).toEqual([]); + expect(summary.coverage).toMatchObject({ + version: 2, + expectedRecurringRounds: 2, + startedRecurringRounds: 2, + verifiedRecurringRounds: 2, + primeRequests: 2, + primeResponseP99Ms: 30, + deliveryP99Ms: 50, + complete: true + }); + expect(summary.coverage.surfaces[0].issuedCorrelationSha256).toBe( + summary.coverage.surfaces[0].verifiedCorrelationSha256 + ); + }); + + it('rejects one digest reused across exact routes', () => { + const shared = receipt( + 1, + '2026-08-02T00:01:00.000Z', + '2026-08-02T00:01:00.020Z', + '2026-08-02T00:01:00.040Z', + 'shared-nonce' + ); + const input = evidence([shared]); + input.surfaces.push({ + ...input.surfaces[0], + tenantId: 'customer-b', + surface: 'api-b', + route: '/customer/customer-b/tenant/b/graphql', + receipts: [{ ...shared }] + }); + + const summary = summarizeRealtimeReceiptEvidence(input); + expect(summary.coverage.complete).toBe(false); + expect(summary.failures).toContain( + 'reused realtime receipt digest: customer-b/api-b' + ); + }); + + it('rejects a response or event timestamp preceding nonce issue', () => { + const summary = summarizeRealtimeReceiptEvidence(evidence([ + receipt( + 1, + '2026-08-02T00:01:00.100Z', + '2026-08-02T00:01:00.000Z', + '2026-08-02T00:01:00.050Z' + ) + ])); + + expect(summary.coverage.complete).toBe(false); + expect(summary.failures).toContain( + 'realtime verified receipt count mismatch: customer-a/api-a' + ); + }); + + it('rejects a self-issued deadline after the externally scheduled slot', () => { + const summary = summarizeRealtimeReceiptEvidence(evidence([ + receipt( + 1, + '2026-08-02T00:02:00.000Z', + '2026-08-02T00:02:00.020Z', + '2026-08-02T00:02:00.040Z' + ) + ])); + + expect(summary.coverage.complete).toBe(false); + expect(summary.failures).toContain( + 'invalid realtime receipt schedule deadline: customer-a/api-a' + ); + }); +}); diff --git a/packages/perf-harness/src/__tests__/realtime.test.ts b/packages/perf-harness/src/__tests__/realtime.test.ts new file mode 100644 index 0000000000..23a9dc1838 --- /dev/null +++ b/packages/perf-harness/src/__tests__/realtime.test.ts @@ -0,0 +1,575 @@ +import { + createRealtimeDriver, + type RealtimeClientFactoryInput, + realtimeHeaders, + realtimeWebSocketUrl +} from '../realtime'; +import type { GraphqlSurface, TenantTarget } from '../types'; + +const surface = ( + customer: string, + tenant: string, + foreignCustomer: string +): GraphqlSurface => { + const payload = `${customer}:${tenant}:resident`; + const physicalDatabaseIdentity = `database-${customer}`; + return { + name: `api-${tenant}`, + buildContract: `${customer}-${tenant}`, + url: `http://127.0.0.1:3410/customer/${customer}/tenant/${tenant}/graphql`, + headers: { 'accept-language': 'es' }, + warmup: { name: 'warm', capability: 'generated', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'generated', query: '{ __typename }' }], + canaries: [{ + name: 'isolation', + query: '{ tenantToken }', + requiredMatches: [{ path: '/data/tenantToken', value: tenant }], + forbiddenMatches: [{ path: '/data/tenantToken', value: 'foreign' }] + }], + realtime: { + headersFromEnvironment: { authorization: 'CPERF_TEST_TOKEN' }, + subscription: { + query: 'subscription Resident { changed { tenantId physicalDatabaseIdentity payload } }', + requiredMatches: [ + { path: '/data/changed/tenantId', value: tenant }, + { + path: '/data/changed/physicalDatabaseIdentity', + value: physicalDatabaseIdentity + } + ], + forbiddenMatches: [ + { path: '/data/changed/tenantId', value: 'foreign' }, + { + path: '/data/changed/physicalDatabaseIdentity', + value: `database-${foreignCustomer}` + } + ] + }, + prime: { + query: 'mutation Prime($payload: String!) { prime(payload: $payload) { tenantId physicalDatabaseIdentity payload } }', + variables: { payload }, + requiredMatches: [ + { path: '/data/prime/tenantId', value: tenant }, + { + path: '/data/prime/physicalDatabaseIdentity', + value: physicalDatabaseIdentity + } + ], + forbiddenMatches: [ + { path: '/data/prime/tenantId', value: 'foreign' }, + { + path: '/data/prime/physicalDatabaseIdentity', + value: `database-${foreignCustomer}` + } + ] + }, + correlation: { + primeVariable: 'payload', + primeResponsePath: '/data/prime/payload', + subscriptionEventPath: '/data/changed/payload' + } + } + }; +}; + +const fleet = (): TenantTarget[] => [ + { id: 'customer-1', surfaces: [surface('customer-1', 'a', 'customer-2')] }, + { id: 'customer-2', surfaces: [surface('customer-2', 'b', 'customer-1')] } +]; + +const waitFor = async (predicate: () => boolean, timeoutMs = 1_000): Promise => { + const deadline = Date.now() + timeoutMs; + while (!predicate() && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 2)); + } + if (!predicate()) throw new Error('TEST_WAIT_TIMEOUT'); +}; + +describe('external realtime driver', () => { + it('uses each exact route, verifies its event, and keeps credentials out of evidence', async () => { + const clients = new Map(); + const created: string[] = []; + const requested: string[] = []; + const previousCorrelationByRoute = new Map(); + const clientFactory = (input: RealtimeClientFactoryInput) => { + created.push(input.url); + const state = { input, unsubscribed: 0, disposed: 0 } as { + input: RealtimeClientFactoryInput; + sink?: { next(value: unknown): void }; + unsubscribed: number; + disposed: number; + }; + clients.set(input.url, state); + return { + subscribe: (_payload: unknown, sink: { next(value: unknown): void }) => { + state.sink = sink; + queueMicrotask(input.onConnected); + return () => { state.unsubscribed++; }; + }, + dispose: async () => { + state.disposed++; + input.onClosed(); + } + }; + }; + const fetchImpl = async (url: string | URL | Request, init?: RequestInit) => { + const href = String(url); + requested.push(href); + expect((init?.headers as Record).authorization).toBe('driver-secret'); + const parsedBody = JSON.parse(String(init?.body)); + const payload = parsedBody.variables.payload as string; + const tenant = href.includes('/tenant/a/') ? 'a' : 'b'; + const customer = href.includes('/customer/customer-1/') ? 'customer-1' : 'customer-2'; + const wsUrl = href.replace(/^http:/, 'ws:'); + // A cursor replay from the same exact tenant/database is legitimate, but + // even a nonce that proved the prior round must not prove this one. + clients.get(wsUrl)!.sink!.next({ + data: { + changed: { + tenantId: tenant, + physicalDatabaseIdentity: `database-${customer}`, + payload: previousCorrelationByRoute.get(href) + ?? 'earlier-valid-event' + } + } + }); + clients.get(wsUrl)!.sink!.next({ + data: { + changed: { + tenantId: tenant, + physicalDatabaseIdentity: `database-${customer}`, + payload + } + } + }); + previousCorrelationByRoute.set(href, payload); + return new Response(JSON.stringify({ + data: { + prime: { + tenantId: tenant, + physicalDatabaseIdentity: `database-${customer}`, + payload + } + } + }), { status: 200, headers: { 'content-type': 'application/json' } }); + }; + const driver = createRealtimeDriver(fleet(), { + concurrency: 1, + timeoutMs: 1_000 + }, { + clientFactory, + fetch: fetchImpl as typeof fetch, + environment: { CPERF_TEST_TOKEN: 'driver-secret' }, + correlationFactory: (surfaceKey, sequence) => + `test-correlation:${surfaceKey}:${sequence}:fresh-round`, + sleep: async () => undefined + }); + + await driver.startAndVerify(); + expect(created).toEqual([ + 'ws://127.0.0.1:3410/customer/customer-1/tenant/a/graphql', + 'ws://127.0.0.1:3410/customer/customer-2/tenant/b/graphql' + ]); + expect(requested).toEqual([ + 'http://127.0.0.1:3410/customer/customer-1/tenant/a/graphql', + 'http://127.0.0.1:3410/customer/customer-2/tenant/b/graphql' + ]); + expect(driver.snapshot()).toMatchObject({ + expected: 2, + active: 2, + verified: 2, + deliveryIntervalMs: 60_000, + deliveryEvents: 2, + deliveryRoundsStarted: 2, + deliveryRoundsVerified: 2, + deliveryRoundsPending: 0, + errors: [] + }); + expect(JSON.stringify(driver.snapshot())).not.toContain('driver-secret'); + driver.assertHealthy(); + + // Later legitimate workload writes change the payload but must preserve + // the permanent tenant/database invariants. + clients.get(created[0])!.sink!.next({ + data: { + changed: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload: 'tenant-a-workload-update' + } + } + }); + driver.assertHealthy(); + + await driver.verifyDeliveryNow(); + expect(driver.snapshot()).toMatchObject({ + deliveryEvents: 4, + deliveryRoundsStarted: 4, + deliveryRoundsVerified: 4, + deliveryRoundsPending: 0 + }); + for (const configured of driver.snapshot().surfaces) { + expect(configured.correlationReceipts).toHaveLength(2); + expect(configured.correlationReceipts.every((receipt) => + receipt.issuedSha256 === receipt.primeResponseSha256 + && receipt.issuedSha256 === receipt.eventSha256 + )).toBe(true); + } + + await driver.dispose(); + expect([...clients.values()].every((client) => + client.unsubscribed === 1 && client.disposed === 1 + )).toBe(true); + expect(driver.snapshot().active).toBe(0); + await driver.dispose(); + expect([...clients.values()].every((client) => client.disposed === 1)).toBe(true); + }); + + it('periodically requires a fresh matching event and never overlaps rounds', async () => { + let sink: { next(value: unknown): void } | null = null; + let primeCalls = 0; + let activePrimeCalls = 0; + let maximumActivePrimeCalls = 0; + let releasePeriodicPrime: (() => void) | null = null; + const periodicPrimeStarted = new Promise((resolve) => { + releasePeriodicPrime = resolve; + }); + let allowPeriodicPrimeToFinish: (() => void) | null = null; + const periodicPrimeCanFinish = new Promise((resolve) => { + allowPeriodicPrimeToFinish = resolve; + }); + const driver = createRealtimeDriver([fleet()[0]], { + concurrency: 1, + timeoutMs: 1_000, + deliveryIntervalMs: 20 + }, { + environment: { CPERF_TEST_TOKEN: 'driver-secret' }, + clientFactory: (input) => ({ + subscribe: (_payload, nextSink) => { + sink = nextSink; + queueMicrotask(input.onConnected); + return () => undefined; + }, + dispose: async () => undefined + }), + fetch: (async (_url, init) => { + primeCalls++; + activePrimeCalls++; + maximumActivePrimeCalls = Math.max(maximumActivePrimeCalls, activePrimeCalls); + const payload = JSON.parse(String(init?.body)).variables.payload; + if (primeCalls === 2) { + releasePeriodicPrime!(); + await periodicPrimeCanFinish; + } + sink!.next({ + data: { + changed: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }); + activePrimeCalls--; + return new Response(JSON.stringify({ + data: { + prime: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }), { status: 200 }); + }) as typeof fetch + }); + + try { + await driver.startAndVerify(); + driver.beginTimedCoverage(80); + await periodicPrimeStarted; + expect(driver.snapshot()).toMatchObject({ + deliveryEvents: 1, + deliveryRoundsStarted: 2, + deliveryRoundsVerified: 1, + deliveryRoundsPending: 1 + }); + + expect(primeCalls).toBe(2); + allowPeriodicPrimeToFinish!(); + await waitFor(() => + driver.snapshot().timedCoverage?.verifiedRecurringRounds === 3 + ); + await new Promise((resolve) => setTimeout(resolve, 25)); + const coverage = await driver.finishTimedCoverage(); + + expect(maximumActivePrimeCalls).toBe(1); + expect(coverage).toMatchObject({ + version: 2, + expectedRecurringRounds: 3, + startedRecurringRounds: 3, + verifiedRecurringRounds: 3, + deadlineLateRecurringRounds: 0, + complete: true, + primeRequests: 3, + surfaces: [{ + tenantId: 'customer-1', + surface: 'api-a', + expectedRecurringRounds: 3, + startedRecurringRounds: 3, + verifiedRecurringRounds: 3 + }] + }); + expect(coverage.surfaces[0].issuedCorrelationSha256).toMatch(/^[a-f0-9]{64}$/); + expect(coverage.surfaces[0].verifiedCorrelationSha256).toBe( + coverage.surfaces[0].issuedCorrelationSha256 + ); + expect(coverage.deliveryP99Ms).toBeGreaterThanOrEqual(0); + expect(driver.snapshot()).toMatchObject({ + deliveryEvents: 4, + deliveryRoundsStarted: 4, + deliveryRoundsVerified: 4, + deliveryRoundsPending: 0, + errors: [] + }); + } finally { + await driver.dispose(); + } + }); + + it('fails when a later round receives no matching event', async () => { + let sink: { next(value: unknown): void } | null = null; + let primeCalls = 0; + const driver = createRealtimeDriver([fleet()[0]], { + concurrency: 1, + timeoutMs: 35, + // Leave enough scheduling headroom that this exercises the event timeout, + // rather than the separate missed-deadline path on a busy test runner. + deliveryIntervalMs: 50 + }, { + environment: { CPERF_TEST_TOKEN: 'driver-secret' }, + clientFactory: (input) => ({ + subscribe: (_payload, nextSink) => { + sink = nextSink; + queueMicrotask(input.onConnected); + return () => undefined; + }, + dispose: async () => undefined + }), + fetch: (async (_url, init) => { + primeCalls++; + const payload = JSON.parse(String(init?.body)).variables.payload; + if (primeCalls === 1) { + sink!.next({ + data: { + changed: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }); + } + return new Response(JSON.stringify({ + data: { + prime: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }), { status: 200 }); + }) as typeof fetch + }); + + try { + await driver.startAndVerify(); + driver.beginTimedCoverage(200); + await waitFor(() => driver.snapshot().errors.length > 0); + expect(primeCalls).toBe(2); + expect(driver.snapshot()).toMatchObject({ + verified: 1, + deliveryEvents: 1, + deliveryRoundsStarted: 2, + deliveryRoundsVerified: 1, + deliveryRoundsPending: 0 + }); + await expect(driver.verifyDeliveryNow()).rejects.toThrow( + 'CPERF_REALTIME_EVENT_TIMEOUT:customer-1/api-a' + ); + } finally { + await driver.dispose(); + } + }); + + it('fails conclusively when the event came from another physical database', async () => { + let sink: { next(value: unknown): void } | null = null; + const driver = createRealtimeDriver([fleet()[0]], { + concurrency: 1, + timeoutMs: 1_000 + }, { + environment: { CPERF_TEST_TOKEN: 'driver-secret' }, + clientFactory: (input) => ({ + subscribe: (_payload, nextSink) => { + sink = nextSink; + queueMicrotask(input.onConnected); + return () => undefined; + }, + dispose: async () => undefined + }), + fetch: (async (_url, init) => { + const payload = JSON.parse(String(init?.body)).variables.payload; + sink!.next({ + data: { + changed: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-2', + payload + } + } + }); + return new Response(JSON.stringify({ + data: { + prime: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }), { status: 200 }); + }) as typeof fetch, + sleep: async () => undefined + }); + + await expect(driver.startAndVerify()).rejects.toThrow( + 'CPERF_REALTIME_FOREIGN_PAYLOAD:customer-1/api-a' + ); + expect(driver.snapshot().verified).toBe(0); + await driver.dispose(); + }); + + it('rejects a correlation digest reused by another exact route', async () => { + const sinks = new Map(); + const reused = 'same-correlation-across-all-routes'; + const driver = createRealtimeDriver(fleet(), { + concurrency: 1, + timeoutMs: 1_000 + }, { + environment: { CPERF_TEST_TOKEN: 'driver-secret' }, + correlationFactory: () => reused, + clientFactory: (input) => ({ + subscribe: (_payload, sink) => { + sinks.set(input.url, sink); + queueMicrotask(input.onConnected); + return () => undefined; + }, + dispose: async () => undefined + }), + fetch: (async (url, init) => { + const href = String(url); + const payload = JSON.parse(String(init?.body)).variables.payload; + const tenant = href.includes('/tenant/a/') ? 'a' : 'b'; + const customer = href.includes('/customer/customer-1/') + ? 'customer-1' + : 'customer-2'; + sinks.get(href.replace(/^http:/, 'ws:'))!.next({ + data: { + changed: { + tenantId: tenant, + physicalDatabaseIdentity: `database-${customer}`, + payload + } + } + }); + return new Response(JSON.stringify({ + data: { + prime: { + tenantId: tenant, + physicalDatabaseIdentity: `database-${customer}`, + payload + } + } + }), { status: 200 }); + }) as typeof fetch + }); + + await expect(driver.startAndVerify()).rejects.toThrow( + 'CPERF_REALTIME_CORRELATION_REUSED:customer-2/api-b' + ); + await driver.dispose(); + }); + + it('records a post-verification drop and fails the health check', async () => { + let callbacks: RealtimeClientFactoryInput | null = null; + let sink: { next(value: unknown): void } | null = null; + const target = fleet()[0]; + const driver = createRealtimeDriver([target], { + concurrency: 1, + timeoutMs: 1_000 + }, { + environment: { CPERF_TEST_TOKEN: 'driver-secret' }, + clientFactory: (input) => { + callbacks = input; + return { + subscribe: (_payload, nextSink) => { + sink = nextSink; + queueMicrotask(input.onConnected); + return () => undefined; + }, + dispose: async () => undefined + }; + }, + fetch: (async (_url, init) => { + const payload = JSON.parse(String(init?.body)).variables.payload; + sink!.next({ + data: { + changed: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }); + return new Response(JSON.stringify({ + data: { + prime: { + tenantId: 'a', + physicalDatabaseIdentity: 'database-customer-1', + payload + } + } + }), { status: 200 }); + }) as typeof fetch, + sleep: async () => undefined + }); + + await driver.startAndVerify(); + callbacks!.onClosed(); + expect(() => driver.assertHealthy()).toThrow( + 'CPERF_REALTIME_TRANSPORT_DROPPED:customer-1/api-a' + ); + await driver.dispose(); + }); + + it('requires secret headers from the runtime environment', () => { + const configured = fleet()[0].surfaces[0]; + expect(() => realtimeHeaders(configured, {})).toThrow( + 'CPERF_REALTIME_HEADER_ENV_MISSING:api-a:CPERF_TEST_TOKEN' + ); + expect(realtimeWebSocketUrl(configured.url)).toBe( + 'ws://127.0.0.1:3410/customer/customer-1/tenant/a/graphql' + ); + expect(() => realtimeWebSocketUrl(`${configured.url}?token=secret`)).toThrow( + 'CPERF_REALTIME_SURFACE_URL_INVALID' + ); + expect(() => createRealtimeDriver([], { + concurrency: 1, + timeoutMs: 1_000, + deliveryIntervalMs: 0 + })).toThrow('CPERF_REALTIME_DELIVERY_INTERVAL_INVALID'); + }); +}); diff --git a/packages/perf-harness/src/__tests__/report.test.ts b/packages/perf-harness/src/__tests__/report.test.ts new file mode 100644 index 0000000000..27ad69f4ed --- /dev/null +++ b/packages/perf-harness/src/__tests__/report.test.ts @@ -0,0 +1,681 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { resolveTenants } from '../config'; +import { + bindResultEvidence, + RESULT_RAW_EVIDENCE_FILES, + writeScoreContext +} from '../evidence'; +import { + rejectDuplicatePostgresRunEpochs, + renderReport +} from '../report'; +import { + buildRunSchedule, + scheduleJobsForPlan, + scheduleManifestSha256, + type CampaignScheduleManifestV1 +} from '../schedule'; +import { scoreRun, type ScoreInput } from '../score'; +import type { + AcceptanceGates, + ArmPlan, + DensityPlanV1, + DensityRunResult, + FleetV1, + MemorySnapshot, + PostgresMemorySnapshot, + RealtimeDeliveryCoverage +} from '../types'; + +const artifactRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-report-test-')); +const planSha256 = 'a'.repeat(64); +const fleetSha256 = 'b'.repeat(64); +const CAPACITY_ERROR = `CAPACITY:sha256:${'9'.repeat(64)}`; +const campaignId = '7'.repeat(64); +const cohortSha256 = createHash('sha256') + .update(`${planSha256}\0${fleetSha256}`) + .digest('hex'); + +const gates: AcceptanceGates = { + maxErrorRate: 0.005, + maxP99Ms: 150, + maxPostWarmupHeapGrowthMiBPerHour: 5, + minMedianDensityImprovement: 0.15, + minAdditionalTenantsEveryRun: 1, + maxAlignedMemorySampleGapMs: 900_000, + minAlignedMemoryCoverageRatio: 0.99, + requireZeroBleed: true, + requireNoPostWarmupEvictions: true, + requireNoPostWarmupBuildRefusals: true, + requireNoPostWarmupBuilds: true, + requirePostgresMemoryTelemetry: false, + requireFreshPostgresRunAttestation: false, + requireRetainedMemoryCheckpoints: false, + requirePhysicalDatabaseTelemetry: false, + requireConclusiveCanaries: true, + requireCompletePeriodicCanaryCoverage: false, + requireConclusiveOperationOracles: false, + requireExplicitCustomerTopology: false, + requiredCacheAdmissionMode: null +}; + +const arms: ArmPlan[] = ['cache-governor-stock', 'scoped-introspection'].map( + (name, index) => ({ + name, + command: ['node', 'server.cjs'], + port: 3345 + index, + readinessUrl: `http://127.0.0.1:${3345 + index}/healthz`, + memoryUrl: `http://127.0.0.1:${3345 + index}/debug/memory`, + introspectionMode: 'scoped-required' + }) +); + +const fleet: FleetV1 = { + version: 1, + sourceSha256: fleetSha256, + tenants: [1, 2, 3].map((index) => ({ + id: `tenant-${index}`, + surfaces: [{ + name: 'api', + buildContract: `tenant-${index}-api`, + url: 'http://127.0.0.1:{port}/graphql', + warmup: { + name: 'warm', + capability: 'graphile', + query: '{ __typename }' + }, + operations: [{ + name: 'read', + capability: 'graphile', + query: '{ __typename }' + }], + canaries: [{ + name: 'cross-schema', + query: '{ __typename }', + requiredMatches: [{ path: '/data/tenant', value: `tenant-${index}` }], + forbiddenMatches: [{ path: '/data/tenant', value: 'foreign' }] + }] + }] + })) +}; + +const plan: DensityPlanV1 = { + version: 1, + sourceSha256: planSha256, + fleetFile: 'fleet.json', + artifactDir: artifactRoot, + arms, + heapMiB: [1024], + tenantCounts: [1, 2, 3], + repetitions: 1, + runOrderSeed: 'test-seed', + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + workload: { + durationSec: 900, + rpsPerTenant: 1, + minWorkloadRequestsPerSurface: 1, + requestTimeoutMs: 1_000, + maxInFlight: 4, + canaryIntervalSec: 60, + warmupTimeoutMs: 1_000, + warmupTimeoutPerSurfaceMs: 1_000, + warmupConcurrency: 4 + }, + gates, + qualification: { + baselineArm: 'cache-governor-stock', + requiredHeapMiB: [1024], + minimumRepetitions: 1 + } +}; + +for (const [index, arm] of arms.entries()) { + const runtimeArtifactFingerprint = `sha256:${String(index + 1).repeat(64)}`; + const configurationFingerprint = `sha256:${String(index + 3).repeat(64)}`; + const artifactFile = path.join(artifactRoot, `hostile-${arm.name}.json`); + const artifact = { + version: 1, + kind: 'exact-runtime-hostile-validation-v1', + passed: true, + arm: arm.name, + runtimeArtifactFingerprint, + configurationFingerprint + }; + const bytes = `${JSON.stringify(artifact, null, 2)}\n`; + fs.writeFileSync(artifactFile, bytes, 'utf8'); + plan.qualification!.hostileValidationEvidence ??= {}; + plan.qualification!.hostileValidationEvidence[arm.name] = { + version: 1, + kind: 'exact-runtime-hostile-validation-v1', + artifactFile, + artifactSha256: createHash('sha256').update(bytes).digest('hex'), + runtimeArtifactFingerprint, + configurationFingerprint + }; +} + +let activePlan = plan; +let activeSchedule: ReturnType = []; +let activeScheduleSha256 = ''; +let previousResultPayloadSha256: string | null = null; + +const beginCampaign = (targetPlan: DensityPlanV1): void => { + activePlan = targetPlan; + activeSchedule = buildRunSchedule( + targetPlan, + targetPlan.arms, + targetPlan.heapMiB, + targetPlan.repetitions + ); + const manifest: CampaignScheduleManifestV1 = { + version: 1, + campaignId, + campaignStartedAt: '2026-08-01T23:59:00.000Z', + runOrderSeed: targetPlan.runOrderSeed!, + planSha256, + fleetSha256, + node: process.version, + v8: process.versions.v8, + platform: 'linux', + architecture: 'x64', + jobs: scheduleJobsForPlan(targetPlan, activeSchedule, true) + }; + activeScheduleSha256 = scheduleManifestSha256(manifest); + fs.writeFileSync( + path.join(artifactRoot, `campaign-${campaignId}.json`), + `${JSON.stringify({ + ...manifest, + scheduleSha256: activeScheduleSha256, + evidenceMode: 'qualification', + qualificationBlockers: [] + }, null, 2)}\n`, + 'utf8' + ); + previousResultPayloadSha256 = null; +}; + +const realtimeCoverage = ( + startedAt: string, + endedAt: string, + durationSec: number +): RealtimeDeliveryCoverage => ({ + version: 2, + deliveryIntervalMs: 60_000, + workloadStartedAt: startedAt, + workloadDeadlineAt: new Date(Date.parse(startedAt) + durationSec * 1000).toISOString(), + workloadEndedAt: endedAt, + expectedRecurringRounds: 0, + startedRecurringRounds: 0, + verifiedRecurringRounds: 0, + deadlineLateRecurringRounds: 0, + primeRequests: 0, + primeResponseP99Ms: 0, + deliveryP99Ms: 0, + complete: true, + surfaces: [] +}); + +const memorySnapshot = ( + timestamp: string, + contracts: string[], + nodeRssBytes: number +): MemorySnapshot => ({ + timestamp, + pid: 123, + nodeEnv: 'production', + heapLimitBytes: 1024 * 1024 ** 2, + heapUsedBytes: 100 * 1024 ** 2, + rssBytes: nodeRssBytes, + processPeakRssBytes: nodeRssBytes, + cacheSize: contracts.length, + residentBuildContractFingerprints: contracts, + residentBuildContracts: contracts, + evictions: 0, + buildRefusals: 0, + buildsStarted: contracts.length, + buildsSucceeded: contracts.length, + buildMaxMs: 80, + pgPoolCacheSize: contracts.length, + pgPoolLeasedPools: 0, + pgPoolActiveLeases: 0, + pgPoolCapacityEvictions: 0, + pgPoolCapacityRefusals: 0, + pgPoolDisposalFailures: 0, + cacheCountersAvailable: true, + buildCountersAvailable: true +}); + +const postgresSnapshot = ( + timestamp: string, + postgresBytes: number +): PostgresMemorySnapshot => ({ + timestamp, + usedBytes: postgresBytes, + limitBytes: 8 * 1024 ** 3, + source: 'cgroup-v2', + cgroupV2: { + currentBytes: postgresBytes, + peakBytes: postgresBytes, + maxBytes: 8 * 1024 ** 3, + stat: {}, + events: { oom: 0, oom_kill: 0 } + }, + raw: `${postgresBytes}B / 8GiB` +}); + +let artifactSequence = 0; + +const scoreInput = ( + armName: string, + configuredCustomers: number, + desiredServiceDensity: number, + executionErrors: string[] = [], + options: { runKind?: 'matrix' | 'soak'; durationSec?: number } = {} +): ScoreInput => { + const arm = arms.find((candidate) => candidate.name === armName)!; + const tenants = resolveTenants(fleet.tenants.slice(0, configuredCustomers), arm); + const durationSec = options.durationSec ?? 900; + const runKind = options.runKind ?? 'matrix'; + const repetition = runKind === 'soak' ? activePlan.repetitions + 1 : 1; + const scheduled = scheduleJobsForPlan(activePlan, activeSchedule, true).find((job) => + job.runKind === runKind + && job.arm === armName + && job.heapMiB === 1024 + && job.tenantCount === configuredCustomers + && job.repetition === repetition + ); + if (!scheduled) throw new Error('test coordinate is absent from the active campaign'); + const startedAt = new Date( + Date.parse('2026-08-02T00:00:00.000Z') + + (scheduled.orderIndex - 1) * 10_000_000 + ).toISOString(); + const endedAt = new Date(Date.parse(startedAt) + durationSec * 1000).toISOString(); + const targetServiceBytes = Math.round( + configuredCustomers / desiredServiceDensity * 1024 ** 3 + ); + const nodeRssBytes = Math.max(256 * 1024 ** 2, Math.floor(targetServiceBytes * 0.6)); + const postgresBytes = Math.max(1, targetServiceBytes - nodeRssBytes); + const contracts = tenants.flatMap((tenant) => + tenant.surfaces.map((surface) => surface.buildContract)); + const memorySnapshots = [ + memorySnapshot(startedAt, contracts, nodeRssBytes), + memorySnapshot(endedAt, contracts, nodeRssBytes) + ]; + const postgresSnapshots = [ + postgresSnapshot(startedAt, postgresBytes), + postgresSnapshot(endedAt, postgresBytes) + ]; + const runOrderIndex = scheduled.orderIndex; + const artifactDir = path.join(artifactRoot, String(++artifactSequence)); + const provenance: ScoreInput['provenance'] = { + cwd: '/tmp/repo', + command: ['node', 'server.cjs'], + gitHead: 'c'.repeat(40), + worktreeDirty: false, + gitStatusSha256: 'd'.repeat(64), + lockfilePath: '/tmp/repo/pnpm-lock.yaml', + lockfileSha256: 'e'.repeat(64), + entryPath: '/tmp/repo/server.cjs', + entrySha256: 'f'.repeat(64), + serverPid: 123, + v8Profile: 'stock', + nodeOptions: '--max-old-space-size=1024', + nodeOptionsArgv: ['--max-old-space-size=1024'], + nodeExecArgv: [], + effectiveNodeRuntimeFlags: ['--max-old-space-size=1024'], + planSha256, + fleetSha256, + node: process.version, + v8: process.versions.v8, + platform: 'linux', + architecture: 'x64', + runOrderSeed: 'test-seed', + runOrderIndex, + memoryPolicy: { + configuredMaxOldSpaceMiB: 1024, + expectedV8HeapLimitBytes: 1024 * 1024 ** 2, + graphileCacheMax: null, + graphileCacheInstanceHeapBytes: null, + graphileCacheServerReserveBytes: null, + graphileCacheBuildReserveBytes: null, + graphileCacheRssLimitBytes: null, + graphileCacheRssBuildReserveBytes: null, + graphileCacheCalibrationId: null, + graphileCacheAdmissionMode: null, + graphileBuildMaxConcurrency: null + } + }; + return { + arm: armName, + evidenceMode: 'qualification', + campaignId, + scheduleSha256: activeScheduleSha256, + previousResultPayloadSha256, + qualificationCohortSha256: cohortSha256, + introspectionMode: arm.introspectionMode, + heapMiB: 1024, + repetition, + expectedMatrixRepetitions: 1, + runKind, + runOrderSeed: 'test-seed', + runOrderIndex, + startedAt, + endedAt, + configuredDurationSec: durationSec, + workloadDurationMs: durationSec * 1000, + artifactDir, + tenants, + warmedSurfaces: new Map(tenants.map((tenant) => [ + tenant.id, + new Set(tenant.surfaces.map((surface) => surface.name)) + ])), + warmupLatencies: tenants.map(() => 80), + resolvedWarmupTimeoutMs: 1_000, + offeredLoad: { + mode: 'per-tenant', + configuredRps: 1, + tenantCount: configuredCustomers, + totalRps: configuredCustomers, + rpsPerTenant: 1 + }, + canaryIntervalSec: 60, + periodicCanarySchedule: 'full-sweep', + canarySchedule: null, + minWorkloadRequestsPerSurface: 1, + samples: tenants.flatMap((tenant) => tenant.surfaces.map((surface) => ({ + tenantId: tenant.id, + surface: surface.name, + operation: 'read', + capability: 'graphile', + latencyMs: 25, + status: 200, + ok: true, + phase: 'workload' as const + }))), + canaries: tenants.flatMap((tenant) => tenant.surfaces.map((surface) => ({ + tenantId: tenant.id, + surface: surface.name, + canary: 'cross-schema', + phase: 'initial' as const, + scheduledAt: startedAt, + startedAt, + completedAt: new Date(Date.parse(startedAt) + 20).toISOString(), + latencyMs: 20, + conclusive: true, + violation: false + }))), + memorySnapshots, + postWarmupSnapshots: memorySnapshots, + postWarmupNodeRssSnapshots: memorySnapshots.map((snapshot) => ({ + timestamp: snapshot.timestamp, + pid: 123, + source: 'proc' as const, + rssBytes: snapshot.rssBytes! + })), + retainedMemory: { baseline: null, final: null, errors: [] }, + memorySampleErrors: [], + postgresSnapshots, + postgresSampleErrors: [], + missedArrivals: 0, + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + gates, + serverExit: null, + provenance: { + ...provenance, + runOrderIndex + }, + provenanceErrors: [], + postgresRunAttestation: null, + realtimeDeliveryCoverage: realtimeCoverage(startedAt, endedAt, durationSec), + externalServer: false, + executionErrors + }; +}; + +const writeJson = (file: string, value: unknown): void => { + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, 'utf8'); +}; + +const persistResult = (input: ScoreInput): DensityRunResult => { + fs.mkdirSync(input.artifactDir, { recursive: true }); + writeJson(path.join(input.artifactDir, 'memory.json'), { + snapshots: input.memorySnapshots, + osSnapshots: input.postWarmupNodeRssSnapshots, + errors: input.memorySampleErrors, + warmupIndex: 0, + osWarmupIndex: 0, + osPeakRssBytes: Math.max(...input.postWarmupNodeRssSnapshots.map( + (snapshot) => snapshot.rssBytes + )) + }); + writeJson(path.join(input.artifactDir, 'postgres-memory.json'), { + snapshots: input.postgresSnapshots, + errors: input.postgresSampleErrors + }); + writeJson(path.join(input.artifactDir, 'canaries.json'), input.canaries); + writeJson(path.join(input.artifactDir, 'canary-schedule.json'), input.canarySchedule); + fs.writeFileSync( + path.join(input.artifactDir, 'requests.ndjson'), + `${input.samples.map((sample) => JSON.stringify(sample)).join('\n')}\n`, + 'utf8' + ); + writeJson(path.join(input.artifactDir, 'workload-progress.json'), { + warmedSurfaces: [...input.warmedSurfaces].map(([tenantId, surfaces]) => ({ + tenantId, + surfaces: [...surfaces].sort() + })), + warmupLatencies: input.warmupLatencies, + samples: input.samples.length, + canaries: input.canaries.length, + canarySchedule: input.canarySchedule, + offeredLoad: input.offeredLoad, + resolvedWarmupTimeoutMs: input.resolvedWarmupTimeoutMs, + workloadDurationMs: input.workloadDurationMs + }); + writeJson(path.join(input.artifactDir, 'retained-memory.json'), input.retainedMemory); + writeJson(path.join(input.artifactDir, 'realtime-driver.json'), [{ + phase: 'timed-coverage-complete', + timestamp: input.endedAt, + snapshot: { + expected: 0, + active: 0, + verified: 0, + deliveryIntervalMs: input.realtimeDeliveryCoverage!.deliveryIntervalMs, + deliveryEvents: 0, + deliveryRoundsStarted: 0, + deliveryRoundsVerified: 0, + deliveryRoundsPending: 0, + timedCoverage: input.realtimeDeliveryCoverage, + errors: [], + surfaces: [] + } + }]); + const result = scoreRun(input); + writeScoreContext(input.artifactDir, input, { + planSha256, + fleetSha256, + campaignId: input.campaignId, + scheduleSha256: input.scheduleSha256, + previousResultPayloadSha256: input.previousResultPayloadSha256, + notBeforeEpochMs: Date.parse(input.startedAt) + }); + bindResultEvidence(result); + previousResultPayloadSha256 = result.evidenceBinding!.resultPayloadSha256; + return result; +}; + +const persistConfiguredMatrix = (): DensityRunResult[] => activeSchedule.map((job) => { + const baseline = job.arm.name === 'cache-governor-stock'; + const acceptedBoundary = baseline ? 1 : 2; + return persistResult(scoreInput( + job.arm.name, + job.tenantCount, + baseline ? 1 : 2, + job.tenantCount <= acceptedBoundary ? [] : [CAPACITY_ERROR] + )); +}); + +describe('density report', () => { + beforeEach(() => beginCampaign(plan)); + + it('rejects a semantically edited result even after its public hashes are rebound', () => { + const first = activeSchedule[0]; + const value = persistResult(scoreInput(first.arm.name, first.tenantCount, 1)); + value.p99Ms += 1; + bindResultEvidence(value); + expect(() => renderReport([value], plan, fleet)).toThrow( + 'does not match semantic replay of raw evidence' + ); + + beginCampaign(plan); + const divergent = persistResult(scoreInput(first.arm.name, first.tenantCount, 1)); + fs.appendFileSync(path.join(divergent.artifactDir, 'requests.ndjson'), '{}\n'); + expect(() => renderReport([divergent], plan, fleet)).toThrow( + 'raw evidence file does not match: requests.ndjson' + ); + }); + + it('renders an exactly paired capacity decision from replayed evidence', () => { + const results = persistConfiguredMatrix(); + const report = renderReport(results, plan, fleet); + expect(report).toContain('Customers/aligned service GiB'); + expect(report).toContain('Customer workload RPS'); + expect(report).toContain('Periodic validation RPS'); + expect(report).toContain('Realtime validation RPS'); + expect(report).toContain('matrices are exactly paired: yes'); + expect(report).toContain('Materially better: **yes**'); + }); + + it('rejects reordered, cross-campaign, and overlapping result ledgers', () => { + const results = persistConfiguredMatrix(); + expect(() => renderReport([ + results[1], + results[0], + ...results.slice(2) + ], plan, fleet)).toThrow('campaign schedule or result chain'); + + const spliced = { ...results[1], campaignId: '8'.repeat(64) }; + bindResultEvidence(spliced); + expect(() => renderReport([ + results[0], + spliced, + ...results.slice(2) + ], plan, fleet)).toThrow('campaign schedule or result chain'); + + const overlapping = { + ...results[1], + startedAt: results[0].startedAt + }; + bindResultEvidence(overlapping); + expect(() => renderReport([ + results[0], + overlapping, + ...results.slice(2) + ], plan, fleet)).toThrow('campaign chronology is invalid or overlapping'); + }); + + it('rejects qualification without exact-runtime hostile validation artifacts', () => { + const unboundPlan: DensityPlanV1 = { + ...plan, + qualification: { + baselineArm: plan.qualification!.baselineArm, + requiredHeapMiB: [...plan.qualification!.requiredHeapMiB], + minimumRepetitions: plan.qualification!.minimumRepetitions + } + }; + beginCampaign(unboundPlan); + const first = activeSchedule[0]; + const result = persistResult(scoreInput(first.arm.name, first.tenantCount, 1)); + expect(() => renderReport([result], unboundPlan, fleet)).toThrow( + 'lacks exact-runtime hostile validation evidence' + ); + }); + + it('rejects malformed nested request evidence after rebinding its artifact hash', () => { + const first = activeSchedule[0]; + const result = persistResult(scoreInput(first.arm.name, first.tenantCount, 1)); + const requestsFile = path.join(result.artifactDir, 'requests.ndjson'); + const request = JSON.parse(fs.readFileSync(requestsFile, 'utf8').trim()); + request.latencyMs = null; + fs.writeFileSync(requestsFile, `${JSON.stringify(request)}\n`, 'utf8'); + bindResultEvidence(result); + expect(() => renderReport([result], plan, fleet)).toThrow( + 'latencyMs must be finite' + ); + }); + + it('requires one configured qualifying soak without mixing it into the matrix', () => { + const soakPlan: DensityPlanV1 = { + ...plan, + soak: { + enabled: true, + arm: 'scoped-introspection', + durationSec: 7_200, + tenantCount: 2, + heapMiB: 1024 + } + }; + beginCampaign(soakPlan); + const matrix = persistConfiguredMatrix(); + expect(renderReport(matrix, soakPlan, fleet)).toContain( + 'configured soak=0/1, accepted=no' + ); + const soak = persistResult(scoreInput( + 'scoped-introspection', + 2, + 2, + [], + { runKind: 'soak', durationSec: 7_200 } + )); + const report = renderReport([...matrix, soak], soakPlan, fleet); + expect(report).toContain('configured soak=1/1, accepted=yes'); + expect(report).toContain('Materially better: **yes**'); + }); + + it('rejects reuse of any PostgreSQL container, cluster, clone, or nonce identity', () => { + const first = persistResult(scoreInput( + activeSchedule[0].arm.name, + activeSchedule[0].tenantCount, + 2 + )); + const second = persistResult(scoreInput( + activeSchedule[1].arm.name, + activeSchedule[1].tenantCount, + 2 + )); + const evidence = { + epochId: `sha256:${'1'.repeat(64)}`, + containerId: '2'.repeat(64), + cgroupIdentitySha256: `sha256:${'3'.repeat(64)}`, + postgresSystemIdentifier: '7421234567890123456', + cloneId: 'measurement-clone-1', + cloneAttestationSetSha256: `sha256:${'4'.repeat(64)}`, + cloneNonceSetSha256: `sha256:${'5'.repeat(64)}` + } as DensityRunResult['postgresRunAttestation']; + first.postgresRunAttestation = evidence; + second.postgresRunAttestation = { + ...evidence!, + epochId: `sha256:${'6'.repeat(64)}`, + containerId: '7'.repeat(64), + cgroupIdentitySha256: `sha256:${'8'.repeat(64)}`, + postgresSystemIdentifier: '8421234567890123456', + cloneId: 'measurement-clone-2', + cloneAttestationSetSha256: `sha256:${'9'.repeat(64)}` + }; + const rejected = rejectDuplicatePostgresRunEpochs([first, second]); + expect(rejected.every((run) => !run.accepted)).toBe(true); + expect(rejected.every((run) => run.failures.some((failure) => + failure.includes('clone-nonce-set:') + ))).toBe(true); + }); + + it('binds every fixed raw-evidence file including the score context', () => { + expect(RESULT_RAW_EVIDENCE_FILES).toContain('score-context.json'); + }); +}); diff --git a/packages/perf-harness/src/__tests__/run-attestation.test.ts b/packages/perf-harness/src/__tests__/run-attestation.test.ts new file mode 100644 index 0000000000..f7724086c9 --- /dev/null +++ b/packages/perf-harness/src/__tests__/run-attestation.test.ts @@ -0,0 +1,164 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + collectPostgresRunAttestation, + normalizePostgresRunAttestation, + postgresRunIdentityClaims, + type RunAttestationContext +} from '../run-attestation'; +import type { ArmPlan } from '../types'; + +const canonicalize = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + const record = value as Record; + return Object.fromEntries(Object.keys(record).sort().map((key) => [ + key, + canonicalize(record[key]) + ])); +}; + +const sha256 = (value: unknown): string => `sha256:${createHash('sha256') + .update(JSON.stringify(canonicalize(value))) + .digest('hex')}`; + +const contextFor = (artifactDir: string): RunAttestationContext => ({ + arm: 'candidate', + heapMiB: 2048, + tenantCount: 1, + repetition: 2, + runOrderIndex: 7, + planSha256: 'a'.repeat(64), + fleetSha256: 'b'.repeat(64), + notBeforeEpochMs: Date.parse('2026-08-02T00:00:00.000Z'), + artifactDir +}); + +const envelopeFor = (context: RunAttestationContext) => { + const customerAudits = [{ + customerId: 'customer-1', + databaseContractFingerprint: `sha256:${'c'.repeat(64)}`, + structuralFingerprints: { combined: { sha256: `sha256:${'d'.repeat(64)}` } }, + cloneAttestationSha256: `sha256:${'e'.repeat(64)}`, + cloneNonceSha256: `sha256:${'f'.repeat(64)}` + }]; + const provisionClone = { + version: 1, + id: 'measurement-unique-clone', + purpose: 'measurement', + attestationSetSha256: `sha256:${'1'.repeat(64)}` + }; + const container = { + id: '2'.repeat(64), + startedAt: '2026-08-02T00:00:00.010Z' + }; + const cgroup = { + version: 1, + source: 'container-cgroup-v2', + identitySha256: `sha256:${'3'.repeat(64)}` + }; + const postgres = { + systemIdentifier: '7421234567890123456', + postmasterStartedAt: '2026-08-02T00:00:00.020Z' + }; + const immutableEpoch = { + dockerContainerId: container.id, + dockerStartedAt: container.startedAt, + containerConfigurationSha256: `sha256:${'4'.repeat(64)}`, + cgroupIdentitySha256: cgroup.identitySha256, + postgresSystemIdentifier: postgres.systemIdentifier, + postgresStartedAt: postgres.postmasterStartedAt, + cloneId: provisionClone.id, + cloneAttestationSetSha256: provisionClone.attestationSetSha256, + cloneNonceSetSha256: sha256(customerAudits.map((audit) => ({ + customerId: audit.customerId, + cloneNonceSha256: audit.cloneNonceSha256 + }))), + liveContractSetSha256: sha256(customerAudits.map((audit) => ({ + customerId: audit.customerId, + databaseContractFingerprint: audit.databaseContractFingerprint, + structuralFingerprint: audit.structuralFingerprints.combined.sha256 + }))) + }; + const payload = { + observedAt: '2026-08-02T00:00:01.000Z', + run: { + arm: context.arm, + heapMiB: context.heapMiB, + customerCount: context.tenantCount, + repetition: context.repetition, + runOrderIndex: context.runOrderIndex, + planSha256: `sha256:${context.planSha256}`, + fleetSha256: `sha256:${context.fleetSha256}` + }, + manifestSha256: `sha256:${'5'.repeat(64)}`, + containerTemplateSha256: `sha256:${'6'.repeat(64)}`, + canonicalDatabaseContractFingerprint: `sha256:${'7'.repeat(64)}`, + provisionClone, + container, + cgroup, + postgres, + customerAudits, + immutableEpoch, + epochId: sha256(immutableEpoch), + freshness: { + freshContainerForRun: true, + cgroupV2Verified: true, + notBeforeEpochMs: context.notBeforeEpochMs, + startToleranceMs: 0 + }, + catalogCacheState: 'warmed-by-live-contract-audit' + }; + return { + version: 1, + kind: 'physical-density-measurement-attestation-v1', + payload, + payloadSha256: sha256(payload) + }; +}; + +describe('PostgreSQL run attestation', () => { + it('binds every immutable container, cluster, clone, and live-contract identity', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-attestation-')); + const context = contextFor(directory); + const envelope = envelopeFor(context); + const artifact = path.join(directory, 'attestation.json'); + fs.writeFileSync(artifact, JSON.stringify(envelope)); + const evidence = normalizePostgresRunAttestation(envelope, context, artifact); + expect(evidence.cloneId).toBe('measurement-unique-clone'); + expect(evidence.cloneAttestationSetSha256).toBe(`sha256:${'1'.repeat(64)}`); + expect(postgresRunIdentityClaims(evidence)).toHaveLength(7); + + const tampered = structuredClone(envelope) as any; + tampered.payload.immutableEpoch.cloneId = 'different-clone'; + tampered.payload.epochId = sha256(tampered.payload.immutableEpoch); + tampered.payloadSha256 = sha256(tampered.payload); + expect(() => normalizePostgresRunAttestation(tampered, context, artifact)) + .toThrow('failed exact validation'); + }); + + it('refuses to overwrite a prior per-run attestation artifact', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'cperf-attestation-')); + const context = contextFor(directory); + fs.writeFileSync( + path.join(directory, 'postgres-run-attestation.json'), + '{}' + ); + const arm = { + name: 'candidate', + port: 3345, + readinessUrl: 'http://127.0.0.1:3345/healthz', + memoryUrl: 'http://127.0.0.1:3345/debug/memory', + introspectionMode: 'stock', + postgresRunAttestation: { + command: [process.execPath, '-e', 'process.exit(0)'], + prepareCommand: [process.execPath, '-e', 'process.exit(0)'] + } + } satisfies ArmPlan; + await expect(collectPostgresRunAttestation(arm, context)) + .rejects.toThrow('artifact already exists'); + }); +}); diff --git a/packages/perf-harness/src/__tests__/run.test.ts b/packages/perf-harness/src/__tests__/run.test.ts new file mode 100644 index 0000000000..8630a6650a --- /dev/null +++ b/packages/perf-harness/src/__tests__/run.test.ts @@ -0,0 +1,67 @@ +import { resolveTenants } from '../config'; +import { buildRunSchedule } from '../run'; +import type { ArmPlan, DensityPlanV1, TenantTarget } from '../types'; + +describe('arm-specific fleet resolution', () => { + it('selects the exact build identity for the running arm', () => { + const tenants = [{ + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: 'default-hash', + buildContracts: { + stock: 'stock-hash', + scoped: 'scoped-hash' + }, + url: 'http://127.0.0.1:{port}/{mode}', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [] + }] + }] as TenantTarget[]; + const arm = { + name: 'scoped', + port: 3345, + introspectionMode: 'scoped-required' + } as ArmPlan; + + const resolved = resolveTenants(tenants, arm); + expect(resolved[0].surfaces[0]).toMatchObject({ + buildContract: 'scoped-hash', + url: 'http://127.0.0.1:3345/scoped-required' + }); + }); + + it('uses heap-specific ramps and reproducibly interleaves arms within each cell', () => { + const arms = [ + { name: 'stock' }, + { name: 'scoped' } + ] as ArmPlan[]; + const plan = { + runOrderSeed: 'seed-a', + tenantCounts: [1], + tenantCountsByHeapMiB: { 2048: [2, 4] } + } as unknown as DensityPlanV1; + const first = buildRunSchedule(plan, arms, [1024, 2048], 2); + const second = buildRunSchedule(plan, arms, [1024, 2048], 2); + expect(first.map((job) => ({ + arm: job.arm.name, + heap: job.heapMiB, + tenants: job.tenantCount, + repetition: job.repetition, + order: job.orderIndex + }))).toEqual(second.map((job) => ({ + arm: job.arm.name, + heap: job.heapMiB, + tenants: job.tenantCount, + repetition: job.repetition, + order: job.orderIndex + }))); + expect(first).toHaveLength(12); + expect(first.slice(0, 2).map((job) => job.heapMiB)).toEqual([1024, 1024]); + expect(first.slice(2, 6).map((job) => job.tenantCount)).toEqual([2, 2, 4, 4]); + expect(first.map((job) => job.orderIndex)).toEqual( + Array.from({ length: 12 }, (_, index) => index + 1) + ); + }); +}); diff --git a/packages/perf-harness/src/__tests__/score.test.ts b/packages/perf-harness/src/__tests__/score.test.ts new file mode 100644 index 0000000000..3732047886 --- /dev/null +++ b/packages/perf-harness/src/__tests__/score.test.ts @@ -0,0 +1,1754 @@ +import { createHash } from 'node:crypto'; + +import { rotatingCanaryIndex } from '../http'; +import { + alignedServiceMemoryCoverage, + alignedServiceMemoryPeak, + compareDensity, + heapGrowthMiBPerHour, + percentile, + retainedMemoryGrowth, + type ScoreInput, + scoreRun, + summarizeCapacityBoundaries} from '../score'; +import type { + AcceptanceGates, + ArmProvenance, + DensityRunResult, + MemorySnapshot, + PostgresRunAttestationEvidence, + RetainedMemoryCheckpoint, + TenantTarget +} from '../types'; + +const gates: AcceptanceGates = { + maxErrorRate: 0.005, + maxP99Ms: 150, + maxPostWarmupHeapGrowthMiBPerHour: 5, + minMedianDensityImprovement: 0.15, + minAdditionalTenantsEveryRun: 1, + maxAlignedMemorySampleGapMs: 900_000, + minAlignedMemoryCoverageRatio: 0.99, + requireZeroBleed: true, + requireNoPostWarmupEvictions: true, + requireNoPostWarmupBuildRefusals: true, + requireNoPostWarmupBuilds: true, + requirePostgresMemoryTelemetry: true, + requireFreshPostgresRunAttestation: false, + requireRetainedMemoryCheckpoints: true, + requirePhysicalDatabaseTelemetry: false, + requireConclusiveCanaries: true, + requireCompletePeriodicCanaryCoverage: false, + requireConclusiveOperationOracles: false, + requireExplicitCustomerTopology: false, + requiredCacheAdmissionMode: null +}; + +const tenant: TenantTarget = { + id: 'tenant-a', + surfaces: [{ + name: 'api', + buildContract: 'tenant-a-api', + url: 'http://127.0.0.1:3345/graphql', + warmup: { name: 'warm', capability: 'graphile', query: '{ __typename }' }, + operations: [{ name: 'read', capability: 'graphile', query: '{ __typename }' }], + canaries: [{ + name: 'cross-schema', + query: '{ __typename }', + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b' }], + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a' }] + }] + }] +}; + +const memory = (minute: number, heapMiB: number, evictions = 0): MemorySnapshot => ({ + timestamp: new Date(Date.UTC(2026, 6, 31, 0, minute)).toISOString(), + pid: 42, + nodeEnv: 'production', + heapLimitBytes: 1024 * 1024 ** 2, + heapUsedBytes: heapMiB * 1024 ** 2, + rssBytes: 200 * 1024 ** 2, + processPeakRssBytes: 220 * 1024 ** 2, + cacheSize: 1, + residentBuildContractFingerprints: ['tenant-a-api'], + residentBuildContracts: ['tenant-a-api'], + evictions, + buildRefusals: 0, + buildsStarted: 1, + buildsSucceeded: 1, + buildMaxMs: 80, + pgPoolCacheSize: 2, + pgPoolLeasedPools: 1, + pgPoolActiveLeases: 1, + pgPoolCapacityEvictions: 0, + pgPoolCapacityRefusals: 0, + pgPoolDisposalFailures: 0, + cacheCountersAvailable: true, + buildCountersAvailable: true +}); + +const canonicalJson = (value: unknown): string => { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value && typeof value === 'object') { + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => + `${JSON.stringify(key)}:${canonicalJson(record[key])}` + ).join(',')}}`; + } + return JSON.stringify(value) ?? 'null'; +}; + +const retainedCheckpoint = ( + minute: number, + heapMiB: number, + externalMiB = 10 +): RetainedMemoryCheckpoint => { + const state = { + pid: 42, + graphileInFlight: 0, + residentBuildContracts: ['tenant-a-api'], + residentBuildContractFingerprints: ['tenant-a-api'], + counters: { buildsStarted: 1, evictions: 0 } + }; + const stateSha256 = `sha256:${createHash('sha256') + .update(canonicalJson(state)) + .digest('hex')}`; + const guard = { + pid: 42, + graphileInFlight: 0, + residentBuildContracts: ['tenant-a-api'], + stateSha256, + state + }; + return { + version: 1, + fixture: 'physical-database-density-v1', + pid: 42, + gcRounds: 8, + stableSampleCount: 3, + stable: true, + samples: Array.from({ length: 8 }, (_, index) => ({ + timestamp: new Date(Date.UTC(2026, 6, 31, 0, minute, 0, index)).toISOString(), + monotonicNs: String(BigInt(minute) * 60_000_000_000n + BigInt(index + 1)), + heapUsedBytes: heapMiB * 1024 ** 2, + externalBytes: externalMiB * 1024 ** 2, + arrayBuffersBytes: 2 * 1024 ** 2, + rssBytes: 200 * 1024 ** 2 + })), + guardBefore: guard, + guardAfter: guard, + errors: [] + }; +}; + +const retainedCheckpointWithState = ( + minute: number, + state: Record +): RetainedMemoryCheckpoint => { + const checkpoint = retainedCheckpoint(minute, minute === 0 ? 100 : 101); + const stateHash = `sha256:${createHash('sha256') + .update(canonicalJson(state)) + .digest('hex')}`; + const residentBuildContracts = Array.isArray(state.residentBuildContracts) + ? state.residentBuildContracts as string[] + : ['tenant-a-api']; + const guard = { + pid: 42, + graphileInFlight: 0, + residentBuildContracts, + stateSha256: stateHash, + state + }; + return { ...checkpoint, guardBefore: guard, guardAfter: guard }; +}; + +const retainedPhysicalState = ( + httpRequestsStarted: number, + httpRequestsCompleted: number, + realtimeConnectionsActive = 1 +): Record => ({ + pid: 42, + graphileInFlight: 0, + residentBuildContracts: ['tenant-a-api'], + residentBuildContractFingerprints: ['tenant-a-api'], + cacheCounters: { + httpRequestsStarted, + httpRequestsCompleted, + websocketUpgradesStarted: 1, + websocketUpgradesCompleted: 0, + evictions: { lru: 0 }, + buildRefusals: { resident_capacity: 0 } + }, + realtime: { connectionsExpected: 1, connectionsActive: realtimeConnectionsActive } +}); + +const provenance: ArmProvenance = { + cwd: '/workspace/constructive', + command: ['/usr/bin/node', '/workspace/constructive/dist/server.js'], + gitHead: 'a'.repeat(40), + worktreeDirty: false, + gitStatusSha256: 'b'.repeat(64), + lockfilePath: '/workspace/constructive/pnpm-lock.yaml', + lockfileSha256: 'c'.repeat(64), + entryPath: '/workspace/constructive/dist/server.js', + entrySha256: 'd'.repeat(64), + serverPid: 42, + v8Profile: 'stock', + nodeOptions: '--max-old-space-size=1024', + nodeOptionsArgv: ['--max-old-space-size=1024'], + nodeExecArgv: [], + effectiveNodeRuntimeFlags: ['--max-old-space-size=1024'], + planSha256: 'e'.repeat(64), + fleetSha256: 'f'.repeat(64), + node: process.version, + v8: process.versions.v8, + platform: 'linux', + architecture: 'x64', + runOrderSeed: 'test-seed', + runOrderIndex: 1, + memoryPolicy: { + configuredMaxOldSpaceMiB: 1024, + expectedV8HeapLimitBytes: 1024 * 1024 ** 2, + graphileCacheMax: null, + graphileCacheInstanceHeapBytes: null, + graphileCacheServerReserveBytes: null, + graphileCacheBuildReserveBytes: null, + graphileCacheRssLimitBytes: null, + graphileCacheRssBuildReserveBytes: null, + graphileCacheCalibrationId: null, + graphileCacheAdmissionMode: null, + graphileBuildMaxConcurrency: null + } +}; + +const qualifyingInput = (): ScoreInput => ({ + arm: 'scoped-introspection', + evidenceMode: 'qualification', + campaignId: '8'.repeat(64), + scheduleSha256: '9'.repeat(64), + previousResultPayloadSha256: null, + qualificationCohortSha256: 'a'.repeat(64), + introspectionMode: 'scoped-required', + heapMiB: 1024, + repetition: 1, + expectedMatrixRepetitions: 1, + runKind: 'matrix', + runOrderSeed: 'test-seed', + runOrderIndex: 1, + startedAt: '2026-07-31T00:00:00.000Z', + endedAt: '2026-07-31T00:15:00.000Z', + configuredDurationSec: 900, + workloadDurationMs: 900_000, + artifactDir: '/tmp/result', + tenants: [tenant], + warmedSurfaces: new Map([['tenant-a', new Set(['api'])]]), + warmupLatencies: [80], + resolvedWarmupTimeoutMs: 180_000, + offeredLoad: { + mode: 'per-tenant', + configuredRps: 1, + tenantCount: 1, + totalRps: 1, + rpsPerTenant: 1 + }, + canaryIntervalSec: 60, + periodicCanarySchedule: 'full-sweep', + canarySchedule: null, + minWorkloadRequestsPerSurface: 1, + samples: [{ + tenantId: 'tenant-a', + surface: 'api', + operation: 'read', + capability: 'graphile', + latencyMs: 20, + status: 200, + ok: true, + phase: 'workload' + }], + canaries: [{ + tenantId: 'tenant-a', + surface: 'api', + canary: 'cross-schema', + phase: 'initial', + scheduledAt: '2026-07-31T00:00:00.000Z', + startedAt: '2026-07-31T00:00:00.000Z', + completedAt: '2026-07-31T00:00:00.020Z', + latencyMs: 20, + conclusive: true, + violation: false + }], + memorySnapshots: [memory(0, 100), memory(15, 101)], + postWarmupSnapshots: [memory(0, 100), memory(15, 101)], + postWarmupNodeRssSnapshots: [memory(0, 100), memory(15, 101)].map( + ({ timestamp, rssBytes }) => ({ + timestamp, + pid: 42, + source: 'proc', + rssBytes: rssBytes! + }) + ), + retainedMemory: { + baseline: retainedCheckpoint(0, 100), + final: retainedCheckpoint(15, 101), + errors: [] + }, + memorySampleErrors: [], + postgresSnapshots: [ + { + timestamp: '2026-07-31T00:00:00.100Z', + containerId: '4'.repeat(64), + cgroupIdentitySha256: `sha256:${'5'.repeat(64)}`, + usedBytes: 100, + limitBytes: 1_000, + source: 'cgroup-v2', + cgroupV2: { + currentBytes: 100, + peakBytes: 150, + maxBytes: 1_000, + stat: {}, + events: { oom: 0, oom_kill: 0 } + }, + raw: '100B / 1000B' + }, + { + timestamp: '2026-07-31T00:15:00.100Z', + containerId: '4'.repeat(64), + cgroupIdentitySha256: `sha256:${'5'.repeat(64)}`, + usedBytes: 120, + limitBytes: 1_000, + source: 'cgroup-v2', + cgroupV2: { + currentBytes: 120, + peakBytes: 180, + maxBytes: 1_000, + stat: {}, + events: { oom: 0, oom_kill: 0 } + }, + raw: '120B / 1000B' + } + ], + postgresSampleErrors: [], + missedArrivals: 0, + requiredCapabilities: ['graphile'], + requiredCanaries: ['cross-schema'], + gates, + serverExit: null, + provenance, + provenanceErrors: [], + realtimeDeliveryCoverage: { + version: 2, + deliveryIntervalMs: 60_000, + workloadStartedAt: '2026-07-31T00:00:00.000Z', + workloadDeadlineAt: '2026-07-31T00:15:00.000Z', + workloadEndedAt: '2026-07-31T00:15:00.000Z', + expectedRecurringRounds: 0, + startedRecurringRounds: 0, + verifiedRecurringRounds: 0, + deadlineLateRecurringRounds: 0, + primeRequests: 0, + primeResponseP99Ms: 0, + deliveryP99Ms: 0, + complete: true, + surfaces: [] + }, + externalServer: false, + executionErrors: [] +}); + +const postgresRunAttestation = (): PostgresRunAttestationEvidence => ({ + version: 1, + kind: 'physical-density-measurement-attestation-v1', + artifactPath: '/tmp/result/postgres-run-attestation.json', + artifactSha256: `sha256:${'1'.repeat(64)}`, + payloadSha256: `sha256:${'2'.repeat(64)}`, + epochId: `sha256:${'3'.repeat(64)}`, + arm: 'scoped-introspection', + heapMiB: 1024, + tenantCount: 1, + repetition: 1, + runOrderIndex: 1, + planSha256: `sha256:${'e'.repeat(64)}`, + fleetSha256: `sha256:${'f'.repeat(64)}`, + containerId: '4'.repeat(64), + containerStartedAt: '2026-07-31T00:00:00.000Z', + cgroupIdentitySha256: `sha256:${'5'.repeat(64)}`, + containerConfigurationSha256: `sha256:${'6'.repeat(64)}`, + postgresSystemIdentifier: '7421234567890123456', + postgresStartedAt: '2026-07-31T00:00:00.010Z', + cloneId: 'measurement-unique-clone', + cloneAttestationSetSha256: `sha256:${'7'.repeat(64)}`, + cloneNonceSetSha256: `sha256:${'8'.repeat(64)}`, + liveContractSetSha256: `sha256:${'9'.repeat(64)}`, + manifestSha256: `sha256:${'a'.repeat(64)}`, + containerTemplateSha256: `sha256:${'b'.repeat(64)}`, + canonicalDatabaseContractFingerprint: `sha256:${'c'.repeat(64)}`, + freshContainerForRun: true, + cgroupV2Verified: true, + liveCustomerContractsAudited: 1, + catalogCacheState: 'warmed-by-live-contract-audit' +}); + +const strictCanaryInput = (): ScoreInput => { + const input = qualifyingInput(); + input.tenants = input.tenants.map((configuredTenant) => ({ + ...configuredTenant, + surfaces: configuredTenant.surfaces.map((configuredSurface) => ({ + ...configuredSurface, + canaries: [...configuredSurface.canaries] + })) + })); + const surface = input.tenants[0].surfaces[0]; + surface.canaries = [ + ...surface.canaries, + { + name: 'prepared-reuse', + query: '{ __typename }', + forbiddenMatches: [{ path: '/data/tenantToken', value: 'tenant-b' }], + requiredMatches: [{ path: '/data/tenantToken', value: 'tenant-a' }] + } + ]; + input.requiredCanaries = ['cross-schema', 'prepared-reuse']; + input.canaryIntervalSec = 300; + input.periodicCanarySchedule = 'rotating-one'; + input.gates = { ...input.gates, requireCompletePeriodicCanaryCoverage: true }; + const startedMs = Date.parse('2026-07-31T00:00:00.000Z'); + const evidence = ( + canary: string, + phase: 'initial' | 'periodic' | 'final', + scheduledMs: number, + periodicRound?: number + ) => ({ + tenantId: 'tenant-a', + surface: 'api', + canary, + phase, + ...(periodicRound != null ? { periodicRound } : {}), + scheduledAt: new Date(scheduledMs).toISOString(), + startedAt: new Date(scheduledMs + 10).toISOString(), + completedAt: new Date(scheduledMs + 20).toISOString(), + latencyMs: phase === 'periodic' ? 5_000 : 10, + conclusive: true, + violation: false + }); + const initial = surface.canaries.map((canary) => + evidence(canary.name, 'initial', startedMs - 1_000) + ); + const periodic = [1, 2].map((periodicRound) => { + const canary = surface.canaries[rotatingCanaryIndex( + 'tenant-a', + 'api', + surface.canaries.length, + periodicRound + )]; + return evidence( + canary.name, + 'periodic', + startedMs + periodicRound * 300_000, + periodicRound + ); + }); + const final = surface.canaries.map((canary) => + evidence(canary.name, 'final', startedMs + 900_000) + ); + input.canaries = [...initial, ...periodic, ...final]; + input.canarySchedule = { + schedule: 'rotating-one', + intervalMs: 300_000, + durationMs: 900_000, + canaryConcurrency: 1, + startedAt: new Date(startedMs).toISOString(), + deadlineAt: new Date(startedMs + 900_000).toISOString(), + planned: 2, + started: 2, + completed: 2, + missed: 0, + overlapped: 0, + deadlineLate: 0, + checksPlanned: 2, + checksStarted: 2, + checksCompleted: 2, + rounds: [1, 2].map((periodicRound) => ({ + periodicRound, + plannedAt: new Date(startedMs + periodicRound * 300_000).toISOString(), + startedAt: new Date(startedMs + periodicRound * 300_000 + 10).toISOString(), + completedAt: new Date(startedMs + periodicRound * 300_000 + 20).toISOString(), + targetsPlanned: 1, + targetsStarted: 1, + targetsCompleted: 1, + checksPlanned: 1, + checksStarted: 1, + checksCompleted: 1, + overlapped: false, + deadlineLate: false, + startDelayMs: 10, + durationMs: 10 + })) + }; + return input; +}; + +describe('density scoring', () => { + it('uses nearest-rank percentiles', () => { + expect(percentile([40, 10, 30, 20], 0.5)).toBe(20); + expect(percentile([40, 10, 30, 20], 0.99)).toBe(40); + }); + + it('qualifies only an exact fresh PostgreSQL run/server binding', () => { + const input = qualifyingInput(); + const attestation = postgresRunAttestation(); + input.gates = { ...input.gates, requireFreshPostgresRunAttestation: true }; + input.postgresRunAttestation = attestation; + input.provenance = { + ...input.provenance!, + command: [ + ...input.provenance!.command, + '--expected-manifest-sha256', attestation.manifestSha256, + '--clone-id', attestation.cloneId + ] + }; + expect(scoreRun(input).accepted).toBe(true); + + input.postgresRunAttestation = { + ...attestation, + freshContainerForRun: false + }; + expect(scoreRun(input).failures).toContain( + 'fresh PostgreSQL run attestation is incomplete or mismatched' + ); + input.postgresRunAttestation = null; + expect(scoreRun(input).failures).toContain( + 'fresh PostgreSQL run attestation unavailable' + ); + }); + + it('requires Linux /proc samples for the exact server PID to qualify', () => { + const nonLinux = qualifyingInput(); + nonLinux.provenance = { ...nonLinux.provenance!, platform: 'darwin' }; + expect(scoreRun(nonLinux).failures).toContain( + 'qualification requires exact-PID Linux /proc RSS evidence' + ); + + const wrongPid = qualifyingInput(); + wrongPid.postWarmupNodeRssSnapshots = wrongPid.postWarmupNodeRssSnapshots.map( + (snapshot) => ({ ...snapshot, pid: 999 }) + ); + expect(scoreRun(wrongPid).failures).toContain( + 'qualification requires exact-PID Linux /proc RSS evidence' + ); + + const endpointRss = qualifyingInput(); + endpointRss.postWarmupNodeRssSnapshots = endpointRss.postWarmupNodeRssSnapshots.map( + (snapshot) => ({ ...snapshot, source: 'authenticated-endpoint' }) + ); + expect(scoreRun(endpointRss).failures).toContain( + 'qualification requires exact-PID Linux /proc RSS evidence' + ); + }); + + it('measures a linear heap slope in MiB per hour', () => { + expect(heapGrowthMiBPerHour([memory(0, 100), memory(30, 102), memory(60, 104)])) + .toBeCloseTo(4, 5); + }); + + it('uses conservative converged bookends for retained heap and external growth', () => { + const summary = retainedMemoryGrowth({ + baseline: retainedCheckpoint(0, 100, 10), + final: retainedCheckpoint(15, 101, 10.5), + errors: [] + }, 42, new Set(['tenant-a-api'])); + expect(summary.errors).toEqual([]); + expect(summary.heapMiBPerHour).toBeCloseTo(4, 4); + expect(summary.externalMiBPerHour).toBeCloseTo(2, 4); + expect(summary.durationSec).toBeCloseTo(900, 4); + }); + + it('reports zero and negative retained growth without clamping', () => { + const zero = retainedMemoryGrowth({ + baseline: retainedCheckpoint(0, 100), + final: retainedCheckpoint(15, 100), + errors: [] + }); + const negative = retainedMemoryGrowth({ + baseline: retainedCheckpoint(0, 100), + final: retainedCheckpoint(15, 99), + errors: [] + }); + expect(zero.heapMiBPerHour).toBe(0); + expect(zero.externalMiBPerHour).toBe(0); + expect(negative.heapMiBPerHour).toBeCloseTo(-4, 4); + }); + + it('independently enforces the one-MiB convergence envelope', () => { + const within = retainedCheckpoint(15, 100); + const outside = retainedCheckpoint(15, 100); + within.samples.slice(-3).forEach((sample, index) => { + sample.heapUsedBytes += [0, 0.4, 0.9][index] * 1024 ** 2; + }); + outside.samples.slice(-3).forEach((sample, index) => { + sample.heapUsedBytes += [0, 2, 0][index] * 1024 ** 2; + }); + expect(retainedMemoryGrowth({ + baseline: retainedCheckpoint(0, 100), + final: within, + errors: [] + }).errors).toEqual([]); + expect(retainedMemoryGrowth({ + baseline: retainedCheckpoint(0, 100), + final: outside, + errors: [] + }).errors).toContain('final retained heapUsedBytes samples did not converge'); + }); + + it('keeps raw heap OLS diagnostic and gates on retained bookends', () => { + const input = qualifyingInput(); + input.postWarmupSnapshots = [memory(0, 100), memory(15, 200)]; + input.memorySnapshots = input.postWarmupSnapshots; + const result = scoreRun(input); + expect(result.rawPostWarmupHeapGrowthMiBPerHour).toBeCloseTo(400, 4); + expect(result.retainedHeapGrowthMiBPerHour).toBeCloseTo(4, 4); + expect(result.accepted).toBe(true); + }); + + it('allows only balanced HTTP lifecycle progress between retained bookends', () => { + const baseline = retainedCheckpointWithState(0, retainedPhysicalState(10, 10)); + const balanced = retainedCheckpointWithState(15, retainedPhysicalState(110, 110)); + expect(retainedMemoryGrowth({ baseline, final: balanced, errors: [] }).errors) + .toEqual([]); + + const unbalanced = retainedCheckpointWithState(15, retainedPhysicalState(110, 109)); + expect(retainedMemoryGrowth({ baseline, final: unbalanced, errors: [] }).errors) + .toContain( + 'retained-memory HTTP handler delta is unbalanced: started=100, completed=99' + ); + + const changedTopology = retainedCheckpointWithState( + 15, + retainedPhysicalState(110, 110, 2) + ); + expect(retainedMemoryGrowth({ baseline, final: changedTopology, errors: [] }).errors) + .toContain('retained-memory residency or non-HTTP counters changed across the workload'); + }); + + it('compares physical residency through stable fingerprints, not process-local HMAC keys', () => { + const state = { + pid: 42, + graphileInFlight: 0, + residentBuildContracts: ['graphile:v1:process-local-hmac'], + residentBuildContractFingerprints: ['tenant-a-api'], + counters: { buildsStarted: 1, evictions: 0 } + }; + const summary = retainedMemoryGrowth({ + baseline: retainedCheckpointWithState(0, state), + final: retainedCheckpointWithState(15, state), + errors: [] + }, 42, new Set(['tenant-a-api']), true); + expect(summary.errors).toEqual([]); + + const missingStable = { ...state }; + delete (missingStable as Partial).residentBuildContractFingerprints; + expect(retainedMemoryGrowth({ + baseline: retainedCheckpointWithState(0, missingStable), + final: retainedCheckpointWithState(15, missingStable), + errors: [] + }, 42, new Set(['tenant-a-api']), true).errors).toContain( + 'baseline retained-memory residency set mismatch' + ); + }); + + it('rejects retained external growth even when V8 retained heap passes', () => { + const input = qualifyingInput(); + input.retainedMemory.final = retainedCheckpoint(15, 101, 12); + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures).toContain( + 'retained external-memory growth 8.00MiB/hour exceeds 5' + ); + }); + + it('fails closed when a retained checkpoint is unstable', () => { + const input = qualifyingInput(); + input.retainedMemory.final = { + ...input.retainedMemory.final!, + stable: false, + errors: ['PDCF_RETAINED_HEAP_NOT_CONVERGED'] + }; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures.some((failure) => + failure.includes('retained-memory checkpoint errors') + )).toBe(true); + }); + + it('falls back to the raw heap-growth gate when retained checkpoints are optional', () => { + const input = qualifyingInput(); + input.gates = { ...input.gates, requireRetainedMemoryCheckpoints: false }; + input.retainedMemory = { baseline: null, final: null, errors: [] }; + const accepted = scoreRun(input); + expect(accepted.accepted).toBe(true); + expect(accepted.retainedHeapGrowthMiBPerHour).toBeNull(); + expect(accepted.retainedMemoryCheckpointErrors).toEqual([ + 'baseline retained-memory checkpoint is unavailable', + 'final retained-memory checkpoint is unavailable' + ]); + + input.postWarmupSnapshots = [memory(0, 100), memory(15, 110)]; + input.memorySnapshots = input.postWarmupSnapshots; + const rejected = scoreRun(input); + expect(rejected.accepted).toBe(false); + expect(rejected.failures).toContain('heap growth 40.00MiB/hour exceeds 5'); + }); + + it('scores the service footprint from near-simultaneous current RSS and PostgreSQL samples', () => { + const node = [ + { ...memory(0, 100), rssBytes: 200 }, + { ...memory(1, 100), rssBytes: 250 } + ]; + const postgres = [ + { timestamp: '2026-07-31T00:00:00.100Z', usedBytes: 50, limitBytes: 1_000, raw: '' }, + { timestamp: '2026-07-31T00:01:00.100Z', usedBytes: 80, limitBytes: 1_000, raw: '' } + ]; + expect(alignedServiceMemoryPeak(node, postgres)).toEqual({ + bytes: 330, + nodeRssBytes: 250, + postgresBytes: 80, + timestamp: '2026-07-31T00:01:00.000Z', + samples: 2, + maxSkewMs: 100 + }); + expect(alignedServiceMemoryPeak(node, [{ + ...postgres[0], + timestamp: '2026-07-31T00:10:00.000Z' + }])).toBeNull(); + expect(alignedServiceMemoryCoverage( + node, + postgres, + Date.parse('2026-07-31T00:00:00.000Z'), + 60_000 + )).toMatchObject({ + expectedDurationMs: 60_000, + coveredDurationMs: 60_000, + coverageRatio: 1, + maxGapMs: 60_000 + }); + }); + + it('uses cgroup memory.peak for the conservative denominator and limits current fallback to diagnostics', () => { + const qualifying = scoreRun(qualifyingInput()); + expect(qualifying.serviceMemoryUpperBoundPostgresSource) + .toBe('cgroup-v2-memory.peak'); + expect(qualifying.serviceMemoryUpperBoundBytes) + .toBe(220 * 1024 ** 2 + 180); + + const missingPeak = qualifyingInput(); + missingPeak.postgresSnapshots = missingPeak.postgresSnapshots.map((snapshot) => ({ + ...snapshot, + cgroupV2: { ...snapshot.cgroupV2!, peakBytes: null as number | null } + })); + const rejected = scoreRun(missingPeak); + expect(rejected.serviceMemoryUpperBoundBytes).toBeNull(); + expect(rejected.failures).toEqual(expect.arrayContaining([ + 'PostgreSQL cgroup-v2 memory.peak telemetry unavailable for conservative denominator', + 'conservative service-memory upper bound unavailable' + ])); + + const mismatchedCurrent = qualifyingInput(); + mismatchedCurrent.postgresSnapshots[0] = { + ...mismatchedCurrent.postgresSnapshots[0], + usedBytes: mismatchedCurrent.postgresSnapshots[0].usedBytes + 1 + }; + expect(scoreRun(mismatchedCurrent).failures).toContain( + 'PostgreSQL cgroup-v2 telemetry was incomplete' + ); + + missingPeak.evidenceMode = 'diagnostic'; + const diagnostic = scoreRun(missingPeak); + expect(diagnostic.serviceMemoryUpperBoundPostgresSource) + .toBe('sampled-current-diagnostic'); + expect(diagnostic.serviceMemoryUpperBoundBytes) + .toBe(220 * 1024 ** 2 + 120); + }); + + it('requires aligned cgroup telemetry to cover the entire post-warm workload at bounded cadence', () => { + const input = qualifyingInput(); + input.gates = { + ...input.gates, + maxAlignedMemorySampleGapMs: 1_000, + minAlignedMemoryCoverageRatio: 0.99 + }; + const startedAtMs = Date.parse('2026-07-31T00:00:00.000Z'); + input.postWarmupNodeRssSnapshots = Array.from({ length: 901 }, (_unused, index) => ({ + timestamp: new Date(startedAtMs + index * 1_000).toISOString(), + pid: 42, + source: 'proc' as const, + rssBytes: 200 * 1024 ** 2 + })); + input.postgresSnapshots = Array.from({ length: 901 }, (_unused, index) => ({ + timestamp: new Date(startedAtMs + index * 1_000 + 100).toISOString(), + usedBytes: 100 + index, + limitBytes: 1_000_000, + source: 'cgroup-v2' as const, + cgroupV2: { + currentBytes: 100 + index, + peakBytes: 1_000 + index, + maxBytes: 1_000_000, + stat: {}, + events: { oom: 0, oom_kill: 0 } + }, + raw: '' + })); + const complete = scoreRun(input); + expect(complete.accepted).toBe(true); + expect(complete.alignedServiceMemoryCoverageRatio).toBe(1); + expect(complete.alignedServiceMemoryMaxGapMs).toBe(1_000); + + const densePostgresSnapshots = input.postgresSnapshots; + input.postgresSnapshots = densePostgresSnapshots.filter((_snapshot, index) => + index % 2 === 0 + ); + const sparsePostgres = scoreRun(input); + expect(sparsePostgres.accepted).toBe(false); + expect(sparsePostgres.alignedServiceMemoryMaxGapMs).toBe(2_000); + expect(sparsePostgres.failures).toContain( + 'aligned service-memory maximum sample gap 2000ms exceeds 1000ms' + ); + input.postgresSnapshots = densePostgresSnapshots; + + input.postWarmupNodeRssSnapshots = input.postWarmupNodeRssSnapshots.slice(0, -10); + input.postgresSnapshots = input.postgresSnapshots.slice(0, -10); + const truncated = scoreRun(input); + expect(truncated.accepted).toBe(false); + expect(truncated.failures).toEqual(expect.arrayContaining([ + expect.stringContaining('maximum sample gap 10000ms exceeds 1000ms'), + expect.stringContaining('workload coverage 98.89% is below 99.00%') + ])); + }); + + it('qualifies only a complete, conclusive, resident tenant', () => { + const result = scoreRun(qualifyingInput()); + expect(result.accepted).toBe(true); + expect(result.qualifiedCustomers).toBe(1); + expect(result.qualifiedTenants).toBe(1); + expect(result.tenantsPerConfiguredOldSpaceGiB).toBe(1); + expect(result.configuredCustomersPerAlignedServiceGiB).toBeGreaterThan(0); + expect(result.observedHeapLimitBytes).toBe(1024 * 1024 ** 2); + expect(result).toMatchObject({ + pgPoolCacheSize: 2, + pgPoolLeasedPools: 1, + pgPoolActiveLeases: 1, + postWarmupPgPoolCapacityEvictions: 0, + postWarmupPgPoolCapacityRefusals: 0, + postWarmupPgPoolDisposalFailures: 0 + }); + }); + + it('requires conclusive per-operation coverage evidence when the gate is enabled', () => { + const input = qualifyingInput(); + input.gates = { ...input.gates, requireConclusiveOperationOracles: true }; + input.tenants[0].surfaces[0].operations[0] = { + ...input.tenants[0].surfaces[0].operations[0], + requiredMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-a' + }], + forbiddenMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: 'physical-db-b' + }] + }; + input.samples.unshift({ + tenantId: 'tenant-a', + surface: 'api', + operation: 'read', + capability: 'graphile', + latencyMs: 10, + status: 200, + ok: true, + phase: 'coverage', + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: false + }); + expect(scoreRun(input).accepted).toBe(true); + + input.samples[0] = { + ...input.samples[0], + ok: false, + oracleConclusive: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_MISSING' + }; + const missing = scoreRun(input); + expect(missing.accepted).toBe(false); + expect(missing.operationOracleInconclusive).toBe(1); + expect(missing.missingOperationOracles).toEqual(['tenant-a/api/read']); + expect(missing.failures).toEqual(expect.arrayContaining([ + 'GraphQL operation response oracles inconclusive=1', + 'missing GraphQL operation response oracles: tenant-a/api/read' + ])); + + input.samples[0] = { + ...input.samples[0], + oracleConclusive: true, + oracleViolation: true, + errorCode: 'GRAPHQL_OPERATION_ORACLE_FORBIDDEN' + }; + const forbidden = scoreRun(input); + expect(forbidden.operationOracleViolations).toBe(1); + expect(forbidden.failures).toContain( + 'GraphQL operation response oracle violations=1' + ); + }); + + it('keeps the 0.5% request-error budget without treating unavailable oracles as bleed', () => { + const input = qualifyingInput(); + input.gates = { ...input.gates, requireConclusiveOperationOracles: true }; + input.tenants[0].surfaces[0].operations[0] = { + ...input.tenants[0].surfaces[0].operations[0], + requiredMatches: [{ path: '/data/physicalDatabaseIdentity', value: 'physical-db-a' }], + forbiddenMatches: [{ path: '/data/physicalDatabaseIdentity', value: 'physical-db-b' }] + }; + const baseSample = { + tenantId: 'tenant-a', + surface: 'api', + operation: 'read', + capability: 'graphile', + latencyMs: 20, + status: 200, + ok: true, + phase: 'workload' as const, + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: false, + oracleUnavailable: false + }; + input.samples = [ + { + ...baseSample, + phase: 'coverage', + }, + ...Array.from({ length: 199 }, () => ({ ...baseSample })), + { + ...baseSample, + status: 0, + ok: false, + errorCode: 'TIMEOUT', + oracleConclusive: false, + oracleUnavailable: true + } + ]; + const result = scoreRun(input); + expect(result.accepted).toBe(true); + expect(result.errorRate).toBe(0.005); + expect(result.operationOracleInconclusive).toBe(0); + expect(result.operationOracleViolations).toBe(0); + }); + + it('requires exactly one conclusive coverage result per operation', () => { + const input = qualifyingInput(); + input.gates = { ...input.gates, requireConclusiveOperationOracles: true }; + input.tenants[0].surfaces[0].operations[0] = { + ...input.tenants[0].surfaces[0].operations[0], + requiredMatches: [{ path: '/data/physicalDatabaseIdentity', value: 'physical-db-a' }], + forbiddenMatches: [{ path: '/data/physicalDatabaseIdentity', value: 'physical-db-b' }] + }; + const coverage = { + tenantId: 'tenant-a', + surface: 'api', + operation: 'read', + capability: 'graphile', + latencyMs: 10, + status: 200, + ok: true, + phase: 'coverage' as const, + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: false + }; + input.samples.unshift(coverage, { ...coverage }); + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.missingOperationOracles).toEqual(['tenant-a/api/read']); + }); + + it('requires exact initial, rotating periodic, and final canary evidence', () => { + const accepted = scoreRun(strictCanaryInput()); + expect(accepted.accepted).toBe(true); + expect(accepted).toMatchObject({ + customerWorkloadRps: 1 / 900, + periodicValidationRps: 2 / 900, + achievedRps: 1 / 900, + p99Ms: 20 + }); + expect(accepted.combinedHttpRps).toBeCloseTo(3 / 900, 12); + + for (const phase of ['initial', 'periodic', 'final'] as const) { + const input = strictCanaryInput(); + const removed = input.canaries.findIndex((canary) => canary.phase === phase); + input.canaries.splice(removed, 1); + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures.some((failure) => + failure.includes('missing exact canary evidence') + )).toBe(true); + if (phase === 'periodic') { + expect(result.failures.some((failure) => + failure.includes('periodic canary coverage is incomplete') + )).toBe(true); + expect(result.failures.some((failure) => + failure.includes('periodic target/round evidence mismatch') + )).toBe(true); + } + } + }); + + it('rejects missing, duplicate, and deadline-late periodic rounds', () => { + const missingRound = strictCanaryInput(); + missingRound.canarySchedule!.completed = 1; + missingRound.canarySchedule!.missed = 1; + missingRound.canarySchedule!.rounds[1].completedAt = null; + let result = scoreRun(missingRound); + expect(result.accepted).toBe(false); + expect(result.failures.some((failure) => + failure.includes('periodic canary rounds planned=2 started=2 completed=1 missed=1') + )).toBe(true); + + const duplicate = strictCanaryInput(); + duplicate.canaries.push({ ...duplicate.canaries.find((canary) => + canary.phase === 'periodic' + )! }); + result = scoreRun(duplicate); + expect(result.accepted).toBe(false); + expect(result.failures.some((failure) => + failure.includes('duplicate exact canary evidence') + )).toBe(true); + + const late = strictCanaryInput(); + const deadlineMs = Date.parse(late.canarySchedule!.deadlineAt); + late.canarySchedule!.deadlineLate = 1; + late.canarySchedule!.rounds[1].deadlineLate = true; + late.canarySchedule!.rounds[1].completedAt = new Date(deadlineMs + 1).toISOString(); + const lateResult = late.canaries.find((canary) => + canary.phase === 'periodic' && canary.periodicRound === 2 + )!; + lateResult.completedAt = new Date(deadlineMs + 1).toISOString(); + result = scoreRun(late); + expect(result.accepted).toBe(false); + expect(result.failures.some((failure) => + failure.includes('periodic canary rounds completed after deadline') + )).toBe(true); + }); + + it('fails closed when raw cgroup samples omit OOM event counters', () => { + const input = qualifyingInput(); + input.postgresSnapshots = input.postgresSnapshots.map((snapshot) => ({ + ...snapshot, + source: 'cgroup-v2' as const, + cgroupV2: { + currentBytes: snapshot.usedBytes, + peakBytes: snapshot.usedBytes, + maxBytes: snapshot.limitBytes, + stat: {}, + events: {} + } + })); + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.postgresOomEvents).toBeNull(); + expect(result.failures).toContain('PostgreSQL cgroup OOM event telemetry unavailable'); + }); + + it('never qualifies a smoke-length run or an eviction', () => { + const input = qualifyingInput(); + input.endedAt = '2026-07-31T00:00:05.000Z'; + input.configuredDurationSec = 5; + input.workloadDurationMs = 5_000; + input.memorySnapshots = [memory(0, 100), memory(1, 100, 1)]; + input.postWarmupSnapshots = input.memorySnapshots; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.qualifiedTenants).toBe(0); + expect(result.failures).toEqual(expect.arrayContaining([ + expect.stringContaining('15-minute'), + expect.stringContaining('evictions=1') + ])); + }); + + it('rejects matching cache counts with the wrong resident build identity', () => { + const input = qualifyingInput(); + input.memorySnapshots = input.memorySnapshots.map((snapshot): MemorySnapshot => ({ + ...snapshot, + residentBuildContracts: ['tenant-b-api'] + })); + input.postWarmupSnapshots = input.memorySnapshots; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures).toContain('resident Graphile build contracts missing: tenant-a-api'); + }); + + it('requires successful capability traffic for each configured tenant surface', () => { + const input = qualifyingInput(); + input.tenants = [{ + ...tenant, + surfaces: tenant.surfaces.map((surface) => ({ + ...surface, + operations: [ + ...surface.operations, + { name: 'search', capability: 'bm25', query: '{ search }' } + ] + })) + }]; + input.requiredCapabilities = ['graphile', 'bm25']; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.tenants[0]).toMatchObject({ + missingOperations: ['api/search'], + missingCapabilities: ['api/bm25', 'required/bm25'] + }); + expect(result.missingCapabilities).toEqual([ + 'tenant-a/api/bm25', + 'tenant-a/required/bm25' + ]); + }); + + it('does not let a healthy aggregate hide a surface that exceeds its SLA', () => { + const input = qualifyingInput(); + const apiSurface = input.tenants[0].surfaces[0]; + input.tenants = [{ + ...input.tenants[0], + surfaces: [ + apiSurface, + { ...apiSurface, name: 'admin' } + ] + }]; + input.warmedSurfaces = new Map([['tenant-a', new Set(['api', 'admin'])]]); + input.samples = ['api', 'admin'].flatMap((surface) => + Array.from({ length: 125 }, (_unused, index) => ({ + ...input.samples[0], + surface, + ok: !(surface === 'admin' && index === 0), + status: surface === 'admin' && index === 0 ? 500 : 200 + })) + ); + input.canaries = ['api', 'admin'].map((surface) => ({ + ...input.canaries[0], + surface + })); + const result = scoreRun(input); + expect(result.errorRate).toBe(0.004); + expect(result.errorRate).toBeLessThanOrEqual(input.gates.maxErrorRate); + expect(result.tenants[0].surfaces.find(({ surface }) => surface === 'admin')) + .toMatchObject({ errorRate: 0.008, qualified: false }); + expect(result.accepted).toBe(false); + }); + + it('does not let coverage-only traffic qualify a resident surface', () => { + const input = qualifyingInput(); + input.samples[0].phase = 'coverage'; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.tenants[0]).toMatchObject({ + surfacesWithTraffic: 0, + missingSurfaces: ['api'] + }); + }); + + it('keeps coverage samples out of workload latency and error metrics', () => { + const input = qualifyingInput(); + input.samples.unshift({ + tenantId: 'tenant-a', + surface: 'api', + operation: 'read', + capability: 'graphile', + latencyMs: 5_000, + status: 500, + ok: false, + phase: 'coverage', + errorCode: 'GRAPHQL_ERROR' + }); + const result = scoreRun(input); + expect(result.accepted).toBe(true); + expect(result).toMatchObject({ + coverageRequests: 1, + workloadRequests: 1, + errors: 0, + p99Ms: 20 + }); + }); + + it('rejects missing process RSS, missed arrivals, and incomplete provenance', () => { + const input = qualifyingInput(); + input.memorySnapshots = input.memorySnapshots.map((snapshot): MemorySnapshot => ({ + ...snapshot, + processPeakRssBytes: null, + pgPoolCacheSize: null + })); + input.postWarmupSnapshots = input.memorySnapshots; + input.missedArrivals = 2; + input.provenance = { ...provenance, entrySha256: null }; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures).toEqual(expect.arrayContaining([ + 'load generator missed scheduled arrivals=2', + 'OS process peak RSS telemetry unavailable', + 'PostgreSQL pool-cache telemetry unavailable', + expect.stringContaining('server provenance incomplete: entrySha256') + ])); + }); + + it('rejects post-warmup PostgreSQL pool churn and disposal failures', () => { + const input = qualifyingInput(); + input.postWarmupSnapshots = [ + memory(0, 100), + { + ...memory(15, 101), + pgPoolCapacityEvictions: 1, + pgPoolCapacityRefusals: 1, + pgPoolDisposalFailures: 1 + } + ]; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures).toEqual(expect.arrayContaining([ + 'post-warmup PostgreSQL pool capacity evictions=1', + 'post-warmup PostgreSQL pool capacity refusals=1', + 'post-warmup PostgreSQL pool disposal failures=1' + ])); + }); + + it('rejects cache, build, and pool counters that reset during the workload', () => { + const input = qualifyingInput(); + input.postWarmupSnapshots = [ + { + ...memory(0, 100, 2), + buildRefusals: 2, + buildsStarted: 2, + pgPoolCapacityEvictions: 2, + pgPoolCapacityRefusals: 2, + pgPoolDisposalFailures: 2 + }, + { + ...memory(7, 100, 3), + buildRefusals: 3, + buildsStarted: 3, + pgPoolCapacityEvictions: 3, + pgPoolCapacityRefusals: 3, + pgPoolDisposalFailures: 3 + }, + { + ...memory(15, 101, 2), + buildRefusals: 2, + buildsStarted: 2, + pgPoolCapacityEvictions: 2, + pgPoolCapacityRefusals: 2, + pgPoolDisposalFailures: 2 + } + ]; + const result = scoreRun(input); + expect(result.accepted).toBe(false); + expect(result.failures).toEqual(expect.arrayContaining([ + 'post-warmup evictions=unknown', + 'post-warmup build refusals=unknown', + 'post-warmup builds=unknown', + 'post-warmup PostgreSQL pool capacity evictions=unknown', + 'post-warmup PostgreSQL pool capacity refusals=unknown', + 'post-warmup PostgreSQL pool disposal failures=unknown' + ])); + }); + + it('requires physical database, backend, pool-client, and realtime residency when enabled', () => { + const input = qualifyingInput(); + input.gates = { ...gates, requirePhysicalDatabaseTelemetry: true }; + input.provenance = { + ...provenance, + memoryPolicy: { + ...provenance.memoryPolicy!, + graphileCacheCalibrationId: 'measured-cache-v1' + } + }; + input.tenants = [{ + ...tenant, + databases: [{ + id: 'logical:tenant-a', + physicalDatabase: 'physical_tenant_a', + apis: [{ + id: 'api:tenant-a', + runtimePoolIdentity: 'pg:v1:tenant-a', + physicalSchemas: ['tenant_a'], + routingLabels: ['tenant-a.localhost'], + realtime: true, + surfaces: ['api'] + }] + }] + }]; + input.memorySnapshots = input.memorySnapshots.map((snapshot): MemorySnapshot => ({ + ...snapshot, + cacheConfiguredMax: 3, + cacheBudgetCapacity: 3, + cacheInstanceHeapBytes: 16 * 1024 ** 2, + cacheCalibrationId: 'measured-cache-v1', + physicalDatabases: 1, + postgresContainerDedicated: true, + unexpectedPostgresDatabases: 0, + postgresBackendTotal: 1, + pgPoolTotalClients: 1, + pgPoolIdleClients: 0, + pgPoolWaitingClients: 0, + runtimePoolTelemetryScope: 'runtime-only-exact-identities', + runtimePoolTelemetryAvailable: true, + runtimePoolRequestedMaxUses: null, + runtimePoolEffectiveMaxUses: null, + runtimePoolEffectiveMaxUsesKnown: true, + runtimePoolMaxUsesExact: true, + runtimePoolExpectedPools: 1, + runtimePoolObservedPools: 1, + runtimePoolTotalClients: 1, + runtimePoolIdleClients: 0, + runtimePoolWaitingClients: 0, + realtimeManagersExpected: 1, + realtimeManagersActive: 1, + realtimeTransportsExpected: 1, + realtimeTransportsActive: 1, + realtimeNotificationMode: 'dedicated' + })); + input.postWarmupSnapshots = input.memorySnapshots; + const accepted = scoreRun(input); + expect(accepted.accepted).toBe(true); + expect(accepted).toMatchObject({ + residentPhysicalDatabases: 1, + cacheConfiguredMax: 3, + cacheBudgetCapacity: 3, + cacheCalibrationId: 'measured-cache-v1', + postgresContainerDedicated: true, + unexpectedPostgresDatabases: 0, + postgresBackendPeak: 1, + pgPoolTotalClients: 1, + runtimePoolExpectedPools: 1, + runtimePoolObservedPools: 1, + residentRealtimeManagers: 1, + residentRealtimeTransports: 1 + }); + + input.postWarmupSnapshots = input.memorySnapshots.map((snapshot): MemorySnapshot => ({ + ...snapshot, + runtimePoolRequestedMaxUses: 1, + runtimePoolEffectiveMaxUses: 1 + })); + const singleCheckout = scoreRun(input); + expect(singleCheckout.accepted).toBe(true); + expect(singleCheckout).toMatchObject({ + runtimePoolRequestedMaxUses: 1, + runtimePoolEffectiveMaxUses: 1 + }); + + const exactRuntimePoolSnapshots = input.postWarmupSnapshots; + input.postWarmupSnapshots = exactRuntimePoolSnapshots.map((snapshot, index) => ({ + ...snapshot, + runtimePoolExpectedPools: index === 0 ? 1 : 2, + runtimePoolObservedPools: index === 0 ? 1 : 2 + })); + const inexactCardinality = scoreRun(input); + expect(inexactCardinality.accepted).toBe(false); + expect(inexactCardinality.failures).toContain( + 'exact runtime PostgreSQL pool telemetry unavailable or inconsistent; observed=unknown, expected=1' + ); + + input.postWarmupSnapshots = exactRuntimePoolSnapshots.map((snapshot, index) => ({ + ...snapshot, + runtimePoolIdleClients: index === 0 ? 1 : 0 + })); + expect(scoreRun(input).failures).toContain( + 'runtime PostgreSQL maxUses=1 retained idle clients after warmup' + ); + + input.postWarmupSnapshots = input.memorySnapshots.map((snapshot): MemorySnapshot => ({ + ...snapshot, + postgresBackendTotal: null + })); + const rejected = scoreRun(input); + expect(rejected.accepted).toBe(false); + expect(rejected.failures).toContain('physical PostgreSQL backend telemetry unavailable'); + }); + + it('qualifies shared realtime from exact broker evidence without requiring one backend per API', () => { + const input = qualifyingInput(); + const residentContracts = [ + 'tenant-a-api', + 'tenant-a-admin', + 'tenant-a-private' + ]; + input.gates = { ...gates, requirePhysicalDatabaseTelemetry: true }; + input.provenance = { + ...provenance, + memoryPolicy: { + ...provenance.memoryPolicy!, + graphileCacheCalibrationId: 'measured-cache-v1' + } + }; + input.tenants = [{ + ...tenant, + surfaces: [ + tenant.surfaces[0], + { ...tenant.surfaces[0], name: 'admin', buildContract: 'tenant-a-admin' }, + { ...tenant.surfaces[0], name: 'private', buildContract: 'tenant-a-private' } + ], + databases: [{ + id: 'logical:tenant-a', + physicalDatabase: 'physical_tenant_a', + apis: ['api', 'admin', 'private'].map((name) => ({ + id: `${name}:tenant-a`, + runtimePoolIdentity: `pg:v1:tenant-a:${name}`, + physicalSchemas: [`tenant_a_${name}`], + routingLabels: [`${name}.tenant-a.localhost`], + realtime: true, + surfaces: [name] + })) + }] + }]; + input.warmedSurfaces = new Map([[ + 'tenant-a', + new Set(['api', 'admin', 'private']) + ]]); + input.samples = ['api', 'admin', 'private'].map((surface) => ({ + ...input.samples[0], + surface + })); + input.canaries = ['api', 'admin', 'private'].map((surface) => ({ + ...input.canaries[0], + surface + })); + input.memorySnapshots = input.memorySnapshots.map((snapshot): MemorySnapshot => ({ + ...snapshot, + cacheSize: 3, + residentBuildContractFingerprints: residentContracts, + residentBuildContracts: residentContracts, + cacheConfiguredMax: 3, + cacheBudgetCapacity: 3, + cacheInstanceHeapBytes: 16 * 1024 ** 2, + cacheCalibrationId: 'measured-cache-v1', + physicalDatabases: 1, + postgresContainerDedicated: true, + unexpectedPostgresDatabases: 0, + postgresBackendTotal: 1, + pgPoolTotalClients: 1, + pgPoolIdleClients: 0, + pgPoolWaitingClients: 0, + runtimePoolTelemetryScope: 'runtime-only-exact-identities', + runtimePoolTelemetryAvailable: true, + runtimePoolRequestedMaxUses: null, + runtimePoolEffectiveMaxUses: null, + runtimePoolEffectiveMaxUsesKnown: true, + runtimePoolMaxUsesExact: true, + runtimePoolExpectedPools: 3, + runtimePoolObservedPools: 3, + runtimePoolTotalClients: 1, + runtimePoolIdleClients: 0, + runtimePoolWaitingClients: 0, + realtimeManagersExpected: 3, + realtimeManagersActive: 3, + realtimeTransportsExpected: 3, + realtimeTransportsActive: 3, + realtimeNotificationMode: 'shared-exact' as const, + notificationBrokers: 1, + notificationListenerConnections: 1, + notificationBrokerLeases: 3, + notificationBrokerTopics: 3, + notificationBrokerSubscribers: 3, + notificationBrokerQueueOverflows: 0, + notificationBrokerFatalFailures: 0, + notificationAuditIdentities: 1, + notificationAuditsHealthy: 1, + notificationAuditsFailed: 0, + notificationAuditsStale: 0, + notificationAuditAttempts: 3, + notificationAuditFailures: 0, + notificationAuditActiveDatabaseTargets: 1, + notificationAuditDatabaseConflicts: 0 + })); + input.postWarmupSnapshots = input.memorySnapshots; + const bindRetainedResidency = ( + checkpoint: RetainedMemoryCheckpoint + ): RetainedMemoryCheckpoint => { + const state = { + ...checkpoint.guardAfter.state, + residentBuildContracts: residentContracts, + residentBuildContractFingerprints: residentContracts + }; + const guard = { + ...checkpoint.guardAfter, + residentBuildContracts: residentContracts, + state, + stateSha256: `sha256:${createHash('sha256') + .update(canonicalJson(state)) + .digest('hex')}` + }; + return { ...checkpoint, guardBefore: guard, guardAfter: guard }; + }; + input.retainedMemory = { + baseline: bindRetainedResidency(input.retainedMemory!.baseline!), + final: bindRetainedResidency(input.retainedMemory!.final!), + errors: [] + }; + + const accepted = scoreRun(input); + expect(accepted.failures).toEqual([]); + expect(accepted.accepted).toBe(true); + expect(accepted).toMatchObject({ + realtimeNotificationMode: 'shared-exact', + notificationBrokers: 1, + notificationListenerConnections: 1, + notificationBrokerLeases: 3, + notificationBrokerSubscribers: 3, + postgresBackendPeak: 1, + pgPoolTotalClients: 1 + }); + + input.postWarmupSnapshots = input.memorySnapshots.map((snapshot) => ({ + ...snapshot, + notificationBrokerSubscribers: 2 + })); + const rejected = scoreRun(input); + expect(rejected.accepted).toBe(false); + expect(rejected.failures).toContain( + 'shared realtime broker residency or listener-role attestation is not exact' + ); + }); + + it('binds required cache admission to both live telemetry and pinned provenance', () => { + const input = qualifyingInput(); + input.gates = { + ...input.gates, + requiredCacheAdmissionMode: 'preserve-resident' + }; + input.provenance = { + ...provenance, + memoryPolicy: { + ...provenance.memoryPolicy!, + graphileCacheAdmissionMode: 'preserve-resident' + } + }; + input.memorySnapshots = input.memorySnapshots.map((snapshot) => ({ + ...snapshot, + cacheAdmissionMode: 'preserve-resident' as const + })); + input.postWarmupSnapshots = input.memorySnapshots; + const accepted = scoreRun(input); + expect(accepted.accepted).toBe(true); + expect(accepted.cacheAdmissionMode).toBe('preserve-resident'); + + input.postWarmupSnapshots = input.memorySnapshots.map((snapshot) => ({ + ...snapshot, + cacheAdmissionMode: 'evict-idle' as const + })); + const liveMismatch = scoreRun(input); + expect(liveMismatch.accepted).toBe(false); + expect(liveMismatch.failures).toContain( + 'live Graphile cache admission mode=evict-idle, required preserve-resident' + ); + + input.postWarmupSnapshots = input.memorySnapshots; + input.provenance.memoryPolicy!.graphileCacheAdmissionMode = 'evict-idle'; + const pinnedMismatch = scoreRun(input); + expect(pinnedMismatch.accepted).toBe(false); + expect(pinnedMismatch.failures).toContain( + 'pinned Graphile cache admission mode=evict-idle, required preserve-resident' + ); + }); + + const densityRun = ( + arm: string, + configuredTenants: number, + accepted: boolean, + peakRssDensity: number, + repetition = 1, + expectedMatrixRepetitions = 1, + heapMiB = 1024 + ): DensityRunResult => ({ + schemaVersion: 6, + runKind: 'matrix', + evidenceMode: 'qualification', + qualificationCohortSha256: 'a'.repeat(64), + arm, + repetition, + expectedMatrixRepetitions, + accepted, + configuredCustomers: configuredTenants, + qualifiedCustomers: accepted ? configuredTenants : 0, + qualifiedTenants: accepted ? configuredTenants : 0, + tenantsPerConfiguredOldSpaceGiB: accepted + ? configuredTenants / (heapMiB / 1024) + : 0, + tenantsPerPeakRssGiB: accepted ? peakRssDensity : null, + customersPerAlignedServiceGiB: accepted ? peakRssDensity : null, + customersPerServiceMemoryUpperBoundGiB: accepted ? peakRssDensity : null, + heapMiB, + configuredTenants + } as unknown as DensityRunResult); + + it('requires an all-repetition pass and a higher failure to establish capacity', () => { + const runs = [ + densityRun('scoped-introspection', 1, true, 2, 1, 2), + densityRun('scoped-introspection', 1, true, 2.1, 2, 2), + densityRun('scoped-introspection', 2, true, 3, 1, 2), + densityRun('scoped-introspection', 2, true, 3.1, 2, 2), + densityRun('scoped-introspection', 3, false, 0, 1, 2), + densityRun('scoped-introspection', 3, false, 0, 2, 2) + ]; + expect(summarizeCapacityBoundaries(runs)[0]).toMatchObject({ + highestAllRepetitionsPass: 2, + lowestGreaterFail: 3, + monotonicQualification: true, + capacityBoundaryReached: true, + incompleteTenantCounts: [] + }); + + runs.pop(); + expect(summarizeCapacityBoundaries(runs)[0]).toMatchObject({ + capacityBoundaryReached: false, + incompleteTenantCounts: [3] + }); + }); + + it('rejects non-monotonic and duplicate repetition boundaries', () => { + const nonMonotonic = [ + densityRun('scoped-introspection', 1, false, 0), + densityRun('scoped-introspection', 2, true, 2), + densityRun('scoped-introspection', 3, false, 0) + ]; + expect(summarizeCapacityBoundaries(nonMonotonic)[0]).toMatchObject({ + highestAllRepetitionsPass: 2, + monotonicQualification: false, + capacityBoundaryReached: false + }); + + const duplicateRepetition = [ + densityRun('scoped-introspection', 1, true, 1, 1, 2), + densityRun('scoped-introspection', 1, true, 1, 1, 2), + densityRun('scoped-introspection', 2, false, 0, 1, 2), + densityRun('scoped-introspection', 2, false, 0, 2, 2) + ]; + expect(summarizeCapacityBoundaries(duplicateRepetition)[0]).toMatchObject({ + capacityBoundaryReached: false, + incompleteTenantCounts: [1] + }); + }); + + it('decides improvement from actual service memory and keeps heap metrics diagnostic', () => { + const baseline = [ + densityRun('cache-governor-stock', 1, true, 1), + densityRun('cache-governor-stock', 2, false, 0), + densityRun('cache-governor-stock', 3, false, 0) + ]; + const candidate = [ + densityRun('scoped-introspection', 1, true, 1), + densityRun('scoped-introspection', 2, true, 1.3), + densityRun('scoped-introspection', 3, false, 0) + ]; + expect(compareDensity(baseline, candidate, gates)).toMatchObject({ + materiallyBetter: true, + everyHeapAddsTenants: true, + capacityBoundariesComplete: true, + pairedMatrixComplete: true, + configuredOldSpaceMedianImprovement: 1, + configuredOldSpaceNonRegression: true, + peakRssNonRegression: true + }); + expect(compareDensity(baseline, candidate, gates).peakRssMedianImprovement) + .toBeCloseTo(0.3, 10); + + candidate[1].tenantsPerPeakRssGiB = 0.9; + expect(compareDensity(baseline, candidate, gates)).toMatchObject({ + materiallyBetter: true, + peakRssNonRegression: false + }); + + candidate[1].customersPerAlignedServiceGiB = 0.9; + candidate[1].customersPerServiceMemoryUpperBoundGiB = 0.9; + expect(compareDensity(baseline, candidate, gates)).toMatchObject({ + materiallyBetter: false, + alignedServiceNonRegression: false, + serviceMemoryUpperBoundNonRegression: false + }); + }); + + it('requires the additional-customer gate in every paired repetition', () => { + const matrix = ( + arm: string, + capacities: [number, number] + ): DensityRunResult[] => [1, 2, 3, 4].flatMap((count) => [1, 2].map((repetition) => + densityRun( + arm, + count, + count <= capacities[repetition - 1], + count <= capacities[repetition - 1] ? count : 0, + repetition, + 2 + ) + )); + + const baseline = matrix('cache-governor-stock', [1, 2]); + const aggregateOnlyImprovement = matrix('scoped-introspection', [2, 2]); + const aggregateComparison = compareDensity( + baseline, + aggregateOnlyImprovement, + gates + ); + expect(aggregateComparison.baselineBoundaries[0].highestAllRepetitionsPass).toBe(1); + expect(aggregateComparison.candidateBoundaries[0].highestAllRepetitionsPass).toBe(2); + expect(aggregateComparison.capacityBoundariesComplete).toBe(true); + expect(aggregateComparison.everyHeapAddsTenants).toBe(false); + expect(aggregateComparison.materiallyBetter).toBe(false); + + const everyRepetitionImproves = matrix('scoped-introspection', [2, 3]); + expect(compareDensity(baseline, everyRepetitionImproves, gates)).toMatchObject({ + everyHeapAddsTenants: true, + materiallyBetter: true + }); + }); + + it('rejects an unbracketed per-repetition capacity even when the aggregate boundary exists', () => { + const baseline = [1, 2, 3].flatMap((count) => [1, 2].map((repetition) => + densityRun( + 'cache-governor-stock', + count, + count <= repetition, + count <= repetition ? count : 0, + repetition, + 2 + ) + )); + const candidate = [1, 2, 3].flatMap((count) => [1, 2].map((repetition) => + densityRun( + 'scoped-introspection', + count, + count <= repetition + 1, + count <= repetition + 1 ? count : 0, + repetition, + 2 + ) + )); + expect(summarizeCapacityBoundaries(candidate)[0].capacityBoundaryReached).toBe(true); + expect(compareDensity(baseline, candidate, gates)).toMatchObject({ + everyHeapAddsTenants: false, + materiallyBetter: false + }); + }); + + it('does not call an unbracketed or incomplete matrix materially better', () => { + const baseline = [ + densityRun('cache-governor-stock', 1, true, 1), + densityRun('cache-governor-stock', 2, false, 0) + ]; + const candidate = [ + densityRun('scoped-introspection', 1, true, 1), + densityRun('scoped-introspection', 2, true, 1.3) + ]; + expect(compareDensity(baseline, candidate, gates)).toMatchObject({ + materiallyBetter: false, + capacityBoundariesComplete: false, + pairedMatrixComplete: true + }); + expect(compareDensity(baseline, candidate.slice(0, 1), gates)).toMatchObject({ + materiallyBetter: false, + pairedMatrixComplete: false + }); + }); +}); diff --git a/packages/perf-harness/src/catalog-bench.ts b/packages/perf-harness/src/catalog-bench.ts new file mode 100644 index 0000000000..90fa5f91a2 --- /dev/null +++ b/packages/perf-harness/src/catalog-bench.ts @@ -0,0 +1,3996 @@ +import { execFileSync, spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { getHeapStatistics } from 'node:v8'; + +import { execute, grafast } from 'grafast'; +import { withPgClientFromPgService } from 'graphile-build-pg'; +import { + createGraphileInstance, + type GraphileCacheEntry +} from 'graphile-cache'; +import { + ConstructivePreset, + createGrafastCacheLimitsPreset, + makePgService +} from 'graphile-settings'; +import { + type ExecutionResult, + lexicographicSortSchema, + parse, + printSchema} from 'graphql'; +import { Pool } from 'pg'; +import { getPgEnvOptions } from 'pg-env'; + +import { + nodeFlagsForV8Profile, + replaceMaxOldSpaceSize, + tokenizeNodeOptions +} from './process'; +import type { IntrospectionMode, NodeV8Profile } from './types'; + +const BUILD_TRANSIENT_SAMPLE_INTERVAL_MS = 5; +const BACKEND_MEMORY_SAMPLE_INTERVAL_MS = 10; +const BACKEND_MEMORY_MAX_CONCLUSIVE_GAP_MS = 50; +const BACKEND_MEMORY_SAMPLER_START_TIMEOUT_MS = 5_000; +const BACKEND_MEMORY_SAMPLER_STOP_TIMEOUT_MS = 5_000; +const BACKEND_MEMORY_SAMPLER_TERM_TIMEOUT_MS = 1_000; +const BACKEND_MEMORY_SAMPLER_KILL_TIMEOUT_MS = 1_000; +const BACKEND_START_IDENTITY_TOLERANCE_MS = 1_500; +const BACKEND_RETIREMENT_POLL_INTERVAL_MS = 10; +const BACKEND_RETIREMENT_TIMEOUT_MS = 5_000; +const GIB = 1024 ** 3; +const MIB_PER_GIB = 1024; +const LINUX_CGROUP_V2_DENSITY_AUTHORITY = 'linux-cgroup-v2-memory.current'; +const DESTROYED_BACKEND_LOWER_BOUND_LIMITATION = + 'Without a PostgreSQL pre-destroy acknowledgement, sampling may stop before the ' + + 'backend records its terminal VmHWM; every sampled peak is therefore a diagnostic ' + + 'lower bound, even when cadence and process identity are conclusive.'; +const DOCKER_DESKTOP_BACKEND_SAMPLER_LIMITATION = + 'Docker Desktop transports procfs samples across a Linux VM boundary. The backend ' + + 'trace remains diagnostic, and service-density authority must come from a ' + + 'separately validated Linux cgroup-v2 memory.current measurement.'; +const BACKEND_MEMORY_SAMPLER_SCRIPT = String.raw`set -eu +status_file=$1 +stat_file=$2 +proc_stat_file=$3 +expected_pid=$4 +expected_backend_start_epoch_ms=$5 +start_tolerance_ms=$6 +interval_seconds=$7 + +if [ -x /usr/bin/awk ]; then + awk_command=/usr/bin/awk +elif [ -x /bin/awk ]; then + awk_command=/bin/awk +else + printf 'catalog backend sampler requires /usr/bin/awk or /bin/awk\n' >&2 + exit 43 +fi +if [ -x /usr/bin/sleep ]; then + sleep_command=/usr/bin/sleep +elif [ -x /bin/sleep ]; then + sleep_command=/bin/sleep +else + printf 'catalog backend sampler requires /usr/bin/sleep or /bin/sleep\n' >&2 + exit 43 +fi +if [ -x /usr/bin/getconf ]; then + getconf_command=/usr/bin/getconf +elif [ -x /bin/getconf ]; then + getconf_command=/bin/getconf +else + printf 'catalog backend sampler requires /usr/bin/getconf or /bin/getconf\n' >&2 + exit 43 +fi + +clock_ticks=$($getconf_command CLK_TCK) +case "$clock_ticks" in + ''|*[!0-9]*) printf 'invalid CLK_TCK value\n' >&2; exit 43 ;; +esac + +if [ ! -r "$stat_file" ] || [ ! -r "$proc_stat_file" ]; then + printf 'PostgreSQL backend procfs identity files are unavailable\n' >&2 + exit 42 +fi +initial_identity=$($awk_command -v stat_file="$stat_file" -v proc_stat_file="$proc_stat_file" ' + FILENAME == stat_file { + line=$0 + sub(/^.*\) /, "", line) + count=split(line, fields, / +/) + if (count >= 20) start_ticks=fields[20] + } + FILENAME == proc_stat_file && $1 == "btime" { boot_time=$2 } + END { + if (start_ticks !~ /^[0-9]+$/ || boot_time !~ /^[0-9]+$/) exit 42 + printf "%s %s\n", start_ticks, boot_time + } +' "$stat_file" "$proc_stat_file") +set -- $initial_identity +expected_start_ticks=$1 +expected_boot_time_epoch_seconds=$2 + +sampler_pid= +cleanup_sampler() { + trap - EXIT + if [ -n "$sampler_pid" ]; then + kill "$sampler_pid" 2>/dev/null || true + wait "$sampler_pid" 2>/dev/null || true + sampler_pid= + fi +} +trap cleanup_sampler EXIT +trap 'exit 143' HUP INT TERM + +report_gone() { + IFS=' ' read -r uptime_seconds _ < /proc/uptime || uptime_seconds=0 + printf 'gone\t%s\n' "$uptime_seconds" +} + +sample_backend() { + if [ ! -r "$status_file" ] || [ ! -r "$stat_file" ]; then + report_gone + return 1 + fi + IFS=' ' read -r uptime_seconds _ < /proc/uptime + set +e + $awk_command \ + -v uptime_seconds="$uptime_seconds" \ + -v expected_pid="$expected_pid" \ + -v expected_start_ticks="$expected_start_ticks" \ + -v expected_boot_time="$expected_boot_time_epoch_seconds" \ + -v expected_backend_start_epoch_ms="$expected_backend_start_epoch_ms" \ + -v start_tolerance_ms="$start_tolerance_ms" \ + -v clock_ticks="$clock_ticks" \ + -v status_file="$status_file" \ + -v stat_file="$stat_file" \ + -v proc_stat_file="$proc_stat_file" ' + FILENAME == status_file && /^Name:/ { process_name=$2 } + FILENAME == status_file && /^NSpid:/ { namespace_pid=$NF } + FILENAME == status_file && /^VmRSS:/ { rss=$2 } + FILENAME == status_file && /^VmHWM:/ { hwm=$2 } + FILENAME == stat_file { + line=$0 + sub(/^.*\) /, "", line) + count=split(line, fields, / +/) + if (count >= 20) start_ticks=fields[20] + } + FILENAME == proc_stat_file && $1 == "btime" { boot_time=$2 } + END { + proc_start_epoch_ms=(boot_time + (start_ticks / clock_ticks)) * 1000 + start_delta_ms=proc_start_epoch_ms - expected_backend_start_epoch_ms + if (start_delta_ms < 0) start_delta_ms=-start_delta_ms + if ( + process_name !~ /^postgres/ + || namespace_pid != expected_pid + || rss !~ /^[0-9]+$/ + || hwm !~ /^[0-9]+$/ + || start_ticks !~ /^[0-9]+$/ + || start_ticks != expected_start_ticks + || boot_time != expected_boot_time + || clock_ticks !~ /^[0-9]+$/ + || start_delta_ms > start_tolerance_ms + ) exit 42 + printf "sample\t%s\t%s\t%s\t%s\t%.0f\t%s\t%s\n", \ + uptime_seconds, rss, hwm, start_ticks, proc_start_epoch_ms, \ + boot_time, clock_ticks + } + ' "$status_file" "$stat_file" "$proc_stat_file" + sample_status=$? + set -e + if [ "$sample_status" -ne 0 ]; then + if [ ! -r "$status_file" ] || [ ! -r "$stat_file" ]; then + report_gone + return 1 + fi + printf 'immutable PostgreSQL backend procfs identity validation failed\n' >&2 + exit 42 + fi +} + +sample_backend +( + while "$sleep_command" "$interval_seconds"; do + sample_backend || break + done +) & +sampler_pid=$! +IFS= read -r _ || true +kill "$sampler_pid" 2>/dev/null || true +set +e +wait "$sampler_pid" 2>/dev/null +sampler_status=$? +set -e +sampler_pid= +if [ "$sampler_status" -ne 0 ] && [ "$sampler_status" -ne 143 ]; then + exit "$sampler_status" +fi +set +e +sample_backend +final_sample_status=$? +set -e +if [ "$final_sample_status" -ne 0 ] && [ "$final_sample_status" -ne 1 ]; then + exit "$final_sample_status" +fi +`; + +export type CatalogScopedCatalogTypes = 'all' | 'dependency-closure'; +export type CatalogIntrospectionClientReleaseMode = 'reuse' | 'destroy'; +export type CatalogBackendSamplerMode = 'off' | 'diagnostic-lower-bound'; + +export interface CatalogBenchConfig { + version: 1; + database: string; + mode: IntrospectionMode; + scopedCatalogTypes: CatalogScopedCatalogTypes | null; + introspectionClientReleaseMode: CatalogIntrospectionClientReleaseMode; + postgresBackendSamplerMode: CatalogBackendSamplerMode; + releaseBuildStateAfterValidation: boolean; + schemas: string[]; + /** Ordered exposed schemas per instance; omitted for the legacy one-schema path. */ + schemaSets?: string[][]; + /** Explicit scoped-introspection dependency allowlist for schemaSets mode. */ + allowedDependencySchemas?: string[]; + checkpoints: number[]; + expectedTokens: string[] | null; + heapMiB: number; + repetition: number; + settleMs: number; + warmOperationsPerInstance: number; + warmOperationReplayPasses: number; + grafastCacheLimits: CatalogGrafastCacheLimits; + postgresContainer: string | null; + commit: string | null; + worktreeDirty: boolean | null; + sourceStateSha256: string | null; + lockfileSha256: string | null; + executedEntrySha256: string; + v8Profile: NodeV8Profile; + nodeOptions: string; + nodeOptionsArgv: string[]; + nodeExecArgv: string[]; + effectiveNodeRuntimeFlags: string[]; +} + +export interface CatalogGrafastCacheLimits { + queryCacheMaxLength: number | null; + operationsCacheMaxLength: number | null; + operationOperationPlansCacheMaxLength: number | null; +} + +export interface CatalogMemorySnapshot { + instances: number; + heapUsedBytes: number; + heapDeltaBytes: number; + rssBytes: number; + rssDeltaBytes: number; + externalBytes: number; + externalDeltaBytes: number; + processPeakRssBytes: number; + processPeakRssDeltaBytes: number; + postgresBackendRssBytes: number | null; + postgresBackendRssDeltaBytes: number | null; + postgresBackendHighWaterBytes: number | null; + postgresBackendHighWaterDeltaBytes: number | null; +} + +export interface CatalogBenchProgress { + version: 1; + status: 'in-progress' | 'complete'; + mode: IntrospectionMode; + scopedCatalogTypes: CatalogScopedCatalogTypes | null; + introspectionClientReleaseMode: CatalogIntrospectionClientReleaseMode; + postgresBackendSamplerMode: CatalogBackendSamplerMode; + releaseBuildStateAfterValidation: boolean; + repetition: number; + heapMiB: number; + v8Profile: NodeV8Profile; + nodeOptions: string; + nodeOptionsArgv: string[]; + nodeExecArgv: string[]; + effectiveNodeRuntimeFlags: string[]; + targetInstances: number; + completedInstances: number; + configuredCheckpoints: number[]; + completedCheckpoints: number[]; + buildsCompleted: number; + canariesCompleted: number; + mismatchViolations: number; + crossTenantViolations: number; + lastSnapshot: CatalogMemorySnapshot; + updatedAt: string; +} + +export interface CatalogTenantProxyDensityPoint { + residentSurfaceInstances: number; + fullTenantProxyGroups: number; + remainderSurfaceInstances: number; + configuredOldSpaceMiB: number; + absolutePeakProcessRssBytes: number; + groupsPerConfiguredOldSpaceGiB: number; + groupsPerAbsolutePeakProcessRssGiB: number; +} + +export interface CatalogBackendPidTransition { + introspectionBackendPid: number; + introspectionBackendStartEpochMs: number; + steadyBackendPid: number; + steadyBackendStartEpochMs: number; + introspectionBackendRetired: boolean; +} + +export interface CatalogBuildSample extends CatalogBackendPidTransition { + instance: number; + schema: string; + buildMs: number; + queryMs: number; + token: string | null; + sdlBytes: number; + sdlSha256: string; + queryFields: string[]; + warmOperations: number; + warmOperationLatencyP50Ms: number | null; + warmOperationLatencyP99Ms: number | null; + warmOperationErrors: number; + warmOperationReturnedStrings: number; + warmOperationExactMatches: number; + warmOperationMismatchViolations: number; + warmOperationCrossTenantViolations: number; + warmOperationCorrectnessConclusive: boolean; + warmOperationCorrectnessPassed: boolean; + warmOperationReplayPasses: number; + warmOperationReplayExecutions: number; + warmOperationReplayLatencyP50Ms: number | null; + warmOperationReplayLatencyP99Ms: number | null; + warmOperationReplayErrors: number; + warmOperationReplayReturnedStrings: number; + warmOperationReplayExactMatches: number; + warmOperationReplayMismatchViolations: number; + warmOperationReplayCrossTenantViolations: number; + warmOperationReplayCorrectnessConclusive: boolean; + warmOperationReplayCorrectnessPassed: boolean; + buildBaselineHeapUsedBytes: number; + buildBaselineRssBytes: number; + sampledBuildPeakHeapUsedBytes: number; + sampledBuildPeakHeapDeltaBytes: number; + sampledBuildPeakRssBytes: number; + sampledBuildPeakRssDeltaBytes: number; + processBuildPeakRssBytes: number; + processBuildPeakRssDeltaBytes: number; + buildTransientSampleCount: number; + postgresIntrospectionBackendMemoryLowerBound: + CatalogBackendIntrospectionMemoryLowerBoundMeasurement | null; +} + +export interface CatalogCanarySample { + phase: 'initial' | 'checkpoint'; + residentInstances: number; + instance: number; + schema: string; + expected: string; + actual: string | null; + returnedString: boolean; + exactMatch: boolean; + matchedOtherTenant: boolean; +} + +export interface CatalogBenchResult { + version: 1; + status: 'performance-only'; + database: string; + mode: IntrospectionMode; + scopedCatalogTypes: CatalogScopedCatalogTypes | null; + introspectionClientReleaseMode: CatalogIntrospectionClientReleaseMode; + postgresBackendSamplerMode: CatalogBackendSamplerMode; + releaseBuildStateAfterValidation: boolean; + /** Present only for the explicit multi-schema surface mode. */ + schemaSets?: string[][]; + /** Present only for the explicit multi-schema surface mode. */ + allowedDependencySchemas?: string[]; + repetition: number; + heapMiB: number; + commit: string | null; + worktreeDirty: boolean | null; + sourceStateSha256: string | null; + lockfileSha256: string | null; + executedEntrySha256: string; + v8Profile: NodeV8Profile; + nodeOptions: string; + nodeOptionsArgv: string[]; + nodeExecArgv: string[]; + effectiveNodeRuntimeFlags: string[]; + node: string; + v8: string; + effectiveV8HeapLimitBytes: number; + platform: string; + architecture: string; + startedAt: string; + endedAt: string; + catalog: { + classes: number; + attributes: number; + procs: number; + types: number; + namespaces: number; + }; + runtimeRole: { + name: string; + superuser: boolean; + bypassRls: boolean; + createRole: boolean; + ownsDatabase: boolean; + canCreateInDatabase: boolean; + ownsRequestedSchema: boolean; + canCreateInRequestedSchema: boolean; + }; + catalogWarmth: 'shared-server-not-reset'; + grafastCacheWarmth: { + operationsPerInstance: number; + cacheLimits: CatalogGrafastCacheLimits; + sourceMode: 'grafast-source'; + sourceSetSha256: string | null; + operationExecutions: number; + latencyP50Ms: number | null; + latencyP99Ms: number | null; + errors: number; + returnedStrings: number; + exactMatches: number; + mismatchViolations: number; + crossTenantViolations: number; + correctnessConclusive: boolean; + correctnessPassed: boolean; + replay: { + passesPerInstance: number; + operationExecutions: number; + latencyP50Ms: number | null; + latencyP99Ms: number | null; + errors: number; + returnedStrings: number; + exactMatches: number; + mismatchViolations: number; + crossTenantViolations: number; + correctnessConclusive: boolean; + correctnessPassed: boolean; + }; + }; + buildTransientSampling: { + approximate: true; + intervalMs: number; + limitation: string; + maxSampledHeapDeltaBytes: number; + maxSampledRssDeltaBytes: number; + maxProcessPeakRssDeltaBytes: number; + }; + postgresBackendMeasurement: { + initialBackendPid: number; + initialBackendStartEpochMs: number; + finalSteadyBackendPid: number; + finalSteadyBackendStartEpochMs: number; + expectedRetirementChecks: number; + completedRetirementChecks: number; + allExpectedRetirementsProven: boolean; + steadyBackendRss: { + measured: boolean; + samplePhase: 'shared-introspection-and-steady' | 'post-introspection-replacement'; + deltaBasis: 'initial-backend' | 'replacement-acquisition'; + }; + introspectionBackendMemory: { + sampledLowerBoundMeasured: boolean; + sharedSnapshotMeasured: boolean; + semantics: + | 'diagnostic-lower-bound-without-pre-destroy-acknowledgement' + | 'post-build-shared-backend-snapshot' + | 'unavailable'; + measurementMethod: + | 'dedicated-identity-bound-procfs-sampler' + | 'post-build-shared-backend-procfs' + | 'unavailable'; + expectedBuildMeasurements: number; + completedBuildMeasurements: number; + allBuildCadenceChecksConclusive: boolean; + backendSamplerAuthority: 'diagnostic-only'; + serviceDensityMemoryAuthority: + `separately-validated-${typeof LINUX_CGROUP_V2_DENSITY_AUTHORITY}`; + limitation: string | null; + }; + }; + fixtureFingerprint: string; + builds: CatalogBuildSample[]; + canaries: CatalogCanarySample[]; + snapshots: CatalogMemorySnapshot[]; + heapSlopeBytesPerInstance: number; + rssSlopeBytesPerInstance: number; + allSdlHashesEqualWithinArm: boolean; + tokenCanariesConclusive: boolean; + tokenCanariesPassed: boolean; + tokenMismatchViolations: number; + crossTenantTokenViolations: number; + bleedViolations: number; +} + +export interface CatalogWarmthCliOptions { + warmOperationsPerInstance: number; + warmOperationReplayPasses: number; + grafastCacheLimits: CatalogGrafastCacheLimits; +} + +export interface CatalogSchemaLayout { + /** Compatibility labels: one primary schema name per resident instance. */ + schemas: string[]; + /** Null preserves the legacy one-schema-per-instance configuration shape. */ + schemaSets: string[][] | null; + /** Null preserves legacy makePgService behavior. */ + allowedDependencySchemas: string[] | null; +} + +export interface CatalogWarmOperationResult { + latenciesMs: number[]; + errors: number; + returnedStrings: number; + exactMatches: number; + mismatchViolations: number; + crossTenantViolations: number; + correctnessConclusive: boolean; + correctnessPassed: boolean; +} + +export interface BuildTransientSample { + baselineHeapUsedBytes: number; + baselineRssBytes: number; + sampledPeakHeapUsedBytes: number; + sampledPeakHeapDeltaBytes: number; + sampledPeakRssBytes: number; + sampledPeakRssDeltaBytes: number; + processPeakRssBytes: number; + processPeakRssDeltaBytes: number; + sampleCount: number; +} + +export interface CatalogBackendMemoryPoint { + monotonicMs: number; + rssBytes: number; + highWaterBytes: number; + procStartTicks: number; + procStartEpochMs: number; + bootTimeEpochSeconds: number; + clockTicksPerSecond: number; +} + +interface BackendMemory { + rssBytes: number; + highWaterBytes: number; +} + +export type CatalogBackendMemorySamplerSource = + | 'linux-host-container-procfs' + | 'docker-container-procfs-diagnostic' + | 'local-linux-procfs'; + +export interface CatalogBackendIdentity { + pid: number; + backendStartEpochMs: number; +} + +export interface CatalogDockerContainerIdentity { + requestedName: string; + immutableId: string; + startedAt: string; + initHostPid: number; +} + +export interface CatalogBackendIntrospectionMemoryLowerBoundMeasurement { + backendPid: number; + backendStartEpochMs: number; + baselineRssBytes: number; + baselineHighWaterBytes: number; + sampledPeakRssLowerBoundBytes: number; + sampledHighWaterLowerBoundBytes: number; + sampledPeakRssDeltaLowerBoundBytes: number; + sampledHighWaterDeltaLowerBoundBytes: number; + sampleCount: number; + targetExitedBeforeStop: boolean; + targetExitedAtMonotonicMs: number | null; + timing: { + configuredIntervalMs: number; + maximumConclusiveGapMs: number; + firstSampleMonotonicMs: number; + lastSampleMonotonicMs: number; + maximumObservedGapMs: number | null; + samplerStartedAt: string; + samplerReadyAt: string; + buildStartedAt: string; + buildCompletedAt: string; + samplerStoppedAt: string; + buildDurationMs: number; + samplerDurationMs: number; + coveredBuildWindow: boolean; + cadenceConclusive: boolean; + samplerLaunchToReadyMs: number; + samplerStopRequestToCloseMs: number; + }; + observerEffect: { + samplerProcessCount: 1; + correctionApplied: false; + pairedComparisonSupported: true; + pairedComparisonFlag: '--postgres-backend-sampler'; + measuredLaunchToReadyMs: number; + measuredStopRequestToCloseMs: number; + limitation: string; + }; + provenance: { + samplerProcess: 'dedicated-external-procfs-loop'; + samplerPid: number; + source: CatalogBackendMemorySamplerSource; + postgresContainer: string | null; + containerIdentity: CatalogDockerContainerIdentity | null; + clientPlatform: string; + clientArchitecture: string; + backendIdentity: { + sqlBackendStartEpochMs: number; + procStartTicks: number; + procStartEpochMs: number; + bootTimeEpochSeconds: number; + clockTicksPerSecond: number; + toleranceMs: number; + }; + dockerInitialExecEnvironment: + | 'not-applicable' + | 'may-inherit-container-config-before-env-i'; + samplerShellEnvironment: 'env-i-path-only'; + hostEnvironmentVariableNames: string[]; + semantics: 'diagnostic-lower-bound-without-pre-destroy-acknowledgement'; + backendSamplerAuthority: 'diagnostic-only'; + serviceDensityMemoryAuthority: + `separately-validated-${typeof LINUX_CGROUP_V2_DENSITY_AUTHORITY}`; + limitation: string | null; + }; +} + +export interface CatalogBackendMemorySamplerHandle { + stop(input: { + buildStartedAt: string; + buildCompletedAt: string; + buildDurationMs: number; + }): Promise; +} + +export interface CatalogBackendPidLifecycleDependencies { + waitForRetirement(identity: CatalogBackendIdentity): Promise; + acquireBackendIdentity(): Promise; +} + +const flag = (args: string[], name: string): string | undefined => { + const index = args.indexOf(`--${name}`); + return index >= 0 ? args[index + 1] : undefined; +}; + +const requireFlag = (args: string[], name: string): string => { + const value = flag(args, name); + if (!value) throw new Error(`catalog-bench requires --${name}`); + return value; +}; + +const parsePositiveInteger = (value: string, label: string): number => { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${label} must be a positive integer`); + } + return parsed; +}; + +const strictOptionalFlag = (args: string[], name: string): string | undefined => { + const flagName = `--${name}`; + const indexes = args.flatMap((value, index) => value === flagName ? [index] : []); + if (indexes.length > 1) throw new Error(`${flagName} may only be specified once`); + if (indexes.length === 0) return undefined; + const value = args[indexes[0] + 1]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flagName} requires a value`); + } + return value; +}; + +export const validateCatalogV8Profile: ( + value: unknown +) => asserts value is NodeV8Profile = (value) => { + if ( + value !== 'stock' + && value !== 'optimize-for-size' + && value !== 'baseline-optimize-for-size' + && value !== 'jitless-optimize-for-size' + ) { + throw new Error( + "v8Profile must be 'stock', 'optimize-for-size', " + + "'baseline-optimize-for-size', or 'jitless-optimize-for-size'" + ); + } +}; + +export const parseCatalogV8Profile = (args: string[]): NodeV8Profile => { + const value = strictOptionalFlag(args, 'v8-profile') ?? 'stock'; + validateCatalogV8Profile(value); + return value; +}; + +interface CatalogRuntimeFlags { + nodeOptions: string; + nodeOptionsArgv: string[]; + nodeExecArgv: string[]; + effectiveNodeRuntimeFlags: string[]; +} + +const CATALOG_MANAGED_V8_OPTION = + /^--(?:no[-_])?(?:jitless|optimize[-_]for[-_]size|max[-_]opt)(?:=.*)?$/; +const CATALOG_MAX_OLD_SPACE_OPTION = + /^--max(?:-|_)old(?:-|_)space(?:-|_)size(?:=.*)?$/; + +export const validateCatalogRuntimeFlags = ( + config: Pick< + CatalogBenchConfig, + | 'heapMiB' + | 'v8Profile' + | 'nodeOptions' + | 'nodeOptionsArgv' + | 'nodeExecArgv' + | 'effectiveNodeRuntimeFlags' + >, + actual: CatalogRuntimeFlags = { + nodeOptions: process.env.NODE_OPTIONS ?? '', + nodeOptionsArgv: tokenizeNodeOptions(process.env.NODE_OPTIONS ?? ''), + nodeExecArgv: [...process.execArgv], + effectiveNodeRuntimeFlags: [ + ...tokenizeNodeOptions(process.env.NODE_OPTIONS ?? ''), + ...process.execArgv + ] + } +): void => { + validateCatalogV8Profile(config.v8Profile); + const expectedExecArgv = [ + ...nodeFlagsForV8Profile(config.v8Profile), + '--expose-gc' + ]; + const configuredNodeOptionsArgv = tokenizeNodeOptions(config.nodeOptions); + const expectedEffective = [ + ...config.nodeOptionsArgv, + ...config.nodeExecArgv + ]; + const maxOldSpace = config.nodeOptionsArgv.filter((option) => + CATALOG_MAX_OLD_SPACE_OPTION.test(option) + ); + const managedInNodeOptions = config.nodeOptionsArgv.some((option) => + CATALOG_MANAGED_V8_OPTION.test(option) + ); + if ( + JSON.stringify(config.nodeOptionsArgv) !== JSON.stringify(configuredNodeOptionsArgv) + || JSON.stringify(config.nodeExecArgv) !== JSON.stringify(expectedExecArgv) + || JSON.stringify(config.effectiveNodeRuntimeFlags) !== JSON.stringify(expectedEffective) + || maxOldSpace.length !== 1 + || maxOldSpace[0] !== `--max-old-space-size=${config.heapMiB}` + || managedInNodeOptions + ) { + throw new Error('catalog-bench configured Node runtime flags are inconsistent'); + } + for (const [label, configured, observed] of [ + ['NODE_OPTIONS', config.nodeOptions, actual.nodeOptions], + ['NODE_OPTIONS argv', config.nodeOptionsArgv, actual.nodeOptionsArgv], + ['process.execArgv', config.nodeExecArgv, actual.nodeExecArgv], + [ + 'effective Node runtime flags', + config.effectiveNodeRuntimeFlags, + actual.effectiveNodeRuntimeFlags + ] + ] as const) { + if (JSON.stringify(configured) !== JSON.stringify(observed)) { + throw new Error(`catalog-bench ${label} does not match the pinned worker config`); + } + } +}; + +const parseStrictInteger = ( + value: string, + label: string, + allowZero: boolean +): number => { + if (!/^(0|[1-9][0-9]*)$/.test(value)) { + throw new Error(`${label} must be ${allowZero ? 'a non-negative' : 'a positive'} integer`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || (allowZero ? parsed < 0 : parsed <= 0)) { + throw new Error(`${label} must be ${allowZero ? 'a non-negative' : 'a positive'} safe integer`); + } + return parsed; +}; + +export const validateCatalogScopedCatalogTypes = ( + mode: IntrospectionMode, + value: CatalogScopedCatalogTypes | null +): void => { + if (mode !== 'stock' && mode !== 'scoped-required') { + throw new Error("catalog-bench mode must be 'stock' or 'scoped-required'"); + } + if (mode === 'stock') { + if (value !== null) { + throw new Error('scopedCatalogTypes must be null for stock introspection'); + } + return; + } + if (value !== 'all' && value !== 'dependency-closure') { + throw new Error( + "scopedCatalogTypes must be 'all' or 'dependency-closure' for scoped-required introspection" + ); + } +}; + +export const parseCatalogScopedCatalogTypes = ( + args: string[], + mode: IntrospectionMode +): CatalogScopedCatalogTypes | null => { + const value = strictOptionalFlag(args, 'scoped-catalog-types'); + if (mode === 'stock') { + if (value !== undefined) { + throw new Error('--scoped-catalog-types requires --mode scoped-required'); + } + return null; + } + const parsed = value ?? 'all'; + if (parsed !== 'all' && parsed !== 'dependency-closure') { + throw new Error( + "--scoped-catalog-types must be 'all' or 'dependency-closure'" + ); + } + return parsed; +}; + +export const validateCatalogIntrospectionClientReleaseMode: ( + value: unknown +) => asserts value is CatalogIntrospectionClientReleaseMode = (value) => { + if (value !== 'reuse' && value !== 'destroy') { + throw new Error( + "introspectionClientReleaseMode must be 'reuse' or 'destroy'" + ); + } +}; + +export const parseCatalogIntrospectionClientReleaseMode = ( + args: string[] +): CatalogIntrospectionClientReleaseMode => { + const value = strictOptionalFlag(args, 'introspection-client-release-mode') + ?? 'reuse'; + validateCatalogIntrospectionClientReleaseMode(value); + return value; +}; + +export const validateCatalogBackendSamplerMode: ( + value: unknown +) => asserts value is CatalogBackendSamplerMode = (value) => { + if (value !== 'off' && value !== 'diagnostic-lower-bound') { + throw new Error( + "postgresBackendSamplerMode must be 'off' or 'diagnostic-lower-bound'" + ); + } +}; + +export const parseCatalogBackendSamplerMode = ( + args: string[] +): CatalogBackendSamplerMode => { + const value = strictOptionalFlag(args, 'postgres-backend-sampler') + ?? 'diagnostic-lower-bound'; + validateCatalogBackendSamplerMode(value); + return value; +}; + +export const catalogIntrospectionBuildIdentity = ( + mode: IntrospectionMode, + scopedCatalogTypes: CatalogScopedCatalogTypes | null, + releaseBuildStateAfterValidation = false, + introspectionClientReleaseMode: CatalogIntrospectionClientReleaseMode = 'reuse' +): string => { + validateCatalogScopedCatalogTypes(mode, scopedCatalogTypes); + validateCatalogIntrospectionClientReleaseMode(introspectionClientReleaseMode); + return `${mode}:scoped-catalog-types=${scopedCatalogTypes ?? 'not-applicable'}` + + `:release-build-state=${releaseBuildStateAfterValidation}` + + `:introspection-client-release=${introspectionClientReleaseMode}`; +}; + +export const resolveCatalogBackendPidAfterBuild = async ( + introspectionClientReleaseMode: CatalogIntrospectionClientReleaseMode, + introspectionBackendIdentity: CatalogBackendIdentity, + dependencies: CatalogBackendPidLifecycleDependencies +): Promise => { + validateCatalogIntrospectionClientReleaseMode(introspectionClientReleaseMode); + validateCatalogBackendIdentity(introspectionBackendIdentity); + if (introspectionClientReleaseMode === 'destroy') { + await dependencies.waitForRetirement(introspectionBackendIdentity); + } + const steadyBackendIdentity = await dependencies.acquireBackendIdentity(); + validateCatalogBackendIdentity(steadyBackendIdentity); + if ( + introspectionClientReleaseMode === 'destroy' + && steadyBackendIdentity.pid === introspectionBackendIdentity.pid + ) { + throw new Error( + `destroyed PostgreSQL introspection backend ${introspectionBackendIdentity.pid} was reused` + ); + } + if ( + introspectionClientReleaseMode === 'reuse' + && ( + steadyBackendIdentity.pid !== introspectionBackendIdentity.pid + || steadyBackendIdentity.backendStartEpochMs + !== introspectionBackendIdentity.backendStartEpochMs + ) + ) { + throw new Error( + `PostgreSQL benchmark backend identity changed from ` + + `${introspectionBackendIdentity.pid}@${introspectionBackendIdentity.backendStartEpochMs} ` + + `to ${steadyBackendIdentity.pid}@${steadyBackendIdentity.backendStartEpochMs}` + ); + } + return { + introspectionBackendPid: introspectionBackendIdentity.pid, + introspectionBackendStartEpochMs: + introspectionBackendIdentity.backendStartEpochMs, + steadyBackendPid: steadyBackendIdentity.pid, + steadyBackendStartEpochMs: steadyBackendIdentity.backendStartEpochMs, + introspectionBackendRetired: introspectionClientReleaseMode === 'destroy' + }; +}; + +export const parseCatalogBuildStateRetirement = (args: string[]): boolean => { + const flagName = '--release-build-state-after-validation'; + const count = args.filter((value) => value === flagName).length; + if (count > 1) throw new Error(`${flagName} may only be specified once`); + return count === 1; +}; + +export const parseCatalogTenantProxySurfaces = (args: string[]): number | null => { + const value = strictOptionalFlag(args, 'tenant-proxy-surfaces'); + return value === undefined + ? null + : parseStrictInteger(value, 'tenant-proxy-surfaces', false); +}; + +export const parseCatalogWarmthCliOptions = ( + args: string[] +): CatalogWarmthCliOptions => { + const warmOperations = strictOptionalFlag(args, 'warm-operations-per-instance'); + const replayPasses = strictOptionalFlag(args, 'warm-operation-replay-passes'); + const queryCacheMax = strictOptionalFlag(args, 'grafast-query-cache-max'); + const operationsCacheMax = strictOptionalFlag(args, 'grafast-operations-cache-max'); + const operationPlansCacheMax = strictOptionalFlag( + args, + 'grafast-operation-plans-cache-max' + ); + const options: CatalogWarmthCliOptions = { + warmOperationsPerInstance: warmOperations === undefined + ? 0 + : parseStrictInteger( + warmOperations, + 'warm-operations-per-instance', + true + ), + warmOperationReplayPasses: replayPasses === undefined + ? 0 + : parseStrictInteger( + replayPasses, + 'warm-operation-replay-passes', + true + ), + grafastCacheLimits: { + queryCacheMaxLength: queryCacheMax === undefined + ? null + : parseStrictInteger(queryCacheMax, 'grafast-query-cache-max', false), + operationsCacheMaxLength: operationsCacheMax === undefined + ? null + : parseStrictInteger(operationsCacheMax, 'grafast-operations-cache-max', false), + operationOperationPlansCacheMaxLength: operationPlansCacheMax === undefined + ? null + : parseStrictInteger( + operationPlansCacheMax, + 'grafast-operation-plans-cache-max', + false + ) + } + }; + validateCatalogWarmthConfig(options); + return options; +}; + +export const validateCatalogWarmthConfig = ( + options: CatalogWarmthCliOptions +): void => { + if ( + !Number.isSafeInteger(options.warmOperationsPerInstance) + || options.warmOperationsPerInstance < 0 + ) { + throw new Error('warmOperationsPerInstance must be a non-negative safe integer'); + } + if ( + !Number.isSafeInteger(options.warmOperationReplayPasses) + || options.warmOperationReplayPasses < 0 + ) { + throw new Error('warmOperationReplayPasses must be a non-negative safe integer'); + } + if ( + options.warmOperationReplayPasses > 0 + && options.warmOperationsPerInstance === 0 + ) { + throw new Error( + 'warmOperationReplayPasses requires warmOperationsPerInstance to be greater than zero' + ); + } + if (!Number.isSafeInteger( + options.warmOperationsPerInstance * options.warmOperationReplayPasses + )) { + throw new Error('warm operation replay execution count must be a safe integer'); + } + if (!options.grafastCacheLimits || typeof options.grafastCacheLimits !== 'object') { + throw new Error('grafastCacheLimits must define all three cache limit fields'); + } + for (const [key, value] of Object.entries(options.grafastCacheLimits)) { + if (value !== null && (!Number.isSafeInteger(value) || value < 2)) { + throw new Error(`grafastCacheLimits.${key} must be null or a safe integer of at least 2`); + } + } + const requiredKeys: Array = [ + 'queryCacheMaxLength', + 'operationsCacheMaxLength', + 'operationOperationPlansCacheMaxLength' + ]; + if (requiredKeys.some((key) => !(key in options.grafastCacheLimits))) { + throw new Error('grafastCacheLimits must define all three cache limit fields'); + } +}; + +const configuredGrafastCacheLimits = ( + limits: CatalogGrafastCacheLimits +): { + queryCacheMaxLength?: number; + operationsCacheMaxLength?: number; + operationOperationPlansCacheMaxLength?: number; +} => ({ + ...(limits.queryCacheMaxLength === null + ? {} + : { queryCacheMaxLength: limits.queryCacheMaxLength }), + ...(limits.operationsCacheMaxLength === null + ? {} + : { operationsCacheMaxLength: limits.operationsCacheMaxLength }), + ...(limits.operationOperationPlansCacheMaxLength === null + ? {} + : { + operationOperationPlansCacheMaxLength: + limits.operationOperationPlansCacheMaxLength + }) +}); + +const parseList = (value: string): string[] => value + .split(',') + .map((item) => item.trim()) + .filter(Boolean); + +const validateSchemaNames = ( + value: unknown, + label: string, + allowEmpty = false +): string[] => { + if (!Array.isArray(value) || (!allowEmpty && value.length === 0)) { + throw new Error(`${label} must contain at least one schema name`); + } + if (value.length === 0) return []; + const names = value.map((name, index) => { + if ( + typeof name !== 'string' + || name.length === 0 + || name.trim() !== name + || name.includes('\0') + ) { + throw new Error(`${label}[${index}] must be a nonempty exact schema name`); + } + return name; + }); + if (new Set(names).size !== names.length) { + throw new Error(`${label} must contain unique schema names`); + } + return names; +}; + +const parseStrictSchemaList = ( + value: string, + label: string, + allowEmpty = false +): string[] => { + if (allowEmpty && value.length === 0) return []; + const raw = value.split(','); + if (raw.some((name) => name.trim().length === 0)) { + throw new Error(`${label} must not contain empty schema names`); + } + return validateSchemaNames(raw.map((name) => name.trim()), label); +}; + +export const parseCatalogSchemaLayout = ( + args: string[], + maxInstances: number +): CatalogSchemaLayout => { + const legacyValue = strictOptionalFlag(args, 'schemas'); + const surfaceValue = strictOptionalFlag(args, 'surface-schemas'); + const dependencyValue = strictOptionalFlag(args, 'allowed-dependency-schemas'); + if (legacyValue !== undefined && surfaceValue !== undefined) { + throw new Error('--schemas and --surface-schemas are mutually exclusive'); + } + if (surfaceValue === undefined) { + if (dependencyValue !== undefined) { + throw new Error('--allowed-dependency-schemas requires --surface-schemas'); + } + if (legacyValue === undefined) { + throw new Error('catalog-bench requires --schemas or --surface-schemas'); + } + const schemas = parseList(legacyValue); + if (schemas.length !== maxInstances || new Set(schemas).size !== schemas.length) { + throw new Error(`--schemas must contain exactly ${maxInstances} unique entries`); + } + return { schemas, schemaSets: null, allowedDependencySchemas: null }; + } + if (maxInstances !== 1) { + throw new Error('--surface-schemas requires exactly one resident instance'); + } + if (dependencyValue === undefined) { + throw new Error('--surface-schemas requires --allowed-dependency-schemas'); + } + const surfaceSchemas = parseStrictSchemaList(surfaceValue, '--surface-schemas'); + const allowedDependencySchemas = parseStrictSchemaList( + dependencyValue, + '--allowed-dependency-schemas', + true + ); + const overlap = surfaceSchemas.filter((schema) => + allowedDependencySchemas.includes(schema) + ); + if (overlap.length > 0) { + throw new Error( + `surface and dependency schema lists must not overlap: ${overlap.join(', ')}` + ); + } + return { + schemas: [surfaceSchemas[0]], + schemaSets: [surfaceSchemas], + allowedDependencySchemas + }; +}; + +export const resolveCatalogSchemaLayout = ( + config: Pick< + CatalogBenchConfig, + 'schemas' | 'schemaSets' | 'allowedDependencySchemas' | 'checkpoints' + > +): CatalogSchemaLayout => { + const maxInstances = Math.max(...config.checkpoints); + if (!Number.isSafeInteger(maxInstances) || maxInstances <= 0) { + throw new Error('checkpoints must contain a positive resident instance count'); + } + const schemas = validateSchemaNames(config.schemas, 'schemas'); + if (config.schemaSets === undefined) { + if (config.allowedDependencySchemas !== undefined) { + throw new Error('allowedDependencySchemas requires schemaSets'); + } + if (schemas.length !== maxInstances) { + throw new Error(`schemas must contain exactly ${maxInstances} entries`); + } + return { schemas, schemaSets: null, allowedDependencySchemas: null }; + } + if ( + !Array.isArray(config.schemaSets) + || maxInstances !== 1 + || config.schemaSets.length !== 1 + || schemas.length !== 1 + ) { + throw new Error('schemaSets mode requires exactly one resident instance'); + } + const schemaSet = validateSchemaNames(config.schemaSets[0], 'schemaSets[0]'); + if (schemas[0] !== schemaSet[0]) { + throw new Error('schemas[0] must equal the first ordered schemaSets[0] entry'); + } + const allowedDependencySchemas = validateSchemaNames( + config.allowedDependencySchemas, + 'allowedDependencySchemas', + true + ); + const overlap = schemaSet.filter((schema) => allowedDependencySchemas.includes(schema)); + if (overlap.length > 0) { + throw new Error( + `schemaSets and allowedDependencySchemas must not overlap: ${overlap.join(', ')}` + ); + } + return { + schemas, + schemaSets: [schemaSet], + allowedDependencySchemas + }; +}; + +export const catalogSchemaContractIdentity = ( + schemas: string[], + allowedDependencySchemas: string[] +): string => { + const exposed = validateSchemaNames(schemas, 'schemas'); + const dependencies = validateSchemaNames( + allowedDependencySchemas, + 'allowedDependencySchemas', + true + ); + const overlap = exposed.filter((schema) => dependencies.includes(schema)); + if (overlap.length > 0) { + throw new Error(`schema contract lists must not overlap: ${overlap.join(', ')}`); + } + return createHash('sha256').update(JSON.stringify({ + schemas: exposed, + allowedDependencySchemas: dependencies + })).digest('hex'); +}; + +const parseCheckpoints = (value: string): number[] => { + const parsed = parseList(value).map((item) => parsePositiveInteger(item, 'instances')); + return [...new Set(parsed)].sort((a, b) => a - b); +}; + +const median = (values: number[]): number => { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; +}; + +export const catalogPercentile = ( + values: readonly number[], + probability: number +): number | null => { + if (values.length === 0) return null; + if (!Number.isFinite(probability) || probability <= 0 || probability > 1) { + throw new Error('percentile probability must be greater than zero and at most one'); + } + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * probability) - 1)]; +}; + +export const makeCatalogWarmOperationSource = (operationIndex: number): string => { + if (!Number.isSafeInteger(operationIndex) || operationIndex <= 0) { + throw new Error('operationIndex must be a positive safe integer'); + } + return `query CatalogWarm${operationIndex} { warmTenantToken: tenantToken }`; +}; + +export const projectCatalogTenantDensity = (input: { + tenantProxySurfaces: number; + configuredOldSpaceMiB: number; + snapshot: Pick< + CatalogMemorySnapshot, + 'instances' | 'processPeakRssBytes' | 'processPeakRssDeltaBytes' + >; +}): CatalogTenantProxyDensityPoint => { + const { + tenantProxySurfaces, + configuredOldSpaceMiB, + snapshot + } = input; + if (!Number.isSafeInteger(tenantProxySurfaces) || tenantProxySurfaces <= 0) { + throw new Error('tenantProxySurfaces must be a positive safe integer'); + } + if (!Number.isSafeInteger(configuredOldSpaceMiB) || configuredOldSpaceMiB <= 0) { + throw new Error('configuredOldSpaceMiB must be a positive safe integer'); + } + if (!Number.isSafeInteger(snapshot.instances) || snapshot.instances < 0) { + throw new Error('snapshot.instances must be a non-negative safe integer'); + } + if (!Number.isFinite(snapshot.processPeakRssBytes) || snapshot.processPeakRssBytes <= 0) { + throw new Error('snapshot.processPeakRssBytes must be a positive finite number'); + } + const fullTenantProxyGroups = Math.floor(snapshot.instances / tenantProxySurfaces); + return { + residentSurfaceInstances: snapshot.instances, + fullTenantProxyGroups, + remainderSurfaceInstances: snapshot.instances % tenantProxySurfaces, + configuredOldSpaceMiB, + absolutePeakProcessRssBytes: snapshot.processPeakRssBytes, + groupsPerConfiguredOldSpaceGiB: + fullTenantProxyGroups / (configuredOldSpaceMiB / MIB_PER_GIB), + groupsPerAbsolutePeakProcessRssGiB: + fullTenantProxyGroups / (snapshot.processPeakRssBytes / GIB) + }; +}; + +export const catalogProgressPath = (resultFile: string): string => + path.join(path.dirname(resultFile), 'progress.json'); + +/** + * Persist a small checkpoint without serializing resident schemas or build + * samples. The same-directory rename is atomic, so an OOM can leave either the + * preceding valid checkpoint or the new one, never a truncated JSON artifact. + */ +export const writeCatalogProgress = ( + resultFile: string, + progress: CatalogBenchProgress +): void => { + const progressFile = catalogProgressPath(resultFile); + const temporaryFile = `${progressFile}.${process.pid}.tmp`; + fs.mkdirSync(path.dirname(progressFile), { recursive: true }); + fs.writeFileSync(temporaryFile, `${JSON.stringify(progress, null, 2)}\n`, 'utf8'); + fs.renameSync(temporaryFile, progressFile); +}; + +const maxOrNull = (values: Array): number | null => { + const measured = values.filter((value): value is number => value !== null); + return measured.length > 0 ? Math.max(...measured) : null; +}; + +const medianOrNull = (values: Array): number | null => { + const measured = values.filter((value): value is number => value !== null); + return measured.length > 0 ? median(measured) : null; +}; + +const sha256File = (file: string): string => + createHash('sha256').update(fs.readFileSync(file)).digest('hex'); + +const GIT_PROVENANCE_MAX_BUFFER_BYTES = 64 * 1024 ** 2; + +const readGitProvenance = (): { + commit: string | null; + worktreeDirty: boolean | null; + sourceStateSha256: string | null; +} => { + try { + const commit = execFileSync('git', ['rev-parse', 'HEAD'], { + encoding: 'utf8', + maxBuffer: GIT_PROVENANCE_MAX_BUFFER_BYTES + }).trim(); + const status = execFileSync( + 'git', + ['status', '--porcelain=v1', '--untracked-files=all'], + { + encoding: 'utf8', + maxBuffer: GIT_PROVENANCE_MAX_BUFFER_BYTES + } + ); + const hash = createHash('sha256').update(commit).update('\0').update(status); + hash.update(execFileSync('git', ['diff', '--binary', 'HEAD'], { + encoding: 'buffer', + maxBuffer: GIT_PROVENANCE_MAX_BUFFER_BYTES + })); + const untracked = execFileSync( + 'git', + ['ls-files', '--others', '--exclude-standard', '-z'], + { + encoding: 'buffer', + maxBuffer: GIT_PROVENANCE_MAX_BUFFER_BYTES + } + ).toString('utf8').split('\0').filter(Boolean).sort(); + for (const relativeFile of untracked) { + hash.update('\0').update(relativeFile).update('\0'); + hash.update(fs.readFileSync(path.resolve(relativeFile))); + } + return { + commit, + worktreeDirty: status.length > 0, + sourceStateSha256: hash.digest('hex') + }; + } catch { + return { commit: null, worktreeDirty: null, sourceStateSha256: null }; + } +}; + +const slope = (points: { x: number; y: number }[]): number => { + if (points.length < 2) return 0; + const meanX = points.reduce((sum, point) => sum + point.x, 0) / points.length; + const meanY = points.reduce((sum, point) => sum + point.y, 0) / points.length; + const numerator = points.reduce( + (sum, point) => sum + (point.x - meanX) * (point.y - meanY), + 0 + ); + const denominator = points.reduce( + (sum, point) => sum + (point.x - meanX) ** 2, + 0 + ); + return denominator === 0 ? 0 : numerator / denominator; +}; + +const forceGc = async (settleMs: number): Promise => { + if (typeof global.gc !== 'function') { + throw new Error('catalog-bench worker requires Node --expose-gc'); + } + for (let index = 0; index < 3; index++) { + global.gc(); + await new Promise((resolve) => setImmediate(resolve)); + } + if (settleMs > 0) { + await new Promise((resolve) => setTimeout(resolve, settleMs)); + global.gc(); + } +}; + +interface TransientMemoryPoint { + heapUsedBytes: number; + rssBytes: number; + processPeakRssBytes: number; +} + +export const summarizeBuildTransientSamples = ( + baseline: TransientMemoryPoint, + samples: readonly TransientMemoryPoint[] +): BuildTransientSample => { + if (samples.length === 0) throw new Error('build transient sampling requires a sample'); + const sampledPeakHeapUsedBytes = Math.max(...samples.map((sample) => sample.heapUsedBytes)); + const sampledPeakRssBytes = Math.max(...samples.map((sample) => sample.rssBytes)); + const processPeakRssBytes = Math.max(...samples.map((sample) => sample.processPeakRssBytes)); + return { + baselineHeapUsedBytes: baseline.heapUsedBytes, + baselineRssBytes: baseline.rssBytes, + sampledPeakHeapUsedBytes, + sampledPeakHeapDeltaBytes: Math.max( + 0, + sampledPeakHeapUsedBytes - baseline.heapUsedBytes + ), + sampledPeakRssBytes, + sampledPeakRssDeltaBytes: Math.max(0, sampledPeakRssBytes - baseline.rssBytes), + processPeakRssBytes, + processPeakRssDeltaBytes: Math.max( + 0, + processPeakRssBytes - baseline.processPeakRssBytes + ), + sampleCount: samples.length + }; +}; + +const readTransientMemoryPoint = (): TransientMemoryPoint => { + const memory = process.memoryUsage(); + return { + heapUsedBytes: memory.heapUsed, + rssBytes: memory.rss, + processPeakRssBytes: process.resourceUsage().maxRSS * 1024 + }; +}; + +const measureBuildTransient = async ( + operation: () => Promise +): Promise<{ value: T; transient: BuildTransientSample }> => { + const baseline = readTransientMemoryPoint(); + const samples: TransientMemoryPoint[] = [baseline]; + const sample = () => samples.push(readTransientMemoryPoint()); + const timer = setInterval(sample, BUILD_TRANSIENT_SAMPLE_INTERVAL_MS); + timer.unref(); + try { + const value = await operation(); + sample(); + return { + value, + transient: summarizeBuildTransientSamples(baseline, samples) + }; + } finally { + clearInterval(timer); + } +}; + +const validateCatalogBackendPid = (backendPid: number): void => { + if (!Number.isSafeInteger(backendPid) || backendPid <= 0) { + throw new Error('PostgreSQL backend PID must be a positive safe integer'); + } +}; + +export const validateCatalogBackendIdentity = ( + identity: CatalogBackendIdentity +): void => { + validateCatalogBackendPid(identity.pid); + if ( + !Number.isSafeInteger(identity.backendStartEpochMs) + || identity.backendStartEpochMs <= 0 + ) { + throw new Error( + 'PostgreSQL backend start timestamp must be a positive safe epoch millisecond' + ); + } +}; + +export const validateCatalogPostgresContainer = (container: string): void => { + if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(container)) { + throw new Error(`invalid PostgreSQL container name '${container}'`); + } +}; + +export const parseCatalogDockerContainerIdentity = ( + output: string, + requestedName: string +): CatalogDockerContainerIdentity => { + validateCatalogPostgresContainer(requestedName); + const fields = output.trim().split('\t'); + if (fields.length !== 3 || !/^[a-f0-9]{64}$/i.test(fields[0])) { + throw new Error('Docker inspect did not return an immutable container ID'); + } + if (!Number.isFinite(Date.parse(fields[1]))) { + throw new Error('Docker inspect did not return a valid container start timestamp'); + } + const initHostPid = Number(fields[2]); + if (!Number.isSafeInteger(initHostPid) || initHostPid <= 0) { + throw new Error('Docker inspect did not return a positive container init PID'); + } + return { + requestedName, + immutableId: fields[0].toLowerCase(), + startedAt: new Date(fields[1]).toISOString(), + initHostPid + }; +}; + +export const assertCatalogDockerContainerIdentity = ( + expected: CatalogDockerContainerIdentity, + actual: CatalogDockerContainerIdentity +): void => { + if ( + actual.requestedName !== expected.requestedName + || actual.immutableId !== expected.immutableId + || actual.startedAt !== expected.startedAt + || actual.initHostPid !== expected.initHostPid + ) { + throw new Error( + `PostgreSQL container '${expected.requestedName}' changed immutable identity ` + + 'during backend sampling' + ); + } +}; + +const parseProcStatusKiB = (value: string | undefined, label: string): number => { + if (value === undefined || !/^\d+$/.test(value)) { + throw new Error(`PostgreSQL backend status did not contain a valid ${label}`); + } + const bytes = Number(value) * 1024; + if (!Number.isSafeInteger(bytes)) { + throw new Error(`PostgreSQL backend ${label} exceeds the safe integer range`); + } + return bytes; +}; + +export const parseCatalogBackendProcStatus = ( + status: string, + expectedPid: number +): BackendMemory => { + validateCatalogBackendPid(expectedPid); + const fields = new Map(); + for (const line of status.split(/\r?\n/)) { + const match = /^([A-Za-z]+):\s*(.*?)\s*$/.exec(line); + if (match) fields.set(match[1], match[2]); + } + const processName = fields.get('Name')?.split(/\s+/)[0]; + const namespacePids = fields.get('NSpid')?.split(/\s+/).filter(Boolean) ?? []; + const namespacePid = Number(namespacePids.at(-1)); + if (!processName?.startsWith('postgres') || namespacePid !== expectedPid) { + throw new Error( + `PostgreSQL backend status identity did not match exact PID ${expectedPid}` + ); + } + return { + rssBytes: parseProcStatusKiB(fields.get('VmRSS')?.split(/\s+/)[0], 'VmRSS'), + highWaterBytes: parseProcStatusKiB(fields.get('VmHWM')?.split(/\s+/)[0], 'VmHWM') + }; +}; + +const maxCatalogBackendSampleGapMs = ( + samples: readonly CatalogBackendMemoryPoint[] +): number | null => { + if (samples.length < 2) return null; + let maximum = 0; + for (let index = 1; index < samples.length; index++) { + const gap = samples[index].monotonicMs - samples[index - 1].monotonicMs; + if (!Number.isFinite(gap) || gap < 0) { + throw new Error('PostgreSQL backend sampler timestamps must be monotonic'); + } + maximum = Math.max(maximum, gap); + } + return maximum; +}; + +export const summarizeCatalogBackendMemorySamples = (input: { + backendIdentity: CatalogBackendIdentity; + samplerPid: number; + source: CatalogBackendMemorySamplerSource; + postgresContainer: string | null; + containerIdentity?: CatalogDockerContainerIdentity | null; + hostEnvironmentVariableNames?: string[]; + samples: readonly CatalogBackendMemoryPoint[]; + targetExitedBeforeStop: boolean; + targetExitedAtMonotonicMs?: number | null; + samplerStartedAt: string; + samplerReadyAt: string; + buildStartedAt: string; + buildCompletedAt: string; + samplerStopRequestedAt: string; + samplerStoppedAt: string; + buildDurationMs: number; + clientPlatform?: string; + clientArchitecture?: string; +}): CatalogBackendIntrospectionMemoryLowerBoundMeasurement => { + validateCatalogBackendIdentity(input.backendIdentity); + if (!Number.isSafeInteger(input.samplerPid) || input.samplerPid <= 0) { + throw new Error('PostgreSQL backend sampler PID must be a positive safe integer'); + } + if (input.samples.length === 0) { + throw new Error('PostgreSQL backend sampler produced no memory samples'); + } + for (const sample of input.samples) { + if ( + !Number.isFinite(sample.monotonicMs) + || sample.monotonicMs < 0 + || !Number.isSafeInteger(sample.rssBytes) + || sample.rssBytes <= 0 + || !Number.isSafeInteger(sample.highWaterBytes) + || sample.highWaterBytes < sample.rssBytes + || !Number.isSafeInteger(sample.procStartTicks) + || sample.procStartTicks <= 0 + || !Number.isSafeInteger(sample.procStartEpochMs) + || sample.procStartEpochMs <= 0 + || !Number.isSafeInteger(sample.bootTimeEpochSeconds) + || sample.bootTimeEpochSeconds <= 0 + || !Number.isSafeInteger(sample.clockTicksPerSecond) + || sample.clockTicksPerSecond <= 0 + ) { + throw new Error('PostgreSQL backend sampler produced an invalid memory sample'); + } + } + if (!Number.isFinite(input.buildDurationMs) || input.buildDurationMs < 0) { + throw new Error('PostgreSQL backend sampler requires a finite build duration'); + } + const baseline = input.samples[0]; + for (const sample of input.samples) { + if ( + sample.procStartTicks !== baseline.procStartTicks + || sample.bootTimeEpochSeconds !== baseline.bootTimeEpochSeconds + || sample.clockTicksPerSecond !== baseline.clockTicksPerSecond + || Math.abs( + sample.procStartEpochMs - input.backendIdentity.backendStartEpochMs + ) > BACKEND_START_IDENTITY_TOLERANCE_MS + ) { + throw new Error( + 'PostgreSQL backend sampler observed a changed or mismatched process start identity' + ); + } + } + const timestamps = { + samplerStarted: Date.parse(input.samplerStartedAt), + samplerReady: Date.parse(input.samplerReadyAt), + buildStarted: Date.parse(input.buildStartedAt), + buildCompleted: Date.parse(input.buildCompletedAt), + samplerStopRequested: Date.parse(input.samplerStopRequestedAt), + samplerStopped: Date.parse(input.samplerStoppedAt) + }; + if (Object.values(timestamps).some((value) => !Number.isFinite(value))) { + throw new Error('PostgreSQL backend sampler timing contains an invalid timestamp'); + } + const coveredBuildWindow = + timestamps.samplerStarted <= timestamps.samplerReady + && timestamps.samplerReady <= timestamps.buildStarted + && timestamps.buildStarted <= timestamps.buildCompleted + && timestamps.buildCompleted <= timestamps.samplerStopRequested + && timestamps.samplerStopRequested <= timestamps.samplerStopped + && timestamps.buildCompleted <= timestamps.samplerStopped; + if (input.targetExitedBeforeStop !== (input.targetExitedAtMonotonicMs != null)) { + throw new Error('PostgreSQL backend sampler target-exit provenance is inconsistent'); + } + if ( + input.targetExitedAtMonotonicMs !== undefined + && input.targetExitedAtMonotonicMs !== null + && ( + !Number.isFinite(input.targetExitedAtMonotonicMs) + || input.targetExitedAtMonotonicMs < input.samples.at(-1)!.monotonicMs + ) + ) { + throw new Error('PostgreSQL backend sampler target-exit time is invalid'); + } + const observedTimingPoints = input.targetExitedAtMonotonicMs === undefined + || input.targetExitedAtMonotonicMs === null + ? input.samples + : [ + ...input.samples, + { + ...input.samples.at(-1)!, + monotonicMs: input.targetExitedAtMonotonicMs + } + ]; + const maximumObservedGapMs = maxCatalogBackendSampleGapMs(observedTimingPoints); + const cadenceConclusive = input.samples.length >= 2 + && coveredBuildWindow + && maximumObservedGapMs !== null + && maximumObservedGapMs <= BACKEND_MEMORY_MAX_CONCLUSIVE_GAP_MS; + const limitations: string[] = []; + limitations.push(DESTROYED_BACKEND_LOWER_BOUND_LIMITATION); + if (!cadenceConclusive) { + limitations.push( + 'The identity-bound sampler did not cover the build with at least two samples ' + + `and a maximum observed gap of ${BACKEND_MEMORY_MAX_CONCLUSIVE_GAP_MS}ms.` + ); + } + if (input.source === 'docker-container-procfs-diagnostic') { + limitations.push(DOCKER_DESKTOP_BACKEND_SAMPLER_LIMITATION); + } + const sampledPeakRssLowerBoundBytes = Math.max( + ...input.samples.map((sample) => sample.rssBytes) + ); + const sampledHighWaterLowerBoundBytes = Math.max( + ...input.samples.map((sample) => sample.highWaterBytes) + ); + const samplerLaunchToReadyMs = Math.max( + 0, + timestamps.samplerReady - timestamps.samplerStarted + ); + const samplerStopRequestToCloseMs = Math.max( + 0, + timestamps.samplerStopped - timestamps.samplerStopRequested + ); + return { + backendPid: input.backendIdentity.pid, + backendStartEpochMs: input.backendIdentity.backendStartEpochMs, + baselineRssBytes: baseline.rssBytes, + baselineHighWaterBytes: baseline.highWaterBytes, + sampledPeakRssLowerBoundBytes, + sampledHighWaterLowerBoundBytes, + sampledPeakRssDeltaLowerBoundBytes: Math.max( + 0, + sampledPeakRssLowerBoundBytes - baseline.rssBytes + ), + sampledHighWaterDeltaLowerBoundBytes: Math.max( + 0, + sampledHighWaterLowerBoundBytes - baseline.highWaterBytes + ), + sampleCount: input.samples.length, + targetExitedBeforeStop: input.targetExitedBeforeStop, + targetExitedAtMonotonicMs: input.targetExitedAtMonotonicMs ?? null, + timing: { + configuredIntervalMs: BACKEND_MEMORY_SAMPLE_INTERVAL_MS, + maximumConclusiveGapMs: BACKEND_MEMORY_MAX_CONCLUSIVE_GAP_MS, + firstSampleMonotonicMs: baseline.monotonicMs, + lastSampleMonotonicMs: input.samples.at(-1)!.monotonicMs, + maximumObservedGapMs, + samplerStartedAt: input.samplerStartedAt, + samplerReadyAt: input.samplerReadyAt, + buildStartedAt: input.buildStartedAt, + buildCompletedAt: input.buildCompletedAt, + samplerStoppedAt: input.samplerStoppedAt, + buildDurationMs: input.buildDurationMs, + samplerDurationMs: Math.max( + 0, + timestamps.samplerStopped - timestamps.samplerStarted + ), + coveredBuildWindow, + cadenceConclusive, + samplerLaunchToReadyMs, + samplerStopRequestToCloseMs + }, + observerEffect: { + samplerProcessCount: 1, + correctionApplied: false, + pairedComparisonSupported: true, + pairedComparisonFlag: '--postgres-backend-sampler', + measuredLaunchToReadyMs: samplerLaunchToReadyMs, + measuredStopRequestToCloseMs: samplerStopRequestToCloseMs, + limitation: 'Only sampler launch and shutdown wall time is measured; sampling ' + + 'CPU/I/O interference is not corrected. Compare paired runs using ' + + "'--postgres-backend-sampler off' and " + + "'--postgres-backend-sampler diagnostic-lower-bound'." + }, + provenance: { + samplerProcess: 'dedicated-external-procfs-loop', + samplerPid: input.samplerPid, + source: input.source, + postgresContainer: input.postgresContainer, + containerIdentity: input.containerIdentity ?? null, + clientPlatform: input.clientPlatform ?? os.platform(), + clientArchitecture: input.clientArchitecture ?? os.arch(), + backendIdentity: { + sqlBackendStartEpochMs: input.backendIdentity.backendStartEpochMs, + procStartTicks: baseline.procStartTicks, + procStartEpochMs: baseline.procStartEpochMs, + bootTimeEpochSeconds: baseline.bootTimeEpochSeconds, + clockTicksPerSecond: baseline.clockTicksPerSecond, + toleranceMs: BACKEND_START_IDENTITY_TOLERANCE_MS + }, + dockerInitialExecEnvironment: + input.source === 'docker-container-procfs-diagnostic' + ? 'may-inherit-container-config-before-env-i' + : 'not-applicable', + samplerShellEnvironment: 'env-i-path-only', + hostEnvironmentVariableNames: + [...(input.hostEnvironmentVariableNames ?? [])].sort(), + semantics: 'diagnostic-lower-bound-without-pre-destroy-acknowledgement', + backendSamplerAuthority: 'diagnostic-only', + serviceDensityMemoryAuthority: + `separately-validated-${LINUX_CGROUP_V2_DENSITY_AUTHORITY}`, + limitation: limitations.length === 0 ? null : limitations.join(' ') + } + }; +}; + +export const catalogBackendSamplerEnvironment = ( + environment: NodeJS.ProcessEnv = process.env +): NodeJS.ProcessEnv => { + const allowed = [ + 'PATH', + 'HOME', + 'LANG', + 'LC_ALL', + 'TZ', + 'TMPDIR', + 'DOCKER_HOST', + 'DOCKER_CONTEXT', + 'DOCKER_TLS_VERIFY', + 'DOCKER_CERT_PATH', + 'DOCKER_CONFIG', + 'XDG_RUNTIME_DIR' + ]; + return Object.fromEntries(allowed.flatMap((name) => { + const value = environment[name]; + return value === undefined ? [] : [[name, value]]; + })); +}; + +const readCatalogDockerContainerIdentity = ( + requestedName: string +): CatalogDockerContainerIdentity => { + validateCatalogPostgresContainer(requestedName); + const output = execFileSync( + 'docker', + [ + 'inspect', + '--format', + '{{.Id}}\t{{.State.StartedAt}}\t{{.State.Pid}}', + requestedName + ], + { + encoding: 'utf8', + env: catalogBackendSamplerEnvironment() + } + ); + return parseCatalogDockerContainerIdentity(output, requestedName); +}; + +export interface CatalogBackendSamplerLaunchSpec { + command: string; + args: string[]; + environment: NodeJS.ProcessEnv; + hostEnvironmentVariableNames: string[]; + source: CatalogBackendMemorySamplerSource; + statusPath: string; + statPath: string; + procStatPath: string; +} + +const catalogSamplerShellArgs = ( + statusPath: string, + statPath: string, + procStatPath: string, + backendIdentity: CatalogBackendIdentity +): string[] => [ + '/bin/sh', + '-c', + BACKEND_MEMORY_SAMPLER_SCRIPT, + 'catalog-backend-sampler', + statusPath, + statPath, + procStatPath, + String(backendIdentity.pid), + String(backendIdentity.backendStartEpochMs), + String(BACKEND_START_IDENTITY_TOLERANCE_MS), + String(BACKEND_MEMORY_SAMPLE_INTERVAL_MS / 1000) +]; + +export const makeCatalogBackendSamplerLaunchSpec = (input: { + backendIdentity: CatalogBackendIdentity; + containerIdentity?: CatalogDockerContainerIdentity | null; + clientPlatform?: string; + environment?: NodeJS.ProcessEnv; +}): CatalogBackendSamplerLaunchSpec | null => { + validateCatalogBackendIdentity(input.backendIdentity); + const clientPlatform = input.clientPlatform ?? os.platform(); + const environment = catalogBackendSamplerEnvironment(input.environment); + const hostEnvironmentVariableNames = Object.keys(environment).sort(); + const containerIdentity = input.containerIdentity ?? null; + if (containerIdentity) { + const containerProcRoot = `/proc/${containerIdentity.initHostPid}/root/proc`; + const hostStatusPath = `${containerProcRoot}/${input.backendIdentity.pid}/status`; + const hostStatPath = `${containerProcRoot}/${input.backendIdentity.pid}/stat`; + const hostProcStatPath = `${containerProcRoot}/stat`; + if ( + clientPlatform === 'linux' + && fs.existsSync(hostStatusPath) + && fs.existsSync(hostStatPath) + && fs.existsSync(hostProcStatPath) + ) { + return { + command: fs.existsSync('/usr/bin/env') ? '/usr/bin/env' : '/bin/env', + args: [ + '-i', + 'PATH=/usr/bin:/bin', + ...catalogSamplerShellArgs( + hostStatusPath, + hostStatPath, + hostProcStatPath, + input.backendIdentity + ) + ], + environment, + hostEnvironmentVariableNames, + source: 'linux-host-container-procfs', + statusPath: hostStatusPath, + statPath: hostStatPath, + procStatPath: hostProcStatPath + }; + } + const statusPath = `/proc/${input.backendIdentity.pid}/status`; + const statPath = `/proc/${input.backendIdentity.pid}/stat`; + return { + command: 'docker', + args: [ + 'exec', + '-i', + containerIdentity.immutableId, + '/usr/bin/env', + '-i', + 'PATH=/usr/bin:/bin', + ...catalogSamplerShellArgs( + statusPath, + statPath, + '/proc/stat', + input.backendIdentity + ) + ], + environment, + hostEnvironmentVariableNames, + source: 'docker-container-procfs-diagnostic', + statusPath, + statPath, + procStatPath: '/proc/stat' + }; + } + const statusPath = `/proc/${input.backendIdentity.pid}/status`; + const statPath = `/proc/${input.backendIdentity.pid}/stat`; + if ( + clientPlatform !== 'linux' + || !fs.existsSync(statusPath) + || !fs.existsSync(statPath) + || !fs.existsSync('/proc/stat') + ) return null; + return { + command: fs.existsSync('/usr/bin/env') ? '/usr/bin/env' : '/bin/env', + args: [ + '-i', + 'PATH=/usr/bin:/bin', + ...catalogSamplerShellArgs( + statusPath, + statPath, + '/proc/stat', + input.backendIdentity + ) + ], + environment, + hostEnvironmentVariableNames, + source: 'local-linux-procfs', + statusPath, + statPath, + procStatPath: '/proc/stat' + }; +}; + +const parseBackendSamplerOutputLine = ( + line: string, + expectedBackendIdentity: CatalogBackendIdentity +): CatalogBackendMemoryPoint | { targetExitedAtMonotonicMs: number } => { + const fields = line.split('\t'); + if (fields[0] === 'gone' && fields.length === 2) { + const targetExitedAtMonotonicMs = Number(fields[1]) * 1000; + if (!Number.isFinite(targetExitedAtMonotonicMs) || targetExitedAtMonotonicMs < 0) { + throw new Error(`invalid PostgreSQL backend sampler output '${line}'`); + } + return { targetExitedAtMonotonicMs }; + } + if (fields[0] !== 'sample' || fields.length !== 8) { + throw new Error(`unexpected PostgreSQL backend sampler output '${line}'`); + } + const monotonicMs = Number(fields[1]) * 1000; + const rssBytes = Number(fields[2]) * 1024; + const highWaterBytes = Number(fields[3]) * 1024; + const procStartTicks = Number(fields[4]); + const procStartEpochMs = Number(fields[5]); + const bootTimeEpochSeconds = Number(fields[6]); + const clockTicksPerSecond = Number(fields[7]); + if ( + !Number.isFinite(monotonicMs) + || monotonicMs < 0 + || !Number.isSafeInteger(rssBytes) + || rssBytes <= 0 + || !Number.isSafeInteger(highWaterBytes) + || highWaterBytes < rssBytes + || !Number.isSafeInteger(procStartTicks) + || procStartTicks <= 0 + || !Number.isSafeInteger(procStartEpochMs) + || procStartEpochMs <= 0 + || !Number.isSafeInteger(bootTimeEpochSeconds) + || bootTimeEpochSeconds <= 0 + || !Number.isSafeInteger(clockTicksPerSecond) + || clockTicksPerSecond <= 0 + || Math.abs(procStartEpochMs - expectedBackendIdentity.backendStartEpochMs) + > BACKEND_START_IDENTITY_TOLERANCE_MS + ) { + throw new Error(`invalid PostgreSQL backend sampler output '${line}'`); + } + return { + monotonicMs, + rssBytes, + highWaterBytes, + procStartTicks, + procStartEpochMs, + bootTimeEpochSeconds, + clockTicksPerSecond + }; +}; + +const withCatalogBackendSamplerTimeout = async ( + operation: Promise, + timeoutMs: number, + message: string +): Promise => new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(message)), timeoutMs); + operation.then((value) => { + clearTimeout(timer); + resolve(value); + }, (error) => { + clearTimeout(timer); + reject(error); + }); +}); + +export type CatalogBackendSamplerStopOutcome = + | 'already-exited' + | 'graceful' + | 'sigterm' + | 'sigkill'; + +export const stopCatalogBackendSamplerProcessTree = async (input: { + requestGracefulStop(): void; + waitForTreeExit(timeoutMs: number): Promise; + signalProcessGroup(signal: 'SIGTERM' | 'SIGKILL'): void; + gracefulTimeoutMs?: number; + termTimeoutMs?: number; + killTimeoutMs?: number; +}): Promise => { + if (await input.waitForTreeExit(0)) return 'already-exited'; + let gracefulStopError: unknown; + try { + input.requestGracefulStop(); + } catch (error) { + gracefulStopError = error; + } + const finish = (outcome: CatalogBackendSamplerStopOutcome) => { + if (gracefulStopError !== undefined) throw gracefulStopError; + return outcome; + }; + if (await input.waitForTreeExit( + input.gracefulTimeoutMs ?? BACKEND_MEMORY_SAMPLER_STOP_TIMEOUT_MS + )) return finish('graceful'); + input.signalProcessGroup('SIGTERM'); + if (await input.waitForTreeExit( + input.termTimeoutMs ?? BACKEND_MEMORY_SAMPLER_TERM_TIMEOUT_MS + )) return finish('sigterm'); + input.signalProcessGroup('SIGKILL'); + if (await input.waitForTreeExit( + input.killTimeoutMs ?? BACKEND_MEMORY_SAMPLER_KILL_TIMEOUT_MS + )) return finish('sigkill'); + throw new Error('PostgreSQL backend sampler process tree survived SIGKILL'); +}; + +const startCatalogBackendMemorySampler = async ( + container: string | null, + backendIdentity: CatalogBackendIdentity +): Promise => { + validateCatalogBackendIdentity(backendIdentity); + const containerIdentity = container === null + ? null + : readCatalogDockerContainerIdentity(container); + const launch = makeCatalogBackendSamplerLaunchSpec({ + backendIdentity, + containerIdentity + }); + if (!launch) return null; + + const samplerStartedAt = new Date().toISOString(); + const detached = process.platform !== 'win32'; + const child = spawn(launch.command, launch.args, { + detached, + env: launch.environment, + stdio: ['pipe', 'pipe', 'pipe'] + }); + const samplerPid = child.pid; + if (!samplerPid) { + child.kill('SIGKILL'); + throw new Error('PostgreSQL backend sampler did not receive a process PID'); + } + const samples: CatalogBackendMemoryPoint[] = []; + let targetExitedBeforeStop = false; + let targetExitedAtMonotonicMs: number | null = null; + let stdoutBuffer = ''; + let stderr = ''; + let stopRequested = false; + let stdinEnded = false; + let childClosed = false; + let exitCode: number | null = null; + let exitSignal: NodeJS.Signals | null = null; + let processError: Error | null = null; + let protocolError: Error | null = null; + let resolveReady!: () => void; + let rejectReady!: (error: Error) => void; + const ready = new Promise((resolve, reject) => { + resolveReady = resolve; + rejectReady = reject; + }); + let resolveClosed!: () => void; + const closed = new Promise((resolve) => { + resolveClosed = resolve; + }); + const requestGracefulStop = (): void => { + stopRequested = true; + if (!stdinEnded) { + stdinEnded = true; + child.stdin.end('stop\n'); + } + }; + const signalProcessGroup = (signal: 'SIGTERM' | 'SIGKILL'): void => { + try { + if (detached) process.kill(-samplerPid, signal); + else child.kill(signal); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; + } + }; + const processGroupExists = (): boolean => { + if (!detached) return !childClosed; + try { + process.kill(-samplerPid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== 'ESRCH'; + } + }; + const waitForTreeExit = async (timeoutMs: number): Promise => { + const deadline = performance.now() + timeoutMs; + while (true) { + if (childClosed && !processGroupExists()) return true; + if (performance.now() >= deadline) return false; + await Promise.race([ + closed, + new Promise((resolve) => { + const timer = setTimeout(resolve, Math.min(10, timeoutMs)); + timer.unref(); + }) + ]); + } + }; + let cleanupPromise: Promise | null = null; + const cleanupTree = (): Promise => { + cleanupPromise ??= stopCatalogBackendSamplerProcessTree({ + requestGracefulStop, + waitForTreeExit, + signalProcessGroup + }); + return cleanupPromise; + }; + const failProtocol = (error: Error): void => { + protocolError ??= error; + rejectReady(error); + requestGracefulStop(); + }; + const consumeLine = (rawLine: string): void => { + const line = rawLine.trim(); + if (!line) return; + try { + const parsed = parseBackendSamplerOutputLine(line, backendIdentity); + if ('targetExitedAtMonotonicMs' in parsed) { + targetExitedBeforeStop = true; + targetExitedAtMonotonicMs ??= parsed.targetExitedAtMonotonicMs; + } else { + const baseline = samples[0]; + if (baseline && ( + parsed.procStartTicks !== baseline.procStartTicks + || parsed.bootTimeEpochSeconds !== baseline.bootTimeEpochSeconds + || parsed.clockTicksPerSecond !== baseline.clockTicksPerSecond + )) { + throw new Error('PostgreSQL backend procfs start identity changed mid-sample'); + } + samples.push(parsed); + if (samples.length === 1) resolveReady(); + } + } catch (error) { + failProtocol(error instanceof Error ? error : new Error(String(error))); + } + }; + child.stdout.on('data', (chunk: Buffer | string) => { + stdoutBuffer += chunk.toString(); + const lines = stdoutBuffer.split('\n'); + stdoutBuffer = lines.pop() ?? ''; + lines.forEach(consumeLine); + }); + child.stderr.on('data', (chunk: Buffer | string) => { + if (stderr.length < 4_096) stderr += chunk.toString().slice(0, 4_096 - stderr.length); + }); + child.stdin.on('error', (error) => { + if (!stopRequested) failProtocol(error); + }); + child.once('error', (error) => { + processError = error; + rejectReady(error); + }); + child.once('close', (code, signal) => { + if (stdoutBuffer.trim()) consumeLine(stdoutBuffer); + childClosed = true; + exitCode = code; + exitSignal = signal; + if (samples.length === 0 && !processError && !protocolError) { + const detail = stderr.trim() ? `: ${stderr.trim()}` : ''; + rejectReady(new Error( + `PostgreSQL backend sampler exited code=${code} signal=${signal}${detail}` + )); + } + resolveClosed(); + }); + try { + await withCatalogBackendSamplerTimeout( + ready, + BACKEND_MEMORY_SAMPLER_START_TIMEOUT_MS, + `PostgreSQL backend sampler for PID ${backendIdentity.pid} did not produce a baseline` + ); + } catch (error) { + try { + await cleanupTree(); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + 'PostgreSQL backend sampler startup and cleanup both failed' + ); + } + throw error; + } + const samplerReadyAt = new Date().toISOString(); + let stopPromise: + Promise | null = null; + return { + stop(input) { + if (stopPromise) return stopPromise; + stopPromise = (async () => { + const samplerStopRequestedAt = new Date().toISOString(); + const stopOutcome = await cleanupTree(); + const samplerStoppedAt = new Date().toISOString(); + if (containerIdentity) { + assertCatalogDockerContainerIdentity( + containerIdentity, + readCatalogDockerContainerIdentity(containerIdentity.requestedName) + ); + } + if (protocolError) throw protocolError; + if (processError) throw processError; + if (exitCode !== 0 || exitSignal !== null) { + const detail = stderr.trim() ? `: ${stderr.trim()}` : ''; + throw new Error( + `PostgreSQL backend sampler exited code=${exitCode} ` + + `signal=${exitSignal} cleanup=${stopOutcome}${detail}` + ); + } + const measurement = summarizeCatalogBackendMemorySamples({ + backendIdentity, + samplerPid, + source: launch.source, + postgresContainer: container, + containerIdentity, + hostEnvironmentVariableNames: launch.hostEnvironmentVariableNames, + samples, + targetExitedBeforeStop, + targetExitedAtMonotonicMs, + samplerStartedAt, + samplerReadyAt, + buildStartedAt: input.buildStartedAt, + buildCompletedAt: input.buildCompletedAt, + samplerStopRequestedAt, + samplerStoppedAt, + buildDurationMs: input.buildDurationMs + }); + if (!measurement.timing.cadenceConclusive) { + throw new Error( + `PostgreSQL introspection backend ${backendIdentity.pid} sampling cadence ` + + `was inconclusive: ${measurement.provenance.limitation}` + ); + } + return measurement; + })(); + return stopPromise; + } + }; +}; + +export const measureCatalogBuildWithBackendSampler = async (input: { + startSampler(): Promise; + build(): Promise; +}): Promise<{ + value: T; + backendMemoryLowerBound: + CatalogBackendIntrospectionMemoryLowerBoundMeasurement | null; + buildDurationMs: number; + }> => { + const sampler = await input.startSampler(); + const buildStartedAt = new Date().toISOString(); + const started = performance.now(); + let value: T | undefined; + let buildError: unknown; + try { + value = await input.build(); + } catch (error) { + buildError = error; + } + const buildDurationMs = performance.now() - started; + const buildCompletedAt = new Date().toISOString(); + let backendMemoryLowerBound: + CatalogBackendIntrospectionMemoryLowerBoundMeasurement | null = null; + let samplerError: unknown; + if (sampler) { + try { + backendMemoryLowerBound = await sampler.stop({ + buildStartedAt, + buildCompletedAt, + buildDurationMs + }); + } catch (error) { + samplerError = error; + } + } + if (buildError !== undefined && samplerError !== undefined) { + throw new AggregateError( + [buildError, samplerError], + 'Graphile build and PostgreSQL introspection backend sampling both failed' + ); + } + if (samplerError !== undefined) throw samplerError; + if (buildError !== undefined) throw buildError; + return { value: value!, backendMemoryLowerBound, buildDurationMs }; +}; + +const readBackendMemory = ( + container: string | null, + backendPid: number +): BackendMemory | null => { + validateCatalogBackendPid(backendPid); + const statusPath = `/proc/${backendPid}/status`; + if (!container) { + if (os.platform() !== 'linux' || !fs.existsSync(statusPath)) return null; + try { + return parseCatalogBackendProcStatus( + fs.readFileSync(statusPath, 'utf8'), + backendPid + ); + } catch { + return null; + } + } + validateCatalogPostgresContainer(container); + const output = execFileSync( + 'docker', + [ + 'exec', + container, + 'cat', + statusPath + ], + { encoding: 'utf8' } + ); + return parseCatalogBackendProcStatus(output, backendPid); +}; + +interface CatalogExecutionContext { + schema: Awaited>; + resolvedPreset: ReturnType; + contextValue: Record; +} + +const getCatalogExecutionContext = async ( + entry: GraphileCacheEntry +): Promise => { + const schema = await entry.pgl.getSchema(); + const resolvedPreset = entry.pgl.getResolvedPreset(); + type BenchPgService = Parameters[0] & { + withPgClientKey?: string; + }; + const pgService = ( + resolvedPreset.pgServices as readonly BenchPgService[] | undefined + )?.[0]; + if (!pgService) throw new Error('built PostGraphile instance has no PostgreSQL service'); + const contextValue: Record = { pgSettings: {} }; + contextValue[pgService.withPgClientKey ?? 'withPgClient'] = withPgClientFromPgService.bind( + null, + pgService + ); + return { schema, resolvedPreset, contextValue }; +}; + +const executeTokenQuery = async ( + entry: GraphileCacheEntry +): Promise<{ token: string | null; elapsedMs: number }> => { + const { schema, resolvedPreset, contextValue } = await getCatalogExecutionContext(entry); + const started = performance.now(); + const result = await execute({ + schema, + document: parse('{ tenantToken }'), + contextValue, + resolvedPreset + }) as ExecutionResult<{ tenantToken?: unknown }>; + const elapsedMs = performance.now() - started; + if (result.errors?.length) { + throw new AggregateError(result.errors, 'tenant token query failed'); + } + const data = result.data as { tenantToken?: unknown } | null | undefined; + return { + token: typeof data?.tenantToken === 'string' ? data.tenantToken : null, + elapsedMs + }; +}; + +const executeWarmOperations = async ( + entry: GraphileCacheEntry, + sources: readonly string[], + passes: number, + expectedToken: string, + allExpectedTokens: readonly string[] +): Promise => { + const { schema, resolvedPreset, contextValue } = await getCatalogExecutionContext(entry); + const latenciesMs: number[] = []; + let errors = 0; + let returnedStrings = 0; + let exactMatches = 0; + let mismatchViolations = 0; + let crossTenantViolations = 0; + for (let pass = 0; pass < passes; pass++) { + for (const source of sources) { + const started = performance.now(); + try { + const result = await grafast({ + schema, + source, + contextValue, + resolvedPreset + }) as ExecutionResult<{ warmTenantToken?: unknown }>; + latenciesMs.push(performance.now() - started); + if (result.errors?.length) { + errors++; + continue; + } + const token = result.data?.warmTenantToken; + if (typeof token !== 'string') continue; + returnedStrings++; + if (token === expectedToken) { + exactMatches++; + } else { + mismatchViolations++; + if (allExpectedTokens.some((candidate) => candidate === token)) { + crossTenantViolations++; + } + } + } catch { + latenciesMs.push(performance.now() - started); + errors++; + } + } + } + const executionCount = sources.length * passes; + const correctnessConclusive = errors === 0 && returnedStrings === executionCount; + return { + latenciesMs, + errors, + returnedStrings, + exactMatches, + mismatchViolations, + crossTenantViolations, + correctnessConclusive, + correctnessPassed: correctnessConclusive && exactMatches === executionCount + }; +}; + +const emptyWarmOperationResult = (): CatalogWarmOperationResult => ({ + latenciesMs: [], + errors: 0, + returnedStrings: 0, + exactMatches: 0, + mismatchViolations: 0, + crossTenantViolations: 0, + correctnessConclusive: true, + correctnessPassed: true +}); + +const checkTokenCanary = async ( + entry: GraphileCacheEntry, + instanceIndex: number, + phase: CatalogCanarySample['phase'], + residentInstances: number, + schemas: string[], + expectedTokens: string[] | null, + canaries: CatalogCanarySample[] +): Promise<{ token: string | null; elapsedMs: number }> => { + const query = await executeTokenQuery(entry); + if (!expectedTokens) return query; + const expected = expectedTokens[instanceIndex]; + const matchedOtherTenant = query.token !== null && expectedTokens.some( + (candidate, tokenIndex) => tokenIndex !== instanceIndex && candidate === query.token + ); + canaries.push({ + phase, + residentInstances, + instance: instanceIndex + 1, + schema: schemas[instanceIndex], + expected, + actual: query.token, + returnedString: query.token !== null, + exactMatch: query.token === expected, + matchedOtherTenant + }); + return query; +}; + +const memorySnapshot = ( + instances: number, + baseline: NodeJS.MemoryUsage, + baselinePeakRssBytes: number, + baselineBackend: BackendMemory | null, + backend: BackendMemory | null, + reportBackendHighWater: boolean +): CatalogMemorySnapshot => { + const memory = process.memoryUsage(); + const processPeakRssBytes = process.resourceUsage().maxRSS * 1024; + return { + instances, + heapUsedBytes: memory.heapUsed, + heapDeltaBytes: memory.heapUsed - baseline.heapUsed, + rssBytes: memory.rss, + rssDeltaBytes: memory.rss - baseline.rss, + externalBytes: memory.external, + externalDeltaBytes: memory.external - baseline.external, + processPeakRssBytes, + processPeakRssDeltaBytes: Math.max(0, processPeakRssBytes - baselinePeakRssBytes), + postgresBackendRssBytes: backend?.rssBytes ?? null, + postgresBackendRssDeltaBytes: backend && baselineBackend + ? backend.rssBytes - baselineBackend.rssBytes + : null, + postgresBackendHighWaterBytes: reportBackendHighWater + ? backend?.highWaterBytes ?? null + : null, + postgresBackendHighWaterDeltaBytes: reportBackendHighWater + && backend + && baselineBackend + ? Math.max(0, backend.highWaterBytes - baselineBackend.highWaterBytes) + : null + }; +}; + +const baselineMemorySnapshot = ( + baseline: NodeJS.MemoryUsage, + baselinePeakRssBytes: number, + backend: BackendMemory | null, + reportBackendHighWater: boolean +): CatalogMemorySnapshot => ({ + instances: 0, + heapUsedBytes: baseline.heapUsed, + heapDeltaBytes: 0, + rssBytes: baseline.rss, + rssDeltaBytes: 0, + externalBytes: baseline.external, + externalDeltaBytes: 0, + processPeakRssBytes: baselinePeakRssBytes, + processPeakRssDeltaBytes: 0, + postgresBackendRssBytes: backend?.rssBytes ?? null, + postgresBackendRssDeltaBytes: backend ? 0 : null, + postgresBackendHighWaterBytes: reportBackendHighWater + ? backend?.highWaterBytes ?? null + : null, + postgresBackendHighWaterDeltaBytes: reportBackendHighWater && backend ? 0 : null +}); + +const readCatalogBackendIdentity = async ( + pool: Pool +): Promise => { + const result = await pool.query<{ + backend_pid: number; + backend_start_epoch_ms: string; + }>(`select + activity.pid as backend_pid, + floor(pg_catalog.extract(epoch from activity.backend_start) * 1000)::bigint::text + as backend_start_epoch_ms + from pg_catalog.pg_stat_activity as activity + where activity.pid = pg_catalog.pg_backend_pid()`); + const row = result.rows[0]; + const identity = { + pid: row?.backend_pid, + backendStartEpochMs: Number(row?.backend_start_epoch_ms) + }; + validateCatalogBackendIdentity(identity); + return identity; +}; + +const waitForBackendPidRetirement = async ( + controlPool: Pool, + backendIdentity: CatalogBackendIdentity +): Promise => { + validateCatalogBackendIdentity(backendIdentity); + const deadline = performance.now() + BACKEND_RETIREMENT_TIMEOUT_MS; + while (true) { + const result = await controlPool.query<{ backend_exists: boolean }>( + `select exists ( + select 1 + from pg_catalog.pg_stat_activity + where pid = $1 + and floor(pg_catalog.extract(epoch from backend_start) * 1000)::bigint = $2 + ) as backend_exists`, + [backendIdentity.pid, backendIdentity.backendStartEpochMs] + ); + if (result.rows[0]?.backend_exists === false) return; + if (performance.now() >= deadline) { + throw new Error( + `PostgreSQL introspection backend ${backendIdentity.pid}` + + `@${backendIdentity.backendStartEpochMs} did not retire within ` + + `${BACKEND_RETIREMENT_TIMEOUT_MS}ms` + ); + } + await new Promise((resolve) => { + const timer = setTimeout(resolve, BACKEND_RETIREMENT_POLL_INTERVAL_MS); + timer.unref(); + }); + } +}; + +const assertBackendIdentity = async ( + pool: Pool, + expected: CatalogBackendIdentity +): Promise => { + const actual = await readCatalogBackendIdentity(pool); + if ( + actual.pid !== expected.pid + || actual.backendStartEpochMs !== expected.backendStartEpochMs + ) { + throw new Error( + `PostgreSQL benchmark backend identity changed from ` + + `${expected.pid}@${expected.backendStartEpochMs} to ` + + `${actual.pid}@${actual.backendStartEpochMs}` + ); + } +}; + +const cleanupEntries = async (entries: GraphileCacheEntry[]): Promise => { + for (const entry of entries.reverse()) { + await entry.pgl.release(); + } +}; + +export const runCatalogBenchWorker = async ( + configFile: string, + resultFile: string +): Promise => { + const config = JSON.parse(fs.readFileSync(configFile, 'utf8')) as CatalogBenchConfig; + validateCatalogRuntimeFlags(config); + if (!Object.prototype.hasOwnProperty.call(config, 'scopedCatalogTypes')) { + config.scopedCatalogTypes = config.mode === 'scoped-required' ? 'all' : null; + } + validateCatalogScopedCatalogTypes(config.mode, config.scopedCatalogTypes); + config.introspectionClientReleaseMode ??= 'reuse'; + validateCatalogIntrospectionClientReleaseMode( + config.introspectionClientReleaseMode + ); + config.postgresBackendSamplerMode ??= 'diagnostic-lower-bound'; + validateCatalogBackendSamplerMode(config.postgresBackendSamplerMode); + config.releaseBuildStateAfterValidation ??= false; + if (typeof config.releaseBuildStateAfterValidation !== 'boolean') { + throw new Error('releaseBuildStateAfterValidation must be boolean'); + } + const schemaLayout = resolveCatalogSchemaLayout(config); + const schemaSets = schemaLayout.schemaSets + ?? schemaLayout.schemas.map((schema) => [schema]); + const requestedSchemaNames = [...new Set([ + ...schemaSets.flat(), + ...(schemaLayout.allowedDependencySchemas ?? []) + ])]; + config.warmOperationReplayPasses ??= 0; + validateCatalogWarmthConfig({ + warmOperationsPerInstance: config.warmOperationsPerInstance, + warmOperationReplayPasses: config.warmOperationReplayPasses, + grafastCacheLimits: config.grafastCacheLimits + }); + if ( + config.warmOperationsPerInstance > 0 + && ( + config.expectedTokens === null + || config.expectedTokens.length !== Math.max(...config.checkpoints) + ) + ) { + throw new Error( + 'catalog-bench warmth requires one expected token per instance' + ); + } + const startedAt = new Date().toISOString(); + const connectionOptions = { + ...getPgEnvOptions({ database: config.database }), + max: 1, + idleTimeoutMillis: 0 + }; + const controlPool = new Pool(connectionOptions); + const pool = new Pool(connectionOptions); + const entries: GraphileCacheEntry[] = []; + let backendPid = 0; + let backendIdentity: CatalogBackendIdentity | null = null; + try { + const metadata = await controlPool.query<{ + classes: string; + attributes: string; + procs: string; + types: string; + namespaces: string; + database_oid: string; + max_class_oid: string; + max_attribute_relation_oid: string; + max_proc_oid: string; + max_type_oid: string; + proc_signature_hash: string; + pg_version: string; + server_version_num: string; + jit: string; + }>(`select + (select count(*) from pg_catalog.pg_class)::text as classes, + (select count(*) from pg_catalog.pg_attribute)::text as attributes, + (select count(*) from pg_catalog.pg_proc)::text as procs, + (select count(*) from pg_catalog.pg_type)::text as types, + (select count(*) from pg_catalog.pg_namespace)::text as namespaces, + (select oid::text from pg_catalog.pg_database where datname = current_database()) as database_oid, + (select max(oid)::text from pg_catalog.pg_class) as max_class_oid, + (select max(attrelid)::text from pg_catalog.pg_attribute) as max_attribute_relation_oid, + (select max(oid)::text from pg_catalog.pg_proc) as max_proc_oid, + (select max(oid)::text from pg_catalog.pg_type) as max_type_oid, + (select md5(coalesce(string_agg( + md5(row( + pg_proc.oid, + pg_proc.pronamespace, + pg_proc.proname, + pg_proc.proowner, + pg_proc.prolang, + pg_proc.prokind, + pg_proc.prosecdef, + pg_proc.proleakproof, + pg_proc.proisstrict, + pg_proc.proretset, + pg_proc.provolatile, + pg_proc.proparallel, + pg_proc.pronargs, + pg_proc.pronargdefaults, + pg_proc.prorettype, + pg_proc.proargtypes, + pg_proc.proallargtypes, + pg_proc.proargmodes, + pg_proc.proargnames, + pg_proc.proconfig, + pg_proc.proacl, + pg_catalog.obj_description(pg_proc.oid, 'pg_proc') + )::text), + '' order by pg_proc.oid + ), '')) from pg_catalog.pg_proc) as proc_signature_hash, + version() as pg_version, + current_setting('server_version_num') as server_version_num, + current_setting('jit') as jit`); + const roleResult = await controlPool.query<{ + rolname: string; + rolsuper: boolean; + rolbypassrls: boolean; + rolcreaterole: boolean; + owns_database: boolean; + can_create_in_database: boolean; + }>(`select + pg_roles.rolname, + pg_roles.rolsuper, + pg_roles.rolbypassrls, + pg_roles.rolcreaterole, + pg_database.datdba = pg_roles.oid as owns_database, + pg_catalog.has_database_privilege( + pg_roles.rolname, + pg_database.oid, + 'CREATE' + ) as can_create_in_database + from pg_catalog.pg_roles + inner join pg_catalog.pg_database + on pg_database.datname = pg_catalog.current_database() + where pg_roles.rolname = current_user`); + const schemaSafety = await controlPool.query<{ + requested_schema_count: string; + owns_schema: boolean | null; + can_create: boolean | null; + }>(`select + count(*)::text as requested_schema_count, + bool_or(pg_catalog.pg_get_userbyid(pg_namespace.nspowner) = current_user) as owns_schema, + bool_or(pg_catalog.has_schema_privilege(current_user, pg_namespace.oid, 'CREATE')) as can_create + from pg_catalog.pg_namespace + where pg_namespace.nspname = any($1::text[])`, [requestedSchemaNames]); + const role = roleResult.rows[0]; + const schemaRole = schemaSafety.rows[0]; + if (!role) throw new Error('runtime role metadata was not returned'); + if ( + role.rolsuper + || role.rolbypassrls + || role.rolcreaterole + || role.owns_database + || role.can_create_in_database + ) { + throw new Error(`unsafe benchmark runtime role '${role.rolname}'`); + } + if (Number(schemaRole.requested_schema_count) !== requestedSchemaNames.length) { + throw new Error('one or more requested schemas do not exist'); + } + if (schemaRole.owns_schema || schemaRole.can_create) { + throw new Error(`runtime role '${role.rolname}' owns or can create in a requested schema`); + } + + backendIdentity = await readCatalogBackendIdentity(pool); + backendPid = backendIdentity.pid; + const initialBackendIdentity = { ...backendIdentity }; + + await forceGc(config.settleMs); + const baseline = process.memoryUsage(); + const baselinePeakRssBytes = process.resourceUsage().maxRSS * 1024; + const baselineBackend = readBackendMemory(config.postgresContainer, backendPid); + let steadyBackendBaseline = baselineBackend; + const reportIntrospectionBackendHighWater = + config.introspectionClientReleaseMode === 'reuse'; + let completedRetirementChecks = 0; + const snapshots: CatalogMemorySnapshot[] = [baselineMemorySnapshot( + baseline, + baselinePeakRssBytes, + baselineBackend, + reportIntrospectionBackendHighWater + )]; + const builds: CatalogBuildSample[] = []; + const canaries: CatalogCanarySample[] = []; + const allWarmOperationLatenciesMs: number[] = []; + const allWarmOperationReplayLatenciesMs: number[] = []; + const maxInstances = Math.max(...config.checkpoints); + const persistProgress = (status: CatalogBenchProgress['status']): void => { + const lastSnapshot = snapshots[snapshots.length - 1]; + writeCatalogProgress(resultFile, { + version: 1, + status, + mode: config.mode, + scopedCatalogTypes: config.scopedCatalogTypes, + introspectionClientReleaseMode: + config.introspectionClientReleaseMode, + postgresBackendSamplerMode: config.postgresBackendSamplerMode, + releaseBuildStateAfterValidation: + config.releaseBuildStateAfterValidation, + repetition: config.repetition, + heapMiB: config.heapMiB, + v8Profile: config.v8Profile, + nodeOptions: config.nodeOptions, + nodeOptionsArgv: [...config.nodeOptionsArgv], + nodeExecArgv: [...process.execArgv], + effectiveNodeRuntimeFlags: [ + ...config.nodeOptionsArgv, + ...process.execArgv + ], + targetInstances: maxInstances, + completedInstances: builds.length, + configuredCheckpoints: [...config.checkpoints], + completedCheckpoints: snapshots + .map((snapshot) => snapshot.instances) + .filter((instances) => instances > 0), + buildsCompleted: builds.length, + canariesCompleted: canaries.length, + mismatchViolations: canaries.filter((canary) => !canary.exactMatch).length, + crossTenantViolations: canaries.filter( + (canary) => canary.matchedOtherTenant + ).length, + lastSnapshot, + updatedAt: new Date().toISOString() + }); + }; + persistProgress('in-progress'); + const warmOperationSources = Array.from( + { length: config.warmOperationsPerInstance }, + (_, operationIndex) => makeCatalogWarmOperationSource(operationIndex + 1) + ); + const grafastCacheLimits = configuredGrafastCacheLimits(config.grafastCacheLimits); + const hasGrafastCacheLimits = Object.keys(grafastCacheLimits).length > 0; + const presetExtensions = hasGrafastCacheLimits + ? [ConstructivePreset, createGrafastCacheLimitsPreset(grafastCacheLimits)] + : [ConstructivePreset]; + const cacheLimitIdentity = hasGrafastCacheLimits + ? createHash('sha256').update(JSON.stringify(config.grafastCacheLimits)).digest('hex') + : null; + const introspectionBuildIdentity = catalogIntrospectionBuildIdentity( + config.mode, + config.scopedCatalogTypes, + config.releaseBuildStateAfterValidation, + config.introspectionClientReleaseMode + ); + + for (let index = 0; index < maxInstances; index++) { + const schemaName = schemaLayout.schemas[index]; + const instanceSchemas = schemaSets[index]; + const pgService = makePgService({ + pool, + schemas: instanceSchemas, + introspectionMode: config.mode, + introspectionClientReleaseMode: + config.introspectionClientReleaseMode, + ...(schemaLayout.allowedDependencySchemas === null + ? {} + : { + introspectionAllowedDependencySchemas: + schemaLayout.allowedDependencySchemas + }), + ...(config.scopedCatalogTypes === null + ? {} + : { introspectionScopedCatalogTypes: config.scopedCatalogTypes }) + }); + const preset = { + extends: presetExtensions, + schema: { + releaseBuildStateAfterValidation: + config.releaseBuildStateAfterValidation + }, + pgServices: [pgService] + }; + const schemaContractIdentity = schemaLayout.schemaSets === null + ? schemaName + : catalogSchemaContractIdentity( + instanceSchemas, + schemaLayout.allowedDependencySchemas! + ); + const buildCacheIdentity = schemaLayout.schemaSets === null + ? schemaName + : schemaContractIdentity; + await forceGc(0); + const sampledBuild = await measureCatalogBuildWithBackendSampler({ + startSampler: () => config.introspectionClientReleaseMode === 'destroy' + && config.postgresBackendSamplerMode === 'diagnostic-lower-bound' + ? startCatalogBackendMemorySampler( + config.postgresContainer, + backendIdentity! + ) + : Promise.resolve(null), + build: () => measureBuildTransient(() => createGraphileInstance({ + preset, + cacheKey: cacheLimitIdentity + ? `${introspectionBuildIdentity}:${cacheLimitIdentity}:${buildCacheIdentity}` + : `${introspectionBuildIdentity}:${buildCacheIdentity}`, + serviceKey: schemaLayout.schemaSets === null + ? schemaName + : `catalog:${schemaContractIdentity}` + })) + }); + const measuredBuild = sampledBuild.value; + const buildMs = sampledBuild.buildDurationMs; + const entry = measuredBuild.value; + entries.push(entry); + // The dedicated sampler is stopped and awaited by the helper before the + // destroyed PID is checked or a replacement checkout can be acquired. + const backendTransition = await resolveCatalogBackendPidAfterBuild( + config.introspectionClientReleaseMode, + backendIdentity, + { + waitForRetirement: (identity) => + waitForBackendPidRetirement(controlPool, identity), + acquireBackendIdentity: () => readCatalogBackendIdentity(pool) + } + ); + if ( + sampledBuild.backendMemoryLowerBound + && ( + sampledBuild.backendMemoryLowerBound.backendPid + !== backendTransition.introspectionBackendPid + || sampledBuild.backendMemoryLowerBound.backendStartEpochMs + !== backendTransition.introspectionBackendStartEpochMs + ) + ) { + throw new Error( + 'PostgreSQL introspection backend sampler identity did not match the ' + + 'retirement lifecycle identity' + ); + } + backendPid = backendTransition.steadyBackendPid; + backendIdentity = { + pid: backendTransition.steadyBackendPid, + backendStartEpochMs: backendTransition.steadyBackendStartEpochMs + }; + if (backendTransition.introspectionBackendRetired) { + completedRetirementChecks++; + steadyBackendBaseline = readBackendMemory( + config.postgresContainer, + backendPid + ); + } + + const query = await checkTokenCanary( + entry, + index, + 'initial', + index + 1, + config.schemas, + config.expectedTokens, + canaries + ); + const warmOperations = config.warmOperationsPerInstance > 0 + ? await executeWarmOperations( + entry, + warmOperationSources, + 1, + config.expectedTokens![index], + config.expectedTokens! + ) + : emptyWarmOperationResult(); + const warmOperationReplay = config.warmOperationReplayPasses > 0 + ? await executeWarmOperations( + entry, + warmOperationSources, + config.warmOperationReplayPasses, + config.expectedTokens![index], + config.expectedTokens! + ) + : emptyWarmOperationResult(); + allWarmOperationLatenciesMs.push(...warmOperations.latenciesMs); + allWarmOperationReplayLatenciesMs.push(...warmOperationReplay.latenciesMs); + await assertBackendIdentity(pool, backendIdentity); + const schema = await entry.pgl.getSchema(); + const sdl = printSchema(lexicographicSortSchema(schema)); + builds.push({ + instance: index + 1, + schema: schemaName, + ...backendTransition, + buildMs, + queryMs: query.elapsedMs, + token: query.token, + sdlBytes: Buffer.byteLength(sdl), + sdlSha256: createHash('sha256').update(sdl).digest('hex'), + queryFields: Object.keys(schema.getQueryType()?.getFields() ?? {}).sort(), + warmOperations: config.warmOperationsPerInstance, + warmOperationLatencyP50Ms: catalogPercentile(warmOperations.latenciesMs, 0.5), + warmOperationLatencyP99Ms: catalogPercentile(warmOperations.latenciesMs, 0.99), + warmOperationErrors: warmOperations.errors, + warmOperationReturnedStrings: warmOperations.returnedStrings, + warmOperationExactMatches: warmOperations.exactMatches, + warmOperationMismatchViolations: warmOperations.mismatchViolations, + warmOperationCrossTenantViolations: warmOperations.crossTenantViolations, + warmOperationCorrectnessConclusive: warmOperations.correctnessConclusive, + warmOperationCorrectnessPassed: warmOperations.correctnessPassed, + warmOperationReplayPasses: config.warmOperationReplayPasses, + warmOperationReplayExecutions: + config.warmOperationsPerInstance * config.warmOperationReplayPasses, + warmOperationReplayLatencyP50Ms: catalogPercentile( + warmOperationReplay.latenciesMs, + 0.5 + ), + warmOperationReplayLatencyP99Ms: catalogPercentile( + warmOperationReplay.latenciesMs, + 0.99 + ), + warmOperationReplayErrors: warmOperationReplay.errors, + warmOperationReplayReturnedStrings: warmOperationReplay.returnedStrings, + warmOperationReplayExactMatches: warmOperationReplay.exactMatches, + warmOperationReplayMismatchViolations: + warmOperationReplay.mismatchViolations, + warmOperationReplayCrossTenantViolations: + warmOperationReplay.crossTenantViolations, + warmOperationReplayCorrectnessConclusive: + warmOperationReplay.correctnessConclusive, + warmOperationReplayCorrectnessPassed: + warmOperationReplay.correctnessPassed, + buildBaselineHeapUsedBytes: measuredBuild.transient.baselineHeapUsedBytes, + buildBaselineRssBytes: measuredBuild.transient.baselineRssBytes, + sampledBuildPeakHeapUsedBytes: measuredBuild.transient.sampledPeakHeapUsedBytes, + sampledBuildPeakHeapDeltaBytes: measuredBuild.transient.sampledPeakHeapDeltaBytes, + sampledBuildPeakRssBytes: measuredBuild.transient.sampledPeakRssBytes, + sampledBuildPeakRssDeltaBytes: measuredBuild.transient.sampledPeakRssDeltaBytes, + processBuildPeakRssBytes: measuredBuild.transient.processPeakRssBytes, + processBuildPeakRssDeltaBytes: measuredBuild.transient.processPeakRssDeltaBytes, + buildTransientSampleCount: measuredBuild.transient.sampleCount, + postgresIntrospectionBackendMemoryLowerBound: + sampledBuild.backendMemoryLowerBound + }); + + if (config.checkpoints.includes(index + 1)) { + for (let residentIndex = 0; residentIndex < entries.length; residentIndex++) { + await checkTokenCanary( + entries[residentIndex], + residentIndex, + 'checkpoint', + index + 1, + config.schemas, + config.expectedTokens, + canaries + ); + } + await assertBackendIdentity(pool, backendIdentity); + await forceGc(config.settleMs); + snapshots.push(memorySnapshot( + index + 1, + baseline, + baselinePeakRssBytes, + steadyBackendBaseline, + readBackendMemory(config.postgresContainer, backendPid), + reportIntrospectionBackendHighWater + )); + // Serialize only a compact post-GC checkpoint after the measurement. + // If the next build OOMs, the parent can still recover the last + // conclusively resident point and bracket the capacity boundary. + persistProgress('in-progress'); + } + } + + const catalogRow = metadata.rows[0]; + const hashes = new Set(builds.map((build) => build.sdlSha256)); + const tokenMismatchViolations = canaries.filter((canary) => !canary.exactMatch).length; + const crossTenantTokenViolations = canaries.filter( + (canary) => canary.matchedOtherTenant + ).length; + const expectedCanaryCount = config.expectedTokens + ? maxInstances + config.checkpoints.reduce((sum, checkpoint) => sum + checkpoint, 0) + : 0; + const tokenCanariesConclusive = config.expectedTokens !== null + && canaries.length === expectedCanaryCount + && canaries.every((canary) => canary.returnedString); + const warmOperationExecutions = builds.reduce( + (sum, build) => sum + build.warmOperations, + 0 + ); + const warmOperationErrors = builds.reduce( + (sum, build) => sum + build.warmOperationErrors, + 0 + ); + const warmOperationReturnedStrings = builds.reduce( + (sum, build) => sum + build.warmOperationReturnedStrings, + 0 + ); + const warmOperationExactMatches = builds.reduce( + (sum, build) => sum + build.warmOperationExactMatches, + 0 + ); + const warmOperationMismatchViolations = builds.reduce( + (sum, build) => sum + build.warmOperationMismatchViolations, + 0 + ); + const warmOperationCrossTenantViolations = builds.reduce( + (sum, build) => sum + build.warmOperationCrossTenantViolations, + 0 + ); + const warmOperationCorrectnessConclusive = config.warmOperationsPerInstance === 0 + || ( + warmOperationErrors === 0 + && warmOperationReturnedStrings === warmOperationExecutions + ); + const warmOperationCorrectnessPassed = warmOperationCorrectnessConclusive + && warmOperationExactMatches === warmOperationExecutions; + const warmOperationReplayExecutions = builds.reduce( + (sum, build) => sum + build.warmOperationReplayExecutions, + 0 + ); + const warmOperationReplayErrors = builds.reduce( + (sum, build) => sum + build.warmOperationReplayErrors, + 0 + ); + const warmOperationReplayReturnedStrings = builds.reduce( + (sum, build) => sum + build.warmOperationReplayReturnedStrings, + 0 + ); + const warmOperationReplayExactMatches = builds.reduce( + (sum, build) => sum + build.warmOperationReplayExactMatches, + 0 + ); + const warmOperationReplayMismatchViolations = builds.reduce( + (sum, build) => sum + build.warmOperationReplayMismatchViolations, + 0 + ); + const warmOperationReplayCrossTenantViolations = builds.reduce( + (sum, build) => sum + build.warmOperationReplayCrossTenantViolations, + 0 + ); + const warmOperationReplayCorrectnessConclusive = + config.warmOperationReplayPasses === 0 + || ( + warmOperationReplayErrors === 0 + && warmOperationReplayReturnedStrings === warmOperationReplayExecutions + ); + const warmOperationReplayCorrectnessPassed = + warmOperationReplayCorrectnessConclusive + && warmOperationReplayExactMatches === warmOperationReplayExecutions; + const fixtureFingerprint = createHash('sha256').update(JSON.stringify({ + database: config.database, + schemas: config.schemas, + ...(schemaLayout.schemaSets === null + ? {} + : { + schemaSets: schemaLayout.schemaSets, + allowedDependencySchemas: schemaLayout.allowedDependencySchemas + }), + classes: catalogRow.classes, + attributes: catalogRow.attributes, + procs: catalogRow.procs, + types: catalogRow.types, + namespaces: catalogRow.namespaces, + databaseOid: catalogRow.database_oid, + maxClassOid: catalogRow.max_class_oid, + maxAttributeRelationOid: catalogRow.max_attribute_relation_oid, + maxProcOid: catalogRow.max_proc_oid, + maxTypeOid: catalogRow.max_type_oid, + procSignatureHash: catalogRow.proc_signature_hash, + pgVersion: catalogRow.pg_version, + serverVersionNum: catalogRow.server_version_num, + jit: catalogRow.jit + })).digest('hex'); + const introspectionBackendMeasurements = builds.flatMap((build) => + build.postgresIntrospectionBackendMemoryLowerBound + ? [build.postgresIntrospectionBackendMemoryLowerBound] + : [] + ); + const expectedIntrospectionBackendMeasurements = + config.introspectionClientReleaseMode === 'destroy' + && config.postgresBackendSamplerMode === 'diagnostic-lower-bound' + ? builds.length + : 0; + const allIntrospectionBackendCadenceChecksConclusive = + expectedIntrospectionBackendMeasurements === 0 + || ( + introspectionBackendMeasurements.length === builds.length + && introspectionBackendMeasurements.every( + (measurement) => measurement.timing.cadenceConclusive + ) + ); + const introspectionMeasurementLimitations = [...new Set( + introspectionBackendMeasurements.flatMap((measurement) => + measurement.provenance.limitation + ? [measurement.provenance.limitation] + : [] + ) + )]; + if ( + expectedIntrospectionBackendMeasurements > 0 + && introspectionBackendMeasurements.length !== 0 + && introspectionBackendMeasurements.length !== builds.length + ) { + throw new Error( + 'PostgreSQL introspection backend measurement was only recorded for ' + + `${introspectionBackendMeasurements.length} of ${builds.length} builds` + ); + } + const result: CatalogBenchResult = { + version: 1, + status: 'performance-only', + database: config.database, + mode: config.mode, + scopedCatalogTypes: config.scopedCatalogTypes, + introspectionClientReleaseMode: + config.introspectionClientReleaseMode, + postgresBackendSamplerMode: config.postgresBackendSamplerMode, + releaseBuildStateAfterValidation: + config.releaseBuildStateAfterValidation, + ...(schemaLayout.schemaSets === null + ? {} + : { + schemaSets: schemaLayout.schemaSets, + allowedDependencySchemas: schemaLayout.allowedDependencySchemas + }), + repetition: config.repetition, + heapMiB: config.heapMiB, + commit: config.commit, + worktreeDirty: config.worktreeDirty, + sourceStateSha256: config.sourceStateSha256, + lockfileSha256: config.lockfileSha256, + executedEntrySha256: config.executedEntrySha256, + v8Profile: config.v8Profile, + nodeOptions: config.nodeOptions, + nodeOptionsArgv: [...config.nodeOptionsArgv], + nodeExecArgv: [...process.execArgv], + effectiveNodeRuntimeFlags: [ + ...config.nodeOptionsArgv, + ...process.execArgv + ], + node: process.version, + v8: process.versions.v8, + effectiveV8HeapLimitBytes: getHeapStatistics().heap_size_limit, + platform: os.platform(), + architecture: os.arch(), + startedAt, + endedAt: new Date().toISOString(), + catalog: { + classes: Number(catalogRow.classes), + attributes: Number(catalogRow.attributes), + procs: Number(catalogRow.procs), + types: Number(catalogRow.types), + namespaces: Number(catalogRow.namespaces) + }, + runtimeRole: { + name: role.rolname, + superuser: role.rolsuper, + bypassRls: role.rolbypassrls, + createRole: role.rolcreaterole, + ownsDatabase: role.owns_database, + canCreateInDatabase: role.can_create_in_database, + ownsRequestedSchema: schemaRole.owns_schema, + canCreateInRequestedSchema: schemaRole.can_create + }, + catalogWarmth: 'shared-server-not-reset', + grafastCacheWarmth: { + operationsPerInstance: config.warmOperationsPerInstance, + cacheLimits: config.grafastCacheLimits, + sourceMode: 'grafast-source', + sourceSetSha256: warmOperationSources.length === 0 + ? null + : createHash('sha256').update(warmOperationSources.join('\0')).digest('hex'), + operationExecutions: warmOperationExecutions, + latencyP50Ms: catalogPercentile(allWarmOperationLatenciesMs, 0.5), + latencyP99Ms: catalogPercentile(allWarmOperationLatenciesMs, 0.99), + errors: warmOperationErrors, + returnedStrings: warmOperationReturnedStrings, + exactMatches: warmOperationExactMatches, + mismatchViolations: warmOperationMismatchViolations, + crossTenantViolations: warmOperationCrossTenantViolations, + correctnessConclusive: warmOperationCorrectnessConclusive, + correctnessPassed: warmOperationCorrectnessPassed, + replay: { + passesPerInstance: config.warmOperationReplayPasses, + operationExecutions: warmOperationReplayExecutions, + latencyP50Ms: catalogPercentile(allWarmOperationReplayLatenciesMs, 0.5), + latencyP99Ms: catalogPercentile(allWarmOperationReplayLatenciesMs, 0.99), + errors: warmOperationReplayErrors, + returnedStrings: warmOperationReplayReturnedStrings, + exactMatches: warmOperationReplayExactMatches, + mismatchViolations: warmOperationReplayMismatchViolations, + crossTenantViolations: warmOperationReplayCrossTenantViolations, + correctnessConclusive: warmOperationReplayCorrectnessConclusive, + correctnessPassed: warmOperationReplayCorrectnessPassed + } + }, + buildTransientSampling: { + approximate: true, + intervalMs: BUILD_TRANSIENT_SAMPLE_INTERVAL_MS, + limitation: 'Event-loop sampling can miss synchronous heap/RSS peaks; process RSS high-water is also captured.', + maxSampledHeapDeltaBytes: Math.max( + 0, + ...builds.map((build) => build.sampledBuildPeakHeapDeltaBytes) + ), + maxSampledRssDeltaBytes: Math.max( + 0, + ...builds.map((build) => build.sampledBuildPeakRssDeltaBytes) + ), + maxProcessPeakRssDeltaBytes: Math.max( + 0, + ...builds.map((build) => build.processBuildPeakRssDeltaBytes) + ) + }, + postgresBackendMeasurement: { + initialBackendPid: initialBackendIdentity.pid, + initialBackendStartEpochMs: + initialBackendIdentity.backendStartEpochMs, + finalSteadyBackendPid: backendPid, + finalSteadyBackendStartEpochMs: backendIdentity.backendStartEpochMs, + expectedRetirementChecks: + config.introspectionClientReleaseMode === 'destroy' + ? builds.length + : 0, + completedRetirementChecks, + allExpectedRetirementsProven: + completedRetirementChecks === ( + config.introspectionClientReleaseMode === 'destroy' + ? builds.length + : 0 + ), + steadyBackendRss: { + measured: baselineBackend !== null, + samplePhase: config.introspectionClientReleaseMode === 'destroy' + ? 'post-introspection-replacement' + : 'shared-introspection-and-steady', + deltaBasis: config.introspectionClientReleaseMode === 'destroy' + ? 'replacement-acquisition' + : 'initial-backend' + }, + introspectionBackendMemory: { + sampledLowerBoundMeasured: + config.introspectionClientReleaseMode === 'destroy' + ? introspectionBackendMeasurements.length === builds.length + : false, + sharedSnapshotMeasured: + config.introspectionClientReleaseMode === 'reuse' + && baselineBackend !== null + && reportIntrospectionBackendHighWater, + semantics: config.introspectionClientReleaseMode === 'destroy' + ? introspectionBackendMeasurements.length === builds.length + ? 'diagnostic-lower-bound-without-pre-destroy-acknowledgement' + : 'unavailable' + : baselineBackend !== null + ? 'post-build-shared-backend-snapshot' + : 'unavailable', + measurementMethod: config.introspectionClientReleaseMode === 'destroy' + ? introspectionBackendMeasurements.length === builds.length + ? 'dedicated-identity-bound-procfs-sampler' + : 'unavailable' + : baselineBackend !== null + ? 'post-build-shared-backend-procfs' + : 'unavailable', + expectedBuildMeasurements: expectedIntrospectionBackendMeasurements, + completedBuildMeasurements: introspectionBackendMeasurements.length, + allBuildCadenceChecksConclusive: + allIntrospectionBackendCadenceChecksConclusive, + backendSamplerAuthority: 'diagnostic-only', + serviceDensityMemoryAuthority: + `separately-validated-${LINUX_CGROUP_V2_DENSITY_AUTHORITY}`, + limitation: config.introspectionClientReleaseMode === 'destroy' + ? introspectionBackendMeasurements.length === 0 + ? config.postgresBackendSamplerMode === 'off' + ? 'Backend sampler disabled for an observer-effect comparison; no ' + + 'replacement-backend value was substituted.' + : 'No identity-bound PostgreSQL backend procfs target was available; ' + + 'no replacement-backend value was substituted.' + : introspectionMeasurementLimitations.length === 0 + ? null + : introspectionMeasurementLimitations.join(' ') + : baselineBackend === null + ? 'No identity-bound PostgreSQL backend procfs target was available.' + : 'This is a post-build snapshot of a reused backend, not a destroyed ' + + 'backend peak or a service-memory authority.' + } + }, + fixtureFingerprint, + builds, + canaries, + snapshots, + heapSlopeBytesPerInstance: slope( + snapshots.map((snapshot) => ({ x: snapshot.instances, y: snapshot.heapDeltaBytes })) + ), + rssSlopeBytesPerInstance: slope( + snapshots.map((snapshot) => ({ x: snapshot.instances, y: snapshot.rssDeltaBytes })) + ), + allSdlHashesEqualWithinArm: hashes.size === 1, + tokenCanariesConclusive, + tokenCanariesPassed: tokenCanariesConclusive && tokenMismatchViolations === 0, + tokenMismatchViolations, + crossTenantTokenViolations, + bleedViolations: tokenMismatchViolations + crossTenantTokenViolations + }; + fs.mkdirSync(path.dirname(resultFile), { recursive: true }); + fs.writeFileSync(resultFile, `${JSON.stringify(result, null, 2)}\n`, 'utf8'); + persistProgress('complete'); + } finally { + await cleanupEntries(entries); + await pool.end(); + await controlPool.end(); + } +}; + +const waitForChild = ( + command: string, + commandArgs: string[], + logFile: string, + env: NodeJS.ProcessEnv +): Promise => new Promise((resolve, reject) => { + const log = fs.createWriteStream(logFile, { flags: 'w' }); + const child = spawn(command, commandArgs, { + env, + stdio: ['ignore', 'pipe', 'pipe'] + }); + child.stdout?.pipe(log); + child.stderr?.pipe(log); + child.once('error', reject); + child.once('exit', (code, signal) => { + log.end(); + if (code === 0) resolve(); + else reject(new Error(`catalog worker exited code=${code} signal=${signal}; see ${logFile}`)); + }); +}); + +export const runCatalogBench = async (args: string[]): Promise => { + const database = requireFlag(args, 'database'); + const mode = requireFlag(args, 'mode') as IntrospectionMode; + if (mode !== 'stock' && mode !== 'scoped-required') { + throw new Error("--mode must be 'stock' or 'scoped-required'"); + } + const scopedCatalogTypes = parseCatalogScopedCatalogTypes(args, mode); + const introspectionClientReleaseMode = + parseCatalogIntrospectionClientReleaseMode(args); + const postgresBackendSamplerMode = parseCatalogBackendSamplerMode(args); + const releaseBuildStateAfterValidation = parseCatalogBuildStateRetirement(args); + const v8Profile = parseCatalogV8Profile(args); + const checkpoints = parseCheckpoints(flag(args, 'instances') ?? '1'); + const maxInstances = Math.max(...checkpoints); + const schemaLayout = parseCatalogSchemaLayout(args, maxInstances); + const schemas = schemaLayout.schemas; + const tokenFlag = flag(args, 'expected-tokens'); + const expectedTokens = tokenFlag ? parseList(tokenFlag) : null; + if (expectedTokens && ( + expectedTokens.length !== maxInstances + || new Set(expectedTokens).size !== expectedTokens.length + )) { + throw new Error('--expected-tokens must contain one unique value per instance'); + } + const warmth = parseCatalogWarmthCliOptions(args); + const tenantProxySurfaces = parseCatalogTenantProxySurfaces(args); + if (warmth.warmOperationsPerInstance > 0 && expectedTokens === null) { + throw new Error( + '--expected-tokens is required when --warm-operations-per-instance is greater than zero' + ); + } + const heapMiB = parsePositiveInteger(flag(args, 'heap-mib') ?? '2048', 'heap-mib'); + const nodeOptions = replaceMaxOldSpaceSize(process.env.NODE_OPTIONS, heapMiB); + const nodeOptionsArgv = tokenizeNodeOptions(nodeOptions); + const nodeExecArgv = [ + ...nodeFlagsForV8Profile(v8Profile), + '--expose-gc' + ]; + const effectiveNodeRuntimeFlags = [...nodeOptionsArgv, ...nodeExecArgv]; + const repetitions = parsePositiveInteger(flag(args, 'repetitions') ?? '3', 'repetitions'); + const settleMs = parsePositiveInteger(flag(args, 'settle-ms') ?? '100', 'settle-ms'); + const outputRoot = path.resolve(requireFlag(args, 'out')); + const postgresContainer = flag(args, 'postgres-container') ?? null; + if (fs.existsSync(outputRoot) && fs.readdirSync(outputRoot).length > 0) { + throw new Error(`catalog-bench refuses to overwrite nonempty output directory '${outputRoot}'`); + } + const provenance = readGitProvenance(); + const lockfile = path.resolve('pnpm-lock.yaml'); + const lockfileSha256 = fs.existsSync(lockfile) ? sha256File(lockfile) : null; + const executedEntrySha256 = sha256File(process.argv[1]); + fs.mkdirSync(outputRoot, { recursive: true }); + const results: CatalogBenchResult[] = []; + + for (let repetition = 1; repetition <= repetitions; repetition++) { + const repetitionDir = path.join(outputRoot, `rep-${repetition}`); + fs.mkdirSync(repetitionDir, { recursive: true }); + const config: CatalogBenchConfig = { + version: 1, + database, + mode, + scopedCatalogTypes, + introspectionClientReleaseMode, + postgresBackendSamplerMode, + releaseBuildStateAfterValidation, + schemas: schemas.slice(0, maxInstances), + ...(schemaLayout.schemaSets === null + ? {} + : { + schemaSets: schemaLayout.schemaSets, + allowedDependencySchemas: schemaLayout.allowedDependencySchemas! + }), + checkpoints, + expectedTokens: expectedTokens?.slice(0, maxInstances) ?? null, + heapMiB, + repetition, + settleMs, + warmOperationsPerInstance: warmth.warmOperationsPerInstance, + warmOperationReplayPasses: warmth.warmOperationReplayPasses, + grafastCacheLimits: warmth.grafastCacheLimits, + postgresContainer, + commit: provenance.commit, + worktreeDirty: provenance.worktreeDirty, + sourceStateSha256: provenance.sourceStateSha256, + lockfileSha256, + executedEntrySha256, + v8Profile, + nodeOptions, + nodeOptionsArgv, + nodeExecArgv, + effectiveNodeRuntimeFlags + }; + const configFile = path.join(repetitionDir, 'config.json'); + const resultFile = path.join(repetitionDir, 'result.json'); + fs.writeFileSync(configFile, `${JSON.stringify(config, null, 2)}\n`, 'utf8'); + await waitForChild( + process.execPath, + [ + ...nodeExecArgv, + process.argv[1], + '__catalog-worker', + '--config', + configFile, + '--result', + resultFile + ], + path.join(repetitionDir, 'worker.log'), + { + ...process.env, + NODE_ENV: 'production', + GRAPHILE_ENV: 'production', + NODE_OPTIONS: nodeOptions + } + ); + results.push(JSON.parse(fs.readFileSync(resultFile, 'utf8')) as CatalogBenchResult); + } + + const finalCheckpoint = maxInstances; + const finalSnapshots = results.map((result) => + result.snapshots.find((snapshot) => snapshot.instances === finalCheckpoint)! + ); + const postgresSampledHighWaterLowerBoundByRepetition = results.map((result) => + result.introspectionClientReleaseMode === 'destroy' + ? maxOrNull(result.builds.map( + (build) => build.postgresIntrospectionBackendMemoryLowerBound + ?.sampledHighWaterLowerBoundBytes ?? null + )) + : null + ); + const postgresSampledHighWaterDeltaLowerBoundByRepetition = results.map( + (result) => result.introspectionClientReleaseMode === 'destroy' + ? maxOrNull(result.builds.map( + (build) => build.postgresIntrospectionBackendMemoryLowerBound + ?.sampledHighWaterDeltaLowerBoundBytes ?? null + )) + : null + ); + const postgresSharedBackendSnapshotHighWaterByRepetition = results.map( + (result) => result.introspectionClientReleaseMode === 'reuse' + ? maxOrNull(result.snapshots.map( + (snapshot) => snapshot.postgresBackendHighWaterBytes + )) + : null + ); + const postgresSteadyRssByRepetition = finalSnapshots.map( + (snapshot) => snapshot.postgresBackendRssBytes + ); + const postgresSteadyRssDeltaByRepetition = finalSnapshots.map( + (snapshot) => snapshot.postgresBackendRssDeltaBytes + ); + const fixtureFingerprints = [...new Set(results.map((result) => result.fixtureFingerprint))]; + const effectiveV8HeapLimitBytes = results.map( + (result) => result.effectiveV8HeapLimitBytes + ); + const runtimeFlagsConsistent = results.every((result) => + result.v8Profile === v8Profile + && result.nodeOptions === nodeOptions + && JSON.stringify(result.nodeOptionsArgv) === JSON.stringify(nodeOptionsArgv) + && JSON.stringify(result.nodeExecArgv) === JSON.stringify(nodeExecArgv) + && JSON.stringify(result.effectiveNodeRuntimeFlags) + === JSON.stringify(effectiveNodeRuntimeFlags) + ); + if (!runtimeFlagsConsistent) { + throw new Error('catalog-bench worker runtime-flag provenance is inconsistent'); + } + const tenantDensityByRepetition = tenantProxySurfaces === null + ? null + : results.map((result, index) => ({ + repetition: result.repetition, + ...projectCatalogTenantDensity({ + tenantProxySurfaces, + configuredOldSpaceMiB: heapMiB, + snapshot: finalSnapshots[index] + }) + })); + const groupsPerConfiguredOldSpaceGiB = tenantDensityByRepetition?.map( + (density) => density.groupsPerConfiguredOldSpaceGiB + ) ?? []; + const groupsPerAbsolutePeakProcessRssGiB = tenantDensityByRepetition?.map( + (density) => density.groupsPerAbsolutePeakProcessRssGiB + ) ?? []; + const summary = { + version: 1, + status: 'performance-only', + mode, + scopedCatalogTypes, + introspectionClientReleaseMode, + postgresBackendSamplerMode, + releaseBuildStateAfterValidation, + v8Profile, + ...(schemaLayout.schemaSets === null + ? {} + : { + schemaSets: schemaLayout.schemaSets, + allowedDependencySchemas: schemaLayout.allowedDependencySchemas + }), + database, + heapMiB, + repetitions, + checkpoints, + v8Heap: { + configuredMaxOldSpaceMiB: heapMiB, + effectiveHeapLimitBytes: effectiveV8HeapLimitBytes, + effectiveHeapLimitConsistent: new Set(effectiveV8HeapLimitBytes).size === 1 + }, + nodeRuntimeFlags: { + nodeOptions, + nodeOptionsArgv, + nodeExecArgv, + effectiveNodeRuntimeFlags, + consistent: runtimeFlagsConsistent + }, + tenantDensityProjection: tenantProxySurfaces === null + ? null + : { + kind: 'synthetic-surface-instance-equivalent', + measuredCompleteTenants: false, + capacityBoundaryReached: false, + tenantProxySurfaces, + checkpoint: 'final-scheduled-checkpoint', + perRepetition: tenantDensityByRepetition, + medianGroupsPerConfiguredOldSpaceGiB: median( + groupsPerConfiguredOldSpaceGiB + ), + worstCaseGroupsPerConfiguredOldSpaceGiB: Math.min( + ...groupsPerConfiguredOldSpaceGiB + ), + medianGroupsPerAbsolutePeakProcessRssGiB: median( + groupsPerAbsolutePeakProcessRssGiB + ), + worstCaseGroupsPerAbsolutePeakProcessRssGiB: Math.min( + ...groupsPerAbsolutePeakProcessRssGiB + ) + }, + grafastCacheWarmth: { + operationsPerInstance: warmth.warmOperationsPerInstance, + cacheLimits: warmth.grafastCacheLimits, + sourceMode: 'grafast-source', + operationExecutions: results.map( + (result) => result.grafastCacheWarmth.operationExecutions + ), + latencyP50Ms: results.map((result) => result.grafastCacheWarmth.latencyP50Ms), + medianLatencyP50Ms: medianOrNull( + results.map((result) => result.grafastCacheWarmth.latencyP50Ms) + ), + latencyP99Ms: results.map((result) => result.grafastCacheWarmth.latencyP99Ms), + medianLatencyP99Ms: medianOrNull( + results.map((result) => result.grafastCacheWarmth.latencyP99Ms) + ), + correctnessConclusive: results.every( + (result) => result.grafastCacheWarmth.correctnessConclusive + ), + correctnessPassed: results.every( + (result) => result.grafastCacheWarmth.correctnessPassed + ), + errors: results.reduce( + (sum, result) => sum + result.grafastCacheWarmth.errors, + 0 + ), + mismatchViolations: results.reduce( + (sum, result) => sum + result.grafastCacheWarmth.mismatchViolations, + 0 + ), + crossTenantViolations: results.reduce( + (sum, result) => sum + result.grafastCacheWarmth.crossTenantViolations, + 0 + ), + replay: { + passesPerInstance: warmth.warmOperationReplayPasses, + sourceSet: 'same-exact-sources-as-population', + operationExecutions: results.map( + (result) => result.grafastCacheWarmth.replay.operationExecutions + ), + latencyP50Ms: results.map( + (result) => result.grafastCacheWarmth.replay.latencyP50Ms + ), + medianLatencyP50Ms: medianOrNull( + results.map((result) => result.grafastCacheWarmth.replay.latencyP50Ms) + ), + latencyP99Ms: results.map( + (result) => result.grafastCacheWarmth.replay.latencyP99Ms + ), + medianLatencyP99Ms: medianOrNull( + results.map((result) => result.grafastCacheWarmth.replay.latencyP99Ms) + ), + correctnessConclusive: results.every( + (result) => result.grafastCacheWarmth.replay.correctnessConclusive + ), + correctnessPassed: results.every( + (result) => result.grafastCacheWarmth.replay.correctnessPassed + ), + errors: results.reduce( + (sum, result) => sum + result.grafastCacheWarmth.replay.errors, + 0 + ), + mismatchViolations: results.reduce( + (sum, result) => + sum + result.grafastCacheWarmth.replay.mismatchViolations, + 0 + ), + crossTenantViolations: results.reduce( + (sum, result) => + sum + result.grafastCacheWarmth.replay.crossTenantViolations, + 0 + ) + } + }, + buildTransientSampling: { + approximate: true, + intervalMs: BUILD_TRANSIENT_SAMPLE_INTERVAL_MS, + limitation: 'Event-loop sampling can miss synchronous heap/RSS peaks; process RSS high-water is also captured.', + maxSampledHeapDeltaBytes: results.map( + (result) => result.buildTransientSampling.maxSampledHeapDeltaBytes + ), + medianMaxSampledHeapDeltaBytes: median( + results.map((result) => result.buildTransientSampling.maxSampledHeapDeltaBytes) + ), + maxSampledRssDeltaBytes: results.map( + (result) => result.buildTransientSampling.maxSampledRssDeltaBytes + ), + medianMaxSampledRssDeltaBytes: median( + results.map((result) => result.buildTransientSampling.maxSampledRssDeltaBytes) + ), + maxProcessPeakRssDeltaBytes: results.map( + (result) => result.buildTransientSampling.maxProcessPeakRssDeltaBytes + ), + medianMaxProcessPeakRssDeltaBytes: median( + results.map((result) => result.buildTransientSampling.maxProcessPeakRssDeltaBytes) + ) + }, + postgresBackendSamplerObserverEffect: { + mode: postgresBackendSamplerMode, + correctionApplied: false, + pairedComparisonSupported: true, + comparisonValues: ['off', 'diagnostic-lower-bound'], + measuredLaunchToReadyMs: results.map((result) => result.builds.flatMap( + (build) => build.postgresIntrospectionBackendMemoryLowerBound + ? [build.postgresIntrospectionBackendMemoryLowerBound + .observerEffect.measuredLaunchToReadyMs] + : [] + )), + measuredStopRequestToCloseMs: results.map((result) => result.builds.flatMap( + (build) => build.postgresIntrospectionBackendMemoryLowerBound + ? [build.postgresIntrospectionBackendMemoryLowerBound + .observerEffect.measuredStopRequestToCloseMs] + : [] + )), + limitation: 'Only sampler launch and shutdown wall time is recorded; sampling ' + + 'CPU/I/O interference is not corrected. Use paired runs with ' + + "'--postgres-backend-sampler off' and " + + "'--postgres-backend-sampler diagnostic-lower-bound'." + }, + measurementProtocol: { + process: 'fresh-node-process-per-repetition', + heap: 'three-forced-gc-cycles-after-resident-reprobe', + build: 'forced-gc-resident-baseline-then-createGraphileInstance-through-schema-readiness', + buildTransient: 'approximate-five-millisecond-event-loop-sampling-plus-process-rss-high-water', + operationWarmth: warmth.warmOperationsPerInstance > 0 + ? 'distinct-named-source-queries-through-grafast' + : 'disabled', + operationReplay: warmth.warmOperationReplayPasses > 0 + ? 'exact-population-source-set-replayed-through-grafast' + : 'disabled', + postgres: results.every( + (result) => ( + result.postgresBackendMeasurement.introspectionBackendMemory + .sampledLowerBoundMeasured + || result.postgresBackendMeasurement.introspectionBackendMemory + .sharedSnapshotMeasured + ) + ) + ? introspectionClientReleaseMode === 'destroy' + ? 'dedicated-identity-bound-procfs-diagnostic-lower-bound-before-retirement' + : 'shared-introspection-backend-post-build-procfs-baseline-relative' + : 'not-measured-no-replacement-backend-substitution', + postgresBackendSamplerAuthority: 'diagnostic-only', + postgresDensityMemoryAuthority: + `separately-validated-${LINUX_CGROUP_V2_DENSITY_AUTHORITY}`, + catalogWarmth: 'shared-server-not-reset', + historicalMethodologyComparable: false + }, + provenance: { + commit: provenance.commit, + worktreeDirty: provenance.worktreeDirty, + sourceStateSha256: provenance.sourceStateSha256, + lockfileSha256, + executedEntrySha256, + v8Profile, + nodeOptions, + nodeOptionsArgv, + nodeExecArgv, + effectiveNodeRuntimeFlags, + benchmarkConfiguration: { + scopedCatalogTypes, + introspectionClientReleaseMode, + postgresBackendSamplerMode, + releaseBuildStateAfterValidation, + ...(schemaLayout.schemaSets === null + ? {} + : { + schemaSets: schemaLayout.schemaSets, + allowedDependencySchemas: schemaLayout.allowedDependencySchemas + }), + warmOperationsPerInstance: warmth.warmOperationsPerInstance, + warmOperationReplayPasses: warmth.warmOperationReplayPasses, + grafastCacheLimits: warmth.grafastCacheLimits, + tenantProxySurfaces, + v8Profile + } + }, + catalog: results[0].catalog, + fixtureFingerprint: fixtureFingerprints.length === 1 ? fixtureFingerprints[0] : null, + fixtureFingerprintConsistent: fixtureFingerprints.length === 1, + freshProcessFirstBuildReadyMs: results.map((result) => result.builds[0].buildMs), + medianFreshProcessFirstBuildReadyMs: median( + results.map((result) => result.builds[0].buildMs) + ), + freshProcessFirstQueryMs: results.map((result) => result.builds[0].queryMs), + medianFreshProcessFirstQueryMs: median(results.map((result) => result.builds[0].queryMs)), + finalForcedGcHeapDeltaBytes: finalSnapshots.map((snapshot) => snapshot.heapDeltaBytes), + medianFinalForcedGcHeapDeltaBytes: median( + finalSnapshots.map((snapshot) => snapshot.heapDeltaBytes) + ), + finalRssDeltaBytes: finalSnapshots.map((snapshot) => snapshot.rssDeltaBytes), + medianFinalRssDeltaBytes: median(finalSnapshots.map((snapshot) => snapshot.rssDeltaBytes)), + medianHeapSlopeBytesPerInstance: median( + results.map((result) => result.heapSlopeBytesPerInstance) + ), + medianRssSlopeBytesPerInstance: median( + results.map((result) => result.rssSlopeBytesPerInstance) + ), + peakProcessRssBytes: Math.max(...results.flatMap((result) => + result.snapshots.map((snapshot) => snapshot.processPeakRssBytes) + )), + medianPeakProcessRssDeltaBytes: median(results.map((result) => + Math.max(...result.snapshots.map((snapshot) => snapshot.processPeakRssDeltaBytes)) + )), + peakProcessRssDeltaBytes: Math.max(...results.flatMap((result) => + result.snapshots.map((snapshot) => snapshot.processPeakRssDeltaBytes) + )), + postgresMemoryMeasured: postgresSteadyRssByRepetition.every( + (value) => value !== null + ), + postgresSteadyBackendRssBytes: postgresSteadyRssByRepetition, + medianPostgresSteadyBackendRssBytes: medianOrNull( + postgresSteadyRssByRepetition + ), + peakPostgresSteadyBackendRssBytes: maxOrNull( + postgresSteadyRssByRepetition + ), + postgresSteadyBackendRssDeltaBytes: + postgresSteadyRssDeltaByRepetition, + medianPostgresSteadyBackendRssDeltaBytes: medianOrNull( + postgresSteadyRssDeltaByRepetition + ), + postgresIntrospectionBackendSampledLowerBoundMeasured: + postgresSampledHighWaterLowerBoundByRepetition.every( + (value) => value !== null + ), + postgresIntrospectionBackendSampledLowerBoundSemantics: + 'diagnostic-lower-bound-without-pre-destroy-acknowledgement', + postgresIntrospectionBackendSampledLowerBoundLimitation: [...new Set( + results.flatMap((result) => { + const limitation = + result.postgresBackendMeasurement.introspectionBackendMemory.limitation; + return limitation ? [limitation] : []; + }) + )].join(' ') || null, + postgresBackendLifecycle: results.map( + (result) => result.postgresBackendMeasurement + ), + postgresIntrospectionBackendSampledHighWaterLowerBoundBytes: + postgresSampledHighWaterLowerBoundByRepetition, + peakPostgresIntrospectionBackendSampledHighWaterLowerBoundBytes: maxOrNull( + postgresSampledHighWaterLowerBoundByRepetition + ), + postgresIntrospectionBackendSampledHighWaterDeltaLowerBoundBytes: + postgresSampledHighWaterDeltaLowerBoundByRepetition, + medianPostgresIntrospectionBackendSampledHighWaterDeltaLowerBoundBytes: + medianOrNull(postgresSampledHighWaterDeltaLowerBoundByRepetition), + peakPostgresIntrospectionBackendSampledHighWaterDeltaLowerBoundBytes: + maxOrNull(postgresSampledHighWaterDeltaLowerBoundByRepetition), + postgresSharedBackendSnapshotHighWaterBytes: + postgresSharedBackendSnapshotHighWaterByRepetition, + peakPostgresSharedBackendSnapshotHighWaterBytes: maxOrNull( + postgresSharedBackendSnapshotHighWaterByRepetition + ), + allSdlHashesEqualWithinArm: results.every( + (result) => result.allSdlHashesEqualWithinArm + ), + tokenCanariesConclusive: results.every((result) => result.tokenCanariesConclusive), + tokenCanariesPassed: results.every((result) => result.tokenCanariesPassed), + tokenMismatchViolations: results.reduce( + (sum, result) => sum + result.tokenMismatchViolations, + 0 + ), + crossTenantTokenViolations: results.reduce( + (sum, result) => sum + result.crossTenantTokenViolations, + 0 + ), + bleedViolations: results.reduce((sum, result) => sum + result.bleedViolations, 0), + resultFiles: results.map((_, index) => `rep-${index + 1}/result.json`) + }; + fs.writeFileSync( + path.join(outputRoot, 'summary.json'), + `${JSON.stringify(summary, null, 2)}\n`, + 'utf8' + ); + process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); +}; diff --git a/packages/perf-harness/src/config.ts b/packages/perf-harness/src/config.ts new file mode 100644 index 0000000000..6a810b39e9 --- /dev/null +++ b/packages/perf-harness/src/config.ts @@ -0,0 +1,1207 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import type { + AcceptanceGates, + ArmPlan, + DensityPlanV1, + FleetV1, + TenantTarget, + WorkloadPlan +} from './types'; + +const RESERVED_PORTS = new Set([3000, 3001, 3002, 5432, 9000]); +const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]']); + +export const DEFAULT_RUN_ORDER_SEED = 'graphile-density-v1'; +export const DEFAULT_SOAK_ARM = 'scoped-introspection'; + +export const soakArmName = ( + plan: Pick +): string => plan.soak?.arm ?? DEFAULT_SOAK_ARM; + +export const hasExactHostileValidationEvidence = ( + plan: Pick +): boolean => { + const evidence = plan.qualification?.hostileValidationEvidence; + if (!evidence) return false; + const expectedArms = plan.arms.map((arm) => arm.name).sort(); + if (JSON.stringify(Object.keys(evidence).sort()) !== JSON.stringify(expectedArms)) { + return false; + } + return expectedArms.every((arm) => { + const binding = evidence[arm]; + return binding?.version === 1 + && binding.kind === 'exact-runtime-hostile-validation-v1' + && typeof binding.artifactFile === 'string' + && binding.artifactFile.length > 0 + && /^[a-f0-9]{64}$/.test(binding.artifactSha256) + && /^sha256:[a-f0-9]{64}$/.test(binding.runtimeArtifactFingerprint) + && /^sha256:[a-f0-9]{64}$/.test(binding.configurationFingerprint); + }); +}; + +const requirePositive = (value: unknown, label: string): void => { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) { + throw new Error(`${label} must be positive`); + } +}; + +const requireNonNegative = (value: unknown, label: string): void => { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new Error(`${label} must be non-negative`); + } +}; + +const requireBoolean = (value: unknown, label: string): void => { + if (typeof value !== 'boolean') throw new Error(`${label} must be boolean`); +}; + +export const validateAcceptanceGates = (gates: AcceptanceGates): void => { + if (!gates || typeof gates !== 'object' || Array.isArray(gates)) { + throw new Error('plan.gates is missing'); + } + requireNonNegative(gates.maxErrorRate, 'plan.gates.maxErrorRate'); + if (gates.maxErrorRate > 1) throw new Error('plan.gates.maxErrorRate must be at most 1'); + requirePositive(gates.maxP99Ms, 'plan.gates.maxP99Ms'); + requireNonNegative( + gates.maxPostWarmupHeapGrowthMiBPerHour, + 'plan.gates.maxPostWarmupHeapGrowthMiBPerHour' + ); + requireNonNegative( + gates.minMedianDensityImprovement, + 'plan.gates.minMedianDensityImprovement' + ); + requireNonNegative( + gates.minAdditionalTenantsEveryRun, + 'plan.gates.minAdditionalTenantsEveryRun' + ); + if (!Number.isSafeInteger(gates.minAdditionalTenantsEveryRun)) { + throw new Error('plan.gates.minAdditionalTenantsEveryRun must be a safe integer'); + } + if (gates.maxAlignedMemorySampleGapMs != null) { + requirePositive( + gates.maxAlignedMemorySampleGapMs, + 'plan.gates.maxAlignedMemorySampleGapMs' + ); + } + if (gates.minAlignedMemoryCoverageRatio != null) { + requirePositive( + gates.minAlignedMemoryCoverageRatio, + 'plan.gates.minAlignedMemoryCoverageRatio' + ); + if (gates.minAlignedMemoryCoverageRatio > 1) { + throw new Error('plan.gates.minAlignedMemoryCoverageRatio must be at most 1'); + } + } + const booleans: Array = [ + 'requireZeroBleed', + 'requireNoPostWarmupEvictions', + 'requireNoPostWarmupBuildRefusals', + 'requireNoPostWarmupBuilds', + 'requirePostgresMemoryTelemetry', + 'requireFreshPostgresRunAttestation', + 'requireRetainedMemoryCheckpoints', + 'requirePhysicalDatabaseTelemetry', + 'requireConclusiveCanaries', + 'requireCompletePeriodicCanaryCoverage', + 'requireConclusiveOperationOracles', + 'requireExplicitCustomerTopology' + ]; + for (const key of booleans) requireBoolean(gates[key], `plan.gates.${key}`); + if ( + gates.requiredCacheAdmissionMode !== null + && gates.requiredCacheAdmissionMode !== 'evict-idle' + && gates.requiredCacheAdmissionMode !== 'preserve-resident' + ) { + throw new Error( + 'plan.gates.requiredCacheAdmissionMode must be null, evict-idle, or preserve-resident' + ); + } +}; + +const SHA256 = /^[a-f0-9]{64}$/i; +const NODE_V8_PROFILES = new Set([ + 'stock', + 'optimize-for-size', + 'baseline-optimize-for-size', + 'jitless-optimize-for-size' +]); +const MANAGED_V8_FLAG = + /^--(?:no[-_])?(?:jitless|optimize[-_]for[-_]size|max[-_]opt)(?:=.*)?$/; + +const fileSha256 = (value: Buffer): string => createHash('sha256').update(value).digest('hex'); + +const validateCountRamp: ( + counts: unknown, + label: string +) => asserts counts is number[] = (counts, label) => { + if (!Array.isArray(counts) || counts.length === 0) { + throw new Error(`${label} must be a nonempty array`); + } + let previous = 0; + for (const count of counts) { + requirePositive(count, label); + if (!Number.isSafeInteger(count)) throw new Error(`${label} must contain safe integers`); + if (count <= previous) throw new Error(`${label} must be strictly increasing`); + previous = count; + } +}; + +export const tenantCountsForHeap = ( + plan: Pick, + heapMiB: number +): number[] => { + const specific = plan.tenantCountsByHeapMiB?.[String(heapMiB)]; + const counts = specific ?? plan.tenantCounts; + if (!counts?.length) { + throw new Error(`no tenant-count ramp is configured for heapMiB=${heapMiB}`); + } + return [...counts]; +}; + +export const armEnvironmentForHeap = ( + arm: Pick, + heapMiB: number +): Record => ({ + ...(arm.env ?? {}), + ...(arm.envByHeapMiB?.[String(heapMiB)] ?? {}) +}); + +/** + * Resolve the arm-specific identities and route without copying credentials + * into a benchmark artifact. The runner and semantic evidence replay must use + * this same transformation or a result could be scored against a different + * build/pool contract from the one that was exercised. + */ +export const resolveTenants = ( + tenants: TenantTarget[], + arm: ArmPlan +): TenantTarget[] => tenants.map((tenant) => ({ + ...tenant, + databases: tenant.databases?.map((database) => ({ + ...database, + apis: database.apis.map((api) => ({ + ...api, + runtimePoolIdentity: + api.runtimePoolIdentities?.[arm.name] ?? api.runtimePoolIdentity + })) + })), + surfaces: tenant.surfaces.map((surface) => ({ + ...surface, + buildContract: surface.buildContracts?.[arm.name] ?? surface.buildContract, + url: resolveTemplate(surface.url, { + port: arm.port, + mode: arm.introspectionMode + }) + })) +})); + +export const validateWorkloadPlan = (workload: WorkloadPlan): void => { + if (!workload || typeof workload !== 'object') throw new Error('plan.workload is missing'); + requirePositive(workload.durationSec, 'workload.durationSec'); + const hasFixedRps = workload.rps != null; + const hasPerTenantRps = workload.rpsPerTenant != null; + if (hasFixedRps === hasPerTenantRps) { + throw new Error('workload must define exactly one of rps or rpsPerTenant'); + } + requirePositive( + hasFixedRps ? workload.rps : workload.rpsPerTenant, + hasFixedRps ? 'workload.rps' : 'workload.rpsPerTenant' + ); + requirePositive( + workload.minWorkloadRequestsPerSurface, + 'workload.minWorkloadRequestsPerSurface' + ); + if (!Number.isSafeInteger(workload.minWorkloadRequestsPerSurface)) { + throw new Error('workload.minWorkloadRequestsPerSurface must be a safe integer'); + } + requirePositive(workload.maxInFlight, 'workload.maxInFlight'); + if (!Number.isSafeInteger(workload.maxInFlight)) { + throw new Error('workload.maxInFlight must be a safe integer'); + } + requirePositive(workload.canaryIntervalSec, 'workload.canaryIntervalSec'); + if ( + workload.periodicCanarySchedule != null + && workload.periodicCanarySchedule !== 'full-sweep' + && workload.periodicCanarySchedule !== 'rotating-one' + ) { + throw new Error( + "workload.periodicCanarySchedule must be 'full-sweep' or 'rotating-one'" + ); + } + if (workload.canaryConcurrency != null) { + requirePositive(workload.canaryConcurrency, 'workload.canaryConcurrency'); + if (!Number.isSafeInteger(workload.canaryConcurrency)) { + throw new Error('workload.canaryConcurrency must be a safe integer'); + } + } + requirePositive(workload.requestTimeoutMs, 'workload.requestTimeoutMs'); + requirePositive(workload.warmupTimeoutMs, 'workload.warmupTimeoutMs'); + requirePositive( + workload.warmupTimeoutPerSurfaceMs, + 'workload.warmupTimeoutPerSurfaceMs' + ); + if (workload.warmupConcurrency != null) { + requirePositive(workload.warmupConcurrency, 'workload.warmupConcurrency'); + if (!Number.isSafeInteger(workload.warmupConcurrency)) { + throw new Error('workload.warmupConcurrency must be a safe integer'); + } + } +}; + +const assertJsonPathMatches = (value: unknown, label: string): void => { + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${label} must contain at least one typed JSON-path match`); + } + for (const [index, match] of value.entries()) { + if (!match || typeof match !== 'object') { + throw new Error(`${label}[${index}] must be an object`); + } + const record = match as Record; + if (typeof record.path !== 'string' || (record.path !== '' && !record.path.startsWith('/'))) { + throw new Error(`${label}[${index}].path must be an RFC 6901 JSON pointer`); + } + if (!Object.prototype.hasOwnProperty.call(record, 'value')) { + throw new Error(`${label}[${index}] must define value`); + } + } +}; + +const assertJsonPathInvariants = (value: unknown, label: string): void => { + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${label} must contain at least one JSON-path invariant`); + } + for (const [index, invariant] of value.entries()) { + if (!invariant || typeof invariant !== 'object') { + throw new Error(`${label}[${index}] must be an object`); + } + const record = invariant as Record; + if (typeof record.path !== 'string' || (record.path !== '' && !record.path.startsWith('/'))) { + throw new Error(`${label}[${index}].path must be an RFC 6901 JSON pointer`); + } + if (!Object.prototype.hasOwnProperty.call(record, 'everyEquals')) { + throw new Error(`${label}[${index}] must define everyEquals`); + } + if (!Number.isSafeInteger(record.min) || (record.min as number) <= 0) { + throw new Error(`${label}[${index}].min must be a positive safe integer`); + } + if ( + record.max != null + && ( + !Number.isSafeInteger(record.max) + || (record.max as number) < (record.min as number) + ) + ) { + throw new Error(`${label}[${index}].max must be a safe integer at least min`); + } + } +}; + +const GRAPHQL_VARIABLE_NAME = /^[_A-Za-z][_0-9A-Za-z]*$/; + +const assertResponseVariableBindings = ( + value: unknown, + staticVariables: unknown, + label: string +): void => { + if (value == null) return; + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + const entries = Object.entries(value as Record); + if (entries.length === 0) throw new Error(`${label} must not be empty`); + const configuredStaticVariables = staticVariables && typeof staticVariables === 'object' + && !Array.isArray(staticVariables) + ? staticVariables as Record + : {}; + for (const [name, pointer] of entries) { + if (!GRAPHQL_VARIABLE_NAME.test(name)) { + throw new Error(`${label} has invalid GraphQL variable '${name}'`); + } + if (Object.prototype.hasOwnProperty.call(configuredStaticVariables, name)) { + throw new Error(`${label}.${name} collides with a static variable`); + } + if (typeof pointer !== 'string' || (pointer !== '' && !pointer.startsWith('/'))) { + throw new Error(`${label}.${name} must be an RFC 6901 JSON pointer`); + } + } +}; + +const assertOptionalOperationOracle = ( + operation: TenantTarget['surfaces'][number]['warmup'], + label: string +): void => { + const hasRequired = operation.requiredMatches != null; + const hasForbidden = operation.forbiddenMatches != null; + if (hasRequired !== hasForbidden) { + throw new Error(`${label} must configure requiredMatches and forbiddenMatches together`); + } + if (hasRequired) { + assertJsonPathMatches(operation.requiredMatches, `${label}.requiredMatches`); + assertJsonPathMatches(operation.forbiddenMatches, `${label}.forbiddenMatches`); + } + if (operation.invariants != null) { + assertJsonPathInvariants(operation.invariants, `${label}.invariants`); + } + const verification = operation.postCoverageVerification; + if (verification != null) { + if (typeof verification.query !== 'string' || !verification.query.trim()) { + throw new Error(`${label}.postCoverageVerification has no query`); + } + assertJsonPathMatches( + verification.requiredMatches, + `${label}.postCoverageVerification.requiredMatches` + ); + assertJsonPathMatches( + verification.forbiddenMatches, + `${label}.postCoverageVerification.forbiddenMatches` + ); + if (verification.invariants != null) { + assertJsonPathInvariants( + verification.invariants, + `${label}.postCoverageVerification.invariants` + ); + } + assertResponseVariableBindings( + verification.variablesFromResponse, + verification.variables, + `${label}.postCoverageVerification.variablesFromResponse` + ); + } +}; + +const hasConclusiveOperationOracle = ( + operation: TenantTarget['surfaces'][number]['warmup'] +): boolean => Boolean( + (operation.requiredMatches?.length && operation.forbiddenMatches?.length) + || ( + operation.postCoverageVerification?.requiredMatches.length + && operation.postCoverageVerification.forbiddenMatches.length + ) +); + +const ENVIRONMENT_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; +const isExactJsonPointer = (value: unknown): value is string => + typeof value === 'string' + && value.startsWith('/') + && value.split('/').slice(1).every((segment) => + segment !== '*' && !/~(?:[^01]|$)/.test(segment) + ); +const FORBIDDEN_DRIVER_HEADERS = new Set([ + 'connection', + 'content-length', + 'host', + 'sec-websocket-accept', + 'sec-websocket-extensions', + 'sec-websocket-key', + 'sec-websocket-protocol', + 'sec-websocket-version', + 'transfer-encoding', + 'upgrade' +]); +const SENSITIVE_HEADERS = new Set(['authorization', 'cookie', 'proxy-authorization']); + +const assertRealtimeProbe = ( + surface: TenantTarget['surfaces'][number], + label: string +): void => { + if (!surface.realtime || typeof surface.realtime !== 'object') { + throw new Error(`${label} has no realtime probe`); + } + for (const operationName of ['subscription', 'prime'] as const) { + const operation = surface.realtime[operationName]; + if (!operation || typeof operation.query !== 'string' || !operation.query.trim()) { + throw new Error(`${label}.realtime.${operationName} has no query`); + } + assertJsonPathMatches( + operation.requiredMatches, + `${label}.realtime.${operationName}.requiredMatches` + ); + assertJsonPathMatches( + operation.forbiddenMatches, + `${label}.realtime.${operationName}.forbiddenMatches` + ); + } + const correlation = surface.realtime.correlation; + if ( + !correlation + || typeof correlation !== 'object' + || !/^[_A-Za-z][_0-9A-Za-z]*$/.test(correlation.primeVariable ?? '') + || !isExactJsonPointer(correlation.primeResponsePath) + || !isExactJsonPointer(correlation.subscriptionEventPath) + || !Object.prototype.hasOwnProperty.call( + surface.realtime.prime.variables ?? {}, + correlation.primeVariable + ) + ) { + throw new Error(`${label}.realtime.correlation is invalid`); + } + if ( + surface.realtime.prime.requiredMatches.some( + (match) => match.path === correlation.primeResponsePath + ) + || surface.realtime.subscription.requiredMatches.some( + (match) => match.path === correlation.subscriptionEventPath + ) + ) { + throw new Error( + `${label}.realtime.correlation paths must not carry a static required match` + ); + } + const inlineHeaders = new Set(); + for (const [name, value] of Object.entries(surface.headers ?? {})) { + const normalized = name.trim().toLowerCase(); + if (!normalized || typeof value !== 'string') { + throw new Error(`${label}.headers must contain nonempty string values`); + } + if (FORBIDDEN_DRIVER_HEADERS.has(normalized)) { + throw new Error(`${label}.headers cannot override '${normalized}'`); + } + if (SENSITIVE_HEADERS.has(normalized)) { + throw new Error( + `${label}.${normalized} must use realtime.headersFromEnvironment` + ); + } + if (inlineHeaders.has(normalized)) { + throw new Error(`${label}.headers contains duplicate '${normalized}'`); + } + inlineHeaders.add(normalized); + } + const environmentHeaders = new Set(); + const mappings = surface.realtime.headersFromEnvironment ?? {}; + if (!mappings || typeof mappings !== 'object' || Array.isArray(mappings)) { + throw new Error(`${label}.realtime.headersFromEnvironment must be an object`); + } + for (const [name, environmentName] of Object.entries(mappings)) { + const normalized = name.trim().toLowerCase(); + if ( + !normalized + || FORBIDDEN_DRIVER_HEADERS.has(normalized) + || typeof environmentName !== 'string' + || !ENVIRONMENT_NAME.test(environmentName) + ) { + throw new Error(`${label}.realtime.headersFromEnvironment is invalid`); + } + if (inlineHeaders.has(normalized) || environmentHeaders.has(normalized)) { + throw new Error(`${label} configures header '${normalized}' more than once`); + } + environmentHeaders.add(normalized); + } +}; + +export const assertIsolatedPort = (port: number, allowReserved = false): void => { + requirePositive(port, 'arm.port'); + if (!allowReserved && RESERVED_PORTS.has(port)) { + throw new Error(`refusing reserved shared-workspace port ${port}`); + } +}; + +export const assertLoopbackObservabilityUrl = (value: string, expectedPort: number): void => { + let url: URL; + try { + url = new URL(value); + } catch { + throw new Error(`memoryUrl is not a valid URL: ${value}`); + } + const valid = url.protocol === 'http:' + && LOOPBACK_HOSTS.has(url.hostname.toLowerCase()) + && Number(url.port) === expectedPort + && !url.username + && !url.password + && url.pathname === '/debug/memory' + && !url.search + && !url.hash; + if (!valid) { + throw new Error( + `memoryUrl must be the credential-free URL http://127.0.0.1:${expectedPort}/debug/memory ` + + '(localhost and ::1 are also accepted)' + ); + } +}; + +export const assertLoopbackRetainedHeapCheckpointUrl = ( + value: string, + expectedPort: number +): void => { + let url: URL; + try { + url = new URL(value); + } catch { + throw new Error(`retainedHeapCheckpointUrl is not a valid URL: ${value}`); + } + const valid = url.protocol === 'http:' + && LOOPBACK_HOSTS.has(url.hostname.toLowerCase()) + && Number(url.port) === expectedPort + && !url.username + && !url.password + && url.pathname === '/__cperf/retained-memory-checkpoint' + && !url.search + && !url.hash; + if (!valid) { + throw new Error( + 'retainedHeapCheckpointUrl must be the credential-free URL ' + + `http://127.0.0.1:${expectedPort}/__cperf/retained-memory-checkpoint ` + + '(localhost and ::1 are also accepted)' + ); + } +}; + +export const loadPlan = (file: string, allowReserved = false): DensityPlanV1 => { + const planPath = path.resolve(file); + const planBytes = fs.readFileSync(planPath); + const plan = JSON.parse(planBytes.toString('utf8')) as DensityPlanV1; + plan.sourceSha256 = fileSha256(planBytes); + if (plan.version !== 1) throw new Error('density plan version must be 1'); + if (!Array.isArray(plan.arms) || plan.arms.length === 0) throw new Error('plan.arms is empty'); + if (!Array.isArray(plan.heapMiB) || plan.heapMiB.length === 0) throw new Error('plan.heapMiB is empty'); + requirePositive(plan.repetitions, 'plan.repetitions'); + if (!Number.isInteger(plan.repetitions)) throw new Error('plan.repetitions must be an integer'); + validateWorkloadPlan(plan.workload); + validateAcceptanceGates(plan.gates); + if (!Array.isArray(plan.requiredCapabilities) || plan.requiredCapabilities.length === 0) { + throw new Error('plan.requiredCapabilities is empty'); + } + if (!Array.isArray(plan.requiredCanaries) || plan.requiredCanaries.length === 0) { + throw new Error('plan.requiredCanaries is empty'); + } + if (new Set(plan.heapMiB).size !== plan.heapMiB.length) { + throw new Error('heapMiB must not contain duplicates'); + } + for (const heap of plan.heapMiB) { + requirePositive(heap, 'heapMiB'); + if (!Number.isInteger(heap)) throw new Error('heapMiB must contain integers'); + validateCountRamp(tenantCountsForHeap(plan, heap), `tenant counts for heapMiB=${heap}`); + } + if (plan.qualification != null) { + if ( + typeof plan.qualification !== 'object' + || Array.isArray(plan.qualification) + || typeof plan.qualification.baselineArm !== 'string' + || !plan.qualification.baselineArm + ) { + throw new Error('plan.qualification.baselineArm must be a nonempty string'); + } + if (!plan.arms.some((arm) => arm.name === plan.qualification!.baselineArm)) { + throw new Error( + `plan.qualification.baselineArm '${plan.qualification.baselineArm}' is not configured` + ); + } + validateCountRamp( + plan.qualification.requiredHeapMiB, + 'plan.qualification.requiredHeapMiB' + ); + for (const heap of plan.qualification.requiredHeapMiB) { + if (!plan.heapMiB.includes(heap)) { + throw new Error(`qualification heap ${heap}MiB is not configured in plan.heapMiB`); + } + } + requirePositive( + plan.qualification.minimumRepetitions, + 'plan.qualification.minimumRepetitions' + ); + if (!Number.isSafeInteger(plan.qualification.minimumRepetitions)) { + throw new Error('plan.qualification.minimumRepetitions must be a safe integer'); + } + if (plan.repetitions < plan.qualification.minimumRepetitions) { + throw new Error( + `plan.repetitions=${plan.repetitions} is below qualification minimum=${plan.qualification.minimumRepetitions}` + ); + } + const hostileEvidence = plan.qualification.hostileValidationEvidence; + if (hostileEvidence != null) { + if (!hasExactHostileValidationEvidence(plan)) { + throw new Error( + 'plan.qualification.hostileValidationEvidence must bind every exact arm' + ); + } + for (const arm of plan.arms) { + const binding = hostileEvidence[arm.name]; + const artifactFile = path.resolve(path.dirname(planPath), binding.artifactFile); + const stat = fs.lstatSync(artifactFile); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new Error(`hostile validation artifact for '${arm.name}' is not a regular file`); + } + const bytes = fs.readFileSync(artifactFile); + if (fileSha256(bytes) !== binding.artifactSha256) { + throw new Error(`hostile validation artifact for '${arm.name}' has the wrong SHA-256`); + } + const report = JSON.parse(bytes.toString('utf8')) as Record; + if ( + report.version !== 1 + || report.kind !== binding.kind + || report.passed !== true + || report.arm !== arm.name + || report.runtimeArtifactFingerprint !== binding.runtimeArtifactFingerprint + || report.configurationFingerprint !== binding.configurationFingerprint + ) { + throw new Error( + `hostile validation artifact for '${arm.name}' does not bind its exact runtime/config` + ); + } + binding.artifactFile = artifactFile; + } + } + } + if (plan.soak != null) { + if (typeof plan.soak !== 'object' || Array.isArray(plan.soak)) { + throw new Error('plan.soak must be an object'); + } + requireBoolean(plan.soak.enabled, 'plan.soak.enabled'); + if (plan.soak.enabled) { + requirePositive(plan.soak.durationSec, 'plan.soak.durationSec'); + requirePositive(plan.soak.tenantCount, 'plan.soak.tenantCount'); + requirePositive(plan.soak.heapMiB, 'plan.soak.heapMiB'); + for (const [value, label] of [ + [plan.soak.durationSec, 'plan.soak.durationSec'], + [plan.soak.tenantCount, 'plan.soak.tenantCount'], + [plan.soak.heapMiB, 'plan.soak.heapMiB'] + ] as const) { + if (!Number.isSafeInteger(value)) throw new Error(`${label} must be a safe integer`); + } + if (!plan.heapMiB.includes(plan.soak.heapMiB)) { + throw new Error(`plan.soak.heapMiB=${plan.soak.heapMiB} is not configured`); + } + if (plan.soak.arm != null && ( + typeof plan.soak.arm !== 'string' || !plan.soak.arm.trim() + )) { + throw new Error('plan.soak.arm must be a nonempty string'); + } + const armName = soakArmName(plan); + if (!plan.arms.some((arm) => arm.name === armName)) { + throw new Error(`plan.soak.arm '${armName}' is not configured`); + } + } + } + if (plan.tenantCountsByHeapMiB != null) { + if ( + typeof plan.tenantCountsByHeapMiB !== 'object' + || Array.isArray(plan.tenantCountsByHeapMiB) + ) { + throw new Error('tenantCountsByHeapMiB must be an object'); + } + const configuredHeaps = new Set(plan.heapMiB.map(String)); + for (const [heap, counts] of Object.entries(plan.tenantCountsByHeapMiB)) { + if (!configuredHeaps.has(heap)) { + throw new Error(`tenantCountsByHeapMiB contains unconfigured heap '${heap}'`); + } + validateCountRamp(counts, `tenantCountsByHeapMiB.${heap}`); + } + } + plan.runOrderSeed ??= DEFAULT_RUN_ORDER_SEED; + if (!plan.runOrderSeed.trim()) throw new Error('runOrderSeed must not be empty'); + const armNames = new Set(); + const armPorts = new Set(); + for (const arm of plan.arms) { + if (!arm.name || armNames.has(arm.name)) throw new Error(`duplicate or empty arm name '${arm.name}'`); + if (armPorts.has(arm.port)) throw new Error(`duplicate arm port ${arm.port}`); + armNames.add(arm.name); + armPorts.add(arm.port); + arm.v8Profile ??= 'stock'; + if (!NODE_V8_PROFILES.has(arm.v8Profile)) { + throw new Error(`arm '${arm.name}' has unknown v8Profile '${arm.v8Profile}'`); + } + if (arm.command?.some((argument) => MANAGED_V8_FLAG.test(argument))) { + throw new Error( + `arm '${arm.name}' must configure managed V8 flags through v8Profile` + ); + } + for (const [heap, environment] of [ + ['default', arm.env], + ...Object.entries(arm.envByHeapMiB ?? {}) + ] as Array<[string, Record | undefined]>) { + const nodeOptions = environment?.NODE_OPTIONS?.split(/\s+/) ?? []; + if (nodeOptions.some((argument) => MANAGED_V8_FLAG.test(argument))) { + throw new Error( + `arm '${arm.name}' NODE_OPTIONS for ${heap} must configure managed V8 flags through v8Profile` + ); + } + } + if ( + arm.v8Profile !== 'stock' + && arm.command?.length + && !['node', 'node.exe'].includes(path.basename(arm.command[0]).toLowerCase()) + ) { + throw new Error(`arm '${arm.name}' non-stock v8Profile requires a Node command`); + } + assertIsolatedPort(arm.port, allowReserved); + if (!Array.isArray(arm.command) && !arm.readinessUrl) { + throw new Error(`arm '${arm.name}' needs command or readinessUrl`); + } + if (arm.command?.length && !arm.commit) { + throw new Error(`arm '${arm.name}' must pin commit for a spawned run`); + } + if (arm.command?.length && plan.gates.requireRetainedMemoryCheckpoints) { + if (!arm.command.includes('--expose-gc')) { + throw new Error(`arm '${arm.name}' must launch Node with --expose-gc`); + } + if (!arm.retainedHeapCheckpointUrl) { + throw new Error(`arm '${arm.name}' needs retainedHeapCheckpointUrl`); + } + const checkpointUrl = resolveTemplate(arm.retainedHeapCheckpointUrl!, { + heapMiB: plan.heapMiB[0], + port: arm.port, + artifactDir: plan.artifactDir, + mode: arm.introspectionMode, + tenantCount: tenantCountsForHeap(plan, plan.heapMiB[0])[0] + }); + assertLoopbackRetainedHeapCheckpointUrl(checkpointUrl, arm.port); + for (const heapMiB of plan.heapMiB) { + if ( + armEnvironmentForHeap(arm, heapMiB) + .GRAPHQL_CPERF_RETAINED_HEAP_ENABLED !== 'true' + ) { + throw new Error( + `arm '${arm.name}' must set GRAPHQL_CPERF_RETAINED_HEAP_ENABLED=true for heap ${heapMiB}` + ); + } + } + } + if (arm.entrySha256 && !SHA256.test(arm.entrySha256)) { + throw new Error(`arm '${arm.name}' entrySha256 must be a SHA-256 hex digest`); + } + if (arm.lockfileSha256 && !SHA256.test(arm.lockfileSha256)) { + throw new Error(`arm '${arm.name}' lockfileSha256 must be a SHA-256 hex digest`); + } + if (arm.envByHeapMiB != null) { + if (typeof arm.envByHeapMiB !== 'object' || Array.isArray(arm.envByHeapMiB)) { + throw new Error(`arm '${arm.name}' envByHeapMiB must be an object`); + } + const configuredHeaps = new Set(plan.heapMiB.map(String)); + for (const heap of configuredHeaps) { + if (!Object.prototype.hasOwnProperty.call(arm.envByHeapMiB, heap)) { + throw new Error(`arm '${arm.name}' envByHeapMiB is missing heap '${heap}'`); + } + } + for (const [heap, environment] of Object.entries(arm.envByHeapMiB)) { + if (!configuredHeaps.has(heap)) { + throw new Error(`arm '${arm.name}' envByHeapMiB contains unconfigured heap '${heap}'`); + } + if ( + !environment + || typeof environment !== 'object' + || Array.isArray(environment) + || Object.entries(environment).some(([key, value]) => + !key || typeof value !== 'string' + ) + ) { + throw new Error(`arm '${arm.name}' envByHeapMiB.${heap} must contain string values`); + } + } + } + if (plan.gates.requirePostgresMemoryTelemetry && !arm.postgresContainer) { + throw new Error(`arm '${arm.name}' needs postgresContainer for required PostgreSQL telemetry`); + } + if (plan.gates.requireFreshPostgresRunAttestation) { + const attestation = arm.postgresRunAttestation; + if (!attestation || !Array.isArray(attestation.command) || attestation.command.length === 0) { + throw new Error( + `arm '${arm.name}' needs postgresRunAttestation.command for fresh PostgreSQL evidence` + ); + } + if ( + attestation.command.some((part) => typeof part !== 'string' || !part) + || !Array.isArray(attestation.prepareCommand) + || attestation.prepareCommand.length === 0 + || attestation.prepareCommand.some((part) => typeof part !== 'string' || !part) + || ( + attestation.timeoutMs != null + && ( + !Number.isSafeInteger(attestation.timeoutMs) + || attestation.timeoutMs <= 0 + ) + ) + ) { + throw new Error(`arm '${arm.name}' has invalid postgresRunAttestation config`); + } + const requiredServerTemplates = [ + '{postgresManifestFile}', + '{postgresSecretsFile}', + '{postgresManifestSha256}', + '{postgresCloneId}' + ]; + const requiredPrepareTemplates = [ + '{postgresFixtureDir}', + '{arm}', + '{heapMiB}', + '{tenantCount}', + '{repetition}', + '{runOrderIndex}' + ]; + const requiredAuditTemplates = [ + '{postgresManifestFile}', + '{postgresSecretsFile}', + '{attestationFile}', + '{planSha256}', + '{fleetSha256}', + '{notBeforeEpochMs}' + ]; + if ( + !arm.command?.length + || requiredServerTemplates.some((template) => !arm.command!.includes(template)) + || requiredPrepareTemplates.some((template) => + !attestation.prepareCommand.includes(template) + ) + || requiredAuditTemplates.some((template) => + !attestation.command.includes(template) + ) + ) { + throw new Error( + `arm '${arm.name}' does not bind the fresh PostgreSQL fixture into its server command` + ); + } + } + } + plan.fleetFile = path.resolve(path.dirname(planPath), plan.fleetFile); + plan.artifactDir = path.resolve(path.dirname(planPath), plan.artifactDir); + return plan; +}; + +export const loadFleet = (file: string): FleetV1 => { + const fleetBytes = fs.readFileSync(path.resolve(file)); + const fleet = JSON.parse(fleetBytes.toString('utf8')) as FleetV1; + fleet.sourceSha256 = fileSha256(fleetBytes); + if (fleet.version !== 1) throw new Error('fleet version must be 1'); + if (!Array.isArray(fleet.tenants) || fleet.tenants.length === 0) { + throw new Error('fleet.tenants is empty'); + } + const tenantIds = new Set(); + for (const tenant of fleet.tenants) { + if (!tenant.id || tenantIds.has(tenant.id)) throw new Error(`duplicate or empty tenant id '${tenant.id}'`); + tenantIds.add(tenant.id); + if (!Array.isArray(tenant.surfaces) || tenant.surfaces.length === 0) { + throw new Error(`tenant '${tenant.id}' has no surfaces`); + } + const surfaceNames = new Set(); + for (const surface of tenant.surfaces) { + if (!surface.name || surfaceNames.has(surface.name)) { + throw new Error(`tenant '${tenant.id}' has duplicate or empty surface '${surface.name}'`); + } + surfaceNames.add(surface.name); + const armContracts = surface.buildContracts; + if (!surface.buildContract && !armContracts) { + throw new Error( + `tenant '${tenant.id}' surface '${surface.name}' has no buildContract or buildContracts` + ); + } + if (armContracts && ( + Object.keys(armContracts).length === 0 + || Object.values(armContracts).some((contract) => !contract) + )) { + throw new Error( + `tenant '${tenant.id}' surface '${surface.name}' has incomplete buildContracts` + ); + } + if (!surface.url || !surface.warmup || surface.operations.length === 0) { + throw new Error(`tenant '${tenant.id}' surface '${surface.name}' is incomplete`); + } + if (!surface.warmup.name || !surface.warmup.capability || !surface.warmup.query) { + throw new Error(`tenant '${tenant.id}' surface '${surface.name}' has an incomplete warmup`); + } + assertOptionalOperationOracle( + surface.warmup, + `tenant '${tenant.id}' surface '${surface.name}'.warmup` + ); + const operationNames = new Set(); + for (const operation of surface.operations) { + if (!operation.name || operationNames.has(operation.name)) { + throw new Error(`tenant '${tenant.id}' surface '${surface.name}' has duplicate or empty operation '${operation.name}'`); + } + operationNames.add(operation.name); + if (!operation.capability || !operation.query) { + throw new Error(`operation '${operation.name}' is incomplete`); + } + if (operation.weight != null && (!Number.isFinite(operation.weight) || operation.weight <= 0)) { + throw new Error(`operation '${operation.name}' weight must be positive`); + } + assertOptionalOperationOracle( + operation, + `tenant '${tenant.id}' surface '${surface.name}' operation '${operation.name}'` + ); + } + if (!Array.isArray(surface.canaries) || surface.canaries.length === 0) { + throw new Error(`tenant '${tenant.id}' surface '${surface.name}' has no isolation canaries`); + } + const canaryNames = new Set(); + for (const canary of surface.canaries) { + if (!canary.name || canaryNames.has(canary.name)) { + throw new Error(`tenant '${tenant.id}' surface '${surface.name}' has duplicate or empty canary '${canary.name}'`); + } + canaryNames.add(canary.name); + if (!canary.query) throw new Error(`canary '${canary.name}' has no query`); + assertJsonPathMatches( + canary.forbiddenMatches, + `canary '${canary.name}'.forbiddenMatches` + ); + assertJsonPathMatches( + canary.requiredMatches, + `canary '${canary.name}'.requiredMatches` + ); + if (canary.invariants != null) { + assertJsonPathInvariants( + canary.invariants, + `canary '${canary.name}'.invariants` + ); + } + } + if (surface.realtime) { + assertRealtimeProbe( + surface, + `tenant '${tenant.id}' surface '${surface.name}'` + ); + } + } + validateCustomerTopology(tenant); + } + return fleet; +}; + +export const validateCustomerTopology = (customer: TenantTarget): void => { + if (customer.databases == null) return; + if (!Array.isArray(customer.databases) || customer.databases.length === 0) { + throw new Error(`customer '${customer.id}' has an empty database topology`); + } + const configuredSurfaces = new Set(customer.surfaces.map((surface) => surface.name)); + const mappedSurfaces = new Set(); + const databaseIds = new Set(); + const apiIds = new Set(); + for (const database of customer.databases) { + if (!database.id || databaseIds.has(database.id)) { + throw new Error(`customer '${customer.id}' has duplicate or empty database id '${database.id}'`); + } + databaseIds.add(database.id); + if (!database.physicalDatabase?.trim()) { + throw new Error(`customer '${customer.id}' database '${database.id}' has no physical database`); + } + if (!Array.isArray(database.apis) || database.apis.length === 0) { + throw new Error(`customer '${customer.id}' database '${database.id}' has no APIs`); + } + for (const api of database.apis) { + if (!api.id || apiIds.has(api.id)) { + throw new Error(`customer '${customer.id}' has duplicate or empty API id '${api.id}'`); + } + apiIds.add(api.id); + if (!api.runtimePoolIdentity && !api.runtimePoolIdentities) { + throw new Error(`customer '${customer.id}' API '${api.id}' has no runtime pool identity`); + } + if (api.runtimePoolIdentities && ( + Object.keys(api.runtimePoolIdentities).length === 0 + || Object.values(api.runtimePoolIdentities).some((identity) => !identity) + )) { + throw new Error(`customer '${customer.id}' API '${api.id}' has incomplete runtime pool identities`); + } + if ( + !Array.isArray(api.physicalSchemas) + || api.physicalSchemas.length === 0 + || api.physicalSchemas.some((schema) => typeof schema !== 'string' || !schema) + || new Set(api.physicalSchemas).size !== api.physicalSchemas.length + ) { + throw new Error(`customer '${customer.id}' API '${api.id}' has invalid physical schemas`); + } + if ( + !Array.isArray(api.routingLabels) + || api.routingLabels.length === 0 + || api.routingLabels.some((label) => typeof label !== 'string' || !label) + || new Set(api.routingLabels).size !== api.routingLabels.length + ) { + throw new Error(`customer '${customer.id}' API '${api.id}' has invalid routing labels`); + } + if (typeof api.realtime !== 'boolean') { + throw new Error(`customer '${customer.id}' API '${api.id}' has no explicit realtime flag`); + } + if ( + !Array.isArray(api.surfaces) + || api.surfaces.length === 0 + || new Set(api.surfaces).size !== api.surfaces.length + ) { + throw new Error(`customer '${customer.id}' API '${api.id}' has invalid surfaces`); + } + for (const surface of api.surfaces) { + if (!configuredSurfaces.has(surface)) { + throw new Error(`customer '${customer.id}' API '${api.id}' maps unknown surface '${surface}'`); + } + if (mappedSurfaces.has(surface)) { + throw new Error(`customer '${customer.id}' maps surface '${surface}' more than once`); + } + const configuredSurface = customer.surfaces.find((candidate) => + candidate.name === surface + ); + if (api.realtime !== Boolean(configuredSurface?.realtime)) { + throw new Error( + `customer '${customer.id}' API '${api.id}' realtime topology disagrees with surface '${surface}'` + ); + } + mappedSurfaces.add(surface); + } + } + } + const missingSurfaces = [...configuredSurfaces].filter((surface) => !mappedSurfaces.has(surface)); + if (missingSurfaces.length > 0) { + throw new Error( + `customer '${customer.id}' topology omits surfaces: ${missingSurfaces.join(', ')}` + ); + } +}; + +export const validateCoverage = (plan: DensityPlanV1, fleet: FleetV1): void => { + const failures: string[] = []; + const databaseOwners = new Map(); + const apiOwners = new Map(); + const matrixCounts = plan.heapMiB?.flatMap((heap) => tenantCountsForHeap(plan, heap)) + ?? plan.tenantCounts + ?? []; + const maxTenantCount = Math.max(0, ...matrixCounts, plan.soak?.tenantCount ?? 0); + if (fleet.tenants.length < maxTenantCount) { + failures.push(`fleet has ${fleet.tenants.length} tenants but the matrix requests ${maxTenantCount}`); + } + if ( + plan.gates?.requireCompletePeriodicCanaryCoverage + && (plan.workload.periodicCanarySchedule ?? 'full-sweep') === 'rotating-one' + ) { + const timedRounds = Math.max( + 0, + Math.ceil(plan.workload.durationSec / plan.workload.canaryIntervalSec) - 1 + ); + const selectedFleet = fleet.tenants.slice(0, maxTenantCount || fleet.tenants.length); + const maxConfiguredCanaries = Math.max( + 0, + ...selectedFleet.flatMap((tenant) => + tenant.surfaces.map((surface) => surface.canaries.length) + ) + ); + if (timedRounds < maxConfiguredCanaries) { + failures.push( + `rotating periodic canary schedule has ${timedRounds} timed rounds but ` + + `a qualifying surface configures ${maxConfiguredCanaries} canaries` + ); + } + } + for (const tenant of fleet.tenants) { + if (plan.gates?.requireExplicitCustomerTopology && !tenant.databases) { + failures.push(`${tenant.id} has no explicit customer -> database -> API topology`); + } + const capabilities = new Set(tenant.surfaces.flatMap((surface) => + surface.operations.map((operation) => operation.capability) + )); + const missingCapabilities = plan.requiredCapabilities.filter((capability) => + !capabilities.has(capability) + ); + if (missingCapabilities.length > 0) { + failures.push(`${tenant.id} has no operations for capabilities: ${missingCapabilities.join(', ')}`); + } + for (const surface of tenant.surfaces) { + if (plan.gates?.requireConclusiveOperationOracles) { + if (!hasConclusiveOperationOracle(surface.warmup)) { + failures.push( + `${tenant.id}/${surface.name} warmup has no conclusive response oracle` + ); + } + const missingOperationOracles = surface.operations + .filter((operation) => !hasConclusiveOperationOracle(operation)) + .map((operation) => operation.name); + if (missingOperationOracles.length > 0) { + failures.push( + `${tenant.id}/${surface.name} operations lack conclusive response oracles: ` + + missingOperationOracles.join(', ') + ); + } + } + if (surface.buildContracts) { + const missingArms = (plan.arms ?? []) + .filter((arm) => !surface.buildContracts?.[arm.name]) + .map((arm) => arm.name); + if (missingArms.length > 0) { + failures.push( + `${tenant.id}/${surface.name} lacks exact build contracts for arms: ${missingArms.join(', ')}` + ); + } + } + const canaries = new Set(surface.canaries.map((canary) => canary.name)); + const missing = plan.requiredCanaries.filter((canary) => !canaries.has(canary)); + if (missing.length > 0) { + failures.push(`${tenant.id}/${surface.name} lacks canaries: ${missing.join(', ')}`); + } + } + for (const database of tenant.databases ?? []) { + const databaseOwner = databaseOwners.get(database.id); + if (databaseOwner && databaseOwner !== tenant.id) { + failures.push( + `logical database id '${database.id}' is reused across customers '${databaseOwner}' and '${tenant.id}'` + ); + } else { + databaseOwners.set(database.id, tenant.id); + } + for (const api of database.apis) { + const apiOwner = apiOwners.get(api.id); + if (apiOwner && apiOwner !== tenant.id) { + failures.push( + `API id '${api.id}' is reused across customers '${apiOwner}' and '${tenant.id}'` + ); + } else { + apiOwners.set(api.id, tenant.id); + } + if (api.runtimePoolIdentities) { + const missingArms = (plan.arms ?? []) + .filter((arm) => !api.runtimePoolIdentities?.[arm.name]) + .map((arm) => arm.name); + if (missingArms.length > 0) { + failures.push( + `${tenant.id}/${database.id}/${api.id} lacks exact runtime pool identities for arms: ${missingArms.join(', ')}` + ); + } + } + } + } + } + const arms = plan.arms?.length + ? plan.arms.map((arm) => arm.name) + : ['default']; + for (const armName of arms) { + const owners = new Map(); + const poolOwners = new Map(); + for (const tenant of fleet.tenants) { + for (const surface of tenant.surfaces) { + const identity = armName === 'default' + ? surface.buildContract + : surface.buildContracts?.[armName] ?? surface.buildContract; + if (!identity) continue; + const owner = owners.get(identity); + if (owner && owner !== tenant.id) { + failures.push( + `build contract '${identity}' for arm '${armName}' is reused across tenants '${owner}' and '${tenant.id}'` + ); + } else { + owners.set(identity, tenant.id); + } + } + for (const database of tenant.databases ?? []) { + for (const api of database.apis) { + const identity = armName === 'default' + ? api.runtimePoolIdentity + : api.runtimePoolIdentities?.[armName] ?? api.runtimePoolIdentity; + if (!identity) continue; + const owner = poolOwners.get(identity); + if (owner && owner !== tenant.id) { + failures.push( + `runtime pool identity '${identity}' for arm '${armName}' is reused across customers '${owner}' and '${tenant.id}'` + ); + } else { + poolOwners.set(identity, tenant.id); + } + } + } + } + } + if (failures.length > 0) { + throw new Error(`density fixture is not qualification-complete:\n- ${failures.join('\n- ')}`); + } +}; + +export const resolveTemplate = ( + value: string, + vars: Record +): string => value.replace(/\{([a-zA-Z][a-zA-Z0-9]*)\}/g, (_match, key: string) => { + if (!(key in vars)) throw new Error(`unknown template variable '${key}'`); + return String(vars[key]); +}); diff --git a/packages/perf-harness/src/evidence.ts b/packages/perf-harness/src/evidence.ts new file mode 100644 index 0000000000..664aed4e78 --- /dev/null +++ b/packages/perf-harness/src/evidence.ts @@ -0,0 +1,1781 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; + +import { + DEFAULT_RUN_ORDER_SEED, + resolveTenants +} from './config'; +import { resolveOfferedLoad, resolveWarmupTimeoutMs } from './http'; +import { normalizeRetainedMemoryCheckpoint } from './memory'; +import { summarizeRealtimeReceiptEvidence } from './realtime-evidence'; +import { normalizePostgresRunAttestation } from './run-attestation'; +import { scoreRun, type ScoreInput } from './score'; +import type { RealtimeDriverSnapshot } from './realtime'; +import type { + ArmProvenance, + CanaryResult, + CanaryScheduleSummary, + DensityPlanV1, + DensityRunResult, + FleetV1, + MemorySnapshot, + NodeRssSnapshot, + PostgresMemorySnapshot, + PostgresRunAttestationEvidence, + RealtimeDeliveryCoverage, + ResolvedOfferedLoad, + RetainedMemoryCheckpointPair, + RequestSample +} from './types'; + +const SHA256 = /^[a-f0-9]{64}$/; +const SANITIZED_EXECUTION_ERROR = /^[A-Z][A-Z0-9_]*:sha256:[a-f0-9]{64}$/; + +/** + * These files contain every variable input to scoreRun that is not supplied by + * the exact plan and fleet bytes. score-context.json deliberately contains no + * fleet, operation, header, environment, or tenant credential material. + */ +export const RESULT_RAW_EVIDENCE_FILES = [ + 'memory.json', + 'postgres-memory.json', + 'canaries.json', + 'canary-schedule.json', + 'requests.ndjson', + 'workload-progress.json', + 'retained-memory.json', + 'realtime-driver.json', + 'score-context.json' +] as const; + +const SCORE_CONTEXT_KEYS = [ + 'version', + 'planSha256', + 'fleetSha256', + 'campaignId', + 'scheduleSha256', + 'previousResultPayloadSha256', + 'evidenceMode', + 'runKind', + 'arm', + 'heapMiB', + 'configuredCustomers', + 'repetition', + 'runOrderIndex', + 'notBeforeEpochMs', + 'startedAt', + 'endedAt', + 'configuredDurationSec', + 'serverExit', + 'externalServer', + 'executionErrors', + 'provenance', + 'provenanceErrors', + 'postgresRunAttestation' +] as const; + +export interface DensityScoreContextV1 { + version: 1; + planSha256: string; + fleetSha256: string; + campaignId: string; + scheduleSha256: string; + previousResultPayloadSha256: string | null; + evidenceMode: 'qualification' | 'diagnostic'; + runKind: 'matrix' | 'soak'; + arm: string; + heapMiB: number; + configuredCustomers: number; + repetition: number; + runOrderIndex: number; + notBeforeEpochMs: number; + startedAt: string; + endedAt: string; + configuredDurationSec: number; + serverExit: DensityRunResult['serverExit']; + externalServer: boolean; + executionErrors: string[]; + provenance: ArmProvenance | null; + provenanceErrors: string[]; + postgresRunAttestation: PostgresRunAttestationEvidence | null; +} + +export interface DensityScoreContextMetadata { + planSha256: string; + fleetSha256: string; + campaignId: string; + scheduleSha256: string; + previousResultPayloadSha256: string | null; + notBeforeEpochMs: number; + /** Compared in-memory only and never serialized into score-context.json. */ + knownRuntimeSecretValues?: readonly string[]; +} + +interface MemoryEvidence { + snapshots: MemorySnapshot[]; + osSnapshots: NodeRssSnapshot[]; + errors: string[]; + warmupIndex: number; + osWarmupIndex: number; + osPeakRssBytes: number | null; +} + +interface PostgresMemoryEvidence { + snapshots: PostgresMemorySnapshot[]; + errors: string[]; +} + +interface WorkloadProgressEvidence { + warmedSurfaces: Array<{ tenantId: string; surfaces: string[] }>; + warmupLatencies: number[]; + samples: number; + canaries: number; + canarySchedule: CanaryScheduleSummary | null; + offeredLoad: ResolvedOfferedLoad | null; + resolvedWarmupTimeoutMs: number | null; + workloadDurationMs: number | null; +} + +interface RealtimeEvidenceEntry { + phase: string; + timestamp: string; + snapshot: RealtimeDriverSnapshot; +} + +const sha256 = (value: string | Buffer): string => createHash('sha256') + .update(value) + .digest('hex'); + +const sourceSha256 = (value: DensityPlanV1 | FleetV1): string => + value.sourceSha256 ?? sha256(JSON.stringify(value)); + +const requireRecord = (value: unknown, label: string): Record => { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value as Record; +}; + +const requireExactKeys = ( + value: Record, + expected: readonly string[], + label: string +): void => { + const actual = Object.keys(value); + const missing = expected.filter((key) => !Object.prototype.hasOwnProperty.call(value, key)); + const unexpected = actual.filter((key) => !expected.includes(key)); + if (missing.length > 0 || unexpected.length > 0) { + throw new Error( + `${label} has an invalid shape; missing=${missing.join(',') || 'none'}; ` + + `unexpected=${unexpected.join(',') || 'none'}` + ); + } +}; + +const requireStringArray = (value: unknown, label: string): string[] => { + if (!Array.isArray(value) || value.some((item) => typeof item !== 'string')) { + throw new Error(`${label} must be a string array`); + } + return value; +}; + +const requireSanitizedExecutionErrors = (value: unknown, label: string): string[] => { + const errors = requireStringArray(value, label); + if (errors.some((error) => !SANITIZED_EXECUTION_ERROR.test(error))) { + throw new Error(`${label} must contain only code-and-SHA-256 evidence`); + } + return errors; +}; + +const requireFinite = (value: unknown, label: string): number => { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`${label} must be finite`); + } + return value; +}; + +const requireSafeInteger = ( + value: unknown, + label: string, + minimum = 0 +): number => { + if (!Number.isSafeInteger(value) || (value as number) < minimum) { + throw new Error(`${label} must be a safe integer >= ${minimum}`); + } + return value as number; +}; + +const requireCanonicalTimestamp = (value: unknown, label: string): string => { + if (typeof value !== 'string') throw new Error(`${label} must be a timestamp`); + const parsed = Date.parse(value); + if (!Number.isFinite(parsed) || new Date(parsed).toISOString() !== value) { + throw new Error(`${label} must be a canonical ISO timestamp`); + } + return value; +}; + +const requireBoolean = (value: unknown, label: string): boolean => { + if (typeof value !== 'boolean') throw new Error(`${label} must be boolean`); + return value; +}; + +const requireNonEmptyString = (value: unknown, label: string): string => { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`${label} must be a nonempty string`); + } + return value; +}; + +const requireAllowedKeys = ( + value: Record, + allowed: readonly string[], + label: string +): void => { + const unexpected = Object.keys(value).filter((key) => !allowed.includes(key)); + if (unexpected.length > 0) { + throw new Error(`${label} has unexpected fields: ${unexpected.join(',')}`); + } +}; + +const requireNullableNonNegativeFinite = (value: unknown, label: string): void => { + if (value == null) return; + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new Error(`${label} must be null or a finite non-negative number`); + } +}; + +/** Read one immutable regular file without following a final symlink. */ +export const readRegularEvidenceFile = (file: string): Buffer => { + const before = fs.lstatSync(file); + if (before.isSymbolicLink() || !before.isFile()) { + throw new Error(`evidence is not a regular non-symlink file: ${path.basename(file)}`); + } + const descriptor = fs.openSync( + file, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) + ); + try { + const opened = fs.fstatSync(descriptor); + if (!opened.isFile() || opened.dev !== before.dev || opened.ino !== before.ino) { + throw new Error(`evidence changed while opening: ${path.basename(file)}`); + } + const bytes = fs.readFileSync(descriptor); + const after = fs.fstatSync(descriptor); + if ( + after.dev !== opened.dev + || after.ino !== opened.ino + || after.size !== opened.size + || after.mtimeMs !== opened.mtimeMs + || after.ctimeMs !== opened.ctimeMs + ) { + throw new Error(`evidence changed while reading: ${path.basename(file)}`); + } + return bytes; + } finally { + fs.closeSync(descriptor); + } +}; + +const parseJsonEvidence = (artifactDir: string, name: string): unknown => { + try { + return JSON.parse(readRegularEvidenceFile(path.join(artifactDir, name)).toString('utf8')); + } catch (error) { + throw new Error( + `invalid ${name}: ${error instanceof Error ? error.message : String(error)}` + ); + } +}; + +const artifactNamesForResult = (result: DensityRunResult): string[] => [ + ...RESULT_RAW_EVIDENCE_FILES, + ...(result.postgresRunAttestation ? ['postgres-run-attestation.json'] : []) +]; + +const resultPayload = (result: DensityRunResult): Omit => { + const payload = { ...result }; + delete payload.evidenceBinding; + return payload; +}; + +const resultPayloadSha256 = (result: DensityRunResult): string => + sha256(JSON.stringify(resultPayload(result))); + +export const bindResultEvidence = (result: DensityRunResult): void => { + const artifacts = artifactNamesForResult(result).map((name) => ({ + name, + sha256: sha256(readRegularEvidenceFile(path.join(result.artifactDir, name))) + })); + result.evidenceBinding = { + version: 2, + algorithm: 'sha256', + resultPayloadSha256: resultPayloadSha256(result), + artifacts + }; +}; + +export const validateResultEvidenceBinding = ( + result: DensityRunResult, + label: string +): void => { + const binding = result.evidenceBinding; + if ( + !binding + || binding.version !== 2 + || binding.algorithm !== 'sha256' + || !SHA256.test(binding.resultPayloadSha256) + || !Array.isArray(binding.artifacts) + ) { + throw new Error(`${label} evidence binding is missing or invalid`); + } + if (binding.resultPayloadSha256 !== resultPayloadSha256(result)) { + throw new Error(`${label} result payload does not match its evidence binding`); + } + const expectedNames = artifactNamesForResult(result); + const byName = new Map(binding.artifacts.map((artifact) => [artifact.name, artifact])); + if (byName.size !== expectedNames.length || binding.artifacts.length !== expectedNames.length) { + throw new Error(`${label} raw evidence binding is incomplete or duplicated`); + } + for (const name of expectedNames) { + const artifact = byName.get(name); + if (!artifact || !SHA256.test(artifact.sha256)) { + throw new Error(`${label} raw evidence binding is missing ${name}`); + } + let bytes: Buffer; + try { + bytes = readRegularEvidenceFile(path.join(result.artifactDir, name)); + } catch { + throw new Error(`${label} raw evidence file is unavailable or unsafe: ${name}`); + } + if (sha256(bytes) !== artifact.sha256) { + throw new Error(`${label} raw evidence file does not match: ${name}`); + } + } +}; + +const SENSITIVE_ENVIRONMENT_NAME = /(?:^|_)(?:API_KEY|AUTHORIZATION|COOKIE|DATABASE_URL|DSN|PASSWORD|PASSWD|PGURL|PRIVATE_KEY|SECRET|TOKEN)$/i; +const SENSITIVE_COMMAND_FLAG = /^--(?:[a-z0-9]+[-_])*(?:api[-_]?key|authorization|cookie|database[-_]?url|dsn|password|passwd|private[-_]?key|secret|token)(?:[-_][a-z0-9]+)*(?:=|$)/i; +const SAFE_SECRET_FILE_FLAG = /^--(?:[a-z0-9]+[-_])*(?:credential|secret)s?(?:[-_]file)?$/i; +const SENSITIVE_ASSIGNMENT = /^(?:DATABASE_URL|PGPASSWORD|PGURL|[^=]*(?:PASSWORD|PASSWD|PRIVATE_KEY|SECRET|TOKEN))=/i; +const USERINFO_URL = /\b(?:https?|postgres(?:ql)?|wss?):\/\/[^\s/:@]+:[^\s/@]+@/i; +const BEARER_VALUE = /^Bearer\s+\S+/i; +const SENSITIVE_QUERY_PARAMETER = /[?&](?:api[-_]?key|authorization|cookie|password|passwd|private[-_]?key|secret|token)=[^&#\s]+/i; +const SENSITIVE_HEADER_VALUE = /^(?:authorization|cookie|proxy-authorization)\s*:\s*\S+/i; +const SENSITIVE_PROVENANCE_KEY = /(?:^|[-_])(?:api[-_]?key|authorization|cookie|database[-_]?url|dsn|password|passwd|pgurl|private[-_]?key|secret|token)(?:$|[-_])/i; +const SAFE_SECRET_REFERENCE_KEY = /(?:file|path)$/i; + +const knownRuntimeSecrets = ( + environment: Readonly> = process.env +): string[] => [...new Set(Object.entries(environment) + .filter(([name, value]) => SENSITIVE_ENVIRONMENT_NAME.test(name) && Boolean(value)) + .map(([_name, value]) => value!))]; + +/** + * Provenance must describe how the process was started, but it must never turn + * into a second credential store. Secret files and environment-variable names + * are safe; literal credential arguments and URL userinfo are rejected. + */ +export const assertScoreContextCredentialSafe = ( + context: DensityScoreContextV1, + secretValues: readonly string[] = knownRuntimeSecrets() +): void => { + const command = context.provenance?.command ?? []; + const sensitiveString = (value: string): boolean => + USERINFO_URL.test(value) + || BEARER_VALUE.test(value) + || SENSITIVE_QUERY_PARAMETER.test(value) + || SENSITIVE_HEADER_VALUE.test(value) + || SENSITIVE_ASSIGNMENT.test(value) + || secretValues.some((secret) => secret.length > 0 && value.includes(secret)); + const inspectStrings = (value: unknown, pathValue: string, ancestors = new Set()): void => { + if (typeof value === 'string') { + if (sensitiveString(value)) { + throw new Error(`score-context provenance contains credential material at ${pathValue}`); + } + return; + } + if (!value || typeof value !== 'object') return; + if (ancestors.has(value)) throw new Error('score-context provenance contains a cycle'); + ancestors.add(value); + if (Array.isArray(value)) { + value.forEach((item, index) => inspectStrings(item, `${pathValue}[${index}]`, ancestors)); + } else { + for (const [key, item] of Object.entries(value)) { + const normalizedKey = key.replace(/([a-z0-9])([A-Z])/g, '$1-$2'); + if ( + typeof item === 'string' + && item.length > 0 + && SENSITIVE_PROVENANCE_KEY.test(normalizedKey) + && !SAFE_SECRET_REFERENCE_KEY.test(normalizedKey) + ) { + throw new Error( + `score-context provenance contains credential material at ${pathValue}.${key}` + ); + } + inspectStrings(item, `${pathValue}.${key}`, ancestors); + } + } + ancestors.delete(value); + }; + inspectStrings( + context.provenance == null + ? null + : { ...context.provenance, command: [] }, + 'provenance' + ); + for (let index = 0; index < command.length; index += 1) { + const argument = command[index]; + const sensitiveFlag = SENSITIVE_COMMAND_FLAG.test(argument) + && !SAFE_SECRET_FILE_FLAG.test(argument); + if ( + sensitiveString(argument) + || (sensitiveFlag && argument.includes('=')) + || (sensitiveFlag && command[index + 1] != null) + ) { + throw new Error('score-context provenance command contains credential material'); + } + } +}; + +export const scoreContextFromInput = ( + input: ScoreInput, + metadata: DensityScoreContextMetadata +): DensityScoreContextV1 => { + requireSanitizedExecutionErrors(input.executionErrors, 'ScoreInput.executionErrors'); + const context: DensityScoreContextV1 = { + version: 1, + planSha256: metadata.planSha256, + fleetSha256: metadata.fleetSha256, + campaignId: metadata.campaignId, + scheduleSha256: metadata.scheduleSha256, + previousResultPayloadSha256: metadata.previousResultPayloadSha256, + evidenceMode: input.evidenceMode, + runKind: input.runKind, + arm: input.arm, + heapMiB: input.heapMiB, + configuredCustomers: input.tenants.length, + repetition: input.repetition, + runOrderIndex: input.runOrderIndex, + notBeforeEpochMs: metadata.notBeforeEpochMs, + startedAt: input.startedAt, + endedAt: input.endedAt, + configuredDurationSec: input.configuredDurationSec, + serverExit: input.serverExit ? { ...input.serverExit } : null, + externalServer: input.externalServer, + executionErrors: [...input.executionErrors], + provenance: input.provenance == null ? null : { ...input.provenance }, + provenanceErrors: [...input.provenanceErrors], + postgresRunAttestation: input.postgresRunAttestation == null + ? null + : { ...input.postgresRunAttestation } + }; + assertScoreContextCredentialSafe( + context, + metadata.knownRuntimeSecretValues ?? knownRuntimeSecrets() + ); + return context; +}; + +export const writeScoreContext = ( + artifactDir: string, + input: ScoreInput, + metadata: DensityScoreContextMetadata +): DensityScoreContextV1 => { + const context = scoreContextFromInput(input, metadata); + fs.writeFileSync( + path.join(artifactDir, 'score-context.json'), + `${JSON.stringify(context, null, 2)}\n`, + { encoding: 'utf8', flag: 'wx' } + ); + return context; +}; + +export const readScoreContextEvidence = (artifactDir: string): DensityScoreContextV1 => { + const record = requireRecord( + parseJsonEvidence(artifactDir, 'score-context.json'), + 'score-context.json' + ); + requireExactKeys(record, SCORE_CONTEXT_KEYS, 'score-context.json'); + if (record.version !== 1) throw new Error('score-context.json version must be 1'); + if (typeof record.planSha256 !== 'string' || !SHA256.test(record.planSha256)) { + throw new Error('score-context.json planSha256 is invalid'); + } + if (typeof record.fleetSha256 !== 'string' || !SHA256.test(record.fleetSha256)) { + throw new Error('score-context.json fleetSha256 is invalid'); + } + for (const key of ['campaignId', 'scheduleSha256'] as const) { + if (typeof record[key] !== 'string' || !SHA256.test(record[key] as string)) { + throw new Error(`score-context.json ${key} is invalid`); + } + } + if ( + record.previousResultPayloadSha256 != null + && ( + typeof record.previousResultPayloadSha256 !== 'string' + || !SHA256.test(record.previousResultPayloadSha256) + ) + ) { + throw new Error('score-context.json previousResultPayloadSha256 is invalid'); + } + if (!['qualification', 'diagnostic'].includes(String(record.evidenceMode))) { + throw new Error('score-context.json evidenceMode is invalid'); + } + if (!['matrix', 'soak'].includes(String(record.runKind))) { + throw new Error('score-context.json runKind is invalid'); + } + if (typeof record.arm !== 'string' || record.arm.length === 0) { + throw new Error('score-context.json arm is invalid'); + } + requireSafeInteger(record.heapMiB, 'score-context.json heapMiB', 1); + requireSafeInteger( + record.configuredCustomers, + 'score-context.json configuredCustomers', + 1 + ); + requireSafeInteger(record.repetition, 'score-context.json repetition', 1); + requireSafeInteger(record.runOrderIndex, 'score-context.json runOrderIndex', 1); + requireSafeInteger(record.notBeforeEpochMs, 'score-context.json notBeforeEpochMs', 1); + for (const key of ['startedAt', 'endedAt'] as const) { + requireCanonicalTimestamp(record[key], `score-context.json ${key}`); + } + if (Date.parse(record.endedAt as string) < Date.parse(record.startedAt as string)) { + throw new Error('score-context.json endedAt precedes startedAt'); + } + requireFinite(record.configuredDurationSec, 'score-context.json configuredDurationSec'); + if (typeof record.externalServer !== 'boolean') { + throw new Error('score-context.json externalServer is invalid'); + } + requireSanitizedExecutionErrors( + record.executionErrors, + 'score-context.json executionErrors' + ); + requireStringArray(record.provenanceErrors, 'score-context.json provenanceErrors'); + if (record.provenance != null) requireRecord(record.provenance, 'score-context.json provenance'); + if (record.postgresRunAttestation != null) { + requireRecord( + record.postgresRunAttestation, + 'score-context.json postgresRunAttestation' + ); + } + if (record.serverExit != null) { + const exit = requireRecord(record.serverExit, 'score-context.json serverExit'); + requireExactKeys(exit, ['code', 'signal'], 'score-context.json serverExit'); + if (exit.code != null && !Number.isSafeInteger(exit.code)) { + throw new Error('score-context.json serverExit.code is invalid'); + } + if (exit.signal != null && typeof exit.signal !== 'string') { + throw new Error('score-context.json serverExit.signal is invalid'); + } + } + const context = record as unknown as DensityScoreContextV1; + assertScoreContextCredentialSafe(context); + return context; +}; + +const parseMemoryEvidence = (artifactDir: string): MemoryEvidence => { + const record = requireRecord(parseJsonEvidence(artifactDir, 'memory.json'), 'memory.json'); + requireExactKeys( + record, + ['snapshots', 'osSnapshots', 'errors', 'warmupIndex', 'osWarmupIndex', 'osPeakRssBytes'], + 'memory.json' + ); + if (!Array.isArray(record.snapshots) || !Array.isArray(record.osSnapshots)) { + throw new Error('memory.json snapshots are invalid'); + } + const memoryRequiredKeys = [ + 'timestamp', + 'pid', + 'nodeEnv', + 'heapLimitBytes', + 'heapUsedBytes', + 'rssBytes', + 'processPeakRssBytes', + 'cacheSize', + 'residentBuildContracts', + 'evictions', + 'buildRefusals', + 'buildsStarted', + 'buildsSucceeded', + 'buildMaxMs', + 'pgPoolCacheSize', + 'pgPoolLeasedPools', + 'pgPoolActiveLeases', + 'pgPoolCapacityEvictions', + 'pgPoolCapacityRefusals', + 'pgPoolDisposalFailures', + 'cacheCountersAvailable', + 'buildCountersAvailable' + ] as const; + const optionalNumericKeys = [ + 'cacheConfiguredMax', + 'cacheBudgetCapacity', + 'cacheInstanceHeapBytes', + 'pgPoolTotalClients', + 'pgPoolIdleClients', + 'pgPoolWaitingClients', + 'runtimePoolRequestedMaxUses', + 'runtimePoolEffectiveMaxUses', + 'runtimePoolExpectedPools', + 'runtimePoolObservedPools', + 'runtimePoolTotalClients', + 'runtimePoolIdleClients', + 'runtimePoolWaitingClients', + 'postgresBackendTotal', + 'postgresBackendActive', + 'postgresBackendIdle', + 'postgresBackendIdleInTransaction', + 'physicalDatabases', + 'unexpectedPostgresDatabases', + 'realtimeManagersExpected', + 'realtimeManagersActive', + 'realtimeTransportsExpected', + 'realtimeTransportsActive', + 'notificationBrokers', + 'notificationListenerConnections', + 'notificationBrokerLeases', + 'notificationBrokerTopics', + 'notificationBrokerSubscribers', + 'notificationBrokerQueueOverflows', + 'notificationBrokerFatalFailures', + 'notificationAuditIdentities', + 'notificationAuditsHealthy', + 'notificationAuditsFailed', + 'notificationAuditsStale', + 'notificationAuditAttempts', + 'notificationAuditFailures', + 'notificationAuditActiveDatabaseTargets', + 'notificationAuditDatabaseConflicts' + ] as const; + const requiredNumericKeys = [ + 'heapLimitBytes', + 'heapUsedBytes', + 'rssBytes', + 'processPeakRssBytes', + 'cacheSize', + 'evictions', + 'buildRefusals', + 'buildsStarted', + 'buildsSucceeded', + 'buildMaxMs', + 'pgPoolCacheSize', + 'pgPoolLeasedPools', + 'pgPoolActiveLeases', + 'pgPoolCapacityEvictions', + 'pgPoolCapacityRefusals', + 'pgPoolDisposalFailures' + ] as const; + const optionalBooleanKeys = [ + 'runtimePoolTelemetryAvailable', + 'runtimePoolEffectiveMaxUsesKnown', + 'runtimePoolMaxUsesExact', + 'postgresContainerDedicated' + ] as const; + const allowedMemoryKeys = [ + ...memoryRequiredKeys, + ...optionalNumericKeys, + ...optionalBooleanKeys, + 'cacheCalibrationId', + 'cacheAdmissionMode', + 'residentBuildContractFingerprints', + 'runtimePoolTelemetryScope', + 'realtimeNotificationMode', + 'raw' + ]; + const snapshots = record.snapshots.map((raw, index): MemorySnapshot => { + const label = `memory.json snapshots[${index}]`; + const snapshot = requireRecord(raw, label); + requireAllowedKeys(snapshot, allowedMemoryKeys, label); + for (const key of memoryRequiredKeys) { + if (!Object.prototype.hasOwnProperty.call(snapshot, key)) { + throw new Error(`${label} is missing ${key}`); + } + } + requireCanonicalTimestamp(snapshot.timestamp, `${label}.timestamp`); + if (snapshot.pid != null) requireSafeInteger(snapshot.pid, `${label}.pid`, 1); + if (snapshot.nodeEnv != null && typeof snapshot.nodeEnv !== 'string') { + throw new Error(`${label}.nodeEnv is invalid`); + } + for (const key of [...requiredNumericKeys, ...optionalNumericKeys]) { + requireNullableNonNegativeFinite(snapshot[key], `${label}.${key}`); + } + if ( + snapshot.residentBuildContracts != null + && ( + !Array.isArray(snapshot.residentBuildContracts) + || snapshot.residentBuildContracts.some((item) => + typeof item !== 'string' || item.length === 0) + || new Set(snapshot.residentBuildContracts).size + !== snapshot.residentBuildContracts.length + ) + ) { + throw new Error(`${label}.residentBuildContracts is invalid`); + } + if ( + snapshot.residentBuildContractFingerprints != null + && ( + !Array.isArray(snapshot.residentBuildContractFingerprints) + || snapshot.residentBuildContractFingerprints.some((item) => + typeof item !== 'string' || item.length === 0) + || new Set(snapshot.residentBuildContractFingerprints).size + !== snapshot.residentBuildContractFingerprints.length + ) + ) { + throw new Error(`${label}.residentBuildContractFingerprints is invalid`); + } + requireBoolean(snapshot.cacheCountersAvailable, `${label}.cacheCountersAvailable`); + requireBoolean(snapshot.buildCountersAvailable, `${label}.buildCountersAvailable`); + for (const key of optionalBooleanKeys) { + if (snapshot[key] != null && typeof snapshot[key] !== 'boolean') { + throw new Error(`${label}.${key} is invalid`); + } + } + if ( + snapshot.cacheCalibrationId != null + && typeof snapshot.cacheCalibrationId !== 'string' + ) { + throw new Error(`${label}.cacheCalibrationId is invalid`); + } + if ( + snapshot.cacheAdmissionMode != null + && !['evict-idle', 'preserve-resident'].includes(String(snapshot.cacheAdmissionMode)) + ) { + throw new Error(`${label}.cacheAdmissionMode is invalid`); + } + if ( + snapshot.runtimePoolTelemetryScope != null + && snapshot.runtimePoolTelemetryScope !== 'runtime-only-exact-identities' + ) { + throw new Error(`${label}.runtimePoolTelemetryScope is invalid`); + } + if ( + snapshot.realtimeNotificationMode != null + && !['dedicated', 'shared-exact'].includes(String(snapshot.realtimeNotificationMode)) + ) { + throw new Error(`${label}.realtimeNotificationMode is invalid`); + } + return snapshot as unknown as MemorySnapshot; + }); + const osSnapshots = record.osSnapshots.map((raw, index): NodeRssSnapshot => { + const label = `memory.json osSnapshots[${index}]`; + const snapshot = requireRecord(raw, label); + requireExactKeys(snapshot, ['timestamp', 'pid', 'source', 'rssBytes'], label); + requireCanonicalTimestamp(snapshot.timestamp, `${label}.timestamp`); + requireSafeInteger(snapshot.pid, `${label}.pid`, 1); + requireSafeInteger(snapshot.rssBytes, `${label}.rssBytes`, 1); + if (!['proc', 'authenticated-endpoint'].includes(String(snapshot.source))) { + throw new Error(`${label}.source is invalid`); + } + return snapshot as unknown as NodeRssSnapshot; + }); + const errors = requireStringArray(record.errors, 'memory.json errors'); + const warmupIndex = requireSafeInteger(record.warmupIndex, 'memory.json warmupIndex', -1); + const osWarmupIndex = requireSafeInteger( + record.osWarmupIndex, + 'memory.json osWarmupIndex', + -1 + ); + if (warmupIndex > snapshots.length || osWarmupIndex > osSnapshots.length) { + throw new Error('memory.json warmup index is out of range'); + } + if (record.osPeakRssBytes != null) { + requireSafeInteger(record.osPeakRssBytes, 'memory.json osPeakRssBytes', 1); + if ( + osSnapshots.length > 0 + && record.osPeakRssBytes !== Math.max(...osSnapshots.map((snapshot) => snapshot.rssBytes)) + ) { + throw new Error('memory.json osPeakRssBytes does not match its raw snapshots'); + } + } + return { + snapshots, + osSnapshots, + errors, + warmupIndex, + osWarmupIndex, + osPeakRssBytes: record.osPeakRssBytes as number | null + }; +}; + +const parsePostgresMemoryEvidence = (artifactDir: string): PostgresMemoryEvidence => { + const record = requireRecord( + parseJsonEvidence(artifactDir, 'postgres-memory.json'), + 'postgres-memory.json' + ); + requireExactKeys(record, ['snapshots', 'errors'], 'postgres-memory.json'); + if (!Array.isArray(record.snapshots)) { + throw new Error('postgres-memory.json snapshots are invalid'); + } + const snapshots = record.snapshots.map((raw, index): PostgresMemorySnapshot => { + const label = `postgres-memory.json snapshots[${index}]`; + const snapshot = requireRecord(raw, label); + requireAllowedKeys(snapshot, [ + 'timestamp', + 'containerId', + 'cgroupIdentitySha256', + 'usedBytes', + 'limitBytes', + 'source', + 'workingSetBytes', + 'sampleStartedAt', + 'sampleEndedAt', + 'sampleDurationMs', + 'cgroupV2', + 'raw' + ], label); + for (const key of ['timestamp', 'usedBytes', 'limitBytes', 'raw']) { + if (!Object.prototype.hasOwnProperty.call(snapshot, key)) { + throw new Error(`${label} is missing ${key}`); + } + } + requireCanonicalTimestamp(snapshot.timestamp, `${label}.timestamp`); + requireSafeInteger(snapshot.usedBytes, `${label}.usedBytes`); + requireSafeInteger(snapshot.limitBytes, `${label}.limitBytes`); + if (typeof snapshot.raw !== 'string') throw new Error(`${label}.raw is invalid`); + if (snapshot.containerId != null && ( + typeof snapshot.containerId !== 'string' || !/^[a-f0-9]{64}$/.test(snapshot.containerId) + )) { + throw new Error(`${label}.containerId is invalid`); + } + if (snapshot.cgroupIdentitySha256 != null && ( + typeof snapshot.cgroupIdentitySha256 !== 'string' + || !/^sha256:[a-f0-9]{64}$/.test(snapshot.cgroupIdentitySha256) + )) { + throw new Error(`${label}.cgroupIdentitySha256 is invalid`); + } + if (snapshot.source != null && !['cgroup-v2', 'docker-stats'].includes(String(snapshot.source))) { + throw new Error(`${label}.source is invalid`); + } + if (snapshot.workingSetBytes != null) { + requireSafeInteger(snapshot.workingSetBytes, `${label}.workingSetBytes`); + } + if ( + snapshot.sampleStartedAt != null + || snapshot.sampleEndedAt != null + || snapshot.sampleDurationMs != null + ) { + const startedAt = requireCanonicalTimestamp( + snapshot.sampleStartedAt, + `${label}.sampleStartedAt` + ); + const endedAt = requireCanonicalTimestamp( + snapshot.sampleEndedAt, + `${label}.sampleEndedAt` + ); + const durationMs = requireSafeInteger( + snapshot.sampleDurationMs, + `${label}.sampleDurationMs` + ); + if (Date.parse(endedAt) < Date.parse(startedAt)) { + throw new Error(`${label} sample chronology is invalid`); + } + if (Math.abs((Date.parse(endedAt) - Date.parse(startedAt)) - durationMs) > 1) { + throw new Error(`${label}.sampleDurationMs is inconsistent`); + } + } + if (snapshot.cgroupV2 != null) { + const cgroup = requireRecord(snapshot.cgroupV2, `${label}.cgroupV2`); + requireExactKeys( + cgroup, + ['currentBytes', 'peakBytes', 'maxBytes', 'stat', 'events'], + `${label}.cgroupV2` + ); + requireSafeInteger(cgroup.currentBytes, `${label}.cgroupV2.currentBytes`); + if (cgroup.peakBytes != null) { + requireSafeInteger(cgroup.peakBytes, `${label}.cgroupV2.peakBytes`); + } + if (cgroup.maxBytes != null) { + requireSafeInteger(cgroup.maxBytes, `${label}.cgroupV2.maxBytes`); + } + for (const field of ['stat', 'events'] as const) { + const values = requireRecord(cgroup[field], `${label}.cgroupV2.${field}`); + for (const [key, value] of Object.entries(values)) { + if (!key) throw new Error(`${label}.cgroupV2.${field} has an empty key`); + requireSafeInteger(value, `${label}.cgroupV2.${field}.${key}`); + } + } + if (snapshot.source !== 'cgroup-v2' || snapshot.usedBytes !== cgroup.currentBytes) { + throw new Error(`${label} cgroup-v2 source does not match current bytes`); + } + } else if (snapshot.source === 'cgroup-v2') { + throw new Error(`${label} cgroup-v2 source has no cgroup payload`); + } + return snapshot as unknown as PostgresMemorySnapshot; + }); + return { + snapshots, + errors: requireStringArray(record.errors, 'postgres-memory.json errors') + }; +}; + +function parseCanarySchedule( + raw: unknown, + label: string +): CanaryScheduleSummary | null { + if (raw == null) return null; + const schedule = requireRecord(raw, label); + requireExactKeys(schedule, [ + 'schedule', + 'intervalMs', + 'durationMs', + 'canaryConcurrency', + 'startedAt', + 'deadlineAt', + 'planned', + 'started', + 'completed', + 'missed', + 'overlapped', + 'deadlineLate', + 'checksPlanned', + 'checksStarted', + 'checksCompleted', + 'rounds' + ], label); + if (!['full-sweep', 'rotating-one'].includes(String(schedule.schedule))) { + throw new Error(`${label}.schedule is invalid`); + } + const intervalMs = requireSafeInteger(schedule.intervalMs, `${label}.intervalMs`, 1); + const durationMs = requireSafeInteger(schedule.durationMs, `${label}.durationMs`, 1); + requireSafeInteger(schedule.canaryConcurrency, `${label}.canaryConcurrency`, 1); + const startedAt = requireCanonicalTimestamp(schedule.startedAt, `${label}.startedAt`); + const deadlineAt = requireCanonicalTimestamp(schedule.deadlineAt, `${label}.deadlineAt`); + if (Date.parse(deadlineAt) - Date.parse(startedAt) !== durationMs) { + throw new Error(`${label}.deadlineAt does not match durationMs`); + } + const counterKeys = [ + 'planned', + 'started', + 'completed', + 'missed', + 'overlapped', + 'deadlineLate', + 'checksPlanned', + 'checksStarted', + 'checksCompleted' + ] as const; + for (const key of counterKeys) requireSafeInteger(schedule[key], `${label}.${key}`); + if (!Array.isArray(schedule.rounds)) throw new Error(`${label}.rounds must be an array`); + const rounds = schedule.rounds.map((rawRound, index) => { + const roundLabel = `${label}.rounds[${index}]`; + const round = requireRecord(rawRound, roundLabel); + requireExactKeys(round, [ + 'periodicRound', + 'plannedAt', + 'startedAt', + 'completedAt', + 'targetsPlanned', + 'targetsStarted', + 'targetsCompleted', + 'checksPlanned', + 'checksStarted', + 'checksCompleted', + 'overlapped', + 'deadlineLate', + 'startDelayMs', + 'durationMs' + ], roundLabel); + const periodicRound = requireSafeInteger( + round.periodicRound, + `${roundLabel}.periodicRound`, + 1 + ); + if (periodicRound !== index + 1) { + throw new Error(`${roundLabel}.periodicRound is not contiguous`); + } + const plannedAt = requireCanonicalTimestamp(round.plannedAt, `${roundLabel}.plannedAt`); + if (Date.parse(plannedAt) !== Date.parse(startedAt) + periodicRound * intervalMs) { + throw new Error(`${roundLabel}.plannedAt does not match its schedule slot`); + } + for (const key of [ + 'targetsPlanned', + 'targetsStarted', + 'targetsCompleted', + 'checksPlanned', + 'checksStarted', + 'checksCompleted' + ] as const) { + requireSafeInteger(round[key], `${roundLabel}.${key}`); + } + requireBoolean(round.overlapped, `${roundLabel}.overlapped`); + requireBoolean(round.deadlineLate, `${roundLabel}.deadlineLate`); + const roundStartedAt = round.startedAt == null + ? null + : requireCanonicalTimestamp(round.startedAt, `${roundLabel}.startedAt`); + const roundCompletedAt = round.completedAt == null + ? null + : requireCanonicalTimestamp(round.completedAt, `${roundLabel}.completedAt`); + if (round.startDelayMs != null) { + requireFinite(round.startDelayMs, `${roundLabel}.startDelayMs`); + if ((round.startDelayMs as number) < 0) { + throw new Error(`${roundLabel}.startDelayMs must be non-negative`); + } + } + if (round.durationMs != null) { + requireFinite(round.durationMs, `${roundLabel}.durationMs`); + if ((round.durationMs as number) < 0) { + throw new Error(`${roundLabel}.durationMs must be non-negative`); + } + } + if ( + (roundStartedAt == null) !== (round.startDelayMs == null) + || (roundCompletedAt == null) !== (round.durationMs == null) + || (roundCompletedAt != null && roundStartedAt == null) + || ( + roundStartedAt != null + && Date.parse(roundStartedAt) < Date.parse(plannedAt) + ) + || ( + roundCompletedAt != null + && Date.parse(roundCompletedAt) < Date.parse(roundStartedAt!) + ) + ) { + throw new Error(`${roundLabel} chronology is invalid`); + } + return round; + }); + const aggregate = rounds.reduce<{ + started: number; + completed: number; + overlapped: number; + deadlineLate: number; + checksPlanned: number; + checksStarted: number; + checksCompleted: number; + }>((summary, round) => ({ + started: summary.started + (round.startedAt == null ? 0 : 1), + completed: summary.completed + (round.completedAt == null ? 0 : 1), + overlapped: summary.overlapped + (round.overlapped ? 1 : 0), + deadlineLate: summary.deadlineLate + (round.deadlineLate ? 1 : 0), + checksPlanned: summary.checksPlanned + Number(round.checksPlanned), + checksStarted: summary.checksStarted + Number(round.checksStarted), + checksCompleted: summary.checksCompleted + Number(round.checksCompleted) + }), { + started: 0, + completed: 0, + overlapped: 0, + deadlineLate: 0, + checksPlanned: 0, + checksStarted: 0, + checksCompleted: 0 + }); + if ( + Number(schedule.planned) !== rounds.length + || Number(schedule.started) !== aggregate.started + || Number(schedule.completed) !== aggregate.completed + || Number(schedule.missed) !== rounds.length - aggregate.completed + || Number(schedule.overlapped) !== aggregate.overlapped + || Number(schedule.deadlineLate) !== aggregate.deadlineLate + || Number(schedule.checksPlanned) !== aggregate.checksPlanned + || Number(schedule.checksStarted) !== aggregate.checksStarted + || Number(schedule.checksCompleted) !== aggregate.checksCompleted + ) { + throw new Error(`${label} aggregate counters do not match its rounds`); + } + return schedule as unknown as CanaryScheduleSummary; +} + +const parseWorkloadProgress = (artifactDir: string): WorkloadProgressEvidence => { + const record = requireRecord( + parseJsonEvidence(artifactDir, 'workload-progress.json'), + 'workload-progress.json' + ); + requireExactKeys(record, [ + 'warmedSurfaces', + 'warmupLatencies', + 'samples', + 'canaries', + 'canarySchedule', + 'offeredLoad', + 'resolvedWarmupTimeoutMs', + 'workloadDurationMs' + ], 'workload-progress.json'); + if (!Array.isArray(record.warmedSurfaces)) { + throw new Error('workload-progress.json warmedSurfaces is invalid'); + } + const warmedSurfaces = record.warmedSurfaces.map((raw, index) => { + const entry = requireRecord(raw, `workload-progress.json warmedSurfaces[${index}]`); + requireExactKeys( + entry, + ['tenantId', 'surfaces'], + `workload-progress.json warmedSurfaces[${index}]` + ); + if (typeof entry.tenantId !== 'string' || !entry.tenantId) { + throw new Error(`workload-progress.json warmedSurfaces[${index}].tenantId is invalid`); + } + const surfaces = requireStringArray( + entry.surfaces, + `workload-progress.json warmedSurfaces[${index}].surfaces` + ); + if (new Set(surfaces).size !== surfaces.length) { + throw new Error(`workload-progress.json warmedSurfaces[${index}] is duplicated`); + } + return { tenantId: entry.tenantId, surfaces }; + }); + if (new Set(warmedSurfaces.map((entry) => entry.tenantId)).size !== warmedSurfaces.length) { + throw new Error('workload-progress.json contains duplicate tenant warmup entries'); + } + if (!Array.isArray(record.warmupLatencies)) { + throw new Error('workload-progress.json warmupLatencies is invalid'); + } + const warmupLatencies = record.warmupLatencies.map((value, index) => + requireFinite(value, `workload-progress.json warmupLatencies[${index}]`)); + const samples = requireSafeInteger(record.samples, 'workload-progress.json samples'); + const canaries = requireSafeInteger(record.canaries, 'workload-progress.json canaries'); + let offeredLoad: ResolvedOfferedLoad | null = null; + if (record.offeredLoad != null) { + const resolved = requireRecord( + record.offeredLoad, + 'workload-progress.json offeredLoad' + ); + requireExactKeys(resolved, [ + 'mode', + 'configuredRps', + 'tenantCount', + 'totalRps', + 'rpsPerTenant' + ], 'workload-progress.json offeredLoad'); + if (!['fixed-total', 'per-tenant'].includes(String(resolved.mode))) { + throw new Error('workload-progress.json offeredLoad.mode is invalid'); + } + requireFinite(resolved.configuredRps, 'workload-progress.json offeredLoad.configuredRps'); + requireSafeInteger(resolved.tenantCount, 'workload-progress.json offeredLoad.tenantCount', 1); + requireFinite(resolved.totalRps, 'workload-progress.json offeredLoad.totalRps'); + requireFinite(resolved.rpsPerTenant, 'workload-progress.json offeredLoad.rpsPerTenant'); + if ( + (resolved.configuredRps as number) <= 0 + || (resolved.totalRps as number) <= 0 + || (resolved.rpsPerTenant as number) <= 0 + ) { + throw new Error('workload-progress.json offeredLoad rates must be positive'); + } + offeredLoad = resolved as unknown as ResolvedOfferedLoad; + } + if (record.resolvedWarmupTimeoutMs != null) { + requireSafeInteger( + record.resolvedWarmupTimeoutMs, + 'workload-progress.json resolvedWarmupTimeoutMs', + 1 + ); + } + if (record.workloadDurationMs != null) { + requireFinite(record.workloadDurationMs, 'workload-progress.json workloadDurationMs'); + } + return { + warmedSurfaces, + warmupLatencies, + samples, + canaries, + canarySchedule: parseCanarySchedule( + record.canarySchedule, + 'workload-progress.json canarySchedule' + ), + offeredLoad, + resolvedWarmupTimeoutMs: record.resolvedWarmupTimeoutMs as number | null, + workloadDurationMs: record.workloadDurationMs as number | null + }; +}; + +const parseRequestSample = (raw: unknown, label: string): RequestSample => { + const sample = requireRecord(raw, label); + requireAllowedKeys(sample, [ + 'tenantId', + 'surface', + 'operation', + 'capability', + 'latencyMs', + 'status', + 'ok', + 'phase', + 'scheduledAtMs', + 'errorCode', + 'oracleConfigured', + 'oracleConclusive', + 'oracleViolation', + 'oracleUnavailable', + 'postCoverageVerification' + ], label); + for (const key of [ + 'tenantId', + 'surface', + 'operation', + 'capability', + 'latencyMs', + 'status', + 'ok', + 'phase' + ]) { + if (!Object.prototype.hasOwnProperty.call(sample, key)) { + throw new Error(`${label} is missing ${key}`); + } + } + for (const key of ['tenantId', 'surface', 'operation', 'capability'] as const) { + requireNonEmptyString(sample[key], `${label}.${key}`); + } + const latencyMs = requireFinite(sample.latencyMs, `${label}.latencyMs`); + if (latencyMs < 0) throw new Error(`${label}.latencyMs must be non-negative`); + const status = requireSafeInteger(sample.status, `${label}.status`); + if (status > 599) throw new Error(`${label}.status is invalid`); + requireBoolean(sample.ok, `${label}.ok`); + if (!['coverage', 'workload'].includes(String(sample.phase))) { + throw new Error(`${label}.phase is invalid`); + } + if (sample.scheduledAtMs != null) { + requireFinite(sample.scheduledAtMs, `${label}.scheduledAtMs`); + } + if (sample.errorCode != null && typeof sample.errorCode !== 'string') { + throw new Error(`${label}.errorCode is invalid`); + } + for (const key of [ + 'oracleConfigured', + 'oracleConclusive', + 'oracleViolation', + 'oracleUnavailable', + 'postCoverageVerification' + ] as const) { + if (sample[key] != null && typeof sample[key] !== 'boolean') { + throw new Error(`${label}.${key} is invalid`); + } + } + return sample as unknown as RequestSample; +}; + +const parseCanaryResult = (raw: unknown, label: string): CanaryResult => { + const canary = requireRecord(raw, label); + requireAllowedKeys(canary, [ + 'tenantId', + 'surface', + 'canary', + 'phase', + 'periodicRound', + 'scheduledAt', + 'startedAt', + 'completedAt', + 'latencyMs', + 'conclusive', + 'violation', + 'detail' + ], label); + for (const key of [ + 'tenantId', + 'surface', + 'canary', + 'phase', + 'scheduledAt', + 'startedAt', + 'completedAt', + 'latencyMs', + 'conclusive', + 'violation' + ]) { + if (!Object.prototype.hasOwnProperty.call(canary, key)) { + throw new Error(`${label} is missing ${key}`); + } + } + for (const key of ['tenantId', 'surface', 'canary'] as const) { + requireNonEmptyString(canary[key], `${label}.${key}`); + } + if (!['initial', 'periodic', 'final'].includes(String(canary.phase))) { + throw new Error(`${label}.phase is invalid`); + } + if (canary.phase === 'periodic') { + requireSafeInteger(canary.periodicRound, `${label}.periodicRound`, 1); + } else if (canary.periodicRound != null) { + throw new Error(`${label}.periodicRound is only valid for periodic canaries`); + } + const scheduledAt = requireCanonicalTimestamp(canary.scheduledAt, `${label}.scheduledAt`); + const startedAt = requireCanonicalTimestamp(canary.startedAt, `${label}.startedAt`); + const completedAt = requireCanonicalTimestamp(canary.completedAt, `${label}.completedAt`); + if ( + Date.parse(startedAt) < Date.parse(scheduledAt) + || Date.parse(completedAt) < Date.parse(startedAt) + ) { + throw new Error(`${label} chronology is invalid`); + } + const latencyMs = requireFinite(canary.latencyMs, `${label}.latencyMs`); + if (latencyMs < 0) throw new Error(`${label}.latencyMs must be non-negative`); + requireBoolean(canary.conclusive, `${label}.conclusive`); + requireBoolean(canary.violation, `${label}.violation`); + if (canary.detail != null && typeof canary.detail !== 'string') { + throw new Error(`${label}.detail is invalid`); + } + return canary as unknown as CanaryResult; +}; + +const parseRequests = (artifactDir: string): RequestSample[] => { + const text = readRegularEvidenceFile(path.join(artifactDir, 'requests.ndjson')) + .toString('utf8'); + return text.split('\n').map((line) => line.trim()).filter(Boolean).map((line, index) => { + try { + return parseRequestSample( + JSON.parse(line), + `requests.ndjson line ${index + 1}` + ); + } catch (error) { + throw new Error( + `invalid requests.ndjson line ${index + 1}: ` + + `${error instanceof Error ? error.message : String(error)}` + ); + } + }); +}; + +const realtimeSurfaceKey = (surface: RealtimeDriverSnapshot['surfaces'][number]): string => + `${surface.tenantId}\0${surface.surface}\0${surface.route}`; + +const deriveRealtimeCoverage = ( + snapshot: RealtimeDriverSnapshot, + label: string +): RealtimeDeliveryCoverage | null => { + const reported = snapshot.timedCoverage; + if (reported == null) return null; + const summary = summarizeRealtimeReceiptEvidence({ + deliveryIntervalMs: snapshot.deliveryIntervalMs, + workloadStartedAt: reported.workloadStartedAt, + workloadDeadlineAt: reported.workloadDeadlineAt, + workloadEndedAt: reported.workloadEndedAt, + surfaces: snapshot.surfaces.map((surface) => ({ + tenantId: surface.tenantId, + surface: surface.surface, + route: surface.route, + expectedRecurringRounds: surface.timedRoundsExpected, + startedRecurringRounds: surface.timedRoundsStarted, + verifiedRecurringRounds: surface.timedRoundsVerified, + deadlineLateRecurringRounds: surface.timedRoundsDeadlineLate, + receipts: surface.correlationReceipts + })) + }); + const structurallyInvalid = summary.failures.find((failure) => + failure.startsWith('duplicate realtime surface:') + || failure.startsWith('invalid realtime receipt sequence:') + || failure.startsWith('invalid realtime receipt digest:') + || failure.startsWith('reused realtime receipt digest:') + || failure.startsWith('invalid realtime prime digest:') + || failure.startsWith('invalid realtime event digest:')); + if (structurallyInvalid) { + throw new Error(`${label} contains invalid receipt evidence: ${structurallyInvalid}`); + } + if (!isDeepStrictEqual(summary.coverage, reported)) { + throw new Error(`${label} reported coverage does not match its raw receipts`); + } + return summary.coverage; +}; + +const assertRealtimeHistoryAppendOnly = ( + previous: RealtimeDriverSnapshot, + current: RealtimeDriverSnapshot, + label: string +): void => { + const previousBySurface = new Map(previous.surfaces.map((surface) => [ + realtimeSurfaceKey(surface), + surface + ])); + const currentBySurface = new Map(current.surfaces.map((surface) => [ + realtimeSurfaceKey(surface), + surface + ])); + if ( + previousBySurface.size !== previous.surfaces.length + || currentBySurface.size !== current.surfaces.length + || previousBySurface.size !== currentBySurface.size + || [...previousBySurface.keys()].some((key) => !currentBySurface.has(key)) + ) { + throw new Error(`${label} realtime surface set changed`); + } + const monotonicSnapshotCounters: Array = [ + 'deliveryEvents', + 'deliveryRoundsStarted', + 'deliveryRoundsVerified' + ]; + for (const key of monotonicSnapshotCounters) { + if ((current[key] as number) < (previous[key] as number)) { + throw new Error(`${label} realtime aggregate counter regressed: ${key}`); + } + } + if (current.deliveryIntervalMs !== previous.deliveryIntervalMs) { + throw new Error(`${label} realtime delivery interval changed`); + } + for (const [key, prior] of previousBySurface) { + const next = currentBySurface.get(key)!; + for (const counter of [ + 'deliveryEvents', + 'deliveryRoundsStarted', + 'deliveryRoundsVerified', + 'timedRoundsExpected', + 'timedRoundsStarted', + 'timedRoundsVerified', + 'timedRoundsDeadlineLate' + ] as const) { + if (next[counter] < prior[counter]) { + throw new Error(`${label} realtime surface counter regressed: ${counter}`); + } + } + if ( + prior.correlationReceipts.length > next.correlationReceipts.length + || !prior.correlationReceipts.every((receipt, index) => + isDeepStrictEqual(receipt, next.correlationReceipts[index])) + ) { + throw new Error(`${label} realtime receipt history is not append-only`); + } + } + if ( + previous.errors.length > current.errors.length + || !previous.errors.every((error, index) => error === current.errors[index]) + ) { + throw new Error(`${label} realtime error history is not append-only`); + } +}; + +export const readRealtimeCoverageEvidence = ( + artifactDir: string +): RealtimeDeliveryCoverage | null => { + const raw = parseJsonEvidence(artifactDir, 'realtime-driver.json'); + if (!Array.isArray(raw)) throw new Error('realtime-driver.json must be an array'); + const entries = raw.map((value, index) => { + const entry = requireRecord(value, `realtime-driver.json[${index}]`); + requireExactKeys(entry, ['phase', 'timestamp', 'snapshot'], `realtime-driver.json[${index}]`); + if (typeof entry.phase !== 'string' || typeof entry.timestamp !== 'string') { + throw new Error(`realtime-driver.json[${index}] metadata is invalid`); + } + if ( + !Number.isFinite(Date.parse(entry.timestamp)) + || new Date(Date.parse(entry.timestamp)).toISOString() !== entry.timestamp + ) { + throw new Error(`realtime-driver.json[${index}] timestamp is invalid`); + } + const snapshot = requireRecord(entry.snapshot, `realtime-driver.json[${index}].snapshot`); + if (!Object.prototype.hasOwnProperty.call(snapshot, 'timedCoverage')) { + throw new Error(`realtime-driver.json[${index}].snapshot has no timedCoverage`); + } + return entry as unknown as RealtimeEvidenceEntry; + }); + const completed = entries.filter((entry) => entry.phase === 'timed-coverage-complete'); + if (completed.length > 1) { + throw new Error('realtime-driver.json has duplicate timed-coverage-complete records'); + } + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]; + deriveRealtimeCoverage(entry.snapshot, `realtime-driver.json[${index}]`); + if (index > 0) { + if (Date.parse(entry.timestamp) < Date.parse(entries[index - 1].timestamp)) { + throw new Error('realtime-driver.json timestamps regressed'); + } + assertRealtimeHistoryAppendOnly( + entries[index - 1].snapshot, + entry.snapshot, + `realtime-driver.json[${index}]` + ); + } + } + if (completed.length === 0) return null; + const completedIndex = entries.indexOf(completed[0]); + const coverage = deriveRealtimeCoverage( + completed[0].snapshot, + `realtime-driver.json[${completedIndex}]` + ); + if (!coverage || coverage.workloadEndedAt == null) { + throw new Error('timed-coverage-complete is not a terminal coverage transition'); + } + for (const [index, entry] of entries.entries()) { + if (index <= completedIndex || entry.snapshot.timedCoverage == null) continue; + const later = deriveRealtimeCoverage(entry.snapshot, `realtime-driver.json[${index}]`); + if (!isDeepStrictEqual(later, coverage)) { + throw new Error('timed realtime evidence changed after terminal coverage transition'); + } + } + return coverage; +}; + +const assertArtifactDirectory = (artifactDir: string, plan: DensityPlanV1): void => { + const root = fs.realpathSync(plan.artifactDir); + const stat = fs.lstatSync(artifactDir); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error('result artifactDir is not a regular directory'); + } + const realArtifactDir = fs.realpathSync(artifactDir); + const relative = path.relative(root, realArtifactDir); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative) || path.dirname(relative) !== '.') { + throw new Error('result artifactDir is outside the configured artifact root'); + } +}; + +const assertContextMatchesResult = ( + context: DensityScoreContextV1, + result: DensityRunResult, + plan: DensityPlanV1, + fleet: FleetV1 +): void => { + const planSha256 = sourceSha256(plan); + const fleetSha256 = sourceSha256(fleet); + const pairs: Array<[unknown, unknown, string]> = [ + [context.planSha256, planSha256, 'plan SHA-256'], + [context.fleetSha256, fleetSha256, 'fleet SHA-256'], + [context.campaignId, result.campaignId, 'campaign identity'], + [context.scheduleSha256, result.scheduleSha256, 'schedule SHA-256'], + [ + context.previousResultPayloadSha256, + result.previousResultPayloadSha256, + 'previous result payload SHA-256' + ], + [context.evidenceMode, result.evidenceMode, 'evidence mode'], + [context.runKind, result.runKind, 'run kind'], + [context.arm, result.arm, 'arm'], + [context.heapMiB, result.heapMiB, 'heap'], + [context.configuredCustomers, result.configuredCustomers, 'configured customers'], + [context.repetition, result.repetition, 'repetition'], + [context.runOrderIndex, result.runOrderIndex, 'run order'], + [context.startedAt, result.startedAt, 'start timestamp'], + [context.endedAt, result.endedAt, 'end timestamp'], + [context.serverExit, result.serverExit, 'server exit'], + [context.provenance, result.provenance, 'provenance'], + [context.provenanceErrors, result.provenanceErrors, 'provenance errors'], + [ + context.postgresRunAttestation, + result.postgresRunAttestation ?? null, + 'PostgreSQL run attestation' + ] + ]; + for (const [left, right, label] of pairs) { + if (!isDeepStrictEqual(left, right)) { + throw new Error(`score-context.json ${label} does not match the result/plan`); + } + } +}; + +const validatePostgresAttestation = ( + context: DensityScoreContextV1, + artifactDir: string +): void => { + const evidence = context.postgresRunAttestation; + if (!evidence) return; + const artifactPath = path.join(artifactDir, 'postgres-run-attestation.json'); + if (path.resolve(evidence.artifactPath) !== path.resolve(artifactPath)) { + throw new Error('PostgreSQL run attestation points outside the result artifact'); + } + const raw = parseJsonEvidence(artifactDir, 'postgres-run-attestation.json'); + const normalized = normalizePostgresRunAttestation(raw, { + arm: context.arm, + heapMiB: context.heapMiB, + tenantCount: context.configuredCustomers, + repetition: context.repetition, + runOrderIndex: context.runOrderIndex, + planSha256: context.planSha256, + fleetSha256: context.fleetSha256, + notBeforeEpochMs: context.notBeforeEpochMs, + artifactDir + }, artifactPath); + if (!isDeepStrictEqual(normalized, evidence)) { + throw new Error('PostgreSQL run attestation does not normalize to score-context evidence'); + } +}; + +export const reconstructScoreInput = ( + result: DensityRunResult, + plan: DensityPlanV1, + fleet: FleetV1 +): ScoreInput => { + assertArtifactDirectory(result.artifactDir, plan); + const context = readScoreContextEvidence(result.artifactDir); + assertContextMatchesResult(context, result, plan, fleet); + validatePostgresAttestation(context, result.artifactDir); + const arm = plan.arms.find((candidate) => candidate.name === context.arm); + if (!arm) throw new Error(`score-context.json uses unknown arm '${context.arm}'`); + if (context.externalServer !== !arm.command?.length) { + throw new Error('score-context.json external-server state contradicts the arm plan'); + } + if (context.evidenceMode === 'qualification') { + const configuredDurationSec = context.runKind === 'soak' + ? plan.soak?.durationSec + : plan.workload.durationSec; + if (configuredDurationSec == null || context.configuredDurationSec !== configuredDurationSec) { + throw new Error('score-context.json qualification duration contradicts the plan'); + } + } + if (context.configuredCustomers > fleet.tenants.length) { + throw new Error('score-context.json configured customer count exceeds the fleet'); + } + const memory = parseMemoryEvidence(result.artifactDir); + const postgres = parsePostgresMemoryEvidence(result.artifactDir); + const samples = parseRequests(result.artifactDir); + const canariesRaw = parseJsonEvidence(result.artifactDir, 'canaries.json'); + if (!Array.isArray(canariesRaw)) throw new Error('canaries.json must be an array'); + const canaries = canariesRaw.map((canary, index) => + parseCanaryResult(canary, `canaries.json[${index}]`)); + const canarySchedule = parseCanarySchedule( + parseJsonEvidence(result.artifactDir, 'canary-schedule.json'), + 'canary-schedule.json' + ); + const workload = parseWorkloadProgress(result.artifactDir); + const retainedRaw = requireRecord( + parseJsonEvidence(result.artifactDir, 'retained-memory.json'), + 'retained-memory.json' + ); + requireExactKeys(retainedRaw, ['baseline', 'final', 'errors'], 'retained-memory.json'); + requireStringArray(retainedRaw.errors, 'retained-memory.json errors'); + const retainedMemory: RetainedMemoryCheckpointPair = { + baseline: retainedRaw.baseline == null + ? null + : normalizeRetainedMemoryCheckpoint(retainedRaw.baseline), + final: retainedRaw.final == null + ? null + : normalizeRetainedMemoryCheckpoint(retainedRaw.final), + errors: retainedRaw.errors as string[] + }; + if ( + (retainedRaw.baseline != null && retainedMemory.baseline == null) + || (retainedRaw.final != null && retainedMemory.final == null) + || ( + retainedMemory.baseline != null + && !isDeepStrictEqual(retainedMemory.baseline, retainedRaw.baseline) + ) + || ( + retainedMemory.final != null + && !isDeepStrictEqual(retainedMemory.final, retainedRaw.final) + ) + ) { + throw new Error('retained-memory.json checkpoint shape is invalid'); + } + if ( + workload.samples !== samples.length + || workload.canaries !== canaries.length + || !isDeepStrictEqual(workload.canarySchedule, canarySchedule) + ) { + throw new Error('workload-progress.json counters or canary schedule are inconsistent'); + } + if ( + context.executionErrors.length === 0 + && ( + workload.offeredLoad == null + || workload.resolvedWarmupTimeoutMs == null + || workload.workloadDurationMs == null + ) + ) { + throw new Error('successful run evidence has incomplete workload progress'); + } + const tenants = resolveTenants( + fleet.tenants.slice(0, context.configuredCustomers), + arm + ); + const selectedWorkload = context.evidenceMode === 'diagnostic' + && context.configuredDurationSec === 5 + ? { + ...plan.workload, + durationSec: 5, + ...(plan.workload.rps != null + ? { rps: Math.min(plan.workload.rps, 5), rpsPerTenant: undefined } + : { + rps: undefined, + rpsPerTenant: Math.min( + plan.workload.rpsPerTenant!, + 5 / context.configuredCustomers + ) + }) + } + : { ...plan.workload, durationSec: context.configuredDurationSec }; + const offeredLoad = resolveOfferedLoad(selectedWorkload, context.configuredCustomers); + const surfaceCount = tenants.reduce((sum, tenant) => sum + tenant.surfaces.length, 0); + const resolvedWarmupTimeoutMs = resolveWarmupTimeoutMs(plan.workload, surfaceCount); + if ( + workload.offeredLoad != null + && !isDeepStrictEqual(workload.offeredLoad, offeredLoad) + ) { + throw new Error('workload-progress.json offered load contradicts the plan/fleet'); + } + if ( + workload.resolvedWarmupTimeoutMs != null + && workload.resolvedWarmupTimeoutMs !== resolvedWarmupTimeoutMs + ) { + throw new Error('workload-progress.json warmup timeout contradicts the plan/fleet'); + } + const tenantById = new Map(tenants.map((tenant) => [tenant.id, tenant])); + const warmedSurfaces = new Map>(); + for (const entry of workload.warmedSurfaces) { + const tenant = tenantById.get(entry.tenantId); + if (!tenant) throw new Error(`warmup evidence contains unknown tenant '${entry.tenantId}'`); + const configured = new Set(tenant.surfaces.map((surface) => surface.name)); + if (entry.surfaces.some((surface) => !configured.has(surface))) { + throw new Error(`warmup evidence contains an unknown surface for '${entry.tenantId}'`); + } + warmedSurfaces.set(entry.tenantId, new Set(entry.surfaces)); + } + const planSha256 = sourceSha256(plan); + const fleetSha256 = sourceSha256(fleet); + const realtimeDeliveryCoverage = readRealtimeCoverageEvidence(result.artifactDir); + if (context.executionErrors.length === 0 && realtimeDeliveryCoverage == null) { + throw new Error('successful run evidence has no timed realtime terminal transition'); + } + return { + arm: arm.name, + evidenceMode: context.evidenceMode, + campaignId: context.campaignId, + scheduleSha256: context.scheduleSha256, + previousResultPayloadSha256: context.previousResultPayloadSha256, + qualificationCohortSha256: sha256(`${planSha256}\0${fleetSha256}`), + commit: arm.commit, + introspectionMode: arm.introspectionMode, + heapMiB: context.heapMiB, + repetition: context.repetition, + expectedMatrixRepetitions: plan.repetitions, + runKind: context.runKind, + runOrderSeed: plan.runOrderSeed ?? DEFAULT_RUN_ORDER_SEED, + runOrderIndex: context.runOrderIndex, + startedAt: context.startedAt, + endedAt: context.endedAt, + configuredDurationSec: context.configuredDurationSec, + workloadDurationMs: workload.workloadDurationMs ?? 0, + artifactDir: result.artifactDir, + tenants, + warmedSurfaces, + warmupLatencies: workload.warmupLatencies, + resolvedWarmupTimeoutMs, + offeredLoad, + canaryIntervalSec: plan.workload.canaryIntervalSec, + periodicCanarySchedule: plan.workload.periodicCanarySchedule ?? 'full-sweep', + canarySchedule, + minWorkloadRequestsPerSurface: plan.workload.minWorkloadRequestsPerSurface, + samples, + canaries, + memorySnapshots: memory.snapshots, + postWarmupSnapshots: memory.snapshots.slice(Math.max(0, memory.warmupIndex)), + postWarmupNodeRssSnapshots: memory.osSnapshots.slice( + Math.max(0, memory.osWarmupIndex) + ), + retainedMemory, + memorySampleErrors: memory.errors, + postgresSnapshots: postgres.snapshots, + postgresSampleErrors: postgres.errors, + missedArrivals: samples.filter( + (sample) => sample.errorCode === 'LOAD_GENERATOR_MISSED_ARRIVAL' + ).length, + requiredCapabilities: [...plan.requiredCapabilities], + requiredCanaries: [...plan.requiredCanaries], + gates: plan.gates, + serverExit: context.serverExit, + provenance: context.provenance, + provenanceErrors: context.provenanceErrors, + postgresRunAttestation: context.postgresRunAttestation, + realtimeDeliveryCoverage, + externalServer: context.externalServer, + executionErrors: context.executionErrors + }; +}; + +export const assertResultSemanticReplay = ( + result: DensityRunResult, + plan: DensityPlanV1, + fleet: FleetV1, + label: string +): void => { + let replayed: DensityRunResult; + try { + replayed = scoreRun(reconstructScoreInput(result, plan, fleet)); + } catch (error) { + throw new Error( + `${label} semantic replay failed: ` + + `${error instanceof Error ? error.message : String(error)}` + ); + } + if (!isDeepStrictEqual(resultPayload(result), replayed)) { + throw new Error(`${label} result does not match semantic replay of raw evidence`); + } +}; diff --git a/packages/perf-harness/src/http.ts b/packages/perf-harness/src/http.ts new file mode 100644 index 0000000000..5a0c3be511 --- /dev/null +++ b/packages/perf-harness/src/http.ts @@ -0,0 +1,888 @@ +import { isDeepStrictEqual } from 'node:util'; + +import type { + CanaryRoundSummary, + CanaryResult, + CanaryScheduleSummary, + GraphqlOperation, + GraphqlSurface, + IsolationCanary, + JsonPathInvariant, + JsonPathMatch, + PeriodicCanarySchedule, + RequestSample, + ResolvedOfferedLoad, + TenantTarget, + WorkloadPlan +} from './types'; + +export interface WorkloadResult { + samples: RequestSample[]; + canaries: CanaryResult[]; + canarySchedule: CanaryScheduleSummary; + warmedSurfaces: Map>; + warmupLatencies: number[]; + capabilities: Set; + capabilitiesByTenantSurface: Map>; + missedArrivals: number; + workloadDurationMs: number; + offeredLoad: ResolvedOfferedLoad; + resolvedWarmupTimeoutMs: number; + warmupSurfaceCount: number; + warmupConcurrency: number; +} + +export interface WorkloadCapture { + samples: RequestSample[]; + canaries: CanaryResult[]; + canarySchedule: CanaryScheduleSummary | null; + warmedSurfaces: Map>; + warmupLatencies: number[]; + capabilities: Set; + capabilitiesByTenantSurface: Map>; +} + +export const createWorkloadCapture = (): WorkloadCapture => ({ + samples: [], + canaries: [], + canarySchedule: null, + warmedSurfaces: new Map(), + warmupLatencies: [], + capabilities: new Set(), + capabilitiesByTenantSurface: new Map() +}); + +export const resolveOfferedLoad = ( + plan: Pick, + tenantCount: number +): ResolvedOfferedLoad => { + if (!Number.isSafeInteger(tenantCount) || tenantCount <= 0) { + throw new Error('tenantCount must be a positive safe integer'); + } + const fixed = plan.rps; + const perTenant = plan.rpsPerTenant; + if ((fixed == null) === (perTenant == null)) { + throw new Error('workload must define exactly one of rps or rpsPerTenant'); + } + const configuredRps = fixed ?? perTenant!; + if (!Number.isFinite(configuredRps) || configuredRps <= 0) { + throw new Error('configured workload RPS must be positive'); + } + const totalRps = fixed ?? perTenant! * tenantCount; + if (!Number.isFinite(totalRps) || totalRps <= 0) { + throw new Error('resolved workload RPS must be positive'); + } + return { + mode: fixed == null ? 'per-tenant' : 'fixed-total', + configuredRps, + tenantCount, + totalRps, + rpsPerTenant: fixed == null ? perTenant! : fixed / tenantCount + }; +}; + +export const resolveWarmupTimeoutMs = ( + plan: Pick< + WorkloadPlan, + 'warmupTimeoutMs' | 'warmupTimeoutPerSurfaceMs' | 'warmupConcurrency' + >, + surfaceCount: number +): number => { + if (!Number.isSafeInteger(surfaceCount) || surfaceCount <= 0) { + throw new Error('warmup surface count must be a positive safe integer'); + } + const concurrency = plan.warmupConcurrency ?? 1; + if (!Number.isSafeInteger(concurrency) || concurrency <= 0) { + throw new Error('warmup concurrency must be a positive safe integer'); + } + if (!Number.isFinite(plan.warmupTimeoutMs) || plan.warmupTimeoutMs <= 0) { + throw new Error('warmupTimeoutMs must be positive'); + } + if ( + !Number.isFinite(plan.warmupTimeoutPerSurfaceMs) + || plan.warmupTimeoutPerSurfaceMs <= 0 + ) { + throw new Error('warmupTimeoutPerSurfaceMs must be positive'); + } + const waves = Math.ceil(surfaceCount / concurrency); + return Math.max(plan.warmupTimeoutMs, waves * plan.warmupTimeoutPerSurfaceMs); +}; + +interface GraphqlResponse { + status: number; + latencyMs: number; + body: unknown; + text: string; + ok: boolean; + errorCode?: string; + retryAfterMs: number; + oracleConfigured: boolean; + oracleConclusive: boolean; + oracleViolation: boolean; + oracleUnavailable: boolean; + postCoverageVerification?: boolean; +} + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +const decodePointerSegment = (segment: string): string => + segment.replace(/~1/g, '/').replace(/~0/g, '~'); + +/** Resolve an RFC 6901 pointer; `*` selects every child at that segment. */ +export const jsonPointerValues = (root: unknown, pointer: string): unknown[] => { + if (pointer === '') return [root]; + if (!pointer.startsWith('/')) return []; + let values: unknown[] = [root]; + for (const rawSegment of pointer.slice(1).split('/')) { + const segment = decodePointerSegment(rawSegment); + const next: unknown[] = []; + for (const value of values) { + if (segment === '*') { + if (Array.isArray(value)) next.push(...value); + else if (value && typeof value === 'object') next.push(...Object.values(value)); + continue; + } + if (Array.isArray(value)) { + const index = /^(0|[1-9]\d*)$/.test(segment) ? Number(segment) : -1; + if (index >= 0 && index < value.length) next.push(value[index]); + } else if ( + value && + typeof value === 'object' && + Object.prototype.hasOwnProperty.call(value, segment) + ) { + next.push((value as Record)[segment]); + } + } + values = next; + if (values.length === 0) break; + } + return values; +}; + +const matchesJsonPath = (body: unknown, match: JsonPathMatch): boolean => + jsonPointerValues(body, match.path).some((value) => isDeepStrictEqual(value, match.value)); + +const evaluateInvariant = ( + body: unknown, + invariant: JsonPathInvariant +): 'missing' | 'unexpected' | null => { + const values = jsonPointerValues(body, invariant.path); + if (values.length < invariant.min) return 'missing'; + if (invariant.max != null && values.length > invariant.max) return 'unexpected'; + return values.every((value) => isDeepStrictEqual(value, invariant.everyEquals)) + ? null + : 'unexpected'; +}; + +const applyResponseOracle = ( + response: Omit< + GraphqlResponse, + 'oracleConfigured' | 'oracleConclusive' | 'oracleViolation' | 'oracleUnavailable' + >, + operation: Pick +): GraphqlResponse => { + const configured = operation.requiredMatches != null + || operation.forbiddenMatches != null + || operation.invariants != null; + if (!configured) { + return { + ...response, + oracleConfigured: false, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: false + }; + } + if ( + !operation.requiredMatches?.length + || !operation.forbiddenMatches?.length + || (operation.invariants != null && operation.invariants.length === 0) + ) { + return { + ...response, + ok: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_INVALID', + oracleConfigured: true, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: false + }; + } + const forbidden = operation.forbiddenMatches.find((match) => + matchesJsonPath(response.body, match) + ); + const missing = operation.requiredMatches.find((match) => + !matchesJsonPath(response.body, match) + ); + const unexpectedInvariant = operation.invariants?.find((invariant) => + evaluateInvariant(response.body, invariant) === 'unexpected' + ); + const missingInvariant = operation.invariants?.find((invariant) => + evaluateInvariant(response.body, invariant) === 'missing' + ); + if (forbidden) { + return { + ...response, + ok: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_FORBIDDEN', + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: true, + oracleUnavailable: false + }; + } + if (unexpectedInvariant) { + return { + ...response, + ok: false, + errorCode: 'GRAPHQL_OPERATION_ORACLE_INVARIANT_UNEXPECTED', + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: true, + oracleUnavailable: false + }; + } + if (!response.ok) { + return { + ...response, + oracleConfigured: true, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: true + }; + } + if (missing || missingInvariant) { + return { + ...response, + ok: false, + errorCode: missingInvariant + ? 'GRAPHQL_OPERATION_ORACLE_INVARIANT_MISSING' + : 'GRAPHQL_OPERATION_ORACLE_MISSING', + oracleConfigured: true, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: false + }; + } + return { + ...response, + oracleConfigured: true, + oracleConclusive: true, + oracleViolation: false, + oracleUnavailable: false + }; +}; + +export const mapWithConcurrency = async ( + items: readonly T[], + concurrency: number, + worker: (item: T, index: number) => Promise +): Promise => { + if (items.length === 0) return; + let cursor = 0; + const workers = Array.from({ length: Math.min(items.length, Math.max(1, concurrency)) }, async () => { + while (cursor < items.length) { + const index = cursor++; + await worker(items[index], index); + } + }); + await Promise.all(workers); +}; + +const requestGraphql = async ( + surface: GraphqlSurface, + operation: Pick< + GraphqlOperation, + 'query' | 'variables' | 'requiredMatches' | 'forbiddenMatches' | 'invariants' + >, + timeoutMs: number +): Promise => { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + const startedAt = performance.now(); + try { + const response = await fetch(surface.url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(surface.headers ?? {}) + }, + body: JSON.stringify({ query: operation.query, variables: operation.variables ?? {} }), + signal: controller.signal + }); + const text = await response.text(); + let body: any = null; + try { + body = JSON.parse(text); + } catch { + body = null; + } + const graphqlError = Array.isArray(body?.errors) && body.errors.length > 0; + const errorCode = graphqlError + ? body.errors[0]?.extensions?.code ?? 'GRAPHQL_ERROR' + : undefined; + const retryAfter = Number.parseInt(response.headers.get('retry-after') ?? '', 10); + return applyResponseOracle({ + status: response.status, + latencyMs: performance.now() - startedAt, + body, + text, + ok: response.ok && !graphqlError, + errorCode, + retryAfterMs: Number.isFinite(retryAfter) ? retryAfter * 1000 : 0 + }, operation); + } catch (error) { + return applyResponseOracle({ + status: 0, + latencyMs: performance.now() - startedAt, + body: null, + text: error instanceof Error ? error.message : String(error), + ok: false, + errorCode: error instanceof Error && error.name === 'AbortError' ? 'TIMEOUT' : 'NETWORK_ERROR', + retryAfterMs: 0 + }, operation); + } finally { + clearTimeout(timer); + } +}; + +const warmSurface = async ( + surface: GraphqlSurface, + deadline: number, + requestTimeoutMs: number +): Promise => { + let last: GraphqlResponse | null = null; + while (Date.now() < deadline) { + const remainingMs = Math.max(1, deadline - Date.now()); + last = await requestGraphql( + surface, + surface.warmup, + Math.min(requestTimeoutMs, remainingMs) + ); + if (last.ok) return last; + if (last.status !== 503) return last; + const retryDelayMs = Math.min( + Math.max(100, last.retryAfterMs), + Math.max(0, deadline - Date.now()) + ); + if (retryDelayMs > 0) await sleep(retryDelayMs); + } + return last ?? { + status: 0, + latencyMs: 0, + body: null, + text: 'global warmup deadline elapsed before this surface could start', + ok: false, + errorCode: 'WARMUP_DEADLINE', + retryAfterMs: 0, + oracleConfigured: surface.warmup.requiredMatches != null + || surface.warmup.forbiddenMatches != null + || surface.warmup.invariants != null, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: true + }; +}; + +const runCanary = async ( + tenant: TenantTarget, + surface: GraphqlSurface, + configuredCanaries: readonly IsolationCanary[], + timeoutMs: number, + metadata: { + phase: CanaryResult['phase']; + periodicRound?: number; + scheduledAt: string; + onCheckStarted?: () => void; + } +): Promise => { + const results: CanaryResult[] = []; + // A physical-density surface may have one pool slot permanently leased by + // realtime. Queueing every hostile probe at once behind the remaining slot + // lets the validation rig create head-of-line blocking for customer traffic. + // Submit probes one at a time so workload requests can interleave while the + // exact same fail-closed canary set is still exercised. + for (const canary of configuredCanaries) { + metadata.onCheckStarted?.(); + const startedAt = new Date().toISOString(); + const response = await requestGraphql(surface, canary, timeoutMs); + const completedAt = new Date().toISOString(); + const evidence = { + tenantId: tenant.id, + surface: surface.name, + canary: canary.name, + phase: metadata.phase, + ...(metadata.periodicRound != null + ? { periodicRound: metadata.periodicRound } + : {}), + scheduledAt: metadata.scheduledAt, + startedAt, + completedAt, + latencyMs: response.latencyMs + }; + results.push({ + ...evidence, + conclusive: response.oracleConclusive, + violation: response.oracleViolation, + ...(!response.ok ? { + detail: response.errorCode ?? `HTTP_${response.status}` + } : {}) + }); + } + return results; +}; + +/** + * Timed slots are strictly inside the workload window. A 15-minute workload + * with a 60-second interval therefore has rounds 1..14, never one at t=0 or + * exactly at the deadline (those boundaries belong to the full sweeps). + */ +export const periodicCanaryRoundCount = ( + durationMs: number, + intervalMs: number +): number => { + if (!Number.isFinite(durationMs) || durationMs < 0) { + throw new Error('canary schedule duration must be non-negative'); + } + if (!Number.isFinite(intervalMs) || intervalMs <= 0) { + throw new Error('canary schedule interval must be positive'); + } + return Math.max(0, Math.ceil(durationMs / intervalMs) - 1); +}; + +const stableOffset = (namespace: string, values: readonly string[], count: number): number => { + if (!Number.isSafeInteger(count) || count <= 0) return 0; + let hash = 0x811c9dc5; + for (const character of [namespace, ...values].join('\0')) { + hash ^= character.charCodeAt(0); + hash = Math.imul(hash, 0x01000193); + } + return (hash >>> 0) % count; +}; + +export const deterministicCanaryOffset = ( + tenantId: string, + surfaceName: string, + canaryCount: number +): number => stableOffset('canary', [tenantId, surfaceName], canaryCount); + +/** Return the selected zero-based canary index for a one-based periodic round. */ +export const rotatingCanaryIndex = ( + tenantId: string, + surfaceName: string, + canaryCount: number, + periodicRound: number +): number => { + if (!Number.isSafeInteger(canaryCount) || canaryCount <= 0) { + throw new Error('rotating canary selection requires at least one canary'); + } + if (!Number.isSafeInteger(periodicRound) || periodicRound <= 0) { + throw new Error('periodic canary round must be a positive safe integer'); + } + return ( + deterministicCanaryOffset(tenantId, surfaceName, canaryCount) + + periodicRound - 1 + ) % canaryCount; +}; + +const weightedOperations = (surface: GraphqlSurface): GraphqlOperation[] => { + const expanded: GraphqlOperation[] = []; + for (const operation of surface.operations) { + const weight = Math.max(1, Math.round((operation.weight ?? 1) * 10)); + for (let index = 0; index < weight; index++) expanded.push(operation); + } + return expanded; +}; + +/** + * Give each tenant/surface a reproducible position in its weighted operation + * schedule. Starting every surface at index zero creates fleet-wide operation + * waves that exaggerate one query shape at a time instead of exercising a + * mixed customer workload. + */ +export const deterministicOperationOffset = ( + tenantId: string, + surfaceName: string, + operationCount: number +): number => { + if (!Number.isSafeInteger(operationCount) || operationCount <= 0) return 0; + let hash = 0x811c9dc5; + for (const character of `${tenantId}\0${surfaceName}`) { + hash ^= character.charCodeAt(0); + hash = Math.imul(hash, 0x01000193); + } + return (hash >>> 0) % operationCount; +}; + +export const runWorkload = async ( + tenants: TenantTarget[], + plan: WorkloadPlan, + onWarmBoundary?: () => void | Promise, + capture: WorkloadCapture = createWorkloadCapture() +): Promise => { + const { + warmedSurfaces, + warmupLatencies, + capabilities, + capabilitiesByTenantSurface, + samples, + canaries + } = capture; + const surfaceTargets = tenants.flatMap((tenant) => tenant.surfaces.map((surface) => { + const operations = weightedOperations(surface); + return { + tenant, + surface, + operations, + cursor: deterministicOperationOffset(tenant.id, surface.name, operations.length) + }; + })); + + const warmupConcurrency = plan.warmupConcurrency ?? 1; + const resolvedWarmupTimeoutMs = resolveWarmupTimeoutMs(plan, surfaceTargets.length); + const warmupDeadline = Date.now() + resolvedWarmupTimeoutMs; + await mapWithConcurrency(surfaceTargets, warmupConcurrency, async ({ tenant, surface }) => { + const response = await warmSurface( + surface, + warmupDeadline, + plan.requestTimeoutMs + ); + warmupLatencies.push(response.latencyMs); + if (response.ok) { + const warmed = warmedSurfaces.get(tenant.id) ?? new Set(); + warmed.add(surface.name); + warmedSurfaces.set(tenant.id, warmed); + } + }); + let missedArrivals = 0; + + const recordResponse = ( + target: typeof surfaceTargets[number], + operation: GraphqlOperation, + response: GraphqlResponse, + phase: RequestSample['phase'], + scheduledAtMs?: number + ): void => { + if (response.ok) { + capabilities.add(operation.capability); + const capabilityKey = `${target.tenant.id}/${target.surface.name}`; + const localCapabilities = capabilitiesByTenantSurface.get(capabilityKey) ?? new Set(); + localCapabilities.add(operation.capability); + capabilitiesByTenantSurface.set(capabilityKey, localCapabilities); + } + samples.push({ + tenantId: target.tenant.id, + surface: target.surface.name, + operation: operation.name, + capability: operation.capability, + // Workload latency starts at the scheduled open-loop arrival, not when + // fetch happened to begin. This includes scheduler/event-loop delay and + // prevents coordinated omission from making a saturated arm look fast. + latencyMs: phase === 'workload' && scheduledAtMs != null + ? Math.max(response.latencyMs, performance.now() - scheduledAtMs) + : response.latencyMs, + status: response.status, + ok: response.ok, + phase, + oracleConfigured: response.oracleConfigured, + oracleConclusive: response.oracleConclusive, + oracleViolation: response.oracleViolation, + oracleUnavailable: response.oracleUnavailable, + ...(response.postCoverageVerification + ? { postCoverageVerification: true } + : {}), + ...(scheduledAtMs != null ? { scheduledAtMs } : {}), + ...(!response.ok ? { errorCode: response.errorCode ?? `HTTP_${response.status}` } : {}) + }); + }; + + const coverage = surfaceTargets.flatMap((target) => + target.surface.operations.map((operation) => ({ target, operation })) + ); + await mapWithConcurrency(coverage, warmupConcurrency, async ({ target, operation }) => { + const primary = await requestGraphql( + target.surface, + operation, + plan.requestTimeoutMs + ); + if (!primary.ok || !operation.postCoverageVerification) { + recordResponse(target, operation, primary, 'coverage'); + return; + } + const responseVariables: Record = {}; + for (const [name, pointer] of Object.entries( + operation.postCoverageVerification.variablesFromResponse ?? {} + )) { + const values = jsonPointerValues(primary.body, pointer); + if (values.length !== 1) { + recordResponse(target, operation, { + ...primary, + ok: false, + errorCode: values.length === 0 + ? 'GRAPHQL_POST_COVERAGE_VARIABLE_MISSING' + : 'GRAPHQL_POST_COVERAGE_VARIABLE_AMBIGUOUS', + oracleConfigured: true, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: false, + postCoverageVerification: true + }, 'coverage'); + return; + } + responseVariables[name] = values[0]; + } + const verification = await requestGraphql( + target.surface, + { + ...operation.postCoverageVerification, + variables: { + ...(operation.postCoverageVerification.variables ?? {}), + ...responseVariables + } + }, + plan.requestTimeoutMs + ); + recordResponse(target, operation, { + ...verification, + latencyMs: primary.latencyMs + verification.latencyMs, + postCoverageVerification: true + }, 'coverage'); + }); + + const canaryConcurrency = plan.canaryConcurrency + ?? Math.min(plan.maxInFlight, warmupConcurrency); + const runFullCanarySweep = async ( + phase: 'initial' | 'final' + ): Promise => { + const scheduledAt = new Date().toISOString(); + await mapWithConcurrency( + surfaceTargets, + canaryConcurrency, + async ({ tenant, surface }) => { + canaries.push(...await runCanary( + tenant, + surface, + surface.canaries, + plan.requestTimeoutMs, + { phase, scheduledAt } + )); + } + ); + }; + await runFullCanarySweep('initial'); + // The warm boundary is the first point at which every resident surface has + // been built, every configured capability has been exercised, and the + // initial isolation sweep has completed. The caller may perform additional + // awaited setup (for example, establishing realtime transports) before it + // marks memory warm. A callback failure must prevent timed traffic. + await onWarmBoundary?.(); + + const startedAt = performance.now(); + const startedWallMs = Date.now(); + const durationMs = plan.durationSec * 1000; + const deadline = startedAt + durationMs; + const deadlineWallMs = startedWallMs + durationMs; + const canaryIntervalMs = plan.canaryIntervalSec * 1000; + const periodicSchedule: PeriodicCanarySchedule = + plan.periodicCanarySchedule ?? 'full-sweep'; + const plannedPeriodicRounds = periodicCanaryRoundCount( + durationMs, + canaryIntervalMs + ); + const checksPerFullSweep = surfaceTargets.reduce( + (sum, target) => sum + target.surface.canaries.length, + 0 + ); + const checksPerRound = periodicSchedule === 'rotating-one' + ? surfaceTargets.length + : checksPerFullSweep; + const canarySchedule: CanaryScheduleSummary = { + schedule: periodicSchedule, + intervalMs: canaryIntervalMs, + durationMs, + canaryConcurrency, + startedAt: new Date(startedWallMs).toISOString(), + deadlineAt: new Date(deadlineWallMs).toISOString(), + planned: plannedPeriodicRounds, + started: 0, + completed: 0, + missed: plannedPeriodicRounds, + overlapped: 0, + deadlineLate: 0, + checksPlanned: plannedPeriodicRounds * checksPerRound, + checksStarted: 0, + checksCompleted: 0, + rounds: Array.from({ length: plannedPeriodicRounds }, (_unused, index): CanaryRoundSummary => ({ + periodicRound: index + 1, + plannedAt: new Date(startedWallMs + (index + 1) * canaryIntervalMs).toISOString(), + startedAt: null, + completedAt: null, + targetsPlanned: surfaceTargets.length, + targetsStarted: 0, + targetsCompleted: 0, + checksPlanned: checksPerRound, + checksStarted: 0, + checksCompleted: 0, + overlapped: false, + deadlineLate: false, + startDelayMs: null, + durationMs: null + })) + }; + capture.canarySchedule = canarySchedule; + + // This is a finite serialized schedule, not a setInterval callback. If a + // round overlaps the next slot it is recorded and drained before the next + // round starts; no validation round disappears behind a boolean guard. Each + // request has requestTimeoutMs, so the finite set of planned probes also + // gives the post-deadline drain a deterministic upper bound. + const periodicCanaries = (async (): Promise => { + let previousCompletedAt = startedAt; + for (const round of canarySchedule.rounds) { + const plannedAt = startedAt + round.periodicRound * canaryIntervalMs; + const waitMs = plannedAt - performance.now(); + if (waitMs > 0) await sleep(waitMs); + const roundStartedAt = performance.now(); + round.overlapped = round.periodicRound > 1 && previousCompletedAt > plannedAt; + round.startedAt = new Date().toISOString(); + round.startDelayMs = Math.max(0, roundStartedAt - plannedAt); + canarySchedule.started++; + if (round.overlapped) canarySchedule.overlapped++; + + await mapWithConcurrency( + surfaceTargets, + canaryConcurrency, + async ({ tenant, surface }) => { + round.targetsStarted++; + const selectedCanaries = periodicSchedule === 'rotating-one' + ? [surface.canaries[rotatingCanaryIndex( + tenant.id, + surface.name, + surface.canaries.length, + round.periodicRound + )]] + : surface.canaries; + const results = await runCanary( + tenant, + surface, + selectedCanaries, + plan.requestTimeoutMs, + { + phase: 'periodic', + periodicRound: round.periodicRound, + scheduledAt: round.plannedAt, + onCheckStarted: () => { + round.checksStarted++; + canarySchedule.checksStarted++; + } + } + ); + canaries.push(...results); + round.checksCompleted += results.length; + canarySchedule.checksCompleted += results.length; + round.targetsCompleted++; + } + ); + + previousCompletedAt = performance.now(); + round.completedAt = new Date().toISOString(); + round.durationMs = previousCompletedAt - roundStartedAt; + round.deadlineLate = previousCompletedAt > deadline; + canarySchedule.completed++; + canarySchedule.missed = canarySchedule.planned - canarySchedule.completed; + if (round.deadlineLate) canarySchedule.deadlineLate++; + } + })(); + + const offeredLoad = resolveOfferedLoad(plan, tenants.length); + const intervalMs = 1000 / offeredLoad.totalRps; + let nextAt = startedAt; + let sequence = 0; + const inFlight = new Set>(); + + const nextScheduledOperation = (): { + target: typeof surfaceTargets[number]; + operation: GraphqlOperation; + } => { + const target = surfaceTargets[sequence % surfaceTargets.length]; + sequence++; + const operation = target.operations[target.cursor % target.operations.length]; + target.cursor++; + return { target, operation }; + }; + + const dispatch = ( + target: typeof surfaceTargets[number], + operation: GraphqlOperation, + scheduledAtMs: number + ): void => { + const pending = requestGraphql(target.surface, operation, plan.requestTimeoutMs) + .then((response) => recordResponse( + target, + operation, + response, + 'workload', + scheduledAtMs + )) + .finally(() => inFlight.delete(pending)); + inFlight.add(pending); + }; + + while (performance.now() < deadline) { + const now = performance.now(); + if (now >= nextAt) { + // Advance directly to the next future arrival. Overdue arrivals become + // explicit failed samples, so saturation/event-loop stalls cannot hide + // latency through coordinated omission and never trigger a catch-up burst. + const due = Math.floor((now - nextAt) / intervalMs) + 1; + const canDispatchLatest = inFlight.size < plan.maxInFlight; + for (let slot = 0; slot < due; slot++) { + const scheduledAtMs = nextAt + slot * intervalMs; + const { target, operation } = nextScheduledOperation(); + if (canDispatchLatest && slot === due - 1) { + dispatch(target, operation, scheduledAtMs); + continue; + } + missedArrivals++; + recordResponse(target, operation, { + status: 0, + latencyMs: Math.max(plan.requestTimeoutMs, now - scheduledAtMs), + body: null, + text: 'scheduled arrival missed before dispatch', + ok: false, + errorCode: 'LOAD_GENERATOR_MISSED_ARRIVAL', + retryAfterMs: 0, + oracleConfigured: operation.requiredMatches != null + || operation.forbiddenMatches != null + || operation.invariants != null, + oracleConclusive: false, + oracleViolation: false, + oracleUnavailable: true + }, 'workload', scheduledAtMs); + } + nextAt += due * intervalMs; + continue; + } + await sleep(Math.min(20, Math.max(1, nextAt - now))); + } + const workloadDurationMs = performance.now() - startedAt; + await Promise.all(inFlight); + await periodicCanaries; + await runFullCanarySweep('final'); + + return { + samples, + canaries, + canarySchedule, + warmedSurfaces, + warmupLatencies, + capabilities, + capabilitiesByTenantSurface, + missedArrivals, + workloadDurationMs, + offeredLoad, + resolvedWarmupTimeoutMs, + warmupSurfaceCount: surfaceTargets.length, + warmupConcurrency + }; +}; diff --git a/packages/perf-harness/src/index.ts b/packages/perf-harness/src/index.ts new file mode 100644 index 0000000000..8b7aacf941 --- /dev/null +++ b/packages/perf-harness/src/index.ts @@ -0,0 +1,127 @@ +#!/usr/bin/env node +import { runCatalogBench, runCatalogBenchWorker } from './catalog-bench'; +import { loadFleet, loadPlan, validateCoverage } from './config'; +import { writeReport } from './report'; +import { runDensityPlan } from './run'; + +const parseList = (value: string | undefined): string[] | undefined => value + ? value.split(',').map((item) => item.trim()).filter(Boolean) + : undefined; + +const parseNumbers = (value: string | undefined): number[] | undefined => parseList(value)?.map((item) => { + const parsed = Number(item); + if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`invalid positive integer '${item}'`); + return parsed; +}); + +const parsePositiveInteger = (value: string | undefined, label: string): number | undefined => { + if (value == null) return undefined; + const parsed = Number.parseInt(value, 10); + if (!Number.isInteger(parsed) || parsed <= 0 || String(parsed) !== value) { + throw new Error(`${label} must be a positive integer`); + } + return parsed; +}; + +const flag = (name: string): string | undefined => { + const index = process.argv.indexOf(`--${name}`); + return index >= 0 ? process.argv[index + 1] : undefined; +}; + +const hasFlag = (name: string): boolean => process.argv.includes(`--${name}`); + +const usage = (): void => { + process.stdout.write(`cperf — local Graphile tenant-density research harness + + cperf validate --plan [--allow-reserved-ports] + cperf run --plan [--arm a,b] [--heaps 1024,2048] [--tenants 1,5] [--repetitions 3] [--smoke] + cperf report --plan --results --out + cperf catalog-bench --database --mode stock|scoped-required --schemas a,b --instances 1,2 --out + cperf catalog-bench --database --mode stock|scoped-required --surface-schemas a,b --allowed-dependency-schemas deps,private --instances 1 --out + [--scoped-catalog-types all|dependency-closure] + [--introspection-client-release-mode reuse|destroy] + [--release-build-state-after-validation] + [--v8-profile stock|optimize-for-size|baseline-optimize-for-size|jitless-optimize-for-size] + [--warm-operations-per-instance 500] [--expected-tokens token-a,token-b] + [--warm-operation-replay-passes 3] + [--grafast-query-cache-max 8] [--grafast-operations-cache-max 8] + [--grafast-operation-plans-cache-max 8] + [--tenant-proxy-surfaces 5] + +Full runs honor the plan's 15-minute matrix and optional two-hour soak. --smoke +forces one five-second run and can never satisfy the qualification gates. +`); +}; + +const main = async (): Promise => { + const command = process.argv[2]; + if (command === '__catalog-worker') { + await runCatalogBenchWorker( + requireFlagForWorker('config'), + requireFlagForWorker('result') + ); + return 0; + } + if (command === 'catalog-bench') { + await runCatalogBench(process.argv.slice(3)); + return 0; + } + const planFile = flag('plan'); + if (!command || !planFile || hasFlag('help')) { + usage(); + return command && hasFlag('help') ? 0 : 1; + } + const plan = loadPlan(planFile, hasFlag('allow-reserved-ports')); + const fleet = loadFleet(plan.fleetFile); + validateCoverage(plan, fleet); + if (command === 'validate') { + process.stdout.write( + `valid plan: ${plan.arms.length} arms, ${fleet.tenants.length} tenants, ` + + `${plan.heapMiB.length} heaps, ${plan.repetitions} repetitions\n` + ); + return 0; + } + if (command === 'run') { + await runDensityPlan(plan, fleet, { + arms: parseList(flag('arm')), + heaps: parseNumbers(flag('heaps')), + tenantCounts: parseNumbers(flag('tenants')), + repetitions: parsePositiveInteger(flag('repetitions'), 'repetitions'), + smoke: hasFlag('smoke') + }); + return 0; + } + if (command === 'report') { + const results = flag('results'); + const output = flag('out'); + if (!results || !output) throw new Error('report requires --results and --out'); + writeReport(results, output, plan, fleet); + return 0; + } + usage(); + return 1; +}; + +const requireFlagForWorker = (name: string): string => { + const value = flag(name); + if (!value) throw new Error(`catalog worker requires --${name}`); + return value; +}; + +void main().then((code) => { + process.exitCode = code; +}, (error) => { + process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`); + process.exitCode = 1; +}); + +export * from './catalog-bench'; +export * from './config'; +export * from './http'; +export * from './memory'; +export * from './postgres'; +export * from './report'; +export * from './run'; +export * from './run-attestation'; +export * from './score'; +export * from './types'; diff --git a/packages/perf-harness/src/memory.ts b/packages/perf-harness/src/memory.ts new file mode 100644 index 0000000000..7b95ddbbd2 --- /dev/null +++ b/packages/perf-harness/src/memory.ts @@ -0,0 +1,650 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import type { + MemorySnapshot, + NodeRssSnapshot, + RetainedMemoryCheckpoint, + RetainedMemoryGuard, + RetainedMemorySample +} from './types'; + +const finiteNumber = (value: unknown): number | null => typeof value === 'number' + && Number.isFinite(value) + ? value + : null; + +const positiveNumber = (value: unknown): number | null => { + const parsed = finiteNumber(value); + return parsed != null && parsed > 0 ? parsed : null; +}; + +const sumNumbers = (value: unknown): number | null => { + if (typeof value === 'number' && Number.isFinite(value)) return value; + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const numbers = Object.values(value).map(sumNumbers); + return numbers.every((item): item is number => item != null) + ? numbers.reduce((sum, item) => sum + item, 0) + : null; +}; + +const stringArray = (value: unknown): string[] | null => Array.isArray(value) + && value.every((item) => typeof item === 'string') + ? [...value] + : null; + +const booleanValue = (value: unknown): boolean | null => typeof value === 'boolean' + ? value + : null; + +const cacheAdmissionMode = ( + value: unknown +): MemorySnapshot['cacheAdmissionMode'] => value === 'evict-idle' + || value === 'preserve-resident' + ? value + : null; + +const realtimeNotificationMode = ( + value: unknown +): MemorySnapshot['realtimeNotificationMode'] => value === 'dedicated' + || value === 'shared-exact' + ? value + : null; + +const runtimePoolTelemetryScope = ( + value: unknown +): MemorySnapshot['runtimePoolTelemetryScope'] => + value === 'runtime-only-exact-identities' ? value : null; + +const maxUsesValue = (value: unknown): number | null => + Number.isSafeInteger(value) && (value as number) > 0 ? value as number : null; + +/** Convert Node's process.resourceUsage().maxRSS KiB value to bytes. */ +const resourcePeakRssBytes = (raw: any): number | null => { + const maxRssKiB = positiveNumber(raw?.resourceUsage?.maxRSS); + return maxRssKiB == null ? null : maxRssKiB * 1024; +}; + +export const normalizeMemorySnapshot = (raw: any): MemorySnapshot => ({ + timestamp: typeof raw?.timestamp === 'string' ? raw.timestamp : new Date().toISOString(), + pid: Number.isSafeInteger(raw?.pid) && raw.pid > 0 ? raw.pid : null, + nodeEnv: typeof raw?.nodeEnv === 'string' ? raw.nodeEnv : null, + heapLimitBytes: positiveNumber(raw?.v8?.heapStatistics?.heap_size_limit), + heapUsedBytes: finiteNumber(raw?.memory?.heapUsedBytes), + rssBytes: positiveNumber(raw?.memory?.rssBytes), + processPeakRssBytes: resourcePeakRssBytes(raw), + cacheSize: finiteNumber(raw?.graphileCache?.size), + cacheConfiguredMax: finiteNumber(raw?.graphileCache?.max), + cacheBudgetCapacity: finiteNumber(raw?.graphileCache?.budgetCapacity), + cacheInstanceHeapBytes: finiteNumber(raw?.graphileCache?.instanceHeapBytes), + cacheCalibrationId: typeof raw?.graphileCache?.calibration?.id === 'string' + ? raw.graphileCache.calibration.id + : null, + cacheAdmissionMode: cacheAdmissionMode(raw?.graphileCache?.admissionMode), + residentBuildContractFingerprints: stringArray( + raw?.physicalDatabaseFixture?.contractEvidence + ?.residentGraphileBuildFingerprints + ), + residentBuildContracts: stringArray(raw?.graphileCache?.keys), + evictions: sumNumbers(raw?.graphileCacheCounters?.evictions), + buildRefusals: sumNumbers(raw?.graphileCacheCounters?.buildRefusals), + buildsStarted: finiteNumber(raw?.graphileGovernor?.buildsStarted + ?? raw?.graphileBuilds?.started), + buildsSucceeded: finiteNumber(raw?.graphileBuilds?.succeeded), + buildMaxMs: finiteNumber(raw?.graphileBuilds?.maxMs), + pgPoolCacheSize: finiteNumber(raw?.pgCache?.size), + pgPoolLeasedPools: finiteNumber(raw?.pgCache?.leasedPools), + pgPoolActiveLeases: finiteNumber(raw?.pgCache?.activeLeases), + pgPoolCapacityEvictions: finiteNumber(raw?.pgCache?.capacityEvictions), + pgPoolCapacityRefusals: finiteNumber(raw?.pgCache?.capacityRefusals), + pgPoolDisposalFailures: finiteNumber(raw?.pgCache?.disposalFailures), + pgPoolTotalClients: finiteNumber( + raw?.pgCache?.totalClients + ?? raw?.physicalDatabaseFixture?.pools?.totalClients + ), + pgPoolIdleClients: finiteNumber( + raw?.pgCache?.idleClients + ?? raw?.physicalDatabaseFixture?.pools?.idleClients + ), + pgPoolWaitingClients: finiteNumber( + raw?.pgCache?.waitingClients + ?? raw?.physicalDatabaseFixture?.pools?.waitingClients + ), + runtimePoolTelemetryScope: runtimePoolTelemetryScope( + raw?.physicalDatabaseFixture?.pools?.scope + ), + runtimePoolTelemetryAvailable: booleanValue( + raw?.physicalDatabaseFixture?.pools?.available + ), + runtimePoolRequestedMaxUses: maxUsesValue( + raw?.physicalDatabaseFixture?.pools?.requestedMaxUses + ), + runtimePoolEffectiveMaxUses: maxUsesValue( + raw?.physicalDatabaseFixture?.pools?.effectiveMaxUses + ), + runtimePoolEffectiveMaxUsesKnown: booleanValue( + raw?.physicalDatabaseFixture?.pools?.effectiveMaxUsesKnown + ), + runtimePoolMaxUsesExact: booleanValue( + raw?.physicalDatabaseFixture?.pools?.maxUsesExact + ), + runtimePoolExpectedPools: finiteNumber( + raw?.physicalDatabaseFixture?.pools?.expectedPools + ), + runtimePoolObservedPools: finiteNumber( + raw?.physicalDatabaseFixture?.pools?.observedPools + ), + runtimePoolTotalClients: finiteNumber( + raw?.physicalDatabaseFixture?.pools?.totalClients + ), + runtimePoolIdleClients: finiteNumber( + raw?.physicalDatabaseFixture?.pools?.idleClients + ), + runtimePoolWaitingClients: finiteNumber( + raw?.physicalDatabaseFixture?.pools?.waitingClients + ), + postgresBackendTotal: finiteNumber(raw?.physicalDatabaseFixture?.backends?.total), + postgresBackendActive: finiteNumber(raw?.physicalDatabaseFixture?.backends?.active), + postgresBackendIdle: finiteNumber(raw?.physicalDatabaseFixture?.backends?.idle), + postgresBackendIdleInTransaction: finiteNumber( + raw?.physicalDatabaseFixture?.backends?.idleInTransaction + ), + physicalDatabases: finiteNumber(raw?.physicalDatabaseFixture?.physicalDatabases), + postgresContainerDedicated: booleanValue( + raw?.physicalDatabaseFixture?.containerScope?.dedicated + ), + unexpectedPostgresDatabases: finiteNumber( + raw?.physicalDatabaseFixture?.containerScope?.unexpectedDatabases + ), + realtimeManagersExpected: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.managersExpected + ), + realtimeManagersActive: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.managersActive + ), + realtimeTransportsExpected: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.transportsExpected + ), + realtimeTransportsActive: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.transportsActive + ), + realtimeNotificationMode: realtimeNotificationMode( + raw?.physicalDatabaseFixture?.realtime?.notificationMode + ), + notificationBrokers: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.brokers + ), + notificationListenerConnections: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.listenerConnections + ), + notificationBrokerLeases: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.leases + ), + notificationBrokerTopics: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.topics + ), + notificationBrokerSubscribers: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.subscribers + ), + notificationBrokerQueueOverflows: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.queueOverflows + ), + notificationBrokerFatalFailures: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationBrokers?.fatalFailures + ), + notificationAuditIdentities: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.identities + ), + notificationAuditsHealthy: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.healthy + ), + notificationAuditsFailed: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.failed + ), + notificationAuditsStale: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.stale + ), + notificationAuditAttempts: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.catalogAuditAttempts + ), + notificationAuditFailures: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.catalogAuditFailures + ), + notificationAuditActiveDatabaseTargets: finiteNumber( + raw?.physicalDatabaseFixture?.realtime?.notificationRoleAudits?.activeDatabaseTargets + ), + notificationAuditDatabaseConflicts: finiteNumber( + raw?.physicalDatabaseFixture?.realtime + ?.notificationRoleAudits?.databaseConfigurationConflicts + ), + cacheCountersAvailable: sumNumbers(raw?.graphileCacheCounters?.evictions) != null + && sumNumbers(raw?.graphileCacheCounters?.buildRefusals) != null, + buildCountersAvailable: finiteNumber( + raw?.graphileGovernor?.buildsStarted ?? raw?.graphileBuilds?.started + ) != null, + raw +}); + +const normalizeRetainedMemorySample = (raw: any): RetainedMemorySample | null => { + const heapUsedBytes = finiteNumber(raw?.heapUsedBytes); + const externalBytes = finiteNumber(raw?.externalBytes); + const arrayBuffersBytes = finiteNumber(raw?.arrayBuffersBytes); + const rssBytes = positiveNumber(raw?.rssBytes); + if ( + typeof raw?.timestamp !== 'string' + || typeof raw?.monotonicNs !== 'string' + || !/^\d+$/.test(raw.monotonicNs) + || heapUsedBytes == null + || heapUsedBytes < 0 + || externalBytes == null + || externalBytes < 0 + || arrayBuffersBytes == null + || arrayBuffersBytes < 0 + || rssBytes == null + ) return null; + return { + timestamp: raw.timestamp, + monotonicNs: raw.monotonicNs, + heapUsedBytes, + externalBytes, + arrayBuffersBytes, + rssBytes + }; +}; + +const normalizeRetainedMemoryGuard = (raw: any): RetainedMemoryGuard | null => { + if ( + !Number.isSafeInteger(raw?.pid) + || raw.pid <= 0 + || !Number.isSafeInteger(raw?.graphileInFlight) + || raw.graphileInFlight < 0 + || !Array.isArray(raw?.residentBuildContracts) + || raw.residentBuildContracts.some((value: unknown) => typeof value !== 'string') + || typeof raw?.stateSha256 !== 'string' + || !/^sha256:[a-f0-9]{64}$/.test(raw.stateSha256) + || !raw?.state + || typeof raw.state !== 'object' + || Array.isArray(raw.state) + ) return null; + return { + pid: raw.pid, + graphileInFlight: raw.graphileInFlight, + residentBuildContracts: [...raw.residentBuildContracts], + stateSha256: raw.stateSha256, + state: raw.state + }; +}; + +export const normalizeRetainedMemoryCheckpoint = ( + raw: any +): RetainedMemoryCheckpoint | null => { + const samples: Array = Array.isArray(raw?.samples) + ? raw.samples.map(normalizeRetainedMemorySample) + : []; + const guardBefore = normalizeRetainedMemoryGuard(raw?.guardBefore); + const guardAfter = normalizeRetainedMemoryGuard(raw?.guardAfter); + if ( + raw?.version !== 1 + || typeof raw?.fixture !== 'string' + || !Number.isSafeInteger(raw?.pid) + || raw.pid <= 0 + || !Number.isSafeInteger(raw?.gcRounds) + || raw.gcRounds < 5 + || raw.gcRounds > 8 + || !Number.isSafeInteger(raw?.stableSampleCount) + || raw.stableSampleCount !== 3 + || typeof raw?.stable !== 'boolean' + || samples.length !== raw.gcRounds + || samples.some((sample) => sample == null) + || !guardBefore + || !guardAfter + || !Array.isArray(raw?.errors) + || raw.errors.some((error: unknown) => typeof error !== 'string') + ) return null; + return { + version: 1, + fixture: raw.fixture, + pid: raw.pid, + gcRounds: raw.gcRounds, + stableSampleCount: 3, + stable: raw.stable, + samples: samples as RetainedMemorySample[], + guardBefore, + guardAfter, + errors: [...raw.errors] + }; +}; + +export interface LinuxProcessMemory { + rssBytes: number | null; + peakRssBytes: number | null; +} + +const statusKiB = (status: string, field: 'VmRSS' | 'VmHWM'): number | null => { + const match = new RegExp(`^${field}:\\s+(\\d+)\\s+kB$`, 'm').exec(status); + return match ? Number(match[1]) * 1024 : null; +}; + +/** Read current and cumulative peak RSS for one exact Linux process. */ +export const readLinuxProcessMemory = ( + pid: number, + procRoot = '/proc', + onError?: (message: string) => void +): LinuxProcessMemory | null => { + try { + const status = fs.readFileSync(path.join(procRoot, String(pid), 'status'), 'utf8'); + const rssBytes = statusKiB(status, 'VmRSS'); + const peakRssBytes = statusKiB(status, 'VmHWM'); + if (rssBytes == null) onError?.(`OS RSS proc status for pid ${pid} omitted VmRSS`); + if (peakRssBytes == null) onError?.(`OS RSS proc status for pid ${pid} omitted VmHWM`); + if (rssBytes == null && peakRssBytes == null) return null; + return { rssBytes, peakRssBytes }; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + onError?.(`OS RSS proc read failed for pid ${pid}: ${detail}`); + return null; + } +}; + +export interface MemorySamplerOptions { + intervalMs?: number; + osSampleIntervalMs?: number; + expectedPid?: number | null; + expectedHeapLimitBytes?: number | null; + procRoot?: string; + /** Auto uses /proc on Linux and the authenticated memory endpoint elsewhere. */ + currentRssSource?: 'auto' | 'proc' | 'authenticated-endpoint'; + /** Ephemeral request headers; callers must never persist bearer credentials. */ + headers?: Readonly>; +} + +export interface MemorySampler { + snapshots: MemorySnapshot[]; + /** High-frequency, harness-timestamped current RSS samples. */ + osSnapshots: NodeRssSnapshot[]; + errors: string[]; + ready: Promise; + markWarmupComplete(): Promise; + stop(): Promise; + warmupIndex: number; + osWarmupIndex: number; + osPeakRssBytes: number | null; +} + +const samplerOptions = (options: number | MemorySamplerOptions): Required> & Pick & { + currentRssSource: 'proc' | 'authenticated-endpoint'; + headers: Readonly>; +} => { + if (typeof options === 'number') { + const currentRssSource = process.platform === 'linux' + ? 'proc' + : 'authenticated-endpoint'; + return { + intervalMs: options, + osSampleIntervalMs: Math.min( + currentRssSource === 'proc' ? 100 : 250, + options + ), + expectedPid: null, + expectedHeapLimitBytes: null, + procRoot: '/proc', + currentRssSource, + headers: Object.freeze({}) + }; + } + const requestedSource = options.currentRssSource ?? 'auto'; + if (!['auto', 'proc', 'authenticated-endpoint'].includes(requestedSource)) { + throw new Error(`unknown current RSS source '${String(requestedSource)}'`); + } + const currentRssSource = requestedSource === 'auto' + ? (options.procRoot != null || process.platform === 'linux' + ? 'proc' + : 'authenticated-endpoint') + : requestedSource; + return { + intervalMs: options.intervalMs ?? 1000, + osSampleIntervalMs: options.osSampleIntervalMs + ?? (currentRssSource === 'proc' ? 100 : 250), + expectedPid: options.expectedPid ?? null, + expectedHeapLimitBytes: options.expectedHeapLimitBytes ?? null, + procRoot: options.procRoot ?? '/proc', + currentRssSource, + headers: Object.freeze({ ...(options.headers ?? {}) }) + }; +}; + +const hasBearerAuthorization = ( + headers: Readonly> +): boolean => Object.entries(headers).some(([name, value]) => + name.toLowerCase() === 'authorization' && /^Bearer\s+\S+$/.test(value) +); + +export const startMemorySampler = ( + url: string, + options: number | MemorySamplerOptions = {} +): MemorySampler => { + const resolved = samplerOptions(options); + if (!Number.isFinite(resolved.intervalMs) || resolved.intervalMs <= 0) { + throw new Error(`memory sample interval must be positive, received ${resolved.intervalMs}`); + } + if (!Number.isFinite(resolved.osSampleIntervalMs) || resolved.osSampleIntervalMs <= 0) { + throw new Error(`OS memory sample interval must be positive, received ${resolved.osSampleIntervalMs}`); + } + + const snapshots: MemorySnapshot[] = []; + const osSnapshots: NodeRssSnapshot[] = []; + const errors: string[] = []; + const observedErrors = new Set(); + let stopped = false; + let inFlight: Promise | null = null; + let osInFlight: Promise | null = null; + let warmupIndex = -1; + let osWarmupIndex = -1; + let osPeakRssBytes: number | null = null; + + const recordError = (message: string): void => { + if (observedErrors.has(message)) return; + observedErrors.add(message); + errors.push(message); + }; + + const validateIdentity = (snapshot: MemorySnapshot): void => { + if (resolved.expectedPid == null) recordError('expected server pid is unavailable'); + else if (snapshot.pid == null) recordError('memory endpoint pid is unavailable'); + else if (snapshot.pid !== resolved.expectedPid) { + recordError(`memory endpoint pid mismatch: expected ${resolved.expectedPid}, observed ${snapshot.pid}`); + } + if (snapshot.nodeEnv !== 'production') { + recordError(`memory endpoint NODE_ENV must be production, observed ${snapshot.nodeEnv ?? 'unknown'}`); + } + if (resolved.expectedHeapLimitBytes == null) recordError('expected V8 heap limit is unavailable'); + else if (snapshot.heapLimitBytes == null) recordError('memory endpoint V8 heap limit is unavailable'); + else if (snapshot.heapLimitBytes !== resolved.expectedHeapLimitBytes) { + recordError( + `V8 heap limit mismatch: expected ${resolved.expectedHeapLimitBytes}, observed ${snapshot.heapLimitBytes}` + ); + } + }; + + const validate = (snapshot: MemorySnapshot): void => { + validateIdentity(snapshot); + if (snapshot.heapUsedBytes == null) recordError('memory endpoint heap usage is unavailable'); + if (snapshot.rssBytes == null) recordError('memory endpoint RSS is unavailable'); + if (snapshot.processPeakRssBytes == null && osPeakRssBytes == null) { + recordError('process peak RSS is unavailable'); + } + if ( + snapshot.pgPoolCacheSize == null + || snapshot.pgPoolLeasedPools == null + || snapshot.pgPoolActiveLeases == null + || snapshot.pgPoolCapacityEvictions == null + || snapshot.pgPoolCapacityRefusals == null + || snapshot.pgPoolDisposalFailures == null + ) { + recordError('PostgreSQL pool-cache telemetry is unavailable'); + } + }; + + const sampleOs = async (): Promise => { + if (resolved.expectedPid == null) return; + if (resolved.currentRssSource === 'authenticated-endpoint') { + if (!hasBearerAuthorization(resolved.headers)) { + recordError('authenticated memory-endpoint RSS sampling requires bearer authorization'); + return; + } + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5_000); + const startedAtMs = Date.now(); + try { + const response = await fetch(url, { + signal: controller.signal, + headers: resolved.headers + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const snapshot = normalizeMemorySnapshot(await response.json()); + const endedAtMs = Date.now(); + validateIdentity(snapshot); + if (snapshot.pid !== resolved.expectedPid || snapshot.rssBytes == null) { + recordError('authenticated memory-endpoint current RSS sample is unavailable'); + return; + } + osSnapshots.push({ + timestamp: new Date( + startedAtMs + ((endedAtMs - startedAtMs) / 2) + ).toISOString(), + pid: resolved.expectedPid, + source: 'authenticated-endpoint', + rssBytes: snapshot.rssBytes + }); + if (snapshot.processPeakRssBytes != null) { + osPeakRssBytes = Math.max( + osPeakRssBytes ?? 0, + snapshot.processPeakRssBytes + ); + } + } catch (error) { + recordError( + `authenticated memory-endpoint RSS sample failed: ${error instanceof Error ? error.message : String(error)}` + ); + } finally { + clearTimeout(timeout); + } + return; + } + const startedAtMs = Date.now(); + const processMemory = readLinuxProcessMemory( + resolved.expectedPid, + resolved.procRoot, + recordError + ); + const endedAtMs = Date.now(); + if (processMemory?.rssBytes != null) { + osSnapshots.push({ + timestamp: new Date(startedAtMs + ((endedAtMs - startedAtMs) / 2)).toISOString(), + pid: resolved.expectedPid, + source: 'proc', + rssBytes: processMemory.rssBytes + }); + } + if (processMemory?.peakRssBytes != null) { + osPeakRssBytes = Math.max(osPeakRssBytes ?? 0, processMemory.peakRssBytes); + } else if (processMemory?.rssBytes != null) { + osPeakRssBytes = Math.max(osPeakRssBytes ?? 0, processMemory.rssBytes); + } + const lastSnapshot = snapshots[snapshots.length - 1]; + if (lastSnapshot && osPeakRssBytes != null) { + lastSnapshot.processPeakRssBytes = Math.max( + lastSnapshot.processPeakRssBytes ?? 0, + osPeakRssBytes + ); + } + }; + + const runOsSample = (): Promise => { + if (osInFlight) return osInFlight; + let pending: Promise; + pending = sampleOs().finally(() => { + if (osInFlight === pending) osInFlight = null; + }); + osInFlight = pending; + return pending; + }; + + const sample = async (): Promise => { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5_000); + try { + await runOsSample(); + const response = await fetch(url, { + signal: controller.signal, + headers: resolved.headers + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const snapshot = normalizeMemorySnapshot(await response.json()); + if (osPeakRssBytes != null) { + snapshot.processPeakRssBytes = Math.max(snapshot.processPeakRssBytes ?? 0, osPeakRssBytes); + } + validate(snapshot); + snapshots.push(snapshot); + } catch (error) { + recordError(error instanceof Error ? error.message : String(error)); + } finally { + clearTimeout(timeout); + } + }; + + const runSample = (): Promise => { + if (inFlight) return inFlight; + let pending: Promise; + pending = sample().finally(() => { + if (inFlight === pending) inFlight = null; + }); + inFlight = pending; + return pending; + }; + + const timer = setInterval(() => { + void runSample(); + }, resolved.intervalMs); + const osTimer = setInterval(() => { + void runOsSample(); + }, resolved.osSampleIntervalMs); + const ready = runSample(); + + return { + snapshots, + osSnapshots, + errors, + ready, + get warmupIndex() { + return warmupIndex; + }, + get osWarmupIndex() { + return osWarmupIndex; + }, + get osPeakRssBytes() { + return osPeakRssBytes; + }, + async markWarmupComplete(): Promise { + if (inFlight) await inFlight; + if (osInFlight) await osInFlight; + warmupIndex = snapshots.length; + osWarmupIndex = osSnapshots.length; + await runOsSample(); + await runSample(); + }, + async stop(): Promise { + if (stopped) return; + stopped = true; + clearInterval(timer); + clearInterval(osTimer); + if (inFlight) await inFlight; + if (osInFlight) await osInFlight; + await runOsSample(); + await runSample(); + } + }; +}; diff --git a/packages/perf-harness/src/postgres.ts b/packages/perf-harness/src/postgres.ts new file mode 100644 index 0000000000..45c05638ab --- /dev/null +++ b/packages/perf-harness/src/postgres.ts @@ -0,0 +1,466 @@ +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import type { PostgresMemorySnapshot } from './types'; + +const UNIT_BYTES: Record = { + B: 1, + KiB: 1024, + MiB: 1024 ** 2, + GiB: 1024 ** 3, + TiB: 1024 ** 4 +}; + +const CGROUP_FILE_MARKER = '__CPERF_CGROUP_FILE__ '; +const CGROUP_FILES = [ + 'memory.current', + 'memory.peak', + 'memory.max', + 'memory.stat', + 'memory.events' +] as const; + +const CGROUP_V2_SCRIPT = ` +set -eu +base=/sys/fs/cgroup +if [ ! -r "$base/memory.current" ]; then + relative=$(awk -F: '$1 == "0" { print $3; exit }' /proc/self/cgroup) + if [ -n "$relative" ] && [ -r "$base$relative/memory.current" ]; then + base="$base$relative" + fi +fi +for file in memory.current memory.peak memory.max memory.stat memory.events; do + if [ -r "$base/$file" ]; then + printf '${CGROUP_FILE_MARKER}%s\n' "$file" + sed -n '1,256p' "$base/$file" + fi +done +`; + +const CGROUP_IDENTITY_SCRIPT = [ + 'set -eu', + 'test -r /sys/fs/cgroup/memory.current', + 'test -r /sys/fs/cgroup/memory.events', + 'printf "membership="', + 'cat /proc/1/cgroup', + 'printf "mount="', + 'stat -c "%d:%i" /sys/fs/cgroup' +].join('\n'); + +const CONTAINER_ID = /^[a-f0-9]{64}$/; +const PREFIXED_SHA256 = /^sha256:[a-f0-9]{64}$/; + +const canonicalStringSha256 = (value: string): string => `sha256:${createHash('sha256') + .update(JSON.stringify(value)) + .digest('hex')}`; + +export const parseDockerBytes = (value: string): number | null => { + const match = /^\s*([\d.]+)\s*(B|KiB|MiB|GiB|TiB)\s*$/.exec(value); + if (!match) return null; + const parsed = Number.parseFloat(match[1]); + return Number.isFinite(parsed) ? Math.round(parsed * UNIT_BYTES[match[2]]) : null; +}; + +const parseNonNegativeInteger = (value: string | undefined): number | null => { + if (!value || !/^\d+$/.test(value.trim())) return null; + const parsed = Number(value.trim()); + return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : null; +}; + +export const parseCgroupKeyValues = (value: string): Record => { + const result: Record = {}; + for (const line of value.split(/\r?\n/)) { + const match = /^([^\s]+)\s+(\d+)$/.exec(line.trim()); + if (!match) continue; + const parsed = Number(match[2]); + if (Number.isSafeInteger(parsed) && parsed >= 0) result[match[1]] = parsed; + } + return result; +}; + +const cgroupSections = (raw: string): Map => { + const sections = new Map(); + let current: string | null = null; + for (const line of raw.split(/\r?\n/)) { + if (line.startsWith(CGROUP_FILE_MARKER)) { + const file = line.slice(CGROUP_FILE_MARKER.length).trim(); + current = (CGROUP_FILES as readonly string[]).includes(file) ? file : null; + if (current && !sections.has(current)) sections.set(current, []); + } else if (current) { + sections.get(current)!.push(line); + } + } + return new Map([...sections].map(([file, lines]) => [file, lines.join('\n').trim()])); +}; + +export interface ParsedCgroupV2Memory { + currentBytes: number; + peakBytes: number | null; + maxBytes: number | null; + stat: Record; + events: Record; +} + +export const parseCgroupV2Memory = (raw: string): ParsedCgroupV2Memory | null => { + const sections = cgroupSections(raw); + const currentBytes = parseNonNegativeInteger(sections.get('memory.current')); + if (currentBytes == null) return null; + const maxRaw = sections.get('memory.max')?.trim(); + return { + currentBytes, + peakBytes: parseNonNegativeInteger(sections.get('memory.peak')), + maxBytes: maxRaw === 'max' ? null : parseNonNegativeInteger(maxRaw), + stat: parseCgroupKeyValues(sections.get('memory.stat') ?? ''), + events: parseCgroupKeyValues(sections.get('memory.events') ?? '') + }; +}; + +const execFileText = ( + command: string, + args: string[], + timeout: number +): Promise => new Promise((resolve, reject) => { + execFile(command, args, { timeout }, (error, stdout, stderr) => { + if (error) { + const detail = String(stderr).trim(); + reject(new Error(detail ? `${error.message}: ${detail}` : error.message)); + return; + } + resolve(String(stdout)); + }); +}); + +interface DockerContainerIdentity { + id: string; + startedAt: string; + cgroupIdentitySha256: string; +} + +const inspectDockerContainer = async (container: string): Promise<{ + id: string; + startedAt: string; +}> => { + const raw = await execFileText('docker', ['inspect', container], 10_000); + const records = JSON.parse(raw) as Array<{ + Id?: unknown; + State?: { Running?: unknown; StartedAt?: unknown }; + }>; + const record = Array.isArray(records) && records.length === 1 ? records[0] : null; + const startedAtMs = Date.parse( + typeof record?.State?.StartedAt === 'string' ? record.State.StartedAt : '' + ); + if ( + !record + || !CONTAINER_ID.test(String(record.Id ?? '')) + || record.State?.Running !== true + || !Number.isSafeInteger(startedAtMs) + ) { + throw new Error('PostgreSQL sampler container identity is invalid'); + } + return { + id: String(record.Id), + startedAt: new Date(startedAtMs).toISOString() + }; +}; + +const inspectContainerCgroupIdentity = async (containerId: string): Promise => { + const raw = await execFileText('docker', [ + 'exec', + containerId, + '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin', + 'sh', '-ceu', CGROUP_IDENTITY_SCRIPT + ], 10_000); + return canonicalStringSha256(raw.trim()); +}; + +const resolveDockerContainerIdentity = async ( + container: string, + expected: Pick< + PostgresMemorySamplerOptions, + 'expectedContainerId' | 'expectedContainerStartedAt' | 'expectedCgroupIdentitySha256' + > +): Promise => { + const inspected = await inspectDockerContainer(container); + if (expected.expectedContainerId && inspected.id !== expected.expectedContainerId) { + throw new Error( + `PostgreSQL sampler container ID mismatch: expected ${expected.expectedContainerId}, observed ${inspected.id}` + ); + } + if ( + expected.expectedContainerStartedAt + && inspected.startedAt !== expected.expectedContainerStartedAt + ) { + throw new Error( + 'PostgreSQL sampler container start time does not match run attestation' + ); + } + const cgroupIdentitySha256 = await inspectContainerCgroupIdentity(inspected.id); + if ( + expected.expectedCgroupIdentitySha256 + && cgroupIdentitySha256 !== expected.expectedCgroupIdentitySha256 + ) { + throw new Error('PostgreSQL sampler cgroup identity does not match run attestation'); + } + return { ...inspected, cgroupIdentitySha256 }; +}; + +const dockerWorkingSet = async (container: string): Promise<{ + usedBytes: number; + limitBytes: number; + raw: string; +}> => { + const raw = (await execFileText( + 'docker', + ['stats', '--no-stream', '--format', '{{.MemUsage}}', container], + 10_000 + )).trim(); + const parts = raw.split('/').map((part) => part.trim()); + const usedBytes = parseDockerBytes(parts[0] ?? ''); + const limitBytes = parseDockerBytes(parts[1] ?? ''); + if (usedBytes == null || limitBytes == null) { + throw new Error(`unrecognized docker memory value '${raw}'`); + } + return { usedBytes, limitBytes, raw }; +}; + +const dockerCgroupV2 = async (container: string): Promise<{ + parsed: ParsedCgroupV2Memory; + raw: string; +}> => { + const raw = await execFileText( + 'docker', + ['exec', container, 'sh', '-c', CGROUP_V2_SCRIPT], + 10_000 + ); + const parsed = parseCgroupV2Memory(raw); + if (!parsed) throw new Error('container does not expose readable cgroup-v2 memory.current'); + return { parsed, raw }; +}; + +type CgroupReader = () => Promise<{ parsed: ParsedCgroupV2Memory; raw: string }>; + +const readHostCgroupV2 = async (base: string): Promise<{ + parsed: ParsedCgroupV2Memory; + raw: string; +}> => { + const sections: string[] = []; + for (const file of CGROUP_FILES) { + try { + const value = await fs.readFile(path.join(base, file), 'utf8'); + sections.push(`${CGROUP_FILE_MARKER}${file}\n${value.trim()}\n`); + } catch (error) { + if (file === 'memory.current') throw error; + } + } + const raw = sections.join(''); + const parsed = parseCgroupV2Memory(raw); + if (!parsed) throw new Error('host does not expose readable cgroup-v2 memory.current'); + return { parsed, raw }; +}; + +const resolveCgroupReader = async (container: string): Promise => { + if (process.platform === 'linux') { + try { + const pidRaw = (await execFileText( + 'docker', + ['inspect', '--format', '{{.State.Pid}}', container], + 10_000 + )).trim(); + const pid = Number(pidRaw); + if (!Number.isSafeInteger(pid) || pid <= 0) { + throw new Error(`invalid container pid '${pidRaw}'`); + } + const membership = await fs.readFile(`/proc/${pid}/cgroup`, 'utf8'); + const relative = membership + .split(/\r?\n/) + .map((line) => /^0::(.+)$/.exec(line)?.[1]) + .find(Boolean); + if (!relative) throw new Error('container process has no cgroup-v2 membership'); + const cgroupRoot = path.resolve('/sys/fs/cgroup'); + const base = path.resolve(cgroupRoot, `.${relative}`); + if (base !== cgroupRoot && !base.startsWith(`${cgroupRoot}${path.sep}`)) { + throw new Error('container cgroup path escaped the cgroup-v2 root'); + } + await readHostCgroupV2(base); + return () => readHostCgroupV2(base); + } catch { + // Docker Desktop and rootless engines may hide the host cgroup. The + // container namespace fallback remains correct, though less frequent. + } + } + return () => dockerCgroupV2(container); +}; + +export interface PostgresMemorySamplerOptions { + intervalMs?: number; + requireCgroupV2?: boolean; + expectedContainerId?: string; + expectedContainerStartedAt?: string; + expectedCgroupIdentitySha256?: string; +} + +export interface PostgresMemorySampler { + snapshots: PostgresMemorySnapshot[]; + errors: string[]; + ready: Promise; + stop(): Promise; +} + +export const startPostgresMemorySampler = ( + container: string, + options: number | PostgresMemorySamplerOptions = {} +): PostgresMemorySampler => { + const intervalMs = typeof options === 'number' ? options : options.intervalMs ?? 250; + const requireCgroupV2 = typeof options === 'number' + ? false + : options.requireCgroupV2 ?? false; + const identityOptions: PostgresMemorySamplerOptions = typeof options === 'number' + ? {} + : options; + if ( + identityOptions.expectedContainerId != null + && !CONTAINER_ID.test(identityOptions.expectedContainerId) + ) { + throw new Error('expected PostgreSQL container ID must be a 64-character digest'); + } + if ( + identityOptions.expectedCgroupIdentitySha256 != null + && !PREFIXED_SHA256.test(identityOptions.expectedCgroupIdentitySha256) + ) { + throw new Error('expected PostgreSQL cgroup identity must be a prefixed SHA-256'); + } + if (!Number.isFinite(intervalMs) || intervalMs <= 0) { + throw new Error(`PostgreSQL sample interval must be positive, received ${intervalMs}`); + } + const snapshots: PostgresMemorySnapshot[] = []; + const errors: string[] = []; + let cgroupErrorRecorded = false; + let cgroupReader: CgroupReader | null = null; + let latestWorkingSet: Awaited> | null = null; + let latestWorkingSetError: string | null = null; + let workingSetInFlight: Promise | null = null; + let inFlight: Promise | null = null; + let stopped = false; + let immutableIdentity: DockerContainerIdentity | null = null; + const exactContainer = (): string => { + if (!immutableIdentity) throw new Error('PostgreSQL sampler identity is unavailable'); + return immutableIdentity.id; + }; + const revalidateIdentity = async (): Promise => { + if (!immutableIdentity) throw new Error('PostgreSQL sampler identity is unavailable'); + const observed = await resolveDockerContainerIdentity(immutableIdentity.id, { + expectedContainerId: immutableIdentity.id, + expectedContainerStartedAt: immutableIdentity.startedAt, + expectedCgroupIdentitySha256: immutableIdentity.cgroupIdentitySha256 + }); + if ( + observed.id !== immutableIdentity.id + || observed.startedAt !== immutableIdentity.startedAt + || observed.cgroupIdentitySha256 !== immutableIdentity.cgroupIdentitySha256 + ) { + throw new Error('PostgreSQL sampler immutable identity changed during the run'); + } + }; + const sampleWorkingSet = (): Promise => { + if (workingSetInFlight) return workingSetInFlight; + let pending: Promise; + pending = dockerWorkingSet(exactContainer()).then((snapshot) => { + latestWorkingSet = snapshot; + latestWorkingSetError = null; + }, (error) => { + // Docker's cache-subtracted working set is diagnostic when the raw + // cgroup-v2 reader is healthy. Keep its failure in the sample payload, + // but do not disqualify an otherwise complete raw-memory measurement. + latestWorkingSetError = error instanceof Error ? error.message : String(error); + }).finally(() => { + if (workingSetInFlight === pending) workingSetInFlight = null; + }); + workingSetInFlight = pending; + return pending; + }; + const sample = async (): Promise => { + const startedAtMs = Date.now(); + let cgroup: Awaited> | null = null; + let cgroupError: string | null = null; + try { + cgroup = await cgroupReader!(); + } catch (error) { + cgroupError = error instanceof Error ? error.message : String(error); + if (requireCgroupV2 && !cgroupErrorRecorded) { + cgroupErrorRecorded = true; + errors.push(`cgroup-v2 telemetry unavailable: ${cgroupError}`); + } + } + const endedAtMs = Date.now(); + const workingSet = latestWorkingSet; + if (!cgroup && !workingSet) { + errors.push('PostgreSQL memory telemetry produced neither cgroup nor Docker data'); + return; + } + const midpointMs = startedAtMs + ((endedAtMs - startedAtMs) / 2); + snapshots.push({ + timestamp: new Date(midpointMs).toISOString(), + containerId: immutableIdentity!.id, + cgroupIdentitySha256: immutableIdentity!.cgroupIdentitySha256, + sampleStartedAt: new Date(startedAtMs).toISOString(), + sampleEndedAt: new Date(endedAtMs).toISOString(), + sampleDurationMs: endedAtMs - startedAtMs, + usedBytes: cgroup?.parsed.currentBytes ?? workingSet!.usedBytes, + ...(workingSet ? { workingSetBytes: workingSet.usedBytes } : {}), + limitBytes: cgroup?.parsed.maxBytes ?? workingSet?.limitBytes ?? 0, + source: cgroup ? 'cgroup-v2' : 'docker-stats', + ...(cgroup ? { cgroupV2: cgroup.parsed } : {}), + raw: JSON.stringify({ + dockerStats: workingSet?.raw ?? null, + dockerStatsError: latestWorkingSetError, + cgroupV2: cgroup?.raw ?? null, + cgroupError + }) + }); + }; + const runSample = (): Promise => { + if (inFlight) return inFlight; + let pending: Promise; + pending = sample().finally(() => { + if (inFlight === pending) inFlight = null; + }); + inFlight = pending; + return pending; + }; + let timer: ReturnType | null = null; + let workingSetTimer: ReturnType | null = null; + const ready = (async () => { + immutableIdentity = await resolveDockerContainerIdentity(container, identityOptions); + cgroupReader = await resolveCgroupReader(immutableIdentity.id); + await sampleWorkingSet(); + await runSample(); + if (stopped) return; + timer = setInterval(() => { + void runSample(); + }, intervalMs); + timer.unref?.(); + workingSetTimer = setInterval(() => { + void sampleWorkingSet(); + }, Math.max(1_000, intervalMs * 4)); + workingSetTimer.unref?.(); + })(); + return { + snapshots, + errors, + ready, + async stop(): Promise { + if (stopped) return; + stopped = true; + if (timer) clearInterval(timer); + if (workingSetTimer) clearInterval(workingSetTimer); + await ready; + if (inFlight) await inFlight; + if (workingSetInFlight) await workingSetInFlight; + await sampleWorkingSet(); + await runSample(); + await revalidateIdentity(); + } + }; +}; diff --git a/packages/perf-harness/src/process.ts b/packages/perf-harness/src/process.ts new file mode 100644 index 0000000000..d8f33503ad --- /dev/null +++ b/packages/perf-harness/src/process.ts @@ -0,0 +1,434 @@ +import { type ChildProcess, spawn, spawnSync } from 'node:child_process'; +import { createHash, randomBytes } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { armEnvironmentForHeap, resolveTemplate } from './config'; +import type { ArmPlan, ArmProvenance, NodeV8Profile } from './types'; + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); +const MAX_OLD_SPACE_OPTION = /^--max(?:-|_)old(?:-|_)space(?:-|_)size(?:=(.*))?$/; +const MANAGED_V8_OPTION = + /^--(?:no[-_])?(?:jitless|optimize[-_]for[-_]size|max[-_]opt)(?:=.*)?$/; +const V8_PROFILE_FLAGS: Readonly> = Object.freeze({ + stock: Object.freeze([]), + 'optimize-for-size': Object.freeze(['--optimize-for-size']), + 'baseline-optimize-for-size': Object.freeze([ + '--max-opt=1', + '--optimize-for-size' + ]), + 'jitless-optimize-for-size': Object.freeze(['--jitless', '--optimize-for-size']) +}); + +export interface ArmProcess { + pid: number | null; + external: boolean; + exit: { code: number | null; signal: NodeJS.Signals | null } | null; + expectedHeapLimitBytes: number | null; + observabilityHeaders: Readonly>; + provenance: ArmProvenance; + provenanceErrors: string[]; + stop(): Promise; +} + +interface ProvenanceResult { + provenance: ArmProvenance; + errors: string[]; +} + +/** Ephemeral credentials for one spawned arm; never serialize this object. */ +export const createObservabilityHeaders = (): Readonly> => Object.freeze({ + Authorization: `Bearer ${randomBytes(32).toString('base64url')}` +}); + +const sha256 = (value: string | Buffer): string => createHash('sha256').update(value).digest('hex'); + +export const tokenizeNodeOptions = (input: string): string[] => { + const tokens: string[] = []; + let token = ''; + let quote: '"' | "'" | null = null; + let escaped = false; + let started = false; + + for (const character of input) { + if (escaped) { + token += character; + escaped = false; + started = true; + } else if (character === '\\') { + escaped = true; + started = true; + } else if (quote) { + if (character === quote) quote = null; + else token += character; + } else if (character === '"' || character === "'") { + quote = character; + started = true; + } else if (/\s/.test(character)) { + if (started) { + tokens.push(token); + token = ''; + started = false; + } + } else { + token += character; + started = true; + } + } + if (escaped || quote) throw new Error('NODE_OPTIONS contains an unterminated escape or quote'); + if (started) tokens.push(token); + return tokens; +}; + +const quoteNodeOption = (option: string): string => { + if (/^[^\s"'\\]+$/.test(option)) return option; + return `"${option.replace(/(["\\])/g, '\\$1')}"`; +}; + +export const nodeFlagsForV8Profile = (profile: NodeV8Profile): string[] => { + const flags = V8_PROFILE_FLAGS[profile]; + if (!flags) throw new Error(`unknown Node V8 profile '${profile}'`); + return [...flags]; +}; + +/** Remove every inherited old-space flag before installing the requested limit. */ +export const replaceMaxOldSpaceSize = (nodeOptions: string | undefined, heapMiB: number): string => { + if (!Number.isSafeInteger(heapMiB) || heapMiB <= 0) { + throw new Error(`heapMiB must be a positive integer, received ${heapMiB}`); + } + const input = nodeOptions?.trim() ? tokenizeNodeOptions(nodeOptions) : []; + const retained: string[] = []; + for (let index = 0; index < input.length; index += 1) { + const match = MAX_OLD_SPACE_OPTION.exec(input[index]); + if (MANAGED_V8_OPTION.test(input[index])) continue; + if (!match) { + retained.push(input[index]); + continue; + } + if (match[1] === undefined && index + 1 < input.length && /^\d+$/.test(input[index + 1])) { + index += 1; + } + } + retained.push(`--max-old-space-size=${heapMiB}`); + return retained.map(quoteNodeOption).join(' '); +}; + +export const expectedHeapLimitForNodeOptions = ( + nodeOptions: string, + nodeExecutable = process.execPath, + directNodeFlags: readonly string[] = [] +): number => { + const result = spawnSync( + nodeExecutable, + [ + ...directNodeFlags, + '-e', + 'process.stdout.write(String(require("node:v8").getHeapStatistics().heap_size_limit))' + ], + { + encoding: 'utf8', + env: { ...process.env, NODE_OPTIONS: nodeOptions }, + timeout: 15_000 + } + ); + const limit = Number(result.stdout?.trim()); + if (result.status !== 0 || !Number.isSafeInteger(limit) || limit <= 0) { + const detail = result.error?.message || result.stderr?.trim() || `exit=${result.status}`; + throw new Error(`could not resolve expected V8 heap limit: ${detail}`); + } + return limit; +}; + +const gitOutput = (cwd: string, args: string[]): string | null => { + const result = spawnSync('git', ['-C', cwd, ...args], { + encoding: 'utf8', + timeout: 30_000 + }); + return result.status === 0 ? result.stdout.trimEnd() : null; +}; + +const regularFile = (file: string): boolean => { + try { + return fs.statSync(file).isFile(); + } catch { + return false; + } +}; + +const resolveEntryPath = (command: string[], cwd: string): string | null => { + const executable = path.basename(command[0] ?? '').toLowerCase(); + const candidates = executable === 'node' || executable === 'node.exe' + ? command.slice(1).filter((part) => !part.startsWith('-')) + : command; + for (const candidate of candidates) { + const resolved = path.isAbsolute(candidate) ? candidate : path.resolve(cwd, candidate); + if (regularFile(resolved)) return fs.realpathSync(resolved); + } + return null; +}; + +const directNodeExecArgv = (command: string[], cwd: string): string[] => { + const executable = path.basename(command[0] ?? '').toLowerCase(); + if (executable !== 'node' && executable !== 'node.exe') return []; + for (let index = 1; index < command.length; index++) { + const candidate = command[index]; + if (candidate.startsWith('-')) continue; + const resolved = path.isAbsolute(candidate) ? candidate : path.resolve(cwd, candidate); + if (regularFile(resolved)) return command.slice(1, index); + } + return command.slice(1).filter((argument) => argument.startsWith('-')); +}; + +interface NodeRuntimeProvenance { + v8Profile: NodeV8Profile; + nodeOptions: string | null; + nodeOptionsArgv: string[]; + nodeExecArgv: string[]; +} + +export const collectArmProvenance = ( + cwd: string, + command: string[], + serverPid: number | null, + runtime: NodeRuntimeProvenance = { + v8Profile: 'stock', + nodeOptions: null, + nodeOptionsArgv: [], + nodeExecArgv: directNodeExecArgv(command, cwd) + } +): ProvenanceResult => { + const errors: string[] = []; + const repoRoot = gitOutput(cwd, ['rev-parse', '--show-toplevel']); + const gitHead = gitOutput(cwd, ['rev-parse', 'HEAD']); + const gitStatus = gitOutput(cwd, ['status', '--porcelain=v1', '--untracked-files=all']); + if (!repoRoot) errors.push(`could not resolve git worktree root for ${cwd}`); + if (!gitHead) errors.push(`could not resolve git HEAD for ${cwd}`); + if (gitStatus == null) errors.push(`could not resolve git status for ${cwd}`); + + const lockfileCandidate = repoRoot ? path.join(repoRoot, 'pnpm-lock.yaml') : null; + const lockfilePath = lockfileCandidate && regularFile(lockfileCandidate) + ? fs.realpathSync(lockfileCandidate) + : null; + if (!lockfilePath) errors.push('workspace pnpm-lock.yaml was not found'); + + const entryPath = resolveEntryPath(command, cwd); + if (!entryPath) errors.push(`could not resolve an executed entry from command: ${command.join(' ')}`); + + return { + provenance: { + cwd, + command: [...command], + gitHead, + worktreeDirty: gitStatus == null ? null : gitStatus.length > 0, + gitStatusSha256: gitStatus == null ? null : sha256(gitStatus), + lockfilePath, + lockfileSha256: lockfilePath ? sha256(fs.readFileSync(lockfilePath)) : null, + entryPath, + entrySha256: entryPath ? sha256(fs.readFileSync(entryPath)) : null, + serverPid, + v8Profile: runtime.v8Profile, + nodeOptions: runtime.nodeOptions, + nodeOptionsArgv: [...runtime.nodeOptionsArgv], + nodeExecArgv: [...runtime.nodeExecArgv], + effectiveNodeRuntimeFlags: [ + ...runtime.nodeOptionsArgv, + ...runtime.nodeExecArgv + ], + planSha256: null, + fleetSha256: null, + node: process.version, + v8: process.versions.v8, + platform: process.platform, + architecture: process.arch, + runOrderSeed: null, + runOrderIndex: null, + memoryPolicy: null + }, + errors + }; +}; + +const assertPinnedProvenance = (arm: ArmPlan, provenance: ArmProvenance): void => { + if (arm.commit && !provenance.gitHead?.startsWith(arm.commit)) { + throw new Error(`arm commit mismatch: expected ${arm.commit}, observed ${provenance.gitHead ?? 'unknown'}`); + } + if (arm.lockfileSha256 && provenance.lockfileSha256 !== arm.lockfileSha256) { + throw new Error( + `arm lockfile mismatch: expected ${arm.lockfileSha256}, observed ${provenance.lockfileSha256 ?? 'unknown'}` + ); + } + if (arm.entrySha256 && provenance.entrySha256 !== arm.entrySha256) { + throw new Error( + `arm entry mismatch: expected ${arm.entrySha256}, observed ${provenance.entrySha256 ?? 'unknown'}` + ); + } +}; + +const waitForReady = async ( + url: string, + timeoutMs: number, + child?: ChildProcess, + getChildError?: () => Error | null +): Promise => { + const deadline = Date.now() + timeoutMs; + let lastError = 'not ready'; + while (Date.now() < deadline) { + const childError = getChildError?.(); + if (childError) throw new Error(`server process failed before readiness: ${childError.message}`); + if (child?.exitCode != null || child?.signalCode != null) { + throw new Error(`server exited before readiness: code=${child.exitCode} signal=${child.signalCode}`); + } + try { + const response = await fetch(url); + if (response.ok) return; + lastError = `HTTP ${response.status}`; + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + await sleep(250); + } + throw new Error(`server readiness timed out after ${timeoutMs}ms: ${lastError}`); +}; + +export const startArmProcess = async ( + arm: ArmPlan, + heapMiB: number, + artifactDir: string, + tenantCount: number, + attestedPostgresVariables: Record = {} +): Promise => { + const vars = { + heapMiB, + port: arm.port, + artifactDir, + mode: arm.introspectionMode, + tenantCount, + ...attestedPostgresVariables + }; + const readinessUrl = resolveTemplate(arm.readinessUrl, vars); + const cwd = path.resolve(arm.cwd ? resolveTemplate(arm.cwd, vars) : process.cwd()); + if (!arm.command?.length) { + await waitForReady(readinessUrl, arm.startupTimeoutMs ?? 120_000); + const collected = collectArmProvenance(cwd, [], null); + return { + pid: null, + external: true, + exit: null, + expectedHeapLimitBytes: null, + observabilityHeaders: Object.freeze({}), + provenance: collected.provenance, + provenanceErrors: collected.errors, + stop: async () => undefined + }; + } + + const configuredCommand = arm.command.map((part) => resolveTemplate(part, vars)); + const v8Profile = arm.v8Profile ?? 'stock'; + const profileFlags = nodeFlagsForV8Profile(v8Profile); + if (configuredCommand.some((argument) => MANAGED_V8_OPTION.test(argument))) { + throw new Error('managed V8 flags must be selected through v8Profile'); + } + const isNodeCommand = ['node', 'node.exe'].includes( + path.basename(configuredCommand[0]).toLowerCase() + ); + if (profileFlags.length > 0 && !isNodeCommand) { + throw new Error(`v8Profile '${v8Profile}' requires a Node command`); + } + const command = isNodeCommand + ? [configuredCommand[0], ...profileFlags, ...configuredCommand.slice(1)] + : configuredCommand; + const armEnvironment = armEnvironmentForHeap(arm, heapMiB); + const nodeOptions = replaceMaxOldSpaceSize( + armEnvironment.NODE_OPTIONS ?? process.env.NODE_OPTIONS, + heapMiB + ); + const nodeExecutable = ['node', 'node.exe'].includes(path.basename(command[0]).toLowerCase()) + ? command[0] + : process.execPath; + const expectedHeapLimitBytes = expectedHeapLimitForNodeOptions( + nodeOptions, + nodeExecutable, + profileFlags + ); + const nodeOptionsArgv = tokenizeNodeOptions(nodeOptions); + const collected = collectArmProvenance(cwd, command, null, { + v8Profile, + nodeOptions, + nodeOptionsArgv, + nodeExecArgv: directNodeExecArgv(command, cwd) + }); + assertPinnedProvenance(arm, collected.provenance); + const observabilityHeaders = createObservabilityHeaders(); + const observabilityToken = observabilityHeaders.Authorization.slice('Bearer '.length); + + fs.mkdirSync(artifactDir, { recursive: true }); + const logStream = fs.createWriteStream(path.join(artifactDir, 'server.log'), { flags: 'a' }); + const samplerDir = path.join(artifactDir, 'debug-sampler'); + const child = spawn(command[0], command.slice(1), { + cwd, + env: { + ...process.env, + ...armEnvironment, + NODE_ENV: 'production', + GRAPHILE_CACHE_TTL_MS: armEnvironment.GRAPHILE_CACHE_TTL_MS ?? '21600000', + NODE_OPTIONS: nodeOptions, + GRAPHILE_INTROSPECTION_MODE: arm.introspectionMode, + GRAPHQL_OBSERVABILITY_ENABLED: 'true', + GRAPHQL_OBSERVABILITY_TOKEN: observabilityToken, + GRAPHQL_DEBUG_SAMPLER_ENABLED: 'true', + GRAPHQL_DEBUG_SAMPLER_INTERVAL_MS: '1000', + GRAPHQL_DEBUG_SAMPLER_DIR: samplerDir + }, + stdio: ['ignore', 'pipe', 'pipe'] + }); + child.stdout?.pipe(logStream); + child.stderr?.pipe(logStream); + collected.provenance.serverPid = child.pid ?? null; + + let exit: ArmProcess['exit'] = null; + let childError: Error | null = null; + child.once('error', (error) => { + childError = error; + }); + child.once('exit', (code, signal) => { + exit = { code, signal }; + logStream.end(); + }); + const stopChild = async (): Promise => { + if (exit) return; + if (child.pid == null) { + logStream.end(); + return; + } + child.kill('SIGTERM'); + const deadline = Date.now() + 15_000; + while (!exit && Date.now() < deadline) await sleep(100); + if (!exit) child.kill('SIGKILL'); + const killDeadline = Date.now() + 2_000; + while (!exit && Date.now() < killDeadline) await sleep(50); + }; + try { + await waitForReady( + readinessUrl, + arm.startupTimeoutMs ?? 120_000, + child, + () => childError + ); + } catch (error) { + await stopChild(); + throw error; + } + + return { + pid: child.pid ?? null, + external: false, + expectedHeapLimitBytes, + observabilityHeaders, + provenance: collected.provenance, + provenanceErrors: collected.errors, + get exit() { + return exit; + }, + stop: stopChild + }; +}; diff --git a/packages/perf-harness/src/realtime-evidence.ts b/packages/perf-harness/src/realtime-evidence.ts new file mode 100644 index 0000000000..1cf02dadf6 --- /dev/null +++ b/packages/perf-harness/src/realtime-evidence.ts @@ -0,0 +1,243 @@ +import { createHash } from 'node:crypto'; + +import type { + RealtimeCorrelationReceipt, + RealtimeDeliveryCoverage, + RealtimeDeliverySurfaceCoverage +} from './types'; + +const SHA256 = /^[a-f0-9]{64}$/; +const EMPTY_SHA256 = createHash('sha256').update('').digest('hex'); + +const timestamp = (value: string | null): number => { + if (value == null) return Number.NaN; + const parsed = Date.parse(value); + return Number.isFinite(parsed) && new Date(parsed).toISOString() === value + ? parsed + : Number.NaN; +}; + +const orderedDigestSha256 = (digests: string[]): string => digests.length === 0 + ? EMPTY_SHA256 + : createHash('sha256').update(digests.join('\n')).digest('hex'); + +export interface RealtimeReceiptSurfaceEvidence { + tenantId: string; + surface: string; + route: string; + expectedRecurringRounds: number; + startedRecurringRounds: number; + verifiedRecurringRounds: number; + deadlineLateRecurringRounds: number; + receipts: RealtimeCorrelationReceipt[]; +} + +export interface RealtimeReceiptEvidenceInput { + deliveryIntervalMs: number; + workloadStartedAt: string; + workloadDeadlineAt: string; + workloadEndedAt: string | null; + surfaces: RealtimeReceiptSurfaceEvidence[]; +} + +export interface RealtimeReceiptEvidenceSummary { + coverage: RealtimeDeliveryCoverage; + failures: string[]; +} + +const receiptIsVerified = (receipt: RealtimeCorrelationReceipt): boolean => { + const deadlineAt = timestamp(receipt.deadlineAt); + const issuedAt = timestamp(receipt.issuedAt); + const primeAt = timestamp(receipt.primeResponseAt); + const eventAt = timestamp(receipt.eventAt); + return Number.isSafeInteger(receipt.sequence) + && receipt.sequence > 0 + && SHA256.test(receipt.issuedSha256) + && receipt.primeResponseSha256 === receipt.issuedSha256 + && receipt.eventSha256 === receipt.issuedSha256 + && Number.isFinite(deadlineAt) + && Number.isFinite(issuedAt) + && Number.isFinite(primeAt) + && Number.isFinite(eventAt) + && issuedAt <= primeAt + && issuedAt <= eventAt + && primeAt <= deadlineAt + && eventAt <= deadlineAt; +}; + +const percentile = (values: number[], fraction: number): number => { + if (values.length === 0) return 0; + const sorted = [...values].sort((left, right) => left - right); + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)]; +}; + +export const summarizeRealtimeReceiptEvidence = ( + input: RealtimeReceiptEvidenceInput +): RealtimeReceiptEvidenceSummary => { + const failures: string[] = []; + const workloadStartedAtMs = timestamp(input.workloadStartedAt); + const workloadDeadlineAtMs = timestamp(input.workloadDeadlineAt); + const workloadEndedAtMs = timestamp(input.workloadEndedAt); + const globalDigests = new Set(); + const surfaceKeys = new Set(); + const allPrimeLatencies: number[] = []; + const allDeliveryLatencies: number[] = []; + const surfaces: RealtimeDeliverySurfaceCoverage[] = input.surfaces.map((surface) => { + const key = `${surface.tenantId}\0${surface.surface}`; + if (surfaceKeys.has(key)) failures.push(`duplicate realtime surface: ${surface.tenantId}/${surface.surface}`); + surfaceKeys.add(key); + const sequences = new Set(); + for (let index = 0; index < surface.receipts.length; index += 1) { + const receipt = surface.receipts[index]; + if ( + !Number.isSafeInteger(receipt.sequence) + || receipt.sequence !== index + 1 + || sequences.has(receipt.sequence) + ) failures.push(`invalid realtime receipt sequence: ${surface.tenantId}/${surface.surface}`); + sequences.add(receipt.sequence); + if (!SHA256.test(receipt.issuedSha256)) { + failures.push(`invalid realtime receipt digest: ${surface.tenantId}/${surface.surface}`); + } else if (globalDigests.has(receipt.issuedSha256)) { + failures.push(`reused realtime receipt digest: ${surface.tenantId}/${surface.surface}`); + } + if ( + receipt.primeResponseSha256 != null + && !SHA256.test(receipt.primeResponseSha256) + ) failures.push(`invalid realtime prime digest: ${surface.tenantId}/${surface.surface}`); + if (receipt.eventSha256 != null && !SHA256.test(receipt.eventSha256)) { + failures.push(`invalid realtime event digest: ${surface.tenantId}/${surface.surface}`); + } + globalDigests.add(receipt.issuedSha256); + } + const timed = surface.receipts.filter((receipt) => receipt.timed); + const verified = timed.filter((receipt, index) => { + const scheduledAt = workloadStartedAtMs + (index + 1) * input.deliveryIntervalMs; + const slotDeadline = Math.min( + workloadDeadlineAtMs, + scheduledAt + input.deliveryIntervalMs + ); + const issuedAt = timestamp(receipt.issuedAt); + const receiptDeadline = timestamp(receipt.deadlineAt); + const scheduleBound = Number.isFinite(scheduledAt) + && Number.isFinite(slotDeadline) + && Number.isFinite(issuedAt) + && Number.isFinite(receiptDeadline) + // Node timers and wall-clock serialization can differ by a few + // milliseconds. This tolerance cannot extend the externally derived + // slot deadline and therefore cannot bless a late delivery. + && issuedAt >= scheduledAt - 5 + && issuedAt < slotDeadline + && receiptDeadline >= issuedAt + && receiptDeadline <= slotDeadline + && receiptDeadline <= workloadDeadlineAtMs; + if (!scheduleBound) { + failures.push( + `invalid realtime receipt schedule deadline: ${surface.tenantId}/${surface.surface}` + ); + } + return scheduleBound && receiptIsVerified(receipt); + }); + if (timed.length !== surface.startedRecurringRounds) { + failures.push(`realtime receipt count mismatch: ${surface.tenantId}/${surface.surface}`); + } + if (verified.length !== surface.verifiedRecurringRounds) { + failures.push(`realtime verified receipt count mismatch: ${surface.tenantId}/${surface.surface}`); + } + if ( + !Number.isSafeInteger(surface.expectedRecurringRounds) + || !Number.isSafeInteger(surface.startedRecurringRounds) + || !Number.isSafeInteger(surface.verifiedRecurringRounds) + || !Number.isSafeInteger(surface.deadlineLateRecurringRounds) + || surface.expectedRecurringRounds < 0 + || surface.startedRecurringRounds < 0 + || surface.verifiedRecurringRounds < 0 + || surface.deadlineLateRecurringRounds < 0 + ) failures.push(`invalid realtime counters: ${surface.tenantId}/${surface.surface}`); + const primeLatencies = verified.map((receipt) => + timestamp(receipt.primeResponseAt) - timestamp(receipt.issuedAt) + ); + const deliveryLatencies = verified.map((receipt) => + timestamp(receipt.eventAt) - timestamp(receipt.issuedAt) + ); + allPrimeLatencies.push(...primeLatencies); + allDeliveryLatencies.push(...deliveryLatencies); + return { + tenantId: surface.tenantId, + surface: surface.surface, + route: surface.route, + expectedRecurringRounds: surface.expectedRecurringRounds, + startedRecurringRounds: surface.startedRecurringRounds, + verifiedRecurringRounds: surface.verifiedRecurringRounds, + issuedCorrelationSha256: orderedDigestSha256( + timed.map((receipt) => receipt.issuedSha256) + ), + verifiedCorrelationSha256: orderedDigestSha256( + verified.map((receipt) => receipt.issuedSha256) + ), + primeRequests: timed.length, + primeResponseP99Ms: percentile(primeLatencies, 0.99), + deliveryP99Ms: percentile(deliveryLatencies, 0.99) + }; + }); + const expectedRecurringRounds = surfaces.reduce( + (sum, surface) => sum + surface.expectedRecurringRounds, + 0 + ); + const startedRecurringRounds = surfaces.reduce( + (sum, surface) => sum + surface.startedRecurringRounds, + 0 + ); + const verifiedRecurringRounds = surfaces.reduce( + (sum, surface) => sum + surface.verifiedRecurringRounds, + 0 + ); + const deadlineLateRecurringRounds = input.surfaces.reduce( + (sum, surface) => sum + surface.deadlineLateRecurringRounds, + 0 + ); + const primeRequests = surfaces.reduce( + (sum, surface) => sum + surface.primeRequests, + 0 + ); + if ( + !Number.isSafeInteger(input.deliveryIntervalMs) + || input.deliveryIntervalMs <= 0 + || !Number.isFinite(workloadStartedAtMs) + || !Number.isFinite(workloadDeadlineAtMs) + || ( + input.workloadEndedAt != null + && !Number.isFinite(workloadEndedAtMs) + ) + ) failures.push('invalid realtime coverage window'); + if (surfaces.length > 0 && expectedRecurringRounds === 0) { + failures.push('realtime coverage has no recurring rounds'); + } + const complete = failures.length === 0 + && input.workloadEndedAt != null + && workloadEndedAtMs >= workloadDeadlineAtMs + && startedRecurringRounds === expectedRecurringRounds + && verifiedRecurringRounds === expectedRecurringRounds + && deadlineLateRecurringRounds === 0 + && surfaces.every((surface) => + surface.issuedCorrelationSha256 === surface.verifiedCorrelationSha256 + ); + return { + coverage: { + version: 2, + deliveryIntervalMs: input.deliveryIntervalMs, + workloadStartedAt: input.workloadStartedAt, + workloadDeadlineAt: input.workloadDeadlineAt, + workloadEndedAt: input.workloadEndedAt, + expectedRecurringRounds, + startedRecurringRounds, + verifiedRecurringRounds, + deadlineLateRecurringRounds, + primeRequests, + primeResponseP99Ms: percentile(allPrimeLatencies, 0.99), + deliveryP99Ms: percentile(allDeliveryLatencies, 0.99), + complete, + surfaces + }, + failures + }; +}; diff --git a/packages/perf-harness/src/realtime.ts b/packages/perf-harness/src/realtime.ts new file mode 100644 index 0000000000..e4a91008dc --- /dev/null +++ b/packages/perf-harness/src/realtime.ts @@ -0,0 +1,764 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { isDeepStrictEqual } from 'node:util'; + +import { createClient } from 'graphql-ws'; +import { WebSocket } from 'ws'; + +import { jsonPointerValues, mapWithConcurrency } from './http'; +import { summarizeRealtimeReceiptEvidence } from './realtime-evidence'; +import type { + GraphqlSurface, + JsonPathMatch, + RealtimeCorrelationReceipt, + RealtimeDeliveryCoverage, + RealtimeGraphqlOperation, + TenantTarget +} from './types'; + +interface RealtimeSink { + next(value: any): void; + error(error: unknown): void; + complete(): void; +} + +interface DriverRealtimeClient { + subscribe( + payload: { query: string; variables?: Record }, + sink: RealtimeSink + ): () => void; + dispose(): Promise; +} + +export interface RealtimeClientFactoryInput { + url: string; + headers: Readonly>; + onConnected(): void; + onClosed(): void; + onError(): void; +} + +export type RealtimeClientFactory = ( + input: RealtimeClientFactoryInput +) => DriverRealtimeClient; + +export interface RealtimeDriverDependencies { + clientFactory?: RealtimeClientFactory; + fetch?: typeof fetch; + environment?: Readonly>; + sleep?: (ms: number) => Promise; + correlationFactory?: (surfaceKey: string, sequence: number) => string; +} + +export interface RealtimeDriverOptions { + concurrency: number; + timeoutMs: number; + deliveryIntervalMs?: number; +} + +export interface RealtimeDriverSnapshot { + expected: number; + active: number; + verified: number; + deliveryIntervalMs: number; + deliveryEvents: number; + deliveryRoundsStarted: number; + deliveryRoundsVerified: number; + deliveryRoundsPending: number; + timedCoverage: RealtimeDeliveryCoverage | null; + errors: string[]; + surfaces: Array<{ + tenantId: string; + surface: string; + route: string; + active: boolean; + verified: boolean; + deliveryEvents: number; + deliveryRoundsStarted: number; + deliveryRoundsVerified: number; + deliveryRoundPending: boolean; + timedRoundsExpected: number; + timedRoundsStarted: number; + timedRoundsVerified: number; + timedRoundsDeadlineLate: number; + correlationReceipts: RealtimeCorrelationReceipt[]; + }>; +} + +interface RealtimeTargetState { + tenantId: string; + surface: GraphqlSurface; + key: string; + route: string; + active: boolean; + verified: boolean; + deliveryEvents: number; + deliveryRoundsStarted: number; + deliveryRoundsVerified: number; + deliveryRoundPending: boolean; + timedRoundsExpected: number; + timedRoundsStarted: number; + timedRoundsVerified: number; + timedRoundsDeadlineLate: number; + correlationSequence: number; + pendingCorrelation: { + value: string; + sha256: string; + receipt: RealtimeCorrelationReceipt; + } | null; + correlationReceipts: RealtimeCorrelationReceipt[]; + client: DriverRealtimeClient | null; + unsubscribe: (() => void) | null; + errors: Set; +} + +const DEFAULT_SLEEP = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); +const DEFAULT_DELIVERY_INTERVAL_MS = 60_000; + +const matches = (body: unknown, match: JsonPathMatch): boolean => + jsonPointerValues(body, match.path).some((value) => + isDeepStrictEqual(value, match.value) + ); + +const exactCorrelationValue = ( + body: unknown, + path: string, + expected: string +): string | null => { + const selected = jsonPointerValues(body, path); + return selected.length === 1 + && typeof selected[0] === 'string' + && isDeepStrictEqual(selected[0], expected) + ? selected[0] + : null; +}; + +const firstForbiddenMatch = ( + body: unknown, + operation: RealtimeGraphqlOperation +): JsonPathMatch | undefined => operation.forbiddenMatches.find((match) => + matches(body, match) +); + +const firstMissingMatch = ( + body: unknown, + operation: RealtimeGraphqlOperation +): JsonPathMatch | undefined => operation.requiredMatches.find((match) => + !matches(body, match) +); + +export const realtimeWebSocketUrl = (surfaceUrl: string): string => { + const parsed = new URL(surfaceUrl); + if ( + !['http:', 'https:'].includes(parsed.protocol) + || parsed.username + || parsed.password + || parsed.search + || parsed.hash + ) { + throw new Error('CPERF_REALTIME_SURFACE_URL_INVALID'); + } + parsed.protocol = parsed.protocol === 'https:' ? 'wss:' : 'ws:'; + return parsed.toString(); +}; + +const surfaceRoute = (surfaceUrl: string): string => { + const parsed = new URL(surfaceUrl); + return parsed.pathname; +}; + +export const realtimeHeaders = ( + surface: GraphqlSurface, + environment: Readonly> = process.env +): Record => { + const headers: Record = { ...(surface.headers ?? {}) }; + for (const [name, environmentName] of Object.entries( + surface.realtime?.headersFromEnvironment ?? {} + )) { + const value = environment[environmentName]; + if (!value) { + throw new Error( + `CPERF_REALTIME_HEADER_ENV_MISSING:${surface.name}:${environmentName}` + ); + } + headers[name] = value; + } + return headers; +}; + +const defaultClientFactory: RealtimeClientFactory = ({ + url, + headers, + onConnected, + onClosed, + onError +}) => { + class HeaderWebSocket extends WebSocket { + constructor(address: string | URL, protocols?: string | string[]) { + super(address, protocols, { headers }); + } + } + const client = createClient({ + url, + webSocketImpl: HeaderWebSocket, + retryAttempts: 0, + connectionAckWaitTimeout: 10_000, + on: { + connected: onConnected, + closed: onClosed, + error: onError + } + }); + return { + subscribe: (payload, sink) => client.subscribe(payload, sink), + dispose: async () => { await client.dispose(); } + }; +}; + +const stateFailure = (state: RealtimeTargetState): string | null => + state.errors.values().next().value ?? null; + +export interface RealtimeDriver { + startAndVerify(): Promise; + beginTimedCoverage(durationMs: number): void; + finishTimedCoverage(): Promise; + verifyDeliveryNow(): Promise; + assertHealthy(): void; + snapshot(): RealtimeDriverSnapshot; + dispose(): Promise; +} + +export const createRealtimeDriver = ( + tenants: TenantTarget[], + options: RealtimeDriverOptions, + dependencies: RealtimeDriverDependencies = {} +): RealtimeDriver => { + if (!Number.isSafeInteger(options.concurrency) || options.concurrency <= 0) { + throw new Error('CPERF_REALTIME_CONCURRENCY_INVALID'); + } + if (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0) { + throw new Error('CPERF_REALTIME_TIMEOUT_INVALID'); + } + const deliveryIntervalMs = options.deliveryIntervalMs + ?? DEFAULT_DELIVERY_INTERVAL_MS; + if ( + !Number.isSafeInteger(deliveryIntervalMs) + || deliveryIntervalMs <= 0 + ) { + throw new Error('CPERF_REALTIME_DELIVERY_INTERVAL_INVALID'); + } + const clientFactory = dependencies.clientFactory ?? defaultClientFactory; + const fetchImpl = dependencies.fetch ?? fetch; + const environment = dependencies.environment ?? process.env; + const sleep = dependencies.sleep ?? DEFAULT_SLEEP; + const correlationFactory = dependencies.correlationFactory + ?? ((_surfaceKey: string, sequence: number) => + `cperf-realtime-v1:${sequence}:${randomUUID()}`); + const states: RealtimeTargetState[] = tenants.flatMap((tenant) => + tenant.surfaces.filter((surface) => surface.realtime).map( + (surface): RealtimeTargetState => ({ + tenantId: tenant.id, + surface, + key: `${tenant.id}/${surface.name}`, + route: surfaceRoute(surface.url), + active: false, + verified: false, + deliveryEvents: 0, + deliveryRoundsStarted: 0, + deliveryRoundsVerified: 0, + deliveryRoundPending: false, + timedRoundsExpected: 0, + timedRoundsStarted: 0, + timedRoundsVerified: 0, + timedRoundsDeadlineLate: 0, + correlationSequence: 0, + pendingCorrelation: null, + correlationReceipts: [], + client: null, + unsubscribe: null, + errors: new Set() + }) + ) + ); + let started = false; + let disposing = false; + let disposed = false; + let deliveryTimer: ReturnType | null = null; + let deliveryRound: Promise | null = null; + let timedCoverage: { + startedAtMs: number; + deadlineAtMs: number; + endedAtMs: number | null; + expectedRounds: number; + nextRound: number; + } | null = null; + const primeAbortControllers = new Set(); + const issuedCorrelations = new Set(); + + const recordError = (state: RealtimeTargetState, code: string): void => { + if (!disposing) state.errors.add(`${code}:${state.key}`); + }; + + const assertState = (state: RealtimeTargetState): void => { + const failure = stateFailure(state); + if (failure) throw new Error(failure); + }; + + const waitUntil = async ( + state: RealtimeTargetState, + predicate: () => boolean, + deadline: number, + timeoutCode: string + ): Promise => { + while (!predicate() && Date.now() < deadline) { + if (disposing || disposed) throw new Error('CPERF_REALTIME_DISPOSED'); + assertState(state); + await sleep(Math.min(25, Math.max(1, deadline - Date.now()))); + } + assertState(state); + if (!predicate()) throw new Error(`${timeoutCode}:${state.key}`); + }; + + const startState = async (state: RealtimeTargetState): Promise => { + const probe = state.surface.realtime!; + const headers = realtimeHeaders(state.surface, environment); + state.client = clientFactory({ + url: realtimeWebSocketUrl(state.surface.url), + headers, + onConnected: () => { state.active = true; }, + onClosed: () => { + state.active = false; + recordError(state, 'CPERF_REALTIME_TRANSPORT_DROPPED'); + }, + onError: () => recordError(state, 'CPERF_REALTIME_TRANSPORT_ERROR') + }); + state.unsubscribe = state.client.subscribe( + { query: probe.subscription.query, variables: probe.subscription.variables }, + { + next: (value) => { + if (Array.isArray(value?.errors) && value.errors.length > 0) { + recordError(state, 'CPERF_REALTIME_GRAPHQL_ERROR'); + return; + } + if (firstForbiddenMatch(value, probe.subscription)) { + recordError(state, 'CPERF_REALTIME_FOREIGN_PAYLOAD'); + return; + } + if (firstMissingMatch(value, probe.subscription)) { + recordError(state, 'CPERF_REALTIME_EVENT_INVARIANT_FAILED'); + return; + } + const pending = state.pendingCorrelation; + const verifiedCorrelation = pending && exactCorrelationValue( + value, + probe.correlation.subscriptionEventPath, + pending.value + ); + if (!pending || !verifiedCorrelation) { + // Cursor-backed delivery is at-least-once, so a valid replay for + // this exact tenant/database may arrive before the event caused by + // this round's fresh nonce. Permanent identity violations above + // still fail closed, but an old event cannot satisfy this round. + return; + } + pending.receipt.eventAt = new Date().toISOString(); + pending.receipt.eventSha256 = createHash('sha256') + .update(verifiedCorrelation) + .digest('hex'); + state.pendingCorrelation = null; + state.deliveryEvents++; + state.verified = true; + }, + error: () => recordError(state, 'CPERF_REALTIME_GRAPHQL_ERROR'), + complete: () => recordError(state, 'CPERF_REALTIME_SUBSCRIPTION_ENDED') + } + ); + await waitUntil( + state, + () => state.active, + Date.now() + options.timeoutMs, + 'CPERF_REALTIME_CONNECT_TIMEOUT' + ); + }; + + const primeOnce = async ( + state: RealtimeTargetState, + deadline: number, + correlation: string, + receipt: RealtimeCorrelationReceipt + ): Promise => { + const probe = state.surface.realtime!; + const controller = new AbortController(); + primeAbortControllers.add(controller); + const timeout = setTimeout( + () => controller.abort(), + Math.max(1, deadline - Date.now()) + ); + try { + const response = await fetchImpl(state.surface.url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...realtimeHeaders(state.surface, environment) + }, + body: JSON.stringify({ + query: probe.prime.query, + variables: { + ...(probe.prime.variables ?? {}), + [probe.correlation.primeVariable]: correlation + } + }), + signal: controller.signal + }); + const body = await response.json().catch((): null => null); + if (!response.ok || Array.isArray((body as any)?.errors)) { + throw new Error(`CPERF_REALTIME_PRIME_FAILED:${state.key}:HTTP_${response.status}`); + } + if (firstForbiddenMatch(body, probe.prime)) { + throw new Error(`CPERF_REALTIME_PRIME_FOREIGN_PAYLOAD:${state.key}`); + } + if (firstMissingMatch(body, probe.prime)) { + throw new Error(`CPERF_REALTIME_PRIME_INCONCLUSIVE:${state.key}`); + } + const responseCorrelation = exactCorrelationValue( + body, + probe.correlation.primeResponsePath, + correlation + ); + if (!responseCorrelation) { + throw new Error(`CPERF_REALTIME_PRIME_CORRELATION_MISMATCH:${state.key}`); + } + if (receipt.primeResponseAt == null) { + receipt.primeResponseAt = new Date().toISOString(); + receipt.primeResponseSha256 = createHash('sha256') + .update(responseCorrelation) + .digest('hex'); + } + } finally { + clearTimeout(timeout); + primeAbortControllers.delete(controller); + } + }; + + const verifyStateDelivery = async ( + state: RealtimeTargetState, + deadline = Date.now() + options.timeoutMs, + timed = false + ): Promise => { + const requiredEventCount = state.deliveryEvents + 1; + const correlation = correlationFactory(state.key, ++state.correlationSequence); + if ( + typeof correlation !== 'string' + || correlation.length < 24 + || correlation.length > 1024 + ) { + throw new Error(`CPERF_REALTIME_CORRELATION_INVALID:${state.key}`); + } + const correlationSha256 = createHash('sha256').update(correlation).digest('hex'); + if (issuedCorrelations.has(correlationSha256)) { + throw new Error(`CPERF_REALTIME_CORRELATION_REUSED:${state.key}`); + } + issuedCorrelations.add(correlationSha256); + const receipt: RealtimeCorrelationReceipt = { + sequence: state.correlationSequence, + timed, + deadlineAt: new Date(deadline).toISOString(), + issuedAt: new Date().toISOString(), + issuedSha256: correlationSha256, + primeResponseAt: null, + primeResponseSha256: null, + eventAt: null, + eventSha256: null + }; + state.correlationReceipts.push(receipt); + state.pendingCorrelation = { + value: correlation, + sha256: correlationSha256, + receipt + }; + state.deliveryRoundsStarted++; + if (timed) state.timedRoundsStarted++; + state.deliveryRoundPending = true; + try { + assertState(state); + await primeOnce(state, deadline, correlation, receipt); + await waitUntil( + state, + () => state.deliveryEvents >= requiredEventCount, + deadline, + 'CPERF_REALTIME_EVENT_TIMEOUT' + ); + state.deliveryRoundsVerified++; + if (timed) { + state.timedRoundsVerified++; + if (Date.now() > deadline) state.timedRoundsDeadlineLate++; + } + } finally { + if (state.pendingCorrelation?.sha256 === correlationSha256) { + state.pendingCorrelation = null; + } + state.deliveryRoundPending = false; + } + }; + + const runDeliveryRound = async ( + deadline = Date.now() + options.timeoutMs, + timed = false + ): Promise => { + const failures: Error[] = []; + await mapWithConcurrency(states, options.concurrency, async (state) => { + try { + await verifyStateDelivery( + state, + timed ? Math.min(deadline, Date.now() + options.timeoutMs) : deadline, + timed + ); + } catch (error) { + const failure = error instanceof Error + ? error + : new Error(`CPERF_REALTIME_DELIVERY_FAILED:${state.key}`); + if (!disposing) state.errors.add(failure.message); + failures.push(failure); + } + }); + if (failures.length > 0) throw failures[0]; + }; + + const clearDeliveryTimer = (): void => { + if (deliveryTimer) clearTimeout(deliveryTimer); + deliveryTimer = null; + }; + + const scheduleDeliveryRound = (): void => { + if ( + disposing + || disposed + || states.length === 0 + || deliveryTimer + || deliveryRound + ) return; + const coverage = timedCoverage; + if (!coverage || coverage.nextRound > coverage.expectedRounds) return; + const scheduledAt = coverage.startedAtMs + + coverage.nextRound * deliveryIntervalMs; + deliveryTimer = setTimeout(() => { + deliveryTimer = null; + const current = timedCoverage; + if (!current || current.nextRound > current.expectedRounds) return; + current.nextRound++; + const deadline = Math.min( + current.deadlineAtMs, + scheduledAt + deliveryIntervalMs + ); + if (Date.now() >= deadline) { + for (const state of states) { + state.timedRoundsStarted++; + state.timedRoundsDeadlineLate++; + } + scheduleDeliveryRound(); + return; + } + void launchDeliveryRound(deadline, true).catch((): void => undefined); + }, Math.max(0, scheduledAt - Date.now())); + }; + + const launchDeliveryRound = ( + deadline = Date.now() + options.timeoutMs, + timed = false + ): Promise => { + if (deliveryRound) return deliveryRound; + if (disposing || disposed) { + return Promise.reject(new Error('CPERF_REALTIME_DISPOSED')); + } + const round = runDeliveryRound(deadline, timed); + deliveryRound = round; + void round.then( + () => { + if (deliveryRound === round) deliveryRound = null; + scheduleDeliveryRound(); + }, + () => { + if (deliveryRound === round) deliveryRound = null; + } + ); + return round; + }; + + const coverageSnapshot = (): RealtimeDeliveryCoverage | null => { + if (!timedCoverage) return null; + return summarizeRealtimeReceiptEvidence({ + deliveryIntervalMs, + workloadStartedAt: new Date(timedCoverage.startedAtMs).toISOString(), + workloadDeadlineAt: new Date(timedCoverage.deadlineAtMs).toISOString(), + workloadEndedAt: timedCoverage.endedAtMs == null + ? null + : new Date(timedCoverage.endedAtMs).toISOString(), + surfaces: states.map((state) => ({ + tenantId: state.tenantId, + surface: state.surface.name, + route: state.route, + expectedRecurringRounds: state.timedRoundsExpected, + startedRecurringRounds: state.timedRoundsStarted, + verifiedRecurringRounds: state.timedRoundsVerified, + deadlineLateRecurringRounds: state.timedRoundsDeadlineLate, + receipts: state.correlationReceipts + })) + }).coverage; + }; + + const snapshot = (): RealtimeDriverSnapshot => ({ + expected: states.length, + active: states.filter((state) => state.active).length, + verified: states.filter((state) => state.verified).length, + deliveryIntervalMs, + deliveryEvents: states.reduce((sum, state) => sum + state.deliveryEvents, 0), + deliveryRoundsStarted: states.reduce( + (sum, state) => sum + state.deliveryRoundsStarted, + 0 + ), + deliveryRoundsVerified: states.reduce( + (sum, state) => sum + state.deliveryRoundsVerified, + 0 + ), + deliveryRoundsPending: states.filter((state) => + state.deliveryRoundPending + ).length, + timedCoverage: coverageSnapshot(), + errors: states.flatMap((state) => [...state.errors]).sort(), + surfaces: states.map((state) => ({ + tenantId: state.tenantId, + surface: state.surface.name, + route: state.route, + active: state.active, + verified: state.verified, + deliveryEvents: state.deliveryEvents, + deliveryRoundsStarted: state.deliveryRoundsStarted, + deliveryRoundsVerified: state.deliveryRoundsVerified, + deliveryRoundPending: state.deliveryRoundPending, + timedRoundsExpected: state.timedRoundsExpected, + timedRoundsStarted: state.timedRoundsStarted, + timedRoundsVerified: state.timedRoundsVerified, + timedRoundsDeadlineLate: state.timedRoundsDeadlineLate, + correlationReceipts: state.correlationReceipts.map((receipt) => ({ + ...receipt + })) + })) + }); + + const assertHealthy = (): void => { + const current = snapshot(); + if (current.errors.length > 0) throw new Error(current.errors[0]); + if ( + current.active !== current.expected + || current.verified !== current.expected + ) { + throw new Error( + `CPERF_REALTIME_NOT_HEALTHY:${current.active}:${current.verified}:${current.expected}` + ); + } + }; + + return { + async startAndVerify(): Promise { + if (started) throw new Error('CPERF_REALTIME_ALREADY_STARTED'); + started = true; + const failures: Error[] = []; + await mapWithConcurrency(states, options.concurrency, async (state) => { + try { + await startState(state); + } catch (error) { + failures.push(error instanceof Error ? error : new Error(String(error))); + } + }); + if (failures.length > 0) throw failures[0]; + await runDeliveryRound(); + assertHealthy(); + }, + beginTimedCoverage(durationMs: number): void { + if (!started) throw new Error('CPERF_REALTIME_NOT_STARTED'); + if (timedCoverage) throw new Error('CPERF_REALTIME_TIMED_COVERAGE_ALREADY_STARTED'); + if (!Number.isSafeInteger(durationMs) || durationMs <= 0) { + throw new Error('CPERF_REALTIME_TIMED_DURATION_INVALID'); + } + const startedAtMs = Date.now(); + const expectedRounds = Math.max( + 0, + Math.ceil(durationMs / deliveryIntervalMs) - 1 + ); + timedCoverage = { + startedAtMs, + deadlineAtMs: startedAtMs + durationMs, + endedAtMs: null, + expectedRounds, + nextRound: 1 + }; + for (const state of states) state.timedRoundsExpected = expectedRounds; + scheduleDeliveryRound(); + }, + async finishTimedCoverage(): Promise { + if (!timedCoverage) throw new Error('CPERF_REALTIME_TIMED_COVERAGE_NOT_STARTED'); + clearDeliveryTimer(); + if (deliveryRound) await deliveryRound; + clearDeliveryTimer(); + while (timedCoverage.nextRound <= timedCoverage.expectedRounds) { + timedCoverage.nextRound++; + for (const state of states) { + state.timedRoundsStarted++; + state.timedRoundsDeadlineLate++; + } + } + timedCoverage.endedAtMs = Date.now(); + const coverage = coverageSnapshot()!; + return coverage; + }, + async verifyDeliveryNow(): Promise { + if (!started) throw new Error('CPERF_REALTIME_NOT_STARTED'); + if (disposing || disposed) throw new Error('CPERF_REALTIME_DISPOSED'); + clearDeliveryTimer(); + const pendingRound = deliveryRound; + if (pendingRound) { + await pendingRound; + } else { + assertHealthy(); + await launchDeliveryRound(); + } + assertHealthy(); + }, + assertHealthy, + snapshot, + async dispose(): Promise { + if (disposed) return; + disposing = true; + clearDeliveryTimer(); + for (const controller of primeAbortControllers) controller.abort(); + const pendingRound = deliveryRound; + if (pendingRound) { + try { + await pendingRound; + } catch { + // Disposal intentionally aborts an in-flight prime or event wait. + } + } + for (const state of states) { + try { + state.unsubscribe?.(); + } catch { + state.errors.add(`CPERF_REALTIME_UNSUBSCRIBE_FAILED:${state.key}`); + } + } + const results = await Promise.allSettled(states.map((state) => state.client?.dispose())); + results.forEach((result, index) => { + if (result.status === 'rejected') { + states[index].errors.add(`CPERF_REALTIME_DISPOSE_FAILED:${states[index].key}`); + } + }); + disposed = true; + for (const state of states) state.active = false; + const disposalFailure = states.flatMap((state) => [...state.errors]).find((error) => + error.startsWith('CPERF_REALTIME_UNSUBSCRIBE_FAILED:') + || error.startsWith('CPERF_REALTIME_DISPOSE_FAILED:') + ); + if (disposalFailure) throw new Error(disposalFailure); + } + }; +}; diff --git a/packages/perf-harness/src/report.ts b/packages/perf-harness/src/report.ts new file mode 100644 index 0000000000..e9d9ac8297 --- /dev/null +++ b/packages/perf-harness/src/report.ts @@ -0,0 +1,1006 @@ +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { + DEFAULT_RUN_ORDER_SEED, + hasExactHostileValidationEvidence, + resolveTemplate, + soakArmName, + tenantCountsForHeap +} from './config'; +import { + assertResultSemanticReplay, + bindResultEvidence, + readRegularEvidenceFile, + RESULT_RAW_EVIDENCE_FILES, + validateResultEvidenceBinding +} from './evidence'; +import { postgresRunIdentityClaims } from './run-attestation'; +import { + buildRunSchedule, + scheduleJobsForPlan, + scheduleManifestSha256, + type CampaignScheduleJob, + type CampaignScheduleManifestV1 +} from './schedule'; +import { compareDensity, percentile, summarizeCapacityBoundaries } from './score'; +import type { DensityPlanV1, DensityRunResult, FleetV1 } from './types'; + +const GIB = 1024 ** 3; + +const SHA256 = /^[a-f0-9]{64}$/; + +const sha256 = (value: string | Buffer): string => createHash('sha256') + .update(value) + .digest('hex'); + +export { bindResultEvidence, RESULT_RAW_EVIDENCE_FILES }; + +const RESULT_V6_REQUIRED_KEYS = ` +schemaVersion runKind evidenceMode campaignId scheduleSha256 previousResultPayloadSha256 +qualificationCohortSha256 arm commit +introspectionMode heapMiB configuredCustomers configuredTenants fleetShape +repetition expectedMatrixRepetitions runOrderSeed runOrderIndex startedAt endedAt +durationSec warmupMaxMs resolvedWarmupTimeoutMs offeredLoad requests +coverageRequests workloadRequests errors customerWorkloadRps periodicValidationRps realtimeValidationRps +combinedHttpRps achievedRps missedArrivals errorRate p50Ms p95Ms p99Ms +peakHeapBytes peakRssBytes observedHeapLimitBytes residentInstances +expectedResidentInstances cacheConfiguredMax cacheBudgetCapacity cacheInstanceHeapBytes +cacheCalibrationId cacheAdmissionMode warmObservedHeapDeltaPerInstanceBytes +postWarmupHeapGrowthMiBPerHour rawPostWarmupHeapGrowthMiBPerHour +retainedHeapGrowthMiBPerHour retainedExternalGrowthMiBPerHour +retainedMemoryDurationSec retainedHeapBaselineBytes retainedHeapFinalBytes +retainedExternalBaselineBytes retainedExternalFinalBytes retainedMemoryCheckpointErrors +postWarmupEvictions postWarmupBuildRefusals postWarmupBuilds pgPoolCacheSize +pgPoolLeasedPools pgPoolActiveLeases postWarmupPgPoolCapacityEvictions +postWarmupPgPoolCapacityRefusals postWarmupPgPoolDisposalFailures coldBuildMaxMs +memorySampleErrors postgresBaselineBytes postgresWarmBoundaryBytes postgresPeakBytes +postgresWorkingSetPeakBytes postgresCgroupV2PeakBytes postgresCgroupV2Samples +postgresOomEvents postgresBackendPeak residentPhysicalDatabases postgresContainerDedicated +unexpectedPostgresDatabases pgPoolTotalClients pgPoolIdleClients pgPoolWaitingClients +runtimePoolRequestedMaxUses runtimePoolEffectiveMaxUses runtimePoolExpectedPools +runtimePoolObservedPools runtimePoolTotalClients runtimePoolIdleClients +runtimePoolWaitingClients +residentRealtimeManagers residentRealtimeTransports realtimeNotificationMode +realtimeDeliveryCoverage notificationBrokers notificationListenerConnections +notificationBrokerLeases notificationBrokerTopics notificationBrokerSubscribers +notificationBrokerQueueOverflows notificationBrokerFatalFailures notificationAuditIdentities +notificationAuditsHealthy notificationAuditsFailed notificationAuditsStale +notificationAuditAttempts notificationAuditFailures notificationAuditActiveDatabaseTargets +notificationAuditDatabaseConflicts postgresColdBuildSpikeBytes postgresSampleErrors +alignedServicePeakBytes alignedServicePeakNodeRssBytes alignedServicePeakPostgresBytes +alignedServicePeakTimestamp alignedServiceMemorySamples alignedServiceMemoryMaxSkewMs +alignedServiceMemoryCoverageRatio alignedServiceMemoryCoveredDurationMs +alignedServiceMemoryExpectedDurationMs alignedServiceMemoryMaxGapMs +serviceMemoryUpperBoundBytes serviceMemoryUpperBoundPostgresSource capabilitiesExercised +missingCapabilities missingCanaries canarySchedule canaryChecks canaryInconclusive +bleedViolations operationOracleChecks operationOracleInconclusive operationOracleViolations +missingOperationOracles tenants qualifiedCustomers qualifiedTenants +tenantsPerConfiguredOldSpaceGiB tenantsPerPeakRssGiB customersPerAlignedServiceGiB +customersPerServiceMemoryUpperBoundGiB configuredCustomersPerAlignedServiceGiB +configuredCustomersPerServiceMemoryUpperBoundGiB accepted failures serverExit provenance +provenanceErrors postgresRunAttestation evidenceBinding artifactDir +`.trim().split(/\s+/); + +export const readResults = (file: string): unknown[] => fs.readFileSync(file, 'utf8') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .map((line, index) => { + try { + return JSON.parse(line); + } catch (error) { + throw new Error( + `invalid result JSONL record ${index + 1}: ${error instanceof Error ? error.message : String(error)}` + ); + } + }); + +const finite = (value: unknown): value is number => typeof value === 'number' + && Number.isFinite(value); + +const closeEnough = (left: number, right: number): boolean => + Math.abs(left - right) <= Math.max(1e-9, Math.abs(right) * 1e-9); + +export interface ResultMatrixValidation { + complete: boolean; + expectedCoordinates: number; + observedCoordinates: number; + missing: string[]; + duplicates: string[]; + diagnostic: string[]; + soakExpected: boolean; + soakObserved: number; + soakComplete: boolean; +} + +const resultCoordinate = (result: Pick< +DensityRunResult, +'arm' | 'heapMiB' | 'configuredTenants' | 'repetition' +>): string => [ + result.arm, + result.heapMiB, + result.configuredTenants, + result.repetition +].join('/'); + +const expectedMatrixCoordinates = (plan: DensityPlanV1): string[] => plan.arms.flatMap( + (arm) => plan.heapMiB.flatMap((heapMiB) => tenantCountsForHeap(plan, heapMiB).flatMap( + (configuredTenants) => Array.from({ length: plan.repetitions }, (_unused, index) => [ + arm.name, + heapMiB, + configuredTenants, + index + 1 + ].join('/')) + )) +); + +interface CampaignEvidence { + manifest: CampaignScheduleManifestV1; + scheduleSha256: string; + evidenceMode: 'qualification' | 'diagnostic'; + qualificationBlockers: string[]; +} + +const canonicalIsoTimestamp = (value: unknown): value is string => { + if (typeof value !== 'string') return false; + const parsed = Date.parse(value); + return Number.isFinite(parsed) && new Date(parsed).toISOString() === value; +}; + +const requireCampaignJob = (value: unknown, label: string): CampaignScheduleJob => { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + const record = value as Record; + const expectedKeys = [ + 'runKind', + 'arm', + 'heapMiB', + 'tenantCount', + 'repetition', + 'orderIndex' + ]; + if ( + Object.keys(record).length !== expectedKeys.length + || expectedKeys.some((key) => !Object.prototype.hasOwnProperty.call(record, key)) + ) { + throw new Error(`${label} has an invalid shape`); + } + if ( + !['matrix', 'soak'].includes(String(record.runKind)) + || typeof record.arm !== 'string' + || record.arm.length === 0 + || !Number.isSafeInteger(record.heapMiB) + || (record.heapMiB as number) <= 0 + || !Number.isSafeInteger(record.tenantCount) + || (record.tenantCount as number) <= 0 + || !Number.isSafeInteger(record.repetition) + || (record.repetition as number) <= 0 + || !Number.isSafeInteger(record.orderIndex) + || (record.orderIndex as number) <= 0 + ) { + throw new Error(`${label} is invalid`); + } + return record as unknown as CampaignScheduleJob; +}; + +const readCampaignEvidence = ( + plan: DensityPlanV1, + campaignId: string +): CampaignEvidence => { + if (!SHA256.test(campaignId)) throw new Error('campaign identity is invalid'); + const root = fs.realpathSync(plan.artifactDir); + const file = path.join(root, `campaign-${campaignId}.json`); + const relative = path.relative(root, path.resolve(file)); + if (relative.startsWith('..') || path.isAbsolute(relative)) { + throw new Error('campaign manifest escaped the configured artifact root'); + } + let raw: unknown; + try { + raw = JSON.parse(readRegularEvidenceFile(file).toString('utf8')); + } catch (error) { + throw new Error( + `campaign manifest is unavailable or invalid: ` + + `${error instanceof Error ? error.message : String(error)}` + ); + } + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error('campaign manifest must be an object'); + } + const record = raw as Record; + const expectedKeys = [ + 'version', + 'campaignId', + 'campaignStartedAt', + 'runOrderSeed', + 'planSha256', + 'fleetSha256', + 'node', + 'v8', + 'platform', + 'architecture', + 'jobs', + 'scheduleSha256', + 'evidenceMode', + 'qualificationBlockers' + ]; + if ( + Object.keys(record).length !== expectedKeys.length + || expectedKeys.some((key) => !Object.prototype.hasOwnProperty.call(record, key)) + ) { + throw new Error('campaign manifest has an invalid shape'); + } + if ( + record.version !== 1 + || record.campaignId !== campaignId + || !canonicalIsoTimestamp(record.campaignStartedAt) + || typeof record.runOrderSeed !== 'string' + || record.runOrderSeed.length === 0 + || typeof record.planSha256 !== 'string' + || !SHA256.test(record.planSha256) + || typeof record.fleetSha256 !== 'string' + || !SHA256.test(record.fleetSha256) + || typeof record.node !== 'string' + || record.node.length === 0 + || typeof record.v8 !== 'string' + || record.v8.length === 0 + || typeof record.platform !== 'string' + || record.platform.length === 0 + || typeof record.architecture !== 'string' + || record.architecture.length === 0 + || typeof record.scheduleSha256 !== 'string' + || !SHA256.test(record.scheduleSha256) + || !['qualification', 'diagnostic'].includes(String(record.evidenceMode)) + || !Array.isArray(record.qualificationBlockers) + || record.qualificationBlockers.some((blocker) => + typeof blocker !== 'string' || blocker.length === 0) + || new Set(record.qualificationBlockers).size !== record.qualificationBlockers.length + || !Array.isArray(record.jobs) + || record.jobs.length === 0 + ) { + throw new Error('campaign manifest is invalid'); + } + const jobs = record.jobs.map((job, index) => + requireCampaignJob(job, `campaign manifest job ${index + 1}`)); + if (jobs.some((job, index) => job.orderIndex !== index + 1)) { + throw new Error('campaign manifest run order is not contiguous'); + } + const manifest: CampaignScheduleManifestV1 = { + version: 1, + campaignId, + campaignStartedAt: record.campaignStartedAt as string, + runOrderSeed: record.runOrderSeed as string, + planSha256: record.planSha256 as string, + fleetSha256: record.fleetSha256 as string, + node: record.node as string, + v8: record.v8 as string, + platform: record.platform as NodeJS.Platform, + architecture: record.architecture as string, + jobs + }; + if (scheduleManifestSha256(manifest) !== record.scheduleSha256) { + throw new Error('campaign manifest does not match its schedule SHA-256'); + } + return { + manifest, + scheduleSha256: record.scheduleSha256, + evidenceMode: record.evidenceMode as CampaignEvidence['evidenceMode'], + qualificationBlockers: record.qualificationBlockers as string[] + }; +}; + +const exactHostileEvidenceAvailable = (plan: DensityPlanV1): boolean => { + if (!hasExactHostileValidationEvidence(plan)) return false; + return plan.arms.every((arm) => { + const binding = plan.qualification!.hostileValidationEvidence![arm.name]; + if (!path.isAbsolute(binding.artifactFile)) return false; + try { + const bytes = readRegularEvidenceFile(binding.artifactFile); + if (sha256(bytes) !== binding.artifactSha256) return false; + const raw = JSON.parse(bytes.toString('utf8')) as Record; + return raw.version === 1 + && raw.kind === binding.kind + && raw.passed === true + && raw.arm === arm.name + && raw.runtimeArtifactFingerprint === binding.runtimeArtifactFingerprint + && raw.configurationFingerprint === binding.configurationFingerprint; + } catch { + return false; + } + }); +}; + +export const validateResultSet = ( + input: unknown[], + plan: DensityPlanV1, + fleet: FleetV1 +): { results: DensityRunResult[]; matrix: ResultMatrixValidation } => { + if (input.length === 0) throw new Error('result set contains no campaign records'); + const planSha256 = plan.sourceSha256 ?? sha256(JSON.stringify(plan)); + const fleetSha256 = fleet.sourceSha256 ?? sha256(JSON.stringify(fleet)); + const cohortSha256 = sha256(`${planSha256}\0${fleetSha256}`); + const armByName = new Map(plan.arms.map((arm) => [arm.name, arm])); + const expected = new Set(expectedMatrixCoordinates(plan)); + const firstRecord = input[0] && typeof input[0] === 'object' && !Array.isArray(input[0]) + ? input[0] as Record + : null; + if (!firstRecord || typeof firstRecord.campaignId !== 'string') { + throw new Error('result record 1 has no campaign identity'); + } + const campaign = readCampaignEvidence(plan, firstRecord.campaignId); + const canonicalMatrix = buildRunSchedule( + plan, + plan.arms, + plan.heapMiB, + plan.repetitions + ); + const canonicalJobs = scheduleJobsForPlan(plan, canonicalMatrix, true); + const canonicalQualificationSchedule = JSON.stringify(campaign.manifest.jobs) + === JSON.stringify(canonicalJobs); + const hostileEvidenceReady = exactHostileEvidenceAvailable(plan); + if ( + campaign.manifest.planSha256 !== planSha256 + || campaign.manifest.fleetSha256 !== fleetSha256 + || campaign.manifest.runOrderSeed !== (plan.runOrderSeed ?? DEFAULT_RUN_ORDER_SEED) + ) { + throw new Error('campaign manifest does not match the plan/fleet cohort'); + } + if (campaign.evidenceMode === 'qualification') { + if (campaign.manifest.platform !== 'linux') { + throw new Error('qualification campaign was not executed on Linux'); + } + if (campaign.qualificationBlockers.length > 0) { + throw new Error('qualification campaign contains prerequisite blockers'); + } + if (!canonicalQualificationSchedule) { + throw new Error('qualification campaign schedule is not the exact configured schedule'); + } + if (!hostileEvidenceReady) { + throw new Error( + 'qualification campaign lacks exact-runtime hostile validation evidence' + ); + } + } + let previousResultPayloadSha256: string | null = null; + let previousEndedAtMs = Date.parse(campaign.manifest.campaignStartedAt); + const seen = new Map(); + const diagnostic: string[] = []; + let soakObserved = 0; + const results = input.map((raw, index): DensityRunResult => { + const label = `result record ${index + 1}`; + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error(`${label} must be an object`); + } + const rawRecord = raw as Record; + const missingKeys = RESULT_V6_REQUIRED_KEYS.filter((key) => + !Object.prototype.hasOwnProperty.call(rawRecord, key) + ); + const unexpectedKeys = Object.keys(rawRecord).filter((key) => + !RESULT_V6_REQUIRED_KEYS.includes(key) + ); + if (missingKeys.length > 0 || unexpectedKeys.length > 0) { + throw new Error( + `${label} does not match the complete result-v6 shape; ` + + `missing=${missingKeys.join(',') || 'none'}; ` + + `unexpected=${unexpectedKeys.join(',') || 'none'}` + ); + } + const result = raw as DensityRunResult; + const arm = armByName.get(result.arm); + const scheduled = campaign.manifest.jobs[index]; + if (result.schemaVersion !== 6) throw new Error(`${label} schemaVersion must be 6`); + if (!scheduled) throw new Error(`${label} exceeds the campaign schedule`); + if ( + result.campaignId !== campaign.manifest.campaignId + || result.scheduleSha256 !== campaign.scheduleSha256 + || result.evidenceMode !== campaign.evidenceMode + || result.previousResultPayloadSha256 !== previousResultPayloadSha256 + || result.runOrderIndex !== index + 1 + || scheduled.orderIndex !== result.runOrderIndex + || scheduled.runKind !== result.runKind + || scheduled.arm !== result.arm + || scheduled.heapMiB !== result.heapMiB + || scheduled.tenantCount !== result.configuredTenants + || scheduled.repetition !== result.repetition + ) { + throw new Error(`${label} does not match the campaign schedule or result chain`); + } + if ( + !canonicalIsoTimestamp(result.startedAt) + || !canonicalIsoTimestamp(result.endedAt) + || Date.parse(result.startedAt) < previousEndedAtMs + || Date.parse(result.endedAt) < Date.parse(result.startedAt) + ) { + throw new Error(`${label} campaign chronology is invalid or overlapping`); + } + if (typeof result.artifactDir !== 'string' || result.artifactDir.length === 0) { + throw new Error(`${label} artifactDir is invalid`); + } + validateResultEvidenceBinding(result, label); + previousResultPayloadSha256 = result.evidenceBinding!.resultPayloadSha256; + previousEndedAtMs = Date.parse(result.endedAt); + if (!arm) throw new Error(`${label} uses unconfigured arm '${String(result.arm)}'`); + assertResultSemanticReplay(result, plan, fleet, label); + if (!['matrix', 'soak'].includes(result.runKind)) throw new Error(`${label} runKind is invalid`); + if (!['qualification', 'diagnostic'].includes(result.evidenceMode)) { + throw new Error(`${label} evidenceMode is invalid`); + } + if (result.qualificationCohortSha256 !== cohortSha256) { + throw new Error(`${label} qualification cohort does not match plan/fleet bytes`); + } + if ( + !Number.isSafeInteger(result.heapMiB) + || !plan.heapMiB.includes(result.heapMiB) + || !Number.isSafeInteger(result.configuredTenants) + || result.configuredTenants <= 0 + || result.configuredTenants !== result.configuredCustomers + || result.configuredTenants > fleet.tenants.length + || !Number.isSafeInteger(result.repetition) + || result.repetition <= 0 + || result.expectedMatrixRepetitions !== plan.repetitions + ) { + throw new Error(`${label} matrix coordinate is invalid`); + } + if (result.runKind === 'matrix') { + if (!tenantCountsForHeap(plan, result.heapMiB).includes(result.configuredTenants)) { + throw new Error(`${label} tenant count is not configured for heap ${result.heapMiB}`); + } + const coordinate = resultCoordinate(result); + seen.set(coordinate, (seen.get(coordinate) ?? 0) + 1); + if (result.evidenceMode !== 'qualification') diagnostic.push(coordinate); + } else { + const soak = plan.soak; + if (!soak?.enabled) { + throw new Error(`${label} contains soak evidence but plan.soak is not enabled`); + } + soakObserved++; + const expectedSoakArm = soakArmName(plan); + if ( + result.arm !== expectedSoakArm + || result.heapMiB !== soak.heapMiB + || result.configuredTenants !== soak.tenantCount + || result.repetition !== plan.repetitions + 1 + || ( + result.evidenceMode === 'qualification' + && result.runOrderIndex !== expected.size + 1 + ) + ) { + throw new Error(`${label} does not match the configured soak coordinate`); + } + if ( + result.accepted + && ( + !finite(result.durationSec) + || result.durationSec < soak.durationSec * 0.99 + || result.durationSec > soak.durationSec * 1.01 + ) + ) { + throw new Error(`${label} accepted soak duration does not match plan.soak.durationSec`); + } + } + const provenance = result.provenance; + if ( + !provenance + || provenance.planSha256 !== planSha256 + || provenance.fleetSha256 !== fleetSha256 + || provenance.runOrderSeed !== (plan.runOrderSeed ?? DEFAULT_RUN_ORDER_SEED) + || provenance.runOrderIndex !== result.runOrderIndex + || provenance.worktreeDirty !== false + || !provenance.gitHead + || !provenance.gitStatusSha256 + || !provenance.entrySha256 + || !provenance.lockfileSha256 + || !provenance.node + || !provenance.v8 + || provenance.node !== campaign.manifest.node + || provenance.v8 !== campaign.manifest.v8 + || provenance.platform !== campaign.manifest.platform + || provenance.architecture !== campaign.manifest.architecture + ) { + throw new Error(`${label} provenance is incomplete or does not match the plan/fleet cohort`); + } + if ( + (arm.commit && !provenance.gitHead.startsWith(arm.commit)) + || (arm.entrySha256 && provenance.entrySha256 !== arm.entrySha256) + || (arm.lockfileSha256 && provenance.lockfileSha256 !== arm.lockfileSha256) + || result.commit !== (arm.commit ?? null) + || result.introspectionMode !== arm.introspectionMode + ) { + throw new Error(`${label} arm provenance does not match its configured arm`); + } + if (!Array.isArray(result.provenanceErrors) || result.provenanceErrors.length > 0) { + throw new Error(`${label} contains provenance validation errors`); + } + if ( + typeof result.accepted !== 'boolean' + || !Array.isArray(result.failures) + || !Number.isSafeInteger(result.qualifiedCustomers) + || result.qualifiedCustomers < 0 + || result.qualifiedCustomers > result.configuredTenants + || result.qualifiedCustomers !== result.qualifiedTenants + || result.accepted !== (result.failures.length === 0) + || (result.accepted && ( + result.failures.length > 0 || result.qualifiedCustomers !== result.configuredTenants + )) + || (!result.accepted && result.qualifiedCustomers !== 0) + ) { + throw new Error(`${label} acceptance fields are internally inconsistent`); + } + const expectedOldSpaceDensity = result.qualifiedCustomers / (result.heapMiB / 1024); + if (!finite(result.tenantsPerConfiguredOldSpaceGiB) + || !closeEnough(result.tenantsPerConfiguredOldSpaceGiB, expectedOldSpaceDensity)) { + throw new Error(`${label} configured-old-space density is inconsistent`); + } + const densityPairs: Array<[number | null, number | null]> = [ + [result.alignedServicePeakBytes, result.customersPerAlignedServiceGiB], + [result.serviceMemoryUpperBoundBytes, result.customersPerServiceMemoryUpperBoundGiB], + [result.peakRssBytes, result.tenantsPerPeakRssGiB] + ]; + for (const [bytes, density] of densityPairs) { + if (bytes == null) { + if (density != null) throw new Error(`${label} density exists without its memory denominator`); + } else if ( + !finite(bytes) + || bytes <= 0 + || !finite(density) + || !closeEnough(density, result.qualifiedCustomers / (bytes / GIB)) + ) { + throw new Error(`${label} density does not match qualified customers and memory bytes`); + } + } + const configuredDensityPairs: Array<[number | null, number | null]> = [ + [result.alignedServicePeakBytes, result.configuredCustomersPerAlignedServiceGiB], + [ + result.serviceMemoryUpperBoundBytes, + result.configuredCustomersPerServiceMemoryUpperBoundGiB + ] + ]; + for (const [bytes, density] of configuredDensityPairs) { + if (bytes == null) { + if (density != null) throw new Error(`${label} diagnostic density has no denominator`); + } else if ( + !finite(bytes) + || bytes <= 0 + || !finite(density) + || !closeEnough(density, result.configuredCustomers / (bytes / GIB)) + ) { + throw new Error(`${label} configured-customer diagnostic density is inconsistent`); + } + } + if ( + result.requests !== result.workloadRequests + || result.achievedRps !== result.customerWorkloadRps + || !closeEnough( + result.combinedHttpRps, + result.customerWorkloadRps + + result.periodicValidationRps + + result.realtimeValidationRps + ) + || !finite(result.errorRate) + || !closeEnough( + result.errorRate, + result.requests > 0 ? result.errors / result.requests : 1 + ) + ) { + throw new Error(`${label} workload counters or rates are internally inconsistent`); + } + const realtime = result.realtimeDeliveryCoverage; + if (realtime != null) { + const startedAtMs = Date.parse(realtime.workloadStartedAt); + const deadlineAtMs = Date.parse(realtime.workloadDeadlineAt); + const endedAtMs = realtime.workloadEndedAt == null + ? NaN + : Date.parse(realtime.workloadEndedAt); + const expectedRoundsPerSurface = Math.max( + 0, + Math.ceil((deadlineAtMs - startedAtMs) / realtime.deliveryIntervalMs) - 1 + ); + const aggregate = realtime.surfaces.reduce((summary, surface) => ({ + expected: summary.expected + surface.expectedRecurringRounds, + started: summary.started + surface.startedRecurringRounds, + verified: summary.verified + surface.verifiedRecurringRounds, + primeRequests: summary.primeRequests + surface.primeRequests + }), { expected: 0, started: 0, verified: 0, primeRequests: 0 }); + const complete = ( + Number.isFinite(endedAtMs) + && endedAtMs >= deadlineAtMs + && realtime.startedRecurringRounds === realtime.expectedRecurringRounds + && realtime.verifiedRecurringRounds === realtime.expectedRecurringRounds + && realtime.deadlineLateRecurringRounds === 0 + && realtime.surfaces.every((surface) => + surface.issuedCorrelationSha256 === surface.verifiedCorrelationSha256 + ) + ); + const surfaceKeys = realtime.surfaces.map((surface) => + `${surface.tenantId}\0${surface.surface}\0${surface.route}` + ); + const expectedRealtimeSurfaceKeys = fleet.tenants + .slice(0, result.configuredTenants) + .flatMap((tenant) => tenant.surfaces + .filter((surface) => surface.realtime != null) + .map((surface) => { + const url = resolveTemplate(surface.url, { + port: arm.port, + mode: arm.introspectionMode + }); + return `${tenant.id}\0${surface.name}\0${new URL(url).pathname}`; + })) + .sort(); + if ( + realtime.version !== 2 + || !Number.isSafeInteger(realtime.deliveryIntervalMs) + || realtime.deliveryIntervalMs <= 0 + || !Number.isSafeInteger(realtime.primeRequests) + || realtime.primeRequests < 0 + || !finite(realtime.primeResponseP99Ms) + || realtime.primeResponseP99Ms < 0 + || !finite(realtime.deliveryP99Ms) + || realtime.deliveryP99Ms < 0 + || !Number.isFinite(startedAtMs) + || !Number.isFinite(deadlineAtMs) + || deadlineAtMs <= startedAtMs + || !Array.isArray(realtime.surfaces) + || new Set(surfaceKeys).size !== surfaceKeys.length + || JSON.stringify([...surfaceKeys].sort()) + !== JSON.stringify(expectedRealtimeSurfaceKeys) + || realtime.surfaces.some((surface) => + surface.expectedRecurringRounds !== expectedRoundsPerSurface + || !Number.isSafeInteger(surface.startedRecurringRounds) + || !Number.isSafeInteger(surface.verifiedRecurringRounds) + || surface.startedRecurringRounds < surface.verifiedRecurringRounds + || !Number.isSafeInteger(surface.primeRequests) + || surface.primeRequests < 0 + || !finite(surface.primeResponseP99Ms) + || surface.primeResponseP99Ms < 0 + || !finite(surface.deliveryP99Ms) + || surface.deliveryP99Ms < 0 + || !/^[a-f0-9]{64}$/.test(surface.issuedCorrelationSha256) + || !/^[a-f0-9]{64}$/.test(surface.verifiedCorrelationSha256) + ) + || aggregate.expected !== realtime.expectedRecurringRounds + || aggregate.started !== realtime.startedRecurringRounds + || aggregate.verified !== realtime.verifiedRecurringRounds + || aggregate.primeRequests !== realtime.primeRequests + || !closeEnough( + result.realtimeValidationRps, + result.durationSec > 0 ? realtime.primeRequests / result.durationSec : 0 + ) + || realtime.complete !== complete + || (result.accepted && !complete) + ) { + throw new Error(`${label} recurring realtime coverage is inconsistent`); + } + } else if (result.accepted) { + throw new Error(`${label} accepted without recurring realtime coverage evidence`); + } + if ( + !Array.isArray(result.tenants) + || (result.accepted && result.tenants.length !== result.configuredTenants) + || (!result.accepted && ![0, result.configuredTenants].includes(result.tenants.length)) + ) { + throw new Error(`${label} does not contain one scored result per configured customer`); + } + for (const tenant of result.tenants) { + if ( + !Array.isArray(tenant.surfaces) + || tenant.surfaces.length !== tenant.surfacesConfigured + || (tenant.qualified && !tenant.surfaces.every((surface) => surface.qualified)) + ) { + throw new Error(`${label} customer/surface qualification evidence is inconsistent`); + } + } + if (plan.gates.requireFreshPostgresRunAttestation) { + const attestation = result.postgresRunAttestation; + if ( + !attestation + || attestation.planSha256 !== `sha256:${planSha256}` + || attestation.fleetSha256 !== `sha256:${fleetSha256}` + || attestation.arm !== result.arm + || attestation.heapMiB !== result.heapMiB + || attestation.tenantCount !== result.configuredTenants + || attestation.repetition !== result.repetition + || attestation.runOrderIndex !== result.runOrderIndex + ) { + throw new Error(`${label} PostgreSQL attestation does not match its matrix coordinate`); + } + } + return result; + }); + const missing = [...expected].filter((coordinate) => !seen.has(coordinate)).sort(); + const duplicates = [...seen] + .filter(([_coordinate, count]) => count !== 1) + .map(([coordinate]) => coordinate) + .sort(); + const soakExpected = plan.soak?.enabled === true; + const soakResults = results.filter((result) => result.runKind === 'soak'); + const soakComplete = !soakExpected + ? soakObserved === 0 + : soakObserved === 1 + && soakResults[0].evidenceMode === 'qualification' + && soakResults[0].accepted; + return { + results, + matrix: { + complete: plan.qualification != null + && missing.length === 0 + && duplicates.length === 0 + && diagnostic.length === 0 + && seen.size === expected.size + && input.length === campaign.manifest.jobs.length + && canonicalQualificationSchedule + && campaign.evidenceMode === 'qualification' + && campaign.manifest.platform === 'linux' + && campaign.qualificationBlockers.length === 0 + && hostileEvidenceReady + && soakComplete, + expectedCoordinates: expected.size, + observedCoordinates: seen.size, + missing, + duplicates, + diagnostic: diagnostic.sort(), + soakExpected, + soakObserved, + soakComplete + } + }; +}; + +const formatNumber = (value: number | null, digits = 2): string => value == null + ? 'n/a' + : Number.isFinite(value) ? value.toFixed(digits) : String(value); + +const medianNullable = (values: Array): number | null => { + const available = values.filter((value): value is number => value != null); + return available.length > 0 ? percentile(available, 0.5) : null; +}; + +const toMiB = (value: number | null): number | null => value == null + ? null + : value / 1024 ** 2; + +export const rejectDuplicatePostgresRunEpochs = ( + input: DensityRunResult[] +): DensityRunResult[] => { + const counts = new Map(); + for (const run of input) { + const evidence = run.postgresRunAttestation; + if (!evidence) continue; + for (const claim of postgresRunIdentityClaims(evidence)) { + counts.set(claim, (counts.get(claim) ?? 0) + 1); + } + } + const duplicates = new Set([...counts] + .filter(([_claim, count]) => count > 1) + .map(([claim]) => claim)); + return input.map((run) => { + const evidence = run.postgresRunAttestation; + if (!evidence) return run; + const reused = postgresRunIdentityClaims(evidence) + .filter((claim) => duplicates.has(claim)); + if (reused.length === 0) return run; + const failure = `PostgreSQL container/clone identities reused across matrix: ${reused.join(', ')}`; + return { + ...run, + accepted: false, + qualifiedCustomers: 0, + qualifiedTenants: 0, + tenantsPerConfiguredOldSpaceGiB: 0, + tenantsPerPeakRssGiB: 0, + customersPerAlignedServiceGiB: 0, + customersPerServiceMemoryUpperBoundGiB: 0, + failures: run.failures.includes(failure) + ? run.failures + : [...run.failures, failure] + }; + }); +}; + +export const renderReport = ( + inputResults: unknown[], + plan: DensityPlanV1, + fleet: FleetV1 +): string => { + const validated = validateResultSet(inputResults, plan, fleet); + const results = rejectDuplicatePostgresRunEpochs(validated.results); + const matrixResults = results.filter((result) => result.runKind === 'matrix'); + const soakResults = results.filter((result) => result.runKind === 'soak'); + const soakComplete = !validated.matrix.soakExpected + ? soakResults.length === 0 + : soakResults.length === 1 + && soakResults[0].evidenceMode === 'qualification' + && soakResults[0].accepted; + const qualificationEvidenceComplete = validated.matrix.complete && soakComplete; + const gates = plan.gates; + const groups = new Map(); + for (const result of matrixResults) { + const key = `${result.arm}|${result.heapMiB}|${result.configuredTenants}`; + const group = groups.get(key) ?? []; + group.push(result); + groups.set(key, group); + } + const lines = [ + '# Graphile customer-density results', + '', + `Generated: ${new Date().toISOString()}`, + '', + `Evidence mode: **${qualificationEvidenceComplete ? 'qualification' : 'diagnostic'}**. Full configured matrix: ${validated.matrix.observedCoordinates}/${validated.matrix.expectedCoordinates} coordinates; missing=${validated.matrix.missing.length}; duplicates=${validated.matrix.duplicates.length}; diagnostic-only=${validated.matrix.diagnostic.length}; configured soak=${validated.matrix.soakExpected ? `${validated.matrix.soakObserved}/1, accepted=${soakComplete ? 'yes' : 'no'}` : 'disabled'}.`, + '', + 'A customer counts only when every declared GraphQL surface and realtime transport stays resident and serves the full qualification workload, all isolation canaries are conclusive, bleed is zero, error rate and p99 meet their gates, required capabilities ran, PostgreSQL telemetry completed, and post-warmup Graphile and PostgreSQL pool eviction/refusal/build/disposal counters remain unchanged. The primary memory denominator is the maximum post-warmup time-aligned sum of current Node RSS and raw PostgreSQL cgroup-v2 memory charge when available; the all-phase high-water upper bound, Docker working set, configured V8, and Node-only RSS remain diagnostics.', + '', + 'Qualifying physical-database runs use a full live DDL/ACL audit before the Graphile timer starts. That audit intentionally warms PostgreSQL catalogs, so the build column is post-attestation warm-catalog latency, not a pristine-catalog cold-build claim. Every arm receives the same audit, and reused container/clone epochs are rejected across the complete result set.', + '', + '| Arm | Old-space MiB | Customers | Physical DBs | Dedicated PG | Runs | Accepted | Warm observed heap delta MiB/instance | Post-attestation build ms | PG baseline MiB | PG warm-boundary MiB | PG spike MiB | PG raw peak MiB | PG working-set peak MiB | Node peak RSS MiB | Aligned Node+PG peak MiB | Conservative service upper bound MiB | Offered RPS | Customer workload RPS | Periodic validation RPS | Realtime validation RPS | Combined HTTP RPS | workload p99 ms | PG pools | PG active leases | PG backends | Pool clients | Realtime managers | Realtime transports | Qualified customers/aligned service GiB | Qualified customers/service upper-bound GiB | Configured customers/aligned service GiB (diagnostic) | Configured customers/service upper-bound GiB (diagnostic) | Customers/configured old-space GiB | Customers/Node peak RSS GiB |', + '|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|' + ]; + for (const group of [...groups.values()].sort((a, b) => { + const left = `${a[0].arm}:${a[0].heapMiB}:${a[0].configuredTenants}`; + const right = `${b[0].arm}:${b[0].heapMiB}:${b[0].configuredTenants}`; + return left.localeCompare(right); + })) { + const first = group[0]; + lines.push([ + `| ${first.arm}`, + first.heapMiB, + first.configuredTenants, + formatNumber(medianNullable(group.map( + (run) => run.residentPhysicalDatabases ?? null + )), 0), + group.every((run) => run.postgresContainerDedicated === true) ? 'yes' : 'no', + group.length, + group.filter((run) => run.accepted).length, + formatNumber(toMiB(medianNullable(group.map( + (run) => run.warmObservedHeapDeltaPerInstanceBytes + ))), 1), + formatNumber(medianNullable(group.map((run) => run.coldBuildMaxMs)), 1), + formatNumber(toMiB(medianNullable(group.map((run) => run.postgresBaselineBytes))), 1), + formatNumber(toMiB(medianNullable(group.map( + (run) => run.postgresWarmBoundaryBytes + ))), 1), + formatNumber(toMiB(medianNullable(group.map((run) => run.postgresColdBuildSpikeBytes))), 1), + formatNumber(toMiB(medianNullable(group.map((run) => run.postgresPeakBytes))), 1), + formatNumber(toMiB(medianNullable(group.map( + (run) => run.postgresWorkingSetPeakBytes ?? null + ))), 1), + formatNumber(toMiB(medianNullable(group.map((run) => run.peakRssBytes))), 1), + formatNumber(toMiB(medianNullable(group.map((run) => run.alignedServicePeakBytes))), 1), + formatNumber(toMiB(medianNullable(group.map((run) => run.serviceMemoryUpperBoundBytes))), 1), + formatNumber(percentile(group.map((run) => run.offeredLoad.totalRps), 0.5), 1), + formatNumber(percentile(group.map( + (run) => run.customerWorkloadRps ?? run.achievedRps + ), 0.5), 1), + formatNumber(percentile(group.map( + (run) => run.periodicValidationRps ?? 0 + ), 0.5), 1), + formatNumber(percentile(group.map( + (run) => run.realtimeValidationRps ?? 0 + ), 0.5), 1), + formatNumber(percentile(group.map((run) => + run.combinedHttpRps + ?? (run.customerWorkloadRps ?? run.achievedRps) + + (run.periodicValidationRps ?? 0) + + (run.realtimeValidationRps ?? 0) + ), 0.5), 1), + formatNumber(percentile(group.map((run) => run.p99Ms), 0.5), 1), + formatNumber(medianNullable(group.map((run) => run.pgPoolCacheSize)), 0), + formatNumber(medianNullable(group.map((run) => run.pgPoolActiveLeases)), 0), + formatNumber(medianNullable(group.map((run) => run.postgresBackendPeak ?? null)), 0), + formatNumber(medianNullable(group.map((run) => run.pgPoolTotalClients ?? null)), 0), + formatNumber(medianNullable(group.map((run) => run.residentRealtimeManagers ?? null)), 0), + formatNumber(medianNullable(group.map((run) => run.residentRealtimeTransports ?? null)), 0), + formatNumber(medianNullable(group.map((run) => run.customersPerAlignedServiceGiB))), + formatNumber(medianNullable(group.map( + (run) => run.customersPerServiceMemoryUpperBoundGiB + ))), + formatNumber(medianNullable(group.map( + (run) => run.configuredCustomersPerAlignedServiceGiB + ))), + formatNumber(medianNullable(group.map( + (run) => run.configuredCustomersPerServiceMemoryUpperBoundGiB + ))), + formatNumber(percentile(group.map( + (run) => run.tenantsPerConfiguredOldSpaceGiB + ), 0.5)), + `${formatNumber(medianNullable(group.map((run) => run.tenantsPerPeakRssGiB)))} |` + ].join(' | ')); + } + + const boundaries = summarizeCapacityBoundaries(matrixResults); + lines.push( + '', + '## Capacity boundaries', + '', + '| Arm | Old-space MiB | Highest all-repetitions customer pass | Lowest greater fail | Monotonic | Boundary reached | Incomplete counts | Customers/aligned service GiB | Customers/service upper-bound GiB | Customers/configured old-space GiB | Customers/Node peak RSS GiB |', + '|---|---:|---:|---:|---:|---:|---|---:|---:|---:|---:|', + ...boundaries.map((boundary) => [ + `| ${boundary.arm}`, + boundary.heapMiB, + boundary.highestAllRepetitionsPass ?? 'n/a', + boundary.lowestGreaterFail ?? 'n/a', + boundary.monotonicQualification ? 'yes' : 'no', + boundary.capacityBoundaryReached ? 'yes' : 'no', + boundary.incompleteTenantCounts.join(',') || 'none', + formatNumber(boundary.medianCustomersPerAlignedServiceGiB), + formatNumber(boundary.medianCustomersPerServiceMemoryUpperBoundGiB), + formatNumber(boundary.medianTenantsPerConfiguredOldSpaceGiB), + `${formatNumber(boundary.medianTenantsPerPeakRssGiB)} |` + ].join(' | ')), + '', + '## Candidate decisions', + '', + ); + const baselineArm = plan.qualification?.baselineArm ?? plan.arms[0]?.name; + const baseline = matrixResults.filter((result) => result.arm === baselineArm); + const candidateArms = plan.arms + .map((arm) => arm.name) + .filter((arm) => + arm !== baselineArm && matrixResults.some((result) => result.arm === arm) + ); + if (!plan.qualification) { + lines.push('This plan is diagnostic-only; it has no qualification contract.', ''); + } else if (candidateArms.length === 0) { + lines.push('No configured candidate arms were executed.', ''); + } else { + for (const candidateArm of candidateArms) { + const candidate = matrixResults.filter((result) => result.arm === candidateArm); + const comparison = compareDensity(baseline, candidate, gates); + const materiallyBetter = qualificationEvidenceComplete && comparison.materiallyBetter; + lines.push( + `### ${candidateArm} vs ${baselineArm}`, + '', + `Materially better: **${materiallyBetter ? 'yes' : 'no'}**. Median aligned Node+PostgreSQL density improvement: ${formatNumber(comparison.alignedServiceMedianImprovement * 100, 1)}%; conservative service upper-bound density improvement: ${formatNumber(comparison.serviceMemoryUpperBoundMedianImprovement * 100, 1)}%; both actual service-memory measures avoid per-heap regression: ${comparison.alignedServiceNonRegression && comparison.serviceMemoryUpperBoundNonRegression ? 'yes' : 'no'}; configured-old-space diagnostic improvement: ${formatNumber(comparison.configuredOldSpaceMedianImprovement * 100, 1)}%; Node-only peak-RSS diagnostic improvement: ${formatNumber(comparison.peakRssMedianImprovement * 100, 1)}%; every paired heap adds the required customer count: ${comparison.everyHeapAddsTenants ? 'yes' : 'no'}; capacity boundaries are complete: ${comparison.capacityBoundariesComplete ? 'yes' : 'no'}; matrices are exactly paired: ${comparison.pairedMatrixComplete ? 'yes' : 'no'}; full configured qualification evidence, including soak when enabled, is present: ${qualificationEvidenceComplete ? 'yes' : 'no'}.`, + '' + ); + } + } + lines.push( + 'Failed and incomplete runs remain in the denominator. Missing arms, heaps, tenant counts, repetitions, short smoke workloads, unavailable memory telemetry, and inconclusive canaries are not treated as passing evidence.', + '', + '## Soak runs', + '', + 'Soak records validate the selected maximum-density candidate over time; they are excluded from every matrix median, capacity boundary, and candidate comparison above.', + '', + '| Arm | Old-space MiB | Customers | Duration sec | Accepted | workload p99 ms | Aligned Node+PG peak MiB | Customers/aligned service GiB | Heap growth MiB/hour |', + '|---|---:|---:|---:|---:|---:|---:|---:|---:|' + ); + if (soakResults.length === 0) { + lines.push('| none | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a |'); + } else { + for (const result of soakResults) { + lines.push([ + `| ${result.arm}`, + result.heapMiB, + result.configuredTenants, + formatNumber(result.durationSec, 0), + result.accepted ? 'yes' : 'no', + formatNumber(result.p99Ms, 1), + formatNumber(toMiB(result.alignedServicePeakBytes), 1), + formatNumber(result.customersPerAlignedServiceGiB), + `${formatNumber(result.retainedHeapGrowthMiBPerHour)} |` + ].join(' | ')); + } + } + lines.push( + '', + '## Run failures', + '' + ); + const failed = results.filter((result) => !result.accepted); + if (failed.length === 0) lines.push('None.'); + else for (const result of failed) { + lines.push(`- ${result.arm} h${result.heapMiB} t${result.configuredTenants} r${result.repetition}: ${result.failures.join('; ')}`); + } + return `${lines.join('\n')}\n`; +}; + +export const writeReport = ( + resultsFile: string, + outputFile: string, + plan: DensityPlanV1, + fleet: FleetV1 +): void => { + const report = renderReport(readResults(resultsFile), plan, fleet); + fs.mkdirSync(path.dirname(path.resolve(outputFile)), { recursive: true }); + fs.writeFileSync(path.resolve(outputFile), report, 'utf8'); +}; diff --git a/packages/perf-harness/src/run-attestation.ts b/packages/perf-harness/src/run-attestation.ts new file mode 100644 index 0000000000..9c195b4d9a --- /dev/null +++ b/packages/perf-harness/src/run-attestation.ts @@ -0,0 +1,364 @@ +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { resolveTemplate } from './config'; +import type { + ArmPlan, + PostgresRunAttestationEvidence +} from './types'; + +const SHA256 = /^sha256:[a-f0-9]{64}$/; +const CONTAINER_ID = /^[a-f0-9]{64}$/; +const KIND = 'physical-density-measurement-attestation-v1'; +const COMMAND_KILL_GRACE_MS = 2_000; + +export interface RunAttestationContext { + arm: string; + heapMiB: number; + tenantCount: number; + repetition: number; + runOrderIndex: number; + planSha256: string; + fleetSha256: string; + notBeforeEpochMs: number; + artifactDir: string; +} + +export const postgresRunIdentityClaims = ( + evidence: PostgresRunAttestationEvidence +): string[] => [ + `epoch:${evidence.epochId}`, + `container:${evidence.containerId}`, + `cgroup:${evidence.cgroupIdentitySha256}`, + `postgres-system:${evidence.postgresSystemIdentifier}`, + `clone:${evidence.cloneId}`, + `clone-attestation-set:${evidence.cloneAttestationSetSha256}`, + `clone-nonce-set:${evidence.cloneNonceSetSha256}` +]; + +const canonicalize = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + const record = value as Record; + return Object.fromEntries(Object.keys(record).sort().map((key) => [ + key, + canonicalize(record[key]) + ])); +}; + +const canonicalSha256 = (value: unknown): string => `sha256:${createHash('sha256') + .update(JSON.stringify(canonicalize(value))) + .digest('hex')}`; + +const readRegularFile = (file: string): Buffer => { + const before = fs.lstatSync(file); + if (before.isSymbolicLink() || !before.isFile()) { + throw new Error('PostgreSQL run evidence must be a regular non-symlink file'); + } + const descriptor = fs.openSync( + file, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) + ); + try { + const opened = fs.fstatSync(descriptor); + if ( + !opened.isFile() + || opened.dev !== before.dev + || opened.ino !== before.ino + ) { + throw new Error('PostgreSQL run evidence changed while it was opened'); + } + return fs.readFileSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +}; + +const fileSha256 = (file: string): string => `sha256:${createHash('sha256') + .update(readRegularFile(file)) + .digest('hex')}`; + +const runCommand = async ( + command: string[], + cwd: string, + timeoutMs: number +): Promise => { + if (command.length === 0) throw new Error('PostgreSQL run command is empty'); + await new Promise((resolve, reject) => { + const child = spawn(command[0], command.slice(1), { + cwd, + env: process.env, + detached: process.platform !== 'win32', + stdio: 'ignore' + }); + let settled = false; + let timedOut = false; + let forceTimer: NodeJS.Timeout | null = null; + const signalTree = (signal: NodeJS.Signals): void => { + if (child.pid && process.platform !== 'win32') { + try { + process.kill(-child.pid, signal); + return; + } catch { + // Fall through to the direct child as a best-effort Windows/fork + // fallback. The child exit remains the completion boundary. + } + } + child.kill(signal); + }; + const clearTimers = (): void => { + clearTimeout(timer); + if (forceTimer) clearTimeout(forceTimer); + }; + const timer = setTimeout(() => { + if (settled) return; + timedOut = true; + signalTree('SIGTERM'); + forceTimer = setTimeout(() => signalTree('SIGKILL'), COMMAND_KILL_GRACE_MS); + }, timeoutMs); + child.once('error', (error) => { + if (settled) return; + settled = true; + clearTimers(); + reject(error); + }); + child.once('exit', (code, signal) => { + if (settled) return; + settled = true; + clearTimers(); + if (timedOut) reject(new Error('PostgreSQL run command timed out')); + else if (code === 0 && signal == null) resolve(); + else reject(new Error( + `PostgreSQL run command failed: code=${code ?? 'null'} signal=${signal ?? 'null'}` + )); + }); + }); +}; + +const requireRecord = (value: unknown, label: string): Record => { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`PostgreSQL run attestation ${label} is invalid`); + } + return value as Record; +}; + +export const normalizePostgresRunAttestation = ( + raw: unknown, + context: RunAttestationContext, + artifactPath: string, + artifactSha256 = fileSha256(artifactPath) +): PostgresRunAttestationEvidence => { + const envelope = requireRecord(raw, 'envelope'); + const payload = requireRecord(envelope.payload, 'payload'); + const run = requireRecord(payload.run, 'run binding'); + const freshness = requireRecord(payload.freshness, 'freshness'); + const immutableEpoch = requireRecord(payload.immutableEpoch, 'immutable epoch'); + const container = requireRecord(payload.container, 'container'); + const cgroup = requireRecord(payload.cgroup, 'cgroup'); + const postgres = requireRecord(payload.postgres, 'PostgreSQL cluster'); + const customerAudits = payload.customerAudits; + const provisionClone = requireRecord(payload.provisionClone, 'provision clone'); + const orderedCustomerAudits = Array.isArray(customerAudits) + ? [...customerAudits].sort((left, right) => + String(left?.customerId).localeCompare(String(right?.customerId))) + : []; + const cloneNonceSetSha256 = canonicalSha256(orderedCustomerAudits.map((audit) => ({ + customerId: audit.customerId, + cloneNonceSha256: audit.cloneNonceSha256 + }))); + const liveContractSetSha256 = canonicalSha256(orderedCustomerAudits.map((audit) => ({ + customerId: audit.customerId, + databaseContractFingerprint: audit.databaseContractFingerprint, + structuralFingerprint: audit.structuralFingerprints?.combined?.sha256 + }))); + const observedAtMs = Date.parse(payload.observedAt ?? ''); + const containerStartedAtMs = Date.parse(container.startedAt ?? ''); + const postgresStartedAtMs = Date.parse(postgres.postmasterStartedAt ?? ''); + if ( + envelope.version !== 1 + || envelope.kind !== KIND + || !SHA256.test(envelope.payloadSha256 ?? '') + || canonicalSha256(payload) !== envelope.payloadSha256 + || run.arm !== context.arm + || run.heapMiB !== context.heapMiB + || run.customerCount !== context.tenantCount + || run.repetition !== context.repetition + || run.runOrderIndex !== context.runOrderIndex + || run.planSha256 !== `sha256:${context.planSha256}` + || run.fleetSha256 !== `sha256:${context.fleetSha256}` + || !SHA256.test(payload.epochId ?? '') + || payload.epochId !== canonicalSha256(immutableEpoch) + || !CONTAINER_ID.test(container.id ?? '') + || typeof container.startedAt !== 'string' + || !Number.isSafeInteger(observedAtMs) + || !Number.isSafeInteger(containerStartedAtMs) + || !Number.isSafeInteger(postgresStartedAtMs) + || observedAtMs < context.notBeforeEpochMs + || containerStartedAtMs > observedAtMs + || postgresStartedAtMs > observedAtMs + || !SHA256.test(cgroup.identitySha256 ?? '') + || cgroup.version !== 1 + || cgroup.source !== 'container-cgroup-v2' + || typeof freshness.freshContainerForRun !== 'boolean' + || freshness.freshContainerForRun + !== (containerStartedAtMs >= context.notBeforeEpochMs) + || freshness.cgroupV2Verified !== true + || freshness.notBeforeEpochMs !== context.notBeforeEpochMs + || freshness.startToleranceMs !== 0 + || payload.catalogCacheState !== 'warmed-by-live-contract-audit' + || provisionClone.purpose !== 'measurement' + || provisionClone.version !== 1 + || typeof provisionClone.id !== 'string' + || !provisionClone.id + || !SHA256.test(provisionClone.attestationSetSha256 ?? '') + || !SHA256.test(payload.manifestSha256 ?? '') + || !SHA256.test(payload.containerTemplateSha256 ?? '') + || !SHA256.test(payload.canonicalDatabaseContractFingerprint ?? '') + || !/^\d+$/.test(postgres.systemIdentifier ?? '') + || !Array.isArray(customerAudits) + || customerAudits.length !== context.tenantCount + || new Set(customerAudits.map((audit: any) => audit?.customerId)).size + !== context.tenantCount + || customerAudits.some((audit: any) => + typeof audit?.customerId !== 'string' + || !audit.customerId + || !SHA256.test(audit?.databaseContractFingerprint ?? '') + || !SHA256.test(audit?.structuralFingerprints?.combined?.sha256 ?? '') + || !SHA256.test(audit?.cloneAttestationSha256 ?? '') + || !SHA256.test(audit?.cloneNonceSha256 ?? '') + ) + || immutableEpoch.dockerContainerId !== container.id + || immutableEpoch.dockerStartedAt !== container.startedAt + || !SHA256.test(immutableEpoch.containerConfigurationSha256 ?? '') + || immutableEpoch.cgroupIdentitySha256 !== cgroup.identitySha256 + || immutableEpoch.postgresSystemIdentifier !== postgres.systemIdentifier + || immutableEpoch.postgresStartedAt !== postgres.postmasterStartedAt + || immutableEpoch.cloneId !== provisionClone.id + || immutableEpoch.cloneAttestationSetSha256 + !== provisionClone.attestationSetSha256 + || immutableEpoch.cloneNonceSetSha256 !== cloneNonceSetSha256 + || immutableEpoch.liveContractSetSha256 !== liveContractSetSha256 + || !SHA256.test(immutableEpoch.cloneNonceSetSha256 ?? '') + || !SHA256.test(immutableEpoch.liveContractSetSha256 ?? '') + ) { + throw new Error('PostgreSQL run attestation failed exact validation'); + } + return { + version: 1, + kind: KIND, + artifactPath, + artifactSha256, + payloadSha256: envelope.payloadSha256, + epochId: payload.epochId, + arm: run.arm, + heapMiB: run.heapMiB, + tenantCount: run.customerCount, + repetition: run.repetition, + runOrderIndex: run.runOrderIndex, + planSha256: run.planSha256, + fleetSha256: run.fleetSha256, + containerId: container.id, + containerStartedAt: container.startedAt, + cgroupIdentitySha256: cgroup.identitySha256, + containerConfigurationSha256: + immutableEpoch.containerConfigurationSha256, + postgresSystemIdentifier: immutableEpoch.postgresSystemIdentifier, + postgresStartedAt: immutableEpoch.postgresStartedAt, + cloneId: provisionClone.id, + cloneAttestationSetSha256: immutableEpoch.cloneAttestationSetSha256, + cloneNonceSetSha256: immutableEpoch.cloneNonceSetSha256, + liveContractSetSha256: immutableEpoch.liveContractSetSha256, + manifestSha256: payload.manifestSha256, + containerTemplateSha256: payload.containerTemplateSha256, + canonicalDatabaseContractFingerprint: + payload.canonicalDatabaseContractFingerprint, + freshContainerForRun: freshness.freshContainerForRun, + cgroupV2Verified: freshness.cgroupV2Verified, + liveCustomerContractsAudited: customerAudits.length, + catalogCacheState: payload.catalogCacheState + }; +}; + +export const collectPostgresRunAttestation = async ( + arm: ArmPlan, + context: RunAttestationContext +): Promise => { + const configured = arm.postgresRunAttestation; + if (!configured) return null; + const artifactPath = path.join( + context.artifactDir, + 'postgres-run-attestation.json' + ); + const postgresFixtureDir = path.join(context.artifactDir, 'postgres-fixture'); + const postgresManifestFile = path.join(postgresFixtureDir, 'provision.json'); + const postgresSecretsFile = path.join(postgresFixtureDir, 'runtime-secrets.json'); + if (fs.existsSync(artifactPath)) { + throw new Error('PostgreSQL run attestation artifact already exists'); + } + const variables = { + arm: context.arm, + heapMiB: context.heapMiB, + tenantCount: context.tenantCount, + repetition: context.repetition, + runOrderIndex: context.runOrderIndex, + planSha256: `sha256:${context.planSha256}`, + fleetSha256: `sha256:${context.fleetSha256}`, + notBeforeEpochMs: context.notBeforeEpochMs, + artifactDir: context.artifactDir, + attestationFile: artifactPath, + postgresFixtureDir, + postgresManifestFile, + postgresSecretsFile, + port: arm.port, + mode: arm.introspectionMode + }; + const cwd = path.resolve(arm.cwd + ? resolveTemplate(arm.cwd, variables) + : process.cwd()); + const timeoutMs = configured.timeoutMs ?? 900_000; + if (configured.prepareCommand?.length) { + await runCommand( + configured.prepareCommand.map((part) => resolveTemplate(part, variables)), + cwd, + timeoutMs + ); + } + await runCommand( + configured.command.map((part) => resolveTemplate(part, variables)), + cwd, + timeoutMs + ); + const stat = fs.lstatSync(artifactPath); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new Error('PostgreSQL run attestation must be a regular non-symlink file'); + } + const manifestStat = fs.lstatSync(postgresManifestFile); + const secretsStat = fs.lstatSync(postgresSecretsFile); + if ( + manifestStat.isSymbolicLink() + || !manifestStat.isFile() + || secretsStat.isSymbolicLink() + || !secretsStat.isFile() + || (secretsStat.mode & 0o777) !== 0o600 + ) { + throw new Error('PostgreSQL run fixture inputs failed private-file validation'); + } + const artifactBytes = readRegularFile(artifactPath); + const artifactSha256 = `sha256:${createHash('sha256') + .update(artifactBytes) + .digest('hex')}`; + const raw = JSON.parse(artifactBytes.toString('utf8')) as unknown; + const evidence = normalizePostgresRunAttestation( + raw, + context, + artifactPath, + artifactSha256 + ); + const manifestSha256 = fileSha256(postgresManifestFile); + if (manifestSha256 !== evidence.manifestSha256) { + throw new Error('PostgreSQL run manifest does not match its attestation'); + } + return evidence; +}; diff --git a/packages/perf-harness/src/run.ts b/packages/perf-harness/src/run.ts new file mode 100644 index 0000000000..07a746cccb --- /dev/null +++ b/packages/perf-harness/src/run.ts @@ -0,0 +1,856 @@ +import { createHash, randomBytes } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { + armEnvironmentForHeap, + assertLoopbackObservabilityUrl, + assertLoopbackRetainedHeapCheckpointUrl, + DEFAULT_RUN_ORDER_SEED, + hasExactHostileValidationEvidence, + resolveTenants, + resolveTemplate, + soakArmName, + tenantCountsForHeap, + validateCoverage +} from './config'; +import { + createWorkloadCapture, + resolveOfferedLoad, + resolveWarmupTimeoutMs, + runWorkload, + type WorkloadCapture, + type WorkloadResult +} from './http'; +import { bindResultEvidence, writeScoreContext } from './evidence'; +import { + normalizeRetainedMemoryCheckpoint, + startMemorySampler +} from './memory'; +import { startPostgresMemorySampler } from './postgres'; +import { startArmProcess } from './process'; +import { createRealtimeDriver, type RealtimeDriverSnapshot } from './realtime'; +import { + buildRunSchedule, + sameRunSchedule, + scheduleJobsForPlan, + scheduleManifestSha256, + type CampaignScheduleManifestV1 +} from './schedule'; +import { + collectPostgresRunAttestation, + postgresRunIdentityClaims +} from './run-attestation'; +import { + scoreRun, + summarizeCapacityBoundaries, + type ScoreInput +} from './score'; +import type { + ArmPlan, + ArmProvenance, + DensityPlanV1, + DensityRunResult, + FleetV1, + PostgresRunAttestationEvidence, + ResolvedMemoryPolicy, + RetainedMemoryCheckpoint, + RetainedMemoryCheckpointPair, + RealtimeDeliveryCoverage, + WorkloadPlan +} from './types'; + +export interface RunSelection { + arms?: string[]; + heaps?: number[]; + tenantCounts?: number[]; + repetitions?: number; + smoke?: boolean; +} + +export { buildRunSchedule } from './schedule'; + +interface RunContext { + expectedMatrixRepetitions: number; + runKind: 'matrix' | 'soak'; + evidenceMode: 'qualification' | 'diagnostic'; + campaignId: string; + scheduleSha256: string; + previousResultPayloadSha256: string | null; + qualificationCohortSha256: string; + runOrderSeed: string; + runOrderIndex: number; + planSha256: string; + fleetSha256: string; + notBeforeEpochMs: number; + claimPostgresRunIdentity( + evidence: PostgresRunAttestationEvidence + ): string | null; +} + +const sha256 = (value: string): string => createHash('sha256').update(value).digest('hex'); + +const executionErrorEvidence = (error: unknown): string => { + const message = error instanceof Error ? error.message : String(error); + const code = message.match(/(?:^|\b)([A-Z][A-Z0-9_]{2,})(?=\b|:)/)?.[1] + ?? 'CPERF_EXECUTION_FAILED'; + return `${code}:sha256:${sha256(message)}`; +}; + +const writeResult = ( + root: string, + result: DensityRunResult, + input: ScoreInput, + context: Pick< + RunContext, + | 'planSha256' + | 'fleetSha256' + | 'notBeforeEpochMs' + | 'campaignId' + | 'scheduleSha256' + | 'previousResultPayloadSha256' + > +): void => { + fs.mkdirSync(result.artifactDir, { recursive: true }); + writeScoreContext(result.artifactDir, input, context); + bindResultEvidence(result); + fs.writeFileSync( + path.join(result.artifactDir, 'result.json'), + `${JSON.stringify(result, null, 2)}\n`, + 'utf8' + ); + fs.mkdirSync(root, { recursive: true }); + const serialized = `${JSON.stringify(result)}\n`; + fs.appendFileSync(path.join(root, 'results.ndjson'), serialized, 'utf8'); + fs.appendFileSync( + path.join(root, `results-${context.campaignId}.ndjson`), + serialized, + 'utf8' + ); +}; + +const writeJson = (file: string, value: unknown): void => { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, 'utf8'); +}; + +const writeExclusiveJson = (file: string, value: unknown): void => { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, { + encoding: 'utf8', + flag: 'wx' + }); +}; + +const invokePostWarmupHook = async ( + arm: ArmPlan, + heapMiB: number, + tenantCount: number, + artifactDir: string, + headers: Readonly> +): Promise => { + if (!arm.postWarmupUrl) return; + const url = resolveTemplate(arm.postWarmupUrl, { + heapMiB, + port: arm.port, + artifactDir, + mode: arm.introspectionMode, + tenantCount + }); + const parsed = new URL(url); + if ( + parsed.protocol !== 'http:' + || !['127.0.0.1', 'localhost', '::1'].includes(parsed.hostname) + || Number(parsed.port) !== arm.port + || parsed.username + || parsed.password + || parsed.search + || parsed.hash + ) { + throw new Error(`postWarmupUrl must be an authenticated loopback URL on port ${arm.port}`); + } + const response = await fetch(url, { + method: 'POST', + headers, + signal: AbortSignal.timeout(60_000) + }); + const responseText = await response.text(); + if (!response.ok) { + throw new Error( + `post-warmup hook failed with HTTP ${response.status}: ${responseText.slice(0, 512)}` + ); + } + let responseBody: unknown = responseText; + try { + responseBody = responseText ? JSON.parse(responseText) : null; + } catch { + // The hook contract permits a diagnostic text response. + } + writeJson(path.join(artifactDir, 'post-warmup-hook.json'), { + timestamp: new Date().toISOString(), + url: parsed.pathname, + response: responseBody + }); +}; + +const invokeRetainedMemoryCheckpoint = async ( + arm: ArmPlan, + heapMiB: number, + tenantCount: number, + headers: Readonly>, + errors: string[] +): Promise => { + if (!arm.retainedHeapCheckpointUrl) return null; + const url = resolveTemplate(arm.retainedHeapCheckpointUrl, { + heapMiB, + port: arm.port, + artifactDir: '', + mode: arm.introspectionMode, + tenantCount + }); + assertLoopbackRetainedHeapCheckpointUrl(url, arm.port); + try { + const response = await fetch(url, { + method: 'POST', + headers, + signal: AbortSignal.timeout(60_000) + }); + const responseText = await response.text(); + let body: unknown = null; + try { + body = responseText ? JSON.parse(responseText) : null; + } catch { + errors.push('retained-memory checkpoint returned non-JSON data'); + } + const checkpoint = normalizeRetainedMemoryCheckpoint(body); + if (!checkpoint) { + const serverMessage = typeof (body as any)?.error?.message === 'string' + ? `: ${(body as any).error.message}` + : ''; + errors.push( + `retained-memory checkpoint response was invalid (HTTP ${response.status})${serverMessage}` + ); + return null; + } + if (!response.ok) { + errors.push(`retained-memory checkpoint failed with HTTP ${response.status}`); + } + return checkpoint; + } catch (error) { + errors.push( + `retained-memory checkpoint request failed: ${error instanceof Error ? error.message : String(error)}` + ); + return null; + } +}; + +const resolvedMemoryPolicy = ( + arm: ArmPlan, + heapMiB: number, + expectedV8HeapLimitBytes: number | null +): ResolvedMemoryPolicy => { + const env = { ...process.env, ...armEnvironmentForHeap(arm, heapMiB) }; + const value = (name: string): string | null => env[name]?.trim() || null; + return { + configuredMaxOldSpaceMiB: heapMiB, + expectedV8HeapLimitBytes, + graphileCacheMax: value('GRAPHILE_CACHE_MAX'), + graphileCacheInstanceHeapBytes: value('GRAPHILE_CACHE_INSTANCE_HEAP_BYTES'), + graphileCacheServerReserveBytes: value('GRAPHILE_CACHE_SERVER_RESERVE_BYTES'), + graphileCacheBuildReserveBytes: value('GRAPHILE_CACHE_BUILD_RESERVE_BYTES'), + graphileCacheRssLimitBytes: value('GRAPHILE_CACHE_RSS_LIMIT_BYTES'), + graphileCacheRssBuildReserveBytes: value('GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES'), + graphileCacheCalibrationId: value('GRAPHILE_CACHE_CALIBRATION_ID'), + graphileCacheAdmissionMode: value('GRAPHILE_CACHE_ADMISSION_MODE'), + graphileBuildMaxConcurrency: value('GRAPHILE_BUILD_MAX_CONCURRENCY') + }; +}; + +const contextualProvenance = ( + provenance: ArmProvenance, + arm: ArmPlan, + heapMiB: number, + expectedV8HeapLimitBytes: number | null, + context: RunContext +): ArmProvenance => ({ + ...provenance, + planSha256: context.planSha256, + fleetSha256: context.fleetSha256, + runOrderSeed: context.runOrderSeed, + runOrderIndex: context.runOrderIndex, + memoryPolicy: resolvedMemoryPolicy(arm, heapMiB, expectedV8HeapLimitBytes) +}); + +const persistPartialArtifacts = ( + artifactDir: string, + memory: ReturnType | null, + postgresMemory: ReturnType | null, + capture: WorkloadCapture, + workloadResult: WorkloadResult | null, + retainedMemory: RetainedMemoryCheckpointPair +): void => { + writeJson(path.join(artifactDir, 'memory.json'), { + snapshots: memory?.snapshots ?? [], + osSnapshots: memory?.osSnapshots ?? [], + errors: memory?.errors ?? [], + warmupIndex: memory?.warmupIndex ?? -1, + osWarmupIndex: memory?.osWarmupIndex ?? -1, + osPeakRssBytes: memory?.osPeakRssBytes ?? null + }); + writeJson(path.join(artifactDir, 'postgres-memory.json'), { + snapshots: postgresMemory?.snapshots ?? [], + errors: postgresMemory?.errors ?? [] + }); + writeJson(path.join(artifactDir, 'canaries.json'), capture.canaries); + writeJson(path.join(artifactDir, 'canary-schedule.json'), capture.canarySchedule); + fs.writeFileSync( + path.join(artifactDir, 'requests.ndjson'), + capture.samples.length > 0 + ? `${capture.samples.map((sample) => JSON.stringify(sample)).join('\n')}\n` + : '', + 'utf8' + ); + writeJson(path.join(artifactDir, 'workload-progress.json'), { + warmedSurfaces: [...capture.warmedSurfaces].map(([tenantId, surfaces]) => ({ + tenantId, + surfaces: [...surfaces].sort() + })), + warmupLatencies: capture.warmupLatencies, + samples: capture.samples.length, + canaries: capture.canaries.length, + canarySchedule: capture.canarySchedule, + offeredLoad: workloadResult?.offeredLoad ?? null, + resolvedWarmupTimeoutMs: workloadResult?.resolvedWarmupTimeoutMs ?? null, + workloadDurationMs: workloadResult?.workloadDurationMs ?? null + }); + writeJson(path.join(artifactDir, 'retained-memory.json'), retainedMemory); +}; + +const artifactName = ( + arm: ArmPlan, + heapMiB: number, + tenantCount: number, + repetition: number, + suffix = 'matrix' +): string => [suffix, arm.name, `h${heapMiB}`, `t${tenantCount}`, `r${repetition}`] + .map((part) => part.replace(/[^a-zA-Z0-9_.-]+/g, '-')) + .join('-'); + +const runOne = async ( + plan: DensityPlanV1, + fleet: FleetV1, + arm: ArmPlan, + heapMiB: number, + tenantCount: number, + repetition: number, + workload: WorkloadPlan, + context: RunContext, + suffix = 'matrix' +): Promise => { + const runId = `${new Date().toISOString().replace(/[:.]/g, '-')}-${process.pid}`; + const artifactDir = path.join( + plan.artifactDir, + `${artifactName(arm, heapMiB, tenantCount, repetition, suffix)}-${runId}` + ); + fs.mkdirSync(artifactDir, { recursive: true }); + let server: Awaited> | null = null; + let memory: ReturnType | null = null; + let postgresMemory: ReturnType | null = null; + let workloadResult: WorkloadResult | null = null; + let provenance: ArmProvenance | null = null; + let postgresRunAttestation: PostgresRunAttestationEvidence | null = null; + let realtimeDeliveryCoverage: RealtimeDeliveryCoverage | null = null; + const realtimeEvidence: Array<{ + phase: string; + timestamp: string; + snapshot: RealtimeDriverSnapshot; + }> = []; + const retainedMemory: RetainedMemoryCheckpointPair = { + baseline: null, + final: null, + errors: [] + }; + const capture = createWorkloadCapture(); + const tenants = resolveTenants(fleet.tenants.slice(0, tenantCount), arm); + const realtime = createRealtimeDriver(tenants, { + // Keep connection/prime transients bounded independently of schema-build + // concurrency; one-at-a-time setup is unnecessarily slow at 500+ surfaces. + concurrency: Math.min(8, workload.maxInFlight), + timeoutMs: workload.requestTimeoutMs + }); + const recordRealtime = (phase: string): void => { + realtimeEvidence.push({ + phase, + timestamp: new Date().toISOString(), + snapshot: realtime.snapshot() + }); + writeJson(path.join(artifactDir, 'realtime-driver.json'), realtimeEvidence); + }; + const startedAt = new Date().toISOString(); + const makeScoreInput = ( + endedAt: string, + executionErrors: string[] + ): ScoreInput => { + const memorySnapshots = memory?.snapshots ?? []; + const osSnapshots = memory?.osSnapshots ?? []; + return { + arm: arm.name, + evidenceMode: context.evidenceMode, + campaignId: context.campaignId, + scheduleSha256: context.scheduleSha256, + previousResultPayloadSha256: context.previousResultPayloadSha256, + qualificationCohortSha256: context.qualificationCohortSha256, + commit: arm.commit, + introspectionMode: arm.introspectionMode, + heapMiB, + repetition, + expectedMatrixRepetitions: context.expectedMatrixRepetitions, + runKind: context.runKind, + runOrderSeed: context.runOrderSeed, + runOrderIndex: context.runOrderIndex, + startedAt, + endedAt, + configuredDurationSec: workload.durationSec, + workloadDurationMs: workloadResult?.workloadDurationMs ?? 0, + artifactDir, + tenants, + warmedSurfaces: capture.warmedSurfaces, + warmupLatencies: capture.warmupLatencies, + resolvedWarmupTimeoutMs: workloadResult?.resolvedWarmupTimeoutMs + ?? resolveWarmupTimeoutMs( + workload, + tenants.reduce((sum, tenant) => sum + tenant.surfaces.length, 0) + ), + offeredLoad: workloadResult?.offeredLoad + ?? resolveOfferedLoad(workload, tenants.length), + canaryIntervalSec: workload.canaryIntervalSec, + periodicCanarySchedule: workload.periodicCanarySchedule ?? 'full-sweep', + canarySchedule: capture.canarySchedule, + minWorkloadRequestsPerSurface: workload.minWorkloadRequestsPerSurface, + samples: capture.samples, + canaries: capture.canaries, + memorySnapshots, + postWarmupSnapshots: memorySnapshots.slice( + Math.max(0, memory?.warmupIndex ?? -1) + ), + postWarmupNodeRssSnapshots: osSnapshots.slice( + Math.max(0, memory?.osWarmupIndex ?? -1) + ), + retainedMemory, + memorySampleErrors: memory?.errors ?? [], + postgresSnapshots: postgresMemory?.snapshots ?? [], + postgresSampleErrors: postgresMemory?.errors ?? [], + missedArrivals: capture.samples.filter( + (sample) => sample.errorCode === 'LOAD_GENERATOR_MISSED_ARRIVAL' + ).length, + requiredCapabilities: plan.requiredCapabilities, + requiredCanaries: plan.requiredCanaries, + gates: plan.gates, + serverExit: server?.exit ?? null, + provenance, + provenanceErrors: (server?.provenanceErrors ?? []).map( + executionErrorEvidence + ), + postgresRunAttestation, + realtimeDeliveryCoverage, + externalServer: server?.external ?? false, + executionErrors + }; + }; + try { + if (plan.gates.requireFreshPostgresRunAttestation) { + postgresRunAttestation = await collectPostgresRunAttestation(arm, { + arm: arm.name, + heapMiB, + tenantCount, + repetition, + runOrderIndex: context.runOrderIndex, + planSha256: context.planSha256, + fleetSha256: context.fleetSha256, + notBeforeEpochMs: context.notBeforeEpochMs, + artifactDir + }); + if (postgresRunAttestation) { + const reusedIdentity = context.claimPostgresRunIdentity( + postgresRunAttestation + ); + if (reusedIdentity) { + throw new Error( + `PostgreSQL run identity was reused: ${reusedIdentity}` + ); + } + } + } + const memoryUrl = resolveTemplate(arm.memoryUrl, { + heapMiB, + port: arm.port, + artifactDir, + mode: arm.introspectionMode + }); + assertLoopbackObservabilityUrl(memoryUrl, arm.port); + server = await startArmProcess( + arm, + heapMiB, + artifactDir, + tenantCount, + postgresRunAttestation ? { + postgresFixtureDir: path.join(artifactDir, 'postgres-fixture'), + postgresManifestFile: path.join( + artifactDir, + 'postgres-fixture', + 'provision.json' + ), + postgresSecretsFile: path.join( + artifactDir, + 'postgres-fixture', + 'runtime-secrets.json' + ), + postgresManifestSha256: postgresRunAttestation.manifestSha256, + postgresCloneId: postgresRunAttestation.cloneId + } : {} + ); + provenance = contextualProvenance( + server.provenance, + arm, + heapMiB, + server.expectedHeapLimitBytes, + context + ); + writeJson(path.join(artifactDir, 'provenance.json'), { + ...provenance, + expectedHeapLimitBytes: server.expectedHeapLimitBytes, + errors: server.provenanceErrors + }); + memory = startMemorySampler(memoryUrl, { + expectedPid: server.pid, + expectedHeapLimitBytes: server.expectedHeapLimitBytes, + currentRssSource: context.evidenceMode === 'qualification' ? 'proc' : 'auto', + headers: server.observabilityHeaders + }); + if (arm.postgresContainer) { + postgresMemory = startPostgresMemorySampler(arm.postgresContainer, { + requireCgroupV2: arm.requirePostgresCgroupV2, + ...(postgresRunAttestation ? { + expectedContainerId: postgresRunAttestation.containerId, + expectedContainerStartedAt: postgresRunAttestation.containerStartedAt, + expectedCgroupIdentitySha256: postgresRunAttestation.cgroupIdentitySha256 + } : {}) + }); + } + await Promise.all([memory.ready, postgresMemory?.ready]); + workloadResult = await runWorkload( + tenants, + workload, + async () => { + // runWorkload invokes this at the final pre-load boundary, after its + // schema warmups, capability coverage, and initial hostile canaries. + // The driver owns all graphql-ws client objects, so their heap/RSS is + // outside the measured server child. The server hook only verifies + // that every exact-route inbound connection and manager is resident. + await realtime.startAndVerify(); + recordRealtime('verified-before-baseline'); + await invokePostWarmupHook( + arm, + heapMiB, + tenantCount, + artifactDir, + server!.observabilityHeaders + ); + realtime.assertHealthy(); + if (arm.retainedHeapCheckpointUrl) { + retainedMemory.baseline = await invokeRetainedMemoryCheckpoint( + arm, + heapMiB, + tenantCount, + server!.observabilityHeaders, + retainedMemory.errors + ); + } + // Begin natural post-warm RSS/heap accounting after the benchmark-only + // baseline GC. The sampler itself and the PostgreSQL sampler remain + // live throughout both bookends. + await memory!.markWarmupComplete(); + realtime.beginTimedCoverage(workload.durationSec * 1000); + }, + capture + ); + realtimeDeliveryCoverage = await realtime.finishTimedCoverage(); + recordRealtime('timed-coverage-complete'); + await realtime.verifyDeliveryNow(); + realtime.assertHealthy(); + recordRealtime('healthy-after-workload'); + if (arm.retainedHeapCheckpointUrl) { + retainedMemory.final = await invokeRetainedMemoryCheckpoint( + arm, + heapMiB, + tenantCount, + server.observabilityHeaders, + retainedMemory.errors + ); + } + realtime.assertHealthy(); + recordRealtime('healthy-after-final-checkpoint'); + await memory.stop(); + if (postgresMemory) await postgresMemory.stop(); + realtime.assertHealthy(); + recordRealtime('healthy-before-disposal'); + await realtime.dispose(); + recordRealtime('disposed'); + persistPartialArtifacts( + artifactDir, + memory, + postgresMemory, + capture, + workloadResult, + retainedMemory + ); + const endedAt = new Date().toISOString(); + const scoreInput = makeScoreInput(endedAt, []); + const result = scoreRun(scoreInput); + writeResult(plan.artifactDir, result, scoreInput, context); + return result; + } catch (error) { + const executionErrors = [executionErrorEvidence(error)]; + recordRealtime('failed'); + if (memory) { + try { + await memory.stop(); + } catch (stopError) { + memory.errors.push(stopError instanceof Error ? stopError.message : String(stopError)); + } + } + if (postgresMemory) { + try { + await postgresMemory.stop(); + } catch (stopError) { + postgresMemory.errors.push( + stopError instanceof Error ? stopError.message : String(stopError) + ); + } + } + try { + await realtime.dispose(); + recordRealtime('disposed-after-failure'); + } catch (disposeError) { + executionErrors.push(executionErrorEvidence(disposeError)); + recordRealtime('dispose-failed'); + } + persistPartialArtifacts( + artifactDir, + memory, + postgresMemory, + capture, + workloadResult, + retainedMemory + ); + const scoreInput = makeScoreInput( + new Date().toISOString(), + executionErrors + ); + const result = scoreRun(scoreInput); + writeResult(plan.artifactDir, result, scoreInput, context); + return result; + } finally { + if (memory) await memory.stop(); + if (postgresMemory) await postgresMemory.stop(); + try { + await realtime.dispose(); + } catch { + // Disposal was already attempted and recorded in the main path. + } + await server?.stop(); + } +}; + +export const runDensityPlan = async ( + plan: DensityPlanV1, + fleet: FleetV1, + selection: RunSelection = {} +): Promise => { + validateCoverage(plan, fleet); + const arms = plan.arms.filter((arm) => !selection.arms || selection.arms.includes(arm.name)); + if (arms.length === 0) throw new Error('run selection contains no known arms'); + const heaps = selection.heaps ?? (selection.smoke ? plan.heapMiB.slice(0, 1) : plan.heapMiB); + const repetitions = selection.smoke ? 1 : selection.repetitions ?? plan.repetitions; + if (!Number.isInteger(repetitions) || repetitions <= 0) { + throw new Error('run repetitions must be a positive integer'); + } + const tenantCountsOverride = selection.tenantCounts + ?? (selection.smoke ? tenantCountsForHeap(plan, heaps[0]).slice(0, 1) : undefined); + const schedule = buildRunSchedule( + plan, + arms, + heaps, + repetitions, + tenantCountsOverride + ); + const configuredSchedule = buildRunSchedule( + plan, + plan.arms, + plan.heapMiB, + plan.repetitions + ); + const exactConfiguredMatrix = sameRunSchedule(schedule, configuredSchedule); + for (const job of schedule) { + if (job.tenantCount > fleet.tenants.length) { + throw new Error(`fleet has ${fleet.tenants.length} tenants, requested ${job.tenantCount}`); + } + } + const results: DensityRunResult[] = []; + const runOrderSeed = plan.runOrderSeed ?? DEFAULT_RUN_ORDER_SEED; + const planSha256 = plan.sourceSha256 ?? sha256(JSON.stringify(plan)); + const fleetSha256 = fleet.sourceSha256 ?? sha256(JSON.stringify(fleet)); + const qualificationCohortSha256 = sha256(`${planSha256}\0${fleetSha256}`); + const hostileEvidenceReady = hasExactHostileValidationEvidence(plan); + const evidenceMode = plan.qualification + && exactConfiguredMatrix + && !selection.smoke + && process.platform === 'linux' + && hostileEvidenceReady + ? 'qualification' + : 'diagnostic'; + const campaignId = randomBytes(32).toString('hex'); + const campaignStartedAt = new Date().toISOString(); + const scheduleManifest: CampaignScheduleManifestV1 = { + version: 1, + campaignId, + campaignStartedAt, + runOrderSeed, + planSha256, + fleetSha256, + node: process.version, + v8: process.versions.v8, + platform: process.platform, + architecture: process.arch, + jobs: scheduleJobsForPlan(plan, schedule, !selection.smoke) + }; + const scheduleSha256 = scheduleManifestSha256(scheduleManifest); + writeExclusiveJson(path.join(plan.artifactDir, `campaign-${campaignId}.json`), { + ...scheduleManifest, + scheduleSha256, + evidenceMode, + qualificationBlockers: [ + ...(!plan.qualification ? ['qualification-plan-missing'] : []), + ...(!exactConfiguredMatrix ? ['noncanonical-or-partial-schedule'] : []), + ...(selection.smoke ? ['smoke-run'] : []), + ...(process.platform !== 'linux' ? ['linux-required'] : []), + ...(!hostileEvidenceReady ? ['exact-hostile-validation-evidence-required'] : []) + ] + }); + if (evidenceMode === 'diagnostic' && plan.qualification) { + process.stdout.write( + `[cperf] campaign=${campaignId} diagnostic qualification prerequisites were not met\n` + ); + } + const claimedPostgresRunIdentities = new Set(); + const claimPostgresRunIdentity = ( + evidence: PostgresRunAttestationEvidence + ): string | null => { + const claims = postgresRunIdentityClaims(evidence); + const reused = claims.find((claim) => claimedPostgresRunIdentities.has(claim)); + if (reused) return reused; + for (const claim of claims) claimedPostgresRunIdentities.add(claim); + return null; + }; + + let previousResultPayloadSha256: string | null = null; + for (const job of schedule) { + const workload: WorkloadPlan = selection.smoke + ? { + ...plan.workload, + durationSec: 5, + ...(plan.workload.rps != null + ? { rps: Math.min(plan.workload.rps, 5), rpsPerTenant: undefined } + : { + rps: undefined, + rpsPerTenant: Math.min(plan.workload.rpsPerTenant!, 5 / job.tenantCount) + }) + } + : plan.workload; + const result = await runOne( + plan, + fleet, + job.arm, + job.heapMiB, + job.tenantCount, + job.repetition, + workload, + { + expectedMatrixRepetitions: plan.repetitions, + runKind: 'matrix', + evidenceMode, + campaignId, + scheduleSha256, + previousResultPayloadSha256, + qualificationCohortSha256, + runOrderSeed, + runOrderIndex: job.orderIndex, + planSha256, + fleetSha256, + notBeforeEpochMs: Date.now(), + claimPostgresRunIdentity + } + ); + results.push(result); + previousResultPayloadSha256 = result.evidenceBinding?.resultPayloadSha256 ?? null; + if (!previousResultPayloadSha256) { + throw new Error('persisted result is missing its evidence payload binding'); + } + process.stdout.write( + `[cperf] order=${job.orderIndex} ${job.arm.name} heap=${job.heapMiB} ` + + `customers=${job.tenantCount} run=${job.repetition} accepted=${result.accepted} ` + + `customersPerAlignedServiceGiB=${result.customersPerAlignedServiceGiB?.toFixed(2) ?? 'n/a'}\n` + ); + } + + if (!selection.smoke && plan.soak?.enabled) { + const configuredSoakArm = soakArmName(plan); + const candidate = arms.find((arm) => arm.name === configuredSoakArm); + if (!candidate) { + throw new Error(`soak enabled but arm '${configuredSoakArm}' is not selected`); + } + results.push(await runOne( + plan, + fleet, + candidate, + plan.soak.heapMiB, + plan.soak.tenantCount, + plan.repetitions + 1, + { ...plan.workload, durationSec: plan.soak.durationSec }, + { + expectedMatrixRepetitions: plan.repetitions, + runKind: 'soak', + evidenceMode, + campaignId, + scheduleSha256, + previousResultPayloadSha256, + qualificationCohortSha256, + runOrderSeed, + runOrderIndex: schedule.length + 1, + planSha256, + fleetSha256, + notBeforeEpochMs: Date.now(), + claimPostgresRunIdentity + }, + 'soak' + )); + previousResultPayloadSha256 = results[results.length - 1] + .evidenceBinding?.resultPayloadSha256 ?? null; + if (!previousResultPayloadSha256) { + throw new Error('persisted soak result is missing its evidence payload binding'); + } + } + writeJson( + path.join(plan.artifactDir, `capacity-boundaries-${campaignId}.json`), + { + version: 1, + runOrderSeed, + planSha256, + fleetSha256, + campaignId, + scheduleSha256, + boundaries: summarizeCapacityBoundaries(results) + } + ); + return results; +}; diff --git a/packages/perf-harness/src/schedule.ts b/packages/perf-harness/src/schedule.ts new file mode 100644 index 0000000000..c1bcfb8f08 --- /dev/null +++ b/packages/perf-harness/src/schedule.ts @@ -0,0 +1,133 @@ +import { createHash } from 'node:crypto'; + +import { + DEFAULT_RUN_ORDER_SEED, + soakArmName, + tenantCountsForHeap +} from './config'; +import type { ArmPlan, DensityPlanV1 } from './types'; + +export interface DensityRunJob { + arm: ArmPlan; + heapMiB: number; + tenantCount: number; + repetition: number; + orderIndex: number; +} + +export interface CampaignScheduleJob { + runKind: 'matrix' | 'soak'; + arm: string; + heapMiB: number; + tenantCount: number; + repetition: number; + orderIndex: number; +} + +export interface CampaignScheduleManifestV1 { + version: 1; + campaignId: string; + campaignStartedAt: string; + runOrderSeed: string; + planSha256: string; + fleetSha256: string; + node: string; + v8: string; + platform: NodeJS.Platform; + architecture: string; + jobs: CampaignScheduleJob[]; +} + +const sha256 = (value: string): string => createHash('sha256').update(value).digest('hex'); + +const deterministicArmOrder = ( + arms: ArmPlan[], + seed: string, + repetition: number, + heapMiB: number, + tenantCount: number +): ArmPlan[] => [...arms].sort((left, right) => { + const prefix = `${seed}\0${repetition}\0${heapMiB}\0${tenantCount}\0`; + return sha256(`${prefix}${left.name}`).localeCompare(sha256(`${prefix}${right.name}`)); +}); + +export const buildRunSchedule = ( + plan: DensityPlanV1, + arms: ArmPlan[], + heaps: number[], + repetitions: number, + tenantCountsOverride?: number[] +): DensityRunJob[] => { + const jobs: DensityRunJob[] = []; + const seed = plan.runOrderSeed ?? DEFAULT_RUN_ORDER_SEED; + for (let repetition = 1; repetition <= repetitions; repetition++) { + for (const heapMiB of heaps) { + const counts = tenantCountsOverride ?? tenantCountsForHeap(plan, heapMiB); + for (const tenantCount of counts) { + for (const arm of deterministicArmOrder( + arms, + seed, + repetition, + heapMiB, + tenantCount + )) { + jobs.push({ + arm, + heapMiB, + tenantCount, + repetition, + orderIndex: jobs.length + 1 + }); + } + } + } + } + return jobs; +}; + +export const scheduleJobsForPlan = ( + plan: DensityPlanV1, + matrix: DensityRunJob[], + includeSoak: boolean +): CampaignScheduleJob[] => { + const jobs: CampaignScheduleJob[] = matrix.map((job) => ({ + runKind: 'matrix', + arm: job.arm.name, + heapMiB: job.heapMiB, + tenantCount: job.tenantCount, + repetition: job.repetition, + orderIndex: job.orderIndex + })); + if (includeSoak && plan.soak?.enabled) { + jobs.push({ + runKind: 'soak', + arm: soakArmName(plan), + heapMiB: plan.soak.heapMiB, + tenantCount: plan.soak.tenantCount, + repetition: plan.repetitions + 1, + orderIndex: matrix.length + 1 + }); + } + return jobs; +}; + +export const scheduleManifestSha256 = ( + manifest: CampaignScheduleManifestV1 +): string => sha256(JSON.stringify(manifest)); + +export const sameRunSchedule = ( + left: DensityRunJob[], + right: DensityRunJob[] +): boolean => JSON.stringify(left.map((job) => [ + job.arm.name, + job.heapMiB, + job.tenantCount, + job.repetition, + job.orderIndex +])) === JSON.stringify(right.map((job) => [ + job.arm.name, + job.heapMiB, + job.tenantCount, + job.repetition, + job.orderIndex +])); diff --git a/packages/perf-harness/src/score.ts b/packages/perf-harness/src/score.ts new file mode 100644 index 0000000000..bad31b7c8c --- /dev/null +++ b/packages/perf-harness/src/score.ts @@ -0,0 +1,2634 @@ +import { createHash } from 'node:crypto'; + +import { periodicCanaryRoundCount, rotatingCanaryIndex } from './http'; +import type { + AcceptanceGates, + ArmProvenance, + CanaryResult, + CanaryScheduleSummary, + CustomerFleetShape, + DensityCapacityBoundary, + DensityRunResult, + MemorySnapshot, + NodeRssSnapshot, + PeriodicCanarySchedule, + PostgresMemorySnapshot, + PostgresRunAttestationEvidence, + RequestSample, + RealtimeDeliveryCoverage, + ResolvedOfferedLoad, + RetainedMemoryCheckpoint, + RetainedMemoryCheckpointPair, + RetainedMemoryGuard, + SurfaceResult, + TenantResult, + TenantTarget +} from './types'; + +const GIB = 1024 ** 3; +const MIB = 1024 ** 2; +const DEFAULT_MEMORY_ALIGNMENT_SKEW_MS = 500; +export const DEFAULT_MAX_ALIGNED_MEMORY_SAMPLE_GAP_MS = 1_000; +export const DEFAULT_MIN_ALIGNED_MEMORY_COVERAGE_RATIO = 0.99; + +export interface AlignedServiceMemoryPeak { + bytes: number; + nodeRssBytes: number; + postgresBytes: number; + timestamp: string; + samples: number; + maxSkewMs: number; +} + +interface AlignedServiceMemorySample { + bytes: number; + nodeRssBytes: number; + postgresBytes: number; + timestamp: string; + timeMs: number; + skewMs: number; +} + +export interface AlignedServiceMemoryCoverage { + expectedDurationMs: number; + coveredDurationMs: number; + coverageRatio: number; + maxGapMs: number; + maxPairedSampleGapMs: number; + maxNodeSampleGapMs: number; + maxPostgresSampleGapMs: number; + firstSampleTimestamp: string | null; + lastSampleTimestamp: string | null; +} + +const alignedServiceMemorySamples = ( + memorySnapshots: Array>, + postgresSnapshots: PostgresMemorySnapshot[], + allowedSkewMs: number +): AlignedServiceMemorySample[] => { + const postgres = postgresSnapshots + .map((snapshot) => ({ snapshot, timeMs: Date.parse(snapshot.timestamp) })) + .filter(({ snapshot, timeMs }) => + Number.isFinite(timeMs) + && Number.isFinite(snapshot.usedBytes) + && snapshot.usedBytes >= 0 + ) + .sort((left, right) => left.timeMs - right.timeMs); + if (postgres.length === 0) return []; + + let postgresIndex = 0; + const samples: AlignedServiceMemorySample[] = []; + const node = memorySnapshots + .map((snapshot) => ({ snapshot, timeMs: Date.parse(snapshot.timestamp) })) + .filter(({ snapshot, timeMs }) => + Number.isFinite(timeMs) + && snapshot.rssBytes != null + && Number.isFinite(snapshot.rssBytes) + && snapshot.rssBytes > 0 + ) + .sort((left, right) => left.timeMs - right.timeMs); + + for (const { snapshot, timeMs } of node) { + while ( + postgresIndex + 1 < postgres.length + && Math.abs(postgres[postgresIndex + 1].timeMs - timeMs) + <= Math.abs(postgres[postgresIndex].timeMs - timeMs) + ) postgresIndex += 1; + const candidate = postgres[postgresIndex]; + const skewMs = Math.abs(candidate.timeMs - timeMs); + if (skewMs > allowedSkewMs) continue; + samples.push({ + bytes: snapshot.rssBytes! + candidate.snapshot.usedBytes, + nodeRssBytes: snapshot.rssBytes!, + postgresBytes: candidate.snapshot.usedBytes, + timestamp: snapshot.timestamp, + timeMs, + skewMs + }); + } + return samples; +}; + +export const alignedServiceMemoryCoverage = ( + memorySnapshots: Array>, + postgresSnapshots: PostgresMemorySnapshot[], + expectedStartMs: number, + expectedDurationMs: number, + allowedSkewMs = DEFAULT_MEMORY_ALIGNMENT_SKEW_MS +): AlignedServiceMemoryCoverage | null => { + if (!Number.isFinite(allowedSkewMs) || allowedSkewMs < 0) { + throw new Error(`memory alignment skew must be non-negative, received ${allowedSkewMs}`); + } + if (!Number.isFinite(expectedStartMs) || !Number.isFinite(expectedDurationMs) + || expectedDurationMs <= 0) return null; + const expectedEndMs = expectedStartMs + expectedDurationMs; + const samples = alignedServiceMemorySamples( + memorySnapshots, + postgresSnapshots, + allowedSkewMs + ).filter((sample) => + sample.timeMs >= expectedStartMs - allowedSkewMs + && sample.timeMs <= expectedEndMs + allowedSkewMs + ); + if (samples.length === 0) return null; + const first = samples[0]; + const last = samples.at(-1)!; + const coveredStartMs = Math.max(expectedStartMs, first.timeMs); + const coveredEndMs = Math.min(expectedEndMs, last.timeMs); + const coveredDurationMs = Math.max(0, coveredEndMs - coveredStartMs); + const cadenceGap = (timestamps: number[]): number => { + const times = [...new Set(timestamps + .filter((timeMs) => + Number.isFinite(timeMs) + && timeMs >= expectedStartMs - allowedSkewMs + && timeMs <= expectedEndMs + allowedSkewMs + ) + .map((timeMs) => Math.max(expectedStartMs, Math.min(expectedEndMs, timeMs))))] + .sort((left, right) => left - right); + if (times.length === 0) return expectedDurationMs; + let gap = Math.max(times[0] - expectedStartMs, expectedEndMs - times.at(-1)!); + for (let index = 1; index < times.length; index += 1) { + gap = Math.max(gap, times[index] - times[index - 1]); + } + return gap; + }; + const maxPairedSampleGapMs = cadenceGap(samples.map((sample) => sample.timeMs)); + const maxNodeSampleGapMs = cadenceGap(memorySnapshots + .filter((snapshot) => Number.isFinite(snapshot.rssBytes) && snapshot.rssBytes! > 0) + .map((snapshot) => Date.parse(snapshot.timestamp))); + const maxPostgresSampleGapMs = cadenceGap(postgresSnapshots + .filter((snapshot) => Number.isFinite(snapshot.usedBytes) && snapshot.usedBytes >= 0) + .map((snapshot) => Date.parse(snapshot.timestamp))); + return { + expectedDurationMs, + coveredDurationMs, + coverageRatio: coveredDurationMs / expectedDurationMs, + maxGapMs: Math.max( + maxPairedSampleGapMs, + maxNodeSampleGapMs, + maxPostgresSampleGapMs + ), + maxPairedSampleGapMs, + maxNodeSampleGapMs, + maxPostgresSampleGapMs, + firstSampleTimestamp: first.timestamp, + lastSampleTimestamp: last.timestamp + }; +}; + +/** + * Pair each current Node RSS sample with the nearest PostgreSQL cgroup sample. + * Cumulative process HWM is deliberately excluded because adding a historical + * Node peak to current PostgreSQL usage would not be a simultaneous service + * footprint. + */ +export const alignedServiceMemoryPeak = ( + memorySnapshots: Array>, + postgresSnapshots: PostgresMemorySnapshot[], + allowedSkewMs = DEFAULT_MEMORY_ALIGNMENT_SKEW_MS +): AlignedServiceMemoryPeak | null => { + if (!Number.isFinite(allowedSkewMs) || allowedSkewMs < 0) { + throw new Error(`memory alignment skew must be non-negative, received ${allowedSkewMs}`); + } + const aligned = alignedServiceMemorySamples( + memorySnapshots, + postgresSnapshots, + allowedSkewMs + ); + if (aligned.length === 0) return null; + let peak: Omit | null = null; + for (const sample of aligned) { + if (!peak || sample.bytes > peak.bytes) { + peak = { + bytes: sample.bytes, + nodeRssBytes: sample.nodeRssBytes, + postgresBytes: sample.postgresBytes, + timestamp: sample.timestamp + }; + } + } + return peak ? { + ...peak, + samples: aligned.length, + maxSkewMs: Math.max(...aligned.map((sample) => sample.skewMs)) + } : null; +}; + +export const percentile = (values: number[], fraction: number): number => { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)]; +}; + +export const summarizeCustomerFleet = ( + customers: TenantTarget[] +): CustomerFleetShape => { + const physicalDatabases = new Set(); + const routingLabels = new Set(); + const buildContracts = new Set(); + const runtimePoolIdentities = new Set(); + let logicalDatabases = 0; + let apis = 0; + let realtimeApis = 0; + let surfaces = 0; + let physicalSchemaBindings = 0; + for (const customer of customers) { + surfaces += customer.surfaces.length; + for (const surface of customer.surfaces) buildContracts.add(surface.buildContract); + for (const database of customer.databases ?? []) { + logicalDatabases += 1; + physicalDatabases.add(database.physicalDatabase); + for (const api of database.apis) { + apis += 1; + if (api.realtime) realtimeApis += 1; + physicalSchemaBindings += api.physicalSchemas.length; + for (const label of api.routingLabels) routingLabels.add(label); + runtimePoolIdentities.add(api.runtimePoolIdentity); + } + } + } + return { + topologyComplete: customers.every((customer) => customer.databases != null), + customers: customers.length, + logicalDatabases, + physicalDatabases: physicalDatabases.size, + apis, + realtimeApis, + surfaces, + physicalSchemaBindings, + routingLabels: routingLabels.size, + uniqueBuildContracts: buildContracts.size, + uniqueRuntimePoolIdentities: runtimePoolIdentities.size + }; +}; + +const counterDelta = ( + snapshots: MemorySnapshot[], + field: 'evictions' | 'buildRefusals' | 'buildsStarted' +): number | null => { + if (snapshots.length < 2) return null; + const available = field === 'buildsStarted' + ? snapshots.every((snapshot) => snapshot.buildCountersAvailable) + : snapshots.every((snapshot) => snapshot.cacheCountersAvailable); + if (!available) return null; + const values = snapshots.map((snapshot) => snapshot[field]); + if (values.some((value) => !Number.isSafeInteger(value) || value! < 0)) return null; + if (values.some((value, index) => index > 0 && value! < values[index - 1]!)) { + return null; + } + return values.at(-1)! - values[0]!; +}; + +type PgPoolCounterField = + | 'pgPoolCapacityEvictions' + | 'pgPoolCapacityRefusals' + | 'pgPoolDisposalFailures'; + +const pgPoolCounterDelta = ( + snapshots: MemorySnapshot[], + field: PgPoolCounterField +): number | null => { + if (snapshots.length < 2) return null; + const values = snapshots.map((snapshot) => snapshot[field]); + if (values.some((value) => !Number.isSafeInteger(value) || value! < 0)) return null; + if (values.some((value, index) => index > 0 && value! < values[index - 1]!)) { + return null; + } + return values.at(-1)! - values[0]!; +}; + +export const heapGrowthMiBPerHour = (snapshots: MemorySnapshot[]): number | null => { + if (snapshots.length < 2 || snapshots.some((snapshot) => snapshot.heapUsedBytes == null)) { + return null; + } + const points = snapshots.map((snapshot) => ({ + x: new Date(snapshot.timestamp).getTime() / 3_600_000, + y: snapshot.heapUsedBytes! / MIB + })); + const meanX = points.reduce((sum, point) => sum + point.x, 0) / points.length; + const meanY = points.reduce((sum, point) => sum + point.y, 0) / points.length; + const numerator = points.reduce((sum, point) => sum + (point.x - meanX) * (point.y - meanY), 0); + const denominator = points.reduce((sum, point) => sum + (point.x - meanX) ** 2, 0); + return denominator === 0 ? null : numerator / denominator; +}; + +export interface RetainedMemoryGrowthSummary { + heapMiBPerHour: number | null; + externalMiBPerHour: number | null; + durationSec: number | null; + heapBaselineBytes: number | null; + heapFinalBytes: number | null; + externalBaselineBytes: number | null; + externalFinalBytes: number | null; + errors: string[]; +} + +const canonicalJson = (value: unknown): string => { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value && typeof value === 'object') { + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => + `${JSON.stringify(key)}:${canonicalJson(record[key])}` + ).join(',')}}`; + } + return JSON.stringify(value); +}; + +const stateSha256 = (value: unknown): string => + `sha256:${createHash('sha256').update(canonicalJson(value)).digest('hex')}`; + +const asRecord = (value: unknown): Record | null => value + && typeof value === 'object' + && !Array.isArray(value) + ? value as Record + : null; + +const lifecycleCounter = ( + counters: Record, + name: string +): number | null => Number.isSafeInteger(counters[name]) + && (counters[name] as number) >= 0 + ? counters[name] as number + : null; + +/** + * The physical fixture hashes its complete residency and monotonic counter + * state. A single GC checkpoint must remain byte-identical, but normal HTTP + * traffic advances its started/completed counters between the two bookends. + * Permit only a balanced monotonic HTTP delta; every other field remains an + * exact topology/counter comparison. WebSocket lifecycle changes are rejected + * because qualifying transports must remain continuously resident. + */ +const validateCrossWorkloadGuardState = ( + baseline: RetainedMemoryGuard, + final: RetainedMemoryGuard, + errors: string[] +): void => { + const baselineState = asRecord(baseline.state); + const finalState = asRecord(final.state); + if (!baselineState || !finalState) { + errors.push('retained-memory workload guard state is invalid'); + return; + } + const baselineCounters = asRecord(baselineState.cacheCounters); + const finalCounters = asRecord(finalState.cacheCounters); + // Legacy/non-physical checkpoint producers do not expose handler lifecycle + // counters, so retain their former exact-state requirement. + if (!baselineCounters && !finalCounters) { + if (baseline.stateSha256 !== final.stateSha256) { + errors.push('retained-memory residency or counters changed across the workload'); + } + return; + } + if (!baselineCounters || !finalCounters) { + errors.push('retained-memory handler counters changed shape across the workload'); + return; + } + const names = [ + 'httpRequestsStarted', + 'httpRequestsCompleted', + 'websocketUpgradesStarted', + 'websocketUpgradesCompleted' + ] as const; + const before = Object.fromEntries(names.map((name) => [ + name, + lifecycleCounter(baselineCounters, name) + ])) as Record<(typeof names)[number], number | null>; + const after = Object.fromEntries(names.map((name) => [ + name, + lifecycleCounter(finalCounters, name) + ])) as Record<(typeof names)[number], number | null>; + if (names.some((name) => before[name] == null || after[name] == null)) { + errors.push('retained-memory handler counters are invalid'); + return; + } + const delta = Object.fromEntries(names.map((name) => [ + name, + after[name]! - before[name]! + ])) as Record<(typeof names)[number], number>; + if (names.some((name) => delta[name] < 0)) { + errors.push('retained-memory handler counters regressed across the workload'); + } + if (delta.httpRequestsStarted !== delta.httpRequestsCompleted) { + errors.push( + `retained-memory HTTP handler delta is unbalanced: started=${delta.httpRequestsStarted}, completed=${delta.httpRequestsCompleted}` + ); + } + if (delta.websocketUpgradesStarted !== 0 || delta.websocketUpgradesCompleted !== 0) { + errors.push( + `retained-memory WebSocket lifecycle changed across the workload: started=${delta.websocketUpgradesStarted}, completed=${delta.websocketUpgradesCompleted}` + ); + } + const withoutHttpLifecycle = ( + state: Record + ): Record => { + const counters = { ...asRecord(state.cacheCounters) }; + delete counters.httpRequestsStarted; + delete counters.httpRequestsCompleted; + return { ...state, cacheCounters: counters }; + }; + if ( + stateSha256(withoutHttpLifecycle(baselineState)) + !== stateSha256(withoutHttpLifecycle(finalState)) + ) { + errors.push('retained-memory residency or non-HTTP counters changed across the workload'); + } +}; + +const stableTail = (checkpoint: RetainedMemoryCheckpoint) => + checkpoint.samples.slice(-checkpoint.stableSampleCount); + +const medianNumber = (values: number[]): number => percentile(values, 0.5); + +const validateCheckpoint = ( + label: string, + checkpoint: RetainedMemoryCheckpoint | null, + expectedPid: number | null, + errors: string[] +): checkpoint is RetainedMemoryCheckpoint => { + if (!checkpoint) { + errors.push(`${label} retained-memory checkpoint is unavailable`); + return false; + } + let structurallyUsable = true; + if (!checkpoint.stable) errors.push(`${label} retained-memory checkpoint is unstable`); + errors.push(...checkpoint.errors.map((error) => `${label}: ${error}`)); + if (checkpoint.samples.length < 5 || checkpoint.samples.length > 8) { + errors.push(`${label} retained-memory checkpoint has invalid GC sample count`); + structurallyUsable = false; + } + if (checkpoint.stableSampleCount !== 3) { + errors.push(`${label} retained-memory checkpoint must use three stable samples`); + structurallyUsable = false; + } + if ( + checkpoint.pid !== checkpoint.guardBefore.pid + || checkpoint.pid !== checkpoint.guardAfter.pid + || (expectedPid != null && checkpoint.pid !== expectedPid) + ) { + errors.push(`${label} retained-memory checkpoint PID mismatch`); + } + for (const [guardLabel, guard] of [ + ['before', checkpoint.guardBefore], + ['after', checkpoint.guardAfter] + ] as const) { + if (guard.graphileInFlight !== 0) { + errors.push(`${label} retained-memory ${guardLabel} guard has in-flight Graphile work`); + } + if (stateSha256(guard.state) !== guard.stateSha256) { + errors.push(`${label} retained-memory ${guardLabel} state hash mismatch`); + } + const state = guard.state as Record; + const stateContracts = Array.isArray(state.residentBuildContracts) + ? state.residentBuildContracts + : null; + if ( + state.pid !== guard.pid + || state.graphileInFlight !== guard.graphileInFlight + || !stateContracts + || stateContracts.length !== guard.residentBuildContracts.length + || guard.residentBuildContracts.some( + (contract, index) => stateContracts[index] !== contract + ) + ) { + errors.push(`${label} retained-memory ${guardLabel} guard summary mismatch`); + } + } + if (checkpoint.guardBefore.stateSha256 !== checkpoint.guardAfter.stateSha256) { + errors.push(`${label} retained-memory residency or counters changed during GC`); + } + const tail = stableTail(checkpoint); + for (const field of ['heapUsedBytes', 'externalBytes'] as const) { + const values = tail.map((sample) => sample[field]); + const spread = Math.max(...values) - Math.min(...values); + const threshold = Math.max(MIB, Math.ceil(Math.max(...values) * 0.0025)); + if (spread > threshold) { + errors.push(`${label} retained ${field} samples did not converge`); + } + } + const monotonic = checkpoint.samples.map((sample) => { + try { + return BigInt(sample.monotonicNs); + } catch { + return null; + } + }); + if ( + monotonic.some((value) => value == null) + || monotonic.some((value, index) => + index > 0 && value! <= monotonic[index - 1]! + ) + ) { + errors.push(`${label} retained-memory monotonic timestamps are invalid`); + structurallyUsable = false; + } + return structurallyUsable; +}; + +export const retainedMemoryGrowth = ( + checkpoints: RetainedMemoryCheckpointPair, + expectedPid: number | null = null, + expectedResidentBuildContracts: ReadonlySet | null = null, + requireStableResidentBuildFingerprints = false +): RetainedMemoryGrowthSummary => { + const errors = [...checkpoints.errors]; + const baselineValid = validateCheckpoint( + 'baseline', checkpoints.baseline, expectedPid, errors + ); + const finalValid = validateCheckpoint( + 'final', checkpoints.final, expectedPid, errors + ); + if (!baselineValid || !finalValid) { + return { + heapMiBPerHour: null, + externalMiBPerHour: null, + durationSec: null, + heapBaselineBytes: null, + heapFinalBytes: null, + externalBaselineBytes: null, + externalFinalBytes: null, + errors + }; + } + const baseline = checkpoints.baseline!; + const final = checkpoints.final!; + if (baseline.fixture !== final.fixture) { + errors.push('retained-memory checkpoint fixture changed'); + } + validateCrossWorkloadGuardState( + baseline.guardAfter, + final.guardBefore, + errors + ); + if (expectedResidentBuildContracts) { + const expected = [...expectedResidentBuildContracts].sort(); + for (const [label, checkpoint] of [ + ['baseline', baseline], + ['final', final] + ] as const) { + const stateFingerprints = checkpoint.guardAfter.state + .residentBuildContractFingerprints; + const stableFingerprints = Array.isArray(stateFingerprints) + && stateFingerprints.every((value) => typeof value === 'string') + ? stateFingerprints as string[] + : null; + const resident = [ + ...(stableFingerprints ?? checkpoint.guardAfter.residentBuildContracts) + ].sort(); + if ( + (requireStableResidentBuildFingerprints && stableFingerprints == null) + || new Set(resident).size !== resident.length + || expected.length !== resident.length + || expected.some((contract, index) => contract !== resident[index]) + ) { + errors.push(`${label} retained-memory residency set mismatch`); + } + } + } + const baselineTail = stableTail(baseline); + const finalTail = stableTail(final); + const baselineNs = BigInt(baseline.samples.at(-1)!.monotonicNs); + const finalNs = BigInt(final.samples.at(-1)!.monotonicNs); + const durationSec = Number(finalNs - baselineNs) / 1e9; + if (!Number.isFinite(durationSec) || durationSec <= 0) { + errors.push('retained-memory checkpoint duration is invalid'); + } + const heapBaselineValues = baselineTail.map((sample) => sample.heapUsedBytes); + const heapFinalValues = finalTail.map((sample) => sample.heapUsedBytes); + const externalBaselineValues = baselineTail.map((sample) => sample.externalBytes); + const externalFinalValues = finalTail.map((sample) => sample.externalBytes); + const heapBaselineBytes = medianNumber(heapBaselineValues); + const heapFinalBytes = medianNumber(heapFinalValues); + const externalBaselineBytes = medianNumber(externalBaselineValues); + const externalFinalBytes = medianNumber(externalFinalValues); + const durationHours = durationSec / 3_600; + return { + heapMiBPerHour: durationHours > 0 + ? (Math.max(...heapFinalValues) - Math.min(...heapBaselineValues)) / MIB + / durationHours + : null, + externalMiBPerHour: durationHours > 0 + ? (Math.max(...externalFinalValues) - Math.min(...externalBaselineValues)) / MIB + / durationHours + : null, + durationSec: durationSec > 0 ? durationSec : null, + heapBaselineBytes, + heapFinalBytes, + externalBaselineBytes, + externalFinalBytes, + errors + }; +}; + +const tenantResult = ( + tenant: TenantTarget, + samples: RequestSample[], + canaries: CanaryResult[], + warmed: Set, + requiredCapabilities: string[], + minWorkloadRequestsPerSurface: number, + gates: AcceptanceGates +): TenantResult => { + const localSamples = samples.filter((sample) => sample.tenantId === tenant.id); + const workloadSamples = localSamples.filter((sample) => sample.phase === 'workload'); + const successfulCoverage = localSamples.filter((sample) => sample.ok); + const localCanaries = canaries.filter((canary) => canary.tenantId === tenant.id); + const errors = workloadSamples.filter((sample) => !sample.ok).length; + const errorRate = workloadSamples.length > 0 ? errors / workloadSamples.length : 1; + const p99Ms = percentile(workloadSamples.map((sample) => sample.latencyMs), 0.99); + const canaryInconclusive = localCanaries.filter((canary) => !canary.conclusive).length; + const bleedViolations = localCanaries.filter((canary) => canary.violation).length; + const successfulOperationKeys = new Set(successfulCoverage.map((sample) => + `${sample.surface}/${sample.operation}` + )); + const successfulCapabilities = new Set(successfulCoverage.map((sample) => sample.capability)); + const successfulCapabilityKeys = new Set(successfulCoverage.map((sample) => + `${sample.surface}/${sample.capability}` + )); + const operationOracleSamples = localSamples.filter((sample) => + sample.oracleConfigured === true + ); + const operationOracleInconclusive = operationOracleSamples.filter((sample) => + sample.oracleUnavailable !== true + && sample.oracleConclusive !== true + ).length; + const operationOracleViolations = operationOracleSamples.filter((sample) => + sample.oracleViolation === true + ).length; + const coverageOracleSamplesByKey = new Map(); + for (const sample of localSamples.filter((candidate) => candidate.phase === 'coverage')) { + const key = `${sample.surface}/${sample.operation}`; + const evidence = coverageOracleSamplesByKey.get(key) ?? []; + evidence.push(sample); + coverageOracleSamplesByKey.set(key, evidence); + } + const conclusiveCoverageOracleKeys = new Set([...coverageOracleSamplesByKey] + .filter(([, evidence]) => evidence.length === 1 && ( + evidence[0].ok + && evidence[0].oracleConfigured === true + && evidence[0].oracleConclusive === true + && evidence[0].oracleViolation !== true + )) + .map(([key]) => key)); + const surfaceResults: SurfaceResult[] = tenant.surfaces.map((surface) => { + const surfaceSamples = localSamples.filter((sample) => sample.surface === surface.name); + const surfaceWorkload = surfaceSamples.filter((sample) => sample.phase === 'workload'); + const surfaceSuccessfulWorkload = surfaceWorkload.filter((sample) => + sample.ok && sample.errorCode !== 'LOAD_GENERATOR_MISSED_ARRIVAL' + ); + const surfaceCoverage = surfaceSamples.filter((sample) => sample.ok); + const operationNames = surface.operations.map((operation) => operation.name); + const exercisedOperations = new Set(surfaceCoverage.map((sample) => sample.operation)); + const missingOperations = operationNames.filter((operation) => + !exercisedOperations.has(operation) + ); + const configuredCapabilities = [...new Set(surface.operations.map( + (operation) => operation.capability + ))]; + const exercisedCapabilities = new Set(surfaceCoverage.map((sample) => sample.capability)); + const missingCapabilities = configuredCapabilities.filter((capability) => + !exercisedCapabilities.has(capability) + ); + const missingOperationOracles = gates.requireConclusiveOperationOracles + ? operationNames.filter((operation) => + !conclusiveCoverageOracleKeys.has(`${surface.name}/${operation}`) + ) + : []; + const surfaceCanaries = localCanaries.filter((canary) => canary.surface === surface.name); + const surfaceOracleSamples = surfaceSamples.filter((sample) => + sample.oracleConfigured === true + ); + const surfaceOracleInconclusive = surfaceOracleSamples.filter((sample) => + sample.oracleUnavailable !== true && sample.oracleConclusive !== true + ).length; + const surfaceOracleViolations = surfaceOracleSamples.filter((sample) => + sample.oracleViolation === true + ).length; + const surfaceErrors = surfaceWorkload.filter((sample) => !sample.ok).length; + const surfaceErrorRate = surfaceWorkload.length > 0 + ? surfaceErrors / surfaceWorkload.length + : 1; + const surfaceP99Ms = percentile(surfaceWorkload.map((sample) => sample.latencyMs), 0.99); + const surfaceCanaryInconclusive = surfaceCanaries.filter( + (canary) => !canary.conclusive + ).length; + const surfaceBleedViolations = surfaceCanaries.filter( + (canary) => canary.violation + ).length; + const surfaceQualified = warmed.has(surface.name) + && surfaceSuccessfulWorkload.length >= minWorkloadRequestsPerSurface + && missingOperations.length === 0 + && missingCapabilities.length === 0 + && missingOperationOracles.length === 0 + && surfaceErrorRate <= gates.maxErrorRate + && surfaceP99Ms <= gates.maxP99Ms + && (!gates.requireConclusiveCanaries || surfaceCanaryInconclusive === 0) + && (!gates.requireZeroBleed || surfaceBleedViolations === 0) + && ( + !gates.requireConclusiveOperationOracles + || (surfaceOracleInconclusive === 0 && surfaceOracleViolations === 0) + ); + return { + surface: surface.name, + warmed: warmed.has(surface.name), + workloadRequests: surfaceWorkload.length, + successfulWorkloadRequests: surfaceSuccessfulWorkload.length, + errors: surfaceErrors, + errorRate: surfaceErrorRate, + p99Ms: surfaceP99Ms, + operationsConfigured: operationNames.length, + operationsExercised: operationNames.length - missingOperations.length, + canaryChecks: surfaceCanaries.length, + canaryInconclusive: surfaceCanaryInconclusive, + bleedViolations: surfaceBleedViolations, + operationOracleChecks: surfaceOracleSamples.length, + operationOracleInconclusive: surfaceOracleInconclusive, + operationOracleViolations: surfaceOracleViolations, + missingOperations, + missingCapabilities, + missingOperationOracles, + qualified: surfaceQualified + }; + }); + const trafficSurfaceNames = new Set(surfaceResults + .filter((surface) => surface.successfulWorkloadRequests >= minWorkloadRequestsPerSurface) + .map((surface) => surface.surface)); + const missingSurfaces = surfaceResults + .filter((surface) => !surface.warmed || !trafficSurfaceNames.has(surface.surface)) + .map((surface) => surface.surface); + const configuredOperations = tenant.surfaces.flatMap((surface) => + surface.operations.map((operation) => `${surface.name}/${operation.name}`) + ); + const missingOperations = configuredOperations.filter((operation) => + !successfulOperationKeys.has(operation) + ); + const missingOperationOracles = gates.requireConclusiveOperationOracles + ? configuredOperations.filter((operation) => + !conclusiveCoverageOracleKeys.has(operation) + ) + : []; + const configuredCapabilities = [...new Set(tenant.surfaces.flatMap((surface) => + surface.operations.map((operation) => `${surface.name}/${operation.capability}`) + ))]; + const missingCapabilities = [ + ...configuredCapabilities.filter((capability) => !successfulCapabilityKeys.has(capability)), + ...requiredCapabilities + .filter((capability) => !successfulCapabilities.has(capability)) + .map((capability) => `required/${capability}`) + ]; + const qualified = warmed.size === tenant.surfaces.length + && surfaceResults.every((surface) => surface.qualified) + && missingSurfaces.length === 0 + && missingOperations.length === 0 + && missingCapabilities.length === 0 + && missingOperationOracles.length === 0 + && errorRate <= gates.maxErrorRate + && p99Ms <= gates.maxP99Ms + && (!gates.requireConclusiveCanaries || canaryInconclusive === 0) + && (!gates.requireZeroBleed || bleedViolations === 0) + && ( + !gates.requireConclusiveOperationOracles + || ( + operationOracleInconclusive === 0 + && operationOracleViolations === 0 + ) + ); + return { + tenantId: tenant.id, + surfacesConfigured: tenant.surfaces.length, + surfacesWarmed: warmed.size, + surfacesWithTraffic: trafficSurfaceNames.size, + operationsConfigured: configuredOperations.length, + operationsExercised: configuredOperations.length - missingOperations.length, + requests: workloadSamples.length, + errors, + errorRate, + p99Ms, + canaryChecks: localCanaries.length, + canaryInconclusive, + bleedViolations, + operationOracleChecks: operationOracleSamples.length, + operationOracleInconclusive, + operationOracleViolations, + missingSurfaces, + missingOperations, + missingCapabilities, + missingOperationOracles, + surfaces: surfaceResults, + qualified + }; +}; + +export interface ScoreInput { + arm: string; + evidenceMode: 'qualification' | 'diagnostic'; + campaignId: string; + scheduleSha256: string; + previousResultPayloadSha256: string | null; + qualificationCohortSha256: string; + commit?: string; + introspectionMode: 'stock' | 'scoped-required'; + heapMiB: number; + repetition: number; + expectedMatrixRepetitions: number; + runKind: 'matrix' | 'soak'; + runOrderSeed: string; + runOrderIndex: number; + startedAt: string; + endedAt: string; + configuredDurationSec: number; + workloadDurationMs: number; + artifactDir: string; + tenants: TenantTarget[]; + warmedSurfaces: Map>; + warmupLatencies: number[]; + resolvedWarmupTimeoutMs: number; + offeredLoad: ResolvedOfferedLoad; + canaryIntervalSec: number; + periodicCanarySchedule: PeriodicCanarySchedule; + canarySchedule: CanaryScheduleSummary | null; + minWorkloadRequestsPerSurface: number; + samples: RequestSample[]; + canaries: CanaryResult[]; + memorySnapshots: MemorySnapshot[]; + postWarmupNodeRssSnapshots: NodeRssSnapshot[]; + memorySampleErrors: string[]; + retainedMemory: RetainedMemoryCheckpointPair; + postgresSnapshots: PostgresMemorySnapshot[]; + postgresSampleErrors: string[]; + postWarmupSnapshots: MemorySnapshot[]; + missedArrivals: number; + requiredCapabilities: string[]; + requiredCanaries: string[]; + gates: AcceptanceGates; + serverExit: DensityRunResult['serverExit']; + provenance: ArmProvenance | null; + provenanceErrors: string[]; + postgresRunAttestation?: PostgresRunAttestationEvidence | null; + realtimeDeliveryCoverage: RealtimeDeliveryCoverage | null; + externalServer: boolean; + executionErrors: string[]; +} + +const SHA256 = /^[a-f0-9]{64}$/; +const EMPTY_SHA256 = createHash('sha256').update('').digest('hex'); + +const validateRealtimeDeliveryCoverage = (input: ScoreInput): string[] => { + const coverage = input.realtimeDeliveryCoverage; + if (!coverage) return ['coverage record is unavailable']; + const failures: string[] = []; + const startedAtMs = Date.parse(coverage.workloadStartedAt); + const deadlineAtMs = Date.parse(coverage.workloadDeadlineAt); + const endedAtMs = coverage.workloadEndedAt == null + ? Number.NaN + : Date.parse(coverage.workloadEndedAt); + if ( + coverage.version !== 2 + || !Number.isSafeInteger(coverage.deliveryIntervalMs) + || coverage.deliveryIntervalMs <= 0 + || !Number.isFinite(startedAtMs) + || !Number.isFinite(deadlineAtMs) + || !Number.isFinite(endedAtMs) + || deadlineAtMs - startedAtMs !== input.configuredDurationSec * 1000 + || endedAtMs < deadlineAtMs + ) failures.push('coverage timing is invalid'); + + const expectedRoundsPerSurface = Math.max( + 0, + Math.ceil( + input.configuredDurationSec * 1000 / coverage.deliveryIntervalMs + ) - 1 + ); + const expectedSurfaces = new Map(); + for (const tenant of input.tenants) { + for (const surface of tenant.surfaces.filter((candidate) => candidate.realtime)) { + expectedSurfaces.set( + `${tenant.id}\0${surface.name}`, + new URL(surface.url).pathname + ); + } + } + const observed = new Set(); + if (expectedSurfaces.size > 0 && expectedRoundsPerSurface === 0) { + failures.push('no recurring round fits inside the workload'); + } + for (const surface of coverage.surfaces) { + const key = `${surface.tenantId}\0${surface.surface}`; + if (observed.has(key)) failures.push(`duplicate surface ${surface.tenantId}/${surface.surface}`); + observed.add(key); + if (expectedSurfaces.get(key) !== surface.route) { + failures.push(`unexpected route ${surface.tenantId}/${surface.surface}`); + } + if ( + surface.expectedRecurringRounds !== expectedRoundsPerSurface + || surface.startedRecurringRounds !== expectedRoundsPerSurface + || surface.verifiedRecurringRounds !== expectedRoundsPerSurface + || surface.primeRequests !== expectedRoundsPerSurface + ) failures.push(`incomplete surface ${surface.tenantId}/${surface.surface}`); + if ( + !SHA256.test(surface.issuedCorrelationSha256) + || !SHA256.test(surface.verifiedCorrelationSha256) + || surface.issuedCorrelationSha256 !== surface.verifiedCorrelationSha256 + || ( + expectedRoundsPerSurface === 0 + && surface.issuedCorrelationSha256 !== EMPTY_SHA256 + ) + || !Number.isFinite(surface.primeResponseP99Ms) + || surface.primeResponseP99Ms < 0 + || !Number.isFinite(surface.deliveryP99Ms) + || surface.deliveryP99Ms < 0 + ) failures.push(`correlation mismatch ${surface.tenantId}/${surface.surface}`); + } + if ( + observed.size !== expectedSurfaces.size + || [...expectedSurfaces.keys()].some((key) => !observed.has(key)) + ) failures.push('configured realtime surface set is incomplete'); + + const expectedTotal = expectedRoundsPerSurface * expectedSurfaces.size; + if ( + coverage.expectedRecurringRounds !== expectedTotal + || coverage.startedRecurringRounds !== expectedTotal + || coverage.verifiedRecurringRounds !== expectedTotal + || coverage.deadlineLateRecurringRounds !== 0 + || coverage.primeRequests !== expectedTotal + || !Number.isFinite(coverage.primeResponseP99Ms) + || coverage.primeResponseP99Ms < 0 + || !Number.isFinite(coverage.deliveryP99Ms) + || coverage.deliveryP99Ms < 0 + || coverage.complete !== true + ) failures.push('aggregate recurring delivery counters are incomplete'); + return failures; +}; + +type CanaryTuple = readonly [ + tenantId: string, + surface: string, + canary: string, + phase: CanaryResult['phase'], + periodicRound: number | null +]; + +const canaryTupleKey = (tuple: CanaryTuple): string => JSON.stringify(tuple); + +const increment = (counts: Map, key: string): void => { + counts.set(key, (counts.get(key) ?? 0) + 1); +}; + +const shortList = (values: string[]): string => { + const limit = 8; + return values.length <= limit + ? values.join(', ') + : `${values.slice(0, limit).join(', ')} (+${values.length - limit} more)`; +}; + +/** + * Validate isolation evidence from the fleet contract, not from artifact + * counters. JSON-encoded tuples preserve boundaries even when tenant, surface, + * or canary names themselves contain slashes. + */ +const validateStrictCanarySchedule = (input: ScoreInput): string[] => { + if (!input.gates.requireCompletePeriodicCanaryCoverage) return []; + const failures: string[] = []; + const summary = input.canarySchedule; + const expectedRoundCount = periodicCanaryRoundCount( + input.configuredDurationSec * 1000, + input.canaryIntervalSec * 1000 + ); + const expected = new Map(); + const configuredCanaries = new Map(); + const expectedChecksByTargetRound = new Map(); + const expectedTargetsPerRound = input.tenants.reduce( + (sum, tenant) => sum + tenant.surfaces.length, + 0 + ); + + for (const tenant of input.tenants) { + for (const surface of tenant.surfaces) { + for (const canary of surface.canaries) { + configuredCanaries.set( + JSON.stringify([tenant.id, surface.name, canary.name]), + [tenant.id, surface.name, canary.name] + ); + for (const phase of ['initial', 'final'] as const) { + const tuple: CanaryTuple = [tenant.id, surface.name, canary.name, phase, null]; + expected.set(canaryTupleKey(tuple), tuple); + } + } + for (let periodicRound = 1; periodicRound <= expectedRoundCount; periodicRound++) { + const selected = input.periodicCanarySchedule === 'rotating-one' + ? [surface.canaries[rotatingCanaryIndex( + tenant.id, + surface.name, + surface.canaries.length, + periodicRound + )]] + : surface.canaries; + expectedChecksByTargetRound.set( + JSON.stringify([tenant.id, surface.name, periodicRound]), + selected.length + ); + for (const canary of selected) { + const tuple: CanaryTuple = [ + tenant.id, + surface.name, + canary.name, + 'periodic', + periodicRound + ]; + expected.set(canaryTupleKey(tuple), tuple); + } + } + } + } + + const actualCounts = new Map(); + const periodicCoverage = new Set(); + const actualChecksByTargetRound = new Map(); + for (const result of input.canaries) { + const round = result.phase === 'periodic' + ? result.periodicRound ?? null + : null; + increment(actualCounts, canaryTupleKey([ + result.tenantId, + result.surface, + result.canary, + result.phase, + round + ])); + if (result.phase === 'periodic' && result.periodicRound != null) { + periodicCoverage.add(JSON.stringify([ + result.tenantId, + result.surface, + result.canary + ])); + increment(actualChecksByTargetRound, JSON.stringify([ + result.tenantId, + result.surface, + result.periodicRound + ])); + } + const scheduledAt = Date.parse(result.scheduledAt); + const startedAt = Date.parse(result.startedAt); + const completedAt = Date.parse(result.completedAt); + if ( + (result.phase === 'periodic') !== (result.periodicRound != null) + || (result.periodicRound != null && ( + !Number.isSafeInteger(result.periodicRound) + || result.periodicRound <= 0 + )) + || !Number.isFinite(scheduledAt) + || !Number.isFinite(startedAt) + || !Number.isFinite(completedAt) + || startedAt + 2 < scheduledAt + || completedAt < startedAt + || !Number.isFinite(result.latencyMs) + || result.latencyMs < 0 + ) { + failures.push( + `canary evidence has invalid timing ${JSON.stringify([ + result.tenantId, + result.surface, + result.canary, + result.phase, + round + ])}` + ); + } + } + + const missing = [...expected].filter(([key]) => !actualCounts.has(key)).map(([key]) => key); + const duplicates = [...expected].filter(([key]) => (actualCounts.get(key) ?? 0) !== 1) + .filter(([key]) => actualCounts.has(key)) + .map(([key]) => `${key} x${actualCounts.get(key)}`); + const unexpected = [...actualCounts].filter(([key]) => !expected.has(key)) + .map(([key, count]) => `${key} x${count}`); + if (missing.length > 0) failures.push(`missing exact canary evidence: ${shortList(missing)}`); + if (duplicates.length > 0) { + failures.push(`duplicate exact canary evidence: ${shortList(duplicates)}`); + } + if (unexpected.length > 0) { + failures.push(`unexpected canary evidence: ${shortList(unexpected)}`); + } + + const missingPeriodicCoverage = [...configuredCanaries] + .filter(([key]) => !periodicCoverage.has(key)) + .map(([key]) => key); + if (missingPeriodicCoverage.length > 0) { + failures.push( + `periodic canary coverage is incomplete: ${shortList(missingPeriodicCoverage)}` + ); + } + const targetRoundMismatches = [...expectedChecksByTargetRound] + .filter(([key, count]) => (actualChecksByTargetRound.get(key) ?? 0) !== count) + .map(([key, count]) => + `${key} expected=${count} actual=${actualChecksByTargetRound.get(key) ?? 0}` + ); + if (targetRoundMismatches.length > 0) { + failures.push( + `periodic target/round evidence mismatch: ${shortList(targetRoundMismatches)}` + ); + } + + if (!summary) { + failures.push('periodic canary schedule summary is unavailable'); + return failures; + } + if (summary.schedule !== input.periodicCanarySchedule) { + failures.push( + `periodic canary schedule=${summary.schedule}, expected ${input.periodicCanarySchedule}` + ); + } + if ( + summary.intervalMs !== input.canaryIntervalSec * 1000 + || summary.durationMs !== input.configuredDurationSec * 1000 + ) { + failures.push('periodic canary schedule timing does not match the workload plan'); + } + if ( + summary.planned !== expectedRoundCount + || summary.started !== expectedRoundCount + || summary.completed !== expectedRoundCount + || summary.missed !== 0 + ) { + failures.push( + `periodic canary rounds planned=${summary.planned} started=${summary.started} ` + + `completed=${summary.completed} missed=${summary.missed}, expected ${expectedRoundCount}` + ); + } + const expectedChecks = [...expected.values()].filter((tuple) => tuple[3] === 'periodic').length; + if ( + summary.checksPlanned !== expectedChecks + || summary.checksStarted !== expectedChecks + || summary.checksCompleted !== expectedChecks + ) { + failures.push( + `periodic canary checks planned=${summary.checksPlanned} ` + + `started=${summary.checksStarted} completed=${summary.checksCompleted}, ` + + `expected ${expectedChecks}` + ); + } + + const scheduleStart = Date.parse(summary.startedAt); + const scheduleDeadline = Date.parse(summary.deadlineAt); + if ( + !Number.isFinite(scheduleStart) + || !Number.isFinite(scheduleDeadline) + || scheduleDeadline !== scheduleStart + summary.durationMs + ) { + failures.push('periodic canary schedule boundary timestamps are invalid'); + } + const wrongPeriodicSlots = input.canaries.filter((result) => + result.phase === 'periodic' + && result.periodicRound != null + && Date.parse(result.scheduledAt) + !== scheduleStart + result.periodicRound * summary.intervalMs + ); + if (wrongPeriodicSlots.length > 0) { + failures.push( + `periodic canary evidence has wrong scheduled slots: ${shortList( + wrongPeriodicSlots.map((result) => JSON.stringify([ + result.tenantId, + result.surface, + result.canary, + result.periodicRound + ])) + )}` + ); + } + const roundsByNumber = new Map(summary.rounds.map((round) => [round.periodicRound, round])); + if (summary.rounds.length !== expectedRoundCount || roundsByNumber.size !== expectedRoundCount) { + failures.push('periodic canary round summaries are incomplete or duplicated'); + } + let recomputedDeadlineLate = 0; + const recomputedOverlapped = summary.rounds.filter((round) => round.overlapped).length; + if (summary.overlapped !== recomputedOverlapped) { + failures.push('periodic canary overlap count does not match round summaries'); + } + for (let periodicRound = 1; periodicRound <= expectedRoundCount; periodicRound++) { + const round = roundsByNumber.get(periodicRound); + if (!round) continue; + const plannedAt = Date.parse(round.plannedAt); + const startedAt = round.startedAt == null ? NaN : Date.parse(round.startedAt); + const completedAt = round.completedAt == null ? NaN : Date.parse(round.completedAt); + const expectedRoundChecks = [...expectedChecksByTargetRound] + .filter(([key]) => (JSON.parse(key) as [string, string, number])[2] === periodicRound) + .reduce((sum, [, count]) => sum + count, 0); + if ( + !Number.isFinite(plannedAt) + || plannedAt !== scheduleStart + periodicRound * summary.intervalMs + || !Number.isFinite(startedAt) + || !Number.isFinite(completedAt) + || completedAt < startedAt + || round.targetsPlanned !== expectedTargetsPerRound + || round.targetsStarted !== round.targetsPlanned + || round.targetsCompleted !== round.targetsPlanned + || round.checksPlanned !== expectedRoundChecks + || round.checksStarted !== expectedRoundChecks + || round.checksCompleted !== expectedRoundChecks + ) { + failures.push(`periodic canary round ${periodicRound} summary is incomplete`); + } + if (completedAt > scheduleDeadline || round.deadlineLate) recomputedDeadlineLate++; + } + const periodicResultsLate = input.canaries.filter((result) => + result.phase === 'periodic' + && Date.parse(result.completedAt) > scheduleDeadline + ).length; + if ( + summary.deadlineLate !== 0 + || recomputedDeadlineLate !== 0 + || periodicResultsLate !== 0 + ) { + failures.push( + `periodic canary rounds completed after deadline=${Math.max( + summary.deadlineLate, + recomputedDeadlineLate + )}; late checks=${periodicResultsLate}` + ); + } + return failures; +}; + +export const scoreRun = (input: ScoreInput): DensityRunResult => { + const coverageSamples = input.samples.filter((sample) => sample.phase === 'coverage'); + const workloadSamples = input.samples.filter((sample) => sample.phase === 'workload'); + const latencies = workloadSamples.map((sample) => sample.latencyMs); + const errors = workloadSamples.filter((sample) => !sample.ok).length; + const errorRate = workloadSamples.length > 0 ? errors / workloadSamples.length : 1; + const tenantResults = input.tenants.map((tenant) => tenantResult( + tenant, + input.samples, + input.canaries, + input.warmedSurfaces.get(tenant.id) ?? new Set(), + input.requiredCapabilities, + input.minWorkloadRequestsPerSurface, + input.gates + )); + const fleetShape = summarizeCustomerFleet(input.tenants); + const evictions = counterDelta(input.postWarmupSnapshots, 'evictions'); + const buildRefusals = counterDelta(input.postWarmupSnapshots, 'buildRefusals'); + const postWarmupBuilds = counterDelta(input.postWarmupSnapshots, 'buildsStarted'); + const postWarmupPgPoolCapacityEvictions = pgPoolCounterDelta( + input.postWarmupSnapshots, + 'pgPoolCapacityEvictions' + ); + const postWarmupPgPoolCapacityRefusals = pgPoolCounterDelta( + input.postWarmupSnapshots, + 'pgPoolCapacityRefusals' + ); + const postWarmupPgPoolDisposalFailures = pgPoolCounterDelta( + input.postWarmupSnapshots, + 'pgPoolDisposalFailures' + ); + const completePgPoolTelemetry = input.postWarmupSnapshots.length > 0 + && input.postWarmupSnapshots.every((snapshot) => + snapshot.pgPoolCacheSize != null + && snapshot.pgPoolLeasedPools != null + && snapshot.pgPoolActiveLeases != null + && snapshot.pgPoolCapacityEvictions != null + && snapshot.pgPoolCapacityRefusals != null + && snapshot.pgPoolDisposalFailures != null + ); + const pgPoolCacheSize = completePgPoolTelemetry + ? Math.max(...input.postWarmupSnapshots.map((snapshot) => snapshot.pgPoolCacheSize!)) + : null; + const pgPoolLeasedPools = completePgPoolTelemetry + ? Math.max(...input.postWarmupSnapshots.map((snapshot) => snapshot.pgPoolLeasedPools!)) + : null; + const pgPoolActiveLeases = completePgPoolTelemetry + ? Math.max(...input.postWarmupSnapshots.map((snapshot) => snapshot.pgPoolActiveLeases!)) + : null; + const maximumCompleteValue = (key: keyof MemorySnapshot): number | null => { + const values = input.postWarmupSnapshots + .map((snapshot) => snapshot[key]) + .filter((value): value is number => typeof value === 'number'); + return values.length === input.postWarmupSnapshots.length && values.length > 0 + ? Math.max(...values) + : null; + }; + const minimumCompleteValue = (key: keyof MemorySnapshot): number | null => { + const values = input.postWarmupSnapshots + .map((snapshot) => snapshot[key]) + .filter((value): value is number => typeof value === 'number'); + return values.length === input.postWarmupSnapshots.length && values.length > 0 + ? Math.min(...values) + : null; + }; + const postgresBackendPeak = maximumCompleteValue('postgresBackendTotal'); + const physicalDatabaseValues = input.postWarmupSnapshots + .map((snapshot) => snapshot.physicalDatabases) + .filter((value): value is number => typeof value === 'number'); + const residentPhysicalDatabases = physicalDatabaseValues.length + === input.postWarmupSnapshots.length && physicalDatabaseValues.length > 0 + ? Math.min(...physicalDatabaseValues) + : null; + const postgresContainerScopeValues = input.postWarmupSnapshots + .map((snapshot) => snapshot.postgresContainerDedicated) + .filter((value): value is boolean => typeof value === 'boolean'); + const postgresContainerDedicated = postgresContainerScopeValues.length + === input.postWarmupSnapshots.length && postgresContainerScopeValues.length > 0 + ? postgresContainerScopeValues.every(Boolean) + : null; + const unexpectedPostgresDatabases = maximumCompleteValue( + 'unexpectedPostgresDatabases' + ); + const pgPoolTotalClients = maximumCompleteValue('pgPoolTotalClients'); + const pgPoolIdleClients = maximumCompleteValue('pgPoolIdleClients'); + const pgPoolWaitingClients = maximumCompleteValue('pgPoolWaitingClients'); + const completeRuntimePoolTelemetry = input.postWarmupSnapshots.length > 0 + && input.postWarmupSnapshots.every((snapshot) => + snapshot.runtimePoolTelemetryScope === 'runtime-only-exact-identities' + && snapshot.runtimePoolTelemetryAvailable === true + && snapshot.runtimePoolEffectiveMaxUsesKnown === true + && snapshot.runtimePoolMaxUsesExact === true + && snapshot.runtimePoolExpectedPools === fleetShape.apis + && snapshot.runtimePoolObservedPools === fleetShape.apis + && Number.isSafeInteger(snapshot.runtimePoolTotalClients) + && snapshot.runtimePoolTotalClients! >= 0 + && Number.isSafeInteger(snapshot.runtimePoolIdleClients) + && snapshot.runtimePoolIdleClients! >= 0 + && Number.isSafeInteger(snapshot.runtimePoolWaitingClients) + && snapshot.runtimePoolWaitingClients! >= 0 + && ( + snapshot.runtimePoolRequestedMaxUses == null + || ( + Number.isSafeInteger(snapshot.runtimePoolRequestedMaxUses) + && snapshot.runtimePoolRequestedMaxUses > 0 + ) + ) + && snapshot.runtimePoolEffectiveMaxUses + === snapshot.runtimePoolRequestedMaxUses + ); + const requestedMaxUsesValues = completeRuntimePoolTelemetry + ? [...new Set(input.postWarmupSnapshots.map( + (snapshot) => snapshot.runtimePoolRequestedMaxUses ?? null + ))] + : []; + const effectiveMaxUsesValues = completeRuntimePoolTelemetry + ? [...new Set(input.postWarmupSnapshots.map( + (snapshot) => snapshot.runtimePoolEffectiveMaxUses ?? null + ))] + : []; + const runtimePoolRequestedMaxUses = requestedMaxUsesValues.length === 1 + ? requestedMaxUsesValues[0] + : null; + const runtimePoolEffectiveMaxUses = effectiveMaxUsesValues.length === 1 + ? effectiveMaxUsesValues[0] + : null; + const runtimePoolExpectedPools = completeRuntimePoolTelemetry + ? minimumCompleteValue('runtimePoolExpectedPools') + : null; + const runtimePoolObservedPools = completeRuntimePoolTelemetry + ? minimumCompleteValue('runtimePoolObservedPools') + : null; + const runtimePoolTotalClients = completeRuntimePoolTelemetry + ? maximumCompleteValue('runtimePoolTotalClients') + : null; + const runtimePoolIdleClients = completeRuntimePoolTelemetry + ? maximumCompleteValue('runtimePoolIdleClients') + : null; + const runtimePoolWaitingClients = completeRuntimePoolTelemetry + ? maximumCompleteValue('runtimePoolWaitingClients') + : null; + const residentRealtimeManagers = minimumCompleteValue('realtimeManagersActive'); + const residentRealtimeTransports = minimumCompleteValue('realtimeTransportsActive'); + const notificationModes = input.postWarmupSnapshots + .map((snapshot) => snapshot.realtimeNotificationMode) + .filter((value): value is 'dedicated' | 'shared-exact' => + value === 'dedicated' || value === 'shared-exact' + ); + const realtimeNotificationMode = notificationModes.length + === input.postWarmupSnapshots.length + && notificationModes.length > 0 + && new Set(notificationModes).size === 1 + ? notificationModes[0] + : null; + const notificationBrokers = minimumCompleteValue('notificationBrokers'); + const notificationListenerConnections = minimumCompleteValue( + 'notificationListenerConnections' + ); + const notificationBrokerLeases = minimumCompleteValue('notificationBrokerLeases'); + const notificationBrokerTopics = minimumCompleteValue('notificationBrokerTopics'); + const notificationBrokerSubscribers = minimumCompleteValue( + 'notificationBrokerSubscribers' + ); + const notificationBrokerQueueOverflows = maximumCompleteValue( + 'notificationBrokerQueueOverflows' + ); + const notificationBrokerFatalFailures = maximumCompleteValue( + 'notificationBrokerFatalFailures' + ); + const notificationAuditIdentities = minimumCompleteValue( + 'notificationAuditIdentities' + ); + const notificationAuditsHealthy = minimumCompleteValue( + 'notificationAuditsHealthy' + ); + const notificationAuditsFailed = maximumCompleteValue('notificationAuditsFailed'); + const notificationAuditsStale = maximumCompleteValue('notificationAuditsStale'); + const notificationAuditAttempts = maximumCompleteValue('notificationAuditAttempts'); + const notificationAuditFailures = maximumCompleteValue('notificationAuditFailures'); + const notificationAuditActiveDatabaseTargets = minimumCompleteValue( + 'notificationAuditActiveDatabaseTargets' + ); + const notificationAuditDatabaseConflicts = maximumCompleteValue( + 'notificationAuditDatabaseConflicts' + ); + const cacheConfiguredMax = minimumCompleteValue('cacheConfiguredMax'); + const cacheBudgetCapacity = minimumCompleteValue('cacheBudgetCapacity'); + const cacheInstanceHeapBytes = maximumCompleteValue('cacheInstanceHeapBytes'); + const cacheCalibrationIds = input.postWarmupSnapshots + .map((snapshot) => snapshot.cacheCalibrationId) + .filter((value): value is string => typeof value === 'string' && value.length > 0); + const cacheCalibrationId = cacheCalibrationIds.length + === input.postWarmupSnapshots.length + && new Set(cacheCalibrationIds).size === 1 + ? cacheCalibrationIds[0] + : null; + const cacheAdmissionModes = input.postWarmupSnapshots + .map((snapshot) => snapshot.cacheAdmissionMode) + .filter((value): value is NonNullable => + value === 'evict-idle' || value === 'preserve-resident' + ); + const cacheAdmissionMode = cacheAdmissionModes.length + === input.postWarmupSnapshots.length + && cacheAdmissionModes.length > 0 + && new Set(cacheAdmissionModes).size === 1 + ? cacheAdmissionModes[0] + : null; + const rawHeapGrowth = heapGrowthMiBPerHour(input.postWarmupSnapshots); + const postgresPeakBytes = input.postgresSnapshots.length + ? Math.max(...input.postgresSnapshots.map((snapshot) => snapshot.usedBytes)) + : null; + const postgresBaselineBytes = input.postgresSnapshots[0]?.usedBytes ?? null; + const postgresWorkingSetValues = input.postgresSnapshots + .map((snapshot) => snapshot.workingSetBytes) + .filter((value): value is number => value != null); + const postgresWorkingSetPeakBytes = postgresWorkingSetValues.length > 0 + ? Math.max(...postgresWorkingSetValues) + : null; + const postgresCgroupV2Samples = input.postgresSnapshots.filter( + (snapshot) => snapshot.source === 'cgroup-v2' && snapshot.cgroupV2 != null + ).length; + const completePostgresCgroupV2CurrentTelemetry = input.postgresSnapshots.length > 0 + && input.postgresSnapshots.every((snapshot) => + snapshot.source === 'cgroup-v2' + && snapshot.cgroupV2 != null + && Number.isSafeInteger(snapshot.cgroupV2.currentBytes) + && snapshot.cgroupV2.currentBytes >= 0 + && snapshot.usedBytes === snapshot.cgroupV2.currentBytes + ); + const postgresCgroupPeakValues = input.postgresSnapshots + .map((snapshot) => snapshot.cgroupV2?.peakBytes) + .filter((value): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 + ); + const postgresCgroupV2PeakBytes = postgresCgroupPeakValues.length > 0 + ? Math.max(...postgresCgroupPeakValues) + : null; + const completePostgresCgroupV2PeakTelemetry = completePostgresCgroupV2CurrentTelemetry + && postgresCgroupPeakValues.length === input.postgresSnapshots.length; + const firstPostgresEvents = input.postgresSnapshots[0]?.cgroupV2?.events; + const lastPostgresEvents = input.postgresSnapshots.at(-1)?.cgroupV2?.events; + const completePostgresOomEvents = firstPostgresEvents + && lastPostgresEvents + && typeof firstPostgresEvents.oom === 'number' + && typeof firstPostgresEvents.oom_kill === 'number' + && typeof lastPostgresEvents.oom === 'number' + && typeof lastPostgresEvents.oom_kill === 'number'; + const postgresOomEvents = completePostgresOomEvents + ? Math.max(0, lastPostgresEvents.oom - firstPostgresEvents.oom) + + Math.max( + 0, + lastPostgresEvents.oom_kill - firstPostgresEvents.oom_kill + ) + : null; + const postWarmupStartedAtMs = input.postWarmupSnapshots[0] + ? Date.parse(input.postWarmupSnapshots[0].timestamp) + : Number.NaN; + const coldBuildPostgresSnapshots = Number.isFinite(postWarmupStartedAtMs) + ? input.postgresSnapshots.filter( + (snapshot) => Date.parse(snapshot.timestamp) <= postWarmupStartedAtMs + ) + : []; + const postgresColdBuildPeakBytes = coldBuildPostgresSnapshots.length > 0 + ? Math.max(...coldBuildPostgresSnapshots.map((snapshot) => snapshot.usedBytes)) + : null; + const postgresColdBuildSpikeBytes = postgresColdBuildPeakBytes != null + && input.postgresSnapshots.length >= 2 + ? Math.max(0, postgresColdBuildPeakBytes - input.postgresSnapshots[0].usedBytes) + : null; + const postWarmupPostgresSnapshots = Number.isFinite(postWarmupStartedAtMs) + ? input.postgresSnapshots.filter( + // Retain the nearest preceding cgroup sample for sub-second alignment. + (snapshot) => Date.parse(snapshot.timestamp) >= postWarmupStartedAtMs - 1_000 + ) + : []; + const alignedPostgresSnapshots = input.evidenceMode === 'qualification' + ? postWarmupPostgresSnapshots.filter( + (snapshot) => snapshot.source === 'cgroup-v2' && snapshot.cgroupV2 != null + ).map((snapshot) => ({ + ...snapshot, + usedBytes: snapshot.cgroupV2!.currentBytes + })) + : postWarmupPostgresSnapshots; + const postgresWarmBoundaryBytes = Number.isFinite(postWarmupStartedAtMs) + && input.postgresSnapshots.length > 0 + ? input.postgresSnapshots.reduce((nearest, snapshot) => + Math.abs(Date.parse(snapshot.timestamp) - postWarmupStartedAtMs) + < Math.abs(Date.parse(nearest.timestamp) - postWarmupStartedAtMs) + ? snapshot + : nearest + ).usedBytes + : null; + const alignedServicePeak = alignedServiceMemoryPeak( + input.postWarmupNodeRssSnapshots, + alignedPostgresSnapshots + ); + const alignedServiceCoverage = alignedServiceMemoryCoverage( + input.postWarmupNodeRssSnapshots, + alignedPostgresSnapshots, + postWarmupStartedAtMs, + input.workloadDurationMs + ); + const expectedResidentInstances = new Set(input.tenants.flatMap((tenant) => + tenant.surfaces.map((surface) => surface.buildContract) + )).size; + const expectedResidentBuildContracts = new Set(input.tenants.flatMap((tenant) => + tenant.surfaces.map((surface) => surface.buildContract) + )); + const retainedGrowth = retainedMemoryGrowth( + input.retainedMemory, + input.provenance?.serverPid ?? null, + expectedResidentBuildContracts, + input.gates.requirePhysicalDatabaseTelemetry + ); + const residentCounts = input.postWarmupSnapshots + .map((snapshot) => snapshot.cacheSize) + .filter((value): value is number => value != null); + const residentInstances = residentCounts.length === input.postWarmupSnapshots.length + && residentCounts.length > 0 + ? Math.min(...residentCounts) + : null; + const baselineHeapBytes = input.memorySnapshots[0]?.heapUsedBytes; + const warmHeapBytes = input.postWarmupSnapshots[0]?.heapUsedBytes; + const warmCacheSize = input.postWarmupSnapshots[0]?.cacheSize; + const warmObservedHeapDeltaPerInstanceBytes = baselineHeapBytes != null + && warmHeapBytes != null + && warmCacheSize > 0 + ? Math.max(0, warmHeapBytes - baselineHeapBytes) / warmCacheSize + : null; + const successfulSamples = input.samples.filter((sample) => sample.ok); + const capabilitiesExercised = [...new Set(successfulSamples.map((sample) => sample.capability))] + .sort(); + const missingCapabilities = tenantResults.flatMap((tenant) => + tenant.missingCapabilities.map((capability) => `${tenant.tenantId}/${capability}`) + ); + const checkedCanaries = new Set(input.canaries.map((canary) => + JSON.stringify([canary.tenantId, canary.surface, canary.canary]) + )); + const missingCanaries = input.tenants.flatMap((tenant) => tenant.surfaces.flatMap((surface) => + input.requiredCanaries + .filter((canary) => !checkedCanaries.has(JSON.stringify([ + tenant.id, + surface.name, + canary + ]))) + .map((canary) => `${tenant.id}/${surface.name}/${canary}`) + )); + const operationOracleChecks = tenantResults.reduce( + (sum, tenant) => sum + tenant.operationOracleChecks, + 0 + ); + const operationOracleInconclusive = tenantResults.reduce( + (sum, tenant) => sum + tenant.operationOracleInconclusive, + 0 + ); + const operationOracleViolations = tenantResults.reduce( + (sum, tenant) => sum + tenant.operationOracleViolations, + 0 + ); + const missingOperationOracles = tenantResults.flatMap((tenant) => + tenant.missingOperationOracles.map((operation) => + `${tenant.tenantId}/${operation}` + ) + ); + const failures: string[] = validateStrictCanarySchedule(input); + if (!/^[a-f0-9]{64}$/.test(input.campaignId)) { + failures.push('campaign identity is unavailable or invalid'); + } + if (!/^[a-f0-9]{64}$/.test(input.scheduleSha256)) { + failures.push('campaign schedule binding is unavailable or invalid'); + } + if ( + input.previousResultPayloadSha256 != null + && !/^[a-f0-9]{64}$/.test(input.previousResultPayloadSha256) + ) { + failures.push('campaign result-chain pointer is invalid'); + } + if (input.configuredDurationSec < 900) failures.push('workload shorter than the 15-minute qualification floor'); + if (input.workloadDurationMs < input.configuredDurationSec * 1000 * 0.99) { + failures.push(`measured workload duration ${(input.workloadDurationMs / 1000).toFixed(2)}s is short`); + } + if (input.memorySnapshots.length === 0) failures.push('memory snapshots unavailable'); + if (input.memorySampleErrors.length > 0) failures.push(`memory sampler errors=${input.memorySampleErrors.length}`); + if (input.provenanceErrors.length > 0) { + failures.push(`provenance validation errors: ${input.provenanceErrors.join('; ')}`); + } + if (!input.provenance) { + failures.push('server provenance unavailable'); + } else { + const missingProvenance = [ + !input.provenance.cwd ? 'cwd' : null, + input.provenance.command.length === 0 ? 'command' : null, + !input.provenance.gitHead ? 'gitHead' : null, + input.provenance.worktreeDirty !== false ? 'cleanWorktree' : null, + !input.provenance.gitStatusSha256 ? 'gitStatusSha256' : null, + !input.provenance.lockfileSha256 ? 'lockfileSha256' : null, + !input.provenance.entrySha256 ? 'entrySha256' : null, + input.provenance.serverPid == null ? 'serverPid' : null, + !input.provenance.v8Profile ? 'v8Profile' : null, + !input.provenance.nodeOptions ? 'nodeOptions' : null, + !Array.isArray(input.provenance.nodeOptionsArgv) ? 'nodeOptionsArgv' : null, + !Array.isArray(input.provenance.nodeExecArgv) ? 'nodeExecArgv' : null, + !Array.isArray(input.provenance.effectiveNodeRuntimeFlags) + ? 'effectiveNodeRuntimeFlags' + : null, + !input.provenance.planSha256 ? 'planSha256' : null, + !input.provenance.fleetSha256 ? 'fleetSha256' : null, + !input.provenance.node ? 'node' : null, + !input.provenance.v8 ? 'v8' : null, + !input.provenance.runOrderSeed ? 'runOrderSeed' : null, + !input.provenance.memoryPolicy ? 'memoryPolicy' : null + ].filter((value): value is string => value != null); + if (missingProvenance.length > 0) { + failures.push(`server provenance incomplete: ${missingProvenance.join(', ')}`); + } + if ( + input.evidenceMode === 'qualification' + && ( + input.provenance.platform !== 'linux' + || input.postWarmupNodeRssSnapshots.length === 0 + || input.postWarmupNodeRssSnapshots.some((snapshot) => + snapshot.source !== 'proc' + || snapshot.pid !== input.provenance!.serverPid + ) + ) + ) { + failures.push('qualification requires exact-PID Linux /proc RSS evidence'); + } + const expectedProfileFlags: Record = { + stock: [], + 'optimize-for-size': ['--optimize-for-size'], + 'baseline-optimize-for-size': ['--max-opt=1', '--optimize-for-size'], + 'jitless-optimize-for-size': ['--jitless', '--optimize-for-size'] + }; + const expectedFlags = expectedProfileFlags[input.provenance.v8Profile]; + const managedFlags = input.provenance.nodeExecArgv.filter((flag) => + flag === '--jitless' + || flag === '--optimize-for-size' + || flag === '--max-opt=1' + ); + const expectedEffectiveFlags = [ + ...input.provenance.nodeOptionsArgv, + ...input.provenance.nodeExecArgv + ]; + if ( + !expectedFlags + || JSON.stringify(managedFlags) !== JSON.stringify(expectedFlags) + || input.provenance.nodeOptionsArgv.some((flag) => + flag === '--jitless' + || flag === '--optimize-for-size' + || flag === '--max-opt=1' + ) + || JSON.stringify(input.provenance.effectiveNodeRuntimeFlags) + !== JSON.stringify(expectedEffectiveFlags) + ) { + failures.push('server V8 runtime-flag provenance is inconsistent'); + } + } + if (input.gates.requireFreshPostgresRunAttestation) { + const attestation = input.postgresRunAttestation; + const provenanceCommand = input.provenance?.command ?? []; + const argumentAfter = (flag: string): string | null => { + const index = provenanceCommand.indexOf(flag); + return index >= 0 ? provenanceCommand[index + 1] ?? null : null; + }; + const exactRunBinding = attestation != null + && attestation.arm === input.arm + && attestation.heapMiB === input.heapMiB + && attestation.tenantCount === input.tenants.length + && attestation.repetition === input.repetition + && attestation.runOrderIndex === input.runOrderIndex + && attestation.planSha256 === `sha256:${input.provenance?.planSha256}` + && attestation.fleetSha256 === `sha256:${input.provenance?.fleetSha256}` + && argumentAfter('--expected-manifest-sha256') === attestation.manifestSha256 + && argumentAfter('--clone-id') === attestation.cloneId; + if (!attestation) { + failures.push('fresh PostgreSQL run attestation unavailable'); + } else if ( + !exactRunBinding + || attestation.freshContainerForRun !== true + || attestation.cgroupV2Verified !== true + || attestation.liveCustomerContractsAudited !== input.tenants.length + || attestation.catalogCacheState !== 'warmed-by-live-contract-audit' + || !/^sha256:[a-f0-9]{64}$/.test(attestation.containerConfigurationSha256) + || !/^sha256:[a-f0-9]{64}$/.test(attestation.cloneAttestationSetSha256) + || !/^sha256:[a-f0-9]{64}$/.test(attestation.cloneNonceSetSha256) + || !/^sha256:[a-f0-9]{64}$/.test(attestation.liveContractSetSha256) + ) { + failures.push('fresh PostgreSQL run attestation is incomplete or mismatched'); + } + if ( + attestation + && ( + input.postgresSnapshots.length === 0 + || input.postgresSnapshots.some((snapshot) => + snapshot.containerId !== attestation.containerId + || snapshot.cgroupIdentitySha256 !== attestation.cgroupIdentitySha256 + ) + ) + ) { + failures.push('PostgreSQL memory samples do not match the attested immutable container'); + } + } + if (input.gates.requiredCacheAdmissionMode) { + const requiredMode = input.gates.requiredCacheAdmissionMode; + const pinnedMode = input.provenance?.memoryPolicy + ?.graphileCacheAdmissionMode ?? null; + if (cacheAdmissionMode !== requiredMode) { + failures.push( + `live Graphile cache admission mode=${cacheAdmissionMode ?? 'unknown'}, required ${requiredMode}` + ); + } + if (pinnedMode !== requiredMode) { + failures.push( + `pinned Graphile cache admission mode=${pinnedMode ?? 'unknown'}, required ${requiredMode}` + ); + } + } + if (input.gates.requirePostgresMemoryTelemetry && input.postgresSnapshots.length < 2) failures.push('PostgreSQL memory telemetry unavailable'); + if (input.gates.requirePostgresMemoryTelemetry && input.postgresSampleErrors.length > 0) failures.push(`PostgreSQL memory sampler errors=${input.postgresSampleErrors.length}`); + if ( + input.gates.requirePostgresMemoryTelemetry + && input.evidenceMode === 'qualification' + && !completePostgresCgroupV2CurrentTelemetry + ) { + failures.push('PostgreSQL cgroup-v2 telemetry was incomplete'); + } + if ( + input.gates.requirePostgresMemoryTelemetry + && input.evidenceMode === 'qualification' + && !completePostgresCgroupV2PeakTelemetry + ) { + failures.push( + 'PostgreSQL cgroup-v2 memory.peak telemetry unavailable for conservative denominator' + ); + } + if (postgresOomEvents != null && postgresOomEvents > 0) { + failures.push(`PostgreSQL cgroup recorded OOM events=${postgresOomEvents}`); + } + if ( + input.gates.requirePostgresMemoryTelemetry + && postgresCgroupV2Samples === input.postgresSnapshots.length + && postgresCgroupV2Samples > 0 + && postgresOomEvents == null + ) { + failures.push('PostgreSQL cgroup OOM event telemetry unavailable'); + } + if ( + input.gates.requirePostgresMemoryTelemetry + && (!alignedServicePeak || alignedServicePeak.samples < 2) + ) { + failures.push('aligned Node and PostgreSQL service-memory telemetry unavailable'); + } + if (input.gates.requirePostgresMemoryTelemetry && input.evidenceMode === 'qualification') { + const maxGapMs = input.gates.maxAlignedMemorySampleGapMs + ?? DEFAULT_MAX_ALIGNED_MEMORY_SAMPLE_GAP_MS; + const minCoverageRatio = input.gates.minAlignedMemoryCoverageRatio + ?? DEFAULT_MIN_ALIGNED_MEMORY_COVERAGE_RATIO; + if (!alignedServiceCoverage) { + failures.push('aligned service-memory workload coverage unavailable'); + } else { + if (alignedServiceCoverage.maxGapMs > maxGapMs) { + failures.push( + `aligned service-memory maximum sample gap ${alignedServiceCoverage.maxGapMs.toFixed(0)}ms exceeds ${maxGapMs}ms` + ); + } + if (alignedServiceCoverage.coverageRatio < minCoverageRatio) { + failures.push( + `aligned service-memory workload coverage ${(alignedServiceCoverage.coverageRatio * 100).toFixed(2)}% is below ${(minCoverageRatio * 100).toFixed(2)}%` + ); + } + } + } + if (residentInstances == null || residentInstances < expectedResidentInstances) { + failures.push(`resident Graphile instances=${residentInstances ?? 'unknown'}, expected at least ${expectedResidentInstances}`); + } + const expectedBuildContracts = new Set(input.tenants.flatMap((tenant) => + tenant.surfaces.map((surface) => surface.buildContract) + )); + const identityUnavailable = input.postWarmupSnapshots.some((snapshot) => + input.gates.requirePhysicalDatabaseTelemetry + ? snapshot.residentBuildContractFingerprints == null + : snapshot.residentBuildContracts == null + ); + const missingResidentBuilds = new Set(); + for (const snapshot of input.postWarmupSnapshots) { + const resident = new Set( + (input.gates.requirePhysicalDatabaseTelemetry + ? snapshot.residentBuildContractFingerprints + : snapshot.residentBuildContracts) + ?? [] + ); + for (const contract of expectedBuildContracts) { + if (!resident.has(contract)) missingResidentBuilds.add(contract); + } + } + if (identityUnavailable || input.postWarmupSnapshots.length === 0) { + failures.push(input.gates.requirePhysicalDatabaseTelemetry + ? 'resident Graphile build-contract fingerprints unavailable' + : 'resident Graphile build-contract identities unavailable'); + } else if (missingResidentBuilds.size > 0) { + failures.push(input.gates.requirePhysicalDatabaseTelemetry + ? `resident Graphile build fingerprints missing: ${[...missingResidentBuilds].join(', ')}` + : `resident Graphile build contracts missing: ${[...missingResidentBuilds].join(', ')}`); + } + if (input.missedArrivals > 0) { + failures.push(`load generator missed scheduled arrivals=${input.missedArrivals}`); + } + if (errorRate > input.gates.maxErrorRate) failures.push(`error rate ${errorRate} exceeds ${input.gates.maxErrorRate}`); + if (percentile(latencies, 0.99) > input.gates.maxP99Ms) failures.push(`p99 exceeds ${input.gates.maxP99Ms}ms`); + if (input.gates.requireNoPostWarmupEvictions && evictions !== 0) failures.push(`post-warmup evictions=${evictions ?? 'unknown'}`); + if (input.gates.requireNoPostWarmupBuildRefusals && buildRefusals !== 0) failures.push(`post-warmup build refusals=${buildRefusals ?? 'unknown'}`); + if (input.gates.requireNoPostWarmupBuilds && postWarmupBuilds !== 0) failures.push(`post-warmup builds=${postWarmupBuilds ?? 'unknown'}`); + if (!completePgPoolTelemetry) failures.push('PostgreSQL pool-cache telemetry unavailable'); + const expectedRealtimeApis = fleetShape.realtimeApis; + if (input.gates.requirePhysicalDatabaseTelemetry) { + const expectedCalibrationId = input.provenance?.memoryPolicy + ?.graphileCacheCalibrationId ?? null; + if ( + !expectedCalibrationId + || cacheCalibrationId !== expectedCalibrationId + ) { + failures.push( + `Graphile cache calibration identity=${cacheCalibrationId ?? 'unknown'}, expected ${expectedCalibrationId ?? 'pinned provenance identity'}` + ); + } + if (cacheConfiguredMax == null || cacheConfiguredMax < expectedResidentInstances) { + failures.push( + `Graphile configured cache max=${cacheConfiguredMax ?? 'unknown'}, expected at least ${expectedResidentInstances}` + ); + } + if (cacheBudgetCapacity == null || cacheBudgetCapacity < expectedResidentInstances) { + failures.push( + `Graphile heap budget capacity=${cacheBudgetCapacity ?? 'unknown'}, expected at least ${expectedResidentInstances}` + ); + } + if (postgresContainerDedicated !== true) { + failures.push( + `dedicated PostgreSQL container scope not proven; unexpected databases=${unexpectedPostgresDatabases ?? 'unknown'}` + ); + } + if ( + residentPhysicalDatabases == null + || physicalDatabaseValues.some((value) => value !== fleetShape.physicalDatabases) + ) { + failures.push( + `resident physical databases=${residentPhysicalDatabases ?? 'unknown'}, expected exactly ${fleetShape.physicalDatabases}` + ); + } + if (postgresBackendPeak == null) { + failures.push('physical PostgreSQL backend telemetry unavailable'); + } + if ( + pgPoolTotalClients == null + || pgPoolIdleClients == null + || pgPoolWaitingClients == null + ) { + failures.push('physical PostgreSQL pool-client telemetry unavailable'); + } + if ( + !completeRuntimePoolTelemetry + || requestedMaxUsesValues.length !== 1 + || effectiveMaxUsesValues.length !== 1 + || runtimePoolExpectedPools !== fleetShape.apis + || runtimePoolObservedPools !== fleetShape.apis + ) { + failures.push( + `exact runtime PostgreSQL pool telemetry unavailable or inconsistent; observed=${runtimePoolObservedPools ?? 'unknown'}, expected=${fleetShape.apis}` + ); + } else if ( + runtimePoolRequestedMaxUses === 1 + && runtimePoolEffectiveMaxUses === 1 + && input.postWarmupSnapshots.some((snapshot) => + snapshot.runtimePoolIdleClients !== 0 + ) + ) { + failures.push( + 'runtime PostgreSQL maxUses=1 retained idle clients after warmup' + ); + } + if (expectedRealtimeApis > 0 && realtimeNotificationMode === 'shared-exact') { + const expectedBrokers = fleetShape.physicalDatabases; + const exactSharedRealtime = input.postWarmupSnapshots.length > 0 + && input.postWarmupSnapshots.every((snapshot) => + snapshot.realtimeNotificationMode === 'shared-exact' + && snapshot.notificationBrokers === expectedBrokers + && snapshot.notificationListenerConnections === expectedBrokers + && snapshot.notificationBrokerLeases === expectedRealtimeApis + && snapshot.notificationBrokerTopics === expectedRealtimeApis + && snapshot.notificationBrokerSubscribers === expectedRealtimeApis + && snapshot.notificationBrokerQueueOverflows === 0 + && snapshot.notificationBrokerFatalFailures === 0 + && snapshot.notificationAuditIdentities === expectedBrokers + && snapshot.notificationAuditsHealthy === expectedBrokers + && snapshot.notificationAuditsFailed === 0 + && snapshot.notificationAuditsStale === 0 + && snapshot.notificationAuditAttempts != null + && snapshot.notificationAuditAttempts >= expectedRealtimeApis + && snapshot.notificationAuditFailures === 0 + && snapshot.notificationAuditActiveDatabaseTargets === expectedBrokers + && snapshot.notificationAuditDatabaseConflicts === 0 + ); + if (!exactSharedRealtime) { + failures.push( + 'shared realtime broker residency or listener-role attestation is not exact' + ); + } + } else if (expectedRealtimeApis > 0) { + if (realtimeNotificationMode !== 'dedicated') { + failures.push('realtime notification mode telemetry unavailable or inconsistent'); + } + if (postgresBackendPeak != null && postgresBackendPeak < expectedRealtimeApis) { + failures.push( + `physical PostgreSQL backends=${postgresBackendPeak}, expected at least ${expectedRealtimeApis} dedicated realtime APIs` + ); + } + if (pgPoolTotalClients != null && pgPoolTotalClients < expectedRealtimeApis) { + failures.push( + `physical PostgreSQL pool clients=${pgPoolTotalClients}, expected at least ${expectedRealtimeApis} dedicated realtime APIs` + ); + } + } + } + if (expectedRealtimeApis > 0) { + const expectedManagers = minimumCompleteValue('realtimeManagersExpected'); + const expectedTransports = minimumCompleteValue('realtimeTransportsExpected'); + if ( + expectedManagers == null + || expectedManagers < expectedRealtimeApis + || residentRealtimeManagers == null + || residentRealtimeManagers < expectedManagers + ) { + failures.push( + `resident realtime managers=${residentRealtimeManagers ?? 'unknown'}, expected ${expectedManagers ?? expectedRealtimeApis}` + ); + } + if ( + expectedTransports == null + || expectedTransports < expectedRealtimeApis + || residentRealtimeTransports == null + || residentRealtimeTransports < expectedTransports + ) { + failures.push( + `resident realtime transports=${residentRealtimeTransports ?? 'unknown'}, expected ${expectedTransports ?? expectedRealtimeApis}` + ); + } + } + if (postWarmupPgPoolCapacityEvictions !== 0) { + failures.push( + `post-warmup PostgreSQL pool capacity evictions=${postWarmupPgPoolCapacityEvictions ?? 'unknown'}` + ); + } + if (postWarmupPgPoolCapacityRefusals !== 0) { + failures.push( + `post-warmup PostgreSQL pool capacity refusals=${postWarmupPgPoolCapacityRefusals ?? 'unknown'}` + ); + } + if (postWarmupPgPoolDisposalFailures !== 0) { + failures.push( + `post-warmup PostgreSQL pool disposal failures=${postWarmupPgPoolDisposalFailures ?? 'unknown'}` + ); + } + if (input.gates.requireRetainedMemoryCheckpoints) { + if (retainedGrowth.errors.length > 0) { + failures.push( + `retained-memory checkpoint errors: ${retainedGrowth.errors.join('; ')}` + ); + } + if (retainedGrowth.heapMiBPerHour == null) { + failures.push('retained heap growth could not be measured'); + } else if ( + retainedGrowth.heapMiBPerHour + > input.gates.maxPostWarmupHeapGrowthMiBPerHour + ) { + failures.push( + `retained heap growth ${retainedGrowth.heapMiBPerHour.toFixed(2)}MiB/hour exceeds ${input.gates.maxPostWarmupHeapGrowthMiBPerHour}` + ); + } + if (retainedGrowth.externalMiBPerHour == null) { + failures.push('retained external-memory growth could not be measured'); + } else if ( + retainedGrowth.externalMiBPerHour + > input.gates.maxPostWarmupHeapGrowthMiBPerHour + ) { + failures.push( + `retained external-memory growth ${retainedGrowth.externalMiBPerHour.toFixed(2)}MiB/hour exceeds ${input.gates.maxPostWarmupHeapGrowthMiBPerHour}` + ); + } + } else if (rawHeapGrowth == null) { + failures.push('post-warmup heap growth could not be measured'); + } else if ( + rawHeapGrowth > input.gates.maxPostWarmupHeapGrowthMiBPerHour + ) { + failures.push( + `heap growth ${rawHeapGrowth.toFixed(2)}MiB/hour exceeds ${input.gates.maxPostWarmupHeapGrowthMiBPerHour}` + ); + } + if (input.gates.requireConclusiveCanaries && input.canaries.some((canary) => !canary.conclusive)) failures.push('isolation canary was inconclusive'); + if (input.gates.requireZeroBleed && input.canaries.some((canary) => canary.violation)) failures.push('cross-tenant bleed detected'); + if (input.gates.requireConclusiveOperationOracles) { + if (operationOracleInconclusive > 0) { + failures.push( + `GraphQL operation response oracles inconclusive=${operationOracleInconclusive}` + ); + } + if (operationOracleViolations > 0) { + failures.push( + `GraphQL operation response oracle violations=${operationOracleViolations}` + ); + } + if (missingOperationOracles.length > 0) { + failures.push( + `missing GraphQL operation response oracles: ${shortList(missingOperationOracles)}` + ); + } + } + if (missingCapabilities.length > 0) failures.push(`missing capabilities: ${missingCapabilities.join(', ')}`); + if (missingCanaries.length > 0) failures.push(`missing canaries: ${missingCanaries.join(', ')}`); + if (tenantResults.some((tenant) => !tenant.qualified)) failures.push('one or more complete tenants failed qualification'); + if (input.serverExit) failures.push(`server exited code=${input.serverExit.code} signal=${input.serverExit.signal}`); + const realtimeFailures = validateRealtimeDeliveryCoverage(input); + if (realtimeFailures.length > 0) { + failures.push( + `deadline-bounded recurring realtime delivery coverage is incomplete: ${realtimeFailures.join('; ')}` + ); + } else if ( + input.realtimeDeliveryCoverage + && ( + input.realtimeDeliveryCoverage.deliveryP99Ms > input.gates.maxP99Ms + || input.realtimeDeliveryCoverage.primeResponseP99Ms > input.gates.maxP99Ms + ) + ) { + failures.push( + `realtime prime or delivery p99 exceeds ${input.gates.maxP99Ms}ms` + ); + } + if (input.externalServer) { + failures.push('external server reuse cannot qualify as a fresh-arm run'); + } + failures.push(...input.executionErrors.map((error) => `execution failed: ${error}`)); + + const heapValues = input.memorySnapshots + .map((snapshot) => snapshot.heapUsedBytes) + .filter((value): value is number => value != null); + const peakHeapBytes = heapValues.length === input.memorySnapshots.length && heapValues.length > 0 + ? Math.max(...heapValues) + : null; + const peakRssValues = input.memorySnapshots + .map((snapshot) => snapshot.processPeakRssBytes) + .filter((value): value is number => value != null); + const peakRssBytes = peakRssValues.length > 0 + ? Math.max(...peakRssValues) + : null; + const serviceMemoryUpperBoundPostgresBytes = completePostgresCgroupV2PeakTelemetry + ? postgresCgroupV2PeakBytes + : input.evidenceMode === 'diagnostic' + ? postgresPeakBytes + : null; + const serviceMemoryUpperBoundPostgresSource = completePostgresCgroupV2PeakTelemetry + ? 'cgroup-v2-memory.peak' as const + : input.evidenceMode === 'diagnostic' && postgresPeakBytes != null + ? 'sampled-current-diagnostic' as const + : null; + const serviceMemoryUpperBoundBytes = peakRssBytes != null + && serviceMemoryUpperBoundPostgresBytes != null + ? peakRssBytes + serviceMemoryUpperBoundPostgresBytes + : null; + const buildMaxValues = input.memorySnapshots + .map((snapshot) => snapshot.buildMaxMs) + .filter((value): value is number => value != null); + const buildMaxMs = buildMaxValues.length === input.memorySnapshots.length && buildMaxValues.length > 0 + ? Math.max(...buildMaxValues) + : null; + const elapsedSec = input.workloadDurationMs / 1000; + const heapLimits = [...new Set(input.memorySnapshots + .map((snapshot) => snapshot.heapLimitBytes) + .filter((value): value is number => value != null))]; + const observedHeapLimitBytes = heapLimits.length === 1 ? heapLimits[0] : null; + if (peakHeapBytes == null) failures.push('heap-used telemetry unavailable'); + if (peakRssBytes == null) failures.push('OS process peak RSS telemetry unavailable'); + if (input.evidenceMode === 'qualification' && serviceMemoryUpperBoundBytes == null) { + failures.push('conservative service-memory upper bound unavailable'); + } + if (observedHeapLimitBytes == null) failures.push('effective V8 heap limit telemetry unavailable or inconsistent'); + const globallyQualified = failures.length === 0; + const qualifiedCustomers = globallyQualified + ? tenantResults.filter((tenant) => tenant.qualified).length + : 0; + const dispatchedWorkloadRequests = workloadSamples.filter((sample) => + sample.phase === 'workload' + && sample.errorCode !== 'LOAD_GENERATOR_MISSED_ARRIVAL' + ).length; + const periodicValidationRequests = input.canaries.filter( + (canary) => canary.phase === 'periodic' + ).length; + const customerWorkloadRps = elapsedSec > 0 + ? dispatchedWorkloadRequests / elapsedSec + : 0; + const periodicValidationRps = elapsedSec > 0 + ? periodicValidationRequests / elapsedSec + : 0; + const realtimeValidationRps = elapsedSec > 0 + ? (input.realtimeDeliveryCoverage?.primeRequests ?? 0) / elapsedSec + : 0; + + return { + schemaVersion: 6, + runKind: input.runKind, + evidenceMode: input.evidenceMode, + campaignId: input.campaignId, + scheduleSha256: input.scheduleSha256, + previousResultPayloadSha256: input.previousResultPayloadSha256, + qualificationCohortSha256: input.qualificationCohortSha256, + arm: input.arm, + commit: input.commit ?? null, + introspectionMode: input.introspectionMode, + heapMiB: input.heapMiB, + configuredCustomers: input.tenants.length, + configuredTenants: input.tenants.length, + fleetShape, + repetition: input.repetition, + expectedMatrixRepetitions: input.expectedMatrixRepetitions, + runOrderSeed: input.runOrderSeed, + runOrderIndex: input.runOrderIndex, + startedAt: input.startedAt, + endedAt: input.endedAt, + durationSec: elapsedSec, + warmupMaxMs: percentile(input.warmupLatencies, 1), + resolvedWarmupTimeoutMs: input.resolvedWarmupTimeoutMs, + offeredLoad: input.offeredLoad, + requests: workloadSamples.length, + coverageRequests: coverageSamples.length, + workloadRequests: workloadSamples.length, + errors, + customerWorkloadRps, + periodicValidationRps, + realtimeValidationRps, + combinedHttpRps: + customerWorkloadRps + periodicValidationRps + realtimeValidationRps, + achievedRps: customerWorkloadRps, + missedArrivals: input.missedArrivals, + errorRate, + p50Ms: percentile(latencies, 0.5), + p95Ms: percentile(latencies, 0.95), + p99Ms: percentile(latencies, 0.99), + peakHeapBytes, + peakRssBytes, + observedHeapLimitBytes, + residentInstances, + expectedResidentInstances, + cacheConfiguredMax, + cacheBudgetCapacity, + cacheInstanceHeapBytes, + cacheCalibrationId, + cacheAdmissionMode, + warmObservedHeapDeltaPerInstanceBytes, + postWarmupHeapGrowthMiBPerHour: rawHeapGrowth, + rawPostWarmupHeapGrowthMiBPerHour: rawHeapGrowth, + retainedHeapGrowthMiBPerHour: retainedGrowth.heapMiBPerHour, + retainedExternalGrowthMiBPerHour: retainedGrowth.externalMiBPerHour, + retainedMemoryDurationSec: retainedGrowth.durationSec, + retainedHeapBaselineBytes: retainedGrowth.heapBaselineBytes, + retainedHeapFinalBytes: retainedGrowth.heapFinalBytes, + retainedExternalBaselineBytes: retainedGrowth.externalBaselineBytes, + retainedExternalFinalBytes: retainedGrowth.externalFinalBytes, + retainedMemoryCheckpointErrors: retainedGrowth.errors, + postWarmupEvictions: evictions, + postWarmupBuildRefusals: buildRefusals, + postWarmupBuilds, + pgPoolCacheSize, + pgPoolLeasedPools, + pgPoolActiveLeases, + postWarmupPgPoolCapacityEvictions, + postWarmupPgPoolCapacityRefusals, + postWarmupPgPoolDisposalFailures, + coldBuildMaxMs: buildMaxMs, + memorySampleErrors: input.memorySampleErrors, + postgresBaselineBytes, + postgresWarmBoundaryBytes, + postgresPeakBytes, + postgresWorkingSetPeakBytes, + postgresCgroupV2PeakBytes, + postgresCgroupV2Samples, + postgresOomEvents, + postgresBackendPeak, + residentPhysicalDatabases, + postgresContainerDedicated, + unexpectedPostgresDatabases, + pgPoolTotalClients, + pgPoolIdleClients, + pgPoolWaitingClients, + runtimePoolRequestedMaxUses, + runtimePoolEffectiveMaxUses, + runtimePoolExpectedPools, + runtimePoolObservedPools, + runtimePoolTotalClients, + runtimePoolIdleClients, + runtimePoolWaitingClients, + residentRealtimeManagers, + residentRealtimeTransports, + realtimeNotificationMode, + realtimeDeliveryCoverage: input.realtimeDeliveryCoverage, + notificationBrokers, + notificationListenerConnections, + notificationBrokerLeases, + notificationBrokerTopics, + notificationBrokerSubscribers, + notificationBrokerQueueOverflows, + notificationBrokerFatalFailures, + notificationAuditIdentities, + notificationAuditsHealthy, + notificationAuditsFailed, + notificationAuditsStale, + notificationAuditAttempts, + notificationAuditFailures, + notificationAuditActiveDatabaseTargets, + notificationAuditDatabaseConflicts, + postgresColdBuildSpikeBytes, + postgresSampleErrors: input.postgresSampleErrors, + alignedServicePeakBytes: alignedServicePeak?.bytes ?? null, + alignedServicePeakNodeRssBytes: alignedServicePeak?.nodeRssBytes ?? null, + alignedServicePeakPostgresBytes: alignedServicePeak?.postgresBytes ?? null, + alignedServicePeakTimestamp: alignedServicePeak?.timestamp ?? null, + alignedServiceMemorySamples: alignedServicePeak?.samples ?? 0, + alignedServiceMemoryMaxSkewMs: alignedServicePeak?.maxSkewMs ?? null, + alignedServiceMemoryCoverageRatio: alignedServiceCoverage?.coverageRatio ?? null, + alignedServiceMemoryCoveredDurationMs: alignedServiceCoverage?.coveredDurationMs ?? null, + alignedServiceMemoryExpectedDurationMs: alignedServiceCoverage?.expectedDurationMs ?? null, + alignedServiceMemoryMaxGapMs: alignedServiceCoverage?.maxGapMs ?? null, + serviceMemoryUpperBoundBytes, + serviceMemoryUpperBoundPostgresSource, + capabilitiesExercised, + missingCapabilities, + missingCanaries, + canarySchedule: input.canarySchedule, + canaryChecks: input.canaries.length, + canaryInconclusive: input.canaries.filter((canary) => !canary.conclusive).length, + bleedViolations: input.canaries.filter((canary) => canary.violation).length, + operationOracleChecks, + operationOracleInconclusive, + operationOracleViolations, + missingOperationOracles, + tenants: tenantResults, + qualifiedCustomers, + qualifiedTenants: qualifiedCustomers, + tenantsPerConfiguredOldSpaceGiB: qualifiedCustomers / (input.heapMiB / 1024), + tenantsPerPeakRssGiB: peakRssBytes && peakRssBytes > 0 + ? qualifiedCustomers / (peakRssBytes / GIB) + : null, + customersPerAlignedServiceGiB: alignedServicePeak && alignedServicePeak.bytes > 0 + ? qualifiedCustomers / (alignedServicePeak.bytes / GIB) + : null, + customersPerServiceMemoryUpperBoundGiB: + serviceMemoryUpperBoundBytes && serviceMemoryUpperBoundBytes > 0 + ? qualifiedCustomers / (serviceMemoryUpperBoundBytes / GIB) + : null, + configuredCustomersPerAlignedServiceGiB: + alignedServicePeak && alignedServicePeak.bytes > 0 + ? input.tenants.length / (alignedServicePeak.bytes / GIB) + : null, + configuredCustomersPerServiceMemoryUpperBoundGiB: + serviceMemoryUpperBoundBytes && serviceMemoryUpperBoundBytes > 0 + ? input.tenants.length / (serviceMemoryUpperBoundBytes / GIB) + : null, + accepted: globallyQualified, + failures, + serverExit: input.serverExit, + provenance: input.provenance, + provenanceErrors: input.provenanceErrors, + postgresRunAttestation: input.postgresRunAttestation ?? null, + artifactDir: input.artifactDir + }; +}; + +const median = (values: number[]): number => percentile(values, 0.5); + +export const summarizeCapacityBoundaries = ( + runs: DensityRunResult[] +): DensityCapacityBoundary[] => { + const groups = new Map(); + for (const run of runs) { + if (run.runKind === 'soak') continue; + const key = `${run.arm}\0${run.heapMiB}`; + const group = groups.get(key) ?? []; + group.push(run); + groups.set(key, group); + } + return [...groups.values()].map((group) => { + const { arm, heapMiB } = group[0]; + const expectedRepetitions = Math.max( + ...group.map((run) => run.expectedMatrixRepetitions ?? 1) + ); + const byCount = new Map(); + for (const run of group) { + const local = byCount.get(run.configuredTenants) ?? []; + local.push(run); + byCount.set(run.configuredTenants, local); + } + const testedTenantCounts = [...byCount.keys()].sort((a, b) => a - b); + const completeCounts = testedTenantCounts.filter((count) => { + const countRuns = byCount.get(count)!; + const repetitions = new Set(countRuns.map((run) => run.repetition)); + return countRuns.length === expectedRepetitions + && repetitions.size === expectedRepetitions + && countRuns.every((run) => run.evidenceMode === 'qualification') + && new Set(countRuns.map((run) => run.qualificationCohortSha256)).size === 1 + && countRuns.every((run) => run.expectedMatrixRepetitions === expectedRepetitions) + && Array.from( + { length: expectedRepetitions }, + (_unused, index) => index + 1 + ).every((repetition) => repetitions.has(repetition)); + }); + const incompleteTenantCounts = testedTenantCounts.filter( + (count) => !completeCounts.includes(count) + ); + const passingCounts = completeCounts.filter((count) => + byCount.get(count)!.every((run) => run.accepted) + ); + const highestAllRepetitionsPass = passingCounts.length > 0 + ? Math.max(...passingCounts) + : null; + const failingCounts = completeCounts.filter((count) => + byCount.get(count)!.some((run) => !run.accepted) + ); + const greaterFailures = highestAllRepetitionsPass == null + ? [] + : failingCounts.filter((count) => count > highestAllRepetitionsPass); + const monotonicQualification = highestAllRepetitionsPass == null + ? failingCounts.length === completeCounts.length + : failingCounts.every((count) => count > highestAllRepetitionsPass); + const lowestGreaterFail = greaterFailures.length > 0 + ? Math.min(...greaterFailures) + : null; + const boundaryRuns = highestAllRepetitionsPass == null + ? [] + : byCount.get(highestAllRepetitionsPass)!; + const peakRssDensities = boundaryRuns + .map((run) => run.tenantsPerPeakRssGiB) + .filter((value): value is number => value != null); + const alignedServiceDensities = boundaryRuns + .map((run) => run.customersPerAlignedServiceGiB) + .filter((value): value is number => value != null); + const serviceUpperBoundDensities = boundaryRuns + .map((run) => run.customersPerServiceMemoryUpperBoundGiB) + .filter((value): value is number => value != null); + return { + arm, + heapMiB, + expectedRepetitions, + testedTenantCounts, + incompleteTenantCounts, + highestAllRepetitionsPass, + lowestGreaterFail, + monotonicQualification, + capacityBoundaryReached: + highestAllRepetitionsPass != null + && lowestGreaterFail != null + && monotonicQualification + && incompleteTenantCounts.length === 0, + medianTenantsPerConfiguredOldSpaceGiB: boundaryRuns.length > 0 + ? median(boundaryRuns.map((run) => run.tenantsPerConfiguredOldSpaceGiB)) + : null, + medianTenantsPerPeakRssGiB: + boundaryRuns.length > 0 && peakRssDensities.length === boundaryRuns.length + ? median(peakRssDensities) + : null, + medianCustomersPerAlignedServiceGiB: + boundaryRuns.length > 0 && alignedServiceDensities.length === boundaryRuns.length + ? median(alignedServiceDensities) + : null, + medianCustomersPerServiceMemoryUpperBoundGiB: + boundaryRuns.length > 0 && serviceUpperBoundDensities.length === boundaryRuns.length + ? median(serviceUpperBoundDensities) + : null + }; + }).sort((a, b) => a.arm.localeCompare(b.arm) || a.heapMiB - b.heapMiB); +}; + +const relativeImprovement = (baseline: number, candidate: number): number => + baseline === 0 ? (candidate > 0 ? Infinity : 0) : (candidate - baseline) / baseline; + +const bracketedPassingCountForRepetition = ( + runs: DensityRunResult[], + arm: string, + heapMiB: number, + repetition: number +): number | null => { + const coordinate = runs.filter((run) => + run.runKind !== 'soak' + && run.arm === arm + && run.heapMiB === heapMiB + && run.repetition === repetition + ); + const byCount = new Map(); + for (const run of coordinate) { + if (byCount.has(run.configuredTenants)) return null; + if (run.evidenceMode !== 'qualification') return null; + byCount.set(run.configuredTenants, run); + } + const counts = [...byCount.keys()].sort((left, right) => left - right); + const passing = counts.filter((count) => byCount.get(count)!.accepted); + if (passing.length === 0) return null; + const highestPassing = Math.max(...passing); + if (counts.some((count) => count < highestPassing && !byCount.get(count)!.accepted)) { + return null; + } + const greaterFailures = counts.filter((count) => + count > highestPassing && !byCount.get(count)!.accepted + ); + return greaterFailures.length > 0 ? highestPassing : null; +}; + +export const compareDensity = ( + baseline: DensityRunResult[], + candidate: DensityRunResult[], + gates: AcceptanceGates +): { + materiallyBetter: boolean; + configuredOldSpaceMedianImprovement: number; + peakRssMedianImprovement: number; + alignedServiceMedianImprovement: number; + serviceMemoryUpperBoundMedianImprovement: number; + configuredOldSpaceNonRegression: boolean; + peakRssNonRegression: boolean; + alignedServiceNonRegression: boolean; + serviceMemoryUpperBoundNonRegression: boolean; + everyHeapAddsTenants: boolean; + capacityBoundariesComplete: boolean; + pairedMatrixComplete: boolean; + baselineBoundaries: DensityCapacityBoundary[]; + candidateBoundaries: DensityCapacityBoundary[]; +} => { + const baselineBoundaries = summarizeCapacityBoundaries(baseline); + const candidateBoundaries = summarizeCapacityBoundaries(candidate); + if (baseline.length === 0 || candidate.length === 0) { + return { + materiallyBetter: false, + configuredOldSpaceMedianImprovement: 0, + peakRssMedianImprovement: 0, + alignedServiceMedianImprovement: 0, + serviceMemoryUpperBoundMedianImprovement: 0, + configuredOldSpaceNonRegression: false, + peakRssNonRegression: false, + alignedServiceNonRegression: false, + serviceMemoryUpperBoundNonRegression: false, + everyHeapAddsTenants: false, + capacityBoundariesComplete: false, + pairedMatrixComplete: false, + baselineBoundaries, + candidateBoundaries + }; + } + const matrixKeys = (runs: DensityRunResult[]): string[] => runs + .filter((run) => run.runKind !== 'soak') + .map((run) => [ + run.qualificationCohortSha256, + run.evidenceMode, + run.heapMiB, + run.configuredTenants, + run.repetition + ].join(':')) + .sort(); + const pairedMatrixComplete = JSON.stringify(matrixKeys(baseline)) === JSON.stringify(matrixKeys(candidate)); + const baselineByHeap = new Map(baselineBoundaries.map((boundary) => [ + boundary.heapMiB, + boundary + ])); + const candidateByHeap = new Map(candidateBoundaries.map((boundary) => [ + boundary.heapMiB, + boundary + ])); + const pairedHeaps = [...candidateByHeap.keys()].filter((heap) => baselineByHeap.has(heap)); + const capacityBoundariesComplete = pairedMatrixComplete + && pairedHeaps.length === baselineByHeap.size + && pairedHeaps.length === candidateByHeap.size + && [...baselineBoundaries, ...candidateBoundaries].every( + (boundary) => boundary.capacityBoundaryReached + ); + const densityPairs = pairedHeaps.map((heap) => ({ + baseline: baselineByHeap.get(heap)!, + candidate: candidateByHeap.get(heap)! + })); + const completeDensityPairs = densityPairs.filter(({ baseline: prior, candidate: next }) => + prior.medianTenantsPerConfiguredOldSpaceGiB != null + && next.medianTenantsPerConfiguredOldSpaceGiB != null + && prior.medianTenantsPerPeakRssGiB != null + && next.medianTenantsPerPeakRssGiB != null + && prior.medianCustomersPerAlignedServiceGiB != null + && next.medianCustomersPerAlignedServiceGiB != null + && prior.medianCustomersPerServiceMemoryUpperBoundGiB != null + && next.medianCustomersPerServiceMemoryUpperBoundGiB != null + ); + const configuredImprovements = completeDensityPairs.map(({ baseline: prior, candidate: next }) => + relativeImprovement( + prior.medianTenantsPerConfiguredOldSpaceGiB!, + next.medianTenantsPerConfiguredOldSpaceGiB! + ) + ); + const peakRssImprovements = completeDensityPairs.map(({ baseline: prior, candidate: next }) => + relativeImprovement( + prior.medianTenantsPerPeakRssGiB!, + next.medianTenantsPerPeakRssGiB! + ) + ); + const alignedServiceImprovements = completeDensityPairs.map(({ + baseline: prior, + candidate: next + }) => relativeImprovement( + prior.medianCustomersPerAlignedServiceGiB!, + next.medianCustomersPerAlignedServiceGiB! + )); + const serviceMemoryUpperBoundImprovements = completeDensityPairs.map(({ + baseline: prior, + candidate: next + }) => relativeImprovement( + prior.medianCustomersPerServiceMemoryUpperBoundGiB!, + next.medianCustomersPerServiceMemoryUpperBoundGiB! + )); + const metricsComplete = completeDensityPairs.length === densityPairs.length + && densityPairs.length > 0; + const configuredOldSpaceMedianImprovement = metricsComplete + ? median(configuredImprovements) + : 0; + const peakRssMedianImprovement = metricsComplete ? median(peakRssImprovements) : 0; + const alignedServiceMedianImprovement = metricsComplete + ? median(alignedServiceImprovements) + : 0; + const serviceMemoryUpperBoundMedianImprovement = metricsComplete + ? median(serviceMemoryUpperBoundImprovements) + : 0; + const configuredOldSpaceNonRegression = metricsComplete + && configuredImprovements.every((improvement) => improvement >= 0); + const peakRssNonRegression = metricsComplete + && peakRssImprovements.every((improvement) => improvement >= 0); + const alignedServiceNonRegression = metricsComplete + && alignedServiceImprovements.every((improvement) => improvement >= 0); + const serviceMemoryUpperBoundNonRegression = metricsComplete + && serviceMemoryUpperBoundImprovements.every((improvement) => improvement >= 0); + const everyHeapAddsTenants = capacityBoundariesComplete + && densityPairs.every(({ baseline: prior, candidate: next }) => { + if (prior.expectedRepetitions !== next.expectedRepetitions) return false; + return Array.from( + { length: prior.expectedRepetitions }, + (_unused, index) => index + 1 + ).every((repetition) => { + const priorCapacity = bracketedPassingCountForRepetition( + baseline, + prior.arm, + prior.heapMiB, + repetition + ); + const nextCapacity = bracketedPassingCountForRepetition( + candidate, + next.arm, + next.heapMiB, + repetition + ); + return priorCapacity != null + && nextCapacity != null + && nextCapacity >= priorCapacity + gates.minAdditionalTenantsEveryRun; + }); + }); + return { + materiallyBetter: everyHeapAddsTenants + && alignedServiceNonRegression + && serviceMemoryUpperBoundNonRegression + && alignedServiceMedianImprovement >= gates.minMedianDensityImprovement + && serviceMemoryUpperBoundMedianImprovement >= gates.minMedianDensityImprovement, + configuredOldSpaceMedianImprovement, + peakRssMedianImprovement, + alignedServiceMedianImprovement, + serviceMemoryUpperBoundMedianImprovement, + configuredOldSpaceNonRegression, + peakRssNonRegression, + alignedServiceNonRegression, + serviceMemoryUpperBoundNonRegression, + everyHeapAddsTenants, + capacityBoundariesComplete, + pairedMatrixComplete, + baselineBoundaries, + candidateBoundaries + }; +}; diff --git a/packages/perf-harness/src/types.ts b/packages/perf-harness/src/types.ts new file mode 100644 index 0000000000..8b9a2fdb60 --- /dev/null +++ b/packages/perf-harness/src/types.ts @@ -0,0 +1,962 @@ +export type IntrospectionMode = 'stock' | 'scoped-required'; +export type CacheAdmissionMode = 'evict-idle' | 'preserve-resident'; +export type NodeV8Profile = + | 'stock' + | 'optimize-for-size' + | 'baseline-optimize-for-size' + | 'jitless-optimize-for-size'; + +export interface GraphqlResponseOracle { + /** Every match must be present in the response for the operation to count. */ + requiredMatches: JsonPathMatch[]; + /** Any match is an isolation violation, even when required matches are present. */ + forbiddenMatches: JsonPathMatch[]; + /** Exhaustive assertions over every value selected by a wildcard-capable pointer. */ + invariants?: JsonPathInvariant[]; +} + +export interface GraphqlPostCoverageVerification extends GraphqlResponseOracle { + query: string; + variables?: Record; + /** GraphQL variable name -> JSON pointer in the primary operation response. */ + variablesFromResponse?: Record; +} + +export interface GraphqlOperation { + name: string; + capability: string; + weight?: number; + query: string; + variables?: Record; + /** Optional fail-closed response oracle, evaluated for every invocation. */ + requiredMatches?: JsonPathMatch[]; + /** Must be configured together with requiredMatches. */ + forbiddenMatches?: JsonPathMatch[]; + /** Cardinality-bounded assertions evaluated for every invocation. */ + invariants?: JsonPathInvariant[]; + /** + * An untimed verification query run after this operation during coverage. + * This is for mutations whose production payload cannot echo a database- + * stamped value; it is never injected into the production GraphQL API. + */ + postCoverageVerification?: GraphqlPostCoverageVerification; +} + +export type JsonValue = + | null + | boolean + | number + | string + | JsonValue[] + | { [key: string]: JsonValue }; + +/** RFC 6901 JSON pointer; `*` may select every array/object child. */ +export interface JsonPathMatch { + path: string; + value: JsonValue; +} + +/** + * Every selected value must equal `everyEquals`, and the selection cardinality + * must stay within the inclusive bounds. A positive `min` makes empty + * collection responses fail closed instead of vacuously passing. + */ +export interface JsonPathInvariant { + path: string; + everyEquals: JsonValue; + min: number; + max?: number; +} + +export interface IsolationCanary { + name: string; + query: string; + variables?: Record; + forbiddenMatches: JsonPathMatch[]; + requiredMatches: JsonPathMatch[]; + invariants?: JsonPathInvariant[]; +} + +export interface RealtimeGraphqlOperation { + query: string; + variables?: Record; + /** Every match must be present for the response/event to be conclusive. */ + requiredMatches: JsonPathMatch[]; + /** Any match is an isolation failure. */ + forbiddenMatches: JsonPathMatch[]; +} + +export type RealtimeSubscriptionOperation = RealtimeGraphqlOperation; + +export interface RealtimeRoundTripCorrelation { + /** Top-level GraphQL variable replaced with a fresh opaque nonce per round. */ + primeVariable: string; + /** Exact JSON pointer where the prime mutation must return that nonce. */ + primeResponsePath: string; + /** Exact JSON pointer where the subscription event must return that nonce. */ + subscriptionEventPath: string; +} + +export interface RealtimeProbe { + subscription: RealtimeSubscriptionOperation; + prime: RealtimeGraphqlOperation; + /** Proves every delivery round was caused by its own fresh mutation. */ + correlation: RealtimeRoundTripCorrelation; + /** + * Header name -> environment variable name. This lets the driver authenticate + * both the HTTP prime and WebSocket upgrade without serializing credentials + * into a fleet or artifact. + */ + headersFromEnvironment?: Record; +} + +export interface GraphqlSurface { + name: string; + /** Default opaque identity when every arm is expected to produce the same build. */ + buildContract: string; + /** + * Exact opaque identity by arm name. When present, every configured arm must + * have an entry and the default is never used for that arm. + */ + buildContracts?: Record; + url: string; + headers?: Record; + warmup: GraphqlOperation; + operations: GraphqlOperation[]; + canaries: IsolationCanary[]; + /** Driver-owned subscription used to prove this exact surface stays live. */ + realtime?: RealtimeProbe; +} + +export interface CustomerApiTopology { + /** Stable control-plane API identity, never a host/service routing label. */ + id: string; + /** Opaque credential-sensitive runtime pool identity. */ + runtimePoolIdentity: string; + /** Exact opaque pool identity by arm when arms use different credentials. */ + runtimePoolIdentities?: Record; + /** Ordered physical schemas compiled into this exact API build. */ + physicalSchemas: string[]; + /** Host/service labels are reported for routing coverage, never isolation. */ + routingLabels: string[]; + /** Whether qualification must keep a realtime transport resident for this API. */ + realtime: boolean; + /** Names from TenantTarget.surfaces served by this API. */ + surfaces: string[]; +} + +export interface CustomerDatabaseTopology { + /** Stable logical database identity used by the Graphile build contract. */ + id: string; + /** Credential-free physical database label for fleet-shape accounting. */ + physicalDatabase: string; + apis: CustomerApiTopology[]; +} + +export interface TenantTarget { + id: string; + /** + * Explicit customer -> logical database -> API mapping. Legacy diagnostic + * fleets may omit it, but a qualifying plan can require it fail-closed. + */ + databases?: CustomerDatabaseTopology[]; + surfaces: GraphqlSurface[]; +} + +export interface FleetV1 { + version: 1; + tenants: TenantTarget[]; + /** Populated by loadFleet; not part of the fleet JSON contract. */ + sourceSha256?: string; +} + +export interface CustomerFleetShape { + topologyComplete: boolean; + customers: number; + logicalDatabases: number; + physicalDatabases: number; + apis: number; + realtimeApis: number; + surfaces: number; + physicalSchemaBindings: number; + routingLabels: number; + uniqueBuildContracts: number; + uniqueRuntimePoolIdentities: number; +} + +export interface ArmPlan { + name: 'origin-main' | 'runtime-boundary-stock' | 'cache-governor-stock' | 'scoped-introspection' | string; + commit?: string; + cwd?: string; + command?: string[]; + port: number; + readinessUrl: string; + memoryUrl: string; + /** + * Authenticated loopback endpoint that performs a benchmark-only full-GC + * checkpoint. Spawned qualification arms must expose this explicitly. + */ + retainedHeapCheckpointUrl?: string; + /** + * Optional authenticated loopback hook invoked after every configured + * surface has warmed, but before post-warmup memory accounting starts. + * Physical-density fixtures use this to assert server-side realtime residency. + */ + postWarmupUrl?: string; + /** Dedicated PostgreSQL container used for cold-build memory telemetry. */ + postgresContainer?: string; + /** + * Outside-process live database/ACL audit bound to one fresh container and + * full matrix coordinate. An optional prepare command must create that run's + * container/clone after the harness establishes its not-before boundary. + */ + postgresRunAttestation?: { + command: string[]; + prepareCommand: string[]; + timeoutMs?: number; + }; + /** Fail the run unless raw cgroup-v2 PostgreSQL memory telemetry is present. */ + requirePostgresCgroupV2?: boolean; + introspectionMode: IntrospectionMode; + /** Closed set of benchmarked V8 flag combinations; defaults to stock. */ + v8Profile?: NodeV8Profile; + env?: Record; + /** + * Heap-specific environment overrides. This is intentionally explicit in + * the plan so governor calibration cannot silently reuse one reserve across + * materially different V8 pressure points. + */ + envByHeapMiB?: Record>; + startupTimeoutMs?: number; + /** Optional pin for the built JavaScript entry executed by command. */ + entrySha256?: string; + /** Optional pin for the workspace pnpm-lock.yaml. */ + lockfileSha256?: string; +} + +export interface AcceptanceGates { + maxErrorRate: number; + maxP99Ms: number; + maxPostWarmupHeapGrowthMiBPerHour: number; + minMedianDensityImprovement: number; + minAdditionalTenantsEveryRun: number; + /** Maximum uncovered boundary or internal gap in aligned service-memory telemetry. */ + maxAlignedMemorySampleGapMs?: number; + /** Minimum fraction of the post-warm workload covered by aligned samples. */ + minAlignedMemoryCoverageRatio?: number; + requireZeroBleed: boolean; + requireNoPostWarmupEvictions: boolean; + requireNoPostWarmupBuildRefusals: boolean; + requireNoPostWarmupBuilds: boolean; + requirePostgresMemoryTelemetry: boolean; + /** Require a unique, fresh, live-audited PostgreSQL epoch for every run. */ + requireFreshPostgresRunAttestation: boolean; + /** Require authenticated forced-GC bookends for retained-memory gating. */ + requireRetainedMemoryCheckpoints: boolean; + /** Require physical database, backend, and concrete pg.Pool client counts. */ + requirePhysicalDatabaseTelemetry: boolean; + requireConclusiveCanaries: boolean; + /** + * Require exact initial/final sweeps plus complete, deadline-bounded + * periodic coverage of every configured canary. + */ + requireCompletePeriodicCanaryCoverage: boolean; + /** Require exact response or post-coverage evidence for every operation. */ + requireConclusiveOperationOracles: boolean; + requireExplicitCustomerTopology: boolean; + /** Require the live cache and pinned process environment to use this mode. */ + requiredCacheAdmissionMode: CacheAdmissionMode | null; +} + +export interface DensityQualificationPlan { + /** Arm used as the denominator for every configured candidate comparison. */ + baselineArm: string; + /** Mandatory curve checkpoints; extra configured checkpoints are permitted. */ + requiredHeapMiB: number[]; + /** Every configured matrix point must contain at least this many repetitions. */ + minimumRepetitions: number; + /** + * One real induced-hostile report for every exact arm runtime/configuration. + * Passive GraphQL identity probes do not satisfy this publication boundary. + */ + hostileValidationEvidence?: Record; +} + +export interface ExactHostileValidationBinding { + version: 1; + kind: 'exact-runtime-hostile-validation-v1'; + artifactFile: string; + /** SHA-256 over the exact artifact bytes, without a prefix. */ + artifactSha256: string; + runtimeArtifactFingerprint: string; + configurationFingerprint: string; +} + +export type PeriodicCanarySchedule = 'full-sweep' | 'rotating-one'; + +export interface WorkloadPlan { + durationSec: number; + /** Fixed process-wide offered load. Mutually exclusive with rpsPerTenant. */ + rps?: number; + /** Offered load multiplied by the number of tenants in this run. */ + rpsPerTenant?: number; + /** Every surface must receive at least this many workload-phase requests. */ + minWorkloadRequestsPerSurface: number; + requestTimeoutMs: number; + maxInFlight: number; + canaryIntervalSec: number; + /** Defaults to the legacy full-fleet/full-canary periodic sweep. */ + periodicCanarySchedule?: PeriodicCanarySchedule; + /** Parallelism across surfaces; probes within one surface stay sequential. */ + canaryConcurrency?: number; + /** Minimum whole-fleet warmup allowance. */ + warmupTimeoutMs: number; + /** Additional scaling budget, applied once per warmup-concurrency wave. */ + warmupTimeoutPerSurfaceMs: number; + /** Maximum simultaneous schema warmups; defaults to one. */ + warmupConcurrency?: number; +} + +export interface DensityPlanV1 { + version: 1; + fleetFile: string; + artifactDir: string; + arms: ArmPlan[]; + heapMiB: number[]; + /** Legacy count ramp used for every heap unless a heap-specific ramp exists. */ + tenantCounts?: number[]; + /** Heap-specific ramps, keyed by configured old-space MiB. */ + tenantCountsByHeapMiB?: Record; + repetitions: number; + /** Reproducible arm interleaving seed. */ + runOrderSeed?: string; + requiredCapabilities: string[]; + requiredCanaries: string[]; + workload: WorkloadPlan; + gates: AcceptanceGates; + /** Omit for diagnostic-only plans that cannot make a qualification claim. */ + qualification?: DensityQualificationPlan; + soak?: { + enabled: boolean; + /** Candidate arm to soak; defaults to scoped-introspection for compatibility. */ + arm?: string; + durationSec: number; + tenantCount: number; + heapMiB: number; + }; + /** Populated by loadPlan; not part of the plan JSON contract. */ + sourceSha256?: string; +} + +export interface RequestSample { + tenantId: string; + surface: string; + operation: string; + capability: string; + latencyMs: number; + status: number; + ok: boolean; + phase: 'coverage' | 'workload'; + scheduledAtMs?: number; + errorCode?: string; + /** True when this request was checked against a configured response oracle. */ + oracleConfigured?: boolean; + /** True only when every required match was observed. */ + oracleConclusive?: boolean; + /** True when at least one forbidden match was observed. */ + oracleViolation?: boolean; + /** Transport/HTTP/GraphQL failure prevented evidence evaluation. */ + oracleUnavailable?: boolean; + /** Whether an untimed post-coverage side-effect verification was used. */ + postCoverageVerification?: boolean; +} + +export interface CanaryResult { + tenantId: string; + surface: string; + canary: string; + phase: 'initial' | 'periodic' | 'final'; + /** One-based planned round number, present only for periodic probes. */ + periodicRound?: number; + scheduledAt: string; + startedAt: string; + completedAt: string; + latencyMs: number; + conclusive: boolean; + violation: boolean; + detail?: string; +} + +export interface CanaryRoundSummary { + /** One-based planned round number. */ + periodicRound: number; + plannedAt: string; + startedAt: string | null; + completedAt: string | null; + targetsPlanned: number; + targetsStarted: number; + targetsCompleted: number; + checksPlanned: number; + checksStarted: number; + checksCompleted: number; + /** The preceding serialized round was still running at this round's slot. */ + overlapped: boolean; + /** This round completed after the workload deadline. */ + deadlineLate: boolean; + startDelayMs: number | null; + durationMs: number | null; +} + +export interface CanaryScheduleSummary { + schedule: PeriodicCanarySchedule; + intervalMs: number; + durationMs: number; + canaryConcurrency: number; + startedAt: string; + deadlineAt: string; + /** Round counts. */ + planned: number; + started: number; + completed: number; + missed: number; + overlapped: number; + deadlineLate: number; + /** Probe counts across periodic rounds only. */ + checksPlanned: number; + checksStarted: number; + checksCompleted: number; + rounds: CanaryRoundSummary[]; +} + +export interface MemorySnapshot { + timestamp: string; + pid: number | null; + nodeEnv: string | null; + heapLimitBytes: number | null; + heapUsedBytes: number | null; + rssBytes: number | null; + processPeakRssBytes: number | null; + cacheSize: number | null; + cacheConfiguredMax?: number | null; + cacheBudgetCapacity?: number | null; + cacheInstanceHeapBytes?: number | null; + cacheCalibrationId?: string | null; + cacheAdmissionMode?: CacheAdmissionMode | null; + /** Stable credential-free evidence for cross-process fleet comparison. */ + residentBuildContractFingerprints?: string[] | null; + /** Process-local keyed identities used only for same-process accounting. */ + residentBuildContracts: string[] | null; + evictions: number | null; + buildRefusals: number | null; + buildsStarted: number | null; + buildsSucceeded: number | null; + buildMaxMs: number | null; + pgPoolCacheSize: number | null; + pgPoolLeasedPools: number | null; + pgPoolActiveLeases: number | null; + pgPoolCapacityEvictions: number | null; + pgPoolCapacityRefusals: number | null; + pgPoolDisposalFailures: number | null; + pgPoolTotalClients?: number | null; + pgPoolIdleClients?: number | null; + pgPoolWaitingClients?: number | null; + runtimePoolTelemetryScope?: 'runtime-only-exact-identities' | null; + runtimePoolTelemetryAvailable?: boolean | null; + runtimePoolRequestedMaxUses?: number | null; + runtimePoolEffectiveMaxUses?: number | null; + runtimePoolEffectiveMaxUsesKnown?: boolean | null; + runtimePoolMaxUsesExact?: boolean | null; + runtimePoolExpectedPools?: number | null; + runtimePoolObservedPools?: number | null; + runtimePoolTotalClients?: number | null; + runtimePoolIdleClients?: number | null; + runtimePoolWaitingClients?: number | null; + postgresBackendTotal?: number | null; + postgresBackendActive?: number | null; + postgresBackendIdle?: number | null; + postgresBackendIdleInTransaction?: number | null; + physicalDatabases?: number | null; + postgresContainerDedicated?: boolean | null; + unexpectedPostgresDatabases?: number | null; + realtimeManagersExpected?: number | null; + realtimeManagersActive?: number | null; + realtimeTransportsExpected?: number | null; + realtimeTransportsActive?: number | null; + realtimeNotificationMode?: 'dedicated' | 'shared-exact' | null; + notificationBrokers?: number | null; + notificationListenerConnections?: number | null; + notificationBrokerLeases?: number | null; + notificationBrokerTopics?: number | null; + notificationBrokerSubscribers?: number | null; + notificationBrokerQueueOverflows?: number | null; + notificationBrokerFatalFailures?: number | null; + notificationAuditIdentities?: number | null; + notificationAuditsHealthy?: number | null; + notificationAuditsFailed?: number | null; + notificationAuditsStale?: number | null; + notificationAuditAttempts?: number | null; + notificationAuditFailures?: number | null; + notificationAuditActiveDatabaseTargets?: number | null; + notificationAuditDatabaseConflicts?: number | null; + cacheCountersAvailable: boolean; + buildCountersAvailable: boolean; + raw?: unknown; +} + +/** + * High-frequency, harness-timestamped current RSS sample for the exact server + * PID. Linux reads /proc; other hosts use the authenticated memory endpoint. + */ +export interface NodeRssSnapshot { + timestamp: string; + /** Exact child PID read by the harness. */ + pid: number; + source: 'proc' | 'authenticated-endpoint'; + rssBytes: number; +} + +export interface RetainedMemorySample { + timestamp: string; + /** Monotonic process time serialized as decimal nanoseconds. */ + monotonicNs: string; + heapUsedBytes: number; + externalBytes: number; + arrayBuffersBytes: number; + rssBytes: number; +} + +export interface RetainedMemoryGuard { + pid: number; + graphileInFlight: number; + residentBuildContracts: string[]; + stateSha256: string; + /** Credential-free residency and monotonic process-counter state. */ + state: Record; +} + +export interface RetainedMemoryCheckpoint { + version: 1; + fixture: string; + pid: number; + gcRounds: number; + stableSampleCount: number; + stable: boolean; + samples: RetainedMemorySample[]; + guardBefore: RetainedMemoryGuard; + guardAfter: RetainedMemoryGuard; + errors: string[]; +} + +export interface RetainedMemoryCheckpointPair { + baseline: RetainedMemoryCheckpoint | null; + final: RetainedMemoryCheckpoint | null; + errors: string[]; +} + +export interface PostgresMemorySnapshot { + timestamp: string; + /** Immutable 64-character Docker ID sampled for this record. */ + containerId?: string; + /** Attested container cgroup identity revalidated by the sampler. */ + cgroupIdentitySha256?: string; + /** Raw cgroup-v2 charge when available; Docker working set otherwise. */ + usedBytes: number; + limitBytes: number; + source?: 'cgroup-v2' | 'docker-stats'; + workingSetBytes?: number; + sampleStartedAt?: string; + sampleEndedAt?: string; + sampleDurationMs?: number; + cgroupV2?: { + currentBytes: number; + peakBytes: number | null; + maxBytes: number | null; + stat: Record; + events: Record; + }; + raw: string; +} + +export interface SurfaceResult { + surface: string; + warmed: boolean; + workloadRequests: number; + successfulWorkloadRequests: number; + errors: number; + errorRate: number; + p99Ms: number; + operationsConfigured: number; + operationsExercised: number; + canaryChecks: number; + canaryInconclusive: number; + bleedViolations: number; + operationOracleChecks: number; + operationOracleInconclusive: number; + operationOracleViolations: number; + missingOperations: string[]; + missingCapabilities: string[]; + missingOperationOracles: string[]; + qualified: boolean; +} + +export interface TenantResult { + tenantId: string; + surfacesConfigured: number; + surfacesWarmed: number; + surfacesWithTraffic: number; + operationsConfigured: number; + operationsExercised: number; + requests: number; + errors: number; + errorRate: number; + p99Ms: number; + canaryChecks: number; + canaryInconclusive: number; + bleedViolations: number; + operationOracleChecks: number; + operationOracleInconclusive: number; + operationOracleViolations: number; + missingSurfaces: string[]; + missingOperations: string[]; + missingCapabilities: string[]; + missingOperationOracles: string[]; + surfaces: SurfaceResult[]; + qualified: boolean; +} + +export interface ResolvedOfferedLoad { + mode: 'fixed-total' | 'per-tenant'; + configuredRps: number; + tenantCount: number; + totalRps: number; + rpsPerTenant: number; +} + +export interface RealtimeDeliverySurfaceCoverage { + tenantId: string; + surface: string; + route: string; + expectedRecurringRounds: number; + startedRecurringRounds: number; + verifiedRecurringRounds: number; + issuedCorrelationSha256: string; + verifiedCorrelationSha256: string; + primeRequests: number; + primeResponseP99Ms: number; + deliveryP99Ms: number; +} + +export interface RealtimeDeliveryCoverage { + version: 2; + deliveryIntervalMs: number; + workloadStartedAt: string; + workloadDeadlineAt: string; + workloadEndedAt: string | null; + expectedRecurringRounds: number; + startedRecurringRounds: number; + verifiedRecurringRounds: number; + deadlineLateRecurringRounds: number; + primeRequests: number; + primeResponseP99Ms: number; + deliveryP99Ms: number; + complete: boolean; + surfaces: RealtimeDeliverySurfaceCoverage[]; +} + +/** Credential-free proof that one fresh prime nonce reached one subscription. */ +export interface RealtimeCorrelationReceipt { + sequence: number; + timed: boolean; + deadlineAt: string; + issuedAt: string; + issuedSha256: string; + primeResponseAt: string | null; + primeResponseSha256: string | null; + eventAt: string | null; + eventSha256: string | null; +} + +export interface DensityResultEvidenceBinding { + version: 2; + algorithm: 'sha256'; + resultPayloadSha256: string; + artifacts: Array<{ + name: string; + sha256: string; + }>; +} + +export interface ResolvedMemoryPolicy { + configuredMaxOldSpaceMiB: number; + expectedV8HeapLimitBytes: number | null; + graphileCacheMax: string | null; + graphileCacheInstanceHeapBytes: string | null; + graphileCacheServerReserveBytes: string | null; + graphileCacheBuildReserveBytes: string | null; + graphileCacheRssLimitBytes: string | null; + graphileCacheRssBuildReserveBytes: string | null; + graphileCacheCalibrationId: string | null; + graphileCacheAdmissionMode: string | null; + graphileBuildMaxConcurrency: string | null; +} + +export interface ArmProvenance { + cwd: string | null; + command: string[]; + gitHead: string | null; + worktreeDirty: boolean | null; + gitStatusSha256: string | null; + lockfilePath: string | null; + lockfileSha256: string | null; + entryPath: string | null; + entrySha256: string | null; + serverPid: number | null; + /** Named allowlisted profile from the plan. */ + v8Profile: NodeV8Profile; + /** Exact sanitized NODE_OPTIONS string installed for the child. */ + nodeOptions: string | null; + /** Exact tokenization of nodeOptions. */ + nodeOptionsArgv: string[]; + /** Exact direct Node flags before the executed entry file. */ + nodeExecArgv: string[]; + /** NODE_OPTIONS followed by direct flags, in effective precedence order. */ + effectiveNodeRuntimeFlags: string[]; + planSha256: string | null; + fleetSha256: string | null; + node: string; + v8: string; + platform: NodeJS.Platform; + architecture: string; + runOrderSeed: string | null; + runOrderIndex: number | null; + memoryPolicy: ResolvedMemoryPolicy | null; +} + +export interface PostgresRunAttestationEvidence { + version: 1; + kind: 'physical-density-measurement-attestation-v1'; + artifactPath: string; + artifactSha256: string; + payloadSha256: string; + epochId: string; + arm: string; + heapMiB: number; + tenantCount: number; + repetition: number; + runOrderIndex: number; + planSha256: string; + fleetSha256: string; + containerId: string; + containerStartedAt: string; + cgroupIdentitySha256: string; + containerConfigurationSha256: string; + postgresSystemIdentifier: string; + postgresStartedAt: string; + cloneId: string; + cloneAttestationSetSha256: string; + cloneNonceSetSha256: string; + liveContractSetSha256: string; + manifestSha256: string; + containerTemplateSha256: string; + canonicalDatabaseContractFingerprint: string; + freshContainerForRun: boolean; + cgroupV2Verified: boolean; + liveCustomerContractsAudited: number; + /** Full pre-run audit intentionally warms PostgreSQL's catalogs. */ + catalogCacheState: 'warmed-by-live-contract-audit'; +} + +export interface DensityRunResult { + schemaVersion: 6; + runKind: 'matrix' | 'soak'; + /** Only a full, unmodified configured matrix may carry qualification evidence. */ + evidenceMode: 'qualification' | 'diagnostic'; + /** Random per-invocation campaign identity; never derived from plan bytes. */ + campaignId: string; + /** Hash of the exact ordered schedule manifest for this invocation. */ + scheduleSha256: string; + /** Hash-chain pointer to the prior result payload in exact run order. */ + previousResultPayloadSha256: string | null; + /** SHA-256 over the exact plan and fleet byte identities. */ + qualificationCohortSha256: string; + arm: string; + commit: string | null; + introspectionMode: IntrospectionMode; + heapMiB: number; + /** Complete customer bundles selected from the explicit fleet manifest. */ + configuredCustomers: number; + /** @deprecated Compatibility alias for configuredCustomers. */ + configuredTenants: number; + fleetShape: CustomerFleetShape; + repetition: number; + expectedMatrixRepetitions: number; + runOrderSeed: string; + runOrderIndex: number; + startedAt: string; + endedAt: string; + durationSec: number; + warmupMaxMs: number; + resolvedWarmupTimeoutMs: number; + offeredLoad: ResolvedOfferedLoad; + requests: number; + coverageRequests: number; + workloadRequests: number; + errors: number; + /** Dispatched customer workload requests divided by measured load duration. */ + customerWorkloadRps: number; + /** Completed periodic isolation probes divided by measured load duration. */ + periodicValidationRps: number; + /** Timed realtime prime mutations divided by measured load duration. */ + realtimeValidationRps: number; + /** Customer workload plus periodic and realtime validation HTTP requests per second. */ + combinedHttpRps: number; + /** @deprecated Compatibility alias for customerWorkloadRps. */ + achievedRps: number; + missedArrivals: number; + errorRate: number; + p50Ms: number; + p95Ms: number; + p99Ms: number; + peakHeapBytes: number | null; + peakRssBytes: number | null; + observedHeapLimitBytes: number | null; + residentInstances: number | null; + expectedResidentInstances: number; + cacheConfiguredMax: number | null; + cacheBudgetCapacity: number | null; + cacheInstanceHeapBytes: number | null; + cacheCalibrationId: string | null; + cacheAdmissionMode: CacheAdmissionMode | null; + warmObservedHeapDeltaPerInstanceBytes: number | null; + /** Raw heapUsed OLS trend; diagnostic only because normal GC is sawtoothed. */ + postWarmupHeapGrowthMiBPerHour: number | null; + rawPostWarmupHeapGrowthMiBPerHour: number | null; + retainedHeapGrowthMiBPerHour: number | null; + retainedExternalGrowthMiBPerHour: number | null; + retainedMemoryDurationSec: number | null; + retainedHeapBaselineBytes: number | null; + retainedHeapFinalBytes: number | null; + retainedExternalBaselineBytes: number | null; + retainedExternalFinalBytes: number | null; + retainedMemoryCheckpointErrors: string[]; + postWarmupEvictions: number | null; + postWarmupBuildRefusals: number | null; + postWarmupBuilds: number | null; + pgPoolCacheSize: number | null; + pgPoolLeasedPools: number | null; + pgPoolActiveLeases: number | null; + postWarmupPgPoolCapacityEvictions: number | null; + postWarmupPgPoolCapacityRefusals: number | null; + postWarmupPgPoolDisposalFailures: number | null; + coldBuildMaxMs: number | null; + memorySampleErrors: string[]; + postgresBaselineBytes: number | null; + postgresWarmBoundaryBytes: number | null; + postgresPeakBytes: number | null; + postgresWorkingSetPeakBytes: number | null; + postgresCgroupV2PeakBytes: number | null; + postgresCgroupV2Samples: number; + postgresOomEvents: number | null; + postgresBackendPeak: number | null; + residentPhysicalDatabases: number | null; + postgresContainerDedicated: boolean | null; + unexpectedPostgresDatabases: number | null; + pgPoolTotalClients: number | null; + pgPoolIdleClients: number | null; + pgPoolWaitingClients: number | null; + runtimePoolRequestedMaxUses: number | null; + runtimePoolEffectiveMaxUses: number | null; + runtimePoolExpectedPools: number | null; + runtimePoolObservedPools: number | null; + runtimePoolTotalClients: number | null; + runtimePoolIdleClients: number | null; + runtimePoolWaitingClients: number | null; + residentRealtimeManagers: number | null; + residentRealtimeTransports: number | null; + realtimeNotificationMode: 'dedicated' | 'shared-exact' | null; + /** Deadline-bounded fresh-event coverage during the timed workload. */ + realtimeDeliveryCoverage?: RealtimeDeliveryCoverage | null; + notificationBrokers: number | null; + notificationListenerConnections: number | null; + notificationBrokerLeases: number | null; + notificationBrokerTopics: number | null; + notificationBrokerSubscribers: number | null; + notificationBrokerQueueOverflows: number | null; + notificationBrokerFatalFailures: number | null; + notificationAuditIdentities: number | null; + notificationAuditsHealthy: number | null; + notificationAuditsFailed: number | null; + notificationAuditsStale: number | null; + notificationAuditAttempts: number | null; + notificationAuditFailures: number | null; + notificationAuditActiveDatabaseTargets: number | null; + notificationAuditDatabaseConflicts: number | null; + postgresColdBuildSpikeBytes: number | null; + postgresSampleErrors: string[]; + /** Maximum near-simultaneous Node current RSS + PostgreSQL cgroup usage. */ + alignedServicePeakBytes: number | null; + alignedServicePeakNodeRssBytes: number | null; + alignedServicePeakPostgresBytes: number | null; + alignedServicePeakTimestamp: string | null; + alignedServiceMemorySamples: number; + alignedServiceMemoryMaxSkewMs: number | null; + alignedServiceMemoryCoverageRatio?: number | null; + alignedServiceMemoryCoveredDurationMs?: number | null; + alignedServiceMemoryExpectedDurationMs?: number | null; + alignedServiceMemoryMaxGapMs?: number | null; + /** Conservative non-simultaneous upper bound: Node RSS HWM + PostgreSQL peak. */ + serviceMemoryUpperBoundBytes: number | null; + serviceMemoryUpperBoundPostgresSource?: + | 'cgroup-v2-memory.peak' + | 'sampled-current-diagnostic' + | null; + capabilitiesExercised: string[]; + missingCapabilities: string[]; + missingCanaries: string[]; + canarySchedule: CanaryScheduleSummary | null; + canaryChecks: number; + canaryInconclusive: number; + bleedViolations: number; + operationOracleChecks: number; + operationOracleInconclusive: number; + operationOracleViolations: number; + missingOperationOracles: string[]; + tenants: TenantResult[]; + qualifiedCustomers: number; + /** @deprecated Compatibility alias for qualifiedCustomers. */ + qualifiedTenants: number; + tenantsPerConfiguredOldSpaceGiB: number; + tenantsPerPeakRssGiB: number | null; + customersPerAlignedServiceGiB: number | null; + customersPerServiceMemoryUpperBoundGiB: number | null; + /** Diagnostic only: configured customers divided by aligned service memory. */ + configuredCustomersPerAlignedServiceGiB: number | null; + /** Diagnostic only: configured customers divided by the service upper bound. */ + configuredCustomersPerServiceMemoryUpperBoundGiB: number | null; + accepted: boolean; + failures: string[]; + serverExit: { code: number | null; signal: NodeJS.Signals | null } | null; + provenance: ArmProvenance | null; + provenanceErrors: string[]; + postgresRunAttestation?: PostgresRunAttestationEvidence | null; + /** Hash binding over the complete result payload and persisted raw evidence. */ + evidenceBinding?: DensityResultEvidenceBinding; + artifactDir: string; +} + +export interface DensityCapacityBoundary { + arm: string; + heapMiB: number; + expectedRepetitions: number; + testedTenantCounts: number[]; + incompleteTenantCounts: number[]; + highestAllRepetitionsPass: number | null; + lowestGreaterFail: number | null; + /** False when a lower tenant count failed but a higher count passed. */ + monotonicQualification: boolean; + capacityBoundaryReached: boolean; + medianTenantsPerConfiguredOldSpaceGiB: number | null; + medianTenantsPerPeakRssGiB: number | null; + medianCustomersPerAlignedServiceGiB: number | null; + medianCustomersPerServiceMemoryUpperBoundGiB: number | null; +} diff --git a/packages/perf-harness/tsconfig.esm.json b/packages/perf-harness/tsconfig.esm.json new file mode 100644 index 0000000000..6bff62dc07 --- /dev/null +++ b/packages/perf-harness/tsconfig.esm.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "ESNext", + "moduleResolution": "bundler", + "outDir": "dist/esm" + } +} diff --git a/packages/perf-harness/tsconfig.json b/packages/perf-harness/tsconfig.json new file mode 100644 index 0000000000..9c8a7d7c10 --- /dev/null +++ b/packages/perf-harness/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ecb8038fa5..5f9a4a727a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2660,6 +2660,52 @@ importers: version: 0.3.0 publishDirectory: dist + packages/perf-harness: + dependencies: + grafast: + specifier: 1.0.2 + version: 1.0.2(graphql@16.13.0) + graphile-build-pg: + specifier: 5.0.2 + version: 5.0.2(patch_hash=869428d66480f2dba792fb7055538a827edeacf0e34631da8f16b648005cb624)(@dataplan/pg@1.0.3(patch_hash=1580ee5de9b0792b9892233f0616ea4e2801b7f387a3fb73468de1e4dfaa76cb)(@dataplan/json@1.0.0(grafast@1.0.2(graphql@16.13.0)))(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.0.2(graphql@16.13.0))(graphile-build@5.0.2(patch_hash=f53ea77ecb2de5d2a48441d8f9ce57f161bffba9a6f6e22d0dc853dccea5fce6)(grafast@1.0.2(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0))(graphile-config@1.0.1)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + graphile-cache: + specifier: workspace:^ + version: link:../../graphile/graphile-cache/dist + graphile-settings: + specifier: workspace:^ + version: link:../../graphile/graphile-settings/dist + graphql: + specifier: 16.13.0 + version: 16.13.0 + graphql-ws: + specifier: ^6.0.8 + version: 6.0.8(graphql@16.13.0)(ws@8.20.1) + pg: + specifier: ^8.21.0 + version: 8.21.0 + pg-env: + specifier: workspace:^ + version: link:../../postgres/pg-env/dist + ws: + specifier: ^8.20.0 + version: 8.20.1 + devDependencies: + '@types/node': + specifier: ^22.19.11 + version: 22.19.19 + '@types/pg': + specifier: ^8.20.0 + version: 8.20.0 + '@types/ws': + specifier: ^8.18.1 + version: 8.18.1 + makage: + specifier: ^0.3.0 + version: 0.3.0 + ts-node: + specifier: ^10.9.2 + version: 10.9.2(@types/node@22.19.19)(typescript@5.9.3) + packages/postmaster: dependencies: 12factor-env: @@ -23852,6 +23898,24 @@ snapshots: v8-compile-cache-lib: 3.0.1 yn: 3.1.1 + ts-node@10.9.2(@types/node@22.19.19)(typescript@5.9.3): + dependencies: + '@cspotcode/source-map-support': 0.8.1 + '@tsconfig/node10': 1.0.12 + '@tsconfig/node12': 1.0.11 + '@tsconfig/node14': 1.0.3 + '@tsconfig/node16': 1.0.4 + '@types/node': 22.19.19 + acorn: 8.15.0 + acorn-walk: 8.3.4 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + typescript: 5.9.3 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + ts-node@10.9.2(@types/node@25.9.1)(typescript@5.9.3): dependencies: '@cspotcode/source-map-support': 0.8.1 diff --git a/research/graphile-density/IMPLEMENTATION-NOTES.md b/research/graphile-density/IMPLEMENTATION-NOTES.md new file mode 100644 index 0000000000..90962d3387 --- /dev/null +++ b/research/graphile-density/IMPLEMENTATION-NOTES.md @@ -0,0 +1,44 @@ +# Implementation notes and unknowns ledger + +## Map retained + +- Optimize complete warm tenants per GiB, not isolated schema build size. +- Keep one PostGraphile instance per exact physical tenant/API build contract. +- Treat RLS as row-level defense in depth; do not use it to justify SQL schema rewriting. +- Keep work local, use a fresh `origin/main` worktree, and never modify `constructive-db`. + +## Territory-driven deviations + +- #1332's SQL-text substitution was replaced by a parameterized query generator and explicit Graphile service option. +- #1333/#1334 were not ported. Dedicated physical-schema instances make BM25 and other `regclass`-like values ordinary physical names and remove the rewrite coverage problem. +- #1335's blueprint-oriented result model was replaced. A run counts tenants only after every declared build contract is resident, every surface is warm, all required operations and canaries ran, and the 15-minute telemetry gates pass. +- Reverse inheritance closure was removed after a partition fixture showed it would traverse from a shared parent into unrelated tenant child schemas. +- The checked-in fleet is an intentionally failing template because real tokens, credentials, APIs, S3, realtime events, and hostile probes do not exist safely inside this repository alone. + +## Unknowns ledger + +| Type | Item | Resolution/status | +|---|---|---| +| Known known | SQL rewrite routes objects before RLS evaluates rows. | Rewrite pooling rejected. | +| Known known | `SET LOCAL` restores the prior session value at transaction end. | Checkout uses `DISCARD ALL`; each request seeds the full security-GUC set including empty claims, role, and read-only state. | +| Known known | Graphile/node-postgres retain client-side prepared-statement bookkeeping. | Both installed adaptor structures are cleared after `DISCARD ALL`; sanitation failure destroys the client. | +| Known known | Actual scoped retained heap and cold-build cost at 62,298 `pg_class`. | Clean three-repetition single-surface median: 6.55 MiB retained heap, 47.77 MiB final RSS delta, and 130.34 ms cold build. Stock was 449.42 MiB, 1,106.94 MiB, and 3,495.95 ms. Complete-customer density remains unmeasured. | +| Known unknown | Full least-privilege grants and RLS/FORCE-RLS behavior for deployed tenant schemas. | The disposable A/B/C production-shaped fixture passed all 56 hostile checks, but it cannot prove every real tenant table/policy. A production-schema policy manifest remains a blocker. | +| Unknown found | Parent→child inheritance closure can include unrelated tenant partitions. | Removed; small byte-equivalence regression passed. | +| Unknown found | Storage and public-key plugins had null-context/fallback paths. | Constructive now preloads an immutable exact-build storage snapshot, the generic fallback is exact/request-scoped/fail-closed, and public-key plans require request settings and the native Graphile transaction client. Remaining deliberate system/build lanes are recorded in `PLUGIN-SQL-AUDIT.md`. | +| Unknown found | Schema-level checks did not catch a runtime/request role that owns an individual table and therefore bypasses RLS. | Top-of-stack safety query now rejects relation, sequence, view, function, and type ownership in exposed schemas. | +| Unknown found | Cache instance samples are supported but not automatically measured by the server. | Operators must feed a benchmarked `GRAPHILE_CACHE_INSTANCE_HEAP_BYTES`; automatic self-tuning is not claimed safe. | +| Unknown found | Docker memory telemetry cannot attribute a sub-second spike to one PostgreSQL backend. | Harness records a coarse dedicated-container delta and labels it accordingly. | +| Unknown found | Capability labels cannot prove semantic coverage by themselves. | The completed fleet and hostile query definitions require human review; the placeholder fails validation rather than producing a score. | +| Unknown found | Legacy Node module resolution in `@constructive-io/graphql-query` loses the PostGraphile adaptor augmentation after the scoped service wrapper is emitted. | The package now uses the same NodeNext/bundler split as the newer Graphile query package; CJS, ESM, tests, and the monorepo build pass without a service-type cast. | +| Unknown found | Route and security settings are fresh but are read by independent statements with no common revision. | Exact identities prevent arbitrary A/B pool aliasing, but handover/revocation can leave one stale HTTP request and an accepted WebSocket can retain stale authorization indefinitely. An atomic `TenantSecurityContractV1` plus socket/generation retirement is a production blocker. | +| Unknown found | Graphile caller plugins are unrestricted in-process code; a fingerprint is not a sandbox or code signature. | Production now rejects non-empty caller presets/plugins by default. Any explicit trust opt-in requires pinned code review and full requalification; built-ins and the runtime credential resolver remain TCB. | +| Unknown found | CAPTCHA admission trusted a client-controlled operation label and did not cover every transport/body shape. | Admission now classifies root mutation fields from the selected AST, parses supported HTTP bodies first, rejects ambiguous/unclassifiable inputs, blocks protected WebSocket mutations, and fails closed on a missing production/strict secret. The protected-field allowlist and Google hostname/action/timeout policy still need release ownership. | + +## Conservative continuation policy + +Do not enable `scoped-required` in production until upstream review, the atomic +security-contract gap, production-schema policy proof, and the full fixture +matrix pass. Do not configure the governor directly from the 6.55 MiB +single-surface result. Any failed or missing telemetry, semantic capability, +surface canary, resident instance, or paired matrix point remains a failed run. diff --git a/research/graphile-density/ORIGINAL-STACK.md b/research/graphile-density/ORIGINAL-STACK.md new file mode 100644 index 0000000000..88d7e41cac --- /dev/null +++ b/research/graphile-density/ORIGINAL-STACK.md @@ -0,0 +1,16 @@ +# Original PR stack provenance + +Fetched on 2026-07-31 after fetching `origin/main`. The local worktree base is `a10ea246fcc45b025713024e131fafb908171149`. + +| PR | GitHub head | Logical base branch | Disposition | +|---|---|---|---| +| #1330 | `6786cc1f77bc5a6b9412b0d5ba4cbb3031de76d1` | `main` | Reimplemented selectively. Its four-commit PR history contains the plugin fixes, quoting migration, lockfile normalization, and a final description-only cleanup; comparing only its head against `cdf155e8` hides the earlier useful commits. | +| #1331 | `161c067c6616a5e121e6757b8921ff28e743179d` | `feat/scale-s1-plugin-fixes` | Hardened and separated from runtime identity work. | +| #1332 | `b8d0c20a20c8a5363e22d8f4285a73d86202b2b5` | `feat/scale-s2-cache-hardening` | SQL substitution implementation rejected and replaced. | +| #1333 | `269b23c2ff82b0b6295cb1822f5affddfbc511cb` | `feat/scale-s3-introspection-filter` | Rejected as a production design. | +| #1334 | `b3d0021b80896baaafaee04c16bd08da35ef2d26` | `feat/scale-s4-pooling-core` | Rejected as a production design. | +| #1335 | `de35e09d67702b6ce3b4ed77e743e449fdc5ea2f` | `feat/scale-s5-pooling-integration` | Harness concepts refreshed; old results retained only as historical claims. | + +The old stack is not one simple linear range: #1330 ends one commit after `cdf155e8`, while #1331 also has `cdf155e8` as its parent and #1332–#1335 form the linear five-commit chain beginning at #1331. Review therefore uses two range-diffs: #1330's full PR range against local branch 1, and #1331–#1335 against local branches 2–5. Lockfile normalization is reviewed independently rather than replayed. + +No branch or PR was pushed or modified. diff --git a/research/graphile-density/PLUGIN-SQL-AUDIT.md b/research/graphile-density/PLUGIN-SQL-AUDIT.md new file mode 100644 index 0000000000..c52614471e --- /dev/null +++ b/research/graphile-density/PLUGIN-SQL-AUDIT.md @@ -0,0 +1,38 @@ +# Graphile plugin SQL and request-scope audit + +This audit covers product packages under `graphile/*/src`, excluding test helpers. Its security model is the refreshed one: each tenant/API build uses its physical schemas and its own exact pool/build identity, so there is no SQL schema rewrite to make plugin SQL “poolable.” Identifiers still need correct quoting, values need binds, and runtime SQL needs the complete request settings. + +Graphile plugins are unrestricted Node.js code, not declarative SQL fragments. +They can use the configured runtime service for raw SQL, open their own +connections, or access process I/O. Production therefore rejects every non-empty +caller `extends`/`preset` by default with +`GRAPHILE_CALLER_PRESET_NOT_TRUSTED`. An explicit +`trustCallerPresetsInProduction: true` opt-in admits that code into the trusted +computing base; the exact build fingerprint separates its cache identity but +does not sandbox it, sign it, or attest mutable closure state. + +| Package/path family | SQL path | Result | +|---|---|---| +| `graphile-i18n` | Raw runtime translation query | Fixed: schema/table/type/column identifiers use `@pgsql/quotes`; values are bound; execution uses request `pgSettings`; locale type state is per build. | +| `graphile-llm` agent discovery | Raw control-metadata query and cache | Fixed: database ID is required, query-filtered, and part of the cache key. Missing identity fails closed. | +| `graphile-llm` RAG/metering | Raw chunk search and usage SQL | Fixed/retained: chunk relations are schema-qualified with quoted identifiers; vector and limits are values; runtime calls carry request `pgSettings`. | +| `graphile-search` BM25 | `pg-sql2` expressions with index-name bind values | Fixed: `to_bm25query` receives the physical schema-qualified index name. BM25 remains enabled because no tenant-schema rewrite occurs. | +| `graphile-search` tsvector/trigram/vector | `pg-sql2` expressions | Acceptable by construction: catalog-derived relation/column/type names use `sql.identifier`; request terms, vectors, limits, and thresholds use values. Must still run in the capability fixture. | +| `graphile-ltree`, `graphile-postgis`, `graphile-connection-filter`, `graphile-pg-aggregates` | Generated `pg-sql2` fragments | Acceptable by construction: dynamic identifiers use `sql.identifier`, while user inputs use `sql.value`. Must still run against quoted physical schemas. | +| `graphile-bulk-mutations` | Generated mutation plans and relation fragments | No tenant routing substitution exists; identifiers come from the introspected build. Full insert/update/upsert/delete behavior remains an integration gate. | +| `graphile-history` | Raw SELECT/UPDATE/INSERT SQL | Request-scoped and parameterized; schema/table/columns are quoted from the current build's codec/tag metadata. Integration coverage remains required. | +| `graphile-function-bindings` | QueryBuilder insert into configured invocation relation | Runtime inserts are request-scoped; schema/table and columns originate in the database-scoped compute-module configuration and values are bound. The optional generic gather fallback has one build-lane `withPgClientFromPgService(pgService, null, ...)` call against the exact build service, but Constructive supplies an authoritative control-plane binding snapshot and does not enter that branch. Integration coverage remains required. | +| `graphile-bucket-provisioner-plugin` | Raw storage metadata and bucket SQL | Metadata and bucket authorization run with request settings and are database-ID filtered; table identifiers are qualified with `@pgsql/quotes`. One request-triggered system-lane call records `physical_name` after the request-scoped lookup and S3 provision: the exact qualified table, authorized bucket UUID, physical pool, and `physical_name IS NULL` guard bound the write, while the baseline role supplies the deliberate bookkeeping privilege. S3 side effects and those grants remain an explicit integration gate. | +| `graphile-presigned-url-plugin` | Raw metadata, bucket, and file SQL | Fixed: Constructive supplies an immutable exact-build control-plane snapshot, including an authoritative empty list when storage is absent. The generic package fallback now carries the exact request `pgSettings`; missing context/settings, database identity, module metadata, bucket visibility, and lookup errors all fail before signing, and a metadata failure cannot select process-global S3 configuration. Global S3 values may fill nullable fields only after a tenant module and persisted physical bucket coordinate have resolved. One deliberate system-lane call remains to record a newly provisioned bucket's `physical_name`, with the same exact-table/UUID/null-guard constraints as the provisioner plugin. Hostile A/B/C storage and S3-side-effect tests remain a production gate. | +| `graphile-settings/PublicKeySignature` | Three raw auth-function paths | Fixed: every plan reads Grafast `pgSettings`, copies the complete request GUC map, explicitly overrides only `role` to `anonymous`, and fails closed if either request settings or `withPgClient` is absent. Queries now use the native Graphile client API inside the transaction established by `withPgClient`, so there is no nested manual transaction or null-context checkout. Identifier validation and qualification remain in place; real auth-function/RLS integration is still required. | +| `graphile-meta` | Build-time Graphile metadata collection | Current `main` already replaced the old module-global table array with schema/build-local state during extraction to `graphile-meta`; no additional port was needed. | + +The post-fix null-context inventory is three deliberate product-source lanes: + +- `graphile-presigned-url-plugin` performs one request-triggered system write that records an already-authorized bucket's persisted physical coordinate. +- `graphile-bucket-provisioner-plugin` uses the same system write pattern for explicit and automatic provisioning. +- `graphile-function-bindings` has one schema-gather fallback against the exact build's PostgreSQL service; Constructive's preloaded control-plane snapshot bypasses it. + +No request metadata, file/bucket authorization, signing, public-key auth, history, search, i18n, LLM/RAG, realtime visibility, or function-invocation path uses a null context. Test utilities use null or synthetic settings by design and are outside the product-source inventory. + +Prepared-statement sanitation was checked against the installed `@dataplan/pg` adaptor: it stores its LRU at `connection._graphilePreparedStatementCache` and node-postgres stores names at `connection.parsedStatements`, which are the two client-side structures cleared after `DISCARD ALL`. The performance cost of discarding prepared plans on every checkout remains a benchmark question, not a claimed free safety measure. diff --git a/research/graphile-density/RANGE-DIFF.md b/research/graphile-density/RANGE-DIFF.md new file mode 100644 index 0000000000..9cc0c09d73 --- /dev/null +++ b/research/graphile-density/RANGE-DIFF.md @@ -0,0 +1,45 @@ +# Old PR stack to local research stack + +The old commits and current-main implementations are different enough that `git range-diff` correctly reports delete/add pairs rather than pretending they are textual rebases. The semantic mapping below comes from that result plus per-file review. + +## Commands and raw correspondence + +```text +git range-diff \ + ed31ed2aa63fcd2d42acec6f27e672dd300a3959..6786cc1f77bc5a6b9412b0d5ba4cbb3031de76d1 \ + a10ea246fcc45b025713024e131fafb908171149..7ef0502666d71d33bd7275bd27588698a205f3ee + +1: ad54f1fbd < -: --------- tenant-isolation/correctness fixes +2: c96a87c84 < -: --------- whole-lockfile normalization +3: cdf155e84 < -: --------- @pgsql/quotes conversion +4: 6786cc1f7 < -: --------- description punctuation cleanup +-: --------- > 1: 7ef050266 current-main plugin scope/quoting implementation +``` + +```text +git range-diff \ + cdf155e84370153b6f5db9a5e7061efd8b9c0329..de35e09d67702b6ce3b4ed77e743e449fdc5ea2f \ + 7ef0502666d71d33bd7275bd27588698a205f3ee..848bfe2e7 + +1: 161c067c6 < -: --------- old cache/governor +2: b8d0c20a2 < -: --------- old introspection text filter +3: 269b23c2f < -: --------- blueprint rewrite core +4: b3d0021b8 < -: --------- blueprint pooling integration +5: de35e09d6 < -: --------- old cperf/scale validation +-: --------- > 1: de92be5a9 runtime boundary +-: --------- > 2: 5fd90ee2b hardened cache governor +-: --------- > 3: f41e480d5 parameterized scoped introspection +-: --------- > 4: 848bfe2e7 refreshed cperf and final audit hardening +``` + +## Semantic disposition + +| Old work | Local replacement | Per-file conclusion | +|---|---|---| +| #1330 | `7ef050266` | Reapplied quoting and tenant/build scoping in the current i18n, LLM/RAG, search/BM25, and cache APIs. The current `graphile-meta` WeakMap/build boundary supersedes its old global-cache edits. The lockfile rewrite and punctuation-only cleanup were intentionally omitted. | +| #1331 | `de92be5a9` + `5fd90ee2b` | Split identity/security from memory policy. Pool/build identities, runtime credentials, GUC initialization, and checkout reset live below the hardened governor rather than being implicit cache-key behavior. | +| #1332 | `f41e480d5` plus the top-branch closure regression | Replaced query-string substitution with `{ text, values }`, an explicit service mode, required-schema assertions, and dependency closure. A partition fixture then removed unsafe parent-to-unrelated-child expansion. | +| #1333/#1334 | none | Intentionally absent. No SQL rewrite seam or blueprint-sharing flag exists in the local stack. Dedicated instances compile against their actual physical schemas. | +| #1335 | `848bfe2e7` | Rebuilt around complete tenants and all surfaces, four fresh-process arms, hostile canaries, mandatory telemetry, paired density scoring, and immutable artifacts. Old blueprint figures remain labeled historical. | + +This is a reconstruction on `origin/main`, not a claim that old patches replayed cleanly. Review should compare the behavior and trust boundaries above, then use the focused files listed in `PLUGIN-SQL-AUDIT.md`, `UPSTREAM-REVIEW.md`, and `REPORT.md`. diff --git a/research/graphile-density/REPORT.md b/research/graphile-density/REPORT.md new file mode 100644 index 0000000000..2ba795208a --- /dev/null +++ b/research/graphile-density/REPORT.md @@ -0,0 +1,163 @@ +# Graphile tenant-density research spike + +The useful non-rewrite work has been rebuilt on current `origin/main` and the +memory result is now measured: on a clean three-repetition 62,298-`pg_class` +fixture, scoped dependency introspection reduced median retained heap from +449.42 MiB to 6.55 MiB and cold build time from 3,495.95 ms to 130.34 ms. The +latest disposable PostgreSQL 18 A/B/C hostile run passed all 56 checks with zero +cross-tenant tokens. The default remains stock introspection and blueprint/SQL +rewrite pooling is absent. + +This is still not production-qualified. A final audit found a real +route/security revision race: exact identities prevent arbitrary cross-tenant +pool aliasing, but a domain handover or revocation can leave an in-flight HTTP +request on the old authorization snapshot, and an accepted WebSocket can retain +it indefinitely. `SECURITY-AUDIT.md` records the required atomic versioned +contract and the remaining release gates. + +## Local stack + +| Branch | Commit at report time | Outcome | +|---|---|---| +| `research/graphile-density-01-plugin-scope` | `7ef0502666d71d33bd7275bd27588698a205f3ee` | Reapplied i18n, LLM/RAG, BM25, quoting, and cache-scope fixes to current plugin architecture; current `graphile-meta` already has build-local state. | +| `research/graphile-density-02-runtime-boundary` | `de92be5a9f459079f75e94f23d35abcb92365d16` | Added exact pool/build identities, optional least-privilege runtime credentials, complete request-GUC initialization, checkout sanitation, and runtime-role safety checks. | +| `research/graphile-density-03-cache-governor` | `5fd90ee2b7aa5c52984cb53269ab4e1ec16422f0` | Hardened disposal, draining, build coalescing/admission, heap budgeting, stable 503 refusal codes, timers, and debug counters. | +| `research/graphile-density-04-scoped-introspection` | `f41e480d55dfee99a68567dc12145b40f5356bd7` | Replaced SQL text substitution with a bind-parameter query API and fail-closed Graphile service mode; stock remains the default. | +| `research/graphile-density-05-cperf` | this report's branch | Refreshed the harness around complete tenants, physical build contracts, hostile per-surface canaries, fresh processes, four arms, immutable run directories, and fail-closed scoring. The audit also extends runtime safety from schema ownership to relation/sequence/view/function/type ownership. | +| `research/graphile-density-06-measured-optimization` | current local branch | Measured the production-shaped catalog, released build-only state, hardened routing/credentials/plugins/storage/auth/GUC boundaries, and ran the complete A/B/C hostile gate. | + +All branches and artifacts are local. No push, PR edit, deployment, or `constructive-db` change was made. + +## Original PR disposition + +| PR | Disposition | Reason | +|---|---|---| +| #1330 | Reimplemented | The SQL quoting and tenant-scoped plugin fixes remain valid. The large old lockfile normalization was not replayed; current package manifests only add dependencies actually used. | +| #1331 | Hardened and split | Cache pressure work remains valuable, but pool/build identity, credentials, GUCs, and checkout sanitation were made an earlier trust boundary rather than mixed into eviction policy. | +| #1332 | Replaced | The old implementation substituted text inside a generated catalog query and could silently fall back. The candidate preserves `makeIntrospectionQuery()`, passes names as values, computes dependency closure, asserts requested schemas, and has no scoped-mode fallback. | +| #1333/#1334 | Rejected for production | Rewriting qualified SQL makes tenant routing depend on exhaustive interception of generated plans, plugins, raw SQL, prepared statements, metadata, functions, sequences, and extension-specific bind values. RLS does not prove that routing layer correct. | +| #1335 | Refreshed with corrected provenance | Process isolation, open-loop load, canaries, and artifacts were retained. Blueprint-specific baselines and success claims are historical and are not accepted as evidence for the dedicated-instance candidate. | + +The exact remote heads and unusual #1330/#1331 ancestry are recorded in `ORIGINAL-STACK.md`. + +## Decisions on the raised concerns + +Zhi's isolation concern is valid, so the aggressive rewrite design is out. Every resident instance compiles against ordered physical schemas, and host/service names are routing labels rather than cache identities. RLS remains valuable for row filtering, but metadata, schema objects, functions, sequences, owner/BYPASS roles, and privileged paths have their own hostile gates. + +The owner check now covers objects as well as schemas: a runtime login or reachable request role that owns a relation, sequence, view, function, or type in an exposed schema is rejected. Admission also rejects `SECURITY DEFINER`, owner-rights views, foreign/materialized views, unsafe stored-expression dependencies, unexpected inherited/`SET ROLE` paths, and privileges on unapproved objects. The real PostgreSQL integration suite created these unsafe grants and proved that they are rejected. + +Dan's API-separation concern is addressed by isolating `makeSchemaScopedIntrospectionQuery(schemas): { text, values }` from Constructive policy. Graphile selects it through a service option; Constructive only supplies the explicit `stock|scoped-required` setting. It is not presented as a general Graphile plugin because the narrow upstream seam belongs beside `makeIntrospectionQuery()` and the gather layer. + +BM25 stays enabled. Its query builder passes the physical schema-qualified index name, so there is no rewrite exclusion or canonical index alias. `SET LOCAL` does restore the prior session state, which is why request initialization alone is insufficient: a reused checkout runs `DISCARD ALL`, clears node-postgres and `@dataplan/pg` prepared bookkeeping, and destroys the connection if reset fails. The hostile fixture proved this on the same backend for A, B, and C. + +The plugin audit closed the earlier storage and `PublicKeySignature` request-context gaps. Constructive now supplies one immutable exact-build storage snapshot; the generic presigned fallback is database/API filtered, request-scoped, ambiguity-failing, and non-sliding. Public-key plans require request `pgSettings` and use the native Graphile transaction client. Arbitrary plugins remain unsandboxed Node code, so production rejects all caller presets by default; an explicit trust opt-in makes that code part of the process and database trusted computing base. See `PLUGIN-SQL-AUDIT.md`. + +The remaining authorization gap is control-plane atomicity rather than SQL rewrite. Route, RLS, auth, feature, CORS, public-key, and WebAuthn fields are read through separate statements with no shared revision. A handover can allow one stale HTTP request to the former tenant, while a WebSocket can retain a stale route/session until the generation is retired. Exact pool/build identities keep the data path on that captured tenant and prevent an A/B mixture, but production needs a revisioned, single-snapshot contract and long-lived transport revalidation. + +## Evidence produced + +The clean production-shaped catalog run used 62,298 `pg_class` rows, 346,369 +attributes, 8,496 procedures, 23,709 types, and 4,036 namespaces. Across three +fresh-process repetitions, stock introspection retained a median 471,256,024 +bytes (449.42 MiB) after forced GC, ended 1,160,708,096 bytes (1,106.94 MiB) +above the RSS baseline, and reached first-build readiness in 3,495.95 ms. Scoped +dependency introspection retained 6,869,560 bytes (6.55 MiB), ended 50,085,888 +bytes (47.77 MiB) above the RSS baseline, and built in 130.34 ms. That is 68.60× +less retained heap, 23.17× less final RSS, and a 26.82× faster cold build. + +The clean arms had identical source/lockfile/entry provenance within their +comparison, identical catalog fingerprints, and zero recorded errors, +mismatches, or cross-tenant tokens. The broader comparison produced an identical +17,976-byte GraphQL SDL with SHA-256 +`5fb82f96153815b23820a9ccf10322a20c864e49605ef5781cd33422b3b31020`. +These are performance-only results for one complete Graphile surface, not a +complete-customer density qualification. + +The small disposable PostgreSQL fixtures also proved stock-query byte +stability, bind-only schema names, fail-closed missing schemas, cross-schema +type/FK closure, safe partition direction, and byte-identical Constructive SDL. +The latest full-capability A/B/C fixture then passed all 56 hostile checks on a +fresh PostgreSQL 18 database with three distinct least-privilege logins and +realtime-resident instances. It recorded same-backend prepared reset for every +tenant, serialized cold builds, ten alternating connection rounds, and zero +cross-tenant tokens. The disposable container was stopped and removed; existing +local PostgreSQL containers were not modified. + +The new cperf package compiles and its current unit/in-process integration suite passes. It rejects the checked-in placeholder with a precise list of the missing 47 tenants, capabilities, and per-surface canaries, so a five-second smoke or a mislabeled `__typename` query cannot become a qualifying result. Each real run records request samples, canary results, Node/cache telemetry, coarse PostgreSQL container telemetry, server logs, and an immutable scored result. + +## Validation performed + +The final top-of-stack root build completed all 119 participating workspace +packages (of 120 total). The GraphQL server's CJS/ESM build passed and its full +suite passed 399 tests with two skips. +GraphQL environment passed 42 tests, request context passed 41, pg-cache passed +119 with five environment-gated skips, cperf passed 233, and the focused +Graphile settings security/capability suites passed 22. Separate live PostgreSQL +sanitizer and runtime-role suites passed two checks each. + +The complete `graphile-llm` suite additionally passed 53 tests and failed 11 +live-provider cases because no Ollama endpoint was available; the changed +discovery/RAG SQL tests pass, but the unavailable model capability remains an +external integration gate rather than being silently skipped. The full Graphile +settings suite has one credential-gated cross-database BM25 case that cannot use +the passwordless local default; the focused changed suites and the independent +A/B/C BM25 fixture pass. + +The final admission pass also found and fixed a CAPTCHA bypass: admission now +classifies the selected mutation from its AST rather than trusting a +client-controlled operation label, rejects malformed/ambiguous/batched requests, +parses all supported HTTP body formats before admission, rejects protected +WebSocket mutations, and fails closed on a missing production/strict secret. +The focused HTTP/WebSocket suite passed all 36 tests. Caller Graphile presets are +now denied by default in production; the focused composition/contract suites +passed all 21 tests. + +The root lint command is not a usable acceptance gate on current `origin/main` +because ESLint 9 cannot find a flat `eslint.config.*`; this predates the spike +and was not papered over locally. + +The lockfile contains only the changed `pg-introspection` patch hash and the new cperf workspace importer with its TypeScript toolchain snapshot. No unrelated lockfile normalization from the old PR stack was replayed. + +## Historical results are not current evidence + +#1335 documents approximately 14.7 MiB retained heap per instance, 417 ms cold builds, a 17 MiB PostgreSQL spike, an 87× reduction, and high same-blueprint tenant density. Those numbers were collected with the rejected SQL-rewrite/blueprint-pooling system and are not reproduced here. Its five-hour soak also recorded one inconclusive isolation canary; the refreshed gate requires zero inconclusive checks, so that run would not qualify under this spike's rules. + +The safe dedicated-instance candidate now beats the old retained-heap and cold +build targets on the production-shaped single-surface fixture: 6.55 MiB and +130.34 ms. That does not reproduce the old blueprint density claim because the +accepted design intentionally keeps tenant/API instances separate. Complete +customers per GiB still requires the multi-surface ramp and soak. + +## Gates still open + +- Introduce an atomic `TenantSecurityContractV1` revision spanning route, + exposure, role, feature, and auth policy. Carry it through runtime/build and + WebSocket contracts, reject mismatches, retire resident generations, and close + stale subscriptions. The present multi-statement reads allow bounded stale + HTTP access and potentially unbounded stale WebSocket authorization after a + handover/revocation. +- Prove the intended RLS/FORCE-RLS policy manifest, request roles, runtime login, + and dependency-schema object allowlist against disposable copies of the real + production tenant schemas. Add an authoritative per-API `authRequired` + contract so missing auth metadata cannot silently become anonymous access. +- Run the four fresh-process arms at 1/2/4 GiB, every ramp point, three repetitions, 15 minutes each, then the two-hour maximum-density churn soak. A candidate passes only if every heap/repetition ramp adds at least one complete tenant and median maximum density improves at least 15%. +- Obtain Graphile maintainer review of the source-level introspection API and dependency closure; remove the temporary package dist patches before production. +- Re-run throughput and p99 on the final security code; authoritative metadata + and role admission add real PostgreSQL work per request and may not be removed + to improve the benchmark. Complete multipart storage byte roundtrips and the + intended provider gates. +- Feed the governor a conservative validated instance-cost value from the final + complete-customer runs. The 6.55 MiB single-surface result is not automatically + a safe production capacity setting. + +## Reviewer checklist + +1. Can any routing label, hostname, or service key affect cache isolation? It should not; only the exact build contract hash can. +2. What happens when scoped introspection misses a configured schema? The build fails; it never retries stock. +3. Does RLS make a wrong physical schema safe? No; the design avoids rewrite routing and tests non-row objects separately. +4. Can a resident tenant count after an eviction, rebuild, build refusal, missing telemetry, or inconclusive canary? No. +5. Are the old 87× and 14.7 MiB figures current evidence? No. The current clean + result is 68.60× retained-heap improvement and 6.55 MiB for one dedicated + production-shaped surface; complete-customer density is still pending. +6. Can a committed handover/revocation invalidate every in-flight HTTP and + WebSocket operation? Not yet; this is the production-blocking revision gap. diff --git a/research/graphile-density/SECURITY-AUDIT.md b/research/graphile-density/SECURITY-AUDIT.md new file mode 100644 index 0000000000..2470ac8847 --- /dev/null +++ b/research/graphile-density/SECURITY-AUDIT.md @@ -0,0 +1,154 @@ +# Graphile tenant-density security audit + +Audit date: 2026-08-02. Current branch: +`research/graphile-density-06-measured-optimization`. + +## Verdict + +Zhi's concern is correct for the old blueprint-pooling design, so that design is +not part of this candidate. The executable candidate does not rewrite runtime +SQL and does not share one PostGraphile instance between tenants. It builds one +instance against each exact physical database, login, API, ordered schema set, +role set, plugin/settings contract, and surface configuration. Scoped +introspection reduces catalog input during schema construction; it is not a +routing or authorization mechanism. + +The current data plane passed the latest disposable PostgreSQL 18 A/B/C hostile +gate: 56 of 56 checks passed, including generated/plugin SQL, metadata, +functions, sequences, owner/BYPASS rejection, poisoned GUCs, rollback, +same-backend prepared-statement reuse, schema drift, cache invalidation, +serialized builds, realtime-resident instances, and ten alternating connection +reuse rounds with zero cross-tenant tokens. That is strong evidence for the +exact physical boundary, but it is not a production approval. + +Production remains blocked on an atomic, versioned route-and-security contract. +Today a request resolves the route and then reads RLS/auth/features through +separate statements. Exact database/API identities prevent this from becoming +an arbitrary A-to-B pool/cache alias, but a domain handover or revocation can +leave one in-flight HTTP request using the old tenant snapshot. An accepted +WebSocket can retain the old route/session indefinitely because operations and +subscriptions are not checked against a control-plane revision. This is a real +stale-authorization defect and must be fixed before production. + +## Claim-by-claim disposition + +| Concern | Finding | Current protection and remaining limit | +|---|---|---| +| SQL is rewritten | True only of rejected #1333/#1334; false in the current runtime. | The current path gives the exact runtime pool and physical schema names directly to `makePgService`. There is no canonical-schema substitution, rewrite pool, or blueprint pool in runtime source. | +| RLS cannot prove SQL routing | Correct. | The candidate does not use RLS to justify routing. Dedicated logins/pools, schema/object ACLs, exact build identities, and runtime role audits enforce the physical boundary. RLS remains row-level defense in depth. | +| A missed identifier or raw plugin SQL can hit another tenant | Correct for rewrite pooling. | There is no rewrite coverage list to miss. Built-in raw SQL paths were audited and run against exact tenant pools. Arbitrary plugins are unsandboxed trusted code, so production now rejects all caller presets/plugins unless explicitly admitted. An admitted plugin remains part of the process and database trusted computing base. | +| RLS does not protect metadata, functions, sequences, indexes, or privileged code | Correct. | Runtime admission rejects cross-schema relation, sequence, function, and type privileges; object ownership; `SUPERUSER`, `BYPASSRLS`, `CREATEROLE`, `CREATEDB`, and replication; unexpected inherited/`SET ROLE` paths; `SECURITY DEFINER`; owner-rights views; foreign/materialized views; and unsafe stored-expression dependencies. PostgreSQL catalog names are still generally visible to connected roles, so the claim is non-use/non-exposure through GraphQL, not catalog-name confidentiality. | +| Fail-closed behavior is required | Correct. | Missing/ambiguous route rows, database/API IDs, physical schemas, roles, feature contracts, runtime credentials, scoped schemas, unsafe roles, protected preset overrides, untrusted production plugins, and invalid internal headers all fail closed. The remaining fail-closed gap is coherent revisioning across route/security reads and long-lived WebSockets. | +| Prepared statements can cross tenant state | Correct in a shared pool. | Pools are split by an opaque HMAC over endpoint, TLS, database, login, password, driver, pool settings, purpose, and sanitation mode. Reused checkouts run `DISCARD ALL`, then clear node-postgres and Graphile prepared-statement bookkeeping; reset failure destroys the client. The real same-backend A/B/C test passed. | +| The fingerprint groups tenants into one Graphile instance | False for the current candidate. | The HMAC build contract includes opaque pool identity, database/API IDs, ordered schemas, roles, resolved plugins/settings, compute/storage bindings, surface flags, and introspection mode. Each exact tenant/API contract gets its own instance. The hash is a cache identity, not a code signature or sandbox. | +| More separate APIs improve reuse | False under the secure design. | API ID participates in the build contract, so a separate API intentionally gets a separate instance. Density comes from lowering each dedicated instance's retained memory and governing builds, not weakening isolation through reuse. | +| BM25 is skipped | False in the current candidate; true of the old rewrite design's compatibility exclusions. | BM25 stays enabled and binds the physical schema-qualified index name. The A/B/C fixture built and exercised BM25 together with tsvector, trigram, vector, PostGIS, and ltree. | +| GUC values can revert to an earlier tenant | `SET LOCAL` restores the prior value after transaction end, so the concern is correct. | Every request writes the full security-GUC allowlist, including empty values, plus role, read-only state, `row_security`, and a pinned `search_path`. Checkout sanitation removes any earlier session and prepared state first. | +| Plugins that reference RLS/schema objects may bypass the design | Plugins are trusted code, so the concern is correct in principle. | Built-ins receive the exact pool, physical schemas, and request `pgSettings`; metadata loaders use same-database/API joins and quoted identifiers. Production caller presets are denied by default. Any explicit opt-in requires pinned dependencies, code review, and requalification because a plugin can open its own connection or use process I/O. | + +## Evidence + +### Latest hostile A/B/C execution + +The final rerun used a disposable `constructiveio/postgres-plus:18` container, a +fresh database, three distinct `LOGIN NOINHERIT` roles, forced-RLS tenant tables, +denied cross-database/schema privileges, exact per-tenant pools, scoped-required +introspection, and realtime-resident instances. It completed in 3.9 seconds and +recorded 56 passing checks with `crossTenantTokens: 0`. The fixture is +deliberately marked `customerQualified: false`: it proves hostile isolation, not +the 15-minute workload/provider/density gates. + +Evidence artifact: +`complete-tenant-fixture/generated/hostile-validation.json`. + +Separate real PostgreSQL integration tests passed 2 of 2 checkout-sanitizer +checks and 2 of 2 runtime-role checks. Those tests deliberately reused one +backend after poisoning session/prepared state and created actual inherited +owner, `BYPASSRLS`, object-ownership, `SECURITY DEFINER`, and cross-schema +relation/sequence/function/type violations that admission had to reject. + +### Relevant source boundaries + +- `graphql/server/src/middleware/graphile.ts` resolves one exact runtime pool, + passes physical schemas to `makePgService`, audits the role boundary on every + resident request, and keys the resident instance by the exact build contract. +- `graphql/server/src/middleware/graphile-build-contract.ts` builds and HMACs the + exact contract. Function source and exact in-process identity participate, but + mutable closure state and supply-chain integrity cannot be attested by a hash. +- `graphql/server/src/middleware/runtime-pg-config.ts` accepts only explicit + credential data, matches the routed physical database and network/TLS target, + and keeps raw credentials in a request-keyed `WeakMap`. +- `graphql/server/src/middleware/runtime-role-safety.ts` performs the catalog + privilege, ownership, role-reachability, privileged-object, and stored + dependency audit. Successful-result reuse defaults to zero milliseconds. +- `postgres/pg-cache/src/pg.ts` defines exact pool identity, executes + `DISCARD ALL`, clears both prepared-statement caches, pins the safe baseline, + and destroys a client on sanitation failure. +- `packages/express-context/src/pg-settings.ts` initializes every security GUC, + role, read-only state, `row_security`, and the allowlisted search path for each + Graphile transaction. +- `graphql/server/src/middleware/graphile-preset-composition.ts` rejects + production caller plugins by default and prevents admitted presets from + replacing server-owned PostgreSQL services, request context, transport/error + policy, build-state policy, or protected plugins. This is admission, not a + sandbox. +- `research/graphile-density/PLUGIN-SQL-AUDIT.md` records the built-in plugin/raw + SQL review and the remaining deliberate system/build lanes. + +### Performance result retained under the secure architecture + +On the clean three-repetition 62,298-`pg_class` single-surface fixture, stock +introspection retained a median 449.42 MiB heap, ended 1,106.94 MiB above the +RSS baseline, and built in 3,495.95 ms. Scoped dependency introspection retained +6.55 MiB heap, ended 47.77 MiB above the RSS baseline, and built in 130.34 ms: +68.60 times less retained heap, 23.17 times less final RSS, and a 26.82 times +faster cold build. The compared schema was byte-equivalent and the recorded +operations had zero errors, mismatches, or cross-tenant tokens. + +These are performance-only instance measurements, not a complete-customer +tenants-per-GiB qualification. Security hardening added authoritative metadata +reads and role admission work, so final throughput/p99 must be remeasured; no +security check may be removed to recover a benchmark. + +## Production blockers + +1. **Atomic security contract.** Add `TenantSecurityContractV1` with an immutable + revision. Resolve route plus routing-plane security/exposure fields in one + parameterized read-only snapshot. Publish tenant-local auth settings under + that revision, then atomically activate it in routing; absence or mismatch + must fail closed. Carry the revision through `ApiStructure`, runtime resolver + input, the Graphile build contract, and WebSocket admission. +2. **Revocation semantics for long-lived transports.** Recheck + `(selector, apiId, databaseId, revision)` before every WebSocket operation, + retire the resident generation on mismatch, and terminate existing + subscriptions through event-driven invalidation plus an authoritative + fallback. Decide and document whether an HTTP request owns an admission + snapshot or must recheck immediately before execution. +3. **Production database policy proof.** Against disposable production-shaped + databases, verify the exact runtime login and both request roles for every + exposed/dependency object, and assert the intended RLS/policy/FORCE-RLS + manifest for shared-row tables. The fixture proves the mechanism, not every + deployed tenant schema. +4. **Explicit auth-required contract.** `strictAuth=false` permits an API with no + RLS module to proceed anonymously. Public/no-auth APIs may be intentional, so + production needs an authoritative per-API `authRequired` field rather than a + process-wide guess; missing required auth metadata must fail closed. +5. **Upstream introspection review.** `scoped-required` depends on source-level + Graphile patches and dependency-closure semantics. It must stay off by + default until Graphile maintainers review the isolated API and the patches + are replaced by supported upstream code. +6. **Operational trust boundary.** The internal-header secret must be stripped + at public ingress and carried only over authenticated encrypted service hops. + `X-Meta-Schema` is a deliberate cross-tenant administration capability, + disabled by default, and must use a separate private ingress and safe roles + if enabled. Runtime credential resolution and every explicitly admitted + plugin remain trusted code. +7. **Release qualification.** Rerun the 15-minute repeated complete-customer + density matrix and two-hour churn soak on the final security code, including + multipart upload/storage byte roundtrips and intended external providers. + +Until blockers 1–5 are closed, the production decision is **no-go**. The +dedicated-instance/scoped-introspection architecture remains the right candidate +because its measured memory gain does not depend on SQL rewrite or weakened +tenant isolation. + diff --git a/research/graphile-density/UNIFORM-DENSITY-FIXTURE.md b/research/graphile-density/UNIFORM-DENSITY-FIXTURE.md new file mode 100644 index 0000000000..3aeb78f817 --- /dev/null +++ b/research/graphile-density/UNIFORM-DENSITY-FIXTURE.md @@ -0,0 +1,63 @@ +# Uniform Graphile density fixture + +`graphile_density_uniform_20260801_a` is a performance-only routing-canary +fixture. It measures Graphile memory density across a uniform 4,000-tenant +catalog; it does not prove database-enforced tenant isolation or qualify a +complete customer surface. The shared `gd_runtime_20260801_a` login can read +every tenant schema by design. + +The fixture was physically cloned from `graphile_density_20260801_a` without +modifying or dropping the source. Tenants 401 through 4,000 were populated in +100-tenant transactions, and 7,920 complete disposable noise tables were +removed in 100-table transactions. The exact `pg_class` accounting is: + +| Catalog bucket | Rows | +| --- | ---: | +| 4,000 tenant schemas, seven direct classes each | 28,000 | +| Remaining `gd_noise` tables, sequences, and indexes | 10,094 | +| Tenant, noise, and system TOAST tables/indexes | 22,808 | +| System classes outside `pg_toast` | 337 | +| **Total** | **61,239** | + +Each tenant has two tables, two identity sequences, three indexes, four owned +TOAST classes, one stable `tenant_token()` function, and nine PostgreSQL 18 +constraints (including cataloged `NOT NULL` constraints). Objects are owned by +`postgres`; the runtime role has schema `USAGE`, table `SELECT`, sequence +`SELECT, USAGE`, and function `EXECUTE`, but no schema/database `CREATE` and no +table write privileges. + +## Reproduce locally + +Run with an already configured PostgreSQL administrator environment; neither +script contains credentials: + +```bash +psql -X -v ON_ERROR_STOP=1 -d postgres \ + -f research/graphile-density/create-uniform-density-fixture.sql + +psql -X -v ON_ERROR_STOP=1 -d postgres \ + -f research/graphile-density/validate-uniform-density-fixture.sql +``` + +The creator fails when the fixed target already exists and never drops a +database. Its explicit `-v resume=1` path is limited to an existing clone that +still passes the asserted 61,239-row heterogeneous source shape before any +tenant DDL runs. + +## Recorded validation + +The final standalone validation completed all 40 least-privilege runtime +batches and reported: + +```text +database_name: graphile_density_uniform_20260801_a +pg_class_count: 61239 +tenant_schema_count: 4000 +distinct_tenant_shapes: 1 +logical_pg_class_fingerprint ec670a0d19a77919732f544d54bb34c9 +tenant_shape_fingerprint: 91910068fdc30af0dc304390ee3a605a +``` + +The logical class fingerprint normalizes OID-derived TOAST names and excludes +volatile physical statistics, so it records catalog shape rather than clone +OID allocation or post-benchmark `ANALYZE` state. diff --git a/research/graphile-density/UPSTREAM-REVIEW.md b/research/graphile-density/UPSTREAM-REVIEW.md new file mode 100644 index 0000000000..5b37372f9e --- /dev/null +++ b/research/graphile-density/UPSTREAM-REVIEW.md @@ -0,0 +1,51 @@ +# Upstream review packet: schema-scoped PostgreSQL introspection + +No upstream contact has been made. Production use of `scoped-required` remains blocked on Graphile maintainer review. + +## Proposed API + +Keep `makeIntrospectionQuery(): string` byte-for-byte unchanged and add: + +```ts +makeSchemaScopedIntrospectionQuery( + schemas: readonly string[] +): { text: string; values: [string[]] } +``` + +The requested names are carried only in `$1::text[]`. Empty, NUL-containing, `pg_*`, and `information_schema` scopes are rejected before SQL execution. The Graphile service option is `introspectionMode: 'stock' | 'scoped-required'`; scoped mode parses the result normally and then asserts that every requested service schema was found. There is no fallback to stock. + +## Dependency closure + +The recursive namespace graph follows dependencies required to parse selected objects: + +- foreign-key source relation → referenced relation; +- relation attribute → attribute type; +- function namespace → argument, OUT-argument, and return types; +- type → base, element, and array types; +- range type → subtype; +- inheritance child → parent. + +`pg_catalog` is returned in the namespace payload and its types remain available as in stock introspection. Inheritance deliberately does not follow parent → child: stock introspection emits `pg_inherits` rows only when the child class is selected, and reverse closure pulled a shared partition parent into unrelated tenant child schemas. The disposable partition regression preserved byte-identical SDL while returning only `density_shared` and `pg_catalog`. + +## Local evidence + +- Stock query stability: 7,332 bytes, SHA-256 `c0ed817b912f78e1ea68c70d89ff4b7f9cb4c02d88112a69ac4109d5b996e4c5`. +- Cross-schema FK/domain/enum/function fixture: stock and scoped Constructive SDL are both 22,538 bytes with SHA-256 `2d899a6f9abcea107987a0aa932f18dd8d1466bca3f4ddd340199316aea1f238`. +- Partition-parent fixture: stock and scoped SDL are both 20,201 bytes with SHA-256 `8f32cab18fa54c8d4c6afa7ab05bed73be770f98c29aa9b1bcd4f29e6f54d532`; hidden tenant child schemas are absent from scoped introspection. +- A missing requested schema fails with `Schema-scoped introspection for service 'main' did not find required schema(s): density_missing`. + +The exact evidence is in `artifacts/scoped-introspection-smoke.json`. These are small PostgreSQL 18.4 fixtures, not the required PostgreSQL 17+/61k-catalog benchmark. + +## Review questions + +1. Is namespace closure the right upstream seam, or should filtering occur by object OID after the stock query generator has expressed every catalog dependency? +2. Which additional OID dependencies must be closed for extensions, composite/domain types, cross-schema defaults/sequences, partitioned tables, procedures, policies, and future introspection versions? +3. Should referenced schemas be included in raw introspection but remain absent from the configured GraphQL surface, as this candidate does? +4. Should missing configured schemas fail in the gather layer, and how should watch-mode schema creation/deletion invalidate a cached failed gather? +5. Can the option live on `PgServiceConfiguration`, and should it take explicit schema names, a callback, or an upstream-defined scope object? +6. What PostgreSQL versions and extension catalogs should upstream CI cover, especially PostgreSQL 17+, PostGIS, vector, BM25, ltree, and partitioning? +7. Can upstream source generate both stock and scoped queries so Constructive can remove its temporary dist patches? + +## Requested upstream tests + +The upstream change should preserve the stock query byte, compare parsed introspection and emitted GraphQL SDL for the dependency cases above, prove bind-only schema names including quotes, fail on missing schemas, and benchmark catalog rows returned, PostgreSQL peak memory, cold-build time, and retained Node heap at the 61k catalog. Constructive's full plugin and hostile tenant matrix remains a separate downstream responsibility. diff --git a/research/graphile-density/artifacts/scoped-introspection-smoke.json b/research/graphile-density/artifacts/scoped-introspection-smoke.json new file mode 100644 index 0000000000..429fe85da7 --- /dev/null +++ b/research/graphile-density/artifacts/scoped-introspection-smoke.json @@ -0,0 +1,73 @@ +{ + "schemaVersion": 1, + "recordedAt": "2026-07-31T17:21:37Z", + "classification": "small disposable-fixture smoke; not a density benchmark or production acceptance run", + "environment": { + "postgresVersion": "18.4", + "database": "codex_graphile_density_20260731_rerun", + "databaseDroppedAfterRun": true, + "constructiveDbUsed": false + }, + "catalogFixture": { + "requestedSchemas": ["density_a"], + "features": [ + "cross-schema foreign key", + "shared enum", + "shared domain", + "identity sequences", + "SQL function" + ], + "stockQuery": { + "bytes": 7332, + "sha256": "c0ed817b912f78e1ea68c70d89ff4b7f9cb4c02d88112a69ac4109d5b996e4c5" + }, + "stockIntrospection": { + "bytes": 404110, + "namespaces": 6, + "classes": 8, + "procedures": 1, + "types": 481 + }, + "scopedIntrospection": { + "bytes": 403639, + "namespaces": 4, + "classes": 8, + "procedures": 1, + "types": 481, + "bindValues": [["density_a"]] + }, + "stockSdl": { + "bytes": 22538, + "sha256": "2d899a6f9abcea107987a0aa932f18dd8d1466bca3f4ddd340199316aea1f238" + }, + "scopedSdl": { + "bytes": 22538, + "sha256": "2d899a6f9abcea107987a0aa932f18dd8d1466bca3f4ddd340199316aea1f238" + }, + "sdlByteEquivalent": true, + "missingSchemaError": "Schema-scoped introspection for service 'main' did not find required schema(s): density_missing" + }, + "partitionFixture": { + "database": "codex_graphile_density_partition_20260731", + "databaseDroppedAfterRun": true, + "requestedSchema": "density_shared", + "hiddenChildSchemas": ["density_a", "density_b"], + "scopedNamespaces": ["density_shared", "pg_catalog"], + "scopedClasses": ["events", "events_pkey"], + "stockSdl": { + "bytes": 20201, + "sha256": "8f32cab18fa54c8d4c6afa7ab05bed73be770f98c29aa9b1bcd4f29e6f54d532" + }, + "scopedSdl": { + "bytes": 20201, + "sha256": "8f32cab18fa54c8d4c6afa7ab05bed73be770f98c29aa9b1bcd4f29e6f54d532" + }, + "sdlByteEquivalent": true, + "finding": "child-to-parent inheritance closure is required; parent-to-child closure over-expands into unrelated tenant partitions and was removed" + }, + "limitations": [ + "The catalog is tiny, so the byte reduction is not representative of the 61k-pg_class target.", + "This does not exercise RLS, runtime credentials, hostile tenants, every plugin, throughput, RSS density, or soak behavior.", + "The first fixture's forced database cleanup emitted expected 57P01 messages from adaptor clients after schema release; the database was confirmed absent afterward. The partition rerun used explicit pools and drained cleanly." + ] +} diff --git a/research/graphile-density/complete-tenant-fixture/.gitignore b/research/graphile-density/complete-tenant-fixture/.gitignore new file mode 100644 index 0000000000..df5372de97 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/.gitignore @@ -0,0 +1,3 @@ +/artifacts/ +/generated/ +/qualification-artifacts/ diff --git a/research/graphile-density/complete-tenant-fixture/README.md b/research/graphile-density/complete-tenant-fixture/README.md new file mode 100644 index 0000000000..2d6b52b526 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/README.md @@ -0,0 +1,192 @@ +# Complete-tenant A/B/C research fixture + +This fixture is the correctness half of the tenant-density spike. The uniform +catalog fixture finds the memory-capacity curve; this fixture asks whether a +candidate can keep a small, production-shaped A/B/C fleet complete and isolated. +Three tenants are too few to establish tenants per GiB, so no performance claim +may be derived from this fixture alone. + +The A/B/C database and generated GraphQL operation names were validated end to +end on 2026-08-02 against a disposable PostgreSQL 18 `postgres-plus` instance. +The latest-source hostile run passed all 56 checks, including all declared +capability operations, realtime-resident instances, same-backend sanitation, +schema drift/rebuild, and alternating connection reuse with zero cross-tenant +tokens. Its local evidence is `generated/hostile-validation.json`. Unsupported +or renamed fields still fail the run; nothing is silently skipped. This is an +offline hostile-isolation result, not complete-customer density or production +provider qualification. + +## Isolation model + +`schema.sql` creates three physical tenant schemas (`ctf_a`, `ctf_b`, `ctf_c`) +with identical objects, forced RLS policies, and distinct canary values. It +requires three distinct `LOGIN NOINHERIT` runtime roles. Each role receives +USAGE and object privileges for exactly one tenant schema, while shared access is +limited to the audited `ctf_extensions` and `jwt_private` dependency schemas. +Configured request roles must not reach parent roles through either `INHERIT` +or `SET`; the startup audit evaluates each request role as a separate execution +root, so a privilege or ownership path that appears only after `SET ROLE` fails +closed. +Each role also receives USAGE and EXECUTE on its own `ctf__realtime` cursor +schema, with PUBLIC and both foreign tenant roles denied. The fixture setup +asserts that ACL matrix before it succeeds. + +`server.cjs` creates one PostGraphile instance and one dedicated runtime pool +for each tenant. The default non-realtime lane uses `max=1`; realtime fails +closed unless the pool has at least two slots because its cursor manager keeps +one client resident. A live build contract includes the exact credential-sensitive, +process-keyed pool identity and physical schema, so host labels cannot alias cache +entries. Cross-process evidence uses a separate deterministic credential-free +contract fingerprint and proves its live role/database/schema mapping at runtime. +Runtime checkouts use `DISCARD ALL`; each request then sets the complete security-GUC +allowlist, role, read-only state, RLS state, and pinned search path. The hostile +probe deliberately reuses a named statement with different SQL after checkout +to verify both PostgreSQL and node-postgres prepared-statement bookkeeping were +cleared. + +Schema drift is deliberately outside the runtime boundary. The runtime roles +have no USAGE or EXECUTE access to `ctf_control`; the loopback control endpoint +uses the separately configured control-plane `PG*` login and a random in-memory +token. RLS remains defense in depth for rows, while schema ACLs and dedicated +logins enforce the physical routing boundary. + +The metadata canary covers GraphQL schema and introspection isolation. It does +not claim that tenant schema names are confidential inside PostgreSQL: +system-catalog object names are generally visible to connected roles, and +hiding those names would require a stronger database/process boundary. The +security claim here is that another tenant's objects cannot be used or exposed +through the GraphQL build, not that their catalog names cannot be observed. + +## What the offline lane covers + +The candidate fleet includes generated CRUD/function plans, i18n, deterministic +LLM/RAG, BM25, tsvector, trigram, pgvector, PostGIS, ltree, presigned-upload +metadata/signing, bulk mutations, realtime-tagged writes, and preloaded function +bindings. BM25 stays enabled because every instance compiles against its real +physical schema; there is no SQL schema rewrite. + +`hostile-validation.cjs` checks every declared canary plus dynamic session +poisoning, savepoint rollback, prepared-statement reset, schema drift and cache +invalidation, serialized cold builds, and alternating A/B/C connection reuse. +Every dynamic identity and authenticated control response must also return the +caller-supplied `current_database()` identity, and every tenant runs a negative +role-safety probe that must reject the control-plane role. +The exact roles and pools prevent cross-tenant session reuse by construction; +the reuse checks prove sanitation within each tenant pool and distinct build +contracts prove that pools cannot alias. Any unavailable or inconclusive probe +exits non-zero. + +Realtime mutations exercise the tagged database write and NOTIFY trigger. With +`--enable-realtime`, every cached instance keeps a cursor manager resident +against its exact tenant cursor schema and exposes a no-server Grafserv upgrade +handler. The outer fixture server selects that handler only after an exact +tenant path match, and disposal terminates that generation's long-lived sockets +before releasing its pool. The physical-density wrapper keeps one +`graphql-transport-ws` subscription resident per surface and requires a real, +tenant-specific event before the surface can count. + +## External-provider boundary + +The exact fixture currently injects a deterministic LLM and uses a signing-only +S3 client. Those paths exercise plugin and database integration but prove +neither model semantics nor an object-storage byte roundtrip. Consequently: + +- `--class offline-research` may pass local gates but always records + `customerQualified: false`. +- `--class production` currently fails with + `CTF_PRODUCTION_EQUIVALENCE_NOT_IMPLEMENTED`, even when provider arguments are + supplied. Missing arguments fail earlier with + `CTF_EXTERNAL_PROVIDER_GATES_UNSATISFIED`. +- Production support requires wiring and testing the intended Ollama-compatible + models plus disposable S3/MinIO PUT, HEAD/GET, and cleanup paths. The manifest + requires `--ollama-url`, `--embedding-model`, `--chat-model`, `--s3-endpoint`, + and `--s3-bucket`; provider credentials stay in the environment and never in + artifacts. + +## Disposable local setup + +Create the three runtime logins separately under an administrator. They must be +distinct, `LOGIN NOINHERIT`, and must not be superuser, `BYPASSRLS`, +`CREATEROLE`, `CREATEDB`, replication, a tenant-schema owner, or able to CREATE +in a tenant schema. The fixture intentionally does not create or alter roles. + +```bash +createdb graphile_complete_tenant_spike +psql --set=ON_ERROR_STOP=1 \ + --set=runtime_role_a=ctf_runtime_a \ + --set=runtime_role_b=ctf_runtime_b \ + --set=runtime_role_c=ctf_runtime_c \ + --dbname=graphile_complete_tenant_spike \ + --file=research/graphile-density/complete-tenant-fixture/schema.sql +``` + +Keep the ordinary `PGHOST`, `PGPORT`, `PGDATABASE`, `PGUSER`, and `PGPASSWORD` +pointed at the fixture owner/control login. Supply runtime credentials only in +`CTF_RUNTIME_A_PGPASSWORD`, `CTF_RUNTIME_B_PGPASSWORD`, and +`CTF_RUNTIME_C_PGPASSWORD`; `GRAPHQL_RUNTIME_PGPASSWORD` is an optional shared +password fallback. Role names are non-secret command arguments. + +Start the candidate and run the hostile gate with a control token of at least 32 +bytes: + +```bash +export CTF_CONTROL_TOKEN="$(openssl rand -hex 32)" +export GRAPHILE_CACHE_MAX=3 PG_CACHE_MAX=4 PG_POOL_MAX=1 PG_POOL_MAX_USES=0 +export DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE=100 + +node research/graphile-density/complete-tenant-fixture/server.cjs \ + --port 3391 --arm local-complete-tenant --mode scoped-required \ + --runtime-role-a ctf_runtime_a \ + --runtime-role-b ctf_runtime_b \ + --runtime-role-c ctf_runtime_c + +node research/graphile-density/complete-tenant-fixture/hostile-validation.cjs \ + --base-url http://127.0.0.1:3391 \ + --expected-physical-database-identity graphile_complete_tenant_spike \ + --arm local-complete-tenant --mode scoped-required +``` + +For the realtime-resident lane, add `--enable-realtime true +--runtime-pool-max 2`, keep process-global `PG_POOL_MAX=1`, and set +`PG_CACHE_MAX` high enough for the three dedicated runtime identities plus the +control identity. Runtime capacity is explicit per pool and must not leak into +the control-plane baseline. + +Generate credential-free cperf inputs only after the server reports exact, +unique `graphile:v1:` contracts: + +```bash +node research/graphile-density/complete-tenant-fixture/generate-inputs.cjs \ + --port 3391 --postgres-container postgres \ + --runtime-role-a ctf_runtime_a \ + --runtime-role-b ctf_runtime_b \ + --runtime-role-c ctf_runtime_c +``` + +The one-command research gate is explicitly offline and runs three 15-minute +repetitions at a 4-GiB V8 old-space setting, followed by the mandatory repository +suites. It is a completeness gate, not a capacity search: + +```bash +node research/graphile-density/complete-tenant-fixture/qualification-runner.cjs \ + --class offline-research \ + --postgres-container postgres \ + --runtime-role-a ctf_runtime_a \ + --runtime-role-b ctf_runtime_b \ + --runtime-role-c ctf_runtime_c +``` + +The perf harness rejects a dirty server provenance. Commit the local research +branches and confirm `git status --short` is empty before a qualifying run; the +fixture ignores only its generated inputs and run-artifact directories so those +outputs do not invalidate provenance. No commit or push is performed by these +scripts. + +Build the affected packages before starting the fixture. Its exact runtime +`dist` artifact fingerprint is part of every Graphile build contract and the +generated benchmark provenance; a stale build-contract API fails closed instead +of silently measuring old code. + +Before using any result, inspect `qualification.json`. The offline lane is valid +only when `localPassed` is true, and `customerQualified` must remain false until +the provider-backed production runner exists and passes. diff --git a/research/graphile-density/complete-tenant-fixture/coverage-manifest.json b/research/graphile-density/complete-tenant-fixture/coverage-manifest.json new file mode 100644 index 0000000000..fe6e4cce09 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/coverage-manifest.json @@ -0,0 +1,78 @@ +{ + "version": 1, + "fixture": "complete-tenant-abc-v1", + "qualificationClass": "offline-research-candidate", + "tenants": ["a", "b", "c"], + "surfacesPerTenant": ["api"], + "performanceClaimsAllowed": false, + "productionQualificationImplemented": false, + "runtimeIsolation": { + "model": "dedicated-login-and-pool-per-tenant", + "sharedRuntimePool": false, + "runtimeSchemaDriftControl": false, + "checkoutSanitationRequired": true + }, + "localCapabilities": [ + { "id": "graphile-generated", "status": "candidate-runtime-required", "evidence": "candidate connection and primary-key operations must compile and execute for A/B/C" }, + { "id": "i18n", "status": "candidate-runtime-required", "evidence": "candidate localeStrings operation must execute the parameterized i18n SQL path" }, + { "id": "llm-deterministic", "status": "offline-only", "evidence": "candidate embedText operation uses an injected deterministic 3-D provider" }, + { "id": "rag-deterministic", "status": "offline-only", "evidence": "candidate ragQuery operation uses deterministic embedding/chat plus pgvector chunk SQL" }, + { "id": "bm25", "status": "candidate-runtime-required", "evidence": "candidate BM25 filter and score operation; BM25 remains enabled" }, + { "id": "tsvector", "status": "candidate-runtime-required", "evidence": "candidate tsvector filter and rank operation" }, + { "id": "trigram", "status": "candidate-runtime-required", "evidence": "candidate pg_trgm filter and similarity operation" }, + { "id": "vector", "status": "candidate-runtime-required", "evidence": "candidate pgvector cosine filter and distance operation" }, + { "id": "postgis", "status": "candidate-runtime-required", "evidence": "candidate PostGIS codec operation using schema-qualified extension SQL" }, + { "id": "ltree", "status": "candidate-runtime-required", "evidence": "candidate ltree filter using schema-qualified operator and cast SQL" }, + { "id": "uploads-storage-presign-only", "status": "offline-only", "evidence": "candidate metadata insert and offline SigV4 signing path; no object bytes are transferred" }, + { "id": "bulk-mutations", "status": "candidate-runtime-required", "evidence": "candidate bulk upsert operation" }, + { "id": "realtime-tagged-write", "status": "candidate-runtime-required", "evidence": "candidate generated update must fire the fixture NOTIFY trigger; delivery is a separate mandatory suite" }, + { "id": "function-bindings", "status": "candidate-runtime-required", "evidence": "candidate preloaded binding must insert an RLS-visible invocation" }, + { "id": "security-session", "status": "candidate-runtime-required", "evidence": "protected controls poison and roll back state; the next exact-tenant checkout must sanitize and initialize every request GUC" } + ], + "hostileCanaries": [ + "cross-schema-identifiers", + "metadata", + "functions", + "sequences", + "prepared-statement-reuse", + "poisoned-gucs", + "rollback-savepoints", + "plugin-raw-sql", + "owner-bypass-role", + "schema-drift", + "cache-invalidation", + "concurrent-builds", + "connection-reuse" + ], + "metadataBoundary": "GraphQL schema/introspection isolation only; PostgreSQL system catalogs may reveal object names to any connected login", + "mandatoryRepositorySuites": [ + { + "id": "realtime-websocket-delivery", + "cwd": "graphile/graphile-realtime-test", + "command": ["pnpm", "exec", "jest", "--runInBand", "__tests__/realtime-websocket.integration.test.ts"] + }, + { + "id": "plugin-capability-closure", + "cwd": "graphile/graphile-settings", + "command": ["pnpm", "exec", "jest", "--runInBand", "__tests__/scoped-introspection-capability-closure.integration.test.ts"] + } + ], + "externalProviderGates": [ + { + "id": "ollama-real-semantic", + "status": "blocking", + "requiredArguments": ["ollama-url", "embedding-model", "chat-model"], + "offlineSurrogate": ["llm-deterministic", "rag-deterministic"] + }, + { + "id": "object-storage-byte-roundtrip", + "status": "blocking", + "requiredArguments": ["s3-endpoint", "s3-bucket"], + "offlineSurrogate": ["uploads-storage-presign-only"], + "repositorySuite": { + "cwd": "graphql/server-test", + "command": ["pnpm", "exec", "jest", "--runInBand", "__tests__/upload.integration.test.ts"] + } + } + ] +} diff --git a/research/graphile-density/complete-tenant-fixture/generate-inputs.cjs b/research/graphile-density/complete-tenant-fixture/generate-inputs.cjs new file mode 100644 index 0000000000..b872a4ca41 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/generate-inputs.cjs @@ -0,0 +1,260 @@ +'use strict'; + +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + FIXTURE_DIR, + REPO_ROOT, + TENANTS, + assertCredentialFree, + assertLoopbackBaseUrl, + makeFleet, + makePlan, + parseArgs, + parsePositiveInteger, + requireString, + validateIntrospectionClientReleaseMode, +} = require('./lib.cjs'); + +const fetchJson = async (url, fetchImpl = fetch) => { + const response = await fetchImpl(url); + if (!response.ok) throw new Error(`CTF_STATUS_HTTP_${response.status}`); + return response.json(); +}; + +const validateServerStatus = ( + status, + { arm, mode, introspectionClientReleaseMode = 'destroy' }, +) => { + validateIntrospectionClientReleaseMode(introspectionClientReleaseMode); + if (status?.version !== 1 || status?.fixture !== 'complete-tenant-abc-v1') { + throw new Error('CTF_SERVER_STATUS_IDENTITY_MISMATCH'); + } + if (status.arm !== arm) throw new Error(`CTF_SERVER_ARM_MISMATCH:${status.arm}`); + if (status.introspectionMode !== mode) { + throw new Error(`CTF_SERVER_MODE_MISMATCH:${status.introspectionMode}`); + } + if (status.introspectionClientReleaseMode !== introspectionClientReleaseMode) { + throw new Error( + 'CTF_SERVER_INTROSPECTION_CLIENT_RELEASE_MODE_MISMATCH:' + + `${status.introspectionClientReleaseMode ?? 'missing'}` + ); + } + if (status.releaseBuildStateAfterValidation !== true) { + throw new Error('CTF_SERVER_BUILD_STATE_RETIREMENT_REQUIRED'); + } + if ( + status.physicalIsolation !== 'dedicated-login-and-pool-per-tenant' + || status.sharedRuntimePool !== false + || status.runtimeSafety?.passed !== true + || status.runtimeSafety?.rolesDistinct !== true + ) { + throw new Error('CTF_SERVER_RUNTIME_BOUNDARY_UNSAFE'); + } + if (!/^sha256:[0-9a-f]{64}$/.test(status.runtimeArtifactFingerprint ?? '')) { + throw new Error('CTF_SERVER_RUNTIME_FINGERPRINT_INVALID'); + } + if ( + status.liveIdentityScope !== 'process-local-keyed-hmac-v1' + || !/^graphile-configuration:ctf:v1:[a-f0-9]{64}$/.test( + status.configurationIdentity ?? '' + ) + ) { + throw new Error('CTF_SERVER_CONFIGURATION_IDENTITY_INVALID'); + } + const contracts = status.buildContracts; + if (!contracts || typeof contracts !== 'object') { + throw new Error('CTF_SERVER_CONTRACTS_MISSING'); + } + const values = TENANTS.map((tenant) => contracts[tenant.id]); + if (values.some((value) => typeof value !== 'string' || !value.startsWith('graphile:v1:'))) { + throw new Error('CTF_SERVER_CONTRACT_INVALID'); + } + if (new Set(values).size !== TENANTS.length) { + throw new Error('CTF_SERVER_CONTRACT_COLLISION'); + } + if (typeof status.physicalDatabase !== 'string' || !status.physicalDatabase.trim()) { + throw new Error('CTF_SERVER_PHYSICAL_DATABASE_MISSING'); + } + const runtimePoolIdentities = status.runtimePoolIdentities; + const poolValues = TENANTS.map((tenant) => runtimePoolIdentities?.[tenant.id]); + if (poolValues.some((value) => + typeof value !== 'string' || !/^pg:v1:[a-f0-9]{64}$/i.test(value) + )) { + throw new Error('CTF_SERVER_POOL_IDENTITY_INVALID'); + } + if (new Set(poolValues).size !== TENANTS.length) { + throw new Error('CTF_SERVER_POOL_IDENTITY_COLLISION'); + } + const evidence = status.contractEvidence; + if ( + evidence?.version !== 1 + || evidence.credentialFree !== true + || evidence.configurationIdentity !== status.configurationIdentity + ) { + throw new Error('CTF_SERVER_CONTRACT_EVIDENCE_INVALID'); + } + assertCredentialFree(evidence); + for (const tenant of TENANTS) { + const pool = evidence.runtimePools?.[tenant.id]; + const build = evidence.graphileBuilds?.[tenant.id]; + const binding = status.runtimeBindings?.[tenant.id]; + if ( + !/^pg-contract-evidence:v1:[a-f0-9]{64}$/.test(pool?.fingerprint ?? '') + || !/^graphile-contract-evidence:v1:[a-f0-9]{64}$/.test( + build?.fingerprint ?? '' + ) + || pool?.input?.databaseName !== status.physicalDatabase + || pool?.input?.role !== binding?.role + || binding?.databaseId !== tenant.databaseId + || binding?.databaseName !== status.physicalDatabase + || JSON.stringify(binding?.schemas) !== JSON.stringify([tenant.schema]) + ) { + throw new Error(`CTF_SERVER_CONTRACT_EVIDENCE_INVALID:${tenant.id}`); + } + } + return Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + evidence.graphileBuilds[tenant.id].fingerprint, + ])); +}; + +const atomicWriteJson = (file, value) => { + const serialized = `${JSON.stringify(value, null, 2)}\n`; + assertCredentialFree(value); + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); + const temporary = `${file}.${process.pid}.tmp`; + fs.writeFileSync(temporary, serialized, { encoding: 'utf8', mode: 0o600 }); + fs.renameSync(temporary, file); +}; + +const currentCommit = () => execFileSync( + 'git', + ['rev-parse', 'HEAD'], + { cwd: REPO_ROOT, encoding: 'utf8' }, +).trim(); + +const validateGeneratedInputs = (planFile, fleetFile) => { + const perfConfigPath = path.join(REPO_ROOT, 'packages/perf-harness/dist/config.js'); + if (!fs.existsSync(perfConfigPath)) { + throw new Error('CTF_BUILD_ARTIFACT_MISSING:packages/perf-harness/dist/config.js'); + } + const { loadFleet, loadPlan, validateCoverage } = require(perfConfigPath); + const plan = loadPlan(planFile); + const fleet = loadFleet(fleetFile); + validateCoverage(plan, fleet); +}; + +const generateInputs = async ({ + arm = 'local-complete-tenant', + mode = 'scoped-required', + introspectionClientReleaseMode = 'destroy', + port = 3391, + baseUrl = `http://127.0.0.1:${port}`, + postgresContainer, + runtimeRoles, + durationSec = 900, + outputDir = path.join(FIXTURE_DIR, 'generated'), + commit = currentCommit(), + fetchImpl = fetch, + validate = true, +} = {}) => { + if (!postgresContainer) throw new Error('CTF_ARGUMENT_REQUIRED:postgres-container'); + if (!['stock', 'scoped-required'].includes(mode)) { + throw new Error(`CTF_INTROSPECTION_MODE_INVALID:${mode}`); + } + validateIntrospectionClientReleaseMode(introspectionClientReleaseMode); + const localBaseUrl = assertLoopbackBaseUrl(baseUrl); + const status = await fetchJson(`${localBaseUrl}/__ctf/status`, fetchImpl); + const buildContracts = validateServerStatus(status, { + arm, + mode, + introspectionClientReleaseMode, + }); + const fleet = makeFleet({ + arm, + port, + buildContracts, + runtimePoolIdentities: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + status.contractEvidence.runtimePools[tenant.id].fingerprint, + ])), + physicalDatabase: status.physicalDatabase, + }); + const plan = makePlan({ + arm, + port, + postgresContainer, + commit, + durationSec, + cwd: REPO_ROOT, + introspectionMode: mode, + introspectionClientReleaseMode, + runtimeRoles, + }); + assertCredentialFree(fleet); + assertCredentialFree(plan); + fs.mkdirSync(outputDir, { recursive: true, mode: 0o700 }); + const fleetFile = path.join(outputDir, 'fleet.json'); + const planFile = path.join(outputDir, 'plan.json'); + atomicWriteJson(fleetFile, fleet); + atomicWriteJson(planFile, plan); + if (validate) validateGeneratedInputs(planFile, fleetFile); + const provenance = { + version: 1, + generatedAt: new Date().toISOString(), + arm, + mode, + introspectionClientReleaseMode, + commit, + customerQualified: false, + reason: 'inputs-only; workload and external provider gates have not run', + files: { + fleet: path.relative(REPO_ROOT, fleetFile), + plan: path.relative(REPO_ROOT, planFile), + }, + }; + atomicWriteJson(path.join(outputDir, 'generation.json'), provenance); + return { fleetFile, planFile, provenance }; +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + const port = parsePositiveInteger(args.port ?? '3391', 'port'); + const runtimeRoles = Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + requireString(args, tenant.runtimeRoleArgument), + ])); + const result = await generateInputs({ + arm: requireString(args, 'arm', 'local-complete-tenant'), + mode: requireString(args, 'mode', 'scoped-required'), + introspectionClientReleaseMode: requireString( + args, + 'introspection-client-release-mode', + 'destroy', + ), + port, + baseUrl: requireString(args, 'base-url', `http://127.0.0.1:${port}`), + postgresContainer: requireString(args, 'postgres-container'), + runtimeRoles, + durationSec: parsePositiveInteger(args['duration-sec'] ?? '900', 'duration-sec'), + outputDir: path.resolve(requireString(args, 'output-dir', path.join(FIXTURE_DIR, 'generated'))), + }); + process.stdout.write(`${JSON.stringify(result.provenance)}\n`); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + atomicWriteJson, + generateInputs, + validateGeneratedInputs, + validateServerStatus, +}; diff --git a/research/graphile-density/complete-tenant-fixture/generate-inputs.test.cjs b/research/graphile-density/complete-tenant-fixture/generate-inputs.test.cjs new file mode 100644 index 0000000000..3bf06ecb39 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/generate-inputs.test.cjs @@ -0,0 +1,216 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); + +const { TENANTS } = require('./lib.cjs'); +const { + atomicWriteJson, + generateInputs, + validateServerStatus, +} = require('./generate-inputs.cjs'); + +const contracts = () => Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `graphile:v1:${tenant.id.repeat(64)}`, +])); + +const poolIdentities = () => Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `pg:v1:${tenant.id.repeat(64)}`, +])); + +const configurationIdentity = + `graphile-configuration:ctf:v1:${'e'.repeat(64)}`; + +const runtimeBindings = () => Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + databaseId: tenant.databaseId, + databaseName: 'graphile_complete_tenant_spike', + role: `ctf_runtime_${tenant.id}`, + schemas: [tenant.schema], + }, +])); + +const contractEvidence = () => ({ + version: 1, + credentialFree: true, + configurationIdentity, + realtimeListener: null, + runtimePools: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + version: 1, + fingerprint: `pg-contract-evidence:v1:${tenant.id.repeat(64)}`, + input: { + databaseName: 'graphile_complete_tenant_spike', + role: `ctf_runtime_${tenant.id}`, + }, + }, + ])), + graphileBuilds: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + version: 1, + fingerprint: `graphile-contract-evidence:v1:${tenant.id.repeat(64)}`, + input: {}, + }, + ])), + residentGraphileBuildFingerprints: [], +}); + +const evidenceContracts = () => Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `graphile-contract-evidence:v1:${tenant.id.repeat(64)}`, +])); + +const evidencePoolIdentities = () => Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `pg-contract-evidence:v1:${tenant.id.repeat(64)}`, +])); + +const status = (overrides = {}) => ({ + version: 1, + fixture: 'complete-tenant-abc-v1', + arm: 'fixture-arm', + introspectionMode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: true, + physicalIsolation: 'dedicated-login-and-pool-per-tenant', + sharedRuntimePool: false, + runtimeSafety: { passed: true, rolesDistinct: true }, + runtimeArtifactFingerprint: `sha256:${'f'.repeat(64)}`, + configurationIdentity, + liveIdentityScope: 'process-local-keyed-hmac-v1', + physicalDatabase: 'graphile_complete_tenant_spike', + runtimePoolIdentities: poolIdentities(), + runtimeBindings: runtimeBindings(), + buildContracts: contracts(), + contractEvidence: contractEvidence(), + ...overrides, +}); + +test('status validation requires strict physical isolation and unique exact contracts', () => { + assert.deepEqual( + validateServerStatus(status(), { arm: 'fixture-arm', mode: 'scoped-required' }), + evidenceContracts(), + ); + assert.throws( + () => validateServerStatus(status({ introspectionMode: 'stock' }), { + arm: 'fixture-arm', + mode: 'scoped-required', + }), + /CTF_SERVER_MODE_MISMATCH:stock/, + ); + assert.throws( + () => validateServerStatus(status({ introspectionClientReleaseMode: 'reuse' }), { + arm: 'fixture-arm', + mode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + }), + /CTF_SERVER_INTROSPECTION_CLIENT_RELEASE_MODE_MISMATCH:reuse/, + ); + assert.throws( + () => validateServerStatus(status({ sharedRuntimePool: true }), { + arm: 'fixture-arm', + mode: 'scoped-required', + }), + /CTF_SERVER_RUNTIME_BOUNDARY_UNSAFE/, + ); + assert.throws( + () => validateServerStatus(status({ releaseBuildStateAfterValidation: false }), { + arm: 'fixture-arm', + mode: 'scoped-required', + }), + /CTF_SERVER_BUILD_STATE_RETIREMENT_REQUIRED/, + ); + const collided = contracts(); + collided.c = collided.a; + assert.throws( + () => validateServerStatus(status({ buildContracts: collided }), { + arm: 'fixture-arm', + mode: 'scoped-required', + }), + /CTF_SERVER_CONTRACT_COLLISION/, + ); + const unresolved = contracts(); + unresolved.b = 'ctf:unresolved:v1:b'; + assert.throws( + () => validateServerStatus(status({ buildContracts: unresolved }), { + arm: 'fixture-arm', + mode: 'scoped-required', + }), + /CTF_SERVER_CONTRACT_INVALID/, + ); +}); + +test('input generation writes exact credential-free contracts atomically', async (context) => { + const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctf-inputs-')); + context.after(() => fs.rmSync(outputDir, { recursive: true, force: true })); + const runtimeRoles = { a: 'ctf_runtime_a', b: 'ctf_runtime_b', c: 'ctf_runtime_c' }; + const result = await generateInputs({ + arm: 'fixture-arm', + mode: 'scoped-required', + port: 3391, + postgresContainer: 'ctf-postgres', + runtimeRoles, + durationSec: 60, + outputDir, + commit: '0123456789abcdef', + validate: false, + fetchImpl: async () => ({ ok: true, json: async () => status() }), + }); + const fleet = JSON.parse(fs.readFileSync(result.fleetFile, 'utf8')); + const plan = JSON.parse(fs.readFileSync(result.planFile, 'utf8')); + assert.equal(fleet.tenants[0].surfaces[0].buildContract, evidenceContracts().a); + assert.equal( + fleet.tenants[0].databases[0].apis[0].runtimePoolIdentity, + evidencePoolIdentities().a, + ); + assert.equal(plan.arms[0].env.PG_POOL_MAX, '1'); + assert.equal(plan.arms[0].env.PG_POOL_MAX_USES, '0'); + assert.equal(plan.arms[0].env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE, '100'); + const releaseModeIndex = plan.arms[0].command.indexOf( + '--introspection-client-release-mode', + ); + assert.ok(releaseModeIndex >= 0); + assert.equal(plan.arms[0].command[releaseModeIndex + 1], 'destroy'); + assert.equal(result.provenance.introspectionClientReleaseMode, 'destroy'); + assert.equal(result.provenance.customerQualified, false); + assert.doesNotMatch( + fs.readdirSync(outputDir).map((file) => fs.readFileSync(path.join(outputDir, file), 'utf8')).join('\n'), + /password|secretAccessKey|authorization|bearer/i, + ); +}); + +test('input generation rejects unsupported introspection client release modes', async () => { + await assert.rejects(() => generateInputs({ + introspectionClientReleaseMode: 'best-effort', + postgresContainer: 'ctf-postgres', + runtimeRoles: { a: 'ctf_runtime_a', b: 'ctf_runtime_b', c: 'ctf_runtime_c' }, + validate: false, + }), /CTF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:best-effort/); +}); + +test('atomic artifact writes reject credential markers', (context) => { + const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctf-atomic-')); + context.after(() => fs.rmSync(outputDir, { recursive: true, force: true })); + assert.throws( + () => atomicWriteJson(path.join(outputDir, 'unsafe.json'), { password: 'value' }), + /CTF_ARTIFACT_CONTAINS_CREDENTIAL_MARKER/, + ); + assert.equal(fs.existsSync(path.join(outputDir, 'unsafe.json')), false); + assert.doesNotThrow(() => atomicWriteJson(path.join(outputDir, 'safe-error.json'), { + failure: 'CTF_RUNTIME_PASSWORD_REQUIRED:CTF_RUNTIME_A_PGPASSWORD', + })); + assert.throws( + () => atomicWriteJson(path.join(outputDir, 'unsafe-url.json'), { + failure: 'postgres://runtime:credential@127.0.0.1/fixture', + }), + /CTF_ARTIFACT_CONTAINS_CREDENTIAL_MARKER/, + ); +}); diff --git a/research/graphile-density/complete-tenant-fixture/hostile-validation.cjs b/research/graphile-density/complete-tenant-fixture/hostile-validation.cjs new file mode 100644 index 0000000000..abeccb0b34 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/hostile-validation.cjs @@ -0,0 +1,589 @@ +'use strict'; + +const path = require('node:path'); + +const { + FIXTURE_DIR, + TENANTS, + assertLoopbackBaseUrl, + evaluateCanaryResponse, + makeFleet, + parseArgs, + requireString, +} = require('./lib.cjs'); +const { atomicWriteJson, validateServerStatus } = require('./generate-inputs.cjs'); + +const DIAGNOSTIC_TEXT_LIMIT = 512; +const CUSTOMER_ID_PATTERN = /^[a-z0-9-]+$/; + +const requirePhysicalDatabaseIdentity = (value) => { + if (typeof value !== 'string' || !value.trim()) { + throw new Error('CTF_EXPECTED_PHYSICAL_DATABASE_IDENTITY_REQUIRED'); + } + return value.trim(); +}; + +const assertCustomerPathPrefix = (pathPrefix, expectedCustomerId) => { + if ( + typeof expectedCustomerId !== 'string' + || !CUSTOMER_ID_PATTERN.test(expectedCustomerId) + || pathPrefix !== `/customer/${expectedCustomerId}` + ) { + throw new Error('CTF_CUSTOMER_PATH_PREFIX_MISMATCH'); + } + return pathPrefix; +}; + +const requestUrl = (baseUrl, pathPrefix, pathname) => + `${baseUrl}${pathPrefix}${pathname}`; + +const collectDiagnosticSecrets = (value, secrets = [], seen = new Set()) => { + if (typeof value === 'string') { + if (value.length >= 4) secrets.push(value); + return secrets; + } + if (!value || typeof value !== 'object' || seen.has(value)) return secrets; + seen.add(value); + if (Array.isArray(value)) { + for (const entry of value) collectDiagnosticSecrets(entry, secrets, seen); + } else { + for (const entry of Object.values(value)) collectDiagnosticSecrets(entry, secrets, seen); + } + return secrets; +}; + +const redactDiagnosticText = (value, secrets = []) => { + let text = typeof value === 'string' ? value : String(value ?? ''); + text = text + .replace(/\bbearer\s+[^\s,'"}]+/gi, 'Bearer [REDACTED]') + .replace(/\b(postgres(?:ql)?):\/\/[^@\s/]+@/gi, '$1://[REDACTED]@') + .replace( + /((?:password|passwd|pwd|token|secret|api[-_]?key|authorization)\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;]+)/gi, + '$1[REDACTED]', + ); + for (const secret of [...new Set(secrets)].sort((left, right) => right.length - left.length)) { + text = text.split(secret).join('[REDACTED]'); + } + return text.replace(/\s+/g, ' ').trim().slice(0, DIAGNOSTIC_TEXT_LIMIT); +}; + +const diagnosticCode = (value, fallback) => { + const normalized = redactDiagnosticText(value ?? fallback) + .replace(/[^A-Za-z0-9_.-]+/g, '_') + .slice(0, 80); + return normalized || fallback; +}; + +const diagnosticRequestTarget = (value) => { + try { + const parsed = new URL(value); + return `${parsed.protocol}//${parsed.host}${parsed.pathname}`; + } catch { + return 'invalid-url'; + } +}; + +const graphqlOperationName = (operation) => { + const declaredName = typeof operation?.name === 'string' ? operation.name : null; + const parsedName = typeof operation?.query === 'string' + ? /\b(?:query|mutation|subscription)\s+([_A-Za-z][_0-9A-Za-z]*)/.exec(operation.query)?.[1] + : null; + return diagnosticCode(declaredName ?? parsedName, 'anonymous-operation'); +}; + +const requestJson = async (url, { + method = 'GET', + body, + token, + headers = {}, + fetchImpl = fetch, +} = {}) => { + const secrets = collectDiagnosticSecrets({ body, token, headers }); + const target = diagnosticRequestTarget(url); + let response; + try { + response = await fetchImpl(url, { + method, + redirect: 'error', + headers: { + ...(body === undefined ? {} : { 'content-type': 'application/json' }), + ...(token ? { authorization: `Bearer ${token}` } : {}), + ...headers, + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + } catch (error) { + const detail = redactDiagnosticText( + error instanceof Error ? error.message : error, + secrets, + ) || 'transport-error'; + throw new Error(`CTF_HTTP_REQUEST_FAILED:${target}:${detail}`); + } + let parsed = null; + try { + parsed = await response.json(); + } catch { + parsed = null; + } + if (!response.ok) { + const code = diagnosticCode(parsed?.error?.code, `HTTP_${response.status}`); + const detail = redactDiagnosticText(parsed?.error?.message ?? '', secrets); + throw new Error( + `CTF_HTTP_FAILURE:${code}:status=${response.status}:target=${target}` + + (detail ? `:message=${detail}` : ''), + ); + } + return parsed; +}; + +const postGraphql = async ( + baseUrl, + pathPrefix, + tenantId, + operation, + fetchImpl = fetch, +) => { + const body = await requestJson(requestUrl( + baseUrl, + pathPrefix, + `/tenant/${tenantId}/graphql`, + ), { + method: 'POST', + headers: { 'accept-language': 'es' }, + body: { + query: operation.query, + variables: operation.variables ?? {}, + }, + fetchImpl, + }); + if (Array.isArray(body?.errors) && body.errors.length > 0) { + const first = body.errors[0] ?? {}; + const code = diagnosticCode(first?.extensions?.code, 'GRAPHQL_ERROR'); + const operationName = graphqlOperationName(operation); + const secrets = collectDiagnosticSecrets(operation?.variables); + const detail = redactDiagnosticText(first?.message ?? '', secrets) || 'no-message'; + const errorPath = Array.isArray(first?.path) + ? first.path.map((entry) => diagnosticCode(entry, 'unknown')).join('.') + : 'none'; + const location = Array.isArray(first?.locations) && first.locations.length > 0 + ? `${Number(first.locations[0]?.line) || 0}:${Number(first.locations[0]?.column) || 0}` + : 'none'; + throw new Error( + `CTF_GRAPHQL_FAILURE:${diagnosticCode(tenantId, 'unknown-tenant')}:${code}` + + `:operation=${operationName}:path=${errorPath}:location=${location}` + + `:errors=${body.errors.length}:message=${detail}`, + ); + } + if (!body || typeof body !== 'object' || !('data' in body)) { + throw new Error(`CTF_GRAPHQL_RESPONSE_INVALID:${tenantId}`); + } + return body; +}; + +const assertPhysicalDatabaseIdentity = ( + observed, + expectedPhysicalDatabaseIdentity, + label, +) => { + const expected = requirePhysicalDatabaseIdentity(expectedPhysicalDatabaseIdentity); + if (observed !== expected) { + throw new Error( + `CTF_PHYSICAL_DATABASE_IDENTITY_MISMATCH:${diagnosticCode(label, 'unknown')}`, + ); + } +}; + +const control = async ( + baseUrl, + pathPrefix, + token, + action, + tenant, + expectedPhysicalDatabaseIdentity, + fetchImpl, +) => { + const response = await requestJson(requestUrl(baseUrl, pathPrefix, '/__ctf/control'), { + method: 'POST', + token, + body: { action, ...(tenant ? { tenant } : {}) }, + fetchImpl, + }); + assertPhysicalDatabaseIdentity( + response?.physicalDatabaseIdentity, + expectedPhysicalDatabaseIdentity, + `${action}:${tenant ?? 'fleet'}`, + ); + return response; +}; + +const identityOperation = { + query: 'query HostileTenantIdentity { tenantIdentity requestIdentity physicalDatabaseIdentity }', +}; + +const physicalIdentityOperation = { + query: 'query HostilePhysicalDatabaseIdentity { physicalDatabaseIdentity }', +}; + +const assertIdentity = (tenant, body, expectedPhysicalDatabaseIdentity) => { + const expectedRequestIdentity = `${tenant.token}:${tenant.databaseId}`; + if ( + body?.data?.tenantIdentity !== tenant.token + || body?.data?.requestIdentity !== expectedRequestIdentity + ) { + throw new Error(`CTF_TENANT_IDENTITY_MISMATCH:${tenant.id}`); + } + assertPhysicalDatabaseIdentity( + body?.data?.physicalDatabaseIdentity, + expectedPhysicalDatabaseIdentity, + `graphql-identity:${tenant.id}`, + ); + const serialized = JSON.stringify(body); + for (const other of TENANTS) { + if (other.id !== tenant.id && serialized.includes(other.token)) { + throw new Error(`CTF_CROSS_TENANT_TOKEN:${tenant.id}:${other.id}`); + } + } +}; + +const runHostileValidation = async ({ + baseUrl = 'http://127.0.0.1:3391', + pathPrefix = '', + expectedCustomerId, + expectedPhysicalDatabaseIdentity, + controlToken, + arm = 'local-complete-tenant', + mode = 'scoped-required', + fetchImpl = fetch, + outputFile, +} = {}) => { + if (typeof controlToken !== 'string' || Buffer.byteLength(controlToken) < 32) { + throw new Error('CTF_CONTROL_TOKEN_REQUIRED'); + } + expectedPhysicalDatabaseIdentity = requirePhysicalDatabaseIdentity( + expectedPhysicalDatabaseIdentity, + ); + baseUrl = assertLoopbackBaseUrl(baseUrl); + if (pathPrefix !== '' || expectedCustomerId !== undefined) { + pathPrefix = assertCustomerPathPrefix(pathPrefix, expectedCustomerId); + } + const startedAt = new Date().toISOString(); + const checks = []; + const record = (name, detail = {}) => checks.push({ name, passed: true, ...detail }); + const status = await requestJson(requestUrl(baseUrl, pathPrefix, '/__ctf/status'), { + fetchImpl, + }); + const contracts = validateServerStatus(status, { arm, mode }); + assertPhysicalDatabaseIdentity( + status.physicalDatabase, + expectedPhysicalDatabaseIdentity, + 'status', + ); + if (status.controlAvailable !== true) throw new Error('CTF_CONTROL_ENDPOINT_UNAVAILABLE'); + record('runtime-boundary', { + physicalIsolation: status.physicalIsolation, + sharedRuntimePool: status.sharedRuntimePool, + }); + + const fleet = makeFleet({ arm, buildContracts: contracts }); + for (const tenantTarget of fleet.tenants) { + const tenantId = tenantTarget.id.slice('complete-tenant-'.length); + for (const canary of tenantTarget.surfaces[0].canaries) { + const response = await postGraphql( + baseUrl, + pathPrefix, + tenantId, + canary, + fetchImpl, + ); + const result = evaluateCanaryResponse(canary, response); + if (!result.conclusive || result.violation) { + throw new Error( + `CTF_CANARY_FAILED:${tenantId}:${canary.name}:${result.detail ?? 'inconclusive'}`, + ); + } + const physicalResponse = await postGraphql( + baseUrl, + pathPrefix, + tenantId, + physicalIdentityOperation, + fetchImpl, + ); + assertPhysicalDatabaseIdentity( + physicalResponse?.data?.physicalDatabaseIdentity, + expectedPhysicalDatabaseIdentity, + `canary:${tenantId}:${canary.name}`, + ); + record(`canary:${tenantId}:${canary.name}`); + } + } + + for (const tenant of TENANTS) { + const poisoned = await control( + baseUrl, + pathPrefix, + controlToken, + 'poison', + tenant.id, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + if (poisoned?.ok !== true) throw new Error(`CTF_POISON_PROBE_FAILED:${tenant.id}`); + const response = await postGraphql( + baseUrl, + pathPrefix, + tenant.id, + identityOperation, + fetchImpl, + ); + assertIdentity(tenant, response, expectedPhysicalDatabaseIdentity); + record(`checkout-sanitization:${tenant.id}`); + + const rollback = await control( + baseUrl, + pathPrefix, + controlToken, + 'rollback-savepoint', + tenant.id, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + if (rollback?.ok !== true || rollback.observed !== tenant.databaseId) { + throw new Error(`CTF_ROLLBACK_SAVEPOINT_FAILED:${tenant.id}`); + } + const afterRollback = await postGraphql( + baseUrl, + pathPrefix, + tenant.id, + identityOperation, + fetchImpl, + ); + assertIdentity(tenant, afterRollback, expectedPhysicalDatabaseIdentity); + record(`rollback-savepoint:${tenant.id}`); + + const prepared = await control( + baseUrl, + pathPrefix, + controlToken, + 'prepared-reset', + tenant.id, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + if ( + prepared?.ok !== true + || prepared.first !== tenant.token + || prepared.second !== `${tenant.token}:${tenant.databaseId}` + || prepared.runtimeRole !== status.runtimeBindings?.[tenant.id]?.role + || prepared.backend?.exact !== true + || prepared.backend?.expected !== ( + status.runtimePoolMaxUses === 1 ? 'rotated-client' : 'same-client' + ) + || prepared.backend?.observed !== prepared.backend?.expected + || !Number.isSafeInteger(prepared.backend?.firstBackendPid) + || !Number.isSafeInteger(prepared.backend?.secondBackendPid) + ) { + throw new Error(`CTF_PREPARED_RESET_FAILED:${tenant.id}`); + } + record(`prepared-statement-reset:${tenant.id}`, { + backendBehavior: prepared.backend.observed, + firstBackendPid: prepared.backend.firstBackendPid, + secondBackendPid: prepared.backend.secondBackendPid, + }); + + const badRole = await control( + baseUrl, + pathPrefix, + controlToken, + 'bad-role-expected-failure', + tenant.id, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + if ( + badRole?.ok !== true + || badRole.rejectedCode !== 'GRAPHILE_UNSAFE_RUNTIME_ROLE' + ) { + throw new Error(`CTF_BAD_ROLE_ACCEPTED:${tenant.id}`); + } + record(`bad-role-expected-failure:${tenant.id}`); + } + + const driftTenant = TENANTS[0]; + let driftApplied = false; + try { + const applied = await control( + baseUrl, + pathPrefix, + controlToken, + 'drift-apply', + driftTenant.id, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + if (applied?.ok !== true) throw new Error('CTF_SCHEMA_DRIFT_APPLY_FAILED'); + driftApplied = true; + const driftResponse = await postGraphql(baseUrl, pathPrefix, driftTenant.id, { + query: 'query DriftApplied { schemaEpoch physicalDatabaseIdentity __type(name: "Document") { fields { name } } }', + }, fetchImpl); + const driftFields = driftResponse?.data?.__type?.fields?.map((field) => field.name) ?? []; + if (driftResponse?.data?.schemaEpoch !== 2 || !driftFields.includes('driftProbe')) { + throw new Error('CTF_SCHEMA_DRIFT_NOT_REBUILT'); + } + assertPhysicalDatabaseIdentity( + driftResponse?.data?.physicalDatabaseIdentity, + expectedPhysicalDatabaseIdentity, + 'schema-drift-applied', + ); + record('schema-drift-apply-and-rebuild'); + } finally { + if (driftApplied) { + const reverted = await control( + baseUrl, + pathPrefix, + controlToken, + 'drift-revert', + driftTenant.id, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + if (reverted?.ok !== true) throw new Error('CTF_SCHEMA_DRIFT_REVERT_FAILED'); + } + } + const revertedResponse = await postGraphql(baseUrl, pathPrefix, driftTenant.id, { + query: 'query DriftReverted { schemaEpoch physicalDatabaseIdentity __type(name: "Document") { fields { name } } }', + }, fetchImpl); + const revertedFields = revertedResponse?.data?.__type?.fields?.map((field) => field.name) ?? []; + if (revertedResponse?.data?.schemaEpoch !== 1 || revertedFields.includes('driftProbe')) { + throw new Error('CTF_SCHEMA_DRIFT_REVERT_NOT_REBUILT'); + } + assertPhysicalDatabaseIdentity( + revertedResponse?.data?.physicalDatabaseIdentity, + expectedPhysicalDatabaseIdentity, + 'schema-drift-reverted', + ); + record('schema-drift-revert-and-rebuild'); + + const beforeConcurrent = await requestJson( + requestUrl(baseUrl, pathPrefix, '/__ctf/status'), + { fetchImpl }, + ); + assertPhysicalDatabaseIdentity( + beforeConcurrent?.physicalDatabase, + expectedPhysicalDatabaseIdentity, + 'before-concurrent-rebuild-status', + ); + await control( + baseUrl, + pathPrefix, + controlToken, + 'invalidate-all', + null, + expectedPhysicalDatabaseIdentity, + fetchImpl, + ); + await Promise.all(TENANTS.map(async (tenant) => { + const response = await postGraphql( + baseUrl, + pathPrefix, + tenant.id, + identityOperation, + fetchImpl, + ); + assertIdentity(tenant, response, expectedPhysicalDatabaseIdentity); + })); + const afterConcurrent = await requestJson( + requestUrl(baseUrl, pathPrefix, '/__ctf/status'), + { fetchImpl }, + ); + assertPhysicalDatabaseIdentity( + afterConcurrent?.physicalDatabase, + expectedPhysicalDatabaseIdentity, + 'after-concurrent-rebuild-status', + ); + if (afterConcurrent?.builds?.maxConcurrent !== 1) { + throw new Error(`CTF_BUILD_SERIALIZATION_FAILED:${afterConcurrent?.builds?.maxConcurrent}`); + } + for (const tenant of TENANTS) { + const before = beforeConcurrent?.builds?.byTenant?.[tenant.id] ?? 0; + const after = afterConcurrent?.builds?.byTenant?.[tenant.id] ?? 0; + if (after !== before + 1) { + throw new Error(`CTF_CONCURRENT_REBUILD_COUNT_FAILED:${tenant.id}:${before}:${after}`); + } + } + record('concurrent-build-serialization', { maxConcurrentBuilds: 1 }); + + for (let iteration = 0; iteration < 10; iteration += 1) { + for (const tenant of TENANTS) { + const response = await postGraphql( + baseUrl, + pathPrefix, + tenant.id, + identityOperation, + fetchImpl, + ); + assertIdentity(tenant, response, expectedPhysicalDatabaseIdentity); + } + } + record('prepared-and-connection-reuse', { rounds: 10, crossTenantTokens: 0 }); + + const report = { + version: 2, + fixture: 'complete-tenant-abc-v1', + startedAt, + endedAt: new Date().toISOString(), + arm, + mode, + pathPrefix, + expectedCustomerId: expectedCustomerId ?? null, + physicalDatabaseIdentity: expectedPhysicalDatabaseIdentity, + passed: true, + customerQualified: false, + customerQualificationReason: 'hostile validation alone does not satisfy provider and workload gates', + checks, + }; + if (outputFile) atomicWriteJson(outputFile, report); + return report; +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + const output = args.output + ? path.resolve(requireString(args, 'output')) + : path.join(FIXTURE_DIR, 'generated', 'hostile-validation.json'); + const report = await runHostileValidation({ + baseUrl: requireString(args, 'base-url', 'http://127.0.0.1:3391'), + pathPrefix: args['path-prefix'] === undefined + ? '' + : requireString(args, 'path-prefix'), + expectedCustomerId: args['customer-id'], + expectedPhysicalDatabaseIdentity: requireString( + args, + 'expected-physical-database-identity', + ), + controlToken: process.env.CTF_CONTROL_TOKEN, + arm: requireString(args, 'arm', 'local-complete-tenant'), + mode: requireString(args, 'mode', 'scoped-required'), + outputFile: output, + }); + process.stdout.write(`${JSON.stringify(report)}\n`); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + assertCustomerPathPrefix, + assertIdentity, + assertPhysicalDatabaseIdentity, + control, + diagnosticRequestTarget, + identityOperation, + postGraphql, + redactDiagnosticText, + requestJson, + runHostileValidation, +}; diff --git a/research/graphile-density/complete-tenant-fixture/hostile-validation.test.cjs b/research/graphile-density/complete-tenant-fixture/hostile-validation.test.cjs new file mode 100644 index 0000000000..5992541e97 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/hostile-validation.test.cjs @@ -0,0 +1,187 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); + +const { TENANTS } = require('./lib.cjs'); +const { + assertCustomerPathPrefix, + assertIdentity, + assertPhysicalDatabaseIdentity, + postGraphql, + requestJson, + runHostileValidation, +} = require('./hostile-validation.cjs'); + +const response = (body, { ok = true, status = 200 } = {}) => ({ + ok, + status, + json: async () => body, +}); + +test('GraphQL helper rejects transport, GraphQL, and malformed success responses', async () => { + let requestOptions; + await requestJson('http://127.0.0.1/no-redirect', { + fetchImpl: async (_url, options) => { + requestOptions = options; + return response({ ok: true }); + }, + }); + assert.equal(requestOptions.redirect, 'error'); + await assert.rejects( + () => requestJson('http://127.0.0.1/failure', { + fetchImpl: async () => response({ error: { code: 'DENIED' } }, { ok: false, status: 403 }), + }), + /CTF_HTTP_FAILURE:DENIED/, + ); + await assert.rejects( + () => postGraphql('http://127.0.0.1', '', 'a', { query: 'query Test { x }' }, async () => + response({ errors: [{ message: 'unsupported field' }] }) + ), + /CTF_GRAPHQL_FAILURE:a:GRAPHQL_ERROR:operation=Test:path=none:location=none:errors=1:message=unsupported field/, + ); + await assert.rejects( + () => postGraphql('http://127.0.0.1', '', 'a', { query: 'query Test { x }' }, async () => + response({ ok: true }) + ), + /CTF_GRAPHQL_RESPONSE_INVALID:a/, + ); +}); + +test('failure diagnostics identify the operation and redact request secrets', async () => { + const variableSecret = 'fixture-variable-secret-value'; + const bearerSecret = 'fixture-bearer-credential-value'; + const databaseSecret = 'fixture-database-password'; + let graphQlError; + try { + await postGraphql('http://127.0.0.1', '', 'a', { + name: 'localized-post-read', + query: 'query LocalizedPostRead($value: String!) { x(value: $value) }', + variables: { value: variableSecret }, + }, async () => response({ + errors: [{ + message: `invalid ${variableSecret}; Bearer ${bearerSecret}; postgresql://runtime:${databaseSecret}@127.0.0.1/db`, + path: ['posts', 'nodes', 0, 'localeStrings'], + locations: [{ line: 2, column: 7 }], + extensions: { code: 'GRAPHQL_VALIDATION_FAILED' }, + }], + })); + } catch (error) { + graphQlError = error; + } + assert.ok(graphQlError instanceof Error); + assert.match( + graphQlError.message, + /operation=localized-post-read:path=posts\.nodes\.0\.localeStrings:location=2:7/, + ); + assert.match(graphQlError.message, /\[REDACTED\]/); + assert.doesNotMatch( + graphQlError.message, + new RegExp([variableSecret, bearerSecret, databaseSecret].join('|')), + ); + + const controlSecret = 'fixture-control-token-value'; + let httpError; + try { + await requestJson( + 'http://runtime:fixture-url-password@127.0.0.1/failure?token=fixture-query-token', + { + token: controlSecret, + fetchImpl: async () => response({ + error: { code: 'DENIED', message: `authorization=${controlSecret}` }, + }, { ok: false, status: 403 }), + }, + ); + } catch (error) { + httpError = error; + } + assert.ok(httpError instanceof Error); + assert.match(httpError.message, /CTF_HTTP_FAILURE:DENIED:status=403:target=http:\/\/127\.0\.0\.1\/failure/); + assert.doesNotMatch( + httpError.message, + /fixture-control-token-value|fixture-url-password|fixture-query-token/, + ); +}); + +test('identity oracle detects wrong identities and any foreign tenant token', () => { + const tenant = TENANTS[0]; + const physicalDatabaseIdentity = 'fixture_database_a'; + assert.doesNotThrow(() => assertIdentity(tenant, { + data: { + tenantIdentity: tenant.token, + requestIdentity: `${tenant.token}:${tenant.databaseId}`, + physicalDatabaseIdentity, + }, + }, physicalDatabaseIdentity)); + assert.throws( + () => assertIdentity(tenant, { + data: { + tenantIdentity: tenant.token, + requestIdentity: `${tenant.token}:${tenant.databaseId}`, + physicalDatabaseIdentity, + unexpected: TENANTS[1].token, + }, + }, physicalDatabaseIdentity), + /CTF_CROSS_TENANT_TOKEN:a:b/, + ); + assert.throws( + () => assertIdentity(tenant, { + data: { + tenantIdentity: 'guc-mismatch', + requestIdentity: null, + physicalDatabaseIdentity, + }, + }, physicalDatabaseIdentity), + /CTF_TENANT_IDENTITY_MISMATCH:a/, + ); + assert.throws( + () => assertIdentity(tenant, { + data: { + tenantIdentity: tenant.token, + requestIdentity: `${tenant.token}:${tenant.databaseId}`, + physicalDatabaseIdentity: 'fixture_database_b', + }, + }, physicalDatabaseIdentity), + /CTF_PHYSICAL_DATABASE_IDENTITY_MISMATCH:graphql-identity_a/, + ); +}); + +test('mounted hostile routes require the exact manifest customer path', () => { + assert.equal( + assertCustomerPathPrefix( + '/customer/physical-customer-0001', + 'physical-customer-0001', + ), + '/customer/physical-customer-0001', + ); + for (const value of [ + '/customer/physical-customer-0002', + '/customer/physical-customer-0001/', + '/customer/%70hysical-customer-0001', + '/customer/physical-customer-0001?tenant=other', + ]) { + assert.throws( + () => assertCustomerPathPrefix(value, 'physical-customer-0001'), + /CTF_CUSTOMER_PATH_PREFIX_MISMATCH/, + ); + } +}); + +test('physical identity is mandatory before hostile validation performs I/O', async () => { + assert.throws( + () => assertPhysicalDatabaseIdentity('database-a', undefined, 'probe'), + /CTF_EXPECTED_PHYSICAL_DATABASE_IDENTITY_REQUIRED/, + ); + let requested = false; + await assert.rejects(() => runHostileValidation({ + baseUrl: 'http://127.0.0.1:3391', + pathPrefix: '/customer/physical-customer-0001', + expectedCustomerId: 'physical-customer-0001', + controlToken: 'c'.repeat(32), + fetchImpl: async () => { + requested = true; + return response({}); + }, + }), /CTF_EXPECTED_PHYSICAL_DATABASE_IDENTITY_REQUIRED/); + assert.equal(requested, false); +}); diff --git a/research/graphile-density/complete-tenant-fixture/lib.cjs b/research/graphile-density/complete-tenant-fixture/lib.cjs new file mode 100644 index 0000000000..d79535ccb9 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/lib.cjs @@ -0,0 +1,718 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const FIXTURE_DIR = __dirname; +const REPO_ROOT = path.resolve(FIXTURE_DIR, '../../..'); +const LOOPBACK_URL_HOSTS = new Set(['127.0.0.1', '[::1]', 'localhost']); + +const TENANTS = Object.freeze([ + Object.freeze({ + id: 'a', + schema: 'ctf_a', + runtimeRoleArgument: 'runtime-role-a', + runtimePasswordEnvironment: 'CTF_RUNTIME_A_PGPASSWORD', + token: 'tenant-a-canary', + databaseId: '10000000-0000-4000-8000-00000000000a', + apiId: '20000000-0000-4000-8000-00000000000a', + metadataField: 'metadataA', + foreignSchema: 'ctf_b', + foreignToken: 'tenant-b-canary', + }), + Object.freeze({ + id: 'b', + schema: 'ctf_b', + runtimeRoleArgument: 'runtime-role-b', + runtimePasswordEnvironment: 'CTF_RUNTIME_B_PGPASSWORD', + token: 'tenant-b-canary', + databaseId: '10000000-0000-4000-8000-00000000000b', + apiId: '20000000-0000-4000-8000-00000000000b', + metadataField: 'metadataB', + foreignSchema: 'ctf_c', + foreignToken: 'tenant-c-canary', + }), + Object.freeze({ + id: 'c', + schema: 'ctf_c', + runtimeRoleArgument: 'runtime-role-c', + runtimePasswordEnvironment: 'CTF_RUNTIME_C_PGPASSWORD', + token: 'tenant-c-canary', + databaseId: '10000000-0000-4000-8000-00000000000c', + apiId: '20000000-0000-4000-8000-00000000000c', + metadataField: 'metadataC', + foreignSchema: 'ctf_a', + foreignToken: 'tenant-a-canary', + }), +]); + +const REQUIRED_CAPABILITIES = Object.freeze([ + 'graphile-generated', + 'i18n', + 'llm-deterministic', + 'rag-deterministic', + 'bm25', + 'tsvector', + 'trigram', + 'vector', + 'postgis', + 'ltree', + 'uploads-storage-presign-only', + 'bulk-mutations', + 'realtime-tagged-write', + 'function-bindings', + 'security-session', +]); + +const REQUIRED_CANARIES = Object.freeze([ + 'cross-schema-identifiers', + 'metadata', + 'functions', + 'sequences', + 'prepared-statement-reuse', + 'poisoned-gucs', + 'rollback-savepoints', + 'plugin-raw-sql', + 'owner-bypass-role', + 'schema-drift', + 'cache-invalidation', + 'concurrent-builds', + 'connection-reuse', +]); + +const parseArgs = (argv) => { + const result = { positional: [] }; + for (let index = 0; index < argv.length; index += 1) { + const value = argv[index]; + if (!value.startsWith('--')) { + result.positional.push(value); + continue; + } + const key = value.slice(2); + const next = argv[index + 1]; + if (!next || next.startsWith('--')) { + result[key] = true; + } else { + result[key] = next; + index += 1; + } + } + return result; +}; + +const requireString = (args, name, fallback) => { + const value = args[name] ?? fallback; + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`CTF_ARGUMENT_REQUIRED:${name}`); + } + return value.trim(); +}; + +const assertLoopbackBaseUrl = (value) => { + let parsed; + try { + parsed = new URL(value); + } catch { + throw new Error('CTF_LOOPBACK_BASE_URL_REQUIRED'); + } + if ( + parsed.protocol !== 'http:' + || !LOOPBACK_URL_HOSTS.has(parsed.hostname) + || parsed.username + || parsed.password + || (parsed.pathname !== '/' && parsed.pathname !== '') + || parsed.search + || parsed.hash + ) { + throw new Error('CTF_LOOPBACK_BASE_URL_REQUIRED'); + } + return parsed.origin; +}; + +const parsePositiveInteger = (value, label) => { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`CTF_INVALID_POSITIVE_INTEGER:${label}`); + } + return parsed; +}; + +const validateIntrospectionClientReleaseMode = (value = 'destroy') => { + if (value !== 'reuse' && value !== 'destroy') { + throw new Error(`CTF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:${value}`); + } + return value; +}; + +const fileSha256 = (file) => crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); + +const readManifest = () => { + const manifest = JSON.parse(fs.readFileSync(path.join(FIXTURE_DIR, 'coverage-manifest.json'), 'utf8')); + validateManifest(manifest); + return manifest; +}; + +const uniqueStrings = (values, label) => { + if (!Array.isArray(values) || values.length === 0 || values.some((value) => typeof value !== 'string' || !value)) { + throw new Error(`CTF_MANIFEST_INVALID:${label}`); + } + if (new Set(values).size !== values.length) throw new Error(`CTF_MANIFEST_DUPLICATE:${label}`); +}; + +const validateManifest = (manifest) => { + if (!manifest || manifest.version !== 1 || manifest.fixture !== 'complete-tenant-abc-v1') { + throw new Error('CTF_MANIFEST_INVALID:identity'); + } + if ( + manifest.performanceClaimsAllowed !== false + || manifest.productionQualificationImplemented !== false + || manifest.runtimeIsolation?.model !== 'dedicated-login-and-pool-per-tenant' + || manifest.runtimeIsolation?.sharedRuntimePool !== false + || manifest.runtimeIsolation?.runtimeSchemaDriftControl !== false + || manifest.runtimeIsolation?.checkoutSanitationRequired !== true + ) { + throw new Error('CTF_MANIFEST_INVALID:failClosedBoundary'); + } + uniqueStrings(manifest.tenants, 'tenants'); + uniqueStrings(manifest.surfacesPerTenant, 'surfacesPerTenant'); + uniqueStrings(manifest.hostileCanaries, 'hostileCanaries'); + const capabilities = manifest.localCapabilities?.map((entry) => entry.id); + uniqueStrings(capabilities, 'localCapabilities'); + for (const capability of REQUIRED_CAPABILITIES) { + if (!capabilities.includes(capability)) throw new Error(`CTF_MANIFEST_MISSING_CAPABILITY:${capability}`); + } + for (const canary of REQUIRED_CANARIES) { + if (!manifest.hostileCanaries.includes(canary)) throw new Error(`CTF_MANIFEST_MISSING_CANARY:${canary}`); + } + for (const gate of manifest.externalProviderGates ?? []) { + if (gate.status !== 'blocking' || !Array.isArray(gate.requiredArguments) || gate.requiredArguments.length === 0) { + throw new Error(`CTF_MANIFEST_INVALID_GATE:${gate.id ?? 'unknown'}`); + } + } +}; + +const assertProviderGates = (manifest, qualificationClass, args) => { + if (qualificationClass === 'offline-research') { + return { + customerQualified: false, + unresolved: manifest.externalProviderGates.map((gate) => gate.id), + }; + } + if (qualificationClass !== 'production') { + throw new Error(`CTF_UNKNOWN_QUALIFICATION_CLASS:${qualificationClass}`); + } + const missing = []; + for (const gate of manifest.externalProviderGates) { + for (const argument of gate.requiredArguments) { + if (typeof args[argument] !== 'string' || !args[argument].trim()) { + missing.push(`${gate.id}:${argument}`); + } + } + } + if (missing.length > 0) { + throw new Error(`CTF_EXTERNAL_PROVIDER_GATES_UNSATISFIED:${missing.join(',')}`); + } + return { customerQualified: true, unresolved: [] }; +}; + +const operation = (name, capability, query, variables, weight = 1) => ({ + name, + capability, + weight, + query, + ...(variables ? { variables } : {}), +}); + +const canary = (name, query, requiredMatches, forbiddenMatches, variables) => ({ + name, + query, + ...(variables ? { variables } : {}), + requiredMatches, + forbiddenMatches, +}); + +const operationsFor = (tenant) => [ + operation( + 'generated-document-read', + 'graphile-generated', + 'query GeneratedDocumentRead { documents(first: 1) { nodes { id tenantId title } } }', + ), + operation( + 'localized-post-read', + 'i18n', + 'query LocalizedPostRead { posts(first: 1, where: { id: { equalTo: 1 } }) { nodes { tenantId localeStrings { langCode title body } } } }', + ), + operation( + 'deterministic-embed', + 'llm-deterministic', + 'query DeterministicEmbed { embedText(text: "tenant fixture") { vector dimensions } }', + ), + operation( + 'deterministic-rag', + 'rag-deterministic', + 'query DeterministicRag { ragQuery(prompt: "machine learning tenant fixture", contextLimit: 2) { answer tokensUsed sources { content similarity tableName parentId } } }', + undefined, + 0.5, + ), + operation( + 'bm25-search', + 'bm25', + 'query Bm25Search { documents(where: { bm25Body: { query: "machine learning intelligence" } }) { nodes { tenantId title bodyBm25Score } } }', + ), + operation( + 'tsvector-search', + 'tsvector', + 'query TsvectorSearch { documents(where: { tsvTsv: "machine learning" }) { nodes { tenantId title tsvRank } } }', + ), + operation( + 'trigram-search', + 'trigram', + 'query TrigramSearch { documents(where: { trgmTitle: { value: "Machne Lerning", threshold: 0.05 } }) { nodes { tenantId title titleTrgmSimilarity } } }', + ), + operation( + 'vector-search', + 'vector', + 'query VectorSearch { documents(where: { vectorEmbedding: { vector: [1, 0, 0], metric: COSINE } }) { nodes { tenantId title embeddingVectorDistance } } }', + ), + operation( + 'postgis-read', + 'postgis', + 'query PostgisRead { documents(first: 1) { nodes { tenantId location { geojson } } } }', + ), + operation( + 'ltree-filter', + 'ltree', + 'query LtreeFilter { documents(where: { path: { within: "/root" } }) { nodes { tenantId title path } } }', + ), + operation( + 'presigned-upload', + 'uploads-storage-presign-only', + 'mutation PresignedUpload($input: UploadAppFileInput!) { uploadAppFile(input: $input) { fileId key deduplicated expiresAt uploadUrl } }', + { + input: { + bucketKey: 'private', + contentHash: crypto.createHash('sha256').update(`complete-tenant-${tenant.id}`).digest('hex'), + contentType: 'text/plain', + size: 32, + filename: `${tenant.id}.txt`, + }, + }, + 0.25, + ), + operation( + 'bulk-upsert', + 'bulk-mutations', + 'mutation BulkUpsert($name: String!) { bulkUpsertBulkItems(input: { values: [{ name: $name, quantity: 1 }], onConflict: { constraint: BULK_ITEMS_NAME_KEY } }) { affectedCount } }', + { name: `${tenant.token}-bulk` }, + 0.5, + ), + operation( + 'realtime-tagged-update', + 'realtime-tagged-write', + 'mutation RealtimeTaggedUpdate($payload: String!) { updateRealtimeItem(input: { id: 1, realtimeItemPatch: { payload: $payload } }) { realtimeItem { id tenantId payload } } }', + { payload: `${tenant.token}-realtime` }, + 0.5, + ), + operation( + 'bound-function-invocation', + 'function-bindings', + 'mutation BoundFunctionInvocation($payload: JSON!) { fixtureTask(input: { payload: $payload }) { invocationId status } }', + { payload: { tenant: tenant.id, source: 'complete-tenant-fixture' } }, + 0.25, + ), + operation( + 'security-context-read', + 'security-session', + 'query SecurityContextRead { requestIdentity }', + undefined, + 0.25, + ), +]; + +const tokenMatch = (pathValue, value) => ({ path: pathValue, value }); + +const canariesFor = (tenant) => { + const ownIdentity = `${tenant.token}:${tenant.databaseId}`; + const identityQuery = 'query TenantIdentity { tenantIdentity }'; + return [ + canary( + 'cross-schema-identifiers', + 'query CrossSchemaIdentifier($schemaName: String!) { foreignAccessState(targetSchema: $schemaName) }', + [tokenMatch('/data/foreignAccessState', 'acl-denied')], + [tokenMatch('/data/foreignAccessState', 'visible')], + { schemaName: tenant.foreignSchema }, + ), + canary( + 'metadata', + 'query MetadataIsolation { __type(name: "Query") { fields { name } } }', + [tokenMatch('/data/__type/fields/*/name', tenant.metadataField)], + TENANTS.filter((candidate) => candidate.id !== tenant.id) + .map((candidate) => tokenMatch('/data/__type/fields/*/name', candidate.metadataField)), + ), + canary( + 'functions', + identityQuery, + [tokenMatch('/data/tenantIdentity', tenant.token)], + [tokenMatch('/data/tenantIdentity', tenant.foreignToken)], + ), + canary( + 'sequences', + 'mutation SequenceIsolation($secret: String!) { createTenantCanary(input: { tenantCanary: { secret: $secret } }) { tenantCanary { tenantId secret } } }', + [tokenMatch('/data/createTenantCanary/tenantCanary/tenantId', tenant.token)], + [tokenMatch('/data/createTenantCanary/tenantCanary/tenantId', tenant.foreignToken)], + { secret: `${tenant.token}-sequence` }, + ), + canary( + 'prepared-statement-reuse', + identityQuery, + [tokenMatch('/data/tenantIdentity', tenant.token)], + [tokenMatch('/data/tenantIdentity', tenant.foreignToken)], + ), + canary( + 'poisoned-gucs', + 'query RequestIdentity { requestIdentity }', + [tokenMatch('/data/requestIdentity', ownIdentity)], + [tokenMatch('/data/requestIdentity', `${tenant.foreignToken}:${tenant.databaseId}`)], + ), + canary( + 'rollback-savepoints', + 'query RequestIdentityAfterRollbackProbe { requestIdentity }', + [tokenMatch('/data/requestIdentity', ownIdentity)], + [tokenMatch('/data/requestIdentity', `poisoned-savepoint:${tenant.databaseId}`)], + ), + canary( + 'plugin-raw-sql', + 'query I18nRawSql { posts(first: 1, where: { id: { equalTo: 1 } }) { nodes { localeStrings { title } } } }', + [tokenMatch('/data/posts/nodes/0/localeStrings/title', `${tenant.token} español`)], + [tokenMatch('/data/posts/nodes/0/localeStrings/title', `${tenant.foreignToken} español`)], + ), + canary( + 'owner-bypass-role', + 'query RuntimeRoleSafety { runtimeRoleSafe }', + [tokenMatch('/data/runtimeRoleSafe', true)], + [tokenMatch('/data/runtimeRoleSafe', false)], + ), + canary( + 'schema-drift', + 'query SchemaEpoch { schemaEpoch }', + [tokenMatch('/data/schemaEpoch', 1)], + [tokenMatch('/data/schemaEpoch', 0)], + ), + canary( + 'cache-invalidation', + identityQuery, + [tokenMatch('/data/tenantIdentity', tenant.token)], + [tokenMatch('/data/tenantIdentity', tenant.foreignToken)], + ), + canary( + 'concurrent-builds', + identityQuery, + [tokenMatch('/data/tenantIdentity', tenant.token)], + [tokenMatch('/data/tenantIdentity', tenant.foreignToken)], + ), + canary( + 'connection-reuse', + 'query RequestIdentity { requestIdentity }', + [tokenMatch('/data/requestIdentity', ownIdentity)], + [tokenMatch('/data/requestIdentity', `${tenant.foreignToken}:${tenant.databaseId}`)], + ), + ]; +}; + +const fallbackBuildContract = (arm, tenant) => + `ctf:unresolved:v1:${arm}:${tenant.id}:api`; + +const fallbackPoolIdentity = (arm, tenant) => + `ctf:unresolved-pool:v1:${arm}:${tenant.id}:api`; + +const makeFleet = ({ + arm = 'local-complete-tenant', + port = 3391, + buildContracts = {}, + runtimePoolIdentities = {}, + physicalDatabase = 'ctf-unresolved-physical-database', +} = {}) => ({ + version: 1, + tenants: TENANTS.map((tenant) => ({ + id: `complete-tenant-${tenant.id}`, + databases: [{ + id: tenant.databaseId, + physicalDatabase, + apis: [{ + id: tenant.apiId, + runtimePoolIdentity: runtimePoolIdentities[tenant.id] + ?? fallbackPoolIdentity(arm, tenant), + runtimePoolIdentities: { + [arm]: runtimePoolIdentities[tenant.id] + ?? fallbackPoolIdentity(arm, tenant), + }, + physicalSchemas: [tenant.schema], + routingLabels: [`ctf-${tenant.id}-api`], + realtime: false, + surfaces: ['api'], + }], + }], + surfaces: [{ + name: 'api', + buildContract: buildContracts[tenant.id] ?? fallbackBuildContract(arm, tenant), + buildContracts: { + [arm]: buildContracts[tenant.id] ?? fallbackBuildContract(arm, tenant), + }, + url: `http://127.0.0.1:{port}/tenant/${tenant.id}/graphql`, + headers: { 'accept-language': 'es' }, + warmup: operation('warm-tenant-identity', 'graphile-generated', 'query WarmTenantIdentity { tenantIdentity }'), + operations: operationsFor(tenant), + canaries: canariesFor(tenant), + }], + })), +}); + +const makePlan = ({ + arm = 'local-complete-tenant', + port = 3391, + postgresContainer, + commit, + durationSec = 900, + cwd = REPO_ROOT, + introspectionMode = 'scoped-required', + introspectionClientReleaseMode = 'destroy', + runtimeRoles, +} = {}) => { + if (!postgresContainer) throw new Error('CTF_ARGUMENT_REQUIRED:postgres-container'); + if (!commit) throw new Error('CTF_ARGUMENT_REQUIRED:commit'); + if (!['stock', 'scoped-required'].includes(introspectionMode)) { + throw new Error(`CTF_INTROSPECTION_MODE_INVALID:${introspectionMode}`); + } + validateIntrospectionClientReleaseMode(introspectionClientReleaseMode); + for (const tenant of TENANTS) { + if (typeof runtimeRoles?.[tenant.id] !== 'string' || !runtimeRoles[tenant.id].trim()) { + throw new Error(`CTF_ARGUMENT_REQUIRED:${tenant.runtimeRoleArgument}`); + } + } + return { + version: 1, + fleetFile: 'fleet.json', + artifactDir: '../artifacts', + arms: [{ + name: arm, + commit, + cwd, + command: [ + 'node', + path.join(FIXTURE_DIR, 'server.cjs'), + '--port', + '{port}', + '--arm', + arm, + '--mode', + '{mode}', + '--introspection-client-release-mode', + introspectionClientReleaseMode, + '--runtime-pool-max', + '1', + '--runtime-pool-max-uses', + 'unlimited', + ...TENANTS.flatMap((tenant) => [ + `--${tenant.runtimeRoleArgument}`, + runtimeRoles[tenant.id], + ]), + ], + port, + readinessUrl: `http://127.0.0.1:{port}/healthz`, + memoryUrl: `http://127.0.0.1:{port}/debug/memory`, + postgresContainer, + introspectionMode, + entrySha256: fileSha256(path.join(FIXTURE_DIR, 'server.cjs')), + lockfileSha256: fileSha256(path.join(REPO_ROOT, 'pnpm-lock.yaml')), + env: { + GRAPHILE_CACHE_MAX: '3', + GRAPHILE_CACHE_INSTANCE_HEAP_BYTES: String(64 * 1024 * 1024), + GRAPHILE_CACHE_SERVER_RESERVE_BYTES: String(256 * 1024 * 1024), + GRAPHILE_CACHE_BUILD_RESERVE_BYTES: String(768 * 1024 * 1024), + GRAPHILE_BUILD_MAX_CONCURRENCY: '1', + GRAPHILE_BUILD_CONCURRENCY: '1', + GRAPHILE_BUILD_QUEUE_MAX: '8', + PG_CACHE_MAX: '4', + PG_POOL_MAX: '1', + PG_POOL_MAX_USES: '0', + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: '100', + }, + }], + heapMiB: [4096], + tenantCountsByHeapMiB: { 4096: [3] }, + repetitions: 3, + runOrderSeed: 'complete-tenant-abc-v1', + requiredCapabilities: [...REQUIRED_CAPABILITIES], + requiredCanaries: [...REQUIRED_CANARIES], + workload: { + durationSec, + rpsPerTenant: 1, + minWorkloadRequestsPerSurface: 30, + requestTimeoutMs: 30000, + maxInFlight: 32, + canaryIntervalSec: 60, + warmupTimeoutMs: 180000, + warmupTimeoutPerSurfaceMs: 30000, + warmupConcurrency: 3, + }, + gates: { + maxErrorRate: 0.005, + maxP99Ms: 150, + maxPostWarmupHeapGrowthMiBPerHour: 5, + minMedianDensityImprovement: 0.15, + minAdditionalTenantsEveryRun: 1, + requireZeroBleed: true, + requireNoPostWarmupEvictions: true, + requireNoPostWarmupBuildRefusals: true, + requireNoPostWarmupBuilds: true, + requirePostgresMemoryTelemetry: true, + requireFreshPostgresRunAttestation: false, + requireRetainedMemoryCheckpoints: false, + requirePhysicalDatabaseTelemetry: false, + requireConclusiveCanaries: true, + requireCompletePeriodicCanaryCoverage: true, + requireConclusiveOperationOracles: false, + requireExplicitCustomerTopology: true, + requiredCacheAdmissionMode: 'evict-idle', + }, + }; +}; + +const decodePointerSegment = (segment) => segment + .replace(/~1/g, '/') + .replace(/~0/g, '~'); + +const jsonPointerValues = (root, pointer) => { + if (pointer === '') return [root]; + if (typeof pointer !== 'string' || !pointer.startsWith('/')) return []; + let values = [root]; + for (const rawSegment of pointer.slice(1).split('/')) { + const segment = decodePointerSegment(rawSegment); + const next = []; + for (const value of values) { + if (segment === '*') { + if (Array.isArray(value)) next.push(...value); + else if (value && typeof value === 'object') next.push(...Object.values(value)); + } else if (Array.isArray(value) && /^(0|[1-9]\d*)$/.test(segment)) { + const index = Number(segment); + if (index < value.length) next.push(value[index]); + } else if ( + value + && typeof value === 'object' + && Object.prototype.hasOwnProperty.call(value, segment) + ) { + next.push(value[segment]); + } + } + values = next; + if (values.length === 0) break; + } + return values; +}; + +const deepEqualJson = (left, right) => { + if (Object.is(left, right)) return true; + if (Array.isArray(left) && Array.isArray(right)) { + return left.length === right.length + && left.every((value, index) => deepEqualJson(value, right[index])); + } + if ( + left + && right + && typeof left === 'object' + && typeof right === 'object' + && !Array.isArray(left) + && !Array.isArray(right) + ) { + const leftKeys = Object.keys(left).sort(); + const rightKeys = Object.keys(right).sort(); + return deepEqualJson(leftKeys, rightKeys) + && leftKeys.every((key) => deepEqualJson(left[key], right[key])); + } + return false; +}; + +const evaluateCanaryResponse = (canary, responseBody) => { + const forbidden = canary.forbiddenMatches.find((match) => + jsonPointerValues(responseBody, match.path).some((value) => + deepEqualJson(value, match.value) + ) + ); + const missing = canary.requiredMatches.find((match) => + !jsonPointerValues(responseBody, match.path).some((value) => + deepEqualJson(value, match.value) + ) + ); + return { + conclusive: !missing, + violation: Boolean(forbidden), + ...(forbidden ? { detail: `forbidden match at '${forbidden.path}' was returned` } : {}), + ...(!forbidden && missing + ? { detail: `required match at '${missing.path}' was absent` } + : {}), + }; +}; + +const assertCredentialFree = (value) => { + const reject = () => { + throw new Error('CTF_ARTIFACT_CONTAINS_CREDENTIAL_MARKER'); + }; + const visit = (candidate, ancestors = new Set()) => { + if (typeof candidate === 'string') { + if ( + /\bbearer\s+[a-z0-9._~+/-]{16,}/i.test(candidate) + || /postgres(?:ql)?:\/\/[^:@/\s]+:[^@/\s]+@/i.test(candidate) + ) reject(); + return; + } + if (!candidate || typeof candidate !== 'object') return; + if (ancestors.has(candidate)) return; + const nextAncestors = new Set(ancestors).add(candidate); + if (Array.isArray(candidate)) { + for (const entry of candidate) visit(entry, nextAncestors); + return; + } + for (const [key, entry] of Object.entries(candidate)) { + if (/password|secretAccessKey|authorization|controlToken|observabilityToken|accessKeyId/i.test(key)) { + reject(); + } + visit(entry, nextAncestors); + } + }; + if (typeof value === 'string') { + try { + visit(JSON.parse(value)); + } catch (error) { + if (error?.message === 'CTF_ARTIFACT_CONTAINS_CREDENTIAL_MARKER') throw error; + visit(value); + } + } else { + visit(value); + } +}; + +module.exports = { + FIXTURE_DIR, + REPO_ROOT, + REQUIRED_CANARIES, + REQUIRED_CAPABILITIES, + TENANTS, + assertCredentialFree, + assertLoopbackBaseUrl, + assertProviderGates, + canariesFor, + evaluateCanaryResponse, + fallbackBuildContract, + fileSha256, + makeFleet, + makePlan, + operationsFor, + parseArgs, + parsePositiveInteger, + jsonPointerValues, + readManifest, + requireString, + validateIntrospectionClientReleaseMode, + validateManifest, +}; diff --git a/research/graphile-density/complete-tenant-fixture/lib.test.cjs b/research/graphile-density/complete-tenant-fixture/lib.test.cjs new file mode 100644 index 0000000000..efdcbb22b5 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/lib.test.cjs @@ -0,0 +1,165 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); + +const { + REQUIRED_CANARIES, + REQUIRED_CAPABILITIES, + TENANTS, + assertLoopbackBaseUrl, + assertProviderGates, + evaluateCanaryResponse, + jsonPointerValues, + makeFleet, + makePlan, + operationsFor, + readManifest, +} = require('./lib.cjs'); + +test('manifest is complete and production provider gates fail closed', () => { + const manifest = readManifest(); + assert.throws( + () => assertProviderGates(manifest, 'production', {}), + /CTF_EXTERNAL_PROVIDER_GATES_UNSATISFIED/, + ); + assert.deepEqual(assertProviderGates(manifest, 'offline-research', {}), { + customerQualified: false, + unresolved: ['ollama-real-semantic', 'object-storage-byte-roundtrip'], + }); +}); + +test('each A/B/C fleet member covers every capability and hostile canary', () => { + const fleet = makeFleet({ arm: 'test-arm', port: 3391 }); + assert.equal(fleet.tenants.length, TENANTS.length); + + const contracts = new Set(); + for (const tenant of fleet.tenants) { + assert.equal(tenant.databases.length, 1); + assert.equal(tenant.databases[0].apis.length, 1); + assert.deepEqual(tenant.databases[0].apis[0].surfaces, ['api']); + assert.equal(tenant.surfaces.length, 1); + const surface = tenant.surfaces[0]; + assert.equal(contracts.has(surface.buildContract), false); + contracts.add(surface.buildContract); + assert.deepEqual( + [...new Set(surface.operations.map((entry) => entry.capability))].sort(), + [...REQUIRED_CAPABILITIES].sort(), + ); + assert.deepEqual( + surface.canaries.map((entry) => entry.name).sort(), + [...REQUIRED_CANARIES].sort(), + ); + for (const entry of surface.canaries) { + assert.ok(entry.requiredMatches.length > 0, `${entry.name} has a positive oracle`); + assert.ok(entry.forbiddenMatches.length > 0, `${entry.name} has a negative oracle`); + } + } +}); + +test('fixture operations match the NoUniqueLookup GraphQL surface', () => { + const tenant = TENANTS[0]; + const operations = operationsFor(tenant); + const localizedRead = operations.find((entry) => entry.name === 'localized-post-read'); + const realtimeUpdate = operations.find((entry) => entry.name === 'realtime-tagged-update'); + const rawSqlCanary = makeFleet().tenants[0].surfaces[0].canaries + .find((entry) => entry.name === 'plugin-raw-sql'); + + assert.match(localizedRead.query, /posts\(first: 1, where: \{ id: \{ equalTo: 1 \} \}\)/); + assert.match(realtimeUpdate.query, /updateRealtimeItem\(input:/); + assert.doesNotMatch( + JSON.stringify({ operations, rawSqlCanary }), + /postById|updateRealtimeItemById/, + ); + assert.deepEqual(rawSqlCanary.requiredMatches, [{ + path: '/data/posts/nodes/0/localeStrings/title', + value: `${tenant.token} español`, + }]); +}); + +test('generated fleet is credential-free and does not call offline surrogates production', () => { + const serialized = JSON.stringify(makeFleet({ arm: 'test-arm', port: 3391 })); + assert.doesNotMatch(serialized, /password|secretAccessKey|authorization|bearer/i); + assert.match(serialized, /llm-deterministic/); + assert.match(serialized, /uploads-storage-presign-only/); +}); + +test('exact server build contracts replace unresolved fixture placeholders', () => { + const contracts = Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `graphile:v1:${tenant.id.repeat(64)}`, + ])); + const fleet = makeFleet({ arm: 'exact-arm', port: 3391, buildContracts: contracts }); + for (const tenant of fleet.tenants) { + const tenantId = tenant.id.slice('complete-tenant-'.length); + const surface = tenant.surfaces[0]; + assert.equal(surface.buildContract, contracts[tenantId]); + assert.equal(surface.buildContracts['exact-arm'], contracts[tenantId]); + assert.doesNotMatch(surface.buildContract, /^ctf:unresolved:/); + } +}); + +test('canary pointer matching is positive, negative, and wildcard aware', () => { + const response = { + data: { + values: [{ name: 'metadataA' }, { name: 'safe' }], + }, + }; + assert.deepEqual(jsonPointerValues(response, '/data/values/*/name'), ['metadataA', 'safe']); + assert.deepEqual(evaluateCanaryResponse({ + requiredMatches: [{ path: '/data/values/*/name', value: 'metadataA' }], + forbiddenMatches: [{ path: '/data/values/*/name', value: 'metadataB' }], + }, response), { conclusive: true, violation: false }); + assert.deepEqual(evaluateCanaryResponse({ + requiredMatches: [{ path: '/data/values/*/name', value: 'missing' }], + forbiddenMatches: [], + }, response), { + conclusive: false, + violation: false, + detail: "required match at '/data/values/*/name' was absent", + }); +}); + +test('plan carries non-secret role names and no runtime credentials', () => { + const runtimeRoles = { a: 'ctf_runtime_a', b: 'ctf_runtime_b', c: 'ctf_runtime_c' }; + const plan = makePlan({ + arm: 'exact-arm', + postgresContainer: 'ctf-postgres', + commit: '0123456789abcdef', + durationSec: 60, + runtimeRoles, + }); + const command = plan.arms[0].command; + for (const tenant of TENANTS) { + assert.ok(command.includes(`--${tenant.runtimeRoleArgument}`)); + assert.ok(command.includes(runtimeRoles[tenant.id])); + } + const releaseModeIndex = command.indexOf('--introspection-client-release-mode'); + assert.ok(releaseModeIndex >= 0); + assert.equal(command[releaseModeIndex + 1], 'destroy'); + assert.throws(() => makePlan({ + arm: 'invalid-release-arm', + postgresContainer: 'ctf-postgres', + commit: '0123456789abcdef', + introspectionClientReleaseMode: 'best-effort', + runtimeRoles, + }), /CTF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:best-effort/); + assert.doesNotMatch(JSON.stringify(plan), /password|secretAccessKey|authorization|bearer/i); +}); + +test('control-bearing fixture URLs are restricted to credential-free loopback HTTP', () => { + assert.equal(assertLoopbackBaseUrl('http://127.0.0.1:3391'), 'http://127.0.0.1:3391'); + assert.equal(assertLoopbackBaseUrl('http://[::1]:3391'), 'http://[::1]:3391'); + assert.throws( + () => assertLoopbackBaseUrl('https://example.com'), + /CTF_LOOPBACK_BASE_URL_REQUIRED/, + ); + assert.throws( + () => assertLoopbackBaseUrl('http://user:credential@127.0.0.1:3391'), + /CTF_LOOPBACK_BASE_URL_REQUIRED/, + ); + assert.throws( + () => assertLoopbackBaseUrl('http://127.0.0.1:3391/prefix'), + /CTF_LOOPBACK_BASE_URL_REQUIRED/, + ); +}); diff --git a/research/graphile-density/complete-tenant-fixture/qualification-runner.cjs b/research/graphile-density/complete-tenant-fixture/qualification-runner.cjs new file mode 100644 index 0000000000..9f3a36ceb5 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/qualification-runner.cjs @@ -0,0 +1,350 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { spawn } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + FIXTURE_DIR, + REPO_ROOT, + TENANTS, + assertProviderGates, + parseArgs, + parsePositiveInteger, + readManifest, + requireString, +} = require('./lib.cjs'); +const { atomicWriteJson, generateInputs } = require('./generate-inputs.cjs'); +const { runHostileValidation } = require('./hostile-validation.cjs'); +const { createFixtureServer, parseServerOptions } = require('./server.cjs'); + +const IN_PROCESS_ENVIRONMENT_KEYS = Object.freeze([ + 'NODE_ENV', + 'DATABASE_URL', + 'PGHOST', + 'PGPORT', + 'PGDATABASE', + 'PGUSER', + 'PGPASSWORD', + 'PGSSLMODE', + 'PGSSLROOTCERT', + 'PGSSLCERT', + 'PGSSLKEY', + 'GRAPHQL_RUNTIME_PGPASSWORD', + ...TENANTS.flatMap((tenant) => [ + tenant.runtimePasswordEnvironment, + `CTF_RUNTIME_${tenant.id.toUpperCase()}_PGUSER`, + ]), + 'CTF_CONTROL_TOKEN', + 'GRAPHQL_OBSERVABILITY_ENABLED', + 'GRAPHQL_OBSERVABILITY_TOKEN', + 'GRAPHILE_CACHE_MAX', + 'GRAPHILE_CACHE_INSTANCE_HEAP_BYTES', + 'GRAPHILE_CACHE_SERVER_RESERVE_BYTES', + 'GRAPHILE_CACHE_BUILD_RESERVE_BYTES', + 'GRAPHILE_BUILD_MAX_CONCURRENCY', + 'GRAPHILE_BUILD_CONCURRENCY', + 'GRAPHILE_BUILD_QUEUE_MAX', + 'PG_CACHE_MAX', + 'PG_POOL_MAX', + 'PG_POOL_MAX_USES', + 'DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE', +]); + +const installProcessEnvironment = ( + source, + keys = IN_PROCESS_ENVIRONMENT_KEYS, +) => { + const previous = new Map(); + for (const key of keys) { + previous.set(key, Object.prototype.hasOwnProperty.call(process.env, key) + ? { present: true, value: process.env[key] } + : { present: false }); + const value = source[key]; + if (value === undefined || value === null) delete process.env[key]; + else process.env[key] = String(value); + } + let restored = false; + return () => { + if (restored) return; + restored = true; + for (const [key, state] of previous) { + if (state.present) process.env[key] = state.value; + else delete process.env[key]; + } + }; +}; + +const runCommand = (command, cwd, environment = process.env) => new Promise((resolve, reject) => { + const child = spawn(command[0], command.slice(1), { + cwd, + env: environment, + stdio: 'inherit', + }); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (code === 0) resolve(); + else reject(new Error( + `CTF_COMMAND_FAILED:${command[0]}:code=${code ?? 'null'}:signal=${signal ?? 'null'}`, + )); + }); +}); + +const runRepositorySuites = async (manifest, environment) => { + const results = []; + for (const suite of manifest.mandatoryRepositorySuites) { + await runCommand(suite.command, path.join(REPO_ROOT, suite.cwd), environment); + results.push({ id: suite.id, passed: true }); + } + return results; +}; + +const summarizeQualification = ({ + qualificationClass, + providerState, + hostilePassed, + repositorySuites, + densityResults, + productionEquivalent = false, + error = null, +}) => { + const localPassed = error === null + && hostilePassed + && repositorySuites.length > 0 + && repositorySuites.every((suite) => suite.passed) + && densityResults.length > 0 + && densityResults.every((result) => result.accepted === true); + const customerQualified = qualificationClass === 'production' + && productionEquivalent === true + && providerState.customerQualified === true + && localPassed; + return { + localPassed, + customerQualified, + unresolvedExternalGates: [...providerState.unresolved], + ...(error ? { failure: error instanceof Error ? error.message : String(error) } : {}), + }; +}; + +const runQualification = async ({ + qualificationClass, + arm, + mode, + port, + postgresContainer, + runtimeRoles, + durationSec, + outputDir, + providerArguments = {}, + environment = process.env, +} = {}) => { + const manifest = readManifest(); + fs.mkdirSync(outputDir, { recursive: true, mode: 0o700 }); + const reportFile = path.join(outputDir, 'qualification.json'); + let providerState = { + customerQualified: false, + unresolved: manifest.externalProviderGates.map((gate) => gate.id), + }; + let failure = null; + try { + providerState = assertProviderGates( + manifest, + qualificationClass, + providerArguments, + ); + if (qualificationClass === 'production') { + throw new Error( + 'CTF_PRODUCTION_EQUIVALENCE_NOT_IMPLEMENTED:the exact fixture still uses deterministic LLM and signing-only storage paths', + ); + } + } catch (error) { + failure = error; + } + const controlToken = crypto.randomBytes(32).toString('hex'); + const observabilityToken = crypto.randomBytes(32).toString('hex'); + const runEnvironment = { + ...environment, + NODE_ENV: 'production', + CTF_CONTROL_TOKEN: controlToken, + GRAPHQL_OBSERVABILITY_ENABLED: 'true', + GRAPHQL_OBSERVABILITY_TOKEN: observabilityToken, + GRAPHILE_CACHE_MAX: '3', + GRAPHILE_CACHE_INSTANCE_HEAP_BYTES: String(64 * 1024 * 1024), + GRAPHILE_CACHE_SERVER_RESERVE_BYTES: String(256 * 1024 * 1024), + GRAPHILE_CACHE_BUILD_RESERVE_BYTES: String(768 * 1024 * 1024), + GRAPHILE_BUILD_MAX_CONCURRENCY: '1', + GRAPHILE_BUILD_CONCURRENCY: '1', + GRAPHILE_BUILD_QUEUE_MAX: '8', + PG_CACHE_MAX: '4', + PG_POOL_MAX: '1', + PG_POOL_MAX_USES: '0', + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: '100', + }; + const serverArgs = [ + '--host', + '127.0.0.1', + '--port', + String(port), + '--arm', + arm, + '--mode', + mode, + '--runtime-pool-max', + '1', + '--runtime-pool-max-uses', + 'unlimited', + ...TENANTS.flatMap((tenant) => [ + `--${tenant.runtimeRoleArgument}`, + runtimeRoles[tenant.id], + ]), + ]; + let server = null; + let hostilePassed = false; + let repositorySuites = []; + let densityResults = []; + let generated = null; + let localExecutionStarted = false; + let restoreProcessEnvironment = null; + try { + if (!failure) { + // The server loads pg-cache and graphile-cache lazily, but those modules + // snapshot governor and pool settings from process.env at first require. + // Install the exact run environment before that load and retain it for + // the child cperf arm, which inherits runtime credentials from this + // process without ever writing them into the generated plan. + restoreProcessEnvironment = installProcessEnvironment(runEnvironment); + localExecutionStarted = true; + server = await createFixtureServer(parseServerOptions(serverArgs, runEnvironment), runEnvironment); + await server.listen(); + generated = await generateInputs({ + arm, + mode, + port, + postgresContainer, + runtimeRoles, + durationSec, + outputDir: path.join(outputDir, 'generated'), + }); + const generatedFleet = JSON.parse(fs.readFileSync(generated.fleetFile, 'utf8')); + const expectedPhysicalDatabaseIdentity = + generatedFleet.tenants?.[0]?.databases?.[0]?.physicalDatabase; + await runHostileValidation({ + baseUrl: `http://127.0.0.1:${port}`, + expectedPhysicalDatabaseIdentity, + controlToken, + arm, + mode, + outputFile: path.join(outputDir, 'hostile-validation.json'), + }); + hostilePassed = true; + await server.close(); + server = null; + + repositorySuites = await runRepositorySuites(manifest, runEnvironment); + const perfHarness = require(path.join(REPO_ROOT, 'packages/perf-harness/dist/index.js')); + const plan = perfHarness.loadPlan(generated.planFile); + const fleet = perfHarness.loadFleet(generated.fleetFile); + perfHarness.validateCoverage(plan, fleet); + densityResults = await perfHarness.runDensityPlan(plan, fleet); + } + } catch (error) { + failure ??= error; + } finally { + try { + if (server) await server.close().catch(() => undefined); + } finally { + restoreProcessEnvironment?.(); + } + } + + const summary = summarizeQualification({ + qualificationClass, + providerState, + hostilePassed, + repositorySuites, + densityResults, + productionEquivalent: false, + error: failure, + }); + const report = { + version: 1, + fixture: manifest.fixture, + qualificationClass, + startedLocally: localExecutionStarted, + productionEquivalent: false, + endedAt: new Date().toISOString(), + arm, + mode, + durationSec, + ...summary, + providerGates: manifest.externalProviderGates.map((gate) => ({ + id: gate.id, + passed: false, + blocking: true, + })), + hostileValidation: { passed: hostilePassed }, + repositorySuites, + densityRuns: densityResults.map((result) => ({ + arm: result.arm, + heapMiB: result.heapMiB, + configuredTenants: result.configuredTenants, + repetition: result.repetition, + accepted: result.accepted, + artifactDir: path.relative(REPO_ROOT, result.artifactDir), + })), + generatedInputs: generated ? { + plan: path.relative(REPO_ROOT, generated.planFile), + fleet: path.relative(REPO_ROOT, generated.fleetFile), + } : null, + }; + atomicWriteJson(reportFile, report); + if (failure) throw failure; + if (!summary.localPassed) throw new Error('CTF_OFFLINE_RESEARCH_GATES_FAILED'); + return { reportFile, report }; +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + const qualificationClass = requireString(args, 'class', 'production'); + const runtimeRoles = Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + requireString(args, tenant.runtimeRoleArgument), + ])); + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const result = await runQualification({ + qualificationClass, + arm: requireString(args, 'arm', 'local-complete-tenant'), + mode: requireString(args, 'mode', 'scoped-required'), + port: parsePositiveInteger(args.port ?? '3391', 'port'), + postgresContainer: requireString(args, 'postgres-container'), + runtimeRoles, + durationSec: parsePositiveInteger(args['duration-sec'] ?? '900', 'duration-sec'), + outputDir: path.resolve(requireString( + args, + 'output-dir', + path.join(FIXTURE_DIR, 'qualification-artifacts', timestamp), + )), + providerArguments: args, + }); + process.stdout.write(`${JSON.stringify({ + reportFile: result.reportFile, + localPassed: result.report.localPassed, + customerQualified: result.report.customerQualified, + })}\n`); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + IN_PROCESS_ENVIRONMENT_KEYS, + installProcessEnvironment, + runCommand, + runQualification, + summarizeQualification, +}; diff --git a/research/graphile-density/complete-tenant-fixture/qualification-runner.test.cjs b/research/graphile-density/complete-tenant-fixture/qualification-runner.test.cjs new file mode 100644 index 0000000000..51b84d9dbd --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/qualification-runner.test.cjs @@ -0,0 +1,74 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); + +const { + installProcessEnvironment, + runQualification, + summarizeQualification, +} = require('./qualification-runner.cjs'); + +test('offline success remains explicitly unqualified for customers', () => { + const localEvidence = { + hostilePassed: true, + repositorySuites: [{ id: 'suite', passed: true }], + densityResults: [{ accepted: true }], + }; + assert.deepEqual(summarizeQualification({ + qualificationClass: 'offline-research', + providerState: { customerQualified: false, unresolved: ['provider'] }, + ...localEvidence, + }), { + localPassed: true, + customerQualified: false, + unresolvedExternalGates: ['provider'], + }); + assert.equal(summarizeQualification({ + qualificationClass: 'production', + providerState: { customerQualified: true, unresolved: [] }, + ...localEvidence, + }).customerQualified, false); + assert.equal(summarizeQualification({ + qualificationClass: 'production', + providerState: { customerQualified: true, unresolved: [] }, + productionEquivalent: true, + ...localEvidence, + }).customerQualified, true); +}); + +test('temporary in-process environment installation is exactly reversible', () => { + const key = `CTF_TEST_ENV_${process.pid}`; + delete process.env[key]; + const restore = installProcessEnvironment({ [key]: 'fixture-value' }, [key]); + assert.equal(process.env[key], 'fixture-value'); + restore(); + assert.equal(Object.prototype.hasOwnProperty.call(process.env, key), false); + restore(); +}); + +test('production preflight failure still writes a fail-closed report', async (context) => { + const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctf-qualification-')); + context.after(() => fs.rmSync(outputDir, { recursive: true, force: true })); + await assert.rejects(() => runQualification({ + qualificationClass: 'production', + arm: 'fixture-arm', + mode: 'scoped-required', + port: 3391, + postgresContainer: 'ctf-postgres', + runtimeRoles: { a: 'ctf_runtime_a', b: 'ctf_runtime_b', c: 'ctf_runtime_c' }, + durationSec: 60, + outputDir, + providerArguments: {}, + environment: {}, + }), /CTF_EXTERNAL_PROVIDER_GATES_UNSATISFIED/); + const report = JSON.parse(fs.readFileSync(path.join(outputDir, 'qualification.json'), 'utf8')); + assert.equal(report.localPassed, false); + assert.equal(report.customerQualified, false); + assert.equal(report.startedLocally, false); + assert.match(report.failure, /^CTF_EXTERNAL_PROVIDER_GATES_UNSATISFIED:/); + assert.equal(report.generatedInputs, null); +}); diff --git a/research/graphile-density/complete-tenant-fixture/schema.sql b/research/graphile-density/complete-tenant-fixture/schema.sql new file mode 100644 index 0000000000..8f8660e087 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/schema.sql @@ -0,0 +1,709 @@ +\set ON_ERROR_STOP on + +\if :{?runtime_role_a} +\else + \echo 'CTF_SETUP_RUNTIME_ROLE_REQUIRED: pass --set=runtime_role_a=' + \quit 3 +\endif +\if :{?runtime_role_b} +\else + \echo 'CTF_SETUP_RUNTIME_ROLE_REQUIRED: pass --set=runtime_role_b=' + \quit 3 +\endif +\if :{?runtime_role_c} +\else + \echo 'CTF_SETUP_RUNTIME_ROLE_REQUIRED: pass --set=runtime_role_c=' + \quit 3 +\endif + +-- psql deliberately does not interpolate variables inside dollar-quoted PL/pgSQL +-- bodies. Materialize the three safely quoted values as data before entering any +-- DO block, then read them through pg_temp below. +CREATE TEMP TABLE ctf_runtime_roles ( + ordinal smallint PRIMARY KEY, + role_name name NOT NULL UNIQUE +); +INSERT INTO ctf_runtime_roles (ordinal, role_name) VALUES + (1, :'runtime_role_a'), + (2, :'runtime_role_b'), + (3, :'runtime_role_c'); + +DO $roles$ +DECLARE + runtime_roles text[]; + runtime_role text; + role_record pg_catalog.pg_roles%ROWTYPE; +BEGIN + SELECT pg_catalog.array_agg(role_name::text ORDER BY ordinal) + INTO runtime_roles + FROM pg_temp.ctf_runtime_roles; + IF pg_catalog.array_length(runtime_roles, 1) <> 3 + OR (SELECT pg_catalog.count(DISTINCT value) FROM pg_catalog.unnest(runtime_roles) AS value) <> 3 THEN + RAISE EXCEPTION 'CTF_RUNTIME_ROLES_MUST_BE_DISTINCT'; + END IF; + + FOREACH runtime_role IN ARRAY runtime_roles + LOOP + SELECT * INTO role_record + FROM pg_catalog.pg_roles + WHERE rolname = runtime_role; + IF NOT FOUND THEN + RAISE EXCEPTION 'CTF_RUNTIME_ROLE_NOT_FOUND:%', runtime_role; + END IF; + IF NOT role_record.rolcanlogin + OR role_record.rolinherit + OR role_record.rolsuper + OR role_record.rolbypassrls + OR role_record.rolcreaterole + OR role_record.rolcreatedb + OR role_record.rolreplication THEN + RAISE EXCEPTION 'CTF_RUNTIME_ROLE_UNSAFE:%', runtime_role; + END IF; + END LOOP; +END +$roles$; + +CREATE SCHEMA ctf_extensions; +CREATE EXTENSION vector WITH SCHEMA ctf_extensions; +CREATE EXTENSION pg_trgm WITH SCHEMA ctf_extensions; +CREATE EXTENSION pg_textsearch WITH SCHEMA ctf_extensions; +CREATE EXTENSION ltree WITH SCHEMA ctf_extensions; +CREATE EXTENSION postgis WITH SCHEMA ctf_extensions; + +-- A shared notification login must be able to CONNECT and LISTEN without +-- inheriting PostgreSQL's default PUBLIC access to application metadata or +-- extension routines. Runtime roles receive the exact extension capabilities +-- they need explicitly below. +REVOKE ALL ON SCHEMA public FROM PUBLIC; +REVOKE ALL ON SCHEMA ctf_extensions FROM PUBLIC; +REVOKE ALL ON ALL TABLES IN SCHEMA ctf_extensions FROM PUBLIC; +REVOKE ALL ON ALL SEQUENCES IN SCHEMA ctf_extensions FROM PUBLIC; +REVOKE ALL ON ALL FUNCTIONS IN SCHEMA ctf_extensions FROM PUBLIC; + +-- PostGIS creates these compatibility views with PostgreSQL's historical +-- owner-rights default. They are dependency metadata, not part of a tenant API; +-- keep any future access under the caller's privileges and remove the PUBLIC +-- read grant before approving ctf_extensions as a runtime dependency schema. +ALTER VIEW ctf_extensions.geometry_columns SET (security_invoker = true); +ALTER VIEW ctf_extensions.geography_columns SET (security_invoker = true); +REVOKE ALL ON ctf_extensions.geometry_columns FROM PUBLIC; +REVOKE ALL ON ctf_extensions.geography_columns FROM PUBLIC; + +DO $fixture$ +DECLARE + extension_name text; + extension_schema text; +BEGIN + FOREACH extension_name IN ARRAY ARRAY['vector', 'pg_trgm', 'pg_textsearch', 'ltree', 'postgis'] + LOOP + SELECT n.nspname + INTO extension_schema + FROM pg_catalog.pg_extension e + JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace + WHERE e.extname = extension_name; + IF extension_schema IS DISTINCT FROM 'ctf_extensions' THEN + RAISE EXCEPTION 'CTF_EXTENSION_SCHEMA_MISMATCH:%:%', extension_name, extension_schema; + END IF; + END LOOP; +END +$fixture$; + +SET search_path TO pg_catalog, ctf_extensions; + +CREATE SCHEMA jwt_private; +REVOKE ALL ON SCHEMA jwt_private FROM PUBLIC; + +CREATE FUNCTION jwt_private.current_database_id() +RETURNS uuid +LANGUAGE sql +STABLE +SET search_path = pg_catalog +AS $function$ + SELECT nullif(current_setting('jwt.claims.database_id', true), '')::uuid +$function$; +REVOKE ALL ON FUNCTION jwt_private.current_database_id() FROM PUBLIC; + +CREATE SCHEMA ctf_control; +REVOKE ALL ON SCHEMA ctf_control FROM PUBLIC; + +CREATE PROCEDURE pg_temp.create_complete_tenant( + schema_name text, + tenant_token text, + database_id uuid, + runtime_role text, + metadata_function text, + storage_module_id uuid, + bucket_id uuid, + binding_id uuid, + definition_id uuid +) +LANGUAGE plpgsql +AS $procedure$ +DECLARE + table_name text; + fn_body text; + realtime_schema_name text; +BEGIN + IF schema_name NOT IN ('ctf_a', 'ctf_b', 'ctf_c') THEN + RAISE EXCEPTION 'CTF_UNKNOWN_TENANT_SCHEMA:%', schema_name; + END IF; + + EXECUTE format('CREATE SCHEMA %I', schema_name); + EXECUTE format('REVOKE ALL ON SCHEMA %I FROM PUBLIC', schema_name); + + realtime_schema_name := schema_name || '_realtime'; + EXECUTE format('CREATE SCHEMA %I', realtime_schema_name); + EXECUTE format('REVOKE ALL ON SCHEMA %I FROM PUBLIC', realtime_schema_name); + EXECUTE format( + 'CREATE FUNCTION %I.touch_listener(node_id text) RETURNS void LANGUAGE sql VOLATILE SECURITY INVOKER SET search_path = pg_catalog AS %L', + realtime_schema_name, + 'SELECT NULL::void' + ); + EXECUTE format( + 'CREATE FUNCTION %I.drain_changes(node_id text, batch_limit integer) RETURNS SETOF jsonb LANGUAGE sql VOLATILE SECURITY INVOKER SET search_path = pg_catalog AS %L', + realtime_schema_name, + 'SELECT NULL::jsonb WHERE false' + ); + EXECUTE format( + 'CREATE FUNCTION %I.cleanup_ephemeral(node_id text) RETURNS void LANGUAGE sql VOLATILE SECURITY INVOKER SET search_path = pg_catalog AS %L', + realtime_schema_name, + 'SELECT NULL::void' + ); + EXECUTE format( + 'REVOKE ALL ON ALL FUNCTIONS IN SCHEMA %I FROM PUBLIC', + realtime_schema_name + ); + EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', realtime_schema_name, runtime_role); + EXECUTE format( + 'GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', + realtime_schema_name, + runtime_role + ); + + EXECUTE format($sql$ + CREATE TABLE %I.tenant_canary ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + secret text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() + ) + $sql$, schema_name, database_id, tenant_token); + + EXECUTE format($sql$ + CREATE TABLE %I.documents ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + title text NOT NULL, + body text NOT NULL, + tsv tsvector NOT NULL, + embedding ctf_extensions.vector(3) NOT NULL, + location ctf_extensions.geometry(Point, 4326) NOT NULL, + path ctf_extensions.ltree NOT NULL, + attachment text + ) + $sql$, schema_name, database_id, tenant_token); + EXECUTE format('COMMENT ON COLUMN %I.documents.attachment IS %L', schema_name, E'@upload'); + EXECUTE format('CREATE INDEX documents_tsv_idx ON %I.documents USING gin (tsv)', schema_name); + EXECUTE format( + 'CREATE INDEX documents_embedding_idx ON %I.documents USING ivfflat (embedding ctf_extensions.vector_cosine_ops) WITH (lists = 1)', + schema_name + ); + EXECUTE format('CREATE INDEX documents_body_bm25_idx ON %I.documents USING bm25 (body) WITH (text_config = %L)', schema_name, 'english'); + EXECUTE format('CREATE INDEX documents_title_trgm_idx ON %I.documents USING gin (title ctf_extensions.gin_trgm_ops)', schema_name); + EXECUTE format('CREATE INDEX documents_location_idx ON %I.documents USING gist (location)', schema_name); + EXECUTE format('CREATE INDEX documents_path_idx ON %I.documents USING gist (path)', schema_name); + + EXECUTE format($sql$ + INSERT INTO %I.documents (id, title, body, tsv, embedding, location, path, attachment) + VALUES ( + 1, + %L, + %L, + to_tsvector('english', %L), + '[1,0,0]'::ctf_extensions.vector, + ctf_extensions.st_setsrid(ctf_extensions.st_makepoint(106.7, 10.8), 4326), + 'root.%s'::ctf_extensions.ltree, + 'fixture://%s/document.txt' + ) + $sql$, + schema_name, + tenant_token || ' Machine Learning', + tenant_token || ' machine learning artificial intelligence', + tenant_token || ' machine learning artificial intelligence', + right(schema_name, 1), + tenant_token + ); + + EXECUTE format($sql$ + CREATE TABLE %I.posts ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + title text NOT NULL, + body text + ) + $sql$, schema_name, database_id, tenant_token); + EXECUTE format('COMMENT ON TABLE %I.posts IS %L', schema_name, E'@i18n posts_translations'); + EXECUTE format($sql$ + CREATE TABLE %I.posts_translations ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + post_id integer NOT NULL REFERENCES %I.posts(id) ON DELETE CASCADE, + lang_code text NOT NULL, + title text NOT NULL, + body text, + UNIQUE (post_id, lang_code) + ) + $sql$, schema_name, database_id, schema_name); + EXECUTE format('INSERT INTO %I.posts (id, title, body) VALUES (1, %L, %L)', schema_name, tenant_token, tenant_token || ' base body'); + EXECUTE format( + 'INSERT INTO %I.posts_translations (post_id, lang_code, title, body) VALUES (1, %L, %L, %L), (1, %L, %L, %L)', + schema_name, + 'en', tenant_token || ' English', tenant_token || ' English body', + 'es', tenant_token || ' español', tenant_token || ' cuerpo español' + ); + + EXECUTE format($sql$ + CREATE TABLE %I.articles ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + title text NOT NULL, + body text NOT NULL, + embedding ctf_extensions.vector(3) NOT NULL + ) + $sql$, schema_name, database_id, tenant_token); + EXECUTE format( + 'COMMENT ON TABLE %I.articles IS %L', + schema_name, + E'@hasChunks {"chunksTable":"articles_chunks","parentFk":"parent_id","parentPk":"id","embeddingField":"embedding","contentField":"content"}' + ); + EXECUTE format($sql$ + CREATE TABLE %I.articles_chunks ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + parent_id integer NOT NULL REFERENCES %I.articles(id) ON DELETE CASCADE, + content text NOT NULL, + embedding ctf_extensions.vector(3) NOT NULL + ) + $sql$, schema_name, database_id, schema_name); + EXECUTE format('CREATE INDEX articles_embedding_idx ON %I.articles USING hnsw (embedding ctf_extensions.vector_cosine_ops)', schema_name); + EXECUTE format('CREATE INDEX articles_chunks_embedding_idx ON %I.articles_chunks USING hnsw (embedding ctf_extensions.vector_cosine_ops)', schema_name); + EXECUTE format( + 'INSERT INTO %I.articles (id, title, body, embedding) VALUES (1, %L, %L, %L::ctf_extensions.vector)', + schema_name, tenant_token || ' article', tenant_token || ' machine learning article', '[1,0,0]' + ); + EXECUTE format( + 'INSERT INTO %I.articles_chunks (id, parent_id, content, embedding) VALUES (1, 1, %L, %L::ctf_extensions.vector)', + schema_name, tenant_token || ' machine learning tenant fixture context', '[0.99,0.01,0]' + ); + + EXECUTE format($sql$ + CREATE TABLE %I.bulk_items ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + name text NOT NULL, + quantity integer NOT NULL DEFAULT 0, + CONSTRAINT bulk_items_name_key UNIQUE (name) + ) + $sql$, schema_name, database_id, tenant_token); + EXECUTE format('COMMENT ON TABLE %I.bulk_items IS %L', schema_name, E'@behavior +bulkInsert +bulkUpsert +bulkUpdate +bulkDelete'); + + EXECUTE format($sql$ + CREATE TABLE %I.realtime_items ( + id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + payload text NOT NULL + ) + $sql$, schema_name, database_id, tenant_token); + EXECUTE format('COMMENT ON TABLE %I.realtime_items IS %L', schema_name, E'@realtime'); + EXECUTE format('INSERT INTO %I.realtime_items (id, payload) VALUES (1, %L)', schema_name, tenant_token || '-initial'); + + fn_body := format($body$ + BEGIN + IF TG_OP = 'DELETE' THEN + PERFORM pg_catalog.pg_notify(%L, TG_OP || ':' || OLD.id::text); + RETURN OLD; + END IF; + PERFORM pg_catalog.pg_notify(%L, TG_OP || ':' || NEW.id::text); + RETURN NEW; + END + $body$, + 'realtime:' || schema_name || '.realtime_items', + 'realtime:' || schema_name || '.realtime_items' + ); + EXECUTE format( + 'CREATE FUNCTION %I.notify_realtime_item() RETURNS trigger LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog AS %L', + schema_name, + fn_body + ); + EXECUTE format( + 'CREATE TRIGGER realtime_items_notify AFTER INSERT OR UPDATE OR DELETE ON %I.realtime_items FOR EACH ROW EXECUTE FUNCTION %I.notify_realtime_item()', + schema_name, + schema_name + ); + + EXECUTE format($sql$ + CREATE TABLE %I.app_buckets ( + id uuid PRIMARY KEY, + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + key text NOT NULL UNIQUE, + type text NOT NULL, + is_public boolean NOT NULL DEFAULT false, + owner_id uuid, + allowed_mime_types text[], + max_file_size integer, + allow_custom_keys boolean NOT NULL DEFAULT false, + physical_name text + ) + $sql$, schema_name, database_id, tenant_token); + EXECUTE format('COMMENT ON TABLE %I.app_buckets IS %L', schema_name, E'@storageBuckets'); + EXECUTE format($sql$ + CREATE TABLE %I.app_files ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + bucket_id uuid NOT NULL REFERENCES %I.app_buckets(id), + key text NOT NULL, + content_hash text NOT NULL, + mime_type text NOT NULL, + size integer NOT NULL, + filename text, + is_public boolean NOT NULL DEFAULT false, + previous_version_id uuid REFERENCES %I.app_files(id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (bucket_id, content_hash) + ) + $sql$, schema_name, database_id, tenant_token, schema_name, schema_name); + EXECUTE format('COMMENT ON TABLE %I.app_files IS %L', schema_name, E'@storageFiles'); + EXECUTE format( + 'INSERT INTO %I.app_buckets (id, key, type, allowed_mime_types, max_file_size, physical_name) VALUES (%L::uuid, %L, %L, ARRAY[%L], 1048576, NULL)', + schema_name, bucket_id, 'private', 'private', 'text/plain' + ); + + EXECUTE format($sql$ + CREATE TABLE %I.function_invocations ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + database_id uuid NOT NULL DEFAULT %L::uuid, + tenant_id text NOT NULL DEFAULT %L, + task_identifier text NOT NULL, + function_definition_id uuid NOT NULL, + api_binding_id uuid NOT NULL, + status text NOT NULL, + payload jsonb, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() + ) + $sql$, schema_name, database_id, tenant_token); + + EXECUTE format($sql$ + CREATE TABLE %I.schema_state ( + id integer PRIMARY KEY CHECK (id = 1), + database_id uuid NOT NULL DEFAULT %L::uuid, + epoch integer NOT NULL + ) + $sql$, schema_name, database_id); + EXECUTE format('INSERT INTO %I.schema_state (id, epoch) VALUES (1, 1)', schema_name); + + FOREACH table_name IN ARRAY ARRAY[ + 'tenant_canary', + 'documents', + 'posts', + 'posts_translations', + 'articles', + 'articles_chunks', + 'bulk_items', + 'realtime_items', + 'app_buckets', + 'app_files', + 'function_invocations', + 'schema_state' + ] + LOOP + EXECUTE format('ALTER TABLE %I.%I ENABLE ROW LEVEL SECURITY', schema_name, table_name); + EXECUTE format('ALTER TABLE %I.%I FORCE ROW LEVEL SECURITY', schema_name, table_name); + EXECUTE format( + 'CREATE POLICY tenant_guard ON %I.%I USING (database_id::text = nullif(current_setting(%L, true), %L)) WITH CHECK (database_id::text = nullif(current_setting(%L, true), %L))', + schema_name, + table_name, + 'jwt.claims.database_id', + '', + 'jwt.claims.database_id', + '' + ); + END LOOP; + + fn_body := format('SELECT CASE WHEN nullif(current_setting(%L, true), %L) = %L THEN %L::text ELSE %L::text END', 'jwt.claims.database_id', '', database_id::text, tenant_token, 'guc-mismatch'); + EXECUTE format('CREATE FUNCTION %I.tenant_identity() RETURNS text LANGUAGE sql STABLE SECURITY INVOKER SET search_path = pg_catalog AS %L', schema_name, fn_body); + EXECUTE format( + 'CREATE FUNCTION %I.physical_database_identity() RETURNS text LANGUAGE sql STABLE PARALLEL SAFE SECURITY INVOKER SET search_path = pg_catalog AS %L', + schema_name, + 'SELECT pg_catalog.current_database()::text' + ); + EXECUTE format('CREATE FUNCTION %I.%I() RETURNS text LANGUAGE sql STABLE SECURITY INVOKER SET search_path = pg_catalog AS %L', schema_name, metadata_function, format('SELECT %L::text', tenant_token)); + EXECUTE format( + 'CREATE FUNCTION %I.request_identity() RETURNS text LANGUAGE sql STABLE SECURITY INVOKER SET search_path = pg_catalog AS %L', + schema_name, + format('SELECT %L || %L || nullif(current_setting(%L, true), %L)', tenant_token, ':', 'jwt.claims.database_id', '') + ); + fn_body := format($body$ + DECLARE + observed text; + BEGIN + BEGIN + PERFORM set_config('jwt.claims.database_id', 'poisoned-savepoint', true); + RAISE EXCEPTION 'fixture subtransaction rollback'; + EXCEPTION WHEN OTHERS THEN + NULL; + END; + observed := nullif(current_setting('jwt.claims.database_id', true), ''); + RETURN observed; + END + $body$); + EXECUTE format('CREATE FUNCTION %I.savepoint_identity() RETURNS text LANGUAGE plpgsql VOLATILE SECURITY INVOKER SET search_path = pg_catalog AS %L', schema_name, fn_body); + EXECUTE format('COMMENT ON FUNCTION %I.savepoint_identity() IS %L', schema_name, E'@behavior -*'); + fn_body := format($body$ + BEGIN + PERFORM set_config('jwt.claims.database_id', %L, false); + PERFORM set_config('jwt.claims.user_id', %L, false); + RETURN 'poisoned'; + END + $body$, 'ffffffff-ffff-4fff-8fff-ffffffffffff', 'poisoned-user'); + EXECUTE format('CREATE FUNCTION %I.poison_session() RETURNS text LANGUAGE plpgsql VOLATILE SECURITY INVOKER SET search_path = pg_catalog AS %L', schema_name, fn_body); + EXECUTE format('COMMENT ON FUNCTION %I.poison_session() IS %L', schema_name, E'@behavior -*'); + fn_body := format($body$ + DECLARE + row_count integer; + BEGIN + IF target_schema NOT IN ('ctf_a', 'ctf_b', 'ctf_c') THEN + RAISE EXCEPTION 'CTF_FOREIGN_SCHEMA_NOT_ALLOWED:%%', target_schema; + END IF; + BEGIN + EXECUTE format('SELECT count(*)::integer FROM %%I.documents', target_schema) INTO row_count; + EXCEPTION WHEN insufficient_privilege THEN + RETURN 'acl-denied'; + END; + RETURN CASE WHEN row_count = 0 THEN 'rls-empty' ELSE 'visible' END; + END + $body$); + EXECUTE format('CREATE FUNCTION %I.foreign_access_state(target_schema text) RETURNS text LANGUAGE plpgsql STABLE SECURITY INVOKER SET search_path = pg_catalog AS %L', schema_name, fn_body); + fn_body := format($body$ + SELECT NOT r.rolsuper + AND NOT r.rolbypassrls + AND NOT r.rolcreaterole + AND NOT pg_has_role(session_user, n.nspowner, 'MEMBER') + AND NOT has_schema_privilege(session_user, %L, 'CREATE') + FROM pg_roles r + JOIN pg_namespace n ON n.nspname = %L + WHERE r.rolname = session_user + $body$, schema_name, schema_name); + EXECUTE format('CREATE FUNCTION %I.runtime_role_safe() RETURNS boolean LANGUAGE sql STABLE SECURITY INVOKER SET search_path = pg_catalog AS %L', schema_name, fn_body); + EXECUTE format( + 'CREATE FUNCTION %I.schema_epoch() RETURNS integer LANGUAGE sql STABLE SECURITY INVOKER SET search_path = pg_catalog AS %L', + schema_name, + format('SELECT epoch FROM %I.schema_state WHERE id = 1', schema_name) + ); + + EXECUTE format('REVOKE ALL ON ALL FUNCTIONS IN SCHEMA %I FROM PUBLIC', schema_name); + EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', schema_name, runtime_role); + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA %I TO %I', schema_name, runtime_role); + EXECUTE format('GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA %I TO %I', schema_name, runtime_role); + EXECUTE format('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', schema_name, runtime_role); +END +$procedure$; + +CALL pg_temp.create_complete_tenant( + 'ctf_a', + 'tenant-a-canary', + '10000000-0000-4000-8000-00000000000a', + :'runtime_role_a', + 'metadata_a', + '30000000-0000-4000-8000-00000000000a', + '40000000-0000-4000-8000-00000000000a', + '50000000-0000-4000-8000-00000000000a', + '60000000-0000-4000-8000-00000000000a' +); +CALL pg_temp.create_complete_tenant( + 'ctf_b', + 'tenant-b-canary', + '10000000-0000-4000-8000-00000000000b', + :'runtime_role_b', + 'metadata_b', + '30000000-0000-4000-8000-00000000000b', + '40000000-0000-4000-8000-00000000000b', + '50000000-0000-4000-8000-00000000000b', + '60000000-0000-4000-8000-00000000000b' +); +CALL pg_temp.create_complete_tenant( + 'ctf_c', + 'tenant-c-canary', + '10000000-0000-4000-8000-00000000000c', + :'runtime_role_c', + 'metadata_c', + '30000000-0000-4000-8000-00000000000c', + '40000000-0000-4000-8000-00000000000c', + '50000000-0000-4000-8000-00000000000c', + '60000000-0000-4000-8000-00000000000c' +); + +DO $realtime_isolation$ +DECLARE + runtime_record record; + target_ordinal integer; + target_schema text; + function_signature text; + function_oid oid; + should_have_access boolean; +BEGIN + FOR runtime_record IN + SELECT ordinal, role_name::text + FROM pg_temp.ctf_runtime_roles + ORDER BY ordinal + LOOP + FOR target_ordinal IN 1..3 + LOOP + target_schema := format( + 'ctf_%s_realtime', + chr(ascii('a') + target_ordinal - 1) + ); + should_have_access := runtime_record.ordinal = target_ordinal; + + IF pg_catalog.has_schema_privilege( + runtime_record.role_name, + target_schema, + 'USAGE' + ) IS DISTINCT FROM should_have_access THEN + RAISE EXCEPTION 'CTF_REALTIME_SCHEMA_ISOLATION_FAILED:%:%', + runtime_record.role_name, + target_schema; + END IF; + IF pg_catalog.has_schema_privilege( + runtime_record.role_name, + target_schema, + 'CREATE' + ) THEN + RAISE EXCEPTION 'CTF_REALTIME_SCHEMA_CREATE_FORBIDDEN:%:%', + runtime_record.role_name, + target_schema; + END IF; + + FOREACH function_signature IN ARRAY ARRAY[ + 'touch_listener(text)', + 'drain_changes(text,integer)', + 'cleanup_ephemeral(text)' + ] + LOOP + function_oid := pg_catalog.to_regprocedure( + format('%I.%s', target_schema, function_signature) + ); + IF function_oid IS NULL THEN + RAISE EXCEPTION 'CTF_REALTIME_FUNCTION_MISSING:%:%', + target_schema, + function_signature; + END IF; + IF pg_catalog.has_function_privilege( + runtime_record.role_name, + function_oid, + 'EXECUTE' + ) IS DISTINCT FROM should_have_access THEN + RAISE EXCEPTION 'CTF_REALTIME_FUNCTION_ISOLATION_FAILED:%:%:%', + runtime_record.role_name, + target_schema, + function_signature; + END IF; + END LOOP; + END LOOP; + END LOOP; +END +$realtime_isolation$; + +CREATE FUNCTION ctf_control.apply_schema_drift(target_schema text) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog +AS $function$ +DECLARE + target_database_id uuid; +BEGIN + target_database_id := CASE target_schema + WHEN 'ctf_a' THEN '10000000-0000-4000-8000-00000000000a'::uuid + WHEN 'ctf_b' THEN '10000000-0000-4000-8000-00000000000b'::uuid + WHEN 'ctf_c' THEN '10000000-0000-4000-8000-00000000000c'::uuid + ELSE NULL + END; + IF target_database_id IS NULL THEN + RAISE EXCEPTION 'CTF_DRIFT_SCHEMA_NOT_ALLOWED:%', target_schema; + END IF; + PERFORM pg_catalog.set_config( + 'jwt.claims.database_id', + target_database_id::text, + true + ); + EXECUTE format('ALTER TABLE %I.documents ADD COLUMN IF NOT EXISTS drift_probe text NOT NULL DEFAULT %L', target_schema, 'drift-applied'); + EXECUTE format('UPDATE %I.schema_state SET epoch = 2 WHERE id = 1', target_schema); +END +$function$; + +CREATE FUNCTION ctf_control.revert_schema_drift(target_schema text) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog +AS $function$ +DECLARE + target_database_id uuid; +BEGIN + target_database_id := CASE target_schema + WHEN 'ctf_a' THEN '10000000-0000-4000-8000-00000000000a'::uuid + WHEN 'ctf_b' THEN '10000000-0000-4000-8000-00000000000b'::uuid + WHEN 'ctf_c' THEN '10000000-0000-4000-8000-00000000000c'::uuid + ELSE NULL + END; + IF target_database_id IS NULL THEN + RAISE EXCEPTION 'CTF_DRIFT_SCHEMA_NOT_ALLOWED:%', target_schema; + END IF; + PERFORM pg_catalog.set_config( + 'jwt.claims.database_id', + target_database_id::text, + true + ); + EXECUTE format('ALTER TABLE %I.documents DROP COLUMN IF EXISTS drift_probe', target_schema); + EXECUTE format('UPDATE %I.schema_state SET epoch = 1 WHERE id = 1', target_schema); +END +$function$; + +REVOKE ALL ON ALL FUNCTIONS IN SCHEMA ctf_control FROM PUBLIC; + +DO $grants$ +DECLARE + runtime_role text; +BEGIN + FOR runtime_role IN + SELECT role_name::text + FROM pg_temp.ctf_runtime_roles + ORDER BY ordinal + LOOP + EXECUTE format('GRANT USAGE ON SCHEMA ctf_extensions, jwt_private TO %I', runtime_role); + EXECUTE format( + 'GRANT SELECT ON ALL TABLES IN SCHEMA ctf_extensions TO %I', + runtime_role + ); + EXECUTE format( + 'GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA ctf_extensions TO %I', + runtime_role + ); + EXECUTE format( + 'GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA ctf_extensions TO %I', + runtime_role + ); + EXECUTE format('GRANT EXECUTE ON FUNCTION jwt_private.current_database_id() TO %I', runtime_role); + END LOOP; +END +$grants$; + +RESET search_path; diff --git a/research/graphile-density/complete-tenant-fixture/schema.test.cjs b/research/graphile-density/complete-tenant-fixture/schema.test.cjs new file mode 100644 index 0000000000..2f58e78a37 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/schema.test.cjs @@ -0,0 +1,95 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); + +const sql = fs.readFileSync(path.join(__dirname, 'schema.sql'), 'utf8'); + +test('schema requires three distinct pre-existing least-privilege logins', () => { + for (const suffix of ['a', 'b', 'c']) { + assert.match(sql, new RegExp(`\\{\\?runtime_role_${suffix}\\}`)); + } + assert.match(sql, /CTF_RUNTIME_ROLES_MUST_BE_DISTINCT/); + assert.match(sql, /NOT role_record\.rolcanlogin/); + assert.match(sql, /role_record\.rolinherit/); + assert.match(sql, /role_record\.rolsuper/); + assert.match(sql, /role_record\.rolbypassrls/); + assert.match(sql, /CREATE TEMP TABLE ctf_runtime_roles/); + assert.doesNotMatch(sql, /CREATE TEMP TABLE ctf_runtime_roles[\s\S]*?ON COMMIT DROP/); + assert.match(sql, /FROM pg_temp\.ctf_runtime_roles/); + const doBlocks = [...sql.matchAll(/DO (\$[^$]+\$)([\s\S]*?)\1;/g)]; + assert.ok(doBlocks.length >= 3); + for (const [, , body] of doBlocks) { + assert.doesNotMatch(body, /:'runtime_role_[abc]'/); + } +}); + +test('runtime grants stay tenant-local and exclude drift control', () => { + assert.match(sql, /GRANT USAGE ON SCHEMA %I TO %I/); + assert.match(sql, /GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA %I TO %I/); + assert.doesNotMatch(sql, /GRANT[^;]+ctf_control[^;]+runtime_role/is); + assert.doesNotMatch(sql, /GRANT[^;]+apply_schema_drift/is); + assert.doesNotMatch(sql, /GRANT[^;]+revert_schema_drift/is); + assert.match(sql, /REVOKE ALL ON ALL FUNCTIONS IN SCHEMA ctf_control FROM PUBLIC/); +}); + +test('realtime startup functions are isolated in per-tenant least-privilege schemas', () => { + assert.match(sql, /realtime_schema_name := schema_name \|\| '_realtime'/); + assert.match(sql, /CREATE FUNCTION %I\.touch_listener\(node_id text\)/); + assert.match(sql, /CREATE FUNCTION %I\.drain_changes\(node_id text, batch_limit integer\) RETURNS SETOF jsonb/); + assert.match(sql, /CREATE FUNCTION %I\.cleanup_ephemeral\(node_id text\)/); + assert.match(sql, /REVOKE ALL ON SCHEMA %I FROM PUBLIC/); + assert.match(sql, /REVOKE ALL ON ALL FUNCTIONS IN SCHEMA %I FROM PUBLIC/); + assert.match(sql, /GRANT USAGE ON SCHEMA %I TO %I/); + assert.match(sql, /GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I/); + assert.match(sql, /CTF_REALTIME_SCHEMA_ISOLATION_FAILED/); + assert.match(sql, /CTF_REALTIME_FUNCTION_ISOLATION_FAILED/); + assert.match(sql, /CTF_REALTIME_SCHEMA_CREATE_FORBIDDEN/); +}); + +test('PostGIS dependency views execute with invoker rights and no public grant', () => { + for (const view of ['geometry_columns', 'geography_columns']) { + assert.match( + sql, + new RegExp(`ALTER VIEW ctf_extensions\\.${view} SET \\(security_invoker = true\\)`), + ); + assert.match(sql, new RegExp(`REVOKE ALL ON ctf_extensions\\.${view} FROM PUBLIC`)); + } +}); + +test('extension defaults do not grant the notification-only role application access', () => { + assert.match(sql, /REVOKE ALL ON SCHEMA public FROM PUBLIC/); + assert.match(sql, /REVOKE ALL ON SCHEMA ctf_extensions FROM PUBLIC/); + assert.match(sql, /REVOKE ALL ON ALL FUNCTIONS IN SCHEMA ctf_extensions FROM PUBLIC/); + assert.match(sql, /GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA ctf_extensions TO %I/); +}); + +test('session mutators are volatile and drift mutators restore an RLS identity', () => { + assert.match(sql, /savepoint_identity\(\) RETURNS text LANGUAGE plpgsql VOLATILE/); + assert.match(sql, /poison_session\(\) RETURNS text LANGUAGE plpgsql VOLATILE/); + assert.match( + sql, + /physical_database_identity\(\) RETURNS text LANGUAGE sql STABLE PARALLEL SAFE SECURITY INVOKER SET search_path = pg_catalog/, + ); + assert.match(sql, /SELECT pg_catalog\.current_database\(\)::text/); + assert.equal((sql.match(/PERFORM pg_catalog\.set_config\(/g) ?? []).length, 2); + assert.equal((sql.match(/SECURITY DEFINER\nSET search_path = pg_catalog/g) ?? []).length, 2); +}); + +test('the first upload must provision a physical bucket through the system lane', () => { + assert.match( + sql, + /INSERT INTO %I\.app_buckets \(id, key, type, allowed_mime_types, max_file_size, physical_name\) VALUES \(%L::uuid, %L, %L, ARRAY\[%L\], 1048576, NULL\)/, + ); + for (const table of ['app_buckets', 'app_files']) { + assert.ok(sql.includes(`'${table}'`)); + } + assert.match(sql, /ALTER TABLE %I\.%I FORCE ROW LEVEL SECURITY/); +}); + +test('realtime trigger returns the correct transition row for DELETE and writes', () => { + assert.match(sql, /IF TG_OP = 'DELETE' THEN[\s\S]+RETURN OLD;[\s\S]+RETURN NEW;/); + assert.doesNotMatch(sql, /coalesce\(NEW, OLD\)/i); +}); diff --git a/research/graphile-density/complete-tenant-fixture/server.cjs b/research/graphile-density/complete-tenant-fixture/server.cjs new file mode 100644 index 0000000000..08988d09d4 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/server.cjs @@ -0,0 +1,2007 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const { createRequire } = require('node:module'); +const path = require('node:path'); + +const { + REPO_ROOT, + TENANTS, + parseArgs, + parsePositiveInteger, + requireString, +} = require('./lib.cjs'); + +const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost']); +const RUNTIME_DEPENDENCY_SCHEMAS = Object.freeze(['ctf_extensions', 'jwt_private']); +const INTROSPECTION_DEPENDENCY_SCHEMAS = Object.freeze(['ctf_extensions']); +const realtimeSchemaFor = (tenant) => `${tenant.schema}_realtime`; +const runtimeDependencySchemasFor = (tenant, enableRealtime) => [ + ...RUNTIME_DEPENDENCY_SCHEMAS, + ...(enableRealtime ? [realtimeSchemaFor(tenant)] : []), +]; +const matchTenantUpgradePath = (rawUrl, pathPrefix = '') => { + if ( + typeof rawUrl !== 'string' + || typeof pathPrefix !== 'string' + || rawUrl.includes('?') + || (pathPrefix !== '' && (!pathPrefix.startsWith('/') || pathPrefix.endsWith('/'))) + || pathPrefix.includes('..') + ) return null; + const basePath = `${pathPrefix}/tenant/`; + if (!rawUrl.startsWith(basePath)) return null; + const parts = rawUrl.slice(basePath.length).split('/'); + return parts.length === 2 && parts[1] === 'graphql' && /^[a-z0-9-]+$/.test(parts[0]) + ? parts[0] + : null; +}; +const GRAFAST_CACHE_LIMITS = Object.freeze({ + queryCacheMaxLength: 8, + operationsCacheMaxLength: 8, + operationOperationPlansCacheMaxLength: 8, +}); +const RELEASE_BUILD_STATE_AFTER_VALIDATION = true; +const FEATURE_SETTINGS = Object.freeze({ + enableAggregates: false, + enablePostgis: true, + enableSearch: true, + enableDirectUploads: true, + enablePresignedUploads: true, + enableManyToMany: true, + enableConnectionFilter: true, + enableLtree: true, + enableLlm: true, + enableRealtime: true, + enableBulk: true, + enableI18n: true, + enableHistory: false, +}); +const RUNTIME_ARTIFACT_PATHS = Object.freeze([ + 'graphile/graphile-cache/dist/index.js', + 'graphile/graphile-cache/dist/create-instance.js', + 'graphile/graphile-cache/dist/graphile-cache.js', + 'graphile/graphile-cache/dist/http-adapter.js', + 'graphile/graphile-cache/dist/preset-services.js', + 'graphile/graphile-cache/dist/realtime-readiness.js', + 'graphile/graphile-realtime-subscriptions/dist/index.js', + 'graphile/graphile-realtime-subscriptions/dist/cursor-tracker.js', + 'graphile/graphile-realtime-subscriptions/dist/realtime-manager.js', + 'graphile/graphile-settings/dist/index.js', + 'postgres/pg-cache/dist/index.js', + 'packages/express-context/dist/index.js', + 'graphile/graphile-llm/dist/index.js', + 'graphile/graphile-function-bindings/dist/index.js', + 'graphile/graphile-presigned-url-plugin/dist/index.js', + 'graphql/server/dist/plugins/auth-cookie-plugin.js', + 'graphql/server/dist/middleware/graphile-build-contract.js', + 'graphql/server/dist/middleware/graphile-build-governor.js', + 'graphql/server/dist/middleware/runtime-role-safety.js', + 'graphql/server/dist/middleware/observability/graphile-build-stats.js', + 'graphql/server/dist/diagnostics/debug-memory-snapshot.js', +]); +const INSTALLED_RUNTIME_ARTIFACT_SPECS = Object.freeze([ + Object.freeze({ + label: 'installed:@dataplan/pg:dist/index.js', + resolveSpecifier: '@dataplan/pg', + relativePath: null, + markers: Object.freeze([ + 'exports.exactClientReleaseCapability = "dataplan-pg-exact-client-destroy-v1";', + ]), + }), + Object.freeze({ + label: 'installed:@dataplan/pg:dist/adaptors/pg.js', + resolveSpecifier: '@dataplan/pg/adaptors/pg', + relativePath: null, + markers: Object.freeze([ + 'const DESTROYABLE_CLIENT_RELEASE_MODES = Object.freeze(["reuse", "destroy"]);', + 'const supportsExactClientDestruction = typeof PgPool === "function" && pool instanceof PgPool;', + 'Exact PostgreSQL client destruction requires a node-postgres Pool', + 'pgClient.release(true);', + '? DESTROYABLE_CLIENT_RELEASE_MODES', + ]), + }), + Object.freeze({ + label: 'installed:@dataplan/pg:dist/pgServices.js', + resolveSpecifier: '@dataplan/pg', + relativePath: 'pgServices.js', + markers: Object.freeze([ + 'withPgClient.supportedClientReleaseModes = originalWithPgClient.supportedClientReleaseModes;', + 'const clientReleaseMode = options?.clientReleaseMode ?? "reuse";', + 'does not support exact client destruction', + ]), + }), + Object.freeze({ + label: 'installed:graphile-build-pg:dist/index.js', + resolveSpecifier: 'graphile-build-pg', + relativePath: null, + markers: Object.freeze([ + 'exports.introspectionClientReleaseCapability = "graphile-build-pg-exact-client-destroy-v1";', + ]), + }), + Object.freeze({ + label: 'installed:graphile-build-pg:dist/plugins/PgIntrospectionPlugin.js', + resolveSpecifier: 'graphile-build-pg', + relativePath: 'plugins/PgIntrospectionPlugin.js', + markers: Object.freeze([ + 'pgService.introspectionClientReleaseMode ?? "reuse"', + 'clientReleaseMode === "reuse" ? undefined : { clientReleaseMode }', + ]), + }), +]); + +const requireBuilt = (relativePath) => { + const absolutePath = path.join(REPO_ROOT, relativePath); + if (!fs.existsSync(absolutePath)) { + throw new Error(`CTF_BUILD_ARTIFACT_MISSING:${relativePath}`); + } + return require(absolutePath); +}; + +const readInstalledRuntimeArtifacts = () => { + const graphileSettingsEntry = path.join( + REPO_ROOT, + 'graphile/graphile-settings/dist/index.js', + ); + const graphileSettingsRequire = createRequire(graphileSettingsEntry); + let postgraphilePgEntry; + try { + postgraphilePgEntry = graphileSettingsRequire.resolve('postgraphile/adaptors/pg'); + } catch { + throw new Error('CTF_INSTALLED_RUNTIME_ARTIFACT_MISSING:postgraphile/adaptors/pg'); + } + const postgraphileRequire = createRequire(postgraphilePgEntry); + + return INSTALLED_RUNTIME_ARTIFACT_SPECS.map((spec) => { + let resolvedEntry; + try { + resolvedEntry = postgraphileRequire.resolve(spec.resolveSpecifier); + } catch { + throw new Error(`CTF_INSTALLED_RUNTIME_ARTIFACT_MISSING:${spec.label}`); + } + const absolutePath = spec.relativePath === null + ? resolvedEntry + : path.join(path.dirname(resolvedEntry), spec.relativePath); + if (!fs.existsSync(absolutePath)) { + throw new Error(`CTF_INSTALLED_RUNTIME_ARTIFACT_MISSING:${spec.label}`); + } + const bytes = fs.readFileSync(absolutePath); + const source = bytes.toString('utf8'); + spec.markers.forEach((marker, markerIndex) => { + if (!source.includes(marker)) { + throw new Error( + `CTF_INSTALLED_RUNTIME_MARKER_MISSING:${spec.label}:${markerIndex}` + ); + } + }); + return { bytes, spec }; + }); +}; + +const installedRuntimeArtifactManifest = () => readInstalledRuntimeArtifacts().map( + ({ bytes, spec }) => ({ + label: spec.label, + sha256: `sha256:${crypto.createHash('sha256').update(bytes).digest('hex')}`, + markerSetSha256: `sha256:${crypto.createHash('sha256') + .update(JSON.stringify(spec.markers)) + .digest('hex')}`, + markerCount: spec.markers.length, + }), +); + +const STATIC_REQUIRE_PATTERN = /\brequire(?:\.resolve)?\(\s*(['"])([^'"\r\n]+)\1\s*\)/g; + +const localDistRelativePath = (absolutePath) => { + let realPath; + try { + realPath = fs.realpathSync(absolutePath); + } catch { + return null; + } + const relativePath = path.relative(REPO_ROOT, realPath); + if ( + relativePath.startsWith('..') + || path.isAbsolute(relativePath) + || relativePath.split(path.sep).includes('node_modules') + || !relativePath.split(path.sep).includes('dist') + || !/\.(?:c|m)?js$/.test(relativePath) + ) return null; + return relativePath.split(path.sep).join('/'); +}; + +const staticRequireSpecifiers = (source) => { + const specifiers = new Set(); + for (const match of source.matchAll(STATIC_REQUIRE_PATTERN)) specifiers.add(match[2]); + return [...specifiers].sort(); +}; + +const resolvedLocalRuntimeArtifactManifest = () => { + const queue = RUNTIME_ARTIFACT_PATHS.map((relativePath) => { + const absolutePath = path.join(REPO_ROOT, relativePath); + if (!fs.existsSync(absolutePath)) { + throw new Error(`CTF_BUILD_ARTIFACT_MISSING:${relativePath}`); + } + return fs.realpathSync(absolutePath); + }); + const artifacts = new Map(); + while (queue.length > 0) { + const absolutePath = queue.shift(); + const relativePath = localDistRelativePath(absolutePath); + if (!relativePath || artifacts.has(relativePath)) continue; + const bytes = fs.readFileSync(absolutePath); + artifacts.set(relativePath, { + path: relativePath, + sha256: `sha256:${crypto.createHash('sha256').update(bytes).digest('hex')}`, + }); + const localRequire = createRequire(absolutePath); + for (const specifier of staticRequireSpecifiers(bytes.toString('utf8'))) { + let resolved; + try { + resolved = localRequire.resolve(specifier); + } catch { + // Generated source can contain documentation examples and optional + // package probes. Only successfully resolved JavaScript can belong to + // the concrete runtime closure for this installation. + continue; + } + if (localDistRelativePath(resolved)) queue.push(fs.realpathSync(resolved)); + } + } + return [...artifacts.values()].sort((left, right) => left.path.localeCompare(right.path)); +}; + +const loadInstalledDataplanPgAdaptor = () => { + const graphileSettingsEntry = path.join( + REPO_ROOT, + 'graphile/graphile-settings/dist/index.js', + ); + const graphileSettingsRequire = createRequire(graphileSettingsEntry); + let postgraphilePgEntry; + try { + postgraphilePgEntry = graphileSettingsRequire.resolve('postgraphile/adaptors/pg'); + } catch { + throw new Error('CTF_INSTALLED_RUNTIME_ARTIFACT_MISSING:postgraphile/adaptors/pg'); + } + const postgraphileRequire = createRequire(postgraphilePgEntry); + let adaptor; + try { + adaptor = postgraphileRequire('@dataplan/pg/adaptors/pg'); + } catch { + throw new Error( + 'CTF_INSTALLED_RUNTIME_ARTIFACT_MISSING:installed:@dataplan/pg:dist/adaptors/pg.js' + ); + } + if (typeof adaptor?.makePgAdaptorWithPgClient !== 'function') { + throw new Error('CTF_DATAPLAN_PREPARED_STATEMENT_ATTESTATION_UNAVAILABLE'); + } + return adaptor; +}; + +let cachedRuntimeArtifactManifest = null; +let cachedRuntimeArtifactFingerprint = null; +const runtimeArtifactManifest = () => { + if (!cachedRuntimeArtifactManifest) { + cachedRuntimeArtifactManifest = Object.freeze({ + version: 2, + roots: Object.freeze([...RUNTIME_ARTIFACT_PATHS]), + localDistClosure: Object.freeze(resolvedLocalRuntimeArtifactManifest()), + installedPatchedArtifacts: Object.freeze(installedRuntimeArtifactManifest()), + }); + } + return cachedRuntimeArtifactManifest; +}; + +const runtimeArtifactFingerprint = () => { + if (!cachedRuntimeArtifactFingerprint) { + cachedRuntimeArtifactFingerprint = `sha256:${crypto.createHash('sha256') + .update(JSON.stringify(runtimeArtifactManifest())) + .digest('hex')}`; + } + return cachedRuntimeArtifactFingerprint; +}; + +const parseBooleanArgument = (value, label, fallback = false) => { + if (value === undefined) return fallback; + if (value === true || value === 'true') return true; + if (value === 'false') return false; + throw new Error(`CTF_INVALID_BOOLEAN:${label}`); +}; + +const provisionAttestationSha256 = ({ + cloneId, + purpose, + customerId, + database, + nonce, +}) => { + const digest = crypto.createHash('sha256'); + for (const value of [ + 'physical-database-density-provision-attestation-v1', + cloneId, + purpose, + customerId, + database, + nonce, + ]) { + digest.update(value); + digest.update('\0'); + } + return `sha256:${digest.digest('hex')}`; +}; + +const validateExpectedProvisionAttestation = (value, customerId, database) => { + if (value == null) return null; + if ( + JSON.stringify(Object.keys(value).sort()) + !== JSON.stringify(['cloneId', 'purpose', 'sha256', 'version']) + || + value?.version !== 1 + || typeof value.cloneId !== 'string' + || !/^[a-z0-9][a-z0-9._-]{0,127}$/i.test(value.cloneId) + || (value.purpose !== 'hostile-preflight' && value.purpose !== 'measurement') + || typeof customerId !== 'string' + || !/^[a-z0-9-]+$/.test(customerId) + || typeof database !== 'string' + || !database + || !/^sha256:[a-f0-9]{64}$/.test(value.sha256 ?? '') + ) { + throw new Error('CTF_PROVISION_ATTESTATION_EXPECTATION_INVALID'); + } + return value; +}; + +const hostileControlEnabledFor = (runPurpose, expectedProvisionAttestation) => + expectedProvisionAttestation == null || runPurpose === 'hostile-preflight'; + +const CONTROL_POOL_MAX = 1; +const PREPARED_STATEMENT_ATTESTATION_KIND = + 'loaded-dataplan-adaptor-behavior-v1'; + +const parseRuntimePoolMaxUses = (value, label = 'runtime-pool-max-uses') => { + if (value === 'unlimited') return null; + if (typeof value !== 'string' || !/^[1-9]\d*$/.test(value)) { + throw new Error(`CTF_INVALID_MAX_USES:${label}`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`CTF_INVALID_MAX_USES:${label}`); + } + return parsed; +}; + +const fixtureConfigurationIdentity = ({ + databaseName, + mode, + introspectionClientReleaseMode, + enableRealtime, + realtimeNotificationMode, + realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs, + runtimeFingerprint, +}) => { + if ( + typeof databaseName !== 'string' + || databaseName.length === 0 + || typeof runtimeFingerprint !== 'string' + || !/^sha256:[a-f0-9]{64}$/.test(runtimeFingerprint) + ) { + throw new Error('CTF_CONFIGURATION_IDENTITY_INPUT_INVALID'); + } + const input = { + version: 1, + fixture: 'complete-tenant-abc-v1', + databaseName, + mode, + introspectionClientReleaseMode, + enableRealtime, + realtimeNotificationMode: enableRealtime ? realtimeNotificationMode : null, + realtimeCursorPollIntervalMs: enableRealtime + ? realtimeCursorPollIntervalMs + : null, + realtimeCursorHeartbeatIntervalMs: enableRealtime + ? realtimeCursorHeartbeatIntervalMs + : null, + runtimeFingerprint, + featureSettings: FEATURE_SETTINGS, + grafastCache: GRAFAST_CACHE_LIMITS, + releaseBuildStateAfterValidation: RELEASE_BUILD_STATE_AFTER_VALIDATION, + }; + return `graphile-configuration:ctf:v1:${crypto.createHash('sha256') + .update(JSON.stringify(input)) + .digest('hex')}`; +}; + +const credentialFreeContractEvidence = (kind, input) => ({ + version: 1, + fingerprint: `${kind}:v1:${crypto.createHash('sha256') + .update(JSON.stringify(input)) + .digest('hex')}`, + input, +}); + +const runtimePoolContractEvidence = ({ + databaseName, + role, + poolMax, + poolMaxUses, + runtimeFingerprint, + purpose = 'runtime', + sanitizeOnCheckout = true, +}) => credentialFreeContractEvidence('pg-contract-evidence', { + version: 1, + databaseName, + role, + pool: { + max: poolMax, + maxUses: poolMaxUses, + }, + purpose, + sanitizeOnCheckout, + runtimeFingerprint, +}); + +const preparedResetBackendEvidence = ( + firstBackendPid, + secondBackendPid, + runtimePoolMaxUses, +) => { + const pidsValid = Number.isSafeInteger(firstBackendPid) + && firstBackendPid > 0 + && Number.isSafeInteger(secondBackendPid) + && secondBackendPid > 0; + const observed = !pidsValid + ? 'invalid' + : firstBackendPid === secondBackendPid + ? 'same-client' + : 'rotated-client'; + const expected = runtimePoolMaxUses === null + ? 'same-client' + : runtimePoolMaxUses === 1 + ? 'rotated-client' + : 'unsupported'; + return { + firstBackendPid: pidsValid ? firstBackendPid : null, + secondBackendPid: pidsValid ? secondBackendPid : null, + observed, + expected, + exact: pidsValid && expected !== 'unsupported' && observed === expected, + }; +}; + +const nativePoolMaxUses = (pool) => { + const value = pool?.options?.maxUses; + if (value === Number.POSITIVE_INFINITY) return { known: true, value: null }; + if (Number.isSafeInteger(value) && value > 0) return { known: true, value }; + return { known: false, value: null }; +}; + +const makeRuntimePoolStats = (pgCache, runtimePoolIdentities, requestedMaxUses) => { + const identities = [...runtimePoolIdentities]; + const distinctIdentities = new Set(identities); + const identitiesUnique = distinctIdentities.size === identities.length; + const recordsAvailable = pgCache?.records instanceof Map; + const records = recordsAvailable + ? identities.map((identity) => pgCache.records.get(identity) ?? null) + : identities.map(() => null); + const pools = records.map((record) => record?.pool ?? null); + const observedPoolObjects = pools.filter(Boolean); + const distinctPoolObjects = new Set(observedPoolObjects); + const poolObjectsUnique = distinctPoolObjects.size === observedPoolObjects.length; + const countsAvailable = pools.every((pool) => + pool + && typeof pool.totalCount === 'number' + && typeof pool.idleCount === 'number' + && typeof pool.waitingCount === 'number' + ); + const effective = pools.map(nativePoolMaxUses); + const effectiveKnown = effective.every((entry) => entry.known); + const effectiveValues = effectiveKnown + ? [...new Set(effective.map((entry) => entry.value))] + : []; + const effectiveMaxUsesKnown = effectiveValues.length === 1; + const effectiveMaxUses = effectiveMaxUsesKnown ? effectiveValues[0] : null; + const observedPools = distinctPoolObjects.size; + const available = recordsAvailable + && identitiesUnique + && poolObjectsUnique + && observedPools === identities.length + && countsAvailable + && effectiveMaxUsesKnown; + return { + scope: 'runtime-only-exact-identities', + available, + requestedMaxUses, + effectiveMaxUses, + effectiveMaxUsesKnown, + maxUsesExact: available && effectiveMaxUses === requestedMaxUses, + identitiesUnique, + poolObjectsUnique, + expectedPools: identities.length, + observedPools, + totalClients: available + ? pools.reduce((sum, pool) => sum + pool.totalCount, 0) + : null, + idleClients: available + ? pools.reduce((sum, pool) => sum + pool.idleCount, 0) + : null, + waitingClients: available + ? pools.reduce((sum, pool) => sum + pool.waitingCount, 0) + : null, + }; +}; + +const preparedStatementCacheRequestFromEnvironment = (environment) => { + const raw = environment.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE; + const normalized = typeof raw === 'string' ? raw.trim() : ''; + const requestedSize = normalized === '' ? 100 : Number(normalized); + if ( + !Number.isSafeInteger(requestedSize) + || requestedSize < 0 + || requestedSize > 10_000 + || (raw != null && ( + typeof raw !== 'string' + || String(requestedSize) !== raw + )) + ) { + throw new Error('CTF_PREPARED_STATEMENT_CACHE_SIZE_INVALID'); + } + return { + environmentValue: typeof raw === 'string' ? raw : null, + requestedSize, + environmentCanonical: typeof raw === 'string' + && String(requestedSize) === raw, + }; +}; + +const attestDataplanPreparedStatementCache = async (adaptor, request) => { + if (typeof adaptor?.makePgAdaptorWithPgClient !== 'function') { + throw new Error('CTF_DATAPLAN_PREPARED_STATEMENT_ATTESTATION_UNAVAILABLE'); + } + const requestedSize = request?.requestedSize; + if (!Number.isSafeInteger(requestedSize) || requestedSize < 0 || requestedSize > 10_000) { + throw new Error('CTF_PREPARED_STATEMENT_CACHE_SIZE_INVALID'); + } + + const parsedStatements = Object.create(null); + const namedQueries = []; + const deallocations = []; + let releases = 0; + const rawClient = { + connection: { parsedStatements }, + addListener() {}, + removeListener() {}, + escapeIdentifier(identifier) { + return `"${String(identifier).replaceAll('"', '""')}"`; + }, + query(query) { + if (typeof query === 'string') { + if (query.startsWith('deallocate ')) { + deallocations.push({ + afterNamedQueries: namedQueries.length, + sql: query, + }); + } + return Promise.resolve({ rows: [], rowCount: 0 }); + } + if (typeof query?.name === 'string') { + namedQueries.push(query.name); + parsedStatements[query.name] = query.text; + } + return Promise.resolve({ rows: [], rowCount: 0 }); + }, + release() { + releases += 1; + }, + }; + const pool = { connect: async () => rawClient }; + const withPgClient = adaptor.makePgAdaptorWithPgClient(pool); + const queryCount = Math.max(1, requestedSize + 1); + await withPgClient(null, async (client) => { + for (let index = 0; index < queryCount; index += 1) { + await client.query({ + text: `select ${index}`, + name: `ctf_prepared_cache_attestation_${index}`, + values: [], + arrayMode: false, + }); + } + }); + // Dataplan's LRU disposer intentionally performs DEALLOCATE asynchronously. + // One turn lets its bookkeeping settle before this proof is published. + await new Promise((resolve) => setImmediate(resolve)); + + const firstEvictionAfterNamedQueries = deallocations[0]?.afterNamedQueries ?? null; + const effectiveSize = namedQueries.length === 0 + ? 0 + : firstEvictionAfterNamedQueries; + const expectedFirstName = 'ctf_prepared_cache_attestation_0'; + const exact = releases === 1 + && ( + requestedSize === 0 + ? namedQueries.length === 0 + && deallocations.length === 0 + && rawClient.connection._graphilePreparedStatementCache == null + : namedQueries.length === queryCount + && deallocations.length === 1 + && firstEvictionAfterNamedQueries === requestedSize + && deallocations[0].sql === `deallocate ${rawClient.escapeIdentifier(expectedFirstName)}` + && rawClient.connection._graphilePreparedStatementCache != null + ); + return { + ...request, + attestation: PREPARED_STATEMENT_ATTESTATION_KIND, + effectiveSize, + effectiveSizeKnown: effectiveSize != null, + exact, + namedQueriesObserved: namedQueries.length, + firstEvictionAfterNamedQueries, + }; +}; + +const parseServerOptions = (argv, environment = process.env) => { + const args = parseArgs(argv); + const host = requireString(args, 'host', '127.0.0.1'); + if (!LOOPBACK_HOSTS.has(host)) throw new Error('CTF_SERVER_LOOPBACK_REQUIRED'); + const mode = requireString(args, 'mode', 'scoped-required'); + if (!['stock', 'scoped-required'].includes(mode)) { + throw new Error(`CTF_INTROSPECTION_MODE_INVALID:${mode}`); + } + const introspectionClientReleaseMode = requireString( + args, + 'introspection-client-release-mode', + 'destroy', + ); + if (!['reuse', 'destroy'].includes(introspectionClientReleaseMode)) { + throw new Error( + `CTF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:${introspectionClientReleaseMode}` + ); + } + const runtimeRoles = Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + requireString( + args, + tenant.runtimeRoleArgument, + environment[`CTF_RUNTIME_${tenant.id.toUpperCase()}_PGUSER`], + ), + ])); + if (new Set(Object.values(runtimeRoles)).size !== TENANTS.length) { + throw new Error('CTF_RUNTIME_ROLES_MUST_BE_DISTINCT'); + } + for (const tenant of TENANTS) { + const password = environment[tenant.runtimePasswordEnvironment] + ?? environment.GRAPHQL_RUNTIME_PGPASSWORD; + if (typeof password !== 'string' || password.length === 0) { + throw new Error(`CTF_RUNTIME_PASSWORD_REQUIRED:${tenant.runtimePasswordEnvironment}`); + } + } + const runtimePoolMax = parsePositiveInteger( + args['runtime-pool-max'] ?? '1', + 'runtime-pool-max', + ); + const runtimePoolMaxUses = parseRuntimePoolMaxUses( + args['runtime-pool-max-uses'] ?? 'unlimited', + ); + const enableRealtime = parseBooleanArgument(args['enable-realtime'], 'enable-realtime'); + const realtimeNotificationMode = requireString( + args, + 'realtime-notification-mode', + 'dedicated', + ); + if (!['dedicated', 'shared-exact'].includes(realtimeNotificationMode)) { + throw new Error( + `CTF_REALTIME_NOTIFICATION_MODE_INVALID:${realtimeNotificationMode}` + ); + } + if (!enableRealtime && realtimeNotificationMode !== 'dedicated') { + throw new Error('CTF_SHARED_REALTIME_REQUIRES_REALTIME'); + } + // Dedicated mode pins the runtime pool's PgSubscriber connection. Shared + // exact mode uses a separate one-client notification pool, so max=1 remains + // a valid runtime density arm. + if (enableRealtime && realtimeNotificationMode === 'dedicated' && runtimePoolMax < 2) { + throw new Error('CTF_REALTIME_REQUIRES_RUNTIME_POOL_MAX_2'); + } + const notificationRole = realtimeNotificationMode === 'shared-exact' + ? requireString( + args, + 'notification-role', + environment.CTF_NOTIFICATION_PGUSER, + ) + : null; + let notificationPasswordAvailable = realtimeNotificationMode === 'shared-exact' + ? environment.CTF_NOTIFICATION_PGPASSWORD + : null; + if ( + realtimeNotificationMode === 'shared-exact' + && ( + typeof notificationPasswordAvailable !== 'string' + || notificationPasswordAvailable.length === 0 + ) + ) { + throw new Error('CTF_NOTIFICATION_PASSWORD_REQUIRED'); + } + if (notificationRole && new Set(Object.values(runtimeRoles)).has(notificationRole)) { + throw new Error('CTF_NOTIFICATION_ROLE_MUST_BE_DISTINCT'); + } + // Keep the listener credential out of the serializable options object. The + // closure exposes exactly one read and cannot enumerate the surrounding + // environment or any runtime-role credential. + const takeNotificationPassword = () => { + const value = notificationPasswordAvailable; + notificationPasswordAvailable = null; + if (realtimeNotificationMode === 'shared-exact' && !value) { + throw new Error('CTF_NOTIFICATION_PASSWORD_ALREADY_CONSUMED'); + } + return value; + }; + return { + host, + port: parsePositiveInteger(args.port ?? '3391', 'port'), + arm: requireString(args, 'arm', 'local-complete-tenant'), + mode, + introspectionClientReleaseMode, + runtimePoolMax, + runtimePoolMaxUses, + preparedStatementCacheRequest: + preparedStatementCacheRequestFromEnvironment(environment), + enableRealtime, + realtimeNotificationMode, + notificationRole, + takeNotificationPassword, + realtimeCursorPollIntervalMs: parsePositiveInteger( + args['realtime-cursor-poll-ms'] ?? '5000', + 'realtime-cursor-poll-ms', + ), + realtimeCursorHeartbeatIntervalMs: parsePositiveInteger( + args['realtime-cursor-heartbeat-ms'] ?? '30000', + 'realtime-cursor-heartbeat-ms', + ), + runtimeRoles, + controlToken: typeof environment.CTF_CONTROL_TOKEN === 'string' + ? environment.CTF_CONTROL_TOKEN + : '', + }; +}; + +const timingSafeTokenEqual = (candidate, expected) => { + if (!candidate || !expected) return false; + const actualBytes = Buffer.from(candidate); + const expectedBytes = Buffer.from(expected); + return actualBytes.length === expectedBytes.length + && crypto.timingSafeEqual(actualBytes, expectedBytes); +}; + +const bearerToken = (request) => { + const value = request.get('authorization'); + return value?.startsWith('Bearer ') ? value.slice('Bearer '.length) : ''; +}; + +const isLoopbackRequest = (request) => { + const address = request.socket?.remoteAddress ?? ''; + return address === '127.0.0.1' || address === '::1' || address === '::ffff:127.0.0.1'; +}; + +const languageCodes = (request) => { + const header = request.get('accept-language') ?? ''; + const parsed = header + .split(',') + .map((part) => part.trim().split(';')[0]?.toLowerCase()) + .filter(Boolean) + .map((part) => part.split('-')[0]); + return [...new Set([...parsed, 'en'])].slice(0, 8); +}; + +const storageModuleFor = (tenant) => ({ + id: `30000000-0000-4000-8000-00000000000${tenant.id}`, + bucketsQualifiedName: `"${tenant.schema}"."app_buckets"`, + filesQualifiedName: `"${tenant.schema}"."app_files"`, + schemaName: tenant.schema, + bucketsTableName: 'app_buckets', + filesTableName: 'app_files', + scope: 'app', + entityTableId: null, + entityQualifiedName: null, + endpoint: null, + publicUrlPrefix: null, + provider: 'minio', + allowedOrigins: ['http://127.0.0.1'], + uploadUrlExpirySeconds: 900, + downloadUrlExpirySeconds: 3600, + defaultMaxFileSize: 1024 * 1024, + maxFilenameLength: 1024, + cacheTtlSeconds: 300, + hasPathShares: false, + maxBulkFiles: 100, + maxBulkTotalSize: 1024 * 1024, +}); + +const computeFor = (tenant) => { + const module = { + schemaName: tenant.schema, + bindingsTableName: 'fixture_preloaded_bindings', + definitionsTableName: 'fixture_preloaded_definitions', + invocationsSchemaName: tenant.schema, + invocationsTableName: 'function_invocations', + invocationsEntityField: 'database_id', + }; + return { + modules: [module], + bindings: [{ + bindingId: `50000000-0000-4000-8000-00000000000${tenant.id}`, + alias: 'fixture_task', + config: { graphql: true }, + functionDefinitionId: `60000000-0000-4000-8000-00000000000${tenant.id}`, + taskIdentifier: `ctf.fixture.${tenant.id}`, + description: 'Complete-tenant fixture task', + payloadArgs: null, + module, + }], + }; +}; + +const pluginComputeFor = (compute) => ({ + modules: compute.modules.map((module) => ({ + computeSchema: module.schemaName, + bindingsTable: module.bindingsTableName, + definitionsTable: module.definitionsTableName, + invocationsSchema: module.invocationsSchemaName, + invocationsTable: module.invocationsTableName, + invocationsEntityField: module.invocationsEntityField, + })), + bindings: compute.bindings.map((binding) => ({ + ...binding, + module: { + computeSchema: binding.module.schemaName, + bindingsTable: binding.module.bindingsTableName, + definitionsTable: binding.module.definitionsTableName, + invocationsSchema: binding.module.invocationsSchemaName, + invocationsTable: binding.module.invocationsTableName, + invocationsEntityField: binding.module.invocationsEntityField, + }, + })), +}); + +const deterministicLlmPlugin = () => ({ + // Downstream LLM plugins declare an ordering dependency on this canonical + // name. The production module plugin is deliberately replaced because this + // lane must not acquire an external provider during an offline run. + name: 'LlmModulePlugin', + version: '1.0.0', + schema: { + hooks: { + build(build) { + const embedder = async () => ({ embedding: [1, 0, 0], promptTokens: 5 }); + const chatCompleter = async (messages) => { + const prompt = messages.find((message) => message.role === 'user')?.content ?? ''; + return { + content: `Deterministic fixture answer: ${prompt}`, + usage: { + input: 10, + output: 10, + reasoning: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 20, + }, + }; + }; + return build.extend(build, { + llmEmbedder: embedder, + llmChatCompleter: chatCompleter, + llmEmbeddingModel: 'ctf-deterministic-3d-v1', + llmChatModel: 'ctf-deterministic-chat-v1', + }, 'Complete-tenant deterministic LLM provider'); + }, + }, + }, +}); + +const loadRuntime = () => { + const express = require(path.join(REPO_ROOT, 'graphql/server/node_modules/express')); + const { S3Client } = require(path.join( + REPO_ROOT, + 'graphql/server/node_modules/@aws-sdk/client-s3', + )); + return { + express, + S3Client, + ...requireBuilt('graphile/graphile-cache/dist/index.js'), + realtimeSubscriptions: requireBuilt( + 'graphile/graphile-realtime-subscriptions/dist/index.js' + ), + graphileSettings: requireBuilt('graphile/graphile-settings/dist/index.js'), + pgCacheApi: requireBuilt('postgres/pg-cache/dist/index.js'), + expressContext: requireBuilt('packages/express-context/dist/index.js'), + llm: requireBuilt('graphile/graphile-llm/dist/index.js'), + functionBindings: requireBuilt('graphile/graphile-function-bindings/dist/index.js'), + presigned: requireBuilt('graphile/graphile-presigned-url-plugin/dist/index.js'), + authCookie: requireBuilt('graphql/server/dist/plugins/auth-cookie-plugin.js'), + dataplanPgAdaptor: loadInstalledDataplanPgAdaptor(), + pgEnv: require(path.join(REPO_ROOT, 'graphql/server/node_modules/pg-env')), + buildContractApi: requireBuilt('graphql/server/dist/middleware/graphile-build-contract.js'), + buildGovernor: requireBuilt('graphql/server/dist/middleware/graphile-build-governor.js'), + roleSafety: requireBuilt('graphql/server/dist/middleware/runtime-role-safety.js'), + buildStats: requireBuilt('graphql/server/dist/middleware/observability/graphile-build-stats.js'), + debugMemory: requireBuilt('graphql/server/dist/diagnostics/debug-memory-snapshot.js'), + }; +}; + +const createFixtureServer = async (options, environment = process.env) => { + const runtime = loadRuntime(); + const preparedStatementCache = await attestDataplanPreparedStatementCache( + runtime.dataplanPgAdaptor, + options.preparedStatementCacheRequest, + ); + if (!preparedStatementCache.exact) { + throw new Error('CTF_DATAPLAN_PREPARED_STATEMENT_CACHE_MISMATCH'); + } + const { + createGraphileInstance, + deleteGraphileCacheEntry, + disposeUncachedEntry, + graphileCache, + invokeEntryHandler, + invokeEntryUpgradeHandler, + prepareCacheForBuild, + getGraphileRealtimeRoleAuditStats, + revalidateEntryRealtimeRole, + } = runtime; + const { + acquirePgPool, + getPgNotificationBrokerIdentity, + getPgNotificationBrokerStats, + getPgPoolIdentity, + pgCache, + teardownPgNotificationBrokers, + teardownPgPools, + } = runtime.pgCacheApi; + const { createGraphileBuildContract, hashGraphileBuildContract } = runtime.buildContractApi; + const { runGraphileBuild, getGraphileGovernorCounters } = runtime.buildGovernor; + const { ensureRuntimeRoleSafety, invalidateRuntimeRoleSafety } = runtime.roleSafety; + const { observeGraphileBuild, getGraphileBuildStats } = runtime.buildStats; + const { getDebugMemorySnapshot } = runtime.debugMemory; + const { buildPgSettings } = runtime.expressContext; + const { + createConstructivePreset, + createGrafastCacheLimitsPreset, + makePgService, + } = runtime.graphileSettings; + const { + createLlmRagPlugin, + createLlmTextMutationPlugin, + createLlmTextSearchPlugin, + } = runtime.llm; + const { createFunctionBindingsPlugin } = runtime.functionBindings; + const { PresignedUrlPreset } = runtime.presigned; + const { AuthCookiePlugin } = runtime.authCookie; + const { + ActivatableGenerationScopedRealtimeSubscriber, + RealtimeTopicCollector, + } = runtime.realtimeSubscriptions; + + const controlPgConfig = { + ...runtime.pgEnv.getPgEnvOptions({}), + // Control-plane queries are trusted and short-lived. Keep their pool shape + // constant so runtime-pool experiments cannot alter the measured baseline. + pool: { max: CONTROL_POOL_MAX, maxUses: 0 }, + }; + const runtimeFingerprint = runtimeArtifactFingerprint(); + const configurationIdentity = fixtureConfigurationIdentity({ + databaseName: controlPgConfig.database, + mode: options.mode, + introspectionClientReleaseMode: options.introspectionClientReleaseMode, + enableRealtime: options.enableRealtime, + realtimeNotificationMode: options.realtimeNotificationMode, + realtimeCursorPollIntervalMs: options.realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs: + options.realtimeCursorHeartbeatIntervalMs, + runtimeFingerprint, + }); + if (!/^graphile-configuration:ctf:v1:[a-f0-9]{64}$/.test(configurationIdentity)) { + throw new Error('CTF_CONFIGURATION_IDENTITY_INVALID'); + } + const runtimePoolOptions = { purpose: 'runtime', sanitizeOnCheckout: true }; + const notificationPassword = options.realtimeNotificationMode === 'shared-exact' + ? options.takeNotificationPassword() + : null; + const notificationPgConfig = options.realtimeNotificationMode === 'shared-exact' + ? { + host: controlPgConfig.host, + port: controlPgConfig.port, + database: controlPgConfig.database, + user: options.notificationRole, + password: notificationPassword, + // The notification broker is long-lived and must never inherit the + // runtime-only single-checkout experiment from process environment. + pool: { max: 1, maxUses: 0 }, + } + : null; + const realtimeListenerIdentity = notificationPgConfig + ? getPgNotificationBrokerIdentity(notificationPgConfig) + : null; + const realtimeListenerContractEvidence = notificationPgConfig + ? runtimePoolContractEvidence({ + databaseName: controlPgConfig.database, + role: options.notificationRole, + poolMax: 1, + poolMaxUses: null, + runtimeFingerprint, + purpose: 'notification-listener', + sanitizeOnCheckout: false, + }) + : null; + const expectedProvisionAttestation = validateExpectedProvisionAttestation( + options.provisionAttestation, + options.provisionCustomerId, + controlPgConfig.database, + ); + if ( + expectedProvisionAttestation + && ( + options.runPurpose !== expectedProvisionAttestation.purpose + || options.cloneId !== expectedProvisionAttestation.cloneId + ) + ) { + throw new Error('CTF_PROVISION_ATTESTATION_RUN_MISMATCH'); + } + const hostileControlEnabled = hostileControlEnabledFor( + options.runPurpose, + expectedProvisionAttestation, + ); + const tenantState = new Map(); + const buildContractEvidenceByLiveIdentity = new Map(); + + for (const tenant of TENANTS) { + const role = options.runtimeRoles[tenant.id]; + const password = environment[tenant.runtimePasswordEnvironment] + ?? environment.GRAPHQL_RUNTIME_PGPASSWORD; + const pgConfig = { + host: controlPgConfig.host, + port: controlPgConfig.port, + database: controlPgConfig.database, + user: role, + password, + pool: { + max: options.runtimePoolMax, + // An explicit zero is pg-cache's unlimited sentinel and prevents an + // ambient PG_POOL_MAX_USES value from contaminating the baseline arm. + maxUses: options.runtimePoolMaxUses ?? 0, + }, + }; + const poolIdentity = getPgPoolIdentity(pgConfig, runtimePoolOptions); + const poolContractEvidence = runtimePoolContractEvidence({ + databaseName: controlPgConfig.database, + role, + poolMax: options.runtimePoolMax, + poolMaxUses: options.runtimePoolMaxUses, + runtimeFingerprint, + }); + const storage = { modules: [storageModuleFor(tenant)] }; + const compute = computeFor(tenant); + const realtimeSchema = realtimeSchemaFor(tenant); + const runtimeDependencySchemas = runtimeDependencySchemasFor( + tenant, + options.enableRealtime, + ); + const contractInput = { + configurationIdentity, + poolIdentity, + databaseId: tenant.databaseId, + databaseName: controlPgConfig.database, + apiId: tenant.apiId, + schemas: [tenant.schema], + authenticatedRole: role, + anonymousRole: role, + pluginSettings: FEATURE_SETTINGS, + graphileSettings: { + releaseBuildStateAfterValidation: RELEASE_BUILD_STATE_AFTER_VALIDATION, + introspectionMode: options.mode, + introspectionClientReleaseMode: options.introspectionClientReleaseMode, + introspectionDependencySchemas: [...INTROSPECTION_DEPENDENCY_SCHEMAS], + grafastCache: GRAFAST_CACHE_LIMITS, + realtimeNotificationMode: options.realtimeNotificationMode, + realtimeCursorPollIntervalMs: options.realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs: + options.realtimeCursorHeartbeatIntervalMs, + fixturePluginConfiguration: { + authCookie: true, + llmProvider: 'deterministic-3d-v1', + storageProvider: 'offline-signing-only-v1', + functionBindings: 'preloaded-v1', + runtimeFingerprint, + }, + }, + compute, + storage, + isPublic: false, + // The complete-tenant lane defaults this off and delegates delivery to + // the mandatory graphql-ws integration suite. Physical-database research + // lanes may opt in after provisioning the required realtime cursor schema. + enableRealtime: options.enableRealtime, + realtimeSchema, + realtimeNotificationMode: options.realtimeNotificationMode, + realtimeListenerPoolIdentity: realtimeListenerIdentity ?? undefined, + realtimeNotificationRoleRevalidationMs: 60_000, + realtimeCursorPollIntervalMs: options.realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs: + options.realtimeCursorHeartbeatIntervalMs, + graphiql: false, + graphiqlOnGraphQLGET: false, + explain: false, + introspectionMode: options.mode, + introspectionClientReleaseMode: options.introspectionClientReleaseMode, + }; + const contract = createGraphileBuildContract(contractInput); + const evidenceContract = createGraphileBuildContract({ + ...contractInput, + poolIdentity: poolContractEvidence.fingerprint, + realtimeListenerPoolIdentity: realtimeListenerContractEvidence?.fingerprint, + }); + const buildContractEvidence = credentialFreeContractEvidence( + 'graphile-contract-evidence', + evidenceContract, + ); + if ( + contract.surface.graphiql !== false + || contract.surface.graphiqlOnGraphQLGET !== false + || contract.surface.realtimeSchema !== (options.enableRealtime ? realtimeSchema : null) + || contract.surface.realtimeNotificationMode + !== (options.enableRealtime ? options.realtimeNotificationMode : null) + || contract.surface.realtimeListenerPoolIdentity + !== (options.enableRealtime && options.realtimeNotificationMode === 'shared-exact' + ? realtimeListenerIdentity + : null) + ) { + throw new Error('CTF_BUILD_CONTRACT_SURFACE_FLAGS_UNSUPPORTED'); + } + const cacheKey = hashGraphileBuildContract(contract); + buildContractEvidenceByLiveIdentity.set( + cacheKey, + buildContractEvidence.fingerprint, + ); + tenantState.set(tenant.id, { + tenant, + role, + pgConfig, + poolIdentity, + storage, + compute, + realtimeSchema, + runtimeDependencySchemas, + realtimeListenerIdentity, + poolContractEvidence, + buildContractEvidence, + cacheKey, + }); + } + + const runtimePoolStats = () => makeRuntimePoolStats( + pgCache, + TENANTS.map((tenant) => tenantState.get(tenant.id).poolIdentity), + options.runtimePoolMaxUses, + ); + const runtimePoolObjects = () => TENANTS.map((tenant) => { + const identity = tenantState.get(tenant.id).poolIdentity; + return pgCache.records instanceof Map + ? pgCache.records.get(identity)?.pool ?? null + : null; + }); + + let activeBuilds = 0; + let maxConcurrentBuilds = 0; + const buildCounts = Object.fromEntries(TENANTS.map((tenant) => [tenant.id, 0])); + const buildGenerations = Object.fromEntries(TENANTS.map((tenant) => [tenant.id, 0])); + const inFlight = new Map(); + + const withLease = async (pgConfig, poolOptions, callback) => { + const lease = acquirePgPool(pgConfig, poolOptions); + try { + return await callback(lease.pool); + } finally { + lease.release(); + } + }; + + const readProvisionAttestation = async () => { + if (!expectedProvisionAttestation) return null; + const result = await withLease( + controlPgConfig, + { purpose: 'control', sanitizeOnCheckout: false }, + (pool) => pool.query(` + SELECT clone_id, + run_purpose, + customer_id, + attestation_nonce, + attestation_sha256, + pg_catalog.current_database()::text AS database + FROM ctf_provision_private.clone_attestation + WHERE singleton = true + `), + ); + if (result.rowCount !== 1) throw new Error('CTF_PROVISION_ATTESTATION_ROW_INVALID'); + const row = result.rows[0]; + if ( + typeof row.clone_id !== 'string' + || !/^[a-z0-9][a-z0-9._-]{0,127}$/i.test(row.clone_id) + || (row.run_purpose !== 'hostile-preflight' && row.run_purpose !== 'measurement') + || typeof row.customer_id !== 'string' + || !/^[a-z0-9-]+$/.test(row.customer_id) + || typeof row.database !== 'string' + || typeof row.attestation_nonce !== 'string' + || !/^[a-f0-9]{64}$/.test(row.attestation_nonce) + || !/^sha256:[a-f0-9]{64}$/.test(row.attestation_sha256 ?? '') + ) { + throw new Error('CTF_PROVISION_ATTESTATION_ROW_INVALID'); + } + const calculatedSha256 = provisionAttestationSha256({ + cloneId: row.clone_id, + purpose: row.run_purpose, + customerId: row.customer_id, + database: row.database, + nonce: row.attestation_nonce, + }); + if ( + row.clone_id !== expectedProvisionAttestation.cloneId + || row.run_purpose !== expectedProvisionAttestation.purpose + || row.customer_id !== options.provisionCustomerId + || row.database !== controlPgConfig.database + || row.attestation_sha256 !== expectedProvisionAttestation.sha256 + || calculatedSha256 !== row.attestation_sha256 + ) { + throw new Error('CTF_PROVISION_ATTESTATION_MISMATCH'); + } + return { + version: 1, + cloneId: row.clone_id, + purpose: row.run_purpose, + customerId: row.customer_id, + database: row.database, + sha256: calculatedSha256, + verified: true, + }; + }; + + // Refuse to publish any physical fixture until its opaque database nonce has + // been queried and matched to the credential-free manifest digest. + const initialProvisionAttestation = await readProvisionAttestation(); + + await Promise.all(TENANTS.map(async (tenant) => { + const state = tenantState.get(tenant.id); + await withLease(state.pgConfig, runtimePoolOptions, (pool) => + ensureRuntimeRoleSafety( + pool, + [state.role], + [tenant.schema], + state.runtimeDependencySchemas, + ) + ); + })); + + const s3Client = new runtime.S3Client({ + endpoint: 'http://127.0.0.1:9', + region: 'us-east-1', + forcePathStyle: true, + credentials: { + accessKeyId: 'ctf-offline-signing-only', + secretAccessKey: 'ctf-offline-signing-only', + }, + }); + + const makePreset = (state, pool, sharedRealtimeBuild = null) => { + const pluginCompute = pluginComputeFor(state.compute); + return { + extends: [ + createConstructivePreset({ + ...FEATURE_SETTINGS, + enableLlm: false, + enablePresignedUploads: false, + preloadedStorageModules: [], + ...(sharedRealtimeBuild ? { + realtimeSubscriptions: { + onTopicsDiscovered: sharedRealtimeBuild.topicCollector.collect, + }, + } : {}), + }), + PresignedUrlPreset({ + s3: { + client: s3Client, + bucket: `ctf-${state.tenant.id}-offline`, + endpoint: 'http://127.0.0.1:9', + region: 'us-east-1', + forcePathStyle: true, + }, + preloadedStorageModules: state.storage.modules, + }), + createGrafastCacheLimitsPreset(GRAFAST_CACHE_LIMITS), + ], + plugins: [ + AuthCookiePlugin, + deterministicLlmPlugin(), + createLlmTextSearchPlugin({ onQuotaExceeded: 'throw' }), + createLlmTextMutationPlugin(), + createLlmRagPlugin({ contextLimit: 2, maxTokens: 256 }), + createFunctionBindingsPlugin({ + apiId: state.tenant.apiId, + modules: pluginCompute.modules, + preloadedBindings: pluginCompute.bindings, + }), + ], + pgServices: [makePgService({ + pool, + schemas: [state.tenant.schema], + introspectionMode: options.mode, + introspectionClientReleaseMode: options.introspectionClientReleaseMode, + introspectionScopedCatalogTypes: options.mode === 'scoped-required' + ? 'dependency-closure' + : undefined, + introspectionAllowedDependencySchemas: [...INTROSPECTION_DEPENDENCY_SCHEMAS], + ...(sharedRealtimeBuild ? { + pubsub: false, + pgSubscriber: sharedRealtimeBuild.subscriber, + } : {}), + })], + schema: { + releaseBuildStateAfterValidation: RELEASE_BUILD_STATE_AFTER_VALIDATION, + }, + grafserv: { + graphqlPath: '/graphql', + graphiql: false, + graphiqlOnGraphQLGET: false, + websockets: options.enableRealtime, + }, + grafast: { + explain: false, + context: (requestContext) => { + const request = requestContext?.expressv4?.req; + const api = request?.api ?? { + dbname: controlPgConfig.database, + schema: [state.tenant.schema], + anonRole: state.role, + roleName: state.role, + databaseId: state.tenant.databaseId, + apiId: state.tenant.apiId, + isPublic: false, + }; + return { + pgSettings: buildPgSettings({ + api, + token: null, + requestId: request?.requestId ?? crypto.randomUUID(), + dependencySchemas: [...INTROSPECTION_DEPENDENCY_SCHEMAS], + }), + langCodes: request ? languageCodes(request) : ['es', 'en'], + }; + }, + }, + }; + }; + + const buildEntry = (state) => { + const resident = graphileCache.get(state.cacheKey); + if (resident && !resident.disposing) return Promise.resolve(resident); + const existing = inFlight.get(state.cacheKey); + if (existing) return existing; + + const buildGeneration = buildGenerations[state.tenant.id]; + const pending = runGraphileBuild(async () => { + await prepareCacheForBuild(); + const lease = acquirePgPool(state.pgConfig, runtimePoolOptions); + let entry = null; + let leaseOwnedByEntry = false; + const sharedRealtimeBuild = options.realtimeNotificationMode === 'shared-exact' + ? { + subscriber: new ActivatableGenerationScopedRealtimeSubscriber(), + topicCollector: new RealtimeTopicCollector(), + } + : null; + let sharedRealtimeOwnedByEntry = false; + activeBuilds += 1; + maxConcurrentBuilds = Math.max(maxConcurrentBuilds, activeBuilds); + buildCounts[state.tenant.id] += 1; + try { + await ensureRuntimeRoleSafety( + lease.pool, + [state.role], + [state.tenant.schema], + state.runtimeDependencySchemas, + ); + entry = await observeGraphileBuild({ + cacheKey: state.cacheKey, + serviceKey: `ctf-${state.tenant.id}-api`, + databaseId: state.tenant.databaseId, + }, () => createGraphileInstance({ + preset: makePreset(state, lease.pool, sharedRealtimeBuild), + cacheKey: state.cacheKey, + poolIdentity: state.poolIdentity, + poolLease: lease, + serviceKey: `ctf-${state.tenant.id}-api`, + databaseId: state.tenant.databaseId, + enableRealtime: options.enableRealtime, + enableWebsockets: options.enableRealtime, + realtimeSchema: state.realtimeSchema, + realtimeSourceSchemas: [state.tenant.schema], + realtimeCursorPollIntervalMs: options.realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs: + options.realtimeCursorHeartbeatIntervalMs, + ...(sharedRealtimeBuild && notificationPgConfig && realtimeListenerIdentity ? { + sharedRealtime: { + ...sharedRealtimeBuild, + listenerPgConfig: notificationPgConfig, + listenerIdentity: realtimeListenerIdentity, + roleRevalidationMs: 60_000, + }, + } : {}), + }), { enabled: true }); + sharedRealtimeOwnedByEntry = Boolean(sharedRealtimeBuild); + leaseOwnedByEntry = true; + if (buildGeneration !== buildGenerations[state.tenant.id]) { + throw new Error(`CTF_BUILD_INVALIDATED:${state.tenant.id}`); + } + graphileCache.set(state.cacheKey, entry); + if (graphileCache.get(state.cacheKey) !== entry) { + throw new Error(`CTF_CACHE_PUBLICATION_FAILED:${state.tenant.id}`); + } + return entry; + } catch (error) { + if (leaseOwnedByEntry && entry) { + await disposeUncachedEntry(entry, state.cacheKey).catch(() => undefined); + } else { + lease.release(); + } + throw error; + } finally { + if (sharedRealtimeBuild && !sharedRealtimeOwnedByEntry) { + await sharedRealtimeBuild.subscriber.release().catch(() => undefined); + } + activeBuilds -= 1; + } + }); + inFlight.set(state.cacheKey, pending); + void pending.finally(() => { + if (inFlight.get(state.cacheKey) === pending) inFlight.delete(state.cacheKey); + }).catch(() => undefined); + return pending; + }; + + const invalidateTenant = async (tenantId) => { + const state = tenantState.get(tenantId); + if (!state) throw new Error(`CTF_UNKNOWN_TENANT:${tenantId}`); + buildGenerations[tenantId] += 1; + const pending = inFlight.get(state.cacheKey); + if (pending) await pending.catch(() => undefined); + const lease = acquirePgPool(state.pgConfig, runtimePoolOptions); + invalidateRuntimeRoleSafety(lease.pool); + lease.release(); + await deleteGraphileCacheEntry(state.cacheKey); + }; + + const handleUpgrade = async (request, socket, head, { pathPrefix = '' } = {}) => { + if (!options.enableRealtime || request.aborted || socket.destroyed) return false; + const tenantId = matchTenantUpgradePath(request.url, pathPrefix); + if (!tenantId) return false; + const state = tenantState.get(tenantId); + if (!state) return false; + const protocols = String(request.headers['sec-websocket-protocol'] ?? '') + .split(',') + .map((value) => value.trim()); + if (!protocols.includes('graphql-transport-ws')) return false; + + request.api = { + apiId: state.tenant.apiId, + databaseId: state.tenant.databaseId, + dbname: controlPgConfig.database, + schema: [state.tenant.schema], + anonRole: state.role, + roleName: state.role, + isPublic: false, + databaseSettings: FEATURE_SETTINGS, + }; + request.token = null; + request.requestId = request.headers['x-request-id'] ?? crypto.randomUUID(); + + for (let attempt = 0; attempt < 2; attempt += 1) { + const entry = graphileCache.get(state.cacheKey) ?? await buildEntry(state); + await ensureRuntimeRoleSafety( + entry.poolLease.pool, + [state.role], + [state.tenant.schema], + state.runtimeDependencySchemas, + ); + await revalidateEntryRealtimeRole(entry); + if (invokeEntryUpgradeHandler(entry, request, socket, head)) return true; + if (request.aborted || socket.destroyed) return true; + } + return false; + }; + + const app = runtime.express(); + app.disable('x-powered-by'); + app.use(runtime.express.json({ limit: '256kb' })); + + app.get('/healthz', (_request, response) => { + const governor = getGraphileGovernorCounters(); + response.status(governor.restartRequired ? 503 : 200).json({ + status: governor.restartRequired ? 'unhealthy' : 'ok', + }); + }); + + app.get('/debug/memory', (request, response) => { + if (!isLoopbackRequest(request)) { + response.status(404).send('Not found'); + return; + } + const configuredToken = environment.GRAPHQL_OBSERVABILITY_TOKEN ?? ''; + if ( + environment.NODE_ENV !== 'development' + && !timingSafeTokenEqual(bearerToken(request), configuredToken) + ) { + response.status(401).json({ error: { code: 'CTF_OBSERVABILITY_UNAUTHORIZED' } }); + return; + } + response.json(getDebugMemorySnapshot()); + }); + + app.get('/__ctf/status', async (request, response, next) => { + if (!isLoopbackRequest(request)) { + response.status(404).send('Not found'); + return; + } + try { + const liveProvisionAttestation = await readProvisionAttestation(); + response.json({ + version: 1, + fixture: 'complete-tenant-abc-v1', + arm: options.arm, + introspectionMode: options.mode, + introspectionClientReleaseMode: options.introspectionClientReleaseMode, + releaseBuildStateAfterValidation: RELEASE_BUILD_STATE_AFTER_VALIDATION, + runtimeArtifactFingerprint: runtimeFingerprint, + configurationIdentity, + liveIdentityScope: 'process-local-keyed-hmac-v1', + physicalIsolation: 'dedicated-login-and-pool-per-tenant', + sharedRuntimePool: false, + runtimePoolMax: options.runtimePoolMax, + runtimePoolMaxUses: options.runtimePoolMaxUses, + runtimePools: runtimePoolStats(), + preparedStatementCache, + enableRealtime: options.enableRealtime, + realtimeNotificationMode: options.realtimeNotificationMode, + realtimeListenerIdentity, + realtimeCursorPollIntervalMs: options.realtimeCursorPollIntervalMs, + realtimeCursorHeartbeatIntervalMs: + options.realtimeCursorHeartbeatIntervalMs, + realtimeNotificationBrokers: getPgNotificationBrokerStats(), + realtimeRoleAudits: getGraphileRealtimeRoleAuditStats(), + realtimeSchemas: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).realtimeSchema, + ])), + physicalDatabase: controlPgConfig.database, + runPurpose: options.runPurpose ?? null, + provisionAttestation: liveProvisionAttestation, + runtimePoolIdentities: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).poolIdentity, + ])), + runtimeBindings: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + databaseId: tenant.databaseId, + databaseName: controlPgConfig.database, + role: tenantState.get(tenant.id).role, + schemas: [tenant.schema], + }, + ])), + controlAvailable: hostileControlEnabled + && Buffer.byteLength(options.controlToken) >= 32, + buildContracts: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).cacheKey, + ])), + residentBuildContracts: [...graphileCache.keys()], + contractEvidence: { + version: 1, + credentialFree: true, + configurationIdentity, + realtimeListener: realtimeListenerContractEvidence, + runtimePools: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).poolContractEvidence, + ])), + graphileBuilds: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).buildContractEvidence, + ])), + residentGraphileBuildFingerprints: [...graphileCache.keys()] + .map((cacheKey) => buildContractEvidenceByLiveIdentity.get(cacheKey)) + .filter(Boolean), + }, + builds: { + active: activeBuilds, + maxConcurrent: maxConcurrentBuilds, + byTenant: { ...buildCounts }, + generations: { ...buildGenerations }, + inFlight: [...inFlight.keys()], + graphile: getGraphileBuildStats(), + }, + runtimeSafety: { + passed: true, + rolesDistinct: true, + dependencySchemasByTenant: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).runtimeDependencySchemas, + ])), + }, + }); + } catch (error) { + next(error); + } + }); + + app.post('/__ctf/control', async (request, response, next) => { + try { + if (!hostileControlEnabled) { + response.status(404).send('Not found'); + return; + } + if ( + !isLoopbackRequest(request) + || Buffer.byteLength(options.controlToken) < 32 + || !timingSafeTokenEqual(bearerToken(request), options.controlToken) + ) { + response.status(404).send('Not found'); + return; + } + const action = request.body?.action; + const tenantId = request.body?.tenant; + if (action === 'invalidate-all') { + await Promise.all(TENANTS.map((tenant) => invalidateTenant(tenant.id))); + const identity = await withLease( + controlPgConfig, + { purpose: 'control', sanitizeOnCheckout: false }, + (pool) => pool.query( + 'SELECT pg_catalog.current_database()::text AS physical_database_identity', + ), + ); + response.json({ + ok: true, + action, + physicalDatabaseIdentity: identity.rows[0]?.physical_database_identity, + }); + return; + } + const state = tenantState.get(tenantId); + if (!state) { + response.status(400).json({ error: { code: 'CTF_UNKNOWN_TENANT' } }); + return; + } + if (action === 'poison') { + const observed = await withLease(state.pgConfig, runtimePoolOptions, async (pool) => { + const client = await pool.connect(); + try { + const result = await client.query( + `SELECT ${state.tenant.schema}.poison_session() AS value, ` + + 'pg_catalog.current_database()::text AS physical_database_identity', + ); + return result.rows[0]; + } finally { + client.release(); + } + }); + response.json({ + ok: observed?.value === 'poisoned', + action, + tenant: tenantId, + physicalDatabaseIdentity: observed?.physical_database_identity, + }); + return; + } + if (action === 'rollback-savepoint') { + const observed = await withLease(state.pgConfig, runtimePoolOptions, async (pool) => { + const client = await pool.connect(); + try { + await client.query('SELECT pg_catalog.set_config($1, $2, false)', [ + 'jwt.claims.database_id', + state.tenant.databaseId, + ]); + const result = await client.query( + `SELECT ${state.tenant.schema}.savepoint_identity() AS value, ` + + 'pg_catalog.current_database()::text AS physical_database_identity', + ); + return result.rows[0]; + } finally { + client.release(); + } + }); + response.json({ + ok: observed?.value === state.tenant.databaseId, + action, + tenant: tenantId, + observed: observed?.value, + physicalDatabaseIdentity: observed?.physical_database_identity, + }); + return; + } + if (action === 'prepared-reset') { + const statementName = `ctf-prepared-reset-${tenantId}`; + const first = await withLease(state.pgConfig, runtimePoolOptions, async (pool) => { + const client = await pool.connect(); + try { + await client.query('SELECT pg_catalog.set_config($1, $2, false)', [ + 'jwt.claims.database_id', + state.tenant.databaseId, + ]); + const result = await client.query({ + name: statementName, + text: `SELECT ${state.tenant.schema}.tenant_identity() AS value, ` + + 'pg_catalog.current_database()::text AS physical_database_identity, ' + + 'pg_catalog.pg_backend_pid()::integer AS backend_pid, ' + + 'current_user::text AS runtime_role', + }); + return result.rows[0]; + } finally { + client.release(); + } + }); + const second = await withLease(state.pgConfig, runtimePoolOptions, async (pool) => { + const client = await pool.connect(); + try { + await client.query('SELECT pg_catalog.set_config($1, $2, false)', [ + 'jwt.claims.database_id', + state.tenant.databaseId, + ]); + const result = await client.query({ + name: statementName, + text: `SELECT ${state.tenant.schema}.request_identity() AS value, ` + + 'pg_catalog.current_database()::text AS physical_database_identity, ' + + 'pg_catalog.pg_backend_pid()::integer AS backend_pid, ' + + 'current_user::text AS runtime_role', + }); + return result.rows[0]; + } finally { + client.release(); + } + }); + const backend = preparedResetBackendEvidence( + first?.backend_pid, + second?.backend_pid, + options.runtimePoolMaxUses, + ); + response.json({ + ok: backend.exact + && first?.value === state.tenant.token + && second?.value === `${state.tenant.token}:${state.tenant.databaseId}` + && first?.physical_database_identity === second?.physical_database_identity + && first?.runtime_role === state.role + && second?.runtime_role === state.role, + action, + tenant: tenantId, + first: first?.value, + second: second?.value, + runtimeRole: first?.runtime_role, + backend, + physicalDatabaseIdentity: first?.physical_database_identity, + }); + return; + } + if (action === 'bad-role-expected-failure') { + const controlIdentity = await withLease( + controlPgConfig, + { purpose: 'control', sanitizeOnCheckout: false }, + (pool) => pool.query( + 'SELECT current_user::text AS role_name, ' + + 'pg_catalog.current_database()::text AS physical_database_identity', + ), + ); + let rejectedCode = null; + try { + await withLease(state.pgConfig, runtimePoolOptions, (pool) => + ensureRuntimeRoleSafety( + pool, + [state.role, controlIdentity.rows[0]?.role_name], + [state.tenant.schema], + state.runtimeDependencySchemas, + ) + ); + } catch (error) { + if (error?.code !== 'GRAPHILE_UNSAFE_RUNTIME_ROLE') throw error; + rejectedCode = error.code; + } + response.json({ + ok: rejectedCode === 'GRAPHILE_UNSAFE_RUNTIME_ROLE', + action, + tenant: tenantId, + rejectedCode, + physicalDatabaseIdentity: + controlIdentity.rows[0]?.physical_database_identity, + }); + return; + } + if (action === 'drift-apply' || action === 'drift-revert') { + const functionName = action === 'drift-apply' + ? 'apply_schema_drift' + : 'revert_schema_drift'; + const result = await withLease( + controlPgConfig, + { purpose: 'control', sanitizeOnCheckout: false }, + (pool) => pool.query( + `SELECT ctf_control.${functionName}($1), ` + + 'pg_catalog.current_database()::text AS physical_database_identity', + [state.tenant.schema], + ), + ); + await invalidateTenant(tenantId); + response.json({ + ok: true, + action, + tenant: tenantId, + physicalDatabaseIdentity: + result.rows[0]?.physical_database_identity, + }); + return; + } + response.status(400).json({ error: { code: 'CTF_UNKNOWN_CONTROL_ACTION' } }); + } catch (error) { + next(error); + } + }); + + for (const tenant of TENANTS) { + const state = tenantState.get(tenant.id); + app.use(`/tenant/${tenant.id}`, async (request, response, next) => { + try { + request.api = { + apiId: tenant.apiId, + databaseId: tenant.databaseId, + dbname: controlPgConfig.database, + schema: [tenant.schema], + anonRole: state.role, + roleName: state.role, + isPublic: false, + databaseSettings: FEATURE_SETTINGS, + }; + request.token = null; + request.requestId = request.get('x-request-id') ?? crypto.randomUUID(); + const entry = graphileCache.get(state.cacheKey) ?? await buildEntry(state); + await ensureRuntimeRoleSafety( + entry.poolLease.pool, + [state.role], + [tenant.schema], + state.runtimeDependencySchemas, + ); + await revalidateEntryRealtimeRole(entry); + if (!invokeEntryHandler(entry, request, response, next) && !response.headersSent) { + response.status(503).json({ error: { code: 'CTF_INSTANCE_ROTATING' } }); + } + } catch (error) { + next(error); + } + }); + } + + app.use((error, _request, response, _next) => { + const code = typeof error?.code === 'string' ? error.code : 'CTF_INTERNAL_ERROR'; + response.status(code === 'GRAPHILE_UNSAFE_RUNTIME_ROLE' ? 503 : 500).json({ + error: { code, message: error instanceof Error ? error.message : String(error) }, + }); + }); + + let httpServer = null; + let upgradeListener = null; + const listen = () => new Promise((resolve, reject) => { + httpServer = app.listen(options.port, options.host, () => resolve(httpServer)); + httpServer.once('error', reject); + if (options.enableRealtime) { + upgradeListener = (request, socket, head) => { + void handleUpgrade(request, socket, head) + .then((handled) => { + if (!handled && !socket.destroyed) socket.destroy(); + }) + .catch(() => socket.destroy()); + }; + httpServer.on('upgrade', upgradeListener); + } + }); + const close = async () => { + if (httpServer && upgradeListener) httpServer.off('upgrade', upgradeListener); + const closeServer = httpServer?.listening + ? new Promise((resolve) => httpServer.close(resolve)) + : Promise.resolve(); + await Promise.all(TENANTS.map((tenant) => + deleteGraphileCacheEntry(tenantState.get(tenant.id).cacheKey) + )); + await closeServer; + await teardownPgNotificationBrokers(); + await teardownPgPools(); + }; + + return { + app, + close, + handleUpgrade, + initialProvisionAttestation, + listen, + options: { + host: options.host, + port: options.port, + arm: options.arm, + mode: options.mode, + runtimeRoles: { ...options.runtimeRoles }, + }, + readProvisionAttestation, + contractEvidence: () => ({ + version: 1, + credentialFree: true, + configurationIdentity, + realtimeListener: realtimeListenerContractEvidence, + runtimePools: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).poolContractEvidence, + ])), + graphileBuilds: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + tenantState.get(tenant.id).buildContractEvidence, + ])), + }), + buildContractFingerprintForLiveIdentity: (cacheKey) => + buildContractEvidenceByLiveIdentity.get(cacheKey) ?? null, + runtimePoolObjects, + runtimePoolStats, + }; +}; + +const main = async () => { + const options = parseServerOptions(process.argv.slice(2)); + const server = await createFixtureServer(options); + await server.listen(); + process.stdout.write( + `${JSON.stringify({ status: 'ready', host: options.host, port: options.port, arm: options.arm })}\n`, + ); + let closing = false; + const shutdown = async () => { + if (closing) return; + closing = true; + await server.close(); + }; + process.once('SIGTERM', () => void shutdown().finally(() => process.exit(0))); + process.once('SIGINT', () => void shutdown().finally(() => process.exit(130))); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + FEATURE_SETTINGS, + GRAFAST_CACHE_LIMITS, + INSTALLED_RUNTIME_ARTIFACT_SPECS, + INTROSPECTION_DEPENDENCY_SCHEMAS, + PREPARED_STATEMENT_ATTESTATION_KIND, + RELEASE_BUILD_STATE_AFTER_VALIDATION, + RUNTIME_ARTIFACT_PATHS, + RUNTIME_DEPENDENCY_SCHEMAS, + attestDataplanPreparedStatementCache, + createFixtureServer, + credentialFreeContractEvidence, + fixtureConfigurationIdentity, + loadInstalledDataplanPgAdaptor, + makeRuntimePoolStats, + parseServerOptions, + parseRuntimePoolMaxUses, + preparedResetBackendEvidence, + preparedStatementCacheRequestFromEnvironment, + matchTenantUpgradePath, + installedRuntimeArtifactManifest, + hostileControlEnabledFor, + realtimeSchemaFor, + provisionAttestationSha256, + resolvedLocalRuntimeArtifactManifest, + runtimeDependencySchemasFor, + runtimeArtifactManifest, + runtimeArtifactFingerprint, + runtimePoolContractEvidence, + timingSafeTokenEqual, +}; diff --git a/research/graphile-density/complete-tenant-fixture/server.test.cjs b/research/graphile-density/complete-tenant-fixture/server.test.cjs new file mode 100644 index 0000000000..605f187148 --- /dev/null +++ b/research/graphile-density/complete-tenant-fixture/server.test.cjs @@ -0,0 +1,543 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const path = require('node:path'); +const test = require('node:test'); + +const { TENANTS } = require('./lib.cjs'); +const { + provisionAttestationSha256: provisionerAttestationSha256, +} = require('../physical-database-density/provision.cjs'); + +const { + INSTALLED_RUNTIME_ARTIFACT_SPECS, + PREPARED_STATEMENT_ATTESTATION_KIND, + RELEASE_BUILD_STATE_AFTER_VALIDATION, + RUNTIME_ARTIFACT_PATHS, + credentialFreeContractEvidence, + fixtureConfigurationIdentity, + hostileControlEnabledFor, + installedRuntimeArtifactManifest, + makeRuntimePoolStats, + matchTenantUpgradePath, + parseRuntimePoolMaxUses, + parseServerOptions, + preparedResetBackendEvidence, + preparedStatementCacheRequestFromEnvironment, + provisionAttestationSha256, + realtimeSchemaFor, + resolvedLocalRuntimeArtifactManifest, + runtimeArtifactManifest, + runtimeArtifactFingerprint, + runtimeDependencySchemasFor, + runtimePoolContractEvidence, + timingSafeTokenEqual, +} = require('./server.cjs'); + +const environment = () => ({ + CTF_RUNTIME_A_PGPASSWORD: 'runtime-a-value', + CTF_RUNTIME_B_PGPASSWORD: 'runtime-b-value', + CTF_RUNTIME_C_PGPASSWORD: 'runtime-c-value', +}); + +const roleArgs = (roles = ['ctf_runtime_a', 'ctf_runtime_b', 'ctf_runtime_c']) => [ + '--runtime-role-a', roles[0], + '--runtime-role-b', roles[1], + '--runtime-role-c', roles[2], +]; + +test('server accepts only loopback with three distinct credentialed runtime roles', () => { + const options = parseServerOptions([ + '--host', '127.0.0.1', + '--port', '3392', + '--mode', 'stock', + ...roleArgs(), + ], environment()); + assert.deepEqual(options.runtimeRoles, { + a: 'ctf_runtime_a', + b: 'ctf_runtime_b', + c: 'ctf_runtime_c', + }); + assert.equal(options.port, 3392); + assert.equal(options.mode, 'stock'); + assert.equal(options.introspectionClientReleaseMode, 'destroy'); + assert.equal(options.runtimePoolMax, 1); + assert.equal(options.runtimePoolMaxUses, null); + assert.equal(options.enableRealtime, false); + const serialized = JSON.stringify(options); + assert.doesNotMatch(serialized, /runtime-[abc]-value/); + + assert.throws( + () => parseServerOptions(['--host', '0.0.0.0', ...roleArgs()], environment()), + /CTF_SERVER_LOOPBACK_REQUIRED/, + ); + assert.throws( + () => parseServerOptions(roleArgs(['same', 'same', 'third']), environment()), + /CTF_RUNTIME_ROLES_MUST_BE_DISTINCT/, + ); + assert.throws( + () => parseServerOptions(roleArgs(), { + CTF_RUNTIME_A_PGPASSWORD: 'only-one', + }), + /CTF_RUNTIME_PASSWORD_REQUIRED:CTF_RUNTIME_B_PGPASSWORD/, + ); +}); + +test('server accepts explicit runtime pool capacity and realtime opt-in', () => { + const options = parseServerOptions([ + '--runtime-pool-max', '4', + '--runtime-pool-max-uses', '1', + '--enable-realtime', + ...roleArgs(), + ], environment()); + assert.equal(options.runtimePoolMax, 4); + assert.equal(options.runtimePoolMaxUses, 1); + assert.equal(options.enableRealtime, true); + + assert.throws( + () => parseServerOptions(['--runtime-pool-max', '0', ...roleArgs()], environment()), + /CTF_INVALID_POSITIVE_INTEGER:runtime-pool-max/, + ); + assert.throws( + () => parseServerOptions([ + '--runtime-pool-max-uses', '0', + ...roleArgs(), + ], environment()), + /CTF_INVALID_MAX_USES:runtime-pool-max-uses/, + ); + for (const value of ['01', '1e2', '0x1', ' 1', '1 ', '', true]) { + assert.throws( + () => parseRuntimePoolMaxUses(value), + /CTF_INVALID_MAX_USES:runtime-pool-max-uses/, + ); + } + assert.throws( + () => parseServerOptions(['--enable-realtime', 'sometimes', ...roleArgs()], environment()), + /CTF_INVALID_BOOLEAN:enable-realtime/, + ); + assert.throws( + () => parseServerOptions([ + '--runtime-pool-max', '1', + '--enable-realtime', + ...roleArgs(), + ], environment()), + /CTF_REALTIME_REQUIRES_RUNTIME_POOL_MAX_2/, + ); +}); + +test('fixture configuration and contract evidence are deterministic and credential-free', () => { + const input = { + databaseName: 'ctf_customer_0001', + mode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + enableRealtime: true, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 30_000, + runtimeFingerprint: `sha256:${'a'.repeat(64)}`, + }; + const configurationIdentity = fixtureConfigurationIdentity(input); + assert.match( + configurationIdentity, + /^graphile-configuration:ctf:v1:[a-f0-9]{64}$/, + ); + assert.equal(fixtureConfigurationIdentity({ ...input }), configurationIdentity); + assert.notEqual( + fixtureConfigurationIdentity({ ...input, databaseName: 'ctf_customer_0002' }), + configurationIdentity, + ); + + const pool = runtimePoolContractEvidence({ + databaseName: input.databaseName, + role: 'ctf_runtime_a', + poolMax: 1, + poolMaxUses: null, + runtimeFingerprint: input.runtimeFingerprint, + }); + assert.match(pool.fingerprint, /^pg-contract-evidence:v1:[a-f0-9]{64}$/); + assert.equal(JSON.stringify(pool).includes('runtime-password-value'), false); + assert.deepEqual( + credentialFreeContractEvidence('fixture-evidence', { a: 1 }), + credentialFreeContractEvidence('fixture-evidence', { a: 1 }), + ); + assert.throws( + () => fixtureConfigurationIdentity({ ...input, runtimeFingerprint: '' }), + /CTF_CONFIGURATION_IDENTITY_INPUT_INVALID/, + ); +}); + +test('prepared reset PID evidence distinguishes sanitation reuse from maxUses rotation', () => { + assert.deepEqual(preparedResetBackendEvidence(101, 101, null), { + firstBackendPid: 101, + secondBackendPid: 101, + observed: 'same-client', + expected: 'same-client', + exact: true, + }); + assert.deepEqual(preparedResetBackendEvidence(101, 202, 1), { + firstBackendPid: 101, + secondBackendPid: 202, + observed: 'rotated-client', + expected: 'rotated-client', + exact: true, + }); + assert.equal(preparedResetBackendEvidence(101, 202, null).exact, false); + assert.equal(preparedResetBackendEvidence(101, 101, 1).exact, false); + assert.deepEqual(preparedResetBackendEvidence(101, 202, 2), { + firstBackendPid: 101, + secondBackendPid: 202, + observed: 'rotated-client', + expected: 'unsupported', + exact: false, + }); +}); + +test('runtime pool telemetry follows exact runtime identities and proves native maxUses', () => { + const nativePool = (maxUses, totalCount, idleCount) => ({ + options: { maxUses }, + totalCount, + idleCount, + waitingCount: 0, + }); + const runtimeA = nativePool(1, 0, 0); + const runtimeB = nativePool(1, 1, 0); + const notification = nativePool(Number.POSITIVE_INFINITY, 1, 1); + const pgCache = { + records: new Map([ + ['runtime-a', { pool: runtimeA }], + ['runtime-b', { pool: runtimeB }], + ['notification', { pool: notification }], + ]), + }; + assert.deepEqual( + makeRuntimePoolStats(pgCache, ['runtime-a', 'runtime-b'], 1), + { + scope: 'runtime-only-exact-identities', + available: true, + requestedMaxUses: 1, + effectiveMaxUses: 1, + effectiveMaxUsesKnown: true, + maxUsesExact: true, + identitiesUnique: true, + poolObjectsUnique: true, + expectedPools: 2, + observedPools: 2, + totalClients: 1, + idleClients: 0, + waitingClients: 0, + }, + ); + + const unlimited = makeRuntimePoolStats( + { records: new Map([['runtime', { pool: notification }]]) }, + ['runtime'], + null, + ); + assert.equal(unlimited.available, true); + assert.equal(unlimited.effectiveMaxUsesKnown, true); + assert.equal(unlimited.effectiveMaxUses, null); + assert.equal(unlimited.maxUsesExact, true); + + const missing = makeRuntimePoolStats(pgCache, ['runtime-a', 'missing'], 1); + assert.equal(missing.available, false); + assert.equal(missing.observedPools, 1); + assert.equal(missing.maxUsesExact, false); + + const duplicateIdentity = makeRuntimePoolStats( + pgCache, + ['runtime-a', 'runtime-a'], + 1, + ); + assert.equal(duplicateIdentity.available, false); + assert.equal(duplicateIdentity.identitiesUnique, false); + assert.equal(duplicateIdentity.observedPools, 1); + assert.equal(duplicateIdentity.maxUsesExact, false); + + const duplicatePoolObject = makeRuntimePoolStats({ + records: new Map([ + ['runtime-a', { pool: runtimeA }], + ['runtime-b', { pool: runtimeA }], + ]), + }, ['runtime-a', 'runtime-b'], 1); + assert.equal(duplicatePoolObject.available, false); + assert.equal(duplicatePoolObject.identitiesUnique, true); + assert.equal(duplicatePoolObject.poolObjectsUnique, false); + assert.equal(duplicatePoolObject.observedPools, 1); +}); + +test('prepared statement cache request accepts only a canonical bounded integer', () => { + assert.deepEqual( + preparedStatementCacheRequestFromEnvironment({ + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: '0', + }), + { + environmentValue: '0', + requestedSize: 0, + environmentCanonical: true, + }, + ); + assert.deepEqual(preparedStatementCacheRequestFromEnvironment({}), { + environmentValue: null, + requestedSize: 100, + environmentCanonical: false, + }); + for (const value of ['01', '1e2', '-1', '10001', 'not-a-number', ' 1', '1 ']) { + assert.throws( + () => preparedStatementCacheRequestFromEnvironment({ + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: value, + }), + /CTF_PREPARED_STATEMENT_CACHE_SIZE_INVALID/, + ); + } +}); + +test('prepared statement cache telemetry attests the loaded Dataplan adaptor behavior', () => { + const serverFile = path.join(__dirname, 'server.cjs'); + const inspect = (size) => JSON.parse(execFileSync(process.execPath, ['-e', ` +const fixture = require(${JSON.stringify(serverFile)}); +(async () => { + const request = fixture.preparedStatementCacheRequestFromEnvironment(process.env); + const adaptor = fixture.loadInstalledDataplanPgAdaptor(); + const proof = await fixture.attestDataplanPreparedStatementCache(adaptor, request); + process.stdout.write(JSON.stringify(proof)); +})().catch((error) => { + process.stderr.write(String(error && error.stack || error)); + process.exit(1); +}); +`], { + cwd: path.resolve(__dirname, '../../..'), + encoding: 'utf8', + env: { + ...process.env, + GRAPHILE_ENV: 'production', + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: String(size), + }, + })); + + const disabled = inspect(0); + assert.equal(disabled.attestation, PREPARED_STATEMENT_ATTESTATION_KIND); + assert.equal(disabled.effectiveSizeKnown, true); + assert.equal(disabled.effectiveSize, 0); + assert.equal(disabled.exact, true); + assert.equal(disabled.namedQueriesObserved, 0); + assert.equal(disabled.firstEvictionAfterNamedQueries, null); + + const bounded = inspect(3); + assert.equal(bounded.attestation, PREPARED_STATEMENT_ATTESTATION_KIND); + assert.equal(bounded.effectiveSizeKnown, true); + assert.equal(bounded.effectiveSize, 3); + assert.equal(bounded.exact, true); + assert.equal(bounded.namedQueriesObserved, 4); + assert.equal(bounded.firstEvictionAfterNamedQueries, 3); +}); + +test('shared exact realtime permits a one-client runtime pool only with a distinct listener login', () => { + const options = parseServerOptions([ + '--runtime-pool-max', '1', + '--enable-realtime', + '--realtime-notification-mode', 'shared-exact', + '--notification-role', 'ctf_notification', + '--realtime-cursor-poll-ms', '30000', + ...roleArgs(), + ], { + ...environment(), + CTF_NOTIFICATION_PGPASSWORD: 'notification-password-value', + }); + assert.equal(options.runtimePoolMax, 1); + assert.equal(options.realtimeNotificationMode, 'shared-exact'); + assert.equal(options.notificationRole, 'ctf_notification'); + assert.equal(options.realtimeCursorPollIntervalMs, 30000); + const serialized = JSON.stringify(options); + assert.doesNotMatch(serialized, /notification-password-value/); + assert.equal(options.takeNotificationPassword(), 'notification-password-value'); + assert.throws( + () => options.takeNotificationPassword(), + /CTF_NOTIFICATION_PASSWORD_ALREADY_CONSUMED/, + ); + + assert.throws(() => parseServerOptions([ + '--runtime-pool-max', '1', + '--enable-realtime', + '--realtime-notification-mode', 'shared-exact', + '--notification-role', 'ctf_runtime_a', + ...roleArgs(), + ], { + ...environment(), + CTF_NOTIFICATION_PGPASSWORD: 'notification-password-value', + }), /CTF_NOTIFICATION_ROLE_MUST_BE_DISTINCT/); + assert.throws(() => parseServerOptions([ + '--runtime-pool-max', '1', + '--enable-realtime', + '--realtime-notification-mode', 'shared-exact', + '--notification-role', 'ctf_notification', + ...roleArgs(), + ], environment()), /CTF_NOTIFICATION_PASSWORD_REQUIRED/); +}); + +test('realtime cursor schemas and runtime safety allowlists remain tenant-exact', () => { + assert.deepEqual(TENANTS.map(realtimeSchemaFor), [ + 'ctf_a_realtime', + 'ctf_b_realtime', + 'ctf_c_realtime', + ]); + + for (const tenant of TENANTS) { + const disabled = runtimeDependencySchemasFor(tenant, false); + const enabled = runtimeDependencySchemasFor(tenant, true); + assert.deepEqual(disabled, ['ctf_extensions', 'jwt_private']); + assert.deepEqual(enabled, [ + 'ctf_extensions', + 'jwt_private', + realtimeSchemaFor(tenant), + ]); + for (const foreignTenant of TENANTS.filter((candidate) => candidate !== tenant)) { + assert.ok(!enabled.includes(realtimeSchemaFor(foreignTenant))); + } + } +}); + +test('websocket upgrade paths select one exact tenant and reject ambiguous routes', () => { + assert.equal(matchTenantUpgradePath('/tenant/a/graphql'), 'a'); + assert.equal( + matchTenantUpgradePath( + '/customer/physical-customer-0001/tenant/c/graphql', + '/customer/physical-customer-0001', + ), + 'c', + ); + assert.equal(matchTenantUpgradePath('/tenant/a/graphql?tenant=b'), null); + assert.equal(matchTenantUpgradePath('/tenant/a/graphql/extra'), null); + assert.equal(matchTenantUpgradePath('/tenant/%61/graphql'), null); + assert.equal(matchTenantUpgradePath('/tenant/a%2F..%2Fb/graphql'), null); + assert.equal(matchTenantUpgradePath('/tenant/a/graphql', '/customer/other'), null); + assert.equal(matchTenantUpgradePath('/tenant/a/graphql', '../customer'), null); +}); + +test('server rejects unknown introspection modes', () => { + assert.throws( + () => parseServerOptions(['--mode', 'fallback', ...roleArgs()], environment()), + /CTF_INTROSPECTION_MODE_INVALID:fallback/, + ); +}); + +test('server validates the introspection-client release mode', () => { + const options = parseServerOptions([ + '--introspection-client-release-mode', 'reuse', + ...roleArgs(), + ], environment()); + assert.equal(options.introspectionClientReleaseMode, 'reuse'); + assert.throws( + () => parseServerOptions([ + '--introspection-client-release-mode', 'best-effort', + ...roleArgs(), + ], environment()), + /CTF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:best-effort/, + ); +}); + +test('control token comparison is exact and timing safe for equal-length values', () => { + const token = 'a'.repeat(64); + assert.equal(timingSafeTokenEqual(token, token), true); + assert.equal(timingSafeTokenEqual(`${'a'.repeat(63)}b`, token), false); + assert.equal(timingSafeTokenEqual('short', token), false); + assert.equal(timingSafeTokenEqual('', token), false); +}); + +test('runtime fingerprint binds every executed built API artifact', () => { + assert.match(runtimeArtifactFingerprint(), /^sha256:[0-9a-f]{64}$/); + assert.equal(runtimeArtifactFingerprint(), runtimeArtifactFingerprint()); + const runtimeManifest = runtimeArtifactManifest(); + const localClosure = resolvedLocalRuntimeArtifactManifest(); + assert.equal(runtimeManifest.version, 2); + assert.ok(localClosure.length > RUNTIME_ARTIFACT_PATHS.length); + assert.deepEqual(runtimeManifest.localDistClosure, localClosure); + assert.ok(localClosure.some((entry) => + entry.path === 'graphile/graphile-settings/dist/presets/constructive-preset.js' + )); + assert.ok(localClosure.some((entry) => + entry.path === 'graphile/graphile-search/dist/index.js' + )); + assert.ok(localClosure.every((entry) => + !path.isAbsolute(entry.path) + && !entry.path.includes('node_modules') + && /^sha256:[0-9a-f]{64}$/.test(entry.sha256) + )); + + const expectedInstalledLabels = [ + 'installed:@dataplan/pg:dist/index.js', + 'installed:@dataplan/pg:dist/adaptors/pg.js', + 'installed:@dataplan/pg:dist/pgServices.js', + 'installed:graphile-build-pg:dist/index.js', + 'installed:graphile-build-pg:dist/plugins/PgIntrospectionPlugin.js', + ]; + const manifest = installedRuntimeArtifactManifest(); + assert.deepEqual( + manifest.map((entry) => entry.label), + expectedInstalledLabels, + ); + assert.deepEqual( + INSTALLED_RUNTIME_ARTIFACT_SPECS.map((entry) => entry.label), + expectedInstalledLabels, + ); + const installedSpecs = Object.fromEntries( + INSTALLED_RUNTIME_ARTIFACT_SPECS.map((entry) => [entry.label, entry]), + ); + assert.ok(installedSpecs['installed:@dataplan/pg:dist/index.js'].markers.includes( + 'exports.exactClientReleaseCapability = "dataplan-pg-exact-client-destroy-v1";', + )); + assert.ok(installedSpecs['installed:@dataplan/pg:dist/adaptors/pg.js'].markers.includes( + 'const supportsExactClientDestruction = typeof PgPool === "function" && pool instanceof PgPool;', + )); + assert.ok(installedSpecs['installed:@dataplan/pg:dist/adaptors/pg.js'].markers.includes( + 'Exact PostgreSQL client destruction requires a node-postgres Pool', + )); + assert.ok(installedSpecs['installed:graphile-build-pg:dist/index.js'].markers.includes( + 'exports.introspectionClientReleaseCapability = "graphile-build-pg-exact-client-destroy-v1";', + )); + assert.ok(manifest.every((entry) => /^sha256:[0-9a-f]{64}$/.test(entry.sha256))); + assert.ok(manifest.every( + (entry) => /^sha256:[0-9a-f]{64}$/.test(entry.markerSetSha256) + && entry.markerCount > 0 + )); + const serialized = JSON.stringify({ + specs: INSTALLED_RUNTIME_ARTIFACT_SPECS, + manifest, + }); + assert.equal(serialized.includes(process.cwd()), false); + assert.doesNotMatch(serialized, /password|credential|secret|authorization|bearer/i); +}); + +test('provisioned measurement servers cannot enable hostile controls', () => { + const attestation = { + version: 1, + cloneId: 'fixture-clone', + purpose: 'measurement', + sha256: `sha256:${'a'.repeat(64)}`, + }; + assert.equal(hostileControlEnabledFor('measurement', attestation), false); + assert.equal(hostileControlEnabledFor('hostile-preflight', { + ...attestation, + purpose: 'hostile-preflight', + }), true); + // The standalone complete fixture keeps its pre-existing local control lane; + // physical runs always carry a database-backed attestation. + assert.equal(hostileControlEnabledFor(undefined, null), true); +}); + +test('live server and provisioner compute the same context-bound attestation', () => { + const input = { + cloneId: 'fixture-clone', + customerId: 'physical-customer-0001', + database: 'pdc_fixture_db_0001', + nonce: 'a'.repeat(64), + }; + assert.equal( + provisionAttestationSha256({ ...input, purpose: 'measurement' }), + provisionerAttestationSha256({ ...input, runPurpose: 'measurement' }), + ); +}); + +test('complete fixture requires post-validation build-state retirement', () => { + assert.equal(RELEASE_BUILD_STATE_AFTER_VALIDATION, true); +}); diff --git a/research/graphile-density/create-uniform-density-fixture.sql b/research/graphile-density/create-uniform-density-fixture.sql new file mode 100644 index 0000000000..1bc3ba8d52 --- /dev/null +++ b/research/graphile-density/create-uniform-density-fixture.sql @@ -0,0 +1,796 @@ +\set ON_ERROR_STOP on +\pset pager off + +-- Rebuilds the local density fixture without modifying its source database. +-- This is a psql script, not generic SQL. Run it as a PostgreSQL administrator: +-- +-- psql -X -d postgres \ +-- -f research/graphile-density/create-uniform-density-fixture.sql +-- +-- The target name is intentionally fixed. The script fails closed when the +-- target already exists; it never replaces or drops a database. `-v resume=1` +-- is only for continuing a clone that passed the pristine-source assertions +-- but stopped before tenant DDL; those assertions run again before mutation. +-- +-- PERFORMANCE-ONLY ROUTING CANARY: the shared gd_runtime_20260801_a login has +-- SELECT/EXECUTE access across every tenant schema. This fixture brackets the +-- Graphile memory-density curve; it does not prove database-enforced tenant +-- isolation and cannot qualify a complete customer surface for production. + +\if :{?resume} +\else + \set resume 0 +\endif + +\echo 'Preflighting source, target, and runtime role' +\echo 'PERFORMANCE_ONLY_ROUTING_CANARY: this fixture is not a tenant-isolation proof' + +SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_database + WHERE datname = 'graphile_density_20260801_a' +) AS source_exists, +EXISTS ( + SELECT 1 FROM pg_catalog.pg_database + WHERE datname = 'graphile_density_uniform_20260801_a' +) AS target_exists, +EXISTS ( + SELECT 1 FROM pg_catalog.pg_roles + WHERE rolname = 'gd_runtime_20260801_a' +) AS runtime_role_exists +\gset + +\if :source_exists +\else + \echo 'GRAPHILE_DENSITY_SOURCE_MISSING: graphile_density_20260801_a' + \quit 3 +\endif + +\if :target_exists + \if :resume + \echo 'Resuming explicitly against the existing asserted-pristine clone' + \else + \echo 'GRAPHILE_DENSITY_TARGET_EXISTS: graphile_density_uniform_20260801_a' + \quit 4 + \endif +\endif + +\if :runtime_role_exists +\else + \echo 'GRAPHILE_DENSITY_RUNTIME_ROLE_MISSING: gd_runtime_20260801_a' + \quit 5 +\endif + +\echo 'Creating immutable physical clone graphile_density_uniform_20260801_a' + +\if :target_exists +\else + CREATE DATABASE graphile_density_uniform_20260801_a + WITH TEMPLATE graphile_density_20260801_a + OWNER postgres; +\endif + +\connect graphile_density_uniform_20260801_a + +SET client_min_messages = warning; +SET statement_timeout = 0; +SET lock_timeout = '30s'; +SET idle_in_transaction_session_timeout = '5min'; + +REVOKE CREATE ON DATABASE graphile_density_uniform_20260801_a + FROM PUBLIC, gd_runtime_20260801_a; +GRANT CONNECT ON DATABASE graphile_density_uniform_20260801_a + TO gd_runtime_20260801_a; + +\echo 'Validating the cloned heterogeneous source shape' + +DO $preflight$ +DECLARE + class_count integer; + tenant_schema_count integer; + full_schema_count integer; + function_only_schema_count integer; +BEGIN + SELECT count(*) INTO class_count FROM pg_catalog.pg_class; + IF class_count <> 61239 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_SOURCE_CLASS_COUNT_MISMATCH: expected 61239, got %', + class_count; + END IF; + + WITH tenant_shapes AS ( + SELECT namespace.nspname, + (SELECT count(*) + FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid) AS class_count, + (SELECT count(*) + FROM pg_catalog.pg_proc AS procedure + WHERE procedure.pronamespace = namespace.oid) AS proc_count, + (SELECT count(*) + FROM pg_catalog.pg_constraint AS constraint_row + WHERE constraint_row.connamespace = namespace.oid) AS constraint_count + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + ) + SELECT count(*), + count(*) FILTER ( + WHERE tenant_shapes.class_count = 7 + AND tenant_shapes.proc_count = 1 + AND tenant_shapes.constraint_count = 9 + ), + count(*) FILTER ( + WHERE tenant_shapes.class_count = 0 + AND tenant_shapes.proc_count = 1 + AND tenant_shapes.constraint_count = 0 + ) + INTO tenant_schema_count, full_schema_count, function_only_schema_count + FROM tenant_shapes; + + IF tenant_schema_count <> 2000 + OR full_schema_count <> 400 + OR function_only_schema_count <> 1600 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_SOURCE_SHAPE_MISMATCH: schemas %, full %, function-only %', + tenant_schema_count, full_schema_count, function_only_schema_count; + END IF; +END +$preflight$; + +-- DDL is deliberately split into 100-tenant transactions. A single +-- transaction would retain locks for tens of thousands of new relations and +-- can exhaust max_locks_per_transaction on an otherwise healthy local server. +CREATE PROCEDURE pg_temp.create_uniform_tenants( + batch_start integer, + batch_end integer +) +LANGUAGE plpgsql +AS $procedure$ +DECLARE + tenant_number integer; + tenant_suffix text; + tenant_schema text; + tenant_token text; +BEGIN + FOR tenant_number IN batch_start..batch_end LOOP + tenant_suffix := CASE + WHEN tenant_number < 1000 + THEN pg_catalog.lpad(tenant_number::text, 3, '0') + ELSE tenant_number::text + END; + tenant_schema := 'gd_t' || tenant_suffix || '_api'; + tenant_token := 'tenant-' || tenant_suffix || '-token'; + + IF tenant_number > 2000 THEN + EXECUTE pg_catalog.format( + 'CREATE SCHEMA %I AUTHORIZATION postgres', + tenant_schema + ); + END IF; + + EXECUTE pg_catalog.format( + 'CREATE OR REPLACE FUNCTION %I.tenant_token()' + ' RETURNS text LANGUAGE sql STABLE AS %L', + tenant_schema, + 'SELECT ' || pg_catalog.quote_literal(tenant_token) || '::text' + ); + + EXECUTE pg_catalog.format( + 'CREATE TABLE %I.tenant_canary (' + ' id bigint GENERATED ALWAYS AS IDENTITY,' + ' tenant_token text NOT NULL,' + ' CONSTRAINT tenant_canary_pkey PRIMARY KEY (id),' + ' CONSTRAINT tenant_canary_tenant_token_key UNIQUE (tenant_token)' + ')', + tenant_schema + ); + + EXECUTE pg_catalog.format( + 'CREATE TABLE %I.widget (' + ' id bigint GENERATED ALWAYS AS IDENTITY,' + ' canary_id bigint NOT NULL,' + ' label text NOT NULL,' + ' CONSTRAINT widget_pkey PRIMARY KEY (id),' + ' CONSTRAINT widget_canary_id_fkey FOREIGN KEY (canary_id)' + ' REFERENCES %I.tenant_canary(id)' + ')', + tenant_schema, + tenant_schema + ); + END LOOP; +END +$procedure$; + +SELECT pg_catalog.format( + 'CALL pg_temp.create_uniform_tenants(%s, %s)', + batch_start, + least(batch_start + 99, 4000) +) +FROM pg_catalog.generate_series(401, 4000, 100) AS batch(batch_start) +\gexec + +DROP PROCEDURE pg_temp.create_uniform_tenants(integer, integer); + +DO $intermediate_count$ +DECLARE + class_count integer; +BEGIN + SELECT count(*) INTO class_count FROM pg_catalog.pg_class; + IF class_count <> 100839 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_UNIFORM_CLASS_DELTA_MISMATCH: expected 100839, got %', + class_count; + END IF; +END +$intermediate_count$; + +\echo 'Normalizing ownership, grants, identity state, and canary rows' + +CREATE PROCEDURE pg_temp.normalize_uniform_tenants( + batch_start integer, + batch_end integer +) +LANGUAGE plpgsql +AS $procedure$ +DECLARE + tenant_number integer; + tenant_suffix text; + tenant_schema text; + tenant_token text; + widget_label text; +BEGIN + FOR tenant_number IN batch_start..batch_end LOOP + tenant_suffix := CASE + WHEN tenant_number < 1000 + THEN pg_catalog.lpad(tenant_number::text, 3, '0') + ELSE tenant_number::text + END; + tenant_schema := 'gd_t' || tenant_suffix || '_api'; + tenant_token := 'tenant-' || tenant_suffix || '-token'; + widget_label := 'tenant-' || tenant_suffix || '-widget'; + + EXECUTE pg_catalog.format('ALTER SCHEMA %I OWNER TO postgres', tenant_schema); + EXECUTE pg_catalog.format( + 'ALTER TABLE %I.tenant_canary OWNER TO postgres', tenant_schema + ); + EXECUTE pg_catalog.format( + 'ALTER TABLE %I.widget OWNER TO postgres', tenant_schema + ); + EXECUTE pg_catalog.format( + 'ALTER FUNCTION %I.tenant_token() OWNER TO postgres', tenant_schema + ); + + EXECUTE pg_catalog.format( + 'TRUNCATE TABLE %I.widget, %I.tenant_canary RESTART IDENTITY', + tenant_schema, + tenant_schema + ); + EXECUTE pg_catalog.format( + 'INSERT INTO %I.tenant_canary (tenant_token) VALUES (%L)', + tenant_schema, + tenant_token + ); + EXECUTE pg_catalog.format( + 'INSERT INTO %I.widget (canary_id, label) VALUES (1, %L)', + tenant_schema, + widget_label + ); + + EXECUTE pg_catalog.format( + 'REVOKE ALL PRIVILEGES ON SCHEMA %I' + ' FROM PUBLIC, postgres, gd_runtime_20260801_a', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT ALL PRIVILEGES ON SCHEMA %I TO postgres', tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT USAGE ON SCHEMA %I TO gd_runtime_20260801_a', tenant_schema + ); + + EXECUTE pg_catalog.format( + 'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I' + ' FROM PUBLIC, postgres, gd_runtime_20260801_a', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I TO postgres', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT SELECT ON ALL TABLES IN SCHEMA %I TO gd_runtime_20260801_a', + tenant_schema + ); + + EXECUTE pg_catalog.format( + 'REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA %I' + ' FROM PUBLIC, postgres, gd_runtime_20260801_a', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA %I TO postgres', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT SELECT, USAGE ON ALL SEQUENCES IN SCHEMA %I' + ' TO gd_runtime_20260801_a', + tenant_schema + ); + + EXECUTE pg_catalog.format( + 'REVOKE ALL PRIVILEGES ON FUNCTION %I.tenant_token()' + ' FROM PUBLIC, postgres, gd_runtime_20260801_a', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT ALL PRIVILEGES ON FUNCTION %I.tenant_token() TO postgres', + tenant_schema + ); + EXECUTE pg_catalog.format( + 'GRANT EXECUTE ON FUNCTION %I.tenant_token()' + ' TO gd_runtime_20260801_a', + tenant_schema + ); + END LOOP; +END +$procedure$; + +SELECT pg_catalog.format( + 'CALL pg_temp.normalize_uniform_tenants(%s, %s)', + batch_start, + least(batch_start + 99, 4000) +) +FROM pg_catalog.generate_series(1, 4000, 100) AS batch(batch_start) +\gexec + +DROP PROCEDURE pg_temp.normalize_uniform_tenants(integer, integer); + +\echo 'Planning a footprint-exact reduction of disposable gd_noise tables' + +CREATE TEMPORARY TABLE noise_drop_plan ON COMMIT PRESERVE ROWS AS +WITH noise_tables AS ( + SELECT class.oid, class.relname, class.reltoastrelid, + 1 + + ( + SELECT count(*) + FROM pg_catalog.pg_index AS table_index + WHERE table_index.indrelid = class.oid + ) + + ( + SELECT count(*) + FROM pg_catalog.pg_depend AS sequence_dependency + JOIN pg_catalog.pg_class AS sequence + ON sequence.oid = sequence_dependency.objid + AND sequence.relkind = 'S' + WHERE sequence_dependency.classid = 'pg_catalog.pg_class'::regclass + AND sequence_dependency.refclassid = 'pg_catalog.pg_class'::regclass + AND sequence_dependency.refobjid = class.oid + AND sequence_dependency.deptype = 'i' + ) + + CASE WHEN class.reltoastrelid = 0 THEN 0 ELSE 1 END + + ( + SELECT count(*) + FROM pg_catalog.pg_index AS toast_index + WHERE toast_index.indrelid = class.reltoastrelid + ) AS class_footprint + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = class.relnamespace + WHERE namespace.nspname = 'gd_noise' + AND class.relkind = 'r' +), candidates AS ( + SELECT relname, class_footprint, + substring(relname FROM '^r_([0-9]+)$')::integer AS noise_number + FROM noise_tables + WHERE class_footprint = 5 +) +SELECT row_number() OVER (ORDER BY noise_number DESC) AS ordinal, + relname, + class_footprint +FROM candidates +ORDER BY noise_number DESC +LIMIT 7920; + +DO $drop_plan$ +DECLARE + candidate_count integer; + planned_footprint integer; +BEGIN + SELECT count(*), sum(class_footprint) + INTO candidate_count, planned_footprint + FROM pg_temp.noise_drop_plan; + + IF candidate_count <> 7920 OR planned_footprint <> 39600 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_NOISE_PLAN_MISMATCH: tables %, pg_class footprint %', + candidate_count, planned_footprint; + END IF; +END +$drop_plan$; + +CREATE PROCEDURE pg_temp.drop_noise_batch( + batch_start integer, + batch_end integer +) +LANGUAGE plpgsql +AS $procedure$ +DECLARE + candidate record; +BEGIN + FOR candidate IN + SELECT relname + FROM pg_temp.noise_drop_plan + WHERE ordinal BETWEEN batch_start AND batch_end + ORDER BY ordinal + LOOP + EXECUTE pg_catalog.format( + 'DROP TABLE gd_noise.%I CASCADE', + candidate.relname + ); + END LOOP; +END +$procedure$; + +SELECT pg_catalog.format( + 'CALL pg_temp.drop_noise_batch(%s, %s)', + batch_start, + least(batch_start + 99, 7920) +) +FROM pg_catalog.generate_series(1, 7920, 100) AS batch(batch_start) +\gexec + +DROP PROCEDURE pg_temp.drop_noise_batch(integer, integer); +DROP TABLE pg_temp.noise_drop_plan; + +\echo 'Analyzing the catalog tables used by Graphile introspection' + +ANALYZE pg_catalog.pg_namespace; +ANALYZE pg_catalog.pg_class; +ANALYZE pg_catalog.pg_attribute; +ANALYZE pg_catalog.pg_constraint; +ANALYZE pg_catalog.pg_proc; +ANALYZE pg_catalog.pg_depend; +ANALYZE pg_catalog.pg_index; + +\echo 'Hard-gating the uniform tenant shape and exact catalog count' + +DO $uniform_shape$ +DECLARE + class_count integer; + tenant_schema_count integer; + distinct_shape_count integer; + missing_schema_count integer; +BEGIN + SELECT count(*) INTO class_count FROM pg_catalog.pg_class; + IF class_count <> 61239 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_FINAL_CLASS_COUNT_MISMATCH: expected 61239, got %', + class_count; + END IF; + + WITH expected AS ( + SELECT 'gd_t' || + CASE WHEN tenant_number < 1000 + THEN pg_catalog.lpad(tenant_number::text, 3, '0') + ELSE tenant_number::text + END || '_api' AS nspname + FROM pg_catalog.generate_series(1, 4000) AS tenant(tenant_number) + ), actual AS ( + SELECT nspname + FROM pg_catalog.pg_namespace + WHERE nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + ), difference AS ( + (SELECT nspname FROM expected EXCEPT SELECT nspname FROM actual) + UNION ALL + (SELECT nspname FROM actual EXCEPT SELECT nspname FROM expected) + ) + SELECT (SELECT count(*) FROM actual), count(*) + INTO tenant_schema_count, missing_schema_count + FROM difference; + + IF tenant_schema_count <> 4000 OR missing_schema_count <> 0 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_SCHEMA_SET_MISMATCH: actual %, symmetric difference %', + tenant_schema_count, missing_schema_count; + END IF; + + WITH tenant_shapes AS ( + SELECT namespace.nspname, + pg_catalog.md5(pg_catalog.concat_ws('|', + namespace.nspowner::regrole::text, + coalesce(namespace.nspacl::text, ''), + ( + SELECT pg_catalog.string_agg( + pg_catalog.concat_ws(':', class.relname, class.relkind, + class.relowner::regrole::text, coalesce(class.relacl::text, '')), + ',' ORDER BY class.relname + ) + FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid + ), + ( + SELECT pg_catalog.string_agg( + pg_catalog.concat_ws(':', class.relname, attribute.attnum, + attribute.attname, + pg_catalog.format_type(attribute.atttypid, attribute.atttypmod), + attribute.attnotnull, attribute.attidentity, + attribute.attgenerated), + ',' ORDER BY class.relname, attribute.attnum + ) + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_attribute AS attribute + ON attribute.attrelid = class.oid + WHERE class.relnamespace = namespace.oid + AND class.relkind = 'r' + AND attribute.attnum > 0 + AND NOT attribute.attisdropped + ), + ( + SELECT pg_catalog.string_agg( + pg_catalog.concat_ws(':', constraint_row.conname, + constraint_row.contype, constraint_row.conkey::text, + constraint_row.confkey::text, + coalesce(referenced_class.relname, '')), + ',' ORDER BY constraint_row.conname + ) + FROM pg_catalog.pg_constraint AS constraint_row + LEFT JOIN pg_catalog.pg_class AS referenced_class + ON referenced_class.oid = constraint_row.confrelid + WHERE constraint_row.connamespace = namespace.oid + ), + ( + SELECT pg_catalog.string_agg( + pg_catalog.concat_ws(':', procedure.proname, + pg_catalog.pg_get_function_identity_arguments(procedure.oid), + pg_catalog.pg_get_function_result(procedure.oid), + procedure.provolatile, procedure.prosecdef, + procedure.proowner::regrole::text, + coalesce(procedure.proacl::text, '')), + ',' ORDER BY procedure.proname + ) + FROM pg_catalog.pg_proc AS procedure + WHERE procedure.pronamespace = namespace.oid + ), + ( + SELECT pg_catalog.string_agg( + pg_catalog.concat_ws(':', sequence.sequencename, + sequence.data_type, sequence.start_value, + sequence.min_value, sequence.max_value, + sequence.increment_by, sequence.cycle, + sequence.cache_size, sequence.last_value), + ',' ORDER BY sequence.sequencename + ) + FROM pg_catalog.pg_sequences AS sequence + WHERE sequence.schemaname = namespace.nspname + ) + )) AS shape_fingerprint + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + ) + SELECT count(DISTINCT shape_fingerprint) + INTO distinct_shape_count + FROM tenant_shapes; + + IF distinct_shape_count <> 1 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_NON_UNIFORM_SHAPE: % distinct fingerprints', + distinct_shape_count; + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND ( + (SELECT count(*) FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid) <> 7 + OR + (SELECT count(*) FROM pg_catalog.pg_proc AS procedure + WHERE procedure.pronamespace = namespace.oid) <> 1 + OR + (SELECT count(*) FROM pg_catalog.pg_constraint AS constraint_row + WHERE constraint_row.connamespace = namespace.oid) <> 9 + ) + ) THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_NON_UNIFORM_COUNTS: expected class/proc/constraint 7/1/9'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = class.relnamespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND class.relowner <> 'postgres'::regrole + ) OR EXISTS ( + SELECT 1 + FROM pg_catalog.pg_proc AS procedure + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = procedure.pronamespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND procedure.proowner <> 'postgres'::regrole + ) THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_OWNER_MISMATCH'; + END IF; +END +$uniform_shape$; + +\echo 'Validating all 4,000 surfaces under the runtime identity' + +CREATE PROCEDURE pg_temp.validate_runtime_batch( + batch_start integer, + batch_end integer +) +LANGUAGE plpgsql +AS $runtime_validation$ +DECLARE + tenant_number integer; + tenant_suffix text; + tenant_schema text; + expected_token text; + expected_label text; + function_token text; + table_token text; + widget_label text; + role_row record; +BEGIN + SELECT * INTO role_row + FROM pg_catalog.pg_roles + WHERE rolname = current_user; + + IF session_user <> 'gd_runtime_20260801_a' + OR current_user <> 'gd_runtime_20260801_a' + OR role_row.rolsuper + OR role_row.rolcreaterole + OR role_row.rolcreatedb + OR role_row.rolbypassrls THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_RUNTIME_ROLE_UNSAFE: %', current_user; + END IF; + + IF NOT pg_catalog.has_database_privilege( + current_user, current_database(), 'CONNECT' + ) OR pg_catalog.has_database_privilege( + current_user, current_database(), 'CREATE' + ) THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_RUNTIME_DATABASE_PRIVILEGE_MISMATCH'; + END IF; + + FOR tenant_number IN batch_start..batch_end LOOP + tenant_suffix := CASE + WHEN tenant_number < 1000 + THEN pg_catalog.lpad(tenant_number::text, 3, '0') + ELSE tenant_number::text + END; + tenant_schema := 'gd_t' || tenant_suffix || '_api'; + expected_token := 'tenant-' || tenant_suffix || '-token'; + expected_label := 'tenant-' || tenant_suffix || '-widget'; + + IF NOT pg_catalog.has_schema_privilege( + current_user, tenant_schema, 'USAGE' + ) OR pg_catalog.has_schema_privilege( + current_user, tenant_schema, 'CREATE' + ) THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_RUNTIME_SCHEMA_PRIVILEGE_MISMATCH: %', + tenant_schema; + END IF; + + IF NOT pg_catalog.has_table_privilege( + current_user, + pg_catalog.format('%I.tenant_canary', tenant_schema), + 'SELECT' + ) OR pg_catalog.has_table_privilege( + current_user, + pg_catalog.format('%I.tenant_canary', tenant_schema), + 'INSERT,UPDATE,DELETE' + ) THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_RUNTIME_TABLE_PRIVILEGE_MISMATCH: %', + tenant_schema; + END IF; + + IF NOT pg_catalog.has_function_privilege( + current_user, + pg_catalog.format('%I.tenant_token()', tenant_schema), + 'EXECUTE' + ) THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_RUNTIME_FUNCTION_PRIVILEGE_MISMATCH: %', + tenant_schema; + END IF; + + EXECUTE pg_catalog.format( + 'SELECT %I.tenant_token()', tenant_schema + ) INTO function_token; + EXECUTE pg_catalog.format( + 'SELECT tenant_token FROM %I.tenant_canary', tenant_schema + ) INTO table_token; + EXECUTE pg_catalog.format( + 'SELECT label FROM %I.widget', tenant_schema + ) INTO widget_label; + + IF function_token <> expected_token + OR table_token <> expected_token + OR widget_label <> expected_label THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_RUNTIME_CANARY_MISMATCH: schema %, function %, table %, widget %', + tenant_schema, function_token, table_token, widget_label; + END IF; + END LOOP; +END +$runtime_validation$; + +REVOKE ALL PRIVILEGES ON PROCEDURE + pg_temp.validate_runtime_batch(integer, integer) FROM PUBLIC; +GRANT EXECUTE ON PROCEDURE + pg_temp.validate_runtime_batch(integer, integer) + TO gd_runtime_20260801_a; + +SET SESSION AUTHORIZATION gd_runtime_20260801_a; + +SELECT pg_catalog.format( + 'CALL pg_temp.validate_runtime_batch(%s, %s)', + batch_start, + least(batch_start + 99, 4000) +) +FROM pg_catalog.generate_series(1, 4000, 100) AS batch(batch_start) +\gexec + +RESET SESSION AUTHORIZATION; + +DROP PROCEDURE pg_temp.validate_runtime_batch(integer, integer); + +\echo 'Recording deterministic logical catalog and tenant-shape fingerprints' + +WITH normalized_classes AS ( + SELECT pg_catalog.concat_ws('|', + CASE WHEN namespace.nspname = 'pg_toast' + THEN 'pg_toast.' + ELSE namespace.nspname || '.' || class.relname + END, + class.relkind, + class.relpersistence, + class.relowner::regrole::text, + coalesce(access_method.amname, ''), + class.relnatts, + class.relchecks, + class.relhasindex, + class.reltoastrelid <> 0, + class.relispartition, + coalesce(class.relacl::text, '') + ) AS logical_class + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = class.relnamespace + LEFT JOIN pg_catalog.pg_am AS access_method + ON access_method.oid = class.relam +), tenant_shapes AS ( + SELECT namespace.nspname, + pg_catalog.concat_ws('|', + (SELECT count(*) FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid), + (SELECT count(*) FROM pg_catalog.pg_proc AS procedure + WHERE procedure.pronamespace = namespace.oid), + (SELECT count(*) FROM pg_catalog.pg_constraint AS constraint_row + WHERE constraint_row.connamespace = namespace.oid), + (SELECT pg_catalog.string_agg( + class.relname || ':' || class.relkind::text, + ',' ORDER BY class.relname) + FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid) + ) AS logical_shape + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' +) +SELECT current_database() AS database_name, + (SELECT count(*) FROM pg_catalog.pg_class) AS pg_class_count, + (SELECT pg_catalog.md5(pg_catalog.string_agg( + logical_class, E'\n' ORDER BY logical_class)) + FROM normalized_classes) AS logical_pg_class_fingerprint, + (SELECT count(*) FROM tenant_shapes) AS tenant_schema_count, + (SELECT count(DISTINCT logical_shape) FROM tenant_shapes) + AS distinct_tenant_shapes, + (SELECT pg_catalog.md5(pg_catalog.string_agg( + nspname || '|' || logical_shape, E'\n' ORDER BY nspname)) + FROM tenant_shapes) AS tenant_shape_fingerprint; + +\echo 'Uniform density fixture completed successfully' diff --git a/research/graphile-density/fleet.example.json b/research/graphile-density/fleet.example.json new file mode 100644 index 0000000000..03ee4b58b3 --- /dev/null +++ b/research/graphile-density/fleet.example.json @@ -0,0 +1,82 @@ +{ + "version": 1, + "tenants": [ + { + "id": "replace-with-tenant-a", + "databases": [ + { + "id": "replace-with-tenant-a-database-id", + "physicalDatabase": "replace-with-tenant-a-physical-database", + "apis": [ + { + "id": "replace-with-tenant-a-api-id", + "runtimePoolIdentity": "replace-with-tenant-a-runtime-pool-identity", + "runtimePoolIdentities": { + "origin-main": "replace-with-origin-main-tenant-a-pool-identity", + "runtime-boundary-stock": "replace-with-runtime-stock-tenant-a-pool-identity", + "cache-governor-stock": "replace-with-governor-stock-tenant-a-pool-identity", + "scoped-introspection": "replace-with-scoped-tenant-a-pool-identity" + }, + "physicalSchemas": [ + "replace-with-tenant-a-api-schema" + ], + "routingLabels": [ + "replace-with-tenant-a-api-host-or-service-label" + ], + "realtime": false, + "surfaces": [ + "api" + ] + } + ] + } + ], + "surfaces": [ + { + "name": "api", + "buildContract": "", + "buildContracts": { + "origin-main": "replace-with-origin-main-tenant-a-api-build-contract", + "runtime-boundary-stock": "replace-with-runtime-stock-tenant-a-api-build-contract", + "cache-governor-stock": "replace-with-governor-stock-tenant-a-api-build-contract", + "scoped-introspection": "replace-with-scoped-tenant-a-api-build-contract" + }, + "url": "http://127.0.0.1:{port}/graphql", + "headers": { + "host": "replace-with-tenant-a-api-host" + }, + "warmup": { + "name": "warm-schema", + "capability": "graphile-generated", + "query": "query { __typename }" + }, + "operations": [ + { + "name": "replace-with-a-real-generated-plan", + "capability": "graphile-generated", + "query": "query { __typename }" + } + ], + "canaries": [ + { + "name": "cross-schema-identifiers", + "query": "query { __typename }", + "forbiddenMatches": [ + { + "path": "/data/tenantToken", + "value": "replace-with-tenant-b-token" + } + ], + "requiredMatches": [ + { + "path": "/data/tenantToken", + "value": "replace-with-tenant-a-token" + } + ] + } + ] + } + ] + } + ] +} diff --git a/research/graphile-density/four-arm-plan.example.json b/research/graphile-density/four-arm-plan.example.json new file mode 100644 index 0000000000..c4d1753db4 --- /dev/null +++ b/research/graphile-density/four-arm-plan.example.json @@ -0,0 +1,121 @@ +{ + "version": 1, + "fleetFile": "fleet.example.json", + "artifactDir": "../../graphile-density-artifacts", + "arms": [ + { + "name": "origin-main", + "commit": "a10ea246fcc45b025713024e131fafb908171149", + "cwd": "/ABSOLUTE/PATH/TO/origin-main-worktree", + "command": ["node", "packages/cli/dist/index.js", "server", "--port", "{port}", "--origin", "*", "--servicesApi"], + "port": 3341, + "readinessUrl": "http://127.0.0.1:{port}/healthz", + "memoryUrl": "http://127.0.0.1:{port}/debug/memory", + "postgresContainer": "REPLACE_WITH_DISPOSABLE_POSTGRES_CONTAINER", + "introspectionMode": "stock" + }, + { + "name": "runtime-boundary-stock", + "commit": "de92be5a9", + "cwd": "/ABSOLUTE/PATH/TO/runtime-boundary-worktree", + "command": ["node", "packages/cli/dist/index.js", "server", "--port", "{port}", "--origin", "*", "--servicesApi"], + "port": 3342, + "readinessUrl": "http://127.0.0.1:{port}/healthz", + "memoryUrl": "http://127.0.0.1:{port}/debug/memory", + "postgresContainer": "REPLACE_WITH_DISPOSABLE_POSTGRES_CONTAINER", + "introspectionMode": "stock" + }, + { + "name": "cache-governor-stock", + "commit": "5fd90ee2b", + "cwd": "/ABSOLUTE/PATH/TO/cache-governor-worktree", + "command": ["node", "packages/cli/dist/index.js", "server", "--port", "{port}", "--origin", "*", "--servicesApi"], + "port": 3343, + "readinessUrl": "http://127.0.0.1:{port}/healthz", + "memoryUrl": "http://127.0.0.1:{port}/debug/memory", + "postgresContainer": "REPLACE_WITH_DISPOSABLE_POSTGRES_CONTAINER", + "introspectionMode": "stock" + }, + { + "name": "scoped-introspection", + "commit": "f41e480d5", + "cwd": "/ABSOLUTE/PATH/TO/scoped-introspection-worktree", + "command": ["node", "packages/cli/dist/index.js", "server", "--port", "{port}", "--origin", "*", "--servicesApi"], + "port": 3344, + "readinessUrl": "http://127.0.0.1:{port}/healthz", + "memoryUrl": "http://127.0.0.1:{port}/debug/memory", + "postgresContainer": "REPLACE_WITH_DISPOSABLE_POSTGRES_CONTAINER", + "introspectionMode": "scoped-required" + } + ], + "heapMiB": [1024, 2048, 4096], + "tenantCountsByHeapMiB": { + "1024": [1, 2, 4, 8, 16, 32, 48, 64], + "2048": [1, 4, 8, 16, 32, 64, 96, 128], + "4096": [1, 8, 16, 32, 64, 128, 192, 256] + }, + "repetitions": 3, + "runOrderSeed": "graphile-density-qualification-v1", + "requiredCapabilities": [ + "graphile-generated", + "i18n", + "llm", + "rag", + "bm25", + "tsvector", + "trigram", + "vector", + "postgis", + "ltree", + "uploads-storage", + "bulk-mutations", + "realtime", + "function-bindings" + ], + "requiredCanaries": [ + "cross-schema-identifiers", + "metadata", + "functions", + "sequences", + "prepared-statement-reuse", + "poisoned-gucs", + "rollback-savepoints", + "plugin-raw-sql", + "owner-bypass-role", + "schema-drift", + "cache-invalidation", + "concurrent-builds", + "connection-reuse" + ], + "workload": { + "durationSec": 900, + "rpsPerTenant": 0.2, + "minWorkloadRequestsPerSurface": 10, + "requestTimeoutMs": 30000, + "maxInFlight": 128, + "canaryIntervalSec": 60, + "warmupTimeoutMs": 180000, + "warmupTimeoutPerSurfaceMs": 2000, + "warmupConcurrency": 1 + }, + "gates": { + "maxErrorRate": 0.005, + "maxP99Ms": 150, + "maxPostWarmupHeapGrowthMiBPerHour": 5, + "minMedianDensityImprovement": 0.15, + "minAdditionalTenantsEveryRun": 1, + "requireZeroBleed": true, + "requireNoPostWarmupEvictions": true, + "requireNoPostWarmupBuildRefusals": true, + "requireNoPostWarmupBuilds": true, + "requirePostgresMemoryTelemetry": true, + "requireConclusiveCanaries": true, + "requireExplicitCustomerTopology": true + }, + "soak": { + "enabled": true, + "durationSec": 7200, + "tenantCount": 48, + "heapMiB": 2048 + } +} diff --git a/research/graphile-density/physical-database-density/.gitignore b/research/graphile-density/physical-database-density/.gitignore new file mode 100644 index 0000000000..b7f1ad81a9 --- /dev/null +++ b/research/graphile-density/physical-database-density/.gitignore @@ -0,0 +1,2 @@ +.local/ +artifacts/ diff --git a/research/graphile-density/physical-database-density/HOSTILE-PREFLIGHT.md b/research/graphile-density/physical-database-density/HOSTILE-PREFLIGHT.md new file mode 100644 index 0000000000..3fb98db36c --- /dev/null +++ b/research/graphile-density/physical-database-density/HOSTILE-PREFLIGHT.md @@ -0,0 +1,62 @@ +# Physical hostile preflight + +Run this validation against a dedicated, unmeasured physical-fixture server before starting cperf. Provision and serve the hostile clone with the same explicit identity and purpose: + +```bash +node research/graphile-density/physical-database-density/provision.cjs \ + --prefix pdc_hostile \ + --customers 8 \ + --out-dir /absolute/path/to/hostile-clone \ + --maintenance-database postgres \ + --run-purpose hostile-preflight \ + --clone-id fresh-preflight-clone-20260802-a + +node --expose-gc research/graphile-density/physical-database-density/server.cjs \ + --manifest /absolute/path/to/hostile-clone/provision.json \ + --secrets /absolute/path/to/hostile-clone/runtime-secrets.json \ + --customers 8 \ + --arm physical-db-idle-1s \ + --mode scoped-required \ + --runtime-pool-max 2 \ + --enable-realtime true \ + --expected-database-contract 'sha256:' \ + --blueprint-compatibility 'sha256:' \ + --run-purpose hostile-preflight \ + --clone-id fresh-preflight-clone-20260802-a +``` + +The provisioner writes a distinct 256-bit nonce into a private schema in every physical database. The nonce never leaves PostgreSQL: the credential-free manifest contains only its context-bound digest, and both the child and aggregate status endpoints query the row live and recompute the digest before reporting `verified: true`. Runtime roles have no privilege on the private schema. + +The validator loads the credential-free manifest and the provisioner's private runtime-secrets file. `--secrets` must name a regular, non-symlink file owned by the current user with no group or other permission bits (normally mode `0600`). The parent reads that file once. Each isolated worker receives exactly the representative customer's three A/B/C passwords through its private process environment, with the selected surface replaced by the probe credential; no worker receives the secrets path or another customer's credential. Neither the secrets path, its contents, nor the authenticated `CTF_CONTROL_TOKEN` is written to the artifact. + +```bash +CTF_CONTROL_TOKEN='' node \ + research/graphile-density/physical-database-density/physical-hostile-preflight.cjs \ + --manifest /absolute/path/to/provision.json \ + --secrets /absolute/path/to/runtime-secrets.json \ + --base-url http://127.0.0.1:3410 \ + --arm physical-db-idle-1s \ + --mode scoped-required \ + --preflight-clone-id fresh-preflight-clone-20260802-a \ + --output /absolute/path/to/physical-hostile-preflight.json +``` + +The validator requires the server's exact hostile-preflight clone ID, live attestation set, manifest customer set, arm, introspection mode, canonical database contract, blueprint fingerprint, and runtime artifact fingerprint. Hostile startup also recomputes every customer's structural and database-contract fingerprints from the live database rather than echoing the provision manifest. It then walks customers sequentially through the exact `/customer/` mount, requires `current_database()` to match that customer's physical identity during every dynamic identity and control sequence, invalidates the whole fleet before rebuilding every surface, and records only credential-free hashes and outcomes. + +Fixture startup admission has one safe-role control followed by an exact 15-case unsafe matrix against a representative hostile physical database. The parent validator first reads the private attestation row and recomputes its context-bound digest, then a live catalog query proves that the temporary roles have exactly the intended profiles: `SUPERUSER`, `BYPASSRLS`, `CREATEROLE`, schema ownership, and schema `CREATE`. The control-plane login, `NODE_OPTIONS`, `NODE_PATH`, and generic Graphile environment are never forwarded to a probe child. Each child validates its exact environment key set, verifies `current_database()` and `current_user` with the selected runtime credential, and substitutes each profile independently into surfaces A, B, and C in a fresh process. All 15 fixture starts must fail with `GRAPHILE_UNSAFE_RUNTIME_ROLE` before the fixture's `buildEntry` or cache publication; the safe control must start successfully with zero builds and zero resident entries. Cleanup is attempted even after an ambiguous setup failure and a live catalog audit must find zero remaining probe roles and schemas. + +This matrix exercises `complete-tenant-fixture.createFixtureServer`, whose role audit is deliberately ahead of its custom build seam. It does not claim to execute the production `graphile()` middleware path. Every worker reports the exact loaded runtime-artifact fingerprint, and all 16 worker results must match the mounted hostile server before the aggregate artifact can pass. + +This command must never be imported by or invoked from the measured cperf process. Provision the measured clone separately with `--run-purpose measurement` and a different `--clone-id`, then pass those exact values to its server. A measurement-purpose server reports `controlAvailable: false` and refuses the entire hostile control endpoint even if a valid control token is accidentally present. The measured clone must reproduce the canonical structural and database-contract fingerprints, but it must not inherit the preflight clone's sessions, PostgreSQL cache state, mutations, nonce, or cgroup history. A passing preflight artifact is security evidence only, so it is explicitly marked `performanceEvidence: false` and `customerQualified: false`. + +The runtime fingerprint hashes the deterministic resolved closure of local `dist` JavaScript reached from the fixture's runtime roots, plus the exact patched installed Graphile artifacts. Changes behind an `index.js` export stub therefore invalidate the evidence without recording absolute paths, source bytes, or credentials. + +Focused tests: + +```bash +node --test \ + research/graphile-density/complete-tenant-fixture/schema.test.cjs \ + research/graphile-density/complete-tenant-fixture/hostile-validation.test.cjs \ + research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.test.cjs \ + research/graphile-density/physical-database-density/physical-hostile-preflight.test.cjs +``` diff --git a/research/graphile-density/physical-database-density/README.md b/research/graphile-density/physical-database-density/README.md new file mode 100644 index 0000000000..9e4a78dc23 --- /dev/null +++ b/research/graphile-density/physical-database-density/README.md @@ -0,0 +1,100 @@ +# Physical-database customer-density fixture + +This fixture measures complete customers per actual GiB consumed; it does not try to fit the service into 1 GiB. The 1, 2, and 4 GiB V8 settings are repeatable pressure points, while the primary denominator is the maximum post-warmup time-aligned sum of current Node RSS and the dedicated PostgreSQL container's raw cgroup-v2 memory charge. + +One logical customer owns one physical PostgreSQL database. Every database has the same three canonical GraphQL surfaces (`ctf_a`, `ctf_b`, and `ctf_c`), the same realtime schemas, build-visible dependency schemas, extension versions, and role-relative ACL shape. Each surface still gets its own least-privilege login, pool identity, and dedicated Graphile instance, so this is the secure production baseline rather than a shared-blueprint implementation. + +The provisioner emits two hashes. The database-contract hash covers normalized schema DDL and ACLs, extension versions, and runtime-role safety flags. The input preflight combines that hash with the exact fixture plugin/settings configuration, dependency closure, source, and built runtime artifact hash. That second hash proves only the structural prerequisites for a future no-rewrite blueprint experiment; it does not authorize sharing, and the fixture never rewrites SQL. + +## Run locally + +Use an existing disposable PostgreSQL 17 container with all fixture extensions available. Input generation only inspects and preflights that exact container. Each later cperf job deliberately removes it and creates a fresh replacement with the same immutable image, loopback port, cgroup resource limits, and narrowly validated `postgres -c name=value` settings; it never inherits data volumes and never accesses `constructive-db`. + +```bash +pnpm build + +PGHOST=127.0.0.1 \ +PGPORT=55432 \ +PGUSER=postgres \ +PGPASSWORD=local-admin-password \ +PGDATABASE=postgres \ +node research/graphile-density/physical-database-density/provision.cjs \ + --prefix pdc_density \ + --customers 64 \ + --out-dir research/graphile-density/physical-database-density/.local \ + --maintenance-database postgres +``` + +Provisioning fails if any target already exists. Replacing this exact disposable prefix requires both `--recreate` and `--yes`; no wildcard or workspace-wide deletion path exists. + +Before generating a density plan, derive the governor calibration from at least three clean, conclusive one-surface `catalog-bench` results produced with the ordered production schema set and explicit dependency allowlist. Every source must have build-state retirement enabled and must prove that its exact PostgreSQL introspection PID disappeared before a different steady-state PID was acquired. The tool takes the maximum measured retained heap, server baseline, build-transient heap, and build-transient RSS across repetitions, applies an explicit safety factor, and binds the retirement proof, source artifact hashes, and provisioned database-contract hash into one calibration identity. + +```bash +node research/graphile-density/physical-database-density/cache-calibration.cjs \ + --results /tmp/catalog/rep-1/result.json,/tmp/catalog/rep-2/result.json,/tmp/catalog/rep-3/result.json \ + --manifest research/graphile-density/physical-database-density/.local/provision.json \ + --safety-factor 1.25 \ + --out research/graphile-density/physical-database-density/.local/cache-calibration.json +``` + +Inconclusive, cross-tenant, legacy one-schema-layout, or scope-mismatched results are rejected. Dirty source state is retained explicitly as `sourceWorktreesClean: false`, which permits local diagnostic sizing but cannot turn the later run into qualifying evidence because cperf independently requires clean server provenance. The calibration is a governor sizing input, not a performance claim; the complete-customer workload still decides qualification. + +Generate the credential-free fleet and plan after provisioning. The secret file remains mode `0600`, and generated plans contain only its path. + +```bash +node research/graphile-density/physical-database-density/generate-inputs.cjs \ + --manifest research/graphile-density/physical-database-density/.local/provision.json \ + --secrets research/graphile-density/physical-database-density/.local/runtime-secrets.json \ + --out-dir research/graphile-density/physical-database-density/.local/inputs \ + --cache-calibration research/graphile-density/physical-database-density/.local/cache-calibration.json \ + --postgres-container postgres-density \ + --arm-profile density-tuning \ + --tenant-counts-by-heap-mib '1024:8,12,16;2048:16,24,32;4096:32,48,64' \ + --heaps 1024,2048,4096 \ + --repetitions 3 \ + --duration-sec 900 + +node packages/perf-harness/dist/index.js validate \ + --plan research/graphile-density/physical-database-density/.local/inputs/plan.json + +node packages/perf-harness/dist/index.js run \ + --plan research/graphile-density/physical-database-density/.local/inputs/plan.json +``` + +The container passed to `--postgres-container` is destructive, disposable fixture state. Generation captures its exact 64-character ID as the only unlabeled container the runner may remove; every replacement must carry the exact fixture, prefix, purpose, image, port, command, and resource-limit contract before it can be removed again. A same-named unrelated container fails closed. The source command may be the image default `postgres`, in which case generation adds and pins a sufficient `max_connections`; otherwise only the checked-in PostgreSQL setting allowlist is accepted, and explicit settings such as `max_connections` and `shared_buffers` are preserved and audited live. + +For each matrix coordinate the prepare wrapper reuses the validated private credential template, but live runtime-pool and Graphile cache identities use a process-random keyed HMAC and intentionally change between preflight and measurement. The fleet carries deterministic, credential-free pool and build-contract fingerprints instead; each process proves the one-to-one mapping from those fingerprints to its live role, database, schema, pool object, and resident cache entry. The wrapper writes a new `0600` secret file and credential-free manifest under that run's artifact directory, provisions a unique run-bound clone and nonce set into the fresh cluster, then runs the full live DDL/ACL/role/extension audit outside the measured Node process. Cperf starts the server with the attested per-run manifest path, manifest hash, and clone ID; a static preflight manifest cannot accidentally satisfy that binding. + +The `density-tuning` profile compares the dedicated-listener baseline with one exact notification broker per physical customer and one-client runtime pools. It isolates stock prepared statements, prepared statements disabled, native single-checkout client retirement (`maxUses=1`), and each V8 size profile before testing a cumulative single-checkout/size arm. Input preflight starts a fresh Node child for every arm, waits for that child to report readiness, validates every customer status plus representative shared realtime, and waits for the child to terminate before starting the next arm. This process boundary matters because Dataplan and pool modules may snapshot environment on first import; setting and restoring `process.env` around multiple in-process servers cannot prove arm isolation. Each child strips ambient Node preload/module-path hooks, runs the arm's exact V8 profile, behaviorally attests the loaded Dataplan prepared-statement cache, and keeps process-global `PG_POOL_MAX=1` plus `PG_POOL_MAX_USES=0`; arm-specific runtime capacity and `maxUses` travel only through exact server options. None is accepted from a microbenchmark alone. The default `idle` profile instead isolates PostgreSQL idle-client retention at 30, 5, and 1 seconds. Every heap checkpoint gets an explicit environment block containing the measured instance cost, server reserve, build reserve, RSS build reserve, calibrated budget capacity as the cache ceiling, `GRAPHILE_CACHE_ADMISSION_MODE=preserve-resident`, and the calibration identity. Input generation resolves Node's effective V8 heap limit and fails before starting runtime status collection when the calibrated resident-plus-next-build budget cannot admit all three surfaces for that heap's requested maximum. The plan records the required residents, calibrated capacity, remaining headroom, and stable boundary refusal reason/code; a 1 GiB checkpoint may still fail under real pressure or correctness gates, but it cannot fail merely because it inherited the old fixed 768 MiB reserve. + +Before scoring begins, every Graphile surface is warm, every configured capability has returned exact customer and physical-database evidence, every realtime manager is running, and one `graphql-transport-ws` subscription per surface has received its configured tenant-specific database event through the exact customer/tenant route. Fixture-only `BEFORE` triggers stamp read/search source rows and upload, bulk, function-binding, and realtime side effects with `current_database()`; i18n translations and RAG source content also carry a database-derived marker. Collection oracles assert every returned row plus nonempty one-row cardinality, and deterministic LLM/RAG, vector, PostGIS, ltree, and search results have operation-specific semantic assertions. + +Every presigned-upload invocation selects a fixture-only VOLATILE mutation sibling that returns `current_database()`, so timed workload calls carry direct physical-database evidence. Coverage then extracts the exact `fileId` returned by that invocation and verifies the stamped `appFiles` row by both ID and content hash, which prevents a reused fixture row from satisfying the check. Buckets start with `physical_name = NULL`, forcing the first upload through the plugin's `withPgClient(null)` provisioning lane while the fixture keeps forced RLS intact. Missing, ambiguous, or foreign evidence fails with stable oracle codes; none of these fixture-only fields or functions belongs to the production API design. + +Realtime verification requires permanent tenant and physical-database invariants plus a one-time prime payload; later legitimate payload changes remain valid, while another selected database is an explicit forbidden match. The subscription clients live in the perf-harness driver process, outside the measured server RSS; the server retains only its real websocket/session state and independently reports one accepted live connection per surface. The post-warmup hook asserts those server-side managers and connections instead of constructing load-generator clients. Driver credentials may be sourced from environment-variable names declared in the fleet, and neither their values nor resolved headers are written to evidence. Post-warmup samples then record Graphile residency and the live budget capacity/calibration identity, concrete `pg.Pool` clients, `pg_stat_activity` backends, raw cgroup-v2 `memory.current`, `memory.peak`, `memory.stat`, `memory.events`, and Docker working set. + +One generated plan owns the full customer-count ramp, and the highest count across all heaps must equal the source manifest's physical database count. Use `--tenant-counts` for one shared ramp or `--tenant-counts-by-heap-mib` for semicolon-separated heap-specific ramps; the latter lets each heap push its own density boundary without forcing the smallest heap to admit the largest fleet. Every matrix coordinate still gets a fresh container provisioned with exactly that coordinate's customer count, so lower-count samples cannot inherit unused databases or catalog cache. Keeping the complete ramp, every configured arm, and every repetition under one immutable plan/fleet cohort lets the report reject spliced or partial evidence while still bracketing the highest passing count with a greater failure. + +## Qualification conditions + +Use one dedicated PostgreSQL container for the measured Node process, with no unrelated databases or traffic. The memory endpoint enumerates `pg_database` and the score fails unless the container contains only the maintenance database and the selected physical customer databases, so extra provisioned-but-unserved databases and shared development databases both make the run non-qualifying. Every scheduled run gets a unique Docker ID, cgroup identity, PostgreSQL system identifier, clone ID, attestation set, and nonce set; reuse of any one identity rejects every affected result, including when separately generated result files are combined for reporting. On Linux the sampler reads the target container's cgroup directly from the host; Docker Desktop falls back to reads inside the container, so publishable numbers should be reproduced on a Linux cgroup-v2 host. + +The perf harness rejects a capacity point unless a greater customer count fails, all repetitions are present, every physical database and realtime transport remains resident, all request-path isolation canaries are conclusive, cross-customer results stay zero, and latency, error, eviction, build, pool, backend, OOM, and heap-growth gates pass. Each 15-minute measured run performs full request-path canary sweeps before and after timed traffic, plus 14 one-canary-per-surface rounds at 60-second intervals. The per-surface rotation is deterministic and staggered, covers all 14 configured canaries exactly once during the timed window, runs at concurrency 16 across surfaces, and never drops an overlapping round; incomplete or deadline-late validation fails the run. These passive probes do not count as induced hostile validation. The current generated physical plan deliberately omits `qualification.hostileValidationEvidence`, so cperf records diagnostic evidence until one immutable `exact-runtime-hostile-validation-v1` report is attached for every exact arm runtime and configuration. Configured V8 old space, Node-only RSS, Docker working set, and cumulative peaks remain diagnostics; customers per aligned Node-plus-PostgreSQL GiB is the decision metric once all qualification prerequisites exist. + +The harness also requires a clean pinned worktree for qualifying evidence. Until these local changes are reviewed and recorded on a local branch, smoke runs can validate mechanics but intentionally cannot count as performance evidence. A smoke still gets a fresh database epoch so it exercises the real lifecycle, but its five-second workload always receives zero qualified customers and cannot enter a capacity or density decision. + +## Offline checks + +```bash +node --test \ + research/graphile-density/physical-database-density/cache-calibration.test.cjs \ + research/graphile-density/physical-database-density/lib.test.cjs \ + research/graphile-density/physical-database-density/inputs.test.cjs \ + research/graphile-density/physical-database-density/prepare-measurement-run.test.cjs \ + research/graphile-density/physical-database-density/measurement-attestation.test.cjs \ + research/graphile-density/physical-database-density/server-realtime.test.cjs \ + research/graphile-density/physical-database-density/server-retained-memory.test.cjs + +pnpm --dir packages/perf-harness exec jest --runInBand +pnpm --dir packages/perf-harness build +``` diff --git a/research/graphile-density/physical-database-density/cache-calibration.cjs b/research/graphile-density/physical-database-density/cache-calibration.cjs new file mode 100644 index 0000000000..b3e0512700 --- /dev/null +++ b/research/graphile-density/physical-database-density/cache-calibration.cjs @@ -0,0 +1,390 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const CALIBRATION_KIND = 'graphile-cache-measured-calibration-v2'; +const SHA256 = /^sha256:[a-f0-9]{64}$/; + +const canonicalize = (value) => { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [ + key, + canonicalize(value[key]), + ])); +}; + +const sha256Canonical = (value) => `sha256:${crypto.createHash('sha256') + .update(JSON.stringify(canonicalize(value))) + .digest('hex')}`; + +const fileSha256 = (file) => `sha256:${crypto.createHash('sha256') + .update(fs.readFileSync(file)) + .digest('hex')}`; + +const positiveSafeInteger = (value, label) => { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`PDCF_CALIBRATION_POSITIVE_INTEGER_REQUIRED:${label}`); + } + return value; +}; + +const validateSafetyFactor = (value) => { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 1 || value > 3) { + throw new Error('PDCF_CALIBRATION_SAFETY_FACTOR_INVALID'); + } + return value; +}; + +const validateCatalogResult = (result, sourceFile) => { + if ( + !result + || result.version !== 1 + || result.status !== 'performance-only' + || !['stock', 'scoped-required'].includes(result.mode) + || result.introspectionClientReleaseMode !== 'destroy' + || result.releaseBuildStateAfterValidation !== true + || typeof result.worktreeDirty !== 'boolean' + || !Array.isArray(result.schemaSets) + || result.schemaSets.length !== 1 + || !Array.isArray(result.schemaSets[0]) + || result.schemaSets[0].length === 0 + || !Array.isArray(result.allowedDependencySchemas) + || !Array.isArray(result.builds) + || result.builds.length !== 1 + || !Array.isArray(result.snapshots) + || result.tokenCanariesConclusive !== true + || result.tokenCanariesPassed !== true + || result.bleedViolations !== 0 + || typeof result.fixtureFingerprint !== 'string' + || result.fixtureFingerprint.length === 0 + || !/^[a-f0-9]{64}$/.test(result.sourceStateSha256 ?? '') + || !/^[a-f0-9]{64}$/.test(result.executedEntrySha256 ?? '') + ) { + throw new Error(`PDCF_CALIBRATION_RESULT_NOT_CONCLUSIVE:${sourceFile}`); + } + const snapshot = result.snapshots.find((candidate) => candidate.instances === 1); + if (!snapshot) throw new Error(`PDCF_CALIBRATION_ONE_SURFACE_SNAPSHOT_REQUIRED:${sourceFile}`); + const build = result.builds[0]; + const introspectionBackendPid = positiveSafeInteger( + build.introspectionBackendPid, + `${sourceFile}:introspectionBackendPid`, + ); + const steadyBackendPid = positiveSafeInteger( + build.steadyBackendPid, + `${sourceFile}:steadyBackendPid`, + ); + if ( + build.introspectionBackendRetired !== true + || introspectionBackendPid === steadyBackendPid + || result.postgresBackendMeasurement?.expectedRetirementChecks !== result.builds.length + || result.postgresBackendMeasurement?.completedRetirementChecks !== result.builds.length + || result.postgresBackendMeasurement?.allExpectedRetirementsProven !== true + ) { + throw new Error(`PDCF_CALIBRATION_INTROSPECTION_RETIREMENT_UNPROVEN:${sourceFile}`); + } + if (!Number.isSafeInteger(build.buildTransientSampleCount) || build.buildTransientSampleCount <= 0) { + throw new Error(`PDCF_CALIBRATION_BUILD_SAMPLES_REQUIRED:${sourceFile}`); + } + const retainedHeapBytes = positiveSafeInteger( + Math.ceil(snapshot.heapDeltaBytes), + `${sourceFile}:retainedHeapBytes`, + ); + const serverBaselineHeapBytes = positiveSafeInteger( + Math.ceil(build.buildBaselineHeapUsedBytes), + `${sourceFile}:serverBaselineHeapBytes`, + ); + const buildTransientHeapBytes = positiveSafeInteger( + Math.ceil(Math.max( + build.sampledBuildPeakHeapDeltaBytes ?? 0, + retainedHeapBytes, + )), + `${sourceFile}:buildTransientHeapBytes`, + ); + const buildTransientRssBytes = positiveSafeInteger( + Math.ceil(Math.max( + build.sampledBuildPeakRssDeltaBytes ?? 0, + build.processBuildPeakRssDeltaBytes ?? 0, + )), + `${sourceFile}:buildTransientRssBytes`, + ); + return { + mode: result.mode, + introspectionClientReleaseMode: result.introspectionClientReleaseMode, + releaseBuildStateAfterValidation: result.releaseBuildStateAfterValidation, + introspectionBackendRetirement: { + conclusive: true, + introspectionBackendPid, + steadyBackendPid, + }, + fixtureFingerprint: result.fixtureFingerprint, + schemaSets: result.schemaSets, + allowedDependencySchemas: result.allowedDependencySchemas, + sourceStateSha256: result.sourceStateSha256, + executedEntrySha256: result.executedEntrySha256, + worktreeDirty: result.worktreeDirty, + retainedHeapBytes, + serverBaselineHeapBytes, + buildTransientHeapBytes, + buildTransientRssBytes, + }; +}; + +const deriveCacheCalibration = ({ + resultFiles, + databaseContractFingerprint, + safetyFactor = 1.25, +}) => { + if (!Array.isArray(resultFiles) || resultFiles.length < 3) { + throw new Error('PDCF_CALIBRATION_THREE_RESULTS_REQUIRED'); + } + if (!SHA256.test(databaseContractFingerprint ?? '')) { + throw new Error('PDCF_CALIBRATION_DATABASE_CONTRACT_REQUIRED'); + } + validateSafetyFactor(safetyFactor); + const sources = resultFiles.map((sourceFile) => { + const absolute = path.resolve(sourceFile); + const result = JSON.parse(fs.readFileSync(absolute, 'utf8')); + return { + fileSha256: fileSha256(absolute), + measurement: validateCatalogResult(result, absolute), + }; + }); + const modes = new Set(sources.map((source) => source.measurement.mode)); + const releaseModes = new Set(sources.map( + (source) => source.measurement.introspectionClientReleaseMode + )); + const buildStateRetirementModes = new Set(sources.map( + (source) => source.measurement.releaseBuildStateAfterValidation + )); + const fixtureFingerprints = new Set( + sources.map((source) => source.measurement.fixtureFingerprint) + ); + const schemaContracts = new Set(sources.map((source) => JSON.stringify({ + schemaSets: source.measurement.schemaSets, + allowedDependencySchemas: source.measurement.allowedDependencySchemas, + }))); + if ( + modes.size !== 1 + || releaseModes.size !== 1 + || !releaseModes.has('destroy') + || buildStateRetirementModes.size !== 1 + || !buildStateRetirementModes.has(true) + || fixtureFingerprints.size !== 1 + || schemaContracts.size !== 1 + ) { + throw new Error('PDCF_CALIBRATION_RESULT_SCOPE_MISMATCH'); + } + const maximum = (field) => Math.max(...sources.map( + (source) => source.measurement[field] + )); + const measured = { + repetitions: sources.length, + retainedHeapPerSurfaceBytes: maximum('retainedHeapBytes'), + serverBaselineHeapBytes: maximum('serverBaselineHeapBytes'), + buildTransientHeapBytes: maximum('buildTransientHeapBytes'), + buildTransientRssBytes: maximum('buildTransientRssBytes'), + }; + const configured = { + instanceHeapBytes: Math.ceil(measured.retainedHeapPerSurfaceBytes * safetyFactor), + serverReserveBytes: Math.ceil(measured.serverBaselineHeapBytes * safetyFactor), + buildReserveBytes: Math.ceil(measured.buildTransientHeapBytes * safetyFactor), + rssBuildReserveBytes: Math.ceil(measured.buildTransientRssBytes * safetyFactor), + }; + const identityPayload = { + kind: CALIBRATION_KIND, + databaseContractFingerprint, + introspectionMode: [...modes][0], + introspectionClientReleaseMode: [...releaseModes][0], + releaseBuildStateAfterValidation: [...buildStateRetirementModes][0], + introspectionBackendRetirementConclusive: sources.every( + (source) => source.measurement.introspectionBackendRetirement.conclusive === true + ), + fixtureFingerprint: [...fixtureFingerprints][0], + schemaContract: JSON.parse([...schemaContracts][0]), + safetyFactor, + measured, + configured, + sourceWorktreesClean: sources.every( + (source) => source.measurement.worktreeDirty === false + ), + sources: sources.map(({ fileSha256: sourceSha256, measurement }) => ({ + sourceSha256, + sourceStateSha256: measurement.sourceStateSha256, + executedEntrySha256: measurement.executedEntrySha256, + worktreeDirty: measurement.worktreeDirty, + introspectionBackendRetirement: measurement.introspectionBackendRetirement, + })), + }; + return { + version: 2, + ...identityPayload, + calibrationId: sha256Canonical(identityPayload), + }; +}; + +const validateCacheCalibration = ( + calibration, + { databaseContractFingerprint, introspectionMode } = {}, +) => { + if ( + !calibration + || calibration.version !== 2 + || calibration.kind !== CALIBRATION_KIND + || !SHA256.test(calibration.calibrationId ?? '') + || !SHA256.test(calibration.databaseContractFingerprint ?? '') + || !Array.isArray(calibration.sources) + || calibration.sources.length < 3 + || !['stock', 'scoped-required'].includes(calibration.introspectionMode) + || calibration.introspectionClientReleaseMode !== 'destroy' + || calibration.releaseBuildStateAfterValidation !== true + || calibration.introspectionBackendRetirementConclusive !== true + || typeof calibration.fixtureFingerprint !== 'string' + || calibration.fixtureFingerprint.length === 0 + || typeof calibration.sourceWorktreesClean !== 'boolean' + ) { + throw new Error('PDCF_CACHE_CALIBRATION_INVALID'); + } + validateSafetyFactor(calibration.safetyFactor); + for (const field of [ + 'retainedHeapPerSurfaceBytes', + 'serverBaselineHeapBytes', + 'buildTransientHeapBytes', + 'buildTransientRssBytes', + ]) positiveSafeInteger(calibration.measured?.[field], `measured.${field}`); + for (const field of [ + 'instanceHeapBytes', + 'serverReserveBytes', + 'buildReserveBytes', + 'rssBuildReserveBytes', + ]) positiveSafeInteger(calibration.configured?.[field], `configured.${field}`); + if (calibration.measured?.repetitions !== calibration.sources.length) { + throw new Error('PDCF_CACHE_CALIBRATION_REPETITION_MISMATCH'); + } + if (calibration.sources.some((source) => ( + !SHA256.test(source?.sourceSha256 ?? '') + || !/^[a-f0-9]{64}$/.test(source?.sourceStateSha256 ?? '') + || !/^[a-f0-9]{64}$/.test(source?.executedEntrySha256 ?? '') + || source?.introspectionBackendRetirement?.conclusive !== true + || !Number.isSafeInteger( + source?.introspectionBackendRetirement?.introspectionBackendPid + ) + || source.introspectionBackendRetirement.introspectionBackendPid <= 0 + || !Number.isSafeInteger(source?.introspectionBackendRetirement?.steadyBackendPid) + || source.introspectionBackendRetirement.steadyBackendPid <= 0 + || source.introspectionBackendRetirement.introspectionBackendPid + === source.introspectionBackendRetirement.steadyBackendPid + ))) { + throw new Error('PDCF_CACHE_CALIBRATION_SOURCE_INVALID'); + } + if ( + calibration.sourceWorktreesClean + !== calibration.sources.every((source) => source.worktreeDirty === false) + ) { + throw new Error('PDCF_CACHE_CALIBRATION_SOURCE_CLEANLINESS_MISMATCH'); + } + const expectedConfigured = { + instanceHeapBytes: Math.ceil( + calibration.measured.retainedHeapPerSurfaceBytes * calibration.safetyFactor + ), + serverReserveBytes: Math.ceil( + calibration.measured.serverBaselineHeapBytes * calibration.safetyFactor + ), + buildReserveBytes: Math.ceil( + calibration.measured.buildTransientHeapBytes * calibration.safetyFactor + ), + rssBuildReserveBytes: Math.ceil( + calibration.measured.buildTransientRssBytes * calibration.safetyFactor + ), + }; + if (JSON.stringify(expectedConfigured) !== JSON.stringify(calibration.configured)) { + throw new Error('PDCF_CACHE_CALIBRATION_FORMULA_MISMATCH'); + } + const { calibrationId: _calibrationId, version: _version, ...identityPayload } = calibration; + if (sha256Canonical(identityPayload) !== calibration.calibrationId) { + throw new Error('PDCF_CACHE_CALIBRATION_ID_MISMATCH'); + } + if ( + databaseContractFingerprint + && calibration.databaseContractFingerprint !== databaseContractFingerprint + ) { + throw new Error('PDCF_CACHE_CALIBRATION_DATABASE_CONTRACT_MISMATCH'); + } + if (introspectionMode && calibration.introspectionMode !== introspectionMode) { + throw new Error('PDCF_CACHE_CALIBRATION_MODE_MISMATCH'); + } + return calibration; +}; + +const computeCalibratedCapacity = (heapLimitBytes, configured) => { + positiveSafeInteger(heapLimitBytes, 'heapLimitBytes'); + const instance = positiveSafeInteger(configured?.instanceHeapBytes, 'instanceHeapBytes'); + const server = positiveSafeInteger(configured?.serverReserveBytes, 'serverReserveBytes'); + const build = positiveSafeInteger(configured?.buildReserveBytes, 'buildReserveBytes'); + if (server + build > heapLimitBytes) return 0; + const backingCapacity = Math.max( + 1024, + Math.min(65_536, Math.floor(heapLimitBytes / (256 * 1024))), + ); + const byResidency = Math.floor((heapLimitBytes - server) / instance); + const byRebuild = Math.floor((heapLimitBytes - server - build) / instance) + 1; + return Math.max(0, Math.min(backingCapacity, byResidency, byRebuild)); +}; + +const parseArgs = (argv) => { + const result = {}; + for (let index = 0; index < argv.length; index += 1) { + const name = argv[index]; + if (!name.startsWith('--') || index + 1 >= argv.length) { + throw new Error(`PDCF_CALIBRATION_ARGUMENT_INVALID:${name}`); + } + result[name.slice(2)] = argv[++index]; + } + return result; +}; + +const main = () => { + const args = parseArgs(process.argv.slice(2)); + if (!args.results || !args.out) throw new Error('PDCF_CALIBRATION_ARGUMENTS_REQUIRED'); + if (Boolean(args.manifest) === Boolean(args['database-contract'])) { + throw new Error('PDCF_CALIBRATION_REQUIRES_ONE_DATABASE_CONTRACT_SOURCE'); + } + const manifest = args.manifest + ? JSON.parse(fs.readFileSync(path.resolve(args.manifest), 'utf8')) + : null; + const calibration = deriveCacheCalibration({ + resultFiles: args.results.split(',').map((value) => value.trim()).filter(Boolean), + databaseContractFingerprint: manifest?.canonicalDatabaseContractFingerprint + ?? args['database-contract'], + safetyFactor: Number(args['safety-factor'] ?? '1.25'), + }); + const output = path.resolve(args.out); + fs.mkdirSync(path.dirname(output), { recursive: true }); + if (fs.existsSync(output)) throw new Error(`PDCF_CALIBRATION_REFUSES_OVERWRITE:${output}`); + fs.writeFileSync(output, `${JSON.stringify(calibration, null, 2)}\n`, { mode: 0o644 }); + process.stdout.write(`${JSON.stringify({ + status: 'calibrated', + calibrationId: calibration.calibrationId, + output, + })}\n`); +}; + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} + +module.exports = { + CALIBRATION_KIND, + computeCalibratedCapacity, + deriveCacheCalibration, + sha256Canonical, + validateCacheCalibration, + validateCatalogResult, +}; diff --git a/research/graphile-density/physical-database-density/cache-calibration.test.cjs b/research/graphile-density/physical-database-density/cache-calibration.test.cjs new file mode 100644 index 0000000000..e0ed070557 --- /dev/null +++ b/research/graphile-density/physical-database-density/cache-calibration.test.cjs @@ -0,0 +1,146 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { + computeCalibratedCapacity, + deriveCacheCalibration, + validateCacheCalibration, +} = require('./cache-calibration.cjs'); + +const MIB = 1024 ** 2; +const databaseContractFingerprint = `sha256:${'d'.repeat(64)}`; + +const result = (ordinal) => ({ + version: 1, + status: 'performance-only', + mode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: true, + worktreeDirty: false, + schemaSets: [['ctf_a']], + allowedDependencySchemas: ['ctf_extensions'], + fixtureFingerprint: 'fixture-fingerprint-v1', + sourceStateSha256: String(ordinal).repeat(64), + executedEntrySha256: String(ordinal).repeat(64), + tokenCanariesConclusive: true, + tokenCanariesPassed: true, + bleedViolations: 0, + builds: [{ + introspectionBackendPid: 1000 + ordinal, + steadyBackendPid: 2000 + ordinal, + introspectionBackendRetired: true, + buildTransientSampleCount: 2, + buildBaselineHeapUsedBytes: (40 + ordinal) * MIB, + sampledBuildPeakHeapDeltaBytes: (80 + ordinal) * MIB, + sampledBuildPeakRssDeltaBytes: (90 + ordinal) * MIB, + processBuildPeakRssDeltaBytes: (100 + ordinal) * MIB, + }], + snapshots: [{ + instances: 1, + heapDeltaBytes: (10 + ordinal) * MIB, + }], + postgresBackendMeasurement: { + expectedRetirementChecks: 1, + completedRetirementChecks: 1, + allExpectedRetirementsProven: true, + }, +}); + +describe('physical density cache calibration', () => { + it('derives a safety-factored, source-bound calibration from three clean results', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-calibration-')); + const files = [1, 2, 3].map((ordinal) => { + const file = path.join(directory, `result-${ordinal}.json`); + fs.writeFileSync(file, JSON.stringify(result(ordinal))); + return file; + }); + const calibration = deriveCacheCalibration({ + resultFiles: files, + databaseContractFingerprint, + safetyFactor: 1.25, + }); + assert.equal(calibration.measured.repetitions, 3); + assert.equal(calibration.sourceWorktreesClean, true); + assert.equal(calibration.introspectionClientReleaseMode, 'destroy'); + assert.equal(calibration.releaseBuildStateAfterValidation, true); + assert.equal(calibration.introspectionBackendRetirementConclusive, true); + assert.equal(calibration.measured.retainedHeapPerSurfaceBytes, 13 * MIB); + assert.equal(calibration.configured.instanceHeapBytes, Math.ceil(13 * MIB * 1.25)); + assert.equal(calibration.configured.buildReserveBytes, Math.ceil(83 * MIB * 1.25)); + assert.equal(calibration.configured.rssBuildReserveBytes, Math.ceil(103 * MIB * 1.25)); + assert.equal(validateCacheCalibration(calibration, { + databaseContractFingerprint, + introspectionMode: 'scoped-required', + }), calibration); + assert.throws(() => validateCacheCalibration({ + ...calibration, + configured: { ...calibration.configured, buildReserveBytes: 1 }, + }), /CALIBRATION_FORMULA_MISMATCH|CALIBRATION_ID_MISMATCH/); + }); + + it('fails closed on inconclusive or mismatched source measurements', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-calibration-bad-')); + const values = [result(1), result(2), result(3)]; + values[1].tokenCanariesPassed = false; + const files = values.map((value, index) => { + const file = path.join(directory, `result-${index}.json`); + fs.writeFileSync(file, JSON.stringify(value)); + return file; + }); + assert.throws(() => deriveCacheCalibration({ + resultFiles: files, + databaseContractFingerprint, + }), /RESULT_NOT_CONCLUSIVE/); + assert.throws(() => deriveCacheCalibration({ + resultFiles: files.slice(0, 2), + databaseContractFingerprint, + }), /THREE_RESULTS_REQUIRED/); + }); + + it('fails closed when build-state or PostgreSQL introspection retirement is unproven', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-calibration-retire-')); + const writeResults = (values, label) => values.map((value, index) => { + const file = path.join(directory, `${label}-${index}.json`); + fs.writeFileSync(file, JSON.stringify(value)); + return file; + }); + const retained = [result(1), result(2), result(3)]; + retained[1].releaseBuildStateAfterValidation = false; + assert.throws(() => deriveCacheCalibration({ + resultFiles: writeResults(retained, 'retained'), + databaseContractFingerprint, + }), /RESULT_NOT_CONCLUSIVE/); + + const reused = [result(1), result(2), result(3)]; + reused[1].builds[0].steadyBackendPid = reused[1].builds[0].introspectionBackendPid; + assert.throws(() => deriveCacheCalibration({ + resultFiles: writeResults(reused, 'reused'), + databaseContractFingerprint, + }), /INTROSPECTION_RETIREMENT_UNPROVEN/); + + const unproven = [result(1), result(2), result(3)]; + unproven[1].postgresBackendMeasurement.allExpectedRetirementsProven = false; + assert.throws(() => deriveCacheCalibration({ + resultFiles: writeResults(unproven, 'unproven'), + databaseContractFingerprint, + }), /INTROSPECTION_RETIREMENT_UNPROVEN/); + }); + + it('computes admission capacity with both resident and next-build budgets', () => { + assert.equal(computeCalibratedCapacity(1024 * MIB, { + instanceHeapBytes: 16 * MIB, + serverReserveBytes: 64 * MIB, + buildReserveBytes: 128 * MIB, + }), 53); + assert.equal(computeCalibratedCapacity(1024 * MIB, { + instanceHeapBytes: 16 * MIB, + serverReserveBytes: 256 * MIB, + buildReserveBytes: 768 * MIB, + }), 1); + }); +}); diff --git a/research/graphile-density/physical-database-density/generate-inputs.cjs b/research/graphile-density/physical-database-density/generate-inputs.cjs new file mode 100644 index 0000000000..92aa92219b --- /dev/null +++ b/research/graphile-density/physical-database-density/generate-inputs.cjs @@ -0,0 +1,976 @@ +'use strict'; + +const { execFileSync, spawn } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + DEFAULT_IDLE_ARMS, + DENSITY_TUNING_ARMS, + FIXTURE_DIR, + PROCESS_GLOBAL_POOL_MAX, + REPO_ROOT, + atomicWriteJson, + loadProvision, + makeCacheCapacityProofByHeapMiB, + makeFleet, + makePlan, + cursorHeartbeatMsForArm, + cursorPollMsForArm, + notificationModeForArm, + preparedStatementCacheSizeForArm, + runtimePoolMaxForArm, + runtimePoolMaxUsesForArm, + validateCustomerCountMatrix, +} = require('./lib.cjs'); +const { + TENANTS, + parseArgs, + parsePositiveInteger, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); +const completeFixtureServer = require('../complete-tenant-fixture/server.cjs'); +const { validateCacheCalibration } = require('./cache-calibration.cjs'); +const { + captureContainerTemplate, + inspectDockerContainer, +} = require('./prepare-measurement-run.cjs'); + +const loadRealtimeDriver = () => require(path.join( + REPO_ROOT, + 'packages/perf-harness/dist/realtime.js', +)).createRealtimeDriver; + +const fileSha256 = (file) => crypto.createHash('sha256') + .update(fs.readFileSync(file)) + .digest('hex'); + +const expectedHeapLimitBytes = (heapMiB) => { + const output = execFileSync(process.execPath, [ + `--max-old-space-size=${heapMiB}`, + '-e', + 'process.stdout.write(String(require("node:v8").getHeapStatistics().heap_size_limit))', + ], { + encoding: 'utf8', + env: { ...process.env, NODE_OPTIONS: '' }, + }).trim(); + const value = Number(output); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`PDCF_EXPECTED_HEAP_LIMIT_INVALID:${heapMiB}:${output}`); + } + return value; +}; + +const canonicalize = (value) => { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [ + key, + canonicalize(value[key]), + ])); +}; + +const sha256Canonical = (value) => `sha256:${crypto.createHash('sha256') + .update(JSON.stringify(canonicalize(value))) + .digest('hex')}`; + +const makeArmPreflightEnvironment = (arm, environment = process.env) => { + const armEnvironment = { ...environment }; + armEnvironment.PG_POOL_IDLE_TIMEOUT_MS = String(arm.idleTimeoutMs); + armEnvironment.PG_POOL_MAX = String(PROCESS_GLOBAL_POOL_MAX); + // Keep process-global and notification pools reusable. The exact runtime + // pool gets its arm-specific maxUses through the explicit server option. + armEnvironment.PG_POOL_MAX_USES = '0'; + // This child receives the secret-file path and control-plane credentials. + // Ambient preloads and module-resolution paths may execute unreviewed code + // before the fixture can enforce its own boundary, so fail closed here. + armEnvironment.NODE_OPTIONS = ''; + delete armEnvironment.NODE_PATH; + const preparedStatementCacheSize = preparedStatementCacheSizeForArm(arm); + if (preparedStatementCacheSize == null) { + delete armEnvironment.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE; + } else { + armEnvironment.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE = + String(preparedStatementCacheSize); + } + return armEnvironment; +}; + +const V8_PROFILE_FLAGS = Object.freeze({ + stock: Object.freeze([]), + 'optimize-for-size': Object.freeze(['--optimize-for-size']), + 'baseline-optimize-for-size': Object.freeze([ + '--max-opt=1', + '--optimize-for-size', + ]), + 'jitless-optimize-for-size': Object.freeze([ + '--jitless', + '--optimize-for-size', + ]), +}); + +const v8FlagsForArm = (arm) => { + const profile = arm.v8Profile ?? 'stock'; + const flags = V8_PROFILE_FLAGS[profile]; + if (!flags) throw new Error(`PDCF_V8_PROFILE_INVALID:${profile}`); + return [...flags]; +}; + +const makeArmPreflightArgs = ({ + arm, + port, + manifestFile, + secretsFile, + customerCount, + mode, + provisionClone, +}) => [ + '--manifest', manifestFile, + '--secrets', secretsFile, + '--customers', String(customerCount), + '--host', '127.0.0.1', + '--port', String(port), + '--arm', arm.name, + '--mode', mode, + '--introspection-client-release-mode', 'destroy', + '--runtime-pool-max', String(runtimePoolMaxForArm(arm)), + '--runtime-pool-max-uses', runtimePoolMaxUsesForArm(arm) == null + ? 'unlimited' + : String(runtimePoolMaxUsesForArm(arm)), + '--realtime-notification-mode', notificationModeForArm(arm), + '--realtime-cursor-poll-ms', String(cursorPollMsForArm(arm)), + '--realtime-cursor-heartbeat-ms', String(cursorHeartbeatMsForArm(arm)), + '--enable-realtime', 'true', + '--run-purpose', provisionClone.purpose, + '--clone-id', provisionClone.id, +]; + +const waitForArmPreflightReady = ({ + child, + arm, + port, + customerCount, + timeoutMs = 600_000, +}) => new Promise((resolve, reject) => { + let buffer = ''; + let settled = false; + const finish = (error, value) => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.off('error', onError); + child.off('exit', onExit); + child.stdout?.off('data', onData); + child.stdout?.resume(); + if (error) reject(error); + else resolve(value); + }; + const onError = () => finish(new Error( + `PDCF_PREFLIGHT_CHILD_SPAWN_FAILED:${arm.name}` + )); + const onExit = (code, signal) => finish(new Error( + `PDCF_PREFLIGHT_CHILD_EXITED_BEFORE_READY:${arm.name}:${code ?? 'signal'}:${signal ?? 'none'}` + )); + const onData = (chunk) => { + buffer += chunk.toString('utf8'); + if (buffer.length > 64 * 1024) { + finish(new Error(`PDCF_PREFLIGHT_CHILD_READY_OUTPUT_EXCEEDED:${arm.name}`)); + return; + } + let newline = buffer.indexOf('\n'); + while (newline >= 0) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (line) { + let message; + try { + message = JSON.parse(line); + } catch { + message = null; + } + if (message?.status === 'ready') { + if ( + message.fixture !== 'physical-database-density-v1' + || message.host !== '127.0.0.1' + || message.port !== port + || message.arm !== arm.name + || message.customers !== customerCount + ) { + finish(new Error(`PDCF_PREFLIGHT_CHILD_READY_INVALID:${arm.name}`)); + } else { + finish(null, message); + } + return; + } + } + newline = buffer.indexOf('\n'); + } + }; + const timer = setTimeout(() => finish(new Error( + `PDCF_PREFLIGHT_CHILD_READY_TIMEOUT:${arm.name}:${timeoutMs}` + )), timeoutMs); + timer.unref?.(); + child.once('error', onError); + child.once('exit', onExit); + child.stdout?.on('data', onData); +}); + +const waitForChildExit = (child, timeoutMs) => { + if (child.exitCode != null || child.signalCode != null) { + return Promise.resolve(true); + } + return new Promise((resolve) => { + let settled = false; + const finish = (exited) => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.off('exit', onExit); + resolve(exited); + }; + const onExit = () => finish(true); + const timer = setTimeout(() => finish(false), timeoutMs); + timer.unref?.(); + child.once('exit', onExit); + }); +}; + +const terminateArmPreflightChild = async ({ + child, + arm, + timeoutMs = 15_000, +}) => { + if (child.exitCode != null || child.signalCode != null) return; + child.kill('SIGTERM'); + if (await waitForChildExit(child, timeoutMs)) return; + child.kill('SIGKILL'); + if (!await waitForChildExit(child, timeoutMs)) { + throw new Error(`PDCF_PREFLIGHT_CHILD_TERMINATION_TIMEOUT:${arm.name}`); + } +}; + +const startArmPreflightChild = async ({ + arm, + port, + manifestFile, + secretsFile, + customerCount, + mode, + provisionClone, + environment = process.env, + entryFile = path.join(FIXTURE_DIR, 'server.cjs'), + spawnImpl = spawn, + readinessTimeoutMs, +}) => { + const armEnvironment = makeArmPreflightEnvironment(arm, environment); + const child = spawnImpl(process.execPath, [ + ...v8FlagsForArm(arm), + '--expose-gc', + entryFile, + ...makeArmPreflightArgs({ + arm, + port, + manifestFile, + secretsFile, + customerCount, + mode, + provisionClone, + }), + ], { + cwd: REPO_ROOT, + env: armEnvironment, + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stderr?.resume(); + try { + await waitForArmPreflightReady({ + child, + arm, + port, + customerCount, + timeoutMs: readinessTimeoutMs, + }); + return child; + } catch (error) { + await terminateArmPreflightChild({ child, arm }); + throw error; + } +}; + +const parseIntegerList = (value, label) => { + const values = value.split(',').map((item) => parsePositiveInteger(item.trim(), label)); + if (values.length === 0 || new Set(values).size !== values.length) { + throw new Error(`PDCF_INVALID_LIST:${label}`); + } + return values.sort((left, right) => left - right); +}; + +const parseTenantCountsByHeapMiB = (value, heapMiB) => { + if (typeof value !== 'string' || !value.trim()) { + throw new Error('PDCF_TENANT_COUNTS_BY_HEAP_REQUIRED'); + } + const result = {}; + for (const entry of value.split(';')) { + const separator = entry.indexOf(':'); + if (separator <= 0 || separator === entry.length - 1) { + throw new Error('PDCF_TENANT_COUNTS_BY_HEAP_INVALID'); + } + const heap = parsePositiveInteger(entry.slice(0, separator).trim(), 'tenant-count-heap'); + const key = String(heap); + if (result[key]) throw new Error(`PDCF_TENANT_COUNTS_BY_HEAP_DUPLICATE:${heap}`); + result[key] = parseIntegerList( + entry.slice(separator + 1), + `tenant-counts-${heap}`, + ); + } + const configured = new Set(heapMiB.map(String)); + if ( + Object.keys(result).length !== configured.size + || Object.keys(result).some((heap) => !configured.has(heap)) + ) { + throw new Error('PDCF_TENANT_COUNTS_BY_HEAP_COVERAGE_INVALID'); + } + return result; +}; + +const validateChildStatus = (status, { arm, customer, mode }) => { + const expectedMaxUses = runtimePoolMaxUsesForArm(arm); + const expectedPreparedStatementCacheSize = preparedStatementCacheSizeForArm(arm); + const expectedPreparedNamedQueries = expectedPreparedStatementCacheSize === 0 + ? 0 + : expectedPreparedStatementCacheSize + 1; + const expectedPreparedFirstEviction = expectedPreparedStatementCacheSize === 0 + ? null + : expectedPreparedStatementCacheSize; + if ( + status?.version !== 1 + || status.fixture !== 'complete-tenant-abc-v1' + || status.arm !== arm.name + || status.introspectionMode !== mode + || status.introspectionClientReleaseMode !== 'destroy' + || status.releaseBuildStateAfterValidation !== true + || status.physicalDatabase !== customer.database + || status.runtimePoolMax !== runtimePoolMaxForArm(arm) + || status.runtimePoolMaxUses !== expectedMaxUses + || status.runtimePools?.scope !== 'runtime-only-exact-identities' + || status.runtimePools?.available !== true + || status.runtimePools?.requestedMaxUses !== expectedMaxUses + || status.runtimePools?.effectiveMaxUsesKnown !== true + || status.runtimePools?.effectiveMaxUses !== expectedMaxUses + || status.runtimePools?.maxUsesExact !== true + || status.runtimePools?.identitiesUnique !== true + || status.runtimePools?.poolObjectsUnique !== true + || status.runtimePools?.expectedPools !== TENANTS.length + || status.runtimePools?.observedPools !== TENANTS.length + || status.preparedStatementCache?.requestedSize + !== expectedPreparedStatementCacheSize + || status.preparedStatementCache?.effectiveSize + !== expectedPreparedStatementCacheSize + || status.preparedStatementCache?.environmentValue + !== String(expectedPreparedStatementCacheSize) + || status.preparedStatementCache?.environmentCanonical !== true + || status.preparedStatementCache?.attestation + !== completeFixtureServer.PREPARED_STATEMENT_ATTESTATION_KIND + || status.preparedStatementCache?.effectiveSizeKnown !== true + || status.preparedStatementCache?.exact !== true + || status.preparedStatementCache?.namedQueriesObserved + !== expectedPreparedNamedQueries + || status.preparedStatementCache?.firstEvictionAfterNamedQueries + !== expectedPreparedFirstEviction + || status.enableRealtime !== true + || status.realtimeNotificationMode !== notificationModeForArm(arm) + || status.realtimeCursorPollIntervalMs !== cursorPollMsForArm(arm) + || status.realtimeCursorHeartbeatIntervalMs !== cursorHeartbeatMsForArm(arm) + || status.runtimeSafety?.passed !== true + || status.liveIdentityScope !== 'process-local-keyed-hmac-v1' + || !/^graphile-configuration:ctf:v1:[a-f0-9]{64}$/.test( + status.configurationIdentity ?? '' + ) + || status.contractEvidence?.version !== 1 + || status.contractEvidence?.credentialFree !== true + || status.contractEvidence?.configurationIdentity + !== status.configurationIdentity + || !/^sha256:[a-f0-9]{64}$/.test(status.runtimeArtifactFingerprint ?? '') + ) { + throw new Error(`PDCF_PREFLIGHT_STATUS_INVALID:${arm.name}:${customer.id}`); + } + if ( + notificationModeForArm(arm) === 'shared-exact' + && !String(status.realtimeListenerIdentity ?? '') + .startsWith('pg-notification-broker:v1:pg:v1:') + ) { + throw new Error( + `PDCF_PREFLIGHT_LISTENER_IDENTITY_INVALID:${arm.name}:${customer.id}` + ); + } + for (const tenantId of ['a', 'b', 'c']) { + if (!String(status.runtimePoolIdentities?.[tenantId] ?? '').startsWith('pg:v1:')) { + throw new Error(`PDCF_PREFLIGHT_POOL_IDENTITY_INVALID:${arm.name}:${customer.id}:${tenantId}`); + } + if (!String(status.buildContracts?.[tenantId] ?? '').startsWith('graphile:v1:')) { + throw new Error(`PDCF_PREFLIGHT_BUILD_CONTRACT_INVALID:${arm.name}:${customer.id}:${tenantId}`); + } + const poolEvidence = status.contractEvidence?.runtimePools?.[tenantId]; + const buildEvidence = status.contractEvidence?.graphileBuilds?.[tenantId]; + const binding = status.runtimeBindings?.[tenantId]; + if ( + !/^pg-contract-evidence:v1:[a-f0-9]{64}$/.test( + poolEvidence?.fingerprint ?? '' + ) + || !/^graphile-contract-evidence:v1:[a-f0-9]{64}$/.test( + buildEvidence?.fingerprint ?? '' + ) + || poolEvidence?.input?.databaseName !== customer.database + || poolEvidence?.input?.role !== customer.roles?.[tenantId] + || binding?.databaseName !== customer.database + || binding?.role !== customer.roles?.[tenantId] + || JSON.stringify(binding?.schemas) !== JSON.stringify([`ctf_${tenantId}`]) + ) { + throw new Error( + `PDCF_PREFLIGHT_CONTRACT_EVIDENCE_INVALID:${arm.name}:${customer.id}:${tenantId}` + ); + } + if (status.realtimeSchemas?.[tenantId] !== `ctf_${tenantId}_realtime`) { + throw new Error(`PDCF_PREFLIGHT_REALTIME_SCHEMA_INVALID:${arm.name}:${customer.id}:${tenantId}`); + } + const expectedDependencies = [ + ...completeFixtureServer.RUNTIME_DEPENDENCY_SCHEMAS, + `ctf_${tenantId}_realtime`, + ]; + if ( + JSON.stringify(status.runtimeSafety?.dependencySchemasByTenant?.[tenantId]) + !== JSON.stringify(expectedDependencies) + ) { + throw new Error( + `PDCF_PREFLIGHT_RUNTIME_DEPENDENCIES_INVALID:${arm.name}:${customer.id}:${tenantId}` + ); + } + } + if (new Set(Object.values(status.runtimePoolIdentities)).size !== TENANTS.length) { + throw new Error(`PDCF_PREFLIGHT_POOL_IDENTITIES_NOT_UNIQUE:${arm.name}:${customer.id}`); + } + return status; +}; + +const assertUniqueRuntimePoolIdentities = ({ statuses, customers, arm }) => { + const identities = customers.flatMap((customer) => + Object.values(statuses?.[customer.id]?.runtimePoolIdentities ?? {}) + ); + if ( + identities.length !== customers.length * TENANTS.length + || new Set(identities).size !== identities.length + ) { + throw new Error(`PDCF_PREFLIGHT_POOL_IDENTITIES_NOT_UNIQUE:${arm.name}`); + } +}; + +const assertRepresentativeSharedRealtime = ({ + before, + after, + driverSnapshot, + arm, + customer, +}) => { + const expectedContracts = Object.values(before.buildContracts).sort(); + const residentContracts = [...(after.residentBuildContracts ?? [])].sort(); + const buildCounts = after.builds?.byTenant ?? {}; + if ( + driverSnapshot?.expected !== TENANTS.length + || driverSnapshot.active !== TENANTS.length + || driverSnapshot.verified !== TENANTS.length + || driverSnapshot.errors?.length !== 0 + || JSON.stringify(residentContracts) !== JSON.stringify(expectedContracts) + || TENANTS.some((tenant) => buildCounts[tenant.id] !== 1) + ) { + throw new Error( + `PDCF_SHARED_REALTIME_PREFLIGHT_INCOMPLETE:${arm.name}:${customer.id}` + ); + } + return { + customerId: customer.id, + surfacesBuilt: expectedContracts.length, + subscriptionsActive: driverSnapshot.active, + subscriptionsVerified: driverSnapshot.verified, + residentBuildContracts: residentContracts, + }; +}; + +const verifyRepresentativeSharedRealtime = async ({ + arm, + port, + customer, + status, + fetchImpl = fetch, + createRealtimeDriver = loadRealtimeDriver(), +}) => { + const oneCustomerFleet = makeFleet({ + manifest: { customers: [customer] }, + statuses: { [arm.name]: { [customer.id]: status } }, + arms: [arm], + port, + }); + const tenants = oneCustomerFleet.tenants.map((tenant) => ({ + ...tenant, + surfaces: tenant.surfaces.map((surface) => ({ + ...surface, + url: surface.url.replace('{port}', String(port)), + })), + })); + const driver = createRealtimeDriver(tenants, { + concurrency: TENANTS.length, + timeoutMs: 120_000, + }); + try { + await driver.startAndVerify(); + driver.assertHealthy(); + const response = await fetchImpl( + `http://127.0.0.1:${port}/customer/${customer.id}/__ctf/status` + ); + if (!response.ok) { + throw new Error( + `PDCF_SHARED_REALTIME_PREFLIGHT_STATUS_HTTP:${arm.name}:${response.status}` + ); + } + const after = await response.json(); + return assertRepresentativeSharedRealtime({ + before: status, + after, + driverSnapshot: driver.snapshot(), + arm, + customer, + }); + } finally { + await driver.dispose(); + } +}; + +const makeBlueprintCompatibility = ({ + manifest, + statuses, + mode, + arms = DEFAULT_IDLE_ARMS, +}) => { + const expectedCanonicalSchemas = [ + 'ctf_extensions', + ...TENANTS.flatMap((tenant) => [ + tenant.schema, + completeFixtureServer.realtimeSchemaFor(tenant), + ]), + 'jwt_private', + ]; + if (JSON.stringify(manifest.canonicalSchemas) !== JSON.stringify(expectedCanonicalSchemas)) { + throw new Error('PDCF_CANONICAL_SCHEMA_CLOSURE_MISMATCH'); + } + const canonicalDatabaseContractFingerprint = manifest.canonicalDatabaseContractFingerprint; + if (!/^sha256:[a-f0-9]{64}$/.test(canonicalDatabaseContractFingerprint ?? '')) { + throw new Error('PDCF_CANONICAL_DATABASE_CONTRACT_REQUIRED'); + } + if (manifest.customers.some( + (customer) => customer.databaseContractFingerprint + !== canonicalDatabaseContractFingerprint + )) { + throw new Error('PDCF_DATABASE_CONTRACT_MANIFEST_MISMATCH'); + } + for (const arm of arms.filter( + (candidate) => notificationModeForArm(candidate) === 'shared-exact' + )) { + const representative = manifest.customers[0]; + const evidence = statuses?.[arm.name]?.[representative.id] + ?.sharedRealtimePreflight; + if ( + evidence?.customerId !== representative.id + || evidence.surfacesBuilt !== TENANTS.length + || evidence.subscriptionsActive !== TENANTS.length + || evidence.subscriptionsVerified !== TENANTS.length + || !Array.isArray(evidence.residentBuildContracts) + || evidence.residentBuildContracts.length !== TENANTS.length + ) { + throw new Error(`PDCF_SHARED_REALTIME_PREFLIGHT_REQUIRED:${arm.name}`); + } + } + const runtimeArtifactFingerprints = new Set(Object.values(statuses).flatMap( + (armStatuses) => Object.values(armStatuses).map( + (status) => status.runtimeArtifactFingerprint + ) + )); + if (runtimeArtifactFingerprints.size !== 1) { + throw new Error('PDCF_RUNTIME_ARTIFACT_FINGERPRINT_MISMATCH'); + } + const runtimeArtifactFingerprint = [...runtimeArtifactFingerprints][0]; + const fixtureServerSha256 = `sha256:${fileSha256(path.join( + FIXTURE_DIR, + '../complete-tenant-fixture/server.cjs', + ))}`; + const pluginConfiguration = { + settingsSource: 'fixture-static-no-control-plane-overrides', + featureSettings: completeFixtureServer.FEATURE_SETTINGS, + grafastCacheLimits: completeFixtureServer.GRAFAST_CACHE_LIMITS, + introspectionDependencySchemas: + completeFixtureServer.INTROSPECTION_DEPENDENCY_SCHEMAS, + runtimeDependencySchemas: completeFixtureServer.RUNTIME_DEPENDENCY_SCHEMAS, + introspectionMode: mode, + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: + completeFixtureServer.RELEASE_BUILD_STATE_AFTER_VALIDATION, + runtimeProfiles: arms.map((arm) => ({ + name: arm.name, + runtimePoolMax: runtimePoolMaxForArm(arm), + runtimePoolMaxUses: runtimePoolMaxUsesForArm(arm), + realtimeNotificationMode: notificationModeForArm(arm), + realtimeCursorPollIntervalMs: cursorPollMsForArm(arm), + realtimeCursorHeartbeatIntervalMs: cursorHeartbeatMsForArm(arm), + preparedStatementCacheSize: preparedStatementCacheSizeForArm(arm), + v8Profile: arm.v8Profile ?? 'stock', + })), + enableRealtime: true, + tenantBindings: TENANTS.map((tenant) => ({ + id: tenant.id, + schema: tenant.schema, + realtimeSchema: completeFixtureServer.realtimeSchemaFor(tenant), + databaseId: tenant.databaseId, + apiId: tenant.apiId, + })), + fixtureServerSha256, + runtimeArtifactFingerprint, + }; + const pluginSettingsIdentity = sha256Canonical(pluginConfiguration); + const compatibility = { + version: 1, + scope: 'blueprint-prerequisites-only', + dedicatedInstancesRemainBaseline: true, + sqlRewriteEnabled: false, + releaseBuildStateAfterValidation: + completeFixtureServer.RELEASE_BUILD_STATE_AFTER_VALIDATION, + canonicalDatabaseContractFingerprint, + canonicalSchemas: manifest.canonicalSchemas, + pluginSettingsIdentity, + runtimeArtifactFingerprint, + }; + return { + ...compatibility, + sha256: sha256Canonical(compatibility), + }; +}; + +const collectArmStatuses = async ({ + arm, + port, + manifestFile, + secretsFile, + customerCount, + mode, + provisionClone, + environment = process.env, + verifySharedRealtime = verifyRepresentativeSharedRealtime, + fetchImpl = fetch, + entryFile, + spawnImpl, + readinessTimeoutMs, + terminationTimeoutMs, +}) => { + const { manifest } = loadProvision(manifestFile, secretsFile); + if ( + manifest.provisionClone?.id !== provisionClone.id + || manifest.provisionClone?.purpose !== provisionClone.purpose + ) { + throw new Error(`PDCF_PREFLIGHT_CLONE_MISMATCH:${arm.name}`); + } + const customers = manifest.customers.slice(0, customerCount); + if (customers.length !== customerCount) { + throw new Error( + `PDCF_PREFLIGHT_CUSTOMER_COUNT_INVALID:${arm.name}:${customerCount}:${customers.length}` + ); + } + let child = null; + try { + child = await startArmPreflightChild({ + arm, + port, + manifestFile, + secretsFile, + customerCount, + mode, + provisionClone, + environment, + entryFile, + spawnImpl, + readinessTimeoutMs, + }); + const statuses = Object.fromEntries(await Promise.all(customers.map(async (customer) => { + const response = await fetchImpl( + `http://127.0.0.1:${port}/customer/${customer.id}/__ctf/status` + ); + if (!response.ok) { + throw new Error(`PDCF_PREFLIGHT_STATUS_HTTP:${arm.name}:${customer.id}:${response.status}`); + } + const status = validateChildStatus(await response.json(), { arm, customer, mode }); + return [customer.id, status]; + }))); + assertUniqueRuntimePoolIdentities({ statuses, customers, arm }); + if (notificationModeForArm(arm) === 'shared-exact') { + const customer = customers[0]; + const evidence = await verifySharedRealtime({ + arm, + port, + customer, + status: statuses[customer.id], + }); + statuses[customer.id] = { + ...statuses[customer.id], + sharedRealtimePreflight: evidence, + }; + } + return statuses; + } finally { + if (child) await terminateArmPreflightChild({ + child, + arm, + timeoutMs: terminationTimeoutMs, + }); + } +}; + +const generateInputs = async ({ + manifestFile, + secretsFile, + outDir, + postgresContainer, + basePort, + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + repetitions, + durationSec, + mode, + cacheCalibrationFile, + arms = DEFAULT_IDLE_ARMS, + environment = process.env, + resolveHeapLimitBytes = expectedHeapLimitBytes, + collectStatuses = collectArmStatuses, + inspectContainer = inspectDockerContainer, +}) => { + const { manifest } = loadProvision(manifestFile, secretsFile); + const countMatrix = validateCustomerCountMatrix({ + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + }); + const cacheCalibration = validateCacheCalibration( + JSON.parse(fs.readFileSync(path.resolve(cacheCalibrationFile), 'utf8')), + { + databaseContractFingerprint: manifest.canonicalDatabaseContractFingerprint, + introspectionMode: mode, + }, + ); + if (Math.max(...countMatrix.all) !== manifest.customers.length) { + throw new Error( + `PDCF_MAX_COUNT_MANIFEST_REQUIRED:${countMatrix.all.join(',')}:${manifest.customers.length}` + ); + } + const maximumCustomers = Math.max(...countMatrix.all); + if (maximumCustomers > manifest.customers.length) { + throw new Error( + `PDCF_COUNT_RAMP_EXCEEDS_PROVISIONED:${maximumCustomers}:${manifest.customers.length}` + ); + } + // Capacity is a pure, calibrated prerequisite. Resolve it before starting + // any arm server so an impossible qualifying point cannot spend minutes on + // database/runtime status collection before failing. + const heapLimitBytesByHeapMiB = Object.fromEntries(heapMiB.map((value) => [ + String(value), + resolveHeapLimitBytes(value), + ])); + const cacheCapacityByHeapMiB = makeCacheCapacityProofByHeapMiB({ + cacheCalibration, + databaseContractFingerprint: manifest.canonicalDatabaseContractFingerprint, + introspectionMode: mode, + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + heapLimitBytesByHeapMiB, + }); + const statuses = {}; + for (let index = 0; index < arms.length; index += 1) { + const arm = arms[index]; + statuses[arm.name] = await collectStatuses({ + arm, + port: basePort + index, + manifestFile, + secretsFile, + customerCount: maximumCustomers, + mode, + provisionClone: manifest.provisionClone, + environment, + }); + } + const blueprintCompatibility = makeBlueprintCompatibility({ + manifest, + statuses, + mode, + arms, + }); + + const pgHost = environment.PGHOST ?? 'localhost'; + const pgPort = parsePositiveInteger( + String(environment.PGPORT ?? '5432'), + 'PGPORT', + ); + const containerTemplate = captureContainerTemplate({ + inspection: inspectContainer(postgresContainer), + container: postgresContainer, + prefix: manifest.prefix, + pgHost, + pgPort, + minimumMaxConnections: Math.max( + 100, + maximumCustomers * TENANTS.length * 3 + 16, + ), + }); + const containerTemplateFile = path.join(outDir, 'postgres-container-template.json'); + atomicWriteJson(containerTemplateFile, containerTemplate); + const containerTemplateSha256 = `sha256:${fileSha256(containerTemplateFile)}`; + + const entryFile = path.join(FIXTURE_DIR, 'server.cjs'); + const lockfile = path.join(REPO_ROOT, 'pnpm-lock.yaml'); + const commit = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: REPO_ROOT, + encoding: 'utf8', + }).trim(); + const fleet = makeFleet({ manifest, statuses, arms, port: basePort }); + const plan = makePlan({ + manifestFile: path.resolve(manifestFile), + secretsFile: path.resolve(secretsFile), + postgresContainer, + commit, + entrySha256: fileSha256(entryFile), + lockfileSha256: fileSha256(lockfile), + arms, + basePort, + heapMiB, + tenantCounts, + tenantCountsByHeapMiB, + repetitions, + durationSec, + introspectionMode: mode, + databaseContractFingerprint: + blueprintCompatibility.canonicalDatabaseContractFingerprint, + blueprintCompatibilityFingerprint: blueprintCompatibility.sha256, + manifestSha256: `sha256:${fileSha256(manifestFile)}`, + provisionClone: manifest.provisionClone, + cacheCalibration, + heapLimitBytesByHeapMiB, + cacheCapacityByHeapMiB, + postgresContainerTemplateFile: path.resolve(containerTemplateFile), + postgresContainerTemplateSha256: containerTemplateSha256, + }); + const fleetFile = path.join(outDir, 'fleet.json'); + const planFile = path.join(outDir, 'plan.json'); + const preflightFile = path.join(outDir, 'preflight-status.json'); + atomicWriteJson(fleetFile, fleet); + atomicWriteJson(planFile, plan); + atomicWriteJson(preflightFile, { + version: 1, + fixture: 'physical-database-density-v1', + canonicalStructuralFingerprint: + manifest.canonicalStructuralFingerprint?.combined?.sha256 ?? null, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint ?? null, + blueprintCompatibility, + cacheCalibration, + heapLimitBytesByHeapMiB, + cacheCapacityByHeapMiB, + statuses, + }); + return { + containerTemplateFile, + fleet, + fleetFile, + plan, + planFile, + preflightFile, + }; +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + const armProfile = requireString(args, 'arm-profile', 'idle'); + const arms = armProfile === 'idle' + ? DEFAULT_IDLE_ARMS + : armProfile === 'density-tuning' + ? DENSITY_TUNING_ARMS + : null; + if (!arms) throw new Error(`PDCF_ARM_PROFILE_INVALID:${armProfile}`); + const manifestFile = path.resolve(requireString(args, 'manifest')); + const secretsFile = path.resolve(requireString(args, 'secrets')); + const outDir = path.resolve(requireString( + args, + 'out-dir', + path.join(FIXTURE_DIR, '.local', 'inputs'), + )); + const heapMiB = parseIntegerList( + requireString(args, 'heaps', '1024,2048,4096'), + 'heaps', + ); + const tenantCountsByHeapMiB = args['tenant-counts-by-heap-mib'] == null + ? undefined + : parseTenantCountsByHeapMiB( + requireString(args, 'tenant-counts-by-heap-mib'), + heapMiB, + ); + const tenantCounts = args['tenant-counts'] == null + ? undefined + : parseIntegerList(requireString(args, 'tenant-counts'), 'tenant-counts'); + if ((tenantCounts == null) === (tenantCountsByHeapMiB == null)) { + throw new Error('PDCF_EXACTLY_ONE_TENANT_COUNT_MODE_REQUIRED'); + } + const result = await generateInputs({ + manifestFile, + secretsFile, + outDir, + postgresContainer: requireString(args, 'postgres-container'), + basePort: parsePositiveInteger(args['base-port'] ?? '3410', 'base-port'), + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + repetitions: parsePositiveInteger(args.repetitions ?? '3', 'repetitions'), + durationSec: parsePositiveInteger(args['duration-sec'] ?? '900', 'duration-sec'), + mode: requireString(args, 'mode', 'scoped-required'), + cacheCalibrationFile: path.resolve(requireString(args, 'cache-calibration')), + arms, + }); + process.stdout.write(`${JSON.stringify({ + status: 'generated', + fleetFile: result.fleetFile, + planFile: result.planFile, + preflightFile: result.preflightFile, + containerTemplateFile: result.containerTemplateFile, + })}\n`); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + collectArmStatuses, + assertUniqueRuntimePoolIdentities, + assertRepresentativeSharedRealtime, + generateInputs, + parseIntegerList, + parseTenantCountsByHeapMiB, + makeBlueprintCompatibility, + expectedHeapLimitBytes, + sha256Canonical, + makeArmPreflightArgs, + makeArmPreflightEnvironment, + startArmPreflightChild, + terminateArmPreflightChild, + validateChildStatus, + v8FlagsForArm, + verifyRepresentativeSharedRealtime, +}; diff --git a/research/graphile-density/physical-database-density/inputs.test.cjs b/research/graphile-density/physical-database-density/inputs.test.cjs new file mode 100644 index 0000000000..58a018443a --- /dev/null +++ b/research/graphile-density/physical-database-density/inputs.test.cjs @@ -0,0 +1,870 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { + DEFAULT_IDLE_ARMS, + loadProvision, + makeCustomers, + makeSecretResolver, +} = require('./lib.cjs'); +const { + assertRepresentativeSharedRealtime, + assertUniqueRuntimePoolIdentities, + collectArmStatuses, + generateInputs, + makeArmPreflightEnvironment, + makeBlueprintCompatibility, + parseIntegerList, + parseTenantCountsByHeapMiB, + startArmPreflightChild, + terminateArmPreflightChild, + validateChildStatus, + v8FlagsForArm, +} = require('./generate-inputs.cjs'); +const { + CALIBRATION_KIND, + sha256Canonical, +} = require('./cache-calibration.cjs'); +const { + aggregateRuntimePoolStats, + classifyDatabaseScope, + matchPhysicalUpgradeCustomer, + parseServerOptions, + runtimeEnvironmentFor, + tokenEqual, +} = require('./server.cjs'); + +const digest = (character) => `sha256:${character.repeat(64)}`; +const MIB = 1024 ** 2; + +const customer = makeCustomers('pdc_test', 1)[0]; +const childStatus = (armName, runtimeArtifactFingerprint = digest('a')) => ({ + version: 1, + fixture: 'complete-tenant-abc-v1', + arm: armName, + introspectionMode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: true, + physicalDatabase: customer.database, + runtimePoolMax: 2, + runtimePoolMaxUses: null, + runtimePools: { + scope: 'runtime-only-exact-identities', + available: true, + requestedMaxUses: null, + effectiveMaxUses: null, + effectiveMaxUsesKnown: true, + maxUsesExact: true, + identitiesUnique: true, + poolObjectsUnique: true, + expectedPools: 3, + observedPools: 3, + totalClients: 0, + idleClients: 0, + waitingClients: 0, + }, + preparedStatementCache: { + environmentValue: '100', + requestedSize: 100, + environmentCanonical: true, + attestation: 'loaded-dataplan-adaptor-behavior-v1', + effectiveSize: 100, + effectiveSizeKnown: true, + exact: true, + namedQueriesObserved: 101, + firstEvictionAfterNamedQueries: 100, + }, + enableRealtime: true, + realtimeNotificationMode: 'dedicated', + realtimeCursorPollIntervalMs: 5000, + realtimeCursorHeartbeatIntervalMs: 30000, + runtimeArtifactFingerprint, + configurationIdentity: `graphile-configuration:ctf:v1:${'e'.repeat(64)}`, + liveIdentityScope: 'process-local-keyed-hmac-v1', + runtimePoolIdentities: { + a: 'pg:v1:a', + b: 'pg:v1:b', + c: 'pg:v1:c', + }, + buildContracts: { + a: 'graphile:v1:a', + b: 'graphile:v1:b', + c: 'graphile:v1:c', + }, + runtimeBindings: Object.fromEntries(['a', 'b', 'c'].map((tenantId) => [ + tenantId, + { + databaseName: customer.database, + role: customer.roles[tenantId], + schemas: [`ctf_${tenantId}`], + }, + ])), + contractEvidence: { + version: 1, + credentialFree: true, + configurationIdentity: `graphile-configuration:ctf:v1:${'e'.repeat(64)}`, + runtimePools: Object.fromEntries(['a', 'b', 'c'].map((tenantId) => [ + tenantId, + { + version: 1, + fingerprint: `pg-contract-evidence:v1:${tenantId.repeat(64)}`, + input: { + databaseName: customer.database, + role: customer.roles[tenantId], + }, + }, + ])), + graphileBuilds: Object.fromEntries(['a', 'b', 'c'].map((tenantId) => [ + tenantId, + { + version: 1, + fingerprint: `graphile-contract-evidence:v1:${tenantId.repeat(64)}`, + input: {}, + }, + ])), + }, + realtimeSchemas: { + a: 'ctf_a_realtime', + b: 'ctf_b_realtime', + c: 'ctf_c_realtime', + }, + runtimeSafety: { + passed: true, + dependencySchemasByTenant: { + a: ['ctf_extensions', 'jwt_private', 'ctf_a_realtime'], + b: ['ctf_extensions', 'jwt_private', 'ctf_b_realtime'], + c: ['ctf_extensions', 'jwt_private', 'ctf_c_realtime'], + }, + }, +}); + +const manifest = { + version: 1, + fixture: 'physical-database-density-v1', + prefix: 'pdc_test', + canonicalSchemas: [ + 'ctf_extensions', + 'ctf_a', + 'ctf_a_realtime', + 'ctf_b', + 'ctf_b_realtime', + 'ctf_c', + 'ctf_c_realtime', + 'jwt_private', + ], + canonicalDatabaseContractFingerprint: digest('b'), + customers: [{ ...customer, databaseContractFingerprint: digest('b') }], +}; + +const makeInsufficientCalibration = () => { + const measured = { + repetitions: 3, + retainedHeapPerSurfaceBytes: 100 * MIB, + serverBaselineHeapBytes: 400 * MIB, + buildTransientHeapBytes: 400 * MIB, + buildTransientRssBytes: 100 * MIB, + }; + const safetyFactor = 1.25; + const payload = { + kind: CALIBRATION_KIND, + databaseContractFingerprint: manifest.canonicalDatabaseContractFingerprint, + introspectionMode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: true, + introspectionBackendRetirementConclusive: true, + fixtureFingerprint: 'fixture-v1', + schemaContract: { + schemaSets: [['ctf_a']], + allowedDependencySchemas: ['ctf_extensions'], + }, + safetyFactor, + measured, + configured: { + instanceHeapBytes: 125 * MIB, + serverReserveBytes: 500 * MIB, + buildReserveBytes: 500 * MIB, + rssBuildReserveBytes: 125 * MIB, + }, + sourceWorktreesClean: true, + sources: ['1', '2', '3'].map((value) => ({ + sourceSha256: digest(value), + sourceStateSha256: value.repeat(64), + executedEntrySha256: value.repeat(64), + worktreeDirty: false, + introspectionBackendRetirement: { + conclusive: true, + introspectionBackendPid: Number(value), + steadyBackendPid: Number(value) + 10, + }, + })), + }; + return { + version: 2, + ...payload, + calibrationId: sha256Canonical(payload), + }; +}; + +describe('physical database density inputs', () => { + it('maps every supported arm profile to the harness V8 flags', () => { + assert.deepEqual(v8FlagsForArm({ v8Profile: 'stock' }), []); + assert.deepEqual(v8FlagsForArm({ v8Profile: 'optimize-for-size' }), [ + '--optimize-for-size', + ]); + assert.deepEqual(v8FlagsForArm({ v8Profile: 'baseline-optimize-for-size' }), [ + '--max-opt=1', + '--optimize-for-size', + ]); + assert.deepEqual(v8FlagsForArm({ v8Profile: 'jitless-optimize-for-size' }), [ + '--jitless', + '--optimize-for-size', + ]); + assert.throws( + () => v8FlagsForArm({ v8Profile: 'ambient-flags' }), + /PDCF_V8_PROFILE_INVALID:ambient-flags/, + ); + }); + + it('preflights each arm in an isolated child and cleans up success and failure', async () => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-isolated-preflight-')); + const entryFile = path.join(temporary, 'fake-physical-server.cjs'); + const cleanupFile = path.join(temporary, 'cleanup.log'); + const preloadFile = path.join(temporary, 'ambient-preload.cjs'); + const preloadMarker = path.join(temporary, 'ambient-preload-ran'); + const parentPreparedCache = process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE; + const parentPoolMaxUses = process.env.PG_POOL_MAX_USES; + try { + fs.writeFileSync(entryFile, `'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const args = {}; +for (let index = 2; index < process.argv.length; index += 2) { + args[process.argv[index].slice(2)] = process.argv[index + 1]; +} +const outputDirectory = process.env.PDCF_TEST_OUTPUT_DIRECTORY; +fs.writeFileSync(path.join(outputDirectory, args.arm + '.json'), JSON.stringify({ + preparedStatementCacheSize: process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE ?? null, + processPoolMax: process.env.PG_POOL_MAX ?? null, + processPoolMaxUses: process.env.PG_POOL_MAX_USES ?? null, + runtimePoolMaxUses: args['runtime-pool-max-uses'], + nodeOptions: process.env.NODE_OPTIONS ?? null, + nodePath: process.env.NODE_PATH ?? null, + execArgv: process.execArgv, +})); +process.once('SIGTERM', () => { + fs.appendFileSync(path.join(outputDirectory, 'cleanup.log'), args.arm + '\\n'); + process.exit(0); +}); +process.stdout.write(JSON.stringify({ + status: 'ready', + fixture: 'physical-database-density-v1', + host: '127.0.0.1', + port: Number(args.port), + arm: args.arm, + customers: Number(args.customers), +}) + '\\n'); +setInterval(() => {}, 1000); +`); + fs.writeFileSync(preloadFile, `'use strict'; +require('node:fs').writeFileSync(process.env.PDCF_TEST_PRELOAD_MARKER, 'loaded'); +`); + process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE = 'parent-contamination'; + process.env.PG_POOL_MAX_USES = '77'; + const stockArm = { + name: 'isolated-stock', + idleTimeoutMs: 1000, + runtimePoolMax: 1, + }; + const noPrepareArm = { + ...stockArm, + name: 'isolated-no-prepare', + preparedStatementCacheSize: 0, + runtimePoolMaxUses: 1, + v8Profile: 'optimize-for-size', + }; + const common = { + manifestFile: '/not-read-by-fake/manifest.json', + secretsFile: '/not-read-by-fake/secrets.json', + customerCount: 1, + mode: 'scoped-required', + provisionClone: { id: 'measurement-clone', purpose: 'measurement' }, + environment: { + ...process.env, + PDCF_TEST_OUTPUT_DIRECTORY: temporary, + PDCF_TEST_PRELOAD_MARKER: preloadMarker, + NODE_OPTIONS: `--require=${preloadFile}`, + NODE_PATH: '/tmp/pdc-untrusted-node-path', + PG_POOL_MAX: '99', + }, + entryFile, + readinessTimeoutMs: 5000, + }; + const stockChild = await startArmPreflightChild({ + ...common, + arm: stockArm, + port: 3491, + }); + await terminateArmPreflightChild({ child: stockChild, arm: stockArm }); + const noPrepareChild = await startArmPreflightChild({ + ...common, + arm: noPrepareArm, + port: 3492, + }); + await terminateArmPreflightChild({ child: noPrepareChild, arm: noPrepareArm }); + + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(temporary, 'isolated-stock.json'), 'utf8')), + { + preparedStatementCacheSize: '100', + processPoolMax: '1', + processPoolMaxUses: '0', + runtimePoolMaxUses: 'unlimited', + nodeOptions: '', + nodePath: null, + execArgv: ['--expose-gc'], + }, + ); + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(temporary, 'isolated-no-prepare.json'), 'utf8')), + { + preparedStatementCacheSize: '0', + processPoolMax: '1', + processPoolMaxUses: '0', + runtimePoolMaxUses: '1', + nodeOptions: '', + nodePath: null, + execArgv: ['--optimize-for-size', '--expose-gc'], + }, + ); + assert.equal( + process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE, + 'parent-contamination', + ); + assert.equal(process.env.PG_POOL_MAX_USES, '77'); + assert.equal(fs.existsSync(preloadMarker), false); + assert.deepEqual( + fs.readFileSync(cleanupFile, 'utf8').trim().split('\n').sort(), + ['isolated-no-prepare', 'isolated-stock'], + ); + + const manifestFile = path.join(temporary, 'provision.json'); + const secretsFile = path.join(temporary, 'runtime-secrets.json'); + const provisionClone = { + version: 1, + id: 'measurement-clone', + purpose: 'measurement', + attestationSetSha256: digest('e'), + }; + fs.writeFileSync(manifestFile, JSON.stringify({ ...manifest, provisionClone })); + fs.writeFileSync(secretsFile, JSON.stringify({ + version: 1, + fixture: 'physical-database-density-v1', + runtimePasswords: Object.fromEntries(Object.values(customer.roles).map( + (role) => [role, `fixture-password-at-least-24-bytes-${role}`] + )), + notificationPasswords: { + [customer.notificationRole]: + `fixture-password-at-least-24-bytes-${customer.notificationRole}`, + }, + }), { mode: 0o600 }); + const failingArm = { ...stockArm, name: 'isolated-fetch-failure' }; + await assert.rejects(collectArmStatuses({ + arm: failingArm, + port: 3493, + manifestFile, + secretsFile, + customerCount: 1, + mode: 'scoped-required', + provisionClone, + environment: { + ...process.env, + PDCF_TEST_OUTPUT_DIRECTORY: temporary, + }, + entryFile, + readinessTimeoutMs: 5000, + terminationTimeoutMs: 5000, + fetchImpl: async () => { + throw new Error('injected status failure'); + }, + }), /injected status failure/); + assert.match(fs.readFileSync(cleanupFile, 'utf8'), /isolated-fetch-failure/); + assert.deepEqual( + makeArmPreflightEnvironment(noPrepareArm, { + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: 'stale', + NODE_OPTIONS: '--require=/tmp/ambient-preload.cjs', + NODE_PATH: '/tmp/ambient-node-path', + PG_POOL_MAX: '99', + PG_POOL_MAX_USES: '99', + }), + { + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: '0', + NODE_OPTIONS: '', + PG_POOL_IDLE_TIMEOUT_MS: '1000', + PG_POOL_MAX: '1', + PG_POOL_MAX_USES: '0', + }, + ); + } finally { + if (parentPreparedCache == null) { + delete process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE; + } else { + process.env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE = parentPreparedCache; + } + if (parentPoolMaxUses == null) delete process.env.PG_POOL_MAX_USES; + else process.env.PG_POOL_MAX_USES = parentPoolMaxUses; + fs.rmSync(temporary, { recursive: true, force: true }); + } + }); + + it('rejects insufficient calibrated capacity before collecting arm statuses', async () => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-capacity-preflight-')); + let statusCollectionStarted = false; + try { + const manifestFile = path.join(temporary, 'provision.json'); + const secretsFile = path.join(temporary, 'runtime-secrets.json'); + const calibrationFile = path.join(temporary, 'cache-calibration.json'); + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + fs.writeFileSync(secretsFile, JSON.stringify({ + version: 1, + fixture: 'physical-database-density-v1', + runtimePasswords: Object.fromEntries(Object.values(customer.roles).map( + (role) => [role, `fixture-password-at-least-24-bytes-${role}`] + )), + notificationPasswords: { + [customer.notificationRole]: + `fixture-password-at-least-24-bytes-${customer.notificationRole}`, + }, + }), { mode: 0o600 }); + fs.writeFileSync(calibrationFile, JSON.stringify(makeInsufficientCalibration())); + + await assert.rejects(generateInputs({ + manifestFile, + secretsFile, + outDir: path.join(temporary, 'inputs'), + postgresContainer: 'postgres-density', + basePort: 3410, + tenantCounts: [1], + heapMiB: [1024], + repetitions: 1, + durationSec: 5, + mode: 'scoped-required', + cacheCalibrationFile: calibrationFile, + resolveHeapLimitBytes: () => 1024 * MIB, + collectStatuses: async () => { + statusCollectionStarted = true; + throw new Error('status collection must not start'); + }, + }), /PDCF_CALIBRATED_CAPACITY_INSUFFICIENT:1024:1:3/); + assert.equal(statusCollectionStarted, false); + } finally { + fs.rmSync(temporary, { recursive: true, force: true }); + } + }); + + it('parses only loopback server options and binds non-secret compatibility hashes', () => { + const options = parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--host', '127.0.0.1', + '--runtime-pool-max', '2', + '--runtime-pool-max-uses', '1', + '--enable-realtime', 'true', + '--expected-database-contract', digest('b'), + '--blueprint-compatibility', digest('c'), + '--expected-manifest-sha256', digest('d'), + '--run-purpose', 'measurement', + '--clone-id', 'measurement-clone-test', + ]); + assert.equal(options.enableRealtime, true); + assert.equal(options.runtimePoolMax, 2); + assert.equal(options.runtimePoolMaxUses, 1); + assert.equal(options.introspectionClientReleaseMode, 'destroy'); + assert.equal(options.expectedDatabaseContractFingerprint, digest('b')); + assert.equal(options.blueprintCompatibilityFingerprint, digest('c')); + assert.equal(options.expectedManifestSha256, digest('d')); + assert.throws(() => parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--host', '0.0.0.0', + ]), /PDCF_SERVER_LOOPBACK_REQUIRED/); + assert.throws(() => parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--introspection-client-release-mode', 'best-effort', + ]), /PDCF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:best-effort/); + assert.throws(() => parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--runtime-pool-max-uses', '0', + '--run-purpose', 'measurement', + '--clone-id', 'measurement-clone-test', + ]), /PDCF_INVALID_MAX_USES:runtime-pool-max-uses/); + for (const value of ['01', '1e2', '0x1', ' 1', '1 ', '']) { + assert.throws(() => parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--runtime-pool-max-uses', value, + '--run-purpose', 'measurement', + '--clone-id', 'measurement-clone-test', + ]), /PDCF_INVALID_MAX_USES:runtime-pool-max-uses/); + } + }); + + it('maps one physical database to three exact least-privilege credentials', () => { + const rawSecrets = { + version: 1, + fixture: 'physical-database-density-v1', + runtimePasswords: Object.fromEntries( + Object.values(customer.roles).map((role) => [role, `long-fixture-secret-${role}`]) + ), + notificationPasswords: { + [customer.notificationRole]: `long-fixture-secret-${customer.notificationRole}`, + }, + }; + const secretResolver = makeSecretResolver(rawSecrets, manifest); + const environment = runtimeEnvironmentFor( + { PGHOST: 'fixture-host' }, + customer, + secretResolver, + true, + ); + assert.equal(environment.PGDATABASE, customer.database); + assert.equal(environment.PG_POOL_MAX_USES, '0'); + assert.equal(environment.CTF_RUNTIME_A_PGUSER, customer.roles.a); + assert.equal( + environment.CTF_RUNTIME_C_PGPASSWORD, + `long-fixture-secret-${customer.roles.c}`, + ); + assert.equal(environment.CTF_NOTIFICATION_PGUSER, customer.notificationRole); + assert.equal( + environment.CTF_NOTIFICATION_PGPASSWORD, + `long-fixture-secret-${customer.notificationRole}`, + ); + assert.equal(tokenEqual('same-token', 'same-token'), true); + assert.equal(tokenEqual('same-token', 'different-token'), false); + assert.doesNotMatch(JSON.stringify(secretResolver), /long-fixture-secret/); + }); + + it('aggregates only child runtime pools and preserves effective native maxUses evidence', () => { + const runtimeStats = (overrides = {}) => ({ + scope: 'runtime-only-exact-identities', + available: true, + requestedMaxUses: 1, + effectiveMaxUses: 1, + effectiveMaxUsesKnown: true, + maxUsesExact: true, + identitiesUnique: true, + poolObjectsUnique: true, + expectedPools: 3, + observedPools: 3, + totalClients: 1, + idleClients: 0, + waitingClients: 0, + ...overrides, + }); + const firstPools = [{}, {}, {}]; + const secondPools = [{}, {}, {}]; + const stats = aggregateRuntimePoolStats([ + { + child: { + runtimePoolStats: () => runtimeStats(), + runtimePoolObjects: () => firstPools, + }, + }, + { + child: { + runtimePoolStats: () => runtimeStats({ totalClients: 2 }), + runtimePoolObjects: () => secondPools, + }, + }, + ], 1); + assert.deepEqual(stats, { + scope: 'runtime-only-exact-identities', + available: true, + requestedMaxUses: 1, + effectiveMaxUses: 1, + effectiveMaxUsesKnown: true, + maxUsesExact: true, + identitiesUnique: true, + poolObjectsUnique: true, + expectedPools: 6, + observedPools: 6, + totalClients: 3, + idleClients: 0, + waitingClients: 0, + }); + + const mismatch = aggregateRuntimePoolStats([ + { + child: { + runtimePoolStats: () => runtimeStats({ + effectiveMaxUses: null, + maxUsesExact: false, + }), + runtimePoolObjects: () => [{}, {}, {}], + }, + }, + ], 1); + assert.equal(mismatch.available, false); + assert.equal(mismatch.maxUsesExact, false); + assert.equal(mismatch.totalClients, null); + + const sharedPool = {}; + const crossCustomerReuse = aggregateRuntimePoolStats([ + { + child: { + runtimePoolStats: () => runtimeStats(), + runtimePoolObjects: () => [sharedPool, {}, {}], + }, + }, + { + child: { + runtimePoolStats: () => runtimeStats(), + runtimePoolObjects: () => [sharedPool, {}, {}], + }, + }, + ], 1); + assert.equal(crossCustomerReuse.poolObjectsUnique, false); + assert.equal(crossCustomerReuse.available, false); + }); + + it('loads secrets only from a regular, non-symlink 0600 file', () => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-private-secrets-')); + try { + const manifestFile = path.join(temporary, 'provision.json'); + const secretsFile = path.join(temporary, 'runtime-secrets.json'); + const symlinkFile = path.join(temporary, 'runtime-secrets-link.json'); + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + const rawSecrets = { + version: 1, + fixture: 'physical-database-density-v1', + runtimePasswords: Object.fromEntries(Object.values(customer.roles).map( + (role) => [role, `fixture-password-at-least-24-bytes-${role}`] + )), + notificationPasswords: { + [customer.notificationRole]: + `fixture-password-at-least-24-bytes-${customer.notificationRole}`, + }, + }; + fs.writeFileSync(secretsFile, JSON.stringify(rawSecrets), { mode: 0o600 }); + const loaded = loadProvision(manifestFile, secretsFile); + assert.equal(loaded.manifest.fixture, 'physical-database-density-v1'); + assert.doesNotMatch( + JSON.stringify(loaded), + /fixture-password-at-least-24-bytes/, + ); + + fs.chmodSync(secretsFile, 0o640); + assert.throws( + () => loadProvision(manifestFile, secretsFile), + /PDCF_SECRETS_FILE_MODE_MUST_BE_0600/, + ); + fs.chmodSync(secretsFile, 0o600); + fs.symlinkSync(secretsFile, symlinkFile); + assert.throws( + () => loadProvision(manifestFile, symlinkFile), + /PDCF_SECRETS_FILE_MUST_BE_REGULAR/, + ); + } finally { + fs.rmSync(temporary, { recursive: true, force: true }); + } + }); + + it('routes websocket upgrades to one exact physical customer', () => { + assert.equal( + matchPhysicalUpgradeCustomer( + '/customer/physical-customer-0001/tenant/a/graphql', + ), + 'physical-customer-0001', + ); + assert.equal(matchPhysicalUpgradeCustomer( + '/customer/physical-customer-0001/tenant/a/graphql?customer=other', + ), null); + assert.equal(matchPhysicalUpgradeCustomer( + '/customer/%70hysical-customer-0001/tenant/a/graphql', + ), null); + assert.equal(matchPhysicalUpgradeCustomer( + '/customer/physical-customer-0001/tenant/a/graphql/extra', + ), null); + }); + + it('marks shared or incomplete PostgreSQL database sets non-qualifying', () => { + assert.deepEqual( + classifyDatabaseScope( + ['postgres', customer.database], + 'postgres', + [customer.database], + ), + { + dedicated: true, + databasesPresent: 2, + fixtureDatabasesExpected: 1, + fixtureDatabasesPresent: 1, + unexpectedDatabases: 0, + missingFixtureDatabases: 0, + unexpectedDatabaseSetSha256: null, + }, + ); + const shared = classifyDatabaseScope( + ['postgres', customer.database, 'unrelated_app'], + 'postgres', + [customer.database], + ); + assert.equal(shared.dedicated, false); + assert.equal(shared.unexpectedDatabases, 1); + assert.match(shared.unexpectedDatabaseSetSha256, /^sha256:[a-f0-9]{64}$/); + assert.equal(classifyDatabaseScope( + ['postgres'], + 'postgres', + [customer.database], + ).dedicated, false); + }); + + it('fails closed on status dependency drift and runtime artifact drift', () => { + const arm = DEFAULT_IDLE_ARMS[0]; + const status = childStatus(arm.name); + assert.equal(validateChildStatus(status, { + arm, + customer, + mode: 'scoped-required', + }), status); + assert.throws(() => validateChildStatus({ + ...status, + releaseBuildStateAfterValidation: false, + }, { arm, customer, mode: 'scoped-required' }), /PDCF_PREFLIGHT_STATUS_INVALID/); + assert.throws(() => validateChildStatus({ + ...status, + runtimeSafety: { + ...status.runtimeSafety, + dependencySchemasByTenant: { + ...status.runtimeSafety.dependencySchemasByTenant, + a: ['ctf_extensions', 'ctf_a_realtime'], + }, + }, + }, { arm, customer, mode: 'scoped-required' }), /RUNTIME_DEPENDENCIES_INVALID/); + assert.throws(() => validateChildStatus({ + ...status, + preparedStatementCache: { + ...status.preparedStatementCache, + attestation: 'environment-echo', + }, + }, { arm, customer, mode: 'scoped-required' }), /PDCF_PREFLIGHT_STATUS_INVALID/); + assert.throws(() => validateChildStatus({ + ...status, + runtimePoolIdentities: { + ...status.runtimePoolIdentities, + c: status.runtimePoolIdentities.a, + }, + }, { arm, customer, mode: 'scoped-required' }), /POOL_IDENTITIES_NOT_UNIQUE/); + + const secondCustomer = makeCustomers('pdc_test', 2)[1]; + assert.throws(() => assertUniqueRuntimePoolIdentities({ + arm, + customers: [customer, secondCustomer], + statuses: { + [customer.id]: status, + [secondCustomer.id]: { + ...status, + physicalDatabase: secondCustomer.database, + }, + }, + }), /POOL_IDENTITIES_NOT_UNIQUE/); + + const statuses = Object.fromEntries(DEFAULT_IDLE_ARMS.map((candidate, index) => [ + candidate.name, + { [customer.id]: childStatus(candidate.name, digest(index === 2 ? 'c' : 'a')) }, + ])); + assert.throws(() => makeBlueprintCompatibility({ + manifest, + statuses, + mode: 'scoped-required', + }), /RUNTIME_ARTIFACT_FINGERPRINT_MISMATCH/); + }); + + it('requires three built surfaces and three live verified shared subscriptions', () => { + const arm = { + name: 'physical-db-shared-stock', + realtimeNotificationMode: 'shared-exact', + }; + const before = childStatus(arm.name); + const after = { + ...before, + residentBuildContracts: Object.values(before.buildContracts), + builds: { byTenant: { a: 1, b: 1, c: 1 } }, + }; + const snapshot = { + expected: 3, + active: 3, + verified: 3, + errors: [], + }; + assert.deepEqual(assertRepresentativeSharedRealtime({ + before, + after, + driverSnapshot: snapshot, + arm, + customer, + }), { + customerId: customer.id, + surfacesBuilt: 3, + subscriptionsActive: 3, + subscriptionsVerified: 3, + residentBuildContracts: Object.values(before.buildContracts).sort(), + }); + assert.throws(() => assertRepresentativeSharedRealtime({ + before, + after, + driverSnapshot: { ...snapshot, active: 2 }, + arm, + customer, + }), /PDCF_SHARED_REALTIME_PREFLIGHT_INCOMPLETE/); + assert.throws(() => assertRepresentativeSharedRealtime({ + before, + after: { + ...after, + builds: { byTenant: { a: 1, b: 1, c: 0 } }, + }, + driverSnapshot: snapshot, + arm, + customer, + }), /PDCF_SHARED_REALTIME_PREFLIGHT_INCOMPLETE/); + }); + + it('emits a deterministic prerequisite fingerprint without enabling blueprint sharing', () => { + const statuses = Object.fromEntries(DEFAULT_IDLE_ARMS.map((arm) => [ + arm.name, + { [customer.id]: childStatus(arm.name) }, + ])); + const compatibility = makeBlueprintCompatibility({ + manifest, + statuses, + mode: 'scoped-required', + }); + assert.match(compatibility.sha256, /^sha256:[a-f0-9]{64}$/); + assert.equal(compatibility.scope, 'blueprint-prerequisites-only'); + assert.equal(compatibility.dedicatedInstancesRemainBaseline, true); + assert.equal(compatibility.sqlRewriteEnabled, false); + assert.equal(compatibility.releaseBuildStateAfterValidation, true); + assert.deepEqual(parseIntegerList('4,1,2', 'counts'), [1, 2, 4]); + assert.deepEqual( + parseTenantCountsByHeapMiB( + '1024:2,1;2048:4,2;4096:8,4', + [1024, 2048, 4096], + ), + { + '1024': [1, 2], + '2048': [2, 4], + '4096': [4, 8], + }, + ); + assert.throws(() => parseTenantCountsByHeapMiB( + '1024:1,2;2048:2,4', + [1024, 2048, 4096], + ), /PDCF_TENANT_COUNTS_BY_HEAP_COVERAGE_INVALID/); + }); +}); diff --git a/research/graphile-density/physical-database-density/lib.cjs b/research/graphile-density/physical-database-density/lib.cjs new file mode 100644 index 0000000000..0d4b4b1653 --- /dev/null +++ b/research/graphile-density/physical-database-density/lib.cjs @@ -0,0 +1,1357 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const completeFixture = require('../complete-tenant-fixture/lib.cjs'); +const { + computeCalibratedCapacity, + validateCacheCalibration, +} = require('./cache-calibration.cjs'); + +const FIXTURE_DIR = __dirname; +const REPO_ROOT = path.resolve(FIXTURE_DIR, '../../..'); +const PHYSICAL_DATABASE_CANARY = 'physical-database-routing'; +const FIXTURE_ID = 'physical-database-density-v1'; +const QUALIFYING_CACHE_ADMISSION_MODE = 'preserve-resident'; +const DEFAULT_PREPARED_STATEMENT_CACHE_SIZE = 100; +const PROCESS_GLOBAL_POOL_MAX = 1; +const DEFAULT_IDLE_ARMS = Object.freeze([ + Object.freeze({ name: 'physical-db-idle-30s', idleTimeoutMs: 30_000 }), + Object.freeze({ name: 'physical-db-idle-5s', idleTimeoutMs: 5_000 }), + Object.freeze({ name: 'physical-db-idle-1s', idleTimeoutMs: 1_000 }), +]); +const DENSITY_TUNING_ARMS = Object.freeze([ + Object.freeze({ + name: 'physical-db-dedicated-stock', + idleTimeoutMs: 1_000, + runtimePoolMax: 2, + realtimeNotificationMode: 'dedicated', + realtimeCursorPollIntervalMs: 5_000, + realtimeCursorHeartbeatIntervalMs: 30_000, + v8Profile: 'stock', + }), + Object.freeze({ + name: 'physical-db-shared-stock', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + v8Profile: 'stock', + }), + Object.freeze({ + name: 'physical-db-shared-no-prepare', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + preparedStatementCacheSize: 0, + v8Profile: 'stock', + }), + Object.freeze({ + name: 'physical-db-shared-maxuses-1', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + runtimePoolMaxUses: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + v8Profile: 'stock', + }), + Object.freeze({ + name: 'physical-db-shared-size', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + v8Profile: 'optimize-for-size', + }), + Object.freeze({ + name: 'physical-db-shared-baseline-size', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + v8Profile: 'baseline-optimize-for-size', + }), + Object.freeze({ + name: 'physical-db-shared-jitless-size', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + v8Profile: 'jitless-optimize-for-size', + }), + Object.freeze({ + name: 'physical-db-shared-maxuses-1-size', + idleTimeoutMs: 1_000, + runtimePoolMax: 1, + runtimePoolMaxUses: 1, + realtimeNotificationMode: 'shared-exact', + realtimeCursorPollIntervalMs: 30_000, + realtimeCursorHeartbeatIntervalMs: 60_000, + v8Profile: 'optimize-for-size', + }), +]); + +const runtimePoolMaxForArm = (arm) => arm.runtimePoolMax ?? 2; +const runtimePoolMaxUsesForArm = (arm) => arm.runtimePoolMaxUses ?? null; +const notificationModeForArm = (arm) => arm.realtimeNotificationMode ?? 'dedicated'; +const cursorPollMsForArm = (arm) => arm.realtimeCursorPollIntervalMs ?? 5_000; +const cursorHeartbeatMsForArm = (arm) => + arm.realtimeCursorHeartbeatIntervalMs ?? 30_000; +const preparedStatementCacheSizeForArm = (arm) => + arm.preparedStatementCacheSize ?? DEFAULT_PREPARED_STATEMENT_CACHE_SIZE; + +const validateCustomerCountRamp = (counts, label) => { + if ( + !Array.isArray(counts) + || counts.length === 0 + || counts.some((count) => !Number.isSafeInteger(count) || count <= 0) + || new Set(counts).size !== counts.length + || counts.some((count, index) => index > 0 && count <= counts[index - 1]) + ) { + throw new Error(`PDCF_PLAN_CUSTOMER_COUNT_RAMP_INVALID:${label}`); + } + return counts; +}; + +const customerCountsForHeap = ({ tenantCounts, tenantCountsByHeapMiB }, heap) => + tenantCountsByHeapMiB?.[String(heap)] ?? tenantCounts; + +const validateCustomerCountMatrix = ({ + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, +}) => { + if (!Array.isArray(heapMiB) || heapMiB.length === 0) { + throw new Error('PDCF_PLAN_HEAP_RAMP_REQUIRED'); + } + if (tenantCounts != null) { + validateCustomerCountRamp(tenantCounts, 'default'); + } + if (tenantCountsByHeapMiB != null) { + if ( + typeof tenantCountsByHeapMiB !== 'object' + || Array.isArray(tenantCountsByHeapMiB) + || tenantCountsByHeapMiB === null + ) { + throw new Error('PDCF_PLAN_CUSTOMER_COUNT_MATRIX_INVALID'); + } + const configuredHeaps = new Set(heapMiB.map(String)); + if (Object.keys(tenantCountsByHeapMiB).some((heap) => !configuredHeaps.has(heap))) { + throw new Error('PDCF_PLAN_CUSTOMER_COUNT_MATRIX_INVALID'); + } + } + const byHeap = Object.fromEntries(heapMiB.map((heap) => { + const counts = customerCountsForHeap({ tenantCounts, tenantCountsByHeapMiB }, heap); + return [String(heap), validateCustomerCountRamp(counts, String(heap))]; + })); + return { + byHeap, + all: [...new Set(Object.values(byHeap).flat())].sort((left, right) => left - right), + }; +}; +const RESIDENT_SUBSCRIPTION = ` +subscription PhysicalDensityRealtimeResident { + onRealtimeItemChanged { + event + overflow + realtimeItem { id tenantId physicalDatabaseIdentity payload } + } +} +`; +const REALTIME_PRIME_MUTATION = ` +mutation PhysicalDensityRealtimePrime($payload: String!) { + updateRealtimeItem(input: { id: 1, realtimeItemPatch: { payload: $payload } }) { + realtimeItem { id tenantId physicalDatabaseIdentity payload } + } +} +`; + +const strictIdentifier = (value, label) => { + if (typeof value !== 'string' || !/^[a-z][a-z0-9_]*$/.test(value) || value.length > 40) { + throw new Error(`PDCF_INVALID_IDENTIFIER:${label}`); + } + return value; +}; + +const customerSuffix = (ordinal) => String(ordinal).padStart(4, '0'); + +const makeCustomer = (prefix, ordinal) => { + const suffix = customerSuffix(ordinal); + const rolePrefix = `${prefix}_c${suffix}`; + return Object.freeze({ + id: `physical-customer-${suffix}`, + ordinal, + database: `${prefix}_db_${suffix}`, + // The schema-level canary returns current_database(), so the database name + // is both credential-free and conclusive under wrong-database routing. + physicalIdentity: `${prefix}_db_${suffix}`, + roles: Object.freeze(Object.fromEntries(completeFixture.TENANTS.map((tenant) => [ + tenant.id, + `${rolePrefix}_${tenant.id}`, + ]))), + // LISTEN is shared only by the three exact Graphile generations that + // target this physical database. It never executes GraphQL or reads an + // application schema. + notificationRole: `${rolePrefix}_notify`, + }); +}; + +const makeCustomers = (prefix, count) => { + strictIdentifier(prefix, 'prefix'); + if (!Number.isSafeInteger(count) || count <= 0 || count > 9999) { + throw new Error('PDCF_INVALID_CUSTOMER_COUNT'); + } + const customers = Array.from({ length: count }, (_unused, index) => + makeCustomer(prefix, index + 1) + ); + for (const customer of customers) { + strictIdentifier(customer.database, 'database'); + for (const role of Object.values(customer.roles)) strictIdentifier(role, 'role'); + strictIdentifier(customer.notificationRole, 'notification-role'); + } + return customers; +}; + +const validateProvisionManifest = (manifest) => { + if ( + !manifest + || manifest.version !== 1 + || manifest.fixture !== FIXTURE_ID + || typeof manifest.prefix !== 'string' + || !Array.isArray(manifest.customers) + || manifest.customers.length === 0 + ) { + throw new Error('PDCF_MANIFEST_INVALID'); + } + strictIdentifier(manifest.prefix, 'prefix'); + const ids = new Set(); + const databases = new Set(); + const physicalIdentities = new Set(); + const roles = new Set(); + for (const customer of manifest.customers) { + if ( + typeof customer?.id !== 'string' + || ids.has(customer.id) + || !Number.isSafeInteger(customer.ordinal) + || customer.ordinal <= 0 + || typeof customer.physicalIdentity !== 'string' + || !customer.physicalIdentity + ) { + throw new Error('PDCF_MANIFEST_CUSTOMER_INVALID'); + } + ids.add(customer.id); + strictIdentifier(customer.database, 'database'); + if (databases.has(customer.database)) throw new Error('PDCF_DATABASE_DUPLICATE'); + databases.add(customer.database); + if (customer.physicalIdentity !== customer.database) { + throw new Error(`PDCF_PHYSICAL_IDENTITY_DATABASE_MISMATCH:${customer.id}`); + } + if (physicalIdentities.has(customer.physicalIdentity)) { + throw new Error('PDCF_PHYSICAL_IDENTITY_DUPLICATE'); + } + physicalIdentities.add(customer.physicalIdentity); + for (const tenant of completeFixture.TENANTS) { + const role = strictIdentifier(customer.roles?.[tenant.id], 'role'); + if (roles.has(role)) throw new Error('PDCF_ROLE_DUPLICATE'); + roles.add(role); + } + const notificationRole = strictIdentifier( + customer.notificationRole, + 'notification-role', + ); + if (roles.has(notificationRole)) throw new Error('PDCF_ROLE_DUPLICATE'); + roles.add(notificationRole); + } + return manifest; +}; + +const validateMeasurementProvisionClone = (provisionClone) => { + if ( + !provisionClone + || provisionClone.version !== 1 + || typeof provisionClone.id !== 'string' + || !provisionClone.id.trim() + || provisionClone.purpose !== 'measurement' + || !/^sha256:[a-f0-9]{64}$/i.test(provisionClone.attestationSetSha256 ?? '') + ) { + throw new Error('PDCF_MEASUREMENT_PROVISION_CLONE_REQUIRED'); + } + return provisionClone; +}; + +const validateSecrets = (secrets, manifest) => { + if (!secrets || secrets.version !== 1 || secrets.fixture !== FIXTURE_ID) { + throw new Error('PDCF_SECRETS_INVALID'); + } + const requiredRoles = manifest.customers.flatMap((customer) => Object.values(customer.roles)); + for (const role of requiredRoles) { + const password = secrets.runtimePasswords?.[role]; + if (typeof password !== 'string' || Buffer.byteLength(password) < 24) { + throw new Error(`PDCF_RUNTIME_PASSWORD_INVALID:${role}`); + } + } + for (const customer of manifest.customers) { + const password = secrets.notificationPasswords?.[customer.notificationRole]; + if (typeof password !== 'string' || Buffer.byteLength(password) < 24) { + throw new Error(`PDCF_NOTIFICATION_PASSWORD_INVALID:${customer.notificationRole}`); + } + } + return secrets; +}; + +const readJson = (file) => JSON.parse(fs.readFileSync(path.resolve(file), 'utf8')); + +const readPrivateJson = (file) => { + const absolute = path.resolve(file); + let before; + try { + before = fs.lstatSync(absolute); + } catch { + throw new Error('PDCF_SECRETS_FILE_UNREADABLE'); + } + if (before.isSymbolicLink() || !before.isFile()) { + throw new Error('PDCF_SECRETS_FILE_MUST_BE_REGULAR'); + } + if ((before.mode & 0o777) !== 0o600) { + throw new Error('PDCF_SECRETS_FILE_MODE_MUST_BE_0600'); + } + + let descriptor; + try { + descriptor = fs.openSync( + absolute, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + const opened = fs.fstatSync(descriptor); + if ( + !opened.isFile() + || opened.dev !== before.dev + || opened.ino !== before.ino + || (opened.mode & 0o777) !== 0o600 + ) { + throw new Error('PDCF_SECRETS_FILE_CHANGED_DURING_OPEN'); + } + return JSON.parse(fs.readFileSync(descriptor, 'utf8')); + } catch (error) { + if (error instanceof Error && error.message.startsWith('PDCF_')) throw error; + throw new Error('PDCF_SECRETS_FILE_UNREADABLE'); + } finally { + if (descriptor != null) fs.closeSync(descriptor); + } +}; + +const makeSecretResolver = (rawSecrets, manifest) => { + const secrets = validateSecrets(rawSecrets, manifest); + const runtimePasswords = new Map(Object.entries(secrets.runtimePasswords)); + const notificationPasswords = new Map(Object.entries(secrets.notificationPasswords)); + return Object.freeze({ + runtimePasswordFor(role) { + const value = runtimePasswords.get(role); + if (typeof value !== 'string') throw new Error(`PDCF_RUNTIME_PASSWORD_REQUIRED:${role}`); + return value; + }, + notificationPasswordFor(role) { + const value = notificationPasswords.get(role); + if (typeof value !== 'string') { + throw new Error(`PDCF_NOTIFICATION_PASSWORD_REQUIRED:${role}`); + } + return value; + }, + toJSON() { + return { kind: 'physical-density-secret-resolver', redacted: true }; + }, + }); +}; + +const loadProvision = (manifestFile, secretsFile) => { + const absoluteManifest = path.resolve(manifestFile); + const absoluteSecrets = path.resolve(secretsFile); + const manifestStat = fs.statSync(absoluteManifest); + const secretsStat = fs.lstatSync(absoluteSecrets); + if (manifestStat.dev === secretsStat.dev && manifestStat.ino === secretsStat.ino) { + throw new Error('PDCF_MANIFEST_SECRETS_MUST_BE_DISTINCT'); + } + const manifest = validateProvisionManifest(readJson(absoluteManifest)); + const secretResolver = makeSecretResolver(readPrivateJson(absoluteSecrets), manifest); + return { manifest, secretResolver }; +}; + +const physicalCanary = (customer, otherCustomers) => ({ + name: PHYSICAL_DATABASE_CANARY, + query: 'query PhysicalDatabaseRouting { physicalDatabaseIdentity }', + requiredMatches: [{ + path: '/data/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }], + // A one-customer mechanics run has no foreign database name to enumerate, + // but a null identity is still an explicit routing failure. Keep that + // negative oracle for every fleet, then add the concrete foreign identities + // when the density point contains multiple customers. + forbiddenMatches: [ + { + path: '/data/physicalDatabaseIdentity', + value: null, + }, + ...otherCustomers.map((candidate) => ({ + path: '/data/physicalDatabaseIdentity', + value: candidate.physicalIdentity, + })), + ], + invariants: [{ + path: '/data/physicalDatabaseIdentity', + everyEquals: customer.physicalIdentity, + min: 1, + max: 1, + }], +}); + +const statusFor = (statuses, armName, customerId) => { + const status = statuses?.[armName]?.[customerId]; + if (!status || status.physicalDatabase == null) { + throw new Error(`PDCF_STATUS_REQUIRED:${armName}:${customerId}`); + } + return status; +}; + +const runtimePoolContractFingerprintFor = (status, tenantId) => { + const fingerprint = status?.contractEvidence?.runtimePools?.[tenantId]?.fingerprint; + if (!/^pg-contract-evidence:v1:[a-f0-9]{64}$/.test(fingerprint ?? '')) { + throw new Error(`PDCF_RUNTIME_POOL_CONTRACT_EVIDENCE_MISSING:${tenantId}`); + } + return fingerprint; +}; + +const graphileBuildContractFingerprintFor = (status, tenantId) => { + const fingerprint = status?.contractEvidence?.graphileBuilds?.[tenantId]?.fingerprint; + if (!/^graphile-contract-evidence:v1:[a-f0-9]{64}$/.test(fingerprint ?? '')) { + throw new Error(`PDCF_GRAPHILE_BUILD_CONTRACT_EVIDENCE_MISSING:${tenantId}`); + } + return fingerprint; +}; + +const realtimeProbe = (customer, tenant, otherCustomers) => { + const payload = `${customer.physicalIdentity}:${tenant.id}:configured-placeholder`; + const foreignEventMatches = [ + ...completeFixture.TENANTS + .filter((candidate) => candidate.id !== tenant.id) + .map((candidate) => ({ + path: '/data/onRealtimeItemChanged/realtimeItem/tenantId', + value: candidate.token, + })), + ...otherCustomers.map((candidate) => ({ + path: '/data/onRealtimeItemChanged/realtimeItem/physicalDatabaseIdentity', + value: candidate.physicalIdentity, + })), + ]; + return { + subscription: { + query: RESIDENT_SUBSCRIPTION, + requiredMatches: [ + { + path: '/data/onRealtimeItemChanged/realtimeItem/tenantId', + value: tenant.token, + }, + { + path: '/data/onRealtimeItemChanged/realtimeItem/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + ], + forbiddenMatches: foreignEventMatches, + }, + prime: { + query: REALTIME_PRIME_MUTATION, + variables: { payload }, + requiredMatches: [ + { + path: '/data/updateRealtimeItem/realtimeItem/tenantId', + value: tenant.token, + }, + { + path: '/data/updateRealtimeItem/realtimeItem/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + ], + forbiddenMatches: [ + ...completeFixture.TENANTS + .filter((candidate) => candidate.id !== tenant.id) + .map((candidate) => ({ + path: '/data/updateRealtimeItem/realtimeItem/tenantId', + value: candidate.token, + })), + ...otherCustomers.map((candidate) => ({ + path: '/data/updateRealtimeItem/realtimeItem/physicalDatabaseIdentity', + value: candidate.physicalIdentity, + })), + ], + }, + correlation: { + primeVariable: 'payload', + primeResponsePath: '/data/updateRealtimeItem/realtimeItem/payload', + subscriptionEventPath: '/data/onRealtimeItemChanged/realtimeItem/payload', + }, + }; +}; + +const physicalIdentityMatches = (pathValue, customer, otherCustomers) => ({ + requiredMatches: [{ + path: pathValue, + value: customer.physicalIdentity, + }], + forbiddenMatches: [ + { path: pathValue, value: null }, + ...otherCustomers.map((candidate) => ({ + path: pathValue, + value: candidate.physicalIdentity, + })), + ], + invariants: [{ + path: pathValue, + everyEquals: customer.physicalIdentity, + min: 1, + max: 1, + }], +}); + +const exactInvariant = (pathValue, everyEquals, min = 1, max = 1) => ({ + path: pathValue, + everyEquals, + min, + max, +}); + +const physicalOperationsFor = (customer, tenant, otherCustomers) => { + const baseByName = new Map(completeFixture.operationsFor(tenant).map( + (candidate) => [candidate.name, candidate] + )); + const fromBase = (name, overrides) => { + const base = baseByName.get(name); + if (!base) throw new Error(`PDCF_OPERATION_REQUIRED:${name}`); + baseByName.delete(name); + return { ...base, ...overrides }; + }; + const expectedDocumentTitle = `${tenant.token} Machine Learning`; + const documentOracle = ( + extraRequired = [], + extraForbidden = [], + extraInvariants = [] + ) => ({ + ...physicalIdentityMatches( + '/data/documents/nodes/0/physicalDatabaseIdentity', + customer, + otherCustomers + ), + requiredMatches: [ + { + path: '/data/documents/nodes/0/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { path: '/data/documents/nodes/0/tenantId', value: tenant.token }, + { path: '/data/documents/nodes/0/title', value: expectedDocumentTitle }, + ...extraRequired, + ], + forbiddenMatches: [ + ...physicalIdentityMatches( + '/data/documents/nodes/0/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + ...completeFixture.TENANTS.filter((candidate) => candidate.id !== tenant.id) + .map((candidate) => ({ + path: '/data/documents/nodes/0/tenantId', + value: candidate.token, + })), + ...extraForbidden, + ], + invariants: [ + exactInvariant( + '/data/documents/nodes/*/physicalDatabaseIdentity', + customer.physicalIdentity + ), + exactInvariant('/data/documents/nodes/*/tenantId', tenant.token), + exactInvariant('/data/documents/nodes/*/title', expectedDocumentTitle), + ...extraInvariants, + ], + }); + const uploadContentHash = crypto.createHash('sha256') + .update(`${customer.physicalIdentity}:${tenant.id}:upload`) + .digest('hex'); + const operations = [ + fromBase('generated-document-read', { + query: 'query GeneratedDocumentRead { documents(first: 1) { nodes { id tenantId title physicalDatabaseIdentity } } }', + ...documentOracle(), + }), + fromBase('localized-post-read', { + query: 'query LocalizedPostRead { posts(first: 1, where: { id: { equalTo: 1 } }) { nodes { tenantId physicalDatabaseIdentity localeStrings { langCode title body } } } }', + requiredMatches: [ + { + path: '/data/posts/nodes/0/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/posts/nodes/0/localeStrings/title', + value: `${tenant.token} español @${customer.physicalIdentity}`, + }, + ], + forbiddenMatches: [ + ...physicalIdentityMatches( + '/data/posts/nodes/0/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + ...completeFixture.TENANTS.filter((candidate) => candidate.id !== tenant.id) + .map((candidate) => ({ + path: '/data/posts/nodes/0/localeStrings/title', + value: `${candidate.token} español @${customer.physicalIdentity}`, + })), + ], + invariants: [ + exactInvariant( + '/data/posts/nodes/*/physicalDatabaseIdentity', + customer.physicalIdentity + ), + exactInvariant( + '/data/posts/nodes/*/localeStrings/title', + `${tenant.token} español @${customer.physicalIdentity}` + ), + ], + }), + fromBase('deterministic-embed', { + query: 'query DeterministicEmbed { physicalDatabaseIdentity embedText(text: "tenant fixture") { vector dimensions } }', + requiredMatches: [ + { + path: '/data/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { path: '/data/embedText/vector', value: [1, 0, 0] }, + { path: '/data/embedText/dimensions', value: 3 }, + ], + forbiddenMatches: physicalIdentityMatches( + '/data/physicalDatabaseIdentity', customer, otherCustomers + ).forbiddenMatches, + invariants: [ + exactInvariant('/data/physicalDatabaseIdentity', customer.physicalIdentity), + exactInvariant('/data/embedText/vector', [1, 0, 0]), + exactInvariant('/data/embedText/dimensions', 3), + ], + }), + fromBase('deterministic-rag', { + query: 'query DeterministicRag { physicalDatabaseIdentity ragQuery(prompt: "machine learning tenant fixture", contextLimit: 2) { answer tokensUsed sources { content similarity tableName parentId } } }', + requiredMatches: [ + { + path: '/data/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/ragQuery/sources/*/content', + value: `${tenant.token} machine learning tenant fixture context @${customer.physicalIdentity}`, + }, + { + path: '/data/ragQuery/answer', + value: 'Deterministic fixture answer: machine learning tenant fixture', + }, + { path: '/data/ragQuery/tokensUsed', value: 20 }, + { path: '/data/ragQuery/sources/*/tableName', value: 'articles' }, + ], + forbiddenMatches: [ + ...physicalIdentityMatches( + '/data/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + ...otherCustomers.map((candidate) => ({ + path: '/data/ragQuery/sources/*/content', + value: `${tenant.token} machine learning tenant fixture context @${candidate.physicalIdentity}`, + })), + ], + invariants: [ + exactInvariant('/data/physicalDatabaseIdentity', customer.physicalIdentity), + exactInvariant( + '/data/ragQuery/sources/*/content', + `${tenant.token} machine learning tenant fixture context @${customer.physicalIdentity}` + ), + exactInvariant('/data/ragQuery/sources/*/tableName', 'articles'), + exactInvariant( + '/data/ragQuery/answer', + 'Deterministic fixture answer: machine learning tenant fixture' + ), + exactInvariant('/data/ragQuery/tokensUsed', 20), + ], + }), + fromBase('bm25-search', { + query: 'query Bm25Search { documents(where: { bm25Body: { query: "machine learning intelligence" } }) { nodes { tenantId title physicalDatabaseIdentity bodyBm25Score } } }', + ...documentOracle(), + }), + fromBase('tsvector-search', { + query: 'query TsvectorSearch { documents(where: { tsvTsv: "machine learning" }) { nodes { tenantId title physicalDatabaseIdentity tsvRank } } }', + ...documentOracle(), + }), + fromBase('trigram-search', { + query: 'query TrigramSearch { documents(where: { trgmTitle: { value: "Machne Lerning", threshold: 0.05 } }) { nodes { tenantId title physicalDatabaseIdentity titleTrgmSimilarity } } }', + ...documentOracle(), + }), + fromBase('vector-search', { + query: 'query VectorSearch { documents(where: { vectorEmbedding: { vector: [1, 0, 0], metric: COSINE } }) { nodes { tenantId title physicalDatabaseIdentity embeddingVectorDistance } } }', + ...documentOracle( + [{ path: '/data/documents/nodes/0/embeddingVectorDistance', value: 0 }], + [], + [exactInvariant('/data/documents/nodes/*/embeddingVectorDistance', 0)] + ), + }), + fromBase('postgis-read', { + query: 'query PostgisRead { documents(first: 1) { nodes { tenantId title physicalDatabaseIdentity location { geojson } } } }', + ...documentOracle( + [{ + path: '/data/documents/nodes/0/location/geojson', + value: { type: 'Point', coordinates: [106.7, 10.8] }, + }], + [], + [exactInvariant( + '/data/documents/nodes/*/location/geojson', + { type: 'Point', coordinates: [106.7, 10.8] } + )] + ), + }), + fromBase('ltree-filter', { + query: 'query LtreeFilter { documents(where: { path: { within: "/root" } }) { nodes { tenantId title physicalDatabaseIdentity path } } }', + ...documentOracle( + [{ path: '/data/documents/nodes/0/path', value: `/root/${tenant.id}` }], + [], + [exactInvariant('/data/documents/nodes/*/path', `/root/${tenant.id}`)] + ), + }), + fromBase('presigned-upload', { + query: 'mutation PresignedUpload($input: UploadAppFileInput!) { uploadAppFile(input: $input) { fileId key deduplicated expiresAt uploadUrl } physicalDatabaseMutationIdentity(input: {}) { result } }', + variables: { + input: { + bucketKey: 'private', + contentHash: uploadContentHash, + contentType: 'text/plain', + size: 32, + filename: `${customer.id}-${tenant.id}.txt`, + }, + }, + requiredMatches: [{ + path: '/data/physicalDatabaseMutationIdentity/result', + value: customer.physicalIdentity, + }], + forbiddenMatches: [ + { path: '/data/physicalDatabaseMutationIdentity/result', value: null }, + ...otherCustomers.map((candidate) => ({ + path: '/data/physicalDatabaseMutationIdentity/result', + value: candidate.physicalIdentity, + })), + ], + invariants: [exactInvariant( + '/data/physicalDatabaseMutationIdentity/result', + customer.physicalIdentity + )], + postCoverageVerification: { + query: 'query VerifyPresignedUpload($fileId: UUID!, $contentHash: String!) { appFiles(first: 1, where: { id: { equalTo: $fileId }, contentHash: { equalTo: $contentHash } }) { nodes { id tenantId contentHash physicalDatabaseIdentity } } }', + variables: { contentHash: uploadContentHash }, + variablesFromResponse: { + fileId: '/data/uploadAppFile/fileId', + }, + requiredMatches: [ + { + path: '/data/appFiles/nodes/0/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/appFiles/nodes/0/contentHash', + value: uploadContentHash, + }, + { path: '/data/appFiles/nodes/0/tenantId', value: tenant.token }, + ], + forbiddenMatches: physicalIdentityMatches( + '/data/appFiles/nodes/0/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + invariants: [ + exactInvariant( + '/data/appFiles/nodes/*/physicalDatabaseIdentity', + customer.physicalIdentity + ), + exactInvariant('/data/appFiles/nodes/*/tenantId', tenant.token), + exactInvariant('/data/appFiles/nodes/*/contentHash', uploadContentHash), + ], + }, + }), + fromBase('bulk-upsert', { + query: 'mutation BulkUpsert($name: String!) { bulkUpsertBulkItems(input: { values: [{ name: $name, quantity: 1 }], onConflict: { constraint: BULK_ITEMS_NAME_KEY } }) { affectedCount returning { tenantId name physicalDatabaseIdentity } } }', + variables: { name: `${customer.id}-${tenant.token}-bulk` }, + requiredMatches: [ + { + path: '/data/bulkUpsertBulkItems/returning/0/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/bulkUpsertBulkItems/returning/0/name', + value: `${customer.id}-${tenant.token}-bulk`, + }, + { + path: '/data/bulkUpsertBulkItems/returning/0/tenantId', + value: tenant.token, + }, + ], + forbiddenMatches: physicalIdentityMatches( + '/data/bulkUpsertBulkItems/returning/0/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + invariants: [ + exactInvariant( + '/data/bulkUpsertBulkItems/returning/*/physicalDatabaseIdentity', + customer.physicalIdentity + ), + exactInvariant( + '/data/bulkUpsertBulkItems/returning/*/tenantId', + tenant.token + ), + exactInvariant( + '/data/bulkUpsertBulkItems/returning/*/name', + `${customer.id}-${tenant.token}-bulk` + ), + ], + }), + fromBase('realtime-tagged-update', { + query: 'mutation RealtimeTaggedUpdate($payload: String!) { updateRealtimeItem(input: { id: 1, realtimeItemPatch: { payload: $payload } }) { realtimeItem { id tenantId physicalDatabaseIdentity payload } } }', + variables: { payload: `${customer.id}-${tenant.token}-realtime` }, + requiredMatches: [ + { + path: '/data/updateRealtimeItem/realtimeItem/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/updateRealtimeItem/realtimeItem/tenantId', + value: tenant.token, + }, + { + path: '/data/updateRealtimeItem/realtimeItem/payload', + value: `${customer.id}-${tenant.token}-realtime`, + }, + ], + forbiddenMatches: physicalIdentityMatches( + '/data/updateRealtimeItem/realtimeItem/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + }), + fromBase('bound-function-invocation', { + query: 'mutation BoundFunctionInvocation($payload: JSON!) { fixtureTask(input: { payload: $payload }) { invocationId status invocation { tenantId physicalDatabaseIdentity taskIdentifier } } }', + variables: { + payload: { + tenant: tenant.id, + customer: customer.id, + source: 'physical-database-density', + }, + }, + requiredMatches: [ + { + path: '/data/fixtureTask/invocation/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/fixtureTask/invocation/tenantId', + value: tenant.token, + }, + { + path: '/data/fixtureTask/invocation/taskIdentifier', + value: `ctf.fixture.${tenant.id}`, + }, + ], + forbiddenMatches: physicalIdentityMatches( + '/data/fixtureTask/invocation/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + }), + fromBase('security-context-read', { + query: 'query SecurityContextRead { physicalDatabaseIdentity requestIdentity }', + requiredMatches: [ + { + path: '/data/physicalDatabaseIdentity', + value: customer.physicalIdentity, + }, + { + path: '/data/requestIdentity', + value: `${tenant.token}:${tenant.databaseId}`, + }, + ], + forbiddenMatches: [ + ...physicalIdentityMatches( + '/data/physicalDatabaseIdentity', + customer, + otherCustomers + ).forbiddenMatches, + ...completeFixture.TENANTS.filter((candidate) => candidate.id !== tenant.id) + .map((candidate) => ({ + path: '/data/requestIdentity', + value: `${candidate.token}:${tenant.databaseId}`, + })), + ], + }), + ]; + if (baseByName.size > 0) { + throw new Error( + `PDCF_OPERATION_ORACLE_MISSING:${[...baseByName.keys()].sort().join(',')}` + ); + } + return operations; +}; + +const physicalCanariesFor = (customer, tenant, otherCustomers) => + completeFixture.canariesFor(tenant).map((candidate) => { + if (candidate.name !== 'plugin-raw-sql') return candidate; + const expectedTitle = `${tenant.token} español @${customer.physicalIdentity}`; + const databaseCandidates = [customer, ...otherCustomers]; + return { + ...candidate, + requiredMatches: [{ + path: '/data/posts/nodes/0/localeStrings/title', + value: expectedTitle, + }], + forbiddenMatches: [ + { path: '/data/posts/nodes/0/localeStrings/title', value: null }, + ...databaseCandidates.flatMap((databaseCustomer) => + completeFixture.TENANTS + .filter((candidateTenant) => ( + candidateTenant.id !== tenant.id + || databaseCustomer.id !== customer.id + )) + .map((candidateTenant) => ({ + path: '/data/posts/nodes/0/localeStrings/title', + value: `${candidateTenant.token} español @${databaseCustomer.physicalIdentity}`, + })) + ), + ], + invariants: [exactInvariant( + '/data/posts/nodes/*/localeStrings/title', + expectedTitle + )], + }; + }); + +const makeFleet = ({ manifest, statuses, arms = DEFAULT_IDLE_ARMS, port = 3410 }) => ({ + version: 1, + tenants: manifest.customers.map((customer) => { + const otherCustomers = manifest.customers.filter((candidate) => candidate.id !== customer.id); + const firstStatus = statusFor(statuses, arms[0].name, customer.id); + if (firstStatus.physicalDatabase !== customer.database) { + throw new Error(`PDCF_STATUS_DATABASE_MISMATCH:${customer.id}`); + } + return { + id: customer.id, + databases: [{ + id: `logical:${customer.id}`, + physicalDatabase: customer.database, + apis: completeFixture.TENANTS.map((tenant) => ({ + id: `api:${customer.id}:${tenant.id}`, + runtimePoolIdentity: runtimePoolContractFingerprintFor( + firstStatus, + tenant.id, + ), + runtimePoolIdentities: Object.fromEntries(arms.map((arm) => [ + arm.name, + runtimePoolContractFingerprintFor( + statusFor(statuses, arm.name, customer.id), + tenant.id, + ), + ])), + physicalSchemas: [tenant.schema], + routingLabels: [`${customer.id}-${tenant.id}`], + realtime: true, + surfaces: [`api-${tenant.id}`], + })), + }], + surfaces: completeFixture.TENANTS.map((tenant) => ({ + name: `api-${tenant.id}`, + buildContract: graphileBuildContractFingerprintFor(firstStatus, tenant.id), + buildContracts: Object.fromEntries(arms.map((arm) => [ + arm.name, + graphileBuildContractFingerprintFor( + statusFor(statuses, arm.name, customer.id), + tenant.id, + ), + ])), + url: `http://127.0.0.1:{port}/customer/${customer.id}/tenant/${tenant.id}/graphql`, + headers: { 'accept-language': 'es' }, + warmup: { + name: 'warm-physical-database-identity', + capability: 'graphile-generated', + query: 'query WarmPhysicalDatabaseIdentity { physicalDatabaseIdentity }', + ...physicalIdentityMatches( + '/data/physicalDatabaseIdentity', + customer, + otherCustomers + ), + }, + operations: physicalOperationsFor(customer, tenant, otherCustomers), + realtime: realtimeProbe(customer, tenant, otherCustomers), + canaries: [ + ...physicalCanariesFor(customer, tenant, otherCustomers), + physicalCanary(customer, otherCustomers), + ], + })), + }; + }), +}); + +const makeCacheCapacityProofByHeapMiB = ({ + cacheCalibration, + databaseContractFingerprint, + introspectionMode = 'scoped-required', + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + heapLimitBytesByHeapMiB, +}) => { + const countMatrix = validateCustomerCountMatrix({ + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + }); + const calibration = validateCacheCalibration(cacheCalibration, { + databaseContractFingerprint, + introspectionMode, + }); + return Object.fromEntries(heapMiB.map((configuredHeapMiB) => { + const requiredResidentInstances = Math.max( + ...countMatrix.byHeap[String(configuredHeapMiB)] + ) * completeFixture.TENANTS.length; + const heapLimitBytes = heapLimitBytesByHeapMiB?.[String(configuredHeapMiB)]; + if (!Number.isSafeInteger(heapLimitBytes) || heapLimitBytes <= 0) { + throw new Error(`PDCF_HEAP_LIMIT_REQUIRED:${configuredHeapMiB}`); + } + const budgetCapacity = computeCalibratedCapacity( + heapLimitBytes, + calibration.configured, + ); + if (budgetCapacity < requiredResidentInstances) { + throw new Error( + `PDCF_CALIBRATED_CAPACITY_INSUFFICIENT:${configuredHeapMiB}:${budgetCapacity}:${requiredResidentInstances}` + ); + } + return [String(configuredHeapMiB), { + calibrationId: calibration.calibrationId, + expectedHeapLimitBytes: heapLimitBytes, + budgetCapacity, + configuredResidentCapacity: budgetCapacity, + requiredResidentInstances, + residentHeadroomInstances: budgetCapacity - requiredResidentInstances, + admissionMode: QUALIFYING_CACHE_ADMISSION_MODE, + capacityRefusalReason: 'resident_capacity', + capacityResponseCode: 'GRAPHILE_BUILD_RESIDENT_CAPACITY', + preservesExistingResidentsAtCapacity: true, + safetyFactor: calibration.safetyFactor, + measured: calibration.measured, + configured: calibration.configured, + sourceResultSha256: calibration.sources.map((source) => source.sourceSha256), + }]; + })); +}; + +const makePlan = ({ + manifestFile, + secretsFile, + postgresContainer, + commit, + entrySha256, + lockfileSha256, + arms = DEFAULT_IDLE_ARMS, + basePort = 3410, + heapMiB = [1024, 2048, 4096], + tenantCounts, + tenantCountsByHeapMiB, + repetitions = 3, + durationSec = 900, + introspectionMode = 'scoped-required', + databaseContractFingerprint, + blueprintCompatibilityFingerprint, + manifestSha256, + provisionClone, + cacheCalibration, + heapLimitBytesByHeapMiB, + cacheCapacityByHeapMiB, + postgresContainerTemplateFile, + postgresContainerTemplateSha256, +}) => { + if ( + !postgresContainer + || !postgresContainerTemplateFile + || !/^sha256:[a-f0-9]{64}$/.test(postgresContainerTemplateSha256 ?? '') + || !commit + || !entrySha256 + || !lockfileSha256 + ) { + throw new Error('PDCF_PLAN_PROVENANCE_REQUIRED'); + } + validateMeasurementProvisionClone(provisionClone); + if ( + !/^sha256:[a-f0-9]{64}$/.test(databaseContractFingerprint ?? '') + || !/^sha256:[a-f0-9]{64}$/.test(blueprintCompatibilityFingerprint ?? '') + || !/^sha256:[a-f0-9]{64}$/.test(manifestSha256 ?? '') + ) { + throw new Error('PDCF_PLAN_COMPATIBILITY_FINGERPRINT_REQUIRED'); + } + const countMatrix = validateCustomerCountMatrix({ + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + }); + const calibration = validateCacheCalibration(cacheCalibration, { + databaseContractFingerprint, + introspectionMode, + }); + const computedCapacityByHeapMiB = makeCacheCapacityProofByHeapMiB({ + cacheCalibration: calibration, + databaseContractFingerprint, + introspectionMode, + tenantCounts, + tenantCountsByHeapMiB, + heapMiB, + heapLimitBytesByHeapMiB, + }); + if ( + cacheCapacityByHeapMiB + && JSON.stringify(cacheCapacityByHeapMiB) !== JSON.stringify(computedCapacityByHeapMiB) + ) { + throw new Error('PDCF_CACHE_CAPACITY_PROOF_MISMATCH'); + } + const calibrationByHeapMiB = cacheCapacityByHeapMiB ?? computedCapacityByHeapMiB; + return { + version: 1, + fleetFile: 'fleet.json', + artifactDir: '../artifacts', + arms: arms.map((arm, index) => { + const port = basePort + index; + const poolIdentitiesPerCustomer = completeFixture.TENANTS.length + + (notificationModeForArm(arm) === 'shared-exact' ? 1 : 0); + return { + name: arm.name, + commit, + cwd: REPO_ROOT, + command: [ + 'node', + '--expose-gc', + path.join(FIXTURE_DIR, 'server.cjs'), + '--manifest', '{postgresManifestFile}', + '--secrets', '{postgresSecretsFile}', + '--customers', '{tenantCount}', + '--host', '127.0.0.1', + '--port', '{port}', + '--arm', arm.name, + '--mode', '{mode}', + '--introspection-client-release-mode', 'destroy', + '--runtime-pool-max', String(runtimePoolMaxForArm(arm)), + '--runtime-pool-max-uses', runtimePoolMaxUsesForArm(arm) == null + ? 'unlimited' + : String(runtimePoolMaxUsesForArm(arm)), + '--realtime-notification-mode', notificationModeForArm(arm), + '--realtime-cursor-poll-ms', String(cursorPollMsForArm(arm)), + '--realtime-cursor-heartbeat-ms', String(cursorHeartbeatMsForArm(arm)), + '--enable-realtime', 'true', + '--expected-database-contract', databaseContractFingerprint, + '--blueprint-compatibility', blueprintCompatibilityFingerprint, + '--expected-manifest-sha256', '{postgresManifestSha256}', + '--run-purpose', 'measurement', + '--clone-id', '{postgresCloneId}', + ], + port, + readinessUrl: 'http://127.0.0.1:{port}/healthz', + memoryUrl: 'http://127.0.0.1:{port}/debug/memory', + retainedHeapCheckpointUrl: + 'http://127.0.0.1:{port}/__cperf/retained-memory-checkpoint', + postWarmupUrl: 'http://127.0.0.1:{port}/__cperf/post-warmup', + postgresContainer, + requirePostgresCgroupV2: true, + postgresRunAttestation: { + command: [ + 'node', + path.join(FIXTURE_DIR, 'measurement-attestation.cjs'), + '--manifest', '{postgresManifestFile}', + '--secrets', '{postgresSecretsFile}', + '--postgres-container', postgresContainer, + '--container-template', postgresContainerTemplateFile, + '--expected-container-template-sha256', + postgresContainerTemplateSha256, + '--arm', '{arm}', + '--heap-mib', '{heapMiB}', + '--customers', '{tenantCount}', + '--repetition', '{repetition}', + '--run-order-index', '{runOrderIndex}', + '--plan-sha256', '{planSha256}', + '--fleet-sha256', '{fleetSha256}', + '--not-before-epoch-ms', '{notBeforeEpochMs}', + '--out', '{attestationFile}', + ], + prepareCommand: [ + 'node', + path.join(FIXTURE_DIR, 'prepare-measurement-run.cjs'), + '--container-template', postgresContainerTemplateFile, + '--expected-container-template-sha256', + postgresContainerTemplateSha256, + '--manifest-template', manifestFile, + '--secrets-template', secretsFile, + '--expected-manifest-template-sha256', manifestSha256, + '--artifact-dir', '{postgresFixtureDir}', + '--arm', '{arm}', + '--heap-mib', '{heapMiB}', + '--customers', '{tenantCount}', + '--repetition', '{repetition}', + '--run-order-index', '{runOrderIndex}', + ], + timeoutMs: 900_000, + }, + introspectionMode, + v8Profile: arm.v8Profile ?? 'stock', + startupTimeoutMs: 600_000, + entrySha256, + lockfileSha256, + env: { + GRAPHILE_BUILD_MAX_CONCURRENCY: '1', + GRAPHILE_BUILD_CONCURRENCY: '1', + GRAPHILE_BUILD_QUEUE_MAX: '64', + // Keep ambient code-loading hooks out of the measured child too; + // the harness adds only the attested heap limit to NODE_OPTIONS. + NODE_OPTIONS: '', + NODE_PATH: '', + // Runtime pools are exact per surface. Shared realtime adds one exact + // notification pool identity per physical customer, even though that + // identity owns only one backend for all of the customer's surfaces. + PG_CACHE_MAX: String( + Math.max(...countMatrix.all) * poolIdentitiesPerCustomer + 8 + ), + PG_POOL_IDLE_TIMEOUT_MS: String(arm.idleTimeoutMs), + // Control and incidental pools must not change shape with the arm. + // Runtime pools receive their capacity through the explicit option. + PG_POOL_MAX: String(PROCESS_GLOBAL_POOL_MAX), + // maxUses is an exact runtime-pool option; ambient, control, and + // notification pools remain reusable in every arm. + PG_POOL_MAX_USES: '0', + ...(preparedStatementCacheSizeForArm(arm) == null ? {} : { + DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE: + String(preparedStatementCacheSizeForArm(arm)), + }), + }, + envByHeapMiB: Object.fromEntries(Object.entries(calibrationByHeapMiB).map( + ([configuredHeapMiB, proof]) => [configuredHeapMiB, { + GRAPHILE_CACHE_MAX: String(proof.configuredResidentCapacity), + GRAPHILE_CACHE_ADMISSION_MODE: proof.admissionMode, + GRAPHILE_CACHE_INSTANCE_HEAP_BYTES: + String(proof.configured.instanceHeapBytes), + GRAPHILE_CACHE_SERVER_RESERVE_BYTES: + String(proof.configured.serverReserveBytes), + GRAPHILE_CACHE_BUILD_RESERVE_BYTES: + String(proof.configured.buildReserveBytes), + GRAPHILE_CACHE_RSS_BUILD_RESERVE_BYTES: + String(proof.configured.rssBuildReserveBytes), + GRAPHILE_CACHE_CALIBRATION_ID: proof.calibrationId, + GRAPHQL_CPERF_RETAINED_HEAP_ENABLED: 'true', + }] + )), + cacheCalibrationByHeapMiB: calibrationByHeapMiB, + }; + }), + heapMiB, + ...(tenantCounts == null ? {} : { tenantCounts }), + ...(tenantCountsByHeapMiB == null ? {} : { tenantCountsByHeapMiB }), + repetitions, + runOrderSeed: `${FIXTURE_ID}:${manifestFile}:${calibration.calibrationId}`, + cacheCalibration: calibration, + requiredCapabilities: [...completeFixture.REQUIRED_CAPABILITIES], + requiredCanaries: [ + ...completeFixture.REQUIRED_CANARIES, + PHYSICAL_DATABASE_CANARY, + ], + workload: { + durationSec, + rpsPerTenant: 0.2, + minWorkloadRequestsPerSurface: 10, + requestTimeoutMs: 30_000, + maxInFlight: 64, + canaryIntervalSec: 60, + periodicCanarySchedule: 'rotating-one', + canaryConcurrency: 16, + warmupTimeoutMs: 300_000, + warmupTimeoutPerSurfaceMs: 45_000, + warmupConcurrency: 1, + }, + gates: { + maxErrorRate: 0.005, + maxP99Ms: 150, + maxPostWarmupHeapGrowthMiBPerHour: 5, + minMedianDensityImprovement: 0.15, + minAdditionalTenantsEveryRun: 1, + requireZeroBleed: true, + requireNoPostWarmupEvictions: true, + requireNoPostWarmupBuildRefusals: true, + requireNoPostWarmupBuilds: true, + requirePostgresMemoryTelemetry: true, + requireFreshPostgresRunAttestation: true, + requirePhysicalDatabaseTelemetry: true, + requireConclusiveCanaries: true, + requireCompletePeriodicCanaryCoverage: true, + requireConclusiveOperationOracles: true, + requireExplicitCustomerTopology: true, + requireRetainedMemoryCheckpoints: true, + requiredCacheAdmissionMode: QUALIFYING_CACHE_ADMISSION_MODE, + }, + ...(repetitions >= 3 && [1024, 2048, 4096].every((heap) => heapMiB.includes(heap)) + ? { + qualification: { + baselineArm: arms[0].name, + requiredHeapMiB: [1024, 2048, 4096], + minimumRepetitions: 3, + }, + } + : {}), + }; +}; + +const atomicWriteJson = (file, value, mode = 0o644) => { + const absolute = path.resolve(file); + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + const temporary = `${absolute}.${process.pid}.${crypto.randomBytes(6).toString('hex')}.tmp`; + fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode }); + fs.renameSync(temporary, absolute); + fs.chmodSync(absolute, mode); +}; + +module.exports = { + DEFAULT_IDLE_ARMS, + DENSITY_TUNING_ARMS, + FIXTURE_DIR, + FIXTURE_ID, + PHYSICAL_DATABASE_CANARY, + PROCESS_GLOBAL_POOL_MAX, + QUALIFYING_CACHE_ADMISSION_MODE, + REPO_ROOT, + atomicWriteJson, + loadProvision, + makeSecretResolver, + makeCustomers, + makeFleet, + makeCacheCapacityProofByHeapMiB, + makePlan, + cursorHeartbeatMsForArm, + cursorPollMsForArm, + notificationModeForArm, + preparedStatementCacheSizeForArm, + runtimePoolMaxForArm, + runtimePoolMaxUsesForArm, + strictIdentifier, + validateCustomerCountMatrix, + validateCustomerCountRamp, + validateProvisionManifest, + validateSecrets, +}; diff --git a/research/graphile-density/physical-database-density/lib.test.cjs b/research/graphile-density/physical-database-density/lib.test.cjs new file mode 100644 index 0000000000..b6ee37cc29 --- /dev/null +++ b/research/graphile-density/physical-database-density/lib.test.cjs @@ -0,0 +1,702 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { + DEFAULT_IDLE_ARMS, + DENSITY_TUNING_ARMS, + PHYSICAL_DATABASE_CANARY, + makeCustomers, + makeCacheCapacityProofByHeapMiB, + makeFleet, + makePlan, + validateProvisionManifest, +} = require('./lib.cjs'); +const { + DEFAULT_CANONICAL_SCHEMAS, + normalizeSchemaDump, +} = require('./provision.cjs'); +const { + CALIBRATION_KIND, + sha256Canonical, +} = require('./cache-calibration.cjs'); + +const MIB = 1024 ** 2; +const databaseContractFingerprint = `sha256:${'d'.repeat(64)}`; +const calibrationPayload = { + kind: CALIBRATION_KIND, + databaseContractFingerprint, + introspectionMode: 'scoped-required', + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: true, + introspectionBackendRetirementConclusive: true, + fixtureFingerprint: 'fixture-v1', + schemaContract: { + schemaSets: [['ctf_a']], + allowedDependencySchemas: ['ctf_extensions'], + }, + safetyFactor: 1.25, + measured: { + repetitions: 3, + retainedHeapPerSurfaceBytes: 12 * MIB, + serverBaselineHeapBytes: 48 * MIB, + buildTransientHeapBytes: 96 * MIB, + buildTransientRssBytes: 96 * MIB, + }, + configured: { + instanceHeapBytes: 15 * MIB, + serverReserveBytes: 60 * MIB, + buildReserveBytes: 120 * MIB, + rssBuildReserveBytes: 120 * MIB, + }, + sourceWorktreesClean: true, + sources: ['1', '2', '3'].map((value) => ({ + sourceSha256: `sha256:${value.repeat(64)}`, + sourceStateSha256: value.repeat(64), + executedEntrySha256: value.repeat(64), + worktreeDirty: false, + introspectionBackendRetirement: { + conclusive: true, + introspectionBackendPid: Number(value), + steadyBackendPid: Number(value) + 10, + }, + })), +}; +const cacheCalibration = { + version: 2, + ...calibrationPayload, + calibrationId: sha256Canonical(calibrationPayload), +}; +const heapLimitBytesByHeapMiB = { + '1024': 1024 * MIB, + '2048': 2048 * MIB, + '4096': 4096 * MIB, +}; + +const manifest = (count = 2) => ({ + version: 1, + fixture: 'physical-database-density-v1', + prefix: 'pdc_test', + provisionClone: { + version: 1, + id: 'measurement-clone-test', + purpose: 'measurement', + attestationSetSha256: `sha256:${'9'.repeat(64)}`, + }, + customers: makeCustomers('pdc_test', count), +}); + +const statuses = (value) => Object.fromEntries(DEFAULT_IDLE_ARMS.map((arm) => [ + arm.name, + Object.fromEntries(value.customers.map((customer) => [customer.id, { + physicalDatabase: customer.database, + runtimePoolIdentities: { + a: `pg:v1:${arm.name}:${customer.id}:a`, + b: `pg:v1:${arm.name}:${customer.id}:b`, + c: `pg:v1:${arm.name}:${customer.id}:c`, + }, + buildContracts: { + a: `graphile:v1:${arm.name}:${customer.id}:a`, + b: `graphile:v1:${arm.name}:${customer.id}:b`, + c: `graphile:v1:${arm.name}:${customer.id}:c`, + }, + contractEvidence: { + runtimePools: Object.fromEntries(['a', 'b', 'c'].map((tenantId) => [ + tenantId, + { + fingerprint: `pg-contract-evidence:v1:${sha256Canonical({ + kind: 'pool', + arm: arm.name, + customer: customer.id, + tenantId, + }).slice('sha256:'.length)}`, + }, + ])), + graphileBuilds: Object.fromEntries(['a', 'b', 'c'].map((tenantId) => [ + tenantId, + { + fingerprint: `graphile-contract-evidence:v1:${sha256Canonical({ + kind: 'build', + arm: arm.name, + customer: customer.id, + tenantId, + }).slice('sha256:'.length)}`, + }, + ])), + }, + }])), +])); + +describe('physical database density fixture', () => { + it('stamps realtime physical identity inside PostgreSQL before every write', () => { + const sql = fs.readFileSync(path.join(__dirname, 'physical-identity.sql'), 'utf8'); + for (const schema of ['ctf_a', 'ctf_b', 'ctf_c']) { + assert.match( + sql, + new RegExp(`ALTER TABLE ${schema}\\.realtime_items[\\s\\S]*ADD COLUMN physical_database_identity text`) + ); + assert.match( + sql, + new RegExp(`BEFORE INSERT OR UPDATE ON ${schema}\\.realtime_items`) + ); + assert.match( + sql, + new RegExp(`CREATE FUNCTION ${schema}\\.stamp_realtime_physical_database_identity\\(\\)[\\s\\S]*NEW\\.physical_database_identity := pg_catalog\\.current_database\\(\\)::text`) + ); + } + for (const table of [ + 'documents', + 'posts', + 'posts_translations', + 'articles', + 'articles_chunks', + 'bulk_items', + 'app_files', + 'function_invocations', + ]) { + assert.match(sql, new RegExp(`'${table}'`)); + } + assert.match( + sql, + /NEW\.physical_database_identity := pg_catalog\.current_database\(\)::text/ + ); + assert.equal( + (sql.match(/physical_database_mutation_identity\(\)[\s\S]*?LANGUAGE sql[\s\S]*?VOLATILE/g) ?? []).length, + 3, + ); + assert.match( + sql, + /GRANT EXECUTE ON FUNCTION ctf_a\.physical_database_mutation_identity\(\) TO :"runtime_role_a"/, + ); + assert.match(sql, /posts_translations SET title = title \|\|/); + assert.match(sql, /articles_chunks SET content = content \|\|/); + }); + + it('generates distinct databases and cluster-wide least-privilege role identities', () => { + const customers = makeCustomers('pdc_test', 2); + assert.deepEqual(customers.map((customer) => customer.database), [ + 'pdc_test_db_0001', + 'pdc_test_db_0002', + ]); + assert.equal(new Set(customers.flatMap((customer) => Object.values(customer.roles))).size, 6); + assert.equal(new Set(customers.map((customer) => customer.notificationRole)).size, 2); + assert.ok(customers.every((customer) => + !Object.values(customer.roles).includes(customer.notificationRole) + )); + assert.equal(customers[0].physicalIdentity, customers[0].database); + assert.equal(validateProvisionManifest(manifest()).customers.length, 2); + }); + + it('binds every unique physical identity to its exact database label', () => { + const swapped = manifest(); + swapped.customers[0] = { + ...swapped.customers[0], + physicalIdentity: swapped.customers[1].database, + }; + assert.throws( + () => validateProvisionManifest(swapped), + /PDCF_PHYSICAL_IDENTITY_DATABASE_MISMATCH/, + ); + + const duplicate = manifest(); + duplicate.customers[1] = { + ...duplicate.customers[1], + database: duplicate.customers[0].database, + physicalIdentity: duplicate.customers[0].physicalIdentity, + }; + assert.throws( + () => validateProvisionManifest(duplicate), + /PDCF_DATABASE_DUPLICATE|PDCF_PHYSICAL_IDENTITY_DUPLICATE/, + ); + }); + + it('maps one complete customer to one physical database and three resident realtime APIs', () => { + const provision = manifest(); + const armStatuses = statuses(provision); + const fleet = makeFleet({ manifest: provision, statuses: armStatuses }); + assert.equal(fleet.tenants.length, 2); + assert.equal(fleet.tenants[0].databases.length, 1); + assert.equal(fleet.tenants[0].databases[0].physicalDatabase, 'pdc_test_db_0001'); + assert.equal(fleet.tenants[0].databases[0].apis.length, 3); + assert.ok(fleet.tenants[0].databases[0].apis.every((api) => api.realtime)); + assert.equal(fleet.tenants[0].surfaces.length, 3); + assert.ok(fleet.tenants[0].surfaces.every((surface) => + surface.realtime?.subscription?.query.includes('PhysicalDensityRealtimeResident') + && surface.realtime?.prime?.query.includes('PhysicalDensityRealtimePrime') + && surface.realtime.subscription.requiredMatches.length === 2 + && surface.realtime.subscription.forbiddenMatches.length === 3 + && surface.realtime.correlation.primeVariable === 'payload' + && surface.realtime.correlation.primeResponsePath + === '/data/updateRealtimeItem/realtimeItem/payload' + && surface.realtime.correlation.subscriptionEventPath + === '/data/onRealtimeItemChanged/realtimeItem/payload' + )); + assert.ok(fleet.tenants[0].surfaces.every((surface) => + surface.warmup.requiredMatches.some((match) => + match.value === 'pdc_test_db_0001' + ) + && surface.operations.every((operation) => { + const oracle = operation.postCoverageVerification ?? operation; + return oracle.requiredMatches.some((match) => + match.value === 'pdc_test_db_0001' + ) && oracle.forbiddenMatches.some((match) => + match.value === 'pdc_test_db_0002' + ); + }) + )); + const operations = new Map( + fleet.tenants[0].surfaces[0].operations.map((operation) => [operation.name, operation]) + ); + assert.match( + operations.get('deterministic-rag').query, + /physicalDatabaseIdentity.*ragQuery/ + ); + assert.ok( + operations.get('deterministic-rag').requiredMatches.some((match) => + match.path.includes('/sources/') + && match.value.endsWith('@pdc_test_db_0001') + ) + ); + assert.match(operations.get('bulk-upsert').query, /returning \{ tenantId name physicalDatabaseIdentity \}/); + assert.match(operations.get('realtime-tagged-update').query, /physicalDatabaseIdentity/); + assert.match(operations.get('bound-function-invocation').query, /invocation \{ tenantId physicalDatabaseIdentity/); + assert.match( + operations.get('presigned-upload').postCoverageVerification.query, + /appFiles.*physicalDatabaseIdentity/ + ); + assert.match( + operations.get('presigned-upload').query, + /physicalDatabaseMutationIdentity\(input: \{\}\) \{ result \}/, + ); + assert.deepEqual( + operations.get('presigned-upload').postCoverageVerification.variablesFromResponse, + { fileId: '/data/uploadAppFile/fileId' }, + ); + assert.match( + operations.get('presigned-upload').postCoverageVerification.query, + /id: \{ equalTo: \$fileId \}/, + ); + for (const operationName of [ + 'generated-document-read', + 'bm25-search', + 'tsvector-search', + 'trigram-search', + 'vector-search', + 'postgis-read', + 'ltree-filter', + ]) { + assert.ok( + operations.get(operationName).invariants.some((invariant) => + invariant.path === '/data/documents/nodes/*/physicalDatabaseIdentity' + && invariant.everyEquals === 'pdc_test_db_0001' + && invariant.min === 1 + && invariant.max === 1 + ), + ); + } + assert.ok(operations.get('deterministic-embed').requiredMatches.some((match) => + match.path === '/data/embedText/vector' + && JSON.stringify(match.value) === '[1,0,0]' + )); + assert.ok(operations.get('deterministic-rag').requiredMatches.some((match) => + match.path === '/data/ragQuery/answer' + && match.value === 'Deterministic fixture answer: machine learning tenant fixture' + )); + assert.ok(operations.get('postgis-read').requiredMatches.some((match) => + match.path.endsWith('/location/geojson') + && match.value.type === 'Point' + )); + assert.ok(operations.get('ltree-filter').requiredMatches.some((match) => + match.path.endsWith('/path') && match.value === '/root/a' + )); + const rawSqlCanary = fleet.tenants[0].surfaces[0].canaries.find( + (candidate) => candidate.name === 'plugin-raw-sql' + ); + assert.equal( + rawSqlCanary.requiredMatches[0].value, + 'tenant-a-canary español @pdc_test_db_0001', + ); + assert.deepEqual( + fleet.tenants[0].surfaces[0].realtime.subscription.requiredMatches, + [ + { + path: '/data/onRealtimeItemChanged/realtimeItem/tenantId', + value: 'tenant-a-canary', + }, + { + path: '/data/onRealtimeItemChanged/realtimeItem/physicalDatabaseIdentity', + value: 'pdc_test_db_0001', + }, + ] + ); + assert.ok( + fleet.tenants[0].surfaces[0].realtime.subscription.forbiddenMatches.some( + (match) => match.path.endsWith('/physicalDatabaseIdentity') + && match.value === 'pdc_test_db_0002' + ) + ); + const canary = fleet.tenants[0].surfaces[0].canaries.find( + (candidate) => candidate.name === PHYSICAL_DATABASE_CANARY + ); + assert.deepEqual(canary.requiredMatches, [{ + path: '/data/physicalDatabaseIdentity', + value: 'pdc_test_db_0001', + }]); + assert.deepEqual(canary.forbiddenMatches, [ + { + path: '/data/physicalDatabaseIdentity', + value: null, + }, + { + path: '/data/physicalDatabaseIdentity', + value: 'pdc_test_db_0002', + }, + ]); + assert.equal( + fleet.tenants[0].surfaces[0].buildContracts[DEFAULT_IDLE_ARMS[2].name], + armStatuses[DEFAULT_IDLE_ARMS[2].name]['physical-customer-0001'] + .contractEvidence.graphileBuilds.a.fingerprint + ); + }); + + it('keeps a conclusive negative routing oracle for a one-customer smoke', () => { + const provision = manifest(); + provision.customers = provision.customers.slice(0, 1); + const fleet = makeFleet({ manifest: provision, statuses: statuses(provision) }); + const canary = fleet.tenants[0].surfaces[0].canaries.find( + (candidate) => candidate.name === PHYSICAL_DATABASE_CANARY + ); + assert.deepEqual(canary.forbiddenMatches, [{ + path: '/data/physicalDatabaseIdentity', + value: null, + }]); + }); + + it('emits explicit 30s, 5s, and 1s arms with a post-warmup realtime hook', () => { + const plan = makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: manifest().provisionClone, + tenantCounts: [2], + cacheCalibration, + heapLimitBytesByHeapMiB, + }); + assert.deepEqual( + plan.arms.map((arm) => arm.env.PG_POOL_IDLE_TIMEOUT_MS), + ['30000', '5000', '1000'] + ); + assert.ok(plan.arms.every((arm) => arm.requirePostgresCgroupV2)); + assert.ok(plan.arms.every((arm) => arm.v8Profile === 'stock')); + assert.ok(plan.arms.every((arm) => arm.postWarmupUrl.endsWith('/__cperf/post-warmup'))); + assert.ok(plan.arms.every((arm) => + arm.retainedHeapCheckpointUrl + .endsWith('/__cperf/retained-memory-checkpoint') + )); + assert.ok(plan.arms.every((arm) => arm.command.includes('--expose-gc'))); + assert.ok(plan.arms.every((arm) => arm.command.includes('{tenantCount}'))); + assert.ok(plan.arms.every((arm) => { + const index = arm.command.indexOf('--introspection-client-release-mode'); + return index >= 0 && arm.command[index + 1] === 'destroy'; + })); + assert.ok(plan.arms.every((arm) => arm.command.includes(`sha256:${'d'.repeat(64)}`))); + assert.ok(plan.arms.every((arm) => { + const purposeIndex = arm.command.indexOf('--run-purpose'); + const cloneIndex = arm.command.indexOf('--clone-id'); + return purposeIndex >= 0 + && arm.command[purposeIndex + 1] === 'measurement' + && cloneIndex >= 0 + && arm.command[cloneIndex + 1] === '{postgresCloneId}'; + })); + assert.ok(plan.arms.every((arm) => + arm.command.includes('{postgresManifestFile}') + && arm.command.includes('{postgresSecretsFile}') + && arm.command.includes('{postgresManifestSha256}') + && arm.postgresRunAttestation.prepareCommand + .includes('{postgresFixtureDir}') + )); + assert.ok(plan.arms.every((arm) => + arm.envByHeapMiB['1024'].GRAPHILE_CACHE_CALIBRATION_ID + === cacheCalibration.calibrationId + )); + assert.ok(plan.arms.every((arm) => + Number(arm.envByHeapMiB['1024'].GRAPHILE_CACHE_MAX) + === arm.cacheCalibrationByHeapMiB['1024'].budgetCapacity + )); + assert.ok(plan.arms.every((arm) => + arm.envByHeapMiB['1024'].GRAPHILE_CACHE_ADMISSION_MODE === 'preserve-resident' + )); + assert.ok(plan.arms.every((arm) => + arm.envByHeapMiB['1024'].GRAPHQL_CPERF_RETAINED_HEAP_ENABLED === 'true' + )); + assert.equal(plan.gates.requireRetainedMemoryCheckpoints, true); + assert.equal(plan.gates.requireConclusiveOperationOracles, true); + assert.equal(plan.gates.requiredCacheAdmissionMode, 'preserve-resident'); + assert.equal(plan.gates.requireCompletePeriodicCanaryCoverage, true); + assert.equal(plan.workload.periodicCanarySchedule, 'rotating-one'); + assert.equal(plan.workload.canaryConcurrency, 16); + assert.equal( + Math.max( + 0, + Math.ceil(plan.workload.durationSec / plan.workload.canaryIntervalSec) - 1, + ), + 14, + ); + assert.equal(plan.requiredCanaries.length, 14); + const capacityProof = plan.arms[0].cacheCalibrationByHeapMiB['1024']; + assert.equal(capacityProof.requiredResidentInstances, 6); + assert.equal(capacityProof.configuredResidentCapacity, capacityProof.budgetCapacity); + assert.equal( + capacityProof.residentHeadroomInstances, + capacityProof.budgetCapacity - capacityProof.requiredResidentInstances, + ); + assert.equal(capacityProof.capacityRefusalReason, 'resident_capacity'); + assert.equal(capacityProof.capacityResponseCode, 'GRAPHILE_BUILD_RESIDENT_CAPACITY'); + assert.equal(capacityProof.preservesExistingResidentsAtCapacity, true); + const oversizedPayload = { + ...calibrationPayload, + measured: { + ...calibrationPayload.measured, + buildTransientHeapBytes: 720 * MIB, + }, + configured: { + ...calibrationPayload.configured, + buildReserveBytes: 900 * MIB, + }, + }; + const rampPlan = makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: manifest().provisionClone, + tenantCounts: [1, 2], + cacheCalibration, + heapLimitBytesByHeapMiB, + }); + assert.deepEqual(rampPlan.tenantCounts, [1, 2]); + assert.equal( + rampPlan.arms[0].cacheCalibrationByHeapMiB['1024'].requiredResidentInstances, + 6, + ); + const perHeapPlan = makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: manifest().provisionClone, + tenantCountsByHeapMiB: { + '1024': [1], + '2048': [1, 2], + }, + heapMiB: [1024, 2048], + cacheCalibration, + heapLimitBytesByHeapMiB, + }); + assert.deepEqual(perHeapPlan.tenantCountsByHeapMiB, { + '1024': [1], + '2048': [1, 2], + }); + assert.equal( + perHeapPlan.arms[0].cacheCalibrationByHeapMiB['1024'].requiredResidentInstances, + 3, + ); + assert.equal( + perHeapPlan.arms[0].cacheCalibrationByHeapMiB['2048'].requiredResidentInstances, + 6, + ); + assert.throws(() => makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: manifest().provisionClone, + tenantCounts: [2, 1], + cacheCalibration, + heapLimitBytesByHeapMiB, + }), /CUSTOMER_COUNT_RAMP_INVALID/); + assert.throws(() => makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: manifest().provisionClone, + tenantCounts: [2], + heapMiB: [1024], + cacheCalibration: { + version: 2, + ...oversizedPayload, + calibrationId: sha256Canonical(oversizedPayload), + }, + heapLimitBytesByHeapMiB: { '1024': 1024 * MIB }, + }), /CALIBRATED_CAPACITY_INSUFFICIENT/); + assert.throws(() => makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: { + ...manifest().provisionClone, + purpose: 'hostile-preflight', + }, + tenantCounts: [2], + cacheCalibration, + heapLimitBytesByHeapMiB, + }), /MEASUREMENT_PROVISION_CLONE_REQUIRED/); + }); + + it('emits secure shared-listener density arms with one-client runtime pools', () => { + const plan = makePlan({ + manifestFile: '/tmp/pdc/provision.json', + secretsFile: '/tmp/pdc/runtime-secrets.json', + postgresContainer: 'postgres-density', + postgresContainerTemplateFile: '/tmp/pdc/postgres-container-template.json', + postgresContainerTemplateSha256: `sha256:${'1'.repeat(64)}`, + commit: 'a'.repeat(40), + entrySha256: 'b'.repeat(64), + lockfileSha256: 'c'.repeat(64), + databaseContractFingerprint, + blueprintCompatibilityFingerprint: `sha256:${'e'.repeat(64)}`, + manifestSha256: `sha256:${'f'.repeat(64)}`, + provisionClone: manifest().provisionClone, + tenantCounts: [2], + cacheCalibration, + heapLimitBytesByHeapMiB, + arms: DENSITY_TUNING_ARMS, + }); + const shared = plan.arms.filter((arm) => arm.name.includes('-shared-')); + assert.equal(shared.length, 7); + for (const arm of shared) { + const poolIndex = arm.command.indexOf('--runtime-pool-max'); + const maxUsesIndex = arm.command.indexOf('--runtime-pool-max-uses'); + const modeIndex = arm.command.indexOf('--realtime-notification-mode'); + const pollIndex = arm.command.indexOf('--realtime-cursor-poll-ms'); + assert.equal(arm.command[poolIndex + 1], '1'); + assert.equal( + arm.command[maxUsesIndex + 1], + arm.name.includes('-maxuses-1') ? '1' : 'unlimited', + ); + assert.equal(arm.env.PG_POOL_MAX_USES, '0'); + assert.equal(arm.env.PG_POOL_MAX, '1'); + assert.equal(arm.command[modeIndex + 1], 'shared-exact'); + assert.equal(arm.command[pollIndex + 1], '30000'); + assert.equal(arm.env.PG_CACHE_MAX, '16'); + } + assert.ok(plan.arms.every((arm) => arm.env.PG_POOL_MAX === '1')); + assert.ok(plan.arms.every((arm) => arm.env.NODE_OPTIONS === '')); + assert.ok(plan.arms.every((arm) => arm.env.NODE_PATH === '')); + assert.equal(plan.arms[0].env.PG_CACHE_MAX, '14'); + assert.deepEqual(plan.arms.map((arm) => arm.v8Profile), [ + 'stock', + 'stock', + 'stock', + 'stock', + 'optimize-for-size', + 'baseline-optimize-for-size', + 'jitless-optimize-for-size', + 'optimize-for-size', + ]); + assert.equal( + plan.arms.find((arm) => arm.name === 'physical-db-shared-stock') + .env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE, + '100', + ); + assert.equal( + plan.arms.find((arm) => arm.name === 'physical-db-shared-no-prepare') + .env.DATAPLAN_PG_PREPARED_STATEMENT_CACHE_SIZE, + '0', + ); + }); + + it('computes the qualifying capacity proof without runtime status inputs', () => { + const proof = makeCacheCapacityProofByHeapMiB({ + cacheCalibration, + databaseContractFingerprint, + tenantCounts: [2], + heapMiB: [1024], + heapLimitBytesByHeapMiB: { '1024': 1024 * MIB }, + })['1024']; + assert.equal(proof.admissionMode, 'preserve-resident'); + assert.equal(proof.requiredResidentInstances, 6); + assert.ok(proof.budgetCapacity > proof.requiredResidentInstances); + }); + + it('normalizes nondeterministic pg_dump guard and version lines before fingerprinting', () => { + const left = normalizeSchemaDump([ + '-- Dumped from database version 17.1', + '-- Dumped by pg_dump version 17.1', + '\\restrict random-left', + 'CREATE TABLE ctf_a.example(id integer);', + '\\unrestrict random-left', + ].join('\n')); + const right = normalizeSchemaDump([ + '-- Dumped from database version 17.2', + '-- Dumped by pg_dump version 17.2', + '\\restrict random-right', + 'CREATE TABLE ctf_a.example(id integer);', + '\\unrestrict random-right', + ].join('\n')); + assert.equal(left, right); + }); + + it('fingerprints build-visible dependency schemas and normalizes equivalent role ACLs', () => { + assert.ok(DEFAULT_CANONICAL_SCHEMAS.includes('ctf_extensions')); + assert.ok(DEFAULT_CANONICAL_SCHEMAS.includes('jwt_private')); + const left = normalizeSchemaDump( + 'GRANT SELECT ON TABLE ctf_a.item TO pdc_test_c0001_a;\n', + { pdc_test_c0001_a: '__runtime_a__' }, + ); + const right = normalizeSchemaDump( + 'GRANT SELECT ON TABLE ctf_a.item TO pdc_test_c0002_a;\n', + { pdc_test_c0002_a: '__runtime_a__' }, + ); + assert.equal(left, right); + assert.match(left, /__runtime_a__/); + }); +}); diff --git a/research/graphile-density/physical-database-density/measurement-attestation.cjs b/research/graphile-density/physical-database-density/measurement-attestation.cjs new file mode 100644 index 0000000000..d4a013e57d --- /dev/null +++ b/research/graphile-density/physical-database-density/measurement-attestation.cjs @@ -0,0 +1,558 @@ +'use strict'; + +const { execFileSync, spawnSync } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + FIXTURE_ID, + loadProvision, +} = require('./lib.cjs'); +const { + parseArgs, + parsePositiveInteger, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); +const { + inspectCustomerContract, + provisionAttestationSetSha256, +} = require('./provision.cjs'); +const { + buildLiveCloneAuditSql, + validateLiveCloneAudit, +} = require('./unsafe-runtime-startup-probe.cjs'); +const { + postgresSettingsFromCommand, + validateContainerTemplate, + validateRunningContainerAgainstTemplate, +} = require('./prepare-measurement-run.cjs'); + +const ATTESTATION_KIND = 'physical-density-measurement-attestation-v1'; +const SHA256 = /^sha256:[a-f0-9]{64}$/; +const CONTAINER_ID = /^[a-f0-9]{64}$/; +const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost']); +const START_TOLERANCE_MS = 0; + +const canonicalize = (value) => { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [ + key, + canonicalize(value[key]), + ])); +}; + +const canonicalSha256 = (value) => `sha256:${crypto.createHash('sha256') + .update(JSON.stringify(canonicalize(value))) + .digest('hex')}`; + +const readRegularFile = (file) => { + const absolute = path.resolve(file); + const before = fs.lstatSync(absolute); + if (before.isSymbolicLink() || !before.isFile()) { + throw new Error('PDCF_MEASUREMENT_EVIDENCE_FILE_INVALID'); + } + const descriptor = fs.openSync( + absolute, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + try { + const opened = fs.fstatSync(descriptor); + if ( + !opened.isFile() + || opened.dev !== before.dev + || opened.ino !== before.ino + ) { + throw new Error('PDCF_MEASUREMENT_EVIDENCE_FILE_INVALID'); + } + return fs.readFileSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +}; + +const bufferSha256 = (value) => `sha256:${crypto.createHash('sha256') + .update(value) + .digest('hex')}`; + +const writeImmutableJson = (file, value) => { + const absolute = path.resolve(file); + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + const temporary = `${absolute}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`; + try { + fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { + flag: 'wx', + mode: 0o644, + }); + fs.linkSync(temporary, absolute); + } finally { + try { fs.unlinkSync(temporary); } catch { /* Preserve the primary error. */ } + } +}; + +const exactCanonical = (left, right) => + JSON.stringify(canonicalize(left)) === JSON.stringify(canonicalize(right)); + +const runPsqlJson = ({ database, sql, environment = process.env }) => { + const result = spawnSync('psql', [ + '--no-psqlrc', + '--no-align', + '--tuples-only', + '--quiet', + '--set=ON_ERROR_STOP=1', + '--dbname', database, + ], { + cwd: __dirname, + env: environment, + encoding: 'utf8', + input: `${sql}\n`, + maxBuffer: 16 * 1024 * 1024, + timeout: 120_000, + }); + if (result.status !== 0) throw new Error('PDCF_MEASUREMENT_ATTESTATION_SQL_FAILED'); + const output = String(result.stdout ?? '').trim(); + if (!output || output.includes('\n')) { + throw new Error('PDCF_MEASUREMENT_ATTESTATION_SQL_RESULT_INVALID'); + } + try { + return JSON.parse(output); + } catch { + throw new Error('PDCF_MEASUREMENT_ATTESTATION_SQL_RESULT_INVALID'); + } +}; + +const inspectDockerContainer = (container) => { + const output = execFileSync('docker', ['inspect', container], { + encoding: 'utf8', + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }); + const records = JSON.parse(output); + if (!Array.isArray(records) || records.length !== 1) { + throw new Error('PDCF_MEASUREMENT_CONTAINER_INSPECT_INVALID'); + } + return records[0]; +}; + +const inspectContainerCgroup = (container) => { + const script = [ + 'set -eu', + 'test -r /sys/fs/cgroup/memory.current', + 'test -r /sys/fs/cgroup/memory.events', + 'printf "membership="', + 'cat /proc/1/cgroup', + 'printf "mount="', + 'stat -c "%d:%i" /sys/fs/cgroup', + ].join('\n'); + const dockerEnvironment = { + PATH: process.env.PATH ?? '/usr/bin:/bin', + ...(process.env.DOCKER_HOST ? { DOCKER_HOST: process.env.DOCKER_HOST } : {}), + ...(process.env.DOCKER_CONTEXT + ? { DOCKER_CONTEXT: process.env.DOCKER_CONTEXT } + : {}), + }; + const output = execFileSync('docker', [ + 'exec', + container, + '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin', + 'sh', '-ceu', script, + ], { + encoding: 'utf8', + timeout: 30_000, + env: dockerEnvironment, + }); + return { + version: 1, + source: 'container-cgroup-v2', + identitySha256: canonicalSha256(output.trim()), + }; +}; + +const clusterAuditSql = (settingNames = []) => ` +SELECT pg_catalog.jsonb_build_object( + 'systemIdentifier', system_identifier::text, + 'postmasterStartedAt', pg_catalog.pg_postmaster_start_time(), + 'serverVersionNum', pg_catalog.current_setting('server_version_num'), + 'databases', ( + SELECT pg_catalog.jsonb_agg(datname ORDER BY datname) + FROM pg_catalog.pg_database + WHERE NOT datistemplate + ), + 'settings', ( + SELECT pg_catalog.jsonb_object_agg( + name, + pg_catalog.jsonb_build_object('setting', setting, 'source', source) + ORDER BY name + ) + FROM pg_catalog.pg_settings + WHERE name = ANY(ARRAY[${settingNames.map((name) => `'${name}'`).join(', ')}]::text[]) + ) +)::text +FROM pg_catalog.pg_control_system(); +`; + +const validateContainer = ({ + inspection, + container, + environment, + notBeforeEpochMs, +}) => { + const id = inspection?.Id; + const name = String(inspection?.Name ?? '').replace(/^\//, ''); + const startedAt = Date.parse(inspection?.State?.StartedAt ?? ''); + const createdAt = Date.parse(inspection?.Created ?? ''); + if ( + !CONTAINER_ID.test(id ?? '') + || name !== container + || inspection?.State?.Running !== true + || !Number.isSafeInteger(startedAt) + || !Number.isSafeInteger(createdAt) + || startedAt < createdAt + ) { + throw new Error('PDCF_MEASUREMENT_CONTAINER_IDENTITY_INVALID'); + } + const pgHost = environment.PGHOST ?? 'localhost'; + const pgPort = String(environment.PGPORT ?? '5432'); + if (!LOOPBACK_HOSTS.has(pgHost)) { + throw new Error('PDCF_MEASUREMENT_POSTGRES_LOOPBACK_REQUIRED'); + } + const bindings = inspection?.NetworkSettings?.Ports?.['5432/tcp']; + if ( + !Array.isArray(bindings) + || !bindings.some((binding) => String(binding?.HostPort) === pgPort) + ) { + throw new Error('PDCF_MEASUREMENT_CONTAINER_PORT_MISMATCH'); + } + return { + id, + name, + imageId: inspection.Image, + createdAt: new Date(createdAt).toISOString(), + startedAt: new Date(startedAt).toISOString(), + freshForRun: startedAt >= notBeforeEpochMs, + }; +}; + +const validateRunBinding = (run) => { + if ( + typeof run?.arm !== 'string' + || !run.arm + || !Number.isSafeInteger(run.heapMiB) + || run.heapMiB <= 0 + || !Number.isSafeInteger(run.customerCount) + || run.customerCount <= 0 + || !Number.isSafeInteger(run.repetition) + || run.repetition <= 0 + || !Number.isSafeInteger(run.runOrderIndex) + || run.runOrderIndex <= 0 + || !SHA256.test(run.planSha256 ?? '') + || !SHA256.test(run.fleetSha256 ?? '') + ) { + throw new Error('PDCF_MEASUREMENT_RUN_BINDING_INVALID'); + } + return run; +}; + +const validateMeasurementAttestation = (attestation, expected = {}) => { + if ( + attestation?.version !== 1 + || attestation.kind !== ATTESTATION_KIND + || !attestation.payload + || !SHA256.test(attestation.payloadSha256 ?? '') + || canonicalSha256(attestation.payload) !== attestation.payloadSha256 + || !SHA256.test(attestation.payload.epochId ?? '') + || typeof attestation.payload.freshness?.freshContainerForRun !== 'boolean' + || attestation.payload.freshness?.cgroupV2Verified !== true + || attestation.payload.catalogCacheState !== 'warmed-by-live-contract-audit' + ) { + throw new Error('PDCF_MEASUREMENT_ATTESTATION_INVALID'); + } + validateRunBinding(attestation.payload.run); + for (const [key, value] of Object.entries(expected)) { + if (!exactCanonical(attestation.payload[key], value)) { + throw new Error(`PDCF_MEASUREMENT_ATTESTATION_MISMATCH:${key}`); + } + } + return attestation; +}; + +const attestMeasurementRun = ({ + manifestFile, + secretsFile, + postgresContainer, + containerTemplateFile, + expectedContainerTemplateSha256, + run, + notBeforeEpochMs, + outputFile, + environment = process.env, +}, dependencies = {}) => { + validateRunBinding(run); + if (!Number.isSafeInteger(notBeforeEpochMs) || notBeforeEpochMs <= 0) { + throw new Error('PDCF_MEASUREMENT_NOT_BEFORE_INVALID'); + } + const containerTemplateBytes = readRegularFile(containerTemplateFile); + if ( + !SHA256.test(expectedContainerTemplateSha256 ?? '') + || bufferSha256(containerTemplateBytes) !== expectedContainerTemplateSha256 + ) { + throw new Error('PDCF_MEASUREMENT_CONTAINER_TEMPLATE_MISMATCH'); + } + const containerTemplate = validateContainerTemplate(JSON.parse( + containerTemplateBytes.toString('utf8'), + )); + if (containerTemplate.containerName !== postgresContainer) { + throw new Error('PDCF_MEASUREMENT_CONTAINER_TEMPLATE_MISMATCH'); + } + const manifestBytesBeforeLoad = readRegularFile(manifestFile); + const provision = (dependencies.loadProvision ?? loadProvision)( + manifestFile, + secretsFile, + ); + const { manifest } = provision; + const manifestBytesAfterLoad = readRegularFile(manifestFile); + const manifestSha256 = bufferSha256(manifestBytesBeforeLoad); + if ( + bufferSha256(manifestBytesAfterLoad) !== manifestSha256 + || !exactCanonical( + JSON.parse(manifestBytesAfterLoad.toString('utf8')), + manifest, + ) + ) { + throw new Error('PDCF_MEASUREMENT_MANIFEST_CHANGED_DURING_AUDIT'); + } + if ( + manifest.provisionClone?.purpose !== 'measurement' + || run.customerCount !== manifest.customers.length + ) { + throw new Error('PDCF_MEASUREMENT_PROVISION_MISMATCH'); + } + const inspection = (dependencies.inspectDockerContainer ?? inspectDockerContainer)( + postgresContainer, + ); + validateRunningContainerAgainstTemplate(inspection, containerTemplate); + const container = validateContainer({ + inspection, + container: postgresContainer, + environment, + notBeforeEpochMs, + }); + const cgroup = (dependencies.inspectContainerCgroup ?? inspectContainerCgroup)( + postgresContainer, + ); + if (!SHA256.test(cgroup?.identitySha256 ?? '')) { + throw new Error('PDCF_MEASUREMENT_CGROUP_IDENTITY_INVALID'); + } + const queryJson = dependencies.runPsqlJson ?? runPsqlJson; + const maintenanceDatabase = environment.PGDATABASE ?? 'postgres'; + const expectedPostgresSettings = postgresSettingsFromCommand( + containerTemplate.postgresCommand, + ); + const cluster = queryJson({ + database: maintenanceDatabase, + sql: clusterAuditSql(Object.keys(expectedPostgresSettings).sort()), + environment, + }); + const postmasterStartedAtMs = Date.parse(cluster?.postmasterStartedAt ?? ''); + const livePostgresSettings = cluster?.settings; + const expectedSettingNames = Object.keys(expectedPostgresSettings).sort(); + if ( + typeof cluster?.systemIdentifier !== 'string' + || !/^\d+$/.test(cluster.systemIdentifier) + || !Number.isSafeInteger(postmasterStartedAtMs) + || Math.abs(postmasterStartedAtMs - Date.parse(container.startedAt)) > 120_000 + || !Array.isArray(cluster.databases) + || !livePostgresSettings + || JSON.stringify(Object.keys(livePostgresSettings).sort()) + !== JSON.stringify(expectedSettingNames) + || expectedSettingNames.some((name) => + typeof livePostgresSettings[name]?.setting !== 'string' + || livePostgresSettings[name].source !== 'command line' + ) + || Number(livePostgresSettings.max_connections?.setting) + !== Number(expectedPostgresSettings.max_connections) + ) { + throw new Error('PDCF_MEASUREMENT_CLUSTER_IDENTITY_INVALID'); + } + const expectedDatabases = [maintenanceDatabase, ...manifest.customers.map( + (customer) => customer.database + )].sort(); + if (JSON.stringify(cluster.databases) !== JSON.stringify(expectedDatabases)) { + throw new Error('PDCF_MEASUREMENT_DATABASE_INVENTORY_INVALID'); + } + + const inspectContract = dependencies.inspectCustomerContract + ?? inspectCustomerContract; + const customerAudits = manifest.customers.map((customer) => { + const liveContract = inspectContract({ + customer, + canonicalSchemas: manifest.canonicalSchemas, + environment, + }); + if ( + liveContract.databaseContractFingerprint !== customer.databaseContractFingerprint + || !exactCanonical( + liveContract.structuralFingerprints, + customer.structuralFingerprints, + ) + ) { + throw new Error(`PDCF_MEASUREMENT_LIVE_CONTRACT_MISMATCH:${customer.id}`); + } + const rawCloneAudit = queryJson({ + database: customer.database, + sql: buildLiveCloneAuditSql(), + environment, + }); + validateLiveCloneAudit(rawCloneAudit, { manifest, customer }); + return { + customerId: customer.id, + database: customer.database, + databaseContractFingerprint: liveContract.databaseContractFingerprint, + structuralFingerprints: liveContract.structuralFingerprints, + roleSafetyProfileSha256: canonicalSha256(liveContract.roleSafetyProfile), + notificationRoleSafetyProfileSha256: + canonicalSha256(liveContract.notificationRoleSafetyProfile), + extensionVersions: liveContract.extensionVersions, + cloneAttestationSha256: rawCloneAudit.sha256, + cloneNonceSha256: canonicalSha256(rawCloneAudit.nonce), + }; + }).sort((left, right) => left.customerId.localeCompare(right.customerId)); + const cloneAttestationSetSha256 = provisionAttestationSetSha256( + manifest.customers, + ); + if (cloneAttestationSetSha256 !== manifest.provisionClone.attestationSetSha256) { + throw new Error('PDCF_MEASUREMENT_CLONE_SET_MISMATCH'); + } + const immutableEpoch = { + dockerContainerId: container.id, + dockerStartedAt: container.startedAt, + containerConfigurationSha256: canonicalSha256({ + imageId: containerTemplate.imageId, + entrypoint: containerTemplate.entrypoint, + pgHost: containerTemplate.pgHost, + pgPort: containerTemplate.pgPort, + postgresCommand: containerTemplate.postgresCommand, + resourceLimits: containerTemplate.resourceLimits, + }), + cgroupIdentitySha256: cgroup.identitySha256, + postgresSystemIdentifier: cluster.systemIdentifier, + postgresStartedAt: new Date(postmasterStartedAtMs).toISOString(), + cloneId: manifest.provisionClone.id, + cloneAttestationSetSha256, + cloneNonceSetSha256: canonicalSha256(customerAudits.map((audit) => ({ + customerId: audit.customerId, + cloneNonceSha256: audit.cloneNonceSha256, + }))), + liveContractSetSha256: canonicalSha256(customerAudits.map((audit) => ({ + customerId: audit.customerId, + databaseContractFingerprint: audit.databaseContractFingerprint, + structuralFingerprint: audit.structuralFingerprints.combined.sha256, + }))), + }; + const payload = { + fixture: FIXTURE_ID, + observedAt: new Date().toISOString(), + run, + manifestSha256, + containerTemplateSha256: expectedContainerTemplateSha256, + provisionClone: manifest.provisionClone, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint, + canonicalStructuralFingerprint: + manifest.canonicalStructuralFingerprint?.combined?.sha256 ?? null, + container, + cgroup, + postgres: { + systemIdentifier: cluster.systemIdentifier, + postmasterStartedAt: new Date(postmasterStartedAtMs).toISOString(), + serverVersionNum: cluster.serverVersionNum, + databases: cluster.databases, + settings: cluster.settings, + }, + customerAudits, + immutableEpoch, + epochId: canonicalSha256(immutableEpoch), + freshness: { + freshContainerForRun: container.freshForRun, + cgroupV2Verified: true, + notBeforeEpochMs, + startToleranceMs: START_TOLERANCE_MS, + }, + // The required full pg_dump/ACL audit warms PostgreSQL catalogs before the + // Graphile timer starts. Results using this evidence must not call their + // build timing pristine-catalog cold start. + catalogCacheState: 'warmed-by-live-contract-audit', + }; + const attestation = validateMeasurementAttestation({ + version: 1, + kind: ATTESTATION_KIND, + payload, + payloadSha256: canonicalSha256(payload), + }); + if (outputFile) writeImmutableJson(outputFile, attestation); + return attestation; +}; + +const main = () => { + const args = parseArgs(process.argv.slice(2)); + const run = { + arm: requireString(args, 'arm'), + heapMiB: parsePositiveInteger(requireString(args, 'heap-mib'), 'heap-mib'), + customerCount: parsePositiveInteger( + requireString(args, 'customers'), + 'customers', + ), + repetition: parsePositiveInteger( + requireString(args, 'repetition'), + 'repetition', + ), + runOrderIndex: parsePositiveInteger( + requireString(args, 'run-order-index'), + 'run-order-index', + ), + planSha256: requireString(args, 'plan-sha256'), + fleetSha256: requireString(args, 'fleet-sha256'), + }; + const result = attestMeasurementRun({ + manifestFile: path.resolve(requireString(args, 'manifest')), + secretsFile: path.resolve(requireString(args, 'secrets')), + postgresContainer: requireString(args, 'postgres-container'), + containerTemplateFile: path.resolve(requireString(args, 'container-template')), + expectedContainerTemplateSha256: requireString( + args, + 'expected-container-template-sha256', + ), + run, + notBeforeEpochMs: Number(requireString(args, 'not-before-epoch-ms')), + outputFile: path.resolve(requireString(args, 'out')), + }); + process.stdout.write(`${JSON.stringify({ + status: 'attested', + epochId: result.payload.epochId, + payloadSha256: result.payloadSha256, + })}\n`); +}; + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} + +module.exports = { + ATTESTATION_KIND, + attestMeasurementRun, + canonicalSha256, + clusterAuditSql, + inspectContainerCgroup, + inspectDockerContainer, + runPsqlJson, + validateContainer, + validateMeasurementAttestation, + validateRunBinding, + writeImmutableJson, +}; diff --git a/research/graphile-density/physical-database-density/measurement-attestation.test.cjs b/research/graphile-density/physical-database-density/measurement-attestation.test.cjs new file mode 100644 index 0000000000..e7ff11b571 --- /dev/null +++ b/research/graphile-density/physical-database-density/measurement-attestation.test.cjs @@ -0,0 +1,175 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { + attestMeasurementRun, + validateMeasurementAttestation, +} = require('./measurement-attestation.cjs'); +const { + captureContainerTemplate, +} = require('./prepare-measurement-run.cjs'); +const { + provisionAttestationSetSha256, + provisionAttestationSha256, +} = require('./provision.cjs'); + +const digest = (character) => `sha256:${character.repeat(64)}`; +const fileSha256 = (file) => `sha256:${crypto.createHash('sha256') + .update(fs.readFileSync(file)) + .digest('hex')}`; + +const containerInspection = () => ({ + Id: '1'.repeat(64), + Name: '/postgres-density-exact', + Image: digest('2'), + Created: '2026-08-02T00:00:00.001Z', + State: { + Running: true, + StartedAt: '2026-08-02T00:00:00.010Z', + }, + Config: { + Cmd: ['postgres', '-c', 'max_connections=160'], + Labels: { + 'io.constructive.graphile-density.fixture': + 'physical-database-density-v1', + 'io.constructive.graphile-density.prefix': 'pdc_test', + 'io.constructive.graphile-density.purpose': 'measurement', + }, + }, + HostConfig: { + Memory: 1024 ** 3, + MemorySwap: 1024 ** 3, + NanoCpus: 1_000_000_000, + ShmSize: 128 * 1024 ** 2, + }, + NetworkSettings: { + Ports: { '5432/tcp': [{ HostIp: '127.0.0.1', HostPort: '55432' }] }, + }, +}); + +describe('physical measurement attestation', () => { + it('audits the live DDL/ACL contract outside Node and binds a fresh run epoch', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-measurement-')); + const manifestFile = path.join(directory, 'provision.json'); + const secretsFile = path.join(directory, 'runtime-secrets.json'); + const templateFile = path.join(directory, 'container-template.json'); + const outputFile = path.join(directory, 'attestation.json'); + const nonce = '3'.repeat(64); + const customer = { + id: 'physical-customer-0001', + database: 'pdc_test_db_0001', + provisionAttestation: { + version: 1, + cloneId: 'measurement-run-clone', + purpose: 'measurement', + sha256: provisionAttestationSha256({ + cloneId: 'measurement-run-clone', + runPurpose: 'measurement', + customerId: 'physical-customer-0001', + database: 'pdc_test_db_0001', + nonce, + }), + }, + databaseContractFingerprint: digest('4'), + structuralFingerprints: { combined: { sha256: digest('5'), bytes: 100 } }, + }; + const manifest = { + canonicalSchemas: ['ctf_a'], + canonicalDatabaseContractFingerprint: digest('4'), + canonicalStructuralFingerprint: customer.structuralFingerprints, + provisionClone: { + version: 1, + id: 'measurement-run-clone', + purpose: 'measurement', + attestationSetSha256: provisionAttestationSetSha256([customer]), + }, + customers: [customer], + }; + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + fs.writeFileSync(secretsFile, '{}', { mode: 0o600 }); + const template = captureContainerTemplate({ + inspection: containerInspection(), + container: 'postgres-density-exact', + prefix: 'pdc_test', + pgHost: '127.0.0.1', + pgPort: 55432, + minimumMaxConnections: 120, + }); + fs.writeFileSync(templateFile, JSON.stringify(template)); + const run = { + arm: 'candidate', + heapMiB: 2048, + customerCount: 1, + repetition: 1, + runOrderIndex: 2, + planSha256: digest('6'), + fleetSha256: digest('7'), + }; + const liveContract = { + databaseContractFingerprint: customer.databaseContractFingerprint, + structuralFingerprints: customer.structuralFingerprints, + roleSafetyProfile: { safe: true }, + notificationRoleSafetyProfile: { safe: true }, + extensionVersions: [{ name: 'vector', version: '1.0' }], + }; + const attestation = attestMeasurementRun({ + manifestFile, + secretsFile, + postgresContainer: 'postgres-density-exact', + containerTemplateFile: templateFile, + expectedContainerTemplateSha256: fileSha256(templateFile), + run, + notBeforeEpochMs: Date.parse('2026-08-02T00:00:00.000Z'), + outputFile, + environment: { + PGHOST: '127.0.0.1', + PGPORT: '55432', + PGDATABASE: 'postgres', + }, + }, { + loadProvision: () => ({ manifest }), + inspectDockerContainer: containerInspection, + inspectContainerCgroup: () => ({ + version: 1, + source: 'container-cgroup-v2', + identitySha256: digest('8'), + }), + inspectCustomerContract: () => liveContract, + runPsqlJson: ({ database }) => database === 'postgres' + ? { + systemIdentifier: '7421234567890123456', + postmasterStartedAt: '2026-08-02T00:00:00.020Z', + serverVersionNum: '170000', + databases: ['pdc_test_db_0001', 'postgres'], + settings: { + max_connections: { setting: '160', source: 'command line' }, + }, + } + : { + version: 1, + kind: 'unsafe-runtime-live-clone-audit-v1', + cloneId: customer.provisionAttestation.cloneId, + purpose: customer.provisionAttestation.purpose, + customerId: customer.id, + database: customer.database, + nonce, + sha256: customer.provisionAttestation.sha256, + }, + }); + assert.equal(attestation.payload.freshness.freshContainerForRun, true); + assert.equal(attestation.payload.customerAudits.length, 1); + assert.equal( + attestation.payload.immutableEpoch.cloneAttestationSetSha256, + manifest.provisionClone.attestationSetSha256, + ); + assert.match(attestation.payload.epochId, /^sha256:[a-f0-9]{64}$/); + assert.doesNotThrow(() => validateMeasurementAttestation(attestation)); + assert.deepEqual(JSON.parse(fs.readFileSync(outputFile, 'utf8')), attestation); + }); +}); diff --git a/research/graphile-density/physical-database-density/physical-hostile-preflight.cjs b/research/graphile-density/physical-database-density/physical-hostile-preflight.cjs new file mode 100644 index 0000000000..56ed0603a7 --- /dev/null +++ b/research/graphile-density/physical-database-density/physical-hostile-preflight.cjs @@ -0,0 +1,831 @@ +'use strict'; + +const { execFileSync } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + REPO_ROOT, + TENANTS, + assertCredentialFree, + assertLoopbackBaseUrl, + parseArgs, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); +const { + assertCustomerPathPrefix, + assertIdentity, + control, + identityOperation, + postGraphql, + requestJson, + runHostileValidation, +} = require('../complete-tenant-fixture/hostile-validation.cjs'); +const { + FIXTURE_ID, + atomicWriteJson, +} = require('./lib.cjs'); +const { provisionAttestationSetSha256 } = require('./provision.cjs'); +const { + ADMISSION_SCOPE, + CLEANUP_AUDIT_KIND, + PROBE_CAPABILITIES, + PROBE_KIND, + expectedAuditedProfiles, + loadPrivateProvision, + runUnsafeRuntimeStartupMatrix, +} = require('./unsafe-runtime-startup-probe.cjs'); + +const FIXTURE_DIR = __dirname; +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/; +const VALIDATOR_ENTRY_FILES = Object.freeze([ + 'research/graphile-density/physical-database-density/physical-hostile-preflight.cjs', + 'research/graphile-density/physical-database-density/lib.cjs', + 'research/graphile-density/complete-tenant-fixture/hostile-validation.cjs', + 'research/graphile-density/complete-tenant-fixture/generate-inputs.cjs', + 'research/graphile-density/complete-tenant-fixture/lib.cjs', + 'research/graphile-density/complete-tenant-fixture/server.cjs', + 'research/graphile-density/complete-tenant-fixture/schema.sql', + 'research/graphile-density/physical-database-density/server.cjs', + 'research/graphile-density/physical-database-density/physical-identity.sql', + 'research/graphile-density/physical-database-density/provision.cjs', + 'research/graphile-density/physical-database-density/provision-attestation.sql', + 'research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.cjs', +]); + +const sha256Buffer = (value) => `sha256:${crypto.createHash('sha256') + .update(value) + .digest('hex')}`; + +const fileSha256 = (file) => sha256Buffer(fs.readFileSync(file)); + +const canonicalize = (value) => { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [ + key, + canonicalize(value[key]), + ])); +}; + +const canonicalSha256 = (value) => sha256Buffer(JSON.stringify(canonicalize(value))); +const canonicalEqual = (left, right) => + JSON.stringify(canonicalize(left)) === JSON.stringify(canonicalize(right)); + +const exactKeys = (value, expected) => { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + return JSON.stringify(Object.keys(value).sort()) === JSON.stringify([...expected].sort()); +}; + +const requireSha256 = (value, code) => { + if (!SHA256_PATTERN.test(value ?? '')) throw new Error(code); + return value; +}; + +const requireArtifactLabel = (value, code) => { + if ( + typeof value !== 'string' + || !/^[a-z0-9][a-z0-9._-]{0,127}$/i.test(value) + ) { + throw new Error(code); + } + return value; +}; + +const assertOutputDoesNotAliasInputs = (outputFile, inputFiles) => { + if (!outputFile) return null; + const absoluteOutputFile = path.resolve(outputFile); + let outputStat = null; + try { + outputStat = fs.statSync(absoluteOutputFile); + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + for (const inputFile of inputFiles) { + const absoluteInputFile = path.resolve(inputFile); + if (absoluteInputFile === absoluteOutputFile) { + throw new Error('PDCF_HOSTILE_OUTPUT_ALIASES_INPUT'); + } + if (outputStat) { + const inputStat = fs.statSync(absoluteInputFile); + if (inputStat.dev === outputStat.dev && inputStat.ino === outputStat.ino) { + throw new Error('PDCF_HOSTILE_OUTPUT_ALIASES_INPUT'); + } + } + } + return absoluteOutputFile; +}; + +const expectedLiveProvisionAttestation = (manifest, customer) => ({ + version: 1, + cloneId: manifest.provisionClone.id, + purpose: manifest.provisionClone.purpose, + customerId: customer.id, + database: customer.database, + sha256: customer.provisionAttestation.sha256, + verified: true, +}); + +const validatePreflightManifest = (manifest, preflightCloneId) => { + if ( + !exactKeys( + manifest.provisionClone, + ['version', 'id', 'purpose', 'attestationSetSha256'], + ) + || + manifest.provisionClone?.version !== 1 + || manifest.provisionClone.id !== preflightCloneId + || manifest.provisionClone.purpose !== 'hostile-preflight' + || !SHA256_PATTERN.test(manifest.provisionClone.attestationSetSha256 ?? '') + || provisionAttestationSetSha256(manifest.customers) + !== manifest.provisionClone.attestationSetSha256 + || !SHA256_PATTERN.test( + manifest.canonicalStructuralFingerprint?.combined?.sha256 ?? '' + ) + || !SHA256_PATTERN.test(manifest.canonicalDatabaseContractFingerprint ?? '') + ) { + throw new Error('PDCF_HOSTILE_PREFLIGHT_MANIFEST_MISMATCH'); + } + for (const customer of manifest.customers) { + if ( + !exactKeys( + customer.provisionAttestation, + ['version', 'cloneId', 'purpose', 'sha256'], + ) + || + customer.provisionAttestation?.version !== 1 + || customer.provisionAttestation.cloneId !== preflightCloneId + || customer.provisionAttestation.purpose !== 'hostile-preflight' + || !SHA256_PATTERN.test(customer.provisionAttestation.sha256 ?? '') + || !SHA256_PATTERN.test(customer.structuralFingerprints?.combined?.sha256 ?? '') + || !SHA256_PATTERN.test(customer.databaseContractFingerprint ?? '') + ) { + throw new Error(`PDCF_HOSTILE_CUSTOMER_MANIFEST_INVALID:${customer.id}`); + } + } + return manifest; +}; + +const validatePhysicalStatus = (status, { + manifest, + arm, + mode, + preflightCloneId = manifest.provisionClone?.id, +}) => { + if ( + status?.version !== 1 + || status.fixture !== FIXTURE_ID + || status.arm !== arm + || status.introspectionMode !== mode + || status.introspectionClientReleaseMode !== 'destroy' + || status.runPurpose !== 'hostile-preflight' + || status.cloneId !== preflightCloneId + || status.provisionClone?.version !== 1 + || status.provisionClone.id !== preflightCloneId + || status.provisionClone.purpose !== 'hostile-preflight' + || status.provisionClone.attestationSetSha256 + !== manifest.provisionClone?.attestationSetSha256 + || status.provisionClone.verified !== true + || !canonicalEqual( + status.canonicalStructuralFingerprint, + manifest.canonicalStructuralFingerprint, + ) + || status.canonicalDatabaseContractFingerprint + !== manifest.canonicalDatabaseContractFingerprint + || status.realtime?.managersExpected !== manifest.customers.length * TENANTS.length + || status.realtime?.connectionsExpected !== manifest.customers.length * TENANTS.length + || status.realtime?.transportsExpected !== manifest.customers.length * TENANTS.length + || status.realtime?.notificationMode !== 'dedicated' + || status.runtimePoolMax !== 2 + || status.runtimePoolMaxUses !== null + ) { + throw new Error('PDCF_HOSTILE_PHYSICAL_STATUS_MISMATCH'); + } + requireSha256( + status.blueprintCompatibilityFingerprint, + 'PDCF_HOSTILE_BLUEPRINT_FINGERPRINT_REQUIRED', + ); + requireSha256( + manifest.canonicalDatabaseContractFingerprint, + 'PDCF_HOSTILE_CANONICAL_CONTRACT_REQUIRED', + ); + if (!Array.isArray(status.customers)) { + throw new Error('PDCF_HOSTILE_CUSTOMER_SET_MISMATCH'); + } + if ( + status.customers.length !== manifest.customers.length + || status.customers.some((observed, index) => { + const customer = manifest.customers[index]; + return observed.id !== customer.id + || observed.physicalDatabase !== customer.database + || !canonicalEqual( + observed.provisionAttestation, + expectedLiveProvisionAttestation(manifest, customer), + ) + || !canonicalEqual( + observed.structuralFingerprints, + customer.structuralFingerprints, + ) + || observed.canonicalStructuralFingerprint + !== customer.structuralFingerprints?.combined?.sha256 + || observed.databaseContractFingerprint !== customer.databaseContractFingerprint + || observed.contractVerification !== 'live-recomputed'; + }) + ) { + throw new Error('PDCF_HOSTILE_CUSTOMER_SET_MISMATCH'); + } + return status; +}; + +const validateChildStatus = (status, { customer, manifest, arm, mode }) => { + const tenantIds = TENANTS.map((tenant) => tenant.id); + if ( + status?.version !== 1 + || status.fixture !== 'complete-tenant-abc-v1' + || status.arm !== arm + || status.introspectionMode !== mode + || status.introspectionClientReleaseMode !== 'destroy' + || status.releaseBuildStateAfterValidation !== true + || status.physicalDatabase !== customer.physicalIdentity + || status.runPurpose !== 'hostile-preflight' + || !canonicalEqual( + status.provisionAttestation, + expectedLiveProvisionAttestation(manifest, customer), + ) + || status.physicalIsolation !== 'dedicated-login-and-pool-per-tenant' + || status.sharedRuntimePool !== false + || status.runtimePoolMax !== 2 + || status.runtimePoolMaxUses !== null + || status.enableRealtime !== true + || status.realtimeNotificationMode !== 'dedicated' + || status.realtimeCursorPollIntervalMs !== 5_000 + || status.realtimeCursorHeartbeatIntervalMs !== 30_000 + || !exactKeys(status.realtimeSchemas, tenantIds) + || status.controlAvailable !== true + || status.runtimeSafety?.passed !== true + || status.runtimeSafety?.rolesDistinct !== true + || status.liveIdentityScope !== 'process-local-keyed-hmac-v1' + || !/^graphile-configuration:ctf:v1:[a-f0-9]{64}$/.test( + status.configurationIdentity ?? '' + ) + || status.contractEvidence?.version !== 1 + || status.contractEvidence?.credentialFree !== true + || status.contractEvidence?.configurationIdentity + !== status.configurationIdentity + || !exactKeys(status.runtimePoolIdentities, tenantIds) + || !exactKeys(status.buildContracts, tenantIds) + || !exactKeys(status.builds?.byTenant, tenantIds) + ) { + throw new Error(`PDCF_HOSTILE_CHILD_STATUS_MISMATCH:${customer.id}`); + } + requireSha256( + status.runtimeArtifactFingerprint, + `PDCF_HOSTILE_RUNTIME_FINGERPRINT_REQUIRED:${customer.id}`, + ); + for (const tenant of TENANTS) { + if (status.realtimeSchemas[tenant.id] !== `${tenant.schema}_realtime`) { + throw new Error(`PDCF_HOSTILE_CHILD_CONTRACT_MISMATCH:${customer.id}:${tenant.id}`); + } + } + for (const tenantId of tenantIds) { + const poolEvidence = status.contractEvidence?.runtimePools?.[tenantId]; + const buildEvidence = status.contractEvidence?.graphileBuilds?.[tenantId]; + const binding = status.runtimeBindings?.[tenantId]; + if ( + !/^pg:v1:[a-f0-9]{64}$/i.test(status.runtimePoolIdentities[tenantId]) + || !String(status.buildContracts[tenantId]).startsWith('graphile:v1:') + || !/^pg-contract-evidence:v1:[a-f0-9]{64}$/.test( + poolEvidence?.fingerprint ?? '' + ) + || !/^graphile-contract-evidence:v1:[a-f0-9]{64}$/.test( + buildEvidence?.fingerprint ?? '' + ) + || poolEvidence?.input?.databaseName !== customer.database + || poolEvidence?.input?.role !== customer.roles?.[tenantId] + || binding?.databaseName !== customer.database + || binding?.role !== customer.roles?.[tenantId] + || JSON.stringify(binding?.schemas) !== JSON.stringify([`ctf_${tenantId}`]) + || !Number.isSafeInteger(status.builds.byTenant[tenantId]) + || status.builds.byTenant[tenantId] < 0 + ) { + throw new Error(`PDCF_HOSTILE_CHILD_CONTRACT_MISMATCH:${customer.id}:${tenantId}`); + } + } + return status; +}; + +const collectSourceProvenance = () => { + const commit = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: REPO_ROOT, + encoding: 'utf8', + }).trim(); + const worktreeState = execFileSync( + 'git', + ['status', '--porcelain=v1', '--untracked-files=all'], + { cwd: REPO_ROOT, encoding: 'utf8' }, + ); + const validatorEntries = VALIDATOR_ENTRY_FILES.map((relativePath) => ({ + path: relativePath, + sha256: fileSha256(path.join(REPO_ROOT, relativePath)), + })); + const provenance = { + git: { + commit, + worktreeDirty: worktreeState.length > 0, + worktreeStateSha256: sha256Buffer(worktreeState), + }, + lockfileSha256: fileSha256(path.join(REPO_ROOT, 'pnpm-lock.yaml')), + runtime: { + node: process.version, + v8: process.versions.v8, + platform: process.platform, + architecture: process.arch, + }, + validatorEntries, + }; + return { + ...provenance, + sourceStateSha256: canonicalSha256(provenance), + }; +}; + +const validateSourceProvenance = (provenance) => { + const { + sourceStateSha256, + ...sourceState + } = provenance ?? {}; + if ( + !/^[a-f0-9]{40,64}$/.test(provenance?.git?.commit ?? '') + || typeof provenance.git.worktreeDirty !== 'boolean' + || !SHA256_PATTERN.test(provenance.git.worktreeStateSha256 ?? '') + || !SHA256_PATTERN.test(provenance.lockfileSha256 ?? '') + || !SHA256_PATTERN.test(sourceStateSha256 ?? '') + || sourceStateSha256 !== canonicalSha256(sourceState) + || typeof provenance.runtime?.node !== 'string' + || typeof provenance.runtime?.v8 !== 'string' + || typeof provenance.runtime?.platform !== 'string' + || typeof provenance.runtime?.architecture !== 'string' + || !Array.isArray(provenance.validatorEntries) + || provenance.validatorEntries.length === 0 + || new Set(provenance.validatorEntries.map((entry) => entry?.path)).size + !== provenance.validatorEntries.length + || provenance.validatorEntries.some((entry) => + typeof entry?.path !== 'string' + || !entry.path + || !SHA256_PATTERN.test(entry.sha256 ?? '') + ) + ) { + throw new Error('PDCF_HOSTILE_SOURCE_PROVENANCE_INVALID'); + } + return provenance; +}; + +const childStatusUrl = (baseUrl, customer) => + `${baseUrl}${assertCustomerPathPrefix( + `/customer/${customer.id}`, + customer.id, + )}/__ctf/status`; + +const readChildStatus = async (baseUrl, customer, fetchImpl) => + requestJson(childStatusUrl(baseUrl, customer), { fetchImpl }); + +const assertBadRoleCoverage = (report, customer) => { + if (report?.passed !== true) { + throw new Error(`PDCF_HOSTILE_CUSTOMER_VALIDATION_FAILED:${customer.id}`); + } + const checks = new Set((report.checks ?? []).map((check) => check.name)); + for (const tenant of TENANTS) { + if (!checks.has(`bad-role-expected-failure:${tenant.id}`)) { + throw new Error(`PDCF_HOSTILE_BAD_ROLE_CHECK_MISSING:${customer.id}:${tenant.id}`); + } + } +}; + +const validateUnsafeRuntimeStartupAdmission = ( + report, + manifest, + expectedRuntimeArtifactFingerprint, +) => { + const expectedSurfaces = TENANTS.map((tenant) => tenant.id); + const expectedAttempts = PROBE_CAPABILITIES.length * expectedSurfaces.length; + const customer = manifest.customers[0]; + const expectedPairs = new Set(PROBE_CAPABILITIES.flatMap((capability) => + expectedSurfaces.map((tenantId) => `${capability}:${tenantId}`) + )); + const attempts = Array.isArray(report?.attempts) ? report.attempts : []; + const observedPairs = new Set(attempts.map((attempt) => + `${attempt.capability}:${attempt.tenantId}` + )); + if ( + !exactKeys(report, [ + 'version', + 'kind', + 'admissionScope', + 'provisionClone', + 'representativeCustomerId', + 'representativePhysicalDatabase', + 'representativeProvisionAttestationSha256', + 'canonicalDatabaseContractFingerprint', + 'runtimeArtifactFingerprint', + 'liveProvisionAttestation', + 'safeStartupControl', + 'roleProfileAudit', + 'cleanupAudit', + 'capabilities', + 'surfaces', + 'attempts', + 'expectedAttempts', + 'rejectedAttempts', + 'acceptedAttempts', + 'graphileBuildsStarted', + 'residentGraphileEntries', + 'passed', + ]) + || report.version !== 2 + || report.kind !== PROBE_KIND + || report.admissionScope !== ADMISSION_SCOPE + || !exactKeys( + report.provisionClone, + ['version', 'id', 'purpose', 'attestationSetSha256'], + ) + || !canonicalEqual(report.provisionClone, manifest.provisionClone) + || report.representativeCustomerId !== customer?.id + || report.representativePhysicalDatabase !== customer?.physicalIdentity + || report.representativeProvisionAttestationSha256 + !== customer?.provisionAttestation?.sha256 + || report.canonicalDatabaseContractFingerprint + !== manifest.canonicalDatabaseContractFingerprint + || report.runtimeArtifactFingerprint !== expectedRuntimeArtifactFingerprint + || !canonicalEqual( + report.liveProvisionAttestation, + expectedLiveProvisionAttestation(manifest, customer), + ) + || !exactKeys(report.safeStartupControl, [ + 'tenantId', + 'accepted', + 'physicalDatabaseVerifiedBeforeRoleAudit', + 'controlCredentialEnvironmentAbsent', + 'graphileBuildsStarted', + 'residentGraphileEntries', + 'passed', + ]) + || report.safeStartupControl.tenantId !== expectedSurfaces[0] + || report.safeStartupControl.accepted !== true + || report.safeStartupControl.physicalDatabaseVerifiedBeforeRoleAudit !== true + || report.safeStartupControl.controlCredentialEnvironmentAbsent !== true + || report.safeStartupControl.graphileBuildsStarted !== 0 + || report.safeStartupControl.residentGraphileEntries !== 0 + || report.safeStartupControl.passed !== true + || !exactKeys( + report.roleProfileAudit, + ['version', 'kind', 'database', 'profiles', 'passed'], + ) + || report.roleProfileAudit.version !== 1 + || report.roleProfileAudit.kind !== 'unsafe-runtime-role-profile-audit-v1' + || report.roleProfileAudit.database !== customer?.database + || !canonicalEqual( + report.roleProfileAudit.profiles, + expectedAuditedProfiles(), + ) + || report.roleProfileAudit.passed !== true + || !exactKeys(report.cleanupAudit, [ + 'version', + 'kind', + 'database', + 'remainingRoles', + 'remainingSchemas', + 'passed', + ]) + || report.cleanupAudit.version !== 1 + || report.cleanupAudit.kind !== CLEANUP_AUDIT_KIND + || report.cleanupAudit.database !== customer?.database + || report.cleanupAudit.remainingRoles !== 0 + || report.cleanupAudit.remainingSchemas !== 0 + || report.cleanupAudit.passed !== true + || JSON.stringify(report.capabilities) !== JSON.stringify(PROBE_CAPABILITIES) + || JSON.stringify(report.surfaces) !== JSON.stringify(expectedSurfaces) + || report.expectedAttempts !== expectedAttempts + || report.rejectedAttempts !== expectedAttempts + || report.acceptedAttempts !== 0 + || report.graphileBuildsStarted !== 0 + || report.residentGraphileEntries !== 0 + || report.passed !== true + || !Array.isArray(report.attempts) + || report.attempts.length !== expectedAttempts + || observedPairs.size !== expectedPairs.size + || [...expectedPairs].some((pair) => !observedPairs.has(pair)) + || report.attempts.some((attempt) => + !exactKeys(attempt, [ + 'capability', + 'tenantId', + 'rejectedCode', + 'controlCredentialEnvironmentAbsent', + 'graphileBuildsStarted', + 'residentGraphileEntries', + ]) + || attempt.rejectedCode !== 'GRAPHILE_UNSAFE_RUNTIME_ROLE' + || attempt.controlCredentialEnvironmentAbsent !== true + || attempt.graphileBuildsStarted !== 0 + || attempt.residentGraphileEntries !== 0 + ) + ) { + throw new Error('PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID'); + } + return report; +}; + +const runFleetInvalidateAndRebuild = async ({ + baseUrl, + controlToken, + manifest, + arm, + mode, + fetchImpl, +}) => { + const before = {}; + for (const customer of manifest.customers) { + before[customer.id] = validateChildStatus( + await readChildStatus(baseUrl, customer, fetchImpl), + { customer, manifest, arm, mode }, + ); + } + + for (const customer of manifest.customers) { + const pathPrefix = `/customer/${customer.id}`; + await control( + baseUrl, + pathPrefix, + controlToken, + 'invalidate-all', + null, + customer.physicalIdentity, + fetchImpl, + ); + } + + for (const customer of manifest.customers) { + const pathPrefix = `/customer/${customer.id}`; + for (const tenant of TENANTS) { + const response = await postGraphql( + baseUrl, + pathPrefix, + tenant.id, + identityOperation, + fetchImpl, + ); + assertIdentity(tenant, response, customer.physicalIdentity); + } + } + + const after = {}; + for (const customer of manifest.customers) { + after[customer.id] = validateChildStatus( + await readChildStatus(baseUrl, customer, fetchImpl), + { customer, manifest, arm, mode }, + ); + for (const tenant of TENANTS) { + const beforeCount = before[customer.id].builds.byTenant[tenant.id]; + const afterCount = after[customer.id].builds.byTenant[tenant.id]; + if (afterCount !== beforeCount + 1) { + throw new Error( + `PDCF_HOSTILE_FLEET_REBUILD_COUNT_MISMATCH:${customer.id}:${tenant.id}`, + ); + } + } + } + return { + invalidatedCustomers: manifest.customers.length, + rebuiltSurfaces: manifest.customers.length * TENANTS.length, + physicalIdentityMismatches: 0, + }; +}; + +const failureCode = (error) => String( + error instanceof Error ? error.message : error, +).split(':', 1)[0].replace(/[^A-Z0-9_-]/gi, '_').slice(0, 96); + +const runPhysicalHostilePreflight = async ({ + manifestFile, + secretsFile, + baseUrl, + controlToken, + arm, + mode, + preflightCloneId, + outputFile, + fetchImpl = fetch, + runCustomerValidation = runHostileValidation, + runUnsafeRoleMatrix = runUnsafeRuntimeStartupMatrix, + provenanceProvider = collectSourceProvenance, +} = {}) => { + if ( + process.env.GRAPHQL_CPERF_MEASURED_RUN === 'true' + || process.env.GRAPHQL_CPERF_RETAINED_HEAP_ENABLED === 'true' + ) { + throw new Error('PDCF_HOSTILE_MEASURED_PROCESS_FORBIDDEN'); + } + if (typeof controlToken !== 'string' || Buffer.byteLength(controlToken) < 32) { + throw new Error('PDCF_HOSTILE_CONTROL_TOKEN_REQUIRED'); + } + preflightCloneId = requireArtifactLabel( + preflightCloneId, + 'PDCF_HOSTILE_PREFLIGHT_CLONE_ID_REQUIRED', + ); + arm = requireArtifactLabel(arm, 'PDCF_HOSTILE_ARM_REQUIRED'); + if (mode !== 'stock' && mode !== 'scoped-required') { + throw new Error('PDCF_HOSTILE_MODE_INVALID'); + } + if (preflightCloneId === controlToken) { + throw new Error('PDCF_HOSTILE_PREFLIGHT_CLONE_ID_INVALID'); + } + const localBaseUrl = assertLoopbackBaseUrl(baseUrl); + const absoluteManifestFile = path.resolve(manifestFile); + const absoluteSecretsFile = path.resolve(requireString( + { secrets: secretsFile }, + 'secrets', + )); + const absoluteOutputFile = assertOutputDoesNotAliasInputs(outputFile, [ + absoluteManifestFile, + absoluteSecretsFile, + ]); + const manifest = validatePreflightManifest( + loadPrivateProvision(absoluteManifestFile, absoluteSecretsFile).manifest, + preflightCloneId, + ); + const startedAt = new Date().toISOString(); + const sourceProvenance = validateSourceProvenance(provenanceProvider()); + const report = { + version: 1, + kind: 'physical-hostile-preflight-v1', + fixture: FIXTURE_ID, + startedAt, + endedAt: null, + passed: false, + customerQualified: false, + performanceEvidence: false, + arm, + mode, + preflightCloneId, + provisionClone: { + ...manifest.provisionClone, + verifiedByServer: false, + }, + measuredCloneRequirement: { + mustBeFresh: true, + mustBeDistinctFromPreflight: true, + mustMatchCanonicalDatabaseContract: true, + }, + manifest: { + sha256: fileSha256(absoluteManifestFile), + canonicalStructuralFingerprint: + manifest.canonicalStructuralFingerprint?.combined?.sha256 ?? null, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint ?? null, + customerIds: manifest.customers.map((customer) => customer.id), + }, + sourceProvenance, + observedRuntimeArtifactFingerprint: null, + observedBlueprintCompatibilityFingerprint: null, + unsafeRuntimeStartupAdmission: null, + customerValidations: [], + fleetInvalidateAndRebuild: null, + unsupportedChecks: [], + }; + + try { + const physicalStatus = validatePhysicalStatus( + await requestJson(`${localBaseUrl}/__physical/status`, { fetchImpl }), + { manifest, arm, mode, preflightCloneId }, + ); + report.provisionClone.verifiedByServer = true; + report.observedBlueprintCompatibilityFingerprint = + physicalStatus.blueprintCompatibilityFingerprint; + const representativeCustomer = manifest.customers[0]; + const representativeChildStatus = validateChildStatus( + await readChildStatus(localBaseUrl, representativeCustomer, fetchImpl), + { customer: representativeCustomer, manifest, arm, mode }, + ); + report.unsafeRuntimeStartupAdmission = validateUnsafeRuntimeStartupAdmission( + await runUnsafeRoleMatrix({ + manifestFile: absoluteManifestFile, + secretsFile: absoluteSecretsFile, + expectedRuntimeArtifactFingerprint: + representativeChildStatus.runtimeArtifactFingerprint, + mode, + }), + manifest, + representativeChildStatus.runtimeArtifactFingerprint, + ); + + const runtimeFingerprints = new Set(); + for (const customer of manifest.customers) { + const childStatus = validateChildStatus( + await readChildStatus(localBaseUrl, customer, fetchImpl), + { customer, manifest, arm, mode }, + ); + runtimeFingerprints.add(childStatus.runtimeArtifactFingerprint); + const customerReport = await runCustomerValidation({ + baseUrl: localBaseUrl, + pathPrefix: `/customer/${customer.id}`, + expectedCustomerId: customer.id, + expectedPhysicalDatabaseIdentity: customer.physicalIdentity, + controlToken, + arm, + mode, + fetchImpl, + }); + assertBadRoleCoverage(customerReport, customer); + report.customerValidations.push({ + customerId: customer.id, + physicalDatabaseIdentity: customer.physicalIdentity, + provisionAttestationSha256: customer.provisionAttestation.sha256, + checks: customerReport.checks.length, + reportSha256: canonicalSha256(customerReport), + passed: true, + }); + } + if ( + runtimeFingerprints.size !== 1 + || !runtimeFingerprints.has( + report.unsafeRuntimeStartupAdmission.runtimeArtifactFingerprint, + ) + ) { + throw new Error('PDCF_HOSTILE_RUNTIME_FINGERPRINT_MISMATCH'); + } + report.observedRuntimeArtifactFingerprint = [...runtimeFingerprints][0]; + report.fleetInvalidateAndRebuild = await runFleetInvalidateAndRebuild({ + baseUrl: localBaseUrl, + controlToken, + manifest, + arm, + mode, + fetchImpl, + }); + report.passed = true; + report.endedAt = new Date().toISOString(); + report.artifactSha256 = canonicalSha256({ + ...report, + artifactSha256: undefined, + }); + assertCredentialFree(report); + if (absoluteOutputFile) atomicWriteJson(absoluteOutputFile, report, 0o600); + return report; + } catch (error) { + report.endedAt = new Date().toISOString(); + report.failureCode = failureCode(error); + report.artifactSha256 = canonicalSha256({ + ...report, + artifactSha256: undefined, + }); + assertCredentialFree(report); + if (absoluteOutputFile) atomicWriteJson(absoluteOutputFile, report, 0o600); + throw error; + } +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const report = await runPhysicalHostilePreflight({ + manifestFile: path.resolve(requireString(args, 'manifest')), + secretsFile: path.resolve(requireString(args, 'secrets')), + baseUrl: requireString(args, 'base-url'), + controlToken: process.env.CTF_CONTROL_TOKEN, + arm: requireString(args, 'arm'), + mode: requireString(args, 'mode', 'scoped-required'), + preflightCloneId: requireString(args, 'preflight-clone-id'), + outputFile: path.resolve(requireString( + args, + 'output', + path.join(FIXTURE_DIR, '.local', `hostile-preflight-${timestamp}.json`), + )), + }); + process.stdout.write(`${JSON.stringify({ + passed: report.passed, + customers: report.customerValidations.length, + artifactSha256: report.artifactSha256, + })}\n`); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${failureCode(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + VALIDATOR_ENTRY_FILES, + assertBadRoleCoverage, + assertOutputDoesNotAliasInputs, + canonicalSha256, + collectSourceProvenance, + runFleetInvalidateAndRebuild, + runPhysicalHostilePreflight, + validateUnsafeRuntimeStartupAdmission, + validateSourceProvenance, + validateChildStatus, + validatePreflightManifest, + validatePhysicalStatus, +}; diff --git a/research/graphile-density/physical-database-density/physical-hostile-preflight.test.cjs b/research/graphile-density/physical-database-density/physical-hostile-preflight.test.cjs new file mode 100644 index 0000000000..1d8fa62d8e --- /dev/null +++ b/research/graphile-density/physical-database-density/physical-hostile-preflight.test.cjs @@ -0,0 +1,658 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { TENANTS, assertCredentialFree } = require('../complete-tenant-fixture/lib.cjs'); +const { makeCustomers } = require('./lib.cjs'); +const { + provisionAttestationSetSha256, + provisionAttestationSha256, +} = require('./provision.cjs'); +const { + assertProvisionCloneManifest, + parseServerOptions, +} = require('./server.cjs'); +const { + assertOutputDoesNotAliasInputs, + canonicalSha256, + runPhysicalHostilePreflight, + validateUnsafeRuntimeStartupAdmission, + validatePhysicalStatus, +} = require('./physical-hostile-preflight.cjs'); +const { + ADMISSION_SCOPE, + CLEANUP_AUDIT_KIND, + PROBE_CAPABILITIES, + PROBE_KIND, + expectedAuditedProfiles, +} = require('./unsafe-runtime-startup-probe.cjs'); + +const digest = (character) => `sha256:${character.repeat(64)}`; +const arm = 'physical-db-idle-1s'; +const mode = 'scoped-required'; +const preflightCloneId = 'fresh-preflight-clone-20260802-a'; + +const makeManifest = () => { + const canonicalDatabaseContractFingerprint = digest('b'); + const canonicalStructuralFingerprint = { + combined: { sha256: digest('c') }, + }; + const customers = makeCustomers('pdc_preflight', 2).map((customer, index) => ({ + ...customer, + provisionAttestation: { + version: 1, + cloneId: preflightCloneId, + purpose: 'hostile-preflight', + sha256: digest(String(index + 3)), + }, + structuralFingerprints: canonicalStructuralFingerprint, + databaseContractFingerprint: canonicalDatabaseContractFingerprint, + })); + return { + version: 1, + fixture: 'physical-database-density-v1', + prefix: 'pdc_preflight', + createdAt: '2026-08-02T00:00:00.000Z', + provisionClone: { + version: 1, + id: preflightCloneId, + purpose: 'hostile-preflight', + attestationSetSha256: provisionAttestationSetSha256(customers), + }, + canonicalStructuralFingerprint, + canonicalDatabaseContractFingerprint, + customers, + }; +}; + +const physicalStatusFor = (manifest) => ({ + version: 1, + fixture: 'physical-database-density-v1', + arm, + runPurpose: 'hostile-preflight', + cloneId: preflightCloneId, + provisionClone: { + ...manifest.provisionClone, + verified: true, + }, + introspectionMode: mode, + introspectionClientReleaseMode: 'destroy', + runtimePoolMax: 2, + runtimePoolMaxUses: null, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint, + blueprintCompatibilityFingerprint: digest('d'), + canonicalStructuralFingerprint: manifest.canonicalStructuralFingerprint, + realtime: { + managersExpected: manifest.customers.length * TENANTS.length, + connectionsExpected: manifest.customers.length * TENANTS.length, + transportsExpected: manifest.customers.length * TENANTS.length, + notificationMode: 'dedicated', + }, + customers: manifest.customers.map((customer) => ({ + id: customer.id, + physicalDatabase: customer.database, + provisionAttestation: { + ...customer.provisionAttestation, + customerId: customer.id, + database: customer.database, + verified: true, + }, + structuralFingerprints: customer.structuralFingerprints, + canonicalStructuralFingerprint: + customer.structuralFingerprints.combined.sha256, + databaseContractFingerprint: customer.databaseContractFingerprint, + contractVerification: 'live-recomputed', + })), +}); + +const childStatusFor = (manifest, customer, buildCounts) => ({ + version: 1, + fixture: 'complete-tenant-abc-v1', + arm, + introspectionMode: mode, + introspectionClientReleaseMode: 'destroy', + releaseBuildStateAfterValidation: true, + runtimeArtifactFingerprint: digest('e'), + physicalIsolation: 'dedicated-login-and-pool-per-tenant', + sharedRuntimePool: false, + runtimePoolMax: 2, + runtimePoolMaxUses: null, + enableRealtime: true, + realtimeNotificationMode: 'dedicated', + realtimeCursorPollIntervalMs: 5_000, + realtimeCursorHeartbeatIntervalMs: 30_000, + realtimeSchemas: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `${tenant.schema}_realtime`, + ])), + physicalDatabase: customer.physicalIdentity, + runPurpose: 'hostile-preflight', + provisionAttestation: { + ...customer.provisionAttestation, + customerId: customer.id, + database: customer.database, + verified: true, + }, + controlAvailable: true, + runtimePoolIdentities: Object.fromEntries(TENANTS.map((tenant, index) => [ + tenant.id, + `pg:v1:${String(index + 1).repeat(64)}`, + ])), + buildContracts: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + `graphile:v1:${customer.id}:${tenant.id}`, + ])), + configurationIdentity: `graphile-configuration:ctf:v1:${'e'.repeat(64)}`, + liveIdentityScope: 'process-local-keyed-hmac-v1', + runtimeBindings: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + databaseName: customer.database, + role: customer.roles[tenant.id], + schemas: [tenant.schema], + }, + ])), + contractEvidence: { + version: 1, + credentialFree: true, + configurationIdentity: `graphile-configuration:ctf:v1:${'e'.repeat(64)}`, + runtimePools: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + version: 1, + fingerprint: `pg-contract-evidence:v1:${tenant.id.repeat(64)}`, + input: { + databaseName: customer.database, + role: customer.roles[tenant.id], + }, + }, + ])), + graphileBuilds: Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + { + version: 1, + fingerprint: `graphile-contract-evidence:v1:${tenant.id.repeat(64)}`, + input: {}, + }, + ])), + }, + builds: { byTenant: { ...buildCounts } }, + runtimeSafety: { passed: true, rolesDistinct: true }, +}); + +const response = (body, status = 200) => ({ + ok: status >= 200 && status < 300, + status, + json: async () => body, +}); + +const makeProvenance = () => { + const provenance = { + git: { + commit: 'a'.repeat(40), + worktreeDirty: false, + worktreeStateSha256: digest('f'), + }, + lockfileSha256: digest('1'), + runtime: { + node: 'v24.0.0', + v8: '13.6', + platform: 'linux', + architecture: 'x64', + }, + validatorEntries: [{ path: 'validator.cjs', sha256: digest('2') }], + }; + return { + ...provenance, + sourceStateSha256: canonicalSha256(provenance), + }; +}; + +const writeSecrets = (directory, manifest) => { + const secretsFile = path.join(directory, 'runtime-secrets.json'); + fs.writeFileSync(secretsFile, JSON.stringify({ + version: 1, + fixture: 'physical-database-density-v1', + runtimePasswords: Object.fromEntries(manifest.customers.flatMap((customer) => + Object.values(customer.roles).map((role) => [ + role, + `test-runtime-password-${role}`, + ]) + )), + notificationPasswords: Object.fromEntries(manifest.customers.map((customer) => [ + customer.notificationRole, + `test-notification-password-${customer.notificationRole}`, + ])), + }), { mode: 0o600 }); + return secretsFile; +}; + +const unsafeStartupAdmissionFor = (manifest) => { + const attempts = PROBE_CAPABILITIES.flatMap((capability) => TENANTS.map((tenant) => ({ + capability, + tenantId: tenant.id, + rejectedCode: 'GRAPHILE_UNSAFE_RUNTIME_ROLE', + controlCredentialEnvironmentAbsent: true, + graphileBuildsStarted: 0, + residentGraphileEntries: 0, + }))); + return { + version: 2, + kind: PROBE_KIND, + admissionScope: ADMISSION_SCOPE, + provisionClone: { ...manifest.provisionClone }, + representativeCustomerId: manifest.customers[0].id, + representativePhysicalDatabase: manifest.customers[0].physicalIdentity, + representativeProvisionAttestationSha256: + manifest.customers[0].provisionAttestation.sha256, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint, + runtimeArtifactFingerprint: digest('e'), + liveProvisionAttestation: { + ...manifest.customers[0].provisionAttestation, + customerId: manifest.customers[0].id, + database: manifest.customers[0].database, + verified: true, + }, + safeStartupControl: { + tenantId: TENANTS[0].id, + accepted: true, + physicalDatabaseVerifiedBeforeRoleAudit: true, + controlCredentialEnvironmentAbsent: true, + graphileBuildsStarted: 0, + residentGraphileEntries: 0, + passed: true, + }, + roleProfileAudit: { + version: 1, + kind: 'unsafe-runtime-role-profile-audit-v1', + database: manifest.customers[0].database, + profiles: expectedAuditedProfiles(), + passed: true, + }, + cleanupAudit: { + version: 1, + kind: CLEANUP_AUDIT_KIND, + database: manifest.customers[0].database, + remainingRoles: 0, + remainingSchemas: 0, + passed: true, + }, + capabilities: [...PROBE_CAPABILITIES], + surfaces: TENANTS.map((tenant) => tenant.id), + attempts, + expectedAttempts: attempts.length, + rejectedAttempts: attempts.length, + acceptedAttempts: 0, + graphileBuildsStarted: 0, + residentGraphileEntries: 0, + passed: true, + }; +}; + +describe('aggregate physical hostile preflight', () => { + it('binds clone purpose and identity to opaque per-database nonce digests', () => { + const base = { + cloneId: preflightCloneId, + runPurpose: 'hostile-preflight', + customerId: 'physical-customer-0001', + database: 'pdc_preflight_db_0001', + nonce: '1'.repeat(64), + }; + const sha256 = provisionAttestationSha256(base); + assert.match(sha256, /^sha256:[a-f0-9]{64}$/); + assert.equal(sha256, provisionAttestationSha256(base)); + assert.notEqual(sha256, provisionAttestationSha256({ + ...base, + nonce: '2'.repeat(64), + })); + assert.notEqual(sha256, provisionAttestationSha256({ + ...base, + runPurpose: 'measurement', + })); + + const sql = fs.readFileSync(path.join(__dirname, 'provision-attestation.sql'), 'utf8'); + assert.match(sql, /CREATE SCHEMA ctf_provision_private/); + assert.match(sql, /REVOKE ALL ON SCHEMA ctf_provision_private FROM PUBLIC/); + assert.match(sql, /REVOKE ALL ON TABLE ctf_provision_private\.clone_attestation FROM PUBLIC/); + assert.doesNotMatch(sql, /GRANT .*runtime_role/i); + }); + + it('requires explicit purpose and clone identity on every physical server', () => { + const options = parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--run-purpose', 'hostile-preflight', + '--clone-id', preflightCloneId, + ]); + assert.equal(options.runPurpose, 'hostile-preflight'); + assert.equal(options.cloneId, preflightCloneId); + assert.throws(() => parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--run-purpose', 'diagnostic', + '--clone-id', preflightCloneId, + ]), /PDCF_RUN_PURPOSE_INVALID/); + assert.throws(() => parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/runtime-secrets.json', + '--run-purpose', 'measurement', + ]), /CTF_ARGUMENT_REQUIRED:clone-id/); + + const manifest = makeManifest(); + assert.equal(assertProvisionCloneManifest(manifest, options), manifest.provisionClone); + assert.throws(() => assertProvisionCloneManifest(manifest, { + ...options, + runPurpose: 'measurement', + }), /PDCF_PROVISION_CLONE_MISMATCH/); + }); + + it('runs every mounted customer sequentially and rebuilds only after fleet invalidation', async (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-hostile-preflight-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const manifest = makeManifest(); + const manifestFile = path.join(temporary, 'provision.json'); + const outputFile = path.join(temporary, 'hostile-preflight.json'); + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + const secretsFile = writeSecrets(temporary, manifest); + + const counts = Object.fromEntries(manifest.customers.map((customer) => [ + customer.id, + { a: 3, b: 4, c: 5 }, + ])); + const pendingRebuilds = new Map(); + const customerOrder = []; + let activeCustomerValidations = 0; + let maxActiveCustomerValidations = 0; + const fetchImpl = async (url, options = {}) => { + const parsed = new URL(url); + if (parsed.pathname === '/__physical/status') { + return response(physicalStatusFor(manifest)); + } + const route = /^\/customer\/([^/]+)(.*)$/.exec(parsed.pathname); + assert.ok(route, `unexpected route ${parsed.pathname}`); + const customer = manifest.customers.find((candidate) => candidate.id === route[1]); + assert.ok(customer); + if (route[2] === '/__ctf/status') { + return response(childStatusFor(manifest, customer, counts[customer.id])); + } + if (route[2] === '/__ctf/control') { + const body = JSON.parse(options.body); + assert.equal(body.action, 'invalidate-all'); + pendingRebuilds.set(customer.id, new Set(TENANTS.map((tenant) => tenant.id))); + return response({ + ok: true, + action: body.action, + physicalDatabaseIdentity: customer.physicalIdentity, + }); + } + const graphqlRoute = /^\/tenant\/([abc])\/graphql$/.exec(route[2]); + assert.ok(graphqlRoute, `unexpected child route ${route[2]}`); + const tenant = TENANTS.find((candidate) => candidate.id === graphqlRoute[1]); + const pending = pendingRebuilds.get(customer.id); + if (pending?.delete(tenant.id)) counts[customer.id][tenant.id] += 1; + return response({ + data: { + tenantIdentity: tenant.token, + requestIdentity: `${tenant.token}:${tenant.databaseId}`, + physicalDatabaseIdentity: customer.physicalIdentity, + }, + }); + }; + const controlToken = 'preflight-control-value-that-is-never-persisted'; + const report = await runPhysicalHostilePreflight({ + manifestFile, + secretsFile, + baseUrl: 'http://127.0.0.1:3410', + controlToken, + arm, + mode, + preflightCloneId, + outputFile, + fetchImpl, + provenanceProvider: makeProvenance, + runUnsafeRoleMatrix: () => unsafeStartupAdmissionFor(manifest), + runCustomerValidation: async (options) => { + activeCustomerValidations += 1; + maxActiveCustomerValidations = Math.max( + maxActiveCustomerValidations, + activeCustomerValidations, + ); + customerOrder.push(options.expectedCustomerId); + assert.equal( + options.pathPrefix, + `/customer/${options.expectedCustomerId}`, + ); + const customer = manifest.customers.find( + (candidate) => candidate.id === options.expectedCustomerId, + ); + assert.equal(options.expectedPhysicalDatabaseIdentity, customer.physicalIdentity); + await Promise.resolve(); + activeCustomerValidations -= 1; + return { + passed: true, + checks: TENANTS.map((tenant) => ({ + name: `bad-role-expected-failure:${tenant.id}`, + passed: true, + })), + }; + }, + }); + + assert.equal(maxActiveCustomerValidations, 1); + assert.deepEqual(customerOrder, manifest.customers.map((customer) => customer.id)); + assert.equal(report.passed, true); + assert.equal(report.customerQualified, false); + assert.equal(report.performanceEvidence, false); + assert.equal(report.fleetInvalidateAndRebuild.invalidatedCustomers, 2); + assert.equal(report.fleetInvalidateAndRebuild.rebuiltSurfaces, 6); + assert.equal(report.observedRuntimeArtifactFingerprint, digest('e')); + assert.equal(report.observedBlueprintCompatibilityFingerprint, digest('d')); + assert.equal(report.unsafeRuntimeStartupAdmission.rejectedAttempts, 15); + assert.equal(report.unsafeRuntimeStartupAdmission.graphileBuildsStarted, 0); + assert.equal( + report.unsafeRuntimeStartupAdmission.runtimeArtifactFingerprint, + digest('e'), + ); + assert.equal(report.unsafeRuntimeStartupAdmission.cleanupAudit.passed, true); + assert.equal(report.provisionClone.id, preflightCloneId); + assert.equal(report.provisionClone.verifiedByServer, true); + assert.match(report.manifest.sha256, /^sha256:[a-f0-9]{64}$/); + assert.match(report.artifactSha256, /^sha256:[a-f0-9]{64}$/); + + const artifact = fs.readFileSync(outputFile, 'utf8'); + assert.doesNotMatch(artifact, new RegExp(controlToken)); + assert.doesNotThrow(() => assertCredentialFree(artifact)); + }); + + it('fails closed when the mounted server customer set or mode differs', () => { + const manifest = makeManifest(); + const status = physicalStatusFor(manifest); + assert.equal(validatePhysicalStatus(status, { + manifest, + arm, + mode, + preflightCloneId, + }), status); + assert.throws(() => validatePhysicalStatus({ + ...status, + introspectionMode: 'stock', + }, { manifest, arm, mode, preflightCloneId }), /PDCF_HOSTILE_PHYSICAL_STATUS_MISMATCH/); + assert.throws(() => validatePhysicalStatus({ + ...status, + customers: status.customers.slice(0, 1), + }, { manifest, arm, mode, preflightCloneId }), /PDCF_HOSTILE_CUSTOMER_SET_MISMATCH/); + assert.throws(() => validatePhysicalStatus({ + ...status, + customers: [...status.customers].reverse(), + }, { manifest, arm, mode, preflightCloneId }), /PDCF_HOSTILE_CUSTOMER_SET_MISMATCH/); + assert.throws(() => validatePhysicalStatus({ + ...status, + realtime: { + ...status.realtime, + managersExpected: 0, + }, + }, { + manifest, + arm, + mode, + preflightCloneId, + }), /PDCF_HOSTILE_PHYSICAL_STATUS_MISMATCH/); + + assert.throws(() => validatePhysicalStatus({ + ...status, + provisionClone: { + ...status.provisionClone, + verified: false, + }, + }, { + manifest, + arm, + mode, + preflightCloneId, + }), /PDCF_HOSTILE_PHYSICAL_STATUS_MISMATCH/); + }); + + it('persists a credential-free failure skeleton before any customer mutation', async (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-hostile-failure-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const manifest = makeManifest(); + const manifestFile = path.join(temporary, 'provision.json'); + const outputFile = path.join(temporary, 'hostile-preflight.json'); + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + const secretsFile = writeSecrets(temporary, manifest); + const controlToken = 'failure-control-value-that-is-never-persisted'; + let customerValidationStarted = false; + + await assert.rejects(() => runPhysicalHostilePreflight({ + manifestFile, + secretsFile, + baseUrl: 'http://127.0.0.1:3410', + controlToken, + arm, + mode, + preflightCloneId, + outputFile, + fetchImpl: async () => response({ + ...physicalStatusFor(manifest), + introspectionMode: 'stock', + }), + provenanceProvider: makeProvenance, + runUnsafeRoleMatrix: () => unsafeStartupAdmissionFor(manifest), + runCustomerValidation: async () => { + customerValidationStarted = true; + }, + }), /PDCF_HOSTILE_PHYSICAL_STATUS_MISMATCH/); + + assert.equal(customerValidationStarted, false); + const artifactText = fs.readFileSync(outputFile, 'utf8'); + const artifact = JSON.parse(artifactText); + assert.equal(artifact.passed, false); + assert.equal(artifact.customerQualified, false); + assert.equal(artifact.performanceEvidence, false); + assert.equal(artifact.failureCode, 'PDCF_HOSTILE_PHYSICAL_STATUS_MISMATCH'); + assert.deepEqual(artifact.customerValidations, []); + assert.doesNotMatch(artifactText, new RegExp(controlToken)); + assert.doesNotThrow(() => assertCredentialFree(artifact)); + }); + + it('rejects partial or post-publication unsafe-role evidence', () => { + const manifest = makeManifest(); + const valid = unsafeStartupAdmissionFor(manifest); + assert.equal(validateUnsafeRuntimeStartupAdmission(valid, manifest, digest('e')), valid); + assert.throws(() => validateUnsafeRuntimeStartupAdmission({ + ...valid, + attempts: valid.attempts.slice(1), + rejectedAttempts: valid.rejectedAttempts - 1, + }, manifest, digest('e')), /PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID/); + assert.throws(() => validateUnsafeRuntimeStartupAdmission({ + ...valid, + graphileBuildsStarted: 1, + }, manifest, digest('e')), /PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID/); + assert.throws(() => validateUnsafeRuntimeStartupAdmission({ + ...valid, + runtimeArtifactFingerprint: digest('f'), + }, manifest, digest('e')), /PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID/); + assert.throws(() => validateUnsafeRuntimeStartupAdmission({ + ...valid, + cleanupAudit: { + ...valid.cleanupAudit, + remainingRoles: 1, + passed: false, + }, + }, manifest, digest('e')), /PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID/); + assert.throws(() => validateUnsafeRuntimeStartupAdmission({ + ...valid, + safeStartupControl: { + ...valid.safeStartupControl, + accepted: false, + }, + }, manifest, digest('e')), /PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID/); + assert.throws(() => validateUnsafeRuntimeStartupAdmission({ + ...valid, + roleProfileAudit: { + ...valid.roleProfileAudit, + profiles: valid.roleProfileAudit.profiles.map((profile) => + profile.capability === 'bypassrls' + ? { ...profile, bypassRls: false } + : profile + ), + }, + }, manifest, digest('e')), /PDCF_UNSAFE_RUNTIME_STARTUP_ADMISSION_INVALID/); + }); + + it('rejects a non-private runtime secrets input before network access', async (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-hostile-secrets-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const manifest = makeManifest(); + const manifestFile = path.join(temporary, 'provision.json'); + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + const secretsFile = writeSecrets(temporary, manifest); + fs.chmodSync(secretsFile, 0o644); + let fetched = false; + await assert.rejects(() => runPhysicalHostilePreflight({ + manifestFile, + secretsFile, + baseUrl: 'http://127.0.0.1:3410', + controlToken: 'private-secret-test-control-token-value', + arm, + mode, + preflightCloneId, + fetchImpl: async () => { + fetched = true; + return response({}); + }, + provenanceProvider: makeProvenance, + }), /PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE/); + assert.equal(fetched, false); + }); + + it('rejects output paths that alias manifest or secrets inputs', (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-hostile-output-alias-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const manifestFile = path.join(temporary, 'provision.json'); + const secretsFile = path.join(temporary, 'runtime-secrets.json'); + fs.writeFileSync(manifestFile, '{}'); + fs.writeFileSync(secretsFile, '{}', { mode: 0o600 }); + assert.throws( + () => assertOutputDoesNotAliasInputs(manifestFile, [manifestFile, secretsFile]), + /PDCF_HOSTILE_OUTPUT_ALIASES_INPUT/, + ); + assert.throws( + () => assertOutputDoesNotAliasInputs(secretsFile, [manifestFile, secretsFile]), + /PDCF_HOSTILE_OUTPUT_ALIASES_INPUT/, + ); + assert.equal( + assertOutputDoesNotAliasInputs( + path.join(temporary, 'hostile-preflight.json'), + [manifestFile, secretsFile], + ), + path.join(temporary, 'hostile-preflight.json'), + ); + }); +}); diff --git a/research/graphile-density/physical-database-density/physical-identity.sql b/research/graphile-density/physical-database-density/physical-identity.sql new file mode 100644 index 0000000000..c5326b3ea9 --- /dev/null +++ b/research/graphile-density/physical-database-density/physical-identity.sql @@ -0,0 +1,270 @@ +\set ON_ERROR_STOP on + +-- The function body is identical in every customer database. The returned +-- value differs because current_database() is connection-bound, which makes a +-- wrong-database route conclusive without making the canonical schema drift. +CREATE OR REPLACE FUNCTION ctf_a.physical_database_identity() +RETURNS text +LANGUAGE sql +STABLE +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ + SELECT pg_catalog.current_database()::text +$function$; + +CREATE OR REPLACE FUNCTION ctf_b.physical_database_identity() +RETURNS text +LANGUAGE sql +STABLE +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ + SELECT pg_catalog.current_database()::text +$function$; + +CREATE OR REPLACE FUNCTION ctf_c.physical_database_identity() +RETURNS text +LANGUAGE sql +STABLE +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ + SELECT pg_catalog.current_database()::text +$function$; + +-- Graphile exposes VOLATILE functions on the mutation root. Selecting this +-- fixture-only sibling in the same GraphQL mutation as uploadAppFile proves +-- which physical database executed every upload invocation, including timed +-- workload calls whose upload payload cannot carry a table-stamped column. +CREATE OR REPLACE FUNCTION ctf_a.physical_database_mutation_identity() +RETURNS text +LANGUAGE sql +VOLATILE +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ + SELECT pg_catalog.current_database()::text +$function$; + +CREATE OR REPLACE FUNCTION ctf_b.physical_database_mutation_identity() +RETURNS text +LANGUAGE sql +VOLATILE +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ + SELECT pg_catalog.current_database()::text +$function$; + +CREATE OR REPLACE FUNCTION ctf_c.physical_database_mutation_identity() +RETURNS text +LANGUAGE sql +VOLATILE +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ + SELECT pg_catalog.current_database()::text +$function$; + +-- Realtime verification must derive its database oracle inside PostgreSQL. +-- A caller-provided payload can be identical on the wrong physical database, +-- so each row carries an immutable value stamped from current_database(). The +-- BEFORE trigger overwrites both inserts and updates even if raw SQL or a +-- generated GraphQL mutation attempts to supply another value. +ALTER TABLE ctf_a.realtime_items + ADD COLUMN physical_database_identity text; +UPDATE ctf_a.realtime_items +SET physical_database_identity = pg_catalog.current_database()::text; +ALTER TABLE ctf_a.realtime_items + ALTER COLUMN physical_database_identity SET NOT NULL; + +CREATE FUNCTION ctf_a.stamp_realtime_physical_database_identity() +RETURNS trigger +LANGUAGE plpgsql +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ +BEGIN + NEW.physical_database_identity := pg_catalog.current_database()::text; + RETURN NEW; +END +$function$; + +CREATE TRIGGER realtime_items_physical_database_identity +BEFORE INSERT OR UPDATE ON ctf_a.realtime_items +FOR EACH ROW EXECUTE FUNCTION ctf_a.stamp_realtime_physical_database_identity(); + +ALTER TABLE ctf_b.realtime_items + ADD COLUMN physical_database_identity text; +UPDATE ctf_b.realtime_items +SET physical_database_identity = pg_catalog.current_database()::text; +ALTER TABLE ctf_b.realtime_items + ALTER COLUMN physical_database_identity SET NOT NULL; + +CREATE FUNCTION ctf_b.stamp_realtime_physical_database_identity() +RETURNS trigger +LANGUAGE plpgsql +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ +BEGIN + NEW.physical_database_identity := pg_catalog.current_database()::text; + RETURN NEW; +END +$function$; + +CREATE TRIGGER realtime_items_physical_database_identity +BEFORE INSERT OR UPDATE ON ctf_b.realtime_items +FOR EACH ROW EXECUTE FUNCTION ctf_b.stamp_realtime_physical_database_identity(); + +ALTER TABLE ctf_c.realtime_items + ADD COLUMN physical_database_identity text; +UPDATE ctf_c.realtime_items +SET physical_database_identity = pg_catalog.current_database()::text; +ALTER TABLE ctf_c.realtime_items + ALTER COLUMN physical_database_identity SET NOT NULL; + +CREATE FUNCTION ctf_c.stamp_realtime_physical_database_identity() +RETURNS trigger +LANGUAGE plpgsql +SECURITY INVOKER +SET search_path = pg_catalog +AS $function$ +BEGIN + NEW.physical_database_identity := pg_catalog.current_database()::text; + RETURN NEW; +END +$function$; + +CREATE TRIGGER realtime_items_physical_database_identity +BEFORE INSERT OR UPDATE ON ctf_c.realtime_items +FOR EACH ROW EXECUTE FUNCTION ctf_c.stamp_realtime_physical_database_identity(); + +-- Every capability response must carry evidence derived by the physical +-- database that executed its SQL. These columns are fixture-only and are +-- stamped in PostgreSQL, so a request payload cannot forge the oracle. The +-- same trigger also covers the tables written by the upload, bulk-mutation, +-- and function-binding plugins. +CREATE PROCEDURE pg_temp.add_physical_response_oracles(schema_name text) +LANGUAGE plpgsql +AS $procedure$ +DECLARE + table_name text; +BEGIN + IF schema_name NOT IN ('ctf_a', 'ctf_b', 'ctf_c') THEN + RAISE EXCEPTION 'PDCF_UNKNOWN_TENANT_SCHEMA:%', schema_name; + END IF; + + FOREACH table_name IN ARRAY ARRAY[ + 'documents', + 'posts', + 'posts_translations', + 'articles', + 'articles_chunks', + 'bulk_items', + 'app_files', + 'function_invocations' + ] + LOOP + EXECUTE format( + 'ALTER TABLE %I.%I ADD COLUMN physical_database_identity text', + schema_name, + table_name + ); + EXECUTE format( + 'UPDATE %I.%I SET physical_database_identity = pg_catalog.current_database()::text', + schema_name, + table_name + ); + EXECUTE format( + 'ALTER TABLE %I.%I ALTER COLUMN physical_database_identity SET NOT NULL', + schema_name, + table_name + ); + EXECUTE format( + 'ALTER TABLE %I.%I ALTER COLUMN physical_database_identity SET DEFAULT pg_catalog.current_database()::text', + schema_name, + table_name + ); + END LOOP; + + -- The i18n and RAG plugins return derived/custom shapes rather than every + -- source-table column. Stamp their returned text as an operation-specific + -- oracle in addition to the root/database field selected by the probe. + EXECUTE format( + 'UPDATE %I.posts_translations SET title = title || %L || pg_catalog.current_database()::text', + schema_name, + ' @' + ); + EXECUTE format( + 'UPDATE %I.articles_chunks SET content = content || %L || pg_catalog.current_database()::text', + schema_name, + ' @' + ); + + EXECUTE format($ddl$ + CREATE FUNCTION %I.stamp_physical_database_identity() + RETURNS trigger + LANGUAGE plpgsql + SECURITY INVOKER + SET search_path = pg_catalog + AS $function$ + BEGIN + NEW.physical_database_identity := pg_catalog.current_database()::text; + RETURN NEW; + END + $function$ + $ddl$, schema_name); + + FOREACH table_name IN ARRAY ARRAY[ + 'documents', + 'posts', + 'posts_translations', + 'articles', + 'articles_chunks', + 'bulk_items', + 'app_files', + 'function_invocations' + ] + LOOP + EXECUTE format( + 'CREATE TRIGGER %I BEFORE INSERT OR UPDATE ON %I.%I FOR EACH ROW EXECUTE FUNCTION %I.stamp_physical_database_identity()', + table_name || '_physical_database_identity', + schema_name, + table_name, + schema_name + ); + END LOOP; +END +$procedure$; + +CALL pg_temp.add_physical_response_oracles('ctf_a'); +CALL pg_temp.add_physical_response_oracles('ctf_b'); +CALL pg_temp.add_physical_response_oracles('ctf_c'); + +REVOKE ALL ON FUNCTION ctf_a.physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_b.physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_c.physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_a.physical_database_mutation_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_b.physical_database_mutation_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_c.physical_database_mutation_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_a.stamp_realtime_physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_b.stamp_realtime_physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_c.stamp_realtime_physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_a.stamp_physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_b.stamp_physical_database_identity() FROM PUBLIC; +REVOKE ALL ON FUNCTION ctf_c.stamp_physical_database_identity() FROM PUBLIC; + +GRANT EXECUTE ON FUNCTION ctf_a.physical_database_identity() TO :"runtime_role_a"; +GRANT EXECUTE ON FUNCTION ctf_b.physical_database_identity() TO :"runtime_role_b"; +GRANT EXECUTE ON FUNCTION ctf_c.physical_database_identity() TO :"runtime_role_c"; +GRANT EXECUTE ON FUNCTION ctf_a.physical_database_mutation_identity() TO :"runtime_role_a"; +GRANT EXECUTE ON FUNCTION ctf_b.physical_database_mutation_identity() TO :"runtime_role_b"; +GRANT EXECUTE ON FUNCTION ctf_c.physical_database_mutation_identity() TO :"runtime_role_c"; +GRANT EXECUTE ON FUNCTION ctf_a.stamp_realtime_physical_database_identity() TO :"runtime_role_a"; +GRANT EXECUTE ON FUNCTION ctf_b.stamp_realtime_physical_database_identity() TO :"runtime_role_b"; +GRANT EXECUTE ON FUNCTION ctf_c.stamp_realtime_physical_database_identity() TO :"runtime_role_c"; +GRANT EXECUTE ON FUNCTION ctf_a.stamp_physical_database_identity() TO :"runtime_role_a"; +GRANT EXECUTE ON FUNCTION ctf_b.stamp_physical_database_identity() TO :"runtime_role_b"; +GRANT EXECUTE ON FUNCTION ctf_c.stamp_physical_database_identity() TO :"runtime_role_c"; diff --git a/research/graphile-density/physical-database-density/prepare-measurement-run.cjs b/research/graphile-density/physical-database-density/prepare-measurement-run.cjs new file mode 100644 index 0000000000..ac762b393f --- /dev/null +++ b/research/graphile-density/physical-database-density/prepare-measurement-run.cjs @@ -0,0 +1,672 @@ +'use strict'; + +const { execFileSync, spawnSync } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + FIXTURE_ID, + loadProvision, +} = require('./lib.cjs'); +const { + parseArgs, + parsePositiveInteger, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); +const { provision } = require('./provision.cjs'); + +const TEMPLATE_KIND = 'physical-density-postgres-container-template-v1'; +const PREPARE_KIND = 'physical-density-measurement-prepare-v1'; +const CONTAINER_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/; +const CONTAINER_ID = /^[a-f0-9]{64}$/; +const IMAGE_ID = /^sha256:[a-f0-9]{64}$/; +const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost']); +const LABEL_FIXTURE = 'io.constructive.graphile-density.fixture'; +const LABEL_PREFIX = 'io.constructive.graphile-density.prefix'; +const LABEL_PURPOSE = 'io.constructive.graphile-density.purpose'; +const POSTGRES_SETTINGS = new Set([ + 'effective_cache_size', + 'maintenance_work_mem', + 'max_connections', + 'max_locks_per_transaction', + 'shared_buffers', + 'shared_preload_libraries', + 'track_io_timing', + 'work_mem', +]); +const POSTGRES_DATA_DIRECTORY = '/var/lib/postgresql/data'; + +const readRegularFile = (file) => { + const absolute = path.resolve(file); + const before = fs.lstatSync(absolute); + if (before.isSymbolicLink() || !before.isFile()) { + throw new Error('PDCF_MEASUREMENT_TEMPLATE_FILE_INVALID'); + } + const descriptor = fs.openSync( + absolute, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + try { + const opened = fs.fstatSync(descriptor); + if ( + !opened.isFile() + || opened.dev !== before.dev + || opened.ino !== before.ino + ) { + throw new Error('PDCF_MEASUREMENT_TEMPLATE_FILE_INVALID'); + } + return fs.readFileSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +}; + +const fileSha256 = (file) => `sha256:${crypto.createHash('sha256') + .update(readRegularFile(file)) + .digest('hex')}`; +const bufferSha256 = (value) => `sha256:${crypto.createHash('sha256') + .update(value) + .digest('hex')}`; + +const inspectDockerContainer = (container) => { + try { + const output = execFileSync('docker', ['inspect', container], { + encoding: 'utf8', + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }); + const records = JSON.parse(output); + return Array.isArray(records) && records.length === 1 ? records[0] : null; + } catch { + return null; + } +}; + +const requireNonnegativeInteger = (value, label) => { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`PDCF_CONTAINER_TEMPLATE_${label}_INVALID`); + } + return value; +}; + +const normalizePostgresCommand = (command, minimumMaxConnections) => { + const source = Array.isArray(command) && command.length > 0 + ? [...command] + : ['postgres']; + if ( + source[0] !== 'postgres' + || (source.length - 1) % 2 !== 0 + || !Number.isSafeInteger(minimumMaxConnections) + || minimumMaxConnections <= 0 + ) { + throw new Error('PDCF_CONTAINER_POSTGRES_COMMAND_INVALID'); + } + const settings = new Map(); + for (let index = 1; index < source.length; index += 2) { + const flag = source[index]; + const assignment = source[index + 1]; + const match = /^([a-z_]+)=([a-zA-Z0-9_.,:/+-]+)$/.exec(assignment ?? ''); + if ( + flag !== '-c' + || !match + || !POSTGRES_SETTINGS.has(match[1]) + || settings.has(match[1]) + ) { + throw new Error('PDCF_CONTAINER_POSTGRES_COMMAND_INVALID'); + } + settings.set(match[1], match[2]); + } + const configuredMax = Number(settings.get('max_connections')); + if (settings.has('max_connections') && ( + !Number.isSafeInteger(configuredMax) + || configuredMax < minimumMaxConnections + )) { + throw new Error('PDCF_CONTAINER_MAX_CONNECTIONS_INSUFFICIENT'); + } + if (!settings.has('max_connections')) { + settings.set('max_connections', String(minimumMaxConnections)); + } + return [ + 'postgres', + ...[...settings].sort(([left], [right]) => left.localeCompare(right)) + .flatMap(([name, value]) => ['-c', `${name}=${value}`]), + ]; +}; + +const postgresSettingsFromCommand = (command) => Object.fromEntries( + Array.from({ length: (command.length - 1) / 2 }, (_, index) => { + const [name, value] = command[index * 2 + 2].split('=', 2); + return [name, value]; + }), +); + +const captureContainerTemplate = ({ + inspection, + container, + prefix, + pgHost, + pgPort, + minimumMaxConnections, +}) => { + const name = String(inspection?.Name ?? '').replace(/^\//, ''); + const bindings = inspection?.NetworkSettings?.Ports?.['5432/tcp']; + const entrypoint = inspection?.Config?.Entrypoint ?? null; + const mounts = inspection?.Mounts ?? []; + if ( + !CONTAINER_NAME.test(container ?? '') + || !/^[a-z][a-z0-9_]{0,47}$/.test(prefix ?? '') + || !LOOPBACK_HOSTS.has(pgHost) + || !Number.isSafeInteger(pgPort) + || pgPort <= 0 + || pgPort > 65535 + || name !== container + || !CONTAINER_ID.test(inspection?.Id ?? '') + || !IMAGE_ID.test(inspection?.Image ?? '') + || inspection?.State?.Running !== true + || !Array.isArray(bindings) + || !bindings.some((binding) => + String(binding?.HostPort) === String(pgPort) + && LOOPBACK_HOSTS.has(binding?.HostIp ?? '') + ) + || !Array.isArray(mounts) + || mounts.some((mount) => mount?.Destination !== POSTGRES_DATA_DIRECTORY) + || ( + entrypoint != null + && ( + !Array.isArray(entrypoint) + || entrypoint.length !== 1 + || !/^[a-zA-Z0-9_./-]+$/.test(entrypoint[0] ?? '') + ) + ) + ) { + throw new Error('PDCF_CONTAINER_TEMPLATE_SOURCE_INVALID'); + } + const template = { + version: 1, + fixture: FIXTURE_ID, + kind: TEMPLATE_KIND, + containerName: container, + sourceContainerId: inspection.Id, + imageId: inspection.Image, + prefix, + pgHost, + pgPort, + entrypoint, + postgresCommand: normalizePostgresCommand( + inspection.Config?.Cmd, + minimumMaxConnections, + ), + resourceLimits: { + memoryBytes: requireNonnegativeInteger( + inspection.HostConfig?.Memory ?? 0, + 'MEMORY', + ), + memorySwapBytes: requireNonnegativeInteger( + inspection.HostConfig?.MemorySwap ?? 0, + 'MEMORY_SWAP', + ), + nanoCpus: requireNonnegativeInteger( + inspection.HostConfig?.NanoCpus ?? 0, + 'NANO_CPUS', + ), + shmSizeBytes: requireNonnegativeInteger( + inspection.HostConfig?.ShmSize ?? 0, + 'SHM_SIZE', + ), + }, + }; + return validateContainerTemplate(template); +}; + +const validateContainerTemplate = (template) => { + if ( + JSON.stringify(Object.keys(template ?? {}).sort()) !== JSON.stringify([ + 'containerName', + 'entrypoint', + 'fixture', + 'imageId', + 'kind', + 'pgHost', + 'pgPort', + 'postgresCommand', + 'prefix', + 'resourceLimits', + 'sourceContainerId', + 'version', + ]) + || JSON.stringify(Object.keys(template?.resourceLimits ?? {}).sort()) + !== JSON.stringify([ + 'memoryBytes', + 'memorySwapBytes', + 'nanoCpus', + 'shmSizeBytes', + ]) + || + template?.version !== 1 + || template.fixture !== FIXTURE_ID + || template.kind !== TEMPLATE_KIND + || !CONTAINER_NAME.test(template.containerName ?? '') + || !CONTAINER_ID.test(template.sourceContainerId ?? '') + || !IMAGE_ID.test(template.imageId ?? '') + || !/^[a-z][a-z0-9_]{0,47}$/.test(template.prefix ?? '') + || !LOOPBACK_HOSTS.has(template.pgHost) + || !Number.isSafeInteger(template.pgPort) + || template.pgPort <= 0 + || template.pgPort > 65535 + || !template.resourceLimits + || !Array.isArray(template.postgresCommand) + || ( + template.entrypoint != null + && ( + !Array.isArray(template.entrypoint) + || template.entrypoint.length !== 1 + || !/^[a-zA-Z0-9_./-]+$/.test(template.entrypoint[0] ?? '') + ) + ) + ) { + throw new Error('PDCF_CONTAINER_TEMPLATE_INVALID'); + } + for (const [key, value] of Object.entries(template.resourceLimits)) { + requireNonnegativeInteger(value, key.toUpperCase()); + } + normalizePostgresCommand( + template.postgresCommand, + Number(postgresSettingsFromCommand(template.postgresCommand).max_connections), + ); + return template; +}; + +const validateExistingTarget = (inspection, template) => { + if (!inspection) return null; + const name = String(inspection.Name ?? '').replace(/^\//, ''); + const labels = inspection.Config?.Labels ?? {}; + const ownedReplacement = labels[LABEL_FIXTURE] === FIXTURE_ID + && labels[LABEL_PREFIX] === template.prefix + && labels[LABEL_PURPOSE] === 'measurement' + && inspection.Image === template.imageId; + if ( + name !== template.containerName + || !CONTAINER_ID.test(inspection.Id ?? '') + || (inspection.Id !== template.sourceContainerId && !ownedReplacement) + ) { + throw new Error('PDCF_CONTAINER_RECREATE_TARGET_NOT_OWNED'); + } + return inspection.Id; +}; + +const dockerRunArgs = (template, environment) => { + const pgUser = environment.PGUSER; + const pgPassword = environment.PGPASSWORD; + const maintenanceDatabase = environment.PGDATABASE ?? 'postgres'; + if ( + typeof pgUser !== 'string' + || !pgUser + || typeof pgPassword !== 'string' + || !pgPassword + || typeof maintenanceDatabase !== 'string' + || !maintenanceDatabase + ) { + throw new Error('PDCF_CONTAINER_ADMIN_CREDENTIALS_REQUIRED'); + } + const limits = template.resourceLimits; + const args = [ + 'run', '--detach', + '--name', template.containerName, + '--label', `${LABEL_FIXTURE}=${FIXTURE_ID}`, + '--label', `${LABEL_PREFIX}=${template.prefix}`, + '--label', `${LABEL_PURPOSE}=measurement`, + '--publish', `127.0.0.1:${template.pgPort}:5432`, + '--env', 'POSTGRES_USER', + '--env', 'POSTGRES_PASSWORD', + '--env', 'POSTGRES_DB', + ]; + if (template.entrypoint) { + args.push('--entrypoint', template.entrypoint[0]); + } + if (limits.memoryBytes > 0) args.push('--memory', String(limits.memoryBytes)); + if (limits.memorySwapBytes > 0) { + args.push('--memory-swap', String(limits.memorySwapBytes)); + } + if (limits.nanoCpus > 0) args.push('--cpus', String(limits.nanoCpus / 1e9)); + if (limits.shmSizeBytes > 0) args.push('--shm-size', String(limits.shmSizeBytes)); + args.push(template.imageId, ...template.postgresCommand); + return args; +}; + +const validateRunningContainerAgainstTemplate = (inspection, template) => { + const labels = inspection?.Config?.Labels ?? {}; + const bindings = inspection?.NetworkSettings?.Ports?.['5432/tcp']; + if ( + !CONTAINER_ID.test(inspection?.Id ?? '') + || inspection.Image !== template.imageId + || inspection.State?.Running !== true + || JSON.stringify(inspection.Config?.Cmd ?? []) + !== JSON.stringify(template.postgresCommand) + || JSON.stringify(inspection.Config?.Entrypoint ?? null) + !== JSON.stringify(template.entrypoint) + || inspection.HostConfig?.Memory !== template.resourceLimits.memoryBytes + || inspection.HostConfig?.MemorySwap !== template.resourceLimits.memorySwapBytes + || inspection.HostConfig?.NanoCpus !== template.resourceLimits.nanoCpus + || inspection.HostConfig?.ShmSize !== template.resourceLimits.shmSizeBytes + || !Array.isArray(bindings) + || !bindings.some((binding) => + String(binding?.HostPort) === String(template.pgPort) + && LOOPBACK_HOSTS.has(binding?.HostIp ?? '') + ) + || labels[LABEL_FIXTURE] !== FIXTURE_ID + || labels[LABEL_PREFIX] !== template.prefix + || labels[LABEL_PURPOSE] !== 'measurement' + ) { + throw new Error('PDCF_FRESH_POSTGRES_CONTAINER_IDENTITY_INVALID'); + } + return inspection; +}; + +const dockerExec = (args, environment = process.env) => { + try { + return execFileSync('docker', args, { + encoding: 'utf8', + timeout: 120_000, + maxBuffer: 4 * 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + env: environment, + }); + } catch { + // Keep Docker diagnostics generic. The disposable admin password is passed + // only through the child environment and must never enter cperf logs. + throw new Error('PDCF_DOCKER_LIFECYCLE_COMMAND_FAILED'); + } +}; + +const waitForPostgres = ({ environment, timeoutMs = 120_000 }) => { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const result = spawnSync('psql', [ + '--no-psqlrc', + '--no-align', + '--tuples-only', + '--quiet', + '--set=ON_ERROR_STOP=1', + '--command', 'SELECT 1', + ], { + cwd: __dirname, + env: environment, + encoding: 'utf8', + timeout: 5_000, + }); + if (result.status === 0 && String(result.stdout).trim() === '1') return; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 250); + } + throw new Error('PDCF_FRESH_POSTGRES_READINESS_TIMEOUT'); +}; + +const runBindingCloneId = (run, randomBytes = crypto.randomBytes) => { + const coordinate = [ + run.arm, + run.heapMiB, + run.customerCount, + run.repetition, + run.runOrderIndex, + ].join('\0'); + const coordinateHash = crypto.createHash('sha256').update(coordinate).digest('hex'); + return `measurement-${coordinateHash.slice(0, 20)}-${randomBytes(8).toString('hex')}`; +}; + +const publishExclusive = (source, destination, mode) => { + fs.linkSync(source, destination); + if (mode != null) fs.chmodSync(destination, mode); + fs.unlinkSync(source); +}; + +const writeImmutableJson = (file, value, mode = 0o644) => { + const temporary = `${file}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`; + try { + fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { + flag: 'wx', + mode, + }); + publishExclusive(temporary, file, mode); + } finally { + try { fs.unlinkSync(temporary); } catch { /* Preserve the primary error. */ } + } +}; + +const prepareMeasurementRun = ({ + containerTemplateFile, + expectedContainerTemplateSha256, + manifestTemplateFile, + secretsTemplateFile, + expectedManifestTemplateSha256, + artifactDir, + run, + environment = process.env, +}, dependencies = {}) => { + const containerTemplateBytes = readRegularFile(containerTemplateFile); + if ( + !/^sha256:[a-f0-9]{64}$/.test(expectedContainerTemplateSha256 ?? '') + || bufferSha256(containerTemplateBytes) !== expectedContainerTemplateSha256 + ) { + throw new Error('PDCF_CONTAINER_TEMPLATE_SHA256_MISMATCH'); + } + const template = validateContainerTemplate(JSON.parse( + containerTemplateBytes.toString('utf8'), + )); + const manifestTemplateBytes = readRegularFile(manifestTemplateFile); + if ( + !/^sha256:[a-f0-9]{64}$/.test(expectedManifestTemplateSha256 ?? '') + || bufferSha256(manifestTemplateBytes) !== expectedManifestTemplateSha256 + ) { + throw new Error('PDCF_MEASUREMENT_MANIFEST_TEMPLATE_SHA256_MISMATCH'); + } + const absoluteArtifactDir = path.resolve(artifactDir); + fs.mkdirSync(absoluteArtifactDir, { recursive: true, mode: 0o700 }); + const manifestOut = path.join(absoluteArtifactDir, 'provision.json'); + const secretsOut = path.join(absoluteArtifactDir, 'runtime-secrets.json'); + const prepareOut = path.join(absoluteArtifactDir, 'prepare-attestation.json'); + if ([manifestOut, secretsOut, prepareOut].some((file) => fs.existsSync(file))) { + throw new Error('PDCF_FRESH_POSTGRES_ARTIFACT_ALREADY_EXISTS'); + } + if ( + !run + || typeof run.arm !== 'string' + || !run.arm + || !Number.isSafeInteger(run.heapMiB) + || run.heapMiB <= 0 + || !Number.isSafeInteger(run.customerCount) + || run.customerCount <= 0 + || !Number.isSafeInteger(run.repetition) + || run.repetition <= 0 + || !Number.isSafeInteger(run.runOrderIndex) + || run.runOrderIndex <= 0 + ) { + throw new Error('PDCF_MEASUREMENT_PREPARE_RUN_BINDING_INVALID'); + } + const { manifest: sourceManifest, secretResolver } = ( + dependencies.loadProvision ?? loadProvision + )(manifestTemplateFile, secretsTemplateFile); + if (fileSha256(manifestTemplateFile) !== expectedManifestTemplateSha256) { + throw new Error('PDCF_MEASUREMENT_MANIFEST_TEMPLATE_SHA256_MISMATCH'); + } + if ( + sourceManifest.prefix !== template.prefix + || sourceManifest.customers.length < run.customerCount + || sourceManifest.provisionClone?.purpose !== 'measurement' + ) { + throw new Error('PDCF_MEASUREMENT_PREPARE_TEMPLATE_MISMATCH'); + } + const selectedCustomers = sourceManifest.customers.slice(0, run.customerCount); + const credentialTemplate = { + runtimePasswords: Object.fromEntries(selectedCustomers.flatMap((customer) => + Object.values(customer.roles).map((role) => [ + role, + secretResolver.runtimePasswordFor(role), + ]) + )), + notificationPasswords: Object.fromEntries(selectedCustomers.map((customer) => [ + customer.notificationRole, + secretResolver.notificationPasswordFor(customer.notificationRole), + ])), + }; + const inspect = dependencies.inspectDockerContainer ?? inspectDockerContainer; + const removeContainer = dependencies.removeContainer + ?? ((container) => dockerExec(['container', 'rm', '--force', '--volumes', container])); + const startContainer = dependencies.startContainer + ?? ((args) => dockerExec(args, { + ...process.env, + POSTGRES_USER: environment.PGUSER, + POSTGRES_PASSWORD: environment.PGPASSWORD, + POSTGRES_DB: environment.PGDATABASE ?? 'postgres', + })); + const existing = inspect(template.containerName); + const existingTargetId = validateExistingTarget(existing, template); + if (existingTargetId) { + // Remove the immutable ID we just attested, not the mutable container name; + // this closes the inspect/remove name-swap window around a destructive call. + removeContainer(existingTargetId); + } + startContainer(dockerRunArgs(template, environment)); + const postgresEnvironment = { + ...environment, + PGHOST: template.pgHost, + PGPORT: String(template.pgPort), + }; + (dependencies.waitForPostgres ?? waitForPostgres)({ + environment: postgresEnvironment, + }); + + const cloneId = runBindingCloneId(run, dependencies.randomBytes); + const stagingDir = fs.mkdtempSync(path.join(absoluteArtifactDir, '.prepare-')); + let provisioned; + try { + provisioned = (dependencies.provision ?? provision)({ + prefix: template.prefix, + customerCount: run.customerCount, + outDir: stagingDir, + maintenanceDatabase: environment.PGDATABASE ?? 'postgres', + schemaFile: path.join(__dirname, '../complete-tenant-fixture/schema.sql'), + identityFile: path.join(__dirname, 'physical-identity.sql'), + attestationFile: path.join(__dirname, 'provision-attestation.sql'), + cloneId, + runPurpose: 'measurement', + recreate: false, + environment: postgresEnvironment, + canonicalSchemas: sourceManifest.canonicalSchemas, + credentialTemplate, + }); + if ( + provisioned.manifest.canonicalDatabaseContractFingerprint + !== sourceManifest.canonicalDatabaseContractFingerprint + || JSON.stringify(provisioned.manifest.canonicalStructuralFingerprint) + !== JSON.stringify(sourceManifest.canonicalStructuralFingerprint) + || JSON.stringify(provisioned.manifest.canonicalSchemas) + !== JSON.stringify(sourceManifest.canonicalSchemas) + ) { + throw new Error('PDCF_FRESH_POSTGRES_CONTRACT_DRIFT'); + } + publishExclusive(provisioned.secretsFile, secretsOut, 0o600); + publishExclusive(provisioned.manifestFile, manifestOut, 0o644); + fs.rmdirSync(stagingDir); + } catch (error) { + try { + if (fs.existsSync(stagingDir)) fs.rmSync(stagingDir, { recursive: true }); + } catch { + // Preserve the primary preparation error. + } + throw error; + } + const current = validateRunningContainerAgainstTemplate( + inspect(template.containerName), + template, + ); + if ( + current.Id === existing?.Id + ) { + throw new Error('PDCF_FRESH_POSTGRES_CONTAINER_IDENTITY_INVALID'); + } + const result = { + version: 1, + fixture: FIXTURE_ID, + kind: PREPARE_KIND, + preparedAt: new Date().toISOString(), + run, + container: { + name: template.containerName, + id: current.Id, + imageId: current.Image, + }, + cloneId, + manifestFile: manifestOut, + manifestSha256: fileSha256(manifestOut), + secretsFile: secretsOut, + credentialTemplate: 'private-0600-reused-without-serialization', + }; + writeImmutableJson(prepareOut, result); + return result; +}; + +const main = () => { + const args = parseArgs(process.argv.slice(2)); + const result = prepareMeasurementRun({ + containerTemplateFile: path.resolve(requireString(args, 'container-template')), + expectedContainerTemplateSha256: requireString( + args, + 'expected-container-template-sha256', + ), + manifestTemplateFile: path.resolve(requireString(args, 'manifest-template')), + secretsTemplateFile: path.resolve(requireString(args, 'secrets-template')), + expectedManifestTemplateSha256: requireString( + args, + 'expected-manifest-template-sha256', + ), + artifactDir: path.resolve(requireString(args, 'artifact-dir')), + run: { + arm: requireString(args, 'arm'), + heapMiB: parsePositiveInteger(requireString(args, 'heap-mib'), 'heap-mib'), + customerCount: parsePositiveInteger( + requireString(args, 'customers'), + 'customers', + ), + repetition: parsePositiveInteger( + requireString(args, 'repetition'), + 'repetition', + ), + runOrderIndex: parsePositiveInteger( + requireString(args, 'run-order-index'), + 'run-order-index', + ), + }, + }); + process.stdout.write(`${JSON.stringify({ + status: 'prepared', + containerId: result.container.id, + cloneId: result.cloneId, + manifestSha256: result.manifestSha256, + })}\n`); +}; + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} + +module.exports = { + PREPARE_KIND, + TEMPLATE_KIND, + captureContainerTemplate, + dockerRunArgs, + inspectDockerContainer, + normalizePostgresCommand, + postgresSettingsFromCommand, + prepareMeasurementRun, + publishExclusive, + runBindingCloneId, + validateContainerTemplate, + validateExistingTarget, + validateRunningContainerAgainstTemplate, + writeImmutableJson, +}; diff --git a/research/graphile-density/physical-database-density/prepare-measurement-run.test.cjs b/research/graphile-density/physical-database-density/prepare-measurement-run.test.cjs new file mode 100644 index 0000000000..2760498e7a --- /dev/null +++ b/research/graphile-density/physical-database-density/prepare-measurement-run.test.cjs @@ -0,0 +1,240 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { + captureContainerTemplate, + dockerRunArgs, + prepareMeasurementRun, + validateExistingTarget, +} = require('./prepare-measurement-run.cjs'); + +const sha256 = (file) => `sha256:${crypto.createHash('sha256') + .update(fs.readFileSync(file)) + .digest('hex')}`; + +const sourceInspection = ({ id = '1'.repeat(64), labels = {} } = {}) => ({ + Id: id, + Name: '/postgres-density-exact', + Image: `sha256:${'2'.repeat(64)}`, + State: { Running: true }, + Config: { + Cmd: [ + 'postgres', + '-c', 'shared_buffers=256MB', + '-c', 'max_connections=240', + ], + Labels: labels, + }, + HostConfig: { + Memory: 2 * 1024 ** 3, + MemorySwap: 2 * 1024 ** 3, + NanoCpus: 2_000_000_000, + ShmSize: 256 * 1024 ** 2, + }, + NetworkSettings: { + Ports: { '5432/tcp': [{ HostIp: '127.0.0.1', HostPort: '55432' }] }, + }, +}); + +const templateFrom = (inspection = sourceInspection()) => captureContainerTemplate({ + inspection, + container: 'postgres-density-exact', + prefix: 'pdc_test', + pgHost: '127.0.0.1', + pgPort: 55432, + minimumMaxConnections: 120, +}); + +describe('fresh PostgreSQL measurement preparation', () => { + it('preserves validated PostgreSQL settings and never falls back to default max_connections', () => { + const template = templateFrom(); + assert.deepEqual(template.postgresCommand, [ + 'postgres', + '-c', 'max_connections=240', + '-c', 'shared_buffers=256MB', + ]); + const args = dockerRunArgs(template, { + PGUSER: 'fixture_admin', + PGPASSWORD: 'private-admin-password', + PGDATABASE: 'postgres', + }); + assert.deepEqual(args.slice(args.indexOf(template.imageId)), [ + template.imageId, + ...template.postgresCommand, + ]); + assert.ok(args.includes('max_connections=240')); + assert.ok(args.includes('shared_buffers=256MB')); + assert.equal(args.some((argument) => argument.includes('private-admin-password')), false); + + const defaultTemplate = templateFrom({ + ...sourceInspection(), + Config: { Cmd: ['postgres'], Labels: {} }, + }); + assert.ok(defaultTemplate.postgresCommand.includes('max_connections=120')); + assert.throws(() => templateFrom({ + ...sourceInspection(), + Config: { + Cmd: ['postgres', '-c', 'max_connections=80'], + Labels: {}, + }, + }), /PDCF_CONTAINER_MAX_CONNECTIONS_INSUFFICIENT/); + }); + + it('will remove only the captured container or its exact owned replacement', () => { + const template = templateFrom(); + assert.equal(validateExistingTarget(sourceInspection(), template), '1'.repeat(64)); + assert.throws(() => validateExistingTarget(sourceInspection({ + id: '3'.repeat(64), + }), template), /PDCF_CONTAINER_RECREATE_TARGET_NOT_OWNED/); + assert.equal(validateExistingTarget(sourceInspection({ + id: '3'.repeat(64), + labels: { + 'io.constructive.graphile-density.fixture': + 'physical-database-density-v1', + 'io.constructive.graphile-density.prefix': 'pdc_test', + 'io.constructive.graphile-density.purpose': 'measurement', + }, + }), template), '3'.repeat(64)); + }); + + it('publishes run-local inputs with stable private credentials and a unique clone', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-prepare-')); + const artifactDir = path.join(directory, 'artifact'); + const containerTemplateFile = path.join(directory, 'container-template.json'); + const manifestTemplateFile = path.join(directory, 'provision.json'); + const secretsTemplateFile = path.join(directory, 'runtime-secrets.json'); + const template = templateFrom(); + fs.writeFileSync(containerTemplateFile, JSON.stringify(template)); + fs.writeFileSync(manifestTemplateFile, JSON.stringify({ template: true })); + fs.writeFileSync(secretsTemplateFile, '{}', { mode: 0o600 }); + const customer = { + id: 'physical-customer-0001', + roles: { a: 'role_a', b: 'role_b', c: 'role_c' }, + notificationRole: 'role_notify', + }; + const unselectedCustomer = { + id: 'physical-customer-0002', + roles: { a: 'role_2a', b: 'role_2b', c: 'role_2c' }, + notificationRole: 'role_2notify', + }; + const sourceManifest = { + prefix: 'pdc_test', + canonicalSchemas: ['ctf_a'], + canonicalStructuralFingerprint: { combined: { sha256: `sha256:${'4'.repeat(64)}` } }, + canonicalDatabaseContractFingerprint: `sha256:${'5'.repeat(64)}`, + provisionClone: { purpose: 'measurement' }, + customers: [customer, unselectedCustomer], + }; + const passwords = { + role_a: 'stable-password-role-a-123456', + role_b: 'stable-password-role-b-123456', + role_c: 'stable-password-role-c-123456', + role_notify: 'stable-password-notify-12345', + role_2a: 'unselected-password-role-2a', + role_2b: 'unselected-password-role-2b', + role_2c: 'unselected-password-role-2c', + role_2notify: 'unselected-password-notify-2', + }; + let inspectionCount = 0; + const replacementSource = sourceInspection({ + id: '6'.repeat(64), + labels: { + 'io.constructive.graphile-density.fixture': + 'physical-database-density-v1', + 'io.constructive.graphile-density.prefix': 'pdc_test', + 'io.constructive.graphile-density.purpose': 'measurement', + }, + }); + const replacement = { + ...replacementSource, + Config: { + ...replacementSource.Config, + Cmd: template.postgresCommand, + }, + }; + const removed = []; + const result = prepareMeasurementRun({ + containerTemplateFile, + expectedContainerTemplateSha256: sha256(containerTemplateFile), + manifestTemplateFile, + secretsTemplateFile, + expectedManifestTemplateSha256: sha256(manifestTemplateFile), + artifactDir, + run: { + arm: 'candidate', + heapMiB: 2048, + customerCount: 1, + repetition: 1, + runOrderIndex: 3, + }, + environment: { + PGHOST: '127.0.0.1', + PGPORT: '55432', + PGUSER: 'fixture_admin', + PGPASSWORD: 'admin-password', + PGDATABASE: 'postgres', + }, + }, { + inspectDockerContainer: () => inspectionCount++ === 0 + ? sourceInspection() + : replacement, + removeContainer: (container) => removed.push(container), + startContainer: () => undefined, + waitForPostgres: () => undefined, + randomBytes: () => Buffer.alloc(8, 7), + loadProvision: () => ({ + manifest: sourceManifest, + secretResolver: { + runtimePasswordFor: (role) => passwords[role], + notificationPasswordFor: (role) => passwords[role], + }, + }), + provision: (options) => { + assert.equal(options.customerCount, 1); + assert.deepEqual(options.credentialTemplate, { + runtimePasswords: { + role_a: passwords.role_a, + role_b: passwords.role_b, + role_c: passwords.role_c, + }, + notificationPasswords: { role_notify: passwords.role_notify }, + }); + const manifestFile = path.join(options.outDir, 'provision.json'); + const secretsFile = path.join(options.outDir, 'runtime-secrets.json'); + fs.writeFileSync(manifestFile, JSON.stringify({ cloneId: options.cloneId })); + fs.writeFileSync(secretsFile, JSON.stringify(passwords), { mode: 0o600 }); + return { + manifest: { + canonicalSchemas: sourceManifest.canonicalSchemas, + canonicalStructuralFingerprint: + sourceManifest.canonicalStructuralFingerprint, + canonicalDatabaseContractFingerprint: + sourceManifest.canonicalDatabaseContractFingerprint, + }, + manifestFile, + secretsFile, + }; + }, + }); + assert.deepEqual(removed, ['1'.repeat(64)]); + assert.equal(result.container.id, '6'.repeat(64)); + assert.match(result.cloneId, /^measurement-[a-f0-9]{20}-0707070707070707$/); + assert.equal(fs.statSync(result.secretsFile).mode & 0o777, 0o600); + const serialized = JSON.stringify({ + result, + prepare: JSON.parse(fs.readFileSync( + path.join(artifactDir, 'prepare-attestation.json'), + 'utf8', + )), + }); + for (const password of Object.values(passwords)) { + assert.doesNotMatch(serialized, new RegExp(password)); + } + }); +}); diff --git a/research/graphile-density/physical-database-density/provision-attestation.sql b/research/graphile-density/physical-database-density/provision-attestation.sql new file mode 100644 index 0000000000..6f977601de --- /dev/null +++ b/research/graphile-density/physical-database-density/provision-attestation.sql @@ -0,0 +1,35 @@ +\set ON_ERROR_STOP on + +BEGIN; + +CREATE SCHEMA ctf_provision_private; +REVOKE ALL ON SCHEMA ctf_provision_private FROM PUBLIC; + +CREATE TABLE ctf_provision_private.clone_attestation ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + clone_id text NOT NULL CHECK (clone_id ~ '^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$'), + run_purpose text NOT NULL CHECK (run_purpose IN ('hostile-preflight', 'measurement')), + customer_id text NOT NULL CHECK (customer_id ~ '^[a-z0-9-]+$'), + attestation_nonce text NOT NULL CHECK (attestation_nonce ~ '^[a-f0-9]{64}$'), + attestation_sha256 text NOT NULL CHECK (attestation_sha256 ~ '^sha256:[a-f0-9]{64}$') +); + +REVOKE ALL ON TABLE ctf_provision_private.clone_attestation FROM PUBLIC; + +INSERT INTO ctf_provision_private.clone_attestation ( + singleton, + clone_id, + run_purpose, + customer_id, + attestation_nonce, + attestation_sha256 +) VALUES ( + true, + :'clone_id', + :'run_purpose', + :'customer_id', + :'attestation_nonce', + :'attestation_sha256' +); + +COMMIT; diff --git a/research/graphile-density/physical-database-density/provision.cjs b/research/graphile-density/physical-database-density/provision.cjs new file mode 100644 index 0000000000..b6240f5c32 --- /dev/null +++ b/research/graphile-density/physical-database-density/provision.cjs @@ -0,0 +1,837 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + FIXTURE_DIR, + FIXTURE_ID, + atomicWriteJson, + makeCustomers, +} = require('./lib.cjs'); +const { + TENANTS, + parseArgs, + parsePositiveInteger, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); + +const DEFAULT_CANONICAL_SCHEMAS = Object.freeze([ + // These shared schemas are visible to every build through type/function + // dependencies, so a tenant-only fingerprint would be insufficient proof + // that two physical databases are blueprint-compatible. + 'ctf_extensions', + 'ctf_a', + 'ctf_a_realtime', + 'ctf_b', + 'ctf_b_realtime', + 'ctf_c', + 'ctf_c_realtime', + 'jwt_private', +]); +const REQUIRED_EXTENSIONS = Object.freeze([ + 'ltree', + 'pg_textsearch', + 'pg_trgm', + 'postgis', + 'vector', +]); +const RUN_PURPOSES = Object.freeze(['hostile-preflight', 'measurement']); +const CLONE_ID_PATTERN = /^[a-z0-9][a-z0-9._-]{0,127}$/i; +const ATTESTATION_NONCE_PATTERN = /^[a-f0-9]{64}$/; + +const quoteIdentifier = (value) => `"${String(value).replace(/"/g, '""')}"`; +const quoteLiteral = (value) => `'${String(value).replace(/'/g, "''")}'`; + +const requireCloneId = (value) => { + if (typeof value !== 'string' || !CLONE_ID_PATTERN.test(value)) { + throw new Error('PDCF_CLONE_ID_INVALID'); + } + return value; +}; + +const requireRunPurpose = (value) => { + if (!RUN_PURPOSES.includes(value)) throw new Error('PDCF_RUN_PURPOSE_INVALID'); + return value; +}; + +const provisionAttestationSha256 = ({ + cloneId, + runPurpose, + customerId, + database, + nonce, +}) => { + requireCloneId(cloneId); + requireRunPurpose(runPurpose); + if (typeof customerId !== 'string' || !customerId) { + throw new Error('PDCF_ATTESTATION_CUSTOMER_ID_INVALID'); + } + if (typeof database !== 'string' || !database) { + throw new Error('PDCF_ATTESTATION_DATABASE_INVALID'); + } + if (typeof nonce !== 'string' || !ATTESTATION_NONCE_PATTERN.test(nonce)) { + throw new Error('PDCF_ATTESTATION_NONCE_INVALID'); + } + const digest = crypto.createHash('sha256'); + for (const value of [ + 'physical-database-density-provision-attestation-v1', + cloneId, + runPurpose, + customerId, + database, + nonce, + ]) { + digest.update(value); + digest.update('\0'); + } + return `sha256:${digest.digest('hex')}`; +}; + +const provisionAttestationSetSha256 = (customers) => sha256Json( + [...customers].map((customer) => ({ + customerId: customer.id, + database: customer.database, + sha256: customer.provisionAttestation.sha256, + })).sort((left, right) => left.customerId.localeCompare(right.customerId)), +); + +const run = (command, args, options = {}) => { + const result = spawnSync(command, args, { + cwd: options.cwd ?? FIXTURE_DIR, + env: options.env ?? process.env, + encoding: 'utf8', + input: options.input, + maxBuffer: 64 * 1024 * 1024, + }); + if (result.status !== 0) { + const detail = result.stderr?.trim() || result.stdout?.trim() || result.error?.message; + throw new Error(`PDCF_COMMAND_FAILED:${command}:${detail || `exit=${result.status}`}`); + } + return result.stdout ?? ''; +}; + +const psql = (database, sql, environment = process.env) => run( + 'psql', + ['--no-psqlrc', '--set=ON_ERROR_STOP=1', '--dbname', database], + { input: `${sql}\n`, env: environment }, +); + +const applySqlFile = (database, file, roles, environment = process.env) => run( + 'psql', + [ + '--no-psqlrc', + '--set=ON_ERROR_STOP=1', + '--dbname', database, + '--set', `runtime_role_a=${roles.a}`, + '--set', `runtime_role_b=${roles.b}`, + '--set', `runtime_role_c=${roles.c}`, + '--file', file, + ], + { env: environment }, +); + +const applyProvisionAttestation = ( + database, + file, + { cloneId, runPurpose, customerId, nonce, sha256 }, + environment = process.env, +) => run( + 'psql', + [ + '--no-psqlrc', + '--set=ON_ERROR_STOP=1', + '--dbname', database, + '--set', `clone_id=${cloneId}`, + '--set', `run_purpose=${runPurpose}`, + '--set', `customer_id=${customerId}`, + '--set', `attestation_nonce=${nonce}`, + '--set', `attestation_sha256=${sha256}`, + '--file', file, + ], + { env: environment }, +); + +const normalizeSchemaDump = (dump, roleAliases = {}) => { + const normalizedRoles = Object.entries(roleAliases) + .sort(([left], [right]) => right.length - left.length); + const normalizeRoles = (line) => normalizedRoles.reduce((value, [role, alias]) => { + const escaped = role.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return value.replace( + new RegExp(`(^|[^a-zA-Z0-9_])${escaped}(?=$|[^a-zA-Z0-9_])`, 'g'), + `$1${alias}`, + ); + }, line); + return `${dump + .split(/\r?\n/) + .filter((line) => !( + line.startsWith('\\restrict ') + || line.startsWith('\\unrestrict ') + || line.startsWith('-- Dumped from database version') + || line.startsWith('-- Dumped by pg_dump version') + || line.startsWith('-- Started on ') + || line.startsWith('-- Completed on ') + )) + .map(normalizeRoles) + .join('\n') + .trim()}\n`; +}; + +const fingerprintDump = ( + database, + schemas, + environment = process.env, + roleAliases = {}, +) => { + const args = [ + '--schema-only', + '--no-owner', + '--dbname', database, + ...schemas.flatMap((schema) => ['--schema', schema]), + ]; + // ACLs are part of Graphile's effective catalog. Preserve them in the dump, + // but replace per-customer login names with stable tenant slots so equivalent + // least-privilege grants compare byte-for-byte across physical databases. + const normalized = normalizeSchemaDump( + run('pg_dump', args, { env: environment }), + roleAliases, + ); + return { + sha256: `sha256:${crypto.createHash('sha256').update(normalized).digest('hex')}`, + bytes: Buffer.byteLength(normalized), + }; +}; + +const structuralFingerprints = (database, schemas, roles, environment) => { + const roleAliases = Object.fromEntries(TENANTS.map((tenant) => [ + roles[tenant.id], + `__runtime_${tenant.id}__`, + ])); + return { + combined: fingerprintDump(database, schemas, environment, roleAliases), + schemas: Object.fromEntries(schemas.map((schema) => [ + schema, + fingerprintDump(database, [schema], environment, roleAliases), + ])), + }; +}; + +const parseJsonRows = (stdout, label) => { + const value = stdout.trim(); + try { + return value ? JSON.parse(value) : []; + } catch { + throw new Error(`PDCF_${label}_JSON_INVALID`); + } +}; + +const sha256Json = (value) => `sha256:${crypto.createHash('sha256') + .update(JSON.stringify(value)) + .digest('hex')}`; + +const normalizedRoleSafetyProfile = (roles, roleAudit) => TENANTS.map((tenant) => { + const roleName = roles[tenant.id]; + const row = roleAudit.find((candidate) => candidate.role_name === roleName); + if (!row) throw new Error(`PDCF_ROLE_AUDIT_PROFILE_INCOMPLETE:${tenant.id}`); + const { role_name: _roleName, ...flags } = row; + return { slot: tenant.id, ...flags }; +}); + +const inspectCustomerContract = ({ + customer, + canonicalSchemas, + environment = process.env, +}) => { + const roleAudit = auditRuntimeRoles(customer.database, customer.roles, environment); + const roleSafetyProfile = normalizedRoleSafetyProfile(customer.roles, roleAudit); + const notificationRoleAudit = auditNotificationRole( + customer.database, + customer.notificationRole, + environment, + ); + const { + role_name: _notificationRoleName, + ...notificationRoleSafetyProfile + } = notificationRoleAudit; + const extensionVersions = auditExtensionVersions(customer.database, environment); + const fingerprints = structuralFingerprints( + customer.database, + canonicalSchemas, + customer.roles, + environment, + ); + const databaseContractFingerprint = sha256Json({ + structuralFingerprints: fingerprints, + extensionVersions, + roleSafetyProfile, + notificationRoleSafetyProfile, + }); + return { + roleAudit, + roleSafetyProfile, + notificationRoleAudit, + notificationRoleSafetyProfile, + extensionVersions, + structuralFingerprints: fingerprints, + databaseContractFingerprint, + }; +}; + +const auditNotificationRole = (database, role, environment) => { + const sql = ` +COPY ( + SELECT row_to_json(audit) + FROM ( + SELECT r.rolname AS role_name, + r.rolcanlogin AS can_login, + NOT r.rolinherit AS noinherit, + NOT r.rolsuper AS not_superuser, + NOT r.rolbypassrls AS no_bypassrls, + NOT r.rolcreaterole AS no_createrole, + NOT r.rolcreatedb AS no_createdb, + NOT r.rolreplication AS no_replication, + NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_auth_members membership + WHERE membership.member = r.oid OR membership.roleid = r.oid + ) AS no_membership, + pg_catalog.has_database_privilege(r.rolname, ${quoteLiteral(database)}, 'CONNECT') + AS target_connect, + NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_database d + WHERE d.datname <> ${quoteLiteral(database)} + AND pg_catalog.has_database_privilege(r.rolname, d.oid, 'CONNECT') + ) AS no_cross_database_connect, + NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_database d + WHERE d.datname = ${quoteLiteral(database)} + AND d.datdba = r.oid + ) AS not_database_owner, + NOT pg_catalog.has_database_privilege( + r.rolname, ${quoteLiteral(database)}, 'CREATE' + ) AS no_database_create, + NOT pg_catalog.has_database_privilege( + r.rolname, ${quoteLiteral(database)}, 'TEMP' + ) AS no_database_temp, + NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_namespace n + WHERE n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_' + AND ( + n.nspowner = r.oid + OR pg_catalog.has_schema_privilege(r.rolname, n.oid, 'CREATE') + OR pg_catalog.has_schema_privilege(r.rolname, n.oid, 'USAGE') + ) + ) AS no_application_schema_access, + NOT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_' + AND ( + CASE WHEN c.relkind IN ('r', 'p', 'v', 'm', 'f') THEN + pg_catalog.has_table_privilege( + r.rolname, c.oid, + 'SELECT,INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER' + ) + OR pg_catalog.has_any_column_privilege( + r.rolname, c.oid, 'SELECT,INSERT,UPDATE,REFERENCES' + ) + WHEN c.relkind = 'S' THEN + pg_catalog.has_sequence_privilege( + r.rolname, c.oid, 'USAGE,SELECT,UPDATE' + ) + ELSE false END + ) + ) AS no_application_relation_access, + NOT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_' + AND pg_catalog.has_function_privilege(r.rolname, p.oid, 'EXECUTE') + ) AS no_application_function_access + FROM pg_catalog.pg_roles r + WHERE r.rolname = ${quoteLiteral(role)} + ) audit +) TO STDOUT; +`; + const row = parseJsonRows(psql(database, sql, environment), 'NOTIFICATION_ROLE_AUDIT'); + const required = [ + 'can_login', + 'noinherit', + 'not_superuser', + 'no_bypassrls', + 'no_createrole', + 'no_createdb', + 'no_replication', + 'no_membership', + 'target_connect', + 'no_cross_database_connect', + 'not_database_owner', + 'no_database_create', + 'no_database_temp', + 'no_application_schema_access', + 'no_application_relation_access', + 'no_application_function_access', + ]; + if ( + !row + || Array.isArray(row) + || row.role_name !== role + || required.some((field) => row[field] !== true) + ) { + throw new Error(`PDCF_NOTIFICATION_ROLE_AUDIT_FAILED:${database}:${role}`); + } + return row; +}; + +const auditExtensionVersions = (database, environment) => { + const sql = ` +COPY ( + SELECT COALESCE( + pg_catalog.json_agg(row_to_json(extension_row) ORDER BY extension_row.name), + '[]'::json + ) + FROM ( + SELECT e.extname AS name, + e.extversion AS version, + n.nspname AS schema + FROM pg_catalog.pg_extension e + JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace + WHERE e.extname = ANY(ARRAY[${REQUIRED_EXTENSIONS.map(quoteLiteral).join(', ')}]::text[]) + ) extension_row +) TO STDOUT; +`; + const rows = parseJsonRows(psql(database, sql, environment), 'EXTENSION_AUDIT'); + if ( + !Array.isArray(rows) + || rows.length !== REQUIRED_EXTENSIONS.length + || rows.some((row, index) => + row.name !== REQUIRED_EXTENSIONS[index] + || typeof row.version !== 'string' + || row.version.length === 0 + || row.schema !== 'ctf_extensions' + ) + ) { + throw new Error(`PDCF_EXTENSION_AUDIT_FAILED:${database}`); + } + return rows; +}; + +const auditRuntimeRoles = (database, roles, environment) => { + const names = Object.values(roles); + const sql = ` +COPY ( + SELECT pg_catalog.json_agg(row_to_json(audit) ORDER BY audit.role_name) + FROM ( + SELECT r.rolname AS role_name, + r.rolcanlogin AS can_login, + NOT r.rolinherit AS noinherit, + NOT r.rolsuper AS not_superuser, + NOT r.rolbypassrls AS no_bypassrls, + NOT r.rolcreaterole AS no_createrole, + NOT r.rolcreatedb AS no_createdb, + NOT r.rolreplication AS no_replication, + NOT pg_catalog.has_database_privilege(r.rolname, current_database(), 'CREATE') AS no_database_create, + NOT pg_catalog.has_schema_privilege( + r.rolname, + 'ctf_provision_private', + 'USAGE' + ) AS no_provision_attestation_schema_usage, + NOT ( + pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'SELECT' + ) + OR pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'INSERT' + ) + OR pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'UPDATE' + ) + OR pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'DELETE' + ) + OR pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'TRUNCATE' + ) + OR pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'REFERENCES' + ) + OR pg_catalog.has_table_privilege( + r.rolname, + 'ctf_provision_private.clone_attestation', + 'TRIGGER' + ) + ) AS no_provision_attestation_table_privileges, + NOT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace n + WHERE n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_' + AND ( + n.nspowner = r.oid + OR pg_catalog.has_schema_privilege(r.rolname, n.oid, 'CREATE') + ) + ) AS no_schema_owner_or_create + FROM pg_catalog.pg_roles r + WHERE r.rolname = ANY(ARRAY[${names.map(quoteLiteral).join(', ')}]::text[]) + ) audit +) TO STDOUT; +`; + const rows = parseJsonRows(psql(database, sql, environment), 'ROLE_AUDIT'); + if (!Array.isArray(rows) || rows.length !== names.length) { + throw new Error(`PDCF_ROLE_AUDIT_INCOMPLETE:${database}`); + } + const required = [ + 'can_login', + 'noinherit', + 'not_superuser', + 'no_bypassrls', + 'no_createrole', + 'no_createdb', + 'no_replication', + 'no_database_create', + 'no_provision_attestation_schema_usage', + 'no_provision_attestation_table_privileges', + 'no_schema_owner_or_create', + ]; + for (const row of rows) { + if (required.some((field) => row[field] !== true)) { + throw new Error(`PDCF_ROLE_AUDIT_FAILED:${database}:${row.role_name}`); + } + } + return rows; +}; + +const provisionCustomer = ({ + customer, + passwords, + maintenanceDatabase, + schemaFile, + identityFile, + attestationFile, + provisionAttestation, + recreate, + environment, + canonicalSchemas, +}) => { + if (recreate) { + psql(maintenanceDatabase, ` +SELECT pg_catalog.pg_terminate_backend(pid) +FROM pg_catalog.pg_stat_activity +WHERE datname = ${quoteLiteral(customer.database)} + AND pid <> pg_catalog.pg_backend_pid(); +DROP DATABASE IF EXISTS ${quoteIdentifier(customer.database)}; +${Object.values(customer.roles).map((role) => + `DROP ROLE IF EXISTS ${quoteIdentifier(role)};` + ).join('\n')} +DROP ROLE IF EXISTS ${quoteIdentifier(customer.notificationRole)}; +`, environment); + } + + psql(maintenanceDatabase, Object.entries(customer.roles).map(([tenantId, role]) => ` +CREATE ROLE ${quoteIdentifier(role)} + LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD ${quoteLiteral(passwords[role])}; +COMMENT ON ROLE ${quoteIdentifier(role)} IS ${quoteLiteral(`${FIXTURE_ID}:${customer.id}:${tenantId}`)}; +`).join('\n') + ` +CREATE ROLE ${quoteIdentifier(customer.notificationRole)} + LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD ${quoteLiteral(passwords[customer.notificationRole])}; +COMMENT ON ROLE ${quoteIdentifier(customer.notificationRole)} IS ${quoteLiteral( + `${FIXTURE_ID}:${customer.id}:notification-only` + )}; +`, environment); + + psql(maintenanceDatabase, ` +CREATE DATABASE ${quoteIdentifier(customer.database)}; +REVOKE ALL ON DATABASE ${quoteIdentifier(customer.database)} FROM PUBLIC; +GRANT CONNECT ON DATABASE ${quoteIdentifier(customer.database)} TO ${[ + ...Object.values(customer.roles), + customer.notificationRole, + ] + .map(quoteIdentifier) + .join(', ')}; +COMMENT ON DATABASE ${quoteIdentifier(customer.database)} IS ${quoteLiteral(`${FIXTURE_ID}:${customer.id}`)}; +`, environment); + + applySqlFile(customer.database, schemaFile, customer.roles, environment); + applySqlFile(customer.database, identityFile, customer.roles, environment); + applyProvisionAttestation( + customer.database, + attestationFile, + provisionAttestation, + environment, + ); + + const contract = inspectCustomerContract({ + customer, + canonicalSchemas, + environment, + }); + return { + ...customer, + provisionAttestation: { + version: 1, + cloneId: provisionAttestation.cloneId, + purpose: provisionAttestation.runPurpose, + sha256: provisionAttestation.sha256, + }, + ...contract, + }; +}; + +const provision = ({ + prefix, + customerCount, + outDir, + maintenanceDatabase, + schemaFile, + identityFile, + attestationFile, + cloneId, + runPurpose, + recreate, + environment = process.env, + canonicalSchemas = DEFAULT_CANONICAL_SCHEMAS, + credentialTemplate = null, +}) => { + cloneId = requireCloneId(cloneId); + runPurpose = requireRunPurpose(runPurpose); + const customers = makeCustomers(prefix, customerCount); + const requiredRuntimeRoles = customers.flatMap((customer) => + Object.values(customer.roles) + ).sort(); + const requiredNotificationRoles = customers.map((customer) => + customer.notificationRole + ).sort(); + const assertExactPasswords = (value, roles, label) => { + if ( + !value + || typeof value !== 'object' + || Array.isArray(value) + || JSON.stringify(Object.keys(value).sort()) !== JSON.stringify(roles) + || roles.some((role) => + typeof value[role] !== 'string' || Buffer.byteLength(value[role]) < 24 + ) + ) { + throw new Error(`PDCF_${label}_CREDENTIAL_TEMPLATE_INVALID`); + } + return Object.fromEntries(roles.map((role) => [role, value[role]])); + }; + const runtimePasswords = credentialTemplate + ? assertExactPasswords( + credentialTemplate.runtimePasswords, + requiredRuntimeRoles, + 'RUNTIME', + ) + : Object.fromEntries(requiredRuntimeRoles.map((role) => [ + role, + crypto.randomBytes(32).toString('base64url'), + ])); + const notificationPasswords = credentialTemplate + ? assertExactPasswords( + credentialTemplate.notificationPasswords, + requiredNotificationRoles, + 'NOTIFICATION', + ) + : Object.fromEntries(requiredNotificationRoles.map((role) => [ + role, + crypto.randomBytes(32).toString('base64url'), + ])); + const allPasswords = { ...runtimePasswords, ...notificationPasswords }; + + // The notification-role contract rejects CONNECT to every other database. + // This fixture owns a disposable PostgreSQL cluster, so remove PostgreSQL's + // default PUBLIC grants before any per-customer role is audited. + psql(maintenanceDatabase, ` +DO $revoke_public_connect$ +DECLARE + database_record record; +BEGIN + FOR database_record IN SELECT datname FROM pg_catalog.pg_database + LOOP + EXECUTE pg_catalog.format( + 'REVOKE CONNECT ON DATABASE %I FROM PUBLIC', + database_record.datname + ); + END LOOP; +END +$revoke_public_connect$; +`, environment); + const provisioned = customers.map((customer) => { + const nonce = crypto.randomBytes(32).toString('hex'); + const provisionAttestation = { + cloneId, + runPurpose, + customerId: customer.id, + nonce, + sha256: provisionAttestationSha256({ + cloneId, + runPurpose, + customerId: customer.id, + database: customer.database, + nonce, + }), + }; + return provisionCustomer({ + customer, + passwords: allPasswords, + maintenanceDatabase, + schemaFile, + identityFile, + attestationFile, + provisionAttestation, + recreate, + environment, + canonicalSchemas, + }); + }); + const expected = provisioned[0].structuralFingerprints; + const expectedDatabaseContractFingerprint = provisioned[0].databaseContractFingerprint; + for (const customer of provisioned.slice(1)) { + if (customer.databaseContractFingerprint !== expectedDatabaseContractFingerprint) { + throw new Error( + `PDCF_DATABASE_CONTRACT_MISMATCH:${provisioned[0].database}:${customer.database}` + ); + } + if (customer.structuralFingerprints.combined.sha256 !== expected.combined.sha256) { + throw new Error( + `PDCF_CANONICAL_SCHEMA_MISMATCH:${provisioned[0].database}:${customer.database}` + ); + } + for (const schema of canonicalSchemas) { + if ( + customer.structuralFingerprints.schemas[schema].sha256 + !== expected.schemas[schema].sha256 + ) { + throw new Error(`PDCF_CANONICAL_SCHEMA_MISMATCH:${schema}:${customer.database}`); + } + } + } + const manifest = { + version: 1, + fixture: FIXTURE_ID, + prefix, + createdAt: new Date().toISOString(), + provisionClone: { + version: 1, + id: cloneId, + purpose: runPurpose, + attestationSetSha256: provisionAttestationSetSha256(provisioned), + }, + canonicalSchemas, + canonicalStructuralFingerprint: expected, + canonicalDatabaseContractFingerprint: expectedDatabaseContractFingerprint, + pgDumpVersion: run('pg_dump', ['--version'], { env: environment }).trim(), + customers: provisioned, + }; + const secrets = { + version: 1, + fixture: FIXTURE_ID, + runtimePasswords, + notificationPasswords, + }; + const manifestFile = path.join(outDir, 'provision.json'); + const secretsFile = path.join(outDir, 'runtime-secrets.json'); + atomicWriteJson(manifestFile, manifest); + atomicWriteJson(secretsFile, secrets, 0o600); + return { manifest, manifestFile, secretsFile }; +}; + +const main = () => { + const args = parseArgs(process.argv.slice(2)); + const recreate = args.recreate === true; + if (recreate && args.yes !== true) throw new Error('PDCF_RECREATE_REQUIRES_YES'); + const prefix = requireString(args, 'prefix', 'pdc_density'); + const customerCount = parsePositiveInteger(args.customers ?? '3', 'customers'); + const outDir = path.resolve(requireString( + args, + 'out-dir', + path.join(FIXTURE_DIR, '.local'), + )); + const maintenanceDatabase = requireString( + args, + 'maintenance-database', + process.env.PGDATABASE ?? 'postgres', + ); + const schemaFile = path.resolve(requireString( + args, + 'schema-file', + path.join(FIXTURE_DIR, '../complete-tenant-fixture/schema.sql'), + )); + const identityFile = path.resolve(requireString( + args, + 'identity-file', + path.join(FIXTURE_DIR, 'physical-identity.sql'), + )); + const attestationFile = path.resolve(requireString( + args, + 'attestation-file', + path.join(FIXTURE_DIR, 'provision-attestation.sql'), + )); + const result = provision({ + prefix, + customerCount, + outDir, + maintenanceDatabase, + schemaFile, + identityFile, + attestationFile, + cloneId: requireString(args, 'clone-id'), + runPurpose: requireString(args, 'run-purpose'), + recreate, + }); + process.stdout.write(`${JSON.stringify({ + status: 'provisioned', + customers: result.manifest.customers.length, + canonicalStructuralFingerprint: + result.manifest.canonicalStructuralFingerprint.combined.sha256, + provisionClone: result.manifest.provisionClone, + manifestFile: result.manifestFile, + secretsFile: result.secretsFile, + })}\n`); +}; + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + } +} + +module.exports = { + DEFAULT_CANONICAL_SCHEMAS, + REQUIRED_EXTENSIONS, + auditExtensionVersions, + auditNotificationRole, + auditRuntimeRoles, + fingerprintDump, + inspectCustomerContract, + normalizeSchemaDump, + normalizedRoleSafetyProfile, + provision, + provisionAttestationSetSha256, + provisionAttestationSha256, + requireCloneId, + requireRunPurpose, + structuralFingerprints, +}; diff --git a/research/graphile-density/physical-database-density/server-realtime.test.cjs b/research/graphile-density/physical-database-density/server-realtime.test.cjs new file mode 100644 index 0000000000..89ed9f3d36 --- /dev/null +++ b/research/graphile-density/physical-database-density/server-realtime.test.cjs @@ -0,0 +1,85 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { EventEmitter } = require('node:events'); +const { describe, it } = require('node:test'); + +const { + createRealtimeConnectionRegistry, + matchPhysicalUpgradeRoute, +} = require('./server.cjs'); + +const socket = () => Object.assign(new EventEmitter(), { + destroyed: false, + destroy() { this.destroyed = true; }, +}); + +describe('physical density server-side realtime accounting', () => { + it('routes only one exact customer and tenant upgrade path', () => { + assert.deepEqual( + matchPhysicalUpgradeRoute( + '/customer/physical-customer-0007/tenant/b/graphql' + ), + { customerId: 'physical-customer-0007', tenantId: 'b' } + ); + assert.equal(matchPhysicalUpgradeRoute('/customer/c1/tenant/b/graphql?token=x'), null); + assert.equal(matchPhysicalUpgradeRoute('/customer/c1/graphql'), null); + assert.equal(matchPhysicalUpgradeRoute('/customer/c1/tenant/b/other'), null); + }); + + it('counts one accepted live socket per surface without retaining clients', () => { + const registry = createRealtimeConnectionRegistry(['customer-1:a', 'customer-1:b']); + const a = socket(); + const b = socket(); + assert.equal(registry.trackAccepted('customer-1:a', a), true); + assert.equal(registry.trackAccepted('customer-1:b', b), true); + assert.deepEqual(registry.assertResident(), { + connectionsExpected: 2, + connectionsAccepted: 2, + connectionsActive: 2, + connectionDrops: 0, + connectionErrors: 0, + connectionsPerSurface: [ + { + key: 'customer-1:a', + accepted: 1, + active: 1, + peakActive: 1, + drops: 0, + errors: 0, + }, + { + key: 'customer-1:b', + accepted: 1, + active: 1, + peakActive: 1, + drops: 0, + errors: 0, + }, + ], + }); + }); + + it('fails residency after a drop, error, duplicate, or unknown route', () => { + const registry = createRealtimeConnectionRegistry(['customer-1:a']); + const accepted = socket(); + registry.trackAccepted('customer-1:a', accepted); + accepted.emit('error', new Error('reset')); + assert.throws( + () => registry.assertResident(), + /PDCF_REALTIME_CONNECTIONS_NOT_RESIDENT:0:1:1:1/ + ); + + const duplicateRegistry = createRealtimeConnectionRegistry(['customer-1:a']); + duplicateRegistry.trackAccepted('customer-1:a', socket()); + duplicateRegistry.trackAccepted('customer-1:a', socket()); + assert.throws( + () => duplicateRegistry.assertResident(), + /PDCF_REALTIME_CONNECTIONS_NOT_RESIDENT/ + ); + + const unknown = socket(); + assert.equal(duplicateRegistry.trackAccepted('customer-2:a', unknown), false); + assert.equal(unknown.destroyed, true); + }); +}); diff --git a/research/graphile-density/physical-database-density/server-retained-memory.test.cjs b/research/graphile-density/physical-database-density/server-retained-memory.test.cjs new file mode 100644 index 0000000000..b36f327bc2 --- /dev/null +++ b/research/graphile-density/physical-database-density/server-retained-memory.test.cjs @@ -0,0 +1,320 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { describe, it } = require('node:test'); + +const { + authorizeRetainedMemoryCheckpoint, + collectRetainedMemoryCheckpoint, + makeGraphileActivityVector, + makeRetainedMemoryGuard, + parseServerOptions, +} = require('./server.cjs'); + +const MIB = 1024 ** 2; + +const guard = (overrides = {}) => makeRetainedMemoryGuard({ + pid: 42, + graphileInFlight: 0, + residentBuildContracts: ['contract-a'], + graphileActivityByBuildContract: [{ + buildContract: 'contract-a', + inflight: 0, + websocketSockets: 0, + transientHttpInFlight: 0, + }], + cacheCounters: { + httpRequestsStarted: 0, + httpRequestsCompleted: 0, + websocketUpgradesStarted: 0, + websocketUpgradesCompleted: 0, + evictions: 0, + buildRefusals: 0, + }, + realtime: { + managersExpected: 0, + managersActive: 0, + connectionsExpected: 0, + connectionsAccepted: 0, + connectionsActive: 0, + connectionDrops: 0, + connectionErrors: 0, + connectionsPerSurface: [], + }, + buildCounters: { started: 1, succeeded: 1, failed: 0 }, + ...overrides, +}); + +const liveRealtimeGuard = (overrides = {}) => guard({ + graphileActivityByBuildContract: [{ + buildContract: 'contract-a', + inflight: 1, + websocketSockets: 1, + transientHttpInFlight: 0, + }], + cacheCounters: { + httpRequestsStarted: 4, + httpRequestsCompleted: 4, + websocketUpgradesStarted: 1, + websocketUpgradesCompleted: 0, + }, + realtime: { + managersExpected: 1, + managersActive: 1, + connectionsExpected: 1, + connectionsAccepted: 1, + connectionsActive: 1, + connectionDrops: 0, + connectionErrors: 0, + connectionsPerSurface: [{ + key: 'customer-1:a', + accepted: 1, + active: 1, + peakActive: 1, + drops: 0, + errors: 0, + }], + }, + ...overrides, +}); + +describe('physical density retained-memory checkpoint', () => { + it('requires the explicit benchmark flag and matching loopback bearer token', () => { + const request = (address, token) => ({ + socket: { remoteAddress: address }, + get: (name) => name === 'authorization' ? `Bearer ${token}` : undefined, + }); + const options = { + benchmarkRetainedHeapEnabled: true, + observabilityToken: 'checkpoint-secret', + }; + assert.equal(authorizeRetainedMemoryCheckpoint( + request('127.0.0.1', 'checkpoint-secret'), options + ), true); + assert.equal(authorizeRetainedMemoryCheckpoint( + request('10.0.0.2', 'checkpoint-secret'), options + ), false); + assert.equal(authorizeRetainedMemoryCheckpoint( + request('127.0.0.1', 'wrong-secret'), options + ), false); + assert.equal(authorizeRetainedMemoryCheckpoint( + request('127.0.0.1', 'checkpoint-secret'), { + ...options, + benchmarkRetainedHeapEnabled: false, + } + ), false); + }); + + it('parses the benchmark flag from the environment only', () => { + const options = parseServerOptions([ + '--manifest', '/tmp/provision.json', + '--secrets', '/tmp/secrets.json', + '--run-purpose', 'measurement', + '--clone-id', 'measurement-clone-test', + ], { + GRAPHQL_CPERF_RETAINED_HEAP_ENABLED: 'true', + GRAPHQL_OBSERVABILITY_TOKEN: 'checkpoint-secret', + }); + assert.equal(options.benchmarkRetainedHeapEnabled, true); + assert.equal(options.observabilityToken, 'checkpoint-secret'); + }); + + it('runs eight full-GC turns and accepts only converged last-three samples', async () => { + let gcCalls = 0; + let reads = 0; + let monotonic = 0n; + const heapMiB = [120, 112, 106, 103, 101, 100.2, 100.1, 100]; + const checkpoint = await collectRetainedMemoryCheckpoint({ + forceGc: () => { gcCalls += 1; }, + readMemory: () => ({ + heapUsed: Math.round(heapMiB[reads++] * MIB), + external: 10 * MIB, + arrayBuffers: 2 * MIB, + rss: 180 * MIB, + }), + readGuard: () => guard(), + monotonicNow: () => ++monotonic, + yieldTurn: async () => undefined, + }); + assert.equal(gcCalls, 8); + assert.equal(checkpoint.samples.length, 8); + assert.equal(checkpoint.stableSampleCount, 3); + assert.equal(checkpoint.stable, true); + assert.deepEqual(checkpoint.errors, []); + assert.equal(checkpoint.guardBefore.stateSha256, checkpoint.guardAfter.stateSha256); + }); + + it('subtracts expected long-lived sockets from exact per-contract activity', () => { + const vector = makeGraphileActivityVector([ + { + cacheKey: 'contract-b', + inflight: 3, + websocketSockets: new Set([{}, {}]), + }, + { + cacheKey: 'contract-a', + inflight: 1, + websocketSockets: new Set([{}]), + }, + ]); + assert.deepEqual(vector, [ + { + buildContract: 'contract-a', + inflight: 1, + websocketSockets: 1, + transientHttpInFlight: 0, + }, + { + buildContract: 'contract-b', + inflight: 3, + websocketSockets: 2, + transientHttpInFlight: 1, + }, + ]); + }); + + it('permits stable expected realtime sockets during full GC', async () => { + let gcCalls = 0; + let monotonic = 0n; + const checkpoint = await collectRetainedMemoryCheckpoint({ + forceGc: () => { gcCalls += 1; }, + readMemory: () => ({ + heapUsed: 100 * MIB, + external: 10 * MIB, + arrayBuffers: 2 * MIB, + rss: 180 * MIB, + }), + readGuard: () => liveRealtimeGuard(), + monotonicNow: () => ++monotonic, + yieldTurn: async () => undefined, + }); + assert.equal(gcCalls, 8); + assert.equal(checkpoint.stable, true); + assert.equal(checkpoint.guardBefore.graphileInFlight, 0); + assert.equal(checkpoint.guardBefore.graphileWebsocketSockets, 1); + assert.equal(checkpoint.guardBefore.realtimeResident, true); + }); + + it('hashes balanced handler lifecycles that begin and end between reads', () => { + const baseline = guard({ + cacheCounters: { + httpRequestsStarted: 4, + httpRequestsCompleted: 4, + websocketUpgradesStarted: 0, + websocketUpgradesCompleted: 0, + }, + }); + const shortHttpRequest = guard({ + cacheCounters: { + httpRequestsStarted: 5, + httpRequestsCompleted: 5, + websocketUpgradesStarted: 0, + websocketUpgradesCompleted: 0, + }, + }); + const shortWebsocket = guard({ + cacheCounters: { + httpRequestsStarted: 5, + httpRequestsCompleted: 5, + websocketUpgradesStarted: 1, + websocketUpgradesCompleted: 1, + }, + }); + assert.notEqual(baseline.stateSha256, shortHttpRequest.stateSha256); + assert.notEqual(shortHttpRequest.stateSha256, shortWebsocket.stateSha256); + }); + + it('binds pg-cache capacity and failure counters into the stable guard', () => { + const baseline = guard({ + pgCacheMonotonicCounters: { + capacityEvictions: 0, + capacityRefusals: 0, + disposalFailures: 0, + }, + }); + const changed = guard({ + pgCacheMonotonicCounters: { + capacityEvictions: 0, + capacityRefusals: 1, + disposalFailures: 0, + }, + }); + assert.notEqual(baseline.stateSha256, changed.stateSha256); + }); + + it('fails closed when heap convergence or process state changes', async () => { + let reads = 0; + let guards = 0; + const heapMiB = [100, 100, 100, 100, 100, 100, 104, 100]; + const checkpoint = await collectRetainedMemoryCheckpoint({ + forceGc: () => undefined, + readMemory: () => ({ + heapUsed: heapMiB[reads++] * MIB, + external: 10 * MIB, + arrayBuffers: 2 * MIB, + rss: 180 * MIB, + }), + readGuard: () => guard({ counter: guards++ }), + monotonicNow: (() => { + let value = 0n; + return () => ++value; + })(), + yieldTurn: async () => undefined, + }); + assert.equal(checkpoint.stable, false); + assert.ok(checkpoint.errors.some((error) => + error.startsWith('PDCF_RETAINED_HEAP_NOT_CONVERGED:') + )); + assert.ok(checkpoint.errors.includes( + 'PDCF_RETAINED_MEMORY_RESIDENCY_OR_COUNTERS_CHANGED' + )); + }); + + it('does not force GC while Graphile work is in flight', async () => { + let gcCalls = 0; + await assert.rejects(collectRetainedMemoryCheckpoint({ + forceGc: () => { gcCalls += 1; }, + readGuard: () => guard({ graphileInFlight: 1 }), + }), /PDCF_RETAINED_MEMORY_IN_FLIGHT:1/); + assert.equal(gcCalls, 0); + }); + + it('does not force GC when expected realtime sockets are missing', async () => { + let gcCalls = 0; + await assert.rejects(collectRetainedMemoryCheckpoint({ + forceGc: () => { gcCalls += 1; }, + readGuard: () => liveRealtimeGuard({ + graphileActivityByBuildContract: [{ + buildContract: 'contract-a', + inflight: 0, + websocketSockets: 0, + transientHttpInFlight: 0, + }], + cacheCounters: { + httpRequestsStarted: 4, + httpRequestsCompleted: 4, + websocketUpgradesStarted: 1, + websocketUpgradesCompleted: 1, + }, + realtime: { + managersExpected: 1, + managersActive: 1, + connectionsExpected: 1, + connectionsAccepted: 1, + connectionsActive: 0, + connectionDrops: 1, + connectionErrors: 0, + connectionsPerSurface: [{ + key: 'customer-1:a', + accepted: 1, + active: 0, + peakActive: 1, + drops: 1, + errors: 0, + }], + }, + }), + }), /PDCF_RETAINED_MEMORY_REALTIME_NOT_RESIDENT:0:0:1/); + assert.equal(gcCalls, 0); + }); +}); diff --git a/research/graphile-density/physical-database-density/server.cjs b/research/graphile-density/physical-database-density/server.cjs new file mode 100644 index 0000000000..7d5b9889f0 --- /dev/null +++ b/research/graphile-density/physical-database-density/server.cjs @@ -0,0 +1,1299 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + FIXTURE_DIR, + REPO_ROOT, + loadProvision, +} = require('./lib.cjs'); +const { + TENANTS, + parseArgs, + parsePositiveInteger, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); +const completeServer = require('../complete-tenant-fixture/server.cjs'); +const { + inspectCustomerContract, + provisionAttestationSetSha256, + requireCloneId, + requireRunPurpose, +} = require('./provision.cjs'); + +const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost']); +const RETAINED_MEMORY_GC_ROUNDS = 8; +const RETAINED_MEMORY_STABLE_SAMPLES = 3; +const MIB = 1024 ** 2; +const SECURITY_ENVIRONMENT_KEYS = Object.freeze([ + 'PGDATABASE', + 'PG_POOL_MAX_USES', + 'CTF_RUNTIME_A_PGUSER', + 'CTF_RUNTIME_B_PGUSER', + 'CTF_RUNTIME_C_PGUSER', + 'CTF_RUNTIME_A_PGPASSWORD', + 'CTF_RUNTIME_B_PGPASSWORD', + 'CTF_RUNTIME_C_PGPASSWORD', + 'CTF_NOTIFICATION_PGUSER', + 'CTF_NOTIFICATION_PGPASSWORD', +]); + +const requireBuilt = (relativePath) => require(path.join(REPO_ROOT, relativePath)); + +const matchPhysicalUpgradeRoute = (rawUrl) => { + if (typeof rawUrl !== 'string' || rawUrl.includes('?')) return null; + const match = /^\/customer\/([a-z0-9-]+)\/tenant\/([a-z0-9-]+)\/graphql$/.exec(rawUrl); + return match ? { customerId: match[1], tenantId: match[2] } : null; +}; + +const matchPhysicalUpgradeCustomer = (rawUrl) => + matchPhysicalUpgradeRoute(rawUrl)?.customerId ?? null; + +const createRealtimeConnectionRegistry = (surfaceKeys) => { + const states = new Map(surfaceKeys.map((key) => [key, { + key, + accepted: 0, + active: 0, + peakActive: 0, + drops: 0, + errors: 0, + }])); + const snapshot = () => { + const surfaces = [...states.values()].map((state) => ({ ...state })); + return { + connectionsExpected: states.size, + connectionsAccepted: surfaces.reduce((sum, state) => sum + state.accepted, 0), + connectionsActive: surfaces.reduce((sum, state) => sum + state.active, 0), + connectionDrops: surfaces.reduce((sum, state) => sum + state.drops, 0), + connectionErrors: surfaces.reduce((sum, state) => sum + state.errors, 0), + connectionsPerSurface: surfaces, + }; + }; + const trackAccepted = (key, socket) => { + const state = states.get(key); + if (!state) { + socket.destroy(); + return false; + } + state.accepted += 1; + state.active += 1; + state.peakActive = Math.max(state.peakActive, state.active); + let released = false; + const release = (errored) => { + if (released) return; + released = true; + state.active = Math.max(0, state.active - 1); + state.drops += 1; + if (errored) state.errors += 1; + }; + socket.once('error', () => release(true)); + socket.once('close', () => release(false)); + if (socket.destroyed) release(false); + return true; + }; + const assertResident = () => { + const current = snapshot(); + const exact = current.connectionsPerSurface.every((state) => + state.accepted === 1 + && state.active === 1 + && state.peakActive === 1 + && state.drops === 0 + && state.errors === 0 + ); + if (!exact || current.connectionsActive !== current.connectionsExpected) { + throw new Error( + `PDCF_REALTIME_CONNECTIONS_NOT_RESIDENT:${current.connectionsActive}:${current.connectionsExpected}:${current.connectionDrops}:${current.connectionErrors}` + ); + } + return current; + }; + return { assertResident, snapshot, trackAccepted }; +}; + +const parseBoolean = (value, label) => { + if (value === true || value === 'true' || value === '1') return true; + if (value === false || value === 'false' || value === '0' || value == null) return false; + throw new Error(`PDCF_INVALID_BOOLEAN:${label}`); +}; + +const parseRuntimePoolMaxUses = (value) => { + if (value === 'unlimited') return null; + if (typeof value !== 'string' || !/^[1-9]\d*$/.test(value)) { + throw new Error('PDCF_INVALID_MAX_USES:runtime-pool-max-uses'); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error('PDCF_INVALID_MAX_USES:runtime-pool-max-uses'); + } + return parsed; +}; + +const optionalSha256 = (value, label) => { + if (value == null) return null; + if (typeof value !== 'string' || !/^sha256:[a-f0-9]{64}$/.test(value)) { + throw new Error(`PDCF_INVALID_SHA256:${label}`); + } + return value; +}; + +const assertProvisionCloneManifest = (manifest, { cloneId, runPurpose }) => { + const provisionClone = manifest?.provisionClone; + if ( + JSON.stringify(Object.keys(provisionClone ?? {}).sort()) + !== JSON.stringify(['attestationSetSha256', 'id', 'purpose', 'version']) + || + provisionClone?.version !== 1 + || provisionClone.id !== cloneId + || provisionClone.purpose !== runPurpose + || !/^sha256:[a-f0-9]{64}$/.test(provisionClone.attestationSetSha256 ?? '') + ) { + throw new Error('PDCF_PROVISION_CLONE_MISMATCH'); + } + for (const customer of manifest.customers ?? []) { + const attestation = customer.provisionAttestation; + if ( + JSON.stringify(Object.keys(attestation ?? {}).sort()) + !== JSON.stringify(['cloneId', 'purpose', 'sha256', 'version']) + || + attestation?.version !== 1 + || attestation.cloneId !== cloneId + || attestation.purpose !== runPurpose + || !/^sha256:[a-f0-9]{64}$/.test(attestation.sha256 ?? '') + ) { + throw new Error(`PDCF_PROVISION_ATTESTATION_INVALID:${customer.id ?? 'unknown'}`); + } + } + if (provisionAttestationSetSha256(manifest.customers) !== provisionClone.attestationSetSha256) { + throw new Error('PDCF_PROVISION_ATTESTATION_SET_MISMATCH'); + } + return provisionClone; +}; + +const assertCustomerContract = (customer, contract) => { + if ( + contract.databaseContractFingerprint !== customer.databaseContractFingerprint + || JSON.stringify(contract.structuralFingerprints) + !== JSON.stringify(customer.structuralFingerprints) + ) { + throw new Error(`PDCF_LIVE_DATABASE_CONTRACT_MISMATCH:${customer.id}`); + } + return contract; +}; + +const parseServerOptions = (argv, environment = process.env) => { + const args = parseArgs(argv); + const host = requireString(args, 'host', '127.0.0.1'); + if (!LOOPBACK_HOSTS.has(host)) throw new Error('PDCF_SERVER_LOOPBACK_REQUIRED'); + const mode = requireString(args, 'mode', 'scoped-required'); + if (!['stock', 'scoped-required'].includes(mode)) { + throw new Error(`PDCF_INTROSPECTION_MODE_INVALID:${mode}`); + } + const introspectionClientReleaseMode = requireString( + args, + 'introspection-client-release-mode', + 'destroy', + ); + if (!['reuse', 'destroy'].includes(introspectionClientReleaseMode)) { + throw new Error( + `PDCF_INTROSPECTION_CLIENT_RELEASE_MODE_INVALID:${introspectionClientReleaseMode}` + ); + } + const realtimeNotificationMode = requireString( + args, + 'realtime-notification-mode', + 'dedicated', + ); + if (!['dedicated', 'shared-exact'].includes(realtimeNotificationMode)) { + throw new Error( + `PDCF_REALTIME_NOTIFICATION_MODE_INVALID:${realtimeNotificationMode}` + ); + } + const enableRealtime = parseBoolean(args['enable-realtime'], 'enable-realtime'); + if (!enableRealtime && realtimeNotificationMode !== 'dedicated') { + throw new Error('PDCF_SHARED_REALTIME_REQUIRES_REALTIME'); + } + return { + host, + port: parsePositiveInteger(args.port ?? '3410', 'port'), + arm: requireString(args, 'arm', 'physical-db-idle-30s'), + runPurpose: requireRunPurpose(requireString(args, 'run-purpose')), + cloneId: requireCloneId(requireString(args, 'clone-id')), + mode, + introspectionClientReleaseMode, + manifestFile: path.resolve(requireString(args, 'manifest')), + secretsFile: path.resolve(requireString(args, 'secrets')), + customerCount: parsePositiveInteger(args.customers ?? '1', 'customers'), + runtimePoolMax: parsePositiveInteger(args['runtime-pool-max'] ?? '2', 'runtime-pool-max'), + runtimePoolMaxUses: parseRuntimePoolMaxUses( + args['runtime-pool-max-uses'] ?? 'unlimited', + ), + enableRealtime, + realtimeNotificationMode, + realtimeCursorPollIntervalMs: parsePositiveInteger( + args['realtime-cursor-poll-ms'] ?? '5000', + 'realtime-cursor-poll-ms', + ), + realtimeCursorHeartbeatIntervalMs: parsePositiveInteger( + args['realtime-cursor-heartbeat-ms'] ?? '30000', + 'realtime-cursor-heartbeat-ms', + ), + expectedDatabaseContractFingerprint: optionalSha256( + args['expected-database-contract'], + 'expected-database-contract', + ), + blueprintCompatibilityFingerprint: optionalSha256( + args['blueprint-compatibility'], + 'blueprint-compatibility', + ), + expectedManifestSha256: optionalSha256( + args['expected-manifest-sha256'], + 'expected-manifest-sha256', + ), + observabilityToken: environment.GRAPHQL_OBSERVABILITY_TOKEN ?? '', + benchmarkRetainedHeapEnabled: parseBoolean( + environment.GRAPHQL_CPERF_RETAINED_HEAP_ENABLED, + 'GRAPHQL_CPERF_RETAINED_HEAP_ENABLED', + ), + }; +}; + +const withProcessEnvironment = async (overrides, callback) => { + const previous = Object.fromEntries(SECURITY_ENVIRONMENT_KEYS.map((key) => [ + key, + Object.prototype.hasOwnProperty.call(process.env, key) ? process.env[key] : undefined, + ])); + try { + for (const [key, value] of Object.entries(overrides)) { + if (value == null) delete process.env[key]; + else process.env[key] = value; + } + return await callback(); + } finally { + for (const [key, value] of Object.entries(previous)) { + if (value == null) delete process.env[key]; + else process.env[key] = value; + } + } +}; + +const isLoopbackRequest = (request) => { + const address = request.socket?.remoteAddress ?? ''; + return address === '127.0.0.1' || address === '::1' || address === '::ffff:127.0.0.1'; +}; + +const bearerToken = (request) => { + const value = request.get('authorization'); + return value?.startsWith('Bearer ') ? value.slice('Bearer '.length) : ''; +}; + +const tokenEqual = (left, right) => { + if (!left || !right) return false; + const leftBytes = Buffer.from(left); + const rightBytes = Buffer.from(right); + return leftBytes.length === rightBytes.length + && crypto.timingSafeEqual(leftBytes, rightBytes); +}; + +const canonicalJson = (value) => { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value && typeof value === 'object') { + return `{${Object.keys(value).sort().map((key) => + `${JSON.stringify(key)}:${canonicalJson(value[key])}` + ).join(',')}}`; + } + return JSON.stringify(value); +}; + +const makeGraphileActivityVector = (entries) => entries.map((entry) => { + const inflight = entry.inflight ?? 0; + const websocketSockets = entry.websocketSockets?.size ?? 0; + const transientHttpInFlight = inflight - websocketSockets; + if ( + !Number.isSafeInteger(inflight) + || inflight < 0 + || !Number.isSafeInteger(websocketSockets) + || websocketSockets < 0 + || transientHttpInFlight < 0 + ) { + throw new Error(`PDCF_GRAPHILE_ACTIVITY_ACCOUNTING_INVALID:${entry.cacheKey}`); + } + return { + buildContract: entry.cacheKey, + inflight, + websocketSockets, + transientHttpInFlight, + }; +}).sort((left, right) => left.buildContract.localeCompare(right.buildContract)); + +const counterValue = (value, label) => { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`PDCF_RETAINED_MEMORY_COUNTER_INVALID:${label}`); + } + return value; +}; + +const makeRetainedMemoryGuard = (state) => { + const residentBuildContracts = [...state.residentBuildContracts].sort(); + const graphileActivityByBuildContract = [ + ...(state.graphileActivityByBuildContract ?? []), + ].sort((left, right) => left.buildContract.localeCompare(right.buildContract)); + const realtime = state.realtime ?? {}; + const cacheCounters = state.cacheCounters ?? {}; + const graphileWebsocketSockets = graphileActivityByBuildContract.reduce( + (sum, entry) => sum + entry.websocketSockets, + 0, + ); + const graphileTransientHttpInFlight = graphileActivityByBuildContract.reduce( + (sum, entry) => sum + entry.transientHttpInFlight, + 0, + ); + const httpRequestsOutstanding = + counterValue(cacheCounters.httpRequestsStarted, 'httpRequestsStarted') + - counterValue(cacheCounters.httpRequestsCompleted, 'httpRequestsCompleted'); + const websocketUpgradesOutstanding = + counterValue(cacheCounters.websocketUpgradesStarted, 'websocketUpgradesStarted') + - counterValue(cacheCounters.websocketUpgradesCompleted, 'websocketUpgradesCompleted'); + const realtimeConnectionsExpected = counterValue( + realtime.connectionsExpected, + 'realtime.connectionsExpected', + ); + const realtimeConnectionsActive = counterValue( + realtime.connectionsActive, + 'realtime.connectionsActive', + ); + const realtimeConnectionsAccepted = counterValue( + realtime.connectionsAccepted, + 'realtime.connectionsAccepted', + ); + const realtimeConnectionDrops = counterValue( + realtime.connectionDrops, + 'realtime.connectionDrops', + ); + const realtimeConnectionErrors = counterValue( + realtime.connectionErrors, + 'realtime.connectionErrors', + ); + const realtimeManagersExpected = counterValue( + realtime.managersExpected, + 'realtime.managersExpected', + ); + const realtimeManagersActive = counterValue( + realtime.managersActive, + 'realtime.managersActive', + ); + const realtimePerSurface = realtime.connectionsPerSurface ?? []; + const activityContractsExact = + graphileActivityByBuildContract.length === residentBuildContracts.length + && graphileActivityByBuildContract.every( + (entry, index) => entry.buildContract === residentBuildContracts[index], + ); + const realtimeSocketsExactPerContract = realtimeConnectionsExpected === 0 + ? graphileActivityByBuildContract.every((entry) => entry.websocketSockets === 0) + : realtimeConnectionsExpected === graphileActivityByBuildContract.length + && graphileActivityByBuildContract.every((entry) => entry.websocketSockets === 1); + const realtimePerSurfaceExact = realtimePerSurface.length === realtimeConnectionsExpected + && realtimePerSurface.every((surface) => + surface.accepted === 1 + && surface.active === 1 + && surface.peakActive === 1 + && surface.drops === 0 + && surface.errors === 0 + ); + const realtimeResident = + realtimeManagersExpected === realtimeConnectionsExpected + && realtimeManagersActive === realtimeManagersExpected + && realtimeConnectionsAccepted === realtimeConnectionsExpected + && realtimeConnectionsActive === realtimeConnectionsExpected + && realtimeConnectionDrops === 0 + && realtimeConnectionErrors === 0 + && graphileWebsocketSockets === realtimeConnectionsExpected + && realtimeSocketsExactPerContract + && websocketUpgradesOutstanding === graphileWebsocketSockets + && realtimePerSurfaceExact; + const handlerAccountingExact = + activityContractsExact + && httpRequestsOutstanding === graphileTransientHttpInFlight + && websocketUpgradesOutstanding === graphileWebsocketSockets; + const normalizedState = { + ...state, + residentBuildContracts, + graphileActivityByBuildContract, + }; + return { + pid: state.pid, + graphileInFlight: state.graphileInFlight, + residentBuildContracts, + graphileActivityByBuildContract, + graphileTransientHttpInFlight, + graphileWebsocketSockets, + activityContractsExact, + httpRequestsOutstanding, + websocketUpgradesOutstanding, + handlerAccountingExact, + realtimeConnectionsExpected, + realtimeConnectionsActive, + realtimeResident, + stateSha256: `sha256:${crypto.createHash('sha256') + .update(canonicalJson(normalizedState)) + .digest('hex')}`, + state: normalizedState, + }; +}; + +const retainedMemoryGuardErrors = (guard) => { + const errors = []; + if (guard.graphileInFlight !== 0) { + errors.push(`PDCF_RETAINED_MEMORY_IN_FLIGHT:${guard.graphileInFlight}`); + } + if (!guard.handlerAccountingExact) { + errors.push( + `PDCF_RETAINED_MEMORY_HANDLER_ACCOUNTING_MISMATCH:` + + `${guard.httpRequestsOutstanding}:${guard.graphileTransientHttpInFlight}:` + + `${guard.websocketUpgradesOutstanding}:${guard.graphileWebsocketSockets}` + ); + } + if (!guard.realtimeResident) { + errors.push( + `PDCF_RETAINED_MEMORY_REALTIME_NOT_RESIDENT:` + + `${guard.graphileWebsocketSockets}:${guard.realtimeConnectionsActive}:` + + `${guard.realtimeConnectionsExpected}` + ); + } + return errors; +}; + +const memoryRange = (samples, field) => { + const values = samples.map((sample) => sample[field]); + return Math.max(...values) - Math.min(...values); +}; + +const memoryConvergenceThreshold = (samples, field) => Math.max( + MIB, + Math.ceil(Math.max(...samples.map((sample) => sample[field])) * 0.0025), +); + +const collectRetainedMemoryCheckpoint = async ({ + forceGc, + readMemory = () => process.memoryUsage(), + readGuard, + monotonicNow = () => process.hrtime.bigint(), + yieldTurn = () => new Promise((resolve) => setImmediate(resolve)), + rounds = RETAINED_MEMORY_GC_ROUNDS, +}) => { + if (typeof forceGc !== 'function') { + throw new Error('PDCF_RETAINED_MEMORY_GC_UNAVAILABLE'); + } + if (!Number.isSafeInteger(rounds) || rounds < 5 || rounds > 8) { + throw new Error('PDCF_RETAINED_MEMORY_GC_ROUNDS_INVALID'); + } + const guardBefore = readGuard(); + const beforeErrors = retainedMemoryGuardErrors(guardBefore); + if (beforeErrors.length > 0) throw new Error(beforeErrors[0]); + const samples = []; + for (let index = 0; index < rounds; index++) { + forceGc(); + await yieldTurn(); + const memory = readMemory(); + samples.push({ + timestamp: new Date().toISOString(), + monotonicNs: String(monotonicNow()), + heapUsedBytes: memory.heapUsed, + externalBytes: memory.external, + arrayBuffersBytes: memory.arrayBuffers, + rssBytes: memory.rss, + }); + } + const guardAfter = readGuard(); + const stableSamples = samples.slice(-RETAINED_MEMORY_STABLE_SAMPLES); + const heapSpreadBytes = memoryRange(stableSamples, 'heapUsedBytes'); + const externalSpreadBytes = memoryRange(stableSamples, 'externalBytes'); + const heapThresholdBytes = memoryConvergenceThreshold( + stableSamples, + 'heapUsedBytes', + ); + const externalThresholdBytes = memoryConvergenceThreshold( + stableSamples, + 'externalBytes', + ); + const errors = []; + errors.push(...retainedMemoryGuardErrors(guardAfter)); + if (guardBefore.pid !== guardAfter.pid) { + errors.push('PDCF_RETAINED_MEMORY_PID_CHANGED'); + } + if (guardBefore.stateSha256 !== guardAfter.stateSha256) { + errors.push('PDCF_RETAINED_MEMORY_RESIDENCY_OR_COUNTERS_CHANGED'); + } + if (heapSpreadBytes > heapThresholdBytes) { + errors.push( + `PDCF_RETAINED_HEAP_NOT_CONVERGED:${heapSpreadBytes}:${heapThresholdBytes}` + ); + } + if (externalSpreadBytes > externalThresholdBytes) { + errors.push( + `PDCF_RETAINED_EXTERNAL_NOT_CONVERGED:${externalSpreadBytes}:${externalThresholdBytes}` + ); + } + return { + version: 1, + fixture: 'physical-database-density-v1', + pid: guardBefore.pid, + gcRounds: rounds, + stableSampleCount: RETAINED_MEMORY_STABLE_SAMPLES, + stable: errors.length === 0, + samples, + guardBefore, + guardAfter, + errors, + }; +}; + +const authorizeRetainedMemoryCheckpoint = (request, options) => + options.benchmarkRetainedHeapEnabled === true + && isLoopbackRequest(request) + && tokenEqual(bearerToken(request), options.observabilityToken); + +const classifyDatabaseScope = (present, controlDatabase, fixtureDatabases) => { + const expected = new Set([controlDatabase, ...fixtureDatabases]); + const unexpected = present.filter((database) => !expected.has(database)); + const presentSet = new Set(present); + const missingFixture = fixtureDatabases.filter((database) => !presentSet.has(database)); + return { + dedicated: unexpected.length === 0 && missingFixture.length === 0, + databasesPresent: present.length, + fixtureDatabasesExpected: fixtureDatabases.length, + fixtureDatabasesPresent: fixtureDatabases.length - missingFixture.length, + unexpectedDatabases: unexpected.length, + missingFixtureDatabases: missingFixture.length, + unexpectedDatabaseSetSha256: unexpected.length === 0 + ? null + : `sha256:${crypto.createHash('sha256').update(unexpected.join('\0')).digest('hex')}`, + }; +}; + +const aggregateRuntimePoolStats = (children, requestedMaxUses) => { + const childStats = children.map(({ child }) => child.runtimePoolStats()); + const identitiesUnique = childStats.every((stats) => stats?.identitiesUnique === true); + const runtimePoolObjects = children.flatMap(({ child }) => + typeof child.runtimePoolObjects === 'function' + ? child.runtimePoolObjects() + : [] + ); + const effectiveKnown = childStats.every((stats) => + stats?.effectiveMaxUsesKnown === true + ); + const effectiveValues = effectiveKnown + ? [...new Set(childStats.map((stats) => stats.effectiveMaxUses))] + : []; + const effectiveMaxUsesKnown = effectiveValues.length === 1; + const effectiveMaxUses = effectiveMaxUsesKnown ? effectiveValues[0] : null; + const expectedPools = childStats.reduce( + (sum, stats) => sum + (Number.isSafeInteger(stats?.expectedPools) ? stats.expectedPools : 0), + 0, + ); + const observedPools = childStats.reduce( + (sum, stats) => sum + (Number.isSafeInteger(stats?.observedPools) ? stats.observedPools : 0), + 0, + ); + const poolObjectsUnique = childStats.every((stats) => stats?.poolObjectsUnique === true) + && runtimePoolObjects.length === expectedPools + && runtimePoolObjects.every(Boolean) + && new Set(runtimePoolObjects).size === runtimePoolObjects.length; + const available = childStats.length > 0 + && childStats.every((stats) => + stats?.scope === 'runtime-only-exact-identities' + && stats.available === true + && stats.identitiesUnique === true + && stats.poolObjectsUnique === true + && stats.requestedMaxUses === requestedMaxUses + && stats.maxUsesExact === true + ) + && observedPools === expectedPools + && poolObjectsUnique + && effectiveMaxUsesKnown; + const sum = (key) => available + ? childStats.reduce((total, stats) => total + stats[key], 0) + : null; + return { + scope: 'runtime-only-exact-identities', + available, + requestedMaxUses, + effectiveMaxUses, + effectiveMaxUsesKnown, + maxUsesExact: available && effectiveMaxUses === requestedMaxUses, + identitiesUnique, + poolObjectsUnique, + expectedPools, + observedPools, + totalClients: sum('totalClients'), + idleClients: sum('idleClients'), + waitingClients: sum('waitingClients'), + }; +}; + +const runtimeEnvironmentFor = ( + environment, + customer, + secretResolver, + includeNotification = false, +) => ({ + ...environment, + PGDATABASE: customer.database, + // Runtime maxUses is supplied through the explicit per-pool config. Keep + // every ambient/control/notification pool on unlimited reuse. + PG_POOL_MAX_USES: '0', + ...Object.fromEntries(TENANTS.flatMap((tenant) => [ + [`CTF_RUNTIME_${tenant.id.toUpperCase()}_PGUSER`, customer.roles[tenant.id]], + [ + `CTF_RUNTIME_${tenant.id.toUpperCase()}_PGPASSWORD`, + secretResolver.runtimePasswordFor(customer.roles[tenant.id]), + ], + ])), + ...(includeNotification ? { + CTF_NOTIFICATION_PGUSER: customer.notificationRole, + CTF_NOTIFICATION_PGPASSWORD: + secretResolver.notificationPasswordFor(customer.notificationRole), + } : {}), +}); + +const completeServerOptionsFor = (options, customer, environment) => ({ + ...completeServer.parseServerOptions([ + '--host', options.host, + '--port', String(options.port), + '--arm', options.arm, + '--mode', options.mode, + '--introspection-client-release-mode', options.introspectionClientReleaseMode, + '--runtime-pool-max', String(options.runtimePoolMax), + '--runtime-pool-max-uses', options.runtimePoolMaxUses == null + ? 'unlimited' + : String(options.runtimePoolMaxUses), + '--enable-realtime', String(options.enableRealtime), + '--realtime-notification-mode', options.realtimeNotificationMode, + '--realtime-cursor-poll-ms', String(options.realtimeCursorPollIntervalMs), + '--realtime-cursor-heartbeat-ms', + String(options.realtimeCursorHeartbeatIntervalMs), + ...(options.realtimeNotificationMode === 'shared-exact' ? [ + '--notification-role', customer.notificationRole, + ] : []), + ...TENANTS.flatMap((tenant) => [ + `--${tenant.runtimeRoleArgument}`, + customer.roles[tenant.id], + ]), + ], environment), + runPurpose: options.runPurpose, + cloneId: options.cloneId, + provisionCustomerId: customer.id, + provisionAttestation: customer.provisionAttestation, +}); + +const createPhysicalDatabaseServer = async (options, environment = process.env) => { + if ( + options.benchmarkRetainedHeapEnabled + && typeof global.gc !== 'function' + ) { + throw new Error('PDCF_RETAINED_MEMORY_REQUIRES_EXPOSE_GC'); + } + if (options.expectedManifestSha256) { + const actual = `sha256:${crypto.createHash('sha256') + .update(fs.readFileSync(options.manifestFile)) + .digest('hex')}`; + if (actual !== options.expectedManifestSha256) { + throw new Error('PDCF_EXPECTED_MANIFEST_SHA256_MISMATCH'); + } + } + const { manifest, secretResolver } = loadProvision( + options.manifestFile, + options.secretsFile, + ); + const provisionClone = assertProvisionCloneManifest(manifest, options); + if ( + options.expectedDatabaseContractFingerprint + && manifest.canonicalDatabaseContractFingerprint + !== options.expectedDatabaseContractFingerprint + ) { + throw new Error('PDCF_EXPECTED_DATABASE_CONTRACT_MISMATCH'); + } + if (options.customerCount !== manifest.customers.length) { + throw new Error( + `PDCF_CUSTOMER_COUNT_MUST_EQUAL_PROVISIONED:${options.customerCount}:${manifest.customers.length}` + ); + } + if ( + options.enableRealtime + && options.realtimeNotificationMode === 'dedicated' + && options.runtimePoolMax < 2 + ) { + throw new Error('PDCF_REALTIME_REQUIRES_RUNTIME_POOL_MAX_2'); + } + const customers = manifest.customers.slice(0, options.customerCount); + const verifiedContracts = new Map(customers.map((customer) => { + const contract = options.runPurpose === 'hostile-preflight' + ? assertCustomerContract(customer, inspectCustomerContract({ + customer, + canonicalSchemas: manifest.canonicalSchemas, + environment, + })) + : { + structuralFingerprints: customer.structuralFingerprints, + databaseContractFingerprint: customer.databaseContractFingerprint, + }; + return [customer.id, { + ...contract, + verification: options.runPurpose === 'hostile-preflight' + ? 'live-recomputed' + : 'provision-manifest', + }]; + })); + const children = []; + for (const customer of customers) { + const childEnvironment = runtimeEnvironmentFor( + environment, + customer, + secretResolver, + options.realtimeNotificationMode === 'shared-exact', + ); + const childOptions = completeServerOptionsFor(options, customer, childEnvironment); + const processOverrides = Object.fromEntries(SECURITY_ENVIRONMENT_KEYS.map((key) => [ + key, + childEnvironment[key], + ])); + const child = await withProcessEnvironment(processOverrides, () => + completeServer.createFixtureServer(childOptions, childEnvironment) + ); + children.push({ customer, child }); + } + + const express = require(path.join(REPO_ROOT, 'graphql/server/node_modules/express')); + const { Pool } = require(path.join(REPO_ROOT, 'graphql/server/node_modules/pg')); + const { getDebugMemorySnapshot } = requireBuilt( + 'graphql/server/dist/diagnostics/debug-memory-snapshot.js' + ); + const { + getGraphileRealtimeRoleAuditStats, + deleteGraphileCacheEntry, + getCacheCounters, + graphileCache, + } = requireBuilt('graphile/graphile-cache/dist/index.js'); + const { getInFlightCount } = requireBuilt( + 'graphql/server/dist/middleware/graphile.js' + ); + const { getGraphileGovernorCounters } = requireBuilt( + 'graphql/server/dist/middleware/graphile-build-governor.js' + ); + const { getGraphileBuildStats } = requireBuilt( + 'graphql/server/dist/middleware/observability/graphile-build-stats.js' + ); + const { + getPgCacheStats, + getPgNotificationBrokerStats, + } = requireBuilt('postgres/pg-cache/dist/index.js'); + const pgEnv = require(path.join(REPO_ROOT, 'graphql/server/node_modules/pg-env')); + const controlConfig = pgEnv.getPgEnvOptions({ + database: environment.PGDATABASE ?? 'postgres', + }); + const observerPool = new Pool({ + host: controlConfig.host, + port: Number(controlConfig.port), + database: controlConfig.database, + user: controlConfig.user, + password: controlConfig.password, + application_name: 'cperf-physical-database-observer', + max: 1, + idleTimeoutMillis: 0, + connectionTimeoutMillis: 5_000, + }); + observerPool.on('error', () => undefined); + + const realtimeConnections = createRealtimeConnectionRegistry( + options.enableRealtime + ? customers.flatMap((customer) => TENANTS.map((tenant) => + `${customer.id}:${tenant.id}` + )) + : [] + ); + let httpServer = null; + let closing = false; + let retainedMemoryCheckpointRunning = false; + + const cacheEntries = () => [...graphileCache.values()]; + const realtimeStats = () => { + const entries = cacheEntries(); + const connections = realtimeConnections.snapshot(); + const notificationBrokers = getPgNotificationBrokerStats(); + const notificationRoleAudits = getGraphileRealtimeRoleAuditStats(); + return { + managersExpected: connections.connectionsExpected, + managersActive: entries.filter((entry) => entry.realtimeManager?.isRunning).length, + ...connections, + // Compatibility names consumed by the v3 scorer. These now describe + // server-side accepted connections, never client objects in this process. + transportsExpected: connections.connectionsExpected, + transportsActive: connections.connectionsActive, + transportErrors: connections.connectionErrors > 0 + ? ['PDCF_REALTIME_SERVER_CONNECTION_ERROR'] + : [], + notificationMode: options.realtimeNotificationMode, + notificationBrokers, + notificationRoleAudits, + }; + }; + + const poolStats = () => aggregateRuntimePoolStats( + children, + options.runtimePoolMaxUses, + ); + + const buildContractFingerprintForLiveIdentity = (cacheKey) => { + const matches = children + .map(({ child }) => child.buildContractFingerprintForLiveIdentity(cacheKey)) + .filter(Boolean); + if (matches.length !== 1) { + throw new Error(`PDCF_BUILD_CONTRACT_EVIDENCE_MAPPING_INVALID:${matches.length}`); + } + return matches[0]; + }; + + const contractEvidence = () => ({ + version: 1, + credentialFree: true, + liveIdentityScope: 'process-local-keyed-hmac-v1', + customers: Object.fromEntries(children.map(({ customer, child }) => [ + customer.id, + child.contractEvidence(), + ])), + residentGraphileBuildFingerprints: [...graphileCache.keys()] + .map(buildContractFingerprintForLiveIdentity) + .sort(), + }); + + const retainedMemoryGuard = () => { + const residentBuildContracts = [...graphileCache.keys()].sort(); + const residentBuildContractFingerprints = residentBuildContracts + .map(buildContractFingerprintForLiveIdentity) + .sort(); + const graphileActivityByBuildContract = makeGraphileActivityVector(cacheEntries()); + const graphileTransientHttpInFlight = graphileActivityByBuildContract.reduce( + (sum, entry) => sum + entry.transientHttpInFlight, + 0, + ); + const graphileBuildsInFlight = getInFlightCount(); + const builds = getGraphileBuildStats(); + const pgCacheStats = getPgCacheStats(); + const realtime = realtimeStats(); + return makeRetainedMemoryGuard({ + pid: process.pid, + // Long-lived GraphQL WebSocket sockets are the expected resident state, + // so only build work and transient HTTP handlers block a full-GC sample. + graphileInFlight: graphileBuildsInFlight + graphileTransientHttpInFlight, + graphileBuildsInFlight, + graphileTransientHttpInFlight, + graphileActivityByBuildContract, + residentBuildContracts, + residentBuildContractFingerprints, + cacheCounters: getCacheCounters(), + governorCounters: getGraphileGovernorCounters(), + buildCounters: { + started: builds.started, + succeeded: builds.succeeded, + failed: builds.failed, + }, + pgCacheMonotonicCounters: { + capacityEvictions: pgCacheStats.capacityEvictions, + capacityRefusals: pgCacheStats.capacityRefusals, + disposalFailures: pgCacheStats.disposalFailures, + }, + realtime: { + managersExpected: realtime.managersExpected, + managersActive: realtime.managersActive, + connectionsExpected: realtime.connectionsExpected, + connectionsAccepted: realtime.connectionsAccepted, + connectionsActive: realtime.connectionsActive, + connectionDrops: realtime.connectionDrops, + connectionErrors: realtime.connectionErrors, + connectionsPerSurface: realtime.connectionsPerSurface, + }, + }); + }; + + const backendStats = async () => { + const result = await observerPool.query(` + SELECT datname, + COALESCE(state, 'unknown') AS state, + count(*)::integer AS count + FROM pg_catalog.pg_stat_activity + WHERE backend_type = 'client backend' + AND datname = ANY($1::text[]) + GROUP BY datname, COALESCE(state, 'unknown') + ORDER BY datname, state + `, [customers.map((customer) => customer.database)]); + const byDatabase = Object.fromEntries(customers.map((customer) => [ + customer.database, + { total: 0, active: 0, idle: 0, idleInTransaction: 0, other: 0 }, + ])); + for (const row of result.rows) { + const state = byDatabase[row.datname]; + if (!state) continue; + state.total += row.count; + if (row.state === 'active') state.active += row.count; + else if (row.state === 'idle') state.idle += row.count; + else if (row.state === 'idle in transaction') state.idleInTransaction += row.count; + else state.other += row.count; + } + return { + total: Object.values(byDatabase).reduce((sum, value) => sum + value.total, 0), + active: Object.values(byDatabase).reduce((sum, value) => sum + value.active, 0), + idle: Object.values(byDatabase).reduce((sum, value) => sum + value.idle, 0), + idleInTransaction: Object.values(byDatabase) + .reduce((sum, value) => sum + value.idleInTransaction, 0), + other: Object.values(byDatabase).reduce((sum, value) => sum + value.other, 0), + byDatabase, + observerExcluded: true, + }; + }; + + const databaseScope = async () => { + const result = await observerPool.query(` + SELECT datname + FROM pg_catalog.pg_database + WHERE NOT datistemplate + ORDER BY datname + `); + const present = result.rows.map((row) => row.datname); + const fixtureDatabases = customers.map((customer) => customer.database); + return classifyDatabaseScope(present, controlConfig.database, fixtureDatabases); + }; + + const assertRealtimeResident = () => { + const realtime = realtimeStats(); + if (realtime.managersActive !== realtime.managersExpected) { + throw new Error( + `PDCF_REALTIME_MANAGERS_NOT_READY:${realtime.managersActive}:${realtime.managersExpected}` + ); + } + if (options.realtimeNotificationMode === 'shared-exact') { + const expectedBrokers = customers.length; + const expectedLeases = customers.length * TENANTS.length; + const brokers = realtime.notificationBrokers; + const audits = realtime.notificationRoleAudits; + if ( + brokers.brokers !== expectedBrokers + || brokers.listenerConnections !== expectedBrokers + || brokers.leases !== expectedLeases + || brokers.topics !== expectedLeases + || brokers.subscribers !== expectedLeases + || brokers.queueOverflows !== 0 + || brokers.fatalFailures !== 0 + || audits.identities !== expectedBrokers + || audits.healthy !== expectedBrokers + || audits.failed !== 0 + || audits.stale !== 0 + || audits.catalogAuditAttempts < expectedLeases + || audits.catalogAuditFailures !== 0 + || audits.activeDatabaseTargets !== expectedBrokers + || audits.databaseConfigurationConflicts !== 0 + ) { + throw new Error( + `PDCF_SHARED_REALTIME_NOT_EXACT:${JSON.stringify({ brokers, audits })}` + ); + } + } + realtimeConnections.assertResident(); + return realtime; + }; + + const app = express(); + app.disable('x-powered-by'); + + app.get('/healthz', (_request, response) => { + response.json({ + status: 'ok', + fixture: 'physical-database-density-v1', + customers: customers.length, + physicalDatabases: customers.length, + }); + }); + + app.get('/debug/memory', async (request, response) => { + if (!isLoopbackRequest(request)) { + response.status(404).send('Not found'); + return; + } + if ( + environment.NODE_ENV !== 'development' + && !tokenEqual(bearerToken(request), options.observabilityToken) + ) { + response.status(401).json({ error: { code: 'PDCF_OBSERVABILITY_UNAUTHORIZED' } }); + return; + } + try { + const [backends, containerScope] = await Promise.all([ + backendStats(), + databaseScope(), + ]); + response.json({ + ...getDebugMemorySnapshot(), + physicalDatabaseFixture: { + fixture: 'physical-database-density-v1', + customers: customers.length, + physicalDatabases: customers.length, + canonicalStructuralFingerprint: + manifest.canonicalStructuralFingerprint?.combined?.sha256 ?? null, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint ?? null, + blueprintCompatibilityFingerprint: + options.blueprintCompatibilityFingerprint, + pools: poolStats(), + contractEvidence: contractEvidence(), + backends, + containerScope, + realtime: realtimeStats(), + }, + }); + } catch (error) { + response.status(503).json({ + error: { + code: 'PDCF_TELEMETRY_UNAVAILABLE', + message: error instanceof Error ? error.message : String(error), + }, + }); + } + }); + + app.post('/__cperf/post-warmup', async (request, response) => { + if ( + !isLoopbackRequest(request) + || !tokenEqual(bearerToken(request), options.observabilityToken) + ) { + response.status(404).send('Not found'); + return; + } + try { + // The perf-harness process owns and verifies graphql-ws clients. This + // measured process only proves its managers and accepted inbound sockets + // are resident at the exact warm boundary. + response.json({ ok: true, realtime: assertRealtimeResident() }); + } catch (error) { + response.status(503).json({ + error: { + code: 'PDCF_REALTIME_NOT_RESIDENT', + message: error instanceof Error ? error.message : String(error), + }, + }); + } + }); + + app.post('/__cperf/retained-memory-checkpoint', async (request, response) => { + if (!authorizeRetainedMemoryCheckpoint(request, options)) { + response.status(404).send('Not found'); + return; + } + if (retainedMemoryCheckpointRunning) { + response.status(409).json({ + error: { code: 'PDCF_RETAINED_MEMORY_CHECKPOINT_RUNNING' }, + }); + return; + } + retainedMemoryCheckpointRunning = true; + try { + const checkpoint = await collectRetainedMemoryCheckpoint({ + forceGc: global.gc, + readGuard: retainedMemoryGuard, + }); + response.status(checkpoint.stable ? 200 : 503).json(checkpoint); + } catch (error) { + response.status(503).json({ + error: { + code: error instanceof Error + ? error.message.split(':', 1)[0] + : 'PDCF_RETAINED_MEMORY_CHECKPOINT_FAILED', + message: error instanceof Error ? error.message : String(error), + }, + }); + } finally { + retainedMemoryCheckpointRunning = false; + } + }); + + app.get('/__physical/status', async (request, response) => { + if (!isLoopbackRequest(request)) { + response.status(404).send('Not found'); + return; + } + try { + const liveAttestations = await Promise.all(children.map(({ child }) => + child.readProvisionAttestation() + )); + const attestedCustomers = customers.map((customer, index) => ({ + ...customer, + provisionAttestation: liveAttestations[index], + })); + if ( + provisionAttestationSetSha256(attestedCustomers) + !== provisionClone.attestationSetSha256 + ) { + throw new Error('PDCF_LIVE_PROVISION_ATTESTATION_SET_MISMATCH'); + } + response.json({ + version: 1, + fixture: 'physical-database-density-v1', + arm: options.arm, + runPurpose: options.runPurpose, + cloneId: options.cloneId, + provisionClone: { + ...provisionClone, + verified: true, + }, + introspectionMode: options.mode, + introspectionClientReleaseMode: options.introspectionClientReleaseMode, + runtimePoolMax: options.runtimePoolMax, + runtimePoolMaxUses: options.runtimePoolMaxUses, + runtimePools: poolStats(), + contractEvidence: contractEvidence(), + customers: customers.map((customer, index) => { + const contract = verifiedContracts.get(customer.id); + return { + id: customer.id, + physicalDatabase: customer.database, + provisionAttestation: liveAttestations[index], + structuralFingerprints: contract.structuralFingerprints, + canonicalStructuralFingerprint: + contract.structuralFingerprints?.combined?.sha256 ?? null, + databaseContractFingerprint: contract.databaseContractFingerprint ?? null, + contractVerification: contract.verification, + }; + }), + canonicalStructuralFingerprint: + manifest.canonicalStructuralFingerprint ?? null, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint ?? null, + blueprintCompatibilityFingerprint: options.blueprintCompatibilityFingerprint, + retainedMemoryCheckpoint: { + enabled: options.benchmarkRetainedHeapEnabled, + gcExposed: typeof global.gc === 'function', + }, + realtime: realtimeStats(), + }); + } catch (error) { + response.status(503).json({ + error: { + code: error instanceof Error + ? error.message.split(':', 1)[0] + : 'PDCF_STATUS_ATTESTATION_FAILED', + }, + }); + } + }); + + for (const { customer, child } of children) { + app.use(`/customer/${customer.id}`, child.app); + } + + let upgradeListener = null; + const listen = () => new Promise((resolve, reject) => { + httpServer = app.listen(options.port, options.host, () => resolve(httpServer)); + httpServer.once('error', reject); + if (options.enableRealtime) { + upgradeListener = (request, socket, head) => { + const rawUrl = request.url ?? ''; + const route = matchPhysicalUpgradeRoute(rawUrl); + const child = route + ? children.find(({ customer }) => customer.id === route.customerId) + : null; + if (!child) { + socket.destroy(); + return; + } + void child.child.handleUpgrade(request, socket, head, { + pathPrefix: `/customer/${child.customer.id}`, + }).then((handled) => { + if (handled) { + realtimeConnections.trackAccepted( + `${route.customerId}:${route.tenantId}`, + socket + ); + } else if (!socket.destroyed) socket.destroy(); + }).catch(() => socket.destroy()); + }; + httpServer.on('upgrade', upgradeListener); + } + }); + + const close = async () => { + if (closing) return; + closing = true; + if (httpServer && upgradeListener) httpServer.off('upgrade', upgradeListener); + if (httpServer?.listening) { + await new Promise((resolve) => httpServer.close(resolve)); + } + // Dispose every customer's realtime manager while its pool is still live. + // Each child owns the process-global pool registry, so allowing the first + // child close to tear it down would strand later managers on ended pools. + await Promise.all([...graphileCache.keys()].map((key) => + deleteGraphileCacheEntry(key) + )); + for (const { child } of children) await child.close(); + await observerPool.end(); + }; + + return { + app, + children, + close, + customers, + listen, + options, + realtimeStats, + assertRealtimeResident, + }; +}; + +const main = async () => { + const options = parseServerOptions(process.argv.slice(2)); + const server = await createPhysicalDatabaseServer(options); + await server.listen(); + process.stdout.write(`${JSON.stringify({ + status: 'ready', + fixture: 'physical-database-density-v1', + host: options.host, + port: options.port, + arm: options.arm, + customers: server.customers.length, + })}\n`); + let stopping = false; + const shutdown = async (code) => { + if (stopping) return; + stopping = true; + await server.close(); + process.exitCode = code; + }; + process.once('SIGTERM', () => void shutdown(0)); + process.once('SIGINT', () => void shutdown(130)); +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + authorizeRetainedMemoryCheckpoint, + assertCustomerContract, + assertProvisionCloneManifest, + aggregateRuntimePoolStats, + collectRetainedMemoryCheckpoint, + completeServerOptionsFor, + classifyDatabaseScope, + createRealtimeConnectionRegistry, + createPhysicalDatabaseServer, + makeGraphileActivityVector, + makeRetainedMemoryGuard, + matchPhysicalUpgradeCustomer, + matchPhysicalUpgradeRoute, + parseServerOptions, + parseRuntimePoolMaxUses, + runtimeEnvironmentFor, + tokenEqual, +}; diff --git a/research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.cjs b/research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.cjs new file mode 100644 index 0000000000..492a743854 --- /dev/null +++ b/research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.cjs @@ -0,0 +1,1130 @@ +'use strict'; + +const { spawnSync } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + REPO_ROOT, + TENANTS, + assertCredentialFree, + parseArgs, + requireString, +} = require('../complete-tenant-fixture/lib.cjs'); +const completeServer = require('../complete-tenant-fixture/server.cjs'); +const { + FIXTURE_ID, + validateProvisionManifest, + validateSecrets, +} = require('./lib.cjs'); + +const PROBE_KIND = 'unsafe-runtime-fixture-startup-admission-v2'; +const PROBE_VERSION = 2; +const ADMISSION_SCOPE = 'complete-tenant-fixture:createFixtureServer-pre-build-role-audit-v1'; +const CLONE_AUDIT_KIND = 'unsafe-runtime-live-clone-audit-v1'; +const ROLE_AUDIT_KIND = 'unsafe-runtime-role-profile-audit-v1'; +const CLEANUP_AUDIT_KIND = 'unsafe-runtime-role-cleanup-audit-v1'; +const SAFE_CONTROL_CAPABILITY = 'safe-control'; +const PROBE_CAPABILITIES = Object.freeze([ + 'superuser', + 'bypassrls', + 'createrole', + 'schema-owner', + 'schema-create', +]); +const PROBE_ROLE_PATTERNS = Object.freeze({ + superuser: /^ctf_unsafe_super_[a-f0-9]{12}$/, + bypassrls: /^ctf_unsafe_bypass_[a-f0-9]{12}$/, + createrole: /^ctf_unsafe_create_role_[a-f0-9]{12}$/, + 'schema-owner': /^ctf_unsafe_schema_owner_[a-f0-9]{12}$/, + 'schema-create': /^ctf_unsafe_schema_create_[a-f0-9]{12}$/, +}); +const SAFE_LABEL_PATTERN = /^[a-z0-9][a-z0-9-]{0,127}$/; +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/; + +const exactKeys = (value, expected) => + value !== null + && typeof value === 'object' + && !Array.isArray(value) + && JSON.stringify(Object.keys(value).sort()) + === JSON.stringify([...expected].sort()); + +const readJson = (file) => JSON.parse(fs.readFileSync(path.resolve(file), 'utf8')); + +const readPrivateJson = (file) => { + const absoluteFile = path.resolve(file); + let descriptor; + let contents; + try { + const before = fs.lstatSync(absoluteFile); + if ( + before.isSymbolicLink() + || !before.isFile() + || (before.mode & 0o777) !== 0o600 + || (typeof process.getuid === 'function' && before.uid !== process.getuid()) + ) { + throw new Error('PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE'); + } + descriptor = fs.openSync( + absoluteFile, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + const stat = fs.fstatSync(descriptor); + if ( + !stat.isFile() + || stat.dev !== before.dev + || stat.ino !== before.ino + || (stat.mode & 0o777) !== 0o600 + || (typeof process.getuid === 'function' && stat.uid !== process.getuid()) + ) { + throw new Error('PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE'); + } + contents = fs.readFileSync(descriptor, 'utf8'); + } catch (error) { + if (error?.message === 'PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE') throw error; + throw new Error('PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE'); + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + } + try { + return JSON.parse(contents); + } catch { + throw new Error('PDCF_UNSAFE_ROLE_SECRETS_INVALID'); + } +}; + +const loadPrivateProvision = (manifestFile, secretsFile) => { + const absoluteManifestFile = path.resolve(manifestFile); + const absoluteSecretsFile = path.resolve(secretsFile); + let manifestStat; + let secretsStat; + try { + manifestStat = fs.statSync(absoluteManifestFile); + secretsStat = fs.lstatSync(absoluteSecretsFile); + } catch { + throw new Error('PDCF_UNSAFE_ROLE_PROVISION_INPUT_INVALID'); + } + if ( + manifestStat.dev === secretsStat.dev + && manifestStat.ino === secretsStat.ino + ) { + throw new Error('PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE'); + } + const manifest = validateProvisionManifest(readJson(absoluteManifestFile)); + assertCredentialFree(manifest); + const secrets = validateSecrets(readPrivateJson(absoluteSecretsFile), manifest); + return { manifest, secrets }; +}; + +const quoteIdentifier = (value) => `"${String(value).replace(/"/g, '""')}"`; +const quoteLiteral = (value) => `'${String(value).replace(/'/g, "''")}'`; + +const requireSafeLabel = (value, code) => { + if (typeof value !== 'string' || !SAFE_LABEL_PATTERN.test(value)) { + throw new Error(code); + } + return value; +}; + +const requireProbeCapability = (value) => { + if (!PROBE_CAPABILITIES.includes(value)) { + throw new Error('PDCF_UNSAFE_ROLE_CAPABILITY_INVALID'); + } + return value; +}; + +const requireProbeCase = (value) => value === SAFE_CONTROL_CAPABILITY + ? value + : requireProbeCapability(value); + +const probeNames = (nonce) => { + if (!/^[a-f0-9]{12}$/.test(nonce)) { + throw new Error('PDCF_UNSAFE_ROLE_NONCE_INVALID'); + } + return { + roles: { + superuser: `ctf_unsafe_super_${nonce}`, + bypassrls: `ctf_unsafe_bypass_${nonce}`, + createrole: `ctf_unsafe_create_role_${nonce}`, + 'schema-owner': `ctf_unsafe_schema_owner_${nonce}`, + 'schema-create': `ctf_unsafe_schema_create_${nonce}`, + }, + ownerSchema: `ctf_unsafe_owner_${nonce}`, + createSchema: `ctf_unsafe_create_${nonce}`, + }; +}; + +const buildUnsafeRoleSetupSql = ({ database, names, passwords }) => { + const role = Object.fromEntries(Object.entries(names.roles).map(([capability, value]) => [ + capability, + quoteIdentifier(value), + ])); + const password = Object.fromEntries(Object.entries(passwords).map(([capability, value]) => [ + capability, + quoteLiteral(value), + ])); + const databaseIdentifier = quoteIdentifier(database); + return ` +BEGIN; +CREATE ROLE ${role.superuser} + LOGIN NOINHERIT SUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD ${password.superuser}; +CREATE ROLE ${role.bypassrls} + LOGIN NOINHERIT NOSUPERUSER BYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD ${password.bypassrls}; +CREATE ROLE ${role.createrole} + LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS NOCREATEDB CREATEROLE NOREPLICATION + PASSWORD ${password.createrole}; +CREATE ROLE ${role['schema-owner']} + LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD ${password['schema-owner']}; +CREATE ROLE ${role['schema-create']} + LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD ${password['schema-create']}; +GRANT CONNECT ON DATABASE ${databaseIdentifier} TO + ${Object.values(role).join(', ')}; +CREATE SCHEMA ${quoteIdentifier(names.ownerSchema)} + AUTHORIZATION ${role['schema-owner']}; +CREATE SCHEMA ${quoteIdentifier(names.createSchema)}; +REVOKE ALL ON SCHEMA ${quoteIdentifier(names.createSchema)} FROM PUBLIC; +GRANT CREATE ON SCHEMA ${quoteIdentifier(names.createSchema)} + TO ${role['schema-create']}; +COMMIT; +`; +}; + +const buildUnsafeRoleAuditSql = ({ names }) => { + const probes = PROBE_CAPABILITIES.map((capability, index) => + `(${index + 1}, ${quoteLiteral(capability)}, ${quoteLiteral(names.roles[capability])})` + ).join(',\n '); + const schemas = [ + ['owner', names.ownerSchema], + ['create', names.createSchema], + ].map(([label, schema]) => + `(${quoteLiteral(label)}, ${quoteLiteral(schema)})` + ).join(',\n '); + return ` +WITH probes(ordinal, capability, role_name) AS ( + VALUES + ${probes} +), probe_schemas(label, schema_name) AS ( + VALUES + ${schemas} +), profiles AS ( + SELECT p.ordinal, + p.capability, + p.role_name, + r.rolcanlogin AS can_login, + r.rolinherit AS inherits, + r.rolsuper AS superuser, + r.rolbypassrls AS bypass_rls, + r.rolcreaterole AS create_role, + r.rolcreatedb AS create_database, + r.rolreplication AS replication, + CASE WHEN r.oid IS NULL THEN false ELSE pg_catalog.has_database_privilege( + r.oid, + (SELECT oid FROM pg_catalog.pg_database WHERE datname = pg_catalog.current_database()), + 'CONNECT' + ) END AS database_connect, + COALESCE(( + SELECT pg_catalog.jsonb_agg(s.label ORDER BY s.label) + FROM probe_schemas s + JOIN pg_catalog.pg_namespace n ON n.nspname = s.schema_name + WHERE n.nspowner = r.oid + ), '[]'::jsonb) AS owned_probe_schemas, + COALESCE(( + SELECT pg_catalog.jsonb_agg(s.label ORDER BY s.label) + FROM probe_schemas s + JOIN pg_catalog.pg_namespace n ON n.nspname = s.schema_name + WHERE r.oid IS NOT NULL + AND pg_catalog.has_schema_privilege(r.oid, n.oid, 'CREATE') + ), '[]'::jsonb) AS create_on_probe_schemas, + CASE WHEN r.oid IS NULL THEN -1 ELSE ( + SELECT pg_catalog.count(*)::integer + FROM pg_catalog.pg_auth_members m + WHERE m.member = r.oid + ) END AS inherited_memberships + FROM probes p + LEFT JOIN pg_catalog.pg_roles r ON r.rolname = p.role_name +) +SELECT pg_catalog.jsonb_build_object( + 'version', 1, + 'kind', ${quoteLiteral(ROLE_AUDIT_KIND)}, + 'database', pg_catalog.current_database(), + 'profiles', pg_catalog.jsonb_agg(pg_catalog.jsonb_build_object( + 'capability', capability, + 'roleName', role_name, + 'canLogin', can_login, + 'inherits', inherits, + 'superuser', superuser, + 'bypassRls', bypass_rls, + 'createRole', create_role, + 'createDatabase', create_database, + 'replication', replication, + 'databaseConnect', database_connect, + 'ownedProbeSchemas', owned_probe_schemas, + 'createOnProbeSchemas', create_on_probe_schemas, + 'inheritedMemberships', inherited_memberships + ) ORDER BY ordinal) +)::text +FROM profiles; +`; +}; + +const buildLiveCloneAuditSql = () => ` +SELECT pg_catalog.jsonb_build_object( + 'version', 1, + 'kind', ${quoteLiteral(CLONE_AUDIT_KIND)}, + 'cloneId', clone_id, + 'purpose', run_purpose, + 'customerId', customer_id, + 'database', pg_catalog.current_database(), + 'nonce', attestation_nonce, + 'sha256', attestation_sha256 +)::text +FROM ctf_provision_private.clone_attestation +WHERE singleton = true; +`; + +const buildUnsafeRoleCleanupSql = ({ names }) => ` +BEGIN; +DROP SCHEMA IF EXISTS ${quoteIdentifier(names.ownerSchema)} CASCADE; +DROP SCHEMA IF EXISTS ${quoteIdentifier(names.createSchema)} CASCADE; +${Object.values(names.roles).map((role, index) => ` +DO $cleanup_${index}$ +BEGIN + IF EXISTS ( + SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = ${quoteLiteral(role)} + ) THEN + EXECUTE ${quoteLiteral(`DROP OWNED BY ${quoteIdentifier(role)}`)}; + END IF; +END +$cleanup_${index}$; +`).join('\n')} +${Object.values(names.roles).reverse().map((role) => + `DROP ROLE IF EXISTS ${quoteIdentifier(role)};` + ).join('\n')} +COMMIT; +`; + +const buildUnsafeRoleCleanupAuditSql = ({ names }) => ` +SELECT pg_catalog.jsonb_build_object( + 'version', 1, + 'kind', ${quoteLiteral(CLEANUP_AUDIT_KIND)}, + 'database', pg_catalog.current_database(), + 'remainingRoles', ( + SELECT pg_catalog.count(*)::integer + FROM pg_catalog.pg_roles + WHERE rolname = ANY(ARRAY[${Object.values(names.roles).map(quoteLiteral).join(', ')}]::text[]) + ), + 'remainingSchemas', ( + SELECT pg_catalog.count(*)::integer + FROM pg_catalog.pg_namespace + WHERE nspname = ANY(ARRAY[${[ + names.ownerSchema, + names.createSchema, + ].map(quoteLiteral).join(', ')}]::text[]) + ) +)::text; +`; + +const validateUnsafeRoleCleanupAudit = (audit, { database }) => { + if ( + !exactKeys(audit, [ + 'version', + 'kind', + 'database', + 'remainingRoles', + 'remainingSchemas', + ]) + || audit.version !== 1 + || audit.kind !== CLEANUP_AUDIT_KIND + || audit.database !== database + || audit.remainingRoles !== 0 + || audit.remainingSchemas !== 0 + ) { + throw new Error('PDCF_UNSAFE_ROLE_CLEANUP_AUDIT_FAILED'); + } + return { + ...audit, + passed: true, + }; +}; + +const runPsql = ({ database, sql, environment = process.env }) => { + const result = spawnSync('psql', [ + '--no-psqlrc', + '--no-align', + '--tuples-only', + '--quiet', + '--set=ON_ERROR_STOP=1', + '--dbname', database, + ], { + cwd: __dirname, + env: environment, + encoding: 'utf8', + input: sql, + maxBuffer: 16 * 1024 * 1024, + timeout: 120_000, + }); + if (result.status !== 0) { + throw new Error('PDCF_UNSAFE_ROLE_SQL_FAILED'); + } + return result.stdout; +}; + +const runPsqlJson = (input) => { + const output = String(runPsql(input) ?? '').trim(); + if (!output || output.includes('\n')) { + throw new Error('PDCF_UNSAFE_ROLE_AUDIT_RESULT_INVALID'); + } + try { + return JSON.parse(output); + } catch { + throw new Error('PDCF_UNSAFE_ROLE_AUDIT_RESULT_INVALID'); + } +}; + +const validateLiveCloneAudit = (audit, { manifest, customer }) => { + if ( + !exactKeys(audit, [ + 'version', + 'kind', + 'cloneId', + 'purpose', + 'customerId', + 'database', + 'nonce', + 'sha256', + ]) + || audit.version !== 1 + || audit.kind !== CLONE_AUDIT_KIND + || audit.cloneId !== manifest.provisionClone.id + || audit.purpose !== manifest.provisionClone.purpose + || audit.customerId !== customer.id + || audit.database !== customer.database + || !/^[a-f0-9]{64}$/.test(audit.nonce ?? '') + || audit.sha256 !== customer.provisionAttestation.sha256 + || completeServer.provisionAttestationSha256({ + cloneId: audit.cloneId, + purpose: audit.purpose, + customerId: audit.customerId, + database: audit.database, + nonce: audit.nonce, + }) !== audit.sha256 + ) { + throw new Error('PDCF_UNSAFE_ROLE_LIVE_CLONE_AUDIT_INVALID'); + } + return { + version: 1, + cloneId: audit.cloneId, + purpose: audit.purpose, + customerId: audit.customerId, + database: audit.database, + sha256: audit.sha256, + verified: true, + }; +}; + +const expectedAuditedProfiles = () => PROBE_CAPABILITIES.map((capability) => ({ + capability, + canLogin: true, + inherits: false, + superuser: capability === 'superuser', + bypassRls: capability === 'bypassrls', + createRole: capability === 'createrole', + createDatabase: false, + replication: false, + databaseConnect: true, + ownedProbeSchemas: capability === 'schema-owner' ? ['owner'] : [], + createOnProbeSchemas: capability === 'superuser' + ? ['create', 'owner'] + : capability === 'schema-owner' + ? ['owner'] + : capability === 'schema-create' + ? ['create'] + : [], + inheritedMemberships: 0, +})); + +const validateUnsafeRoleAudit = (audit, { database, names }) => { + if ( + !exactKeys(audit, ['version', 'kind', 'database', 'profiles']) + || audit.version !== 1 + || audit.kind !== ROLE_AUDIT_KIND + || audit.database !== database + || !Array.isArray(audit.profiles) + || audit.profiles.length !== PROBE_CAPABILITIES.length + ) { + throw new Error('PDCF_UNSAFE_ROLE_AUDIT_RESULT_INVALID'); + } + const expectedProfiles = expectedAuditedProfiles(); + const normalized = audit.profiles.map((profile, index) => { + const expected = expectedProfiles[index]; + if ( + !exactKeys(profile, [ + 'capability', + 'roleName', + 'canLogin', + 'inherits', + 'superuser', + 'bypassRls', + 'createRole', + 'createDatabase', + 'replication', + 'databaseConnect', + 'ownedProbeSchemas', + 'createOnProbeSchemas', + 'inheritedMemberships', + ]) + || profile.roleName !== names.roles[expected.capability] + ) { + throw new Error('PDCF_UNSAFE_ROLE_AUDIT_RESULT_INVALID'); + } + const { roleName: _roleName, ...credentialFreeProfile } = profile; + if (Object.entries(expected).some(([key, value]) => + JSON.stringify(credentialFreeProfile[key]) !== JSON.stringify(value) + )) { + throw new Error(`PDCF_UNSAFE_ROLE_PROFILE_MISMATCH:${expected.capability}`); + } + return expected; + }); + return { + version: 1, + kind: ROLE_AUDIT_KIND, + database, + profiles: normalized, + passed: true, + }; +}; + +const WORKER_RESULT_KEYS = Object.freeze([ + 'version', + 'kind', + 'admissionScope', + 'customerId', + 'tenantId', + 'capability', + 'cloneId', + 'provisionAttestationSha256', + 'runtimeArtifactFingerprint', + 'physicalDatabaseVerifiedBeforeRoleAudit', + 'controlCredentialEnvironmentAbsent', + 'accepted', + 'rejectedCode', + 'graphileBuildsStarted', + 'residentGraphileEntries', +]); + +const WORKER_ENVIRONMENT_ALLOWLIST = new Set([ + 'PATH', + 'NODE_ENV', + 'TMPDIR', + 'TMP', + 'TEMP', + 'LANG', + 'LC_ALL', + 'LC_CTYPE', + '__CF_USER_TEXT_ENCODING', + 'TZ', + 'PGHOST', + 'PGPORT', + 'PGSSLMODE', + 'PGSSLROOTCERT', + 'PGCHANNELBINDING', +]); +const WORKER_RUNTIME_PASSWORD_KEYS = Object.freeze(TENANTS.map( + (tenant) => tenant.runtimePasswordEnvironment, +)); +const WORKER_EXACT_ENVIRONMENT_KEYS = new Set([ + ...WORKER_ENVIRONMENT_ALLOWLIST, + 'PGDATABASE', + ...WORKER_RUNTIME_PASSWORD_KEYS, +]); + +const parseWorkerResult = (stdout) => { + const lines = String(stdout ?? '').split(/\r?\n/).map((line) => line.trim()).filter(Boolean); + const results = []; + for (const line of lines) { + try { + const value = JSON.parse(line); + if (value?.version === PROBE_VERSION && value.kind === PROBE_KIND) results.push(value); + } catch { + // Runtime logging may precede the final one-line result. Only the exact + // credential-free worker envelope is accepted. + } + } + if (results.length !== 1 || !exactKeys(results[0], WORKER_RESULT_KEYS)) { + throw new Error('PDCF_UNSAFE_ROLE_WORKER_RESULT_INVALID'); + } + return results[0]; +}; + +const makeWorkerEnvironment = (environment) => Object.fromEntries( + Object.entries(environment ?? {}).filter(([key]) => + WORKER_ENVIRONMENT_ALLOWLIST.has(key) + ), +); + +const assertExactWorkerEnvironment = (environment) => { + const keys = Object.keys(environment ?? {}); + if ( + keys.some((key) => !WORKER_EXACT_ENVIRONMENT_KEYS.has(key)) + || typeof environment?.PGDATABASE !== 'string' + || environment.PGDATABASE.length === 0 + || WORKER_RUNTIME_PASSWORD_KEYS.some((key) => + typeof environment[key] !== 'string' + || Buffer.byteLength(environment[key]) < 24 + ) + ) { + throw new Error('PDCF_UNSAFE_ROLE_WORKER_ENVIRONMENT_INVALID'); + } + return true; +}; + +const makeProbeWorkerEnvironment = ({ + environment, + database, + tenantId, + password, + runtimePasswords, +}) => { + const passwordEnvironment = Object.fromEntries(TENANTS.map((tenant) => { + const value = tenant.id === tenantId ? password : runtimePasswords?.[tenant.id]; + if (typeof value !== 'string' || Buffer.byteLength(value) < 24) { + throw new Error(`PDCF_UNSAFE_ROLE_PASSWORD_REQUIRED:${tenant.id}`); + } + return [tenant.runtimePasswordEnvironment, value]; + })); + const workerEnvironment = { + ...makeWorkerEnvironment(environment), + PGDATABASE: database, + ...passwordEnvironment, + }; + assertExactWorkerEnvironment(workerEnvironment); + return workerEnvironment; +}; + +const defaultRunWorker = ({ + manifestFile, + customerId, + database, + tenantId, + capability, + role, + password, + runtimePasswords, + mode, + environment, +}) => { + const workerEnvironment = makeProbeWorkerEnvironment({ + environment, + database, + tenantId, + password, + runtimePasswords, + }); + const result = spawnSync(process.execPath, [ + __filename, + '--worker', + '--manifest', manifestFile, + '--customer-id', customerId, + '--tenant', tenantId, + '--capability', capability, + '--probe-role', role, + '--mode', mode, + ], { + cwd: REPO_ROOT, + env: workerEnvironment, + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + timeout: 180_000, + }); + if (result.status !== 0 || result.signal) { + const workerCode = String(result.stderr ?? '').split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => /^[A-Z][A-Z0-9_]{2,95}$/.test(line)) + .at(-1) ?? 'PDCF_UNSAFE_ROLE_WORKER_UNKNOWN'; + throw new Error( + `PDCF_UNSAFE_ROLE_WORKER_FAILED:${capability}:${tenantId}:${workerCode}`, + ); + } + return parseWorkerResult(result.stdout); +}; + +const validateWorkerRejection = (result, { + customerId, + tenantId, + capability, + cloneId, + provisionAttestationSha256, + runtimeArtifactFingerprint, +}) => { + if ( + !exactKeys(result, WORKER_RESULT_KEYS) + || result.version !== PROBE_VERSION + || result.kind !== PROBE_KIND + || result.admissionScope !== ADMISSION_SCOPE + || result.cloneId !== cloneId + || result.provisionAttestationSha256 !== provisionAttestationSha256 + || result.runtimeArtifactFingerprint !== runtimeArtifactFingerprint + || result.physicalDatabaseVerifiedBeforeRoleAudit !== true + || result.controlCredentialEnvironmentAbsent !== true + || result.customerId !== customerId + || result.tenantId !== tenantId + || result.capability !== capability + || result.accepted !== false + || result.rejectedCode !== 'GRAPHILE_UNSAFE_RUNTIME_ROLE' + || result.graphileBuildsStarted !== 0 + || result.residentGraphileEntries !== 0 + ) { + throw new Error(`PDCF_UNSAFE_ROLE_NOT_REJECTED:${capability}:${tenantId}`); + } + return result; +}; + +const validateWorkerAcceptance = (result, { + customerId, + tenantId, + cloneId, + provisionAttestationSha256, + runtimeArtifactFingerprint, +}) => { + if ( + !exactKeys(result, WORKER_RESULT_KEYS) + || result.version !== PROBE_VERSION + || result.kind !== PROBE_KIND + || result.admissionScope !== ADMISSION_SCOPE + || result.customerId !== customerId + || result.tenantId !== tenantId + || result.capability !== SAFE_CONTROL_CAPABILITY + || result.cloneId !== cloneId + || result.provisionAttestationSha256 !== provisionAttestationSha256 + || result.runtimeArtifactFingerprint !== runtimeArtifactFingerprint + || result.physicalDatabaseVerifiedBeforeRoleAudit !== true + || result.controlCredentialEnvironmentAbsent !== true + || result.accepted !== true + || result.rejectedCode !== null + || result.graphileBuildsStarted !== 0 + || result.residentGraphileEntries !== 0 + ) { + throw new Error('PDCF_SAFE_RUNTIME_ROLE_CONTROL_REJECTED'); + } + return result; +}; + +const runUnsafeRuntimeStartupMatrix = ({ + manifestFile, + secretsFile, + expectedRuntimeArtifactFingerprint, + mode = 'scoped-required', + environment = process.env, + nonce = crypto.randomBytes(6).toString('hex'), + runSql = runPsql, + runCloneAudit = runPsqlJson, + runRoleAudit = runPsqlJson, + runCleanupAudit = runPsqlJson, + runWorker = defaultRunWorker, +} = {}) => { + const absoluteManifestFile = path.resolve(manifestFile); + const absoluteSecretsFile = path.resolve(secretsFile); + const { manifest, secrets } = loadPrivateProvision( + absoluteManifestFile, + absoluteSecretsFile, + ); + if (manifest.provisionClone?.purpose !== 'hostile-preflight') { + throw new Error('PDCF_UNSAFE_ROLE_HOSTILE_CLONE_REQUIRED'); + } + if (mode !== 'stock' && mode !== 'scoped-required') { + throw new Error('PDCF_UNSAFE_ROLE_MODE_INVALID'); + } + if (!SHA256_PATTERN.test(expectedRuntimeArtifactFingerprint ?? '')) { + throw new Error('PDCF_UNSAFE_ROLE_RUNTIME_FINGERPRINT_REQUIRED'); + } + const customer = manifest.customers[0]; + if (!customer) throw new Error('PDCF_UNSAFE_ROLE_CUSTOMER_REQUIRED'); + const liveProvisionAttestation = validateLiveCloneAudit(runCloneAudit({ + database: customer.database, + sql: buildLiveCloneAuditSql(), + environment, + }), { manifest, customer }); + const names = probeNames(nonce); + const passwords = Object.fromEntries(PROBE_CAPABILITIES.map((capability) => [ + capability, + crypto.randomBytes(32).toString('base64url'), + ])); + const safeTenant = TENANTS[0]; + const safeRole = customer.roles[safeTenant.id]; + const runtimePasswords = Object.fromEntries(TENANTS.map((tenant) => [ + tenant.id, + secrets.runtimePasswords[customer.roles[tenant.id]], + ])); + const safeControlResult = validateWorkerAcceptance(runWorker({ + manifestFile: absoluteManifestFile, + customerId: customer.id, + database: customer.database, + tenantId: safeTenant.id, + capability: SAFE_CONTROL_CAPABILITY, + role: safeRole, + password: secrets.runtimePasswords[safeRole], + runtimePasswords, + mode, + environment, + }), { + customerId: customer.id, + tenantId: safeTenant.id, + cloneId: manifest.provisionClone.id, + provisionAttestationSha256: customer.provisionAttestation.sha256, + runtimeArtifactFingerprint: expectedRuntimeArtifactFingerprint, + }); + let setupAttempted = false; + let roleAudit; + let cleanupAudit; + let matrixError = null; + const attempts = []; + try { + setupAttempted = true; + runSql({ + database: customer.database, + sql: buildUnsafeRoleSetupSql({ + database: customer.database, + names, + passwords, + }), + environment, + }); + roleAudit = validateUnsafeRoleAudit(runRoleAudit({ + database: customer.database, + sql: buildUnsafeRoleAuditSql({ names }), + environment, + }), { + database: customer.database, + names, + }); + for (const capability of PROBE_CAPABILITIES) { + for (const tenant of TENANTS) { + const result = validateWorkerRejection(runWorker({ + manifestFile: absoluteManifestFile, + customerId: customer.id, + database: customer.database, + tenantId: tenant.id, + capability, + role: names.roles[capability], + password: passwords[capability], + runtimePasswords, + mode, + environment, + }), { + customerId: customer.id, + tenantId: tenant.id, + capability, + cloneId: manifest.provisionClone.id, + provisionAttestationSha256: customer.provisionAttestation.sha256, + runtimeArtifactFingerprint: expectedRuntimeArtifactFingerprint, + }); + attempts.push({ + capability, + tenantId: tenant.id, + rejectedCode: result.rejectedCode, + controlCredentialEnvironmentAbsent: + result.controlCredentialEnvironmentAbsent, + graphileBuildsStarted: result.graphileBuildsStarted, + residentGraphileEntries: result.residentGraphileEntries, + }); + } + } + } catch (error) { + matrixError = error; + } + let cleanupError = null; + if (setupAttempted) { + try { + runSql({ + database: customer.database, + sql: buildUnsafeRoleCleanupSql({ names }), + environment, + }); + cleanupAudit = validateUnsafeRoleCleanupAudit(runCleanupAudit({ + database: customer.database, + sql: buildUnsafeRoleCleanupAuditSql({ names }), + environment, + }), { database: customer.database }); + } catch (error) { + cleanupError = error; + } + } + if (cleanupError) { + throw new Error('PDCF_UNSAFE_ROLE_CLEANUP_FAILED', { cause: cleanupError }); + } + if (matrixError) throw matrixError; + const report = { + version: PROBE_VERSION, + kind: PROBE_KIND, + admissionScope: ADMISSION_SCOPE, + provisionClone: { + version: manifest.provisionClone.version, + id: manifest.provisionClone.id, + purpose: manifest.provisionClone.purpose, + attestationSetSha256: manifest.provisionClone.attestationSetSha256, + }, + representativeCustomerId: customer.id, + representativePhysicalDatabase: customer.physicalIdentity, + representativeProvisionAttestationSha256: + customer.provisionAttestation.sha256, + canonicalDatabaseContractFingerprint: + manifest.canonicalDatabaseContractFingerprint, + runtimeArtifactFingerprint: expectedRuntimeArtifactFingerprint, + liveProvisionAttestation, + safeStartupControl: { + tenantId: safeControlResult.tenantId, + accepted: safeControlResult.accepted, + physicalDatabaseVerifiedBeforeRoleAudit: + safeControlResult.physicalDatabaseVerifiedBeforeRoleAudit, + controlCredentialEnvironmentAbsent: + safeControlResult.controlCredentialEnvironmentAbsent, + graphileBuildsStarted: safeControlResult.graphileBuildsStarted, + residentGraphileEntries: safeControlResult.residentGraphileEntries, + passed: true, + }, + roleProfileAudit: roleAudit, + cleanupAudit, + capabilities: [...PROBE_CAPABILITIES], + surfaces: TENANTS.map((tenant) => tenant.id), + attempts, + expectedAttempts: PROBE_CAPABILITIES.length * TENANTS.length, + rejectedAttempts: attempts.length, + acceptedAttempts: 0, + graphileBuildsStarted: attempts.reduce( + (sum, attempt) => sum + attempt.graphileBuildsStarted, + 0, + ), + residentGraphileEntries: attempts.reduce( + (sum, attempt) => sum + attempt.residentGraphileEntries, + 0, + ), + passed: attempts.length === PROBE_CAPABILITIES.length * TENANTS.length, + }; + assertCredentialFree(report); + return report; +}; + +const verifyPhysicalDatabaseWithRuntimeCredential = async ({ + database, + role, + password, +}) => { + const { Pool } = require(path.join(REPO_ROOT, 'graphql/server/node_modules/pg')); + const pool = new Pool({ + database, + user: role, + password, + max: 1, + connectionTimeoutMillis: 5_000, + application_name: 'unsafe-runtime-startup-probe', + }); + try { + const result = await pool.query(` + SELECT pg_catalog.current_database()::text AS database, + current_user::text AS role + `); + if ( + result.rowCount !== 1 + || result.rows[0]?.database !== database + || result.rows[0]?.role !== role + ) { + throw new Error('PDCF_UNSAFE_ROLE_PHYSICAL_DATABASE_MISMATCH'); + } + return true; + } finally { + await pool.end(); + } +}; + +const workerProbe = async ({ + manifestFile, + customerId, + tenantId, + capability, + probeRole, + mode, + environment = process.env, +}) => { + assertExactWorkerEnvironment(environment); + const controlCredentialEnvironmentAbsent = true; + requireSafeLabel(customerId, 'PDCF_UNSAFE_ROLE_CUSTOMER_INVALID'); + requireSafeLabel(tenantId, 'PDCF_UNSAFE_ROLE_TENANT_INVALID'); + requireProbeCase(capability); + if (!TENANTS.some((tenant) => tenant.id === tenantId)) { + throw new Error('PDCF_UNSAFE_ROLE_TENANT_INVALID'); + } + if (typeof probeRole !== 'string' || !/^[a-z_][a-z0-9_]{0,62}$/.test(probeRole)) { + throw new Error('PDCF_UNSAFE_ROLE_NAME_INVALID'); + } + const selectedTenant = TENANTS.find((tenant) => tenant.id === tenantId); + const password = environment[selectedTenant.runtimePasswordEnvironment]; + if (typeof password !== 'string' || Buffer.byteLength(password) < 24) { + throw new Error('PDCF_UNSAFE_ROLE_PASSWORD_REQUIRED'); + } + const manifest = validateProvisionManifest(readJson(manifestFile)); + assertCredentialFree(manifest); + if ( + manifest.fixture !== FIXTURE_ID + || manifest.provisionClone?.purpose !== 'hostile-preflight' + ) { + throw new Error('PDCF_UNSAFE_ROLE_HOSTILE_CLONE_REQUIRED'); + } + const customer = manifest.customers.find((candidate) => candidate.id === customerId); + if (!customer) throw new Error('PDCF_UNSAFE_ROLE_CUSTOMER_INVALID'); + if (environment.PGDATABASE !== customer.database) { + throw new Error('PDCF_UNSAFE_ROLE_PHYSICAL_DATABASE_MISMATCH'); + } + if ( + capability === SAFE_CONTROL_CAPABILITY + ? probeRole !== customer.roles[tenantId] + : !PROBE_ROLE_PATTERNS[capability].test(probeRole) + ) { + throw new Error('PDCF_UNSAFE_ROLE_NAME_INVALID'); + } + const runtimeRoles = capability === SAFE_CONTROL_CAPABILITY + ? { ...customer.roles } + : { ...customer.roles, [tenantId]: probeRole }; + const childEnvironment = { + ...makeWorkerEnvironment(environment), + PGDATABASE: customer.database, + ...Object.fromEntries(TENANTS.map((tenant) => [ + tenant.runtimePasswordEnvironment, + environment[tenant.runtimePasswordEnvironment], + ])), + }; + if (environment === process.env) { + Object.assign(process.env, childEnvironment); + } + const physicalDatabaseVerifiedBeforeRoleAudit = + await verifyPhysicalDatabaseWithRuntimeCredential({ + database: customer.database, + role: probeRole, + password, + }); + const options = { + ...completeServer.parseServerOptions([ + '--host', '127.0.0.1', + '--port', '3391', + '--arm', 'unsafe-runtime-startup-probe', + '--mode', mode, + '--introspection-client-release-mode', 'destroy', + '--runtime-pool-max', '2', + '--enable-realtime', 'true', + ...TENANTS.flatMap((tenant) => [ + `--${tenant.runtimeRoleArgument}`, + runtimeRoles[tenant.id], + ]), + ], childEnvironment), + runPurpose: 'hostile-preflight', + cloneId: manifest.provisionClone.id, + }; + let accepted = false; + let rejectedCode = null; + let server = null; + try { + server = await completeServer.createFixtureServer(options, childEnvironment); + accepted = true; + } catch (error) { + rejectedCode = typeof error?.code === 'string' ? error.code : null; + if (rejectedCode !== 'GRAPHILE_UNSAFE_RUNTIME_ROLE') throw error; + } finally { + if (server) { + await server.close(); + } else { + // createFixtureServer has not returned its close handle when startup is + // rejected, but its pre-publication role audit may already have leased + // pools. End them explicitly so each isolated worker exits immediately + // instead of waiting for node-postgres idle timeouts. + await require(path.join( + REPO_ROOT, + 'postgres/pg-cache/dist/index.js', + )).teardownPgPools(); + } + } + const graphileCache = require(path.join( + REPO_ROOT, + 'graphile/graphile-cache/dist/index.js', + )).graphileCache; + const buildStats = require(path.join( + REPO_ROOT, + 'graphql/server/dist/middleware/observability/graphile-build-stats.js', + )).getGraphileBuildStats(); + const result = { + version: PROBE_VERSION, + kind: PROBE_KIND, + admissionScope: ADMISSION_SCOPE, + customerId, + tenantId, + capability, + cloneId: manifest.provisionClone.id, + provisionAttestationSha256: customer.provisionAttestation.sha256, + runtimeArtifactFingerprint: completeServer.runtimeArtifactFingerprint(), + physicalDatabaseVerifiedBeforeRoleAudit, + controlCredentialEnvironmentAbsent, + accepted, + rejectedCode, + graphileBuildsStarted: buildStats.started, + residentGraphileEntries: graphileCache.size, + }; + assertCredentialFree(result); + return result; +}; + +const main = async () => { + const args = parseArgs(process.argv.slice(2)); + if (args.worker !== true) throw new Error('PDCF_UNSAFE_ROLE_WORKER_REQUIRED'); + const result = await workerProbe({ + manifestFile: path.resolve(requireString(args, 'manifest')), + customerId: requireString(args, 'customer-id'), + tenantId: requireString(args, 'tenant'), + capability: requireString(args, 'capability'), + probeRole: requireString(args, 'probe-role'), + mode: requireString(args, 'mode', 'scoped-required'), + }); + process.stdout.write(`${JSON.stringify(result)}\n`); +}; + +if (require.main === module) { + main().catch((error) => { + const code = typeof error?.code === 'string' + ? error.code + : String(error instanceof Error ? error.message : error).split(':', 1)[0]; + process.stderr.write(`${code}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + ADMISSION_SCOPE, + CLEANUP_AUDIT_KIND, + CLONE_AUDIT_KIND, + PROBE_CAPABILITIES, + PROBE_KIND, + ROLE_AUDIT_KIND, + SAFE_CONTROL_CAPABILITY, + assertExactWorkerEnvironment, + buildLiveCloneAuditSql, + buildUnsafeRoleAuditSql, + buildUnsafeRoleCleanupAuditSql, + buildUnsafeRoleCleanupSql, + buildUnsafeRoleSetupSql, + expectedAuditedProfiles, + loadPrivateProvision, + makeProbeWorkerEnvironment, + makeWorkerEnvironment, + parseWorkerResult, + probeNames, + runUnsafeRuntimeStartupMatrix, + validateLiveCloneAudit, + validateUnsafeRoleCleanupAudit, + validateUnsafeRoleAudit, + validateWorkerAcceptance, + validateWorkerRejection, + verifyPhysicalDatabaseWithRuntimeCredential, + workerProbe, +}; diff --git a/research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.test.cjs b/research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.test.cjs new file mode 100644 index 0000000000..4d6c780db4 --- /dev/null +++ b/research/graphile-density/physical-database-density/unsafe-runtime-startup-probe.test.cjs @@ -0,0 +1,435 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { describe, it } = require('node:test'); + +const { + TENANTS, + assertCredentialFree, +} = require('../complete-tenant-fixture/lib.cjs'); +const { makeCustomers } = require('./lib.cjs'); +const { provisionAttestationSha256 } = require('./provision.cjs'); +const { + ADMISSION_SCOPE, + CLEANUP_AUDIT_KIND, + PROBE_CAPABILITIES, + PROBE_KIND, + SAFE_CONTROL_CAPABILITY, + assertExactWorkerEnvironment, + buildLiveCloneAuditSql, + buildUnsafeRoleAuditSql, + buildUnsafeRoleCleanupAuditSql, + buildUnsafeRoleCleanupSql, + buildUnsafeRoleSetupSql, + expectedAuditedProfiles, + makeProbeWorkerEnvironment, + makeWorkerEnvironment, + parseWorkerResult, + probeNames, + runUnsafeRuntimeStartupMatrix, + workerProbe, +} = require('./unsafe-runtime-startup-probe.cjs'); + +const digest = (character) => `sha256:${character.repeat(64)}`; +const runtimeArtifactFingerprint = digest('e'); + +const writeFixture = (directory) => { + const nonce = '1'.repeat(64); + const customer = { + ...makeCustomers('unsafe_probe', 1)[0], + provisionAttestation: { + version: 1, + cloneId: 'unsafe-probe-clone', + purpose: 'hostile-preflight', + sha256: provisionAttestationSha256({ + cloneId: 'unsafe-probe-clone', + runPurpose: 'hostile-preflight', + customerId: 'physical-customer-0001', + database: 'unsafe_probe_db_0001', + nonce, + }), + }, + structuralFingerprints: { combined: { sha256: digest('b') } }, + databaseContractFingerprint: digest('c'), + }; + const manifest = { + version: 1, + fixture: 'physical-database-density-v1', + prefix: 'unsafe_probe', + provisionClone: { + version: 1, + id: 'unsafe-probe-clone', + purpose: 'hostile-preflight', + attestationSetSha256: digest('d'), + }, + canonicalStructuralFingerprint: customer.structuralFingerprints, + canonicalDatabaseContractFingerprint: customer.databaseContractFingerprint, + customers: [customer], + }; + const secrets = { + version: 1, + fixture: 'physical-database-density-v1', + runtimePasswords: Object.fromEntries(Object.values(customer.roles).map((role) => [ + role, + `safe-runtime-password-${role}`, + ])), + notificationPasswords: { + [customer.notificationRole]: + `safe-notification-password-${customer.notificationRole}`, + }, + }; + const manifestFile = path.join(directory, 'provision.json'); + const secretsFile = path.join(directory, 'runtime-secrets.json'); + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + fs.writeFileSync(secretsFile, JSON.stringify(secrets), { mode: 0o600 }); + return { customer, manifestFile, nonce, secretsFile }; +}; + +const liveCloneAuditFor = (customer, nonce) => ({ + version: 1, + kind: 'unsafe-runtime-live-clone-audit-v1', + cloneId: customer.provisionAttestation.cloneId, + purpose: customer.provisionAttestation.purpose, + customerId: customer.id, + database: customer.database, + nonce, + sha256: customer.provisionAttestation.sha256, +}); + +const roleAuditFor = (customer, names) => ({ + version: 1, + kind: 'unsafe-runtime-role-profile-audit-v1', + database: customer.database, + profiles: expectedAuditedProfiles().map((profile) => ({ + ...profile, + roleName: names.roles[profile.capability], + })), +}); + +const workerResultFor = (customer, input, accepted) => ({ + version: 2, + kind: PROBE_KIND, + admissionScope: ADMISSION_SCOPE, + customerId: customer.id, + tenantId: input.tenantId, + capability: input.capability, + cloneId: customer.provisionAttestation.cloneId, + provisionAttestationSha256: customer.provisionAttestation.sha256, + runtimeArtifactFingerprint, + physicalDatabaseVerifiedBeforeRoleAudit: true, + controlCredentialEnvironmentAbsent: true, + accepted, + rejectedCode: accepted ? null : 'GRAPHILE_UNSAFE_RUNTIME_ROLE', + graphileBuildsStarted: 0, + residentGraphileEntries: 0, +}); + +const cleanupAuditFor = (customer) => ({ + version: 1, + kind: CLEANUP_AUDIT_KIND, + database: customer.database, + remainingRoles: 0, + remainingSchemas: 0, +}); + +describe('unsafe runtime startup admission matrix', () => { + it('builds five materially distinct PostgreSQL privilege profiles', () => { + const names = probeNames('012345abcdef'); + const passwords = Object.fromEntries(PROBE_CAPABILITIES.map((capability) => [ + capability, + `password-${capability}`, + ])); + const setup = buildUnsafeRoleSetupSql({ + database: 'unsafe_probe_db_0001', + names, + passwords, + }); + assert.match(setup, /SUPERUSER NOBYPASSRLS/); + assert.match(setup, /NOSUPERUSER BYPASSRLS/); + assert.match(setup, /NOCREATEDB CREATEROLE/); + assert.match(setup, /CREATE SCHEMA "ctf_unsafe_owner_012345abcdef"\s+AUTHORIZATION/); + assert.match(setup, /GRANT CREATE ON SCHEMA "ctf_unsafe_create_012345abcdef"/); + assert.equal((setup.match(/NOINHERIT/g) ?? []).length, 5); + + const audit = buildUnsafeRoleAuditSql({ names }); + assert.match(audit, /pg_catalog\.pg_roles/); + assert.match(audit, /pg_catalog\.pg_auth_members/); + assert.match(audit, /pg_catalog\.has_schema_privilege/); + assert.match(audit, /pg_catalog\.has_database_privilege/); + assert.match(buildLiveCloneAuditSql(), /ctf_provision_private\.clone_attestation/); + + const cleanup = buildUnsafeRoleCleanupSql({ names }); + assert.match(cleanup, /DROP SCHEMA IF EXISTS/); + assert.equal((cleanup.match(/DROP OWNED BY/g) ?? []).length, 5); + assert.equal((cleanup.match(/DROP ROLE IF EXISTS/g) ?? []).length, 5); + assert.match(buildUnsafeRoleCleanupAuditSql({ names }), /remainingRoles/); + assert.match(buildUnsafeRoleCleanupAuditSql({ names }), /remainingSchemas/); + }); + + it('requires all five capabilities to fail before publication on A, B, and C', (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-unsafe-probe-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const { customer, manifestFile, nonce, secretsFile } = writeFixture(temporary); + const sqlCalls = []; + const cloneAuditCalls = []; + const auditCalls = []; + const workerCalls = []; + const names = probeNames('012345abcdef'); + const report = runUnsafeRuntimeStartupMatrix({ + manifestFile, + secretsFile, + expectedRuntimeArtifactFingerprint: runtimeArtifactFingerprint, + nonce: '012345abcdef', + runSql: (input) => sqlCalls.push(input), + runCloneAudit: (input) => { + cloneAuditCalls.push(input); + return liveCloneAuditFor(customer, nonce); + }, + runRoleAudit: (input) => { + auditCalls.push(input); + return roleAuditFor(customer, names); + }, + runCleanupAudit: () => cleanupAuditFor(customer), + runWorker: (input) => { + workerCalls.push(input); + return workerResultFor( + customer, + input, + input.capability === SAFE_CONTROL_CAPABILITY, + ); + }, + }); + + assert.equal(sqlCalls.length, 2); + assert.equal(cloneAuditCalls.length, 1); + assert.match(cloneAuditCalls[0].sql, /clone_attestation/); + assert.match(sqlCalls[0].sql, /CREATE ROLE/); + assert.match(sqlCalls[1].sql, /DROP ROLE IF EXISTS/); + assert.equal(auditCalls.length, 1); + assert.match(auditCalls[0].sql, /unsafe-runtime-role-profile-audit-v1/); + assert.equal(workerCalls.length, 16); + assert.equal(workerCalls[0].capability, SAFE_CONTROL_CAPABILITY); + assert.deepEqual( + new Set(workerCalls.slice(1).map((call) => call.capability)), + new Set(PROBE_CAPABILITIES), + ); + assert.deepEqual( + new Set(workerCalls.slice(1).map((call) => call.tenantId)), + new Set(TENANTS.map((tenant) => tenant.id)), + ); + assert.equal(report.passed, true); + assert.equal(report.expectedAttempts, 15); + assert.equal(report.rejectedAttempts, 15); + assert.equal(report.acceptedAttempts, 0); + assert.equal(report.graphileBuildsStarted, 0); + assert.equal(report.residentGraphileEntries, 0); + assert.equal(report.safeStartupControl.passed, true); + assert.equal(report.safeStartupControl.physicalDatabaseVerifiedBeforeRoleAudit, true); + assert.equal(report.safeStartupControl.controlCredentialEnvironmentAbsent, true); + assert.equal(report.liveProvisionAttestation.verified, true); + assert.equal(report.roleProfileAudit.passed, true); + assert.equal(report.cleanupAudit.passed, true); + assert.equal(report.runtimeArtifactFingerprint, runtimeArtifactFingerprint); + assert.equal(report.admissionScope, ADMISSION_SCOPE); + assert.equal(report.provisionClone.id, 'unsafe-probe-clone'); + assert.equal( + report.representativeProvisionAttestationSha256, + customer.provisionAttestation.sha256, + ); + assert.doesNotThrow(() => assertCredentialFree(JSON.stringify(report))); + for (const call of workerCalls) { + assert.equal(Object.hasOwn(call, 'secretsFile'), false); + assert.deepEqual(Object.keys(call.runtimePasswords).sort(), ['a', 'b', 'c']); + assert.doesNotMatch(JSON.stringify(report), new RegExp(call.password)); + } + }); + + it('cleans up and fails closed if any startup reaches publication admission', (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-unsafe-accepted-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const { customer, manifestFile, nonce, secretsFile } = writeFixture(temporary); + const sqlCalls = []; + const names = probeNames('fedcba987654'); + assert.throws(() => runUnsafeRuntimeStartupMatrix({ + manifestFile, + secretsFile, + expectedRuntimeArtifactFingerprint: runtimeArtifactFingerprint, + nonce: 'fedcba987654', + runSql: (input) => sqlCalls.push(input), + runCloneAudit: () => liveCloneAuditFor(customer, nonce), + runRoleAudit: () => roleAuditFor(customer, names), + runCleanupAudit: () => cleanupAuditFor(customer), + runWorker: (input) => workerResultFor(customer, input, true), + }), /PDCF_UNSAFE_ROLE_NOT_REJECTED/); + assert.equal(sqlCalls.length, 2); + assert.match(sqlCalls[1].sql, /DROP ROLE IF EXISTS/); + }); + + it('attempts idempotent cleanup and audits zero leftovers after ambiguous setup failure', (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-unsafe-ambiguous-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const { customer, manifestFile, nonce, secretsFile } = writeFixture(temporary); + const sqlCalls = []; + assert.throws(() => runUnsafeRuntimeStartupMatrix({ + manifestFile, + secretsFile, + expectedRuntimeArtifactFingerprint: runtimeArtifactFingerprint, + nonce: 'abcdef012345', + runSql: (input) => { + sqlCalls.push(input); + if (sqlCalls.length === 1) throw new Error('PDCF_TEST_SETUP_RESULT_AMBIGUOUS'); + }, + runCloneAudit: () => liveCloneAuditFor(customer, nonce), + runRoleAudit: () => assert.fail('role audit must not run'), + runCleanupAudit: () => cleanupAuditFor(customer), + runWorker: (input) => workerResultFor(customer, input, true), + }), /PDCF_TEST_SETUP_RESULT_AMBIGUOUS/); + assert.equal(sqlCalls.length, 2); + assert.match(sqlCalls[0].sql, /CREATE ROLE/); + assert.match(sqlCalls[1].sql, /DROP ROLE IF EXISTS/); + }); + + it('parses only the final exact worker envelope', () => { + const customer = { + id: 'physical-customer-0001', + provisionAttestation: { + cloneId: 'unsafe-probe-clone', + sha256: digest('a'), + }, + }; + const expected = workerResultFor(customer, { + tenantId: 'a', + capability: 'superuser', + }, false); + assert.deepEqual(parseWorkerResult( + `runtime log\n${JSON.stringify(expected)}\n`, + ), expected); + assert.throws( + () => parseWorkerResult('{"kind":"wrong"}\n'), + /PDCF_UNSAFE_ROLE_WORKER_RESULT_INVALID/, + ); + assert.throws( + () => parseWorkerResult(`${JSON.stringify(expected)}\n${JSON.stringify(expected)}\n`), + /PDCF_UNSAFE_ROLE_WORKER_RESULT_INVALID/, + ); + }); + + it('requires a private regular secrets file before any startup probe', (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-unsafe-private-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const { manifestFile, secretsFile } = writeFixture(temporary); + fs.chmodSync(secretsFile, 0o640); + assert.throws(() => runUnsafeRuntimeStartupMatrix({ + manifestFile, + secretsFile, + runWorker: () => assert.fail('worker must not run'), + }), /PDCF_UNSAFE_ROLE_SECRETS_NOT_PRIVATE/); + }); + + it('rejects a manifest containing credential-shaped fields before any worker starts', (context) => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'pdc-unsafe-manifest-secret-')); + context.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const { manifestFile, secretsFile } = writeFixture(temporary); + const manifest = JSON.parse(fs.readFileSync(manifestFile, 'utf8')); + manifest.password = 'credential-that-must-not-reach-the-worker'; + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + assert.throws(() => runUnsafeRuntimeStartupMatrix({ + manifestFile, + secretsFile, + runWorker: () => assert.fail('worker must not run'), + }), /CTF_ARTIFACT_CONTAINS_CREDENTIAL_MARKER/); + }); + + it('passes only PostgreSQL transport, locale, and inert process settings to workers', () => { + assert.deepEqual(makeWorkerEnvironment({ + PATH: '/bin', + NODE_OPTIONS: '--require=/tmp/worker-injection.cjs', + NODE_PATH: '/tmp/untrusted-modules', + PGHOST: '127.0.0.1', + PGPORT: '5432', + PGSSLMODE: 'verify-full', + PGPASSWORD: 'control-secret', + PGUSER: 'control-user', + PGSERVICE: 'control-service', + PGSERVICEFILE: '/private/control-service-file', + PGPASSFILE: '/private/control-passfile', + PGSSLKEY: '/private/client-key', + PGDATABASE: 'wrong-database', + DATABASE_URL: 'postgresql://control:secret@database/control', + GRAPHILE_CACHE_MAX: '5', + GRAPHQL_OBSERVABILITY_TOKEN: 'must-not-cross-boundary', + GRAPHQL_RUNTIME_PGUSER: 'must-not-cross-boundary', + GRAPHQL_RUNTIME_PGPASSWORD: 'must-not-cross-boundary', + CTF_CONTROL_TOKEN: 'must-not-cross-boundary', + GITHUB_TOKEN: 'must-not-cross-boundary', + }), { + PATH: '/bin', + PGHOST: '127.0.0.1', + PGPORT: '5432', + PGSSLMODE: 'verify-full', + }); + }); + + it('accepts only the exact worker environment key contract', () => { + const valid = { + PGDATABASE: 'unsafe_probe_db_0001', + CTF_RUNTIME_A_PGPASSWORD: 'a'.repeat(24), + CTF_RUNTIME_B_PGPASSWORD: 'b'.repeat(24), + CTF_RUNTIME_C_PGPASSWORD: 'c'.repeat(24), + }; + assert.equal(assertExactWorkerEnvironment(valid), true); + assert.throws(() => assertExactWorkerEnvironment({ + ...valid, + NODE_OPTIONS: '--require=/tmp/worker-injection.cjs', + }), /PDCF_UNSAFE_ROLE_WORKER_ENVIRONMENT_INVALID/); + assert.throws(() => assertExactWorkerEnvironment({ + ...valid, + PGPASSWORD: 'control-secret', + }), /PDCF_UNSAFE_ROLE_WORKER_ENVIRONMENT_INVALID/); + assert.throws(() => assertExactWorkerEnvironment({ + ...valid, + GRAPHILE_CACHE_MAX: '5', + }), /PDCF_UNSAFE_ROLE_WORKER_ENVIRONMENT_INVALID/); + }); + + it('constructs exactly three customer-surface credentials and replaces the probe surface', () => { + const environment = makeProbeWorkerEnvironment({ + environment: { + PGHOST: '127.0.0.1', + NODE_OPTIONS: '--require=/tmp/worker-injection.cjs', + GRAPHILE_CACHE_MAX: '5', + }, + database: 'unsafe_probe_db_0001', + tenantId: 'b', + password: 'probe-b-password'.repeat(2), + runtimePasswords: { + a: 'safe-a-password'.repeat(2), + b: 'safe-b-password'.repeat(2), + c: 'safe-c-password'.repeat(2), + }, + }); + assert.deepEqual(Object.keys(environment).sort(), [ + 'CTF_RUNTIME_A_PGPASSWORD', + 'CTF_RUNTIME_B_PGPASSWORD', + 'CTF_RUNTIME_C_PGPASSWORD', + 'PGDATABASE', + 'PGHOST', + ]); + assert.equal(environment.CTF_RUNTIME_A_PGPASSWORD, 'safe-a-password'.repeat(2)); + assert.equal(environment.CTF_RUNTIME_B_PGPASSWORD, 'probe-b-password'.repeat(2)); + assert.equal(environment.CTF_RUNTIME_C_PGPASSWORD, 'safe-c-password'.repeat(2)); + assert.equal(Object.hasOwn(environment, 'NODE_OPTIONS'), false); + assert.equal(Object.hasOwn(environment, 'GRAPHILE_CACHE_MAX'), false); + }); + + it('fails before file or database access if an unexpected credential reaches a worker', async () => { + await assert.rejects(() => workerProbe({ + environment: { + PGPASSWORD: 'control-secret-that-must-not-reach-the-worker', + }, + }), /PDCF_UNSAFE_ROLE_WORKER_ENVIRONMENT_INVALID/); + }); +}); diff --git a/research/graphile-density/production-shaped-canary.sql b/research/graphile-density/production-shaped-canary.sql new file mode 100644 index 0000000000..477c03a263 --- /dev/null +++ b/research/graphile-density/production-shaped-canary.sql @@ -0,0 +1,22 @@ +\set ON_ERROR_STOP on + +-- Performance-only canary for the local ~62k-catalog fixture. Keeping the +-- field in a real application schema avoids exposing `public`, where PostGIS +-- installs extension-owned catalog views that are not part of this API. +CREATE OR REPLACE FUNCTION "simple-pets-public".tenant_token() +RETURNS text +LANGUAGE sql +STABLE +PARALLEL SAFE +SET search_path = pg_catalog +AS $function$ + SELECT 'production-shaped-token'::text +$function$; + +REVOKE ALL +ON FUNCTION "simple-pets-public".tenant_token() +FROM PUBLIC; + +GRANT EXECUTE +ON FUNCTION "simple-pets-public".tenant_token() +TO gdp_runtime_20260801_a; diff --git a/research/graphile-density/validate-uniform-density-fixture.sql b/research/graphile-density/validate-uniform-density-fixture.sql new file mode 100644 index 0000000000..254a3eba2d --- /dev/null +++ b/research/graphile-density/validate-uniform-density-fixture.sql @@ -0,0 +1,342 @@ +\set ON_ERROR_STOP on +\pset pager off + +-- Standalone, repeatable validation for the local uniform density fixture. +-- It does not mutate persistent objects. The temporary validation procedure +-- runs with invoker rights and is dropped before the session ends. + +\connect graphile_density_uniform_20260801_a + +SET statement_timeout = 0; +SET lock_timeout = '30s'; + +\echo 'PERFORMANCE_ONLY_ROUTING_CANARY: gd_runtime_20260801_a can read every tenant schema' +\echo 'This fixture measures Graphile memory density; it does not prove database-enforced tenant isolation or complete customer qualification.' + +DO $catalog_validation$ +DECLARE + class_count integer; + tenant_schema_count integer; +BEGIN + IF current_database() <> 'graphile_density_uniform_20260801_a' THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_WRONG_DATABASE: %', current_database(); + END IF; + + SELECT count(*) INTO class_count FROM pg_catalog.pg_class; + IF class_count <> 61239 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_FINAL_CLASS_COUNT_MISMATCH: expected 61239, got %', + class_count; + END IF; + + SELECT count(*) INTO tenant_schema_count + FROM pg_catalog.pg_namespace + WHERE nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$'; + IF tenant_schema_count <> 4000 THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_SCHEMA_COUNT_MISMATCH: expected 4000, got %', + tenant_schema_count; + END IF; + + IF EXISTS ( + WITH expected AS ( + SELECT 'gd_t' || + CASE WHEN tenant_number < 1000 + THEN pg_catalog.lpad(tenant_number::text, 3, '0') + ELSE tenant_number::text + END || '_api' AS nspname + FROM pg_catalog.generate_series(1, 4000) AS tenant(tenant_number) + ), actual AS ( + SELECT nspname + FROM pg_catalog.pg_namespace + WHERE nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + ) + (SELECT nspname FROM expected EXCEPT SELECT nspname FROM actual) + UNION ALL + (SELECT nspname FROM actual EXCEPT SELECT nspname FROM expected) + ) THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_SCHEMA_SET_MISMATCH'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND ( + (SELECT pg_catalog.array_agg( + class.relname || ':' || class.relkind::text ORDER BY class.relname) + FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid) + IS DISTINCT FROM ARRAY[ + 'tenant_canary:r', + 'tenant_canary_id_seq:S', + 'tenant_canary_pkey:i', + 'tenant_canary_tenant_token_key:i', + 'widget:r', + 'widget_id_seq:S', + 'widget_pkey:i' + ]::text[] + OR + (SELECT pg_catalog.array_agg( + constraint_row.conname || ':' || constraint_row.contype::text + ORDER BY constraint_row.conname) + FROM pg_catalog.pg_constraint AS constraint_row + WHERE constraint_row.connamespace = namespace.oid) + IS DISTINCT FROM ARRAY[ + 'tenant_canary_id_not_null:n', + 'tenant_canary_pkey:p', + 'tenant_canary_tenant_token_key:u', + 'tenant_canary_tenant_token_not_null:n', + 'widget_canary_id_fkey:f', + 'widget_canary_id_not_null:n', + 'widget_id_not_null:n', + 'widget_label_not_null:n', + 'widget_pkey:p' + ]::text[] + OR + (SELECT pg_catalog.array_agg( + pg_catalog.concat_ws(':', class.relname, attribute.attnum, + attribute.attname, + pg_catalog.format_type(attribute.atttypid, attribute.atttypmod), + attribute.attnotnull, attribute.attidentity) + ORDER BY class.relname, attribute.attnum) + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_attribute AS attribute + ON attribute.attrelid = class.oid + WHERE class.relnamespace = namespace.oid + AND class.relkind = 'r' + AND attribute.attnum > 0 + AND NOT attribute.attisdropped) + IS DISTINCT FROM ARRAY[ + 'tenant_canary:1:id:bigint:t:a', + 'tenant_canary:2:tenant_token:text:t:', + 'widget:1:id:bigint:t:a', + 'widget:2:canary_id:bigint:t:', + 'widget:3:label:text:t:' + ]::text[] + OR + (SELECT count(*) + FROM pg_catalog.pg_proc AS procedure + WHERE procedure.pronamespace = namespace.oid + AND procedure.proname = 'tenant_token' + AND pg_catalog.pg_get_function_identity_arguments(procedure.oid) = '' + AND pg_catalog.pg_get_function_result(procedure.oid) = 'text' + AND procedure.provolatile = 's' + AND NOT procedure.prosecdef) <> 1 + OR + (SELECT count(*) + FROM pg_catalog.pg_class AS table_class + WHERE table_class.relnamespace = namespace.oid + AND table_class.relkind = 'r' + AND table_class.reltoastrelid <> 0) <> 2 + OR + (SELECT sum(1 + ( + SELECT count(*) + FROM pg_catalog.pg_index AS toast_index + WHERE toast_index.indrelid = table_class.reltoastrelid + )) + FROM pg_catalog.pg_class AS table_class + WHERE table_class.relnamespace = namespace.oid + AND table_class.relkind = 'r') <> 4 + ) + ) THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_NON_UNIFORM_SHAPE: expected identical 7/1/9 direct shape plus four TOAST classes'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND ( + namespace.nspowner <> 'postgres'::regrole + OR NOT pg_catalog.has_schema_privilege( + 'gd_runtime_20260801_a', namespace.oid, 'USAGE' + ) + OR pg_catalog.has_schema_privilege( + 'gd_runtime_20260801_a', namespace.oid, 'CREATE' + ) + ) + ) OR EXISTS ( + SELECT 1 + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = class.relnamespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND class.relowner <> 'postgres'::regrole + ) OR EXISTS ( + SELECT 1 + FROM pg_catalog.pg_proc AS procedure + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = procedure.pronamespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' + AND procedure.proowner <> 'postgres'::regrole + ) THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_OWNER_OR_SCHEMA_PRIVILEGE_MISMATCH'; + END IF; + + IF pg_catalog.has_database_privilege( + 'gd_runtime_20260801_a', current_database(), 'CREATE' + ) OR NOT pg_catalog.has_database_privilege( + 'gd_runtime_20260801_a', current_database(), 'CONNECT' + ) THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_RUNTIME_DATABASE_PRIVILEGE_MISMATCH'; + END IF; +END +$catalog_validation$; + +CREATE PROCEDURE pg_temp.validate_runtime_batch( + batch_start integer, + batch_end integer +) +LANGUAGE plpgsql +AS $runtime_validation$ +DECLARE + tenant_number integer; + tenant_suffix text; + tenant_schema text; + expected_token text; + expected_label text; + function_token text; + table_token text; + widget_label text; + role_row record; +BEGIN + SELECT * INTO role_row + FROM pg_catalog.pg_roles + WHERE rolname = current_user; + + IF session_user <> 'gd_runtime_20260801_a' + OR current_user <> 'gd_runtime_20260801_a' + OR role_row.rolsuper + OR role_row.rolcreaterole + OR role_row.rolcreatedb + OR role_row.rolbypassrls THEN + RAISE EXCEPTION 'GRAPHILE_DENSITY_RUNTIME_ROLE_UNSAFE: %', current_user; + END IF; + + FOR tenant_number IN batch_start..batch_end LOOP + tenant_suffix := CASE + WHEN tenant_number < 1000 + THEN pg_catalog.lpad(tenant_number::text, 3, '0') + ELSE tenant_number::text + END; + tenant_schema := 'gd_t' || tenant_suffix || '_api'; + expected_token := 'tenant-' || tenant_suffix || '-token'; + expected_label := 'tenant-' || tenant_suffix || '-widget'; + + IF NOT pg_catalog.has_schema_privilege( + current_user, tenant_schema, 'USAGE' + ) OR pg_catalog.has_schema_privilege( + current_user, tenant_schema, 'CREATE' + ) OR NOT pg_catalog.has_table_privilege( + current_user, + pg_catalog.format('%I.tenant_canary', tenant_schema), + 'SELECT' + ) OR pg_catalog.has_table_privilege( + current_user, + pg_catalog.format('%I.tenant_canary', tenant_schema), + 'INSERT,UPDATE,DELETE' + ) OR NOT pg_catalog.has_function_privilege( + current_user, + pg_catalog.format('%I.tenant_token()', tenant_schema), + 'EXECUTE' + ) THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_RUNTIME_PRIVILEGE_MISMATCH: %', tenant_schema; + END IF; + + EXECUTE pg_catalog.format( + 'SELECT %I.tenant_token()', tenant_schema + ) INTO function_token; + EXECUTE pg_catalog.format( + 'SELECT tenant_token FROM %I.tenant_canary', tenant_schema + ) INTO table_token; + EXECUTE pg_catalog.format( + 'SELECT label FROM %I.widget', tenant_schema + ) INTO widget_label; + + IF function_token <> expected_token + OR table_token <> expected_token + OR widget_label <> expected_label THEN + RAISE EXCEPTION + 'GRAPHILE_DENSITY_RUNTIME_CANARY_MISMATCH: schema %, function %, table %, widget %', + tenant_schema, function_token, table_token, widget_label; + END IF; + END LOOP; +END +$runtime_validation$; + +REVOKE ALL PRIVILEGES ON PROCEDURE + pg_temp.validate_runtime_batch(integer, integer) FROM PUBLIC; +GRANT EXECUTE ON PROCEDURE + pg_temp.validate_runtime_batch(integer, integer) + TO gd_runtime_20260801_a; + +SET SESSION AUTHORIZATION gd_runtime_20260801_a; + +SELECT pg_catalog.format( + 'CALL pg_temp.validate_runtime_batch(%s, %s)', + batch_start, + least(batch_start + 99, 4000) +) +FROM pg_catalog.generate_series(1, 4000, 100) AS batch(batch_start) +\gexec + +RESET SESSION AUTHORIZATION; + +DROP PROCEDURE pg_temp.validate_runtime_batch(integer, integer); + +WITH normalized_classes AS ( + SELECT pg_catalog.concat_ws('|', + CASE WHEN namespace.nspname = 'pg_toast' + THEN 'pg_toast.' + ELSE namespace.nspname || '.' || class.relname + END, + class.relkind, + class.relpersistence, + class.relowner::regrole::text, + coalesce(access_method.amname, ''), + class.relnatts, + class.relchecks, + class.relhasindex, + class.reltoastrelid <> 0, + class.relispartition, + coalesce(class.relacl::text, '') + ) AS logical_class + FROM pg_catalog.pg_class AS class + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = class.relnamespace + LEFT JOIN pg_catalog.pg_am AS access_method + ON access_method.oid = class.relam +), tenant_shapes AS ( + SELECT namespace.nspname, + pg_catalog.concat_ws('|', + (SELECT count(*) FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid), + (SELECT count(*) FROM pg_catalog.pg_proc AS procedure + WHERE procedure.pronamespace = namespace.oid), + (SELECT count(*) FROM pg_catalog.pg_constraint AS constraint_row + WHERE constraint_row.connamespace = namespace.oid), + (SELECT pg_catalog.string_agg( + class.relname || ':' || class.relkind::text, + ',' ORDER BY class.relname) + FROM pg_catalog.pg_class AS class + WHERE class.relnamespace = namespace.oid) + ) AS logical_shape + FROM pg_catalog.pg_namespace AS namespace + WHERE namespace.nspname ~ '^gd_t([0-9]{3}|[0-9]{4})_api$' +) +SELECT current_database() AS database_name, + (SELECT count(*) FROM pg_catalog.pg_class) AS pg_class_count, + (SELECT pg_catalog.md5(pg_catalog.string_agg( + logical_class, E'\n' ORDER BY logical_class)) + FROM normalized_classes) AS logical_pg_class_fingerprint, + (SELECT count(*) FROM tenant_shapes) AS tenant_schema_count, + (SELECT count(DISTINCT logical_shape) FROM tenant_shapes) + AS distinct_tenant_shapes, + (SELECT pg_catalog.md5(pg_catalog.string_agg( + nspname || '|' || logical_shape, E'\n' ORDER BY nspname)) + FROM tenant_shapes) AS tenant_shape_fingerprint; + +\echo 'Uniform density fixture validation completed successfully'