From b671009a7d58c6a8d76d847c5f9ce4facada5e07 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 1 Aug 2026 05:13:39 +0000 Subject: [PATCH] =?UTF-8?q?feat(pgpm):=20change=20granularity=20dial=20?= =?UTF-8?q?=E2=80=94=20one=20change=20per=20alteration=20(planning=20#1342?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/pgpm-projections/README.md | 22 ++-- .../__tests__/projections.e2e.test.ts | 56 +++++---- pgpm/cli/__tests__/diff-e2e.test.ts | 27 +++++ pgpm/cli/__tests__/transform-e2e.test.ts | 46 ++++++++ pgpm/cli/src/commands/diff.ts | 14 ++- pgpm/cli/src/commands/import.ts | 14 ++- pgpm/cli/src/commands/transform.ts | 20 +++- pgpm/cli/src/utils/scratch-db.ts | 2 +- pgpm/import/src/import.ts | 6 +- .../naming-spec/__tests__/naming-spec.test.ts | 4 + pgpm/naming-spec/src/index.ts | 4 + .../__tests__/granularity-driver.test.ts | 71 ++++++++++++ pgpm/transform/__tests__/sub-object.test.ts | 72 ++++++++++++ pgpm/transform/src/granularity-driver.ts | 101 ++++++++++++++++- pgpm/transform/src/index.ts | 4 + pgpm/transform/src/restructure.ts | 15 ++- pgpm/transform/src/semantic-diff-driver.ts | 10 +- pgpm/transform/src/sub-object.ts | 107 ++++++++++++++++++ 18 files changed, 551 insertions(+), 44 deletions(-) create mode 100644 pgpm/transform/__tests__/sub-object.test.ts create mode 100644 pgpm/transform/src/sub-object.ts diff --git a/examples/pgpm-projections/README.md b/examples/pgpm-projections/README.md index 3b1224ddd..e704f4f52 100644 --- a/examples/pgpm-projections/README.md +++ b/examples/pgpm-projections/README.md @@ -24,7 +24,8 @@ one canonical model: | Axis | Projections | What changes | What stays the same | | --- | --- | --- | --- | -| **granularity** | `atomic` · `object` · `consolidated` | how finely a change is split (one `ALTER` per column vs one statement per object vs the whole thing) | the resulting catalog | +| **granularity** | `atomic` · `object` · `consolidated` | the SQL statement shape (one `ALTER` per column vs one statement per object vs the whole thing) | the schema | +| **change granularity** | `object` · `alteration` | the plan-entry shape (one change per object vs one change per column/constraint, each with its own deploy/revert/verify) | the schema | | **partition** | one module vs many | which objects live in which package | the combined schema | | **diff** | `v1 → v2` migration | — | derived, never hand-written | | **output** | pgpm module · linear `.sql` · bundle | the artifact format | the migration | @@ -48,6 +49,10 @@ cd out/blog pgpm transform --granularity atomic pgpm transform --granularity consolidated +# 2b. The fourth dial: one change PER ALTERATION — every column and constraint +# becomes its own plan entry with its own deploy/revert/verify + requires +pgpm transform --granularity atomic --change-granularity alteration --out ../alteration + # 3. Partition the source into app + security modules cd ../.. pgpm import schema/schema.sql --pkg blog-part --partition schema/partition.json --out ./out @@ -74,8 +79,12 @@ The suite ([`__tests__/projections.e2e.test.ts`](./__tests__/projections.e2e.tes runs the real CLI and then compares the emitted artifacts **semantically** — no database needed, because equivalence is checked at the identity-keyed model: -- **granularity-invariant** — `object` and `consolidated` normalize to the same - schema (empty diff). +- **granularity-invariant** — `atomic`, `object`, and `consolidated` all + normalize to the same schema (empty diff), including standalone + `ADD CONSTRAINT` placement. +- **change-granularity-invariant** — one change per alteration (per column / + per constraint plan entries) still normalizes to the same schema (empty + diff). - **partition-invariant** — the `blog-core` + `blog-security` modules recombine to the same schema as the single module (empty diff). - **diff is exact** — `schema.sql → schema-v2.sql` derives precisely the real @@ -92,11 +101,4 @@ covered against live Postgres by the engine's own suites (`pgpm/cli` `transform-e2e` "dial parity" and `diff-e2e`). This example is deliberately database-free and asserts the model-level invariants above. -> Note: `atomic` authorship emits standalone `ALTER TABLE … ADD CONSTRAINT` -> statements. Because Postgres names such constraints at deploy time, the -> semantic normalizer does not yet fold every standalone constraint back into -> its owning table object, so `atomic` is guaranteed **catalog-equivalent** -> rather than AST-identical to `object`/`consolidated`. Tightening that -> normalization is tracked in `constructive-planning`. - It's a normal workspace package, so `pnpm install` at the repo root wires it up. diff --git a/examples/pgpm-projections/__tests__/projections.e2e.test.ts b/examples/pgpm-projections/__tests__/projections.e2e.test.ts index 5a15fa321..0ffb81b60 100644 --- a/examples/pgpm-projections/__tests__/projections.e2e.test.ts +++ b/examples/pgpm-projections/__tests__/projections.e2e.test.ts @@ -3,19 +3,21 @@ // One SQL schema (schema/schema.sql) is the source of truth. We normalize it // to an identity-keyed object set and then *project* it into many shapes: // -// granularity object | consolidated (atomic too — see the note below) -// partition one module vs app + security modules -// diff schema.sql -> schema-v2.sql as a generated migration -// output pgpm module | linear .sql +// granularity atomic | object | consolidated (statement shape) +// change granularity object | alteration (plan-entry shape) +// partition one module vs app + security modules +// diff schema.sql -> schema-v2.sql as a generated migration +// output pgpm module | linear .sql // // The headline guarantee: a projection changes the *representation*, never the // *meaning*. We prove it WITHOUT a database by normalizing each projection back // to its object set and asserting the diff is empty — authoring granularity, -// naming, partitioning, ordering, and whitespace all wash out. +// change granularity, naming, partitioning, ordering, and whitespace all wash +// out. // -// (Deploy-level catalog equivalence for every projection, including `atomic`, -// is proven against live Postgres by the engine's own suites — pgpm/cli -// transform-e2e "dial parity" and diff-e2e. See README.) +// (Deploy-level catalog equivalence for every projection is also proven +// against live Postgres by the engine's own suites — pgpm/cli transform-e2e +// "dial parity" and diff-e2e. See README.) // // This suite is intentionally database-free: it only runs the CLI and compares // the emitted artifacts semantically. @@ -50,6 +52,12 @@ describe('pgpm projections: one schema, every shape, one meaning', () => { pgpm(work, ['transform', '--granularity', 'atomic', '--cwd', path.join(work, 'blog')]); pgpm(work, ['transform', '--granularity', 'consolidated', '--cwd', path.join(work, 'blog')]); + // The fourth dial: one change per alteration (per column / per constraint). + pgpm(work, [ + 'transform', '--granularity', 'atomic', '--change-granularity', 'alteration', + '--cwd', path.join(work, 'blog'), '--out', path.join(work, 'alteration') + ]); + // Partition the same source into app + security modules. pgpm(work, [ 'import', path.join(SCHEMA_DIR, 'schema.sql'), @@ -93,19 +101,25 @@ describe('pgpm projections: one schema, every shape, one meaning', () => { expect(diffChangeSets(object, partitioned).identical).toBe(true); }); - it('emits the atomic projection as a deployable module covering the same schemas', () => { - // atomic explodes objects into per-column / per-constraint statements. It - // deploys to the same catalog as the other granularities (proven against - // live Postgres by pgpm/cli transform-e2e "dial parity"); here we assert it - // is a well-formed module spanning the same schemas. - const atomicDir = path.join(work, 'blog-atomic'); - expect(fs.existsSync(path.join(atomicDir, 'pgpm.plan'))).toBe(true); - const atomicChanges = changesOf(atomicDir); - expect(atomicChanges.length).toBeGreaterThan(0); - - const schemasOf = (changes: ChangeSet): string[] => - [...new Set(changes.map(c => c.name.split('/')[1]).filter(Boolean))].sort(); - expect(schemasOf(atomicChanges)).toEqual(schemasOf(changesOf(path.join(work, 'blog')))); + it('is granularity-invariant: atomic normalizes to the same schema too', () => { + // atomic explodes objects into per-column / per-constraint statements, yet + // constraint-placement normalization folds them back — full semantic + // identity, not just catalog equivalence. + const object = changesOf(path.join(work, 'blog')); + const atomic = changesOf(path.join(work, 'blog-atomic')); + expect(diffChangeSets(object, atomic).identical).toBe(true); + }); + + it('is change-granularity-invariant: one change per alteration, same schema', () => { + const alterationDir = path.join(work, 'alteration', 'blog-atomic'); + const plan = fs.readFileSync(path.join(alterationDir, 'pgpm.plan'), 'utf-8'); + // every column and constraint is its own plan entry... + expect(plan).toMatch(/\/columns\/[a-z_]+\/column/); + expect(plan).toMatch(/\/constraints\/[a-z_]+\/constraint/); + // ...and the meaning is untouched. + const object = changesOf(path.join(work, 'blog')); + const alteration = changesOf(alterationDir); + expect(diffChangeSets(object, alteration).identical).toBe(true); }); it('derives the v1 -> v2 migration: exactly the real changes, nothing guessed', () => { diff --git a/pgpm/cli/__tests__/diff-e2e.test.ts b/pgpm/cli/__tests__/diff-e2e.test.ts index ea1ab7028..aaf3c491d 100644 --- a/pgpm/cli/__tests__/diff-e2e.test.ts +++ b/pgpm/cli/__tests__/diff-e2e.test.ts @@ -324,6 +324,33 @@ describe('pgpm diff e2e', () => { } ); + it('emits an alteration-change-granularity migration that reaches the same v2 catalog', async () => { + const pkg = 'diff-mig-alteration'; + await fixture.runTerminalCommands( + ` + cd ${WS} + pgpm diff diff-v1 diff-v2 --emit-migration . --pkg ${pkg} --granularity atomic --change-granularity alteration + `, + {} + ); + expect(fs.existsSync(path.join(wsDir, pkg, 'pgpm.plan'))).toBe(true); + + const testDb = await fixture.setupTestDatabase(); + await fixture.runTerminalCommands( + ` + cd ${WS}/diff-v1 + pgpm deploy --database ${testDb.name} --package diff-v1 --yes + cd ../${pkg} + pgpm deploy --database ${testDb.name} --package ${pkg} --yes + `, + { database: testDb.name } + ); + + const snapMigrated = await snapshotCatalog(testDb); + const snapV2 = await snapshotCatalog(v2Db); + expect(diffCatalogSnapshots(withoutColumnOrder(snapMigrated), withoutColumnOrder(snapV2))).toEqual([]); + }); + it('--verify proves the emitted migration is a catalog-level oracle match', async () => { await fixture.runTerminalCommands( ` diff --git a/pgpm/cli/__tests__/transform-e2e.test.ts b/pgpm/cli/__tests__/transform-e2e.test.ts index cfeab8d76..776d2a21a 100644 --- a/pgpm/cli/__tests__/transform-e2e.test.ts +++ b/pgpm/cli/__tests__/transform-e2e.test.ts @@ -215,6 +215,52 @@ describe('pgpm transform e2e', () => { } ); + it('--change-granularity alteration splits per column/constraint, deploys equivalently, verifies, and reverts clean', async () => { + await fixture.runTerminalCommands( + ` + cd ${WS}/${MODULE_NAME} + pgpm transform --granularity atomic --change-granularity alteration --out ../alteration-out + `, + {} + ); + + const outDir = path.join(wsDir, 'alteration-out', `${MODULE_NAME}-atomic`); + const plan = fs.readFileSync(path.join(outDir, 'pgpm.plan'), 'utf-8'); + expect(plan).toContain('schemas/tfx_app/tables/users/columns/id/column'); + expect(plan).toContain('schemas/tfx_app/tables/users/columns/email/column'); + expect(plan).toContain('schemas/tfx_app/tables/users/constraints/users_pkey/constraint'); + expect(plan).toContain('schemas/tfx_sec/tables/audit/constraints/audit_user_id_fkey/constraint'); + + // Each alteration has its own deploy/revert/verify triple. + const columnChange = 'schemas/tfx_app/tables/users/columns/email/column'; + expect(fs.readFileSync(path.join(outDir, 'deploy', `${columnChange}.sql`), 'utf-8')).toContain('ADD COLUMN email'); + expect(fs.readFileSync(path.join(outDir, 'revert', `${columnChange}.sql`), 'utf-8')).toContain('DROP COLUMN email'); + expect(fs.readFileSync(path.join(outDir, 'verify', `${columnChange}.sql`), 'utf-8')).toContain('information_schema.columns'); + + const testDb = await fixture.setupTestDatabase(); + await fixture.runTerminalCommands( + ` + cd ${WS}/alteration-out/${MODULE_NAME}-atomic + pgpm deploy --database ${testDb.name} --package ${MODULE_NAME}-atomic --yes + `, + { database: testDb.name } + ); + + const snapOriginal = await snapshotCatalog(originalDb); + const snapTransformed = await snapshotCatalog(testDb); + expect(diffCatalogSnapshots(snapOriginal, snapTransformed)).toEqual([]); + + await fixture.runTerminalCommands( + ` + cd ${WS}/alteration-out/${MODULE_NAME}-atomic + pgpm verify --database ${testDb.name} --package ${MODULE_NAME}-atomic --yes + pgpm revert --database ${testDb.name} --package ${MODULE_NAME}-atomic --yes + `, + { database: testDb.name } + ); + expect(await testDb.exists('schema', 'tfx_app')).toBe(false); + }); + it('rewrites an existing output when --write is passed', async () => { // The refusal path (no --write) exits the process, so it is covered by the // checkOverwrite unit tests; here we prove --write re-generates in place. diff --git a/pgpm/cli/src/commands/diff.ts b/pgpm/cli/src/commands/diff.ts index af8c777b0..c4209c03c 100644 --- a/pgpm/cli/src/commands/diff.ts +++ b/pgpm/cli/src/commands/diff.ts @@ -8,9 +8,12 @@ import { import { Logger } from '@pgpmjs/logger'; import { appendModule, + CHANGE_GRANULARITIES, + ChangeGranularity, diffChangeSets, EXPORT_GRANULARITIES, ExportGranularity, + isChangeGranularity, isExportGranularity, loadModule, SemanticDiffResult, @@ -70,6 +73,9 @@ Options: --pkg Emitted migration package name (default: diff-migration) --granularity Granularity for emitted changes: atomic | object | consolidated (default: object) + --change-granularity + Change-level distribution for emitted changes: + object | alteration (default: object) --naming