diff --git a/.claude/yaml-formats.md b/.claude/yaml-formats.md new file mode 100644 index 000000000..35ecc24dd --- /dev/null +++ b/.claude/yaml-formats.md @@ -0,0 +1,96 @@ +# OpenFn Project YAML Formats + +Two YAML formats are used across the monorepo. The key distinction: **v1** uses objects keyed by ID; **v2** uses arrays. + +## v1 (Lightning app state) + +Used by `packages/deploy` and sent to/from the Lightning API (`Provisioner.Project` type from `@openfn/lexicon/lightning`). + +- `workflows` is a keyed object (`{ [slug]: Workflow }`) +- Each workflow has `jobs`, `triggers`, and `edges` as keyed objects +- Steps are called `jobs`; code is stored in `body` +- Credentials referenced by UUID (`project_credential_id`) +- No version marker — absence of `schema_version`/`cli.version` means v1 + +```yaml +id: abc-123 +name: My Project +project_credentials: + - id: cred-uuid + name: My Credential + owner: admin@openfn.org +workflows: + my-workflow: + id: wf-uuid + name: My Workflow + jobs: + transform-data: + id: job-uuid + name: Transform data + body: 'fn(s => s)' + adaptor: '@openfn/language-common@latest' + project_credential_id: cred-uuid + keychain_credential_id: null + triggers: + webhook: + id: trig-uuid + type: webhook + enabled: true + edges: + trigger->transform-data: + id: edge-uuid + enabled: true + source_trigger_id: trig-uuid + target_job_id: job-uuid +``` + +## v2 (local project state) + +Used by `packages/project` and the CLI project subcommands (`ProjectState` type from `@openfn/lexicon`). + +- Identified by `schema_version` field (current: `'4.0'`) or legacy `cli.version: 2` +- `workflows` is an array +- Each workflow has a `steps` array; triggers are steps with a `type` field +- Code stored in `expression`; edges expressed inline via `next` map on each step +- Credentials referenced by name string (`configuration`) + +```yaml +id: my-project +name: My Project +schema_version: '4.0' +credentials: + - uuid: cred-uuid + name: My Credential + owner: admin@openfn.org +workflows: + - id: my-workflow + name: My Workflow + start: webhook + steps: + - id: webhook + type: webhook + enabled: true + next: + transform-data: + condition: always + - id: transform-data + name: Transform data + expression: 'fn(s => s)' + adaptor: '@openfn/language-common@latest' + configuration: 'admin@openfn.org|My Credential' +``` + +## Detection logic + +Use `detectVersion(data)` from `@openfn/project` — returns `1` or `2`. Accepts YAML/JSON string or pre-parsed object. + +```typescript +import { detectVersion } from '@openfn/project'; +if (detectVersion(json) === 2) { /* v2 */ } +``` + +## Conversion + +- **v2 → v1**: `Project.from('project', json).then(p => p.serialize('state', { format: 'yaml' }))` — see `maybeConvertV2spec` in `packages/cli/src/deploy/handler.ts` +- **v1 → v2**: `Project.from('state', json)` — see `packages/project/src/parse/from-app-state.ts` +- Full conversion logic: `packages/project/src/serialize/to-app-state.ts` (v2→v1) and `packages/project/src/parse/from-app-state.ts` (v1→v2) diff --git a/CLAUDE.md b/CLAUDE.md index c1744b67e..d88984188 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -131,6 +131,7 @@ cd packages/cli && pnpm test:watch # Watch mode The [.claude](.claude) folder contains detailed guides: - **[event-processor.md](.claude/event-processor.md)** - Worker event processing deep-dive (ordering, batching) — companion to `packages/ws-worker/CLAUDE.md` +- **[yaml-formats.md](.claude/yaml-formats.md)** - v1 vs v2 project YAML formats: structure, detection logic, and conversion paths Key packages also carry their own `CLAUDE.md` (runtime, engine-multi, ws-worker), auto-loaded when you work in them. diff --git a/integration-tests/cli/test/deploy.test.ts b/integration-tests/cli/test/deploy.test.ts index 607cd6b82..2425d0177 100644 --- a/integration-tests/cli/test/deploy.test.ts +++ b/integration-tests/cli/test/deploy.test.ts @@ -269,6 +269,52 @@ test.serial('redirect to v2 protocol if openfn.yaml is present', async (t) => { ); }); +test.serial('deploy a v2 spec file', async (t) => { + const testProjectV2 = ` +name: test-project +schema_version: '4.0' +workflows: + - id: my-workflow + name: My Workflow + start: webhook + steps: + - id: webhook + type: webhook + enabled: true + next: + my-job: {} + - id: my-job + name: My Job + expression: 'fn(s => s)' + adaptor: '@openfn/language-common@latest' +`.trim(); + + await fs.writeFile(path.join(tmpDir, 'project.yaml'), testProjectV2); + + t.is(Object.keys(server.state.projects).length, 0); + + const { stdout, stderr } = await run( + `openfn deploy \ + --project-path ${tmpDir}/project.yaml \ + --state-path ${tmpDir}/.state.json \ + --no-confirm \ + --log-json \ + -l debug` + ); + + t.falsy(stderr); + + const logs = extractLogs(stdout); + assertLog(t, logs, /v2 spec/i); + assertLog(t, logs, /Deployed/); + + t.is(Object.keys(server.state.projects).length, 1); + const [project] = Object.values(server.state.projects) as any[]; + t.is(project.name, 'test-project'); + const [workflow] = Object.values(project.workflows) as any[]; + t.is(workflow.name, 'My Workflow'); +}); + test.serial('deploy then pull, changes one workflow, deploy', async (t) => { t.is(Object.keys(server.state.projects).length, 0); diff --git a/packages/cli/src/deploy/handler.ts b/packages/cli/src/deploy/handler.ts index 13b0d7fd7..4c04b9c2d 100644 --- a/packages/cli/src/deploy/handler.ts +++ b/packages/cli/src/deploy/handler.ts @@ -10,7 +10,7 @@ import { DeployOptions } from './command'; import * as beta from '../projects/deploy'; import path from 'node:path'; import { fileExists } from '../util/file-exists'; -import { yamlToJson } from '@openfn/project'; +import Project, { detectVersion, yamlToJson } from '@openfn/project'; import fs from 'node:fs/promises'; export type DeployFn = typeof deploy; @@ -62,6 +62,15 @@ async function deployHandler( config.endpoint = process.env['OPENFN_ENDPOINT']; } + const rawSpec = await fs.readFile(config.specPath, 'utf-8'); + const convertedSpec = await maybeConvertV2spec(rawSpec); + if (convertedSpec !== rawSpec) { + logger.info( + 'Detected v2 spec file - converting to legacy format; validation will be skipped.' + ); + config.spec = convertedSpec; + } + logger.debug('Deploying with config', config); logger.info(`Deploying`); @@ -137,4 +146,13 @@ const redirectTov2 = async ( ); }; +export const maybeConvertV2spec = async (yaml: string): Promise => { + const json = yamlToJson(yaml) as any; + if (detectVersion(json) > 1) { + const project = await Project.from('project', json); + return project.serialize('state', { format: 'yaml' }) as string; + } + return yaml; +}; + export default deployHandler; diff --git a/packages/cli/test/deploy/deploy.test.ts b/packages/cli/test/deploy/deploy.test.ts index 0e9ea411d..f47633d9a 100644 --- a/packages/cli/test/deploy/deploy.test.ts +++ b/packages/cli/test/deploy/deploy.test.ts @@ -3,7 +3,11 @@ import test from 'ava'; import mockfs from 'mock-fs'; import { Logger, createMockLogger } from '@openfn/logger'; -import deployHandler, { DeployFn } from '../../src/deploy/handler'; +import deployHandler, { + DeployFn, + maybeConvertV2spec, +} from '../../src/deploy/handler'; +import { yamlToJson } from '@openfn/project'; import { DeployError, type DeployConfig } from '@openfn/deploy'; import { DeployOptions } from '../../src/deploy/command'; @@ -183,3 +187,96 @@ test.serial('catches DeployErrors', async (t) => { t.is(process.exitCode, 10); process.exitCode = origExitCode; }); + +// maybeConvertV2spec + +const v1Yaml = `id: '1234' +name: My Project +workflows: + my-workflow: + id: job-1 + name: My Workflow + jobs: + transform-data: + id: job-1 + name: Transform data + body: 'fn(s => s)' + adaptor: '@openfn/language-common@latest' + project_credential_id: null + keychain_credential_id: null + triggers: + webhook: + id: trig-1 + type: webhook + enabled: true + edges: + trigger->transform-data: + id: edge-1 + enabled: true + source_trigger_id: trig-1 + target_job_id: job-1 +project_credentials: [] +`; + +const v2Yaml = `id: my-project +name: My Project +schema_version: '4.0' +workflows: + - id: my-workflow + name: My Workflow + start: webhook + steps: + - id: webhook + type: webhook + enabled: true + next: + transform-data: {} + - id: transform-data + name: Transform data + expression: 'fn(s => s)' + adaptor: '@openfn/language-common@latest' +`; + +test('maybeConvertV2spec: returns v1 yaml unchanged', async (t) => { + const result = await maybeConvertV2spec(v1Yaml); + t.is(result, v1Yaml); +}); + +test('maybeConvertV2spec: converts v2 (schema_version) to v1', async (t) => { + const result = await maybeConvertV2spec(v2Yaml); + const json = yamlToJson(result) as any; + + // v1 has workflows as a keyed object + t.is(typeof json.workflows, 'object'); + t.false(Array.isArray(json.workflows)); + + // v1 uses jobs, not steps + const workflow = Object.values(json.workflows)[0] as any; + t.truthy(workflow.jobs); + t.falsy(workflow.steps); + t.truthy(workflow.triggers); + + // no v2 marker + t.falsy(json.schema_version); +}); + +test('maybeConvertV2spec: converts legacy v2 (cli.version: 2) to v1', async (t) => { + const legacyV2Yaml = `id: my-project +name: My Project +cli: + version: 2 +workflows: + - id: my-workflow + name: My Workflow + start: webhook + steps: + - id: webhook + type: webhook + enabled: true +`; + const result = await maybeConvertV2spec(legacyV2Yaml); + const json = yamlToJson(result) as any; + + t.is(typeof json.workflows, 'object'); + t.false(Array.isArray(json.workflows)); +}); diff --git a/packages/deploy/src/index.ts b/packages/deploy/src/index.ts index d5aaad97e..5d3ecc558 100644 --- a/packages/deploy/src/index.ts +++ b/packages/deploy/src/index.ts @@ -2,7 +2,7 @@ import { confirm } from '@inquirer/prompts'; import { inspect } from 'node:util'; import { DeployConfig, ProjectState } from './types'; import { readFile, writeFile } from 'fs/promises'; -import { parseAndValidate } from './validator'; +import { parseAndValidate, parseSpec } from './validator'; import jsondiff from 'json-diff'; import { mergeProjectPayloadIntoState, @@ -108,8 +108,8 @@ export async function getSpec(path: string) { export async function deploy(config: DeployConfig, logger: Logger) { const [state, spec] = await Promise.all([ - getState(config.statePath), - getSpec(config.specPath), + config.state ?? getState(config.statePath), + config.spec ? parseSpec(config.spec) : getSpec(config.specPath), ]); logger.debug('spec', spec); diff --git a/packages/deploy/src/types.ts b/packages/deploy/src/types.ts index d8dbf4444..c53b2886e 100644 --- a/packages/deploy/src/types.ts +++ b/packages/deploy/src/types.ts @@ -193,4 +193,6 @@ export interface DeployConfig { requireConfirmation: boolean; dryRun: boolean; apiKey: string | null; + spec?: string; + state?: ProjectState; } diff --git a/packages/deploy/src/validator.ts b/packages/deploy/src/validator.ts index 6c79750f5..b578ac5a8 100644 --- a/packages/deploy/src/validator.ts +++ b/packages/deploy/src/validator.ts @@ -3,6 +3,10 @@ import { ProjectSpec } from './types'; import { readFile } from 'fs/promises'; import path from 'path'; +export function parseSpec(input: string) { + return { errors: [] as Error[], doc: YAML.parse(input) as ProjectSpec }; +} + export interface Error { context: any; message: string; diff --git a/packages/project/src/index.ts b/packages/project/src/index.ts index f80d48072..a46c440f9 100644 --- a/packages/project/src/index.ts +++ b/packages/project/src/index.ts @@ -24,4 +24,6 @@ export { export { mapWorkflow } from './parse/from-app-state'; +export { default as detectVersion } from './util/detect-version'; + export type { MergeProjectOptions } from './merge/merge-project'; diff --git a/packages/project/src/parse/from-project.ts b/packages/project/src/parse/from-project.ts index 0f2e8f765..0e834f697 100644 --- a/packages/project/src/parse/from-project.ts +++ b/packages/project/src/parse/from-project.ts @@ -4,6 +4,7 @@ import Project from '../Project'; import ensureJson from '../util/ensure-json'; import { Provisioner } from '@openfn/lexicon/lightning'; import fromAppState, { fromAppStateConfig } from './from-app-state'; +import detectVersion from '../util/detect-version'; // Load a project from any JSON or yaml representation // This is backwards-compatible with v1 state.json files @@ -21,11 +22,7 @@ export default ( // first ensure the data is in JSON format let rawJson = ensureJson(data); - if ( - rawJson.schema_version || - rawJson.cli?.version === 2 || - rawJson.version /*deprecated*/ - ) { + if (detectVersion(rawJson) > 1) { return new Project(from_v2(rawJson as SerializedProject), config); } diff --git a/packages/project/src/util/detect-version.ts b/packages/project/src/util/detect-version.ts new file mode 100644 index 000000000..dacbe7417 --- /dev/null +++ b/packages/project/src/util/detect-version.ts @@ -0,0 +1,14 @@ +import ensureJson from './ensure-json'; + +// Detect whether a project spec is v1 (Lightning app state) or v2 (local project state) +// Accepts YAML/JSON strings or a pre-parsed object +export default function detectVersion(projectSpec: string | object): number { + const json = ensureJson(projectSpec); + if (json.schema_version) { + return parseInt(json.schema_version, 10); + } + if (json.cli?.version === 2 || json.version) { + return 2; + } + return 1; +} diff --git a/packages/project/test/util/detect-version.test.ts b/packages/project/test/util/detect-version.test.ts new file mode 100644 index 000000000..d8a20bd1f --- /dev/null +++ b/packages/project/test/util/detect-version.test.ts @@ -0,0 +1,68 @@ +import test from 'ava'; +import detectVersion from '../../src/util/detect-version'; + +test('detects v1 from a JSON object', (t) => { + const project = { + id: '1234', + name: 'My Project', + workflows: {}, + project_credentials: [], + }; + const version = detectVersion(project); + t.is(version, 1); +}); + +test('detects v1 from a YAML string', (t) => { + const project = ` + id: '1234' + name: My Project + workflows: {} + project_credentials: []`; + const version = detectVersion(project); + t.is(version, 1); +}); + +test('detects v1 from a JSON string', (t) => { + const project = JSON.stringify({ + id: '1234', + name: 'My Project', + workflows: {}, + }); + const version = detectVersion(project); + t.is(version, 1); +}); + +test('detects v2 via schema_version from a JSON object', (t) => { + const project = { + id: 'my-project', + name: 'My Project', + schema_version: '4.0', + workflows: [], + }; + const version = detectVersion(project); + t.is(version, 4); +}); + +test('detects v2 via schema_version from a YAML string', (t) => { + const project = `id: my-project\nname: My Project\nschema_version: '4.0'\nworkflows: []\n`; + const version = detectVersion(project); + t.is(version, 4); +}); + +test('detects v2 via cli.version === 2 (legacy format)', (t) => { + const project = { id: 'x', name: 'x', cli: { version: 2 }, workflows: [] }; + const version = detectVersion(project); + t.is(version, 2); +}); + +test('does not detect v2 for cli.version !== 2', (t) => { + const project = { id: 'x', name: 'x', cli: { version: 1 }, workflows: {} }; + const version = detectVersion(project); + t.is(version, 1); +}); + +test('detects v2 via deprecated version field', (t) => { + const project = { id: 'x', name: 'x', version: '1.0', workflows: [] }; + const version = detectVersion(project); + t.is(version, 2); +});