Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
248 changes: 195 additions & 53 deletions tools/deno/api-diff.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,60 +72,170 @@ const normalizeRef = (ref?: string | number) =>

const LEGACY_SPEC_PATH = 'openapi/nexus.json'

async function listSchemaDir(ref: string) {
/**
* A source of schema files. The remote source reads from GitHub; the local
* source reads from the git repo in the current directory. Both expose the
* same primitives so the diff logic doesn't care where schemas come from.
*/
type Source = {
/** Resolve a ref (PR number, SHA, branch, jj revset, or undefined for the default) to a commit id */
resolveCommit: (ref?: string | number) => Promise<string>
/** List schema filenames under openapi/nexus at a commit, or null if the dir is absent */
listSchemaNames: (commit: string) => Promise<string[] | null>
/** Read the filename that nexus-latest.json points to at a commit */
readLatestPointer: (commit: string) => Promise<string>
/** Read spec content at a commit (the remote source follows gitstub references) */
readSpec: (commit: string, specPath: string) => Promise<string>
}

/** Subset of the GitHub contents API entry we rely on */
type GhContent = { name: string; download_url: string }

// the contents listing is hit twice per ref (names + latest pointer), so memoize
const schemaDirCache = new Map<string, GhContent[] | null>()
async function listSchemaDir(ref: string): Promise<GhContent[] | null> {
if (schemaDirCache.has(ref)) return schemaDirCache.get(ref)!
let result: GhContent[] | null
try {
result = await $`gh api ${SPEC_DIR_URL(ref)}`.stderr('null').json()
} catch {
result = null
}
schemaDirCache.set(ref, result)
return result
}

const remoteSource: Source = {
resolveCommit,
listSchemaNames: async (commit) => {
const contents = await listSchemaDir(commit)
return contents?.map((f) => f.name) ?? null
},
readLatestPointer: async (commit) => {
const contents = await listSchemaDir(commit)
const latestLink = contents?.find((f) => f.name === 'nexus-latest.json')
// callers verify nexus-latest.json exists before calling, but guard anyway
if (!latestLink) throw new Error(`nexus-latest.json not found at ${commit}`)
return (await fetch(latestLink.download_url).then((r) => r.text())).trim()
},
readSpec: async (commit, specPath) => {
const url = await resolveSpecUrl(commit, specPath)
const resp = await fetch(url)
if (!resp.ok) {
throw new Error(
`Failed to download ${specPath} at ${commit}: ${resp.status} ${resp.statusText}`
)
}
return resp.text()
},
}

/** Read schemas from the git repo in the current directory (run from an omicron checkout) */
async function createLocalSource(): Promise<Source> {
if (!$.commandExistsSync('git')) throw new Error('--local requires git')
// jj's working copy is always a commit, so in a jj repo @ is the natural
// default and reflects in-progress (even uncommitted) work. Plain git uses HEAD.
const isJj =
$.commandExistsSync('jj') &&
(await $`jj root`.noThrow().stdout('null').stderr('null')).code === 0

const gitShow = (target: string) => $`git show ${target}`.text()

const resolveOne = async (ref: string): Promise<string> => {
try {
if (isJj) {
const out = (
await $`jj log -r ${ref} --no-graph -T commit_id`.stderr('null').text()
).trim()
if (out.includes('\n')) throw new Error(`Revset '${ref}' matches multiple commits`)
return out
}
// pass the peel as a single arg so ^{commit} isn't brace-expanded
const rev = `${ref}^{commit}`
return (await $`git rev-parse --verify ${rev}`.stderr('null').text()).trim()
} catch (e) {
if (e instanceof Error && e.message.startsWith('Revset')) throw e
throw new Error(`Could not resolve '${ref}' in local ${isJj ? 'jj' : 'git'} repo`)
}
}

return {
resolveCommit: async (ref) => {
if (typeof ref === 'number')
throw new Error(
`PR numbers aren't supported in --local mode; pass a git or jj revision`
)
if (ref === undefined) {
const def = isJj ? '@' : 'HEAD'
console.error(`No ref given, defaulting to ${def} (comparing against its parent)`)
return resolveOne(def)
}
return resolveOne(ref)
},
listSchemaNames: async (commit) => {
const out = (
await $`git ls-tree --name-only ${commit} -- openapi/nexus/`.text()
).trim()
if (!out) return null
return out.split('\n').map((p) => p.replace(/^openapi\/nexus\//, ''))
},
readLatestPointer: (commit) =>
gitShow(`${commit}:openapi/nexus/nexus-latest.json`).then((s) => s.trim()),
// local mode only diffs current schemas, which are always real files at
// each commit (the versioned gitstub files are never read), so unlike the
// remote source this doesn't need to follow gitstub references
readSpec: (commit, specPath) => gitShow(`${commit}:${specPath}`),
}
}

/** First parent of a commit, via git (used for local single-ref vs-parent diffs) */
async function gitParent(commit: string): Promise<string> {
const rev = `${commit}^`
try {
return await $`gh api ${SPEC_DIR_URL(ref)}`.stderr('null').json()
return (await $`git rev-parse --verify ${rev}`.stderr('null').text()).trim()
} catch {
return null
throw new Error(`${commit} has no parent commit to compare against`)
}
}

async function getLatestSchema(ref: string) {
const contents = await listSchemaDir(ref)
if (!contents) {
async function getLatestSchema(source: Source, ref: string) {
const names = await source.listSchemaNames(ref)
if (!names) {
console.error(`No openapi/nexus/ dir at ${ref}, falling back to ${LEGACY_SPEC_PATH}`)
return LEGACY_SPEC_PATH
}
const schemaFiles = contents
.map((f: { name: string }) => f.name)
.filter((n: string) => n.startsWith('nexus-'))
.sort()
const latestLink = contents.find((f: { name: string }) => f.name === 'nexus-latest.json')
if (!latestLink) {
if (!names.includes('nexus-latest.json')) {
const schemaFiles = names.filter((n) => n.startsWith('nexus-')).sort()
throw new Error(
`nexus-latest.json not found at ref '${ref}'. ` +
`Available schemas: ${schemaFiles.join(', ') || '(none)'}`
)
}
const latest = (await fetch(latestLink.download_url).then((r) => r.text())).trim()
const latest = await source.readLatestPointer(ref)
return `openapi/nexus/${latest}`
}

/** When diffing a single ref, we diff its latest schema against the previous one */
async function getLatestAndPreviousSchema(ref: string) {
const contents = await listSchemaDir(ref)
if (!contents) {
async function getLatestAndPreviousSchema(source: Source, ref: string) {
const names = await source.listSchemaNames(ref)
if (!names) {
throw new Error(
`No openapi/nexus/ dir at ref '${ref}'. ` +
`Single-ref mode requires the versioned schema directory.`
)
}

const latestLink = contents.find((f: { name: string }) => f.name === 'nexus-latest.json')
const schemaFiles = contents
.filter(
(f: { name: string }) => f.name.startsWith('nexus-') && f.name !== 'nexus-latest.json'
)
.map((f: { name: string }) => f.name)
const schemaFiles = names
.filter((n) => n.startsWith('nexus-') && n !== 'nexus-latest.json')
.sort()

if (!latestLink) {
if (!names.includes('nexus-latest.json')) {
throw new Error(
`nexus-latest.json not found at ref '${ref}'. ` +
`Available schemas: ${schemaFiles.join(', ') || '(none)'}`
)
}
const latest = (await fetch(latestLink.download_url).then((r) => r.text())).trim()
const latest = await source.readLatestPointer(ref)

const latestIndex = schemaFiles.indexOf(latest)
if (latestIndex === -1)
Expand All @@ -138,26 +248,49 @@ async function getLatestAndPreviousSchema(ref: string) {
}
}

async function resolveTarget(ref1?: string | number, ref2?: string): Promise<DiffTarget> {
/** Compare the latest (current) schema at two commits */
async function diffLatest(
source: Source,
baseRef: string,
headRef: string
): Promise<DiffTarget> {
console.error(`Comparing ${baseRef} vs ${headRef}`)
const [baseSchema, headSchema] = await Promise.all([
getLatestSchema(source, baseRef),
getLatestSchema(source, headRef),
])
return { baseCommit: baseRef, baseSchema, headCommit: headRef, headSchema }
}

async function resolveTarget(
source: Source,
local: boolean,
ref1?: string | number,
ref2?: string
): Promise<DiffTarget> {
// PR numbers are a remote concept; locally everything is a git/jj revision
const norm = (ref?: string | number) => (local ? ref : normalizeRef(ref))

// Two refs: compare latest schema on each
if (ref2 !== undefined) {
if (ref1 === undefined)
throw new ValidationError('Provide a base ref when passing two refs')
const [baseRef, headRef] = await Promise.all([
resolveCommit(normalizeRef(ref1)),
resolveCommit(normalizeRef(ref2)),
])
console.error(`Comparing ${baseRef} vs ${headRef}`)
const [baseSchema, headSchema] = await Promise.all([
getLatestSchema(baseRef),
getLatestSchema(headRef),
source.resolveCommit(norm(ref1)),
source.resolveCommit(norm(ref2)),
])
return { baseCommit: baseRef, baseSchema, headCommit: headRef, headSchema }
return diffLatest(source, baseRef, headRef)
}

// Single ref: compare previous schema to latest within that ref
const ref = await resolveCommit(normalizeRef(ref1))
const { previous, latest } = await getLatestAndPreviousSchema(ref)
// Local single ref: compare this commit's current schema against its parent's
if (local) {
const headRef = await source.resolveCommit(norm(ref1))
return diffLatest(source, await gitParent(headRef), headRef)
}

// Remote single ref: compare previous schema to latest within that ref
const ref = await source.resolveCommit(norm(ref1))
const { previous, latest } = await getLatestAndPreviousSchema(source, ref)
return {
baseCommit: ref,
baseSchema: previous,
Expand Down Expand Up @@ -186,20 +319,18 @@ async function resolveSpecUrl(commit: string, specFilename: string): Promise<str
return SPEC_RAW_URL(stubCommit, stubPath)
}

async function ensureSchema(commit: string, specFilename: string, force: boolean) {
async function ensureSchema(
source: Source,
commit: string,
specFilename: string,
force: boolean
) {
const dir = `/tmp/api-diff/${commit}/${specFilename}`
const schemaPath = `${dir}/spec.json`
if (force || !(await exists(schemaPath))) {
await $`mkdir -p ${dir}`
console.error(`Downloading ${specFilename}...`)
const url = await resolveSpecUrl(commit, specFilename)
const resp = await fetch(url)
if (!resp.ok) {
throw new Error(
`Failed to download ${specFilename} at ${commit}: ${resp.status} ${resp.statusText}`
)
}
const content = await resp.text()
console.error(`Loading ${specFilename}...`)
const content = await source.readSpec(commit, specFilename)
await Deno.writeTextFile(schemaPath, content)
}
return schemaPath
Expand All @@ -220,8 +351,6 @@ async function ensureClient(schemaPath: string, force: boolean) {
// ACTUAL SCRIPT FOLLOWS
//////////////////////////////

if (!$.commandExistsSync('gh')) throw Error('Need gh (GitHub CLI)')

/** Run diff with clean labels (version extracted from spec filename) */
async function runDiff(
base: string,
Expand Down Expand Up @@ -250,17 +379,26 @@ await new Command()
`Display changes to API client or schema caused by a given Omicron PR.

Arguments:
No args Interactive PR picker
No args Interactive PR picker (or, with --local, the default ref)
<ref> PR number, commit SHA, branch, or tag
<base> <head> Two refs, compare latest schema on each

With --local, schemas are read from the git repo in the current directory
(run it from your omicron checkout) instead of GitHub, and every diff compares
the current schema (nexus-latest) at each commit:
No args / <ref> Compare a commit's schema against its parent's
(default ref is @ in a jj repo, otherwise HEAD)
<base> <head> Compare the schema at each ref
Refs are git revisions (jj revsets in a jj repo). PR numbers are not available.

Dependencies:
- Deno
- GitHub CLI (gh)
- GitHub CLI (gh), or git in the current directory with --local
- Optional: delta diff pager https://dandavison.github.io/delta/
- Optional: fzf for PR picker https://github.com/junegunn/fzf`
)
.helpOption('-h, --help', 'Show help')
.option('--local', 'Read schemas from the repo in the current directory')
.option('--force', 'Redo everything even if cached')
.type('format', ({ value }) => {
if (value !== 'ts' && value !== 'schema') {
Expand All @@ -274,12 +412,16 @@ Dependencies:
.arguments('[ref1:string] [ref2:string]')
.action(async (options, ref?: string, ref2?: string) => {
try {
const target = await resolveTarget(ref, ref2)
const local = options.local ?? false
if (!local && !$.commandExistsSync('gh')) throw new Error('Need gh (GitHub CLI)')
const source = local ? await createLocalSource() : remoteSource

const target = await resolveTarget(source, local, ref, ref2)
const force = options.force ?? false

const [baseSchema, headSchema] = await Promise.all([
ensureSchema(target.baseCommit, target.baseSchema, force),
ensureSchema(target.headCommit, target.headSchema, force),
ensureSchema(source, target.baseCommit, target.baseSchema, force),
ensureSchema(source, target.headCommit, target.headSchema, force),
])

if (options.format === 'schema') {
Expand Down
Loading