From 798d865a6f84ce724b65e82c79ad0b814cd77587 Mon Sep 17 00:00:00 2001 From: Aviv Keller Date: Mon, 13 Jul 2026 17:10:00 -0700 Subject: [PATCH] add memory benchmarking --- .github/workflows/generate.yml | 5 +- docs/comparators.md | 9 +- scripts/comparators/file-size.mjs | 26 ++++- scripts/comparators/object-assertion.mjs | 18 ++- scripts/comparators/performance.mjs | 116 ++++++++++++++++++ scripts/constants.mjs | 2 + src/__tests__/comparators.test.mjs | 143 +++++++++++++++++++++++ src/threading/index.mjs | 4 +- 8 files changed, 309 insertions(+), 14 deletions(-) create mode 100644 scripts/comparators/performance.mjs create mode 100644 src/__tests__/comparators.test.mjs diff --git a/.github/workflows/generate.yml b/.github/workflows/generate.yml index 079dd636..6ae50930 100644 --- a/.github/workflows/generate.yml +++ b/.github/workflows/generate.yml @@ -146,7 +146,10 @@ jobs: - name: Generate ${{ matrix.target }} run: | - node bin/cli.mjs generate \ + /usr/bin/time \ + --output out/benchmark.json \ + --format '{"elapsedSeconds": %e, "userCpuSeconds": %U, "systemCpuSeconds": %S, "maxRssKiB": %M}' \ + node bin/cli.mjs generate \ -t ${{ matrix.target }} \ -i "${{ matrix.input }}" \ -o out \ diff --git a/docs/comparators.md b/docs/comparators.md index 4370e64c..6f87f379 100644 --- a/docs/comparators.md +++ b/docs/comparators.md @@ -7,7 +7,7 @@ This guide explains how to create build comparison scripts for `@node-core/doc-k Comparators are scripts that: 1. **Compare** generated documentation between two builds (base vs. head) -2. **Identify differences** in content, structure, or file size +2. **Identify differences** in content, structure, file size, or performance 3. **Report results** in a format suitable for CI/CD systems 4. **Help catch regressions** before merging changes @@ -15,6 +15,7 @@ Comparators are scripts that: - **Verify backward compatibility** - Ensure new code produces same output - **Track file size changes** - Monitor bundle size growth +- **Catch performance regressions** - Compare elapsed time, CPU time, and peak memory - **Validate transformations** - Check that refactors don't alter output - **Debug generation issues** - Understand what changed between versions @@ -38,6 +39,12 @@ Comparators can be reused across multiple generators. You specify which comparat - `object-assertion.mjs` can compare JSON output from `legacy-json`, `json-simple`, etc. - `my-comparator.mjs` would be a custom comparator for specific needs +The generation workflow also stores timing, CPU, and peak resident memory in +`benchmark.json`. The built-in comparators include these measurements in their +Markdown report and exclude the metadata file from output comparisons. If a +base artifact predates performance measurement, the output comparison still +runs and the performance section is omitted. + ## Creating a Comparator ### Step 1: Create the Comparator File diff --git a/scripts/comparators/file-size.mjs b/scripts/comparators/file-size.mjs index ba58bf06..affce49a 100644 --- a/scripts/comparators/file-size.mjs +++ b/scripts/comparators/file-size.mjs @@ -1,7 +1,8 @@ import { stat, readdir } from 'node:fs/promises'; import path from 'node:path'; -import { BASE, HEAD, TITLE } from '../constants.mjs'; +import { BASE, BENCHMARK_FILE, HEAD, TITLE } from '../constants.mjs'; +import { comparePerformance } from './performance.mjs'; const UNITS = ['B', 'KB', 'MB', 'GB']; @@ -25,7 +26,7 @@ const formatBytes = bytes => { * @returns {Promise>} Map of filename to size in bytes */ const getStats = async dir => { - const files = await readdir(dir); + const files = (await readdir(dir)).filter(file => file !== BENCHMARK_FILE); return new Map( await Promise.all( files.map(async f => [f, (await stat(path.join(dir, f))).size]) @@ -53,6 +54,8 @@ const changed = [...new Set([...baseStats.keys(), ...headStats.keys()])] .map(toDiffObject) .sort((a, b) => Math.abs(b.diff) - Math.abs(a.diff)); +const sections = []; + // Output markdown table if there are changes if (changed.length) { const rows = changed.map(({ file, base, head, diff }) => { @@ -63,8 +66,19 @@ if (changed.length) { return `| \`${file}\` | ${formatBytes(base)} | ${formatBytes(head)} | ${diffFormatted} |`; }); - console.log(TITLE); - console.log('| File | Base | Head | Diff |'); - console.log('|-|-|-|-|'); - console.log(rows.join('\n') + '\n'); + sections.push( + '### Output size', + '| File | Base | Head | Diff |', + '|-|-|-|-|', + rows.join('\n') + ); +} + +const performance = await comparePerformance(); +if (performance) { + sections.push(performance); +} + +if (sections.length) { + console.log(`${TITLE}\n\n${sections.join('\n\n')}\n`); } diff --git a/scripts/comparators/object-assertion.mjs b/scripts/comparators/object-assertion.mjs index e2d7b4e3..0d670edd 100644 --- a/scripts/comparators/object-assertion.mjs +++ b/scripts/comparators/object-assertion.mjs @@ -2,9 +2,10 @@ import assert from 'node:assert'; import { readdir, readFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { BASE, HEAD, TITLE } from '../constants.mjs'; +import { BASE, BENCHMARK_FILE, HEAD, TITLE } from '../constants.mjs'; +import { comparePerformance } from './performance.mjs'; -const files = await readdir(BASE); +const files = (await readdir(BASE)).filter(file => file !== BENCHMARK_FILE); export const details = (summary, diff) => `
\n${summary}\n\n\`\`\`diff\n${diff}\n\`\`\`\n\n
`; @@ -28,7 +29,16 @@ const results = await Promise.all(files.map(getFileDiff)); const filteredResults = results.filter(Boolean); +const sections = []; if (filteredResults.length) { - console.log(TITLE); - console.log(filteredResults.join('\n') + '\n'); + sections.push('### Output', filteredResults.join('\n')); +} + +const performance = await comparePerformance(); +if (performance) { + sections.push(performance); +} + +if (sections.length) { + console.log(`${TITLE}\n\n${sections.join('\n\n')}\n`); } diff --git a/scripts/comparators/performance.mjs b/scripts/comparators/performance.mjs new file mode 100644 index 00000000..4e2f5b0e --- /dev/null +++ b/scripts/comparators/performance.mjs @@ -0,0 +1,116 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +import { BASE, BENCHMARK_FILE, HEAD } from '../constants.mjs'; + +const UNITS = ['B', 'KB', 'MB', 'GB']; + +const formatBytes = bytes => { + if (!bytes) { + return '0 B'; + } + + const unit = Math.min( + Math.floor(Math.log(Math.abs(bytes)) / Math.log(1024)), + UNITS.length - 1 + ); + return `${(bytes / Math.pow(1024, unit)).toFixed(2)} ${UNITS[unit]}`; +}; + +const formatSeconds = seconds => + Math.abs(seconds) < 1 + ? `${(seconds * 1_000).toFixed(2)} ms` + : `${seconds.toFixed(2)} s`; + +const formatPercent = (base, diff) => { + if (base === 0) { + return 'n/a'; + } + + const percent = (diff / base) * 100; + return `${percent > 0 ? '+' : ''}${percent.toFixed(2)}%`; +}; + +const formatDiff = (base, head, formatter) => { + const diff = head - base; + const value = `${diff > 0 ? '+' : ''}${formatter(diff)}`; + return `${value} (${formatPercent(base, diff)})`; +}; + +const readBenchmark = async directory => { + try { + return JSON.parse( + await readFile(path.join(directory, BENCHMARK_FILE), 'utf8') + ); + } catch (error) { + if (error.code === 'ENOENT') { + return null; + } + + throw error; + } +}; + +const METRICS = [ + { + label: 'Elapsed time', + value: benchmark => benchmark.elapsedSeconds, + format: formatSeconds, + }, + { + label: 'User CPU time', + value: benchmark => benchmark.userCpuSeconds, + format: formatSeconds, + }, + { + label: 'System CPU time', + value: benchmark => benchmark.systemCpuSeconds, + format: formatSeconds, + }, + { + label: 'Peak resident memory', + value: benchmark => benchmark.maxRssKiB * 1024, + format: formatBytes, + }, +]; + +/** + * Builds a Markdown comparison of generation performance. + * + * Missing benchmark files are expected while artifacts created before + * performance measurement are still being used as the comparison base. + * + * @param {string} baseDirectory - Base artifact directory + * @param {string} headDirectory - Head artifact directory + * @returns {Promise} Markdown table, or an empty string + */ +export const comparePerformance = async ( + baseDirectory = BASE, + headDirectory = HEAD +) => { + const [base, head] = await Promise.all( + [baseDirectory, headDirectory].map(readBenchmark) + ); + + if (!base || !head) { + return ''; + } + + const rows = METRICS.map(({ label, value, format }) => { + const baseValue = value(base); + const headValue = value(head); + + if (!Number.isFinite(baseValue) || !Number.isFinite(headValue)) { + throw new TypeError(`Invalid ${label.toLowerCase()} benchmark value`); + } + + return `| ${label} | ${format(baseValue)} | ${format(headValue)} | ${formatDiff(baseValue, headValue, format)} |`; + }); + + return [ + '### Performance', + '| Metric | Base | Head | Diff |', + '|-|-|-|-|', + ...rows, + ].join('\n'); +}; diff --git a/scripts/constants.mjs b/scripts/constants.mjs index 04122c63..4f5a5d56 100644 --- a/scripts/constants.mjs +++ b/scripts/constants.mjs @@ -10,6 +10,8 @@ export const HEAD = export const TITLE = process.env.TITLE || `## \`${process.env.GENERATOR ?? '...'}\` Generator`; +export const BENCHMARK_FILE = 'benchmark.json'; + // MDN Constants export const MDN_COMPAT_URL = 'https://github.com/mdn/browser-compat-data/releases/latest/download/data.json'; diff --git a/src/__tests__/comparators.test.mjs b/src/__tests__/comparators.test.mjs new file mode 100644 index 00000000..a1e7e483 --- /dev/null +++ b/src/__tests__/comparators.test.mjs @@ -0,0 +1,143 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +import { comparePerformance } from '../../scripts/comparators/performance.mjs'; + +const benchmark = { + elapsedSeconds: 2, + userCpuSeconds: 1, + systemCpuSeconds: 0.5, + maxRssKiB: 1024, +}; + +const executeFile = promisify(execFile); +const comparatorDirectory = fileURLToPath( + new URL('../../scripts/comparators/', import.meta.url) +); + +const createDirectories = async t => { + const root = await mkdtemp(path.join(tmpdir(), 'doc-kit-')); + const base = path.join(root, 'base'); + const head = path.join(root, 'head'); + + await Promise.all([ + mkdir(base, { recursive: true }), + mkdir(head, { recursive: true }), + ]); + t.after(() => rm(root, { recursive: true, force: true })); + + return { base, head }; +}; + +const writeBenchmark = (directory, value = benchmark) => + writeFile( + path.join(directory, 'benchmark.json'), + JSON.stringify(value), + 'utf8' + ); + +const runComparator = async (name, base, head) => { + const { stdout } = await executeFile( + process.execPath, + [path.join(comparatorDirectory, `${name}.mjs`)], + { + env: { ...process.env, BASE: base, HEAD: head, GENERATOR: 'test' }, + } + ); + + return stdout; +}; + +test('comparePerformance formats benchmark differences', async t => { + const { base, head } = await createDirectories(t); + + await Promise.all([ + writeBenchmark(base), + writeBenchmark(head, { + elapsedSeconds: 3, + userCpuSeconds: 0.75, + systemCpuSeconds: 0.5, + maxRssKiB: 1536, + }), + ]); + + const result = await comparePerformance(base, head); + + assert.match( + result, + /Elapsed time \| 2\.00 s \| 3\.00 s \| \+1\.00 s \(\+50\.00%\)/ + ); + assert.match( + result, + /User CPU time \| 1\.00 s \| 750\.00 ms \| -250\.00 ms \(-25\.00%\)/ + ); + assert.match( + result, + /Peak resident memory \| 1\.00 MB \| 1\.50 MB \| \+512\.00 KB \(\+50\.00%\)/ + ); +}); + +test('comparePerformance omits results when an artifact has no benchmark', async t => { + const { base, head } = await createDirectories(t); + + await writeBenchmark(head); + + assert.equal(await comparePerformance(base, head), ''); +}); + +test('comparePerformance rejects invalid benchmark values', async t => { + const { base, head } = await createDirectories(t); + + await Promise.all([ + writeBenchmark(base), + writeBenchmark(head, { ...benchmark, maxRssKiB: undefined }), + ]); + + await assert.rejects( + comparePerformance(base, head), + /Invalid peak resident memory benchmark value/ + ); +}); + +test('file-size comparator combines output and performance results', async t => { + const { base, head } = await createDirectories(t); + + await Promise.all([ + writeFile(path.join(base, 'result.txt'), 'base', 'utf8'), + writeFile(path.join(head, 'result.txt'), 'a larger result', 'utf8'), + writeBenchmark(base), + writeBenchmark(head, { ...benchmark, elapsedSeconds: 3 }), + ]); + + const result = await runComparator('file-size', base, head); + + assert.equal(result.match(/## `test` Generator/g)?.length, 1); + assert.match(result, /### Output size/); + assert.match(result, /`result\.txt`/); + assert.match(result, /### Performance/); + assert.doesNotMatch(result, /benchmark\.json/); +}); + +test('object comparator treats benchmark data as metadata', async t => { + const { base, head } = await createDirectories(t); + + await Promise.all([ + writeFile(path.join(base, 'result.json'), '{"value":true}', 'utf8'), + writeFile(path.join(head, 'result.json'), '{"value":true}', 'utf8'), + writeBenchmark(base), + writeBenchmark(head, { ...benchmark, maxRssKiB: 2048 }), + ]); + + const result = await runComparator('object-assertion', base, head); + + assert.equal(result.match(/## `test` Generator/g)?.length, 1); + assert.doesNotMatch(result, /### Output\n/); + assert.match(result, /### Performance/); + assert.doesNotMatch(result, /benchmark\.json/); +}); diff --git a/src/threading/index.mjs b/src/threading/index.mjs index bd7f2dfa..98ba22f4 100644 --- a/src/threading/index.mjs +++ b/src/threading/index.mjs @@ -20,8 +20,8 @@ export default function createWorkerPool(threads) { return new Piscina({ filename: workerScript, - minThreads: threads, + minThreads: 0, maxThreads: threads, - idleTimeout: Infinity, // Keep workers alive + idleTimeout: 1_000, }); }