add memory benchmarking

.github/workflows/generate.yml

@@ -146,7 +146,10 @@ jobs:
 
       - name: Generate ${{ matrix.target }}
         run: |
-          node bin/cli.mjs generate \
+          /usr/bin/time \
+            --output out/benchmark.json \
+            --format '{"elapsedSeconds": %e, "userCpuSeconds": %U, "systemCpuSeconds": %S, "maxRssKiB": %M}' \
+            node bin/cli.mjs generate \
             -t ${{ matrix.target }} \
             -i "${{ matrix.input }}" \
             -o out \

docs/comparators.md

@@ -7,14 +7,15 @@ This guide explains how to create build comparison scripts for `@node-core/doc-k
 Comparators are scripts that:
 
 1. **Compare** generated documentation between two builds (base vs. head)
-2. **Identify differences** in content, structure, or file size
+2. **Identify differences** in content, structure, file size, or performance
 3. **Report results** in a format suitable for CI/CD systems
 4. **Help catch regressions** before merging changes
 
 ### When to Use Comparators
 
 - **Verify backward compatibility** - Ensure new code produces same output
 - **Track file size changes** - Monitor bundle size growth
+- **Catch performance regressions** - Compare elapsed time, CPU time, and peak memory
 - **Validate transformations** - Check that refactors don't alter output
 - **Debug generation issues** - Understand what changed between versions
 
@@ -38,6 +39,12 @@ Comparators can be reused across multiple generators. You specify which comparat
 - `object-assertion.mjs` can compare JSON output from `legacy-json`, `json-simple`, etc.
 - `my-comparator.mjs` would be a custom comparator for specific needs
 
+The generation workflow also stores timing, CPU, and peak resident memory in
+`benchmark.json`. The built-in comparators include these measurements in their
+Markdown report and exclude the metadata file from output comparisons. If a
+base artifact predates performance measurement, the output comparison still
+runs and the performance section is omitted.
+
 ## Creating a Comparator
 
 ### Step 1: Create the Comparator File

scripts/comparators/file-size.mjs

@@ -1,7 +1,8 @@
 import { stat, readdir } from 'node:fs/promises';
 import path from 'node:path';
 
-import { BASE, HEAD, TITLE } from '../constants.mjs';
+import { BASE, BENCHMARK_FILE, HEAD, TITLE } from '../constants.mjs';
+import { comparePerformance } from './performance.mjs';
 
 const UNITS = ['B', 'KB', 'MB', 'GB'];
 
@@ -25,7 +26,7 @@ const formatBytes = bytes => {
  * @returns {Promise<Map<string, number>>} Map of filename to size in bytes
  */
 const getStats = async dir => {
-  const files = await readdir(dir);
+  const files = (await readdir(dir)).filter(file => file !== BENCHMARK_FILE);
   return new Map(
     await Promise.all(
       files.map(async f => [f, (await stat(path.join(dir, f))).size])
@@ -53,6 +54,8 @@ const changed = [...new Set([...baseStats.keys(), ...headStats.keys()])]
   .map(toDiffObject)
   .sort((a, b) => Math.abs(b.diff) - Math.abs(a.diff));
 
+const sections = [];
+
 // Output markdown table if there are changes
 if (changed.length) {
   const rows = changed.map(({ file, base, head, diff }) => {
@@ -63,8 +66,19 @@ if (changed.length) {
     return `| \`${file}\` | ${formatBytes(base)} | ${formatBytes(head)} | ${diffFormatted} |`;
   });
 
-  console.log(TITLE);
-  console.log('| File | Base | Head | Diff |');
-  console.log('|-|-|-|-|');
-  console.log(rows.join('\n') + '\n');
+  sections.push(
+    '### Output size',
+    '| File | Base | Head | Diff |',
+    '|-|-|-|-|',
+    rows.join('\n')
+  );
+}
+
+const performance = await comparePerformance();
+if (performance) {
+  sections.push(performance);
+}
+
+if (sections.length) {
+  console.log(`${TITLE}\n\n${sections.join('\n\n')}\n`);
 }

scripts/comparators/object-assertion.mjs

@@ -2,9 +2,10 @@ import assert from 'node:assert';
 import { readdir, readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 
-import { BASE, HEAD, TITLE } from '../constants.mjs';
+import { BASE, BENCHMARK_FILE, HEAD, TITLE } from '../constants.mjs';
+import { comparePerformance } from './performance.mjs';
 
-const files = await readdir(BASE);
+const files = (await readdir(BASE)).filter(file => file !== BENCHMARK_FILE);
 
 export const details = (summary, diff) =>
   `<details>\n<summary>${summary}</summary>\n\n\`\`\`diff\n${diff}\n\`\`\`\n\n</details>`;
@@ -28,7 +29,16 @@ const results = await Promise.all(files.map(getFileDiff));
 
 const filteredResults = results.filter(Boolean);
 
+const sections = [];
 if (filteredResults.length) {
-  console.log(TITLE);
-  console.log(filteredResults.join('\n') + '\n');
+  sections.push('### Output', filteredResults.join('\n'));
+}
+
+const performance = await comparePerformance();
+if (performance) {
+  sections.push(performance);
+}
+
+if (sections.length) {
+  console.log(`${TITLE}\n\n${sections.join('\n\n')}\n`);
 }

scripts/comparators/performance.mjs

@@ -0,0 +1,116 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+import { BASE, BENCHMARK_FILE, HEAD } from '../constants.mjs';
+
+const UNITS = ['B', 'KB', 'MB', 'GB'];
+
+const formatBytes = bytes => {
+  if (!bytes) {
+    return '0 B';
+  }
+
+  const unit = Math.min(
+    Math.floor(Math.log(Math.abs(bytes)) / Math.log(1024)),
+    UNITS.length - 1
+  );
+  return `${(bytes / Math.pow(1024, unit)).toFixed(2)} ${UNITS[unit]}`;
+};
+
+const formatSeconds = seconds =>
+  Math.abs(seconds) < 1
+    ? `${(seconds * 1_000).toFixed(2)} ms`
+    : `${seconds.toFixed(2)} s`;
+
+const formatPercent = (base, diff) => {
+  if (base === 0) {
+    return 'n/a';
+  }
+
+  const percent = (diff / base) * 100;
+  return `${percent > 0 ? '+' : ''}${percent.toFixed(2)}%`;
+};
+
+const formatDiff = (base, head, formatter) => {
+  const diff = head - base;
+  const value = `${diff > 0 ? '+' : ''}${formatter(diff)}`;
+  return `${value} (${formatPercent(base, diff)})`;
+};
+
+const readBenchmark = async directory => {
+  try {
+    return JSON.parse(
+      await readFile(path.join(directory, BENCHMARK_FILE), 'utf8')
+    );
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      return null;
+    }
+
+    throw error;
+  }
+};
+
+const METRICS = [
+  {
+    label: 'Elapsed time',
+    value: benchmark => benchmark.elapsedSeconds,
+    format: formatSeconds,
+  },
+  {
+    label: 'User CPU time',
+    value: benchmark => benchmark.userCpuSeconds,
+    format: formatSeconds,
+  },
+  {
+    label: 'System CPU time',
+    value: benchmark => benchmark.systemCpuSeconds,
+    format: formatSeconds,
+  },
+  {
+    label: 'Peak resident memory',
+    value: benchmark => benchmark.maxRssKiB * 1024,
+    format: formatBytes,
+  },
+];
+
+/**
+ * Builds a Markdown comparison of generation performance.
+ *
+ * Missing benchmark files are expected while artifacts created before
+ * performance measurement are still being used as the comparison base.
+ *
+ * @param {string} baseDirectory - Base artifact directory
+ * @param {string} headDirectory - Head artifact directory
+ * @returns {Promise<string>} Markdown table, or an empty string
+ */
+export const comparePerformance = async (
+  baseDirectory = BASE,
+  headDirectory = HEAD
+) => {
+  const [base, head] = await Promise.all(
+    [baseDirectory, headDirectory].map(readBenchmark)
+  );
+
+  if (!base || !head) {
+    return '';
+  }
+
+  const rows = METRICS.map(({ label, value, format }) => {
+    const baseValue = value(base);
+    const headValue = value(head);
+
+    if (!Number.isFinite(baseValue) || !Number.isFinite(headValue)) {
+      throw new TypeError(`Invalid ${label.toLowerCase()} benchmark value`);
+    }
+
+    return `| ${label} | ${format(baseValue)} | ${format(headValue)} | ${formatDiff(baseValue, headValue, format)} |`;
+  });
+
+  return [
+    '### Performance',
+    '| Metric | Base | Head | Diff |',
+    '|-|-|-|-|',
+    ...rows,
+  ].join('\n');
+};

scripts/constants.mjs

@@ -10,6 +10,8 @@ export const HEAD =
 export const TITLE =
   process.env.TITLE || `## \`${process.env.GENERATOR ?? '...'}\` Generator`;
 
+export const BENCHMARK_FILE = 'benchmark.json';
+
 // MDN Constants
 export const MDN_COMPAT_URL =
   'https://github.com/mdn/browser-compat-data/releases/latest/download/data.json';

src/__tests__/comparators.test.mjs

@@ -0,0 +1,143 @@
+import assert from 'node:assert/strict';
+import { execFile } from 'node:child_process';
+import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { test } from 'node:test';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+
+import { comparePerformance } from '../../scripts/comparators/performance.mjs';
+
+const benchmark = {
+  elapsedSeconds: 2,
+  userCpuSeconds: 1,
+  systemCpuSeconds: 0.5,
+  maxRssKiB: 1024,
+};
+
+const executeFile = promisify(execFile);
+const comparatorDirectory = fileURLToPath(
+  new URL('../../scripts/comparators/', import.meta.url)
+);
+
+const createDirectories = async t => {
+  const root = await mkdtemp(path.join(tmpdir(), 'doc-kit-'));
+  const base = path.join(root, 'base');
+  const head = path.join(root, 'head');
+
+  await Promise.all([
+    mkdir(base, { recursive: true }),
+    mkdir(head, { recursive: true }),
+  ]);
+  t.after(() => rm(root, { recursive: true, force: true }));
+
+  return { base, head };
+};
+
+const writeBenchmark = (directory, value = benchmark) =>
+  writeFile(
+    path.join(directory, 'benchmark.json'),
+    JSON.stringify(value),
+    'utf8'
+  );
+
+const runComparator = async (name, base, head) => {
+  const { stdout } = await executeFile(
+    process.execPath,
+    [path.join(comparatorDirectory, `${name}.mjs`)],
+    {
+      env: { ...process.env, BASE: base, HEAD: head, GENERATOR: 'test' },
+    }
+  );
+
+  return stdout;
+};
+
+test('comparePerformance formats benchmark differences', async t => {
+  const { base, head } = await createDirectories(t);
+
+  await Promise.all([
+    writeBenchmark(base),
+    writeBenchmark(head, {
+      elapsedSeconds: 3,
+      userCpuSeconds: 0.75,
+      systemCpuSeconds: 0.5,
+      maxRssKiB: 1536,
+    }),
+  ]);
+
+  const result = await comparePerformance(base, head);
+
+  assert.match(
+    result,
+    /Elapsed time \| 2\.00 s \| 3\.00 s \| \+1\.00 s \(\+50\.00%\)/
+  );
+  assert.match(
+    result,
+    /User CPU time \| 1\.00 s \| 750\.00 ms \| -250\.00 ms \(-25\.00%\)/
+  );
+  assert.match(
+    result,
+    /Peak resident memory \| 1\.00 MB \| 1\.50 MB \| \+512\.00 KB \(\+50\.00%\)/
+  );
+});
+
+test('comparePerformance omits results when an artifact has no benchmark', async t => {
+  const { base, head } = await createDirectories(t);
+
+  await writeBenchmark(head);
+
+  assert.equal(await comparePerformance(base, head), '');
+});
+
+test('comparePerformance rejects invalid benchmark values', async t => {
+  const { base, head } = await createDirectories(t);
+
+  await Promise.all([
+    writeBenchmark(base),
+    writeBenchmark(head, { ...benchmark, maxRssKiB: undefined }),
+  ]);
+
+  await assert.rejects(
+    comparePerformance(base, head),
+    /Invalid peak resident memory benchmark value/
+  );
+});
+
+test('file-size comparator combines output and performance results', async t => {
+  const { base, head } = await createDirectories(t);
+
+  await Promise.all([
+    writeFile(path.join(base, 'result.txt'), 'base', 'utf8'),
+    writeFile(path.join(head, 'result.txt'), 'a larger result', 'utf8'),
+    writeBenchmark(base),
+    writeBenchmark(head, { ...benchmark, elapsedSeconds: 3 }),
+  ]);
+
+  const result = await runComparator('file-size', base, head);
+
+  assert.equal(result.match(/## `test` Generator/g)?.length, 1);
+  assert.match(result, /### Output size/);
+  assert.match(result, /`result\.txt`/);
+  assert.match(result, /### Performance/);
+  assert.doesNotMatch(result, /benchmark\.json/);
+});
+
+test('object comparator treats benchmark data as metadata', async t => {
+  const { base, head } = await createDirectories(t);
+
+  await Promise.all([
+    writeFile(path.join(base, 'result.json'), '{"value":true}', 'utf8'),
+    writeFile(path.join(head, 'result.json'), '{"value":true}', 'utf8'),
+    writeBenchmark(base),
+    writeBenchmark(head, { ...benchmark, maxRssKiB: 2048 }),
+  ]);
+
+  const result = await runComparator('object-assertion', base, head);
+
+  assert.equal(result.match(/## `test` Generator/g)?.length, 1);
+  assert.doesNotMatch(result, /### Output\n/);
+  assert.match(result, /### Performance/);
+  assert.doesNotMatch(result, /benchmark\.json/);
+});