feat(bench): seed changesets from the fixture's commit history

scripts/run.sh

@@ -300,6 +300,7 @@ for tool in $TOOLS; do
         # changesets needs resolved node_modules to find workspaces
         if [[ "$tool" == "changesets" && -f "$tmp_dir/package.json" ]]; then
           (cd "$tmp_dir" && npm install --ignore-scripts --no-audit --no-fund &>/dev/null) || true
+          "$SCRIPT_DIR/seed-changesets.sh" "$tmp_dir" || true
         fi
       fi
 

scripts/seed-changesets.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Give `changesets status` the same release to plan as every other tool.
+#
+# Usage: ./seed-changesets.sh <fixture-dir>
+#
+# changesets does not read git history — it plans from `.changeset/*.md` files a
+# human writes by hand. Benchmarking it against a fixture with none measures
+# `npx` plus Node boot and nothing else: its time tracks package count, stays
+# flat across a 100x range of commits, and "wins" the big fixtures against tools
+# that actually walked them. That flatters us on the small fixtures and
+# penalises us on the large ones, and neither number means anything.
+#
+# So write the changesets a maintainer would have written for this history: one
+# per package with a releasable commit since the release tag, at the level
+# FerrFlow derives from the same commits (breaking > feat > fix/perf/refactor,
+# per FerrFlow's src/conventional_commits.rs). Both tools then plan the same
+# release from equivalent input.
+#
+# No-op when the fixture ships changesets by hand, or has no changesets config.
+#
+# Requires: git, jq, awk
+
+FIXTURE_DIR="${1:-}"
+
+if [[ -z "$FIXTURE_DIR" ]]; then
+  echo "Usage: $0 <fixture-dir>" >&2
+  exit 2
+fi
+if [[ ! -d "$FIXTURE_DIR/.git" ]]; then
+  echo "Not a git fixture: $FIXTURE_DIR" >&2
+  exit 2
+fi
+if [[ ! -f "$FIXTURE_DIR/.changeset/config.json" ]]; then
+  exit 0
+fi
+if compgen -G "$FIXTURE_DIR/.changeset/*.md" > /dev/null; then
+  exit 0
+fi
+
+root_pkg="$FIXTURE_DIR/package.json"
+[[ -f "$root_pkg" ]] || exit 0
+
+# A changeset naming a package changesets can't resolve is a hard error, so the
+# package set comes from the workspace rather than from the commit scopes. The
+# single-package fixture scopes its commits by area (core, api, cli...), not by
+# package name, so scopes are only trustworthy in the monorepo layout.
+SINGLE=""
+PKG_LIST=""
+if jq -e '.workspaces' "$root_pkg" >/dev/null 2>&1; then
+  while IFS= read -r p; do
+    name=$(jq -r '.name // empty' "$p" 2>/dev/null || true)
+    [[ -n "$name" ]] && PKG_LIST="$PKG_LIST $name"
+  done < <(find "$FIXTURE_DIR/packages" -mindepth 2 -maxdepth 2 -name package.json 2>/dev/null | sort)
+  [[ -n "$PKG_LIST" ]] || exit 0
+else
+  SINGLE=$(jq -r '.name // empty' "$root_pkg")
+  [[ -n "$SINGLE" ]] || exit 0
+fi
+
+# Every package's v0.1.0 is tagged at the root commit, so "since the last
+# release" and "since the root commit" are the same range here — and reading it
+# from the graph avoids guessing tag names, which differ between the single
+# (`v0.1.0`) and monorepo (`pkg-001@v0.1.0`) layouts.
+base=$(git -C "$FIXTURE_DIR" rev-list --max-parents=0 HEAD | tail -1)
+
+# One awk pass, not a shell read loop: mono-large is 10k commits and the
+# bash-regex version takes minutes.
+mapfile -t PAIRS < <(
+  git -C "$FIXTURE_DIR" log "$base..HEAD" --format='%s' |
+    awk -v single="$SINGLE" -v pkgs="$PKG_LIST" '
+      BEGIN {
+        n = split(pkgs, a, " ")
+        for (i = 1; i <= n; i++) known[a[i]] = 1
+        rank["patch"] = 1; rank["minor"] = 2; rank["major"] = 3
+      }
+      {
+        if (match($0, /^[a-z]+(\([^)]*\))?!?:/) == 0) next
+        head = substr($0, 1, RLENGTH)
+        bang = (head ~ /!:$/)
+
+        type = head
+        sub(/[(!:].*$/, "", type)
+
+        scope = ""
+        if (head ~ /\(/) {
+          scope = head
+          sub(/^[a-z]+\(/, "", scope)
+          sub(/\).*$/, "", scope)
+        }
+
+        if (bang) level = "major"
+        else if (type == "feat") level = "minor"
+        else if (type == "fix" || type == "perf" || type == "refactor") level = "patch"
+        else next
+
+        if (single != "") target = single
+        else if (scope != "" && (scope in known)) target = scope
+        else next
+
+        if (rank[level] > rank[best[target]]) best[target] = level
+      }
+      END { for (t in best) printf "%s %s\n", t, best[t] }
+    ' | sort
+)
+
+count=0
+for pair in "${PAIRS[@]}"; do
+  [[ -n "$pair" ]] || continue
+  pkg="${pair%% *}"
+  level="${pair##* }"
+  file="$FIXTURE_DIR/.changeset/${pkg//\//-}-benchmark.md"
+  {
+    printf -- '---\n'
+    printf -- '"%s": %s\n' "$pkg" "$level"
+    printf -- '---\n\n'
+    printf -- 'Release %s at %s, derived from the conventional commits since the last tag.\n' \
+      "$pkg" "$level"
+  } > "$file"
+  count=$((count + 1))
+done
+
+echo "Seeded $count changeset(s) in $FIXTURE_DIR/.changeset" >&2

tests/seed-changesets.bats

@@ -0,0 +1,139 @@
+#!/usr/bin/env bats
+
+SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)/scripts"
+
+setup() {
+  TMPDIR="$(mktemp -d)"
+  FX="$TMPDIR/fx"
+  mkdir -p "$FX/.changeset"
+  git -C "$FX" init -q -b main 2>/dev/null || (mkdir -p "$FX" && git -C "$FX" init -q -b main)
+  git -C "$FX" config user.email t@t.t
+  git -C "$FX" config user.name t
+  echo '{"$schema":"x"}' > "$FX/.changeset/config.json"
+}
+
+teardown() { rm -rf "$TMPDIR"; }
+
+monorepo() {
+  echo '{"name":"root","private":true,"workspaces":["packages/*"]}' > "$FX/package.json"
+  for p in "$@"; do
+    mkdir -p "$FX/packages/$p"
+    echo "{\"name\":\"$p\",\"version\":\"0.1.0\"}" > "$FX/packages/$p/package.json"
+  done
+  git -C "$FX" add -A
+  git -C "$FX" commit -qm "chore: initial commit"
+}
+
+single_pkg() {
+  echo '{"name":"myapp","version":"0.1.0","private":true}' > "$FX/package.json"
+  git -C "$FX" add -A
+  git -C "$FX" commit -qm "chore: initial commit"
+}
+
+commit() {
+  echo "$RANDOM" >> "$FX/f.txt"
+  git -C "$FX" add -A
+  git -C "$FX" commit -qm "$1"
+}
+
+level_of() { grep -o '": [a-z]*' "$FX/.changeset/$1-benchmark.md" | sed 's/": //'; }
+
+@test "writes one changeset per touched package at the derived level" {
+  monorepo pkg-001 pkg-002
+  commit "feat(pkg-001): add thing"
+  commit "fix(pkg-002): fix thing"
+
+  run "$SCRIPT_DIR/seed-changesets.sh" "$FX"
+  [ "$status" -eq 0 ]
+  [ "$(level_of pkg-001)" = "minor" ]
+  [ "$(level_of pkg-002)" = "patch" ]
+}
+
+# Must match FerrFlow's determine_bump, or the two tools plan different releases
+# and the comparison is meaningless again.
+@test "breaking beats feat beats fix for the same package" {
+  monorepo pkg-001 pkg-002
+  commit "fix(pkg-001): a"
+  commit "feat(pkg-001): b"
+  commit "fix(pkg-002): c"
+  commit "feat(pkg-002): d"
+  commit "chore(pkg-002)!: e"
+
+  run "$SCRIPT_DIR/seed-changesets.sh" "$FX"
+  [ "$status" -eq 0 ]
+  [ "$(level_of pkg-001)" = "minor" ]
+  [ "$(level_of pkg-002)" = "major" ]
+}
+
+@test "perf and refactor bump patch, chore docs ci test do not" {
+  monorepo pkg-001 pkg-002 pkg-003
+  commit "perf(pkg-001): a"
+  commit "refactor(pkg-002): b"
+  commit "chore(pkg-003): c"
+  commit "docs(pkg-003): d"
+  commit "ci(pkg-003): e"
+  commit "test(pkg-003): f"
+
+  run "$SCRIPT_DIR/seed-changesets.sh" "$FX"
+  [ "$status" -eq 0 ]
+  [ "$(level_of pkg-001)" = "patch" ]
+  [ "$(level_of pkg-002)" = "patch" ]
+  [ ! -f "$FX/.changeset/pkg-003-benchmark.md" ]
+}
+
+# The single-package fixture scopes by area (core, api, cli), not by package —
+# naming a package changesets can't resolve is a hard error.
+@test "single-package fixture targets the root package, not the commit scope" {
+  single_pkg
+  commit "feat(core): add thing"
+  commit "fix(api): fix thing"
+
+  run "$SCRIPT_DIR/seed-changesets.sh" "$FX"
+  [ "$status" -eq 0 ]
+  [ "$(level_of myapp)" = "minor" ]
+  [ ! -f "$FX/.changeset/core-benchmark.md" ]
+}
+
+@test "ignores scopes that are not workspace packages" {
+  monorepo pkg-001
+  commit "feat(pkg-001): a"
+  commit "feat(not-a-package): b"
+
+  run "$SCRIPT_DIR/seed-changesets.sh" "$FX"
+  [ "$status" -eq 0 ]
+  [ "$(ls "$FX/.changeset"/*-benchmark.md | wc -l)" -eq 1 ]
+  [ ! -f "$FX/.changeset/not-a-package-benchmark.md" ]
+}
+
+# `single` ships a hand-written changeset on purpose; don't fight it.
+@test "no-op when the fixture already ships a changeset" {
+  monorepo pkg-001
+  commit "feat(pkg-001): a"
+  echo '---' > "$FX/.changeset/handwritten.md"
+
+  run "$SCRIPT_DIR/seed-changesets.sh" "$FX"
+  [ "$status" -eq 0 ]
+  [ ! -f "$FX/.changeset/pkg-001-benchmark.md" ]
+}
+
+@test "no-op when the fixture has no changesets config" {
+  monorepo pkg-001
+  commit "feat(pkg-001): a"
+  rm "$FX/.changeset/config.json"
+
+  run "$SCRIPT_DIR/seed-changesets.sh" "$FX"
+  [ "$status" -eq 0 ]
+  [ ! -f "$FX/.changeset/pkg-001-benchmark.md" ]
+}
+
+@test "commits before the release tag are out of range" {
+  monorepo pkg-001
+  run "$SCRIPT_DIR/seed-changesets.sh" "$FX"
+  [ "$status" -eq 0 ]
+  [ ! -f "$FX/.changeset/pkg-001-benchmark.md" ]
+}
+
+@test "fails on a directory that is not a git repo" {
+  run "$SCRIPT_DIR/seed-changesets.sh" "$TMPDIR/nope"
+  [ "$status" -eq 2 ]
+}