feat(bench): split each cell into startup and work against a floor fixture

scripts/derive-work.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Split each benchmark cell into startup time and work time.
+#
+# Usage: ./derive-work.sh <latest.json> <floor-fixture-name> <output.json>
+#
+# The floor fixture is the smallest real job a tool can be given (one package,
+# one commit since the last tag), run with each tool's normal command. Its
+# median is therefore that tool's fixed cost of being invoked at all — process
+# spawn, runtime boot, module loading, config resolution — on this runner.
+#
+#   work_ms = median_ms - startup_ms
+#
+# `--version` would be the obvious floor and it is wrong: it skips the lazy
+# requires the real command pulls in, undercounting startup and billing the
+# difference as the tool's work. Measured at ~286ms for commit-and-tag-version,
+# and it flatters us, so the floor has to run the real command.
+#
+# Floor cells are removed from `benchmarks` — they are an instrument, not a
+# data point, and the site would otherwise render `floor` as a fixture.
+#
+# Startup is real time the user waits, so median_ms stays untouched as the
+# headline. This only explains where it goes.
+#
+# Requires: jq
+
+INPUT="${1:-}"
+FLOOR="${2:-}"
+OUTPUT="${3:-}"
+
+if [[ -z "$INPUT" || -z "$FLOOR" || -z "$OUTPUT" ]]; then
+  echo "Usage: $0 <latest.json> <floor-fixture-name> <output.json>" >&2
+  exit 2
+fi
+
+if [[ ! -f "$INPUT" ]]; then
+  echo "Not a file: $INPUT" >&2
+  exit 2
+fi
+
+mkdir -p "$(dirname "$OUTPUT")"
+
+jq --arg floor "$FLOOR" '
+  # Floor keys are "<floor>-<tool>-<method>-<cmd>"; benchmark keys are
+  # "<fixture>-<tool>-<method>-<cmd>". Match a cell to its floor by the
+  # "<tool>-<method>-<cmd>" tail, taking the longest match so a tool whose
+  # name is a suffix of another cannot steal it.
+  def floor_key($k; $startup):
+    $startup | keys | map(. as $s | select($k | endswith("-" + $s))) | sort_by(length) | last;
+
+  (.benchmarks // {}) as $all
+  | ($all
+     | to_entries
+     | map(select(.key | startswith($floor + "-")))
+     | map({key: (.key | ltrimstr($floor + "-")), value: .value})
+     | from_entries) as $startup
+  | .benchmarks = (
+      $all
+      | to_entries
+      | map(select(.key | startswith($floor + "-") | not))
+      | map(
+          floor_key(.key; $startup) as $fk
+          | if $fk == null then .
+            else .value += {
+                   startup_ms: $startup[$fk].median_ms,
+                   work_ms: ([(.value.median_ms - $startup[$fk].median_ms), 0] | max)
+                 }
+            end
+        )
+      | from_entries
+    )
+' "$INPUT" > "$OUTPUT"
+
+derived=$(jq '[.benchmarks[] | select(has("work_ms"))] | length' "$OUTPUT")
+total=$(jq '.benchmarks | length' "$OUTPUT")
+echo "Derived work_ms for $derived/$total cell(s) in $OUTPUT" >&2

scripts/run.sh

@@ -20,6 +20,7 @@ SKIP_COMPETITORS=false
 VERBOSE="${VERBOSE:-false}"
 WARMUP=2
 RUNS=10
+FLOOR_FIXTURE="floor"
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -325,7 +326,7 @@ for tool in $TOOLS; do
         # startup overhead instead of the binary's all-cores time.
         par_cmd=""
         if [[ "$tool" == "ferrflow" ]]; then
-          if [[ "$method" == "binary" ]]; then
+          if [[ "$method" == "binary" && "$fixture" != "$FLOOR_FIXTURE" ]]; then
             par_cmd="$full_cmd"
           fi
           full_cmd="$tool_cmd --jobs 1 $cmd"
@@ -344,7 +345,7 @@ for tool in $TOOLS; do
         cache_cmd=""
         if [[ "$tool" == "ferrflow" ]]; then
           prepare_args=(--prepare "rm -rf $tmp_dir/.git/ferrflow-cache")
-          if [[ "$method" == "binary" && "$cmd_name" == "check" ]]; then
+          if [[ "$method" == "binary" && "$cmd_name" == "check" && "$fixture" != "$FLOOR_FIXTURE" ]]; then
             cache_cmd="$full_cmd"
           fi
         fi
@@ -518,6 +519,14 @@ jq -n \
     install_sizes: $install_sizes
   }' > "$RESULTS_DIR/latest.json"
 
+# Split each cell into startup and work against the floor fixture, if this run
+# benchmarked one. No-op when it didn't, so an old fixture set still produces a
+# valid latest.json.
+if [[ -d "$FIXTURES_DIR/$FLOOR_FIXTURE" ]]; then
+  "$SCRIPT_DIR/derive-work.sh" "$RESULTS_DIR/latest.json" "$FLOOR_FIXTURE" "$RESULTS_DIR/latest.derived.json"
+  mv "$RESULTS_DIR/latest.derived.json" "$RESULTS_DIR/latest.json"
+fi
+
 echo "" >&2
 echo "Results saved to $RESULTS_DIR/latest.json" >&2
 

tests/derive-work.bats

@@ -0,0 +1,131 @@
+#!/usr/bin/env bats
+
+SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)/scripts"
+
+setup() { TMPDIR="$(mktemp -d)"; }
+teardown() { rm -rf "$TMPDIR"; }
+
+write_input() {
+  cat > "$TMPDIR/latest.json"
+}
+
+run_derive() {
+  run "$SCRIPT_DIR/derive-work.sh" "$TMPDIR/latest.json" floor "$TMPDIR/out.json"
+}
+
+@test "splits a cell into startup and work using the floor" {
+  write_input <<'EOF'
+{"benchmarks":{
+  "floor-changesets-npm-check":     {"median_ms": 700.0},
+  "mono-large-changesets-npm-check":{"median_ms": 990.0}
+}}
+EOF
+  run_derive
+  [ "$status" -eq 0 ]
+  [ "$(jq '.benchmarks["mono-large-changesets-npm-check"].startup_ms == 700' "$TMPDIR/out.json")" = "true" ]
+  [ "$(jq '.benchmarks["mono-large-changesets-npm-check"].work_ms == 290' "$TMPDIR/out.json")" = "true" ]
+}
+
+# The floor is an instrument, not a data point; leaving it in would render as a
+# fixture on the site.
+@test "drops floor cells from benchmarks" {
+  write_input <<'EOF'
+{"benchmarks":{
+  "floor-ferrflow-binary-check":  {"median_ms": 5.0},
+  "single-ferrflow-binary-check": {"median_ms": 25.0}
+}}
+EOF
+  run_derive
+  [ "$status" -eq 0 ]
+  [ "$(jq '.benchmarks | has("floor-ferrflow-binary-check")' "$TMPDIR/out.json")" = "false" ]
+  [ "$(jq '.benchmarks | length' "$TMPDIR/out.json")" -eq 1 ]
+}
+
+@test "median_ms stays the headline and is never rewritten" {
+  write_input <<'EOF'
+{"benchmarks":{
+  "floor-changesets-npm-check":     {"median_ms": 700.0},
+  "mono-large-changesets-npm-check":{"median_ms": 990.0, "memory_mb": "40.0"}
+}}
+EOF
+  run_derive
+  [ "$status" -eq 0 ]
+  [ "$(jq '.benchmarks["mono-large-changesets-npm-check"].median_ms == 990' "$TMPDIR/out.json")" = "true" ]
+  [ "$(jq -r '.benchmarks["mono-large-changesets-npm-check"].memory_mb' "$TMPDIR/out.json")" = "40.0" ]
+}
+
+# A tool faster than its own floor is noise, not negative work.
+@test "clamps work at zero when a cell lands under the floor" {
+  write_input <<'EOF'
+{"benchmarks":{
+  "floor-changesets-npm-check":  {"median_ms": 700.0},
+  "single-changesets-npm-check": {"median_ms": 690.0}
+}}
+EOF
+  run_derive
+  [ "$status" -eq 0 ]
+  [ "$(jq '.benchmarks["single-changesets-npm-check"].work_ms == 0' "$TMPDIR/out.json")" = "true" ]
+}
+
+# Each tool/method/command has its own floor — a binary's startup is not npx's.
+@test "matches each cell to its own tool, method and command" {
+  write_input <<'EOF'
+{"benchmarks":{
+  "floor-ferrflow-binary-check":         {"median_ms": 5.0},
+  "floor-ferrflow-npm-check":            {"median_ms": 600.0},
+  "floor-ferrflow-binary-release-dry-run":{"median_ms": 8.0},
+  "mono-large-ferrflow-binary-check":    {"median_ms": 1155.0},
+  "mono-large-ferrflow-npm-check":       {"median_ms": 1750.0},
+  "mono-large-ferrflow-binary-release-dry-run":{"median_ms": 1200.0}
+}}
+EOF
+  run_derive
+  [ "$status" -eq 0 ]
+  [ "$(jq '.benchmarks["mono-large-ferrflow-binary-check"].work_ms == 1150' "$TMPDIR/out.json")" = "true" ]
+  [ "$(jq '.benchmarks["mono-large-ferrflow-npm-check"].work_ms == 1150' "$TMPDIR/out.json")" = "true" ]
+  [ "$(jq '.benchmarks["mono-large-ferrflow-binary-release-dry-run"].work_ms == 1192' "$TMPDIR/out.json")" = "true" ]
+}
+
+# Without a floor run there is nothing to subtract; the cell must still ship.
+@test "leaves a cell untouched when its floor is missing" {
+  write_input <<'EOF'
+{"benchmarks":{
+  "floor-ferrflow-binary-check":     {"median_ms": 5.0},
+  "single-semantic-release-npm-check":{"median_ms": 800.0}
+}}
+EOF
+  run_derive
+  [ "$status" -eq 0 ]
+  [ "$(jq '.benchmarks["single-semantic-release-npm-check"] | has("work_ms")' "$TMPDIR/out.json")" = "false" ]
+  [ "$(jq '.benchmarks["single-semantic-release-npm-check"].median_ms == 800' "$TMPDIR/out.json")" = "true" ]
+}
+
+# A shard that never ran the floor still has to produce usable output.
+@test "no floor cells at all is a passthrough" {
+  write_input <<'EOF'
+{"benchmarks":{"single-ferrflow-binary-check":{"median_ms": 25.0}},"runner_cores":4}
+EOF
+  run_derive
+  [ "$status" -eq 0 ]
+  [ "$(jq '.benchmarks | length' "$TMPDIR/out.json")" -eq 1 ]
+  [ "$(jq '.runner_cores' "$TMPDIR/out.json")" -eq 4 ]
+}
+
+@test "run-level metadata survives" {
+  write_input <<'EOF'
+{"ferrflow_version":"ferrflow 5.29.4","warmup":3,"runs":30,
+ "benchmarks":{"floor-ferrflow-binary-check":{"median_ms":5.0},
+               "single-ferrflow-binary-check":{"median_ms":25.0}},
+ "ferrflow_cached":{"single-check":{"median_ms":3.0}}}
+EOF
+  run_derive
+  [ "$status" -eq 0 ]
+  [ "$(jq -r '.ferrflow_version' "$TMPDIR/out.json")" = "ferrflow 5.29.4" ]
+  [ "$(jq '.warmup' "$TMPDIR/out.json")" -eq 3 ]
+  [ "$(jq '.ferrflow_cached["single-check"].median_ms == 3' "$TMPDIR/out.json")" = "true" ]
+}
+
+@test "fails on a missing input file" {
+  run "$SCRIPT_DIR/derive-work.sh" "$TMPDIR/nope.json" floor "$TMPDIR/out.json"
+  [ "$status" -eq 2 ]
+}