diff --git a/scripts/derive-work.sh b/scripts/derive-work.sh new file mode 100755 index 0000000..63f0a47 --- /dev/null +++ b/scripts/derive-work.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Split each benchmark cell into startup time and work time. +# +# Usage: ./derive-work.sh +# +# The floor fixture is the smallest real job a tool can be given (one package, +# one commit since the last tag), run with each tool's normal command. Its +# median is therefore that tool's fixed cost of being invoked at all — process +# spawn, runtime boot, module loading, config resolution — on this runner. +# +# work_ms = median_ms - startup_ms +# +# `--version` would be the obvious floor and it is wrong: it skips the lazy +# requires the real command pulls in, undercounting startup and billing the +# difference as the tool's work. Measured at ~286ms for commit-and-tag-version, +# and it flatters us, so the floor has to run the real command. +# +# Floor cells are removed from `benchmarks` — they are an instrument, not a +# data point, and the site would otherwise render `floor` as a fixture. +# +# Startup is real time the user waits, so median_ms stays untouched as the +# headline. This only explains where it goes. +# +# Requires: jq + +INPUT="${1:-}" +FLOOR="${2:-}" +OUTPUT="${3:-}" + +if [[ -z "$INPUT" || -z "$FLOOR" || -z "$OUTPUT" ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +if [[ ! -f "$INPUT" ]]; then + echo "Not a file: $INPUT" >&2 + exit 2 +fi + +mkdir -p "$(dirname "$OUTPUT")" + +jq --arg floor "$FLOOR" ' + # Floor keys are "---"; benchmark keys are + # "---". Match a cell to its floor by the + # "--" tail, taking the longest match so a tool whose + # name is a suffix of another cannot steal it. + def floor_key($k; $startup): + $startup | keys | map(. as $s | select($k | endswith("-" + $s))) | sort_by(length) | last; + + (.benchmarks // {}) as $all + | ($all + | to_entries + | map(select(.key | startswith($floor + "-"))) + | map({key: (.key | ltrimstr($floor + "-")), value: .value}) + | from_entries) as $startup + | .benchmarks = ( + $all + | to_entries + | map(select(.key | startswith($floor + "-") | not)) + | map( + floor_key(.key; $startup) as $fk + | if $fk == null then . + else .value += { + startup_ms: $startup[$fk].median_ms, + work_ms: ([(.value.median_ms - $startup[$fk].median_ms), 0] | max) + } + end + ) + | from_entries + ) +' "$INPUT" > "$OUTPUT" + +derived=$(jq '[.benchmarks[] | select(has("work_ms"))] | length' "$OUTPUT") +total=$(jq '.benchmarks | length' "$OUTPUT") +echo "Derived work_ms for $derived/$total cell(s) in $OUTPUT" >&2 diff --git a/scripts/run.sh b/scripts/run.sh index d8b8b4f..2b13932 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -20,6 +20,7 @@ SKIP_COMPETITORS=false VERBOSE="${VERBOSE:-false}" WARMUP=2 RUNS=10 +FLOOR_FIXTURE="floor" while [[ $# -gt 0 ]]; do case "$1" in @@ -325,7 +326,7 @@ for tool in $TOOLS; do # startup overhead instead of the binary's all-cores time. par_cmd="" if [[ "$tool" == "ferrflow" ]]; then - if [[ "$method" == "binary" ]]; then + if [[ "$method" == "binary" && "$fixture" != "$FLOOR_FIXTURE" ]]; then par_cmd="$full_cmd" fi full_cmd="$tool_cmd --jobs 1 $cmd" @@ -344,7 +345,7 @@ for tool in $TOOLS; do cache_cmd="" if [[ "$tool" == "ferrflow" ]]; then prepare_args=(--prepare "rm -rf $tmp_dir/.git/ferrflow-cache") - if [[ "$method" == "binary" && "$cmd_name" == "check" ]]; then + if [[ "$method" == "binary" && "$cmd_name" == "check" && "$fixture" != "$FLOOR_FIXTURE" ]]; then cache_cmd="$full_cmd" fi fi @@ -518,6 +519,14 @@ jq -n \ install_sizes: $install_sizes }' > "$RESULTS_DIR/latest.json" +# Split each cell into startup and work against the floor fixture, if this run +# benchmarked one. No-op when it didn't, so an old fixture set still produces a +# valid latest.json. +if [[ -d "$FIXTURES_DIR/$FLOOR_FIXTURE" ]]; then + "$SCRIPT_DIR/derive-work.sh" "$RESULTS_DIR/latest.json" "$FLOOR_FIXTURE" "$RESULTS_DIR/latest.derived.json" + mv "$RESULTS_DIR/latest.derived.json" "$RESULTS_DIR/latest.json" +fi + echo "" >&2 echo "Results saved to $RESULTS_DIR/latest.json" >&2 diff --git a/tests/derive-work.bats b/tests/derive-work.bats new file mode 100755 index 0000000..b955f8b --- /dev/null +++ b/tests/derive-work.bats @@ -0,0 +1,131 @@ +#!/usr/bin/env bats + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)/scripts" + +setup() { TMPDIR="$(mktemp -d)"; } +teardown() { rm -rf "$TMPDIR"; } + +write_input() { + cat > "$TMPDIR/latest.json" +} + +run_derive() { + run "$SCRIPT_DIR/derive-work.sh" "$TMPDIR/latest.json" floor "$TMPDIR/out.json" +} + +@test "splits a cell into startup and work using the floor" { + write_input <<'EOF' +{"benchmarks":{ + "floor-changesets-npm-check": {"median_ms": 700.0}, + "mono-large-changesets-npm-check":{"median_ms": 990.0} +}} +EOF + run_derive + [ "$status" -eq 0 ] + [ "$(jq '.benchmarks["mono-large-changesets-npm-check"].startup_ms == 700' "$TMPDIR/out.json")" = "true" ] + [ "$(jq '.benchmarks["mono-large-changesets-npm-check"].work_ms == 290' "$TMPDIR/out.json")" = "true" ] +} + +# The floor is an instrument, not a data point; leaving it in would render as a +# fixture on the site. +@test "drops floor cells from benchmarks" { + write_input <<'EOF' +{"benchmarks":{ + "floor-ferrflow-binary-check": {"median_ms": 5.0}, + "single-ferrflow-binary-check": {"median_ms": 25.0} +}} +EOF + run_derive + [ "$status" -eq 0 ] + [ "$(jq '.benchmarks | has("floor-ferrflow-binary-check")' "$TMPDIR/out.json")" = "false" ] + [ "$(jq '.benchmarks | length' "$TMPDIR/out.json")" -eq 1 ] +} + +@test "median_ms stays the headline and is never rewritten" { + write_input <<'EOF' +{"benchmarks":{ + "floor-changesets-npm-check": {"median_ms": 700.0}, + "mono-large-changesets-npm-check":{"median_ms": 990.0, "memory_mb": "40.0"} +}} +EOF + run_derive + [ "$status" -eq 0 ] + [ "$(jq '.benchmarks["mono-large-changesets-npm-check"].median_ms == 990' "$TMPDIR/out.json")" = "true" ] + [ "$(jq -r '.benchmarks["mono-large-changesets-npm-check"].memory_mb' "$TMPDIR/out.json")" = "40.0" ] +} + +# A tool faster than its own floor is noise, not negative work. +@test "clamps work at zero when a cell lands under the floor" { + write_input <<'EOF' +{"benchmarks":{ + "floor-changesets-npm-check": {"median_ms": 700.0}, + "single-changesets-npm-check": {"median_ms": 690.0} +}} +EOF + run_derive + [ "$status" -eq 0 ] + [ "$(jq '.benchmarks["single-changesets-npm-check"].work_ms == 0' "$TMPDIR/out.json")" = "true" ] +} + +# Each tool/method/command has its own floor — a binary's startup is not npx's. +@test "matches each cell to its own tool, method and command" { + write_input <<'EOF' +{"benchmarks":{ + "floor-ferrflow-binary-check": {"median_ms": 5.0}, + "floor-ferrflow-npm-check": {"median_ms": 600.0}, + "floor-ferrflow-binary-release-dry-run":{"median_ms": 8.0}, + "mono-large-ferrflow-binary-check": {"median_ms": 1155.0}, + "mono-large-ferrflow-npm-check": {"median_ms": 1750.0}, + "mono-large-ferrflow-binary-release-dry-run":{"median_ms": 1200.0} +}} +EOF + run_derive + [ "$status" -eq 0 ] + [ "$(jq '.benchmarks["mono-large-ferrflow-binary-check"].work_ms == 1150' "$TMPDIR/out.json")" = "true" ] + [ "$(jq '.benchmarks["mono-large-ferrflow-npm-check"].work_ms == 1150' "$TMPDIR/out.json")" = "true" ] + [ "$(jq '.benchmarks["mono-large-ferrflow-binary-release-dry-run"].work_ms == 1192' "$TMPDIR/out.json")" = "true" ] +} + +# Without a floor run there is nothing to subtract; the cell must still ship. +@test "leaves a cell untouched when its floor is missing" { + write_input <<'EOF' +{"benchmarks":{ + "floor-ferrflow-binary-check": {"median_ms": 5.0}, + "single-semantic-release-npm-check":{"median_ms": 800.0} +}} +EOF + run_derive + [ "$status" -eq 0 ] + [ "$(jq '.benchmarks["single-semantic-release-npm-check"] | has("work_ms")' "$TMPDIR/out.json")" = "false" ] + [ "$(jq '.benchmarks["single-semantic-release-npm-check"].median_ms == 800' "$TMPDIR/out.json")" = "true" ] +} + +# A shard that never ran the floor still has to produce usable output. +@test "no floor cells at all is a passthrough" { + write_input <<'EOF' +{"benchmarks":{"single-ferrflow-binary-check":{"median_ms": 25.0}},"runner_cores":4} +EOF + run_derive + [ "$status" -eq 0 ] + [ "$(jq '.benchmarks | length' "$TMPDIR/out.json")" -eq 1 ] + [ "$(jq '.runner_cores' "$TMPDIR/out.json")" -eq 4 ] +} + +@test "run-level metadata survives" { + write_input <<'EOF' +{"ferrflow_version":"ferrflow 5.29.4","warmup":3,"runs":30, + "benchmarks":{"floor-ferrflow-binary-check":{"median_ms":5.0}, + "single-ferrflow-binary-check":{"median_ms":25.0}}, + "ferrflow_cached":{"single-check":{"median_ms":3.0}}} +EOF + run_derive + [ "$status" -eq 0 ] + [ "$(jq -r '.ferrflow_version' "$TMPDIR/out.json")" = "ferrflow 5.29.4" ] + [ "$(jq '.warmup' "$TMPDIR/out.json")" -eq 3 ] + [ "$(jq '.ferrflow_cached["single-check"].median_ms == 3' "$TMPDIR/out.json")" = "true" ] +} + +@test "fails on a missing input file" { + run "$SCRIPT_DIR/derive-work.sh" "$TMPDIR/nope.json" floor "$TMPDIR/out.json" + [ "$status" -eq 2 ] +}