ci(executable): verify the packaged binary per PR and make the release verify actually fail

.github/workflows/executable-check.yml

@@ -0,0 +1,36 @@
+name: Verify Executable
+
+# Per-PR gate: build the packaged executables exactly as the release pipeline
+# does and smoke-test them — but without signing, notarizing or uploading. This
+# catches binary-only breakages (e.g. a bad require produced by the CJS
+# transpile that never shows up in the Node/source test suite) on the PR that
+# introduces them, so they can't reach a release.
+#
+# No secrets are used or needed: scripts/executable.sh skips the macOS
+# signing/notarization block when APPLE_DEV_CERT is unset, which also makes this
+# safe to run on pull requests from forks.
+on:
+  pull_request:
+    branches: [master]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: verify-executable-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  verify:
+    name: Build & verify executable
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 14
+          architecture: x64
+      - name: Build executables (no signing, no upload)
+        run: ./scripts/executable.sh
+      - name: Verify executable
+        run: ./scripts/verify-executable.sh ./percy

.github/workflows/executable.yml

@@ -20,7 +20,7 @@ jobs:
           APPLE_CERT_KEY: ${{secrets.APPLE_CERT_KEY}}
           APPLE_TEAM_ID: ${{secrets.APPLE_TEAM_ID}}
       - name: Verify executable
-        run: ./percy --version
+        run: ./scripts/verify-executable.sh ./percy
       - name: Upload win artifact
         uses: actions/upload-artifact@v4
         with:
@@ -93,7 +93,8 @@ jobs:
           Remove-Item -Force gcp-sa-key.json -ErrorAction SilentlyContinue
           Remove-Item -Force comodo_signing_cert.crt -ErrorAction SilentlyContinue
       - name: Verify executable
-        run: ./percy.exe --version
+        shell: bash
+        run: ./scripts/verify-executable.sh ./percy.exe
       - run: |
           powershell -Command "Compress-Archive -Path 'percy.exe' -DestinationPath 'percy-win.zip'"
       - name: Upload assets

scripts/executable.sh

@@ -44,32 +44,44 @@ gsed -i '/Update NODE_ENV for executable/{s//\nprocess.env.NODE_ENV = "executabl
 npm run build_cjs
 cp -R ./build/* packages/
 
-# Create executables
-pkg ./packages/cli/bin/run.js -d
+# Create executables. (No `-d`/`--debug`: it only adds per-file "included as
+# DISCLOSED code / asset content" logging — thousands of lines — without
+# changing the output binaries.)
+pkg ./packages/cli/bin/run.js
 
 # Rename executables
 mv run-linux percy && chmod +x percy
 mv run-macos percy-osx && chmod +x percy-osx
 mv run-win.exe percy.exe && chmod +x percy.exe
 
-# Sign & Notrize mac app
-echo "$APPLE_DEV_CERT" | base64 -d > AppleDevIDApp.p12
-
-security create-keychain -p percy percy.keychain
-security import AppleDevIDApp.p12 -t agg -k percy.keychain -P $APPLE_CERT_KEY -A
-security list-keychains -s ~/Library/Keychains/percy.keychain
-security default-keychain -s ~/Library/Keychains/percy.keychain
-security unlock-keychain -p "percy" ~/Library/Keychains/percy.keychain
-security set-keychain-settings -t 3600 -l ~/Library/Keychains/percy.keychain
-security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k percy ~/Library/Keychains/percy.keychain-db
-
-codesign  --force --verbose=4 -s "Developer ID Application: BrowserStack Inc ($APPLE_TEAM_ID)" --options runtime --entitlements scripts/files/entitlement.plist --keychain ~/Library/Keychains/percy.keychain percy-osx
-
-# Create zip file for uploading as assets
-zip percy-linux.zip percy
-mv percy-osx percy
-zip percy-osx.zip percy
-
-xcrun notarytool submit --apple-id "$APPLE_ID_USERNAME" --password $APPLE_ID_KEY --team-id $APPLE_TEAM_ID percy-osx.zip --wait
-
-cleanup
+# Sign, notarize and package the assets only when the Apple signing secrets are
+# present. Pull-request builds run without secrets: there we only want to prove
+# the executables build and run (the verify step), not sign or ship them.
+if [ -n "${APPLE_DEV_CERT:-}" ]; then
+  # Sign & Notrize mac app
+  echo "$APPLE_DEV_CERT" | base64 -d > AppleDevIDApp.p12
+
+  security create-keychain -p percy percy.keychain
+  security import AppleDevIDApp.p12 -t agg -k percy.keychain -P $APPLE_CERT_KEY -A
+  security list-keychains -s ~/Library/Keychains/percy.keychain
+  security default-keychain -s ~/Library/Keychains/percy.keychain
+  security unlock-keychain -p "percy" ~/Library/Keychains/percy.keychain
+  security set-keychain-settings -t 3600 -l ~/Library/Keychains/percy.keychain
+  security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k percy ~/Library/Keychains/percy.keychain-db
+
+  codesign  --force --verbose=4 -s "Developer ID Application: BrowserStack Inc ($APPLE_TEAM_ID)" --options runtime --entitlements scripts/files/entitlement.plist --keychain ~/Library/Keychains/percy.keychain percy-osx
+
+  # Create zip file for uploading as assets
+  zip percy-linux.zip percy
+  mv percy-osx percy
+  zip percy-osx.zip percy
+
+  xcrun notarytool submit --apple-id "$APPLE_ID_USERNAME" --password $APPLE_ID_KEY --team-id $APPLE_TEAM_ID percy-osx.zip --wait
+
+  cleanup
+else
+  echo "APPLE_DEV_CERT not set — skipping macOS signing/notarization (PR build)."
+  # Leave ./percy as the macOS binary so the verify step runs natively on the
+  # macOS runner (mirrors the signed path, which ends with percy == percy-osx).
+  mv percy-osx percy
+fi

scripts/verify-executable.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Smoke-test a packaged percy executable.
+#
+# Why this isn't just `./percy --version`: the executables are built on Node 14,
+# where an unhandled promise rejection is reported as a *warning* and the process
+# still exits 0. So a binary that throws on startup (e.g. a bad require produced
+# by the CJS transpile) prints a stack trace yet a bare `--version` exit-code
+# check passes — and the release pipeline happily uploads a broken binary.
+#
+# This script treats the binary as broken if `--version` either exits non-zero,
+# fails to print a real version, or emits any runtime-error marker on stdout/stderr.
+#
+# Usage: scripts/verify-executable.sh [path-to-binary]   (default: ./percy)
+set -u -o pipefail
+
+BIN="${1:-./percy}"
+echo "Verifying: $BIN --version"
+
+# Capture stdout+stderr together; keep the exit code without tripping set -e.
+output="$("$BIN" --version 2>&1)"
+status=$?
+
+echo "----- output -----"
+echo "$output"
+echo "------------------"
+
+if [ "$status" -ne 0 ]; then
+  echo "::error::'$BIN --version' exited with status $status"
+  exit 1
+fi
+
+# Node 14 turns startup crashes into non-fatal warnings, so scan the output for
+# the error signatures a broken binary leaves behind.
+if echo "$output" | grep -qiE 'UnhandledPromiseRejection|is not a function|TypeError|ReferenceError|SyntaxError|Cannot find module|Error:'; then
+  echo "::error::'$BIN --version' emitted a runtime error (binary is broken)"
+  exit 1
+fi
+
+# A healthy binary prints its semver. If it crashed before printing one, fail.
+if ! echo "$output" | grep -qE '[0-9]+\.[0-9]+\.[0-9]+'; then
+  echo "::error::'$BIN --version' did not print a valid version string"
+  exit 1
+fi
+
+echo "OK: $BIN is healthy"