test: add cross-instance hash-consistency harness for bazel-diff serve

.github/workflows/serve-consistency.yml

@@ -0,0 +1,63 @@
+name: Serve Consistency
+
+# Cross-instance hash-consistency check for the `bazel-diff serve` query service. Runs
+# tools/serve_consistency.py, which starts several real `serve` processes against one mock S3 cache
+# tier and verifies they compute identical hashes for the same commit. A failure means target
+# hashes carry host-specific state, which makes every impacted-target answer the fleet gives
+# suspect. See the harness module docstring for the case matrix.
+#
+# Cron-only for now, matching serve-harness.yml's precedent: several concurrent Bazel servers per
+# run is too heavy to gate every PR on until the harness has a healthy track record.
+# `workflow_dispatch` allows on-demand runs from the Actions tab in the meantime. Note that
+# scheduled + dispatched workflows run from the default branch, so this must land on master before
+# the cron fires.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "30 3 * * *"
+
+jobs:
+  Consistency:
+    runs-on: ubuntu-latest
+    # Three instances means three independent Bazel output bases and three Bazel servers, so this
+    # is meaningfully slower than the single-instance serve harness.
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        bazel: [ '8.x', '9.x' ]
+    steps:
+      - name: Setup Java JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+      - name: Setup Go environment
+        uses: actions/setup-go@v5
+        with:
+          go-version: ^1.17
+        id: go
+      - name: Setup Bazelisk
+        run: go install github.com/bazelbuild/bazelisk@latest
+      - uses: actions/checkout@v4
+      - name: Run consistency harness
+        env:
+          USE_BAZEL_VERSION: ${{ matrix.bazel }}
+        # git daemon ships with git (preinstalled on the runner); python3 is preinstalled. The mock
+        # S3 backend is pure stdlib, so no service container is needed.
+        run: |
+          python3 tools/serve_consistency.py \
+            --bazel "$HOME/go/bin/bazelisk" \
+            --instances 3 \
+            --metrics-out "serve-consistency-${{ matrix.bazel }}-metrics.json" \
+            --summary-out "serve-consistency-${{ matrix.bazel }}-summary.md"
+      - name: Publish summary
+        if: always()
+        run: cat "serve-consistency-${{ matrix.bazel }}-summary.md" >> "$GITHUB_STEP_SUMMARY" || true
+      - name: Upload metrics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: serve-consistency-metrics-${{ matrix.bazel }}
+          path: serve-consistency-${{ matrix.bazel }}-metrics.json
+          if-no-files-found: ignore

tools/BUILD

@@ -55,11 +55,12 @@ py_test(
     deps = [":coverage_check_lib"],
 )
 
-# Note: the serve integration harnesses (serve_harness.py, serve_stress.py) are intentionally NOT
-# wrapped in py_binary targets. They shell out to `bazel build //cli:bazel-diff`, which would
-# deadlock on the output-base lock if they were themselves launched via `bazel run` (nested bazel,
-# same output base). Run them directly: `python3 tools/serve_harness.py` /
-# `python3 tools/serve_stress.py` (see their module docstrings). They use only the Python stdlib.
+# Note: the serve integration harnesses (serve_harness.py, serve_stress.py, serve_consistency.py)
+# are intentionally NOT wrapped in py_binary targets. They shell out to
+# `bazel build //cli:bazel-diff`, which would deadlock on the output-base lock if they were
+# themselves launched via `bazel run` (nested bazel, same output base). Run them directly:
+# `python3 tools/serve_harness.py` / `python3 tools/serve_stress.py` /
+# `python3 tools/serve_consistency.py` (see their module docstrings). Python stdlib only.
 #
 # A py_test over their *pure* helpers is fine, though -- it imports the module and calls functions
 # that never shell out, so no nested bazel and no deadlock. serve_stress_test.py covers the CLI
@@ -81,3 +82,39 @@ py_test(
     python_version = "PY3",
     deps = [":serve_stress_lib"],
 )
+
+# mock_s3.py is the one harness-adjacent module that never invokes bazel or git, so its own test
+# can drive the live HTTP server as well as the pure helpers.
+py_library(
+    name = "mock_s3_lib",
+    srcs = ["mock_s3.py"],
+    imports = ["."],
+    visibility = ["//visibility:private"],
+)
+
+py_test(
+    name = "mock_s3_test",
+    srcs = ["mock_s3_test.py"],
+    python_version = "PY3",
+    deps = [":mock_s3_lib"],
+)
+
+# serve_consistency_test.py covers the payload comparator that decides whether a fleet's hashes
+# agree; a false negative there would report consistency that was never actually checked.
+py_library(
+    name = "serve_consistency_lib",
+    srcs = [
+        "mock_s3.py",
+        "serve_consistency.py",
+        "serve_harness.py",  # imported by serve_consistency as `base`
+    ],
+    imports = ["."],
+    visibility = ["//visibility:private"],
+)
+
+py_test(
+    name = "serve_consistency_test",
+    srcs = ["serve_consistency_test.py"],
+    python_version = "PY3",
+    deps = [":serve_consistency_lib"],
+)