fix(orb): repoint the orphaned review_targets reputation signal, sample clock skew in broker mode, and bound the upstream-drift fallback fetch

grafana/dashboards/selfhost.json

@@ -2773,6 +2773,42 @@
         }
       ]
     },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 7200 }
+            ]
+          },
+          "unit": "s"
+        }
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 205 },
+      "id": 206,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "textMode": "auto"
+      },
+      "description": "Seconds since the last successful clock-skew sample (#7000, #9156). Ages continuously from process start when never sampled (#9128) -- LoopoverClockSkewSampleStale fires past 7200s (2h). While this is high, the Clock Skew panel to the left cannot be trusted: it may be reading stale or never-measured data.",
+      "title": "Clock Skew Sample Age (#7000, LoopoverClockSkewSampleStale > 7200s)",
+      "type": "stat",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "(loopover_clock_skew_sample_age_seconds or gittensory_clock_skew_sample_age_seconds)",
+          "legendFormat": "sample age"
+        }
+      ]
+    },
     {
       "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
       "fieldConfig": {

prometheus/rules/alerts.yml

@@ -672,6 +672,24 @@ groups:
           description: "Clock skew has exceeded 120s (sustained 2m), well past the point GitHub App JWT auth (\"Bad credentials\") is expected to start failing fleet-wide."
           runbook: "Same as LoopoverClockSkewWarning, but treat as urgent: fix NTP sync immediately (chronyc sources, chronyc makestep, restart chrony if every source stays at Reach: 0). Check for github_app_jwt_rejected logs to confirm auth impact."
 
+      - alert: LoopoverClockSkewSampleStale
+        # #9156: loopover_clock_skew_seconds' own companion staleness gauge (#7000) was referenced by ZERO
+        # alert rules until this one -- so LoopoverClockSkewWarning/Critical above could read a flattering
+        # "0" (or, pre-#9156, a hard 0 that never distinguished "fine" from "never measured") forever on a
+        # deployment whose sampling path never runs (a brokered self-host before #9156's broker-path fix, or
+        # any future regression that stops installation-token minting). The gauge ages continuously from
+        # process start when never sampled (#9128), so ONE threshold catches both "never sampled at all" and
+        # "was sampled once, then mint activity stalled". Installation tokens mint roughly hourly by design;
+        # a 2x margin tolerates normal mint-timing jitter without masking a real stall.
+        expr: loopover_clock_skew_sample_age_seconds > 7200
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "loopover has not sampled clock skew in over {{ $value | printf \"%.0f\" }}s"
+          description: "loopover_clock_skew_seconds has gone unmeasured for over 2h (sustained 15m) -- an installation-token mint (local App-JWT or brokered) hasn't happened recently, or the sampling call itself broke. While this fires, LoopoverClockSkewWarning/Critical cannot be trusted -- real drift could be going undetected."
+          runbook: "Confirm installation-token minting is still happening (look for github_installation_token_rejected / orb_broker_unavailable logs). If mint activity looks healthy but this stays stale, the sampling call itself (recordClockSkewFromResponse, src/selfhost/clock-skew.ts) may be broken -- check both requestInstallationTokenWithJwt (src/github/app.ts) and fetchBrokeredInstallationToken (src/orb/broker-client.ts)."
+
   # ── Cloudflare D1 (central cloud) size + signal_snapshots dedup regression (#3810) ────
   # loopover_d1_* metrics come from the OPT-IN Cloudflare Management API probe (src/selfhost/
   # d1-size-probe.ts, CLOUDFLARE_D1_MONITOR_* env vars) -- absent/disabled reads -1 on every gauge below,

src/auth/github-oauth.ts

@@ -42,7 +42,11 @@ type GitHubWebOAuthState = {
 
 export async function startGitHubDeviceFlow(env: Env): Promise<GitHubDeviceCodeResponse> {
   if (!env.GITHUB_OAUTH_CLIENT_ID) throw new Error("github_oauth_not_configured");
-  const response = await fetch("https://github.com/login/device/code", {
+  // #9165 sweep: this file's OTHER GitHub calls (createSessionFromGitHubToken, verifyTokenBelongsToApp,
+  // refreshGitHubUserToken below) already route through timeoutFetch -- this bare fetch and the two below it
+  // were the only ones left with no timeout, so a stalled github.com connection during device-flow start/poll
+  // or the web-OAuth code exchange would have hung indefinitely instead of failing bounded.
+  const response = await timeoutFetch("https://github.com/login/device/code", {
     method: "POST",
     headers: {
       accept: "application/json",
@@ -68,7 +72,7 @@ export async function startGitHubDeviceFlow(env: Env): Promise<GitHubDeviceCodeR
 
 export async function pollGitHubDeviceFlow(env: Env, deviceCode: string) {
   if (!env.GITHUB_OAUTH_CLIENT_ID) throw new Error("github_oauth_not_configured");
-  const tokenResponse = await fetch("https://github.com/login/oauth/access_token", {
+  const tokenResponse = await timeoutFetch("https://github.com/login/oauth/access_token", {
     method: "POST",
     headers: {
       accept: "application/json",
@@ -145,7 +149,7 @@ export async function completeGitHubWebOAuth(
   if (!args.cookieState || !(await timingSafeEqual(args.state, args.cookieState))) throw new Error("github_oauth_state_invalid");
   const state = await verifyOAuthState(env, args.state);
   if (!state) throw new Error("github_oauth_state_invalid");
-  const response = await fetch("https://github.com/login/oauth/access_token", {
+  const response = await timeoutFetch("https://github.com/login/oauth/access_token", {
     method: "POST",
     headers: {
       accept: "application/json",

src/integrations/linear-adapter.ts

@@ -2,6 +2,10 @@ import { getDecryptedRepositoryLinearKey } from "../db/repositories";
 import type { ProjectTrackerAdapter, ProjectTrackerAttachResult, ProjectTrackerContext, ProjectTrackerMatch, ProjectTrackerRef } from "./project-tracker-adapter";
 
 const LINEAR_API_URL = "https://api.linear.app/graphql";
+// #9165 sweep: matches src/gittensor/api.ts's GITTENSOR_FETCH_TIMEOUT_MS -- a per-adapter timeout constant
+// on every raw fetch() outside the shared GitHub client helpers, so a stalled Linear connection degrades
+// this adapter's caller (falls back to fuzzy matching) instead of hanging.
+const LINEAR_FETCH_TIMEOUT_MS = 10_000;
 
 // "Open" for Linear means not-yet-completed and not-canceled -- listing the positive set (rather than
 // excluding just "completed" via `neq`) so a canceled project is never mistaken for open. Bounded pagination
@@ -22,6 +26,7 @@ async function linearGraphQl<T>(apiKey: string, query: string, variables: Record
     method: "POST",
     headers: { "Content-Type": "application/json", Authorization: apiKey },
     body: JSON.stringify({ query, variables }),
+    signal: AbortSignal.timeout(LINEAR_FETCH_TIMEOUT_MS),
   });
   if (!response.ok) throw new Error(`Linear API HTTP ${response.status}`);
   const body = (await response.json()) as { data?: T } & LinearGraphQlErrorResponse;

src/orb/broker-client.ts

@@ -7,6 +7,7 @@
 // The signal is the ENROLLMENT SECRET's presence: a brokered self-host sets ORB_ENROLLMENT_SECRET (issued by the
 // operator), cloud never does — so this path is inert on cloud and the deploy is byte-identical there.
 
+import { recordClockSkewFromResponse } from "../selfhost/clock-skew";
 import { incr } from "../selfhost/metrics";
 
 /** The Orb's hosted broker base; override (ORB_BROKER_URL) only to point at a private loopover deployment. */
@@ -72,6 +73,13 @@ export async function fetchBrokeredInstallationToken(
     ...(options.forceRefresh ? { body: JSON.stringify({ forceRefresh: true }) } : {}),
     signal: AbortSignal.timeout(BROKER_TIMEOUT_MS),
   });
+  // #9156: this IS the JWT-mint-equivalent call in broker mode -- a brokered self-host holds no App private
+  // key, so requestInstallationTokenWithJwt's own clock-skew sample (src/github/app.ts, #3811) is UNREACHABLE
+  // by construction whenever isOrbBrokerMode(env) is true (mintInstallationToken's broker branch returns
+  // before ever reaching the JWT path). Sampled unconditionally (both success and failure responses carry a
+  // real `Date` header) so a brokered deployment's clock drift is measured at all, not silently left at the
+  // NaN "never sampled" sentinel forever.
+  recordClockSkewFromResponse(response);
   if (!response.ok) {
     throw new Error(`Orb broker token exchange failed (${response.status}).`);
   }

src/queue/processors.ts

@@ -486,9 +486,11 @@ import {
 import {
   buildVisualBugAnalysisUserPrompt,
   buildVisualRegressionFindings,
+  buildVisualVisionSkippedForReputationFinding,
   buildVisualVisionUserPrompt,
   evaluateVisualVisionGate,
   parseVisualVisionResponse,
+  selectRoutesForVisualVision,
   VISUAL_BUG_ANALYSIS_SYSTEM_PROMPT,
   VISUAL_VISION_SYSTEM_PROMPT,
 } from "../review/visual/visual-findings";
@@ -641,6 +643,7 @@ import { DEFAULT_SCREENSHOT_CONTRACT_MESSAGE, DEFAULT_SCREENSHOT_TABLE_GATE, eva
 import { isSafeHttpUrl } from "../review/content-lane/safe-url";
 import {
   buildScreenshotTableVisionFindings,
+  buildScreenshotTableVisionSkippedForReputationFinding,
   buildScreenshotTableVisionUserPrompt,
   evaluateScreenshotTableVisionGate,
   parseScreenshotTableVisionResponse,
@@ -8774,7 +8777,18 @@ export async function runVisualVisionForAdvisory(
       providerKey: visionProviderKey,
       selfHostVisionAvailable,
     });
-    if (!visionGate.run) return;
+    if (!visionGate.run) {
+      // #9136 compensating hold: a low-reputation skip on a route the pixel-diff threshold already confirmed
+      // changed must not go completely silent -- see buildVisualVisionSkippedForReputationFinding's own doc
+      // comment for why (the #9015 "suspicion buys less scrutiny" shape). Only reachable for low_reputation --
+      // the other two skip reasons (byok_not_configured, no_confirmed_regression) have nothing to compensate
+      // for: either there was no confirmed regression at all, or vision simply isn't configured this run.
+      if (visionGate.reason === "low_reputation") {
+        const skippedFinding = buildVisualVisionSkippedForReputationFinding(selectRoutesForVisualVision(args.routes).length);
+        if (skippedFinding) args.advisory.findings.push(skippedFinding);
+      }
+      return;
+    }
     // evaluateVisualVisionGate only ever returns run:true when providerKey OR selfHostVisionAvailable (the
     // SAME two values resolved above) was truthy -- this is a defensive type-narrowing guard, not a reachable
     // false case: if neither is set here, the gate itself would already have returned run:false above.
@@ -9111,6 +9125,14 @@ export async function runScreenshotTableVisionForAdvisory(
           // summary -- the caller's own "absent means omit" contract for the AI review prompt param.
           evidenceSummary = parseScreenshotTableVisionSummary(visionText);
         }
+      } else if (gate.reason === "low_reputation") {
+        // #9136 compensating hold: mirrors runVisualVisionForAdvisory's own low_reputation compensating
+        // finding above -- a low-reputation submitter's real (different-bytes) screenshot-table pair is
+        // EXACTLY the gaming case this check exists to catch, so skipping it silently reproduces #9015's
+        // "suspicion buys less scrutiny" shape. The other two skip reasons (byok_not_configured,
+        // no_image_pairs) have nothing to compensate for.
+        const skippedFinding = buildScreenshotTableVisionSkippedForReputationFinding(fetchedPairs.length);
+        if (skippedFinding) findings.push(skippedFinding);
       }
     }
     if (findings.length > 0) args.advisory.findings.push(...findings);

src/registry/sync.ts

@@ -8,6 +8,11 @@ import type { RegistrySnapshot } from "../types";
 import { errorMessage, jsonString, nowIso, repoParts } from "../utils/json";
 import { normalizeRegistryPayload } from "./normalize";
 
+// #9165 sweep: this loop's `fallbackUrl` candidate is the same raw-GitHub-fallback shape as
+// src/upstream/ruleset.ts's fetchTrackedSource -- a stalled connection to any candidate (API or raw fallback)
+// hung this whole sequential probe with no bound, since `fetch()` has no default timeout.
+const REGISTRY_SYNC_FETCH_TIMEOUT_MS = 12_000;
+
 const API_CANDIDATES = [
   "https://api.gittensor.io/repositories",
   "https://api.gittensor.io/api/repositories",
@@ -40,6 +45,7 @@ export async function refreshRegistry(env: Env): Promise<RegistrySnapshot> {
             accept: "application/json",
             "user-agent": PRODUCT_USER_AGENT,
           },
+          signal: AbortSignal.timeout(REGISTRY_SYNC_FETCH_TIMEOUT_MS),
         });
         if (!response.ok) {
           warnings.push(`Registry probe failed: ${url} (${response.status})`);

src/review/ams-miner-cohort.ts

@@ -1,9 +1,10 @@
 // AMS-vs-human contributor-mix dashboard panel (#6488), per #6210's decided design: for one repo, classify each
 // recent submitter as "has AMS track-record data" (via the ORB/AMS reputation bridge's pull path, #6485/#6208)
 // vs. not, then compare acceptance rate / review-cycle count / time-to-merge / PR volume between the two
-// cohorts. Reuses `submitter-reputation.ts`'s existing `review_targets`-based outcome classification for the
-// metrics and `ams-reputation-bridge.ts`'s existing fail-safe, timeout-bounded pull for AMS membership — no new
-// identity system, no new network path.
+// cohorts. Reuses `submitter-reputation.ts`'s existing outcome classification (`listSubmitterCohortRows`, #9136:
+// repointed off the orphaned `review_targets` table onto the live `review_audit`/`pull_requests` ledgers — this
+// module needed no changes of its own to pick that up) for the metrics, and `ams-reputation-bridge.ts`'s
+// existing fail-safe, timeout-bounded pull for AMS membership — no new identity system, no new network path.
 //
 // BOUNDED, NOT EXHAUSTIVE: classifying membership needs one live call per distinct submitter login (there is no
 // cached "is this login an AMS miner" flag anywhere yet). An unbounded per-dashboard-load fan-out over every
@@ -33,8 +34,11 @@ export type AmsMinerCohortMetrics = {
   /** merged / (merged + closed) over the cohort's terminal rows, `null` when the cohort has no terminal rows
    *  to divide by (never fabricated as 0, which would misleadingly read as "0% acceptance"). */
   acceptanceRate: number | null;
-  /** Mean of each submitter's own average `attempt_count` (the gate's re-review counter, reused as the
-   *  review-cycle-count proxy per #6488's requirements) — `null` when the cohort is empty. */
+  /** Mean of each submitter's own average review-cycle-count proxy per #6488's requirements — `null` when
+   *  the cohort is empty. Sourced from `pull_requests.merge_attempt_count` (#9136: the pre-repoint
+   *  `review_targets.attempt_count`, the gate's own re-review counter, has no live equivalent; see
+   *  `SubmitterCohortRow.avgAttemptCount`'s own doc comment in submitter-reputation.ts for the documented,
+   *  narrower substitution this reuses). */
   avgReviewCycleCount: number | null;
   /** Mean time-to-merge in ms across the cohort's MERGED rows only, `null` when the cohort has no merges. */
   avgTimeToMergeMs: number | null;

src/review/ops.ts

@@ -547,9 +547,17 @@ export interface ReviewSourceFreshnessCheck {
  *  window. `review_targets` was silently orphaned by the 2026-06-22 convergence cutover (no live writer
  *  anywhere — see src/db/repo-identity-rename.ts / src/review/public-stats.ts's own comments) and nobody
  *  noticed for months; this is the generalizable fix so the NEXT orphaning is loud, not silent.
- *   - review_targets: submitter-reputation.ts's own REPUTATION_WINDOW_DAYS (90) — this table's newest row
- *     is frozen at the 2026-06-22 cutover, so it will fall outside this window (and read permanently stale
- *     here) around 2026-09-20 if nothing else changes; see #9136 for the full scope decision on that module.
+ *   - review_targets: submitter-reputation.ts (#9136) has SINCE been repointed off this table onto the live
+ *     review_audit/pull_requests ledgers — its own REPUTATION_WINDOW_DAYS (90) no longer reads this table at
+ *     all. The remaining live readers are THIS file's own computeAgentHealth (byStatus/byVerdict/failedRows/
+ *     manualRate/stuckRetryable/failed) and computeCalibration (mergedRows/closesByReasonRows/disputedRows) —
+ *     deferred here because review_targets' non-terminal states (queued/reviewing/error/error_retryable) and
+ *     the attempt-exhausted 'failed' bucket have no live per-target equivalent post-cutover: gate_decision's
+ *     own `decision` column only ever records 'merge' | 'close' | 'hold' (parity.ts's GateAction), not the
+ *     full status enum review_targets tracked. This table's newest row is frozen at the 2026-06-22 cutover,
+ *     so ANY windowDays here eventually reads permanently stale — 90 is kept as a conservative, still-fires-
+ *     eventually value, not because a live consumer still uses that exact number. See #9136 for the tracked
+ *     remainder and the reasoning above.
  *   - review_audit: this module's own ANOMALY_WINDOW (7 days) — IS live today (parity-wire.ts writes
  *     'gate_decision' rows, outcomes-wire.ts writes the outcome/reversal types), so this should read fresh
  *     in steady state. If both writers ever stop, this is what catches the alerter going silently inert