fix(selfhost): backlog-vs-fresh fairness, convergence-cap shadowing, and maintenance admission feedback loop

apps/loopover-ui/src/lib/selfhost-env-reference.ts

@@ -337,6 +337,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "MAINTENANCE_ADMISSION_MAX_PENDING",
     firstReference: "src/selfhost/maintenance-admission.ts",
   },
+  {
+    name: "MAINTENANCE_ADMISSION_PRESSURE_CACHE_TTL_MS",
+    firstReference: "src/selfhost/maintenance-admission.ts",
+  },
   {
     name: "MIGRATIONS_DIR",
     firstReference: "src/server.ts",
@@ -729,6 +733,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `MAINTENANCE_ADMISSION_MAX_LIVE_AGE_MS` | `src/selfhost/maintenance-admission.ts` |",
   "| `MAINTENANCE_ADMISSION_MAX_LIVE_PENDING` | `src/selfhost/maintenance-admission.ts` |",
   "| `MAINTENANCE_ADMISSION_MAX_PENDING` | `src/selfhost/maintenance-admission.ts` |",
+  "| `MAINTENANCE_ADMISSION_PRESSURE_CACHE_TTL_MS` | `src/selfhost/maintenance-admission.ts` |",
   "| `MIGRATIONS_DIR` | `src/server.ts` |",
   "| `OBSERVABILITY_SMOKE_POLL_MS` | `scripts/smoke-observability-traces.ts` |",
   "| `OBSERVABILITY_SMOKE_TIMEOUT_MS` | `scripts/smoke-observability-traces.ts` |",

src/queue/processors.ts

@@ -258,7 +258,7 @@ import {
   isRegateSweepDraining,
   selectRegateCandidates,
 } from "../settings/agent-sweep";
-import { selectBacklogConvergenceCandidates } from "../selfhost/backlog-convergence";
+import { BACKLOG_CONVERGENCE_SWEEP_MAX_PRS, sortedBacklogConvergenceCandidates } from "../selfhost/backlog-convergence";
 import {
   LOW_REST_RATE_LIMIT_REMAINING,
   MAINTENANCE_RESERVED_HEADROOM,
@@ -1851,16 +1851,53 @@ export async function sweepRepoBacklogConvergence(
   const sweepInstallationId = repo?.installationId ?? null;
   if (sweepInstallationId == null) return;
   const openPullRequests = await listOpenPullRequests(env, repoFullName);
-  const allCandidates = selectBacklogConvergenceCandidates({ pulls: openPullRequests });
+  // #9154: the exhaustion filter must run BEFORE the `max` cap, not after -- capping first (the previous
+  // shape: selectBacklogConvergenceCandidates already sliced to 5, THEN filtered) let five permanently
+  // repair-exhausted old PRs occupy every slot forever, shadowing every other PR needing convergence behind
+  // them. Walk the FULL oldest-open-first order and skip exhausted candidates as we go, stopping as soon as
+  // `max` actionable ones are found -- bounded to however far into the backlog we actually need to look, not
+  // the whole list, in the common case where most of the head is actionable.
+  //
   // #orb-retry-storm (backlog-convergence half): needsSurfaceConvergence re-fires on the exact same
   // lastPublishedSurfaceSha-mismatch signal as the main sweep's outage-repair priority path, but this sweeper
   // had no memory of prior attempts at all -- a PR whose gate-check finalize kept failing silently got a fresh
   // full re-review dispatched every ~30 minutes indefinitely. Share the same per-SHA attempt budget as the main
   // sweep (isRegateRepairExhausted) rather than adding an independent cap, since both sweeps competing for the
   // same stuck PR would otherwise double the wasted spend the cap exists to prevent.
-  const exhaustedFlags = await Promise.all(allCandidates.map((pr) => isRegateRepairExhausted(env, repoFullName, pr)));
-  const candidates = allCandidates.filter((_pr, index) => !exhaustedFlags[index]);
-  if (candidates.length === 0) return;
+  const orderedCandidates = sortedBacklogConvergenceCandidates(openPullRequests);
+  const candidates: PullRequestRecord[] = [];
+  let examinedCount = 0;
+  for (const pr of orderedCandidates) {
+    if (candidates.length >= BACKLOG_CONVERGENCE_SWEEP_MAX_PRS) break;
+    examinedCount += 1;
+    if (await isRegateRepairExhausted(env, repoFullName, pr)) continue;
+    candidates.push(pr);
+  }
+  if (candidates.length === 0) {
+    // Every candidate examined was repair-exhausted -- the sweep is returning early with nothing to show for
+    // it. Previously silent (the caller just saw an empty array and bailed); log + audit so a permanently
+    // wedged head of the backlog is visible instead of masquerading as "nothing needs convergence".
+    if (orderedCandidates.length > 0) {
+      console.warn(
+        JSON.stringify({
+          level: "warn",
+          event: "backlog_convergence_sweep_all_exhausted",
+          repository: repoFullName,
+          examined: examinedCount,
+          totalCandidates: orderedCandidates.length,
+        }),
+      );
+      await recordAuditEvent(env, {
+        eventType: "agent.sweep.backlog_convergence",
+        actor: "loopover",
+        targetKey: repoFullName,
+        outcome: "denied",
+        detail: `backlog-convergence sweep found ${orderedCandidates.length} open PR(s) needing convergence, all repair-exhausted`,
+        metadata: { repoFullName, examined: examinedCount, totalCandidates: orderedCandidates.length },
+      });
+    }
+    return;
+  }
   // Stamp the backlog-convergence draining marker for EVERY candidate NOW, at dispatch — not in the downstream
   // per-PR job (#4502, mirrors #audit-sweep-dispatch-stamp). This makes getLatestBacklogConvergenceRegatedAt
   // reflect this sweep immediately, so fanOutBacklogConvergenceSweepJobs's in-flight guard skips re-arming this

src/selfhost/backlog-convergence.ts

@@ -32,25 +32,39 @@ export function needsSurfaceConvergence(pr: Pick<PullRequestRecord, "headSha" |
 }
 
 /**
- * Select the open PRs a single repo's backlog-convergence sweep should re-enqueue: drop drafts and anything
- * whose surface is already published at the current head, then take the `max` PRs that have been open longest
- * (oldest `createdAt` first, falling back to the epoch so a PR with no known creation time still sorts
- * deterministically rather than being silently dropped) — this is the explicit "oldest open PRs first" fairness
- * ordering the backlog-drain lane depends on (see queue-fairness.ts, PR2). Ties broken by PR number. Pure +
- * deterministic: same inputs -> same ordered batch.
+ * Every open PR a repo's backlog-convergence sweep considers, in the order it should serve them: drop drafts
+ * and anything whose surface is already published at the current head, then order oldest-open-first (oldest
+ * `createdAt` first, falling back to the epoch so a PR with no known creation time still sorts
+ * deterministically rather than being silently dropped) — this is the explicit "oldest open PRs first"
+ * fairness ordering the backlog-drain lane depends on (see queue-fairness.ts, PR2). Ties broken by PR number.
+ * Deliberately UNSLICED (#9154): selectBacklogConvergenceCandidates below applies the sweep's `max` cap
+ * directly to this order, which shadows every PR behind the first `max` whenever any of THOSE are
+ * permanently repair-exhausted (a caller that also needs to skip exhausted candidates -- see
+ * sweepRepoBacklogConvergence in processors.ts -- must walk this full order and exclude exhausted PRs BEFORE
+ * capping, not after). Pure + deterministic: same inputs -> same ordered list.
  */
-export function selectBacklogConvergenceCandidates(input: {
-  pulls: PullRequestRecord[];
-  max?: number;
-}): PullRequestRecord[] {
-  const max = input.max ?? BACKLOG_CONVERGENCE_SWEEP_MAX_PRS;
+export function sortedBacklogConvergenceCandidates(pulls: PullRequestRecord[]): PullRequestRecord[] {
   const ageKey = (pr: PullRequestRecord): number => {
     const created = pr.createdAt ? Date.parse(pr.createdAt) : Number.NaN;
     return Number.isFinite(created) ? created : 0;
   };
-  return input.pulls
+  return pulls
     .filter((pr) => pr.state === "open" && !pr.isDraft)
     .filter((pr) => needsSurfaceConvergence(pr))
-    .sort((a, b) => ageKey(a) - ageKey(b) || a.number - b.number)
-    .slice(0, Math.max(0, max));
+    .sort((a, b) => ageKey(a) - ageKey(b) || a.number - b.number);
+}
+
+/**
+ * Select the open PRs a single repo's backlog-convergence sweep should re-enqueue: the first `max` PRs (by
+ * default BACKLOG_CONVERGENCE_SWEEP_MAX_PRS) from sortedBacklogConvergenceCandidates' full oldest-open-first
+ * order. This convenience wrapper caps WITHOUT regard to repair-exhaustion — fine for a caller that doesn't
+ * need that filter, but see sortedBacklogConvergenceCandidates' doc comment for why sweepRepoBacklogConvergence
+ * itself calls that function directly instead. Pure + deterministic.
+ */
+export function selectBacklogConvergenceCandidates(input: {
+  pulls: PullRequestRecord[];
+  max?: number;
+}): PullRequestRecord[] {
+  const max = input.max ?? BACKLOG_CONVERGENCE_SWEEP_MAX_PRS;
+  return sortedBacklogConvergenceCandidates(input.pulls).slice(0, Math.max(0, max));
 }

src/selfhost/maintenance-admission.ts

@@ -96,10 +96,13 @@ export interface MaintenancePressureSignals {
    *  keeps finding stale work every sweep). This is the field evaluateMaintenanceAdmission's live_pending_high
    *  check actually gates on. */
   liveRunnableNowCount: number;
-  /** Age in ms of the oldest genuinely-active (processing, or pending AND due) foreground job -- null when
-   *  none qualifies right now. Distinct from oldestLivePendingAgeMs, which is dominated by a job intentionally
-   *  scheduled far in the future and says nothing about how long already-active work has sat unclaimed/running.
-   *  This is the field evaluateMaintenanceAdmission's live_job_age_high check actually gates on. */
+  /** Age in ms of the oldest DUE-AND-UNCLAIMED (status='pending', run_after<=now) foreground job -- null when
+   *  none qualifies right now. Deliberately EXCLUDES 'processing' rows (#9155): a job actively being worked (a
+   *  normal in-flight AI review routinely takes minutes) must not, by merely running, trip live_job_age_high
+   *  and collapse the entire maintenance lane -- including the watchdog/alerter that would report an actual
+   *  overload -- to the 4-hour trickle backstop. Distinct from oldestLivePendingAgeMs, which is dominated by a
+   *  job intentionally scheduled far in the future and says nothing about how long work has sat unclaimed. This
+   *  is the field evaluateMaintenanceAdmission's live_job_age_high check actually gates on. */
   oldestLiveRunnableAgeMs: number | null;
   maintenancePendingCount: number;
   oldestMaintenancePendingAgeMs: number | null;
@@ -129,6 +132,14 @@ export interface MaintenanceAdmissionConfig {
   deferMs: number;
   maxDeferAgeMs: number;
   maintenanceDrainAgeMs: number;
+  /** #9155: how long a computed MaintenancePressureSignals snapshot may be reused across successive claim
+   *  attempts before it must be recomputed. A denied maintenance job returns `true` from processOne(), so the
+   *  pump's drain loop immediately claims the next due maintenance row and re-evaluates admission -- without
+   *  this, a burst of N due maintenance rows means 4N sequential aggregate scans in one tight loop (more
+   *  denials -> more scans -> higher DB/host load -> higher hostLoadAvg1PerCore -> more denials, a positive
+   *  feedback loop). A short TTL, not "once per drain pass": pressure is still re-measured often enough to
+   *  react to a genuinely changing queue instead of latching a stale reading for a whole burst. */
+  pressureSignalsCacheTtlMs: number;
 }
 
 const DEFAULT_MAX_LIVE_PENDING_COUNT = 5;
@@ -143,6 +154,9 @@ const DEFAULT_MAX_BACKLOG_CONVERGENCE_PENDING_COUNT = 10;
 const DEFAULT_DEFER_MS = 3 * 60_000;
 const DEFAULT_MAX_DEFER_AGE_MS = 4 * 60 * 60_000;
 const DEFAULT_MAINTENANCE_DRAIN_AGE_MS = 10 * 60_000;
+// #9155: within the suggested 1-2s memoization window -- long enough to collapse a burst of denials sharing
+// one drain pass, short enough that a real pressure change is still visible within a couple of poll ticks.
+const DEFAULT_PRESSURE_SIGNALS_CACHE_TTL_MS = 1_500;
 
 function maintenanceAdmissionEnabled(): boolean {
   const raw = (process.env.MAINTENANCE_ADMISSION_ENABLED ?? "").trim().toLowerCase();
@@ -195,6 +209,10 @@ export function resolveMaintenanceAdmissionConfig(): MaintenanceAdmissionConfig
     // Never longer than the trickle backstop itself -- a misconfigured drain age above maxDeferAgeMs would be a
     // no-op (the trickle would always win first), so clamp it down rather than let it silently do nothing.
     maintenanceDrainAgeMs: Math.min(requestedDrainAgeMs, maxDeferAgeMs),
+    pressureSignalsCacheTtlMs: parsePositiveIntEnv("MAINTENANCE_ADMISSION_PRESSURE_CACHE_TTL_MS", {
+      min: 0,
+      fallback: DEFAULT_PRESSURE_SIGNALS_CACHE_TTL_MS,
+    }),
   };
 }