fix(orb): webhook redelivery, reputation cadence, DB retention, and gate severity fidelity

migrations/0191_retention_column_indexes.sql

@@ -0,0 +1,33 @@
+-- #9083: no RETENTION_POLICY (src/db/retention.ts) table had an index whose LEADING column is its
+-- retention timestamp -- every existing index on these tables either doesn't exist at all
+-- (webhook_events, signal_snapshots, score_previews, repo_snapshots) or has the timestamp only in a
+-- trailing position (audit_events, ai_usage_events, product_usage_events,
+-- github_rate_limit_observations, agent_context_snapshots, notification_deliveries). Paired with the
+-- pruneExpiredRecords rewrite (PK-ordered range delete instead of a rowid/ctid semi-join), the inner
+-- SELECT in each batched delete is now an index range scan instead of a full table scan.
+CREATE INDEX IF NOT EXISTS idx_webhook_events_received_at ON webhook_events(received_at);
+CREATE INDEX IF NOT EXISTS idx_audit_events_created_at ON audit_events(created_at);
+CREATE INDEX IF NOT EXISTS idx_ai_usage_events_created_at ON ai_usage_events(created_at);
+CREATE INDEX IF NOT EXISTS idx_product_usage_events_occurred_at ON product_usage_events(occurred_at);
+CREATE INDEX IF NOT EXISTS idx_github_rate_limit_observations_observed_at ON github_rate_limit_observations(observed_at);
+CREATE INDEX IF NOT EXISTS idx_signal_snapshots_generated_at ON signal_snapshots(generated_at);
+CREATE INDEX IF NOT EXISTS idx_score_previews_generated_at ON score_previews(generated_at);
+CREATE INDEX IF NOT EXISTS idx_repo_snapshots_fetched_at ON repo_snapshots(fetched_at);
+CREATE INDEX IF NOT EXISTS idx_agent_context_snapshots_created_at ON agent_context_snapshots(created_at);
+CREATE INDEX IF NOT EXISTS idx_notification_deliveries_created_at ON notification_deliveries(created_at);
+CREATE INDEX IF NOT EXISTS idx_predicted_gate_calls_created_at ON predicted_gate_calls(created_at);
+
+-- #9083: the four never-pruned caches (grounding_file_content_cache, ai_review_cache, ai_slop_cache,
+-- linked_issue_satisfaction_cache) newly gained a retention rule in the same change -- index their
+-- timestamp column too so the new prune sweep stays cheap as they grow.
+CREATE INDEX IF NOT EXISTS idx_grounding_file_content_cache_fetched_at ON grounding_file_content_cache(fetched_at);
+CREATE INDEX IF NOT EXISTS idx_ai_review_cache_created_at ON ai_review_cache(created_at);
+CREATE INDEX IF NOT EXISTS idx_ai_slop_cache_created_at ON ai_slop_cache(created_at);
+CREATE INDEX IF NOT EXISTS idx_linked_issue_satisfaction_cache_created_at ON linked_issue_satisfaction_cache(created_at);
+
+-- #9083: the three newly-retained append-only tables (review_audit, decision_records, orb_webhook_events)
+-- each already had an index touching their timestamp column, but only in a trailing/combined position --
+-- add the leading single-column index the new PK-ordered delete needs.
+CREATE INDEX IF NOT EXISTS idx_review_audit_created_at ON review_audit(created_at);
+CREATE INDEX IF NOT EXISTS idx_decision_records_created_at ON decision_records(created_at);
+CREATE INDEX IF NOT EXISTS idx_orb_webhook_events_received_at ON orb_webhook_events(received_at);

migrations/0192_purge_stuck_queued_webhook_events.sql

@@ -0,0 +1,24 @@
+-- #9054: one-off purge of webhook_events rows PERMANENTLY stuck at 'queued'/'superseded' by the dedup-guard
+-- bug fixed alongside this migration (src/github/webhook.ts's enqueueWebhookByEnv). Before that fix, a row
+-- left at 'queued' (the insert happened but WEBHOOKS.send() was lost, or the enqueued job vanished) or
+-- 'superseded' (overwritten by a later coalesced delivery before either was claimed) could NEVER be
+-- redelivered: every GitHub retry and every operator "Redeliver" click carries the identical delivery_id and
+-- (usually) the identical payload hash, hit the dedup guard, and was silently discarded as a no-op
+-- duplicate, forever.
+--
+-- On the live box this incident left 5,547 such rows (4,900 of them check_suite.completed -- the
+-- maybeReReviewOnCiCompletion auto-merge trigger), all from a ~3.5-week cutover-era window that was already
+-- quiescent by the time this was diagnosed. Replaying them is not useful: the code-level fix means any
+-- FUTURE stuck delivery becomes redeliverable after STALE_QUEUED_WEBHOOK_MS (10 minutes) rather than never,
+-- so there is no ongoing gap to backfill, and the PRs behind this historical backlog are long since
+-- resolved one way or another. Simply deleting them (rather than replaying) is what the issue itself called
+-- for, so the stuck-delivery metric is meaningful again going forward.
+--
+-- Applies once, wherever/whenever this migration runs: any 'queued'/'superseded' row still unprocessed a
+-- full day after receipt is dead by definition (real processing completes in well under a second), so this
+-- is safe as a general one-time data-hygiene sweep on any environment, and a no-op on a fresh database with
+-- no history to purge.
+DELETE FROM webhook_events
+ WHERE status IN ('queued', 'superseded')
+   AND processed_at IS NULL
+   AND received_at < datetime('now', '-1 day');

packages/loopover-engine/src/advisory/gate-advisory.ts

@@ -39,6 +39,11 @@ function sanitizeForCheckRun(text: string): string {
 }
 
 const DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE = 0.93;
+// #9085 (host-parity): mirrors CONFIDENCE_WHEN_UNSTATED (src/services/ai-review.ts) -- kept as a local constant
+// rather than an import, matching this file's own divergence design (it deliberately never pulls in the host's
+// signals/services subsystem). An ABSENT `finding.confidence` must never read as maximum certainty (1.0):
+// "silence is not certainty" (#8833).
+const CONFIDENCE_WHEN_UNSTATED = 0.5;
 /** Exported to mirror the src twin (#8224): loopover's LOOSENABLE_KNOBS registry anchors the slop knob's
  *  shipped value on this constant (divided by 100 onto the corpus's confidence scale). Value unchanged. */
 export const DEFAULT_SLOP_BLOCK_THRESHOLD = 60;
@@ -196,12 +201,15 @@ export function buildPullRequestAdvisory(
   const targetKey = pr ? `${repoFullName}#${pr.number}` : `${repoFullName}#unknown`;
   const findings: AdvisoryFinding[] = [];
   if (!repo) {
+    // #9085 (host-parity): renamed from `repo_not_registered` -- one character away from, and with the
+    // OPPOSITE gate consequence of, `repo_unregistered` below (addRepoFindings). See the host copy
+    // (src/rules/advisory.ts) for the full rename rationale.
     findings.push({
-      code: "repo_not_registered",
+      code: "repo_not_cached",
       severity: "warning",
-      title: "Repository registration is unknown",
-      detail: "LoopOver cannot evaluate repo-specific rules until registry data is available.",
-      action: "Refresh the Gittensor registry snapshot.",
+      title: "Repository is not yet cached",
+      detail: "LoopOver has not synced this repository yet, so repo-specific rules cannot be evaluated.",
+      action: "Wait for the next repo sync, or re-deliver the installation webhook.",
     });
   } else {
     addRepoFindings(repo, findings);
@@ -634,7 +642,8 @@ function isEvaluationBlocker(code: string, policy: GateCheckPolicy): boolean {
   // pre_merge_check_unresolved: an enforced path-gated pre-merge check whose changed-file set could not be
   // resolved — loopover cannot evaluate it yet, so the gate is NEUTRAL (held) and re-evaluates on the next
   // sync, rather than auto-merging past the unverified requirement or hard-closing on a transient miss. (#review-audit)
-  if (code === "repo_not_registered" || code === "repo_not_seen" || code === "pr_not_cached" || code === "pre_merge_check_unresolved") return true;
+  // #9085 (host-parity): renamed from `repo_not_registered`.
+  if (code === "repo_not_cached" || code === "repo_not_seen" || code === "pr_not_cached" || code === "pre_merge_check_unresolved") return true;
   // cla_check_unresolved (#2564): the CLA-bot check-run's conclusion could not be resolved. Unlike the codes
   // above (which are never mode-gated), evaluateClaCheck runs for BOTH claGateMode "advisory" and "block" (so
   // the finding surfaces either way) — only "block" should ever HOLD the gate on an unresolved check-run.
@@ -678,7 +687,7 @@ function isConfiguredGateBlocker(finding: AdvisoryFinding, policy: GateCheckPoli
     if (!gatePolicyBlocks(policy.aiReviewGateMode, "advisory")) return false;
     if ((policy.aiReviewLowConfidenceDisposition ?? "hold_for_review") === "advisory_only") {
       const floor = policy.aiReviewCloseConfidence ?? DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE;
-      const confidence = finding.confidence ?? 1;
+      const confidence = finding.confidence ?? CONFIDENCE_WHEN_UNSTATED;
       if (confidence < floor) return false;
     }
     return true;

packages/loopover-engine/src/governor/chokepoint.ts

@@ -35,7 +35,7 @@ import { DEFAULT_SELF_REPUTATION_THRESHOLDS, selfReputationThrottle } from "./re
 import type { OwnSubmissionRecord, SelfPlagiarismCandidate, SelfPlagiarismConfig, SelfPlagiarismVerdict } from "./self-plagiarism.js";
 import { DEFAULT_SELF_PLAGIARISM_CONFIG, selfPlagiarismCheck } from "./self-plagiarism.js";
 import type { WriteRateLimitBackoffStore, WriteRateLimitBucketStore, WriteRateLimitPolicies, WriteRateLimitVerdict } from "./write-rate-limit.js";
-import { evaluateWriteRateLimit } from "./write-rate-limit.js";
+import { DEFAULT_WRITE_RATE_LIMIT_POLICIES, evaluateWriteRateLimit } from "./write-rate-limit.js";
 
 /** Which stage of the precedence ladder produced the final verdict. */
 export type GovernorDecisionStage =
@@ -97,6 +97,13 @@ export type GovernorDecisionDetail = {
   convergence?: PortfolioConvergenceVerdict;
   reputation?: SelfReputationThrottleDecision;
   selfPlagiarism?: SelfPlagiarismVerdict;
+  /** #9062: the rate-limit policies actually used to reach `rateLimit` -- identical to the caller-supplied
+   *  `rateLimitPolicies` (or the engine default) UNLESS a non-floored reputation throttle scaled the
+   *  per-repo window. Present once the rate-limit stage has run (mirrors `rateLimit`'s own presence). The
+   *  stateful miner-lib wrapper (`governor-chokepoint.js`) reads this back to advance the SAME bucket
+   *  arithmetic post-decision that produced this verdict -- reusing the caller's raw, un-scaled policy there
+   *  instead would silently disagree with the decision it is recording state for. */
+  effectiveRateLimitPolicies?: WriteRateLimitPolicies;
 };
 
 export type GovernorDecision = {
@@ -110,6 +117,30 @@ export type GovernorDecision = {
   ledgerEvent: GovernorLedgerEvent;
 };
 
+/** #9062: stretch the PER-REPO rate-limit window for `actionClass` by `1 / cadenceFactor` so a throttled
+ *  miner's write cadence on THIS repo degrades proportionally instead of being denied outright --
+ *  reputation-throttle.ts's own doc comment promises "a recovering ratio restores it -- never a hard
+ *  permanent ban", so consuming `cadenceFactor` here (rather than discarding it, as before) is what actually
+ *  delivers that contract for the common (non-floored) `"throttled"` case. Only the per-repo scope is
+ *  scaled: `RepoOutcomeHistory` is scoped to ONE repo, so a bad ratio there must not also slow the miner's
+ *  unrelated global write velocity across every other repo it works on. Never called with
+ *  `cadenceFactor >= 1` (the caller only invokes this once it has already checked that). */
+function scaleRateLimitPerRepoWindow(policies: WriteRateLimitPolicies, actionClass: string, cadenceFactor: number): WriteRateLimitPolicies {
+  const perRepoConfig = policies.perRepo[actionClass];
+  if (!perRepoConfig) return policies;
+  // Defensive floor: cadenceFactor is documented as "never 0" by resolveSelfReputationThresholds's clamped
+  // minCadenceFactor, but that field has no enforced lower bound of its own -- guard the division regardless
+  // of what a caller-supplied threshold override might set it to.
+  const safeFactor = cadenceFactor > 0 ? cadenceFactor : 0.001;
+  return {
+    ...policies,
+    perRepo: {
+      ...policies.perRepo,
+      [actionClass]: { ...perRepoConfig, windowMs: Math.round(perRepoConfig.windowMs / safeFactor) },
+    },
+  };
+}
+
 function denyResult(input: {
   stage: GovernorDecisionStage;
   reason: string;
@@ -184,6 +215,35 @@ export function evaluateGovernorChokepoint(input: GovernorChokepointInput): Gove
     };
   }
 
+  const isSelfSubmissionAction = SELF_SUBMISSION_ACTION_CLASSES.has(input.actionClass);
+
+  // #9062: computed here -- BEFORE rate-limit -- rather than after budget/convergence (where its own HARD
+  // DENY for `reason === "floored"` still stays, preserving the ladder's documented precedence for that
+  // path) so a non-floored `cadenceFactor` can scale the per-repo rate-limit window below. Discarding
+  // cadenceFactor and hard-denying on ANY throttled verdict (the pre-#9062 behavior) is exactly what turned a
+  // recoverable soft throttle into a permanent self-ban: submissions are the miner's only source of new
+  // decided outcomes, so a miner that can never submit can never dilute a bad ratio back down.
+  let reputation: SelfReputationThrottleDecision | undefined;
+  if (isSelfSubmissionAction && input.reputationHistory !== undefined) {
+    try {
+      reputation = selfReputationThrottle(input.reputationHistory, input.reputationThresholds ?? DEFAULT_SELF_REPUTATION_THRESHOLDS);
+    } catch (error) {
+      return denyResult({
+        stage: "internal_error",
+        reason: `reputation_throttle_calculator_error: ${error instanceof Error ? error.message : String(error)}`,
+        mode,
+        detail: baseDetail,
+        eventType: "denied",
+        actionClass: input.actionClass,
+        repoFullName: input.repoFullName,
+      });
+    }
+  }
+  const rateLimitPolicies =
+    reputation && reputation.cadenceFactor < 1
+      ? scaleRateLimitPerRepoWindow(input.rateLimitPolicies ?? DEFAULT_WRITE_RATE_LIMIT_POLICIES, input.actionClass, reputation.cadenceFactor)
+      : input.rateLimitPolicies;
+
   let rateLimit: WriteRateLimitVerdict;
   try {
     rateLimit = evaluateWriteRateLimit({
@@ -192,7 +252,7 @@ export function evaluateGovernorChokepoint(input: GovernorChokepointInput): Gove
       buckets: input.rateLimitBuckets,
       backoffAttempts: input.rateLimitBackoffAttempts,
       nowMs: input.nowMs,
-      ...(input.rateLimitPolicies ? { policies: input.rateLimitPolicies } : {}),
+      ...(rateLimitPolicies ? { policies: rateLimitPolicies } : {}),
       ...(input.rateLimitRandomFn ? { randomFn: input.rateLimitRandomFn } : {}),
     });
   } catch (error) {
@@ -206,7 +266,11 @@ export function evaluateGovernorChokepoint(input: GovernorChokepointInput): Gove
       repoFullName: input.repoFullName,
     });
   }
-  const detailWithRateLimit: GovernorDecisionDetail = { ...baseDetail, rateLimit };
+  const detailWithRateLimit: GovernorDecisionDetail = {
+    ...baseDetail,
+    rateLimit,
+    effectiveRateLimitPolicies: rateLimitPolicies ?? DEFAULT_WRITE_RATE_LIMIT_POLICIES,
+  };
   if (!rateLimit.allowed) {
     return denyResult({
       stage: "rate_limit",
@@ -216,7 +280,13 @@ export function evaluateGovernorChokepoint(input: GovernorChokepointInput): Gove
       eventType: "throttled",
       actionClass: input.actionClass,
       repoFullName: input.repoFullName,
-      extraPayload: { retryAfterMs: rateLimit.retryAfterMs, blockedBy: rateLimit.blockedBy },
+      extraPayload: {
+        retryAfterMs: rateLimit.retryAfterMs,
+        blockedBy: rateLimit.blockedBy,
+        // Surfaced only when the reputation throttle actually degraded this window, so an operator reading
+        // the ledger can tell "genuinely busy" apart from "a reputation-scaled cadence caught up with it".
+        ...(reputation && reputation.cadenceFactor < 1 ? { reputationCadenceFactor: reputation.cadenceFactor } : {}),
+      },
     });
   }
 
@@ -275,29 +345,17 @@ export function evaluateGovernorChokepoint(input: GovernorChokepointInput): Gove
     });
   }
 
-  const isSelfSubmissionAction = SELF_SUBMISSION_ACTION_CLASSES.has(input.actionClass);
-
+  // #9062: `reputation` was already computed above (before rate-limit, so its cadenceFactor could scale that
+  // stage's per-repo window). Its own hard-deny stays at its documented ladder position -- AFTER budget/
+  // convergence -- but is now reserved for `reason === "floored"` only: the extreme, near-certain-abuse case.
+  // A plain `"throttled"` verdict already had its cadence effect folded into the rate-limit evaluation above,
+  // so it no longer denies here at all -- it either already got caught by the tightened rate limit, or it
+  // didn't need to be, and either way the miner keeps submitting (at a degraded cadence) instead of being
+  // structurally unable to ever generate the new clean outcomes that would dilute its ratio back down.
   let detailWithReputation = detailWithConvergence;
-  // `!== undefined` (not a truthy check): an omitted key means "skip this stage"; any OTHER value the caller
-  // supplied -- including a bad `null` from a malformed upstream source -- must reach the calculator and, if it
-  // cannot handle it, fail closed via the catch below, never silently skip.
-  if (isSelfSubmissionAction && input.reputationHistory !== undefined) {
-    let reputation: SelfReputationThrottleDecision;
-    try {
-      reputation = selfReputationThrottle(input.reputationHistory, input.reputationThresholds ?? DEFAULT_SELF_REPUTATION_THRESHOLDS);
-    } catch (error) {
-      return denyResult({
-        stage: "internal_error",
-        reason: `reputation_throttle_calculator_error: ${error instanceof Error ? error.message : String(error)}`,
-        mode,
-        detail: detailWithConvergence,
-        eventType: "denied",
-        actionClass: input.actionClass,
-        repoFullName: input.repoFullName,
-      });
-    }
+  if (reputation) {
     detailWithReputation = { ...detailWithConvergence, reputation };
-    if (reputation.throttled) {
+    if (reputation.reason === "floored") {
       return denyResult({
         stage: "reputation_throttle",
         reason: reputation.reason,