fix(retention): bound the five unpruned tables that filled the hosted D1

src/db/retention.ts

@@ -25,8 +25,10 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [
   // depending on it, so a shorter window than the audit/usage-log tables above is appropriate.
   { table: "agent_context_snapshots", column: "created_at", days: 30 },
   // One row per inbound webhook delivery (#8381 / unfinished #3896); short-lived idempotency lookups,
-  // not durable history — same 90d window as audit/ai_usage logs.
-  { table: "webhook_events", column: "received_at", days: 90 },
+  // not durable history. Cut 90d -> 14d: the dedup lookups these serve are minutes-old at most, and at the
+  // hosted fleet's ~4.3M-rows/day write rate a 90-day window on a pure idempotency log is the single
+  // largest avoidable contributor to the D1 ceiling (see the block at the end of this policy).
+  { table: "webhook_events", column: "received_at", days: 14 },
   // One row per outbound notification delivery (#8899); same append-only log shape as webhook_events.
   { table: "notification_deliveries", column: "created_at", days: 90 },
   // #9138: one row per loopover_predict_gate/explain_gate_disposition MCP call (unbounded, contributor-
@@ -54,8 +56,25 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [
   // close still has its record, matching product_usage_events' 180-day window for the same reason.
   { table: "decision_records", column: "created_at", days: 180 },
   // #9083: the central Orb App's OWN webhook delivery/dedup log (separate from webhook_events, which is the
-  // per-repo review app's) -- identical short-lived-idempotency shape, same 90d window.
-  { table: "orb_webhook_events", column: "received_at", days: 90 },
+  // per-repo review app's) -- identical short-lived-idempotency shape, same 14d window as webhook_events.
+  { table: "orb_webhook_events", column: "received_at", days: 14 },
+  // The five highest-growth tables below had NO retention rule at all and grew without bound, which is what
+  // actually filled the hosted D1 to its 10GB ceiling: every write then failed with
+  // `D1_ERROR: Exceeded maximum DB size`, including recordOrbWebhookEvent's INSERT, so /v1/orb/webhook
+  // returned 500 to GitHub and inbound webhook delivery stopped fleet-wide until rows were pruned by hand.
+  // Measured at the time of the outage: check_summaries 100,459 rows / 0.30GB of payload_json and
+  // pull_request_files 54,532 rows / 0.20GB were the two largest single consumers in the database.
+  //
+  // All five are re-derivable GitHub mirrors or superseded snapshots, never a contributor's evidentiary
+  // trail (decision_records, 180d above, is the table that carries that): a check-run summary, a PR's file
+  // list, a repo's totals snapshot, and the merged-PR/outcome caches are all re-fetched from GitHub on the
+  // next sync of the PR that needs them. 30 days keeps every window the review paths actually read
+  // (grounding/impact-map reads are head_sha-scoped and days-fresh at most) while bounding the growth.
+  { table: "check_summaries", column: "updated_at", days: 30 },
+  { table: "pull_request_files", column: "updated_at", days: 30 },
+  { table: "repo_github_totals_snapshots", column: "fetched_at", days: 30 },
+  { table: "recent_merged_pull_requests", column: "updated_at", days: 30 },
+  { table: "orb_pr_outcomes", column: "occurred_at", days: 90 },
 ];
 
 // #9083: a real, single-column, indexable primary key for the ordered-range delete below, keyed by table
@@ -95,8 +114,15 @@ export type PruneResult = { table: string; column: string; cutoff: string; delet
 const SAFE_IDENTIFIER = /^[a-z_]+$/;
 const BATCH_SIZE = 1000;
 // Bound work per table per run so a first prune of a large backlog cannot blow the D1 statement budget;
-// the daily cron drains any remainder over subsequent runs.
-const MAX_DELETED_PER_TABLE = 50_000;
+// the cron drains any remainder over subsequent runs.
+//
+// Raised 50k -> 250k because the old ceiling made retention structurally unable to keep up on the hosted
+// fleet and so guaranteed the D1 ceiling would be reached eventually: at 50k/table against a measured
+// ~4.3M rows written per day, a DAILY prune could delete at most ~1.25M rows/day across the whole policy —
+// a permanent ~3.5x deficit that no window tightening alone can close, because the cap (not the cutoff)
+// was the binding constraint. Paired with the hourly cadence in src/index.ts, the policy now drains far
+// faster than the fleet writes, so a backlog converges instead of compounding.
+const MAX_DELETED_PER_TABLE = 250_000;
 const MS_PER_DAY = 86_400_000;
 
 function retentionWhere(rule: RetentionRule): string {

src/index.ts

@@ -317,8 +317,15 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController):
   if (isHourly && hour === 7 && selfHostedReviews && isRiskControlEnabled(env)) {
     jobs.push({ type: "risk-control-recalibrate", requestedBy: "schedule" });
   }
-  // Prune expired log/snapshot rows once a day (03:00 UTC) per the conservative RETENTION_POLICY.
-  if (isHourly && hour === 3) {
+  // Prune expired log/snapshot rows EVERY hour per RETENTION_POLICY. Was once a day (03:00 UTC), which
+  // combined with the old per-table delete cap to put retention permanently behind the hosted fleet's write
+  // rate — the D1 database then reached its 10GB ceiling and every write failed with
+  // `D1_ERROR: Exceeded maximum DB size`, taking inbound GitHub webhook delivery down fleet-wide (the
+  // /v1/orb/webhook handler 500s when it cannot record the delivery). Hourly keeps each run's deletions
+  // small and steady rather than one large daily burst, and 24x the drain rate turns a growing backlog into
+  // a converging one. Cheap when there is nothing to do: each rule is an indexed range delete that matches
+  // zero rows once the table is inside its window.
+  if (isHourly) {
     jobs.push({ type: "prune-retention", requestedBy: "schedule" });
   }
   // Repo-doc refresh sweep (#3003, part of #2993) -- once a day (09:00 UTC, distinct from prune-retention's

test/unit/index.test.ts

@@ -609,6 +609,10 @@ describe("worker entrypoint", () => {
       { type: "refresh-scoring-model", requestedBy: "schedule" },
       { type: "refresh-upstream-drift", requestedBy: "schedule" },
       { type: "rollup-product-usage", requestedBy: "schedule", days: 7 },
+      // prune-retention is on EVERY hourly tick (was 03:00 daily): at the old cadence and per-table delete
+      // cap, retention could not keep up with the fleet's write rate and the hosted D1 reached its maximum
+      // size, after which every write failed and inbound webhook delivery stopped fleet-wide.
+      { type: "prune-retention", requestedBy: "schedule" },
     ]);
   });
 
@@ -679,6 +683,8 @@ describe("worker entrypoint", () => {
       { type: "refresh-scoring-model", requestedBy: "schedule" },
       { type: "refresh-upstream-drift", requestedBy: "schedule" },
       { type: "rollup-product-usage", requestedBy: "schedule", days: 7 },
+      // Hourly, not daily — see the note on the six-hour-window test above.
+      { type: "prune-retention", requestedBy: "schedule" },
       { type: "generate-signal-snapshots", requestedBy: "schedule" },
       { type: "build-burden-forecasts", requestedBy: "schedule" },
       { type: "build-contributor-evidence", requestedBy: "schedule" },
@@ -723,6 +729,7 @@ describe("worker entrypoint", () => {
       "refresh-scoring-model",
       "refresh-upstream-drift",
       "rollup-product-usage",
+      "prune-retention",
       "generate-signal-snapshots",
       "build-burden-forecasts",
       "build-contributor-evidence",

test/unit/retention.test.ts

@@ -16,7 +16,14 @@ async function seed(env: Env) {
   await db.insert(webhookEvents).values([
     { deliveryId: "wh-old-1", eventName: "push", payloadHash: "h", status: "processed", receivedAt: daysAgo(100) },
     { deliveryId: "wh-old-2", eventName: "push", payloadHash: "h", status: "processed", receivedAt: daysAgo(95) },
-    { deliveryId: "wh-recent", eventName: "push", payloadHash: "h", status: "processed", receivedAt: daysAgo(1) },
+    // Anchored to REAL now, not the fixed NOW the `daysAgo` rows use: the pruneExpiredRecords tests below
+    // pass an explicit `nowMs: NOW`, but the processJob and preview-route tests do not and so evaluate the
+    // policy against actual wall-clock time. A `daysAgo(1)` "recent" row is ~45 days before real now, which
+    // survived the old 90-day webhook_events window purely by accident and stopped surviving when that
+    // window tightened to 14 days. Real-now keeps this row genuinely recent under BOTH clocks (it is also
+    // after NOW, so it is never past a NOW-based cutoff either) and keeps each test's "recent rows are
+    // kept" intent independent of how wide the window happens to be.
+    { deliveryId: "wh-recent", eventName: "push", payloadHash: "h", status: "processed", receivedAt: new Date().toISOString() },
   ]);
   // ai_usage_events window = 90d; one old + one recent.
   await db.insert(aiUsageEvents).values([
@@ -121,6 +128,56 @@ describe("pruneExpiredRecords", () => {
     expect(rows.results.map((row) => row.id)).toEqual(["ctx-recent"]);
   });
 
+  // The hosted D1 reached its 10GB ceiling because these five tables had NO retention rule at all: every
+  // write then failed with `D1_ERROR: Exceeded maximum DB size`, including recordOrbWebhookEvent's INSERT,
+  // so /v1/orb/webhook returned 500 to GitHub and inbound webhook delivery stopped fleet-wide. Pinning the
+  // exact column names matters as much as the windows -- pruneExpiredRecords builds `<column> < ?` from
+  // this policy, so a column that does not exist on the table makes the rule a permanent silent no-op and
+  // the table resumes growing without bound exactly as before.
+  it("covers the five previously-unbounded high-growth tables (D1 ceiling regression)", () => {
+    const expected = [
+      { table: "check_summaries", column: "updated_at", days: 30 },
+      { table: "pull_request_files", column: "updated_at", days: 30 },
+      { table: "repo_github_totals_snapshots", column: "fetched_at", days: 30 },
+      { table: "recent_merged_pull_requests", column: "updated_at", days: 30 },
+      { table: "orb_pr_outcomes", column: "occurred_at", days: 90 },
+    ];
+    for (const want of expected) {
+      expect(RETENTION_POLICY).toContainEqual(want);
+    }
+  });
+
+  // Both webhook logs are short-lived idempotency lookups (minutes-old at most), cut 90d -> 14d as the
+  // single largest avoidable contributor to the same ceiling.
+  it("keeps both webhook idempotency logs on the tightened 14-day window", () => {
+    expect(RETENTION_POLICY).toContainEqual({ table: "webhook_events", column: "received_at", days: 14 });
+    expect(RETENTION_POLICY).toContainEqual({ table: "orb_webhook_events", column: "received_at", days: 14 });
+  });
+
+  // check_summaries was the largest single consumer in the database at the time of the outage (100,459 rows
+  // / 0.30GB of payload_json). Exercises the REAL policy window rather than an ad-hoc override, so dropping
+  // or re-widening the entry fails here.
+  it("prunes check_summaries past its policy window and keeps recent rows", async () => {
+    const env = createTestEnv();
+    await env.DB.prepare(
+      `INSERT INTO check_summaries (id, repo_full_name, pull_number, head_sha, name, status, conclusion, payload_json, updated_at)
+       VALUES
+         ('cs-old-1', 'acme/widgets', 1, 'sha1', 'ci', 'completed', 'success', '{}', ?),
+         ('cs-old-2', 'acme/widgets', 2, 'sha2', 'ci', 'completed', 'failure', '{}', ?),
+         ('cs-recent', 'acme/widgets', 3, 'sha3', 'ci', 'completed', 'success', '{}', ?)`,
+    )
+      .bind(daysAgo(60), daysAgo(31), daysAgo(1))
+      .run();
+
+    const rule = RETENTION_POLICY.find((r) => r.table === "check_summaries");
+    expect(rule).toBeDefined();
+
+    const results = await pruneExpiredRecords(env, { nowMs: NOW, policy: [rule as (typeof RETENTION_POLICY)[number]] });
+    expect(results[0]?.deleted).toBe(2);
+    const rows = await env.DB.prepare("SELECT id FROM check_summaries").all<{ id: string }>();
+    expect(rows.results.map((row) => row.id)).toEqual(["cs-recent"]);
+  });
+
   it("the policy only targets append-only/log/snapshot tables (no current-state tables)", () => {
     const tables = RETENTION_POLICY.map((r) => r.table);
     expect(tables).toContain("webhook_events");
@@ -299,7 +356,7 @@ describe("pruneExpiredRecords", () => {
     expect(rows.results.map((row) => row.id)).toEqual(["dr-recent"]);
   });
 
-  it("prunes orb_webhook_events older than its 90-day window and keeps recent rows (#9083)", async () => {
+  it("prunes orb_webhook_events older than its 14-day window and keeps recent rows (#9083)", async () => {
     const env = createTestEnv();
     await env.DB.prepare(
       `INSERT INTO orb_webhook_events (delivery_id, event_name, payload_hash, status, received_at)
@@ -311,7 +368,7 @@ describe("pruneExpiredRecords", () => {
       .run();
 
     const rule = RETENTION_POLICY.find((r) => r.table === "orb_webhook_events");
-    expect(rule).toEqual({ table: "orb_webhook_events", column: "received_at", days: 90 });
+    expect(rule).toEqual({ table: "orb_webhook_events", column: "received_at", days: 14 });
 
     const results = await pruneExpiredRecords(env, { nowMs: NOW, policy: [rule!] });
     expect(results[0]?.deleted).toBe(1);