diff --git a/src/db/retention.ts b/src/db/retention.ts index 896f53be8..761a5de5c 100644 --- a/src/db/retention.ts +++ b/src/db/retention.ts @@ -25,8 +25,10 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [ // depending on it, so a shorter window than the audit/usage-log tables above is appropriate. { table: "agent_context_snapshots", column: "created_at", days: 30 }, // One row per inbound webhook delivery (#8381 / unfinished #3896); short-lived idempotency lookups, - // not durable history — same 90d window as audit/ai_usage logs. - { table: "webhook_events", column: "received_at", days: 90 }, + // not durable history. Cut 90d -> 14d: the dedup lookups these serve are minutes-old at most, and at the + // hosted fleet's ~4.3M-rows/day write rate a 90-day window on a pure idempotency log is the single + // largest avoidable contributor to the D1 ceiling (see the block at the end of this policy). + { table: "webhook_events", column: "received_at", days: 14 }, // One row per outbound notification delivery (#8899); same append-only log shape as webhook_events. { table: "notification_deliveries", column: "created_at", days: 90 }, // #9138: one row per loopover_predict_gate/explain_gate_disposition MCP call (unbounded, contributor- @@ -54,8 +56,25 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [ // close still has its record, matching product_usage_events' 180-day window for the same reason. { table: "decision_records", column: "created_at", days: 180 }, // #9083: the central Orb App's OWN webhook delivery/dedup log (separate from webhook_events, which is the - // per-repo review app's) -- identical short-lived-idempotency shape, same 90d window. - { table: "orb_webhook_events", column: "received_at", days: 90 }, + // per-repo review app's) -- identical short-lived-idempotency shape, same 14d window as webhook_events. + { table: "orb_webhook_events", column: "received_at", days: 14 }, + // The five highest-growth tables below had NO retention rule at all and grew without bound, which is what + // actually filled the hosted D1 to its 10GB ceiling: every write then failed with + // `D1_ERROR: Exceeded maximum DB size`, including recordOrbWebhookEvent's INSERT, so /v1/orb/webhook + // returned 500 to GitHub and inbound webhook delivery stopped fleet-wide until rows were pruned by hand. + // Measured at the time of the outage: check_summaries 100,459 rows / 0.30GB of payload_json and + // pull_request_files 54,532 rows / 0.20GB were the two largest single consumers in the database. + // + // All five are re-derivable GitHub mirrors or superseded snapshots, never a contributor's evidentiary + // trail (decision_records, 180d above, is the table that carries that): a check-run summary, a PR's file + // list, a repo's totals snapshot, and the merged-PR/outcome caches are all re-fetched from GitHub on the + // next sync of the PR that needs them. 30 days keeps every window the review paths actually read + // (grounding/impact-map reads are head_sha-scoped and days-fresh at most) while bounding the growth. + { table: "check_summaries", column: "updated_at", days: 30 }, + { table: "pull_request_files", column: "updated_at", days: 30 }, + { table: "repo_github_totals_snapshots", column: "fetched_at", days: 30 }, + { table: "recent_merged_pull_requests", column: "updated_at", days: 30 }, + { table: "orb_pr_outcomes", column: "occurred_at", days: 90 }, ]; // #9083: a real, single-column, indexable primary key for the ordered-range delete below, keyed by table @@ -95,8 +114,15 @@ export type PruneResult = { table: string; column: string; cutoff: string; delet const SAFE_IDENTIFIER = /^[a-z_]+$/; const BATCH_SIZE = 1000; // Bound work per table per run so a first prune of a large backlog cannot blow the D1 statement budget; -// the daily cron drains any remainder over subsequent runs. -const MAX_DELETED_PER_TABLE = 50_000; +// the cron drains any remainder over subsequent runs. +// +// Raised 50k -> 250k because the old ceiling made retention structurally unable to keep up on the hosted +// fleet and so guaranteed the D1 ceiling would be reached eventually: at 50k/table against a measured +// ~4.3M rows written per day, a DAILY prune could delete at most ~1.25M rows/day across the whole policy — +// a permanent ~3.5x deficit that no window tightening alone can close, because the cap (not the cutoff) +// was the binding constraint. Paired with the hourly cadence in src/index.ts, the policy now drains far +// faster than the fleet writes, so a backlog converges instead of compounding. +const MAX_DELETED_PER_TABLE = 250_000; const MS_PER_DAY = 86_400_000; function retentionWhere(rule: RetentionRule): string { diff --git a/src/index.ts b/src/index.ts index 486ce9ff2..13c9e5529 100644 --- a/src/index.ts +++ b/src/index.ts @@ -317,8 +317,15 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController): if (isHourly && hour === 7 && selfHostedReviews && isRiskControlEnabled(env)) { jobs.push({ type: "risk-control-recalibrate", requestedBy: "schedule" }); } - // Prune expired log/snapshot rows once a day (03:00 UTC) per the conservative RETENTION_POLICY. - if (isHourly && hour === 3) { + // Prune expired log/snapshot rows EVERY hour per RETENTION_POLICY. Was once a day (03:00 UTC), which + // combined with the old per-table delete cap to put retention permanently behind the hosted fleet's write + // rate — the D1 database then reached its 10GB ceiling and every write failed with + // `D1_ERROR: Exceeded maximum DB size`, taking inbound GitHub webhook delivery down fleet-wide (the + // /v1/orb/webhook handler 500s when it cannot record the delivery). Hourly keeps each run's deletions + // small and steady rather than one large daily burst, and 24x the drain rate turns a growing backlog into + // a converging one. Cheap when there is nothing to do: each rule is an indexed range delete that matches + // zero rows once the table is inside its window. + if (isHourly) { jobs.push({ type: "prune-retention", requestedBy: "schedule" }); } // Repo-doc refresh sweep (#3003, part of #2993) -- once a day (09:00 UTC, distinct from prune-retention's diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index 2056d24ec..d6dfd17c1 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -609,6 +609,10 @@ describe("worker entrypoint", () => { { type: "refresh-scoring-model", requestedBy: "schedule" }, { type: "refresh-upstream-drift", requestedBy: "schedule" }, { type: "rollup-product-usage", requestedBy: "schedule", days: 7 }, + // prune-retention is on EVERY hourly tick (was 03:00 daily): at the old cadence and per-table delete + // cap, retention could not keep up with the fleet's write rate and the hosted D1 reached its maximum + // size, after which every write failed and inbound webhook delivery stopped fleet-wide. + { type: "prune-retention", requestedBy: "schedule" }, ]); }); @@ -679,6 +683,8 @@ describe("worker entrypoint", () => { { type: "refresh-scoring-model", requestedBy: "schedule" }, { type: "refresh-upstream-drift", requestedBy: "schedule" }, { type: "rollup-product-usage", requestedBy: "schedule", days: 7 }, + // Hourly, not daily — see the note on the six-hour-window test above. + { type: "prune-retention", requestedBy: "schedule" }, { type: "generate-signal-snapshots", requestedBy: "schedule" }, { type: "build-burden-forecasts", requestedBy: "schedule" }, { type: "build-contributor-evidence", requestedBy: "schedule" }, @@ -723,6 +729,7 @@ describe("worker entrypoint", () => { "refresh-scoring-model", "refresh-upstream-drift", "rollup-product-usage", + "prune-retention", "generate-signal-snapshots", "build-burden-forecasts", "build-contributor-evidence", diff --git a/test/unit/retention.test.ts b/test/unit/retention.test.ts index f2d877f0d..9b322add4 100644 --- a/test/unit/retention.test.ts +++ b/test/unit/retention.test.ts @@ -16,7 +16,14 @@ async function seed(env: Env) { await db.insert(webhookEvents).values([ { deliveryId: "wh-old-1", eventName: "push", payloadHash: "h", status: "processed", receivedAt: daysAgo(100) }, { deliveryId: "wh-old-2", eventName: "push", payloadHash: "h", status: "processed", receivedAt: daysAgo(95) }, - { deliveryId: "wh-recent", eventName: "push", payloadHash: "h", status: "processed", receivedAt: daysAgo(1) }, + // Anchored to REAL now, not the fixed NOW the `daysAgo` rows use: the pruneExpiredRecords tests below + // pass an explicit `nowMs: NOW`, but the processJob and preview-route tests do not and so evaluate the + // policy against actual wall-clock time. A `daysAgo(1)` "recent" row is ~45 days before real now, which + // survived the old 90-day webhook_events window purely by accident and stopped surviving when that + // window tightened to 14 days. Real-now keeps this row genuinely recent under BOTH clocks (it is also + // after NOW, so it is never past a NOW-based cutoff either) and keeps each test's "recent rows are + // kept" intent independent of how wide the window happens to be. + { deliveryId: "wh-recent", eventName: "push", payloadHash: "h", status: "processed", receivedAt: new Date().toISOString() }, ]); // ai_usage_events window = 90d; one old + one recent. await db.insert(aiUsageEvents).values([ @@ -121,6 +128,56 @@ describe("pruneExpiredRecords", () => { expect(rows.results.map((row) => row.id)).toEqual(["ctx-recent"]); }); + // The hosted D1 reached its 10GB ceiling because these five tables had NO retention rule at all: every + // write then failed with `D1_ERROR: Exceeded maximum DB size`, including recordOrbWebhookEvent's INSERT, + // so /v1/orb/webhook returned 500 to GitHub and inbound webhook delivery stopped fleet-wide. Pinning the + // exact column names matters as much as the windows -- pruneExpiredRecords builds ` < ?` from + // this policy, so a column that does not exist on the table makes the rule a permanent silent no-op and + // the table resumes growing without bound exactly as before. + it("covers the five previously-unbounded high-growth tables (D1 ceiling regression)", () => { + const expected = [ + { table: "check_summaries", column: "updated_at", days: 30 }, + { table: "pull_request_files", column: "updated_at", days: 30 }, + { table: "repo_github_totals_snapshots", column: "fetched_at", days: 30 }, + { table: "recent_merged_pull_requests", column: "updated_at", days: 30 }, + { table: "orb_pr_outcomes", column: "occurred_at", days: 90 }, + ]; + for (const want of expected) { + expect(RETENTION_POLICY).toContainEqual(want); + } + }); + + // Both webhook logs are short-lived idempotency lookups (minutes-old at most), cut 90d -> 14d as the + // single largest avoidable contributor to the same ceiling. + it("keeps both webhook idempotency logs on the tightened 14-day window", () => { + expect(RETENTION_POLICY).toContainEqual({ table: "webhook_events", column: "received_at", days: 14 }); + expect(RETENTION_POLICY).toContainEqual({ table: "orb_webhook_events", column: "received_at", days: 14 }); + }); + + // check_summaries was the largest single consumer in the database at the time of the outage (100,459 rows + // / 0.30GB of payload_json). Exercises the REAL policy window rather than an ad-hoc override, so dropping + // or re-widening the entry fails here. + it("prunes check_summaries past its policy window and keeps recent rows", async () => { + const env = createTestEnv(); + await env.DB.prepare( + `INSERT INTO check_summaries (id, repo_full_name, pull_number, head_sha, name, status, conclusion, payload_json, updated_at) + VALUES + ('cs-old-1', 'acme/widgets', 1, 'sha1', 'ci', 'completed', 'success', '{}', ?), + ('cs-old-2', 'acme/widgets', 2, 'sha2', 'ci', 'completed', 'failure', '{}', ?), + ('cs-recent', 'acme/widgets', 3, 'sha3', 'ci', 'completed', 'success', '{}', ?)`, + ) + .bind(daysAgo(60), daysAgo(31), daysAgo(1)) + .run(); + + const rule = RETENTION_POLICY.find((r) => r.table === "check_summaries"); + expect(rule).toBeDefined(); + + const results = await pruneExpiredRecords(env, { nowMs: NOW, policy: [rule as (typeof RETENTION_POLICY)[number]] }); + expect(results[0]?.deleted).toBe(2); + const rows = await env.DB.prepare("SELECT id FROM check_summaries").all<{ id: string }>(); + expect(rows.results.map((row) => row.id)).toEqual(["cs-recent"]); + }); + it("the policy only targets append-only/log/snapshot tables (no current-state tables)", () => { const tables = RETENTION_POLICY.map((r) => r.table); expect(tables).toContain("webhook_events"); @@ -299,7 +356,7 @@ describe("pruneExpiredRecords", () => { expect(rows.results.map((row) => row.id)).toEqual(["dr-recent"]); }); - it("prunes orb_webhook_events older than its 90-day window and keeps recent rows (#9083)", async () => { + it("prunes orb_webhook_events older than its 14-day window and keeps recent rows (#9083)", async () => { const env = createTestEnv(); await env.DB.prepare( `INSERT INTO orb_webhook_events (delivery_id, event_name, payload_hash, status, received_at) @@ -311,7 +368,7 @@ describe("pruneExpiredRecords", () => { .run(); const rule = RETENTION_POLICY.find((r) => r.table === "orb_webhook_events"); - expect(rule).toEqual({ table: "orb_webhook_events", column: "received_at", days: 90 }); + expect(rule).toEqual({ table: "orb_webhook_events", column: "received_at", days: 14 }); const results = await pruneExpiredRecords(env, { nowMs: NOW, policy: [rule!] }); expect(results[0]?.deleted).toBe(1);