From b90f7e5df75ae014f491adbcc0d1e60d1c4869b7 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:22:26 -0700 Subject: [PATCH 1/2] =?UTF-8?q?feat(orb):=20serve=20GET=20/v1/public/eval-?= =?UTF-8?q?scores=20=E2=80=94=20the=20v1=20EvalScoreRecord=20transport=20(?= =?UTF-8?q?#9266)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reshapes the already-computed outcome-confirmed rule precision into the EvalScoreRecord shape spec'd on #9215: per-record digest commitment, corpus-checksum-anchored, independently re-derivable by any consumer without trusting the transport. Same flag/cache posture as /v1/public/stats since this is a reshaping of the same data, not a new scoring surface. Extends PublicRulePrecisionRow with a raw confirmed count (previously only the rounded precision ratio was exposed) so EvalScoreRecord.score.confirmed is exact rather than reconstructed from an already-rounded value. --- apps/loopover-ui/public/openapi.json | 43 ++++++ src/api/routes.ts | 28 ++++ src/openapi/schemas.ts | 2 +- src/openapi/spec.ts | 10 ++ src/review/eval-score-records.ts | 145 ++++++++++++++++++ src/review/public-rule-precision.ts | 8 +- .../public-decision-ledger-routes.test.ts | 1 + .../public-eval-scores-route.test.ts | 102 ++++++++++++ test/unit/eval-score-records.test.ts | 136 ++++++++++++++++ test/unit/public-rule-precision.test.ts | 6 +- 10 files changed, 476 insertions(+), 5 deletions(-) create mode 100644 src/review/eval-score-records.ts create mode 100644 test/integration/public-eval-scores-route.test.ts create mode 100644 test/unit/eval-score-records.test.ts diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index bfc3bf69df..bf235246c0 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -736,11 +736,15 @@ "precision": { "type": "number", "nullable": true + }, + "confirmed": { + "type": "number" } }, "required": [ "ruleId", "decided", + "confirmed", "precision" ] } @@ -19293,6 +19297,45 @@ } ] } + }, + "/v1/public/eval-scores": { + "get": { + "summary": "Fetch EvalScoreRecords (#9215) -- the objective-eval-provider transport, digest-committed and independently re-derivable", + "parameters": [ + { + "schema": { + "type": "string" + }, + "required": false, + "name": "subject", + "in": "query" + }, + { + "schema": { + "type": "string" + }, + "required": false, + "name": "since", + "in": "query" + } + ], + "responses": { + "200": { + "description": "{ records: EvalScoreRecord[] }, optionally filtered by subject id and/or minimum issuedAt -- degrades to an empty array on an internal read error rather than a non-200 status, matching loadPublicRulePrecision's own fail-safe contract" + }, + "404": { + "description": "Public stats disabled (same flag as /v1/public/stats)" + } + }, + "security": [ + { + "LoopOverBearer": [] + }, + { + "LoopOverSessionCookie": [] + } + ] + } } }, "servers": [ diff --git a/src/api/routes.ts b/src/api/routes.ts index 65547d40f3..177d1d0f2a 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -311,6 +311,7 @@ import { backfillContributorGateHistory } from "../review/contributor-gate-histo import { isFairnessAnalyticsEnabled, resolveFairnessAnalyticsManifestOverride } from "../review/contributor-trust-profile-wire"; import { isRagEnabled } from "../review/rag-wire"; import { loadPublicDecisionRecord, verifyDecisionLedger } from "../review/decision-record"; +import { buildEvalScoreRecordsFromRulePrecision, filterEvalScoreRecords } from "../review/eval-score-records"; import { getPublicStats, isPublicStatsEnabled, resolvePublicStatsManifestOverride } from "../review/public-stats"; import { loadPublicAccuracyTrend } from "../services/public-accuracy-trend"; import { loadPublicRulePrecision } from "../review/public-rule-precision"; @@ -1303,6 +1304,30 @@ export function createApp() { return c.json({ record: published.record, recordDigest: published.recordDigest }); }); + // #9266 (epic #8534, spec #9215): the v1 transport for EvalScoreRecords -- the shape a validator (or + // anyone) fetches and independently re-derives rather than trusts. Same flag/cache/error posture as + // /v1/public/stats deliberately (this reshapes the SAME already-computed rule-precision data into a + // per-record, digest-committed form; it is not a new scoring surface). Wired for the + // outcome_confirmed_precision source only today -- a future benchmark_run source (#9265) feeds the same + // endpoint, never a second response format. + app.get("/v1/public/eval-scores", async (c) => { + const publicStatsManifestOverride = await resolvePublicStatsManifestOverride(c.env); + if (!isPublicStatsEnabled(c.env, publicStatsManifestOverride)) return c.json({ error: "not_found" }, 404); + // No try/catch: loadPublicRulePrecision is fail-safe internally (safeAll swallows every read error into + // an empty section, per its own doc comment), and buildEvalScoreRecordsFromRulePrecision/ + // filterEvalScoreRecords are pure -- there is no reachable throw path in this composition today. A future + // IO-touching source (the benchmark_run records from #9265) is exactly where real error handling belongs, + // added when that source actually exists, not as untestable defensive code here. + const precision = await loadPublicRulePrecision(c.env); + const records = await buildEvalScoreRecordsFromRulePrecision(precision, new Date().toISOString()); + const filtered = filterEvalScoreRecords(records, { + subject: c.req.query("subject"), + since: c.req.query("since"), + }); + c.header("Cache-Control", "public, max-age=60, stale-while-revalidate=300"); + return c.json({ records: filtered }); + }); + app.get("/v1/public/github/repos/:owner/:repo/stats", async (c) => { try { const stats = await fetchPublicRepoStats(c.env, c.req.param("owner"), c.req.param("repo")); @@ -6741,6 +6766,9 @@ function requiresApiToken(path: string): boolean { // #9120: this route's own doc comment always claimed "safe unauthenticated" but was missing from this exact // list, so it 401'd in prod — verified live: subnet-interface/stats/quality answered 200, this one 401'd. if (path === "/v1/public/decision-ledger/verify") return false; + // #9266: the eval-scores transport is unauthenticated by the same design as every /v1/public/* sibling + // above -- committed to a corpus checksum, independently re-derivable, nothing gated behind a token. + if (path === "/v1/public/eval-scores") return false; // #9123: the new public decision-record read route — same unauthenticated posture as its ledger-verify // sibling immediately above, added in the SAME PR so the two can never drift apart the way #9120 did. The // pull segment matches any non-slash text (not just digits): an invalid pull number is the ROUTE's 400 to diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index d222f98739..79085ab1e2 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -116,7 +116,7 @@ export const PublicStatsSchema = z * counts and the latest backtest run's corpus checksum (the independently-verifiable freeze point). */ rulePrecision: z.object({ windowDays: z.number(), - rules: z.array(z.object({ ruleId: z.string(), decided: z.number(), precision: z.number().nullable() })), + rules: z.array(z.object({ ruleId: z.string(), decided: z.number(), confirmed: z.number(), precision: z.number().nullable() })), reversals: z.object({ reopened: z.number(), reverted: z.number(), superseded: z.number() }), latestBacktestRun: z.object({ corpusChecksum: z.string(), at: z.string() }).nullable(), }), diff --git a/src/openapi/spec.ts b/src/openapi/spec.ts index 3c606361c2..b5acd25bcb 100644 --- a/src/openapi/spec.ts +++ b/src/openapi/spec.ts @@ -1016,6 +1016,16 @@ export function buildOpenApiSpec() { 404: { description: "No decision record persisted yet for this PR" }, }, }); + registry.registerPath({ + method: "get", + path: "/v1/public/eval-scores", + summary: "Fetch EvalScoreRecords (#9215) -- the objective-eval-provider transport, digest-committed and independently re-derivable", + request: { query: z.object({ subject: z.string().optional(), since: z.string().optional() }) }, + responses: { + 200: { description: "{ records: EvalScoreRecord[] }, optionally filtered by subject id and/or minimum issuedAt -- degrades to an empty array on an internal read error rather than a non-200 status, matching loadPublicRulePrecision's own fail-safe contract" }, + 404: { description: "Public stats disabled (same flag as /v1/public/stats)" }, + }, + }); registry.registerPath({ method: "post", path: "/v1/orb/ingest", diff --git a/src/review/eval-score-records.ts b/src/review/eval-score-records.ts new file mode 100644 index 0000000000..f1b4f7f7a0 --- /dev/null +++ b/src/review/eval-score-records.ts @@ -0,0 +1,145 @@ +// EvalScoreRecord (#9266, epic #8534): the v1 consumable artifact spec'd on #9215 -- the shape a validator +// (or anyone) fetches from GET /v1/public/eval-scores and independently re-derives, rather than trusting. +// This module is pure: it reshapes ALREADY-COMPUTED data (PublicRulePrecision) into the spec'd record shape. +// It adds no new scoring and no new trust -- the numbers are the same ones `/v1/public/stats` already +// publishes, just committed to a corpus checksum and made independently re-derivable per-record. +import { canonicalJson, contentDigest } from "./decision-record"; +import type { PublicRulePrecision } from "./public-rule-precision"; + +export const EVAL_SCORE_RECORD_SCHEMA_VERSION = 1 as const; + +/** Semantics version for the `outcome_confirmed_precision` work-unit kind specifically (#9215 §5) -- + * independent of this module's own package version. Bump ONLY when what the numbers MEAN changes (e.g. the + * reversal-provenance rules), never for an unrelated refactor. */ +export const OUTCOME_CONFIRMED_PRECISION_SCORING_RULE_VERSION = "outcome-confirmed-precision-v1"; + +/** Sentinel subject for `outcome_confirmed_precision` records. This work-unit kind measures ORB's OWN gate + * rule performance -- there is no external agent being scored, unlike a future `benchmark_run` record + * (#9265) whose subject is the agent under evaluation. A fixed, documented id (rather than a null/omitted + * subject) keeps every EvalScoreRecord's shape uniform regardless of work-unit kind. */ +export const ORB_GATE_SUBJECT_ID = "orb-gate"; + +export type EvalScoreRecordSubject = { kind: "agent"; id: string }; + +export type EvalScoreRecordWorkUnit = + | { kind: "outcome_confirmed_precision"; ruleId: string } + | { kind: "benchmark_run"; benchmarkId: string; snapshotRef: string }; + +export type EvalScoreRecordScore = { + decided: number; + confirmed: number; + precision: number | null; + recall: number | null; + coverage: number | null; + abstained: number; +}; + +export type EvalScoreRecordCommitments = { + corpusChecksum: string; + scoringRuleVersion: string; + windowStart: string; + windowEnd: string; + splitSeed: string | null; + heldOutFraction: number | null; +}; + +// Discriminated union, not #9215's own loosely-typed prose (`tier` + an optional `attestation`) -- the wire +// JSON is identical either way (an "attested" record still carries `attestation` as a sibling key), but this +// makes "attested without an envelope" a compile-time impossibility instead of a runtime-checked invariant. +export type EvalScoreRecordTrust = + | { tier: "attested"; attestation: { envelopeDigest: string; measurement: string; runId: string } } + | { tier: "reproducible" | "asserted" }; + +export type EvalScoreRecord = { + schemaVersion: typeof EVAL_SCORE_RECORD_SCHEMA_VERSION; + subject: EvalScoreRecordSubject; + workUnit: EvalScoreRecordWorkUnit; + score: EvalScoreRecordScore; + commitments: EvalScoreRecordCommitments; + trust: EvalScoreRecordTrust; + issuedAt: string; + recordDigest: string; +}; + +/** Everything about an EvalScoreRecord except its own digest -- the exact preimage `contentDigest` hashes, + * so `recordDigest` always commits to the record's OWN content and never to itself. */ +type EvalScoreRecordDigestInput = Omit; + +async function finalizeRecord(input: EvalScoreRecordDigestInput): Promise { + const recordDigest = await contentDigest(input); + return { ...input, recordDigest }; +} + +/** + * Build `EvalScoreRecord`s from the already-computed public rule-precision block. Returns an empty array + * when there is no persisted backtest run yet (`latestBacktestRun === null`) -- per #9215's own requirement, + * a record whose commitments cannot be independently re-derived (no corpus checksum to point at) is not + * publishable, so this deliberately emits nothing rather than a record with a placeholder commitment. + * + * `recall` and `abstained` do not apply to this work-unit kind: ORB's gate rules fire deterministically (no + * agent choosing to abstain) and this data measures precision, not a false-negative rate. `recall` is + * `null` (genuinely inapplicable, never a misleading `0`); `abstained` is `0` (there is no abstention + * concept here, so `0` is the correct value, not a masked null). PURE -- no IO, no clock (the caller + * supplies `issuedAt`). + */ +export async function buildEvalScoreRecordsFromRulePrecision(precision: PublicRulePrecision, issuedAt: string): Promise { + if (!precision.latestBacktestRun) return []; + const { corpusChecksum } = precision.latestBacktestRun; + const windowStart = new Date(Date.parse(issuedAt) - precision.windowDays * 24 * 60 * 60 * 1000).toISOString(); + + const records = await Promise.all( + precision.rules.map((row) => + finalizeRecord({ + schemaVersion: EVAL_SCORE_RECORD_SCHEMA_VERSION, + subject: { kind: "agent", id: ORB_GATE_SUBJECT_ID }, + workUnit: { kind: "outcome_confirmed_precision", ruleId: row.ruleId }, + score: { + decided: row.decided, + confirmed: row.confirmed, + precision: row.precision, + recall: null, + coverage: null, + abstained: 0, + }, + commitments: { + corpusChecksum, + scoringRuleVersion: OUTCOME_CONFIRMED_PRECISION_SCORING_RULE_VERSION, + windowStart, + windowEnd: issuedAt, + splitSeed: null, + heldOutFraction: null, + }, + trust: { tier: "reproducible" }, + issuedAt, + }), + ), + ); + return records; +} + +export type EvalScoreRecordFilter = { subject?: string; since?: string }; + +/** Filter already-built records by subject id and/or a minimum `issuedAt` -- pure, used identically by the + * route handler and by tests, so the two can never apply different filtering logic. An invalid `since` + * (fails to parse) excludes nothing, matching this system's general fail-open posture for optional query + * filters rather than erroring the whole response over one bad parameter. */ +export function filterEvalScoreRecords(records: readonly EvalScoreRecord[], filter: EvalScoreRecordFilter): EvalScoreRecord[] { + const sinceMs = filter.since ? Date.parse(filter.since) : Number.NaN; + return records.filter((record) => { + if (filter.subject && record.subject.id !== filter.subject) return false; + if (!Number.isNaN(sinceMs) && Date.parse(record.issuedAt) < sinceMs) return false; + return true; + }); +} + +/** Recompute a record's digest and compare -- the exact check a consumer runs to verify a fetched record + * without trusting the transport. Exported so both this module's own tests and a future CLI/docs example + * exercise the identical recomputation, never a parallel reimplementation. */ +export async function verifyEvalScoreRecordDigest(record: EvalScoreRecord): Promise { + const { recordDigest, ...rest } = record; + return (await contentDigest(rest)) === recordDigest; +} + +// Re-exported so callers that only import this module never need a second import from decision-record.ts +// just to canonicalize something alongside a record (e.g. logging, or a future signed-bundle wrapper). +export { canonicalJson, contentDigest }; diff --git a/src/review/public-rule-precision.ts b/src/review/public-rule-precision.ts index 0ae483c4e8..18b7d9cc7a 100644 --- a/src/review/public-rule-precision.ts +++ b/src/review/public-rule-precision.ts @@ -26,6 +26,10 @@ const HUMAN_OVERRIDE_EVENT_TYPE_PREFIX = "signal.human_override:"; export type PublicRulePrecisionRow = { ruleId: string; decided: number; + /** Raw count, always defined regardless of the sample floor -- unlike `precision`, this is never rounded + * and never null, so a consumer needing the exact count (eval-score-records.ts, #9266) never has to + * reconstruct it by inverting a value that's already been rounded to 3 decimals. */ + confirmed: number; /** confirmed / decided, rounded to 3 decimals; null below {@link PUBLIC_PRECISION_MIN_DECIDED}. */ precision: number | null; }; @@ -62,10 +66,12 @@ export async function loadPublicRulePrecision(env: Env, nowMs: number = Date.now * future query-shape change, mirroring loadOverrideDayRows' identical note. */ const reversed = row.reversed ?? 0; const decided = row.decided; + const confirmed = decided - reversed; return { ruleId: row.rule_id, decided, - precision: decided >= PUBLIC_PRECISION_MIN_DECIDED ? Math.round(((decided - reversed) / decided) * 1000) / 1000 : null, + confirmed, + precision: decided >= PUBLIC_PRECISION_MIN_DECIDED ? Math.round((confirmed / decided) * 1000) / 1000 : null, }; }) .sort((a, b) => a.ruleId.localeCompare(b.ruleId)); diff --git a/test/integration/public-decision-ledger-routes.test.ts b/test/integration/public-decision-ledger-routes.test.ts index 8ad410ed66..cfb59068cf 100644 --- a/test/integration/public-decision-ledger-routes.test.ts +++ b/test/integration/public-decision-ledger-routes.test.ts @@ -80,6 +80,7 @@ describe("public decision-ledger/decision-records routes answer WITHOUT credenti ["repos/:owner/:repo/badge.svg", "/v1/public/repos/acme/widgets/badge.svg"], ["repos/:owner/:repo/badge.json", "/v1/public/repos/acme/widgets/badge.json"], ["repos/:owner/:repo/quality", "/v1/public/repos/acme/widgets/quality"], + ["eval-scores", "/v1/public/eval-scores"], ]; it.each(exemptPublicRoutes)("%s answers without credentials (never 401)", async (_name, path) => { diff --git a/test/integration/public-eval-scores-route.test.ts b/test/integration/public-eval-scores-route.test.ts new file mode 100644 index 0000000000..adbcccfae5 --- /dev/null +++ b/test/integration/public-eval-scores-route.test.ts @@ -0,0 +1,102 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { createApp } from "../../src/api/routes"; +import { createTestEnv } from "../helpers/d1"; +import { clearPublicStatsManifestOverrideCacheForTest } from "../../src/review/public-stats"; +import { recordAuditEvent } from "../../src/db/repositories"; +import { createSignalStore } from "../../src/review/signal-tracking-wire"; +import { contentDigest } from "../../src/review/decision-record"; +import { ORB_GATE_SUBJECT_ID, type EvalScoreRecord } from "../../src/review/eval-score-records"; + +const NOW = Date.parse("2026-07-27T12:00:00.000Z"); + +async function seedConfirmedPrecisionData(env: Env): Promise { + const store = createSignalStore(env); + for (let i = 0; i < 20; i += 1) { + await store.recordHumanOverride({ + ruleId: "ai_consensus_defect", + targetKey: `acme/widgets#${i + 1}`, + verdict: i < 16 ? "confirmed" : "reversed", + occurredAt: new Date(NOW - 1000 - i).toISOString(), + }); + } + await recordAuditEvent(env, { + eventType: "calibration.logic_backtest_run", + targetKey: "rule", + outcome: "completed", + metadata: { corpusChecksum: "freeze-point-checksum", comparison: {} }, + createdAt: new Date(NOW - 60_000).toISOString(), + }); +} + +describe("GET /v1/public/eval-scores (#9266, epic #8534, spec #9215)", () => { + beforeEach(() => { + clearPublicStatsManifestOverrideCacheForTest(); + }); + + it("404s when LOOPOVER_PUBLIC_STATS is off (default) -- same flag as /v1/public/stats", async () => { + const env = createTestEnv(); + const res = await createApp().request("/v1/public/eval-scores", {}, env); + expect(res.status).toBe(404); + }); + + it("200s with an empty records array when there is no persisted backtest run yet", async () => { + const env = createTestEnv({ LOOPOVER_PUBLIC_STATS: "1" }); + const res = await createApp().request("/v1/public/eval-scores", {}, env); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ records: [] }); + }); + + it("returns records whose recordDigest is independently recomputable from the record's own content", async () => { + const env = createTestEnv({ LOOPOVER_PUBLIC_STATS: "1" }); + await seedConfirmedPrecisionData(env); + + const res = await createApp().request("/v1/public/eval-scores", {}, env); + expect(res.status).toBe(200); + const body = (await res.json()) as { records: EvalScoreRecord[] }; + expect(body.records).toHaveLength(1); + const [record] = body.records; + expect(record?.workUnit).toEqual({ kind: "outcome_confirmed_precision", ruleId: "ai_consensus_defect" }); + expect(record?.score).toEqual({ decided: 20, confirmed: 16, precision: 0.8, recall: null, coverage: null, abstained: 0 }); + expect(record?.commitments.corpusChecksum).toBe("freeze-point-checksum"); + expect(record?.subject).toEqual({ kind: "agent", id: ORB_GATE_SUBJECT_ID }); + + const { recordDigest, ...rest } = record as EvalScoreRecord; + expect(await contentDigest(rest)).toBe(recordDigest); + }); + + it("filters by ?subject=", async () => { + const env = createTestEnv({ LOOPOVER_PUBLIC_STATS: "1" }); + await seedConfirmedPrecisionData(env); + + const matching = await createApp().request(`/v1/public/eval-scores?subject=${ORB_GATE_SUBJECT_ID}`, {}, env); + expect(((await matching.json()) as { records: EvalScoreRecord[] }).records).toHaveLength(1); + + const nonMatching = await createApp().request("/v1/public/eval-scores?subject=some-other-agent", {}, env); + expect(((await nonMatching.json()) as { records: EvalScoreRecord[] }).records).toHaveLength(0); + }); + + it("filters by ?since= (an unparseable value excludes nothing, fail-open on a malformed optional filter)", async () => { + const env = createTestEnv({ LOOPOVER_PUBLIC_STATS: "1" }); + await seedConfirmedPrecisionData(env); + + const future = await createApp().request("/v1/public/eval-scores?since=2099-01-01T00:00:00.000Z", {}, env); + expect(((await future.json()) as { records: EvalScoreRecord[] }).records).toHaveLength(0); + + const malformed = await createApp().request("/v1/public/eval-scores?since=not-a-date", {}, env); + expect(((await malformed.json()) as { records: EvalScoreRecord[] }).records).toHaveLength(1); + }); + + it("sets the same Cache-Control posture as /v1/public/stats", async () => { + const env = createTestEnv({ LOOPOVER_PUBLIC_STATS: "1" }); + const res = await createApp().request("/v1/public/eval-scores", {}, env); + expect(res.headers.get("Cache-Control")).toBe("public, max-age=60, stale-while-revalidate=300"); + }); + + it("degrades to an empty records array (not a thrown error) on a broken store, mirroring loadPublicRulePrecision's own fail-safe contract", async () => { + const env = createTestEnv({ LOOPOVER_PUBLIC_STATS: "1" }); + env.DB = { prepare: () => { throw new Error("boom"); } } as never; + const res = await createApp().request("/v1/public/eval-scores", {}, env); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ records: [] }); + }); +}); diff --git a/test/unit/eval-score-records.test.ts b/test/unit/eval-score-records.test.ts new file mode 100644 index 0000000000..1b8479ddf1 --- /dev/null +++ b/test/unit/eval-score-records.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from "vitest"; +import { + buildEvalScoreRecordsFromRulePrecision, + EVAL_SCORE_RECORD_SCHEMA_VERSION, + filterEvalScoreRecords, + ORB_GATE_SUBJECT_ID, + OUTCOME_CONFIRMED_PRECISION_SCORING_RULE_VERSION, + verifyEvalScoreRecordDigest, + type EvalScoreRecord, +} from "../../src/review/eval-score-records"; +import { contentDigest } from "../../src/review/decision-record"; +import type { PublicRulePrecision } from "../../src/review/public-rule-precision"; + +const ISSUED_AT = "2026-07-27T12:00:00.000Z"; + +const PRECISION_WITH_FREEZE_POINT: PublicRulePrecision = { + windowDays: 90, + rules: [ + { ruleId: "ai_consensus_defect", decided: 25, confirmed: 20, precision: 0.8 }, + { ruleId: "sparse_rule", decided: 9, confirmed: 9, precision: null }, + ], + reversals: { reopened: 2, reverted: 1, superseded: 3 }, + latestBacktestRun: { corpusChecksum: "abc123def456", at: "2026-07-27T10:00:00.000Z" }, +}; + +describe("buildEvalScoreRecordsFromRulePrecision (#9266)", () => { + it("returns an empty array when there is no persisted backtest run to commit to", async () => { + const records = await buildEvalScoreRecordsFromRulePrecision({ ...PRECISION_WITH_FREEZE_POINT, latestBacktestRun: null }, ISSUED_AT); + expect(records).toEqual([]); + }); + + it("builds one record per rule, committed to the freeze point's corpus checksum", async () => { + const records = await buildEvalScoreRecordsFromRulePrecision(PRECISION_WITH_FREEZE_POINT, ISSUED_AT); + expect(records).toHaveLength(2); + for (const record of records) { + expect(record.commitments.corpusChecksum).toBe("abc123def456"); + expect(record.commitments.scoringRuleVersion).toBe(OUTCOME_CONFIRMED_PRECISION_SCORING_RULE_VERSION); + expect(record.commitments.windowEnd).toBe(ISSUED_AT); + expect(record.commitments.windowStart).toBe(new Date(Date.parse(ISSUED_AT) - 90 * 24 * 60 * 60 * 1000).toISOString()); + expect(record.commitments.splitSeed).toBeNull(); + expect(record.commitments.heldOutFraction).toBeNull(); + expect(record.schemaVersion).toBe(EVAL_SCORE_RECORD_SCHEMA_VERSION); + expect(record.subject).toEqual({ kind: "agent", id: ORB_GATE_SUBJECT_ID }); + expect(record.trust).toEqual({ tier: "reproducible" }); + expect(record.issuedAt).toBe(ISSUED_AT); + } + }); + + it("carries decided/confirmed/precision through verbatim, with recall null and abstained 0 (not applicable to this work-unit kind)", async () => { + const records = await buildEvalScoreRecordsFromRulePrecision(PRECISION_WITH_FREEZE_POINT, ISSUED_AT); + const withPrecision = records.find((r) => r.workUnit.kind === "outcome_confirmed_precision" && r.workUnit.ruleId === "ai_consensus_defect"); + expect(withPrecision?.score).toEqual({ decided: 25, confirmed: 20, precision: 0.8, recall: null, coverage: null, abstained: 0 }); + + const sparse = records.find((r) => r.workUnit.kind === "outcome_confirmed_precision" && r.workUnit.ruleId === "sparse_rule"); + // Below the public sample floor: precision is null (never a misleading 0), decided/confirmed still real counts. + expect(sparse?.score).toEqual({ decided: 9, confirmed: 9, precision: null, recall: null, coverage: null, abstained: 0 }); + }); + + it("each record's recordDigest is the sha256 of its own canonical content (independently recomputable)", async () => { + const records = await buildEvalScoreRecordsFromRulePrecision(PRECISION_WITH_FREEZE_POINT, ISSUED_AT); + for (const record of records) { + const { recordDigest, ...rest } = record; + expect(await contentDigest(rest)).toBe(recordDigest); + } + }); + + it("emits records sorted the same way the input rules array is ordered (no re-sort, no reordering surprise)", async () => { + const records = await buildEvalScoreRecordsFromRulePrecision(PRECISION_WITH_FREEZE_POINT, ISSUED_AT); + expect(records.map((r) => (r.workUnit.kind === "outcome_confirmed_precision" ? r.workUnit.ruleId : ""))).toEqual([ + "ai_consensus_defect", + "sparse_rule", + ]); + }); +}); + +describe("filterEvalScoreRecords", () => { + const record = (ruleId: string, issuedAt: string, subjectId = ORB_GATE_SUBJECT_ID): EvalScoreRecord => ({ + schemaVersion: EVAL_SCORE_RECORD_SCHEMA_VERSION, + subject: { kind: "agent", id: subjectId }, + workUnit: { kind: "outcome_confirmed_precision", ruleId }, + score: { decided: 10, confirmed: 8, precision: 0.8, recall: null, coverage: null, abstained: 0 }, + commitments: { + corpusChecksum: "x", + scoringRuleVersion: OUTCOME_CONFIRMED_PRECISION_SCORING_RULE_VERSION, + windowStart: "2026-01-01T00:00:00.000Z", + windowEnd: issuedAt, + splitSeed: null, + heldOutFraction: null, + }, + trust: { tier: "reproducible" }, + issuedAt, + recordDigest: "deadbeef", + }); + + it("returns everything when no filter is given", () => { + const records = [record("a", "2026-07-01T00:00:00.000Z"), record("b", "2026-07-02T00:00:00.000Z")]; + expect(filterEvalScoreRecords(records, {})).toEqual(records); + }); + + it("filters by exact subject id", () => { + const records = [record("a", "2026-07-01T00:00:00.000Z", "orb-gate"), record("b", "2026-07-01T00:00:00.000Z", "some-other-agent")]; + expect(filterEvalScoreRecords(records, { subject: "orb-gate" })).toEqual([records[0]]); + }); + + it("filters out records issued before the since timestamp", () => { + const records = [record("old", "2026-01-01T00:00:00.000Z"), record("new", "2026-07-01T00:00:00.000Z")]; + expect(filterEvalScoreRecords(records, { since: "2026-06-01T00:00:00.000Z" })).toEqual([records[1]]); + }); + + it("excludes nothing on an unparseable since value (fail-open on a malformed optional filter)", () => { + const records = [record("a", "2026-01-01T00:00:00.000Z")]; + expect(filterEvalScoreRecords(records, { since: "not-a-date" })).toEqual(records); + }); + + it("combines subject and since filters", () => { + const records = [ + record("a", "2026-07-01T00:00:00.000Z", "orb-gate"), + record("b", "2026-01-01T00:00:00.000Z", "orb-gate"), + record("c", "2026-07-01T00:00:00.000Z", "other"), + ]; + expect(filterEvalScoreRecords(records, { subject: "orb-gate", since: "2026-06-01T00:00:00.000Z" })).toEqual([records[0]]); + }); +}); + +describe("verifyEvalScoreRecordDigest", () => { + it("returns true for a record whose digest matches its own content", async () => { + const [record] = await buildEvalScoreRecordsFromRulePrecision(PRECISION_WITH_FREEZE_POINT, ISSUED_AT); + expect(await verifyEvalScoreRecordDigest(record as EvalScoreRecord)).toBe(true); + }); + + it("returns false for a record whose content was tampered with after the digest was computed", async () => { + const [record] = await buildEvalScoreRecordsFromRulePrecision(PRECISION_WITH_FREEZE_POINT, ISSUED_AT); + const tampered: EvalScoreRecord = { ...(record as EvalScoreRecord), issuedAt: "2099-01-01T00:00:00.000Z" }; + expect(await verifyEvalScoreRecordDigest(tampered)).toBe(false); + }); +}); diff --git a/test/unit/public-rule-precision.test.ts b/test/unit/public-rule-precision.test.ts index 84f552459b..fdf4e5d85d 100644 --- a/test/unit/public-rule-precision.test.ts +++ b/test/unit/public-rule-precision.test.ts @@ -35,8 +35,8 @@ describe("loadPublicRulePrecision (#8230)", () => { const block = await loadPublicRulePrecision(env, NOW); expect(block.windowDays).toBe(PUBLIC_PRECISION_WINDOW_DAYS); expect(block.rules).toEqual([ - { ruleId: "ai_consensus_defect", decided: 25, precision: 0.8 }, - { ruleId: "linked_issue_scope_mismatch", decided: 12, precision: 0.75 }, + { ruleId: "ai_consensus_defect", decided: 25, confirmed: 20, precision: 0.8 }, + { ruleId: "linked_issue_scope_mismatch", decided: 12, confirmed: 9, precision: 0.75 }, ]); }); @@ -52,7 +52,7 @@ describe("loadPublicRulePrecision (#8230)", () => { }); const block = await loadPublicRulePrecision(env, NOW); - expect(block.rules).toEqual([{ ruleId: "sparse_rule", decided: PUBLIC_PRECISION_MIN_DECIDED - 1, precision: null }]); + expect(block.rules).toEqual([{ ruleId: "sparse_rule", decided: PUBLIC_PRECISION_MIN_DECIDED - 1, confirmed: PUBLIC_PRECISION_MIN_DECIDED - 1, precision: null }]); }); it("counts all three reversal shapes over the window and surfaces the latest backtest run's corpus checksum", async () => { From 41d49ed0ade1c87fb5bee3df657b5668212c9824 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:29:50 -0700 Subject: [PATCH 2/2] fix(orb): explicit undefined in EvalScoreRecordFilter under exactOptionalPropertyTypes --- src/review/eval-score-records.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/review/eval-score-records.ts b/src/review/eval-score-records.ts index f1b4f7f7a0..64e9f9305f 100644 --- a/src/review/eval-score-records.ts +++ b/src/review/eval-score-records.ts @@ -117,7 +117,9 @@ export async function buildEvalScoreRecordsFromRulePrecision(precision: PublicRu return records; } -export type EvalScoreRecordFilter = { subject?: string; since?: string }; +// `| undefined` is explicit (not just `?:`) because the root tsconfig sets exactOptionalPropertyTypes and +// the route handler passes `c.req.query(...)`'s `string | undefined` results straight through. +export type EvalScoreRecordFilter = { subject?: string | undefined; since?: string | undefined }; /** Filter already-built records by subject id and/or a minimum `issuedAt` -- pure, used identically by the * route handler and by tests, so the two can never apply different filtering logic. An invalid `since`