feat(orb): serve GET /v1/public/eval-scores — the v1 EvalScoreRecord transport

apps/loopover-ui/public/openapi.json

@@ -736,11 +736,15 @@
                     "precision": {
                       "type": "number",
                       "nullable": true
+                    },
+                    "confirmed": {
+                      "type": "number"
                     }
                   },
                   "required": [
                     "ruleId",
                     "decided",
+                    "confirmed",
                     "precision"
                   ]
                 }
@@ -19293,6 +19297,45 @@
           }
         ]
       }
+    },
+    "/v1/public/eval-scores": {
+      "get": {
+        "summary": "Fetch EvalScoreRecords (#9215) -- the objective-eval-provider transport, digest-committed and independently re-derivable",
+        "parameters": [
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": false,
+            "name": "subject",
+            "in": "query"
+          },
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": false,
+            "name": "since",
+            "in": "query"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "{ records: EvalScoreRecord[] }, optionally filtered by subject id and/or minimum issuedAt -- degrades to an empty array on an internal read error rather than a non-200 status, matching loadPublicRulePrecision's own fail-safe contract"
+          },
+          "404": {
+            "description": "Public stats disabled (same flag as /v1/public/stats)"
+          }
+        },
+        "security": [
+          {
+            "LoopOverBearer": []
+          },
+          {
+            "LoopOverSessionCookie": []
+          }
+        ]
+      }
     }
   },
   "servers": [

src/api/routes.ts

@@ -311,6 +311,7 @@ import { backfillContributorGateHistory } from "../review/contributor-gate-histo
 import { isFairnessAnalyticsEnabled, resolveFairnessAnalyticsManifestOverride } from "../review/contributor-trust-profile-wire";
 import { isRagEnabled } from "../review/rag-wire";
 import { loadPublicDecisionRecord, verifyDecisionLedger } from "../review/decision-record";
+import { buildEvalScoreRecordsFromRulePrecision, filterEvalScoreRecords } from "../review/eval-score-records";
 import { getPublicStats, isPublicStatsEnabled, resolvePublicStatsManifestOverride } from "../review/public-stats";
 import { loadPublicAccuracyTrend } from "../services/public-accuracy-trend";
 import { loadPublicRulePrecision } from "../review/public-rule-precision";
@@ -1303,6 +1304,30 @@ export function createApp() {
     return c.json({ record: published.record, recordDigest: published.recordDigest });
   });
 
+  // #9266 (epic #8534, spec #9215): the v1 transport for EvalScoreRecords -- the shape a validator (or
+  // anyone) fetches and independently re-derives rather than trusts. Same flag/cache/error posture as
+  // /v1/public/stats deliberately (this reshapes the SAME already-computed rule-precision data into a
+  // per-record, digest-committed form; it is not a new scoring surface). Wired for the
+  // outcome_confirmed_precision source only today -- a future benchmark_run source (#9265) feeds the same
+  // endpoint, never a second response format.
+  app.get("/v1/public/eval-scores", async (c) => {
+    const publicStatsManifestOverride = await resolvePublicStatsManifestOverride(c.env);
+    if (!isPublicStatsEnabled(c.env, publicStatsManifestOverride)) return c.json({ error: "not_found" }, 404);
+    // No try/catch: loadPublicRulePrecision is fail-safe internally (safeAll swallows every read error into
+    // an empty section, per its own doc comment), and buildEvalScoreRecordsFromRulePrecision/
+    // filterEvalScoreRecords are pure -- there is no reachable throw path in this composition today. A future
+    // IO-touching source (the benchmark_run records from #9265) is exactly where real error handling belongs,
+    // added when that source actually exists, not as untestable defensive code here.
+    const precision = await loadPublicRulePrecision(c.env);
+    const records = await buildEvalScoreRecordsFromRulePrecision(precision, new Date().toISOString());
+    const filtered = filterEvalScoreRecords(records, {
+      subject: c.req.query("subject"),
+      since: c.req.query("since"),
+    });
+    c.header("Cache-Control", "public, max-age=60, stale-while-revalidate=300");
+    return c.json({ records: filtered });
+  });
+
   app.get("/v1/public/github/repos/:owner/:repo/stats", async (c) => {
     try {
       const stats = await fetchPublicRepoStats(c.env, c.req.param("owner"), c.req.param("repo"));
@@ -6741,6 +6766,9 @@ function requiresApiToken(path: string): boolean {
   // #9120: this route's own doc comment always claimed "safe unauthenticated" but was missing from this exact
   // list, so it 401'd in prod — verified live: subnet-interface/stats/quality answered 200, this one 401'd.
   if (path === "/v1/public/decision-ledger/verify") return false;
+  // #9266: the eval-scores transport is unauthenticated by the same design as every /v1/public/* sibling
+  // above -- committed to a corpus checksum, independently re-derivable, nothing gated behind a token.
+  if (path === "/v1/public/eval-scores") return false;
   // #9123: the new public decision-record read route — same unauthenticated posture as its ledger-verify
   // sibling immediately above, added in the SAME PR so the two can never drift apart the way #9120 did. The
   // pull segment matches any non-slash text (not just digits): an invalid pull number is the ROUTE's 400 to

src/openapi/schemas.ts

@@ -116,7 +116,7 @@ export const PublicStatsSchema = z
      *  counts and the latest backtest run's corpus checksum (the independently-verifiable freeze point). */
     rulePrecision: z.object({
       windowDays: z.number(),
-      rules: z.array(z.object({ ruleId: z.string(), decided: z.number(), precision: z.number().nullable() })),
+      rules: z.array(z.object({ ruleId: z.string(), decided: z.number(), confirmed: z.number(), precision: z.number().nullable() })),
       reversals: z.object({ reopened: z.number(), reverted: z.number(), superseded: z.number() }),
       latestBacktestRun: z.object({ corpusChecksum: z.string(), at: z.string() }).nullable(),
     }),

src/openapi/spec.ts

@@ -1016,6 +1016,16 @@ export function buildOpenApiSpec() {
       404: { description: "No decision record persisted yet for this PR" },
     },
   });
+  registry.registerPath({
+    method: "get",
+    path: "/v1/public/eval-scores",
+    summary: "Fetch EvalScoreRecords (#9215) -- the objective-eval-provider transport, digest-committed and independently re-derivable",
+    request: { query: z.object({ subject: z.string().optional(), since: z.string().optional() }) },
+    responses: {
+      200: { description: "{ records: EvalScoreRecord[] }, optionally filtered by subject id and/or minimum issuedAt -- degrades to an empty array on an internal read error rather than a non-200 status, matching loadPublicRulePrecision's own fail-safe contract" },
+      404: { description: "Public stats disabled (same flag as /v1/public/stats)" },
+    },
+  });
   registry.registerPath({
     method: "post",
     path: "/v1/orb/ingest",

src/review/eval-score-records.ts

@@ -0,0 +1,147 @@
+// EvalScoreRecord (#9266, epic #8534): the v1 consumable artifact spec'd on #9215 -- the shape a validator
+// (or anyone) fetches from GET /v1/public/eval-scores and independently re-derives, rather than trusting.
+// This module is pure: it reshapes ALREADY-COMPUTED data (PublicRulePrecision) into the spec'd record shape.
+// It adds no new scoring and no new trust -- the numbers are the same ones `/v1/public/stats` already
+// publishes, just committed to a corpus checksum and made independently re-derivable per-record.
+import { canonicalJson, contentDigest } from "./decision-record";
+import type { PublicRulePrecision } from "./public-rule-precision";
+
+export const EVAL_SCORE_RECORD_SCHEMA_VERSION = 1 as const;
+
+/** Semantics version for the `outcome_confirmed_precision` work-unit kind specifically (#9215 §5) --
+ *  independent of this module's own package version. Bump ONLY when what the numbers MEAN changes (e.g. the
+ *  reversal-provenance rules), never for an unrelated refactor. */
+export const OUTCOME_CONFIRMED_PRECISION_SCORING_RULE_VERSION = "outcome-confirmed-precision-v1";
+
+/** Sentinel subject for `outcome_confirmed_precision` records. This work-unit kind measures ORB's OWN gate
+ *  rule performance -- there is no external agent being scored, unlike a future `benchmark_run` record
+ *  (#9265) whose subject is the agent under evaluation. A fixed, documented id (rather than a null/omitted
+ *  subject) keeps every EvalScoreRecord's shape uniform regardless of work-unit kind. */
+export const ORB_GATE_SUBJECT_ID = "orb-gate";
+
+export type EvalScoreRecordSubject = { kind: "agent"; id: string };
+
+export type EvalScoreRecordWorkUnit =
+  | { kind: "outcome_confirmed_precision"; ruleId: string }
+  | { kind: "benchmark_run"; benchmarkId: string; snapshotRef: string };
+
+export type EvalScoreRecordScore = {
+  decided: number;
+  confirmed: number;
+  precision: number | null;
+  recall: number | null;
+  coverage: number | null;
+  abstained: number;
+};
+
+export type EvalScoreRecordCommitments = {
+  corpusChecksum: string;
+  scoringRuleVersion: string;
+  windowStart: string;
+  windowEnd: string;
+  splitSeed: string | null;
+  heldOutFraction: number | null;
+};
+
+// Discriminated union, not #9215's own loosely-typed prose (`tier` + an optional `attestation`) -- the wire
+// JSON is identical either way (an "attested" record still carries `attestation` as a sibling key), but this
+// makes "attested without an envelope" a compile-time impossibility instead of a runtime-checked invariant.
+export type EvalScoreRecordTrust =
+  | { tier: "attested"; attestation: { envelopeDigest: string; measurement: string; runId: string } }
+  | { tier: "reproducible" | "asserted" };
+
+export type EvalScoreRecord = {
+  schemaVersion: typeof EVAL_SCORE_RECORD_SCHEMA_VERSION;
+  subject: EvalScoreRecordSubject;
+  workUnit: EvalScoreRecordWorkUnit;
+  score: EvalScoreRecordScore;
+  commitments: EvalScoreRecordCommitments;
+  trust: EvalScoreRecordTrust;
+  issuedAt: string;
+  recordDigest: string;
+};
+
+/** Everything about an EvalScoreRecord except its own digest -- the exact preimage `contentDigest` hashes,
+ *  so `recordDigest` always commits to the record's OWN content and never to itself. */
+type EvalScoreRecordDigestInput = Omit<EvalScoreRecord, "recordDigest">;
+
+async function finalizeRecord(input: EvalScoreRecordDigestInput): Promise<EvalScoreRecord> {
+  const recordDigest = await contentDigest(input);
+  return { ...input, recordDigest };
+}
+
+/**
+ * Build `EvalScoreRecord`s from the already-computed public rule-precision block. Returns an empty array
+ * when there is no persisted backtest run yet (`latestBacktestRun === null`) -- per #9215's own requirement,
+ * a record whose commitments cannot be independently re-derived (no corpus checksum to point at) is not
+ * publishable, so this deliberately emits nothing rather than a record with a placeholder commitment.
+ *
+ * `recall` and `abstained` do not apply to this work-unit kind: ORB's gate rules fire deterministically (no
+ * agent choosing to abstain) and this data measures precision, not a false-negative rate. `recall` is
+ * `null` (genuinely inapplicable, never a misleading `0`); `abstained` is `0` (there is no abstention
+ * concept here, so `0` is the correct value, not a masked null). PURE -- no IO, no clock (the caller
+ * supplies `issuedAt`).
+ */
+export async function buildEvalScoreRecordsFromRulePrecision(precision: PublicRulePrecision, issuedAt: string): Promise<EvalScoreRecord[]> {
+  if (!precision.latestBacktestRun) return [];
+  const { corpusChecksum } = precision.latestBacktestRun;
+  const windowStart = new Date(Date.parse(issuedAt) - precision.windowDays * 24 * 60 * 60 * 1000).toISOString();
+
+  const records = await Promise.all(
+    precision.rules.map((row) =>
+      finalizeRecord({
+        schemaVersion: EVAL_SCORE_RECORD_SCHEMA_VERSION,
+        subject: { kind: "agent", id: ORB_GATE_SUBJECT_ID },
+        workUnit: { kind: "outcome_confirmed_precision", ruleId: row.ruleId },
+        score: {
+          decided: row.decided,
+          confirmed: row.confirmed,
+          precision: row.precision,
+          recall: null,
+          coverage: null,
+          abstained: 0,
+        },
+        commitments: {
+          corpusChecksum,
+          scoringRuleVersion: OUTCOME_CONFIRMED_PRECISION_SCORING_RULE_VERSION,
+          windowStart,
+          windowEnd: issuedAt,
+          splitSeed: null,
+          heldOutFraction: null,
+        },
+        trust: { tier: "reproducible" },
+        issuedAt,
+      }),
+    ),
+  );
+  return records;
+}
+
+// `| undefined` is explicit (not just `?:`) because the root tsconfig sets exactOptionalPropertyTypes and
+// the route handler passes `c.req.query(...)`'s `string | undefined` results straight through.
+export type EvalScoreRecordFilter = { subject?: string | undefined; since?: string | undefined };
+
+/** Filter already-built records by subject id and/or a minimum `issuedAt` -- pure, used identically by the
+ *  route handler and by tests, so the two can never apply different filtering logic. An invalid `since`
+ *  (fails to parse) excludes nothing, matching this system's general fail-open posture for optional query
+ *  filters rather than erroring the whole response over one bad parameter. */
+export function filterEvalScoreRecords(records: readonly EvalScoreRecord[], filter: EvalScoreRecordFilter): EvalScoreRecord[] {
+  const sinceMs = filter.since ? Date.parse(filter.since) : Number.NaN;
+  return records.filter((record) => {
+    if (filter.subject && record.subject.id !== filter.subject) return false;
+    if (!Number.isNaN(sinceMs) && Date.parse(record.issuedAt) < sinceMs) return false;
+    return true;
+  });
+}
+
+/** Recompute a record's digest and compare -- the exact check a consumer runs to verify a fetched record
+ *  without trusting the transport. Exported so both this module's own tests and a future CLI/docs example
+ *  exercise the identical recomputation, never a parallel reimplementation. */
+export async function verifyEvalScoreRecordDigest(record: EvalScoreRecord): Promise<boolean> {
+  const { recordDigest, ...rest } = record;
+  return (await contentDigest(rest)) === recordDigest;
+}
+
+// Re-exported so callers that only import this module never need a second import from decision-record.ts
+// just to canonicalize something alongside a record (e.g. logging, or a future signed-bundle wrapper).
+export { canonicalJson, contentDigest };

src/review/public-rule-precision.ts

@@ -26,6 +26,10 @@ const HUMAN_OVERRIDE_EVENT_TYPE_PREFIX = "signal.human_override:";
 export type PublicRulePrecisionRow = {
   ruleId: string;
   decided: number;
+  /** Raw count, always defined regardless of the sample floor -- unlike `precision`, this is never rounded
+   *  and never null, so a consumer needing the exact count (eval-score-records.ts, #9266) never has to
+   *  reconstruct it by inverting a value that's already been rounded to 3 decimals. */
+  confirmed: number;
   /** confirmed / decided, rounded to 3 decimals; null below {@link PUBLIC_PRECISION_MIN_DECIDED}. */
   precision: number | null;
 };
@@ -62,10 +66,12 @@ export async function loadPublicRulePrecision(env: Env, nowMs: number = Date.now
        * future query-shape change, mirroring loadOverrideDayRows' identical note. */
       const reversed = row.reversed ?? 0;
       const decided = row.decided;
+      const confirmed = decided - reversed;
       return {
         ruleId: row.rule_id,
         decided,
-        precision: decided >= PUBLIC_PRECISION_MIN_DECIDED ? Math.round(((decided - reversed) / decided) * 1000) / 1000 : null,
+        confirmed,
+        precision: decided >= PUBLIC_PRECISION_MIN_DECIDED ? Math.round((confirmed / decided) * 1000) / 1000 : null,
       };
     })
     .sort((a, b) => a.ruleId.localeCompare(b.ruleId));

test/integration/public-decision-ledger-routes.test.ts

@@ -80,6 +80,7 @@ describe("public decision-ledger/decision-records routes answer WITHOUT credenti
     ["repos/:owner/:repo/badge.svg", "/v1/public/repos/acme/widgets/badge.svg"],
     ["repos/:owner/:repo/badge.json", "/v1/public/repos/acme/widgets/badge.json"],
     ["repos/:owner/:repo/quality", "/v1/public/repos/acme/widgets/quality"],
+    ["eval-scores", "/v1/public/eval-scores"],
   ];
 
   it.each(exemptPublicRoutes)("%s answers without credentials (never 401)", async (_name, path) => {