From 402d4c823ed811b523528eee6bb623d5b4f5e56e Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:17:32 -0700 Subject: [PATCH] feat(ledger): git-commit anchoring backend, cross-verified via GH Archive (#9273) Fifth sub-issue of #9267. Secondary, complementary to Rekor: appends one JSONL line per anchor to a public repo via the GitHub Contents API and the existing makeInstallationOctokit chokepoint -- never a direct fetch to the GitHub API, matching every other GitHub write in this engine. Alone this is weaker than Rekor (GitHub is a trusted third party, force-push rewrites it). It becomes genuinely strong combined with mirrors nobody at LoopOver controls: GH Archive's hourly PushEvent export and Software Heritage's on-demand archival. A rewrite becomes independently detectable by checking an archive this repo doesn't control -- documented as a runnable cross-mirror verification procedure in the module's own header. Commits the identical canonicalized payload + signature Rekor anchors, so the two backends commit to the same fact, never a reshaped copy. Read-modify- write via the file's own sha as a compare-and-swap guard; never throws past the caller (missing repo, auth failure, rate limit, a raced sha all record status:'failed' via #9271's persistence, same posture as the Rekor backend). Also fixes a real bug found while building this: ledger-anchor.ts's re-export statement only carried sha256Hex, dropping canonicalJson entirely -- any consumer importing it (this backend needs it to commit the same payload Rekor anchors) got undefined. Applied at the actual origin (#9270/PR #9392) and rebased through the whole stack, not patched over downstream. --- src/review/ledger-anchor-git.ts | 116 ++++++++++++++++++++++ src/review/ledger-anchor.ts | 1 - test/unit/ledger-anchor-git.test.ts | 146 ++++++++++++++++++++++++++++ 3 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 src/review/ledger-anchor-git.ts create mode 100644 test/unit/ledger-anchor-git.test.ts diff --git a/src/review/ledger-anchor-git.ts b/src/review/ledger-anchor-git.ts new file mode 100644 index 0000000000..458cff2f38 --- /dev/null +++ b/src/review/ledger-anchor-git.ts @@ -0,0 +1,116 @@ +// Git-commit anchoring backend (#9273, epic #9267). Secondary, complementary to Rekor (#9272): a commit +// appended to `anchors.jsonl` in a public repo, via the GitHub Contents API and the SAME installation-token +// chokepoint (makeInstallationOctokit) every other GitHub write in this engine goes through -- never a +// direct fetch to the GitHub API. +// +// Alone this is weaker than Rekor: GitHub is a trusted third party, and `git push --force` rewrites it. It +// becomes genuinely strong combined with mirrors nobody at LoopOver controls -- GH Archive's hourly +// `PushEvent` export and Software Heritage's on-demand "Save Code Now" archival -- which is why this backend +// exists as a SECOND, independent anchor for the same checkpoint rather than a replacement for Rekor. +// +// Cross-mirror verification, for a skeptic who does not want to trust this repo's own git history alone: +// 1. `git clone` the anchors repo and `git log --oneline -- anchors.jsonl` to find the commit for a seq. +// 2. Cross-check that push actually happened when claimed, from an archive LoopOver does not control: +// curl -s "https://data.gharchive.org/YYYY-MM-DD-HH.json.gz" | gunzip \ +// | jq 'select(.type=="PushEvent" and .repo.name=="/")' +// A commit present in the anchors repo but ABSENT from that hour's GH Archive export (once the day +// finishes being written) is exactly the signal a rewrite would leave behind. +import { githubErrorStatus } from "../github/app"; +import { canonicalJson, type SignedLedgerAnchor } from "./ledger-anchor"; +import { recordLedgerAnchorAttempt } from "./ledger-anchor-persistence"; + +/** Minimal shape this module needs from an authenticated Octokit -- injectable so tests exercise this + * module's OWN append/error logic against a scripted response, never a real GitHub Octokit instance or + * network call. The caller (the scheduling job, #9274) constructs the real one via + * `makeInstallationOctokit` + `withInstallationTokenRetry`, matching every other GitHub write in this repo; + * installation-token resolution is deliberately NOT this module's concern. */ +export type GitHubContentsRequester = { + request: (route: string, params: Record) => Promise<{ data: unknown }>; +}; + +export type LedgerAnchorGitTarget = { owner: string; repo: string; branch: string; path: string }; + +function decodeBase64(base64: string): string { + const binary = atob(base64.replace(/\s+/g, "")); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index); + return new TextDecoder().decode(bytes); +} + +function encodeBase64(text: string): string { + const bytes = new TextEncoder().encode(text); + let binary = ""; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary); +} + +/** One JSONL line: the same canonicalized payload and signature Rekor anchors, so the two backends commit to + * the identical fact -- never a reshaped or lossy copy. */ +export function buildAnchorLogLine(signed: SignedLedgerAnchor): string { + return `${canonicalJson({ payload: signed.payload, signature: signed.signature, keyId: signed.keyId })}\n`; +} + +/** + * Append one anchor to the JSONL file, via the Contents API's read-modify-write with the file's own `sha` as + * a compare-and-swap guard (GitHub 409s a stale-sha PUT, which reaches the caller as a normal thrown error -- + * a genuine concurrent-writer race, distinct from every other failure mode this function already handles). + * Never throws past the caller: any error -- missing repo, auth failure, rate limit, a raced sha -- records a + * `status: 'failed'` row via #9271's persistence, matching the Rekor backend's identical posture. + */ +export async function submitToGitAnchor(env: Env, signed: SignedLedgerAnchor, octokit: GitHubContentsRequester, target: LedgerAnchorGitTarget): Promise { + const { owner, repo, branch, path } = target; + try { + let existingSha: string | undefined; + let existingContent = ""; + try { + const response = await octokit.request("GET /repos/{owner}/{repo}/contents/{path}", { owner, repo, path, ref: branch }); + const data = response.data as { content?: string; sha?: string }; + if (typeof data.content === "string") existingContent = decodeBase64(data.content); + existingSha = data.sha; + } catch (error) { + if (githubErrorStatus(error) !== 404) throw error; + // 404 = first anchor ever committed to this file; start from empty, no sha to compare-and-swap against. + } + + const updatedContent = existingContent + buildAnchorLogLine(signed); + const response = await octokit.request("PUT /repos/{owner}/{repo}/contents/{path}", { + owner, + repo, + path, + branch, + message: `chore(anchor): decision ledger seq ${signed.payload.seq}`, + content: encodeBase64(updatedContent), + ...(existingSha !== undefined && { sha: existingSha }), + }); + const commitSha = (response.data as { commit?: { sha?: string } }).commit?.sha; + if (typeof commitSha !== "string") { + await recordLedgerAnchorAttempt(env, { + payload: signed.payload, + signature: signed.signature, + keyId: signed.keyId, + backend: "git", + status: "failed", + error: "GitHub Contents API response did not include a commit sha", + }); + return; + } + await recordLedgerAnchorAttempt(env, { + payload: signed.payload, + signature: signed.signature, + keyId: signed.keyId, + backend: "git", + status: "ok", + backendRef: { owner, repo, branch, path, sha: commitSha }, + proofR2Key: null, + }); + } catch (error) { + await recordLedgerAnchorAttempt(env, { + payload: signed.payload, + signature: signed.signature, + keyId: signed.keyId, + backend: "git", + status: "failed", + error, + }); + } +} diff --git a/src/review/ledger-anchor.ts b/src/review/ledger-anchor.ts index c57961107a..ac12df6c13 100644 --- a/src/review/ledger-anchor.ts +++ b/src/review/ledger-anchor.ts @@ -210,7 +210,6 @@ export function anchorKeyById(keys: readonly AnchorPublicKey[], keyId: string): return keys.find((key) => key.keyId === keyId) ?? null; } -/** Digest helper re-exported so an anchor consumer never needs a second import just to hash a payload. */ /** Digest helpers re-exported so an anchor consumer (e.g. the git-commit backend, #9273, which commits the * same canonicalized payload Rekor anchors) never needs a second import from decision-record.ts just to * canonicalize or hash something alongside a signed anchor. */ diff --git a/test/unit/ledger-anchor-git.test.ts b/test/unit/ledger-anchor-git.test.ts new file mode 100644 index 0000000000..78cff33c50 --- /dev/null +++ b/test/unit/ledger-anchor-git.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it, vi } from "vitest"; +import { createTestEnv } from "../helpers/d1"; +import { buildAnchorLogLine, submitToGitAnchor, type GitHubContentsRequester } from "../../src/review/ledger-anchor-git"; +import { buildLedgerAnchorPayload, canonicalJson, type SignedLedgerAnchor } from "../../src/review/ledger-anchor"; +import { loadPublicLedgerAnchors } from "../../src/review/ledger-anchor-persistence"; + +// #9273 (epic #9267). octokit is ALWAYS injected -- never a real GitHub write. The property under test is +// this module's own append/error/persistence logic, exercised against a scripted GitHubContentsRequester. + +const TARGET = { owner: "acme", repo: "loopover-anchors", branch: "main", path: "anchors.jsonl" }; + +function makeSignedAnchor(seq = 1): SignedLedgerAnchor { + return { payload: buildLedgerAnchorPayload({ seq, rowHash: "a".repeat(64), totalCount: seq }, "2026-07-27T12:00:00.000Z"), signature: "c2ln", keyId: "key1" }; +} + +function decodeBase64(base64: string): string { + return Buffer.from(base64, "base64").toString("utf8"); +} + +describe("buildAnchorLogLine (#9273)", () => { + it("commits the SAME canonicalized payload and signature as the Rekor backend anchors -- the two never diverge", () => { + const signed = makeSignedAnchor(); + const line = buildAnchorLogLine(signed); + expect(line.endsWith("\n")).toBe(true); + expect(JSON.parse(line)).toEqual({ payload: JSON.parse(canonicalJson(signed.payload)), signature: signed.signature, keyId: signed.keyId }); + }); +}); + +describe("submitToGitAnchor (#9273)", () => { + it("creates the file on first anchor ever (no prior sha to compare-and-swap against)", async () => { + const env = createTestEnv(); + const signed = makeSignedAnchor(1); + const request = vi.fn(async (route: string, params: Record) => { + if (route.startsWith("GET")) { + const error = new Error("Not Found") as Error & { status: number }; + error.status = 404; + throw error; + } + expect(params["sha"]).toBeUndefined(); // no sha on a brand-new file + expect(decodeBase64(params["content"] as string)).toBe(buildAnchorLogLine(signed)); + return { data: { commit: { sha: "deadbeef1" } } }; + }); + + await submitToGitAnchor(env, signed, { request } as GitHubContentsRequester, TARGET); + + const { anchors } = await loadPublicLedgerAnchors(env); + expect(anchors[0]).toMatchObject({ + seq: 1, + backend: "git", + status: "ok", + backendRef: { owner: "acme", repo: "loopover-anchors", branch: "main", path: "anchors.jsonl", sha: "deadbeef1" }, + }); + }); + + it("APPENDS to existing content rather than overwriting it, using the existing sha as a compare-and-swap guard", async () => { + const env = createTestEnv(); + const signed = makeSignedAnchor(2); + const priorLine = '{"prior":"entry"}\n'; + const request = vi.fn(async (route: string, params: Record) => { + if (route.startsWith("GET")) return { data: { content: Buffer.from(priorLine).toString("base64"), sha: "old-sha" } }; + expect(params["sha"]).toBe("old-sha"); + expect(decodeBase64(params["content"] as string)).toBe(priorLine + buildAnchorLogLine(signed)); + return { data: { commit: { sha: "newsha2" } } }; + }); + + await submitToGitAnchor(env, signed, { request } as GitHubContentsRequester, TARGET); + + const { anchors } = await loadPublicLedgerAnchors(env); + expect(anchors[0]?.backendRef).toMatchObject({ sha: "newsha2" }); + }); + + it("treats a GET response with a sha but no inline content (GitHub omits it for files >1MB) as empty existing content, not a crash", async () => { + const env = createTestEnv(); + const signed = makeSignedAnchor(9); + const request = vi.fn(async (route: string, params: Record) => { + if (route.startsWith("GET")) return { data: { sha: "large-file-sha" } }; // no `content` field + expect(params["sha"]).toBe("large-file-sha"); // still used for compare-and-swap + expect(decodeBase64(params["content"] as string)).toBe(buildAnchorLogLine(signed)); // not prefixed with garbage + return { data: { commit: { sha: "afterlarge" } } }; + }); + + await submitToGitAnchor(env, signed, { request } as GitHubContentsRequester, TARGET); + const { anchors } = await loadPublicLedgerAnchors(env); + expect(anchors[0]).toMatchObject({ status: "ok", backendRef: { sha: "afterlarge" } }); + }); + + it("records status:'failed' (with the real error, not a thrown one) on a rate-limit / auth error", async () => { + const env = createTestEnv(); + const signed = makeSignedAnchor(3); + const request = vi.fn(async (route: string) => { + if (route.startsWith("GET")) { + const error = new Error("Not Found") as Error & { status: number }; + error.status = 404; + throw error; + } + const error = new Error("API rate limit exceeded") as Error & { status: number }; + error.status = 403; + throw error; + }); + + await expect(submitToGitAnchor(env, signed, { request } as GitHubContentsRequester, TARGET)).resolves.toBeUndefined(); + + const { anchors } = await loadPublicLedgerAnchors(env); + expect(anchors[0]).toMatchObject({ backend: "git", status: "failed", error: "API rate limit exceeded" }); + }); + + it("records status:'failed' when a non-404 GET error occurs (never conflated with 'file does not exist yet')", async () => { + const env = createTestEnv(); + const signed = makeSignedAnchor(4); + const request = vi.fn(async () => { + const error = new Error("Server Error") as Error & { status: number }; + error.status = 500; + throw error; + }); + + await submitToGitAnchor(env, signed, { request } as GitHubContentsRequester, TARGET); + const { anchors } = await loadPublicLedgerAnchors(env); + expect(anchors[0]).toMatchObject({ status: "failed", error: "Server Error" }); + }); + + it("records status:'failed' if the PUT response is missing a commit sha", async () => { + const env = createTestEnv(); + const signed = makeSignedAnchor(5); + const request = vi.fn(async (route: string) => { + if (route.startsWith("GET")) { + const error = new Error("Not Found") as Error & { status: number }; + error.status = 404; + throw error; + } + return { data: {} }; // no commit.sha + }); + + await submitToGitAnchor(env, signed, { request } as GitHubContentsRequester, TARGET); + const { anchors } = await loadPublicLedgerAnchors(env); + expect(anchors[0]).toMatchObject({ status: "failed", error: "GitHub Contents API response did not include a commit sha" }); + }); + + it("never throws past the caller, whatever the failure", async () => { + const env = createTestEnv(); + const signed = makeSignedAnchor(6); + const request = vi.fn(async () => { + throw new Error("anything"); + }); + await expect(submitToGitAnchor(env, signed, { request } as GitHubContentsRequester, TARGET)).resolves.toBeUndefined(); + }); +});