From 2e3559f4c7b895637fd4709eba872b4e76566d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Wed, 10 Jun 2026 22:21:46 +1200 Subject: [PATCH 01/20] docs: Canopy backup integration spec Spec for integrating pgro as a Canopy-mediated restore consumer (drop static AWS keys, fetch short-lived restore creds, report restore- verification = signal 3). Authored alongside the Canopy plan. --- docs/canopy-backup-integration.md | 469 ++++++++++++++++++++++++++++++ 1 file changed, 469 insertions(+) create mode 100644 docs/canopy-backup-integration.md diff --git a/docs/canopy-backup-integration.md b/docs/canopy-backup-integration.md new file mode 100644 index 0000000..b8c6c22 --- /dev/null +++ b/docs/canopy-backup-integration.md @@ -0,0 +1,469 @@ +# Canopy backup-credentials integration + +Implementation spec for the pgro side of the Canopy "backup-credentials" +system. Canopy is BES's backup control plane: it issues short-lived, +per-group S3 credentials for kopia repositories, owns repo maintenance and +retention, and tracks backup health through three signals — *backed up* (1), +*persisted* (2), and **restorable** (3). pgro is the producer of signal 3. + +The authoritative design lives in the canopy repo +(`docs/plans/backup-credentials.md`, section "External restore consumers + +restore-verification (PGRO)"). This document is the pgro-side contract and +implementation plan; it does not re-decide canopy-side shape. + +> **Stage.** This is an *additive, later-stage* change in the overall +> rollout. It depends on canopy having already shipped: per-bucket roles, +> the `restore` session-policy path, repo-password ownership, the +> issues/events alerting, and a first-party (non-device) auth surface. +> Until those exist on the canopy side there is nothing for pgro to call. +> Build pgro's side behind the CRD opt-in below so today's static-Secret +> path keeps working unchanged during migration. + +## What changes, in one paragraph + +Today a `PostgresPhysicalReplica` carries a `kopiaSecretRef` pointing at a +hand-created `kopia-credentials` Secret that holds **long-lived** AWS access +keys plus the kopia repository password (`src/kopia.rs`, +`REQUIRED_KEYS`). The operator copies those values into env vars +(`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `KOPIA_PASSWORD`, …) on every +snapshot-list and restore Job (`src/controllers/replica/resources.rs`, +`src/controllers/restore/builders.rs`). This is exactly the long-lived-creds +pattern the canopy system exists to eliminate. The integration replaces that +static Secret with a **canopy-mediated** flow: the replica references a +**canopy group** instead of a Secret; the operator fetches **short-lived, +read-only restore credentials + target + repo password** from canopy, +refreshing them as needed; and after each restore the operator **reports the +outcome back to canopy** (signal 3, restore-verification). pgro is a +**restore-only** consumer — it never writes to or deletes from the bucket. + +## Why canopy-mediated (not pgro assuming a role directly) + +The relationship is **bidirectional**: creds flow out (canopy → pgro) and +restore-verification reports flow in (pgro → canopy). Routing both over one +first-party channel keeps it a single relationship rather than two disjoint +mechanisms (an AWS role pgro assumes + a separate reporting path). pgro is +trusted first-party infra but is **not** a fleet device and is **not** a +member of the group it reads, so canopy gates pgro behind an +operator-authorized, audited **external-restore grant** ("consumer C may +read group X, read-only") — a deliberate, controlled cousin of the +cross-group restore that is banned for devices. pgro does not implement the +grant; it just authenticates as itself and names the group. + +--- + +## Part 1 — Creds out (Canopy → pgro) + +### 1.1 New canopy client module: `src/canopy.rs` + +A new module owning all HTTP interaction with canopy. Follows the existing +`src/notifications.rs` / `reqwest`-based conventions (the crate already +depends on `reqwest` with `json`, and `jiff` with `serde`). + +Types (wire shapes mirror canopy's device endpoints, which pgro reuses via +the external-restore grant — same response bodies, different auth): + +```rust +/// Short-lived read-only credentials for a group's kopia repo, as returned +/// by canopy's restore-credentials endpoint. Mirrors the AWS SDK +/// `credential_process` output (canopy's `POST /backup-credentials` with +/// purpose=restore), plus the kopia repo password and S3 target that +/// canopy's `GET /backup-target` carries. +#[derive(Debug, Clone, Deserialize)] +pub struct CanopyRestoreCreds { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: String, + /// RFC3339; the operator must refresh before this. + pub expiration: jiff::Timestamp, + pub bucket: String, + pub prefix: String, // normally empty (repo at bucket root) + pub region: String, + pub repository_password: String, +} +``` + +Functions: + +- `async fn fetch_restore_creds(&self, group: &str) -> Result` + — authenticates as pgro (see Part 3), requests `purpose=restore` for + `group`, returns the combined creds+target+password. Canopy maps the group + to its per-bucket role, assumes it cross-account with the read-only restore + **session policy**, and returns prod-account creds for that bucket. pgro + treats a `403`/`409` (grant absent or group unconfigured) as a clear, + surfaced error, not a transient retry. +- `async fn report_restore(&self, report: &RestoreReport) -> Result<()>` + — see Part 2. + +The client holds the canopy base URL + auth material (Tailscale or OIDC; Part +3) read from env/config at startup, on the `Context` (`src/context.rs`) +alongside the existing `http_client`. + +### 1.2 CRD change: reference a canopy group, not a Secret + +Add an alternative to `kopiaSecretRef` on `PostgresPhysicalReplicaSpec` +(`src/types/replica.rs`). The static-Secret path stays valid for migration +and for non-canopy repos (e.g. the minio integration-test repo), so make the +two mutually-exclusive and keep `kopia_secret_ref` optional: + +```rust +/// Reference to a Secret containing kopia repository credentials. +/// Mutually exclusive with `canopyBackup`. One of the two is required. +#[serde(default, skip_serializing_if = "Option::is_none")] +pub kopia_secret_ref: Option, + +/// Fetch short-lived read-only restore credentials from Canopy instead of +/// a static Secret. Mutually exclusive with `kopiaSecretRef`. +#[serde(default, skip_serializing_if = "Option::is_none")] +pub canopy_backup: Option, +``` + +```rust +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct CanopyBackupRef { + /// The Canopy server-group id (UUID) whose backups this replica restores. + pub group: String, + // Canopy base URL + auth come from operator-level config, not per-replica, + // so every replica trusts the same canopy. (Open question 3.) +} +``` + +Making `kopia_secret_ref` `Option` is a CRD change touching every existing +construction site (`controllers/replica/{resources,tests}.rs`, +`controllers/restore/tests.rs`, `controllers/replica/schema_migration.rs`). +Validate "exactly one of `kopiaSecretRef` / `canopyBackup`" in the replica +reconcile (`src/controllers/replica.rs`) and surface a `Warning` event + +`Failed` phase on violation, mirroring how an invalid kopia secret is handled +today (`Error::InvalidKopiaSecret`). + +Per `AGENTS.md`: a CRD spec change **must** update the README CRD tables and +regenerate CRDs (`cargo run --bin gen-crds > crds.yaml`). + +### 1.3 Where the creds are consumed — the materialisation strategy + +Jobs consume creds today as **env vars sourced from the Secret** +(`env_from_secret(... kopia_secret ...)`), then run +`kopia repository connect s3 --access-key=... --secret-access-key=... +--password=...`. The scripts read `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` +/ `KOPIA_PASSWORD` from the environment. + +Short-lived creds add three things the static path didn't need: a +**session token**, an **expiry**, and a **refresh** obligation. Two viable +strategies; **this spec recommends (A)** for snapshot-list and short restores, +with (B) as the escape hatch for long restores (see the 1-hour-cap open +question, 3.x): + +**(A) Operator materialises a transient per-Job Secret.** Just before +building a Job, the operator calls `fetch_restore_creds(group)` and writes a +short-lived k8s Secret (owned by the replica, so it's GC'd) containing the +same keys the Job already expects, plus `sessionToken`. The Job's connect +script gains `AWS_SESSION_TOKEN` and `--session-token` (kopia/AWS SDK pick up +`AWS_SESSION_TOKEN` automatically for the SDK path; for the kopia S3 backend, +pass the session token via the `AWS_SESSION_TOKEN` env which kopia forwards — +**verify**, see open questions). This keeps the Job-builder code path almost +identical to today: `env_from_secret(...)` against a per-Job Secret name +instead of the user-provided one. The `validate_kopia_secret` / +`KopiaCredentials` plumbing in `src/kopia.rs` is reused verbatim with a +`sessionToken`/`AWS_SESSION_TOKEN` field added. + +**(B) Job self-refreshes via a sidecar/credential helper.** The Job itself +calls canopy (it's the AWS `credential_process` model bestool uses on the +device side). Heavier — the Job needs the canopy auth material and network +path — but it is the only thing that survives a restore that runs **longer +than the credential lifetime** (chained AssumeRole sessions are capped at +**1 hour** regardless of role config; see the canopy plan "AWS quirks"). + +Restore Jobs can legitimately exceed an hour (large data dir, slow WAL +replay — the operator already has a configurable +`DEPLOYMENT_READY_TIMEOUT_SECS` defaulting to 30 min and raisable). The +kopia *restore* phase reads from S3 throughout; if its creds expire +mid-restore the restore fails. **This is the single most important design +question for pgro** and is called out in open questions 3.x. snapshot-list is +short and safely fits (A). + +### 1.4 Drop the static keys + +Once a replica uses `canopyBackup`: +- No `accessKeyId` / `secretAccessKey` / `repositoryPassword` ever live in a + user-managed Secret. The transient per-Job Secret (strategy A) holds creds + that expire within the hour and is owned/GC'd by the operator. +- `REQUIRED_KEYS` in `src/kopia.rs` still governs the legacy static path; + the canopy path builds `KopiaCredentials` from the canopy response instead + of `validate_kopia_secret`. + +--- + +## Part 2 — Reports in (pgro → Canopy): Signal 3, restore-verification + +A successful pgro restore *proves the backup is restorable* — the strongest +backup-health signal there is, stronger than signal 2's "a snapshot exists in +the repo". pgro reports each restore outcome to canopy, which records it in +its `backup_restore_checks` table and reconciles it against +`backup_repo_snapshots` / `backup_runs`. A failed or stale restorability +check becomes a high-severity **group-level** alert on the canopy side (the +server-independent incident path, like poisoning detection) — pgro does not +manage alerting, it only reports. + +### 2.1 Report shape (`RestoreReport`) + +pgro already has all of this in `PostgresPhysicalRestoreStatus` and the +`NotificationPayload` machinery (`src/notifications.rs`); this is a new +notification *target*, not new data collection. + +```rust +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RestoreReport { + /// The canopy group this replica restores from (CanopyBackupRef.group). + pub group: String, + /// The kopia snapshot id that was restored — the cross-reference key + /// canopy joins against backup_repo_snapshots / backup_runs. + pub snapshot_id: String, + /// "success" | "failure". + pub outcome: RestoreOutcome, + /// Populated on failure (restore-job failure, deployment-never-ready, + /// detected postgres version mismatch, etc.). + pub error: Option, + /// Replica health after restore: did postgres come up and pass the + /// operator's readiness gate. + pub replica_healthy: bool, + /// Detected postgres major version (status.postgresVersion). + pub postgres_version: Option, + /// When the restore completed (status.restoredAt / activatedAt). + pub observed_at: jiff::Timestamp, +} +``` + +`snapshot_id` is load-bearing: it is the join key canopy uses to close the +loop *backed up → persisted → restorable*. It maps directly to +`PostgresPhysicalRestoreStatus`-tracked snapshot and to the +`canopy@:` kopia source / `canopy-run` tag model canopy uses +for attribution. + +### 2.2 When pgro reports + +The restore controller (`src/controllers/restore.rs`) already drives a +restore through `Pending → Restoring → Ready → Switching → Active → Failed` +and counts `consecutiveRestoreFailures`. Hook the report at the terminal +transitions: +- **success**: when a restore reaches `Active` (or `Ready` and the deployment + passes the readiness gate) — `outcome=success`, `replica_healthy=true`. +- **failure**: when a restore reaches `Failed`, or the deployment fails to + become ready within `DEPLOYMENT_READY_TIMEOUT_SECS` — `outcome=failure`, + `error` populated. + +Report **at most once per restore** and tolerate canopy being unreachable: +record "reported" in the restore status (a new optional +`status.canopyReportedAt: Time`, mirroring `NotificationStatus`), retry on +the next reconcile if it hasn't been sent. Reporting failures must **never** +fail the restore itself (same posture as notifications today — +`NotificationStatus.last_error` is recorded, the restore proceeds). + +This reuses the existing notification pattern almost exactly. Consider +modelling the canopy report as a built-in notification target rather than a +bespoke path, but keep it operator-configured (Part 3 auth), not per-replica +webhook config — the canopy relationship is first-party and trusted, not an +arbitrary webhook. + +--- + +## Part 3 — Cross-cluster first-party auth (DESIGN LATER) + +**pgro runs in a different k8s cluster from canopy.** There is no shared +cluster, no shared ServiceAccount, and no shared Secret to lean on. canopy's +device endpoints use mTLS device identity; pgro is not a device. So pgro +needs a **first-party, non-device** auth path to canopy. Two options to weigh +(the canopy plan defers the decision; pgro should be built so the auth +mechanism is swappable — isolate it behind one trait in `src/canopy.rs`): + +**Option 1 — Tailscale (available today, least new machinery).** Both pgro +and canopy are on the tailnet. pgro reaches canopy's **private** API over +Tailscale, reusing existing Tailscale identity/gating (canopy already gates +its admin/private surface on Tailscale). Lowest lift: no new federation +infrastructure; the auth "material" is just being on the tailnet, and canopy +authorises by tailnet identity. The external-restore grant keys off that +identity. This is the likely **stage-1** choice. + +**Option 2 — OIDC trust / workload-identity-federation (more general, +later).** Canopy becomes an **OIDC relying party** federating pgro's cluster +OIDC issuer (pgro's ServiceAccount projected token). More extensible: the +*same* mechanism generalises to **GitHub Actions** and other internal +automation authenticating to canopy — a first-party automation auth surface +beyond pgro. Likely the better long-term direction. Not designed here; pgro's +side would mount a projected SA token and present it as a bearer assertion, +which the swappable auth trait can accommodate later without touching the +creds/report logic. + +Either way, **one channel carries both directions** (creds out, reports in). +pgro should not build two transports. + +### 3.1 Operator config surface + +Canopy base URL + auth selector come from operator-level config, not the CRD +(open question 3.x: per-replica vs operator-global canopy — recommend +operator-global so every replica trusts one canopy). Plumb via env on the +Deployment (`operator.yaml`) and/or the existing +`postgres-restore-operator-config` ConfigMap that `src/bin/operator.rs` +already watches (`read_config`): +- `CANOPY_BASE_URL` +- `CANOPY_AUTH_MODE` = `tailscale` | `oidc` +- (oidc) projected-token path / audience; (tailscale) nothing beyond tailnet + membership. + +--- + +## IaC / deployment changes (`operator.yaml` + ops repo) + +- **No more user-managed `kopia-credentials` Secret** for canopy-backed + replicas. Operators stop hand-creating it; the static-Secret path remains + only for legacy/non-canopy repos. +- **Network path to canopy.** Tailscale: ensure the pgro pod can reach + canopy's private endpoint over the tailnet (sidecar or host tailnet, per + the cluster's existing Tailscale setup — ops repo). OIDC: expose pgro's + cluster OIDC issuer to canopy and project a SA token into the operator pod + (`operator.yaml` Deployment `volumes` + `serviceAccountToken` projection). +- **RBAC delta in `operator.yaml`.** Strategy (A) writes transient per-Job + Secrets — the ClusterRole already grants full Secret verbs + (`get/list/watch/create/update/patch/delete`), so no RBAC change is needed + for that. No new cluster permissions for the canopy path itself (it's + outbound HTTP). +- The auth-plumbing (OIDC trust config or Tailscale exposure) and the + external-restore grant live in the **canopy + ops** repos, not pgro + (canopy plan: "ops: the auth plumbing … and PGRO's read-only access path"). + +--- + +## Interfaces this component exposes / consumes + +**Consumes (from canopy):** +- `fetch_restore_creds(group)` → `CanopyRestoreCreds` (read-only restore + creds + S3 target + repo password). Canopy-side: restore session policy on + the per-bucket role; gated by the external-restore grant for pgro's + first-party identity. +- First-party auth surface (Tailscale identity or OIDC trust). + +**Provides (to canopy):** +- `RestoreReport` via `report_restore(...)` → canopy `backup_restore_checks` + / signal-3 detection. Cross-referenced by `snapshot_id`. +- The `PostgresPhysicalReplica.spec.canopyBackup.group` field is the operator- + visible contract that binds a replica to a canopy group. + +**Provides (to pgro operators):** +- CRD: `canopyBackup: { group }` as the alternative to `kopiaSecretRef`. +- README CRD-table updates documenting it. + +--- + +## Testing approach (per `AGENTS.md`) + +`AGENTS.md`: add tests with features/fixes; integration tests **don't run +locally** — flag them to the user for CI and **add a matrix entry in +`.github/workflows/integration.yml`** for any new test file. Never write docs +except CRD-table updates. Run `cargo clippy` + `cargo fmt` before committing; +conventional commits; always work on a branch. + +**Unit tests (run locally, alongside the code):** +- `src/canopy.rs`: deserialize a canopy creds response into + `CanopyRestoreCreds`; serialize a `RestoreReport`; expiry parsing; the + "should-refresh" decision (creds near expiry). Mirror the table-driven + style in `src/kopia.rs` / `src/types/replica.rs` tests. +- CRD validation: "exactly one of kopiaSecretRef/canopyBackup" — both-set and + neither-set are errors; round-trip `CanopyBackupRef` through serde + (mirroring `schema_migration_phase_roundtrips_*`). +- Job builder: with `canopyBackup`, the connect script/env carry + `AWS_SESSION_TOKEN` and reference the transient Secret name; legacy path + unchanged. Extend the existing `kopia_connect_args_*` tests for the + session-token arg. +- Report gating: `canopyReportedAt` prevents double-reporting; a report + failure does not fail the restore (assert status transitions unaffected). + +**Integration tests (CI-only; new matrix entry required):** +- A `test-canopy-restore` namespace exercising the canopy path against a + **stub canopy** (a small in-cluster HTTP service returning fixed + creds for the minio repo + accepting reports), so the existing + `tests/fixtures/minio.yaml` + `setup-kopia-repo.yaml` repo can be reused. + Assert: replica reaches `Ready`/`Active` using fetched creds; a + `RestoreReport` with the right `snapshot_id`/`outcome` is POSTed to the + stub. Add the matrix entry in `.github/workflows/integration.yml` and tell + the user it only runs in CI. +- Negative: grant-absent (stub returns 403) → replica surfaces a clear + `Failed`/Warning, doesn't crash-loop. + +--- + +## Open questions / decisions to make + +1. **Credential lifetime vs. long restores (the critical one).** Chained + AssumeRole sessions are capped at **1 hour**. Large restores (slow WAL + replay) can exceed that. The kopia restore reads from S3 for the whole + run, so expired creds mid-restore fail it. Options: + (a) Job self-refreshes (strategy B) — Job calls canopy as a + `credential_process`-style helper; needs canopy network path + auth inside + the Job. (b) Operator watches expiry and rotates the transient Secret + + the kopia env mid-Job — fragile, kopia would need to re-read creds. + (c) canopy issues longer-lived creds to first-party consumers via a + *non-chained* direct web-identity assume (the canopy plan already uses + direct, non-chained cross-account web-identity for **maintenance Jobs**, + sidestepping the 1-hour cap — decision 13). **Recommend pushing canopy to + make (c) available to the pgro external-restore grant**, since pgro is + first-party like the maintenance Jobs. Decide before building the Job + credential path. + +2. **kopia S3 backend + session token.** Confirm kopia's S3 backend honours + `AWS_SESSION_TOKEN` (temporary creds) — the canopy plan assumes the AWS + SDK `credential_process` path on devices, but pgro drives kopia via CLI + flags (`--access-key` / `--secret-access-key`). There is no + `--session-token` flag in the current `kopia_connect_args`; verify whether + kopia reads `AWS_SESSION_TOKEN` from the env (likely yes, via the AWS SDK + it embeds) or whether a different invocation is needed. This gates + strategy (A). + +3. **Per-replica vs operator-global canopy config.** Recommend + operator-global (`CANOPY_BASE_URL` + auth on the Deployment/ConfigMap), so + `CanopyBackupRef` carries only `group`. Revisit if pgro ever needs to talk + to more than one canopy. + +4. **Tailscale vs OIDC for stage 1.** Tailscale is available now and is the + lowest-lift stage-1 path; OIDC is the more general long-term direction + (also unlocks GitHub Actions → canopy). Isolate behind one auth trait so + the choice is swappable. Decision owned jointly with canopy + ops. + +5. **Report transport coupling.** Should the canopy report reuse the existing + notification subsystem (a new built-in target) or be a dedicated path on + the restore controller? Recommend a dedicated, operator-configured path — + the canopy relationship is first-party/trusted and bidirectional, unlike + arbitrary user webhooks — but the `NotificationStatus`/retry shape is the + right model to copy. + +6. **Snapshot-list / availability checks.** snapshot-list Jobs also need repo + creds. They're short and fit strategy (A) cleanly. Confirm canopy is happy + to serve `purpose=restore` creds for the periodic snapshot-list cadence + (it is read-only; no objection expected), and that the read traffic volume + is acceptable. + +7. **Migration / coexistence.** During rollout a replica may switch from + `kopiaSecretRef` to `canopyBackup`. Confirm a clean switchover (the next + reconcile picks up the canopy path; the old Secret can be deleted by the + operator once no replica references it) and that mixed fleets work. + +8. **Region/endpoint for non-AWS repos.** The legacy path supports + `endpoint` + `disableTls` (minio, S3-compatible). Canopy's `backup-target` + serves `bucket/prefix/region` for real S3; if any canopy-backed repo is + ever non-AWS, `CanopyRestoreCreds` would need optional `endpoint`/ + `disableTls` too. Out of scope unless canopy supports non-AWS targets. + +--- + +## Backup types addendum + +Per the Canopy plan's "Backup types": PGRO consumes a **specific type**. + +- PGRO restores the **`tamanu-postgres`** type (the same type bestool + produces). The `PostgresPhysicalReplica` CRD references the canopy group + **and type**. +- The external-restore grant is per `(consumer, group, type)`: + "PGRO may read group X's `tamanu-postgres`, read-only". +- Signal-3 restore-verification is **per type** — PGRO reports which + `(group, type)` snapshot it restored and the outcome; a failed/stale + restorability check for that `(group, type)` is the group-level alert. +- PGRO filters the repo by the `canopy-type` tag to find its snapshots. From 464a28f5165b924195a744e6de9a2c7b379ab142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 16 Jun 2026 15:50:25 +1200 Subject: [PATCH 02/20] docs: canopy implementation status (restore creds chained; longer-lived still owed; raise_group_event signature) The canopy-side restore surface exists as device-shaped chained (1h) creds (PR #224); the longer-lived/non-chained restore-cred decision PGRO needs is still owed. raise_group_event signature for signal-3 is now concrete (PR #225). --- docs/canopy-backup-integration.md | 791 ++++++++++++++++++++++-------- docs/canopy-handoff.md | 483 ++++++++++++++++++ 2 files changed, 1064 insertions(+), 210 deletions(-) create mode 100644 docs/canopy-handoff.md diff --git a/docs/canopy-backup-integration.md b/docs/canopy-backup-integration.md index b8c6c22..941a4d5 100644 --- a/docs/canopy-backup-integration.md +++ b/docs/canopy-backup-integration.md @@ -14,7 +14,7 @@ implementation plan; it does not re-decide canopy-side shape. > **Stage.** This is an *additive, later-stage* change in the overall > rollout. It depends on canopy having already shipped: per-bucket roles, > the `restore` session-policy path, repo-password ownership, the -> issues/events alerting, and a first-party (non-device) auth surface. +> issues/events alerting, and a new non-server-bound device role. > Until those exist on the canopy side there is nothing for pgro to call. > Build pgro's side behind the CRD opt-in below so today's static-Secret > path keeps working unchanged during migration. @@ -23,18 +23,24 @@ implementation plan; it does not re-decide canopy-side shape. Today a `PostgresPhysicalReplica` carries a `kopiaSecretRef` pointing at a hand-created `kopia-credentials` Secret that holds **long-lived** AWS access -keys plus the kopia repository password (`src/kopia.rs`, -`REQUIRED_KEYS`). The operator copies those values into env vars -(`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `KOPIA_PASSWORD`, …) on every -snapshot-list and restore Job (`src/controllers/replica/resources.rs`, -`src/controllers/restore/builders.rs`). This is exactly the long-lived-creds -pattern the canopy system exists to eliminate. The integration replaces that -static Secret with a **canopy-mediated** flow: the replica references a -**canopy group** instead of a Secret; the operator fetches **short-lived, -read-only restore credentials + target + repo password** from canopy, -refreshing them as needed; and after each restore the operator **reports the -outcome back to canopy** (signal 3, restore-verification). pgro is a -**restore-only** consumer — it never writes to or deletes from the bucket. +keys plus the kopia repository password (`src/kopia.rs`, `REQUIRED_KEYS`). The +operator copies those values into env vars and CLI flags +(`--access-key=`/`--secret-access-key=`/`--password=`) on every snapshot-list +and restore Job (`src/controllers/replica/resources.rs`, +`src/controllers/restore/builders.rs`, `src/kopia.rs::kopia_connect_args`). +This is exactly the long-lived-creds pattern the canopy system exists to +eliminate. The integration replaces that static Secret with a +**canopy-mediated, proxy-mediated** flow: the replica references a **canopy +group** instead of a Secret; alongside each kopia Job runs a +**bestool-kopia loopback SigV4 re-signing proxy** (the same S3P proxy +canopy's own maintenance Jobs and bestool's device backups/restores use) +that fetches short-lived restore credentials + the S3 target + repo password +from canopy, refreshes them transparently between requests, and re-signs +each kopia S3 request with the live creds; kopia itself talks only to +`127.0.0.1` with dummy keys, never touching real AWS material. After each +restore the operator **reports the outcome back to canopy** (signal 3, +restore-verification). pgro is a **restore-only** consumer — it never +writes to or deletes from the bucket. ## Why canopy-mediated (not pgro assuming a role directly) @@ -53,50 +59,59 @@ grant; it just authenticates as itself and names the group. ## Part 1 — Creds out (Canopy → pgro) -### 1.1 New canopy client module: `src/canopy.rs` - -A new module owning all HTTP interaction with canopy. Follows the existing -`src/notifications.rs` / `reqwest`-based conventions (the crate already -depends on `reqwest` with `json`, and `jiff` with `serde`). - -Types (wire shapes mirror canopy's device endpoints, which pgro reuses via -the external-restore grant — same response bodies, different auth): - -```rust -/// Short-lived read-only credentials for a group's kopia repo, as returned -/// by canopy's restore-credentials endpoint. Mirrors the AWS SDK -/// `credential_process` output (canopy's `POST /backup-credentials` with -/// purpose=restore), plus the kopia repo password and S3 target that -/// canopy's `GET /backup-target` carries. -#[derive(Debug, Clone, Deserialize)] -pub struct CanopyRestoreCreds { - pub access_key_id: String, - pub secret_access_key: String, - pub session_token: String, - /// RFC3339; the operator must refresh before this. - pub expiration: jiff::Timestamp, - pub bucket: String, - pub prefix: String, // normally empty (repo at bucket root) - pub region: String, - pub repository_password: String, -} -``` - -Functions: - -- `async fn fetch_restore_creds(&self, group: &str) -> Result` - — authenticates as pgro (see Part 3), requests `purpose=restore` for - `group`, returns the combined creds+target+password. Canopy maps the group - to its per-bucket role, assumes it cross-account with the read-only restore - **session policy**, and returns prod-account creds for that bucket. pgro - treats a `403`/`409` (grant absent or group unconfigured) as a clear, - surfaced error, not a transient retry. -- `async fn report_restore(&self, report: &RestoreReport) -> Result<()>` - — see Part 2. - -The client holds the canopy base URL + auth material (Tailscale or OIDC; Part -3) read from env/config at startup, on the `Context` (`src/context.rs`) -alongside the existing `http_client`. +### 1.1 Canopy client: depend on `bestool-canopy` + +The bestool repo already publishes a `bestool-canopy` crate (`crates.io`, +currently `0.4.0`) that owns the canopy HTTP wire types and the +`CanopyClient`. pgro takes a normal `cargo` dependency on it rather than +re-implementing. The same crate is also used by the proxy's credential +provider (`bestool-kopia`, see §1.3), so depending on it is on the path +either way. + +Wire types pgro consumes verbatim from `bestool-canopy::backup`: + +- `BackupCredentials` — the `credential_process`-shaped response from `POST + /backup-credentials`: `Version, AccessKeyId, SecretAccessKey, SessionToken, + Expiration`. PascalCase on the wire; the crate already handles that. +- `BackupTarget` — `{ storage, bucket, prefix, region, repo_password }` from + `GET /backup-target`. +- `BackupReport` — the `POST /backup-report` body. **pgro does not use this + type for signal-3** (see §2 for why a separate endpoint is needed); it is + named here only because it is the de-facto shape of a run report and pgro's + `RestoreReport` will resemble it for the fields they share. +- `Purpose::Restore` — the enum variant pgro passes to `backup_credentials`. + +Client functions pgro calls on the published `CanopyClient`: + +- `client.backup_credentials(base_url, backup_type, Purpose::Restore)` — + authenticates pgro (see Part 3), returns `BackupCredentials`. Canopy maps + the consumer + group + type to its per-bucket role, assumes it + cross-account with the read-only restore **session policy**, and returns + prod-account creds for that bucket. A `403`/`409` (grant absent or group + unconfigured) is a surfaced error, not a transient retry. +- `client.backup_target(base_url)` — returns `BackupTarget` (or `Dormant`). +- The credential provider in the proxy (§1.3) drives both; pgro's operator + code does not call them directly. + +**Caveats forcing canopy-side changes before pgro can use the published +crate as-is:** + +- `CanopyClient` today is device-mTLS-only (constructor takes a + `device_key_pem`) — which **is fine for pgro**, because canopy already + has non-server-bound device roles (`releaser-device`, `admin-device`) + alongside `server-device`. pgro becomes a device of a new role (working + title `backup-restore`); the existing `CanopyClient` constructor works + unchanged. See Part 3 for the auth design. +- `BackupCredentialsRequest` is `{ type, purpose }` — no `group` field. The + external-restore path needs canopy to either add a `group` field or + expose a sibling endpoint that takes one (a `backup-restore`-role device + has no implicit group, unlike `server-device`). Canopy-owed wire change, + not a pgro one. + +The base URL + device key/cert come from operator-level config (see §3.1), +held on the `Context` (`src/context.rs`) alongside the existing +`http_client`. A small `src/canopy.rs` wires the `CanopyClient` into the +operator's context and holds the report path (§2). ### 1.2 CRD change: reference a canopy group, not a Secret @@ -139,57 +154,87 @@ today (`Error::InvalidKopiaSecret`). Per `AGENTS.md`: a CRD spec change **must** update the README CRD tables and regenerate CRDs (`cargo run --bin gen-crds > crds.yaml`). -### 1.3 Where the creds are consumed — the materialisation strategy - -Jobs consume creds today as **env vars sourced from the Secret** -(`env_from_secret(... kopia_secret ...)`), then run -`kopia repository connect s3 --access-key=... --secret-access-key=... ---password=...`. The scripts read `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` -/ `KOPIA_PASSWORD` from the environment. - -Short-lived creds add three things the static path didn't need: a -**session token**, an **expiry**, and a **refresh** obligation. Two viable -strategies; **this spec recommends (A)** for snapshot-list and short restores, -with (B) as the escape hatch for long restores (see the 1-hour-cap open -question, 3.x): - -**(A) Operator materialises a transient per-Job Secret.** Just before -building a Job, the operator calls `fetch_restore_creds(group)` and writes a -short-lived k8s Secret (owned by the replica, so it's GC'd) containing the -same keys the Job already expects, plus `sessionToken`. The Job's connect -script gains `AWS_SESSION_TOKEN` and `--session-token` (kopia/AWS SDK pick up -`AWS_SESSION_TOKEN` automatically for the SDK path; for the kopia S3 backend, -pass the session token via the `AWS_SESSION_TOKEN` env which kopia forwards — -**verify**, see open questions). This keeps the Job-builder code path almost -identical to today: `env_from_secret(...)` against a per-Job Secret name -instead of the user-provided one. The `validate_kopia_secret` / -`KopiaCredentials` plumbing in `src/kopia.rs` is reused verbatim with a -`sessionToken`/`AWS_SESSION_TOKEN` field added. - -**(B) Job self-refreshes via a sidecar/credential helper.** The Job itself -calls canopy (it's the AWS `credential_process` model bestool uses on the -device side). Heavier — the Job needs the canopy auth material and network -path — but it is the only thing that survives a restore that runs **longer -than the credential lifetime** (chained AssumeRole sessions are capped at -**1 hour** regardless of role config; see the canopy plan "AWS quirks"). - -Restore Jobs can legitimately exceed an hour (large data dir, slow WAL -replay — the operator already has a configurable -`DEPLOYMENT_READY_TIMEOUT_SECS` defaulting to 30 min and raisable). The -kopia *restore* phase reads from S3 throughout; if its creds expire -mid-restore the restore fails. **This is the single most important design -question for pgro** and is called out in open questions 3.x. snapshot-list is -short and safely fits (A). +### 1.3 Where the creds are consumed — the proxy-sidecar architecture + +The bestool + canopy family has already solved this and pgro joins the +solution rather than picking a new one. Every kopia caller in the family — +bestool device backups/restores, canopy's maintenance/inspection/s3-metrics +Jobs — runs kopia through a small loopback **SigV4 re-signing proxy** (the +"S3P" spec at `bestool/.workhorse/specs/canopy/s3-sigv4-proxy.md`, +implementation in `bestool/crates/kopia/src/proxy.rs`). The shape: + +- kopia is invoked with `--endpoint=127.0.0.1:`, `--disable-tls`, and + meaningless **dummy** access/secret keys + (`bestool_kopia::PROXY_DUMMY_ACCESS_KEY` / `PROXY_DUMMY_SECRET_KEY`). It + carries no real AWS material at all. +- A small in-process HTTP proxy bound to that loopback port discards kopia's + dummy signature, re-signs each request with the **current** STS creds, and + forwards over TLS to the real S3 host. Streaming bodies (chunked SigV4 + uploads) are re-signed chunk by chunk. +- The proxy's `CredentialProvider` (a trait in `bestool_kopia::proxy`) + refreshes creds out-of-band as they near expiry. bestool's + `CanopyCredentialProvider` + (`bestool/crates/bestool/src/actions/canopy/backup/provider.rs`) hits + canopy's `POST /backup-credentials` directly. pgro's provider plugs the + same trait but hits the **operator's in-cluster credential broker** + (Part 3.1.2) rather than canopy directly — because only the operator + pod is a canopy device. Same ~2-minute refresh-margin shape. +- The kopia repo password (from canopy's `GET /backup-target`) is + fetched once by the operator before Job-spawn and passed into the Job + pod via env / argv. The proxy sidecar passes it to kopia via + `--password=` on `connect`. + +Under this model the credential lifetime is invisible to kopia: a refresh +between two requests changes only the signing key the next one uses. A long +restore is bounded by **canopy reachability**, not by any single credential +lifetime — so the 1-hour STS cap is a non-issue (Open Q 1 collapses), and +kopia's `AWS_SESSION_TOKEN` handling is irrelevant because kopia never sees +real AWS creds (Open Q 2 collapses). + +**Where the proxy runs.** A **sidecar container in each kopia Job** — +mirroring bestool's one-proxy-per-op model. The operator templates the Job +spec to have two containers in one Pod: + +- the kopia container, pointed at `127.0.0.1:` with `--disable-tls`, + the dummy keys, `--bucket`/`--prefix`/`--region` from the canopy target, + and `--password=` set to the canopy repo password; +- a pgro-owned tiny sidecar binary linking `bestool-kopia` (for + `proxy::spawn`) and a pgro-implemented `CredentialProvider` that calls + the **operator's in-cluster credential-broker endpoint** (Part 3.1.2) + rather than canopy directly. The sidecar process binds the loopback + port, runs the proxy until the kopia container exits, and exits + itself. Built into a pgro-published image; both bestool crates are + ordinary cargo deps. + +Loopback-only is a security invariant of S3P (§Security in the spec): the +proxy binds a loopback literal and is never exposed off-host. Sharing a Pod +puts the kopia container and the sidecar in one network namespace, which +satisfies that invariant. A k8s `Service`-fronted proxy would not, and is +ruled out. + +The connect args helper (`src/kopia.rs::kopia_connect_args`) gains a +"canopy" variant emitting `--endpoint=127.0.0.1:`, `--disable-tls`, +the dummy keys, and the canopy bucket/region/prefix — the legacy +`KopiaCredentials` (static keys, optional minio endpoint) stays +unmodified for the non-canopy path. + +Snapshot-list Jobs use the same sidecar shape; the proxy is just as cheap +to spawn for a short op. Confirms Open Q 6. ### 1.4 Drop the static keys Once a replica uses `canopyBackup`: - No `accessKeyId` / `secretAccessKey` / `repositoryPassword` ever live in a - user-managed Secret. The transient per-Job Secret (strategy A) holds creds - that expire within the hour and is owned/GC'd by the operator. + user-managed Secret. The Job container running kopia **never sees real AWS + credentials** — only the loopback proxy endpoint and meaningless dummy + keys. The real creds live in the sidecar's process memory, refreshed + transparently, and are never written to disk or to a k8s object. +- The repo password is fetched per-Job from canopy's `GET /backup-target` + and passed to kopia via `--password=` on `connect`; it is also held only + in the sidecar's process memory (and kopia's argv for the lifetime of the + Job — same posture as bestool). - `REQUIRED_KEYS` in `src/kopia.rs` still governs the legacy static path; - the canopy path builds `KopiaCredentials` from the canopy response instead - of `validate_kopia_secret`. + the canopy path skips `validate_kopia_secret` entirely. --- @@ -204,6 +249,65 @@ check becomes a high-severity **group-level** alert on the canopy side (the server-independent incident path, like poisoning detection) — pgro does not manage alerting, it only reports. +### 2.0 Why not reuse `POST /backup-report` + +Canopy already serves `POST /backup-report` and it already accepts +`{ purpose: "restore", outcome, snapshot_id, error, run_id, ... }` — for +**devices**. The shape looks superficially close to what pgro wants, so it +is worth being explicit about why pgro needs a **new** ingest endpoint +rather than reusing `/backup-report`: + +1. **Identity model is upside-down.** The handler resolves `device_id`, + `server_id`, and `group_id` from the **authenticated mTLS context** + (`canopy/crates/public-server/src/backup.rs:495`), not the request body. + pgro is none of those — it has no `device_id`, no `server_id`, no + implicit group. Wiring a `group_id` into the body would break the + security invariant that an authenticated device cannot report a run as + some *other* group. +2. **The schema is device-shaped.** `backup_runs` has + `device_id UUID NOT NULL REFERENCES devices(id)` and + `group_id NOT NULL REFERENCES server_groups(id)`. pgro reports have no + device; the FK can't be satisfied. The only honest options are pollute + the table with sentinel devices, drop the FK, or add a separate table — + which is exactly the `backup_restore_checks` the canopy plan names. +3. **Two different "restore" meanings collide on `purpose`.** A device with + `purpose=restore` (`bestool canopy restore`, used for clone / DR-test on + the same fleet) writes its own outcome into `backup_runs`. That is NOT a + signal-3 verification — it is a normal device-side restore and should + not raise a group-level "the backup isn't restorable" incident. So + `purpose=restore` alone is not a sufficient discriminator. +4. **Alerting paths diverge.** A `/backup-report` failure feeds **per-server + staleness** (signal 1, server-scoped). Signal 3 must feed **group-scoped** + `raise_group_event(ref = "restore-verification", severity = Error)` + bypassing per-server `is_monitored`. Reusing the endpoint means branching + inside the handler on actor type — at which point you have already forked + it. +5. **Side-effects don't match.** The handler clears `BackupRequest` on every + report (`backup.rs:534`) so the heartbeat stops re-emitting "back up now" + for that server. Irrelevant and possibly harmful for a pgro report; + another conditional. +6. **Payload shape is wrong for signal 3.** `ReportArgs` carries + `bytes_uploaded` and `s3_*_bytes` (pgro's proxy will emit the same + tallies, fine) but lacks `replica_healthy`, postgres major version, and + `observed_at` — the very state that makes signal 3 stronger than signal + 2. Squeezing those into `error: Option` loses structure for the + load-bearing fields. +7. **`run_id` semantics don't transfer.** For devices, `run_id` is the same + UUID across `/backup-credentials` (issuance audit) and `/backup-report`, + minted at run start, duplicate → 409. For pgro the meaningful identity + is the **snapshot being verified**, not the verifier's run UUID; a + pgro-minted run UUID has no cross-table linkage to + `backup_credential_issuances` (pgro's issuances aren't even in that + table — different consumer type). + +By the time canopy has added `group_id` to the body, relaxed (or split off) +the FKs, branched the handler on actor type, routed failures to a different +alerting path, and gated the `BackupRequest::clear` side-effect, the handler +has effectively forked. Cleaner for canopy to expose a sibling endpoint +(working title `POST /restore-verification`, canopy-side naming theirs) +writing to a new `backup_restore_checks` table and routing failures through +`raise_group_event`. The body shape pgro proposes is in §2.1. + ### 2.1 Report shape (`RestoreReport`) pgro already has all of this in `PostgresPhysicalRestoreStatus` and the @@ -267,48 +371,181 @@ arbitrary webhook. --- -## Part 3 — Cross-cluster first-party auth (DESIGN LATER) - -**pgro runs in a different k8s cluster from canopy.** There is no shared -cluster, no shared ServiceAccount, and no shared Secret to lean on. canopy's -device endpoints use mTLS device identity; pgro is not a device. So pgro -needs a **first-party, non-device** auth path to canopy. Two options to weigh -(the canopy plan defers the decision; pgro should be built so the auth -mechanism is swappable — isolate it behind one trait in `src/canopy.rs`): - -**Option 1 — Tailscale (available today, least new machinery).** Both pgro -and canopy are on the tailnet. pgro reaches canopy's **private** API over -Tailscale, reusing existing Tailscale identity/gating (canopy already gates -its admin/private surface on Tailscale). Lowest lift: no new federation -infrastructure; the auth "material" is just being on the tailnet, and canopy -authorises by tailnet identity. The external-restore grant keys off that -identity. This is the likely **stage-1** choice. - -**Option 2 — OIDC trust / workload-identity-federation (more general, -later).** Canopy becomes an **OIDC relying party** federating pgro's cluster -OIDC issuer (pgro's ServiceAccount projected token). More extensible: the -*same* mechanism generalises to **GitHub Actions** and other internal -automation authenticating to canopy — a first-party automation auth surface -beyond pgro. Likely the better long-term direction. Not designed here; pgro's -side would mount a projected SA token and present it as a bearer assertion, -which the swappable auth trait can accommodate later without touching the -creds/report logic. - -Either way, **one channel carries both directions** (creds out, reports in). -pgro should not build two transports. - -### 3.1 Operator config surface - -Canopy base URL + auth selector come from operator-level config, not the CRD -(open question 3.x: per-replica vs operator-global canopy — recommend -operator-global so every replica trusts one canopy). Plumb via env on the -Deployment (`operator.yaml`) and/or the existing -`postgres-restore-operator-config` ConfigMap that `src/bin/operator.rs` -already watches (`read_config`): -- `CANOPY_BASE_URL` -- `CANOPY_AUTH_MODE` = `tailscale` | `oidc` -- (oidc) projected-token path / audience; (tailscale) nothing beyond tailnet - membership. +## Part 3 — First-party auth: a new canopy device role + +pgro runs in a different k8s cluster from canopy. There is no shared +cluster, no shared ServiceAccount, no shared Secret. But canopy already +has the right shape for this. Two independent axes: + +### 3.0 Identity: add a new canopy device role + +Canopy's `securitySchemes` defines `server-device`, `releaser-device`, and +`admin-device` — three mTLS device roles, two of which are **not** bound +to a fleet server (`releaser-device` is used by the packaging pipelines, +`admin-device` by operators). Adding a **fourth role** for pgro (working +title `backup-restore-device` — role `backup-restore` — canopy's +naming) reuses that machinery instead of designing fresh federation. + +- mTLS client cert, no server / group binding. Canopy adds it to the role + enum and the `securitySchemes` block. +- The role gates the restore-credentials + restore-verification endpoints. +- **`purpose=backup` is rejected for this role at the API layer.** A + `backup-restore` device that asks `/backup-credentials` with + `purpose=backup` gets a `403`/`409`, not write-capable creds. The + restore-only contract is enforced server-side; pgro doesn't have to ask + nicely, and a compromised pgro can't pivot to writing/poisoning. +- The external-restore grant `(consumer, group, type)` is keyed by the + device cert's identity, the same way device-bound endpoints key off + `device → server → group_id`. +- pgro mounts the cert + key as a k8s Secret (operator pod + each Job's + proxy sidecar). One identity for both directions (creds out, reports in) + — matching the bestool model where `CanopyClient` carries `device_key`. +- Cert provisioning is one-time and operator-driven (no need for the + TPM-bound `canopy register` device-enrolment flow bestool uses); + matches how `releaser-device` certs are issued to packaging pipelines. +- The role is **generic, not pgro-specific**: any future restore-only + consumer (a separate restore-test harness, an external auditor's + read-only verifier, etc.) can use the same role with its own + `(consumer, group, type)` grant. + +This is the lowest lift on the identity side: canopy's role enum + cert +verification + `securitySchemes` plumbing already exists. The cross-cutting +canopy work is the new role itself and the external-restore grant; both +are pgro-blocking either way. + +OIDC workload-identity-federation is a *different* identity model (bearer +token from a federated issuer rather than mTLS). It would be useful for +*other* consumers later (GitHub Actions → canopy and similar), but pgro +doesn't need that generality. Note it here, don't pursue it for pgro. + +### 3.1 Transport: public-mTLS vs Tailscale + +Canopy exposes its API on two transports, and the **authentication +mechanism differs by transport** — they don't layer: + +- **Public-internet, mTLS.** Identity comes from the presented client + certificate; the device-role lookup keys off the cert. This is how + `server-device` / `releaser-device` reach canopy today. +- **Tailscale.** No client cert is presented; identity comes from the + caller's **tailnet identity**, mapped server-side to a canopy device + record. (Canopy's enrollment flow already accommodates this — see + `BeginArgs.spki` in the openapi: required on the tailnet transport + because there's no cert to read the key from, omitted on the mTLS + path where the cert supplies it.) + +The `backup-restore` device role still applies on either transport — it's +a property of the device record, not of the auth mechanism. The choice +determines what pgro mounts and how `CanopyClient` is constructed. + +**`bestool-canopy::CanopyClient` auto-probes.** It tries the hardcoded +tailnet URL (`canopy.tail53aef.ts.net`, `crates/canopy/src/client.rs:38`) +first; if reachable, it uses the tailnet path with plain HTTPS (auth via +tailnet identity). If unreachable, it falls back to the mTLS endpoint +using the provided `device_key_pem`. So pgro doesn't pick at startup — +the client picks per-probe, and `refresh()` re-evaluates. pgro just needs +to (a) make the tailnet reachable from the pod (sidecar — see §3.1.1) or +(b) provide a device cert — or both, in which case Tailscale wins when +present and mTLS catches the gap when it isn't. + +Recommend the **Tailscale** path for production since the operator + +Jobs are already long-lived in-cluster workloads that can sit on the +tailnet, it keeps canopy's public surface area smaller, and it sidesteps +managing a long-lived mTLS keypair as a k8s Secret. Public-mTLS is +viable as a fallback (provision a cert anyway and `CanopyClient` will +use it automatically when the tailnet is unreachable). + +#### 3.1.1 Tailscale sidecar lives on the operator pod only + +Canopy's tailnet auth identifies each caller by **tailscale node +identity** (`commons-servers/src/device_auth/tailnet.rs:52` — it resolves +the source IP via the tailnet directory and keys into +`devices.tailscale_node_id`, auto-creating an `Untrusted` device row on +first contact). Tags are only a coarse admission gate +(`TAILSCALE_REQUIRED_TAG`), not a role assignment. + +So putting a Tailscale sidecar on every Job pod would mean **every Job +pod creates a new `Untrusted` canopy device row** that nobody promotes +to `backup-restore` — and the rows accumulate forever. That doesn't +work. The only workable shape is: + +- **One Tailscale sidecar, on the operator Pod only.** Stable tailnet + node identity, persisted with `TS_STATE_DIR` on a PVC (or via the + Tailscale k8s operator's `kube:` state mode so a restarted operator + resumes the same identity). Image + `ghcr.io/tailscale/tailscale` in userspace mode (`TS_USERSPACE=true`) + exposing `TS_SOCKS5_SERVER=:1055` on `localhost`. Authkey + OAuth-issued by ops, the node tagged for ACL admission (the tag + itself is just admission, not role). +- **The operator becomes a canopy device.** First contact creates an + `Untrusted` row; an operator (human) promotes it once to + `backup-restore`. One device record total. +- **kopia Job pods carry no Tailscale sidecar** and are not canopy + devices. + +#### 3.1.2 The credential broker: operator mediates for Jobs + +`bestool-kopia::proxy::CredentialProvider` is a trait. bestool's +implementation +(`CanopyCredentialProvider`) calls canopy directly. pgro plugs a +**different provider** in the Job-side proxy sidecar — one that calls +the operator pod over the in-cluster network instead: + +- The operator exposes a small in-cluster HTTP endpoint, e.g. + `POST /internal/restore-creds` taking `{group, type}` and returning + the same `BackupCredentials` shape `bestool_canopy` already + deserializes. Backed by a per-(group, type) cache so concurrent Jobs + don't N×-multiply canopy calls. +- The operator (running on the tailnet via its single sidecar) hits + canopy's `POST /backup-credentials` with `purpose=restore`, returns + the result. Canopy still enforces the role + purpose gate + server-side; the operator-broker is purely a transport reuser. +- The Job's proxy sidecar implements `CredentialProvider` against this + endpoint. Refresh cadence is the same (~2 min before expiry). +- Network gating: the broker endpoint is only exposed within the + cluster (no Service.type=LoadBalancer), gated by a NetworkPolicy + restricting access to pgro's own Job pods (matched by label) and the + operator's namespace. + +Restore reports (§2) also flow through the operator — same justification: +one canopy device, one identity. + +Net effect: the Tailscale + canopy-device-role machinery is a property +of the **operator process**, not of each Job. The Jobs see only an +in-cluster HTTP endpoint that happens to broker canopy creds. + +**ops-repo setup, one-time per cluster** (called out here for spec +completeness; actual implementation lives in the ops repo): +- Install the Tailscale k8s operator into pgro's cluster. +- Mint an OAuth client + ACL tag for pgro's operator pod (tag is just + the ACL admission, not the canopy role). +- ACL allows the tag to reach `canopy.tail53aef.ts.net`. +- Canopy-side: after the operator's first contact creates its + `Untrusted` device row, promote it to `backup-restore` (one-time + manual step, same as `releaser-device` promotion today). + +### 3.2 Operator config surface + +Operator-level config, not per-CRD (open question 3.x: per-replica vs +operator-global canopy — recommend operator-global so every replica +trusts one canopy). Plumb via env on the Deployment (`operator.yaml`) +and/or the existing `postgres-restore-operator-config` ConfigMap that +`src/bin/operator.rs` already watches (`read_config`): +- `CANOPY_BASE_URL` — the public-mTLS endpoint, used by `CanopyClient` + as the mTLS-fallback target. The tailnet hostname is hardcoded in + `bestool-canopy`, not configurable here. +- `CANOPY_DEVICE_CERT_SECRET` — optional. Name of a k8s Secret with a + pgro device cert + key, mounted only into the operator pod. Used by + `CanopyClient` when the tailnet probe fails. Skip entirely if the + operator is Tailscale-only. +- `PGRO_CREDENTIAL_BROKER_ADDR` — the cluster-internal listen address + the operator exposes its `/internal/restore-creds` endpoint on. The + Job builder propagates the matching Service URL into each kopia + Job's proxy sidecar. + +The Tailscale sidecar is wired only on the operator Deployment in +`operator.yaml` — annotation-driven if the Tailscale k8s operator's +`ProxyClass` is used, otherwise as an explicit second container. The +kopia Job spec has **no** Tailscale sidecar. --- @@ -317,18 +554,43 @@ already watches (`read_config`): - **No more user-managed `kopia-credentials` Secret** for canopy-backed replicas. Operators stop hand-creating it; the static-Secret path remains only for legacy/non-canopy repos. -- **Network path to canopy.** Tailscale: ensure the pgro pod can reach - canopy's private endpoint over the tailnet (sidecar or host tailnet, per - the cluster's existing Tailscale setup — ops repo). OIDC: expose pgro's - cluster OIDC issuer to canopy and project a SA token into the operator pod - (`operator.yaml` Deployment `volumes` + `serviceAccountToken` projection). -- **RBAC delta in `operator.yaml`.** Strategy (A) writes transient per-Job - Secrets — the ClusterRole already grants full Secret verbs - (`get/list/watch/create/update/patch/delete`), so no RBAC change is needed - for that. No new cluster permissions for the canopy path itself (it's - outbound HTTP). -- The auth-plumbing (OIDC trust config or Tailscale exposure) and the - external-restore grant live in the **canopy + ops** repos, not pgro +- **New pgro-published sidecar image** linking `bestool-kopia` (proxy) and + `bestool-canopy` (HTTP client + credential provider). Built and pushed + alongside the existing operator image; pinned by tag in `operator.yaml` + via env (e.g. `CANOPY_PROXY_SIDECAR_IMAGE`). The Job builder injects this + container into every kopia Job for canopy-backed replicas. +- **Tailscale sidecar on the operator pod only** — one stable tailnet + node identity → one canopy device. Image + `ghcr.io/tailscale/tailscale` in userspace mode with + `TS_SOCKS5_SERVER=:1055`; auth via OAuth-issued authkey carrying the + pgro ACL tag (admission, not role). Provisioned either by the + Tailscale k8s operator's `ProxyClass` injection (preferred) or as an + explicit second container in the operator Deployment. +- **kopia Job pods have NO Tailscale sidecar**. They reach the + operator's in-cluster credential-broker endpoint + (`/internal/restore-creds`) via a normal cluster Service. The + operator forwards to canopy on their behalf. +- pgro's `reqwest` client (in the operator) is built with + `Proxy::all("socks5://localhost:1055")` so `CanopyClient`'s + auto-probe of `canopy.tail53aef.ts.net` succeeds via the sidecar. +- **Optional public-mTLS fallback**: a `CANOPY_DEVICE_CERT_SECRET` + mounted into the operator pod only. `CanopyClient` uses it + automatically when the tailnet is unreachable. Skip the cert entirely + if pgro is comfortable being Tailscale-only. +- **NetworkPolicy gating** the credential-broker endpoint: only pgro's + own Jobs (matched by label) in the operator's namespace can hit it. +- **RBAC delta in `operator.yaml`.** The new path stops writing transient + per-Job Secrets (that was strategy (A) in the old framing — superseded by + the sidecar). The ClusterRole's existing Secret verbs are still required + for the legacy static-Secret path. No new cluster permissions for the + canopy path itself (it's outbound HTTP from operator + sidecar). +- The auth plumbing lives in the **canopy + ops** repos, not pgro: + canopy adds the `backup-restore` role, the external-restore grant, + the purpose-gate, and the one-time promotion of pgro's operator-pod + device record from `Untrusted` to `backup-restore`; ops installs the + Tailscale k8s operator in pgro's cluster, mints the OAuth client + + ACL tag for pgro's operator pod (admission only, not role), and + writes the ACL allowing that tag to reach `canopy.tail53aef.ts.net` (canopy plan: "ops: the auth plumbing … and PGRO's read-only access path"). --- @@ -336,20 +598,31 @@ already watches (`read_config`): ## Interfaces this component exposes / consumes **Consumes (from canopy):** -- `fetch_restore_creds(group)` → `CanopyRestoreCreds` (read-only restore - creds + S3 target + repo password). Canopy-side: restore session policy on - the per-bucket role; gated by the external-restore grant for pgro's - first-party identity. -- First-party auth surface (Tailscale identity or OIDC trust). +- `POST /backup-credentials` with `(type, purpose=restore)` plus the + external-restore grant context for `group` — returns `BackupCredentials` + (`bestool_canopy::BackupCredentials`). Called from the sidecar's + credential provider, not from the operator process. +- `GET /backup-target` — returns `BackupTarget` (`bestool_canopy::BackupTarget`). + Called once per Job to populate kopia's `--bucket`/`--region`/`--prefix` + and `--password`. +- A canopy device cert for the new non-server-bound role (Part 3). + +**Consumes (from bestool, as published cargo crates):** +- `bestool-kopia` (≥0.3.2): the S3P proxy + `CredentialProvider` trait. +- `bestool-canopy` (≥0.4.0): the canopy wire types and the `CanopyClient` + (constructed with pgro's device key — unchanged from the bestool usage). **Provides (to canopy):** -- `RestoreReport` via `report_restore(...)` → canopy `backup_restore_checks` - / signal-3 detection. Cross-referenced by `snapshot_id`. +- `RestoreReport` via a new canopy-side endpoint (working title `POST + /restore-verification`) → canopy `backup_restore_checks` / signal-3 + detection. Cross-referenced by `snapshot_id`. - The `PostgresPhysicalReplica.spec.canopyBackup.group` field is the operator- visible contract that binds a replica to a canopy group. **Provides (to pgro operators):** - CRD: `canopyBackup: { group }` as the alternative to `kopiaSecretRef`. +- A new pgro-published sidecar image (canopy S3P proxy + credential + provider) used by every kopia Job for canopy-backed replicas. - README CRD-table updates documenting it. --- @@ -363,17 +636,17 @@ except CRD-table updates. Run `cargo clippy` + `cargo fmt` before committing; conventional commits; always work on a branch. **Unit tests (run locally, alongside the code):** -- `src/canopy.rs`: deserialize a canopy creds response into - `CanopyRestoreCreds`; serialize a `RestoreReport`; expiry parsing; the - "should-refresh" decision (creds near expiry). Mirror the table-driven - style in `src/kopia.rs` / `src/types/replica.rs` tests. -- CRD validation: "exactly one of kopiaSecretRef/canopyBackup" — both-set and - neither-set are errors; round-trip `CanopyBackupRef` through serde +- `src/canopy.rs`: serialize a `RestoreReport`; verify the auth-mode + selector dispatches correctly. Wire-shape de/serialisation is covered by + `bestool-canopy`'s own tests — pgro re-uses the types, doesn't re-test + them. +- CRD validation: "exactly one of kopiaSecretRef/canopyBackup" — both-set + and neither-set are errors; round-trip `CanopyBackupRef` through serde (mirroring `schema_migration_phase_roundtrips_*`). -- Job builder: with `canopyBackup`, the connect script/env carry - `AWS_SESSION_TOKEN` and reference the transient Secret name; legacy path - unchanged. Extend the existing `kopia_connect_args_*` tests for the - session-token arg. +- Job builder: with `canopyBackup`, the connect args carry the proxy + endpoint, dummy keys, and `--disable-tls`; the Job spec has the proxy + sidecar container; legacy path unchanged. Extend the existing + `kopia_connect_args_*` tests for the canopy variant. - Report gating: `canopyReportedAt` prevents double-reporting; a report failure does not fail the restore (assert status transitions unaffected). @@ -393,40 +666,37 @@ conventional commits; always work on a branch. ## Open questions / decisions to make -1. **Credential lifetime vs. long restores (the critical one).** Chained - AssumeRole sessions are capped at **1 hour**. Large restores (slow WAL - replay) can exceed that. The kopia restore reads from S3 for the whole - run, so expired creds mid-restore fail it. Options: - (a) Job self-refreshes (strategy B) — Job calls canopy as a - `credential_process`-style helper; needs canopy network path + auth inside - the Job. (b) Operator watches expiry and rotates the transient Secret + - the kopia env mid-Job — fragile, kopia would need to re-read creds. - (c) canopy issues longer-lived creds to first-party consumers via a - *non-chained* direct web-identity assume (the canopy plan already uses - direct, non-chained cross-account web-identity for **maintenance Jobs**, - sidestepping the 1-hour cap — decision 13). **Recommend pushing canopy to - make (c) available to the pgro external-restore grant**, since pgro is - first-party like the maintenance Jobs. Decide before building the Job - credential path. - -2. **kopia S3 backend + session token.** Confirm kopia's S3 backend honours - `AWS_SESSION_TOKEN` (temporary creds) — the canopy plan assumes the AWS - SDK `credential_process` path on devices, but pgro drives kopia via CLI - flags (`--access-key` / `--secret-access-key`). There is no - `--session-token` flag in the current `kopia_connect_args`; verify whether - kopia reads `AWS_SESSION_TOKEN` from the env (likely yes, via the AWS SDK - it embeds) or whether a different invocation is needed. This gates - strategy (A). +1. **Credential lifetime vs. long restores — RESOLVED by the proxy model.** + Chained AssumeRole sessions cap at 1 hour, but the S3P proxy refreshes + creds out-of-band between requests, so kopia is oblivious to credential + lifetime. A long restore is bounded by canopy reachability, not by any + single issuance. Canopy may still want to offer non-chained direct + web-identity creds to first-party consumers for efficiency / fewer + refresh round-trips (and to mirror the maintenance-Job pattern), but it + is a secondary optimisation, not a viability gate. Keep the request on + canopy's list; don't block pgro on it. + +2. **kopia S3 backend + session token — RESOLVED by the proxy model.** + Under the S3P proxy, kopia talks only to `127.0.0.1` with dummy keys; it + never sees a real session token. `--session-token` and + `AWS_SESSION_TOKEN` are both irrelevant on the kopia leg. (For + completeness: kopia *does* support `--session-token`, per the canopy + kopia spike — but the production design doesn't use that path because + the proxy is more general.) 3. **Per-replica vs operator-global canopy config.** Recommend operator-global (`CANOPY_BASE_URL` + auth on the Deployment/ConfigMap), so `CanopyBackupRef` carries only `group`. Revisit if pgro ever needs to talk to more than one canopy. -4. **Tailscale vs OIDC for stage 1.** Tailscale is available now and is the - lowest-lift stage-1 path; OIDC is the more general long-term direction - (also unlocks GitHub Actions → canopy). Isolate behind one auth trait so - the choice is swappable. Decision owned jointly with canopy + ops. +4. **Confirm the new-canopy-device-role + transport choice.** Identity is + a new device role (`backup-restore`, generic and not pgro-specific) + alongside `server`/`releaser`/`admin`, keyed to the external-restore + grant, with `purpose=backup` rejected for the role at the API layer. + The transport choice — canopy's public-mTLS surface or its Tailscale + surface — picks the auth mechanism (mTLS cert vs tailnet identity), not + a layering on top of mTLS. Recommend Tailscale for production. Confirm + both with canopy before pgro builds the auth path. See Part 3. 5. **Report transport coupling.** Should the canopy report reuse the existing notification subsystem (a new built-in target) or be a dedicated path on @@ -435,11 +705,11 @@ conventional commits; always work on a branch. arbitrary user webhooks — but the `NotificationStatus`/retry shape is the right model to copy. -6. **Snapshot-list / availability checks.** snapshot-list Jobs also need repo - creds. They're short and fit strategy (A) cleanly. Confirm canopy is happy - to serve `purpose=restore` creds for the periodic snapshot-list cadence - (it is read-only; no objection expected), and that the read traffic volume - is acceptable. +6. **Snapshot-list / availability checks.** snapshot-list Jobs also need + repo creds; they use the same proxy-sidecar pattern as restores. Confirm + canopy is happy to serve `purpose=restore` creds for the periodic + snapshot-list cadence (it is read-only; no objection expected), and that + the read traffic volume is acceptable. 7. **Migration / coexistence.** During rollout a replica may switch from `kopiaSecretRef` to `canopyBackup`. Confirm a clean switchover (the next @@ -447,10 +717,13 @@ conventional commits; always work on a branch. operator once no replica references it) and that mixed fleets work. 8. **Region/endpoint for non-AWS repos.** The legacy path supports - `endpoint` + `disableTls` (minio, S3-compatible). Canopy's `backup-target` - serves `bucket/prefix/region` for real S3; if any canopy-backed repo is - ever non-AWS, `CanopyRestoreCreds` would need optional `endpoint`/ - `disableTls` too. Out of scope unless canopy supports non-AWS targets. + `endpoint` + `disableTls` (minio, S3-compatible). Canopy's + `GET /backup-target` returns `{ storage, bucket, prefix, region, + repo_password }` — no `endpoint`/`disable_tls` field. If any + canopy-backed repo is ever non-AWS, the canopy wire type would need + optional `endpoint`/`disable_tls`, and the bestool S3P proxy would need + to be willing to point upstream at a non-`s3..amazonaws.com` + host. Out of scope unless canopy supports non-AWS targets. --- @@ -467,3 +740,101 @@ Per the Canopy plan's "Backup types": PGRO consumes a **specific type**. `(group, type)` snapshot it restored and the outcome; a failed/stale restorability check for that `(group, type)` is the group-level alert. - PGRO filters the repo by the `canopy-type` tag to find its snapshots. + +--- + +## Canopy implementation status (re-verified 2026-06-26) + +Where canopy actually stands today, so this spec builds on what exists vs. +what canopy still owes. Re-checked against `canopy/crates/public-server/` +(openapi.json + src), `canopy/migrations/`, and the bestool repo on +2026-06-26: **none of the pgro-blocking surfaces have moved** since this +section was first written 2026-06-16. Canopy work in the intervening two +weeks has been operator-UI and S3-traffic tallies (PRs through #282; +migrations through `add_s3_traffic_to_backup_runs`). + +**Available now (PR #224):** `POST /backup-credentials` with +`{ "type": "tamanu-postgres", "purpose": "restore" }` returns read-only +`credential_process` creds (the restore session policy: `GetObject` + +unconditioned `GetBucketLocation` + prefix-conditioned `ListBucket`), and +`GET /backup-target` returns `{ storage, bucket, prefix, region, +repo_password }`. So the *device-shaped* restore path exists — but it is +`ServerDevice` (mTLS) authenticated and the creds are chained (1-hour cap, +practically a non-issue under the proxy model — see §1.3). + +**Available now in the bestool repo (published crates):** the S3P proxy +(`bestool-kopia` 0.3.2: `bestool_kopia::proxy::{spawn, CredentialProvider}`) +and the canopy HTTP client + wire types (`bestool-canopy` 0.4.0: +`CanopyClient`, `BackupCredentials`, `BackupTarget`, `BackupReport`, +`Purpose`). pgro depends on both as ordinary cargo deps; no git/vendoring. + +**Still owed by canopy (not built — blocks PGRO):** +- **A new device role (`backup-restore` or equivalent).** Canopy already + has `server-device`/`releaser-device`/`admin-device`; pgro's identity is + a fourth, generic restore-only role keyed off the external-restore + grant, with `purpose=backup` server-side rejected for the role. Smaller + lift than net-new auth machinery, but still net-new role+cert plumbing + that canopy must add (role enum, `securitySchemes` entry, route-gating, + purpose-gating, cert-issuance flow). Until this lands, pgro cannot + authenticate to canopy at all. **Biggest blocker.** +- **`CredentialsArgs` / external-restore wire change.** Today's + `{ type, purpose }` body has no `group` field; the new role has no + implicit group, so canopy needs either an additive field or a sibling + endpoint that takes one. The `(consumer, group, type)` external-restore + grant lookup hangs off this. +- **The external-restore grant** (`(consumer, group, type)` authz model, + operator-authorized + audited) — no migration, no code. +- **Restore-verification ingest path.** A new endpoint (working title + `POST /restore-verification`) writing to a new `backup_restore_checks` + table, routing failures through `raise_group_event(ref = + "restore-verification")`. See §2.0 for why reusing `/backup-report` is + not viable. + +**Group-level alert entrypoint is concrete (PR #225).** Signal-3 routes +through `database::backup::alerts::raise_group_event`, whose signature is: +`raise_group_event(conn, group_id, ref: &str, severity: Severity, +description: Option<&str>, message: &str, active: bool) -> Result`. +Use `ref = "restore-verification"` (the constant in `database::backup::refs`), +severity `Error` (group-level, bypasses per-server `is_monitored`); recovery +is the same `(source="canopy", ref)` with `active: false`. Group-scoped issues +are first-class (Option B: `issues.server_group_id`), so no per-server shim is +needed. + +--- + +## bestool-side asks + +What needs to be added to the published bestool crates before pgro can +build against them. All four are additive (no breaking changes to +existing bestool consumers) and each is gated by the corresponding +canopy endpoint shipping first. + +**In `bestool-canopy`:** + +1. **`CanopyClient::restore_credentials(base, type, group)`** — + group-aware variant of `backup_credentials`. The existing method + infers group from `device → server → group_id`, which doesn't apply + to a non-server-bound `backup-restore` device; the new method puts + `group` in the request body to match the canopy wire change. +2. **`CanopyClient::restore_target(base, group)`** — same group issue. +3. **`CanopyClient::restore_verification(base, &RestoreVerification)`** + — posts to canopy's new ingest endpoint (working title + `POST /restore-verification`). See §2.0 for why this is a separate + endpoint from `/backup-report`. +4. **`pub struct RestoreVerification`** in `bestool-canopy::backup` — + the wire request type, mirroring whatever canopy lands. + +**In `bestool-kopia`:** **no changes needed.** `proxy::spawn`, the +`CredentialProvider` trait, the `Credentials` struct, and `TrafficStats` +are already what pgro consumes. pgro implements its own +`CredentialProvider` against the operator's in-cluster broker (§3.1.2), +no changes to the proxy or the trait. + +**Also unchanged:** `CanopyClient::new(... device_key_pem: Option<&str>, +...)` already accepts `None`, so pgro's tailscale-only operator (no +mTLS fallback cert) works without further changes. `Purpose::Restore` +already exists. No new crate — all four asks fit in `bestool-canopy`. + +The `purpose=backup` rejection for the `backup-restore` role is purely +canopy-side enforcement; `bestool-canopy` doesn't know about role-based +purpose gating, it just sends what it's told. diff --git a/docs/canopy-handoff.md b/docs/canopy-handoff.md new file mode 100644 index 0000000..3f5603e --- /dev/null +++ b/docs/canopy-handoff.md @@ -0,0 +1,483 @@ +# Handoff to canopy: PGRO restore-verification integration + +**From:** pgro +**To:** canopy (then canopy → bestool for the appendix) +**Status:** waiting on canopy. pgro will not start building until the +items in §4 land (or are contract-frozen) and the bestool additions in +§A ship in a published crate. + +This document is the actionable subset of pgro's full integration spec +(`pgro/docs/canopy-backup-integration.md`). Read that for the +why-it-looks-like-this; read this for what to build. Anything contentious +here gets bounced back to pgro before implementation. + +--- + +## 1. Context, brief + +pgro restores tamanu-postgres physical backups out of kopia repos into +working postgres replicas. Today it authenticates with hand-set, +long-lived AWS keys + repo password in a k8s Secret — the exact +long-lived-creds pattern the canopy backup-credentials system exists to +eliminate. Bringing pgro under canopy gets two things: + +1. **Eliminates the static keys** on the pgro side. canopy mediates + restore creds the same way it mediates device backup creds. +2. **Closes the lifecycle loop end-to-end.** A successful pgro restore + *proves the snapshot is restorable* — signal 3, the strongest + backup-health signal there is, stronger than signal 2's + "a snapshot exists in the repo". pgro reports per-replica restore + outcomes back to canopy; a failed/stale restorability check becomes + a high-severity group-level alert. + +This is the integration the canopy backup-credentials plan calls out in +§"External restore consumers + restore-verification (PGRO)" — pgro is +ready to build its side once canopy's side exists. + +--- + +## 2. Architecture pgro is building toward + +Read this so the wire-shape and identity choices below make sense in +context. + +- **One stable pgro operator Pod** sits on the tailnet and is the + single canopy device. It speaks to canopy directly. +- **Each kopia restore is a k8s Job** spawned by the operator. Each Job + Pod runs two containers: kopia, and a pgro-published proxy sidecar. +- **The proxy sidecar runs the bestool S3P loopback re-signing proxy** + (`bestool_kopia::proxy::spawn` from the published `bestool-kopia` + crate). kopia is pointed at `127.0.0.1` with dummy keys; the proxy + holds the live STS creds and re-signs each request. Same model as + bestool device backups and canopy's own maintenance jobs. +- **The proxy's `CredentialProvider`** doesn't call canopy directly. + It calls an in-cluster HTTP endpoint on the operator + (`/internal/restore-creds`), and the operator forwards to canopy. + This is forced by the identity model (§3) — Job Pods are not canopy + devices and have no way to authenticate. +- **`bestool-canopy::CanopyClient` auto-probes tailnet vs mTLS.** pgro + uses the tailnet path (via the Tailscale sidecar on the operator + Pod); mTLS is an optional fallback if a device cert is provisioned. + +Consequences worth flagging up front: + +- **The chained-STS 1-hour cap is a non-issue.** The proxy refreshes + creds between requests; long restores are bounded by canopy + reachability, not by any single issuance lifetime. pgro does not need + non-chained / direct-IRSA creds. +- **kopia never sees real AWS credentials.** It carries dummy keys and + talks to `127.0.0.1`. The `--session-token` / `AWS_SESSION_TOKEN` + question is moot. + +--- + +## 3. Identity model: one operator-Pod tailnet device + +canopy's tailnet auth identifies callers by **tailscale node identity** +(`commons-servers/src/device_auth/tailnet.rs:52` — looks up the source +IP via the tailnet directory, keys into `devices.tailscale_node_id`, +auto-creates an `Untrusted` device row on first contact). Tags are only +a coarse admission gate (`TAILSCALE_REQUIRED_TAG`). + +That means **one tailnet node = one canopy device record**. Per-Job +Tailscale sidecars would create one `Untrusted` row per Job pod, +forever — unworkable. + +So pgro will run **exactly one Tailscale sidecar**, on the operator +Pod, and pgro is **one canopy device**: + +- First contact creates an `Untrusted` row. +- canopy (admin) promotes it once to role `backup-restore` (working + name — see §4.1). +- The operator brokers everything for Job Pods over the in-cluster + network, so Job Pods never need their own canopy identity. + +The mTLS path is symmetric — one operator-Pod-mounted device cert, +one canopy device, same identity. Either path works; canopy's auth +mechanism is the only thing that differs. + +--- + +## 4. What canopy needs to build + +Five items. They depend on each other roughly in the order listed. + +### 4.1 New device role: `backup-restore` + +Add a fourth role alongside `server` / `releaser` / `admin`. + +- Generic, not pgro-specific. Any future restore-only consumer (an + external auditor's verifier, a separate test-restore harness) shares + the same role with its own external-restore grant. +- No server / group binding. Like `releaser-device`, the role itself + doesn't imply membership in any group. +- Add to the device-role enum, `securitySchemes` in + `crates/public-server/openapi.json`, route-gating macros, and the + cert-issuance flow (one-off operator-driven cert minting — does not + need the TPM-bound `canopy register` enrolment flow that bestool + servers use; the `releaser-device` provisioning path is the right + model). +- **`purpose=backup` must be rejected at the API layer for this role.** + A `backup-restore`-role caller hitting `/backup-credentials` with + `purpose=backup` gets `403`/`409`, full stop. The role's read-only + contract is server-enforced, not consumer-promised, and a compromised + pgro can't pivot to writing/poisoning. + +This is the biggest single blocker. Until this lands pgro cannot +authenticate at all. + +### 4.2 Group-aware credentials + target endpoints + +For server-bound roles, `device → server → group_id` resolves the group +implicitly. A `backup-restore`-role device has no implicit server and +no implicit group, so the request body has to carry `group`. + +Two viable shapes; canopy picks: + +- **(a) Add `group: Uuid` to the existing `CredentialsArgs` / + `BackupTarget` paths** and accept it only from `backup-restore`-role + callers. Smaller diff; mildly violates the principle that + device-authenticated requests don't put authz fields in the body. +- **(b) Sibling endpoints**: e.g. `POST /restore-credentials` and + `GET /restore-target?group=...`. Clean separation; bestool-canopy + gets two new methods rather than overloaded ones (matches the + appendix bestool deltas). + +pgro lightly prefers (b) for clarity, but defers to canopy. + +Behaviour either way: canopy verifies the `(consumer, group, type)` +external-restore grant (§4.3), then runs the same restore session +policy + per-bucket role + repo-password lookup it does today, and +returns `BackupCredentials` + `BackupTarget` unchanged. + +### 4.3 The external-restore grant + +The operator-authorised, audited authz primitive that says "consumer C +may read group G's type T, read-only." + +- Per `(consumer_device_id, group_id, type)`. New table; canopy picks + the name (`backup_restore_grants` or similar). +- Operator-authorised via the existing private-server UI or `canopy + ctl` CLI; audited. +- Checked at request time for `/restore-credentials` (4.2) and + `/restore-verification` (4.4). Absence is a clear 403, not a + transient error. +- pgro will surface that 403 as a clear `Failed` phase + Warning event + on the replica; the operator who set up the replica diagnoses by + going to canopy and inspecting / creating the grant. + +### 4.4 Restore-verification ingest endpoint + `backup_restore_checks` + +#### 4.4.1 Why NOT reuse `POST /backup-report` + +`/backup-report` already accepts `{ purpose: "restore", outcome, +snapshot_id, error, run_id }` — for **devices**. The shape looks close +to what pgro wants, but reusing it is wrong for seven concrete reasons: + +1. **Identity is auth-context-derived, not body-derived.** The handler + resolves `device_id`, `server_id`, and `group_id` from the + authenticated mTLS context (`crates/public-server/src/backup.rs:495`), + not the body. A `backup-restore`-role caller has no implicit server + or group; threading them through the body would break the invariant + that a device can't report a run as some *other* group. +2. **Schema is device-shaped.** `backup_runs` has `device_id UUID NOT + NULL REFERENCES devices(id)` and `group_id NOT NULL REFERENCES + server_groups(id)`. The pgro device row exists but it's not + "running" a backup for any server; satisfying the FKs requires + either sentinel data or schema changes. +3. **Two different "restore" meanings collide on `purpose`.** A device + with `purpose=restore` (e.g. `bestool canopy restore` for clone / + DR-test on the same fleet) writes to `backup_runs`. That is NOT a + signal-3 verification — it's a normal device-side restore and + should not raise a group-level "the backup isn't restorable" + incident. `purpose=restore` alone is not a sufficient discriminator + between device-restore-runs and signal-3 verifications. +4. **Alerting paths diverge.** `/backup-report` failure feeds per-server + staleness (signal 1, server-scoped). Signal 3 must feed group-scoped + `raise_group_event(ref = "restore-verification")` bypassing + per-server `is_monitored`. +5. **Side-effects don't match.** The handler clears `BackupRequest` + (`backup.rs:534`) so the heartbeat stops re-emitting "back up now" + for that server. Irrelevant for a pgro report. +6. **Payload shape is wrong.** `ReportArgs` carries `bytes_uploaded` + + `s3_*_bytes` (good — pgro's proxy emits those too) but lacks + `replica_healthy`, postgres major version, `observed_at` — the + load-bearing fields that make signal 3 stronger than signal 2. +7. **`run_id` semantics don't transfer.** For devices, `run_id` is the + same UUID across `/backup-credentials` (issuance audit) and + `/backup-report`, minted at run start, dup → 409. pgro's natural + identity is the snapshot being verified, not a per-run UUID; a + pgro-minted run UUID has no cross-table linkage to + `backup_credential_issuances`. + +By the time `/backup-report` has been extended to take `group_id`, +relaxed (or split off) the FKs, branched the handler on actor type, +routed failures differently, and gated the `BackupRequest::clear` +side-effect, the handler has forked. Cleaner to expose a sibling. + +#### 4.4.2 New endpoint + +Working title `POST /restore-verification` (canopy picks the name). +Authenticated as `backup-restore`-role; gated by the external-restore +grant for the body's `(group, type)`. + +Request body (proposed): + +```json +{ + "group": "", + "type": "tamanu-postgres", + "snapshot_id": "", + "outcome": "success" | "failure", + "error": "", + "replica_healthy": true, + "postgres_version": "", + "observed_at": "", + "s3_sent_raw_bytes": 12345, + "s3_sent_payload_bytes": 12300, + "s3_received_raw_bytes": 98765, + "s3_received_payload_bytes": 98700 +} +``` + +- `snapshot_id` is the join key into `backup_repo_snapshots` / + `backup_runs`. Load-bearing for closing the loop *backed up → + persisted → restorable*. +- `outcome=success` with `replica_healthy=true` means kopia restored + successfully AND postgres came up AND the operator's readiness gate + passed. +- `outcome=failure` with an `error` string covers restore-job failure, + deployment-never-ready, postgres-version mismatch, etc. +- S3 byte tallies come from the bestool proxy's `TrafficStats` (already + there in `bestool-kopia`). pgro emits them on success and failure, + same as `/backup-report`. + +#### 4.4.3 New table: `backup_restore_checks` + +Roughly: + +```sql +CREATE TABLE backup_restore_checks ( + id BIGSERIAL PRIMARY KEY, + consumer_device_id UUID NOT NULL REFERENCES devices(id), + group_id UUID NOT NULL REFERENCES server_groups(id), + type TEXT NOT NULL, + snapshot_id TEXT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('success','failure')), + error TEXT, + replica_healthy BOOLEAN NOT NULL, + postgres_version TEXT, + observed_at TIMESTAMPTZ NOT NULL, + s3_sent_raw_bytes BIGINT, + s3_sent_payload_bytes BIGINT, + s3_received_raw_bytes BIGINT, + s3_received_payload_bytes BIGINT, + reported_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX ON backup_restore_checks (group_id, type, observed_at DESC); +CREATE INDEX ON backup_restore_checks (snapshot_id); +``` + +Exact shape is canopy's call. pgro just needs the endpoint to accept +the body in §4.4.2 and reject 4xx clearly on grant/role failure. + +#### 4.4.4 Alert routing + +Plumb `outcome=failure` (and staleness — see §6 / "Open questions") +into: + +```rust +raise_group_event( + conn, group_id, + ref: "restore-verification", // const in database::backup::refs + severity: Severity::Error, // group-level; bypasses per-server is_monitored + description: ..., + message: ..., + active: true, +); +``` + +Already concrete in PR #225, no new plumbing on the alerting side — +just call it from the new handler. Recovery (`active: false`) on the +next successful report for the same `(group, type)`. + +### 4.5 Wire-type stability + +For pgro's side: please freeze the wire shapes for §4.2 and §4.4.2 +before merging the bestool changes (Appendix A). Mid-flight name churn +on `BackupCredentials` / `BackupTarget` fields would also cause +collateral damage — pgro is going to consume `bestool_canopy`'s +existing types verbatim, so renames there propagate. + +--- + +## 5. What pgro is NOT asking for + +These have come up in earlier rounds and pgro has explicitly **decided +against** them: + +- **Non-chained / longer-lived STS creds for pgro.** The proxy refreshes + out-of-band; the 1-hour chained cap is fine in practice. Don't burn + effort here on pgro's account. (canopy may still want it for its own + reasons — that's a canopy call.) +- **Reusing `/backup-report` for signal 3.** §4.4.1 covers why. +- **Server-side cred caching across pgro Jobs.** pgro's operator + already caches in-process for the broker (§Architecture); canopy + doesn't need to. +- **A new auth federation (OIDC).** pgro is happy with mTLS + tailnet. + OIDC would be useful for *other* future first-party consumers and + canopy can pursue it independently, but pgro doesn't need it. + +--- + +## 6. Open questions canopy owns + +Pick before / during implementation; flag back to pgro if any of these +change pgro-visible shape. + +1. **4.2 (a) vs (b):** group in body of existing endpoints, or sibling + `/restore-*` endpoints. pgro mildly prefers (b). +2. **Naming.** Role: `backup-restore` (pgro suggestion) vs whatever + canopy prefers. Endpoint: `/restore-verification` vs + `/backup-restore-check` vs… Table name: `backup_restore_checks` vs + `restore_verifications`. pgro doesn't care, just needs them stable + before bestool ships. +3. **Staleness detection for signal 3.** A successful report is + straightforward. "Stale" (no recent successful verification for a + `(group, type)`) is a periodic check canopy needs to run — out of + pgro's scope, but in scope for the alerting story. Define the + cadence + threshold canopy-side. +4. **`backup_restore_checks` retention.** pgro suggests indefinite + (audit trail, small rows); canopy decides. +5. **Per-Pod identity for audit.** pgro is intentionally one canopy + device; per-Job audit lives in pgro's own k8s record (CRD status, + events). If canopy wants to split per-Pod, pgro can include a + `consumer_instance` opaque string in the body — but the cost is + real and the value is unclear. Default: don't. +6. **Cert-issuance flow for the new role.** pgro will be tailscale-only + in normal operation; mTLS cert is the fallback. If canopy doesn't + want to build cert minting for the new role at all (tailscale-only, + period), pgro is fine with that — just confirm. + +--- + +## 7. Pgro-side commitments (so canopy knows what to expect) + +- pgro will be one canopy device. First contact creates `Untrusted`; + canopy admin promotes once. +- pgro will report `outcome=success` only when the deployment actually + passes the readiness gate (not on bare-kopia-success). Failure + reporting is best-effort and never blocks restore progression. +- pgro will at-most-once-per-restore, with retry across reconciles + until the report lands (status-tracked). +- pgro will not write or delete from any bucket. The proxy is fed by + the restore session policy; even if pgro is compromised it has no + write capability (compounded by §4.1's role-level `purpose=backup` + rejection). + +--- + +## Appendix A — Hand off to bestool + +Once §4 has landed (or shipped to a feature branch with frozen wire +shapes), canopy passes this list to bestool. All four are additive in +the published `bestool-canopy` crate; no breaking changes to existing +consumers, no new crate. `bestool-kopia` needs no changes. + +### A.1 `bestool_canopy::backup::RestoreVerification` (new) + +Public wire type mirroring §4.4.2: + +```rust +#[derive(Debug, Clone, Serialize)] +pub struct RestoreVerification<'a> { + pub group: Uuid, + pub r#type: &'a str, + pub snapshot_id: &'a str, + pub outcome: RunOutcome, // reuse existing enum + pub error: Option<&'a str>, + pub replica_healthy: bool, + pub postgres_version: Option<&'a str>, + pub observed_at: jiff::Timestamp, + pub s3_sent_raw_bytes: Option, + pub s3_sent_payload_bytes: Option, + pub s3_received_raw_bytes: Option, + pub s3_received_payload_bytes: Option, +} +``` + +Field-renaming-via-serde to whatever canopy lands; the Rust shape is +indicative. + +### A.2 `CanopyClient::restore_credentials(base, type, group) -> Result` + +Group-aware variant of `backup_credentials`. Posts to whichever +endpoint canopy picks in §4.2 (a) or (b); the response type is the +existing `BackupCredentials` unchanged. + +```rust +pub async fn restore_credentials( + &self, + base_url: &Url, + backup_type: &str, + group: Uuid, +) -> Result { ... } +``` + +### A.3 `CanopyClient::restore_target(base, group) -> Result` + +Same group issue for target lookup. Response is the existing +`TargetOutcome` (Ready/Dormant) — Dormant maps to grant-absent or +group-unconfigured. + +```rust +pub async fn restore_target( + &self, + base_url: &Url, + group: Uuid, +) -> Result { ... } +``` + +### A.4 `CanopyClient::restore_verification(base, &RestoreVerification) -> Result<()>` + +Posts to canopy's new ingest endpoint. 204 on success; surface +4xx body as error. + +```rust +pub async fn restore_verification( + &self, + base_url: &Url, + report: &RestoreVerification<'_>, +) -> Result<()> { ... } +``` + +### A.5 What does NOT change in bestool + +- `bestool-kopia` — no changes. `proxy::spawn`, `CredentialProvider`, + `Credentials`, `TrafficStats` are exactly what pgro consumes. +- `CanopyClient::new(...)` — already accepts `device_key_pem: + Option<&str>`, so pgro's tailscale-only operator works as-is. +- `Purpose::Restore` — already there. +- `BackupCredentials` / `BackupTarget` shapes — pgro consumes these + verbatim; please don't reshape them mid-flight (see §4.5). + +### A.6 Suggested release shape + +One bestool-canopy minor version bump containing all four additions, +landing after canopy's endpoints exist on at least a feature branch +with frozen wire shapes. Tag and publish; pgro depends on `^X.Y`. + +--- + +## Next round + +Once §4 + Appendix A have shipped, ping pgro. pgro will: + +1. Read the as-implemented wire + types (any drift from this doc is + fine, just needs to be visible). +2. Re-evaluate the open questions in + `pgro/docs/canopy-backup-integration.md` and tighten the spec to + match what canopy actually shipped. +3. Start building Part 1 (canopy client wiring + CRD field + sidecar + image) and Part 2 (the restore-verification reporter) against the + real surfaces. From 8d11b04b581fd2d23c0fe7fe139843c48903f16a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Wed, 1 Jul 2026 08:48:58 +1200 Subject: [PATCH 03/20] docs(canopy): rewrite integration spec for shipped inverted model + sign-off response Canopy and bestool both shipped their sides of the restore-replicas integration. Spec rewritten end-to-end against the as-shipped wire shapes; pgro's role is now a worklist-reconciler with cluster-as-state (labelled namespaces) rather than CRD-driven. Legacy kopiaSecretRef path is untouched. The handoff response captures pgro's sign-off on canopy's design inversion (canopy owns desired state, pgro reconciles) and resolution of the unsupported-intent question (via /restore-capabilities). --- docs/canopy-backup-integration.md | 1386 ++++++++++++----------------- docs/canopy-handoff-response.md | 81 ++ 2 files changed, 673 insertions(+), 794 deletions(-) create mode 100644 docs/canopy-handoff-response.md diff --git a/docs/canopy-backup-integration.md b/docs/canopy-backup-integration.md index 941a4d5..c377302 100644 --- a/docs/canopy-backup-integration.md +++ b/docs/canopy-backup-integration.md @@ -1,840 +1,638 @@ # Canopy backup-credentials integration -Implementation spec for the pgro side of the Canopy "backup-credentials" -system. Canopy is BES's backup control plane: it issues short-lived, -per-group S3 credentials for kopia repositories, owns repo maintenance and -retention, and tracks backup health through three signals — *backed up* (1), -*persisted* (2), and **restorable** (3). pgro is the producer of signal 3. - -The authoritative design lives in the canopy repo -(`docs/plans/backup-credentials.md`, section "External restore consumers + -restore-verification (PGRO)"). This document is the pgro-side contract and -implementation plan; it does not re-decide canopy-side shape. - -> **Stage.** This is an *additive, later-stage* change in the overall -> rollout. It depends on canopy having already shipped: per-bucket roles, -> the `restore` session-policy path, repo-password ownership, the -> issues/events alerting, and a new non-server-bound device role. -> Until those exist on the canopy side there is nothing for pgro to call. -> Build pgro's side behind the CRD opt-in below so today's static-Secret -> path keeps working unchanged during migration. +pgro's side of the Canopy "restore-replicas" system. Canopy is the source +of truth for which postgres replicas should exist; pgro reconciles +cluster state against canopy's declaration set and reports per-replica +restore-verification (signal 3) back. This document is pgro's +implementation contract, grounded against: + +- canopy `crates/public-server/src/restore.rs` on `origin/main` (the + shipped endpoints: `restore-capabilities`, `restore-worklist`, + `restore-credentials`, `restore-verification`). +- `bestool-canopy` 0.4.2 (the published client, our cargo dep). +- `bestool-kopia` 0.3.3 (the S3P loopback re-signing proxy, our cargo + dep — unchanged for this integration). + +History note: this spec was rewritten 2026-06-30 after the design +inverted (canopy became the desired-state owner) and after canopy + +bestool both shipped their sides. The earlier handoff (`canopy-handoff.md`) +and pgro's sign-off (`canopy-handoff-response.md`) record how we got here; +this doc supersedes both for *implementation* purposes. + +--- ## What changes, in one paragraph -Today a `PostgresPhysicalReplica` carries a `kopiaSecretRef` pointing at a -hand-created `kopia-credentials` Secret that holds **long-lived** AWS access -keys plus the kopia repository password (`src/kopia.rs`, `REQUIRED_KEYS`). The -operator copies those values into env vars and CLI flags -(`--access-key=`/`--secret-access-key=`/`--password=`) on every snapshot-list -and restore Job (`src/controllers/replica/resources.rs`, -`src/controllers/restore/builders.rs`, `src/kopia.rs::kopia_connect_args`). -This is exactly the long-lived-creds pattern the canopy system exists to -eliminate. The integration replaces that static Secret with a -**canopy-mediated, proxy-mediated** flow: the replica references a **canopy -group** instead of a Secret; alongside each kopia Job runs a -**bestool-kopia loopback SigV4 re-signing proxy** (the same S3P proxy -canopy's own maintenance Jobs and bestool's device backups/restores use) -that fetches short-lived restore credentials + the S3 target + repo password -from canopy, refreshes them transparently between requests, and re-signs -each kopia S3 request with the live creds; kopia itself talks only to -`127.0.0.1` with dummy keys, never touching real AWS material. After each -restore the operator **reports the outcome back to canopy** (signal 3, -restore-verification). pgro is a **restore-only** consumer — it never -writes to or deletes from the bucket. - -## Why canopy-mediated (not pgro assuming a role directly) - -The relationship is **bidirectional**: creds flow out (canopy → pgro) and -restore-verification reports flow in (pgro → canopy). Routing both over one -first-party channel keeps it a single relationship rather than two disjoint -mechanisms (an AWS role pgro assumes + a separate reporting path). pgro is -trusted first-party infra but is **not** a fleet device and is **not** a -member of the group it reads, so canopy gates pgro behind an -operator-authorized, audited **external-restore grant** ("consumer C may -read group X, read-only") — a deliberate, controlled cousin of the -cross-group restore that is banned for devices. pgro does not implement the -grant; it just authenticates as itself and names the group. +Today a `PostgresPhysicalReplica` CR with a `kopiaSecretRef` is the +operator-authored unit of work. pgro's controllers reconcile against it, +fetch the kopia secret, run snapshot-list + restore Jobs with long-lived +AWS keys + the kopia repo password baked into env vars. The new +"canopy-backed" path inverts ownership: an operator declares a +**restore-replica** in canopy's operator UI (`(group, server | all, +type, intent, name, freshness)`), and pgro discovers that declaration +via `GET /restore-worklist`, materialises a labelled k8s `Namespace` per +declaration, drives a kopia Job whose data-path goes through a +**bestool S3P loopback proxy sidecar** (kopia talks to `[::1]` with +dummy keys; the sidecar holds refreshing STS creds), and reports the +outcome to canopy via `POST /restore-verification` (signal 3, +group-level restorability alert). The legacy `kopiaSecretRef` CR path +is untouched — both coexist; new replicas should be declared in canopy. --- -## Part 1 — Creds out (Canopy → pgro) - -### 1.1 Canopy client: depend on `bestool-canopy` - -The bestool repo already publishes a `bestool-canopy` crate (`crates.io`, -currently `0.4.0`) that owns the canopy HTTP wire types and the -`CanopyClient`. pgro takes a normal `cargo` dependency on it rather than -re-implementing. The same crate is also used by the proxy's credential -provider (`bestool-kopia`, see §1.3), so depending on it is on the path -either way. - -Wire types pgro consumes verbatim from `bestool-canopy::backup`: - -- `BackupCredentials` — the `credential_process`-shaped response from `POST - /backup-credentials`: `Version, AccessKeyId, SecretAccessKey, SessionToken, - Expiration`. PascalCase on the wire; the crate already handles that. -- `BackupTarget` — `{ storage, bucket, prefix, region, repo_password }` from - `GET /backup-target`. -- `BackupReport` — the `POST /backup-report` body. **pgro does not use this - type for signal-3** (see §2 for why a separate endpoint is needed); it is - named here only because it is the de-facto shape of a run report and pgro's - `RestoreReport` will resemble it for the fields they share. -- `Purpose::Restore` — the enum variant pgro passes to `backup_credentials`. - -Client functions pgro calls on the published `CanopyClient`: - -- `client.backup_credentials(base_url, backup_type, Purpose::Restore)` — - authenticates pgro (see Part 3), returns `BackupCredentials`. Canopy maps - the consumer + group + type to its per-bucket role, assumes it - cross-account with the read-only restore **session policy**, and returns - prod-account creds for that bucket. A `403`/`409` (grant absent or group - unconfigured) is a surfaced error, not a transient retry. -- `client.backup_target(base_url)` — returns `BackupTarget` (or `Dormant`). -- The credential provider in the proxy (§1.3) drives both; pgro's operator - code does not call them directly. - -**Caveats forcing canopy-side changes before pgro can use the published -crate as-is:** - -- `CanopyClient` today is device-mTLS-only (constructor takes a - `device_key_pem`) — which **is fine for pgro**, because canopy already - has non-server-bound device roles (`releaser-device`, `admin-device`) - alongside `server-device`. pgro becomes a device of a new role (working - title `backup-restore`); the existing `CanopyClient` constructor works - unchanged. See Part 3 for the auth design. -- `BackupCredentialsRequest` is `{ type, purpose }` — no `group` field. The - external-restore path needs canopy to either add a `group` field or - expose a sibling endpoint that takes one (a `backup-restore`-role device - has no implicit group, unlike `server-device`). Canopy-owed wire change, - not a pgro one. - -The base URL + device key/cert come from operator-level config (see §3.1), -held on the `Context` (`src/context.rs`) alongside the existing -`http_client`. A small `src/canopy.rs` wires the `CanopyClient` into the -operator's context and holds the report path (§2). - -### 1.2 CRD change: reference a canopy group, not a Secret - -Add an alternative to `kopiaSecretRef` on `PostgresPhysicalReplicaSpec` -(`src/types/replica.rs`). The static-Secret path stays valid for migration -and for non-canopy repos (e.g. the minio integration-test repo), so make the -two mutually-exclusive and keep `kopia_secret_ref` optional: +## Architecture overview + +### Identity and transport + +- The operator pod runs a **Tailscale sidecar**. One canopy device, + role `backup-restore` (canopy-side enum value), promoted from + `Untrusted` once by an operator. Kopia Job pods are **not** on the + tailnet and are not canopy devices. +- `bestool-canopy::CanopyClient` auto-probes the tailnet URL + (`canopy.tail53aef.ts.net`) and falls back to mTLS if a + `device_key_pem` was provided. pgro is tailnet-primary; mTLS cert is + optional and used only if provided. +- The operator's reqwest client is built with + `Proxy::all("socks5://[::1]:1055")` so the auto-probe succeeds + via the Tailscale sidecar. + +### Control loop + +A new third controller (`src/controllers/canopy.rs`) ticks on a jittered +interval (~30s), fetches `restore_worklist()`, lists `Namespaces` with +the pgro-managed label, and reconciles the diff: + +- worklist entry without a namespace → provision. +- namespace without a worklist entry → tear down. +- both present, freshness exceeded or snapshot id changed → refresh. + +No CRDs are involved on the canopy-backed path. Cluster state — the +labelled `Namespace` plus the objects inside it — is the runtime model; +the worklist is the desired state. + +### Data path (kopia) + +Each kopia Job pod has two containers: + +- the kopia container, invoked with `--endpoint=[::1]:`, + `--disable-tls`, dummy access/secret keys, the canopy bucket/region/ + prefix from the worklist entry, and `--password=` set to + `RestoreCredentials.repo_password`; +- a pgro-published **proxy sidecar** (new binary, see §6.2) linking + `bestool-kopia::proxy::spawn` with a pgro `CredentialProvider` + implementation that calls the operator's in-cluster + **credential broker** (§4) — *not* canopy directly. The broker hands + back `BackupCredentials` (the AWS `credential_process` shape) that + the proxy uses for re-signing. Refresh margin: ~2 minutes before + expiry, matching bestool. + +Consequences of the proxy model: +- kopia never sees real AWS material — only dummy keys and a loopback + endpoint. +- Long restores survive credential expiry: the proxy refreshes between + requests, so the 1-hour STS cap is bounded by canopy reachability, + not by any single issuance. +- The proxy binds a loopback literal in userspace mode; no `NET_ADMIN` + for the sidecar. + +### Reporting (signal 3) + +After each restore reaches a terminal phase (`Active` / `Failed` / +deployment-never-ready), pgro builds a `RestoreVerification` and POSTs +it to `/restore-verification`. The report is at-most-once-per-restore, +tracked by an annotation on the replica's Namespace; canopy unreachable +→ retry next reconcile, never fail the restore for the report. + +### Capability registration + +On operator startup (and on any later change to the supported set), the +operator POSTs `/restore-capabilities` with the intents pgro implements: +`verify`, `analytics`, `disaster-recovery`. Canopy then only dispatches +worklist entries whose intent is in the set; declarations stranded on +unsupported intents become operator-facing configuration *gaps*, not +restore-health failures. -```rust -/// Reference to a Secret containing kopia repository credentials. -/// Mutually exclusive with `canopyBackup`. One of the two is required. -#[serde(default, skip_serializing_if = "Option::is_none")] -pub kopia_secret_ref: Option, - -/// Fetch short-lived read-only restore credentials from Canopy instead of -/// a static Secret. Mutually exclusive with `kopiaSecretRef`. -#[serde(default, skip_serializing_if = "Option::is_none")] -pub canopy_backup: Option, -``` +--- -```rust -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct CanopyBackupRef { - /// The Canopy server-group id (UUID) whose backups this replica restores. - pub group: String, - // Canopy base URL + auth come from operator-level config, not per-replica, - // so every replica trusts the same canopy. (Open question 3.) -} -``` +## Part 1 — Canopy client wiring + +### 1.1 Operator-level config (`operator.yaml` + ConfigMap) + +Plumb via env on the Deployment and/or the existing +`postgres-restore-operator-config` ConfigMap that `src/bin/operator.rs` +already reads: + +- `CANOPY_BASE_URL` — public-mTLS base URL (used only when the tailnet + probe fails); the tailnet URL is hardcoded in `bestool-canopy`. +- `CANOPY_DEVICE_CERT_SECRET` — optional. Name of a k8s Secret with + cert + key, mounted only into the operator pod. Skip entirely for + tailscale-only operation. +- `CANOPY_RECONCILE_INTERVAL_SECS` — default 30. The worklist-syncer + tick cadence; jittered ±20% to avoid stampedes. + +### 1.2 `bestool-canopy::CanopyClient` construction + +In `src/canopy.rs` (new module): + +- Construct a `reqwest::Client::builder()` with + `Proxy::all("socks5://[::1]:1055")` and pass the closure to + `CanopyClient::new(tamanu_version, device_key_pem_opt, builder)`. +- Stash the `CanopyClient` plus its base `Url` on the `Context` + (`src/context.rs`) alongside the existing `http_client`. +- Expose thin async methods: `worklist()`, `restore_credentials(group, + type)`, `restore_verification(&report)`, `restore_capabilities(&intents)`. + Each is a one-line forward to the underlying `CanopyClient` method; + the wrapper exists for testability (it's the seam where the + worklist-syncer integration tests inject a stub). + +### 1.3 Tailscale sidecar on the operator Pod + +`operator.yaml` Deployment gains a second container, ideally via the +Tailscale k8s operator's `ProxyClass` annotation. Image +`ghcr.io/tailscale/tailscale`, userspace mode (`TS_USERSPACE=true`), +`TS_SOCKS5_SERVER=:1055`, state in `kube:` mode so a restart resumes +the same identity. Authkey OAuth-issued by ops, the node tagged for +ACL admission (tag does not bind canopy role — canopy resolves the +role from the device record after promotion). + +**IPv6 verification point**: pgro's k8s cluster is IPv6-only internally +(`[[ipv6-only-cluster]]`), so the Tailscale sidecar's SOCKS5 listener +must bind v6. `TS_SOCKS5_SERVER=:1055` binds all interfaces by default +in Go's net package (dual-stack), so it should work as-is, but verify +once the sidecar is up that `nc -6 ::1 1055` from the operator +container reaches it. If not, set `TS_SOCKS5_SERVER=[::]:1055` explicitly. -Making `kopia_secret_ref` `Option` is a CRD change touching every existing -construction site (`controllers/replica/{resources,tests}.rs`, -`controllers/restore/tests.rs`, `controllers/replica/schema_migration.rs`). -Validate "exactly one of `kopiaSecretRef` / `canopyBackup`" in the replica -reconcile (`src/controllers/replica.rs`) and surface a `Warning` event + -`Failed` phase on violation, mirroring how an invalid kopia secret is handled -today (`Error::InvalidKopiaSecret`). - -Per `AGENTS.md`: a CRD spec change **must** update the README CRD tables and -regenerate CRDs (`cargo run --bin gen-crds > crds.yaml`). - -### 1.3 Where the creds are consumed — the proxy-sidecar architecture - -The bestool + canopy family has already solved this and pgro joins the -solution rather than picking a new one. Every kopia caller in the family — -bestool device backups/restores, canopy's maintenance/inspection/s3-metrics -Jobs — runs kopia through a small loopback **SigV4 re-signing proxy** (the -"S3P" spec at `bestool/.workhorse/specs/canopy/s3-sigv4-proxy.md`, -implementation in `bestool/crates/kopia/src/proxy.rs`). The shape: - -- kopia is invoked with `--endpoint=127.0.0.1:`, `--disable-tls`, and - meaningless **dummy** access/secret keys - (`bestool_kopia::PROXY_DUMMY_ACCESS_KEY` / `PROXY_DUMMY_SECRET_KEY`). It - carries no real AWS material at all. -- A small in-process HTTP proxy bound to that loopback port discards kopia's - dummy signature, re-signs each request with the **current** STS creds, and - forwards over TLS to the real S3 host. Streaming bodies (chunked SigV4 - uploads) are re-signed chunk by chunk. -- The proxy's `CredentialProvider` (a trait in `bestool_kopia::proxy`) - refreshes creds out-of-band as they near expiry. bestool's - `CanopyCredentialProvider` - (`bestool/crates/bestool/src/actions/canopy/backup/provider.rs`) hits - canopy's `POST /backup-credentials` directly. pgro's provider plugs the - same trait but hits the **operator's in-cluster credential broker** - (Part 3.1.2) rather than canopy directly — because only the operator - pod is a canopy device. Same ~2-minute refresh-margin shape. -- The kopia repo password (from canopy's `GET /backup-target`) is - fetched once by the operator before Job-spawn and passed into the Job - pod via env / argv. The proxy sidecar passes it to kopia via - `--password=` on `connect`. - -Under this model the credential lifetime is invisible to kopia: a refresh -between two requests changes only the signing key the next one uses. A long -restore is bounded by **canopy reachability**, not by any single credential -lifetime — so the 1-hour STS cap is a non-issue (Open Q 1 collapses), and -kopia's `AWS_SESSION_TOKEN` handling is irrelevant because kopia never sees -real AWS creds (Open Q 2 collapses). - -**Where the proxy runs.** A **sidecar container in each kopia Job** — -mirroring bestool's one-proxy-per-op model. The operator templates the Job -spec to have two containers in one Pod: - -- the kopia container, pointed at `127.0.0.1:` with `--disable-tls`, - the dummy keys, `--bucket`/`--prefix`/`--region` from the canopy target, - and `--password=` set to the canopy repo password; -- a pgro-owned tiny sidecar binary linking `bestool-kopia` (for - `proxy::spawn`) and a pgro-implemented `CredentialProvider` that calls - the **operator's in-cluster credential-broker endpoint** (Part 3.1.2) - rather than canopy directly. The sidecar process binds the loopback - port, runs the proxy until the kopia container exits, and exits - itself. Built into a pgro-published image; both bestool crates are - ordinary cargo deps. - -Loopback-only is a security invariant of S3P (§Security in the spec): the -proxy binds a loopback literal and is never exposed off-host. Sharing a Pod -puts the kopia container and the sidecar in one network namespace, which -satisfies that invariant. A k8s `Service`-fronted proxy would not, and is -ruled out. - -The connect args helper (`src/kopia.rs::kopia_connect_args`) gains a -"canopy" variant emitting `--endpoint=127.0.0.1:`, `--disable-tls`, -the dummy keys, and the canopy bucket/region/prefix — the legacy -`KopiaCredentials` (static keys, optional minio endpoint) stays -unmodified for the non-canopy path. - -Snapshot-list Jobs use the same sidecar shape; the proxy is just as cheap -to spawn for a short op. Confirms Open Q 6. - -### 1.4 Drop the static keys - -Once a replica uses `canopyBackup`: -- No `accessKeyId` / `secretAccessKey` / `repositoryPassword` ever live in a - user-managed Secret. The Job container running kopia **never sees real AWS - credentials** — only the loopback proxy endpoint and meaningless dummy - keys. The real creds live in the sidecar's process memory, refreshed - transparently, and are never written to disk or to a k8s object. -- The repo password is fetched per-Job from canopy's `GET /backup-target` - and passed to kopia via `--password=` on `connect`; it is also held only - in the sidecar's process memory (and kopia's argv for the lifetime of the - Job — same posture as bestool). -- `REQUIRED_KEYS` in `src/kopia.rs` still governs the legacy static path; - the canopy path skips `validate_kopia_secret` entirely. +--- + +## Part 2 — Worklist syncer (the new controller) + +### 2.1 Controller shape + +`src/controllers/canopy.rs`. Not kube-rs CRD-watched (no CR for the +canopy path); instead a `tokio::time::interval` loop that: + +1. Calls `ctx.canopy.worklist().await` — returns `Vec`. +2. Lists `Namespaces` with label + `pgro.bes.au/managed-by=pgro-canopy`. Discovery is the same on + every tick; no in-memory cache across ticks beyond what's needed + inside the function. +3. Computes the diff: worklist entries indexed by `replica_id`, + namespaces indexed by `pgro.bes.au/declaration-id` annotation. +4. Concurrently dispatches per-entry reconciliation; bounds concurrency + to avoid k8s-apiserver thundering herd (~8 in-flight). + +The loop is single-threaded by construction; if a tick is still running +when the next one fires, the next is skipped (`tokio::time::interval` +default behaviour is fine). + +### 2.2 Cluster-as-state model + +For each worklist entry, the per-replica state lives in a Namespace: + +**Namespace name**: `-` where: +- `slug` is `WorklistEntry.name` slugified: lowercased, non-alphanumeric + runs collapsed to `-`, trailing `-` trimmed, truncated to 50 chars. The + declaration name is operator-set in canopy's UI (e.g. `Nauru prod + analytics`) — already the human-recognisable label for this replica. +- `hex` is 8 hex chars from `SHA-256(replica_id || server_id)[..4]`. Both + ids are needed because a `server_id=NULL` declaration in canopy expands + to one `WorklistEntry` per live server in the group, all sharing + `replica_id` and `name` (`canopy/crates/public-server/src/restore.rs:175-211`). + +Example: `nauru-prod-analytics-a3f9c2d1`. DNS-1123 valid; ≤59 chars. + +**Labels** (immutable): +- `pgro.bes.au/managed-by=pgro-canopy` — discovery key. +- `pgro.bes.au/declaration-id=` +- `pgro.bes.au/group=` +- `pgro.bes.au/server=` +- `pgro.bes.au/intent=verify|analytics|disaster-recovery` + +**Annotations** (mutable, runtime state): +- `pgro.bes.au/desired-snapshot-id` / `pgro.bes.au/desired-snapshot-at` — + the worklist's snapshot for this entry on the last successful sync. +- `pgro.bes.au/last-restored-snapshot-id` / `pgro.bes.au/last-restored-at` +- `pgro.bes.au/restore-state` = `pending|restoring|active|failed` +- `pgro.bes.au/last-verification-reported-at` (RFC3339) — + at-most-once gate; absent means owed. +- `pgro.bes.au/last-verification-error` — last report failure string. + +Other namespace contents (Deployment, Service, PVC, current restore +Job) are conventional k8s objects; they carry their own pgro labels for +discovery but the Namespace is the authoritative root. + +### 2.3 Provisioning a replica (worklist entry, no namespace) + +1. Create the Namespace with labels + initial annotations + (`restore-state=pending`, `desired-snapshot-id` from worklist). +2. Create a PVC sized per intent (default per-intent table; overridable + by ConfigMap entry). +3. Create the restore Job (§6 — same Job builder shape as legacy, with + the sidecar variant). +4. Subsequent ticks observe the Job and update `restore-state`. On + Job-success: bring up the postgres Deployment, run readiness gate. + On Job-failure: mark `restore-state=failed`, store the error in an + annotation, schedule retry per error policy. + +### 2.4 Refreshing (entry + namespace both present) + +Refresh triggers, in order of precedence: +- `desired-snapshot-id` from worklist differs from + `last-restored-snapshot-id` — newer backup available. +- `now > last_restored_at + freshness_seconds` — overdue. +- Manual: an operator annotates the Namespace with + `pgro.bes.au/force-refresh=now` (last-resort escape hatch). + +The refresh creates a new restore Job alongside the existing +Deployment, drains traffic on success, swaps Service endpoints, deletes +the old PVC. Intent-specific cutover policy lives in the existing +`controllers/restore.rs` switching machinery — reused unchanged. + +### 2.5 Teardown (namespace, no entry) + +The worklist no longer references this replica. Mark +`restore-state=terminating`, drain queries (intent-specific grace +period), delete the Deployment + Service + PVCs, finally delete the +Namespace. A k8s finalizer on the Namespace blocks deletion until pgro +has confirmed teardown; this is the only finalizer pgro adds. + +### 2.6 Worklist-fetch failure handling + +Treat any error from `restore_worklist()` as transient. Log + metric +the failure, skip the tick, continue. **Do not** tear down replicas +because the worklist was empty due to a fetch error; the tick is a +no-op if the worklist can't be retrieved. Persistent failure for >5 +minutes raises a pgro-side metric/alert (no canopy-side alert path +exists for "consumer can't read its worklist"). --- -## Part 2 — Reports in (pgro → Canopy): Signal 3, restore-verification - -A successful pgro restore *proves the backup is restorable* — the strongest -backup-health signal there is, stronger than signal 2's "a snapshot exists in -the repo". pgro reports each restore outcome to canopy, which records it in -its `backup_restore_checks` table and reconciles it against -`backup_repo_snapshots` / `backup_runs`. A failed or stale restorability -check becomes a high-severity **group-level** alert on the canopy side (the -server-independent incident path, like poisoning detection) — pgro does not -manage alerting, it only reports. - -### 2.0 Why not reuse `POST /backup-report` - -Canopy already serves `POST /backup-report` and it already accepts -`{ purpose: "restore", outcome, snapshot_id, error, run_id, ... }` — for -**devices**. The shape looks superficially close to what pgro wants, so it -is worth being explicit about why pgro needs a **new** ingest endpoint -rather than reusing `/backup-report`: - -1. **Identity model is upside-down.** The handler resolves `device_id`, - `server_id`, and `group_id` from the **authenticated mTLS context** - (`canopy/crates/public-server/src/backup.rs:495`), not the request body. - pgro is none of those — it has no `device_id`, no `server_id`, no - implicit group. Wiring a `group_id` into the body would break the - security invariant that an authenticated device cannot report a run as - some *other* group. -2. **The schema is device-shaped.** `backup_runs` has - `device_id UUID NOT NULL REFERENCES devices(id)` and - `group_id NOT NULL REFERENCES server_groups(id)`. pgro reports have no - device; the FK can't be satisfied. The only honest options are pollute - the table with sentinel devices, drop the FK, or add a separate table — - which is exactly the `backup_restore_checks` the canopy plan names. -3. **Two different "restore" meanings collide on `purpose`.** A device with - `purpose=restore` (`bestool canopy restore`, used for clone / DR-test on - the same fleet) writes its own outcome into `backup_runs`. That is NOT a - signal-3 verification — it is a normal device-side restore and should - not raise a group-level "the backup isn't restorable" incident. So - `purpose=restore` alone is not a sufficient discriminator. -4. **Alerting paths diverge.** A `/backup-report` failure feeds **per-server - staleness** (signal 1, server-scoped). Signal 3 must feed **group-scoped** - `raise_group_event(ref = "restore-verification", severity = Error)` - bypassing per-server `is_monitored`. Reusing the endpoint means branching - inside the handler on actor type — at which point you have already forked - it. -5. **Side-effects don't match.** The handler clears `BackupRequest` on every - report (`backup.rs:534`) so the heartbeat stops re-emitting "back up now" - for that server. Irrelevant and possibly harmful for a pgro report; - another conditional. -6. **Payload shape is wrong for signal 3.** `ReportArgs` carries - `bytes_uploaded` and `s3_*_bytes` (pgro's proxy will emit the same - tallies, fine) but lacks `replica_healthy`, postgres major version, and - `observed_at` — the very state that makes signal 3 stronger than signal - 2. Squeezing those into `error: Option` loses structure for the - load-bearing fields. -7. **`run_id` semantics don't transfer.** For devices, `run_id` is the same - UUID across `/backup-credentials` (issuance audit) and `/backup-report`, - minted at run start, duplicate → 409. For pgro the meaningful identity - is the **snapshot being verified**, not the verifier's run UUID; a - pgro-minted run UUID has no cross-table linkage to - `backup_credential_issuances` (pgro's issuances aren't even in that - table — different consumer type). - -By the time canopy has added `group_id` to the body, relaxed (or split off) -the FKs, branched the handler on actor type, routed failures to a different -alerting path, and gated the `BackupRequest::clear` side-effect, the handler -has effectively forked. Cleaner for canopy to expose a sibling endpoint -(working title `POST /restore-verification`, canopy-side naming theirs) -writing to a new `backup_restore_checks` table and routing failures through -`raise_group_event`. The body shape pgro proposes is in §2.1. - -### 2.1 Report shape (`RestoreReport`) - -pgro already has all of this in `PostgresPhysicalRestoreStatus` and the -`NotificationPayload` machinery (`src/notifications.rs`); this is a new -notification *target*, not new data collection. +## Part 3 — Restore-verification reporter (signal 3) -```rust -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct RestoreReport { - /// The canopy group this replica restores from (CanopyBackupRef.group). - pub group: String, - /// The kopia snapshot id that was restored — the cross-reference key - /// canopy joins against backup_repo_snapshots / backup_runs. - pub snapshot_id: String, - /// "success" | "failure". - pub outcome: RestoreOutcome, - /// Populated on failure (restore-job failure, deployment-never-ready, - /// detected postgres version mismatch, etc.). - pub error: Option, - /// Replica health after restore: did postgres come up and pass the - /// operator's readiness gate. - pub replica_healthy: bool, - /// Detected postgres major version (status.postgresVersion). - pub postgres_version: Option, - /// When the restore completed (status.restoredAt / activatedAt). - pub observed_at: jiff::Timestamp, -} -``` +### 3.1 When to report + +The restore controller (`src/controllers/restore.rs`) drives a restore +through `Pending → Restoring → Ready → Switching → Active → Failed`. +Hook the report at terminal transitions: + +- `Active` (or `Ready` + readiness gate passed) — `outcome=success`, + `replica_healthy=true`. +- `Failed`, or deployment-never-ready within + `DEPLOYMENT_READY_TIMEOUT_SECS` — `outcome=failure`, `error` + populated. + +### 3.2 Building the body -`snapshot_id` is load-bearing: it is the join key canopy uses to close the -loop *backed up → persisted → restorable*. It maps directly to -`PostgresPhysicalRestoreStatus`-tracked snapshot and to the -`canopy@:` kopia source / `canopy-run` tag model canopy uses -for attribution. - -### 2.2 When pgro reports - -The restore controller (`src/controllers/restore.rs`) already drives a -restore through `Pending → Restoring → Ready → Switching → Active → Failed` -and counts `consecutiveRestoreFailures`. Hook the report at the terminal -transitions: -- **success**: when a restore reaches `Active` (or `Ready` and the deployment - passes the readiness gate) — `outcome=success`, `replica_healthy=true`. -- **failure**: when a restore reaches `Failed`, or the deployment fails to - become ready within `DEPLOYMENT_READY_TIMEOUT_SECS` — `outcome=failure`, - `error` populated. - -Report **at most once per restore** and tolerate canopy being unreachable: -record "reported" in the restore status (a new optional -`status.canopyReportedAt: Time`, mirroring `NotificationStatus`), retry on -the next reconcile if it hasn't been sent. Reporting failures must **never** -fail the restore itself (same posture as notifications today — -`NotificationStatus.last_error` is recorded, the restore proceeds). - -This reuses the existing notification pattern almost exactly. Consider -modelling the canopy report as a built-in notification target rather than a -bespoke path, but keep it operator-configured (Part 3 auth), not per-replica -webhook config — the canopy relationship is first-party and trusted, not an -arbitrary webhook. +From the Namespace's labels (declaration-id, group, server, intent, +type) and the restore's tracked state (snapshot_id from +`desired-snapshot-id`, postgres major version from the Deployment's +status, observed_at from terminal-transition time). S3 byte tallies +come from the proxy sidecar — the sidecar writes them as Job +annotations on its own Job at exit; the reporter reads them when +present (absent if the Job died before the sidecar could write). + +Reuse the published `bestool_canopy::RestoreVerification<'a>` shape +verbatim. Set `replica_id` from the namespace's +`pgro.bes.au/declaration-id` label. + +### 3.3 At-most-once with retry across reconciles + +Gate on the namespace annotation +`pgro.bes.au/last-verification-reported-at`. If set, report has landed. +If absent and a terminal transition has been observed, the +worklist-syncer's per-replica step calls +`ctx.canopy.restore_verification(...)`; on 2xx, stamp the annotation; +on error, record the error in +`pgro.bes.au/last-verification-error` and retry next tick. Reporting +failure **never** fails the restore — same posture as the existing +`notifications.rs` retry shape, but at a different cadence (worklist +tick rather than in-loop sleep). --- -## Part 3 — First-party auth: a new canopy device role - -pgro runs in a different k8s cluster from canopy. There is no shared -cluster, no shared ServiceAccount, no shared Secret. But canopy already -has the right shape for this. Two independent axes: - -### 3.0 Identity: add a new canopy device role - -Canopy's `securitySchemes` defines `server-device`, `releaser-device`, and -`admin-device` — three mTLS device roles, two of which are **not** bound -to a fleet server (`releaser-device` is used by the packaging pipelines, -`admin-device` by operators). Adding a **fourth role** for pgro (working -title `backup-restore-device` — role `backup-restore` — canopy's -naming) reuses that machinery instead of designing fresh federation. - -- mTLS client cert, no server / group binding. Canopy adds it to the role - enum and the `securitySchemes` block. -- The role gates the restore-credentials + restore-verification endpoints. -- **`purpose=backup` is rejected for this role at the API layer.** A - `backup-restore` device that asks `/backup-credentials` with - `purpose=backup` gets a `403`/`409`, not write-capable creds. The - restore-only contract is enforced server-side; pgro doesn't have to ask - nicely, and a compromised pgro can't pivot to writing/poisoning. -- The external-restore grant `(consumer, group, type)` is keyed by the - device cert's identity, the same way device-bound endpoints key off - `device → server → group_id`. -- pgro mounts the cert + key as a k8s Secret (operator pod + each Job's - proxy sidecar). One identity for both directions (creds out, reports in) - — matching the bestool model where `CanopyClient` carries `device_key`. -- Cert provisioning is one-time and operator-driven (no need for the - TPM-bound `canopy register` device-enrolment flow bestool uses); - matches how `releaser-device` certs are issued to packaging pipelines. -- The role is **generic, not pgro-specific**: any future restore-only - consumer (a separate restore-test harness, an external auditor's - read-only verifier, etc.) can use the same role with its own - `(consumer, group, type)` grant. - -This is the lowest lift on the identity side: canopy's role enum + cert -verification + `securitySchemes` plumbing already exists. The cross-cutting -canopy work is the new role itself and the external-restore grant; both -are pgro-blocking either way. - -OIDC workload-identity-federation is a *different* identity model (bearer -token from a federated issuer rather than mTLS). It would be useful for -*other* consumers later (GitHub Actions → canopy and similar), but pgro -doesn't need that generality. Note it here, don't pursue it for pgro. - -### 3.1 Transport: public-mTLS vs Tailscale - -Canopy exposes its API on two transports, and the **authentication -mechanism differs by transport** — they don't layer: - -- **Public-internet, mTLS.** Identity comes from the presented client - certificate; the device-role lookup keys off the cert. This is how - `server-device` / `releaser-device` reach canopy today. -- **Tailscale.** No client cert is presented; identity comes from the - caller's **tailnet identity**, mapped server-side to a canopy device - record. (Canopy's enrollment flow already accommodates this — see - `BeginArgs.spki` in the openapi: required on the tailnet transport - because there's no cert to read the key from, omitted on the mTLS - path where the cert supplies it.) - -The `backup-restore` device role still applies on either transport — it's -a property of the device record, not of the auth mechanism. The choice -determines what pgro mounts and how `CanopyClient` is constructed. - -**`bestool-canopy::CanopyClient` auto-probes.** It tries the hardcoded -tailnet URL (`canopy.tail53aef.ts.net`, `crates/canopy/src/client.rs:38`) -first; if reachable, it uses the tailnet path with plain HTTPS (auth via -tailnet identity). If unreachable, it falls back to the mTLS endpoint -using the provided `device_key_pem`. So pgro doesn't pick at startup — -the client picks per-probe, and `refresh()` re-evaluates. pgro just needs -to (a) make the tailnet reachable from the pod (sidecar — see §3.1.1) or -(b) provide a device cert — or both, in which case Tailscale wins when -present and mTLS catches the gap when it isn't. - -Recommend the **Tailscale** path for production since the operator + -Jobs are already long-lived in-cluster workloads that can sit on the -tailnet, it keeps canopy's public surface area smaller, and it sidesteps -managing a long-lived mTLS keypair as a k8s Secret. Public-mTLS is -viable as a fallback (provision a cert anyway and `CanopyClient` will -use it automatically when the tailnet is unreachable). - -#### 3.1.1 Tailscale sidecar lives on the operator pod only - -Canopy's tailnet auth identifies each caller by **tailscale node -identity** (`commons-servers/src/device_auth/tailnet.rs:52` — it resolves -the source IP via the tailnet directory and keys into -`devices.tailscale_node_id`, auto-creating an `Untrusted` device row on -first contact). Tags are only a coarse admission gate -(`TAILSCALE_REQUIRED_TAG`), not a role assignment. - -So putting a Tailscale sidecar on every Job pod would mean **every Job -pod creates a new `Untrusted` canopy device row** that nobody promotes -to `backup-restore` — and the rows accumulate forever. That doesn't -work. The only workable shape is: - -- **One Tailscale sidecar, on the operator Pod only.** Stable tailnet - node identity, persisted with `TS_STATE_DIR` on a PVC (or via the - Tailscale k8s operator's `kube:` state mode so a restarted operator - resumes the same identity). Image - `ghcr.io/tailscale/tailscale` in userspace mode (`TS_USERSPACE=true`) - exposing `TS_SOCKS5_SERVER=:1055` on `localhost`. Authkey - OAuth-issued by ops, the node tagged for ACL admission (the tag - itself is just admission, not role). -- **The operator becomes a canopy device.** First contact creates an - `Untrusted` row; an operator (human) promotes it once to - `backup-restore`. One device record total. -- **kopia Job pods carry no Tailscale sidecar** and are not canopy - devices. - -#### 3.1.2 The credential broker: operator mediates for Jobs - -`bestool-kopia::proxy::CredentialProvider` is a trait. bestool's -implementation -(`CanopyCredentialProvider`) calls canopy directly. pgro plugs a -**different provider** in the Job-side proxy sidecar — one that calls -the operator pod over the in-cluster network instead: - -- The operator exposes a small in-cluster HTTP endpoint, e.g. - `POST /internal/restore-creds` taking `{group, type}` and returning - the same `BackupCredentials` shape `bestool_canopy` already - deserializes. Backed by a per-(group, type) cache so concurrent Jobs - don't N×-multiply canopy calls. -- The operator (running on the tailnet via its single sidecar) hits - canopy's `POST /backup-credentials` with `purpose=restore`, returns - the result. Canopy still enforces the role + purpose gate - server-side; the operator-broker is purely a transport reuser. -- The Job's proxy sidecar implements `CredentialProvider` against this - endpoint. Refresh cadence is the same (~2 min before expiry). -- Network gating: the broker endpoint is only exposed within the - cluster (no Service.type=LoadBalancer), gated by a NetworkPolicy - restricting access to pgro's own Job pods (matched by label) and the - operator's namespace. - -Restore reports (§2) also flow through the operator — same justification: -one canopy device, one identity. - -Net effect: the Tailscale + canopy-device-role machinery is a property -of the **operator process**, not of each Job. The Jobs see only an -in-cluster HTTP endpoint that happens to broker canopy creds. - -**ops-repo setup, one-time per cluster** (called out here for spec -completeness; actual implementation lives in the ops repo): -- Install the Tailscale k8s operator into pgro's cluster. -- Mint an OAuth client + ACL tag for pgro's operator pod (tag is just - the ACL admission, not the canopy role). -- ACL allows the tag to reach `canopy.tail53aef.ts.net`. -- Canopy-side: after the operator's first contact creates its - `Untrusted` device row, promote it to `backup-restore` (one-time - manual step, same as `releaser-device` promotion today). - -### 3.2 Operator config surface - -Operator-level config, not per-CRD (open question 3.x: per-replica vs -operator-global canopy — recommend operator-global so every replica -trusts one canopy). Plumb via env on the Deployment (`operator.yaml`) -and/or the existing `postgres-restore-operator-config` ConfigMap that -`src/bin/operator.rs` already watches (`read_config`): -- `CANOPY_BASE_URL` — the public-mTLS endpoint, used by `CanopyClient` - as the mTLS-fallback target. The tailnet hostname is hardcoded in - `bestool-canopy`, not configurable here. -- `CANOPY_DEVICE_CERT_SECRET` — optional. Name of a k8s Secret with a - pgro device cert + key, mounted only into the operator pod. Used by - `CanopyClient` when the tailnet probe fails. Skip entirely if the - operator is Tailscale-only. -- `PGRO_CREDENTIAL_BROKER_ADDR` — the cluster-internal listen address - the operator exposes its `/internal/restore-creds` endpoint on. The - Job builder propagates the matching Service URL into each kopia - Job's proxy sidecar. - -The Tailscale sidecar is wired only on the operator Deployment in -`operator.yaml` — annotation-driven if the Tailscale k8s operator's -`ProxyClass` is used, otherwise as an explicit second container. The -kopia Job spec has **no** Tailscale sidecar. +## Part 4 — Credential broker (operator-side HTTP endpoint) + +### 4.1 The endpoint + +Add `POST /internal/restore-creds` to the existing axum router in +`src/bin/operator.rs:307 build_router`. Body: `{ group: Uuid, type: +String }`. Response: the verbatim `RestoreCredentials` body from +`bestool_canopy::restore_credentials` (creds + repo_password). 4xx +errors mirror canopy's; the broker is a transport reuser, not an +authorizer (canopy's `RestoreReplica::authorizes` is the source of +truth, gating the upstream call). + +### 4.2 Caching + +A small per-(group, type) cache in-process keyed by `(group, type)`, +storing the last response with its `Expiration`. Concurrent Job +sidecars asking close together should not multiply canopy calls. +Cache entries expire 2 minutes before the `Expiration` (same margin as +the sidecar). Cache is best-effort — no persistence; a restart +forgets — that's fine, the sidecar will just see a brief refresh. + +### 4.3 In-cluster gating + +A `NetworkPolicy` in `operator.yaml` restricts ingress to the broker +port to pods carrying `pgro.bes.au/proxy-sidecar=true` in the same +cluster. The Job builder labels the sidecar containers' pods +accordingly; nothing else in the cluster can hit the broker. The +operator's other endpoints (`/metrics`, `/livez`, etc.) live on the +same listener; the policy gates by path is not feasible at the +NetworkPolicy layer — separating ports is the easiest robust gate. +**Decision:** the broker binds a **separate port** from the existing +metrics router (e.g. `:9091` vs `:9090`); NetworkPolicy gates the +broker port to proxy-sidecar pods, leaves metrics open to the +Prometheus selectors. --- -## IaC / deployment changes (`operator.yaml` + ops repo) +## Part 5 — Capability registration + +### 5.1 On startup -- **No more user-managed `kopia-credentials` Secret** for canopy-backed - replicas. Operators stop hand-creating it; the static-Secret path remains - only for legacy/non-canopy repos. -- **New pgro-published sidecar image** linking `bestool-kopia` (proxy) and - `bestool-canopy` (HTTP client + credential provider). Built and pushed - alongside the existing operator image; pinned by tag in `operator.yaml` - via env (e.g. `CANOPY_PROXY_SIDECAR_IMAGE`). The Job builder injects this - container into every kopia Job for canopy-backed replicas. -- **Tailscale sidecar on the operator pod only** — one stable tailnet - node identity → one canopy device. Image - `ghcr.io/tailscale/tailscale` in userspace mode with - `TS_SOCKS5_SERVER=:1055`; auth via OAuth-issued authkey carrying the - pgro ACL tag (admission, not role). Provisioned either by the - Tailscale k8s operator's `ProxyClass` injection (preferred) or as an - explicit second container in the operator Deployment. -- **kopia Job pods have NO Tailscale sidecar**. They reach the - operator's in-cluster credential-broker endpoint - (`/internal/restore-creds`) via a normal cluster Service. The - operator forwards to canopy on their behalf. -- pgro's `reqwest` client (in the operator) is built with - `Proxy::all("socks5://localhost:1055")` so `CanopyClient`'s - auto-probe of `canopy.tail53aef.ts.net` succeeds via the sidecar. -- **Optional public-mTLS fallback**: a `CANOPY_DEVICE_CERT_SECRET` - mounted into the operator pod only. `CanopyClient` uses it - automatically when the tailnet is unreachable. Skip the cert entirely - if pgro is comfortable being Tailscale-only. -- **NetworkPolicy gating** the credential-broker endpoint: only pgro's - own Jobs (matched by label) in the operator's namespace can hit it. -- **RBAC delta in `operator.yaml`.** The new path stops writing transient - per-Job Secrets (that was strategy (A) in the old framing — superseded by - the sidecar). The ClusterRole's existing Secret verbs are still required - for the legacy static-Secret path. No new cluster permissions for the - canopy path itself (it's outbound HTTP from operator + sidecar). -- The auth plumbing lives in the **canopy + ops** repos, not pgro: - canopy adds the `backup-restore` role, the external-restore grant, - the purpose-gate, and the one-time promotion of pgro's operator-pod - device record from `Untrusted` to `backup-restore`; ops installs the - Tailscale k8s operator in pgro's cluster, mints the OAuth client + - ACL tag for pgro's operator pod (admission only, not role), and - writes the ACL allowing that tag to reach `canopy.tail53aef.ts.net` - (canopy plan: "ops: the auth plumbing … and PGRO's read-only access path"). +Right after the operator initializes its `CanopyClient` (post-config +load, pre-controller-start), call `restore_capabilities(&["verify", +"analytics", "disaster-recovery"])`. Failure is logged but +non-fatal — the worklist-syncer will start with an empty (or stale) +set on canopy's side, but the next tick that succeeds in +`restore_capabilities` will repopulate. + +### 5.2 Change handling + +The supported intent set is fixed in the operator binary for now (pgro +implements all three). If the set ever shrinks in a future release, +re-push at startup with the smaller set; canopy turns previously-handled +declarations into config gaps automatically. No runtime +intent-toggling. --- -## Interfaces this component exposes / consumes +## Part 6 — Job builders + proxy sidecar -**Consumes (from canopy):** -- `POST /backup-credentials` with `(type, purpose=restore)` plus the - external-restore grant context for `group` — returns `BackupCredentials` - (`bestool_canopy::BackupCredentials`). Called from the sidecar's - credential provider, not from the operator process. -- `GET /backup-target` — returns `BackupTarget` (`bestool_canopy::BackupTarget`). - Called once per Job to populate kopia's `--bucket`/`--region`/`--prefix` - and `--password`. -- A canopy device cert for the new non-server-bound role (Part 3). - -**Consumes (from bestool, as published cargo crates):** -- `bestool-kopia` (≥0.3.2): the S3P proxy + `CredentialProvider` trait. -- `bestool-canopy` (≥0.4.0): the canopy wire types and the `CanopyClient` - (constructed with pgro's device key — unchanged from the bestool usage). - -**Provides (to canopy):** -- `RestoreReport` via a new canopy-side endpoint (working title `POST - /restore-verification`) → canopy `backup_restore_checks` / signal-3 - detection. Cross-referenced by `snapshot_id`. -- The `PostgresPhysicalReplica.spec.canopyBackup.group` field is the operator- - visible contract that binds a replica to a canopy group. - -**Provides (to pgro operators):** -- CRD: `canopyBackup: { group }` as the alternative to `kopiaSecretRef`. -- A new pgro-published sidecar image (canopy S3P proxy + credential - provider) used by every kopia Job for canopy-backed replicas. -- README CRD-table updates documenting it. +### 6.1 `KopiaSource` enum + +Lift the legacy job-builder code in +`src/controllers/restore/builders.rs:508 build_restore_job` and +`src/controllers/replica/resources.rs:51 build_snapshot_list_job` (the +canopy path doesn't need snapshot-list — canopy provides the snapshot — +so only `build_restore_job` matters here) to take a `KopiaSource`: + +```rust +enum KopiaSource { + /// Legacy: env-from-secret + connect-args from validate_kopia_secret. + Secret { kopia_secret: SecretReference, creds: KopiaCredentials }, + /// Canopy: proxy sidecar + dummy keys + broker URL. + CanopyProxy { + broker_url: String, + repo_password: SecretKeySelector, // password from a per-Job Secret materialised by the syncer + bucket: String, + region: String, + prefix: String, + server_id: Uuid, + }, +} +``` + +`kopia_connect_args` (`src/kopia.rs:211`) gains a parallel +`kopia_connect_args_proxy` that emits `--endpoint=[::1]:`, +`--disable-tls`, dummy keys, `--password=$(REPO_PASSWORD)`, +`--override-username=canopy`, `--override-hostname=`. The +legacy connect-args helper is unchanged. + +### 6.2 The proxy sidecar binary + +A new `[[bin]]` in `Cargo.toml`: `canopy-proxy`. Single small Rust +binary linking `bestool-kopia` (for `proxy::spawn`, +`CredentialProvider`, `Credentials`, `TrafficStats`) and `bestool-canopy` +(only for the `BackupCredentials` shape; the sidecar doesn't call canopy +directly). What it does: + +1. Reads `PGRO_BROKER_URL`, `PGRO_GROUP`, `PGRO_TYPE`, `PGRO_REGION`, + `PGRO_LISTEN_PORT` from env. +2. Constructs a `BrokerCredentialProvider` (new pgro type, implements + `bestool_kopia::proxy::CredentialProvider`) that calls + `/internal/restore-creds` with `{group, type}` and + caches per the same 2-minute margin. +3. Calls `bestool_kopia::proxy::spawn` with that provider, the + `S3ProxyConfig { upstream: "https://s3..amazonaws.com", + upstream_host: "s3..amazonaws.com", region }`, bound to + `[::1]:`. +4. Waits on a graceful shutdown signal (the kopia container exiting + should propagate; sidecar's restart policy lets it exit cleanly + after kopia is done). On exit, writes the proxy's `TrafficStats` to + a known annotation on its own Job so the verification reporter can + read it. + +**Upstream dependency — bestool-kopia bind address.** Today +`bestool_kopia::proxy::spawn` hardcodes +`TcpListener::bind(("127.0.0.1", 0))` at +`bestool/crates/kopia/src/proxy.rs:174`. pgro's k8s cluster is +**IPv6-only internally** (`[[ipv6-only-cluster]]`), so the v4 loopback +bind fails. + +Fixed in `beyondessential/bestool#616` ("fix(kopia): bind proxy to +IPv6 loopback, falling back to IPv4"): a new private `bind_loopback()` +helper tries `::1` first, falls back to `127.0.0.1`. No API change, +no caller-side decision — pgro just depends on whatever +`bestool-kopia` version ships next (≥0.3.4 expected). pgro doesn't +pass a bind option. + +This is the only external blocker for the pgro PR; track #616 to +merge + release. + +### 6.3 Job spec changes + +The canopy-variant Job has two containers in one Pod: +- `kopia` (existing image, connect-args from + `kopia_connect_args_proxy`) +- `canopy-proxy` (new pgro-published image) + +Both containers share `network=Pod` so `[::1]` works. Resource +limits: the sidecar is very small (~50Mi memory typical, single core +adequate even for streaming uploads — kopia restore is GET-heavy and +the proxy just re-signs). + +The Pod gets the label `pgro.bes.au/proxy-sidecar=true` so the +NetworkPolicy in §4.3 admits it. + +### 6.4 Image build + release + +Pgro's existing CD workflow (`.github/workflows/cd.yml:74`) builds a +single binary. Update the matrix to build both binaries +(`operator` + `canopy-proxy`) and to push two images, sharing the +multi-arch cross-compile work. Containerfile gains a second stage with +the new entrypoint; image tag aligns with the operator's tag for +release-version coupling. --- -## Testing approach (per `AGENTS.md`) +## Coexistence with the legacy `kopiaSecretRef` path -`AGENTS.md`: add tests with features/fixes; integration tests **don't run -locally** — flag them to the user for CI and **add a matrix entry in -`.github/workflows/integration.yml`** for any new test file. Never write docs -except CRD-table updates. Run `cargo clippy` + `cargo fmt` before committing; -conventional commits; always work on a branch. - -**Unit tests (run locally, alongside the code):** -- `src/canopy.rs`: serialize a `RestoreReport`; verify the auth-mode - selector dispatches correctly. Wire-shape de/serialisation is covered by - `bestool-canopy`'s own tests — pgro re-uses the types, doesn't re-test - them. -- CRD validation: "exactly one of kopiaSecretRef/canopyBackup" — both-set - and neither-set are errors; round-trip `CanopyBackupRef` through serde - (mirroring `schema_migration_phase_roundtrips_*`). -- Job builder: with `canopyBackup`, the connect args carry the proxy - endpoint, dummy keys, and `--disable-tls`; the Job spec has the proxy - sidecar container; legacy path unchanged. Extend the existing - `kopia_connect_args_*` tests for the canopy variant. -- Report gating: `canopyReportedAt` prevents double-reporting; a report - failure does not fail the restore (assert status transitions unaffected). +Untouched. The CRDs `PostgresPhysicalReplica` / `PostgresPhysicalRestore` +keep their existing shapes; the existing CR-driven controllers +(`controllers/replica.rs`, `controllers/restore.rs`, +`controllers/postgres.rs`, `controllers/jobs.rs`) reconcile them +exactly as today. The new worklist-syncer is **additive** — a third +controller that operates on Namespaces with a different label key, +producing Job specs through the same builder code but via the +`KopiaSource::CanopyProxy` variant. -**Integration tests (CI-only; new matrix entry required):** -- A `test-canopy-restore` namespace exercising the canopy path against a - **stub canopy** (a small in-cluster HTTP service returning fixed - creds for the minio repo + accepting reports), so the existing - `tests/fixtures/minio.yaml` + `setup-kopia-repo.yaml` repo can be reused. - Assert: replica reaches `Ready`/`Active` using fetched creds; a - `RestoreReport` with the right `snapshot_id`/`outcome` is POSTed to the - stub. Add the matrix entry in `.github/workflows/integration.yml` and tell - the user it only runs in CI. -- Negative: grant-absent (stub returns 403) → replica surfaces a clear - `Failed`/Warning, doesn't crash-loop. +There is no migration path between the two — operators wishing to +move a replica off the legacy path delete the `PostgresPhysicalReplica` +CR and declare the equivalent restore-replica in canopy's UI. The +legacy `kopia-credentials` Secret is theirs to delete. --- -## Open questions / decisions to make - -1. **Credential lifetime vs. long restores — RESOLVED by the proxy model.** - Chained AssumeRole sessions cap at 1 hour, but the S3P proxy refreshes - creds out-of-band between requests, so kopia is oblivious to credential - lifetime. A long restore is bounded by canopy reachability, not by any - single issuance. Canopy may still want to offer non-chained direct - web-identity creds to first-party consumers for efficiency / fewer - refresh round-trips (and to mirror the maintenance-Job pattern), but it - is a secondary optimisation, not a viability gate. Keep the request on - canopy's list; don't block pgro on it. - -2. **kopia S3 backend + session token — RESOLVED by the proxy model.** - Under the S3P proxy, kopia talks only to `127.0.0.1` with dummy keys; it - never sees a real session token. `--session-token` and - `AWS_SESSION_TOKEN` are both irrelevant on the kopia leg. (For - completeness: kopia *does* support `--session-token`, per the canopy - kopia spike — but the production design doesn't use that path because - the proxy is more general.) - -3. **Per-replica vs operator-global canopy config.** Recommend - operator-global (`CANOPY_BASE_URL` + auth on the Deployment/ConfigMap), so - `CanopyBackupRef` carries only `group`. Revisit if pgro ever needs to talk - to more than one canopy. - -4. **Confirm the new-canopy-device-role + transport choice.** Identity is - a new device role (`backup-restore`, generic and not pgro-specific) - alongside `server`/`releaser`/`admin`, keyed to the external-restore - grant, with `purpose=backup` rejected for the role at the API layer. - The transport choice — canopy's public-mTLS surface or its Tailscale - surface — picks the auth mechanism (mTLS cert vs tailnet identity), not - a layering on top of mTLS. Recommend Tailscale for production. Confirm - both with canopy before pgro builds the auth path. See Part 3. - -5. **Report transport coupling.** Should the canopy report reuse the existing - notification subsystem (a new built-in target) or be a dedicated path on - the restore controller? Recommend a dedicated, operator-configured path — - the canopy relationship is first-party/trusted and bidirectional, unlike - arbitrary user webhooks — but the `NotificationStatus`/retry shape is the - right model to copy. - -6. **Snapshot-list / availability checks.** snapshot-list Jobs also need - repo creds; they use the same proxy-sidecar pattern as restores. Confirm - canopy is happy to serve `purpose=restore` creds for the periodic - snapshot-list cadence (it is read-only; no objection expected), and that - the read traffic volume is acceptable. - -7. **Migration / coexistence.** During rollout a replica may switch from - `kopiaSecretRef` to `canopyBackup`. Confirm a clean switchover (the next - reconcile picks up the canopy path; the old Secret can be deleted by the - operator once no replica references it) and that mixed fleets work. - -8. **Region/endpoint for non-AWS repos.** The legacy path supports - `endpoint` + `disableTls` (minio, S3-compatible). Canopy's - `GET /backup-target` returns `{ storage, bucket, prefix, region, - repo_password }` — no `endpoint`/`disable_tls` field. If any - canopy-backed repo is ever non-AWS, the canopy wire type would need - optional `endpoint`/`disable_tls`, and the bestool S3P proxy would need - to be willing to point upstream at a non-`s3..amazonaws.com` - host. Out of scope unless canopy supports non-AWS targets. +## Interfaces this component exposes / consumes + +**Consumes (from canopy via `bestool-canopy` 0.4.2):** +- `CanopyClient::restore_capabilities(base, &[intents])` +- `CanopyClient::restore_worklist(base) -> Vec` +- `CanopyClient::restore_credentials(base, type, group) -> RestoreCredentials` +- `CanopyClient::restore_verification(base, &RestoreVerification)` + +**Consumes (from bestool, published cargo crates):** +- `bestool-canopy` ≥0.4.2 — the four methods above + the wire types + (`WorklistEntry`, `RestoreCredentials`, `RestoreVerification`, + `RestoreCapabilitiesRequest`, `RestoreCredentialsRequest`). +- `bestool-kopia` ≥0.3.3 — `proxy::spawn`, `CredentialProvider` trait, + `Credentials`, `TrafficStats`, `PROXY_DUMMY_ACCESS_KEY`/`SECRET_KEY`. + +**Provides internally:** +- `POST /internal/restore-creds` on the operator's broker port — + consumed only by the proxy sidecar in canopy-backed restore Jobs. + +**Provides operationally:** +- A new sidecar image (`ghcr.io/beyondessential/pgro-canopy-proxy`) + alongside the existing operator image. +- Namespaces labelled `pgro.bes.au/managed-by=pgro-canopy` — + observable via `kubectl get ns`. + +--- + +## IaC / deployment changes (`operator.yaml` + ops repo) + +- **Operator Deployment**: add the Tailscale sidecar container (or + the Tailscale-operator `ProxyClass` annotation); add env vars + `CANOPY_BASE_URL`, `CANOPY_RECONCILE_INTERVAL_SECS`, and (optionally) + `CANOPY_DEVICE_CERT_SECRET`; add a second container port for the + broker (`9091`). +- **NetworkPolicy** restricting the broker port to pods labelled + `pgro.bes.au/proxy-sidecar=true` within the cluster (deny ingress + otherwise). +- **ClusterRole delta**: the operator already has full Namespace + + Deployment + Job + Secret verbs cluster-wide for the legacy path; + the same RBAC covers the canopy-backed path. Verify + `verbs=[create, delete]` on Namespaces is present. +- **ops-repo, one-time per cluster**: install the Tailscale k8s + operator; mint an OAuth client + ACL tag for pgro's operator pod; + ACL allows the tag to reach `canopy.tail53aef.ts.net`. After the + operator's first contact creates the canopy `Untrusted` device row, + promote to role `backup-restore` (one-time manual step). --- -## Backup types addendum +## Testing approach (per `AGENTS.md`) -Per the Canopy plan's "Backup types": PGRO consumes a **specific type**. +**Unit tests (local):** +- `src/canopy.rs`: wrapper round-trip against an axum stub server that + serves canned `WorklistEntry` / `RestoreCredentials` JSON; assert + serde + URL routing. `bestool-canopy`'s own tests cover wire-shape + correctness, so pgro only tests its wrapper layer. +- Worklist syncer reconciliation: table-driven tests with synthesized + `Vec` + `Vec` inputs, asserting the + expected provision/refresh/teardown decisions (no apiserver, no + reconcile actions — just the diff function). +- `KopiaSource::CanopyProxy` job-builder snapshot tests, mirroring the + existing `kopia_connect_args_*` tests for the proxy connect-args + shape. +- Reporter at-most-once: assert that a stamped + `last-verification-reported-at` prevents re-report; reset → report + → annotation written. +- Capability registration on startup: the operator pushes + `["verify","analytics","disaster-recovery"]`; transient failure is + retried, fatal config error is surfaced. + +**Integration tests (CI-only; new matrix entry required):** -- PGRO restores the **`tamanu-postgres`** type (the same type bestool - produces). The `PostgresPhysicalReplica` CRD references the canopy group - **and type**. -- The external-restore grant is per `(consumer, group, type)`: - "PGRO may read group X's `tamanu-postgres`, read-only". -- Signal-3 restore-verification is **per type** — PGRO reports which - `(group, type)` snapshot it restored and the outcome; a failed/stale - restorability check for that `(group, type)` is the group-level alert. -- PGRO filters the repo by the `canopy-type` tag to find its snapshots. +- `tests/canopy_integration.rs` — a `test-canopy-restore` namespace + exercising the canopy path against a **stub canopy** (a small + in-cluster HTTP service implementing the four endpoints with canned + responses, reusing the existing `tests/fixtures/minio.yaml` + + `setup-kopia-repo.yaml` as the S3 backing store). Assert: + - worklist tick discovers no namespaces, creates one; restore Job + reaches `Active`; verification stub receives a `RestoreVerification` + with the expected `replica_id`/`snapshot_id`/`outcome=success`. + - worklist entry removed → namespace torn down within ~2 ticks. + - `RestoreReplica::authorizes`-style 403 from the stub → pgro + surfaces a clear failure (annotation + event) and doesn't + crash-loop. +- Matrix entry added in `.github/workflows/integration.yml`. Flag to + the user that this only runs in CI. --- -## Canopy implementation status (re-verified 2026-06-26) - -Where canopy actually stands today, so this spec builds on what exists vs. -what canopy still owes. Re-checked against `canopy/crates/public-server/` -(openapi.json + src), `canopy/migrations/`, and the bestool repo on -2026-06-26: **none of the pgro-blocking surfaces have moved** since this -section was first written 2026-06-16. Canopy work in the intervening two -weeks has been operator-UI and S3-traffic tallies (PRs through #282; -migrations through `add_s3_traffic_to_backup_runs`). - -**Available now (PR #224):** `POST /backup-credentials` with -`{ "type": "tamanu-postgres", "purpose": "restore" }` returns read-only -`credential_process` creds (the restore session policy: `GetObject` + -unconditioned `GetBucketLocation` + prefix-conditioned `ListBucket`), and -`GET /backup-target` returns `{ storage, bucket, prefix, region, -repo_password }`. So the *device-shaped* restore path exists — but it is -`ServerDevice` (mTLS) authenticated and the creds are chained (1-hour cap, -practically a non-issue under the proxy model — see §1.3). - -**Available now in the bestool repo (published crates):** the S3P proxy -(`bestool-kopia` 0.3.2: `bestool_kopia::proxy::{spawn, CredentialProvider}`) -and the canopy HTTP client + wire types (`bestool-canopy` 0.4.0: -`CanopyClient`, `BackupCredentials`, `BackupTarget`, `BackupReport`, -`Purpose`). pgro depends on both as ordinary cargo deps; no git/vendoring. - -**Still owed by canopy (not built — blocks PGRO):** -- **A new device role (`backup-restore` or equivalent).** Canopy already - has `server-device`/`releaser-device`/`admin-device`; pgro's identity is - a fourth, generic restore-only role keyed off the external-restore - grant, with `purpose=backup` server-side rejected for the role. Smaller - lift than net-new auth machinery, but still net-new role+cert plumbing - that canopy must add (role enum, `securitySchemes` entry, route-gating, - purpose-gating, cert-issuance flow). Until this lands, pgro cannot - authenticate to canopy at all. **Biggest blocker.** -- **`CredentialsArgs` / external-restore wire change.** Today's - `{ type, purpose }` body has no `group` field; the new role has no - implicit group, so canopy needs either an additive field or a sibling - endpoint that takes one. The `(consumer, group, type)` external-restore - grant lookup hangs off this. -- **The external-restore grant** (`(consumer, group, type)` authz model, - operator-authorized + audited) — no migration, no code. -- **Restore-verification ingest path.** A new endpoint (working title - `POST /restore-verification`) writing to a new `backup_restore_checks` - table, routing failures through `raise_group_event(ref = - "restore-verification")`. See §2.0 for why reusing `/backup-report` is - not viable. - -**Group-level alert entrypoint is concrete (PR #225).** Signal-3 routes -through `database::backup::alerts::raise_group_event`, whose signature is: -`raise_group_event(conn, group_id, ref: &str, severity: Severity, -description: Option<&str>, message: &str, active: bool) -> Result`. -Use `ref = "restore-verification"` (the constant in `database::backup::refs`), -severity `Error` (group-level, bypasses per-server `is_monitored`); recovery -is the same `(source="canopy", ref)` with `active: false`. Group-scoped issues -are first-class (Option B: `issues.server_group_id`), so no per-server shim is -needed. +## Open questions / decisions + +1. **Per-intent provisioning defaults.** Storage size, postgres + resource requests, retention of the previous Deployment on refresh + — intent-dependent. Worklist entries don't carry these; pgro picks + from a defaults table per intent, with a per-replica ConfigMap + override mechanism. Define the defaults table during Part 2 + implementation. + +2. **Snapshot-list Job retirement.** Canopy supplies the snapshot id, + so `build_snapshot_list_job` is unused for canopy-backed replicas. + It stays for the legacy path. Decide whether to refactor + `controllers/replica/resources.rs` to make the variance explicit, + or leave the legacy path alone and just never call it from the new + syncer. + +3. **Reporter latency vs reconciles.** Verification reports go out on + the next worklist tick after a terminal transition (up to ~30s + delay). If that's too slow, the restore controller can call the + reporter inline at the terminal transition — same idempotency + guard. Default: tick-driven (simpler), inline if real users + complain. + +4. **Per-replica freshness drift.** The worklist's `freshness_seconds` + is canopy's. If a replica is mid-restore when the freshness window + opens for the next refresh, pgro should not start a second restore + in parallel — serialize per-namespace. The existing restore phase + machine enforces this implicitly (one CR + one Job), but the + canopy path needs an equivalent serialization on the Namespace. + +5. **NetworkPolicy in dev/test clusters.** Some integration test + environments don't enforce NetworkPolicy (no CNI plugin). The + broker should be deployable without NetworkPolicy in those + environments; in production, NetworkPolicy is the only thing + between the broker and arbitrary in-cluster callers. Document + this clearly. --- -## bestool-side asks - -What needs to be added to the published bestool crates before pgro can -build against them. All four are additive (no breaking changes to -existing bestool consumers) and each is gated by the corresponding -canopy endpoint shipping first. - -**In `bestool-canopy`:** - -1. **`CanopyClient::restore_credentials(base, type, group)`** — - group-aware variant of `backup_credentials`. The existing method - infers group from `device → server → group_id`, which doesn't apply - to a non-server-bound `backup-restore` device; the new method puts - `group` in the request body to match the canopy wire change. -2. **`CanopyClient::restore_target(base, group)`** — same group issue. -3. **`CanopyClient::restore_verification(base, &RestoreVerification)`** - — posts to canopy's new ingest endpoint (working title - `POST /restore-verification`). See §2.0 for why this is a separate - endpoint from `/backup-report`. -4. **`pub struct RestoreVerification`** in `bestool-canopy::backup` — - the wire request type, mirroring whatever canopy lands. - -**In `bestool-kopia`:** **no changes needed.** `proxy::spawn`, the -`CredentialProvider` trait, the `Credentials` struct, and `TrafficStats` -are already what pgro consumes. pgro implements its own -`CredentialProvider` against the operator's in-cluster broker (§3.1.2), -no changes to the proxy or the trait. - -**Also unchanged:** `CanopyClient::new(... device_key_pem: Option<&str>, -...)` already accepts `None`, so pgro's tailscale-only operator (no -mTLS fallback cert) works without further changes. `Purpose::Restore` -already exists. No new crate — all four asks fit in `bestool-canopy`. - -The `purpose=backup` rejection for the `backup-restore` role is purely -canopy-side enforcement; `bestool-canopy` doesn't know about role-based -purpose gating, it just sends what it's told. +## Sequencing + +One PR. Order within the PR: + +1. `src/kopia.rs` — `kopia_connect_args_proxy` + tests. +2. `src/canopy.rs` — `CanopyClient` wrapper, types re-exported from + `bestool-canopy`. +3. `Cargo.toml` — add `bestool-canopy = "0.4"`, `bestool-kopia = "0.3"`; + add `[[bin]] name = "canopy-proxy"`. +4. `src/bin/canopy_proxy.rs` — the sidecar binary. +5. `src/controllers/canopy.rs` — the worklist-syncer controller. +6. `src/bin/operator.rs` — wire the new controller; register + capabilities; mount the broker route on a separate port. +7. Job-builder `KopiaSource` refactor + sidecar container injection. +8. `tests/canopy_integration.rs` + stub canopy + `integration.yml` + matrix entry. +9. `operator.yaml` — Tailscale sidecar, broker port, NetworkPolicy. +10. `.github/workflows/cd.yml` — second image build + push. + +Legacy CRD path stays compiling at every commit; the canopy path is +behind no feature flag (it's only exercised when canopy hands out a +worklist, which requires a promoted `backup-restore` device — gated by +canopy-side configuration, not pgro code). diff --git a/docs/canopy-handoff-response.md b/docs/canopy-handoff-response.md new file mode 100644 index 0000000..4737cf9 --- /dev/null +++ b/docs/canopy-handoff-response.md @@ -0,0 +1,81 @@ +# PGRO sign-off on canopy's restore-replicas response + +**From:** pgro +**To:** canopy +**Re:** `canopy/docs/plans/pgro-restore-replicas-canopy-response.md` +**Status:** signed off. Canopy may freeze wire shapes and proceed with +PR1 + PR2 as described in §6 of the response. + +## What pgro accepts + +- **§1 corrections.** Noted: four device roles (untrusted included); + role column is plain `TEXT`, no migration; `releaser`-style operator + trust-promotion replaces the imagined "cert-issuance flow"; reason 7 + in §4.4.1 of the handoff was wrong (`run_id` isn't shared between + issuance and report) — the conclusion stands on the other six; + `raise_group_event` + `restore-verification` ref already merged. +- **§2.1 — canopy supplies the snapshot id.** pgro will not list the + repo to discover what to restore; it consumes the snapshot id from + the worklist entry. +- **§2.2 — per-server, not per-group.** Worklist entries, restore + targeting, and `/restore-verification` reports are per-server. + Credentials remain per-(group, type). +- **§3 — the inversion, fully.** Canopy is the source of truth for + *which replicas should exist*, including `analytics` and + `disaster-recovery` intents — not just `verify`. Operator UX is + canopy's, not k8s-CRD's. The boundary canopy proposed (canopy owns + *what/why/how-fresh*; pgro owns *how*) is the contract. + +## What this changes on pgro's side (for canopy's awareness) + +These are pgro-internal consequences of accepting the inversion; canopy +doesn't need to do anything with them, but they shape what pgro will +build later. + +- **pgro becomes a worklist-reconciler.** `GET /restore-worklist` is + the only desired-state input. No CRD for canopy-backed replicas. +- **Cluster state is pgro's runtime model.** Each replica is a + labelled k8s `Namespace` + (`pgro/declaration-id`/`group`/`server`/`type`/`intent`), with + per-replica status in annotations + (`last-restored-snapshot-id`, `last-restored-at`, + `restore-state`). On boot pgro discovers via + `LIST Namespaces label=pgro/managed-by=pgro` and reconciles against + the worklist. No intermediate CR layer. +- **Legacy `kopiaSecretRef` CRD path coexists.** Pre-canopy replicas + keep their existing CRDs and the existing CR-driven controllers; + the canopy-backed path is greenfield and uses the worklist model. + No migration tool — operators decommission old CRDs as they convert + declarations into canopy. + +## Resolved: unsupported intents (was: pgro's open question) + +Canopy took the structured route in §7 of the response: pgro registers +its supported intents via `POST /restore-capabilities` on startup and +on change; canopy persists them, dispatches only matching worklist +entries, and surfaces stranded declarations as operator-facing +configuration *gaps* rather than restore-health incidents. The right +call — conflating capability mismatch with "backup unrestorable" would +page operators for what is a configuration concern. + +Consequences pgro will implement: +- `restore_capabilities` registration on operator startup, re-pushed + on any change to the supported intent set. +- `/restore-verification` outcome stays `success`/`failure` only; no + `unsupported` variant. +- A fifth bestool addition: `CanopyClient::restore_capabilities(base, + &[intents])` in Appendix A. + +## What canopy is unblocked to do + +- Freeze the §4 wire shapes (now per-server with canopy-supplied + snapshot id). +- Build PR1 (`backup-restore` role + declared-replica model + operator + UI + `GET /restore-worklist` + `POST /restore-credentials`). +- Build PR2 (`backup_restore_checks` + `POST /restore-verification` + + alert routing + freshness sweep). +- Restate Appendix A and hand off to bestool. + +When PR1 + PR2 + bestool's additions are merged, ping pgro. pgro will +then rewrite `pgro/docs/canopy-backup-integration.md` against the +as-shipped surfaces and start building. From c08721ea04bd2552ad9bddabfa68caa87b6fb6cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Wed, 1 Jul 2026 10:26:17 +1200 Subject: [PATCH 04/20] chore: silence pre-existing lints (clippy useless_conversion in replica.rs; needless_borrow in switchover.rs) These are pre-existing in the tree; both block 'cargo clippy --all-targets -- -D warnings' which is otherwise clean. Fixing in a single chore so the feature commits that follow have a green clippy baseline. --- src/controllers/replica.rs | 7 +------ tests/switchover.rs | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/controllers/replica.rs b/src/controllers/replica.rs index a7168c8..2512647 100644 --- a/src/controllers/replica.rs +++ b/src/controllers/replica.rs @@ -1096,12 +1096,7 @@ async fn mark_schema_migration_complete( /// `creationTimestamp` as the start; falls back to "not exceeded" if the /// Job has no creation timestamp (which shouldn't happen in practice). fn migration_exceeded_budget(replica: &PostgresPhysicalReplica, job: &Job) -> bool { - let Some(created) = job - .metadata - .creation_timestamp - .as_ref() - .map(|t| Timestamp::from(t.0)) - else { + let Some(created) = job.metadata.creation_timestamp.as_ref().map(|t| t.0) else { return false; }; let elapsed = Timestamp::now().duration_since(created); diff --git a/tests/switchover.rs b/tests/switchover.rs index 03d0564..ffb0db8 100644 --- a/tests/switchover.rs +++ b/tests/switchover.rs @@ -230,7 +230,7 @@ async fn second_restore_and_switchover() { // The second restore's pod should be labeled ready-for-traffic. let pods: kube::Api = - kube::Api::namespaced(client.clone(), &ns); + kube::Api::namespaced(client.clone(), ns); let pod_list = pods .list( &kube::api::ListParams::default() From da99f92763d2e3d213e92c7c5c156273a9ba96c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Wed, 1 Jul 2026 10:26:17 +1200 Subject: [PATCH 05/20] feat(kopia): add canopy-proxy connect-args + bestool deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds kopia_connect_args_proxy emitting --endpoint=[::1]: with dummy keys and TLS disabled — the kopia-side half of the bestool S3P loopback re-signing proxy convention. The legacy kopia_connect_args is untouched for kopiaSecretRef replicas. Pulls in bestool-canopy (HTTP client + wire types) and bestool-kopia (proxy module + dummy-key consts) as workspace deps. Declares the new canopy-proxy binary (currently a stub; real impl in the next commit). --- Cargo.lock | 2602 +++++++++++++++++++++++++++++++-------- Cargo.toml | 6 + src/bin/canopy_proxy.rs | 8 + src/kopia.rs | 108 ++ 4 files changed, 2199 insertions(+), 525 deletions(-) create mode 100644 src/bin/canopy_proxy.rs diff --git a/Cargo.lock b/Cargo.lock index 26fc75e..4430617 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,76 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "age" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a07d86e4272c093c88caf7864a2d09af52a5159180848ca4832a3cdbd7d014d5" +dependencies = [ + "age-core", + "base64 0.21.7", + "bech32", + "chacha20poly1305", + "cookie-factory", + "futures", + "hmac 0.12.1", + "i18n-embed", + "i18n-embed-fl", + "lazy_static", + "memchr", + "nom 7.1.3", + "pin-project", + "rand 0.8.6", + "rust-embed", + "scrypt", + "sha2 0.10.9", + "subtle", + "x25519-dalek", + "zeroize", +] + +[[package]] +name = "age-core" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2bf6a89c984ca9d850913ece2da39e1d200563b0a94b002b253beee4c5acf99" +dependencies = [ + "base64 0.21.7", + "chacha20poly1305", + "cookie-factory", + "hkdf", + "io_tee", + "nom 7.1.3", + "rand 0.8.6", + "secrecy", + "sha2 0.10.9", +] + [[package]] name = "ahash" version = "0.7.8" @@ -35,23 +105,152 @@ dependencies = [ "memchr", ] +[[package]] +name = "algae-cli" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904db2bebc10c4af40c21866dae385b008883ac57994896d8da815896ba3a63f" +dependencies = [ + "age", + "age-core", + "clap", + "clap-markdown", + "dialoguer", + "diceware_wordlists", + "futures", + "indicatif", + "itertools", + "jiff", + "miette", + "pinentry", + "rand 0.10.1", + "tokio", + "tokio-util", + "tracing", + "windows_exe_info", +] + [[package]] name = "allocator-api2" version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "arc-swap" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "async-broadcast" @@ -84,7 +283,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -95,7 +294,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -106,15 +305,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.15.4" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" dependencies = [ "aws-lc-sys", "zeroize", @@ -122,14 +321,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.37.1" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -195,12 +395,117 @@ dependencies = [ "tokio", ] +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "backtrace-ext" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50" +dependencies = [ + "backtrace", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "basic-toml" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a" +dependencies = [ + "serde", +] + +[[package]] +name = "bech32" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" + +[[package]] +name = "bestool-canopy" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cb064d9c74aedc67f722d2bf7cac5a9214525ef6bf12b70c09d5428abe020e2" +dependencies = [ + "algae-cli", + "base64 0.22.1", + "blake3", + "flate2", + "futures", + "getrandom 0.4.3", + "hickory-resolver", + "jiff", + "machine-uid", + "miette", + "rcgen", + "reqwest", + "serde", + "serde_json", + "sysinfo", + "time", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "bestool-kopia" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f03d98bc1690b9adaa51b3e4416ce19c1bc8472a520a269a33224b64abfd4757" +dependencies = [ + "bytes", + "hex", + "hmac 0.13.0", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "jiff", + "miette", + "rustls", + "serde", + "serde_json", + "sha2 0.11.0", + "tokio", + "tracing", + "whoami", +] + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -209,15 +514,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "bitvec" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" dependencies = [ "funty", "radium", @@ -225,6 +530,20 @@ dependencies = [ "wyz", ] +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -245,42 +564,43 @@ dependencies = [ [[package]] name = "borsh" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1da5ab77c1437701eeff7c88d968729e7766172279eab0676857b3d63af7a6f" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" dependencies = [ "borsh-derive", + "bytes", "cfg_aliases", ] [[package]] name = "borsh-derive" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" +checksum = "3ae8fb4fb5740e4b2c4884ff95f5f32f5e8479db1e8fd8eb49ddbe09eb09bb7c" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] name = "bstr" -version = "1.12.1" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +checksum = "5cee35f73844aa3014bb606320a6c1f010249dbdf43342fe54b5a4f6a8ed4b79" dependencies = [ "memchr", - "serde", + "serde_core", ] [[package]] name = "bumpalo" -version = "3.19.1" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytecheck" @@ -316,11 +636,17 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +[[package]] +name = "camino" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2d30e4173c4026932d51d31d6b0613b1fd3014bf3f9f8943d4ba139c437ba0" + [[package]] name = "cc" -version = "1.2.56" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", "jobserver", @@ -328,12 +654,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cesu8" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" - [[package]] name = "cfg-if" version = "1.0.4" @@ -348,20 +668,105 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chacha20" -version = "0.10.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", "cpufeatures 0.3.0", - "rand_core 0.10.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20 0.9.1", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", + "zeroize", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap-markdown" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2a2617956a06d4885b490697b5307ebb09fec10b088afc18c81762d848c2339" +dependencies = [ + "clap", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", ] +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] @@ -372,6 +777,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "combine" version = "4.6.7" @@ -391,12 +802,39 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width 0.2.2", + "windows-sys 0.61.2", +] + [[package]] name = "const-oid" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +dependencies = [ + "futures", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -441,6 +879,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "cronexpr" version = "1.5.0" @@ -448,7 +901,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a366517b317d398f448a2ba76c1b5000108db2ae393c865ec0ed6250dffdce32" dependencies = [ "jiff", - "winnow 1.0.2", + "winnow", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", ] [[package]] @@ -485,6 +956,32 @@ dependencies = [ "cmov", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "darling" version = "0.20.11" @@ -516,7 +1013,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -529,7 +1026,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -540,7 +1037,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -551,14 +1048,14 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] name = "data-encoding" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "defmt" @@ -580,7 +1077,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -592,6 +1089,26 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom 7.1.3", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + [[package]] name = "derive_builder" version = "0.20.2" @@ -610,7 +1127,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -620,7 +1137,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -641,9 +1158,26 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.116", + "syn 2.0.118", +] + +[[package]] +name = "dialoguer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f104b501bf2364e78d0d3974cbc774f738f5865306ed128e1e0d7499c0ad96" +dependencies = [ + "console", + "shell-words", + "zeroize", ] +[[package]] +name = "diceware_wordlists" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9b52b69d268c7a2bc582e3aec5cdfa43ac91cef4fe6b6751b02da2b43d6166" + [[package]] name = "digest" version = "0.10.7" @@ -652,6 +1186,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -666,15 +1201,25 @@ dependencies = [ "ctutils", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.0", + "objc2", +] + [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -698,14 +1243,34 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "embed-resource" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31a88c8d26de40ed18fe748c547845aa39de1db3afd958f8cb91579f3644bcb" +dependencies = [ + "cc", + "memchr", + "rustc_version", + "toml 1.1.2+spec-1.1.0", + "vswhom", + "winreg", +] + +[[package]] +name = "encode_unicode" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" @@ -718,22 +1283,22 @@ dependencies = [ [[package]] name = "enum-ordinalize" -version = "4.3.2" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1091a7bb1f8f2c4b28f1fe2cef4980ca2d410a3d727d67ecc3178c9b0800f0" +checksum = "07f808d588c10e464ea6f7d3eaed500049eff30aaac103460f61828c2d65b3eb" dependencies = [ "enum-ordinalize-derive", ] [[package]] name = "enum-ordinalize-derive" -version = "4.3.2" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" +checksum = "42e528e2d34ba8a67a1a650b86beae8ef69fc5fdb638016f386b973226590432" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -781,9 +1346,24 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-crate" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "59a98bbaacea1c0eb6a0876280051b892eb73594fd90cf3b20e9c817029c57d2" +dependencies = [ + "toml 0.5.11", +] [[package]] name = "find-msvc-tools" @@ -792,16 +1372,64 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] -name = "fnv" -version = "1.0.7" +name = "flate2" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] [[package]] -name = "foldhash" -version = "0.1.5" +name = "fluent" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb74634707bebd0ce645a981148e8fb8c7bccd4c33c652aeffd28bf2f96d555a" +dependencies = [ + "fluent-bundle", + "unic-langid", +] + +[[package]] +name = "fluent-bundle" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe0a21ee80050c678013f82edf4b705fe2f26f1f9877593d13198612503f493" +dependencies = [ + "fluent-langneg", + "fluent-syntax", + "intl-memoizer", + "intl_pluralrules", + "rustc-hash 1.1.0", + "self_cell 0.10.3", + "smallvec", + "unic-langid", +] + +[[package]] +name = "fluent-langneg" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "7eebbe59450baee8282d71676f3bfed5689aeab00b27545e83e5f14b1195e8b0" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "fluent-syntax" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a530c4694a6a8d528794ee9bbd8ba0122e779629ac908d15ad5a7ae7763a33d" +dependencies = [ + "thiserror 1.0.69", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" @@ -886,7 +1514,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -950,25 +1578,29 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "r-efi", - "rand_core 0.10.0", - "wasip2", - "wasip3", + "r-efi 6.0.0", + "rand_core 0.10.1", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "globset" version = "0.4.18" @@ -996,9 +1628,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", @@ -1015,9 +1647,9 @@ dependencies = [ [[package]] name = "handlebars" -version = "6.4.1" +version = "6.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d43ccdfe15a81ab0a8af639e90254227c9a46afd9c5f5b6ec7efaa345c4b0f00" +checksum = "f26569a2763497b7bd3fbd19374b774ea6038c5293678771259cd534d49740ff" dependencies = [ "derive_builder", "log", @@ -1038,15 +1670,6 @@ dependencies = [ "ahash 0.7.8", ] -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash 0.1.5", -] - [[package]] name = "hashbrown" version = "0.16.1" @@ -1055,15 +1678,115 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash 0.2.0", + "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hickory-net" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.1", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.1", + "ring", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot", + "rand 0.10.1", + "resolv-conf", + "smallvec", + "system-configuration", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" @@ -1140,9 +1863,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.8.1" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -1155,7 +1878,6 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -1163,9 +1885,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http", "hyper", @@ -1173,10 +1895,10 @@ dependencies = [ "log", "rustls", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", + "webpki-roots", ] [[package]] @@ -1198,7 +1920,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -1217,14 +1939,81 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "i18n-config" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e06b90c8a0d252e203c94344b21e35a30f3a3a85dc7db5af8f8df9f3e0c63ef" +dependencies = [ + "basic-toml", + "log", + "serde", + "serde_derive", + "thiserror 1.0.69", + "unic-langid", +] + +[[package]] +name = "i18n-embed" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "669ffc2c93f97e6ddf06ddbe999fcd6782e3342978bb85f7d3c087c7978404c4" +dependencies = [ + "arc-swap", + "fluent", + "fluent-langneg", + "fluent-syntax", + "i18n-embed-impl", + "intl-memoizer", + "log", + "parking_lot", + "rust-embed", + "thiserror 1.0.69", + "unic-langid", + "walkdir", +] + +[[package]] +name = "i18n-embed-fl" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04b2969d0b3fc6143776c535184c19722032b43e6a642d710fa3f88faec53c2d" +dependencies = [ + "find-crate", + "fluent", + "fluent-syntax", + "i18n-config", + "i18n-embed", + "proc-macro-error2", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.118", + "unic-langid", +] + +[[package]] +name = "i18n-embed-impl" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2cc0e0523d1fe6fc2c6f66e5038624ea8091b3e7748b5e8e0c84b1698db6c2" +dependencies = [ + "find-crate", + "i18n-config", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -1232,9 +2021,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -1245,9 +2034,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1259,15 +2048,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -1279,15 +2068,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -1298,12 +2087,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -1323,9 +2106,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -1333,33 +2116,116 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", - "serde", - "serde_core", + "hashbrown 0.17.1", +] + +[[package]] +name = "indicatif" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993f007684f2e9727160da8b960ec161264703bfd1af084fd2e34d040c9a0dd4" +dependencies = [ + "console", + "portable-atomic", + "tokio", + "unicode-width 0.2.2", + "unit-prefix", + "web-time", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "intl-memoizer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "310da2e345f5eb861e7a07ee182262e94975051db9e4223e909ba90f392f163f" +dependencies = [ + "type-map", + "unic-langid", +] + +[[package]] +name = "intl_pluralrules" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "io_tee" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b3f7cef34251886990511df1c61443aa928499d598a9473929ab5a90a527304" + +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", ] [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +dependencies = [ + "serde", +] + +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.29" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34f877a98676d2fb664698d74cc6a51ce6c484ce8c770f05d0108ec9090aeb46" +checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" dependencies = [ "defmt", "jiff-static", @@ -1373,20 +2239,20 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.29" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0666b5ab5ecaca213fc2a85b8c0083d9004e84ee2d5f9a7e0017aaf50986f25f" +checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] name = "jiff-tzdb" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" [[package]] name = "jiff-tzdb-platform" @@ -1399,25 +2265,52 @@ dependencies = [ [[package]] name = "jni" -version = "0.21.1" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" dependencies = [ - "cesu8", "cfg-if", "combine", + "jni-macros", "jni-sys", "log", - "thiserror 1.0.69", + "simd_cesu8", + "thiserror 2.0.18", "walkdir", - "windows-sys 0.45.0", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.118", ] [[package]] name = "jni-sys" -version = "0.3.0" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.118", +] [[package]] name = "jobserver" @@ -1431,24 +2324,25 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.85" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ - "once_cell", + "cfg-if", + "futures-util", "wasm-bindgen", ] [[package]] name = "json-patch" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" +checksum = "7421438de105a0827e44fadd05377727847d717c80ce29a229f85fd04c427b72" dependencies = [ "jsonptr", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] @@ -1480,20 +2374,32 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51b326f5219dd55872a72c1b6ddd1b830b8334996c667449c29391d657d78d5e" dependencies = [ - "base64", + "base64 0.22.1", "jiff", "schemars", "serde", "serde_json", ] +[[package]] +name = "k8s-openapi" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c6922f6afe80418dd6019818af5d0d34584c371780ff09b9752370c25b4abb" +dependencies = [ + "base64 0.22.1", + "jiff", + "serde", + "serde_json", +] + [[package]] name = "kube" version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acc5a6a69da2975ed9925d56b5dcfc9cc739b66f37add06785b7c9f6d1e88741" dependencies = [ - "k8s-openapi", + "k8s-openapi 0.27.1", "kube-client", "kube-core", "kube-derive", @@ -1506,7 +2412,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fcaf2d1f1a91e1805d4cd82e8333c022767ae8ffd65909bbef6802733a7dd40" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "either", "futures", @@ -1519,7 +2425,7 @@ dependencies = [ "hyper-util", "jiff", "jsonpath-rust", - "k8s-openapi", + "k8s-openapi 0.27.1", "kube-core", "pem", "rustls", @@ -1547,7 +2453,7 @@ dependencies = [ "http", "jiff", "json-patch", - "k8s-openapi", + "k8s-openapi 0.27.1", "schemars", "serde", "serde-value", @@ -1566,7 +2472,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -1584,7 +2490,7 @@ dependencies = [ "hashbrown 0.16.1", "hostname", "json-patch", - "k8s-openapi", + "k8s-openapi 0.27.1", "kube-client", "parking_lot", "pin-project", @@ -1602,8 +2508,8 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5006102fbd11aa86b196a0c76bda5e2ae0cd71d4c9126c9b8bdfe7c7b7a7158" dependencies = [ - "k8s-openapi", - "nom", + "k8s-openapi 0.28.0", + "nom 8.0.0", "rust_decimal", "thiserror 2.0.18", ] @@ -1614,17 +2520,11 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" -version = "0.2.182" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libmimalloc-sys" @@ -1637,19 +2537,24 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.12" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" dependencies = [ - "bitflags 2.11.0", "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -1662,9 +2567,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru-slab" @@ -1672,6 +2577,17 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "machine-uid" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe0d6336d341b10ae80e099b8f3c59f34bf8aef66ef25e83aa98bf2955c0ae7" +dependencies = [ + "libc", + "windows-registry", + "windows-sys 0.61.2", +] + [[package]] name = "matchers" version = "0.2.0" @@ -1699,9 +2615,39 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "backtrace", + "backtrace-ext", + "cfg-if", + "miette-derive", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size", + "textwrap", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "mimalloc" @@ -1718,17 +2664,66 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" -version = "1.1.1" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -1738,6 +2733,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1747,11 +2751,36 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-modular" -version = "0.6.1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17bb261bf36fa7d83f4c294f834e91256769097b3cb505d44831e0a179ac647f" +checksum = "fc41a1374056e9672221567958a66c16be12d0e2c1b408761e14d901c237d5e0" [[package]] name = "num-order" @@ -1771,13 +2800,61 @@ dependencies = [ "autocfg", ] +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.0", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.0", + "objc2", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-open-directory" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb82bed227edf5201dfedf072bba4015a33d3d4a98519837295a90f0a23f676d" +dependencies = [ + "objc2", + "objc2-core-foundation", + "objc2-foundation", ] [[package]] @@ -1789,11 +2866,45 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl-probe" @@ -1810,6 +2921,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "owo-colors" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" + [[package]] name = "parking" version = "2.2.1" @@ -1839,13 +2956,23 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + [[package]] name = "pem" version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -1885,7 +3012,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -1919,35 +3046,61 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] -name = "pin-utils" -version = "0.1.0" +name = "pinentry" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "014e8043c4a1705b519f37562a413378af00d0da7e423c087c6e4cb9cd770875" +dependencies = [ + "log", + "nom 8.0.0", + "percent-encoding", + "secrecy", + "wait-timeout", + "which", + "zeroize", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "poly1305" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] [[package]] name = "portable-atomic" @@ -1957,9 +3110,9 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] @@ -1970,11 +3123,11 @@ version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" dependencies = [ - "base64", + "base64 0.22.1", "byteorder", "bytes", "fallible-iterator", - "hmac", + "hmac 0.13.0", "md-5", "memchr", "rand 0.10.1", @@ -1988,6 +3141,8 @@ version = "0.3.26" dependencies = [ "anyhow", "axum", + "bestool-canopy", + "bestool-kopia", "bytes", "cronexpr", "futures", @@ -1996,7 +3151,7 @@ dependencies = [ "http", "http-body-util", "jiff", - "k8s-openapi", + "k8s-openapi 0.27.1", "kube", "kube_quantity", "mimalloc", @@ -2031,13 +3186,19 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2048,20 +3209,21 @@ dependencies = [ ] [[package]] -name = "prettyplease" -version = "0.2.37" +name = "prefix-trie" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" dependencies = [ - "proc-macro2", - "syn 2.0.116", + "either", + "ipnet", + "num-traits", ] [[package]] name = "proc-macro-crate" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ "toml_edit", ] @@ -2085,7 +3247,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -2154,16 +3316,16 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.2", "rustls", "socket2", "thiserror 2.0.18", @@ -2174,17 +3336,17 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", - "rustc-hash", + "rustc-hash 2.1.2", "rustls", "rustls-pki-types", "slab", @@ -2210,9 +3372,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.44" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -2223,6 +3385,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radium" version = "0.7.0" @@ -2231,9 +3399,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -2242,9 +3410,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -2256,9 +3424,9 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ - "chacha20", - "getrandom 0.4.1", - "rand_core 0.10.0", + "chacha20 0.10.1", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -2301,9 +3469,23 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.10.0" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rcgen" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] [[package]] name = "redox_syscall" @@ -2311,7 +3493,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.0", ] [[package]] @@ -2331,14 +3513,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -2359,9 +3541,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.9" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "rend" @@ -2378,7 +3560,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -2412,6 +3594,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + [[package]] name = "ring" version = "0.17.14" @@ -2455,6 +3643,40 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rust-embed" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.118", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1" +dependencies = [ + "sha2 0.10.9", + "walkdir", +] + [[package]] name = "rust_decimal" version = "1.42.1" @@ -2465,18 +3687,30 @@ dependencies = [ "borsh", "bytes", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "rkyv", "serde", "serde_json", "wasm-bindgen", ] +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + [[package]] name = "rustc-hash" -version = "2.1.1" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc_version" @@ -2487,11 +3721,33 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.3", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" -version = "0.23.36" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "aws-lc-rs", "log", @@ -2504,9 +3760,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -2516,9 +3772,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "web-time", "zeroize", @@ -2526,9 +3782,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", @@ -2553,9 +3809,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -2575,6 +3831,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2586,9 +3851,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -2616,7 +3881,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -2625,6 +3890,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "seahash" version = "4.1.0" @@ -2642,11 +3918,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.6.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -2655,19 +3931,34 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.16.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321c8673b092a9a42605034a9879d73cb79101ed5fd117bc9a597b89b4e9e61a" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", ] +[[package]] +name = "self_cell" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14e4d63b804dc0c7ec4a1e52bcb63f02c7ac94476755aa579edac21e01f915d" +dependencies = [ + "self_cell 1.2.2", +] + +[[package]] +name = "self_cell" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" + [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" @@ -2706,7 +3997,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -2717,7 +4008,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -2726,6 +4017,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -2744,6 +4036,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2811,11 +4112,17 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" @@ -2827,6 +4134,22 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -2835,9 +4158,9 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "siphasher" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "slab" @@ -2847,9 +4170,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" @@ -2890,6 +4213,27 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "supports-color" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" +dependencies = [ + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e396b6523b11ccb83120b115a0b7366de372751aa6edf19844dfb13a6af97e91" + +[[package]] +name = "supports-unicode" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" + [[package]] name = "syn" version = "1.0.109" @@ -2903,9 +4247,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.116" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df424c70518695237746f84cede799c9c58fcb37450d7b23716568cc8bc69cb" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -2929,7 +4273,22 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", +] + +[[package]] +name = "sysinfo" +version = "0.39.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c8bd2130a9b60bee2581bf82cfe89ee836424d1f37dcfa4ce21509611684673" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "objc2-open-directory", + "windows", ] [[package]] @@ -2938,7 +4297,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.13.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -2953,12 +4312,38 @@ dependencies = [ "libc", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tap" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "unicode-linebreak", + "unicode-width 0.2.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -2985,7 +4370,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -2996,7 +4381,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -3008,21 +4393,52 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e48db7b415311b615f910b3dcaa4557bcd4bf1982379c95c223fd8c2a20e210" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -3035,9 +4451,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -3052,13 +4468,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -3117,42 +4533,73 @@ checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "slab", "tokio", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + [[package]] name = "toml_datetime" -version = "0.7.5+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.23.10+spec-1.0.0" +version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ "indexmap", "toml_datetime", "toml_parser", - "winnow 0.7.14", + "winnow", ] [[package]] name = "toml_parser" -version = "1.0.9+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 0.7.14", + "winnow", ] +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + [[package]] name = "tower" version = "0.5.3" @@ -3176,8 +4623,8 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "base64", - "bitflags 2.11.0", + "base64 0.22.1", + "bitflags 2.13.0", "bytes", "futures-util", "http", @@ -3223,7 +4670,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] @@ -3295,12 +4742,21 @@ dependencies = [ "http", "httparse", "log", - "rand 0.9.2", + "rand 0.9.4", "sha1", "thiserror 2.0.18", "utf-8", ] +[[package]] +name = "type-map" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb30dbbd9036155e74adad6812e9898d03ec374946234fbcebd5dfc7b9187b90" +dependencies = [ + "rustc-hash 2.1.2", +] + [[package]] name = "typenum" version = "1.20.1" @@ -3313,6 +4769,25 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unic-langid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05" +dependencies = [ + "unic-langid-impl", +] + +[[package]] +name = "unic-langid-impl" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" +dependencies = [ + "serde", + "tinystr", +] + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -3325,6 +4800,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + [[package]] name = "unicode-normalization" version = "0.1.25" @@ -3338,13 +4819,35 @@ dependencies = [ name = "unicode-properties" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" [[package]] -name = "unicode-xid" -version = "0.2.6" +name = "universal-hash" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] [[package]] name = "unsafe-libyaml" @@ -3382,13 +4885,21 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" -version = "1.21.0" +version = "1.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" +checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" dependencies = [ + "getrandom 0.4.3", "js-sys", + "serde_core", "wasm-bindgen", ] @@ -3404,6 +4915,35 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vswhom" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be979b7f07507105799e854203b470ff7c78a1639e330a58f183b5fea574608b" +dependencies = [ + "libc", + "vswhom-sys", +] + +[[package]] +name = "vswhom-sys" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb067e4cbd1ff067d1df46c9194b5de0e98efd2810bbc95c5d5e5f25a3231150" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -3440,18 +4980,9 @@ dependencies = [ [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen", ] @@ -3467,9 +4998,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.108" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -3481,23 +5012,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.58" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.108" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3505,65 +5032,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.108" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.108" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags 2.11.0", - "hashbrown 0.15.5", - "indexmap", - "semver", -] - [[package]] name = "web-sys" -version = "0.3.85" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -3581,18 +5074,36 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.6" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d7cd18d4acb58fb3cdfe9ea54e6cd96a4e7d4cc45c56338b236e82dad47248" +dependencies = [ + "libc", +] + [[package]] name = "whoami" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d" +checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d" dependencies = [ "libc", "libredox", @@ -3601,6 +5112,28 @@ dependencies = [ "web-sys", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -3610,12 +5143,95 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + [[package]] name = "windows-registry" version = "0.6.1" @@ -3647,18 +5263,18 @@ dependencies = [ [[package]] name = "windows-sys" -version = "0.45.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.42.2", + "windows-targets 0.52.6", ] [[package]] name = "windows-sys" -version = "0.52.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ "windows-targets 0.52.6", ] @@ -3681,21 +5297,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -3730,10 +5331,13 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" +name = "windows-threading" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] [[package]] name = "windows_aarch64_gnullvm" @@ -3747,12 +5351,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -3766,10 +5364,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" [[package]] -name = "windows_i686_gnu" -version = "0.42.2" +name = "windows_exe_info" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "3a7c2cd292e8e58e012eaf18f18f6b64ef74e0b90677b4f9e1f15bfca24056c7" +dependencies = [ + "camino", + "embed-resource", +] [[package]] name = "windows_i686_gnu" @@ -3795,12 +5397,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -3813,12 +5409,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -3831,12 +5421,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -3849,12 +5433,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -3869,130 +5447,89 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.14" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] [[package]] -name = "winnow" -version = "1.0.2" +name = "winreg" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" +checksum = "cb5a765337c50e9ec252c2069be9bf91c7df47afb103b642ba3a53bf8101be97" dependencies = [ - "memchr", + "cfg-if", + "windows-sys 0.59.0", ] [[package]] name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] -name = "wit-bindgen-rust" -version = "0.51.0" +name = "writeable" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn 2.0.116", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" +name = "wyz" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.116", - "wit-bindgen-core", - "wit-bindgen-rust", + "tap", ] [[package]] -name = "wit-component" -version = "0.244.0" +name = "x25519-dalek" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ - "anyhow", - "bitflags 2.11.0", - "indexmap", - "log", + "curve25519-dalek", + "rand_core 0.6.4", "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", + "zeroize", ] [[package]] -name = "wit-parser" -version = "0.244.0" +name = "x509-parser" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom 7.1.3", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.18", + "time", ] [[package]] -name = "writeable" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" - -[[package]] -name = "wyz" -version = "0.5.1" +name = "yasna" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" dependencies = [ - "tap", + "bit-vec", + "time", ] [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4001,68 +5538,82 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.39" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.39" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -4071,10 +5622,11 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", @@ -4082,13 +5634,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.116", + "syn 2.0.118", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 68bdfcf..20f36ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,8 @@ publish = false [dependencies] anyhow = "1.0.102" axum = "0.8.9" +bestool-canopy = "0.4.3" +bestool-kopia = { version = "0.3.4", features = ["proxy"] } cronexpr = "1.5.0" futures = "0.3.31" globset = "0.4.18" @@ -38,3 +40,7 @@ bytes = "1" http = "1" http-body-util = "0.1" tower = "0.5" + +[[bin]] +name = "canopy-proxy" +path = "src/bin/canopy_proxy.rs" diff --git a/src/bin/canopy_proxy.rs b/src/bin/canopy_proxy.rs new file mode 100644 index 0000000..77b5b0d --- /dev/null +++ b/src/bin/canopy_proxy.rs @@ -0,0 +1,8 @@ +//! Placeholder. Real implementation lands in the next commit (the canopy-proxy +//! sidecar binary: loopback S3P proxy with a broker-backed credential +//! provider). + +fn main() { + eprintln!("canopy-proxy: not yet implemented"); + std::process::exit(1); +} diff --git a/src/kopia.rs b/src/kopia.rs index f2226ef..990fcf5 100644 --- a/src/kopia.rs +++ b/src/kopia.rs @@ -230,6 +230,50 @@ pub fn kopia_connect_args(creds: &KopiaCredentials) -> Vec { args } +/// Parameters for the kopia-via-canopy-proxy connect convention. +/// +/// kopia talks to a loopback bestool S3P proxy with dummy keys; the proxy holds +/// the live STS creds and re-signs each request. Bucket/region/prefix come from +/// the canopy worklist entry; the repo password comes from +/// `/restore-credentials`. +pub struct ProxyConnect<'a> { + /// Loopback endpoint the proxy bound to, including port and IPv6 brackets, + /// e.g. `[::1]:8421`. + pub endpoint: &'a str, + pub bucket: &'a str, + pub region: &'a str, + pub prefix: &'a str, + pub repository_password: &'a str, + /// The server whose snapshots this Job restores. kopia source-host filter: + /// snapshots taken by bestool on that server are tagged with this hostname. + pub server_id: &'a str, +} + +/// Builds the kopia CLI args for connecting to a canopy-managed repo via the +/// S3P proxy sidecar. kopia carries dummy keys + TLS disabled on the loopback +/// leg; the proxy holds the real STS creds. +pub fn kopia_connect_args_proxy(conn: &ProxyConnect<'_>) -> Vec { + vec![ + "repository".to_string(), + "connect".to_string(), + "s3".to_string(), + "--readonly".to_string(), + format!("--bucket={}", conn.bucket), + format!("--prefix={}", conn.prefix), + format!("--region={}", conn.region), + format!("--endpoint={}", conn.endpoint), + "--disable-tls".to_string(), + format!("--access-key={}", bestool_kopia::PROXY_DUMMY_ACCESS_KEY), + format!( + "--secret-access-key={}", + bestool_kopia::PROXY_DUMMY_SECRET_KEY + ), + format!("--password={}", conn.repository_password), + "--override-username=canopy".to_string(), + format!("--override-hostname={}", conn.server_id), + ] +} + #[cfg(test)] mod tests { use super::*; @@ -665,4 +709,68 @@ mod tests { assert_eq!(creds.endpoint, None); assert!(!creds.disable_tls); } + + #[test] + fn kopia_connect_args_proxy_emits_dummy_keys_and_loopback() { + let conn = ProxyConnect { + endpoint: "[::1]:8421", + bucket: "canopy-prod-tongatongaisland", + region: "ap-southeast-2", + prefix: "", + repository_password: "secret-pass", + server_id: "11111111-1111-1111-1111-111111111111", + }; + let args = kopia_connect_args_proxy(&conn); + + assert_eq!(args[0..4], ["repository", "connect", "s3", "--readonly"]); + assert!(args.contains(&"--bucket=canopy-prod-tongatongaisland".to_string())); + assert!(args.contains(&"--region=ap-southeast-2".to_string())); + assert!(args.contains(&"--prefix=".to_string())); + assert!(args.contains(&"--endpoint=[::1]:8421".to_string())); + assert!(args.contains(&"--disable-tls".to_string())); + assert!(args.contains(&format!( + "--access-key={}", + bestool_kopia::PROXY_DUMMY_ACCESS_KEY + ))); + assert!(args.contains(&format!( + "--secret-access-key={}", + bestool_kopia::PROXY_DUMMY_SECRET_KEY + ))); + assert!(args.contains(&"--password=secret-pass".to_string())); + assert!(args.contains(&"--override-username=canopy".to_string())); + assert!( + args.contains(&"--override-hostname=11111111-1111-1111-1111-111111111111".to_string()) + ); + } + + #[test] + fn kopia_connect_args_proxy_omits_tls_verification_flag() { + // On the loopback leg there's no TLS at all, so --disable-tls-verification + // (which applies to the TLS handshake) is irrelevant — only --disable-tls + // is needed. The legacy minio path is the only one that needs both. + let conn = ProxyConnect { + endpoint: "[::1]:1234", + bucket: "b", + region: "r", + prefix: "p", + repository_password: "rp", + server_id: "s", + }; + let args = kopia_connect_args_proxy(&conn); + assert!(!args.contains(&"--disable-tls-verification".to_string())); + } + + #[test] + fn kopia_connect_args_proxy_carries_prefix() { + let conn = ProxyConnect { + endpoint: "[::1]:1234", + bucket: "b", + region: "r", + prefix: "backups/2026", + repository_password: "rp", + server_id: "s", + }; + let args = kopia_connect_args_proxy(&conn); + assert!(args.contains(&"--prefix=backups/2026".to_string())); + } } From f9cc8d5062c94bd7c7c28c802e4784c903d7f9d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Wed, 1 Jul 2026 10:32:41 +1200 Subject: [PATCH 06/20] feat(canopy): client wrapper module + canopy-proxy sidecar binary src/canopy.rs wraps bestool_canopy::CanopyClient with a thin layer that owns SOCKS5-proxy construction (Tailscale sidecar at [::1]:1055 by default) and exposes the four restore-* endpoints pgro consumes (capabilities, worklist, credentials, verification). Wire types are re-exported from bestool-canopy verbatim. src/bin/canopy_proxy.rs is the new sidecar binary that runs the bestool S3P loopback re-signing proxy for one kopia run. Its BrokerCredentialProvider fetches creds from the operator's in-cluster /internal/restore-creds endpoint and refreshes within 2min of expiry (matching bestool's cadence). Bestool's proxy::spawn binds an ephemeral port; the sidecar writes that port to PGRO_PROXY_PORT_FILE (atomic rename) so the sibling kopia container can discover it. On SIGTERM the sidecar writes TrafficStats to PGRO_PROXY_STATS_FILE for the restore-verification reporter to include. Adds an Error::Canopy variant, the uuid crate, and tempfile as a dev-dep for the sidecar's port-file roundtrip test. --- Cargo.lock | 15 +++ Cargo.toml | 2 + src/bin/canopy_proxy.rs | 287 +++++++++++++++++++++++++++++++++++++++- src/canopy.rs | 171 ++++++++++++++++++++++++ src/error.rs | 3 + src/lib.rs | 1 + 6 files changed, 473 insertions(+), 6 deletions(-) create mode 100644 src/canopy.rs diff --git a/Cargo.lock b/Cargo.lock index 4430617..bbfe336 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3164,6 +3164,7 @@ dependencies = [ "serde_json", "serde_yaml", "socket2", + "tempfile", "thiserror 2.0.18", "tokio", "tokio-postgres", @@ -3171,6 +3172,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "uuid", ] [[package]] @@ -4324,6 +4326,19 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "terminal_size" version = "0.4.4" diff --git a/Cargo.toml b/Cargo.toml index 20f36ad..bfebcd7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,11 +34,13 @@ tokio-postgres = "0.7" tower-http = { version = "0.6", features = ["trace"] } tracing = "0.1.44" tracing-subscriber = { version = "0.3.23", features = ["json", "env-filter"] } +uuid = { version = "1.23.4", features = ["v4", "serde"] } [dev-dependencies] bytes = "1" http = "1" http-body-util = "0.1" +tempfile = "3.27.0" tower = "0.5" [[bin]] diff --git a/src/bin/canopy_proxy.rs b/src/bin/canopy_proxy.rs index 77b5b0d..f87097e 100644 --- a/src/bin/canopy_proxy.rs +++ b/src/bin/canopy_proxy.rs @@ -1,8 +1,283 @@ -//! Placeholder. Real implementation lands in the next commit (the canopy-proxy -//! sidecar binary: loopback S3P proxy with a broker-backed credential -//! provider). +//! `canopy-proxy` — sidecar binary that runs the bestool S3P loopback +//! re-signing proxy for one kopia run. +//! +//! Lifecycle: +//! +//! 1. Read configuration from env (broker URL, group, type, region, file paths). +//! 2. Spawn `bestool_kopia::proxy::spawn` with a [`BrokerCredentialProvider`] +//! that fetches creds from the operator's in-cluster broker. +//! 3. Write the ephemeral port to `PGRO_PROXY_PORT_FILE` (atomic rename) so +//! the sibling kopia container can discover it. +//! 4. Wait for SIGTERM (kopia container completing → pod termination). +//! 5. On shutdown, write `TrafficStats` to `PGRO_PROXY_STATS_FILE` for the +//! operator's restore-verification reporter to pick up. +//! +//! The kopia container is responsible for waiting on the port file before +//! invoking kopia (job builder injects a shell wrapper, see Step 7 of the +//! integration spec). -fn main() { - eprintln!("canopy-proxy: not yet implemented"); - std::process::exit(1); +use std::{path::PathBuf, pin::Pin, process::ExitCode, sync::Arc, time::Duration}; + +use bestool_canopy::BackupCredentials; +use bestool_kopia::proxy::{self, BoxError, CredentialProvider, Credentials, S3ProxyConfig}; +use jiff::{Timestamp, ToSpan}; +use serde::Serialize; +use tokio::sync::Mutex; +use tracing::{error, info}; + +/// Refresh creds this far ahead of their `Expiration` (mirrors bestool's +/// 2-minute margin). +const REFRESH_MARGIN_MINUTES: i64 = 2; + +#[derive(Debug)] +struct Config { + broker_url: String, + group: String, + backup_type: String, + region: String, + port_file: PathBuf, + stats_file: PathBuf, +} + +impl Config { + fn from_env() -> Result { + Ok(Self { + broker_url: env_required("PGRO_BROKER_URL")?, + group: env_required("PGRO_GROUP")?, + backup_type: env_required("PGRO_TYPE")?, + region: env_required("PGRO_REGION")?, + port_file: env_or("PGRO_PROXY_PORT_FILE", "/var/run/pgro/proxy-port").into(), + stats_file: env_or("PGRO_PROXY_STATS_FILE", "/var/run/pgro/proxy-stats.json").into(), + }) + } +} + +fn env_required(name: &str) -> Result { + std::env::var(name).map_err(|_| format!("{name} not set")) +} + +fn env_or(name: &str, default: &str) -> String { + std::env::var(name).unwrap_or_else(|_| default.to_string()) +} + +/// Pulls creds from the operator's broker. Caches the response in-process and +/// refreshes within the margin of expiry. +struct BrokerCredentialProvider { + http: reqwest::Client, + url: String, + group: String, + backup_type: String, + cache: Mutex>, +} + +impl BrokerCredentialProvider { + fn new(broker_url: &str, group: &str, backup_type: &str) -> Self { + Self { + http: reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .expect("static reqwest client builder"), + url: format!( + "{}/internal/restore-creds", + broker_url.trim_end_matches('/') + ), + group: group.to_string(), + backup_type: backup_type.to_string(), + cache: Mutex::new(None), + } + } + + async fn fetch(&self) -> Result { + #[derive(Serialize)] + struct Req<'a> { + group: &'a str, + r#type: &'a str, + } + let resp = self + .http + .post(&self.url) + .json(&Req { + group: &self.group, + r#type: &self.backup_type, + }) + .send() + .await? + .error_for_status()?; + // Broker returns `RestoreCredentials { credentials, repo_password }`; + // the sidecar only needs the credentials field (the repo password is + // already in kopia's argv via the Job spec). + #[derive(serde::Deserialize)] + struct Resp { + credentials: BackupCredentials, + } + let body: Resp = resp.json().await?; + Ok(body.credentials) + } +} + +fn needs_refresh(cached: &Option, now: Timestamp) -> bool { + match cached { + None => true, + Some(creds) => creds.expiration <= now + REFRESH_MARGIN_MINUTES.minutes(), + } +} + +impl CredentialProvider for BrokerCredentialProvider { + fn credentials( + &self, + ) -> Pin> + Send + '_>> { + Box::pin(async move { + let mut cached = self.cache.lock().await; + if needs_refresh(&cached, Timestamp::now()) { + let fresh = self.fetch().await?; + *cached = Some(fresh); + } + let creds = cached.as_ref().expect("cache populated above"); + Ok(Credentials { + access_key: creds.access_key_id.clone(), + secret_key: creds.secret_access_key.0.clone(), + session_token: Some(creds.session_token.0.clone()), + }) + }) + } +} + +#[derive(Serialize)] +struct StatsFile { + sent_raw_bytes: u64, + sent_payload_bytes: u64, + received_raw_bytes: u64, + received_payload_bytes: u64, +} + +/// Write `port` to `port_file` atomically (write `.tmp` then rename) so a +/// partial read by the kopia container's wait-loop can't observe a torn value. +fn write_port_atomic(port_file: &std::path::Path, port: u16) -> std::io::Result<()> { + if let Some(parent) = port_file.parent() { + std::fs::create_dir_all(parent)?; + } + let tmp = port_file.with_extension("tmp"); + std::fs::write(&tmp, port.to_string())?; + std::fs::rename(&tmp, port_file) +} + +fn write_stats(stats_file: &std::path::Path, stats: &StatsFile) -> std::io::Result<()> { + if let Some(parent) = stats_file.parent() { + std::fs::create_dir_all(parent)?; + } + let json = serde_json::to_string(stats).expect("stats serialize"); + std::fs::write(stats_file, json) +} + +async fn run(cfg: Config) -> Result<(), String> { + let group = cfg.group.clone(); + let backup_type = cfg.backup_type.clone(); + let provider = Arc::new(BrokerCredentialProvider::new( + &cfg.broker_url, + &group, + &backup_type, + )); + let s3_cfg = S3ProxyConfig { + upstream: format!("https://s3.{}.amazonaws.com", cfg.region), + upstream_host: format!("s3.{}.amazonaws.com", cfg.region), + region: cfg.region.clone(), + }; + let proxy = proxy::spawn(s3_cfg, provider) + .await + .map_err(|err| format!("spawning S3P proxy: {err}"))?; + let addr = proxy.addr(); + let port = addr.port(); + info!(%addr, port, "S3P proxy bound"); + + write_port_atomic(&cfg.port_file, port) + .map_err(|err| format!("writing port file {}: {err}", cfg.port_file.display()))?; + + // Wait for SIGTERM (kopia container completion → pod termination). + tokio::signal::ctrl_c() + .await + .map_err(|err| format!("waiting for shutdown signal: {err}"))?; + info!("shutdown signal received"); + + let traffic = proxy.traffic(); + let stats = StatsFile { + sent_raw_bytes: traffic.sent_raw, + sent_payload_bytes: traffic.sent_payload, + received_raw_bytes: traffic.received_raw, + received_payload_bytes: traffic.received_payload, + }; + if let Err(err) = write_stats(&cfg.stats_file, &stats) { + error!(error = %err, stats_file = %cfg.stats_file.display(), "writing stats file failed"); + } else { + info!(?stats.sent_raw_bytes, ?stats.received_raw_bytes, "wrote stats file"); + } + Ok(()) +} + +#[tokio::main] +async fn main() -> ExitCode { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .with_target(false) + .init(); + + let cfg = match Config::from_env() { + Ok(c) => c, + Err(err) => { + eprintln!("canopy-proxy: {err}"); + return ExitCode::from(2); + } + }; + if let Err(err) = run(cfg).await { + error!(error = %err, "canopy-proxy exiting with error"); + return ExitCode::FAILURE; + } + ExitCode::SUCCESS +} + +#[cfg(test)] +mod tests { + use super::*; + + fn creds_at(expiration: &str) -> BackupCredentials { + // Build via JSON: BackupCredentials' fields use wrapper types + // (Redacted, etc.) that aren't all re-exported at the crate root. + serde_json::from_value(serde_json::json!({ + "Version": 1, + "AccessKeyId": "AKIA", + "SecretAccessKey": "secret", + "SessionToken": "session", + "Expiration": expiration, + })) + .unwrap() + } + + #[test] + fn needs_refresh_when_absent() { + let now: Timestamp = "2026-07-01T00:00:00Z".parse().unwrap(); + assert!(needs_refresh(&None, now)); + } + + #[test] + fn needs_refresh_within_margin() { + let now: Timestamp = "2026-07-01T00:00:00Z".parse().unwrap(); + assert!(needs_refresh(&Some(creds_at("2026-07-01T00:01:00Z")), now)); + } + + #[test] + fn does_not_need_refresh_beyond_margin() { + let now: Timestamp = "2026-07-01T00:00:00Z".parse().unwrap(); + assert!(!needs_refresh(&Some(creds_at("2026-07-01T01:00:00Z")), now)); + } + + #[test] + fn write_port_atomic_roundtrip() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("subdir").join("port"); + write_port_atomic(&path, 12345).unwrap(); + let content = std::fs::read_to_string(&path).unwrap(); + assert_eq!(content, "12345"); + } } diff --git a/src/canopy.rs b/src/canopy.rs new file mode 100644 index 0000000..c6b907d --- /dev/null +++ b/src/canopy.rs @@ -0,0 +1,171 @@ +//! Thin wrapper around `bestool_canopy::CanopyClient`. +//! +//! Owns construction (the SOCKS5 proxy wiring to the Tailscale sidecar, plus +//! optional mTLS fallback) and exposes the four restore-* endpoints pgro +//! consumes: `restore_capabilities`, `restore_worklist`, `restore_credentials`, +//! `restore_verification`. Each is a one-line forward; the wrapper exists as +//! the integration seam tests inject a stub at, and as the place to hang +//! pgro-specific logging / retry / cache concerns later. + +use bestool_canopy::{ + CanopyClient, RestoreCredentials, RestoreVerification, WorklistEntry, client_builder, +}; +use reqwest::Url; +use uuid::Uuid; + +use crate::error::{Error, Result}; + +/// Default SOCKS5 proxy the operator's Tailscale sidecar listens on (IPv6 +/// loopback). Override via `CANOPY_SOCKS5_PROXY` for tests / non-sidecar +/// dev setups; set to empty to disable proxy entirely. +pub const DEFAULT_SOCKS5_PROXY: &str = "socks5://[::1]:1055"; + +/// Configuration for the canopy client at startup. +#[derive(Debug, Clone)] +pub struct CanopyConfig { + /// The public-mTLS base URL. Used by `bestool_canopy::CanopyClient` as + /// the fallback endpoint when the tailnet probe fails. The tailnet URL + /// itself is hardcoded in `bestool-canopy`. + pub base_url: Url, + /// SOCKS5 URL of the Tailscale sidecar's userspace proxy. Empty string + /// means no proxy. + pub socks5_proxy: String, + /// Optional device cert + key (PEM, concatenated). Used only if the + /// tailnet probe fails. + pub device_key_pem: Option, +} + +/// Build the inner `bestool_canopy::CanopyClient`. The SOCKS5 proxy URL is +/// captured into the builder factory so every probe + reconnect uses it. +async fn build_inner(cfg: &CanopyConfig) -> Result { + let socks5 = cfg.socks5_proxy.clone(); + let version = env!("CARGO_PKG_VERSION").to_string(); + let make_builder = move || { + let mut b = client_builder(&version); + if !socks5.is_empty() { + match reqwest::Proxy::all(&socks5) { + Ok(proxy) => b = b.proxy(proxy), + Err(err) => { + tracing::error!(socks5_proxy = %socks5, error = %err, "CanopyConfig: invalid SOCKS5 proxy URL"); + } + } + } + b + }; + + let inner = CanopyClient::new( + env!("CARGO_PKG_VERSION"), + cfg.device_key_pem.as_deref(), + make_builder, + ) + .await + .map_err(|err| Error::Canopy(format!("constructing canopy client: {err}")))? + .ok_or_else(|| { + Error::Canopy( + "canopy client unconfigured: tailnet unreachable and no device cert provided".into(), + ) + })?; + Ok(inner) +} + +/// pgro's canopy client wrapper. Holds the live `bestool_canopy::CanopyClient` +/// plus the public-mTLS base URL — the bestool client uses its own hardcoded +/// tailnet URL on the tailnet path; the base URL is the mTLS-leg fallback. +pub struct Client { + inner: CanopyClient, + base_url: Url, +} + +impl std::fmt::Debug for Client { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("canopy::Client") + .field("base_url", &self.base_url.as_str()) + .finish_non_exhaustive() + } +} + +impl Client { + /// Build a client from operator-level config. Returns `Ok(None)` if no + /// canopy integration is configured (no base URL set) — the operator + /// then runs in legacy-only mode. + pub async fn from_config(cfg: Option) -> Result> { + let Some(cfg) = cfg else { return Ok(None) }; + let inner = build_inner(&cfg).await?; + Ok(Some(Self { + inner, + base_url: cfg.base_url, + })) + } + + /// Register the intents this consumer supports. Replaces the registered + /// set wholesale (per canopy's semantics). + pub async fn restore_capabilities(&self, intents: &[&str]) -> Result<()> { + self.inner + .restore_capabilities(&self.base_url, intents) + .await + .map_err(|err| Error::Canopy(format!("restore_capabilities: {err}"))) + } + + /// Fetch the consumer's desired-state worklist. Each entry is one + /// concrete replica to maintain. + pub async fn worklist(&self) -> Result> { + self.inner + .restore_worklist(&self.base_url) + .await + .map_err(|err| Error::Canopy(format!("restore_worklist: {err}"))) + } + + /// Fetch short-lived read-only STS creds plus the repo password for a + /// `(group, type)`. Authorized iff a declaration covers it. + pub async fn restore_credentials( + &self, + backup_type: &str, + group: Uuid, + ) -> Result { + self.inner + .restore_credentials(&self.base_url, backup_type, group) + .await + .map_err(|err| { + Error::Canopy(format!( + "restore_credentials({backup_type}, {group}): {err}" + )) + }) + } + + /// Report a restore outcome (signal 3, restore-verification). + pub async fn restore_verification(&self, report: &RestoreVerification<'_>) -> Result<()> { + self.inner + .restore_verification(&self.base_url, report) + .await + .map_err(|err| Error::Canopy(format!("restore_verification: {err}"))) + } + + /// Direct access to the public-mTLS base URL the client is configured + /// against. The tailnet path uses its own hardcoded URL inside + /// `bestool-canopy`. + pub fn base_url(&self) -> &Url { + &self.base_url + } +} + +/// Re-export the wire types pgro consumes verbatim from `bestool-canopy`. +pub use bestool_canopy::{ + BackupCredentials as Credentials, Outcome, RestoreCredentials as CredsResponse, + RestoreVerification as Verification, WorklistEntry as Entry, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_socks5_proxy_is_v6_loopback() { + assert_eq!(DEFAULT_SOCKS5_PROXY, "socks5://[::1]:1055"); + } + + #[test] + fn default_socks5_proxy_parses_as_reqwest_proxy() { + // Catches a typo in the constant — reqwest insists on a real URL. + reqwest::Proxy::all(DEFAULT_SOCKS5_PROXY).expect("DEFAULT_SOCKS5_PROXY must parse"); + } +} diff --git a/src/error.rs b/src/error.rs index bcbfb7e..ed90dc1 100644 --- a/src/error.rs +++ b/src/error.rs @@ -43,6 +43,9 @@ pub enum Error { #[error("Database connection error: {0}")] Postgres(#[from] tokio_postgres::Error), + + #[error("Canopy error: {0}")] + Canopy(String), } pub type Result = std::result::Result; diff --git a/src/lib.rs b/src/lib.rs index 740f198..0098134 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +pub mod canopy; pub mod context; pub mod controllers; pub mod error; From 7e3253e8978f8675d516e736d6f2045d293b2b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Wed, 1 Jul 2026 12:31:44 +1200 Subject: [PATCH 07/20] feat(canopy): worklist syncer controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/controllers/canopy.rs is pgro's third top-level controller. Unlike the CRD-watched replica/restore controllers, it ticks periodically (~30s jittered) and reconciles cluster state (labelled Namespaces + annotations) against canopy's worklist. No CRD, no intermediate CR. Provides: - Namespace-name computation: -<8-hex(sha256(replica_id || server_id))>, DNS-1123-safe, ≤63 chars. - Pure diff function producing an Action per (entry, namespace) pair: Provision / Refresh(reason) / Teardown / NoOp. Testable without a cluster (14 unit tests cover slug/hash/diff). - CanopyController::run_forever with tokio-time-interval + ±20% jitter. - Provisioning of the Namespace itself with immutable labels (declaration-id, group, server, type, intent) and initial annotations (restore-state=pending, desired-snapshot-id from worklist). - Teardown deletes the Namespace (cascade handles children). Refresh + full teardown-drain + the actual restore Job spawn are stubs here; they land in the next commit alongside the KopiaSource job-builder refactor. Adds sha2 as a direct dep (already transitive via rustls) and a canopy: Option> field on Context so the syncer can find its client. The field defaults to None; startup wiring lands in the follow-up operator.rs commit. --- Cargo.lock | 1 + Cargo.toml | 1 + src/context.rs | 5 + src/controllers.rs | 1 + src/controllers/canopy.rs | 647 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 655 insertions(+) create mode 100644 src/controllers/canopy.rs diff --git a/Cargo.lock b/Cargo.lock index bbfe336..8ca9a7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3163,6 +3163,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha2 0.10.9", "socket2", "tempfile", "thiserror 2.0.18", diff --git a/Cargo.toml b/Cargo.toml index bfebcd7..ce99589 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ schemars = { version = "1.2.1", features = ["jiff02"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.150" serde_yaml = "0.9.34" +sha2 = "0.10.9" socket2 = "0.6.2" thiserror = "2.0.18" tokio = { version = "1.50.0", features = ["full"] } diff --git a/src/context.rs b/src/context.rs index 19b43bc..881b36d 100644 --- a/src/context.rs +++ b/src/context.rs @@ -21,6 +21,10 @@ pub struct Context { pub kopia_image: Arc>, pub use_port_forward: Arc, pub http_client: reqwest::Client, + /// Canopy integration client — `None` when the operator is running in + /// legacy-only mode (no canopy config provided). Populated at startup + /// by [`crate::canopy::Client::from_config`]. + pub canopy: Option>, /// In-memory store for snapshot-list results POSTed by jobs. pub snapshot_results: Arc, /// In-memory store for schema migration results POSTed by jobs. @@ -60,6 +64,7 @@ impl Context { kopia_image: Arc::new(RwLock::new(kopia_image)), use_port_forward: Arc::new(AtomicBool::new(use_port_forward)), http_client: reqwest::Client::new(), + canopy: None, snapshot_results: Arc::new(CallbackStore::default()), schema_migration_results: Arc::new(CallbackStore::default()), callback_base_url, diff --git a/src/controllers.rs b/src/controllers.rs index c8d6c49..8f5f22c 100644 --- a/src/controllers.rs +++ b/src/controllers.rs @@ -1,6 +1,7 @@ use k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, Pod, SecretKeySelector, SecretReference}; use kube::{Api, Client}; +pub mod canopy; pub mod jobs; pub mod postgres; pub mod replica; diff --git a/src/controllers/canopy.rs b/src/controllers/canopy.rs new file mode 100644 index 0000000..7adf58e --- /dev/null +++ b/src/controllers/canopy.rs @@ -0,0 +1,647 @@ +//! Canopy worklist syncer — pgro's third top-level controller. +//! +//! Ticks periodically (default 30s, jittered ±20%), fetches +//! `GET /restore-worklist`, discovers the pgro-managed Namespaces already +//! in the cluster, and reconciles the diff by provisioning / refreshing / +//! tearing down per-replica Namespaces. Unlike the `replica` and `restore` +//! controllers this one is **not** CRD-watched: cluster state (labelled +//! Namespaces + annotations) is the runtime model, canopy's worklist is +//! the spec, and there is no intermediate CR. +//! +//! Actual Job creation for the restore itself is delegated to the Job +//! builder (see step 7 in the integration spec); this module only writes +//! Namespaces and their labels/annotations, leaving `restore-state` +//! transitions for the follow-up commits. + +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, + time::Duration, +}; + +use bestool_canopy::WorklistEntry; +use futures::stream::{self, StreamExt}; +use k8s_openapi::{ + api::core::v1::Namespace, + apimachinery::pkg::apis::meta::v1::{ObjectMeta, Time}, +}; +use kube::{ + Api, ResourceExt, + api::{DeleteParams, ListParams, PostParams}, +}; +use rand::RngExt; +use sha2::{Digest, Sha256}; +use tracing::{debug, error, info, warn}; +use uuid::Uuid; + +use crate::{context::Context, error::Result}; + +/// How many per-entry reconciliations run concurrently within one tick. +/// Keeps the k8s apiserver from being hit by a stampede when the worklist +/// is large. +const RECONCILE_CONCURRENCY: usize = 8; + +/// Jitter multiplier applied to the reconcile interval each tick (±20%). +const JITTER_RATIO: f64 = 0.2; + +/// Labels applied to canopy-managed Namespaces. +/// +/// Labels are the discovery key: `LIST Namespaces +/// label=pgro.bes.au/managed-by=pgro-canopy` returns every canopy-backed +/// replica in one call. They also carry the immutable identity of the +/// replica (declaration id, group, server, type, intent) — mutable +/// runtime state lives in [`annotations`]. +pub mod labels { + pub const MANAGED_BY: &str = "pgro.bes.au/managed-by"; + pub const MANAGED_BY_VALUE: &str = "pgro-canopy"; + pub const DECLARATION_ID: &str = "pgro.bes.au/declaration-id"; + pub const GROUP: &str = "pgro.bes.au/group"; + pub const SERVER: &str = "pgro.bes.au/server"; + pub const TYPE: &str = "pgro.bes.au/type"; + pub const INTENT: &str = "pgro.bes.au/intent"; +} + +/// Annotations on canopy-managed Namespaces — mutable per-replica state. +pub mod annotations { + /// The snapshot canopy wants restored. Populated on every successful + /// worklist sync; compared against `LAST_RESTORED_SNAPSHOT_ID` to + /// detect the "newer snapshot available" refresh trigger. + pub const DESIRED_SNAPSHOT_ID: &str = "pgro.bes.au/desired-snapshot-id"; + pub const DESIRED_SNAPSHOT_AT: &str = "pgro.bes.au/desired-snapshot-at"; + pub const LAST_RESTORED_SNAPSHOT_ID: &str = "pgro.bes.au/last-restored-snapshot-id"; + pub const LAST_RESTORED_AT: &str = "pgro.bes.au/last-restored-at"; + pub const RESTORE_STATE: &str = "pgro.bes.au/restore-state"; + pub const LAST_VERIFICATION_REPORTED_AT: &str = "pgro.bes.au/last-verification-reported-at"; + pub const LAST_VERIFICATION_ERROR: &str = "pgro.bes.au/last-verification-error"; + /// Operator escape hatch: setting this annotation to any value triggers + /// a refresh on the next tick and is cleared once the refresh Job is + /// spawned. + pub const FORCE_REFRESH: &str = "pgro.bes.au/force-refresh"; +} + +/// Values for the `RESTORE_STATE` annotation. +pub mod restore_state { + pub const PENDING: &str = "pending"; + pub const RESTORING: &str = "restoring"; + pub const ACTIVE: &str = "active"; + pub const FAILED: &str = "failed"; + pub const TERMINATING: &str = "terminating"; +} + +/// The reconciliation decision for one (worklist entry, current namespace) pair. +#[derive(Debug, PartialEq, Eq)] +pub enum Action { + /// Worklist has an entry, no matching namespace exists — create one. + Provision, + /// Both present, refresh needed for the given reason. + Refresh(RefreshReason), + /// Namespace exists but worklist has no entry — tear down. + Teardown, + /// Both present, in sync, healthy — nothing to do this tick. + NoOp, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum RefreshReason { + /// Canopy is offering a snapshot newer than what we last restored. + NewerSnapshot, + /// `last-restored-at` exceeded the declaration's `freshness_seconds`. + FreshnessExpired, + /// Operator set the `force-refresh` annotation. + Forced, +} + +/// Compute the k8s Namespace name for a worklist entry. +/// +/// Format: `-<8-hex(SHA-256(replica_id || server_id))>`. +/// +/// The slug is derived from the operator-set declaration name in canopy so +/// namespaces are human-recognisable; the 8-hex disambiguator covers the +/// case where a group-wide declaration (`server_id=NULL` in canopy) +/// expands to one worklist entry per live server in the group — all +/// carrying the same `replica_id` and `name`, only `server_id` differs. +pub fn namespace_name_for(entry: &WorklistEntry) -> String { + format!( + "{}-{}", + slug(&entry.name), + short_hash(entry.replica_id, entry.server_id), + ) +} + +/// Slugify to DNS-1123-label-safe: lowercased, non-alphanumeric runs → `-`, +/// leading/trailing `-` trimmed, truncated to 50 chars (leaves ≥9 chars for +/// the `-XXXXXXXX` disambiguator suffix in a 63-char label limit). +fn slug(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut prev_dash = true; + for c in s.chars() { + let mapped = if c.is_ascii_alphanumeric() { + c.to_ascii_lowercase() + } else { + '-' + }; + if mapped == '-' { + if !prev_dash { + out.push('-'); + prev_dash = true; + } + } else { + out.push(mapped); + prev_dash = false; + } + } + while out.ends_with('-') { + out.pop(); + } + while out.starts_with('-') { + out.remove(0); + } + if out.is_empty() { + out.push_str("replica"); + } + if out.len() > 50 { + out.truncate(50); + while out.ends_with('-') { + out.pop(); + } + } + out +} + +/// 8-hex-char disambiguator from SHA-256 of `replica_id || server_id`. +/// +/// Not cryptographic — just a stable, DNS-safe way to keep namespaces +/// for different (replica, server) pairs from colliding when they share +/// a `name` (group-wide declaration expanded across servers). +fn short_hash(replica_id: Uuid, server_id: Uuid) -> String { + let mut hasher = Sha256::new(); + hasher.update(replica_id.as_bytes()); + hasher.update(server_id.as_bytes()); + let digest = hasher.finalize(); + let mut out = String::with_capacity(8); + for byte in &digest[..4] { + out.push_str(&format!("{byte:02x}")); + } + out +} + +/// Diff the worklist against the discovered Namespaces and produce one +/// [`Action`] per (namespace, entry) pair. Entries and namespaces are +/// matched by `pgro.bes.au/declaration-id` label / `WorklistEntry.replica_id`; +/// unmatched entries produce a `Provision`, unmatched namespaces produce a +/// `Teardown`. Refresh vs NoOp decisions live in `evaluate_existing`. +/// +/// Pure function; testable without a cluster. +pub fn diff(entries: &[WorklistEntry], namespaces: &[Namespace]) -> Vec<(String, Action)> { + let mut by_replica: HashMap> = HashMap::new(); + for ns in namespaces { + if let Some(replica) = ns + .labels() + .get(labels::DECLARATION_ID) + .and_then(|s| Uuid::parse_str(s).ok()) + { + by_replica.entry(replica).or_default().push(ns.clone()); + } + } + + let mut seen_replicas: HashSet = HashSet::new(); + let mut actions = Vec::new(); + + for entry in entries { + let target_name = namespace_name_for(entry); + let matched = by_replica + .get(&entry.replica_id) + .and_then(|nss| nss.iter().find(|ns| ns.name_any() == target_name).cloned()); + match matched { + Some(existing) => { + let action = evaluate_existing(entry, &existing); + actions.push((target_name, action)); + } + None => actions.push((target_name, Action::Provision)), + } + seen_replicas.insert(entry.replica_id); + } + + // Namespaces with no matching worklist entry → teardown. Match by the + // full derived name so a namespace lingering after a declaration was + // deleted and re-created with the same id but a different server won't + // be reused mid-flight. + for ns in namespaces { + let name = ns.name_any(); + let already = actions.iter().any(|(n, _)| n == &name); + if !already { + actions.push((name, Action::Teardown)); + } + } + + actions +} + +/// Refresh decision for an existing namespace vs its worklist entry. Pure. +fn evaluate_existing(entry: &WorklistEntry, ns: &Namespace) -> Action { + let annos = ns.annotations(); + if annos.get(annotations::FORCE_REFRESH).is_some() { + return Action::Refresh(RefreshReason::Forced); + } + let last_restored = annos.get(annotations::LAST_RESTORED_SNAPSHOT_ID); + if let (Some(desired), Some(last)) = (entry.snapshot_id.as_ref(), last_restored) + && desired != last + { + return Action::Refresh(RefreshReason::NewerSnapshot); + } + if entry.snapshot_id.is_some() && last_restored.is_none() { + // Namespace exists but never completed a restore — treat as still + // in the initial provisioning attempt. The provision path will + // re-observe the Job on the next tick. + return Action::NoOp; + } + if let (Some(fresh_secs), Some(last_at_str)) = ( + entry.freshness_seconds, + annos.get(annotations::LAST_RESTORED_AT), + ) && fresh_secs > 0 + && let Ok(last_at) = last_at_str.parse::() + { + let elapsed_secs = jiff::Timestamp::now() + .duration_since(last_at) + .as_secs() + .max(0); + if (elapsed_secs as u64) > (fresh_secs as u64) { + return Action::Refresh(RefreshReason::FreshnessExpired); + } + } + Action::NoOp +} + +/// The syncer controller. Holds a shared `Context`; runs a periodic tick +/// via [`Self::run_forever`], or one-shot via [`Self::tick`] for tests. +pub struct CanopyController { + ctx: Arc, + interval: Duration, +} + +impl CanopyController { + pub fn new(ctx: Arc, interval: Duration) -> Self { + Self { ctx, interval } + } + + /// Run the syncer forever with jittered ticks. Returns only if the + /// canopy client is not configured on `ctx` (legacy-only mode). + pub async fn run_forever(&self) { + if self.ctx.canopy.is_none() { + info!("canopy client not configured; skipping worklist syncer"); + return; + } + let mut rng = rand::rng(); + loop { + let jitter = rng.random_range(-JITTER_RATIO..JITTER_RATIO); + let delay = self.interval.mul_f64(1.0 + jitter); + tokio::time::sleep(delay).await; + if let Err(err) = self.tick().await { + error!(error = %err, "canopy worklist syncer tick failed"); + } + } + } + + /// One reconciliation pass. Fetches the worklist, lists namespaces, + /// dispatches per-entry actions concurrently. + pub async fn tick(&self) -> Result<()> { + let Some(canopy) = self.ctx.canopy.as_ref() else { + return Ok(()); + }; + let entries = canopy.worklist().await?; + debug!(count = entries.len(), "fetched canopy worklist"); + + let ns_api: Api = Api::all(self.ctx.client.clone()); + let params = ListParams::default().labels(&format!( + "{}={}", + labels::MANAGED_BY, + labels::MANAGED_BY_VALUE + )); + let namespaces = ns_api.list(¶ms).await?.items; + + let actions = diff(&entries, &namespaces); + info!( + worklist_entries = entries.len(), + existing_namespaces = namespaces.len(), + actions = actions.len(), + "canopy worklist syncer tick" + ); + + let entries_by_replica: HashMap = + entries.iter().map(|e| (e.replica_id, e)).collect(); + let ns_by_name: HashMap = + namespaces.iter().map(|n| (n.name_any(), n)).collect(); + + let ctx = self.ctx.clone(); + stream::iter(actions) + .for_each_concurrent(RECONCILE_CONCURRENCY, |(ns_name, action)| { + let ctx = ctx.clone(); + let entry = entries_by_replica + .iter() + .find_map(|(_, e)| { + if namespace_name_for(e) == ns_name { + Some(*e) + } else { + None + } + }) + .cloned(); + let existing = ns_by_name.get(&ns_name).map(|n| (*n).clone()); + async move { + if let Err(err) = dispatch(&ctx, &ns_name, action, entry, existing).await { + warn!( + namespace = %ns_name, + error = %err, + "canopy per-namespace reconciliation failed" + ); + } + } + }) + .await; + + Ok(()) + } +} + +async fn dispatch( + ctx: &Context, + ns_name: &str, + action: Action, + entry: Option, + existing: Option, +) -> Result<()> { + match action { + Action::Provision => { + let Some(entry) = entry else { + return Ok(()); // shouldn't happen; diff produces Provision only with an entry + }; + provision(ctx, ns_name, &entry).await + } + Action::Refresh(reason) => { + let Some(entry) = entry else { return Ok(()) }; + refresh(ctx, ns_name, &entry, reason, existing).await + } + Action::Teardown => teardown(ctx, ns_name, existing).await, + Action::NoOp => Ok(()), + } +} + +/// Create the Namespace for a new worklist entry with immutable labels + the +/// initial mutable annotations. Follow-up commits wire up the PVC / Deployment / +/// restore Job (step 7 in the integration spec); this tick just records the +/// intent by creating the Namespace and marking `restore-state=pending`. +async fn provision(ctx: &Context, ns_name: &str, entry: &WorklistEntry) -> Result<()> { + info!(namespace = %ns_name, replica_id = %entry.replica_id, "canopy: provisioning replica namespace"); + + let mut labels_map = std::collections::BTreeMap::new(); + labels_map.insert(labels::MANAGED_BY.into(), labels::MANAGED_BY_VALUE.into()); + labels_map.insert(labels::DECLARATION_ID.into(), entry.replica_id.to_string()); + labels_map.insert(labels::GROUP.into(), entry.group_id.to_string()); + labels_map.insert(labels::SERVER.into(), entry.server_id.to_string()); + labels_map.insert(labels::TYPE.into(), entry.r#type.to_string()); + labels_map.insert(labels::INTENT.into(), entry.intent.to_string()); + + let mut annos = std::collections::BTreeMap::new(); + annos.insert( + annotations::RESTORE_STATE.into(), + restore_state::PENDING.into(), + ); + if let Some(sid) = &entry.snapshot_id { + annos.insert(annotations::DESIRED_SNAPSHOT_ID.into(), sid.clone()); + } + if let Some(sat) = &entry.snapshot_at { + annos.insert(annotations::DESIRED_SNAPSHOT_AT.into(), sat.clone()); + } + + let ns = Namespace { + metadata: ObjectMeta { + name: Some(ns_name.to_string()), + labels: Some(labels_map), + annotations: Some(annos), + ..Default::default() + }, + ..Default::default() + }; + + let api: Api = Api::all(ctx.client.clone()); + api.create(&PostParams::default(), &ns).await?; + Ok(()) +} + +/// Placeholder — refresh flow lands in the follow-up commit alongside the +/// Job builder. For now, just log the reason and update the desired-snapshot +/// annotation so the next tick can pick up if the entry keeps changing. +async fn refresh( + ctx: &Context, + ns_name: &str, + entry: &WorklistEntry, + reason: RefreshReason, + _existing: Option, +) -> Result<()> { + info!(namespace = %ns_name, replica_id = %entry.replica_id, ?reason, "canopy: refresh needed (Job spawn not yet implemented)"); + let _ = (ctx, ns_name, entry); + Ok(()) +} + +/// Placeholder — teardown flow (drain, delete children, delete namespace) +/// lands in a follow-up commit. For now, mark the namespace terminating so +/// operators can see it via `kubectl` and delete the namespace directly. +async fn teardown(ctx: &Context, ns_name: &str, existing: Option) -> Result<()> { + info!(namespace = %ns_name, "canopy: worklist no longer covers this namespace; tearing down"); + if existing.is_none() { + return Ok(()); + } + let api: Api = Api::all(ctx.client.clone()); + // Namespace deletion cascades to everything inside it; k8s handles the + // child cleanup. + match api.delete(ns_name, &DeleteParams::default()).await { + Ok(_) => Ok(()), + Err(kube::Error::Api(err)) if err.code == 404 => Ok(()), + Err(err) => Err(err.into()), + } +} + +/// Now-timestamp helper for annotation values. Keeps RFC3339 stringification +/// centralised so tests can rely on the format. +pub fn now_rfc3339() -> String { + Time(jiff::Timestamp::now()).0.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(name: &str, replica: Uuid, server: Uuid) -> WorklistEntry { + serde_json::from_value(serde_json::json!({ + "replica_id": replica.to_string(), + "group_id": Uuid::nil().to_string(), + "server_id": server.to_string(), + "type": "tamanu-postgres", + "intent": "verify", + "name": name, + "snapshot_id": null, + "snapshot_at": null, + "storage": "s3", + "bucket": "b", + "prefix": "", + "region": "us-east-1", + })) + .unwrap() + } + + #[test] + fn slug_ascii_alnum_untouched() { + assert_eq!(slug("hello123"), "hello123"); + } + + #[test] + fn slug_lowercases_and_replaces_specials() { + assert_eq!(slug("Nauru Prod Analytics!"), "nauru-prod-analytics"); + } + + #[test] + fn slug_collapses_runs_of_specials() { + assert_eq!(slug("a__b--c/d.e"), "a-b-c-d-e"); + } + + #[test] + fn slug_trims_edges() { + assert_eq!(slug("---weird---"), "weird"); + } + + #[test] + fn slug_empty_becomes_replica() { + assert_eq!(slug(""), "replica"); + assert_eq!(slug("...!!!"), "replica"); + } + + #[test] + fn slug_truncates_at_50_without_trailing_dash() { + let out = slug(&"a".repeat(60)); + assert_eq!(out.len(), 50); + let out = slug(&format!("{}{}", "a".repeat(48), "--")); + assert!(out.len() <= 50); + assert!(!out.ends_with('-')); + } + + #[test] + fn short_hash_deterministic() { + let a = Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(); + let b = Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(); + let h1 = short_hash(a, b); + let h2 = short_hash(a, b); + assert_eq!(h1, h2); + assert_eq!(h1.len(), 8); + } + + #[test] + fn short_hash_differs_by_server() { + let a = Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(); + let b1 = Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(); + let b2 = Uuid::parse_str("33333333-3333-3333-3333-333333333333").unwrap(); + assert_ne!(short_hash(a, b1), short_hash(a, b2)); + } + + #[test] + fn namespace_name_length_under_dns_label_limit() { + let e = entry(&"A".repeat(200), Uuid::nil(), Uuid::nil()); + let name = namespace_name_for(&e); + assert!(name.len() <= 63, "got {} chars: {name}", name.len()); + } + + #[test] + fn diff_missing_namespace_is_provision() { + let e = entry("nauru", Uuid::new_v4(), Uuid::new_v4()); + let actions = diff(std::slice::from_ref(&e), &[]); + assert_eq!(actions.len(), 1); + assert_eq!(actions[0].1, Action::Provision); + } + + #[test] + fn diff_orphan_namespace_is_teardown() { + let ns = Namespace { + metadata: ObjectMeta { + name: Some("orphan".into()), + labels: Some(std::collections::BTreeMap::from([( + labels::MANAGED_BY.into(), + labels::MANAGED_BY_VALUE.into(), + )])), + ..Default::default() + }, + ..Default::default() + }; + let actions = diff(&[], &[ns]); + assert_eq!(actions.len(), 1); + assert_eq!(actions[0].1, Action::Teardown); + } + + #[test] + fn diff_matched_pair_is_noop_when_never_restored_yet() { + let replica = Uuid::new_v4(); + let server = Uuid::new_v4(); + let e = entry("nauru", replica, server); + let ns = Namespace { + metadata: ObjectMeta { + name: Some(namespace_name_for(&e)), + labels: Some(std::collections::BTreeMap::from([ + (labels::MANAGED_BY.into(), labels::MANAGED_BY_VALUE.into()), + (labels::DECLARATION_ID.into(), replica.to_string()), + ])), + ..Default::default() + }, + ..Default::default() + }; + let actions = diff(&[e], &[ns]); + assert_eq!(actions.len(), 1); + assert_eq!(actions[0].1, Action::NoOp); + } + + #[test] + fn diff_newer_snapshot_triggers_refresh() { + let replica = Uuid::new_v4(); + let server = Uuid::new_v4(); + let mut e = entry("nauru", replica, server); + e.snapshot_id = Some("new-snap".into()); + let ns = Namespace { + metadata: ObjectMeta { + name: Some(namespace_name_for(&e)), + labels: Some(std::collections::BTreeMap::from([ + (labels::MANAGED_BY.into(), labels::MANAGED_BY_VALUE.into()), + (labels::DECLARATION_ID.into(), replica.to_string()), + ])), + annotations: Some(std::collections::BTreeMap::from([( + annotations::LAST_RESTORED_SNAPSHOT_ID.into(), + "old-snap".into(), + )])), + ..Default::default() + }, + ..Default::default() + }; + let actions = diff(&[e], &[ns]); + assert_eq!(actions[0].1, Action::Refresh(RefreshReason::NewerSnapshot)); + } + + #[test] + fn diff_forced_refresh_annotation_wins() { + let replica = Uuid::new_v4(); + let server = Uuid::new_v4(); + let e = entry("nauru", replica, server); + let ns = Namespace { + metadata: ObjectMeta { + name: Some(namespace_name_for(&e)), + labels: Some(std::collections::BTreeMap::from([ + (labels::MANAGED_BY.into(), labels::MANAGED_BY_VALUE.into()), + (labels::DECLARATION_ID.into(), replica.to_string()), + ])), + annotations: Some(std::collections::BTreeMap::from([( + annotations::FORCE_REFRESH.into(), + "now".into(), + )])), + ..Default::default() + }, + ..Default::default() + }; + let actions = diff(&[e], &[ns]); + assert_eq!(actions[0].1, Action::Refresh(RefreshReason::Forced)); + } +} From 0062e52078866f59050f7e76b2c93f83096cff71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Wed, 1 Jul 2026 12:37:04 +1200 Subject: [PATCH 08/20] feat(canopy): wire syncer, capability registration, and broker route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/bin/operator.rs now: - constructs a canopy Client at startup from env config (CANOPY_BASE_URL, CANOPY_SOCKS5_PROXY, CANOPY_DEVICE_CERT_SECRET). None-in-env → legacy-only mode with no canopy noise; - registers pgro's supported intents (verify/analytics/disaster-recovery) with canopy via POST /restore-capabilities on a background task with bounded exponential retry; - spawns the worklist syncer with a jittered ~30s tick (configurable via CANOPY_RECONCILE_INTERVAL_SECS) on a background task; - listens on a second axum port (PGRO_BROKER_LISTEN_ADDR, default [::]:9091) for the in-cluster credential broker (POST /internal/restore-creds), with per-(group, type) caching that expires 2min before the STS creds themselves. Also moves ThreadRng use out of the syncer's await-loop into a fn-scoped helper (ThreadRng is !Send). --- src/bin/operator.rs | 246 +++++++++++++++++++++++++++++++++++++- src/controllers/canopy.rs | 14 ++- 2 files changed, 254 insertions(+), 6 deletions(-) diff --git a/src/bin/operator.rs b/src/bin/operator.rs index cea62f3..4a9399b 100644 --- a/src/bin/operator.rs +++ b/src/bin/operator.rs @@ -20,6 +20,7 @@ use tower_http::trace::TraceLayer; use tracing::{debug, info, warn}; use postgres_restore_operator::{ + canopy::{self, DEFAULT_SOCKS5_PROXY}, context::{Context, DEFAULT_DEPLOYMENT_READY_TIMEOUT_SECS, DEFAULT_KOPIA_IMAGE}, controllers, types::{PostgresPhysicalReplica, PostgresPhysicalRestore}, @@ -36,8 +37,14 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; const DEFAULT_MAX_CONCURRENT_RESTORES: usize = 2; const DEFAULT_METRICS_ADDR: &str = "[::]:8080"; const DEFAULT_METRICS_PORT: u16 = 8080; +const DEFAULT_BROKER_ADDR: &str = "[::]:9091"; +const DEFAULT_CANOPY_RECONCILE_INTERVAL_SECS: u64 = 30; const CONFIGMAP_NAME: &str = "postgres-restore-operator-config"; +/// Intent set pgro registers with canopy on startup; only worklist entries +/// with a matching intent will be dispatched. +const PGRO_SUPPORTED_INTENTS: &[&str] = &["verify", "analytics", "disaster-recovery"]; + /// Annotate the operator's own pod with the running version. async fn annotate_own_pod(client: &Client, namespace: &str) { let pod_name = match std::env::var("HOSTNAME") { @@ -192,14 +199,21 @@ async fn main() -> anyhow::Result<()> { "deployment readiness timeout configured" ); - let ctx = Arc::new(Context::new( + let mut ctx = Context::new( client.clone(), max_concurrent_restores, kopia_image, use_port_forward, callback_base_url, deployment_ready_timeout_secs, - )); + ); + ctx.canopy = load_canopy_client(&client, &namespace) + .await + .unwrap_or_else(|err| { + warn!(error = %err, "canopy client not configured; running in legacy-only mode"); + None + }); + let ctx = Arc::new(ctx); // Heartbeat: a background task updates this timestamp every 5s. // If the runtime is deadlocked, the timestamp goes stale and /livez fails. @@ -315,6 +329,48 @@ async fn main() -> anyhow::Result<()> { } }); + // Register pgro's supported intents with canopy + start the worklist + // syncer. Only runs when a canopy client was successfully constructed. + if ctx.canopy.is_some() { + let register_ctx = ctx.clone(); + tokio::spawn(async move { + register_capabilities(register_ctx).await; + }); + + let interval_secs = std::env::var("CANOPY_RECONCILE_INTERVAL_SECS") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_CANOPY_RECONCILE_INTERVAL_SECS); + let syncer_ctx = ctx.clone(); + tokio::spawn(async move { + let syncer = controllers::canopy::CanopyController::new( + syncer_ctx, + Duration::from_secs(interval_secs), + ); + syncer.run_forever().await; + warn!("canopy worklist syncer exited"); + }); + } + + // Broker HTTP server on a separate listener/port, gated by NetworkPolicy + // to accept only the proxy sidecars in canopy-backed Job pods. + let broker_addr = std::env::var("PGRO_BROKER_LISTEN_ADDR") + .unwrap_or_else(|_| DEFAULT_BROKER_ADDR.to_string()); + let broker_ctx = ctx.clone(); + tokio::spawn(async move { + let state = BrokerState::new(broker_ctx); + let app = broker_router(state); + match tokio::net::TcpListener::bind(&broker_addr).await { + Ok(listener) => { + info!(addr = broker_addr, "credential broker listening"); + if let Err(e) = axum::serve(listener, app).await { + tracing::error!(error = %e, "broker exited with error"); + } + } + Err(e) => tracing::error!(error = %e, addr = broker_addr, "broker bind failed"), + } + }); + // Start controllers let replica_api: Api = Api::all(client.clone()); let restore_api: Api = Api::all(client.clone()); @@ -513,6 +569,192 @@ async fn livez(State(state): State) -> (StatusCode, &'static str) { (StatusCode::OK, "ok") } +/// Try to build the canopy client from env config. `Ok(None)` means the +/// integration is intentionally not configured (no `CANOPY_BASE_URL`); pgro +/// runs in legacy-only mode. `Err(_)` means configuration was attempted but +/// failed — logged and downgraded to `None` at the call site. +async fn load_canopy_client( + client: &Client, + operator_namespace: &str, +) -> anyhow::Result>> { + let Ok(base_url_str) = std::env::var("CANOPY_BASE_URL") else { + return Ok(None); + }; + let base_url = reqwest::Url::parse(&base_url_str) + .map_err(|e| anyhow::anyhow!("CANOPY_BASE_URL is not a valid URL: {e}"))?; + + let socks5_proxy = + std::env::var("CANOPY_SOCKS5_PROXY").unwrap_or_else(|_| DEFAULT_SOCKS5_PROXY.to_string()); + + let device_key_pem = if let Ok(secret_name) = std::env::var("CANOPY_DEVICE_CERT_SECRET") { + let secrets: Api = + Api::namespaced(client.clone(), operator_namespace); + let sec = secrets.get(&secret_name).await?; + let data = sec.data.and_then(|d| d.into_iter().next()); + match data { + Some((_, v)) => Some(String::from_utf8(v.0).map_err(|e| { + anyhow::anyhow!("device cert secret {secret_name} not valid UTF-8: {e}") + })?), + None => { + warn!(secret = secret_name, "device cert secret has no data"); + None + } + } + } else { + None + }; + + let cfg = canopy::CanopyConfig { + base_url, + socks5_proxy, + device_key_pem, + }; + let cli = canopy::Client::from_config(Some(cfg)).await?; + Ok(cli.map(Arc::new)) +} + +/// POST `/restore-capabilities` to canopy with the intents pgro implements. +/// Retries on transient failure with exponential-ish backoff up to ~5 min; +/// past that, gives up and logs — the next operator restart will re-attempt. +async fn register_capabilities(ctx: Arc) { + let Some(canopy) = ctx.canopy.as_ref() else { + return; + }; + let mut delay = Duration::from_secs(1); + let max_delay = Duration::from_secs(300); + for attempt in 1..=8u32 { + match canopy.restore_capabilities(PGRO_SUPPORTED_INTENTS).await { + Ok(_) => { + info!( + intents = ?PGRO_SUPPORTED_INTENTS, + "registered supported intents with canopy" + ); + return; + } + Err(err) => { + warn!( + attempt, + error = %err, + "canopy restore_capabilities failed; retrying" + ); + tokio::time::sleep(delay).await; + delay = std::cmp::min(delay * 2, max_delay); + } + } + } + warn!("gave up registering supported intents with canopy after 8 attempts"); +} + +/// State passed to the credential-broker Router. Holds the operator's canopy +/// client and a per-(group, type) cache so concurrent Job sidecars don't +/// multiply upstream canopy calls. +#[derive(Clone)] +struct BrokerState { + ctx: Arc, + cache: Arc>>, +} + +#[derive(Clone)] +struct CachedCreds { + body: serde_json::Value, + /// Cached response expires this long before the STS creds' own expiry + /// so the sidecar's next refresh call gets a fresh cache miss. + expires_at: jiff::Timestamp, +} + +impl BrokerState { + fn new(ctx: Arc) -> Self { + Self { + ctx, + cache: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())), + } + } +} + +fn broker_router(state: BrokerState) -> Router { + Router::new() + .route( + "/internal/restore-creds", + axum::routing::post(post_restore_creds), + ) + .route("/healthz", get(|| async { StatusCode::OK })) + .with_state(state) + .layer(TraceLayer::new_for_http()) +} + +#[derive(serde::Deserialize)] +struct BrokerCredsRequest { + group: uuid::Uuid, + r#type: String, +} + +/// Broker endpoint the proxy sidecar hits to refresh its STS creds. Forwards +/// to canopy's `POST /restore-credentials`, caches the response per-(group, +/// type) up to 2 minutes before its expiry. 4xx failures propagate the +/// upstream status verbatim so a missing external-restore grant surfaces +/// clearly at the sidecar. +async fn post_restore_creds( + State(state): State, + axum::Json(req): axum::Json, +) -> (StatusCode, axum::Json) { + let Some(canopy) = state.ctx.canopy.as_ref() else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + axum::Json(serde_json::json!({ + "error": "canopy client not configured on operator", + })), + ); + }; + + let key = (req.group.to_string(), req.r#type.clone()); + { + let cache = state.cache.lock().await; + if let Some(cached) = cache.get(&key) + && cached.expires_at > jiff::Timestamp::now() + { + return (StatusCode::OK, axum::Json(cached.body.clone())); + } + } + + match canopy.restore_credentials(&req.r#type, req.group).await { + Ok(resp) => { + let expires_at = resp + .credentials + .expiration + .checked_sub(jiff::SignedDuration::from_secs(120)) + .unwrap_or(resp.credentials.expiration); + // bestool-canopy's RestoreCredentials only derives Deserialize; build + // the response JSON manually with the same shape the sidecar expects. + let body = serde_json::json!({ + "credentials": { + "Version": resp.credentials.version, + "AccessKeyId": resp.credentials.access_key_id, + "SecretAccessKey": resp.credentials.secret_access_key.0, + "SessionToken": resp.credentials.session_token.0, + "Expiration": resp.credentials.expiration.to_string(), + }, + "repo_password": resp.repo_password.0, + }); + let mut cache = state.cache.lock().await; + cache.insert( + key, + CachedCreds { + body: body.clone(), + expires_at, + }, + ); + (StatusCode::OK, axum::Json(body)) + } + Err(err) => { + warn!(error = %err, group = %req.group, r#type = %req.r#type, "broker: canopy restore_credentials failed"); + ( + StatusCode::BAD_GATEWAY, + axum::Json(serde_json::json!({ "error": err.to_string() })), + ) + } + } +} + /// Readiness: checks that the heartbeat is fresh (runtime is responsive). async fn readyz(State(state): State) -> (StatusCode, &'static str) { let last = state.heartbeat.load(Ordering::Relaxed); diff --git a/src/controllers/canopy.rs b/src/controllers/canopy.rs index 7adf58e..567e4d2 100644 --- a/src/controllers/canopy.rs +++ b/src/controllers/canopy.rs @@ -168,6 +168,15 @@ fn slug(s: &str) -> String { out } +/// Apply ±20% jitter to a Duration. Scopes the (non-Send) thread rng so the +/// caller can `.await` after receiving the result — `run_forever` needs +/// this since ThreadRng can't be held across an `await`. +fn jittered(base: Duration) -> Duration { + let mut rng = rand::rng(); + let jitter = rng.random_range(-JITTER_RATIO..JITTER_RATIO); + base.mul_f64(1.0 + jitter) +} + /// 8-hex-char disambiguator from SHA-256 of `replica_id || server_id`. /// /// Not cryptographic — just a stable, DNS-safe way to keep namespaces @@ -291,11 +300,8 @@ impl CanopyController { info!("canopy client not configured; skipping worklist syncer"); return; } - let mut rng = rand::rng(); loop { - let jitter = rng.random_range(-JITTER_RATIO..JITTER_RATIO); - let delay = self.interval.mul_f64(1.0 + jitter); - tokio::time::sleep(delay).await; + tokio::time::sleep(jittered(self.interval)).await; if let Err(err) = self.tick().await { error!(error = %err, "canopy worklist syncer tick failed"); } From 6e09f34dcb935cc43c296df5f03f4feb0ae4c4b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Wed, 1 Jul 2026 12:47:04 +1200 Subject: [PATCH 09/20] feat(canopy): job builder + PVC + syncer provisions actual restore Jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/controllers/canopy/builders.rs adds the canopy-path Job + PVC builders: - build_canopy_restore_job produces a Pod with two containers (kopia + the pgro-canopy-proxy sidecar) sharing an emptyDir. Kopia runs a small wrapper shell that waits for the sidecar's port-file, reads the port, then invokes kopia against [::1]: with dummy keys. - build_pgdata_pvc creates a ReadWriteOnce PVC sized per-intent. The Pod carries pgro.bes.au/proxy-sidecar=true so the operator's broker NetworkPolicy (step 10, ops) can admit its ingress. Active deadline is 4h (vs 2h on the CRD path) — the proxy refreshes creds transparently, so restores are reachability-bounded not credential-bounded. The worklist syncer's provision() now fetches creds from canopy and spawns the PVC + Job in the new namespace. Repo password comes from canopy's restore-credentials response; STS creds are refreshed per-request by the sidecar. If canopy hasn't issued a snapshot yet (worklist entry with snapshot_id=None), provisioning stops after creating the namespace and re-checks next tick. Context gains three new fields (canopy_broker_base_url, canopy_proxy_image, canopy_pgdata_pvc_size), all populated at operator startup from env. Sensible defaults; overridable via env or ConfigMap (later). --- src/bin/operator.rs | 24 +- src/context.rs | 15 + src/controllers/canopy.rs | 89 +++++- src/controllers/canopy/builders.rs | 456 +++++++++++++++++++++++++++++ 4 files changed, 581 insertions(+), 3 deletions(-) create mode 100644 src/controllers/canopy/builders.rs diff --git a/src/bin/operator.rs b/src/bin/operator.rs index 4a9399b..e2c0737 100644 --- a/src/bin/operator.rs +++ b/src/bin/operator.rs @@ -21,7 +21,10 @@ use tracing::{debug, info, warn}; use postgres_restore_operator::{ canopy::{self, DEFAULT_SOCKS5_PROXY}, - context::{Context, DEFAULT_DEPLOYMENT_READY_TIMEOUT_SECS, DEFAULT_KOPIA_IMAGE}, + context::{ + Context, DEFAULT_CANOPY_PGDATA_PVC_SIZE, DEFAULT_CANOPY_PROXY_IMAGE, + DEFAULT_DEPLOYMENT_READY_TIMEOUT_SECS, DEFAULT_KOPIA_IMAGE, + }, controllers, types::{PostgresPhysicalReplica, PostgresPhysicalRestore}, }; @@ -213,6 +216,25 @@ async fn main() -> anyhow::Result<()> { warn!(error = %err, "canopy client not configured; running in legacy-only mode"); None }); + ctx.canopy_proxy_image = std::env::var("CANOPY_PROXY_IMAGE") + .unwrap_or_else(|_| DEFAULT_CANOPY_PROXY_IMAGE.to_string()); + ctx.canopy_pgdata_pvc_size = std::env::var("CANOPY_PGDATA_PVC_SIZE") + .unwrap_or_else(|_| DEFAULT_CANOPY_PGDATA_PVC_SIZE.to_string()); + ctx.canopy_broker_base_url = if let Ok(url) = std::env::var("CANOPY_BROKER_BASE_URL") { + url + } else if let Ok(svc) = std::env::var("OPERATOR_SERVICE_NAME") { + // Broker listens on its own port; parse from PGRO_BROKER_LISTEN_ADDR + // (default [::]:9091). + let broker_addr = std::env::var("PGRO_BROKER_LISTEN_ADDR") + .unwrap_or_else(|_| DEFAULT_BROKER_ADDR.to_string()); + let broker_port: u16 = broker_addr + .rsplit_once(':') + .and_then(|(_, p)| p.parse().ok()) + .unwrap_or(9091); + format!("http://{svc}.{namespace}.svc:{broker_port}") + } else { + String::new() + }; let ctx = Arc::new(ctx); // Heartbeat: a background task updates this timestamp every 5s. diff --git a/src/context.rs b/src/context.rs index 881b36d..e3518c1 100644 --- a/src/context.rs +++ b/src/context.rs @@ -11,6 +11,8 @@ use crate::{controllers::jobs::CallbackStore, metrics::Metrics}; pub const DEFAULT_KOPIA_IMAGE: &str = "kopia/kopia:0.22.3"; pub const DEFAULT_DEPLOYMENT_READY_TIMEOUT_SECS: u64 = 30 * 60; +pub const DEFAULT_CANOPY_PROXY_IMAGE: &str = "ghcr.io/beyondessential/pgro-canopy-proxy:latest"; +pub const DEFAULT_CANOPY_PGDATA_PVC_SIZE: &str = "20Gi"; pub struct Context { pub client: Client, @@ -25,6 +27,16 @@ pub struct Context { /// legacy-only mode (no canopy config provided). Populated at startup /// by [`crate::canopy::Client::from_config`]. pub canopy: Option>, + /// Base URL the canopy-path proxy sidecar hits for STS creds, e.g. + /// `http://postgres-restore-operator.pgro-system.svc:9091`. Empty when + /// canopy is not configured. + pub canopy_broker_base_url: String, + /// Image reference for the pgro-canopy-proxy sidecar container. Set + /// by the operator startup from `CANOPY_PROXY_IMAGE`. + pub canopy_proxy_image: String, + /// Default pgdata PVC size used when the operator provisions a + /// canopy-backed replica (per intent). + pub canopy_pgdata_pvc_size: String, /// In-memory store for snapshot-list results POSTed by jobs. pub snapshot_results: Arc, /// In-memory store for schema migration results POSTed by jobs. @@ -65,6 +77,9 @@ impl Context { use_port_forward: Arc::new(AtomicBool::new(use_port_forward)), http_client: reqwest::Client::new(), canopy: None, + canopy_broker_base_url: String::new(), + canopy_proxy_image: DEFAULT_CANOPY_PROXY_IMAGE.to_string(), + canopy_pgdata_pvc_size: DEFAULT_CANOPY_PGDATA_PVC_SIZE.to_string(), snapshot_results: Arc::new(CallbackStore::default()), schema_migration_results: Arc::new(CallbackStore::default()), callback_base_url, diff --git a/src/controllers/canopy.rs b/src/controllers/canopy.rs index 567e4d2..84c8d7e 100644 --- a/src/controllers/canopy.rs +++ b/src/controllers/canopy.rs @@ -36,6 +36,12 @@ use uuid::Uuid; use crate::{context::Context, error::Result}; +mod builders; +pub use builders::{ + CanopyRestoreJobConfig, KOPIA_JOB_NAME, PGDATA_PVC_NAME, PROXY_SIDECAR_POD_LABEL, + build_canopy_restore_job, build_pgdata_pvc, +}; + /// How many per-entry reconciliations run concurrently within one tick. /// Keeps the k8s apiserver from being hit by a stampede when the worklist /// is large. @@ -429,11 +435,90 @@ async fn provision(ctx: &Context, ns_name: &str, entry: &WorklistEntry) -> Resul ..Default::default() }; - let api: Api = Api::all(ctx.client.clone()); - api.create(&PostParams::default(), &ns).await?; + let ns_api: Api = Api::all(ctx.client.clone()); + match ns_api.create(&PostParams::default(), &ns).await { + Ok(_) => info!(namespace = %ns_name, "canopy: created replica namespace"), + Err(kube::Error::Api(err)) if err.code == 409 => { + debug!(namespace = %ns_name, "canopy: namespace already exists"); + } + Err(err) => return Err(err.into()), + } + + // If canopy hasn't yet issued a snapshot for this (server, type) the + // entry has no snapshot_id — nothing to restore this tick. The next + // worklist tick will re-check. + let Some(snapshot_id) = entry.snapshot_id.as_deref() else { + info!(namespace = %ns_name, "canopy: entry has no snapshot yet, waiting"); + return Ok(()); + }; + + // The repo password comes from canopy's restore-credentials response + // (bundled with the STS creds). Fetch it once here; the proxy sidecar + // fetches STS creds itself on each refresh via the broker. + let Some(canopy) = ctx.canopy.as_ref() else { + return Ok(()); + }; + let creds = canopy + .restore_credentials(&entry.r#type.to_string(), entry.group_id) + .await?; + + spawn_pvc_and_job(ctx, ns_name, entry, snapshot_id, &creds.repo_password.0).await?; Ok(()) } +async fn spawn_pvc_and_job( + ctx: &Context, + ns_name: &str, + entry: &WorklistEntry, + snapshot_id: &str, + repo_password: &str, +) -> Result<()> { + let pvc = builders::build_pgdata_pvc(ns_name, &ctx.canopy_pgdata_pvc_size); + let pvc_api: Api = + Api::namespaced(ctx.client.clone(), ns_name); + match pvc_api.create(&PostParams::default(), &pvc).await { + Ok(_) => {} + Err(kube::Error::Api(err)) if err.code == 409 => {} + Err(err) => return Err(err.into()), + } + + let job_name = format!("restore-{}", short_id(snapshot_id)); + let cfg = builders::CanopyRestoreJobConfig { + entry, + namespace: ns_name, + job_name: &job_name, + kopia_image: &ctx.kopia_image(), + canopy_proxy_image: &ctx.canopy_proxy_image, + broker_base_url: &ctx.canopy_broker_base_url, + snapshot_id, + repo_password, + pgdata_pvc_size: &ctx.canopy_pgdata_pvc_size, + }; + let job = builders::build_canopy_restore_job(&cfg); + let job_api: Api = + Api::namespaced(ctx.client.clone(), ns_name); + match job_api.create(&PostParams::default(), &job).await { + Ok(_) => { + info!(namespace = %ns_name, job = %job_name, "canopy: created restore Job"); + } + Err(kube::Error::Api(err)) if err.code == 409 => { + debug!(namespace = %ns_name, job = %job_name, "canopy: restore Job already exists"); + } + Err(err) => return Err(err.into()), + } + Ok(()) +} + +/// First 8 chars of a snapshot id, DNS-safe. Used as a Job name suffix so +/// the Job for the current desired snapshot is stably named. +fn short_id(s: &str) -> String { + s.chars() + .filter(|c| c.is_ascii_alphanumeric()) + .take(8) + .collect::() + .to_ascii_lowercase() +} + /// Placeholder — refresh flow lands in the follow-up commit alongside the /// Job builder. For now, just log the reason and update the desired-snapshot /// annotation so the next tick can pick up if the entry keeps changing. diff --git a/src/controllers/canopy/builders.rs b/src/controllers/canopy/builders.rs new file mode 100644 index 0000000..eb09290 --- /dev/null +++ b/src/controllers/canopy/builders.rs @@ -0,0 +1,456 @@ +//! Canopy-path Job + PVC builders. +//! +//! The CRD path (`src/controllers/restore/builders.rs`) has its own +//! `build_restore_job` deeply tied to `PostgresPhysicalReplica` / +//! `PostgresPhysicalRestore` spec fields. The canopy path has no CRDs — it +//! works from a `WorklistEntry` + labelled Namespace. This module holds the +//! canopy-side equivalents, small enough to live independently rather than +//! forcing a shared abstraction across two different data models. + +use std::collections::BTreeMap; + +use bestool_canopy::WorklistEntry; +use k8s_openapi::{ + api::{ + batch::v1::{Job, JobSpec}, + core::v1::{ + Container, EmptyDirVolumeSource, EnvVar, PersistentVolumeClaim, + PersistentVolumeClaimSpec, PodSpec, PodTemplateSpec, ResourceRequirements, Volume, + VolumeMount, VolumeResourceRequirements, + }, + }, + apimachinery::pkg::api::resource::Quantity, +}; +use kube::api::ObjectMeta; + +use crate::{ + controllers::{canopy::labels, kopia_writable_env}, + kopia::{ProxyConnect, kopia_connect_args_proxy}, +}; + +/// Default kopia image used by the canopy path's restore Job when the +/// operator's dynamic `kopia_image` isn't accessible from the caller. +/// Kept in sync with `context::DEFAULT_KOPIA_IMAGE`; callers should always +/// pass the value from `ctx.kopia_image()`. +pub const KOPIA_JOB_NAME: &str = "restore"; + +/// Name of the canopy-path restore PVC. Namespaces are per-replica so a +/// stable short name is fine. +pub const PGDATA_PVC_NAME: &str = "pgdata"; + +/// Standard label used on the proxy-sidecar-carrying Pod so the operator's +/// broker NetworkPolicy can admit its ingress (see spec §4.3). +pub const PROXY_SIDECAR_POD_LABEL: (&str, &str) = ("pgro.bes.au/proxy-sidecar", "true"); + +/// Config the caller (worklist syncer) supplies to build a canopy restore Job. +pub struct CanopyRestoreJobConfig<'a> { + pub entry: &'a WorklistEntry, + pub namespace: &'a str, + pub job_name: &'a str, + pub kopia_image: &'a str, + pub canopy_proxy_image: &'a str, + /// Base URL the sidecar hits for creds, e.g. + /// `http://postgres-restore-operator.pgro-system.svc:9091`. From + /// `Context::canopy_broker_base_url`. + pub broker_base_url: &'a str, + /// Snapshot the operator wants restored — comes from the worklist + /// entry when Provision or Refresh, but callers pass it explicitly + /// because the syncer may know a more recent value. + pub snapshot_id: &'a str, + pub repo_password: &'a str, + pub pgdata_pvc_size: &'a str, +} + +/// Build the pgdata PVC for a canopy-backed replica. Namespace-scoped, one +/// PVC per replica namespace; sized from `pgdata_pvc_size` (caller decides +/// per intent). +pub fn build_pgdata_pvc(namespace: &str, size: &str) -> PersistentVolumeClaim { + PersistentVolumeClaim { + metadata: ObjectMeta { + name: Some(PGDATA_PVC_NAME.into()), + namespace: Some(namespace.into()), + ..Default::default() + }, + spec: Some(PersistentVolumeClaimSpec { + access_modes: Some(vec!["ReadWriteOnce".into()]), + resources: Some(VolumeResourceRequirements { + requests: Some(BTreeMap::from([("storage".into(), Quantity(size.into()))])), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + } +} + +/// Shell wrapper the kopia container runs on the canopy path. Waits for the +/// proxy sidecar to publish its ephemeral port to `/var/run/pgro/proxy-port` +/// (30s timeout — the sidecar writes it as soon as the proxy binds, which +/// is essentially instant), reads the port, then invokes kopia against the +/// loopback endpoint. +fn kopia_wrapper_script() -> &'static str { + r#"set -e + +mkdir -p /tmp/kopia/config /tmp/kopia/logs /tmp/kopia/cache + +PORT_FILE="/var/run/pgro/proxy-port" +for _ in $(seq 1 30); do + [ -f "$PORT_FILE" ] && break + sleep 1 +done +if [ ! -f "$PORT_FILE" ]; then + echo "ERROR: canopy-proxy sidecar did not write port file within 30s" >&2 + exit 1 +fi +PROXY_PORT=$(cat "$PORT_FILE") +CANOPY_ENDPOINT="[::1]:${PROXY_PORT}" +echo "kopia connecting via canopy proxy at ${CANOPY_ENDPOINT}" + +# Connect via proxy: kopia talks to [::1] with dummy keys, the proxy holds +# the live STS creds. +CONNECT_ARGS_FILE="/tmp/kopia-connect-args" +cat > "$CONNECT_ARGS_FILE" <` via the proxy) and the pgro-published +/// canopy-proxy sidecar. Both share an emptyDir volume for the port-file +/// coordination handshake. +pub fn build_canopy_restore_job(cfg: &CanopyRestoreJobConfig<'_>) -> Job { + // Serialize the kopia connect args once; the wrapper script injects the + // [::1]: endpoint at runtime because the port isn't known until + // the sidecar binds. Placeholder marker: @ENDPOINT@ (replaced by the + // wrapper's sed). + let connect_args = kopia_connect_args_proxy(&ProxyConnect { + endpoint: "@ENDPOINT@", + bucket: &cfg.entry.bucket, + region: &cfg.entry.region, + prefix: &cfg.entry.prefix, + repository_password: cfg.repo_password, + server_id: &cfg.entry.server_id.to_string(), + }); + let connect_args_serialized = connect_args + .into_iter() + .map(|a| shlex_quote(&a)) + .collect::>() + .join(" \\\n "); + + let mut pod_labels = BTreeMap::new(); + pod_labels.insert( + PROXY_SIDECAR_POD_LABEL.0.into(), + PROXY_SIDECAR_POD_LABEL.1.into(), + ); + pod_labels.insert( + labels::DECLARATION_ID.into(), + cfg.entry.replica_id.to_string(), + ); + pod_labels.insert(labels::SERVER.into(), cfg.entry.server_id.to_string()); + pod_labels.insert("pgro.bes.au/job-kind".into(), "canopy-restore".into()); + + let script = kopia_wrapper_script().replace("${CONNECT_ARGS}", &connect_args_serialized); + + let kopia_container = Container { + name: KOPIA_JOB_NAME.into(), + image: Some(cfg.kopia_image.into()), + command: Some(vec!["/bin/sh".into(), "-c".into()]), + args: Some(vec![script]), + env: Some( + [ + vec![EnvVar { + name: "SNAPSHOT_ID".into(), + value: Some(cfg.snapshot_id.into()), + ..Default::default() + }], + kopia_writable_env(), + ] + .concat(), + ), + volume_mounts: Some(vec![ + VolumeMount { + name: "pgdata".into(), + mount_path: "/pgdata".into(), + ..Default::default() + }, + VolumeMount { + name: "kopia-cache".into(), + mount_path: "/tmp/kopia".into(), + ..Default::default() + }, + VolumeMount { + name: "proxy-shared".into(), + mount_path: "/var/run/pgro".into(), + ..Default::default() + }, + ]), + resources: Some(ResourceRequirements { + requests: Some(BTreeMap::from([ + ("cpu".into(), Quantity("500m".into())), + ("memory".into(), Quantity("1Gi".into())), + ])), + limits: Some(BTreeMap::from([ + ("cpu".into(), Quantity("2".into())), + ("memory".into(), Quantity("4Gi".into())), + ])), + ..Default::default() + }), + ..Default::default() + }; + + let sidecar_container = Container { + name: "canopy-proxy".into(), + image: Some(cfg.canopy_proxy_image.into()), + env: Some(vec![ + EnvVar { + name: "PGRO_BROKER_URL".into(), + value: Some(cfg.broker_base_url.into()), + ..Default::default() + }, + EnvVar { + name: "PGRO_GROUP".into(), + value: Some(cfg.entry.group_id.to_string()), + ..Default::default() + }, + EnvVar { + name: "PGRO_TYPE".into(), + value: Some(cfg.entry.r#type.to_string()), + ..Default::default() + }, + EnvVar { + name: "PGRO_REGION".into(), + value: Some(cfg.entry.region.clone()), + ..Default::default() + }, + ]), + volume_mounts: Some(vec![VolumeMount { + name: "proxy-shared".into(), + mount_path: "/var/run/pgro".into(), + ..Default::default() + }]), + resources: Some(ResourceRequirements { + requests: Some(BTreeMap::from([ + ("cpu".into(), Quantity("50m".into())), + ("memory".into(), Quantity("64Mi".into())), + ])), + limits: Some(BTreeMap::from([ + ("cpu".into(), Quantity("500m".into())), + ("memory".into(), Quantity("256Mi".into())), + ])), + ..Default::default() + }), + ..Default::default() + }; + + Job { + metadata: ObjectMeta { + name: Some(cfg.job_name.into()), + namespace: Some(cfg.namespace.into()), + labels: Some(BTreeMap::from([ + ("pgro.bes.au/job-kind".into(), "canopy-restore".into()), + ( + labels::DECLARATION_ID.into(), + cfg.entry.replica_id.to_string(), + ), + ])), + ..Default::default() + }, + spec: Some(JobSpec { + backoff_limit: Some(3), + // Longer than the CRD path's 2h — the proxy refreshes creds so + // long restores aren't credential-bounded, only reachability-bounded. + active_deadline_seconds: Some(14400), // 4 hours + ttl_seconds_after_finished: Some(600), + template: PodTemplateSpec { + metadata: Some(ObjectMeta { + labels: Some(pod_labels), + ..Default::default() + }), + spec: Some(PodSpec { + restart_policy: Some("Never".into()), + // Termination grace: kopia may need a beat to flush kopia + // caches; the sidecar exits within a second of SIGTERM. + termination_grace_period_seconds: Some(30), + containers: vec![kopia_container, sidecar_container], + volumes: Some(vec![ + Volume { + name: "pgdata".into(), + persistent_volume_claim: Some( + k8s_openapi::api::core::v1::PersistentVolumeClaimVolumeSource { + claim_name: PGDATA_PVC_NAME.into(), + ..Default::default() + }, + ), + ..Default::default() + }, + Volume { + name: "kopia-cache".into(), + empty_dir: Some(EmptyDirVolumeSource::default()), + ..Default::default() + }, + Volume { + name: "proxy-shared".into(), + empty_dir: Some(EmptyDirVolumeSource { + medium: Some("Memory".into()), + ..Default::default() + }), + ..Default::default() + }, + ]), + ..Default::default() + }), + }, + // The syncer polls Job status via labels; no need for a + // Selector-based ownership contract with the operator. + ..Default::default() + }), + ..Default::default() + } +} + +/// Very small shell quoter for the connect args. kopia args have no `'` or +/// null bytes; the strings are canopy-supplied bucket/region/etc. + our +/// hardcoded dummy keys, so this is defensive rather than needed for +/// correctness. +fn shlex_quote(s: &str) -> String { + if s.is_empty() + || s.chars() + .any(|c| !(c.is_ascii_alphanumeric() || "-_.=/:[]@".contains(c))) + { + // Wrap in single quotes; there is no way for a `'` to appear in the + // kopia arg space we build, so a naive single-quote wrap is safe. + format!("'{s}'") + } else { + s.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use uuid::Uuid; + + fn worklist_entry() -> WorklistEntry { + serde_json::from_value(serde_json::json!({ + "replica_id": Uuid::new_v4().to_string(), + "group_id": Uuid::new_v4().to_string(), + "server_id": Uuid::new_v4().to_string(), + "type": "tamanu-postgres", + "intent": "verify", + "name": "test", + "snapshot_id": "abc123", + "snapshot_at": "2026-07-01T00:00:00Z", + "storage": "s3", + "bucket": "canopy-test", + "prefix": "", + "region": "ap-southeast-2", + })) + .unwrap() + } + + #[test] + fn build_canopy_restore_job_shape() { + let entry = worklist_entry(); + let cfg = CanopyRestoreJobConfig { + entry: &entry, + namespace: "pgro-r-abc", + job_name: "restore-1", + kopia_image: "kopia/kopia:0.22.3", + canopy_proxy_image: "ghcr.io/beyondessential/pgro-canopy-proxy:latest", + broker_base_url: "http://postgres-restore-operator.pgro-system.svc:9091", + snapshot_id: "abc123", + repo_password: "supersecret", + pgdata_pvc_size: "10Gi", + }; + let job = build_canopy_restore_job(&cfg); + + let spec = job.spec.as_ref().unwrap(); + let pod_spec = spec.template.spec.as_ref().unwrap(); + assert_eq!(pod_spec.containers.len(), 2); + assert_eq!(pod_spec.containers[0].name, "restore"); + assert_eq!(pod_spec.containers[1].name, "canopy-proxy"); + + // Sidecar gets the broker URL. + let sidecar_env = pod_spec.containers[1].env.as_ref().unwrap(); + assert!(sidecar_env.iter().any(|e| e.name == "PGRO_BROKER_URL" + && e.value.as_deref() + == Some("http://postgres-restore-operator.pgro-system.svc:9091"))); + assert!( + sidecar_env + .iter() + .any(|e| e.name == "PGRO_TYPE" && e.value.as_deref() == Some("tamanu-postgres")) + ); + + // Pod is labeled for the broker NetworkPolicy. + let pod_labels = job + .spec + .as_ref() + .unwrap() + .template + .metadata + .as_ref() + .unwrap() + .labels + .as_ref() + .unwrap(); + assert_eq!( + pod_labels + .get(PROXY_SIDECAR_POD_LABEL.0) + .map(String::as_str), + Some(PROXY_SIDECAR_POD_LABEL.1), + ); + + // Kopia container has the SNAPSHOT_ID env and mounts the shared proxy-port emptyDir. + let kopia_env = pod_spec.containers[0].env.as_ref().unwrap(); + assert!( + kopia_env + .iter() + .any(|e| e.name == "SNAPSHOT_ID" && e.value.as_deref() == Some("abc123")) + ); + let kopia_mounts = pod_spec.containers[0].volume_mounts.as_ref().unwrap(); + assert!( + kopia_mounts + .iter() + .any(|m| m.name == "proxy-shared" && m.mount_path == "/var/run/pgro") + ); + + // Shell script wraps the connect args and reads the port file. + let script = pod_spec.containers[0].args.as_ref().unwrap()[0].clone(); + assert!(script.contains("/var/run/pgro/proxy-port")); + assert!(script.contains("kopia snapshot restore")); + assert!(script.contains("--disable-tls")); + } + + #[test] + fn shlex_quote_leaves_simple_strings_bare() { + assert_eq!(shlex_quote("abc123-_"), "abc123-_"); + assert_eq!( + shlex_quote("--endpoint=[::1]:1234"), + "--endpoint=[::1]:1234" + ); + } + + #[test] + fn shlex_quote_wraps_specials() { + assert_eq!(shlex_quote("hello world"), "'hello world'"); + assert_eq!(shlex_quote(""), "''"); + } + + #[test] + fn build_pgdata_pvc_has_size_and_rwo() { + let pvc = build_pgdata_pvc("ns", "20Gi"); + let spec = pvc.spec.unwrap(); + assert_eq!(spec.access_modes, Some(vec!["ReadWriteOnce".into()])); + let req = spec.resources.unwrap().requests.unwrap(); + assert_eq!(req.get("storage"), Some(&Quantity("20Gi".into()))); + assert_eq!(pvc.metadata.name, Some(PGDATA_PVC_NAME.into())); + assert_eq!(pvc.metadata.namespace, Some("ns".into())); + } +} From 82e64dd658b2b4de3a86c46d1cd1b8b4f09baf9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Wed, 1 Jul 2026 12:50:15 +1200 Subject: [PATCH 10/20] feat(canopy): restore-verification reporter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/controllers/canopy/reporter.rs observes each managed namespace's restore Job at the end of every tick and: - transitions the pgro.bes.au/restore-state annotation (pending → restoring → active/failed) based on Job.status; - on the first observation of a terminal state, builds a bestool_canopy::RestoreVerification from the namespace's labels/annotations and POSTs it via ctx.canopy; - gates on pgro.bes.au/last-verification-reported-at so canopy sees the outcome exactly once per terminal state per namespace; - records failures in pgro.bes.au/last-verification-error and retries on the next tick. postgres_version + s3_*_bytes are left null for now — the former needs a Deployment on the canopy path (deferred), the latter needs a durable channel for the sidecar's TrafficStats (the current emptyDir dies with the Pod). --- src/controllers/canopy.rs | 6 + src/controllers/canopy/reporter.rs | 381 +++++++++++++++++++++++++++++ 2 files changed, 387 insertions(+) create mode 100644 src/controllers/canopy/reporter.rs diff --git a/src/controllers/canopy.rs b/src/controllers/canopy.rs index 84c8d7e..f5d1482 100644 --- a/src/controllers/canopy.rs +++ b/src/controllers/canopy.rs @@ -37,6 +37,7 @@ use uuid::Uuid; use crate::{context::Context, error::Result}; mod builders; +mod reporter; pub use builders::{ CanopyRestoreJobConfig, KOPIA_JOB_NAME, PGDATA_PVC_NAME, PROXY_SIDECAR_POD_LABEL, build_canopy_restore_job, build_pgdata_pvc, @@ -371,6 +372,11 @@ impl CanopyController { }) .await; + // Re-list namespaces so the reporter observes any state we just + // wrote. Cheap — one apiserver call for the pgro-canopy label. + let namespaces = ns_api.list(¶ms).await?.items; + reporter::observe_and_report(&self.ctx, &namespaces).await; + Ok(()) } } diff --git a/src/controllers/canopy/reporter.rs b/src/controllers/canopy/reporter.rs new file mode 100644 index 0000000..b81d780 --- /dev/null +++ b/src/controllers/canopy/reporter.rs @@ -0,0 +1,381 @@ +//! Restore-verification reporter (signal 3). +//! +//! On each syncer tick, observes each managed Namespace's restore Job. When +//! the Job reaches a terminal state we transition the Namespace's +//! `restore-state` annotation and — if it hasn't been done yet — post a +//! `RestoreVerification` to canopy. At-most-once-per-terminal-state gated +//! by the `pgro.bes.au/last-verification-reported-at` annotation; failure +//! to report is recorded in `pgro.bes.au/last-verification-error` and +//! retried on the next tick. +//! +//! What the reporter does NOT do (yet): +//! - Postgres version detection. There is no Deployment on the canopy path +//! yet; the field is left null. +//! - S3 traffic tallies. The sidecar writes them to an emptyDir volume that +//! is gone by the time we observe the terminated Pod. Requires either +//! annotation-write RBAC in the sidecar or a broker POST — deferred. + +use std::collections::BTreeMap; + +use bestool_canopy::{Outcome, RestoreVerification, WorklistEntry}; +use k8s_openapi::api::{batch::v1::Job, core::v1::Namespace}; +use kube::{ + Api, ResourceExt, + api::{ListParams, Patch, PatchParams}, +}; +use tracing::{debug, info, warn}; +use uuid::Uuid; + +use crate::{ + context::Context, + controllers::canopy::{annotations, labels, restore_state}, + error::Result, +}; + +/// Kick the reporter for every managed namespace: transition state on +/// terminal Jobs, then emit verification reports where owed. Called from +/// [`super::CanopyController::tick`] after the provision/refresh/teardown +/// dispatch. +pub async fn observe_and_report(ctx: &Context, namespaces: &[Namespace]) { + for ns in namespaces { + if let Err(err) = observe_one(ctx, ns).await { + warn!( + namespace = %ns.name_any(), + error = %err, + "canopy reporter: failed to observe namespace" + ); + } + } +} + +async fn observe_one(ctx: &Context, ns: &Namespace) -> Result<()> { + let ns_name = ns.name_any(); + let annos = ns.annotations(); + let current_state = annos + .get(annotations::RESTORE_STATE) + .map(String::as_str) + .unwrap_or(restore_state::PENDING); + + // If we're in a terminal state and already reported, nothing to do. + let terminal = matches!(current_state, restore_state::ACTIVE | restore_state::FAILED); + let reported = annos.contains_key(annotations::LAST_VERIFICATION_REPORTED_AT); + if terminal && reported { + return Ok(()); + } + + // Otherwise look at the Jobs to figure out where we are. + let job_api: Api = Api::namespaced(ctx.client.clone(), &ns_name); + let jobs = job_api + .list(&ListParams::default().labels("pgro.bes.au/job-kind=canopy-restore")) + .await?; + + let Some(job) = latest_job(&jobs.items) else { + // No Job yet — provision is still pending. Nothing to report. + return Ok(()); + }; + + let (new_state, outcome, error_msg) = classify_job(job); + + // Transition the namespace's state annotation if it changed. + if new_state != current_state { + set_annotations( + ctx, + &ns_name, + &[ + (annotations::RESTORE_STATE, Some(new_state.into())), + match outcome { + Some(Outcome::Success) => ( + annotations::LAST_RESTORED_SNAPSHOT_ID, + annos + .get(annotations::DESIRED_SNAPSHOT_ID) + .cloned() + .or_else(|| snapshot_id_from_job(job).map(String::from)), + ), + _ => (annotations::LAST_RESTORED_SNAPSHOT_ID, None), + }, + ( + annotations::LAST_RESTORED_AT, + if outcome == Some(Outcome::Success) { + Some(super::now_rfc3339()) + } else { + None + }, + ), + ], + ) + .await?; + } + + // Only report from a terminal state. + let Some(outcome) = outcome else { + return Ok(()); + }; + + // If we already reported for this terminal round, nothing more to do. + // (The reported-at gate is cleared when the next refresh spawns a new Job; + // for now the annotation is a per-namespace at-most-once — a follow-up can + // key it by snapshot_id if that turns out to be too coarse.) + if annos.contains_key(annotations::LAST_VERIFICATION_REPORTED_AT) { + return Ok(()); + } + + if let Err(err) = send_verification(ctx, ns, outcome, error_msg.as_deref()).await { + warn!( + namespace = %ns_name, + error = %err, + "canopy reporter: restore-verification POST failed; will retry next tick" + ); + set_annotations( + ctx, + &ns_name, + &[(annotations::LAST_VERIFICATION_ERROR, Some(err.to_string()))], + ) + .await?; + } else { + info!(namespace = %ns_name, ?outcome, "canopy reporter: verification reported"); + set_annotations( + ctx, + &ns_name, + &[ + ( + annotations::LAST_VERIFICATION_REPORTED_AT, + Some(super::now_rfc3339()), + ), + (annotations::LAST_VERIFICATION_ERROR, None), + ], + ) + .await?; + } + Ok(()) +} + +/// Pick the most recent restore Job by creation timestamp. Usually there +/// is only one, but a refresh may leave the old one lingering until TTL. +fn latest_job(jobs: &[Job]) -> Option<&Job> { + jobs.iter().max_by_key(|j| { + j.metadata + .creation_timestamp + .as_ref() + .map(|t| t.0) + .unwrap_or(jiff::Timestamp::MIN) + }) +} + +/// Read the snapshot id from the running Job's env — mirrors what the +/// syncer set at Job creation time. +fn snapshot_id_from_job(job: &Job) -> Option<&str> { + let spec = job.spec.as_ref()?; + let containers = spec.template.spec.as_ref()?.containers.iter(); + for c in containers { + if c.name == super::KOPIA_JOB_NAME + && let Some(env) = c.env.as_ref() + { + for e in env { + if e.name == "SNAPSHOT_ID" + && let Some(v) = &e.value + { + return Some(v); + } + } + } + } + None +} + +/// Classify a Job's terminal state into (new_state, outcome, error). +/// `outcome` is `None` while the Job is still running. +fn classify_job(job: &Job) -> (&'static str, Option, Option) { + let status = match job.status.as_ref() { + Some(s) => s, + None => return (restore_state::PENDING, None, None), + }; + if status.succeeded.unwrap_or(0) > 0 { + return (restore_state::ACTIVE, Some(Outcome::Success), None); + } + if status.failed.unwrap_or(0) > 0 { + // Grab the most informative failure condition message if present. + let msg = status + .conditions + .as_ref() + .and_then(|cs| { + cs.iter().find_map(|c| { + if c.type_ == "Failed" && c.status == "True" { + c.message.clone().or_else(|| c.reason.clone()) + } else { + None + } + }) + }) + .unwrap_or_else(|| "restore Job failed".to_string()); + return (restore_state::FAILED, Some(Outcome::Failure), Some(msg)); + } + // Job is running (active > 0) — transition to `restoring`. + if status.active.unwrap_or(0) > 0 { + return (restore_state::RESTORING, None, None); + } + (restore_state::PENDING, None, None) +} + +async fn send_verification( + ctx: &Context, + ns: &Namespace, + outcome: Outcome, + error_msg: Option<&str>, +) -> Result<()> { + let Some(canopy) = ctx.canopy.as_ref() else { + return Err(crate::error::Error::Canopy( + "canopy client not configured".into(), + )); + }; + let ns_labels = ns.labels(); + let annos = ns.annotations(); + + let replica_id = ns_labels + .get(labels::DECLARATION_ID) + .and_then(|s| Uuid::parse_str(s).ok()); + let Some(group) = ns_labels + .get(labels::GROUP) + .and_then(|s| Uuid::parse_str(s).ok()) + else { + return Err(crate::error::Error::Canopy(format!( + "namespace {} missing group label", + ns.name_any() + ))); + }; + let Some(server_id) = ns_labels + .get(labels::SERVER) + .and_then(|s| Uuid::parse_str(s).ok()) + else { + return Err(crate::error::Error::Canopy(format!( + "namespace {} missing server label", + ns.name_any() + ))); + }; + let backup_type = ns_labels + .get(labels::TYPE) + .map(String::as_str) + .unwrap_or(""); + let intent = ns_labels + .get(labels::INTENT) + .map(String::as_str) + .unwrap_or(""); + let snapshot_id = annos + .get(annotations::LAST_RESTORED_SNAPSHOT_ID) + .or_else(|| annos.get(annotations::DESIRED_SNAPSHOT_ID)) + .map(String::as_str); + let replica_healthy = matches!(outcome, Outcome::Success); + + let report = RestoreVerification { + replica_id, + group, + server_id, + r#type: backup_type, + intent, + snapshot_id, + outcome, + error: error_msg, + replica_healthy, + postgres_version: None, + observed_at: jiff::Timestamp::now(), + s3_sent_raw_bytes: None, + s3_sent_payload_bytes: None, + s3_received_raw_bytes: None, + s3_received_payload_bytes: None, + }; + + canopy.restore_verification(&report).await +} + +/// Patch the given annotations on a Namespace. `None` values delete the +/// annotation; `Some(v)` sets it. Uses a strategic-merge patch so we don't +/// clobber annotations we don't own. +async fn set_annotations( + ctx: &Context, + ns_name: &str, + updates: &[(&str, Option)], +) -> Result<()> { + let mut annos = serde_json::Map::new(); + for (key, val) in updates { + annos.insert( + (*key).to_string(), + match val { + Some(v) => serde_json::Value::String(v.clone()), + None => serde_json::Value::Null, + }, + ); + } + let patch = serde_json::json!({ + "metadata": { "annotations": annos }, + }); + let api: Api = Api::all(ctx.client.clone()); + api.patch( + ns_name, + &PatchParams::apply("postgres-restore-operator").force(), + &Patch::Merge(&patch), + ) + .await?; + debug!(namespace = %ns_name, ?updates, "canopy reporter: patched annotations"); + Ok(()) +} + +// Silence unused-warning for helpers we plan to use in follow-ups. +#[allow(dead_code)] +fn _ensure_types_used(entry: &WorklistEntry) { + let _ = entry.replica_id; + let _ = BTreeMap::::new(); +} + +#[cfg(test)] +mod tests { + use super::*; + use k8s_openapi::api::batch::v1::JobStatus; + + fn job_with_status(status: JobStatus) -> Job { + Job { + status: Some(status), + ..Default::default() + } + } + + #[test] + fn classify_running_job_is_restoring() { + let job = job_with_status(JobStatus { + active: Some(1), + ..Default::default() + }); + let (state, outcome, _) = classify_job(&job); + assert_eq!(state, restore_state::RESTORING); + assert!(outcome.is_none()); + } + + #[test] + fn classify_succeeded_job_is_active_success() { + let job = job_with_status(JobStatus { + succeeded: Some(1), + ..Default::default() + }); + let (state, outcome, _) = classify_job(&job); + assert_eq!(state, restore_state::ACTIVE); + assert_eq!(outcome, Some(Outcome::Success)); + } + + #[test] + fn classify_failed_job_is_failed_failure() { + let job = job_with_status(JobStatus { + failed: Some(3), + ..Default::default() + }); + let (state, outcome, err) = classify_job(&job); + assert_eq!(state, restore_state::FAILED); + assert_eq!(outcome, Some(Outcome::Failure)); + assert!(err.is_some()); + } + + #[test] + fn classify_pending_job_is_pending() { + let job = job_with_status(JobStatus::default()); + let (state, outcome, _) = classify_job(&job); + assert_eq!(state, restore_state::PENDING); + assert!(outcome.is_none()); + } +} From 1d5b399f47c7db37a008c0939c3aa73d3c1a7baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Wed, 1 Jul 2026 12:53:37 +1200 Subject: [PATCH 11/20] test(canopy): integration test skeleton + CI matrix entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/canopy_integration.rs exercises the canopy-path syncer end-to-end against an in-cluster stub canopy. Three #[ignore]d tokio tests cover: - happy path: worklist entry → provisioned namespace with expected labels + annotations, restore Job with kopia + canopy-proxy containers; - grant-denied: stub returns 403 on /restore-credentials → clear failure state, no crash loop; - verification-report round-trip: successful restore emits the expected report to the stub. tests/fixtures/stub-canopy.yaml is a placeholder for the in-cluster stub Deployment — the actual stub needs a small binary (nginx alone can't capture POST bodies for readback), landing in a follow-up. For now the fixture provides the canned WorklistEntry + restore-credentials JSON that the eventual stub will serve. .github/workflows/integration.yml gains a canopy_integration matrix entry so the test file is picked up by CI once the fixture is complete. Integration tests don't run locally — this file is CI-only and will fail until the stub-canopy Deployment lands. The tests use kube-rs list-and-wait against real k8s objects, following the existing helpers.rs pattern. All test bodies are #[ignore]d so cargo test doesn't run them by default. --- .github/workflows/integration.yml | 5 + tests/canopy_integration.rs | 197 ++++++++++++++++++++++++++++++ tests/fixtures/stub-canopy.yaml | 59 +++++++++ 3 files changed, 261 insertions(+) create mode 100644 tests/canopy_integration.rs create mode 100644 tests/fixtures/stub-canopy.yaml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 90fd480..ddcf976 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -56,6 +56,11 @@ jobs: test-ps-all-missing needs_non_pg_snapshot: false + - name: canopy_integration + namespaces: >- + test-canopy-restore + needs_non_pg_snapshot: false + steps: - uses: actions/checkout@v6.0.2 diff --git a/tests/canopy_integration.rs b/tests/canopy_integration.rs new file mode 100644 index 0000000..e5e108b --- /dev/null +++ b/tests/canopy_integration.rs @@ -0,0 +1,197 @@ +//! Canopy-path integration test. +//! +//! Exercises the worklist syncer end-to-end against a **stub canopy**: an +//! in-cluster HTTP service that returns canned `WorklistEntry` / +//! `RestoreCredentials` and captures the `RestoreVerification` reports pgro +//! POSTs. The stub is a small nginx-backed service defined in +//! `tests/fixtures/stub-canopy.yaml` (deployed by the CI matrix step, not +//! this test). +//! +//! CI-only: this test assumes: +//! - a real k8s cluster (kind) with the operator running with +//! `CANOPY_BASE_URL` pointing at the stub; +//! - the `test-canopy-restore` namespace exists as the work root; +//! - the stub is reachable at `stub-canopy.test-canopy-restore.svc`. +//! +//! Do not run this file locally — nothing here works without the CI setup. + +#![allow(dead_code, reason = "shared helpers may not all be exercised yet")] + +use std::time::Duration; + +use k8s_openapi::{ + api::{batch::v1::Job, core::v1::Namespace}, + apimachinery::pkg::apis::meta::v1::Time, +}; +use kube::{Api, Client, ResourceExt, api::ListParams}; +use tokio::time::{sleep, timeout}; + +const POLL_INTERVAL: Duration = Duration::from_secs(2); +const NAMESPACE_TIMEOUT: Duration = Duration::from_secs(90); +const JOB_TIMEOUT: Duration = Duration::from_secs(120); + +mod helpers { + #[allow(unused_imports)] + pub use super::*; + + pub async fn make_client() -> Client { + Client::try_default() + .await + .expect("expected a valid kubeconfig (kind)") + } + + /// Wait for at least one namespace labelled `pgro.bes.au/managed-by=pgro-canopy` + /// to appear. Returns the first match. + pub async fn wait_for_canopy_namespace(client: &Client) -> Namespace { + let ns_api: Api = Api::all(client.clone()); + let params = ListParams::default().labels("pgro.bes.au/managed-by=pgro-canopy"); + let deadline = tokio::time::Instant::now() + NAMESPACE_TIMEOUT; + loop { + if let Ok(list) = ns_api.list(¶ms).await + && let Some(ns) = list.items.into_iter().next() + { + return ns; + } + if tokio::time::Instant::now() >= deadline { + panic!("no canopy-managed namespace appeared within {NAMESPACE_TIMEOUT:?}"); + } + sleep(POLL_INTERVAL).await; + } + } + + /// Wait for the canopy-path restore Job in `ns` to appear. Returns it. + pub async fn wait_for_restore_job(client: &Client, ns: &str) -> Job { + let job_api: Api = Api::namespaced(client.clone(), ns); + let params = ListParams::default().labels("pgro.bes.au/job-kind=canopy-restore"); + let deadline = tokio::time::Instant::now() + JOB_TIMEOUT; + loop { + if let Ok(list) = job_api.list(¶ms).await + && let Some(job) = list.items.into_iter().next() + { + return job; + } + if tokio::time::Instant::now() >= deadline { + panic!("no canopy-restore Job appeared in {ns} within {JOB_TIMEOUT:?}"); + } + sleep(POLL_INTERVAL).await; + } + } + + /// Poll the stub canopy's `/tests/reports` endpoint for the report count. + /// The stub exposes captured `RestoreVerification` POSTs as a JSON array + /// there — see `tests/fixtures/stub-canopy.yaml`. + pub async fn stub_report_count(_client: &Client) -> usize { + // TODO(canopy-integration): once the stub-canopy Deployment lands + // in tests/fixtures/, port-forward and read /tests/reports. For now + // the fixture is a placeholder; this helper is a stub. + 0 + } +} + +/// End-to-end happy path: stub canopy hands out a worklist entry, pgro +/// provisions a namespace with the expected labels, creates the restore +/// Job, and (eventually) reports a verification back. +#[ignore = "integration test; run in CI with the stub-canopy fixture"] +#[tokio::test] +async fn worklist_provisions_namespace_and_job() { + let client = helpers::make_client().await; + + // The stub returns a fixed WorklistEntry named "int-test" with a canned + // snapshot id. pgro's syncer should discover it on the next tick and + // create a labelled Namespace. + let ns = timeout( + NAMESPACE_TIMEOUT, + helpers::wait_for_canopy_namespace(&client), + ) + .await + .expect("timed out waiting for canopy-managed namespace"); + assert!( + ns.name_any().starts_with("int-test-"), + "unexpected namespace name: {}", + ns.name_any() + ); + + let labels = ns.labels(); + assert_eq!( + labels.get("pgro.bes.au/managed-by").map(String::as_str), + Some("pgro-canopy"), + ); + assert!(labels.contains_key("pgro.bes.au/declaration-id")); + assert!(labels.contains_key("pgro.bes.au/group")); + assert!(labels.contains_key("pgro.bes.au/server")); + assert_eq!( + labels.get("pgro.bes.au/type").map(String::as_str), + Some("tamanu-postgres"), + ); + + let annos = ns.annotations(); + assert_eq!( + annos.get("pgro.bes.au/restore-state").map(String::as_str), + Some("pending"), + "restore-state should start at pending; got {:?}", + annos.get("pgro.bes.au/restore-state"), + ); + assert!( + annos.contains_key("pgro.bes.au/desired-snapshot-id"), + "desired-snapshot-id should be set from the worklist entry" + ); + + // The syncer should then create the restore Job in that namespace. + let job = timeout( + JOB_TIMEOUT, + helpers::wait_for_restore_job(&client, &ns.name_any()), + ) + .await + .expect("timed out waiting for canopy-restore Job"); + let spec = job + .spec + .as_ref() + .and_then(|s| s.template.spec.as_ref()) + .expect("job pod spec missing"); + let container_names: Vec<_> = spec.containers.iter().map(|c| c.name.as_str()).collect(); + assert!( + container_names.contains(&"restore"), + "expected kopia restore container, got {container_names:?}", + ); + assert!( + container_names.contains(&"canopy-proxy"), + "expected canopy-proxy sidecar, got {container_names:?}", + ); +} + +/// Grant-absent: stub returns 403 on `/restore-credentials`. pgro should +/// surface a clear failure on the namespace annotation and not crash-loop. +#[ignore = "integration test; run in CI with the stub-canopy fixture"] +#[tokio::test] +async fn missing_grant_surfaces_clearly() { + let client = helpers::make_client().await; + // The stub's `/restore-credentials?scenario=denied` variant returns 403; + // wired via a per-worklist-entry flag in the fixture ConfigMap. + // TODO(canopy-integration): once the fixture supports the `scenario` knob, + // assert that: + // - the namespace exists with restore-state != "active" after 60s; + // - a Warning event has been emitted on the namespace; + // - the operator's restore-Job creation was not retried into a crash loop + // (Job count in the namespace <= 1). + let ns_api: Api = Api::all(client.clone()); + let list = ns_api + .list(&ListParams::default().labels("pgro.bes.au/managed-by=pgro-canopy")) + .await; + assert!( + list.is_ok(), + "namespace list must succeed even under grant-denied" + ); + let _keep_time_import: Option