diff --git a/.env.example b/.env.example deleted file mode 100644 index 2f4ae554..00000000 --- a/.env.example +++ /dev/null @@ -1,7 +0,0 @@ -FLEET_AUTH_ISSUER_URL= -FLEET_AUTH_AUTHORIZE_URL= -FLEET_AUTH_TOKEN_URL= -FLEET_AUTH_CLIENT_ID= -FLEET_AUTH_REDIRECT_URI= -FLEET_AUTH_SCOPE= -FLEET_AUTH_TRUSTED_AUDIENCES= diff --git a/AGENTS.md b/AGENTS.md index bf5525be..e7ebfb36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,3 +92,7 @@ mdbook build ## Conventions Rust edition 2024, resolver v2. Conventional commits (`feat:`/`fix:`/`chore:`/`docs:`/`refactor:`). Small, single-purpose PRs (~200 lines, excluding generated code). License AGPL v3. Lean on the type system — compile-time guarantees over runtime checks; abstractions domain-level, not provider-specific. + +## Dos and Don't + +- Never tag "Co-Authored-By" with an agent name in commit messages. diff --git a/Cargo.lock b/Cargo.lock index 0c7bbac5..5ce839da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2492,6 +2492,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "docker-compose-types" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdfd601efc90644958510466e8904ca90da65a98fdfe4d21f940644a21c026b4" +dependencies = [ + "derive_builder 0.20.2", + "indexmap 2.14.0", + "serde", + "serde_yaml", +] + [[package]] name = "dockerfile_builder" version = "0.1.6" @@ -2818,6 +2830,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "example-compose-java-react" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "clap", + "docker-compose-types", + "env_logger", + "fqdn", + "harmony", + "harmony_cli", + "harmony_types", + "k8s-openapi", + "log", + "serde", + "serde_yaml", + "tempfile", + "thiserror 2.0.18", + "tokio", + "toml", +] + [[package]] name = "example-fleet-auth-callout" version = "0.1.0" @@ -4063,6 +4098,7 @@ dependencies = [ "harmony", "harmony-fleet-auth", "harmony-reconciler-contracts", + "harmony_secret", "serde", "serde_json", "thiserror 2.0.18", @@ -4080,6 +4116,7 @@ dependencies = [ "async-nats", "chrono", "harmony-reconciler-contracts", + "harmony_zitadel_auth", "jsonwebtoken", "reqwest 0.12.28", "serde", @@ -4134,11 +4171,15 @@ dependencies = [ "harmony-fleet-auth", "harmony-fleet-deploy", "harmony-fleet-operator", + "harmony-k8s", "harmony-reconciler-contracts", "harmony_types", + "jsonwebtoken", "k3d-rs", "k8s-openapi", "kube", + "nkeys", + "reqwest 0.12.28", "serde", "serde_json", "tempfile", @@ -4468,6 +4509,7 @@ dependencies = [ "async-trait", "directories", "harmony_secret_derive", + "harmony_zitadel_jwt", "http 1.4.0", "infisical", "inquire 0.7.5", @@ -4537,6 +4579,7 @@ dependencies = [ "base64 0.22.1", "chrono", "harmony_config", + "harmony_zitadel_jwt", "jsonwebtoken", "openidconnect", "rand 0.9.4", @@ -4551,6 +4594,18 @@ dependencies = [ "url", ] +[[package]] +name = "harmony_zitadel_jwt" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "jsonwebtoken", + "reqwest 0.12.28", + "serde", + "serde_json", +] + [[package]] name = "hashbrown" version = "0.12.3" diff --git a/Cargo.toml b/Cargo.toml index 93946321..46da8293 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "private_repos/*", "harmony", "harmony_zitadel_auth", + "harmony_zitadel_jwt", "harmony_types", "harmony_macros", "harmony_tui", diff --git a/ROADMAP/fleet_platform/agent_secret_access_e2e.md b/ROADMAP/fleet_platform/agent_secret_access_e2e.md new file mode 100644 index 00000000..9c6a0738 --- /dev/null +++ b/ROADMAP/fleet_platform/agent_secret_access_e2e.md @@ -0,0 +1,95 @@ +# Agent secret access: full-stack integration e2e + +**Design:** [ADR-025](../../docs/adr/025-fleet-device-secret-access.md) · +[user guide](../../docs/guides/fleet-device-secrets.md) · +prior work: [groups rollout](./secret_access_groups_rollout.md) +**Branch:** `feat/fleet-device-secret-access` +**Written:** 2026-06-12 + +The groups rollout proved OpenBao-side enforcement against minted test +JWTs. This entry closes the loop with the *real* device path, in one VM +e2e: + +- agent connects to NATS through the auth callout with its Zitadel + machine identity (first time callout runs in the e2e harness), +- a deployment in desired state references a secret, +- the agent logs into OpenBao with the same Zitadel JWT (groups claim + from the flattening Action — live verification of the deferred item) + and injects the secret into the container, +- removing the OpenBao policy makes the next fetch fail, visibly. + +The operator is deliberately absent: CR→KV plumbing and grant sync are +already unit/e2e-covered; the test drives desired state via `AdminKv` +and grants via the same `OpenBaoDeploymentSecretGrants` the operator +composes. The axis under test is agent ↔ Zitadel ↔ callout/OpenBao. + +## Phase 1 — agent secret read path (test-first) + +- [x] `PodmanService.secret_env: Vec` + (serde default, wire-compat with old payloads). Secret lives at + `///`, field `value` — + harmony_secret's existing convention. +- [x] `OpenbaoSecretStore`: constructor taking the agent's existing + `ZitadelJwtBearer` (one identity for NATS + OpenBao, no token + cache files); `get_raw` retries once through `refresh_auth` on + permission-denied (batch tokens expire, re-login IS the sync — + ADR-025). +- [x] Agent `[openbao]` config section (url, role, mounts, prefix); + reconciler resolves `secret_env` → env before interpret; fetch + failure → `Phase::Failed` with the OpenBao error (the observable + the negative test asserts). +- [x] Unit tests: wire compat, config parse, resolution + failure + mapping via in-memory `SecretStore`. + +## Phase 2 — e2e stack: callout mode + +`AuthMode::Callout` in `fleet/harmony-fleet-e2e/src/stack.rs`, composed +from the same Scores as `fleet_staging_install` / `fleet_auth_callout`: + +- [x] Zitadel + CoreDNS rewrite (`sso.fleet-e2e.local` → zitadel svc) + + `ZitadelSetupScore`: project, roles-as-groups (`edge-a`, + `edge-b`), `groups_claim_action: true`, machine users per device + (username = device id) + `e2e-admin` (fleet-admin role) for the + host-side NATS client. +- [x] NATS via `FleetNatsScore::callout` + `NatsAuthCalloutScore` + (callout image build/sideload joins images.rs). +- [x] OpenBao + JWT auth: `oidc_discovery_url` = in-cluster issuer, + `groups_claim=groups`, batch, 1h — same shape staging uses. + NodePort service so the VM reaches it via the libvirt gateway. +- [x] k3d cluster gains port mappings 30424 (Zitadel NodePort, with a + `zitadel-e2e` Service whose in-cluster port equals the nodePort + so the issuer `http://sso.fleet-e2e.local:30424` is valid from + host, VM and in-cluster; 8080 was rejected — dev hosts commonly + occupy it) and 30425 (OpenBao). Existing cluster without them → + clear bring-up error telling the user to delete `fleet-e2e`. +- [x] VM device setup: `FleetDeviceAuth` Zitadel machine key + + `hosts_entries` (sso → gateway IP) + `[openbao]` section. +- [x] Host-side admin NATS connect mints a Zitadel token via the + Host-header trick (no /etc/hosts edits on the host). + +## Phase 3 — the tests (`tests/vm_secret_access.rs`) + +Gated by `HARMONY_FLEET_VM_E2E=1`, shared VM stack: + +- [x] **Grant + read**: device granted `edge-a` in Zitadel; secret + written with the root token; `set_deployment_groups([web → + [edge-a]])`; desired state with `secret_env` → `Phase::Running` + and SSH ground truth: `podman inspect` shows the env value. +- [x] **Policy removed**: delete `deployment-web` policy in OpenBao, + bump the spec to force re-apply → fetch denied → + `Phase::Failed`, error names the denial. + +## Phase 4 — verification + +- [x] `./build/check.sh` green +- [x] Existing fleet e2e suite green +- [x] New VM e2e green (x86_64; aarch64 before release) +- [x] Push + +## Deferred (carried from groups rollout) + +- Operator-pod env injection live test (grant sync in-cluster) — still + staging-milestone scope. +- TTL tuning, per-cell sizing — unchanged. +- aarch64 VM run of the new test if the manual x86_64 run is the gate + today. diff --git a/ROADMAP/fleet_platform/device_secret_access_handoff.md b/ROADMAP/fleet_platform/device_secret_access_handoff.md new file mode 100644 index 00000000..32cb4ad9 --- /dev/null +++ b/ROADMAP/fleet_platform/device_secret_access_handoff.md @@ -0,0 +1,463 @@ +# Fleet device secret access — implementation handoff + +**Owner:** TBD +**Design:** [`docs/adr/025-fleet-device-secret-access.md`](../../docs/adr/025-fleet-device-secret-access.md) +**Status:** Superseded by implementation — see ADR-025 and its iteration +trail in [`docs/adr/025-device-secret-access/`](../../docs/adr/025-device-secret-access/) +(this plan targeted iteration [025-1](../../docs/adr/025-device-secret-access/025-1-jwt-deployments-claim.md); +what shipped is [025-2](../../docs/adr/025-device-secret-access/025-2-entity-by-policy.md)) +**Written:** 2026-06-01 + +> **As-built note.** The implementation took a simpler scoping path than +> this plan (entity-by-policy instead of a JWT `deployments` claim + +> `groups_claim`). This six-PR breakdown is kept for history; the parts +> that actually shipped: +> +> | Plan | Shipped | +> |---|---| +> | PR-1 extract JWT-bearer minter | ✅ — now in the leaf crate `harmony_zitadel_jwt` (re-exported by `harmony_zitadel_auth`) | +> | PR-2 `refresh_auth` + `cached_scope` + `decode_deployments_claim` | ⚠️ `refresh_auth` only; scope decoding dropped (scope lives in the OpenBao entity, not the token) | +> | PR-3 `groups_claim`/`bound_claims_json` on `OpenbaoJwtAuth` | ❌ not needed | +> | PR-4 Zitadel deployments-claim Action Score | ❌ not needed | +> | PR-5 membership Score | ✅ as `DeviceSecretAccess` trait + `OpenBaoDeviceSecretAccess` (`harmony/src/modules/openbao/identity.rs`) + operator `sync_secret_access`; OpenBao entity writes only, no Zitadel-metadata write | +> | PR-6 agent before-reconcile refresh-check | ❌ not needed (entity policy changes are seen without re-auth) | + +Read ADR-025 first. This document is the work plan, not the design. + +## Summary + +The fleet agent reads per-deployment secrets through the existing +`harmony_config` chain, which already wraps OpenBao via +`harmony_secret::OpenbaoSecretStore`. `OpenbaoSecretStore`'s auth +ladder is extended with one new rung — Zitadel **JWT-bearer**, keyed +off the device's existing machine keyfile (the same root of trust the +NATS callout uses). Per-deployment scope rides in the JWT itself as a +`deployments` array claim, populated by the operator through Zitadel +user metadata. OpenBao's `groups_claim` binds those values to +auto-created external groups, each with a small policy granting +`read` on `harmony-fleet/data//*`. The agent's NATS +watcher calls `refresh_auth()` on the store before reconciling a +deployment whose secrets are outside the cached scope. + +## Architecture at a glance + +``` + Fleet operator (one per fleet, holds Zitadel admin scope) + │ + ├── on new Deployment CR: + │ 1. FleetDeviceDeploymentMembershipScore + │ ├─ Zitadel: append dep to device.metadata.deployments + │ └─ OpenBao: upsert external group + policy + │ fleet-deployment- → read on + │ harmony-fleet/data//* + │ 2. existing NATS publish (data plane, unchanged) + │ + ▼ + Fleet agent + │ + ├── NATS handler (main.rs): + │ if !secret_store.cached_scope().contains(&dep) { + │ secret_store.refresh_auth().await?; + │ } + │ reconciler.apply(…) + │ + └── reads secrets via harmony_config (StoreSource wrapping + OpenbaoSecretStore with new JWT-bearer auth rung) +``` + +## Pre-flight + +- [ ] Confirm staging Zitadel has a service account for the operator + with **write user metadata** scope. File a sub-task if not. +- [ ] Confirm the staging OpenBao KV mount `harmony-fleet` exists, or + add its creation to `OpenbaoSetupScore`. +- [ ] Validate `DeploymentName`'s character set is safe for KV paths + and Bao external-group names (`[a-zA-Z0-9_-]`). + +## Work breakdown + +Six PRs. Dependencies: + +``` +PR-1 harmony_zitadel_auth: extract JWT-bearer minter + └─ PR-2 harmony_secret: JWT-bearer auth + refresh_auth + cached_scope + └─ PR-6 agent main.rs: inline refresh-check + +PR-3 harmony OpenbaoJwtAuth Score: bound_claims + groups_claim +PR-4 Zitadel Action Score: deployments claim + └─ PR-5 Operator: FleetDeviceDeploymentMembershipScore +PR-3 ─┘ +``` + +PRs 1, 3, 4 are independent and can start in parallel. + +--- + +### PR-1 — Extract Zitadel JWT-bearer minter into `harmony_zitadel_auth` + +**Crate:** `harmony_zitadel_auth` +**Depends on:** nothing +**Blocks:** PR-2 + +Move `MachineKeyFile`, `CachedToken`, `build_assertion*`, `build_scope`, +`build_token_url`, and the mint+cache logic out of +`fleet/harmony-fleet-auth/src/credentials.rs` into a new +`harmony_zitadel_auth/src/jwt_bearer.rs`. Two real consumers (NATS +callout + OpenBao) cross Rule of Three at the second consumer. + +`harmony_secret` cannot depend on `harmony-fleet-auth` (fleet-specific +crate; would invert the dependency graph). `harmony_zitadel_auth` is +neutral and already houses the human-OIDC counterpart. + +**Shape.** + +```rust +// harmony_zitadel_auth/src/jwt_bearer.rs +pub struct ZitadelJwtBearer { + key: MachineKeyFile, + oidc_issuer_url: String, + audience: String, + http: reqwest::Client, + cache: Mutex>, +} + +impl ZitadelJwtBearer { + pub fn new(key: MachineKeyFile, oidc_issuer_url: String, audience: String, + danger_accept_invalid_certs: bool) -> Result; + + /// Cached if comfortably valid, otherwise mints fresh. + pub async fn bearer_token(&self) -> Result; + + /// Force a re-mint on next call. + pub fn invalidate_cache(&self); +} +``` + +`CredentialSource::ZitadelJwt` in `harmony-fleet-auth` becomes a thin +wrapper around `Arc`. Port the existing pure-builder +tests from `credentials.rs` into the new crate. + +**Acceptance.** `cargo test -p harmony_zitadel_auth -p harmony-fleet-auth` +clean. Fleet e2e NATS auth still works. + +--- + +### PR-2 — `OpenbaoSecretStore` JWT-bearer rung + `refresh_auth` + `cached_scope` + +**Crate:** `harmony_secret` (`src/store/openbao.rs`) +**Depends on:** PR-1 +**Blocks:** PR-6 + +Add the fifth rung to `OpenbaoSecretStore::new`'s auth ladder. +Position: between cached-token (rung 2) and Zitadel OIDC device flow +(rung 4). Triggered only when a machine keyfile is configured; if it +fails, fall through to the existing ladder. + +**Constructor refactor.** Knock out the `too_many_arguments` clippy +TODO at `openbao.rs:55` while we're adding more args: + +```rust +pub struct OpenbaoStoreOptions { + pub base_url: String, + pub kv_mount: String, + pub auth_mount: String, + pub skip_tls: bool, + pub token: Option, + pub username: Option, pub password: Option, + pub zitadel_sso_url: Option, pub zitadel_client_id: Option, + pub zitadel_jwt_bearer: Option, // NEW + pub jwt_role: Option, pub jwt_auth_mount: Option, +} + +pub struct ZitadelJwtBearerConfig { + pub key_path: Option, + pub key_json: Option, + pub oidc_issuer_url: String, + pub audience: String, // Zitadel project ID +} +``` + +**Refresh capability.** A single `Mutex` over the refresh-affected +state. Refresh is rare and uncontended; `ArcSwap` would be ceremony. + +```rust +pub struct OpenbaoSecretStore { + inner: Mutex, + kv_mount: String, + auth_mount: String, + jwt_bearer: Option>, + jwt_role: Option, + jwt_auth_mount: Option, + base_url: String, + skip_tls: bool, +} + +struct Inner { + client: VaultClient, + scope: HashSet, // deployment IDs from the Zitadel JWT +} + +impl OpenbaoSecretStore { + pub async fn refresh_auth(&self) -> Result<(), SecretStoreError> { + let bearer = self.jwt_bearer.as_ref().ok_or_else(|| /* err */)?; + bearer.invalidate_cache(); + let jwt = bearer.bearer_token().await?; + let scope = decode_deployments_claim(&jwt)?; // pure JWT decode + let session = jwt_login(&self.base_url, /* … */, &jwt).await?; + let client = build_vault_client(&self.base_url, self.skip_tls, &session.token)?; + *self.inner.lock().await = Inner { client, scope }; + Ok(()) + } + + pub async fn cached_scope(&self) -> HashSet { + self.inner.lock().await.scope.clone() + } +} +``` + +`decode_deployments_claim` is a small pure helper that base64-decodes +the JWT body and reads the `deployments` array. No Bao round-trip. +`get_raw` / `set_raw` lock `inner`, take a `&VaultClient`, perform the +call, release. The lock is uncontended on the hot path. + +**On construction with JWT-bearer.** Wire `jwt_bearer` and call +`refresh_auth()` once before returning. If it fails, fall through to +the next ladder rung. + +**Tests.** + +- Ladder ordering: JWT-bearer present → tried before OIDC; absent → + unchanged. +- `refresh_auth` against a fake HTTP server updates `cached_scope`. +- `decode_deployments_claim` on a hand-crafted JWT returns the + expected set. + +**Acceptance.** `cargo test -p harmony_secret` clean. `examples/openbao` +userpass path unaffected. Manual staging run authenticates via +JWT-bearer. + +--- + +### PR-3 — Extend `OpenbaoJwtAuth` Score with `bound_claims_json` + `groups_claim` + +**Crate:** `harmony` (`src/modules/openbao/setup.rs`) +**Depends on:** nothing + +Two new fields, both defaulting to empty so existing callers are +unaffected: + +```rust +pub struct OpenbaoJwtAuth { + // … existing fields … + #[serde(default)] pub bound_claims_json: String, + #[serde(default)] pub groups_claim: String, +} +``` + +Extend `configure_jwt` (line 435+) to pass each as a `bao write +auth/jwt/role/...` flag when non-empty. + +**Acceptance.** `cargo check -p harmony --all-features` clean. +`examples/openbao` still works. + +--- + +### PR-4 — Zitadel Action Score: surface `deployments` metadata as a claim + +**Crate:** `harmony` (new module under `src/modules/zitadel/` or +wherever existing Zitadel deploy code lives) +**Depends on:** nothing +**Blocks:** PR-5 + +Declarative Score that upserts the post-access-token-creation Action +on the configured Zitadel project. Cannot be avoided: Zitadel emits +`urn:zitadel:iam:user:metadata` as base64-encoded values in a map, +and Bao's `groups_claim` can't consume that shape — the Action +decodes and re-emits as a flat string array. + +**Action script** (canonical text): + +```javascript +function addDeployments(ctx, api) { + const md = ctx.v1.user.getMetadata(); + const entry = md.metadata.find(m => m.key === "deployments"); + if (!entry) return; + try { + const deployments = JSON.parse( + Buffer.from(entry.value, "base64").toString("utf-8") + ); + if (Array.isArray(deployments)) { + api.v1.claims.setClaim("deployments", deployments); + } + } catch (_) { /* malformed → no deployments */ } +} +``` + +**Score shape.** + +```rust +pub struct ZitadelDeploymentsClaimActionScore { + pub project_id: String, + pub action_name: String, // default "fleet-deployments-claim" +} +``` + +Interpret upserts via Zitadel Management API, attaches to +`Complement Token / PreAccessTokenCreation`. Idempotent. + +**Acceptance.** Manual decode of a staging token shows the +`deployments` claim. + +--- + +### PR-5 — `FleetDeviceDeploymentMembershipScore` + operator wiring + +**Crate:** `harmony` (`src/modules/fleet/` or wherever fleet Scores +live) for the Score; `fleet/harmony-fleet-operator` for wiring. +**Depends on:** PR-3 (Bao JWT role + accessor must exist), PR-4 +(claim must surface) + +One Score, two writes, declared in execution order. Per ADR-023 the +operator hands off ordering to the Score rather than sequencing the +external API calls by hand. + +```rust +pub struct FleetDeviceDeploymentMembershipScore { + pub zitadel_project_id: String, + pub device_user_id: String, // Zitadel machine user ID + pub deployments: Vec, // declarative full set + pub openbao_instance: OpenbaoInstance, + pub kv_mount: String, // "harmony-fleet" + pub jwt_auth_accessor: String, // resolved from PR-3 +} +``` + +Interpret runs, in order: + +1. **Zitadel metadata** — declarative replace on + `user.metadata.deployments`. Read current value, write only on + diff. Removal is "declare the new set without the removed entry." +2. **OpenBao external group + policy, per deployment in the set.** For + each `` in `deployments`: + - Policy `fleet-deployment-` granting `read` on + `harmony-fleet/data//*` and + `read, list` on `harmony-fleet/metadata//*`. + - External group `` (type=external) with that policy + attached, alias matching the JWT-auth accessor. + +Idempotent throughout. + +**Operator wiring.** In the existing per-`Deployment` reconciler, +compose this Score before the existing NATS desired-state publish. +If the Score errors, surface as a CR status condition and do not +publish — never tell a device to run a deployment it cannot +authenticate for. + +**Note: NATS publish stays in operator code.** It's data-plane (the +"now do this" half of enrollment), already wired, and changing its +home isn't part of this work. + +**Tests.** + +- Unit on the Score: Zitadel diff logic (no-op on match, full write + on diff); policy text generation; group alias shape. +- Integration: against the real Bao in `examples/openbao` + a fake + Zitadel; second apply is a no-op. + +**Acceptance.** A `Deployment` CR rolled in staging causes both +writes to land before the NATS publish; manual JWT-bearer login at +Bao returns a token with the expected per-deployment policies +attached via external-group bindings. + +--- + +### PR-6 — Agent: inline refresh-check in `main.rs` + +**Crate:** `fleet/harmony-fleet-agent` +**Depends on:** PR-2 + +Around the existing JetStream KV watcher in +`fleet/harmony-fleet-agent/src/main.rs:110-120`, gate +`reconciler.apply` on a scope check: + +```rust +async_nats::jetstream::kv::Operation::Put => { + if let Some(dep) = deployment_from_key(&entry.key) { + if !secret_store.cached_scope().await.contains(dep.as_str()) { + tracing::info!(%dep, "deployment outside cached scope — refreshing"); + if let Err(e) = secret_store.refresh_auth().await { + tracing::warn!(key = %entry.key, error = %e, "refresh failed"); + continue; + } + } + } + if let Err(e) = reconciler.apply(&entry.key, &entry.value).await { + tracing::warn!(key = %entry.key, error = %e, "apply failed"); + } +} +``` + +`secret_store` is the same `Arc` held by the +agent's `ConfigManager` — share it through whatever construction +path `main.rs` already uses. + +**No new module.** One consumer, one site, ~10 lines. Per CLAUDE.md +"Rule of Three: introduce an abstraction at the second real +instance." If/when a second pre-reconcile concern (image-pull creds, +monitoring registration) arrives, extract a layer then. + +**No retry on "still missing after refresh."** The operator's +ordering guarantees Zitadel is consistent before the NATS publish. +If we observe the race in practice, add a single retry then. + +**Tests.** None new — this is wiring between two tested components. +Confirm via the integration milestone below. + +**Acceptance.** Rolling a new deployment to a staging device shows +one "outside cached scope — refreshing" log followed by a clean +reconcile. + +--- + +## Integration milestone — staging dry run + +After PRs 1-6 land: + +- [ ] Apply the new `OpenbaoJwtAuth` config to staging Bao via + `OpenbaoSetupScore`. +- [ ] Apply `ZitadelDeploymentsClaimActionScore` to staging Zitadel. +- [ ] Hand-provision metadata + group/policy for one test deployment + + one test device. Confirm the agent reads its secret. +- [ ] Roll a `Deployment` CR via the operator. Confirm + `FleetDeviceDeploymentMembershipScore` writes Zitadel and Bao + before NATS, and the agent reconciles cleanly. +- [ ] Negative: device not in deployment X gets a hard error reading + X's secret (not a silent fall-through). +- [ ] Negative: a JWT minted before the metadata update cannot read + the new deployment's secret until the agent's inline refresh + runs. + +## Decisions deferred + +1. **Vault token TTL.** ADR-025 recommends 15 min. Confirm in + staging; adjust `OpenbaoJwtAuth::ttl` if needed. +2. **Hard revocation on deployment removal.** Wait for TTL today. + Add a revoke companion only if a real fleet requires it. +3. **Bao down at agent startup.** `OpenbaoSecretStore::new` with + JWT-bearer must not panic; either fall through or surface a + retryable error. Confirm and document in the milestone. + +## Required reading + +- `docs/adr/025-fleet-device-secret-access.md` — design. +- `docs/adr/020-1-zitadel-openbao-secure-config-store.md` — + human-user counterpart of this auth flow. +- `docs/adr/023-deploy-architecture.md` — Score discipline for PRs 3-5. +- `harmony_secret/src/store/openbao.rs` — auth ladder being extended. +- `harmony_config/src/source/store.rs` — wrapper the agent uses. +- `fleet/harmony-fleet-auth/src/credentials.rs` — JWT-bearer mint + path being extracted in PR-1. +- `nats/callout/src/zitadel.rs` — JWT validation shape the Bao role + mirrors (`bound_issuer` + `bound_audiences`). +- `fleet/harmony-fleet-agent/src/main.rs` — site of the PR-6 inline + edit. diff --git a/ROADMAP/fleet_platform/secret_access_groups_rollout.md b/ROADMAP/fleet_platform/secret_access_groups_rollout.md new file mode 100644 index 00000000..bfc167c5 --- /dev/null +++ b/ROADMAP/fleet_platform/secret_access_groups_rollout.md @@ -0,0 +1,139 @@ +# Secret access: groups implementation + branch consolidation + +**Design:** [ADR-025](../../docs/adr/025-fleet-device-secret-access.md) · +[user guide](../../docs/guides/fleet-device-secrets.md) · +[analysis](../../docs/adr/025-device-secret-access/025-3-groups-as-security-boundary.md) +**Branch:** `feat/fleet-device-secret-access` +**Written:** 2026-06-11 + +Two efforts tracked here: (1) TDD implementation of the group-based +secret access design, (2) consolidation of the six stacked chapter +branches that will conflict with it. + +## Part 1 — implementation (test-first) + +Each phase: write the test, watch it fail, implement to green. + +### Phase 1 — OpenBao group enforcement e2e (the security core) + +New `fleet/harmony-fleet-e2e/tests/openbao_groups.rs`, self-contained +(no Zitadel: the JWT auth mount validates against a static test RSA +key via `jwt_validation_pubkeys`; tests mint their own JWTs). Proves +every row of ADR-025's convergence/revocation table: + +- [x] JWT login with `groups: ["edge-a"]` claim binds the device to + the `edge-a` external group; batch token issued +- [x] Member device reads `deployment-web` secrets; non-member + (`groups: ["edge-b"]`) is denied +- [x] Attaching the deployment policy to a group grants **existing** + tokens immediately (request-time) +- [x] Detaching revokes existing tokens immediately +- [x] Membership change binds only at re-login: old token keeps old + groups, fresh login sees new groups +- [x] `disabled=true` on the entity blocks an in-flight **batch** + token (ADR-025-3 open item 1) + +### Phase 2 — OpenBao group-grant backend + +`DeviceSecretAccess` (device → deployments, retired with 025-2) +becomes deployment → allowed-groups: + +- [x] Trait: `set_deployment_groups(&[(DeploymentName, Vec)])` + — absolute, idempotent; empty set detaches everywhere; + in-memory impl for operator tests +- [x] OpenBao impl: upsert policy `deployment-` (read on its + subtree), upsert external group + JWT-mount alias per fleet + group, set group policy lists (add to allowed, remove from + formerly-allowed) +- [x] Phase 1 e2e drives this backend end to end + +### Phase 3 — JWT auth role: groups, batch, TTL + +`OpenbaoJwtAuth` (`harmony/src/modules/openbao/setup.rs`): + +- [x] `groups_claim` (default empty → flag omitted) +- [x] `token_type` (default `service`; fleet role uses `batch`) +- [x] `jwt_validation_pubkeys` as alternative to `oidc_discovery_url` + (what makes Phase 1 self-contained) +- [x] Fleet defaults: `ttl=1h`, no renewal reliance (ADR-025) + +### Phase 4 — operator: groups gate scheduling + drive grants + +`harmony/src/modules/fleet/operator/crd.rs` + +`fleet/harmony-fleet-operator/src/fleet_aggregator.rs`: + +- [x] `allowed_groups: Vec` on `DeploymentSpec` +- [x] Membership source trait (device → groups) with in-memory test + impl; Zitadel impl reads role grants via the existing + Management API client (`users/grants/_search`, + `harmony/src/modules/zitadel/setup.rs`) — roles-as-groups + interim per ADR-025 +- [x] Scheduling = members(allowed_groups) ∩ label selector; unit + tests: out-of-group device never targeted even on label match +- [x] On Deployment upsert/delete: sync deployment → groups to the + Phase 2 backend (replaces per-device `sync_secret_access`) +- [x] Rename the `group` *label* in test fixtures (placement vocab, + device-writable) so it can't be confused with authorization + groups +- [ ] Operator e2e: deployment with `allowedGroups` schedules only + onto in-group devices + +### Phase 5 — Zitadel roles→groups Action + +- [x] Score upserting the flattening Action (roles map → `groups` + string array claim) + flow trigger on the fleet project; + idempotent; unit-tested on request shape +- [ ] Staging verification deferred to the integration milestone + (needs a live Zitadel) + +### Phase 6 — verification + push + +- [x] `./build/check.sh` green +- [x] `HARMONY_FLEET_E2E=1` suite green (incl. existing + `openbao_policy.rs`, `ping.rs`, `operator.rs`) +- [x] Push `feat/fleet-device-secret-access` + +## Part 2 — chapter branch consolidation + +Six branches stack linearly from `97af299d` (pre-secret-access): +`fleet-device-exec-logs` ⊂ `ch1-role-gate` ⊂ `ch2-operator-recovery` +⊂ `ch3-log-streaming` ⊂ `ch4-agent-upgrade` ⊂ +`ch5-graceful-deploy-upgrade` — except ch2's three follow-up fixes +never propagated to ch3–ch5. All will conflict with this branch +(`credentials.rs` was refactored into `harmony_zitadel_jwt`; +`fleet_aggregator.rs` changes in Phase 4). + +Collapse to two feature-scoped branches, rebased onto +`feat/fleet-device-secret-access` after Part 1 lands: + +| New branch | Absorbs | Scope | +|---|---|---| +| `feat/fleet-operator-recovery-and-logs` | exec-logs base fixes + ch1 + ch2 (fixes folded in) + ch3 | Operator reliability & observability: role gate via OIDC scope, aggregator recovery + orphan GC, log selector + live tail, agent auto-start replay | +| `feat/fleet-upgrade-path` | ch4 + ch5 | Agent self-upgrade + auto-rollback (ADR-022) and graceful roll-forward with container-ID identity | + +- [x] Rebase/rewrite branch A; resolve `credentials.rs` → + `harmony_zitadel_jwt` conflicts; ch2 fixes squashed into the + recovery commit +- [x] Rebase branch B on top of A (it builds on ch3 state) +- [x] Both: `./build/check.sh` + fleet e2e green = working state +- [x] Push both; old six branches left untouched for reference until + the new ones are reviewed + +## Deferred to staging integration milestone + +- Flattening Action applied + token decode shows `groups` array + (flow/trigger ids `2`/`4`,`5` assumed from API docs — verify) +- Zitadel membership source verified against live role grants +- TTL = 1 h default revisited with measured login load +- Per-cell sizing guidance (identity-store memory measurement) +- Known gap: a failed grant sync only retries on the next Deployment + change (warn-and-continue). Add a periodic resync if staging shows + real transient failures. +- Operator e2e for the `allowedGroups` gate needs operator-pod env + injection in the harness (`FLEET_DEVICE_GROUPS`); gate currently + unit-tested, enforcement e2e-tested. +- Harness fixes that fell out of this work (on the PR): topology + `ensure_ready` before raw `Score::interpret`, `k3d image import + --mode direct` (tools-node copy fails silently), staging respects + `CARGO_TARGET_DIR`, typed SSA takes field ownership, recovery suite + serialized (in-process aggregators react to each other's CRs). diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 14798669..9b927e98 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -25,6 +25,7 @@ - [Adding Capabilities](./guides/adding-capabilities.md) - [Web Authentication and CSRF Security](./guides/web-auth-security.md) - [Operator Dashboard SSO (Zitadel) — setup](./guides/operator-dashboard-sso.md) +- [Fleet Device Secrets](./guides/fleet-device-secrets.md) ## Configuration @@ -54,3 +55,5 @@ - [019 · Network Bond Setup](./adr/019-Network-bond-setup.md) - [020 · Interactive Configuration Crate](./adr/020-interactive-configuration-crate.md) - [020-1 · Zitadel + OpenBao Secure Config Store](./adr/020-1-zitadel-openbao-secure-config-store.md) + - [025 · Fleet Device Secret Access](./adr/025-fleet-device-secret-access.md) + - [025-3 · Groups as the Security Boundary](./adr/025-device-secret-access/025-3-groups-as-security-boundary.md) diff --git a/docs/adr/025-device-secret-access/025-1-jwt-deployments-claim.md b/docs/adr/025-device-secret-access/025-1-jwt-deployments-claim.md new file mode 100644 index 00000000..737013a7 --- /dev/null +++ b/docs/adr/025-device-secret-access/025-1-jwt-deployments-claim.md @@ -0,0 +1,325 @@ +# ADR 025-1: Fleet Device Secret Access via JWT `deployments` Claim (superseded) + +Initial Author: Jean-Gabriel Gill-Couture + +Initial Date: 2026-06-01 + +## Status + +Superseded by [025-2](./025-2-entity-by-policy.md) during implementation. + +The trust model, threat model, and JWT-bearer auth root survived as +designed and are now in the main +[ADR-025](../025-fleet-device-secret-access.md). What did not survive is +the scoping mechanism below: carrying per-deployment membership as a JWT +`deployments` claim (Zitadel user metadata + a token-customization +Action + OpenBao `groups_claim` + external groups). Preserved verbatim +as the first design iteration. + +## Context + +Fleet agents on devices need to read per-deployment secrets (image-pull +credentials, application secrets, etc.) from OpenBao. The agent already +holds one durable secret: a Zitadel machine-user JWT keyfile dropped by +`FleetDeviceSetupScore`. That key is the basis for the agent's existing +NATS authentication (`nats/callout` validates the Zitadel-minted access +token; `fleet/harmony-fleet-auth/src/credentials.rs` mints it via the +RFC 7523 JWT-bearer flow). + +Three requirements shape the design: + +1. **No new device-side secret.** The Zitadel machine key is already the + single root of trust on a device; the secret-access path must derive + from the same key, not introduce a second one. + +2. **Per-deployment isolation, enforced cryptographically.** A device + enrolled in deployments A and B reads only `A`'s and `B`'s secrets. + A device that hosts no deployments reads nothing. The device cannot + widen its own scope — only the operator can change membership. + +3. **Cross-project safety.** A second Zitadel project (a different + tenant, a different fleet, a malicious org) must not be able to + produce a token that OpenBao accepts. The trust boundary is the + project, not the deployment. + +The kubelet analogy is the architectural north star: the agent is a +small runtime that learns its workload (and the credentials needed to +run it) from a control-plane authority. The agent never decides what it +is allowed to run or read; it presents a signed identity and the +infrastructure decides. + +## Decision + +Three coordinating pieces. + +### 1. OpenBao JWT auth bound to the Zitadel project + +OpenBao's JWT auth method validates incoming tokens against Zitadel's +OIDC discovery URL (JWKS). One auth role per fleet, configured against +**one** Zitadel project: + +``` +bound_issuer = +bound_audiences = +bound_claims = { "urn:zitadel:iam:org:project:roles": "fleet-device" } +user_claim = sub +groups_claim = deployments +``` + +`bound_audiences` is the project boundary. A token minted in any other +Zitadel project has a different `aud` claim and is rejected before any +membership claim is read. This is the same defense +`nats/callout/src/zitadel.rs` already applies via `set_audience`. + +`groups_claim = deployments` instructs OpenBao to read the JWT's +`deployments` array and bind the resulting Vault token to one external +group per element. Each external group carries a per-deployment policy +granting `read` on `harmony-fleet/data//*`. + +### 2. Operator-managed Zitadel metadata as the membership source of truth + +The fleet operator is the only writer of `user.metadata.deployments` +on each device's Zitadel machine user. A Zitadel post-token-creation +**Action** copies that metadata into a top-level `deployments` claim on +the access token. The device never touches its own metadata. + +When a new `Deployment` CR is observed in Kubernetes, the operator +executes three writes in a strict order: + +1. **Zitadel metadata** — append the deployment ID to the device's + `deployments` array (per device targeted by the deployment). +2. **OpenBao external group + policy** — upsert + `identity/group/` (`type=external`, alias matching the + JWT-auth accessor) and policy + `fleet-deployment-` granting + `read` on `harmony-fleet/data//*`. +3. **NATS desired-state** — publish + `desired-state..` with the workload score. + +Reversed, the agent could see the desired-state, attempt a re-auth, +and find the deployment missing from its claims — a "permission denied +for a deployment I was told to run" race that is confusing to debug +and weakens the trust story. Trust state always precedes the workload +signal. + +Removal runs in reverse: NATS delete → (optional) group/policy delete → +metadata removal. Currently-cached Vault tokens retain access until +their short TTL expires; explicit revocation is available via +`bao token revoke` on the device's accessor if hard revocation is +needed. + +### 3. Client side: JWT-bearer in `harmony_secret`, refresh before reconcile in the agent + +The agent does **not** grow a new secrets client. Per ADR-020, +`harmony_config` is the unified config+secret entry point and already +wraps OpenBao via `harmony_secret::OpenbaoSecretStore`. The missing +piece is auth: `OpenbaoSecretStore` supports env token, cached token, +Zitadel OIDC device flow (humans), and userpass — but not Zitadel +**JWT-bearer** for headless machine identity. + +Three additions: + +- A fifth rung on `OpenbaoSecretStore`'s auth ladder takes a Zitadel + machine keyfile + Bao JWT role + audience, mints via RFC 7523, and + POSTs to `/v1/auth/jwt/login`. +- The pure minting moves to `harmony_zitadel_auth` so NATS and + OpenBao auth share one implementation (Rule of Three: NATS callout + + OpenBao auth = two real consumers). +- `OpenbaoSecretStore` gains `refresh_auth()` (re-mint + re-login, + guarded by an internal `Mutex`) and `cached_scope() -> + HashSet` derived from decoding the in-hand Zitadel JWT — + no Bao round-trip needed since the `deployments` claim is already + in the token we just minted. + +In the agent, the NATS KV watcher consults `cached_scope()` before +each `reconciler.apply()`. If the desired deployment isn't covered, +it calls `refresh_auth()` and proceeds. The check is inline in +`main.rs` — about ten lines around the existing watcher loop. No +new module: one consumer, one site, inlining is the right size. + +### Secret path layout + +``` +harmony-fleet/data// +``` + +The Zitadel project ID does **not** appear in the path. Its job is +done at the JWT validation boundary (`bound_audiences`), not repeated +in every key. + +## Rationale + +**Why Zitadel project ID lives in `bound_audiences`, not the path.** +The same trust assertion in two places is duplication, not defense in +depth — both reduce to "the JWT signature is valid for this audience." +Concentrating it at the auth role: + +- gives one source of truth ("which project owns this Bao instance"); +- keeps secret paths readable and operator-friendly; +- decouples secret organization from Zitadel project identity (a + project ID rotation reconfigures one Bao role, not every path). + +**Why user metadata over project roles for deployment membership.** +Project roles in Zitadel live in a flat namespace inside a project. +A handful of roles (`fleet-admin`, `fleet-device`) maps cleanly; one +role per deployment would not — role inventories at hundreds of +deployments per fleet become hard to audit and slow to mutate. +User metadata is a per-machine-user JSON store, naturally +multi-valued, and admin-only-writable. The Zitadel Action that copies +metadata to a claim is a one-time, fleet-wide piece of configuration. + +**Why `groups_claim` over claim-templated paths.** Vault policy +templating (`{{identity.entity.aliases…metadata.}}`) supports +single-value substitution but not iteration over an array. Multiple +deployments per device require either multiple JWT logins (one per +deployment) or one login that resolves to multiple policies. +`groups_claim` + external groups gives the latter cleanly: one login, +N policies attached automatically. + +**Why `harmony_config` / `harmony_secret`, not a fleet-local secrets +client.** ADR-020 is explicit that `harmony_config` is the unified +config+secret entry point and `OpenbaoSecretStore` is the canonical +OpenBao client. Adding a parallel fleet-only client would duplicate +the auth ladder, cache-file layout, and `kv2` plumbing already in +`harmony_secret`. The fleet's need is an *additional auth branch*, +not a different store. + +**Why scope is decoded from the Zitadel JWT, not asked of Bao.** The +agent already holds the JWT it's about to present at login; the +`deployments` claim is right there. A `/v1/auth/token/lookup-self` +round-trip after login would compute the same set from the other +direction, paying a network call to recover information already in +hand. + +## Consequences + +**Pros** + +- One auth root on a device (the existing Zitadel machine key) covers + both NATS and OpenBao access. Rotation, revocation, and inventory + remain centralized. +- The operator owns membership; the agent owns identity. A compromised + device cannot widen its own access. A compromised operator's blast + radius is its own fleet (one Zitadel project, one Bao instance). +- Per-deployment policies are mechanical to generate. Bao policy text + is identical modulo the deployment ID, produced by a small templated + Score. New deployments add one external group + one policy; no + hand-written ACLs. +- The lifecycle layer is a reusable home for future + "before-reconcile" work without further architectural changes. + +**Cons** + +- **Two-token invalidation on membership change.** Both the cached + Zitadel access token and the cached Bao Vault token must be dropped + for new membership to take effect. This is encapsulated in the + `secrets.refresh()` call but is a real round-trip cost (one HTTPS to + Zitadel + one to Bao) on every membership change. Mitigated by the + fact that membership changes are rare relative to secret reads. +- **Removal latency = Vault token TTL.** Removing a device from a + deployment does not immediately revoke its currently-cached Vault + token; access ends at next renewal or TTL expiry. Short TTLs (15 min) + bound the worst case; explicit `bao token revoke -accessor` is + available if needed. +- **Operator gains Zitadel-admin scope.** The operator must hold + credentials that can write user metadata in the Zitadel project. + This is a high-privilege scope and concentrates trust in the + operator. The mitigation is a per-fleet Zitadel project: a + compromised operator can only mutate its own fleet's identities. +- **Zitadel Action required.** Surfacing user metadata as a JWT claim + needs a small Zitadel Action (server-side JavaScript). It is part of + the fleet's Zitadel setup and must be in version control / applied + by the fleet's bootstrap, not configured by hand. (See "Additional + Notes" for the script.) + +## Alternatives considered + +**Project roles for deployment membership.** Rejected: flat namespace +inside a project, no native multi-value semantics, role inventory +explodes at hundreds of deployments per fleet, mutations require +project-admin scope on a coarse-grained API. Kept for the coarse +`fleet-device` / `fleet-admin` distinction the NATS callout already +uses. + +**Project ID embedded in the secret path +(`secrets///...`).** Rejected: the project +isolation is already enforced by `bound_audiences` at the JWT layer. +Encoding it in the path is duplication of the same assertion, couples +the secret tree to a Zitadel ID, and complicates project rotations. +Adds no security: a token that passes `bound_audiences` validation can +read the path regardless; one that fails cannot read anything. + +**Claim-templated single policy +(`{{identity.…metadata.deployment_id}}`).** Rejected for the +multi-deployment case: Vault policy templating does not iterate over +arrays, so a single-policy template can only express "one deployment +per device." Acceptable for a single-deployment-per-device world; the +chosen kubelet-like architecture admits N deployments per device, and +collapsing the chosen `groups_claim` design to this would force +multiple JWT logins per refresh. + +**Static per-device Bao token issued at provisioning.** Rejected: +introduces a second long-lived secret on the device, breaks rotation +(re-provisioning required), and provides no native per-deployment +scoping. + +**OpenBao OIDC code flow.** Rejected: that flow is for human users +with a browser. Devices are headless and already hold a JWT-bearer +identity; using OIDC would re-invent the wheel and require a local +browser-equivalent. + +**Lifecycle layer inside the NATS handler.** Rejected: conflates +transport with domain logic and makes the refresh-then-reconcile +ordering implicit. The dedicated module makes the contract testable +and lets future triggers reuse the same code path. + +## Additional Notes + +### Zitadel Action (token customization) + +A single post-access-token-creation Action per fleet's Zitadel project +copies user metadata `deployments` into a top-level claim: + +```javascript +// Trigger: pre-access-token-creation +function addDeployments(ctx, api) { + const md = ctx.v1.user.getMetadata(); + const entry = md.metadata.find(m => m.key === "deployments"); + if (!entry) return; + try { + const deployments = JSON.parse( + Buffer.from(entry.value, "base64").toString("utf-8") + ); + if (Array.isArray(deployments)) { + api.v1.claims.setClaim("deployments", deployments); + } + } catch (_) { /* malformed metadata is treated as no deployments */ } +} +``` + +The Action lives in Zitadel's "Flows" configuration, attached to the +`Complement Token` flow on the relevant project. A Harmony Score +(`ZitadelTokenCustomizationScore` or similar) is the right home for +applying this declaratively; see plan document for status. + +### Relationship to ADR-016 and ADR-020-1 + +ADR-016 (agent mesh on NATS JetStream) establishes the agent's +existing Zitadel-keyed identity for NATS. This ADR reuses that +identity unchanged. + +ADR-020-1 establishes the human-developer authentication path to +OpenBao via Zitadel's Device Authorization Grant. This ADR is the +machine-user counterpart: same OpenBao, same Zitadel, different +auth-method binding (humans use device code; devices use +JWT-bearer-derived access tokens against `/auth/jwt/login`). + +### Threat model summary + +| Attacker | Capability | Defense | +|---|---|---| +| External (no Zitadel identity) | None | No valid JWT signature; rejected at JWKS validation. | +| Compromised device (key theft) | Full agent scope on its own deployments only | `groups_claim` restricts scope to the device's metadata; Zitadel admin can rotate the machine key and trigger immediate re-issuance. | +| Different Zitadel project (different tenant or malicious org) | Can mint valid Zitadel tokens for its own project | `bound_audiences` rejects at the JWT auth boundary before any claim is read. | +| Compromised operator | Can mutate Zitadel metadata + Bao policies for its fleet | One operator per fleet; operator credentials themselves stored in Bao under a separate auth path; compromise is contained to the operator's project. | +| Compromised Bao | Full access to all stored secrets | Out of scope — Bao is the root of secret trust by definition. ADR-006 covers Bao operational hardening. | diff --git a/docs/adr/025-device-secret-access/025-2-entity-by-policy.md b/docs/adr/025-device-secret-access/025-2-entity-by-policy.md new file mode 100644 index 00000000..375a881d --- /dev/null +++ b/docs/adr/025-device-secret-access/025-2-entity-by-policy.md @@ -0,0 +1,74 @@ +# ADR 025-2: Entity-by-Policy Scoping (as built) + +Initial Author: Jean-Gabriel Gill-Couture + +Initial Date: 2026-06-11 + +## Status + +Retired. First implementation shipped on +`feat/fleet-device-secret-access`, then superseded by the group-based +design (main [ADR-025](../025-fleet-device-secret-access.md)) after +the scale analysis in +[025-3](./025-3-groups-as-security-boundary.md): per-device entity +writes are O(targeted devices) per deployment change, which does not +hold at fleet scale. The identity layer built alongside it +(`harmony_zitadel_jwt`, the JWT-bearer auth rung, `refresh_auth()`) +carries over unchanged. + +## What changed vs 025-1 + +[025-1](./025-1-jwt-deployments-claim.md) carried per-deployment +membership *inside the token*: Zitadel user metadata → token Action → +JWT `deployments` claim → OpenBao `groups_claim` → external groups. +Implementation replaced that whole chain with one server-side write: + +**A device's secret access is its OpenBao identity entity's policy +list.** The operator sets the entity's policies to exactly +`deployment-` for each deployment currently scheduled on the +device. OpenBao resolves identity policies **at request time**, so a +grant or revocation takes effect on the device's very next read — the +device logs in once and keeps its token; no re-auth, no thundering herd +on bulk reassignment. Proven by the e2e test +`entity_policy_grants_existing_openbao_token_access` +(`fleet/harmony-fleet-e2e/tests/openbao_policy.rs`). + +## Why the claim chain lost + +- **Tokens snapshot, entities don't.** Claim-carried scope is frozen at + mint time; every membership change required invalidating two cached + tokens (Zitadel + Bao) per device. Entity policies change server-side + with zero device participation — strictly better for revocation and + for offline devices. +- **Less machinery.** Dropped: the Zitadel token Action (server-side JS + in the security path), `groups_claim` + external groups, the agent's + `cached_scope()` and before-reconcile refresh-check, and the + operator's Zitadel-metadata write scope. The operator now writes only + OpenBao. +- **One source of truth.** Membership derives from the operator's view + of the `Deployment` CRs in its reconcile loop, not from a second copy + in Zitadel user metadata. + +## Shipped shape + +| Piece | Home | +|---|---| +| `DeviceSecretAccess` trait (devices ↔ deployments, backend-agnostic) + in-memory test impl | `harmony/src/modules/fleet/secret_access.rs` | +| `OpenBaoDeviceSecretAccess` (entity resolution via JWT-mount alias, policy-list writes) | `harmony/src/modules/openbao/identity.rs` | +| Operator wiring: `sync_secret_access` after every selector/device change | `fleet/harmony-fleet-operator/src/fleet_aggregator.rs` | +| JWT-bearer minter (RFC 7523), extracted to a leaf crate to break a dependency cycle | `harmony_zitadel_jwt` (re-exported by `harmony_zitadel_auth`) | +| JWT-bearer rung on the auth ladder + `refresh_auth()` | `harmony_secret/src/store/openbao.rs` | + +Entity resolution subtlety: OpenBao auto-provisions an entity (random +name) + alias (name = `user_claim`, i.e. the device id) on first JWT +login. The operator therefore resolves device → entity **via the +alias**, creating entity + alias itself when the device has never +logged in, so access can be granted ahead of first login. + +## What this iteration deliberately does not solve + +Scheduling — and therefore secret access — is gated only by the +deployment's label selector over **device-reported** labels. A device +that can choose its own labels can attract deployments and their +secrets. That gap is the subject of +[025-3](./025-3-groups-as-security-boundary.md). diff --git a/docs/adr/025-device-secret-access/025-3-groups-as-security-boundary.md b/docs/adr/025-device-secret-access/025-3-groups-as-security-boundary.md new file mode 100644 index 00000000..f4bcc131 --- /dev/null +++ b/docs/adr/025-device-secret-access/025-3-groups-as-security-boundary.md @@ -0,0 +1,177 @@ +# ADR 025-3: Groups as the Security Boundary — Scale & Security Analysis + +Initial Author: Jean-Gabriel Gill-Couture + +Initial Date: 2026-06-11 + +## Status + +Accepted — this analysis produced the design in the main +[ADR-025](../025-fleet-device-secret-access.md). Kept as the record of +the entity-vs-groups comparison, the verification work behind it, and +the state-of-the-art survey. + +## Context + +[025-2](./025-2-entity-by-policy.md) (entity-by-policy) shipped first +and is correct at small scale, but two concerns drove a re-design: + +1. **Privilege escalation via labels.** Scheduling — and therefore + secret grants — keyed on a label selector over *device-reported* + labels. A compromised device relabels itself to match any + deployment's selector and receives its secrets. Kubernetes has the + same structural risk and closes it with the `NodeRestriction` + admission plugin; we had no equivalent, and no operator-facing way + to say "this deployment may only ever be seen by the lab fleet." +2. **Write amplification.** The operator rewrote one OpenBao identity + entity per targeted device per deployment change — O(targeted + devices) Raft-backed writes. The fleet ambition is millions of + devices. + +The proposal: split authorization from placement, as Kubernetes splits +RBAC from scheduling. Groups (admin-managed, never self-asserted) gate +who may *view* a deployment; labels only place workloads within the +authorized set. Two enforcement variants were compared: + +- **A — external groups:** Zitadel group membership rides in the token + (`groups_claim`); OpenBao maps it to external identity groups; each + deployment's policy attaches to its allowed groups. +- **B — entity-by-policy + group gate:** keep 025-2's per-device + entity policy lists; groups exist only in the control plane as a + scheduling filter. + +An initial review recommended B (finer granularity, no re-login, +instant server-side revocation). Verification of the underlying +facts — plugin source, documented limits, at-scale precedent — +reversed that recommendation. **A wins at the stated scale.** + +## Verified facts (with sources) + +1. **Token renewal does NOT re-sync external-group membership for + JWT auth.** The plugin's renew handler only checks the role still + exists and extends TTLs; claims are never re-read, `GroupAliases` + never rebuilt. Membership converges **only at fresh login**. + HashiCorp's KB wording ("login or renewal") is true for methods + whose renew path re-queries the IdP (e.g. LDAP), not JWT. + Source: `vault-plugin-auth-jwt/path_login.go` and OpenBao's fork + `builtin/credential/jwt/path_login.go`. +2. **Identity policies resolve at request time.** "The evaluation of + policies applicable to the token through its identity will happen + at request time" (Vault/OpenBao identity concepts). Attaching or + detaching a policy on a group is visible to every member's existing + tokens immediately. +3. **Batch tokens carry identity.** OpenBao marshals `EntityID` into + the encrypted batch-token blob (`vault/token_store.go`), so + login-issued batch tokens resolve entity and external-group + policies per request — with **zero storage writes per login**. + Limits: no renewal, no accessor, not individually revocable. + (Token-backend-created batch tokens have no entity — the source of + conflicting community lore.) +4. **Entity disable is the per-device kill switch.** + `identity/entity` `disabled=true`: "associated tokens cannot be + used, but are not revoked" — request-time blocking, no device + cooperation, works for batch tokens. +5. **Identity store ceiling.** Entities pack into **256 storage + entries** (hard cap ≈ 0.5–1.25M entities at default limits, + ~200 B/entity), the store is memory-resident (go-memdb), and each + entity upsert rewrites its whole shard — per-write cost grows with + fleet size. Community reports: degradation from ~50k identities; + 5.7M entities → ~110 GB RAM and >10 min failover + (hashicorp/vault#8761). **JWT login unavoidably creates one + entity + alias per distinct `user_claim`** — under any design. +6. **Write throughput.** Reference-architecture benchmark ≈ 720 + storage writes/s; >1k req/s is "high volume" guidance for + integrated storage. No bulk-entity-update API exists. +7. **Zitadel.** Machine-JWT token issuance benchmarked ≈ 437/s + (v2.66) to ≈ 852/s (v4-rc); roles claim + (`urn:zitadel:iam:org:project:roles`) is a **map**, while + `groups_claim` requires a **string array** → flattening Action + needed until first-class groups (epic zitadel#9702, in flight, + machine users included) ship a documented array claim. +8. **At-scale precedent — AWS IoT Core.** Policies attach to thing + groups; "policies are dynamically applied to things when they + change group membership"; Greengrass deploys secrets by targeting + thing groups. Group-attached policy *is* the canonical hyperscale + IoT authorization model. Azure scopes to per-device/module + identities; no per-workload grant materialization anywhere. + +## The decisive arithmetic + +Let a deployment target M devices in a fleet cell. + +| | B: entity-by-policy | A: groups | +|---|---|---| +| Writes per deployment change | M entity upserts; each rewrites ~1/256 of all entities → at M = 10⁵, gigabytes of Raft churn, minutes–hours of convergence, partial-failure management | 1 policy attach per allowed group, instant for all existing tokens (fact 2) | +| Writes per membership change | 1 (instant) | 1 Zitadel write; converges at next login ≤ TTL (fact 1) | +| Steady-state auth load | ~0 (long-lived tokens viable) | N/TTL logins/s — 1M devices @ 1 h ≈ 280/s, stateless with batch tokens (fact 3); Zitadel-side token reused across logins (~23 mints/s @ 12 h validity) | +| Per-device emergency cutoff | strip entity policies | disable entity (fact 4) — equivalent | +| Compromised-device blast radius | deployments scheduled on it | all deployments its groups may view — by declaration; granularity = group sizing | +| IdP machinery | none | flattening Action until Zitadel groups GA (fact 7) | + +B's write cost scales with *deployment churn × fleet size*; A's with +*membership churn*. Membership is the rare, per-device event — +deployments are the frequent, fleet-wide one. The data plane already +fans out O(M) per deployment (NATS desired-state KV), but NATS KV +sustains ~10⁵ writes/s where OpenBao identity writes run ~10²–10³/s +and degrade as the store grows: per-device *instructions* scale; +per-device *grants* don't. B's apparent security edge largely +dissolved under verification: per-device hard revocation exists in A +(fact 4), routine de-grouping ≤ TTL is an operational tolerance, and +finer isolation is available by defining finer groups. What remains is +A's group-wide secret visibility — accepted explicitly, since +`allowedGroups` *is* the declared audience. + +A corollary of fact 1: don't renew tokens at all. Short-TTL batch +tokens whose natural expiry forces re-login make the login itself the +membership sync — deleting the re-login-signal mechanism the first +sketch of this proposal included. + +Independent of A vs B, fact 5 caps any single OpenBao at ~10⁵–10⁶ +device entities: "millions of devices" means **cell sharding** (one +Zitadel project + OpenBao pair per cell) under either design. Groups +shard naturally with cells. + +## State of the art surveyed + +- **Vault-ecosystem fleet patterns** — AppRole + trusted orchestrator, + TLS-cert + SCEP (Vault 1.20), SPIFFE auth (Vault 1.21, not in + OpenBao) — all solve device *identity*, which Zitadel already + provides; none address per-workload scoping. Nothing resembling + "a deployment declares which device groups may read its secrets" + exists in that ecosystem; the model is uniformly identity → policy + → path. +- **Kubernetes `NodeAuthorizer`** — per-node grants derived from pod + placement, the inspiration for B — materializes nothing: it is an + in-memory request-time graph in the API server. The faithful OpenBao + equivalent would be a custom request-time authorization plugin, + rejected for coupling secret reads to control-plane liveness. +- **Templated policies** (`claim_mappings` → alias metadata → one + fleet-wide policy `…/{{identity.entity.aliases..metadata.group}}/*`) + achieve zero grant writes ever — AWS IoT's policy-variable trick — + but templating cannot iterate, forcing exactly one group per device + and group-scoped secret paths. Kept as a fallback simplification. +- **SPIFFE/SPIRE** — workload-level (not device-level) identity would + let each deployment's process authenticate as itself, shrinking the + blast radius below anything device-scoped and making grants + O(deployments). Deferred: SPIRE server + per-device agents, weak + node attestation on TPM-less hardware, and no OpenBao SPIFFE auth + method yet. The group/policy layer chosen here survives that + migration — only the auth mount changes. + +## Resolved during review + +- ~~Re-login signal on membership change~~ — unnecessary; TTL expiry + is the sync. +- ~~Where membership lives~~ — Zitadel (roles-as-groups interim); + Device CR fields would create a second authority. +- Vocabulary: the staging fleet's device *label* named `group` must be + renamed or clearly distinguished — labels are placement vocabulary + and device-writable; the authorization group is admin-only. + +## Open items for implementation + +1. Confirm in staging that a disabled entity blocks an in-flight + batch token at request time (fact 4 is documented; prove it e2e). +2. TTL default 1 h — revisit with measured login load. +3. Per-cell sizing guidance once identity-store memory is measured on + OpenBao (Vault-lineage numbers above are the planning basis). diff --git a/docs/adr/025-fleet-device-secret-access.md b/docs/adr/025-fleet-device-secret-access.md new file mode 100644 index 00000000..2a9df3d9 --- /dev/null +++ b/docs/adr/025-fleet-device-secret-access.md @@ -0,0 +1,196 @@ +# Architecture Decision Record: Fleet Device Secret Access + +Initial Author: Jean-Gabriel Gill-Couture + +Initial Date: 2026-06-01 + +Last Updated Date: 2026-06-11 + +## Status + +Accepted. User-facing documentation: +[Fleet device secrets guide](../guides/fleet-device-secrets.md). + +This ADR is the third design iteration; the trail is preserved in +[`025-device-secret-access/`](./025-device-secret-access/): +[025-1](./025-device-secret-access/025-1-jwt-deployments-claim.md) +(JWT claim scoping, superseded) → +[025-2](./025-device-secret-access/025-2-entity-by-policy.md) +(entity-by-policy, first implementation, retired) → +[025-3](./025-device-secret-access/025-3-groups-as-security-boundary.md) +(the scale and security analysis that produced this design). + +## Context + +Fleet agents must read per-deployment secrets (image-pull credentials, +application secrets) from OpenBao. Constraints: + +1. **One root of trust per device.** The agent already holds a Zitadel + machine keyfile for NATS auth; secret access must derive from the + same key, never a second secret. +2. **Store-enforced isolation.** The secret store decides what a device + may read; a device cannot widen its own scope. +3. **Cross-project safety.** A token minted in another Zitadel project + must be rejected outright. +4. **Fleet scale.** The target is millions of devices. Control-plane + write cost must track *membership churn* (rare, per-device), never + *deployment churn × targeted devices* — a deployment rolling out to + a large group must not trigger a per-device write storm. + +## Decision + +**Groups are the security boundary; labels only place workloads.** + +### Identity: one key, one login, short batch tokens + +The device mints a Zitadel access token from its machine keyfile (RFC +7523 JWT-bearer, shared with NATS auth via `harmony_zitadel_jwt`) and +logs into OpenBao's JWT auth method through +`harmony_config`/`OpenbaoSecretStore`. The auth role pins +`bound_issuer` and `bound_audiences = ` — the +project is the trust boundary — with `user_claim = sub` and +`groups_claim = groups`. The role issues **batch tokens** (no storage +writes per login) with **TTL ≈ 1 h, no renewal**. Expiry forces a fresh +login (`refresh_auth()`), and that login is the group-membership sync — +there is no separate re-login flow, signal, or membership watcher. + +### Groups: admin-managed in Zitadel, mirrored as OpenBao external groups + +A fleet group (e.g. `edge-a`, `customer-x`) is membership a device can +never self-assert: admins manage it in Zitadel, and it reaches OpenBao +inside the signed token's `groups` array claim. Each fleet group has an +OpenBao **external group** whose alias matches the JWT mount, so a +login binds the device's entity to its current groups. Interim claim +source until Zitadel's first-class groups are GA: one project role per +group plus the standard flattening Action (Zitadel's roles claim is a +map; `groups_claim` requires a string array). + +### Authorization and scheduling: groups gate, labels place + +A `Deployment` declares `allowedGroups` — the groups permitted to view +it (run it and read its secrets). The operator schedules onto +**members of the allowed groups ∩ the label selector**. Labels remain +device-reported placement vocabulary; they can narrow a deployment's +spread, never widen access. (This is Kubernetes' RBAC-vs-scheduling +split.) + +### Grants: one policy per deployment, attached to groups + +Secrets live under the deployment's subtree of the fleet KV mount; the +policy `deployment-` grants read on exactly that subtree. When a +`Deployment` appears, the operator creates the policy and attaches it +to each allowed group's external group — **O(allowed groups) writes, +regardless of fleet size**. Identity policies resolve at request time, +so every already-issued token of every member device gains (or loses) +access immediately when a policy is attached (or detached). + +### Convergence and revocation + +| Change | Mechanism | Latency | +|---|---|---| +| Deployment created / `allowedGroups` edited | attach/detach policy on group | immediate (request time) | +| Device added to / removed from a group | next login (token expiry) | ≤ token TTL | +| Emergency: cut one device | disable its OpenBao entity | immediate | +| Emergency: cut one deployment or group | detach the policy | immediate | +| Device key compromised | rotate/revoke machine key in Zitadel + disable entity | immediate | + +TTL is therefore an operational tolerance for *routine* de-grouping, +not a security boundary — emergencies never wait on it. + +## Rationale + +The full analysis with sources is +[025-3](./025-device-secret-access/025-3-groups-as-security-boundary.md); +the load-bearing points: + +- **Write cost aligned with event granularity.** The retired + entity-by-policy design (025-2) wrote every targeted device's entity + per deployment change; each identity write rewrites 1/256 of all + entities, so cost grows with fleet size — gigabytes of Raft churn + and hours of convergence at 10⁵–10⁶ targets. Group attachment is one + write and binds instantly. +- **Groups close the label-escalation hole.** Labels are + device-reported; with labels gating secrets, a compromised device + relabels itself to attract any deployment (Kubernetes needs + `NodeRestriction` for exactly this). Group membership is signed by + Zitadel and admin-only. +- **Renewal does not re-sync groups.** Verified in the JWT auth plugin + source (Vault and OpenBao): renew only extends TTLs. Hence + short-lived, non-renewed tokens whose natural expiry is the sync — + batch tokens make this free (1M devices at 1 h TTL ≈ 280 stateless + logins/s; the Zitadel-side token is reused across many logins). +- **Precedent.** AWS IoT Core authorizes fleets exactly this way: + policies attach to thing groups and "are dynamically applied to + things when they change group membership"; Greengrass deploys + secrets by targeting thing groups. There is no Vault-ecosystem + pattern for per-device grant materialization at fleet scale. +- **Cell ceiling is independent of this choice.** OpenBao's identity + store is memory-resident and packs entities into 256 storage entries + (≈0.5–1.25M entities max), and JWT login creates one entity per + device under any design. Past ~10⁵ devices, fleets shard into cells + (one Zitadel project + OpenBao pair each); groups are scoped within + a cell. + +## Consequences + +**Pros** + +- New deployment: no device session touched, no per-device writes, no + admin permission work beyond declaring `allowedGroups`. +- One auth root per device covers NATS and OpenBao; rotation, + inventory, and revocation stay centralized in Zitadel. +- The security boundary is legible: group rosters in Zitadel, + group→deployment grants on the OpenBao group object, both + human-auditable. +- Batch tokens keep steady-state load CPU-bound (JWKS verification), + with zero token-storage writes. + +**Cons / accepted trade-offs** + +- **Group-granular visibility.** Every device in an allowed group can + read the deployment's secrets, scheduled onto it or not. Isolation + granularity is group sizing — need finer isolation, define finer + groups. (Per-workload identity via SPIFFE is the future evolution + that lifts this; the policy layer survives that migration.) +- **De-grouping converges at TTL** (≤ 1 h). Emergencies use the + immediate levers above instead. +- **Zitadel Action required** until first-class groups GA — small, + version-controlled server-side JS in the token path. +- **Batch tokens are not individually revocable**; per-device cutoff + is entity disable, which blocks the token at request time. +- One entity + alias per device still accumulates in OpenBao (the + cell ceiling above). + +## Threat model + +| Attacker | Capability | Defense | +|---|---|---| +| External (no Zitadel identity) | None | JWKS signature validation. | +| Other Zitadel project | Mints valid tokens for its own project | `bound_audiences` rejects before any claim is read. | +| Compromised device (key theft) | Reads secrets of deployments its groups are allowed to view — nothing more. Cannot self-widen: labels don't grant, groups are Zitadel-signed. | Disable entity + rotate key; blast radius bounded by group sizing. | +| Compromised operator | Full control of its fleet's grants (policy attach, scheduling) | Blast radius = one cell (one project, one Bao); operator creds under a separate Bao auth path. | +| Compromised OpenBao | Everything in its cell | Out of scope — root of secret trust (ADR-006); cells bound the radius. | + +## Alternatives considered + +Full detail in the +[iteration trail](./025-device-secret-access/). In brief: **JWT +`deployments` claim** (025-1 — scope frozen at mint time, two-token +invalidation per change, Zitadel metadata as a second source of +truth); **entity-by-policy** (025-2, first implementation — instant +per-device grants but O(targeted devices) writes per deployment, +disqualifying at fleet scale); **one templated policy on alias +metadata** (zero writes ever, but policy templating cannot iterate, so +exactly one group per device — kept as a fallback simplification); +**static per-device Bao tokens** (second long-lived secret, no +scoping); **AppRole / TLS-cert / SPIFFE auth** (solve identity, which +Zitadel already provides; SPIFFE noted as the per-workload evolution +path); **request-time authorization plugin** (true NodeAuthorizer +equivalent, but couples secret reads to control-plane liveness). + +## Relationship to other ADRs + +ADR-016: the Zitadel machine identity reused here. ADR-020/020-1: +`harmony_config` as the single secret entry point; the human +counterpart of this machine auth path. ADR-023: all OpenBao/Zitadel +configuration lands via Scores. diff --git a/docs/adr/README.md b/docs/adr/README.md index d50a291d..b652ec64 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -53,6 +53,10 @@ Every ADR follows this structure: | [020-1](./020-1-zitadel-openbao-secure-config-store.md) | Zitadel + OpenBao Secure Config Store | Accepted | | [020](./020-interactive-configuration-crate.md) | Interactive Configuration Crate | Proposed | | [022](./022-fleet-agent-upgrade.md) | Fleet Agent Upgrade Procedure | Accepted | +| [023](./023-deploy-architecture.md) | Deploy Architecture | Accepted | +| [024](./024-fleet-platform-capability-decomposition.md) | Fleet Platform Capability Decomposition | Accepted | +| [025](./025-fleet-device-secret-access.md) | Fleet Device Secret Access | Accepted | +| [025-3](./025-device-secret-access/025-3-groups-as-security-boundary.md) | Groups as the Security Boundary — Scale & Security Analysis | Accepted | ## Contributing diff --git a/docs/guides/fleet-device-secrets.md b/docs/guides/fleet-device-secrets.md new file mode 100644 index 00000000..1df7e2cc --- /dev/null +++ b/docs/guides/fleet-device-secrets.md @@ -0,0 +1,113 @@ +# Fleet device secrets + +How fleet devices read their deployments' secrets — image-pull +credentials, application config, API keys — from OpenBao, and how you +control who reads what. Design reasoning lives in +[ADR-025](../adr/025-fleet-device-secret-access.md); this page is the +operating manual. + +## The architecture in one minute + +A device proves *who it is* with the Zitadel machine key it already +uses for NATS. *What it may read* is decided by **group membership**, +which only admins can change. *Where it runs* is decided by labels. +The three never mix: + +``` + Admin Zitadel OpenBao + │ assign device to groups ──▶ device ∈ {edge-a, …} │ + │ write secret ──────────────────────────────────────────────▶ secret///… + │ │ + Deployment CR │ + │ allowedGroups: [edge-a] ──▶ operator attaches policy ──────▶ group edge-a may read + │ targetSelector: labels ──▶ operator schedules within │ deployment- subtree + │ authorized devices │ + │ │ + Device │ + └─ machine key ─▶ Zitadel token (groups: [edge-a]) ─▶ JWT login ─▶ reads its secrets +``` + +Three rules to remember: + +| Concept | Who sets it | What it controls | +|---|---|---| +| **Group** | Admin, in Zitadel | Security boundary: which deployments' secrets a device may read | +| **Label** | Device config (self-reported) | Placement only: which authorized devices actually run the workload — narrows, never widens | +| **`allowedGroups`** | Deployment author | Which groups may view (run + read secrets of) the deployment | + +## Storing a secret for a deployment + +Write it under the deployment's subtree of the fleet KV mount: + +```bash +bao kv put secret///db-credentials \ + username=app password=… +``` + +Nothing else to configure — access follows from the deployment's +`allowedGroups` below. Every device in an allowed group can read the +whole `/` subtree, whether or not the workload landed +on it; size your groups to the audience the secret may have. + +## Authorizing and placing a deployment + +```yaml +apiVersion: fleet.nationtech.io/v1alpha1 +kind: Deployment +metadata: + name: hello-web +spec: + allowedGroups: [edge-a] # security: who may view this deployment + targetSelector: + matchLabels: + hw: pi5 # placement: which authorized devices run it + score: … +``` + +The operator schedules onto devices that are **members of an allowed +group AND match the selector**, then attaches the deployment's read +policy to each allowed group. Existing device sessions pick the grant +up immediately — rolling out a new deployment never touches logins, +tokens, or per-device configuration. + +## Managing device group membership + +Until Zitadel's first-class groups are GA, a fleet group is a Zitadel +project role (one role per group, surfaced as a `groups` claim by the +fleet's token Action — applied by the fleet bootstrap, not by hand). +Grant or remove the role on the device's machine user to move it +between groups. + +Membership changes take effect at the device's **next login, at most +one token TTL later (≈ 1 h)**. This is the only eventually-consistent +operation in the system — everything else binds immediately. If that +window matters, use the emergency levers below. + +## Reading secrets on the device + +Nothing to do: the agent reads through `harmony_config`, which +authenticates to OpenBao with the machine keyfile automatically +(JWT-bearer rung of `OpenbaoSecretStore`). Tokens are short-lived +batch tokens; the store re-logins transparently on expiry. + +## Revocation runbook + +| Situation | Action | Effect | +|---|---|---| +| Device leaves a group (routine) | Remove the role/group in Zitadel | Converges at next login, ≤ 1 TTL | +| Device compromised | Disable its OpenBao identity entity, then rotate/delete its Zitadel machine key | Token dead at next request; no new logins | +| Secret/deployment compromised | Detach `deployment-` policy from its groups (or rotate the secret) | All devices lose read immediately | +| Whole group compromised | Detach the group's policies | Immediate, group-wide | + +## Guarantees and limits + +- A device can only ever read secrets of deployments whose + `allowedGroups` intersect its groups. Labels, CR edits, or a + compromised agent cannot widen that — membership is signed into the + Zitadel token and validated against the project (`bound_audiences`), + so other projects' tokens are rejected outright. +- Visibility is **group-granular**: being scheduled is not required to + read. Finer isolation ⇒ finer groups. +- One Zitadel project + OpenBao pair (a *cell*) comfortably serves + ~10⁵ devices; beyond that, shard fleets into cells + ([ADR-025](../adr/025-fleet-device-secret-access.md), Rationale). diff --git a/examples/compose_java_react/Cargo.toml b/examples/compose_java_react/Cargo.toml new file mode 100644 index 00000000..b6d5072f --- /dev/null +++ b/examples/compose_java_react/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "example-compose-java-react" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true +description = "Deploy a Java+React app to Kubernetes with Harmony by importing its existing docker-compose (publish + deploy, fleet pattern). See ADR-026." + +[lib] +name = "example_compose_java_react" +path = "src/lib.rs" + +# `deploy` — converge the app onto a cluster (helm upgrade --install). +[[bin]] +name = "compose-deploy" +path = "src/main.rs" + +# `publish` — build + push the per-service images and the hydrated chart. +[[bin]] +name = "compose-publish" +path = "src/bin/publish.rs" + +[dependencies] +harmony = { path = "../../harmony" } +harmony_cli = { path = "../../harmony_cli" } +harmony_types = { path = "../../harmony_types" } + +docker-compose-types = "0.24" +anyhow = { workspace = true } +async-trait = { workspace = true } +clap = { workspace = true } +fqdn = "0.5.2" +k8s-openapi = { workspace = true } +log = { workspace = true } +env_logger = { workspace = true } +serde = { workspace = true } +serde_yaml = { workspace = true } +tempfile = "3" +thiserror = { workspace = true } +tokio = { workspace = true, features = ["full"] } +toml = { workspace = true } diff --git a/examples/compose_java_react/Harmony.toml b/examples/compose_java_react/Harmony.toml new file mode 100644 index 00000000..7a4add83 --- /dev/null +++ b/examples/compose_java_react/Harmony.toml @@ -0,0 +1,6 @@ +# Identity only (ADR-026 §3): the app *name*, nothing behavioral. +# Namespace, image, and selectors derive by convention from name × context; +# deploy knobs (replicas, storage class, ingress host, rolling strategy) +# live in typed Scores, never here. +[app] +name = "timesheet" diff --git a/examples/compose_java_react/README.md b/examples/compose_java_react/README.md new file mode 100644 index 00000000..120aa3c8 --- /dev/null +++ b/examples/compose_java_react/README.md @@ -0,0 +1,69 @@ +# Deploy a Java + React app from its `docker-compose.yml` + +Demonstrates deploying an existing containerized app to Kubernetes with +Harmony by **importing its `docker-compose.yml`** — no hand-written +manifests, no Argo. The compose file stays the source of truth for the +app's *base* shape; deploy-only concerns live in typed Rust. + +``` +app/ the "customer" project (their existing repo) + docker-compose.yml ← source of truth: images, ports, env, volumes + backend/ (Java + SQLite, Dockerfile) + frontend/ (React + nginx, Dockerfile) +Harmony.toml identity only — the app name (ADR-026 §3) +src/ + compose.rs import docker-compose → typed model (loud on the unsupported) + chart.rs model + deploy knobs → a hydrated helm chart (typed k8s) + publish.rs docker build/push each service + helm package/push the chart + score.rs ComposeAppScore: helm upgrade --install + Ingress + main.rs `compose-deploy` bin + bin/publish `compose-publish` bin +``` + +## Design + +- **compose = base, Score = deploy knobs.** The importer reads only + images, ports, env, and volumes. Replicas, RWX storage class, ingress + host, and rolling strategy are *not* in compose (or `Harmony.toml`) — + they are typed fields on the deploy `Score` (ADR-026 §6). A behavioral + knob in compose or `Harmony.toml` is a defect. +- **Build / publish / deploy split — the fleet pattern.** `publish` + builds + pushes the per-service images and a hydrated helm chart; + `deploy` is a `Score` that `helm upgrade --install`s it and layers the + public Ingress. No `ApplicationScore`, no ArgoCD. +- **Same Scores local → prod.** Only the topology changes + (`K8sAnywhereTopology::from_env`): local k3d to test, a tenant in prod. +- **Loud, never lossy.** Bind mounts and unparseable ports are hard + errors; `depends_on`/`command`/`restart` are warned, never silently + dropped. + +## Run it + +Local (build, don't push; render the chart from compose): + +```sh +cargo run -p example-compose-java-react --bin compose-publish -- --no-push +cargo run -p example-compose-java-react --bin compose-deploy -- \ + --host timesheet.local --storage-class --yes +``` + +CD / prod (push to the registry, then install the published chart): + +```sh +compose-publish --version 0.1.0 +compose-deploy --published --version 0.1.0 \ + --namespace -timesheet --host timesheet..example \ + --storage-class cephfs --cluster-issuer letsencrypt --yes +``` + +`compose-deploy --help` lists every deploy knob. + +## ⚠️ SQLite + RollingUpdate + RWX + +This demo ships `--rolling` (RollingUpdate) with a `ReadWriteMany` PVC, as +requested. During a rollout two pods briefly share the same SQLite file, +which SQLite does not make safe for concurrent writers over a networked +filesystem. It is tolerable only for an effectively single-writer, +low-traffic app. For true safe zero-downtime, either deploy with +`--recreate` (one writer at a time, brief downtime) or migrate the store +to Postgres. diff --git a/examples/compose_java_react/app/README.md b/examples/compose_java_react/app/README.md new file mode 100644 index 00000000..e91899d1 --- /dev/null +++ b/examples/compose_java_react/app/README.md @@ -0,0 +1,15 @@ +# Timesheet (demo compose app) + +Minimal two-tier app used to demo Harmony deploying an existing `docker-compose` +stack. Add timesheet notes; they persist to SQLite. + +- **backend** — JDK `HttpServer` (no framework) on `:8080`, SQLite via sqlite-jdbc. + REST: `GET /api/health`, `GET /api/entries`, `POST /api/entries`. +- **frontend** — React (Vite) served by nginx on `:80`; nginx proxies `/api/` + to the backend (`BACKEND_URL`), so the bundle stays env-agnostic. +- **volume** `timesheet-data` mounts at `/data`; the DB lives at `/data/timesheet.db`. + +```sh +docker compose up --build +# frontend → http://localhost:8081 backend → http://localhost:8080/api/health +``` diff --git a/examples/compose_java_react/app/backend/Dockerfile b/examples/compose_java_react/app/backend/Dockerfile new file mode 100644 index 00000000..1df1e684 --- /dev/null +++ b/examples/compose_java_react/app/backend/Dockerfile @@ -0,0 +1,12 @@ +FROM maven:3-eclipse-temurin-21 AS build +WORKDIR /src +COPY pom.xml . +COPY src ./src +RUN mvn -q -B package + +FROM eclipse-temurin:21-jre +WORKDIR /app +COPY --from=build /src/target/timesheet-backend.jar app.jar +EXPOSE 8080 +USER 1000 +ENTRYPOINT ["java", "-jar", "app.jar"] diff --git a/examples/compose_java_react/app/backend/pom.xml b/examples/compose_java_react/app/backend/pom.xml new file mode 100644 index 00000000..9ea15923 --- /dev/null +++ b/examples/compose_java_react/app/backend/pom.xml @@ -0,0 +1,48 @@ + + + 4.0.0 + + io.nationtech.demo + timesheet-backend + 1.0.0 + jar + + + 21 + UTF-8 + + + + + org.xerial + sqlite-jdbc + 3.49.1.0 + + + + + timesheet-backend + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + package + shade + + + + io.nationtech.demo.Main + + + + + + + + + diff --git a/examples/compose_java_react/app/backend/src/main/java/io/nationtech/demo/Main.java b/examples/compose_java_react/app/backend/src/main/java/io/nationtech/demo/Main.java new file mode 100644 index 00000000..55b1c024 --- /dev/null +++ b/examples/compose_java_react/app/backend/src/main/java/io/nationtech/demo/Main.java @@ -0,0 +1,125 @@ +package io.nationtech.demo; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.time.Instant; + +public class Main { + + static String dbUrl; + + public static void main(String[] args) throws Exception { + String dbPath = System.getenv().getOrDefault("DB_PATH", "/data/timesheet.db"); + dbUrl = "jdbc:sqlite:" + dbPath; + + try (Connection c = DriverManager.getConnection(dbUrl); Statement s = c.createStatement()) { + s.execute("CREATE TABLE IF NOT EXISTS entries (id INTEGER PRIMARY KEY AUTOINCREMENT, note TEXT, created_at TEXT)"); + } + + HttpServer server = HttpServer.create(new InetSocketAddress("0.0.0.0", 8080), 0); + server.createContext("/api/health", ex -> send(ex, 200, "{\"status\":\"ok\"}")); + server.createContext("/api/entries", Main::entries); + server.setExecutor(null); + server.start(); + System.out.println("listening on 0.0.0.0:8080, db=" + dbPath); + } + + static void entries(HttpExchange ex) throws IOException { + try { + switch (ex.getRequestMethod()) { + case "GET" -> send(ex, 200, listEntries()); + case "POST" -> { + String note = parseNote(new String(ex.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)); + send(ex, 201, insertEntry(note)); + } + default -> send(ex, 405, "{\"error\":\"method not allowed\"}"); + } + } catch (Exception e) { + send(ex, 500, "{\"error\":" + jsonStr(e.getMessage()) + "}"); + } + } + + static String listEntries() throws Exception { + StringBuilder sb = new StringBuilder("["); + try (Connection c = DriverManager.getConnection(dbUrl); + Statement s = c.createStatement(); + ResultSet rs = s.executeQuery("SELECT id, note, created_at FROM entries ORDER BY id")) { + boolean first = true; + while (rs.next()) { + if (!first) sb.append(","); + first = false; + sb.append(entryJson(rs.getLong("id"), rs.getString("note"), rs.getString("created_at"))); + } + } + return sb.append("]").toString(); + } + + static String insertEntry(String note) throws Exception { + String createdAt = Instant.now().toString(); + try (Connection c = DriverManager.getConnection(dbUrl); + PreparedStatement ps = c.prepareStatement( + "INSERT INTO entries (note, created_at) VALUES (?, ?)", Statement.RETURN_GENERATED_KEYS)) { + ps.setString(1, note); + ps.setString(2, createdAt); + ps.executeUpdate(); + try (ResultSet keys = ps.getGeneratedKeys()) { + keys.next(); + return entryJson(keys.getLong(1), note, createdAt); + } + } + } + + static String entryJson(long id, String note, String createdAt) { + return "{\"id\":" + id + ",\"note\":" + jsonStr(note) + ",\"created_at\":" + jsonStr(createdAt) + "}"; + } + + // Tiny extractor for {"note":"..."} — sufficient for this demo's single-field body. + static String parseNote(String body) { + int k = body.indexOf("\"note\""); + if (k < 0) return ""; + int colon = body.indexOf(':', k); + int open = body.indexOf('"', colon + 1); + if (open < 0) return ""; + StringBuilder sb = new StringBuilder(); + for (int i = open + 1; i < body.length(); i++) { + char ch = body.charAt(i); + if (ch == '\\' && i + 1 < body.length()) { sb.append(body.charAt(++i)); continue; } + if (ch == '"') break; + sb.append(ch); + } + return sb.toString(); + } + + static String jsonStr(String v) { + if (v == null) return "null"; + StringBuilder sb = new StringBuilder("\""); + for (char ch : v.toCharArray()) { + switch (ch) { + case '"' -> sb.append("\\\""); + case '\\' -> sb.append("\\\\"); + case '\n' -> sb.append("\\n"); + case '\r' -> sb.append("\\r"); + case '\t' -> sb.append("\\t"); + default -> sb.append(ch); + } + } + return sb.append("\"").toString(); + } + + static void send(HttpExchange ex, int status, String body) throws IOException { + byte[] b = body.getBytes(StandardCharsets.UTF_8); + ex.getResponseHeaders().set("Content-Type", "application/json"); + ex.sendResponseHeaders(status, b.length); + ex.getResponseBody().write(b); + ex.close(); + } +} diff --git a/examples/compose_java_react/app/docker-compose.yml b/examples/compose_java_react/app/docker-compose.yml new file mode 100644 index 00000000..e80949bd --- /dev/null +++ b/examples/compose_java_react/app/docker-compose.yml @@ -0,0 +1,27 @@ +# The customer's existing docker-compose — the single source of truth for +# the app's *base* shape (images, ports, env, volumes). Harmony imports +# this and derives the k8s deployment; deploy-only knobs (ingress host, +# storage class, replicas, rolling strategy) live in the deploy crate's +# Score, never here. See ADR-026. +name: timesheet +services: + backend: + build: ./backend + ports: + - "8080:8080" + environment: + - DB_PATH=/data/timesheet.db + volumes: + - timesheet-data:/data + + frontend: + build: ./frontend + ports: + - "8081:80" + environment: + - BACKEND_URL=http://backend:8080 + depends_on: + - backend + +volumes: + timesheet-data: diff --git a/examples/compose_java_react/app/frontend/.dockerignore b/examples/compose_java_react/app/frontend/.dockerignore new file mode 100644 index 00000000..f06235c4 --- /dev/null +++ b/examples/compose_java_react/app/frontend/.dockerignore @@ -0,0 +1,2 @@ +node_modules +dist diff --git a/examples/compose_java_react/app/frontend/Dockerfile b/examples/compose_java_react/app/frontend/Dockerfile new file mode 100644 index 00000000..945f5782 --- /dev/null +++ b/examples/compose_java_react/app/frontend/Dockerfile @@ -0,0 +1,14 @@ +FROM node:22-alpine AS build +WORKDIR /app +COPY package.json ./ +RUN npm install +COPY . . +RUN npm run build + +FROM nginx:alpine +COPY --from=build /app/dist /usr/share/nginx/html +COPY nginx.conf.template /etc/nginx/templates/default.conf.template +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +EXPOSE 80 +ENTRYPOINT ["/entrypoint.sh"] diff --git a/examples/compose_java_react/app/frontend/entrypoint.sh b/examples/compose_java_react/app/frontend/entrypoint.sh new file mode 100644 index 00000000..46afd13a --- /dev/null +++ b/examples/compose_java_react/app/frontend/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -e +envsubst '${BACKEND_URL}' < /etc/nginx/templates/default.conf.template > /etc/nginx/conf.d/default.conf +exec nginx -g 'daemon off;' diff --git a/examples/compose_java_react/app/frontend/index.html b/examples/compose_java_react/app/frontend/index.html new file mode 100644 index 00000000..ba165a85 --- /dev/null +++ b/examples/compose_java_react/app/frontend/index.html @@ -0,0 +1,12 @@ + + + + + + Timesheet + + +
+ + + diff --git a/examples/compose_java_react/app/frontend/nginx.conf.template b/examples/compose_java_react/app/frontend/nginx.conf.template new file mode 100644 index 00000000..ff5a2f62 --- /dev/null +++ b/examples/compose_java_react/app/frontend/nginx.conf.template @@ -0,0 +1,13 @@ +server { + listen 80; + root /usr/share/nginx/html; + index index.html; + + location /api/ { + proxy_pass ${BACKEND_URL}; + } + + location / { + try_files $uri /index.html; + } +} diff --git a/examples/compose_java_react/app/frontend/package.json b/examples/compose_java_react/app/frontend/package.json new file mode 100644 index 00000000..f3925bd0 --- /dev/null +++ b/examples/compose_java_react/app/frontend/package.json @@ -0,0 +1,18 @@ +{ + "name": "timesheet-frontend", + "private": true, + "version": "1.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@vitejs/plugin-react": "^4.3.4", + "vite": "^5.4.11" + } +} diff --git a/examples/compose_java_react/app/frontend/src/App.jsx b/examples/compose_java_react/app/frontend/src/App.jsx new file mode 100644 index 00000000..3b76dab2 --- /dev/null +++ b/examples/compose_java_react/app/frontend/src/App.jsx @@ -0,0 +1,48 @@ +import { useEffect, useState } from "react"; + +export default function App() { + const [entries, setEntries] = useState([]); + const [note, setNote] = useState(""); + + const load = () => + fetch("/api/entries") + .then((r) => r.json()) + .then(setEntries); + + useEffect(() => { + load(); + }, []); + + const submit = async (e) => { + e.preventDefault(); + if (!note.trim()) return; + await fetch("/api/entries", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ note }), + }); + setNote(""); + load(); + }; + + return ( +
+

Timesheet

+
+ setNote(e.target.value)} + placeholder="What did you work on?" + /> + +
+
    + {entries.map((e) => ( +
  • + {e.note} ({e.created_at}) +
    • + ))} +
+
+ ); +} diff --git a/examples/compose_java_react/app/frontend/src/main.jsx b/examples/compose_java_react/app/frontend/src/main.jsx new file mode 100644 index 00000000..ce286a9b --- /dev/null +++ b/examples/compose_java_react/app/frontend/src/main.jsx @@ -0,0 +1,5 @@ +import React from "react"; +import { createRoot } from "react-dom/client"; +import App from "./App.jsx"; + +createRoot(document.getElementById("root")).render(); diff --git a/examples/compose_java_react/app/frontend/vite.config.js b/examples/compose_java_react/app/frontend/vite.config.js new file mode 100644 index 00000000..081c8d9f --- /dev/null +++ b/examples/compose_java_react/app/frontend/vite.config.js @@ -0,0 +1,6 @@ +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; + +export default defineConfig({ + plugins: [react()], +}); diff --git a/examples/compose_java_react/src/bin/publish.rs b/examples/compose_java_react/src/bin/publish.rs new file mode 100644 index 00000000..9fb0b085 --- /dev/null +++ b/examples/compose_java_react/src/bin/publish.rs @@ -0,0 +1,56 @@ +//! `compose-publish` — build + push the per-service images and the +//! hydrated chart from the imported compose. `docker`/`helm` must be on +//! PATH and logged in to the registry. + +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use clap::Parser; + +use example_compose_java_react::{ComposeApp, DeployConfig, HarmonyManifest, publish::publish}; + +#[derive(Parser, Debug)] +#[command( + name = "compose-publish", + about = "Build + push the app images and chart" +)] +struct Cli { + #[arg(long, default_value = env!("CARGO_MANIFEST_DIR"))] + project_dir: PathBuf, + #[arg(long)] + compose_dir: Option, + #[arg(long, default_value = "hub.nationtech.io")] + registry: String, + #[arg(long, default_value = "harmony")] + project: String, + #[arg(long, default_value = "0.1.0")] + version: String, + /// Build + package only; skip both pushes (local k3d smoke-test). + #[arg(long)] + no_push: bool, +} + +fn main() -> Result<()> { + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); + let cli = Cli::parse(); + + let manifest = HarmonyManifest::from_dir(&cli.project_dir)?; + let compose_dir = cli + .compose_dir + .unwrap_or_else(|| cli.project_dir.join("app")); + let app = ComposeApp::from_dir(&compose_dir).context("importing docker-compose")?; + + let deploy = DeployConfig { + app_name: manifest.app.name, + chart_version: cli.version.clone(), + registry: cli.registry, + project: cli.project, + version: cli.version, + storage_class: None, + volume_size: "1Gi".to_string(), + replicas: 1, + rolling: true, + }; + + publish(&app, &deploy, !cli.no_push) +} diff --git a/examples/compose_java_react/src/chart.rs b/examples/compose_java_react/src/chart.rs new file mode 100644 index 00000000..e6288733 --- /dev/null +++ b/examples/compose_java_react/src/chart.rs @@ -0,0 +1,378 @@ +//! Generate a helm chart from an imported [`ComposeApp`]. +//! +//! Typed `k8s_openapi` resources serialized at build time (ADR-018, the +//! fleet operator chart pattern) — no `{{ .Values }}` templating. One +//! Deployment + Service per compose service, one RWX PVC per mounted +//! named volume. Ingress is layered at deploy time by the +//! [`Score`](crate::score), not baked here, so the host stays a deploy +//! knob. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use k8s_openapi::api::apps::v1::{ + Deployment, DeploymentSpec, DeploymentStrategy, RollingUpdateDeployment, +}; +use k8s_openapi::api::core::v1::{ + Container, ContainerPort, EnvVar, PersistentVolumeClaim, PersistentVolumeClaimSpec, + PersistentVolumeClaimVolumeSource, PodSpec, PodTemplateSpec, Service, ServicePort, ServiceSpec, + Volume, VolumeMount, VolumeResourceRequirements, +}; +use k8s_openapi::apimachinery::pkg::api::resource::Quantity; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; + +use harmony::modules::application::helm::{HelmChart, HelmResourceKind}; + +use crate::compose::{ComposeApp, ComposeService}; + +/// Deploy-only knobs (ADR-026 §6) — the typed home for everything the +/// compose file deliberately does not carry. +#[derive(Debug, Clone, serde::Serialize)] +pub struct DeployConfig { + pub app_name: String, + pub chart_version: String, + /// Image coordinates for services that build from source. + pub registry: String, + pub project: String, + pub version: String, + /// RWX-capable storage class for PVCs (e.g. ceph `cephfs`). `None` + /// uses the cluster default — only correct if that default is RWX. + pub storage_class: Option, + pub volume_size: String, + pub replicas: i32, + /// `true` → RollingUpdate (needs RWX so old+new pods share volumes); + /// `false` → Recreate (safe for single-writer stores like sqlite). + pub rolling: bool, +} + +const ACCESS_MODE_RWX: &str = "ReadWriteMany"; + +/// The published image ref a service's Deployment should pull: built +/// services get our registry coordinates, prebuilt `image:` services are +/// used verbatim. +pub fn service_image(cfg: &DeployConfig, svc: &ComposeService) -> String { + if svc.build_context.is_some() { + format!( + "{}/{}/{}-{}:{}", + cfg.registry, cfg.project, cfg.app_name, svc.name, cfg.version + ) + } else { + svc.image + .clone() + .expect("validated: image or build present") + } +} + +/// Build + write the chart to `out_dir`; returns the chart directory +/// `helm install ` wants. +pub fn build_chart( + app: &ComposeApp, + cfg: &DeployConfig, + out_dir: &Path, +) -> Result { + let mut chart = HelmChart::new(cfg.app_name.clone(), cfg.version.clone()); + chart.version = cfg.chart_version.clone(); + chart.description = format!("{} — imported from docker-compose by Harmony", cfg.app_name); + + for svc in &app.services { + chart.add_resource( + HelmResourceKind::from_serializable( + format!("deployment-{}.yaml", svc.name), + &deployment(svc, cfg), + ) + .map_err(|e| format!("serialize deployment {}: {e}", svc.name))?, + ); + if let Some(service) = service(svc) { + chart.add_resource( + HelmResourceKind::from_serializable(format!("service-{}.yaml", svc.name), &service) + .map_err(|e| format!("serialize service {}: {e}", svc.name))?, + ); + } + } + + for vol in app.mounted_volumes() { + chart.add_resource( + HelmResourceKind::from_serializable(format!("pvc-{vol}.yaml"), &pvc(&vol, cfg)) + .map_err(|e| format!("serialize pvc {vol}: {e}"))?, + ); + } + + chart + .write_to(out_dir) + .map_err(|e| format!("writing chart: {e}")) +} + +fn labels(name: &str) -> BTreeMap { + BTreeMap::from([ + ("app.kubernetes.io/name".to_string(), name.to_string()), + ( + "app.kubernetes.io/managed-by".to_string(), + "harmony".to_string(), + ), + ]) +} + +fn deployment(svc: &ComposeService, cfg: &DeployConfig) -> Deployment { + let labels = labels(&svc.name); + + let ports: Vec = svc + .ports + .iter() + .map(|p| ContainerPort { + container_port: i32::from(p.container), + protocol: Some(p.protocol.clone()), + ..Default::default() + }) + .collect(); + + let env: Vec = svc + .env + .iter() + .map(|(k, v)| EnvVar { + name: k.clone(), + value: Some(v.clone()), + ..Default::default() + }) + .collect(); + + let volume_mounts: Vec = svc + .mounts + .iter() + .map(|m| VolumeMount { + name: m.volume.clone(), + mount_path: m.path.clone(), + ..Default::default() + }) + .collect(); + + let volumes: Vec = svc + .mounts + .iter() + .map(|m| Volume { + name: m.volume.clone(), + persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource { + claim_name: m.volume.clone(), + ..Default::default() + }), + ..Default::default() + }) + .collect(); + + let strategy = Some(if cfg.rolling { + DeploymentStrategy { + type_: Some("RollingUpdate".to_string()), + rolling_update: Some(RollingUpdateDeployment { + max_surge: Some(IntOrString::Int(1)), + max_unavailable: Some(IntOrString::Int(0)), + }), + } + } else { + DeploymentStrategy { + type_: Some("Recreate".to_string()), + rolling_update: None, + } + }); + + Deployment { + metadata: ObjectMeta { + name: Some(svc.name.clone()), + labels: Some(labels.clone()), + ..Default::default() + }, + spec: Some(DeploymentSpec { + replicas: Some(cfg.replicas), + strategy, + selector: LabelSelector { + match_labels: Some(labels.clone()), + ..Default::default() + }, + template: PodTemplateSpec { + metadata: Some(ObjectMeta { + labels: Some(labels), + ..Default::default() + }), + spec: Some(PodSpec { + containers: vec![Container { + name: svc.name.clone(), + image: Some(service_image(cfg, svc)), + image_pull_policy: Some("IfNotPresent".to_string()), + ports: (!ports.is_empty()).then_some(ports), + env: (!env.is_empty()).then_some(env), + volume_mounts: (!volume_mounts.is_empty()).then_some(volume_mounts), + ..Default::default() + }], + volumes: (!volumes.is_empty()).then_some(volumes), + ..Default::default() + }), + }, + ..Default::default() + }), + ..Default::default() + } +} + +fn service(svc: &ComposeService) -> Option { + if svc.ports.is_empty() { + return None; + } + let labels = labels(&svc.name); + let ports = svc + .ports + .iter() + .map(|p| ServicePort { + name: Some(format!("port-{}", p.container)), + port: i32::from(p.container), + target_port: Some(IntOrString::Int(i32::from(p.container))), + protocol: Some(p.protocol.clone()), + ..Default::default() + }) + .collect(); + Some(Service { + metadata: ObjectMeta { + name: Some(svc.name.clone()), + labels: Some(labels.clone()), + ..Default::default() + }, + spec: Some(ServiceSpec { + type_: Some("ClusterIP".to_string()), + selector: Some(labels), + ports: Some(ports), + ..Default::default() + }), + ..Default::default() + }) +} + +fn pvc(name: &str, cfg: &DeployConfig) -> PersistentVolumeClaim { + PersistentVolumeClaim { + metadata: ObjectMeta { + name: Some(name.to_string()), + labels: Some(labels(name)), + ..Default::default() + }, + spec: Some(PersistentVolumeClaimSpec { + access_modes: Some(vec![ACCESS_MODE_RWX.to_string()]), + storage_class_name: cfg.storage_class.clone(), + resources: Some(VolumeResourceRequirements { + requests: Some(BTreeMap::from([( + "storage".to_string(), + Quantity(cfg.volume_size.clone()), + )])), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + fn fixture() -> (ComposeApp, DeployConfig, tempfile::TempDir) { + let yaml = "name: t\nservices:\n backend:\n build: ./backend\n ports: [\"8080:8080\"]\n volumes:\n - data:/data\n db:\n image: postgres:16\nvolumes:\n data:\n"; + let app = ComposeApp::parse(yaml, Path::new("/p")).unwrap(); + let cfg = DeployConfig { + app_name: "timesheet".to_string(), + chart_version: "0.1.0".to_string(), + registry: "hub.example".to_string(), + project: "harmony".to_string(), + version: "1.2.3".to_string(), + storage_class: Some("cephfs".to_string()), + volume_size: "2Gi".to_string(), + replicas: 2, + rolling: true, + }; + let tmp = tempfile::tempdir().unwrap(); + build_chart(&app, &cfg, tmp.path()).unwrap(); + (app, cfg, tmp) + } + + fn read(tmp: &tempfile::TempDir, file: &str) -> String { + std::fs::read_to_string(tmp.path().join("timesheet/templates").join(file)).unwrap() + } + + #[test] + fn built_service_gets_registry_image_prebuilt_stays_verbatim() { + let cfg = fixture().1; + let app = ComposeApp::parse( + "services:\n backend:\n build: .\n db:\n image: postgres:16\n", + Path::new("/p"), + ) + .unwrap(); + assert_eq!( + service_image(&cfg, app.service("backend").unwrap()), + "hub.example/harmony/timesheet-backend:1.2.3" + ); + assert_eq!( + service_image(&cfg, app.service("db").unwrap()), + "postgres:16" + ); + } + + #[test] + fn each_service_renders_its_own_deployment_file() { + let (_, _, tmp) = fixture(); + // Per-service filenames — a shared "deployment.yaml" would collide. + assert!( + tmp.path() + .join("timesheet/templates/deployment-backend.yaml") + .exists() + ); + assert!( + tmp.path() + .join("timesheet/templates/deployment-db.yaml") + .exists() + ); + } + + #[test] + fn portless_service_emits_no_service_object() { + let (_, _, tmp) = fixture(); + assert!( + tmp.path() + .join("timesheet/templates/service-backend.yaml") + .exists() + ); + assert!( + !tmp.path() + .join("timesheet/templates/service-db.yaml") + .exists() + ); + } + + #[test] + fn pvc_is_readwritemany_with_storage_class() { + let (_, _, tmp) = fixture(); + let pvc = read(&tmp, "pvc-data.yaml"); + assert!(pvc.contains("ReadWriteMany"), "{pvc}"); + assert!(pvc.contains("cephfs"), "{pvc}"); + assert!(pvc.contains("2Gi"), "{pvc}"); + } + + #[test] + fn rolling_strategy_and_replicas_render() { + let (_, _, tmp) = fixture(); + let dep = read(&tmp, "deployment-backend.yaml"); + assert!(dep.contains("RollingUpdate"), "{dep}"); + assert!(dep.contains("replicas: 2"), "{dep}"); + assert!(dep.contains("claimName: data"), "{dep}"); + } + + #[test] + fn recreate_strategy_when_not_rolling() { + let yaml = "services:\n a:\n build: .\n"; + let app = ComposeApp::parse(yaml, Path::new("/p")).unwrap(); + let mut cfg = fixture().1; + cfg.rolling = false; + let tmp = tempfile::tempdir().unwrap(); + build_chart(&app, &cfg, tmp.path()).unwrap(); + let dep = std::fs::read_to_string(tmp.path().join("timesheet/templates/deployment-a.yaml")) + .unwrap(); + assert!(dep.contains("Recreate"), "{dep}"); + } +} diff --git a/examples/compose_java_react/src/compose.rs b/examples/compose_java_react/src/compose.rs new file mode 100644 index 00000000..19f176b2 --- /dev/null +++ b/examples/compose_java_react/src/compose.rs @@ -0,0 +1,388 @@ +//! Import an existing `docker-compose.yml` into a typed model. +//! +//! The compose file is the source of truth for the app's *base* shape — +//! images, ports, env, volumes (ADR-026). We translate only that subset +//! into typed Rust at the boundary; deploy-only knobs (replicas, storage +//! class, ingress host, rolling strategy) are NOT read from here — they +//! live in the deploy [`Score`](crate::score). This is an adapter, like +//! reading a Dockerfile, not a YAML deployment interface. +//! +//! What we deliberately do NOT support is rejected loudly (bind mounts, +//! unparseable ports) or warned about (depends_on, networks, …) — a +//! deploy tool must never silently drop config it can't honor. + +use std::path::{Path, PathBuf}; + +use docker_compose_types::{Compose, Environment, Ports, PublishedPort, Volumes}; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ComposeError { + #[error("reading {path}: {source}")] + Read { + path: PathBuf, + source: std::io::Error, + }, + #[error("parsing compose yaml: {0}")] + Parse(#[from] serde_yaml::Error), + #[error("service '{service}' has neither `image` nor `build` — nothing to deploy")] + NoImageOrBuild { service: String }, + #[error("service '{service}': port '{spec}' is not parseable")] + BadPort { service: String, spec: String }, + #[error( + "service '{service}': volume '{spec}' is a bind mount; only named volumes map to a k8s PVC" + )] + BindMount { service: String, spec: String }, + #[error("service '{service}': volume source '{volume}' is not a named top-level volume")] + UnknownVolume { service: String, volume: String }, +} + +/// The imported app: an ordered list of services plus the named volumes +/// declared at the compose top level. +#[derive(Debug, Clone, serde::Serialize)] +pub struct ComposeApp { + pub services: Vec, + pub named_volumes: Vec, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct ComposeService { + pub name: String, + /// Explicit prebuilt image (e.g. `postgres:16`) — used as-is, never + /// built. Mutually informative with `build_context`. + pub image: Option, + /// Build context dir (resolved relative to the compose file) — when + /// set, `publish` builds + pushes the image under our registry. + pub build_context: Option, + pub dockerfile: Option, + pub ports: Vec, + pub env: Vec<(String, String)>, + pub mounts: Vec, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct PortMap { + pub container: u16, + pub published: Option, + pub protocol: String, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct VolumeMount { + pub volume: String, + pub path: String, +} + +impl ComposeApp { + /// Parse `/docker-compose.yml` (falling back to `.yaml`). + pub fn from_dir(dir: &Path) -> Result { + let path = ["docker-compose.yml", "docker-compose.yaml", "compose.yaml"] + .iter() + .map(|f| dir.join(f)) + .find(|p| p.exists()) + .unwrap_or_else(|| dir.join("docker-compose.yml")); + let raw = std::fs::read_to_string(&path).map_err(|source| ComposeError::Read { + path: path.clone(), + source, + })?; + Self::parse(&raw, dir) + } + + /// Parse compose YAML, resolving build contexts relative to `base_dir`. + pub fn parse(raw: &str, base_dir: &Path) -> Result { + let compose: Compose = serde_yaml::from_str(raw)?; + let named_volumes: Vec = compose.volumes.0.keys().cloned().collect(); + + let mut services = Vec::new(); + for (name, svc) in compose.services.0 { + let svc = svc.unwrap_or_default(); + + let build_context = match &svc.build_ { + Some(docker_compose_types::BuildStep::Simple(ctx)) => Some(base_dir.join(ctx)), + Some(docker_compose_types::BuildStep::Advanced(a)) => { + Some(base_dir.join(&a.context)) + } + None => None, + }; + let dockerfile = match &svc.build_ { + Some(docker_compose_types::BuildStep::Advanced(a)) => a.dockerfile.clone(), + _ => None, + }; + if svc.image.is_none() && build_context.is_none() { + return Err(ComposeError::NoImageOrBuild { service: name }); + } + + warn_ignored(&name, &svc); + + services.push(ComposeService { + ports: parse_ports(&name, &svc.ports)?, + env: parse_env(&svc.environment), + mounts: parse_volumes(&name, &svc.volumes, &named_volumes)?, + image: svc.image, + build_context, + dockerfile, + name, + }); + } + + Ok(Self { + services, + named_volumes, + }) + } + + pub fn service(&self, name: &str) -> Option<&ComposeService> { + self.services.iter().find(|s| s.name == name) + } + + /// Named volumes actually referenced by a mount — the ones that need + /// a PVC. A declared-but-unmounted volume needs no k8s resource. + pub fn mounted_volumes(&self) -> Vec { + self.named_volumes + .iter() + .filter(|v| { + self.services + .iter() + .any(|s| s.mounts.iter().any(|m| &&m.volume == v)) + }) + .cloned() + .collect() + } +} + +fn parse_ports(service: &str, ports: &Ports) -> Result, ComposeError> { + match ports { + Ports::Short(specs) => specs.iter().map(|s| parse_short_port(service, s)).collect(), + Ports::Long(ports) => Ok(ports + .iter() + .map(|p| PortMap { + container: p.target, + published: match &p.published { + Some(PublishedPort::Single(n)) => Some(*n), + _ => None, + }, + protocol: p.protocol.clone().unwrap_or_else(|| "TCP".to_string()), + }) + .collect()), + } +} + +/// `"8081:80"`, `"80"`, `"127.0.0.1:8080:80"`, `"80/udp"` → the last +/// colon-token is the container port, the previous (if any) the host +/// port, a host IP before that is ignored (k8s has no host binding). +fn parse_short_port(service: &str, spec: &str) -> Result { + let bad = || ComposeError::BadPort { + service: service.to_string(), + spec: spec.to_string(), + }; + let (ports, protocol) = match spec.split_once('/') { + Some((p, proto)) => (p, proto.to_uppercase()), + None => (spec, "TCP".to_string()), + }; + let tokens: Vec<&str> = ports.split(':').collect(); + let container = tokens.last().ok_or_else(bad)?.parse().map_err(|_| bad())?; + let published = match tokens.len() { + 1 => None, + n => tokens[n - 2].parse().ok(), + }; + Ok(PortMap { + container, + published, + protocol, + }) +} + +fn parse_env(env: &Environment) -> Vec<(String, String)> { + match env { + Environment::List(items) => items + .iter() + .map(|item| match item.split_once('=') { + Some((k, v)) => (k.to_string(), v.to_string()), + None => (item.to_string(), String::new()), + }) + .collect(), + Environment::KvPair(map) => map + .iter() + .map(|(k, v)| { + let v = v.as_ref().map(|s| s.to_string()).unwrap_or_default(); + (k.clone(), v) + }) + .collect(), + } +} + +fn parse_volumes( + service: &str, + volumes: &[Volumes], + named: &[String], +) -> Result, ComposeError> { + let mut mounts = Vec::new(); + for vol in volumes { + let (source, target) = match vol { + Volumes::Simple(spec) => { + let (source, target) = + spec.split_once(':') + .ok_or_else(|| ComposeError::BindMount { + service: service.to_string(), + spec: spec.clone(), + })?; + // strip a trailing `:ro`/`:rw` mode if present. + let target = target.split(':').next().unwrap_or(target); + (source.to_string(), target.to_string()) + } + Volumes::Advanced(a) => (a.source.clone().unwrap_or_default(), a.target.clone()), + }; + // A path-like source is a bind mount — not portable to a cluster. + if source.starts_with('.') || source.starts_with('/') { + return Err(ComposeError::BindMount { + service: service.to_string(), + spec: source, + }); + } + if !named.contains(&source) { + return Err(ComposeError::UnknownVolume { + service: service.to_string(), + volume: source, + }); + } + mounts.push(VolumeMount { + volume: source, + path: target, + }); + } + Ok(mounts) +} + +/// Compose keys we recognize but don't translate. A deploy tool that +/// drops these silently is a footgun, so we say so. +fn warn_ignored(name: &str, svc: &docker_compose_types::Service) { + if !svc.depends_on.is_empty() { + log::warn!( + "service '{name}': `depends_on` ignored — k8s ordering is via readiness, not startup order" + ); + } + if svc.command.is_some() { + log::warn!("service '{name}': `command` override ignored — bake it into the image"); + } + if svc.restart.is_some() { + log::warn!("service '{name}': `restart` ignored — k8s restarts via the Deployment policy"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + const COMPOSE: &str = r#" +name: timesheet +services: + backend: + build: ./backend + ports: ["8080:8080"] + environment: + - DB_PATH=/data/timesheet.db + volumes: + - timesheet-data:/data + frontend: + build: ./frontend + ports: ["8081:80"] + environment: + - BACKEND_URL=http://backend:8080 + depends_on: + - backend +volumes: + timesheet-data: +"#; + + fn app() -> ComposeApp { + ComposeApp::parse(COMPOSE, Path::new("/proj")).expect("parse") + } + + #[test] + fn imports_both_services_in_order() { + let app = app(); + let names: Vec<&str> = app.services.iter().map(|s| s.name.as_str()).collect(); + assert_eq!(names, ["backend", "frontend"]); + } + + #[test] + fn build_context_resolves_against_base_dir() { + let backend = app().service("backend").unwrap().clone(); + assert_eq!( + backend.build_context, + Some(PathBuf::from("/proj/./backend")) + ); + assert!(backend.image.is_none()); + } + + #[test] + fn ports_split_published_and_container() { + let app = app(); + let be = &app.service("backend").unwrap().ports[0]; + assert_eq!((be.container, be.published), (8080, Some(8080))); + let fe = &app.service("frontend").unwrap().ports[0]; + assert_eq!((fe.container, fe.published), (80, Some(8081))); + } + + #[test] + fn env_parsed_as_key_value() { + let backend = app().service("backend").unwrap().clone(); + assert_eq!( + backend.env, + vec![("DB_PATH".to_string(), "/data/timesheet.db".to_string())] + ); + } + + #[test] + fn named_volume_becomes_a_mount_and_only_mounted_volumes_need_pvcs() { + let app = app(); + let m = &app.service("backend").unwrap().mounts[0]; + assert_eq!( + (m.volume.as_str(), m.path.as_str()), + ("timesheet-data", "/data") + ); + assert_eq!(app.mounted_volumes(), vec!["timesheet-data".to_string()]); + } + + #[test] + fn short_port_forms_parse() { + assert_eq!(parse_short_port("s", "80").unwrap().container, 80); + let hostip = parse_short_port("s", "127.0.0.1:8080:80").unwrap(); + assert_eq!((hostip.container, hostip.published), (80, Some(8080))); + let udp = parse_short_port("s", "53:53/udp").unwrap(); + assert_eq!(udp.protocol, "UDP"); + assert!(parse_short_port("s", "notaport").is_err()); + } + + #[test] + fn bind_mounts_are_rejected() { + let yaml = "services:\n a:\n image: x\n volumes:\n - ./host:/data\n"; + let err = ComposeApp::parse(yaml, Path::new("/p")).unwrap_err(); + assert!(matches!(err, ComposeError::BindMount { .. }), "{err}"); + } + + #[test] + fn volume_not_declared_at_top_level_is_rejected() { + let yaml = "services:\n a:\n image: x\n volumes:\n - ghost:/data\n"; + let err = ComposeApp::parse(yaml, Path::new("/p")).unwrap_err(); + assert!(matches!(err, ComposeError::UnknownVolume { .. }), "{err}"); + } + + #[test] + fn service_without_image_or_build_is_rejected() { + let yaml = "services:\n a:\n ports: [\"80\"]\n"; + let err = ComposeApp::parse(yaml, Path::new("/p")).unwrap_err(); + assert!(matches!(err, ComposeError::NoImageOrBuild { .. }), "{err}"); + } + + /// The example's real compose must stay importable — it's the demo's + /// source of truth and a contract for the chart tests. + #[test] + fn the_examples_compose_file_imports() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("app"); + let app = ComposeApp::from_dir(&dir).expect("import example compose"); + assert_eq!(app.services.len(), 2); + assert_eq!(app.mounted_volumes(), vec!["timesheet-data".to_string()]); + } +} diff --git a/examples/compose_java_react/src/lib.rs b/examples/compose_java_react/src/lib.rs new file mode 100644 index 00000000..9b648ed1 --- /dev/null +++ b/examples/compose_java_react/src/lib.rs @@ -0,0 +1,40 @@ +//! Deploy a Java+React app to Kubernetes by importing its existing +//! `docker-compose.yml` (ADR-026). The compose file is the source of +//! truth for the app's base shape; deploy-only knobs live in +//! [`DeployConfig`]/[`ComposeAppScore`]. Build/publish/deploy follow the +//! fleet pattern — no `ApplicationScore`, no ArgoCD. + +pub mod chart; +pub mod compose; +pub mod publish; +pub mod score; + +pub use chart::DeployConfig; +pub use compose::ComposeApp; +pub use score::{ComposeAppScore, PublicEndpoint, PublishedChart}; + +use std::path::Path; + +use anyhow::{Context, Result}; +use serde::Deserialize; + +/// Identity-only project manifest (ADR-026 §3): the app *name*, nothing +/// behavioral. A behavioral knob here is a defect — it belongs in a Score. +#[derive(Debug, Deserialize)] +pub struct HarmonyManifest { + pub app: AppIdentity, +} + +#[derive(Debug, Deserialize)] +pub struct AppIdentity { + pub name: String, +} + +impl HarmonyManifest { + pub fn from_dir(dir: &Path) -> Result { + let path = dir.join("Harmony.toml"); + let raw = std::fs::read_to_string(&path) + .with_context(|| format!("reading {}", path.display()))?; + toml::from_str(&raw).with_context(|| format!("parsing {}", path.display())) + } +} diff --git a/examples/compose_java_react/src/main.rs b/examples/compose_java_react/src/main.rs new file mode 100644 index 00000000..f625edd6 --- /dev/null +++ b/examples/compose_java_react/src/main.rs @@ -0,0 +1,127 @@ +//! `compose-deploy` — converge the compose-imported app onto a cluster +//! (`helm upgrade --install`). The same Scores run against local k3d and +//! a remote tenant; only the topology (selected by env) changes. + +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use clap::Parser; +use harmony::inventory::Inventory; +use harmony::topology::K8sAnywhereTopology; +use harmony_cli::Args as HarmonyCliArgs; + +use example_compose_java_react::{ + ComposeApp, ComposeAppScore, DeployConfig, HarmonyManifest, PublicEndpoint, PublishedChart, +}; + +#[derive(Parser, Debug)] +#[command( + name = "compose-deploy", + about = "Deploy a docker-compose app to k8s via Harmony" +)] +struct Cli { + /// Project root holding `Harmony.toml` (defaults to this example). + #[arg(long, default_value = env!("CARGO_MANIFEST_DIR"))] + project_dir: PathBuf, + /// Directory holding `docker-compose.yml` (defaults to `/app`). + #[arg(long)] + compose_dir: Option, + #[arg(long, default_value = "timesheet")] + namespace: String, + #[arg(long, default_value = "hub.nationtech.io")] + registry: String, + #[arg(long, default_value = "harmony")] + project: String, + #[arg(long, default_value = "0.1.0")] + version: String, + /// RWX storage class (e.g. ceph `cephfs`). Omit for the cluster default. + #[arg(long)] + storage_class: Option, + #[arg(long, default_value = "1Gi")] + volume_size: String, + #[arg(long, default_value_t = 2)] + replicas: i32, + /// Recreate instead of RollingUpdate — safe for single-writer sqlite. + #[arg(long)] + recreate: bool, + /// Install the published OCI chart instead of rendering from compose. + #[arg(long)] + published: bool, + /// Compose service to expose via Ingress. + #[arg(long, default_value = "frontend")] + public_service: String, + /// Ingress host; omit to keep the app cluster-internal. + #[arg(long)] + host: Option, + /// cert-manager ClusterIssuer for TLS; omit for plain HTTP. + #[arg(long)] + cluster_issuer: Option, + #[command(flatten)] + harmony_cli: HarmonyCliArgs, +} + +#[tokio::main] +async fn main() -> Result<()> { + harmony_cli::cli_logger::init(); + let cli = Cli::parse(); + + let manifest = HarmonyManifest::from_dir(&cli.project_dir)?; + let compose_dir = cli + .compose_dir + .unwrap_or_else(|| cli.project_dir.join("app")); + let app = ComposeApp::from_dir(&compose_dir).context("importing docker-compose")?; + + let deploy = DeployConfig { + app_name: manifest.app.name.clone(), + chart_version: cli.version.clone(), + registry: cli.registry.clone(), + project: cli.project.clone(), + version: cli.version.clone(), + storage_class: cli.storage_class, + volume_size: cli.volume_size, + replicas: cli.replicas, + rolling: !cli.recreate, + }; + + // The host is a deploy-only knob; the port is read from the compose + // service so it stays the single source of truth for the app shape. + let public_endpoint = cli.host.map(|host| -> Result { + let svc = app + .service(&cli.public_service) + .with_context(|| format!("no compose service '{}'", cli.public_service))?; + let port = svc + .ports + .first() + .with_context(|| format!("service '{}' exposes no port", cli.public_service))? + .container; + Ok(PublicEndpoint { + service: cli.public_service.clone(), + port, + host, + cluster_issuer: cli.cluster_issuer, + }) + }); + let public_endpoint = public_endpoint.transpose()?; + + let score = ComposeAppScore { + namespace: cli.namespace, + release_name: manifest.app.name, + published_chart: cli.published.then_some(PublishedChart { + registry: cli.registry, + project: cli.project, + version: cli.version, + }), + public_endpoint, + app, + deploy, + }; + + harmony_cli::run( + Inventory::autoload(), + K8sAnywhereTopology::from_env(), + vec![Box::new(score)], + Some(cli.harmony_cli), + ) + .await + .map_err(|e| anyhow::anyhow!("{e}")) +} diff --git a/examples/compose_java_react/src/publish.rs b/examples/compose_java_react/src/publish.rs new file mode 100644 index 00000000..d1f4a8b0 --- /dev/null +++ b/examples/compose_java_react/src/publish.rs @@ -0,0 +1,96 @@ +//! Build + push the per-service images and the hydrated chart. +//! +//! The fleet `release.rs` pattern, generalized over the compose +//! services: each service that builds from source is built then pushed +//! with docker; prebuilt `image:` services are left alone; then the +//! chart is hydrated, `helm package`d, and pushed to the OCI registry. +//! Plain `anyhow` — this is binary glue, not library API. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result, bail}; + +use crate::chart::{DeployConfig, build_chart, service_image}; +use crate::compose::ComposeApp; + +/// Build + push every built service image and the chart. With +/// `push = false`, builds + packages only (a local k3d smoke-test). +/// `docker` and `helm` must be on PATH and logged in to the registry. +pub fn publish(app: &ComposeApp, cfg: &DeployConfig, push: bool) -> Result<()> { + for svc in &app.services { + let Some(context) = &svc.build_context else { + log::info!("service '{}' uses prebuilt image, skipping build", svc.name); + continue; + }; + let image = service_image(cfg, svc); + log::info!("docker build {image}"); + let mut cmd = Command::new("docker"); + cmd.arg("build").arg("-t").arg(&image); + if let Some(dockerfile) = &svc.dockerfile { + cmd.arg("-f").arg(context.join(dockerfile)); + } + cmd.arg(context); + run(cmd)?; + if push { + log::info!("docker push {image}"); + run({ + let mut c = Command::new("docker"); + c.arg("push").arg(&image); + c + })?; + } + } + + let tmp = tempfile::tempdir().context("chart tempdir")?; + let chart_dir = + build_chart(app, cfg, tmp.path()).map_err(|e| anyhow::anyhow!("hydrate chart: {e}"))?; + let tgz = helm_package(&chart_dir, tmp.path())?; + if push { + let oci_repo = format!("oci://{}/{}", cfg.registry, cfg.project); + log::info!("helm push {} {oci_repo}", tgz.display()); + run({ + let mut c = Command::new("helm"); + c.arg("push").arg(&tgz).arg(&oci_repo); + c + })?; + } + + log::info!( + "published app={} version={} pushed={push}", + cfg.app_name, + cfg.version + ); + Ok(()) +} + +fn run(mut cmd: Command) -> Result<()> { + let status = cmd + .status() + .with_context(|| format!("spawn {cmd:?} (is it installed and in PATH?)"))?; + if !status.success() { + bail!("`{cmd:?}` failed ({status})"); + } + Ok(()) +} + +fn helm_package(chart_dir: &Path, out_dir: &Path) -> Result { + let output = Command::new("helm") + .args(["package", path_str(chart_dir)?, "-d", path_str(out_dir)?]) + .output() + .context("spawn helm package")?; + if !output.status.success() { + bail!("helm package failed ({})", output.status); + } + // helm prints "…saved it to: "; the last token is the path. + String::from_utf8_lossy(&output.stdout) + .split_whitespace() + .last() + .map(PathBuf::from) + .context("helm package printed no path") +} + +fn path_str(p: &Path) -> Result<&str> { + p.to_str() + .with_context(|| format!("path not utf-8: {}", p.display())) +} diff --git a/examples/compose_java_react/src/score.rs b/examples/compose_java_react/src/score.rs new file mode 100644 index 00000000..f24b39be --- /dev/null +++ b/examples/compose_java_react/src/score.rs @@ -0,0 +1,197 @@ +//! [`ComposeAppScore`] — converge a compose-imported app onto a cluster. +//! +//! Mirrors `FleetOperatorScore`: render a self-contained helm chart from +//! the imported compose at interpret time (or install an already-published +//! OCI chart on the CD path), `helm upgrade --install` via +//! [`HelmChartScore`], then layer the public Ingress on top. No +//! `ApplicationScore`, no ArgoCD — the same build/publish/deploy split the +//! fleet stack uses. + +use std::str::FromStr; + +use async_trait::async_trait; +use harmony_types::id::Id; +use log::info; +use serde::Serialize; + +use harmony::data::Version; +use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}; +use harmony::inventory::Inventory; +use harmony::modules::helm::chart::{HelmChartScore, NonBlankString}; +use harmony::modules::k8s::ingress::K8sIngressScore; +use harmony::score::Score; +use harmony::topology::{HelmCommand, K8sclient, Topology}; + +use crate::chart::{DeployConfig, build_chart}; +use crate::compose::ComposeApp; + +/// The already-published OCI chart to install (the CD path). When set, +/// the chart is pulled from the registry instead of rendered locally. +#[derive(Debug, Clone, Serialize)] +pub struct PublishedChart { + pub registry: String, + pub project: String, + pub version: String, +} + +/// Expose one service through an Ingress (a deploy-only knob — the host +/// is never read from compose). +#[derive(Debug, Clone, Serialize)] +pub struct PublicEndpoint { + /// Compose service name to route to (its Service + port). + pub service: String, + pub port: u16, + pub host: String, + /// cert-manager ClusterIssuer for TLS; `None` serves plain HTTP + /// (the right default on issuer-less k3d). + pub cluster_issuer: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ComposeAppScore { + pub namespace: String, + pub release_name: String, + pub app: ComposeApp, + pub deploy: DeployConfig, + /// `None` renders + installs the chart from the imported compose + /// (dev/e2e); `Some` installs the published OCI chart (CD). + pub published_chart: Option, + pub public_endpoint: Option, +} + +impl Score for ComposeAppScore { + fn create_interpret(&self) -> Box> { + Box::new(ComposeAppInterpret { + score: self.clone(), + }) + } + + fn name(&self) -> String { + format!("ComposeAppScore({})", self.deploy.app_name) + } +} + +#[derive(Debug)] +struct ComposeAppInterpret { + score: ComposeAppScore, +} + +#[async_trait] +impl Interpret for ComposeAppInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + let s = &self.score; + + // Install from the published OCI chart (CD) or a chart rendered + // from the imported compose (dev/e2e). Each branch keeps its + // tempdir alive across the install. + let helm_outcome = if let Some(p) = &s.published_chart { + let chart_ref = format!("oci://{}/{}/{}", p.registry, p.project, s.deploy.app_name); + info!( + "Installing '{}' from published chart {chart_ref}:{}", + s.release_name, p.version + ); + self.install(inventory, topology, &chart_ref, &p.version) + .await? + } else { + let tmp = tempfile::tempdir() + .map_err(|e| InterpretError::new(format!("chart tempdir: {e}")))?; + let chart_path = build_chart(&s.app, &s.deploy, tmp.path()) + .map_err(|e| InterpretError::new(format!("render chart: {e}")))?; + let chart_path = chart_path + .to_str() + .ok_or_else(|| InterpretError::new("chart path not utf-8".to_string()))?; + info!( + "Installing '{}' from rendered chart {chart_path}", + s.release_name + ); + self.install(inventory, topology, chart_path, &s.deploy.chart_version) + .await? + }; + + // Public Ingress, applied after the chart so the backing Service + // exists. Skipped when no endpoint is configured. + if let Some(ep) = &s.public_endpoint { + let to_fqdn = |v: &str| { + fqdn::FQDN::from_str(v) + .map_err(|e| InterpretError::new(format!("invalid fqdn '{v}': {e}"))) + }; + let scheme = if ep.cluster_issuer.is_some() { + "https" + } else { + "http" + }; + info!( + "Exposing service '{}' at {scheme}://{}", + ep.service, ep.host + ); + K8sIngressScore { + name: to_fqdn(&s.deploy.app_name)?, + host: to_fqdn(&ep.host)?, + backend_service: to_fqdn(&ep.service)?, + port: ep.port, + path: None, + path_type: None, + namespace: Some(to_fqdn(&s.namespace)?), + ingress_class_name: None, + cluster_issuer: ep.cluster_issuer.clone(), + } + .interpret(inventory, topology) + .await?; + } + + Ok(Outcome::success_with_details( + helm_outcome.message, + vec![format!("app: {}", s.deploy.app_name)], + )) + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("ComposeAppInterpret") + } + + fn get_version(&self) -> Version { + Version::from("0.1.0").expect("static version literal") + } + + fn get_status(&self) -> InterpretStatus { + InterpretStatus::QUEUED + } + + fn get_children(&self) -> Vec { + vec![] + } +} + +impl ComposeAppInterpret { + async fn install( + &self, + inventory: &Inventory, + topology: &T, + chart_name: &str, + version: &str, + ) -> Result { + let s = &self.score; + HelmChartScore { + namespace: Some(non_blank(&s.namespace, "namespace")?), + release_name: non_blank(&s.release_name, "release_name")?, + chart_name: non_blank(chart_name, "chart_name")?, + chart_version: Some(non_blank(version, "chart_version")?), + values_overrides: None, + values_yaml: None, + create_namespace: true, + install_only: false, + repository: None, + } + .interpret(inventory, topology) + .await + } +} + +fn non_blank(value: &str, field: &str) -> Result { + NonBlankString::from_str(value) + .map_err(|e| InterpretError::new(format!("{field} must be non-blank: {e}"))) +} diff --git a/examples/compose_java_react/tests/helm_render.rs b/examples/compose_java_react/tests/helm_render.rs new file mode 100644 index 00000000..07326a37 --- /dev/null +++ b/examples/compose_java_react/tests/helm_render.rs @@ -0,0 +1,69 @@ +//! Render the example app's chart from its real `docker-compose.yml` and +//! validate it with actual `helm` — the chart our importer emits must be +//! something helm accepts and that renders to valid k8s. Skips when helm +//! isn't on PATH so CI without helm stays green. + +use std::path::Path; +use std::process::Command; + +use example_compose_java_react::{ComposeApp, DeployConfig, chart::build_chart}; + +fn helm_present() -> bool { + Command::new("helm") + .arg("version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +#[test] +fn generated_chart_passes_helm_lint_and_template() { + if !helm_present() { + eprintln!("helm not on PATH — skipping chart validation"); + return; + } + + let app_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("app"); + let app = ComposeApp::from_dir(&app_dir).expect("import compose"); + let cfg = DeployConfig { + app_name: "timesheet".to_string(), + chart_version: "0.1.0".to_string(), + registry: "hub.example".to_string(), + project: "harmony".to_string(), + version: "0.1.0".to_string(), + storage_class: Some("cephfs".to_string()), + volume_size: "1Gi".to_string(), + replicas: 2, + rolling: true, + }; + + let tmp = tempfile::tempdir().unwrap(); + let chart = build_chart(&app, &cfg, tmp.path()).expect("render chart"); + let chart = chart.to_str().unwrap(); + + let lint = Command::new("helm").args(["lint", chart]).output().unwrap(); + assert!( + lint.status.success(), + "helm lint failed:\n{}", + String::from_utf8_lossy(&lint.stdout) + ); + + let out = Command::new("helm") + .args(["template", "timesheet", chart]) + .output() + .unwrap(); + assert!( + out.status.success(), + "helm template failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + let rendered = String::from_utf8_lossy(&out.stdout); + // Both services + the RWX PVC must survive a real render. + assert!(rendered.contains("kind: Deployment"), "{rendered}"); + assert!(rendered.contains("kind: Service"), "{rendered}"); + assert!( + rendered.contains("kind: PersistentVolumeClaim"), + "{rendered}" + ); + assert!(rendered.contains("ReadWriteMany"), "{rendered}"); +} diff --git a/examples/fleet_auth_callout/src/lib.rs b/examples/fleet_auth_callout/src/lib.rs index c2d4c9bf..b993d0bd 100644 --- a/examples/fleet_auth_callout/src/lib.rs +++ b/examples/fleet_auth_callout/src/lib.rs @@ -267,6 +267,7 @@ pub async fn bring_up_stack() -> Result { group: None, }, ], + groups_claim_action: false, machine_users: vec![ ZitadelMachineUser { username: ADMIN_USERNAME.to_string(), @@ -759,7 +760,7 @@ pub async fn mint_access_token( access_token: String, } let tr: TokenResponse = resp.json().await.context("parse token response")?; - if std::env::var("FLEET_AUTH_CALLOUT_DEBUG_TOKENS").is_ok() + if std::env::var("HARMONY_SSO_CALLOUT_DEBUG_TOKENS").is_ok() && let Some(payload_b64) = tr.access_token.split('.').nth(1) { use base64::Engine; diff --git a/examples/fleet_device_enroll/src/main.rs b/examples/fleet_device_enroll/src/main.rs index bf50f0cf..973d323e 100644 --- a/examples/fleet_device_enroll/src/main.rs +++ b/examples/fleet_device_enroll/src/main.rs @@ -302,6 +302,7 @@ async fn main() -> Result<()> { auth, agent_binary_path: agent_binary, hosts_entries: vec![], + openbao: None, }; let setup_score = FleetDeviceSetupScore::new(setup_config); diff --git a/examples/fleet_e2e_demo/src/lib.rs b/examples/fleet_e2e_demo/src/lib.rs index d615ee1b..40b8cd19 100644 --- a/examples/fleet_e2e_demo/src/lib.rs +++ b/examples/fleet_e2e_demo/src/lib.rs @@ -185,6 +185,7 @@ pub async fn bring_up_full_stack(opts: E2eDemoOpts) -> Result { group: None, }, ], + groups_claim_action: false, machine_users: vec![ ZitadelMachineUser { username: ADMIN_USERNAME.to_string(), @@ -354,6 +355,7 @@ async fn provision_device( applications: vec![], api_apps: vec![], roles: vec![], + groups_claim_action: false, machine_users: vec![ZitadelMachineUser { username: username.clone(), name: format!("Fleet Device {device_id}"), @@ -418,6 +420,7 @@ async fn provision_device( ip: opts.libvirt_host_ip.clone(), hostname: ZITADEL_HOST.to_string(), }], + openbao: None, }); // Apply the score over SSH against the VM. Same pattern as diff --git a/examples/fleet_load_test/src/main.rs b/examples/fleet_load_test/src/main.rs index 1cfd09e7..594f8685 100644 --- a/examples/fleet_load_test/src/main.rs +++ b/examples/fleet_load_test/src/main.rs @@ -406,6 +406,7 @@ async fn apply_one_cr( let cr = Deployment::new( &group.cr_name, DeploymentSpec { + allowed_groups: None, target_selector: LabelSelector { match_labels: Some(match_labels), match_expressions: None, @@ -420,6 +421,7 @@ async fn apply_one_cr( image: "docker.io/library/nginx:alpine".to_string(), ports: vec!["8080:80".to_string()], env: vec![], + secret_env: vec![], volumes: vec![], restart_policy: Default::default(), }], diff --git a/examples/fleet_rpi_setup/src/main.rs b/examples/fleet_rpi_setup/src/main.rs index 5cbe0df3..586db968 100644 --- a/examples/fleet_rpi_setup/src/main.rs +++ b/examples/fleet_rpi_setup/src/main.rs @@ -168,6 +168,7 @@ async fn main() -> Result<()> { auth, agent_binary_path: cli.agent_binary.clone(), hosts_entries: vec![], + openbao: None, }); // We have our own clap CLI, so harmony_cli must NOT call diff --git a/examples/fleet_staging_deploy/src/lib.rs b/examples/fleet_staging_deploy/src/lib.rs index e315a46a..47b2f923 100644 --- a/examples/fleet_staging_deploy/src/lib.rs +++ b/examples/fleet_staging_deploy/src/lib.rs @@ -296,6 +296,7 @@ async fn provision_zitadel_project( // No machine users provisioned here — `fleet_rpi_setup` mints // them on demand per device, so the staging deploy stays // device-count-agnostic. + groups_claim_action: false, machine_users: vec![], }; setup diff --git a/examples/fleet_staging_install/src/main.rs b/examples/fleet_staging_install/src/main.rs index a2ffd675..c13e4f62 100644 --- a/examples/fleet_staging_install/src/main.rs +++ b/examples/fleet_staging_install/src/main.rs @@ -22,7 +22,8 @@ use harmony::modules::nats::capability::NatsCluster; use harmony::modules::nats::score_nats_k8s::{AuthCalloutCfg, NatsK8sScore, WebSocketRouteCfg}; use harmony::modules::nats_auth_callout::NatsAuthCalloutScore; use harmony::modules::openbao::{ - OpenbaoInstance, OpenbaoPolicy, OpenbaoScore, OpenbaoSetupScore, cached_root_token, + OpenbaoInstance, OpenbaoJwtAuth, OpenbaoPolicy, OpenbaoScore, OpenbaoSetupScore, + cached_root_token, }; use harmony::modules::zitadel::{ MachineKeyType, ZitadelApiApp, ZitadelAppType, ZitadelApplication, ZitadelClientConfig, @@ -33,7 +34,7 @@ use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology}; use harmony_config::{Config, ConfigClient, StoreSource}; use harmony_fleet_deploy::{FleetDeploySecrets, FleetOperatorScore, OperatorCredentials}; use harmony_k8s::KubernetesDistribution; -use harmony_secret::OpenbaoSecretStore; +use harmony_secret::{OpenbaoSecretStore, OpenbaoStoreOptions}; use nkeys::KeyPair; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -129,6 +130,10 @@ async fn main() -> Result<()> { endpoint: None, admin_org_id: None, namespace: cfg.zitadel_namespace.clone(), + // Devices read per-deployment secrets gated by group membership; + // the Action surfaces role keys as the `groups` claim OpenBao + // binds against (ADR-025). + groups_claim_action: true, // Device-code OIDC app for human admin login from // `fleet_device_enroll`'s SSO flow. The numeric `client_id` Zitadel // generates is read back below and printed for `--admin-oidc-client-id`. @@ -224,6 +229,21 @@ async fn main() -> Result<()> { path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"# .to_string(), }], + // Device login (ADR-025): batch tokens so re-login is free, 1h + // TTL as the group-membership convergence bound, groups bound + // from the Action-flattened `groups` claim. + jwt_auth: Some(OpenbaoJwtAuth { + oidc_discovery_url: format!("https://{zitadel_host}"), + jwt_validation_pubkeys: String::new(), + bound_issuer: format!("https://{zitadel_host}"), + role_name: "fleet-device".to_string(), + bound_audiences: project_id.clone(), + user_claim: "sub".to_string(), + groups_claim: "groups".to_string(), + token_type: "batch".to_string(), + ttl: "1h".to_string(), + max_ttl: "1h".to_string(), + }), ..Default::default() } .interpret(&Inventory::empty(), &topology) @@ -325,19 +345,20 @@ path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"# .await .context("port-forward to OpenBao")?; tokio::time::sleep(std::time::Duration::from_secs(1)).await; - let store = OpenbaoSecretStore::new( - format!("http://127.0.0.1:{}", pf.port()), - "secret".to_string(), - "token".to_string(), - true, - Some(cached_root_token(&openbao).map_err(|e| anyhow::anyhow!(e))?), - None, - None, - None, - None, - None, - None, - ) + let store = OpenbaoSecretStore::new(OpenbaoStoreOptions { + base_url: format!("http://127.0.0.1:{}", pf.port()), + kv_mount: "secret".to_string(), + auth_mount: "token".to_string(), + skip_tls: true, + token: Some(cached_root_token(&openbao).map_err(|e| anyhow::anyhow!(e))?), + username: None, + password: None, + zitadel_sso_url: None, + zitadel_client_id: None, + jwt_role: None, + jwt_auth_mount: None, + zitadel_jwt_bearer: None, + }) .await .context("OpenBao client")?; ConfigClient::new(vec![ diff --git a/examples/fleet_vm_setup/src/main.rs b/examples/fleet_vm_setup/src/main.rs index f3fe71e2..ef5d365c 100644 --- a/examples/fleet_vm_setup/src/main.rs +++ b/examples/fleet_vm_setup/src/main.rs @@ -219,6 +219,7 @@ async fn main() -> Result<()> { }, agent_binary_path: agent_binary, hosts_entries: vec![], + openbao: None, }); run_setup_score(&setup_score, &linux_topology).await?; diff --git a/examples/harmony_apply_deployment/src/main.rs b/examples/harmony_apply_deployment/src/main.rs index 6c46cb49..1b3dd259 100644 --- a/examples/harmony_apply_deployment/src/main.rs +++ b/examples/harmony_apply_deployment/src/main.rs @@ -204,6 +204,7 @@ fn build_cr(cli: &Cli) -> Deployment { image: cli.image.clone(), ports: vec![cli.port.clone()], env, + secret_env: vec![], volumes, restart_policy: cli.restart.into(), }], @@ -226,6 +227,7 @@ fn build_cr(cli: &Cli) -> Deployment { Deployment::new( &cli.name, DeploymentSpec { + allowed_groups: None, target_selector: LabelSelector { match_labels: Some(match_labels), match_expressions: None, diff --git a/examples/harmony_sso/src/main.rs b/examples/harmony_sso/src/main.rs index 96d70766..c15650ef 100644 --- a/examples/harmony_sso/src/main.rs +++ b/examples/harmony_sso/src/main.rs @@ -311,6 +311,7 @@ async fn main() -> anyhow::Result<()> { }], api_apps: vec![], roles: vec![], + groups_claim_action: false, machine_users: vec![], } .interpret(&Inventory::autoload(), &topology) @@ -337,11 +338,13 @@ async fn main() -> anyhow::Result<()> { }], jwt_auth: Some(OpenbaoJwtAuth { oidc_discovery_url: format!("http://{}:{}", ZITADEL_HOST, HTTP_PORT), + jwt_validation_pubkeys: String::new(), bound_issuer: format!("http://{}:{}", ZITADEL_HOST, HTTP_PORT), role_name: "harmony-developer".to_string(), bound_audiences: client_id.clone(), user_claim: "email".to_string(), - policies: vec!["harmony-dev".to_string()], + groups_claim: String::new(), + token_type: "service".to_string(), ttl: "4h".to_string(), max_ttl: "24h".to_string(), }), @@ -366,19 +369,20 @@ async fn main() -> anyhow::Result<()> { let openbao_url = format!("http://127.0.0.1:{}", _pf.port()); let sso_url = format!("http://{}:{}", ZITADEL_HOST, HTTP_PORT); - let store = OpenbaoSecretStore::new( - openbao_url, - "secret".to_string(), - "jwt".to_string(), - true, - None, - None, - None, - Some(sso_url), - Some(client_id), - Some("harmony-developer".to_string()), - Some("jwt".to_string()), - ) + let store = OpenbaoSecretStore::new(harmony_secret::OpenbaoStoreOptions { + base_url: openbao_url, + kv_mount: "secret".to_string(), + auth_mount: "jwt".to_string(), + skip_tls: true, + token: None, + username: None, + password: None, + zitadel_sso_url: Some(sso_url), + zitadel_client_id: Some(client_id), + jwt_role: Some("harmony-developer".to_string()), + jwt_auth_mount: Some("jwt".to_string()), + zitadel_jwt_bearer: None, + }) .await .context("SSO authentication failed")?; diff --git a/examples/openbao/src/main.rs b/examples/openbao/src/main.rs index f5824b3e..79197bbb 100644 --- a/examples/openbao/src/main.rs +++ b/examples/openbao/src/main.rs @@ -101,11 +101,13 @@ async fn main() -> Result<()> { let jwt_auth = (!cfg.oidc_issuer.is_empty() && !cfg.oidc_audience.is_empty()).then(|| OpenbaoJwtAuth { oidc_discovery_url: cfg.oidc_issuer.clone(), + jwt_validation_pubkeys: String::new(), bound_issuer: cfg.oidc_issuer.clone(), role_name: "harmony".to_string(), bound_audiences: cfg.oidc_audience.clone(), user_claim: "sub".to_string(), - policies: vec![HARMONY_CONFIG_POLICY.to_string()], + groups_claim: String::new(), + token_type: "service".to_string(), ttl: "1h".to_string(), max_ttl: "8h".to_string(), }); diff --git a/fleet/harmony-fleet-agent/Cargo.toml b/fleet/harmony-fleet-agent/Cargo.toml index 60c111d1..70dcb1c9 100644 --- a/fleet/harmony-fleet-agent/Cargo.toml +++ b/fleet/harmony-fleet-agent/Cargo.toml @@ -6,6 +6,7 @@ rust-version = "1.85" [dependencies] harmony-fleet-auth = { path = "../harmony-fleet-auth" } +harmony_secret = { path = "../../harmony_secret" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } harmony = { path = "../../harmony", default-features = false, features = ["podman"] } async-nats = { workspace = true } diff --git a/fleet/harmony-fleet-agent/src/main.rs b/fleet/harmony-fleet-agent/src/main.rs index dcc6b766..31886430 100644 --- a/fleet/harmony-fleet-agent/src/main.rs +++ b/fleet/harmony-fleet-agent/src/main.rs @@ -227,6 +227,33 @@ async fn main() -> Result<()> { let creds = credential_source_from_config(&cfg.credentials) .context("building NATS credential source")?; + // Deployment secret store (ADR-025). The OpenBao login reuses the + // NATS path's ZitadelJwtBearer — one machine identity, two + // consumers — so `[openbao]` without `zitadel-jwt` credentials is + // a configuration error, not a fallback. + let secrets = match (&cfg.openbao, creds.as_ref()) { + (Some(ob), CredentialSource::ZitadelJwt { bearer }) => { + let store = harmony_secret::OpenbaoSecretStore::with_zitadel_bearer( + &ob.url, + ob.danger_accept_invalid_certs, + &ob.kv_mount, + &ob.jwt_auth_mount, + &ob.jwt_role, + bearer.clone(), + ) + .map_err(|e| anyhow::anyhow!("building OpenBao secret store: {e}"))?; + tracing::info!(url = %ob.url, prefix = %ob.secret_prefix, "OpenBao secret store ready"); + Some(reconciler::SecretSource { + store: Arc::new(store), + prefix: ob.secret_prefix.clone(), + }) + } + (Some(_), _) => { + anyhow::bail!("[openbao] requires [credentials] type = \"zitadel-jwt\""); + } + (None, _) => None, + }; + let client = connect_nats(&cfg, creds).await.map_err(|e| { let msg = format!("Nats connection FAILED : {e}"); tracing::error!(msg); @@ -266,6 +293,7 @@ async fn main() -> Result<()> { t.clone(), inventory.clone(), Some(fleet.clone()), + secrets, )) }); diff --git a/fleet/harmony-fleet-agent/src/reconciler.rs b/fleet/harmony-fleet-agent/src/reconciler.rs index 3ba5b583..228f8470 100644 --- a/fleet/harmony-fleet-agent/src/reconciler.rs +++ b/fleet/harmony-fleet-agent/src/reconciler.rs @@ -8,11 +8,21 @@ use harmony_reconciler_contracts::{DeploymentName, DeploymentState, Id, Phase}; use tokio::sync::Mutex; use harmony::inventory::Inventory; -use harmony::modules::podman::{PodmanTopology, PodmanV0Score, ReconcileScore}; +use harmony::modules::podman::{EnvVar, PodmanTopology, PodmanV0Score, ReconcileScore}; use harmony::score::Score; +use harmony_secret::SecretStore; use crate::fleet_publisher::FleetPublisher; +/// Where deployment secrets come from (ADR-025). `prefix` scopes the +/// KV reads to `//` — it must match the +/// prefix the grant writer scopes `deployment-` policies to, +/// or every fetch is denied. +pub struct SecretSource { + pub store: Arc, + pub prefix: String, +} + /// Cache key → last-seen state, populated by `apply` and consulted by the /// 30-second periodic tick and the delete path. struct CachedEntry { @@ -40,6 +50,9 @@ pub struct Reconciler { /// Publish surface. Optional so unit tests without a live NATS /// client still work; always populated in the real agent runtime. fleet: Option>, + /// Deployment secret store. `None` = any score referencing + /// `secret_env` fails to apply (loudly, via `Phase::Failed`). + secrets: Option, } impl Reconciler { @@ -48,6 +61,7 @@ impl Reconciler { topology: Arc, inventory: Arc, fleet: Option>, + secrets: Option, ) -> Self { Self { device_id, @@ -56,9 +70,50 @@ impl Reconciler { state: Mutex::new(HashMap::new()), deployments: Mutex::new(HashMap::new()), fleet, + secrets, } } + /// Materialize every `secret_env` reference into a plain env var. + /// Resolution happens once per spec change, never on the periodic + /// tick: secret values bind at apply time (rotation = spec bump or + /// re-apply), which keeps the secret store off the 30s hot path — + /// fleet-scale login load is the ADR-025 constraint. + async fn resolve_secrets( + &self, + deployment: Option<&DeploymentName>, + mut score: PodmanV0Score, + ) -> Result { + for service in &mut score.services { + if service.secret_env.is_empty() { + continue; + } + let src = self.secrets.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "service '{}' references secrets but this device has no secret store configured", + service.name + ) + })?; + let dep = deployment + .ok_or_else(|| anyhow::anyhow!("secret_env requires a deployment-scoped key"))?; + let namespace = format!("{}/{}", src.prefix, dep); + for sref in &service.secret_env { + let bytes = src + .store + .get_raw(&namespace, &sref.secret) + .await + .map_err(|e| { + anyhow::anyhow!("fetching secret '{}/{}': {e}", namespace, sref.secret) + })?; + let value = String::from_utf8(bytes).map_err(|_| { + anyhow::anyhow!("secret '{}/{}' is not valid UTF-8", namespace, sref.secret) + })?; + service.env.push(EnvVar::new(&sref.name, &value)); + } + } + Ok(score) + } + /// Record a new phase for a deployment and, if it changed, write /// the updated [`DeploymentState`] to the KV. Same-phase /// re-confirmations are no-ops so the periodic reconcile tick @@ -139,7 +194,18 @@ impl Reconciler { self.apply_phase(name, Phase::Pending, None).await; } - match self.run_score(key, &incoming).await { + let resolved = match self.resolve_secrets(deployment.as_ref(), incoming).await { + Ok(score) => score, + Err(e) => { + if let Some(name) = &deployment { + self.apply_phase(name, Phase::Failed, Some(short(&e.to_string()))) + .await; + } + return Err(e); + } + }; + + match self.run_score(key, &resolved).await { Ok(()) => { if let Some(name) = &deployment { self.apply_phase(name, Phase::Running, None).await; @@ -155,11 +221,14 @@ impl Reconciler { } let mut state = self.state.lock().await; + // Cache the *resolved* score: the periodic tick re-runs it + // as-is, so converged deployments never re-hit the secret + // store (see `resolve_secrets`). state.insert( key.to_string(), CachedEntry { serialized, - score: incoming, + score: resolved, }, ); Ok(()) @@ -286,6 +355,10 @@ mod tests { use std::path::PathBuf; fn reconciler() -> Reconciler { + reconciler_with_secrets(None) + } + + fn reconciler_with_secrets(secrets: Option) -> Reconciler { let topology = Arc::new( PodmanTopology::from_unix_socket(PathBuf::from("/nonexistent/for-tests")).unwrap(), ); @@ -295,6 +368,7 @@ mod tests { topology, inventory, None, + secrets, ) } @@ -302,6 +376,119 @@ mod tests { DeploymentName::try_new(s).expect("valid test name") } + /// In-memory secret store keyed by `(namespace, key)`. + #[derive(Debug, Default)] + struct MapStore(HashMap<(String, String), String>); + + #[async_trait::async_trait] + impl SecretStore for MapStore { + async fn get_raw( + &self, + namespace: &str, + key: &str, + ) -> Result, harmony_secret::SecretStoreError> { + self.0 + .get(&(namespace.to_string(), key.to_string())) + .map(|v| v.as_bytes().to_vec()) + .ok_or_else(|| harmony_secret::SecretStoreError::Store("permission denied".into())) + } + + async fn set_raw( + &self, + _namespace: &str, + _key: &str, + _value: &[u8], + ) -> Result<(), harmony_secret::SecretStoreError> { + unimplemented!("tests never write") + } + } + + fn secret_score(env_name: &str, secret: &str) -> PodmanV0Score { + use harmony::modules::podman::{PodmanService, SecretEnvVar}; + PodmanV0Score { + services: vec![PodmanService { + name: "web-svc".to_string(), + image: "nginx".to_string(), + ports: vec![], + env: vec![], + secret_env: vec![SecretEnvVar { + name: env_name.to_string(), + secret: secret.to_string(), + }], + volumes: vec![], + restart_policy: Default::default(), + }], + } + } + + fn map_source(entries: &[(&str, &str, &str)]) -> SecretSource { + let map = entries + .iter() + .map(|(ns, k, v)| ((ns.to_string(), k.to_string()), v.to_string())) + .collect(); + SecretSource { + store: Arc::new(MapStore(map)), + prefix: "fleet-test".to_string(), + } + } + + #[tokio::test] + async fn resolve_secrets_injects_env_value() { + let r = reconciler_with_secrets(Some(map_source(&[( + "fleet-test/web", + "db_password", + "hunter2", + )]))); + let resolved = r + .resolve_secrets(Some(&dn("web")), secret_score("DB_PASSWORD", "db_password")) + .await + .expect("resolution succeeds"); + let env = &resolved.services[0].env; + assert_eq!(env.len(), 1); + assert_eq!(env[0], EnvVar::new("DB_PASSWORD", "hunter2")); + } + + #[tokio::test] + async fn resolve_secrets_without_store_fails() { + let r = reconciler(); + let err = r + .resolve_secrets(Some(&dn("web")), secret_score("DB_PASSWORD", "db_password")) + .await + .expect_err("must fail without a store"); + assert!(err.to_string().contains("no secret store"), "{err}"); + } + + #[tokio::test] + async fn apply_marks_failed_when_secret_denied() { + // Store has no entry → MapStore answers "permission denied", + // the same observable a revoked OpenBao policy produces. The + // failure must surface as Phase::Failed *before* any podman + // call (the test topology has no socket). + let r = reconciler_with_secrets(Some(map_source(&[]))); + let payload = serde_json::to_vec(&ReconcileScore::PodmanV0(secret_score( + "DB_PASSWORD", + "db_password", + ))) + .unwrap(); + let err = r + .apply("test-device.web", &payload) + .await + .expect_err("denied fetch fails the apply"); + assert!(err.to_string().contains("permission denied"), "{err}"); + let phases = r.deployments.lock().await; + assert_eq!(phases.get(&dn("web")), Some(&Phase::Failed)); + } + + #[tokio::test] + async fn plain_scores_resolve_without_store() { + // No secret_env → no store needed; resolution is a no-op. + let r = reconciler(); + let score = PodmanV0Score { services: vec![] }; + r.resolve_secrets(Some(&dn("web")), score.clone()) + .await + .expect("no-op resolution"); + } + #[tokio::test] async fn apply_phase_records_new_phase() { let r = reconciler(); diff --git a/fleet/harmony-fleet-auth/Cargo.toml b/fleet/harmony-fleet-auth/Cargo.toml index 7e32f1e4..a99836e3 100644 --- a/fleet/harmony-fleet-auth/Cargo.toml +++ b/fleet/harmony-fleet-auth/Cargo.toml @@ -11,6 +11,7 @@ path = "src/lib.rs" [dependencies] harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } +harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" } async-nats = { workspace = true } anyhow = { workspace = true } chrono = { workspace = true } diff --git a/fleet/harmony-fleet-auth/examples/mint_zitadel_token.rs b/fleet/harmony-fleet-auth/examples/mint_zitadel_token.rs new file mode 100644 index 00000000..73d7c949 --- /dev/null +++ b/fleet/harmony-fleet-auth/examples/mint_zitadel_token.rs @@ -0,0 +1,26 @@ +use anyhow::{Context, Result}; +use harmony_fleet_auth::{CredentialSource, CredentialsSection, NatsCredential}; + +#[tokio::main] +async fn main() -> Result<()> { + let key_path = std::env::var("ZITADEL_KEY_PATH").context("ZITADEL_KEY_PATH missing")?; + let issuer = std::env::var("ZITADEL_ISSUER").context("ZITADEL_ISSUER missing")?; + let audience = std::env::var("ZITADEL_PROJECT_ID").context("ZITADEL_PROJECT_ID missing")?; + + let source = + CredentialSource::credential_source_from_config(&CredentialsSection::ZitadelJwt { + key_path: key_path.into(), + key_json: None, + oidc_issuer_url: issuer, + audience, + danger_accept_invalid_certs: false, + })?; + + match source.next_credential().await? { + NatsCredential::BearerToken(token) => { + println!("{token}"); + Ok(()) + } + NatsCredential::UserPass { .. } => anyhow::bail!("expected bearer token"), + } +} diff --git a/fleet/harmony-fleet-auth/src/agent_config.rs b/fleet/harmony-fleet-auth/src/agent_config.rs index 1d149fc0..3858fec6 100644 --- a/fleet/harmony-fleet-auth/src/agent_config.rs +++ b/fleet/harmony-fleet-auth/src/agent_config.rs @@ -25,6 +25,42 @@ pub struct AgentConfig { /// auto-published `device-id` label. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub labels: BTreeMap, + /// Secret store the device reads deployment secrets from + /// (ADR-025). Absent = deployments referencing `secret_env` fail + /// to apply. Requires `zitadel-jwt` credentials — the same + /// machine identity logs into OpenBao's JWT mount. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub openbao: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OpenbaoSection { + pub url: String, + /// KV prefix deployment subtrees live under + /// (`///`). Must + /// match the prefix the grant writer scopes policies to — no + /// default on purpose. + pub secret_prefix: String, + #[serde(default = "default_jwt_auth_mount")] + pub jwt_auth_mount: String, + #[serde(default = "default_jwt_role")] + pub jwt_role: String, + #[serde(default = "default_kv_mount")] + pub kv_mount: String, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub danger_accept_invalid_certs: bool, +} + +fn default_jwt_auth_mount() -> String { + "jwt".to_string() +} + +fn default_jwt_role() -> String { + "fleet-device".to_string() +} + +fn default_kv_mount() -> String { + "secret".to_string() } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -93,6 +129,54 @@ arch = "aarch64" } } + #[test] + fn openbao_section_parses_with_defaults() { + let raw = r#" +[agent] +device_id = "pi-42" + +[credentials] +type = "toml-shared" +nats_user = "u" +nats_pass = "p" + +[nats] +urls = ["nats://nats:4222"] + +[openbao] +url = "http://10.0.0.1:30425" +secret_prefix = "fleet-e2e" +"#; + let cfg: AgentConfig = toml::from_str(raw).expect("valid config"); + let ob = cfg.openbao.expect("openbao section"); + assert_eq!(ob.url, "http://10.0.0.1:30425"); + assert_eq!(ob.secret_prefix, "fleet-e2e"); + assert_eq!(ob.jwt_auth_mount, "jwt"); + assert_eq!(ob.jwt_role, "fleet-device"); + assert_eq!(ob.kv_mount, "secret"); + assert!(!ob.danger_accept_invalid_certs); + } + + #[test] + fn openbao_section_absent_is_none_and_omitted_on_serialize() { + let raw = r#" +[agent] +device_id = "pi-42" + +[credentials] +type = "toml-shared" +nats_user = "u" +nats_pass = "p" + +[nats] +urls = ["nats://nats:4222"] +"#; + let cfg: AgentConfig = toml::from_str(raw).expect("valid config"); + assert!(cfg.openbao.is_none()); + let rendered = toml::to_string(&cfg).expect("serialize"); + assert!(!rendered.contains("[openbao]"), "got:\n{rendered}"); + } + #[test] fn labels_section_optional_defaults_empty() { let raw = r#" @@ -175,6 +259,7 @@ urls = ["nats://nats:4222"] ("group".to_string(), "site\"a".to_string()), ("arch".to_string(), "aarch64".to_string()), ]), + openbao: None, }; let rendered = toml::to_string(&original).expect("serialize"); let parsed: AgentConfig = toml::from_str(&rendered).expect("deserialize"); @@ -215,6 +300,7 @@ urls = ["nats://nats:4222"] nats_pass: "p".to_string(), }, labels: BTreeMap::new(), + openbao: None, }; let rendered = toml::to_string(&cfg).expect("serialize"); assert!(!rendered.contains("[labels]"), "got:\n{rendered}"); diff --git a/fleet/harmony-fleet-auth/src/credentials.rs b/fleet/harmony-fleet-auth/src/credentials.rs index 82aa799d..36982d55 100644 --- a/fleet/harmony-fleet-auth/src/credentials.rs +++ b/fleet/harmony-fleet-auth/src/credentials.rs @@ -23,14 +23,13 @@ //! cleaner than a Trait + factory. use std::path::Path; -use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::sync::Arc; use anyhow::{Context, Result}; -use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader}; -use serde::Deserialize; -use crate::config::CredentialsSection; +use harmony_zitadel_auth::{MachineKeyFile, ZitadelJwtBearer}; + +use crate::CredentialsSection; /// Material the NATS connector needs to authenticate. Returned per /// (re)connect attempt — the source decides whether to mint fresh. @@ -44,17 +43,8 @@ pub enum NatsCredential { /// from the parsed `[credentials]` section; cloned via Arc into the /// async-nats auth callback. pub enum CredentialSource { - TomlShared { - user: String, - pass: String, - }, - ZitadelJwt { - key: MachineKeyFile, - oidc_issuer_url: String, - audience: String, - http: reqwest::Client, - cache: Mutex>, - }, + TomlShared { user: String, pass: String }, + ZitadelJwt { bearer: Arc }, } impl CredentialSource { @@ -72,256 +62,58 @@ impl CredentialSource { } async fn zitadel_next(&self) -> Result { - // Fast path: lock the cache synchronously, copy out the token if - // it's comfortably valid, drop the lock. Holding a MutexGuard - // across `.await` would make this future !Sync, which - // async-nats's `with_auth_callback` rejects at compile time. - if let Some(token) = self.cached_if_fresh() { - return Ok(NatsCredential::BearerToken(token)); - } - // Slow path: mint outside any lock. Two concurrent (re)connect - // attempts could both reach here and both mint; that's a wasted - // HTTP round-trip in a rare race, not a correctness issue — - // the second writer wins and replaces the first's value. - let fresh = self.zitadel_mint().await?; - let token = fresh.access_token.clone(); - if let Self::ZitadelJwt { - cache, audience, .. - } = self - && let Ok(mut guard) = cache.lock() - { - *guard = Some(fresh); - tracing::info!(audience = %audience, "minted fresh Zitadel access token"); - } + let Self::ZitadelJwt { bearer } = self else { + anyhow::bail!("zitadel_next called on non-ZitadelJwt variant"); + }; + let token = bearer.bearer_token().await?; Ok(NatsCredential::BearerToken(token)) } - fn cached_if_fresh(&self) -> Option { - let Self::ZitadelJwt { cache, .. } = self else { - return None; - }; - let now = chrono::Utc::now().timestamp(); - let guard = cache.lock().ok()?; - let cached = guard.as_ref()?; - if cached.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS > now { - Some(cached.access_token.clone()) - } else { - None - } - } - - async fn zitadel_mint(&self) -> Result { - let Self::ZitadelJwt { - key, - oidc_issuer_url, - audience, - http, - .. - } = self - else { - anyhow::bail!("zitadel_mint called on non-ZitadelJwt variant"); - }; - - let now = chrono::Utc::now().timestamp(); - let assertion = build_assertion(key, oidc_issuer_url, now)?; - let scope = build_scope(audience); - let token_url = build_token_url(oidc_issuer_url); - - let resp = http - .post(&token_url) - .form(&[ - ( - "grant_type", - "urn:ietf:params:oauth:grant-type:jwt-bearer".to_string(), - ), - ("assertion", assertion), - ("scope", scope), - ]) - .send() - .await - .with_context(|| format!("POST {token_url}"))?; - - if !resp.status().is_success() { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - anyhow::bail!("Zitadel token endpoint returned {status}: {body}"); - } - - #[derive(Deserialize)] - struct TokenResponse { - access_token: String, - #[serde(default)] - expires_in: Option, - } - let tr: TokenResponse = resp.json().await.context("parsing token response")?; - // Zitadel typically returns 12h (43200s); be defensive against - // a missing field by assuming a conservative 1h. - let expires_in = tr.expires_in.unwrap_or(3600); - Ok(CachedToken { - access_token: tr.access_token, - expires_at_unix: now + expires_in, - }) - } -} - -/// Build the JWT-bearer assertion. Split out from the network path so -/// the claims + header shape can be unit-tested without an HTTP server, -/// and split internally into the (pure) claim/header builders so they -/// can be unit-tested without an RSA private key fixture. -pub(crate) fn build_assertion( - key: &MachineKeyFile, - oidc_issuer_url: &str, - now: i64, -) -> Result { - let claims = build_assertion_claims(key, oidc_issuer_url, now); - let header = build_assertion_header(key); - let assertion = jsonwebtoken::encode( - &header, - &claims, - &EncodingKey::from_rsa_pem(key.key.as_bytes()) - .context("parsing RSA private key from machine key file")?, - ) - .context("signing JWT assertion")?; - Ok(assertion) -} - -/// Pure claim payload for the JWT-bearer assertion. `iss == sub == userId` -/// is a Zitadel requirement; `aud` is Zitadel itself (the token endpoint -/// is reached via `oidc_issuer_url`); `exp - iat` MUST be ≤ 60 s or -/// Zitadel rejects. -pub(crate) fn build_assertion_claims( - key: &MachineKeyFile, - oidc_issuer_url: &str, - now: i64, -) -> serde_json::Value { - serde_json::json!({ - "iss": key.user_id, - "sub": key.user_id, - "aud": oidc_issuer_url, - "exp": now + ASSERTION_LIFETIME_SECS, - "iat": now, - }) -} - -/// JWT header for the assertion. The `kid` tells Zitadel which of the -/// machine user's registered keys to verify the signature against. -pub(crate) fn build_assertion_header(key: &MachineKeyFile) -> JwtHeader { - let mut header = JwtHeader::new(Algorithm::RS256); - header.kid = Some(key.key_id.clone()); - header -} - -/// Build the OAuth `scope` string for the token-bearer request. -/// -/// Three scopes are needed for the access token to be useful here: -/// -/// * `openid` — base OIDC requirement. -/// * `urn:zitadel:iam:org:projects:roles` (PLURAL "projects") — -/// tells Zitadel to include the role-claim block in the access -/// token. Without this, the callout sees "no authorized role -/// in token" even when the user has a project role grant. -/// * `urn:zitadel:iam:org:project:id::aud` (SINGULAR -/// "project") — adds to the access token's `aud` claim -/// so the callout's audience validation accepts the project -/// ID we're using as the JWT-bearer audience. -/// -/// The plural-vs-singular distinction is a Zitadel convention, -/// not a typo. Both scopes are required. -pub(crate) fn build_scope(audience: &str) -> String { - format!( - "openid \ - urn:zitadel:iam:org:projects:roles \ - urn:zitadel:iam:org:project:id:{audience}:aud" - ) -} - -/// Resolve the token endpoint URL, tolerating a trailing slash on -/// `oidc_issuer_url`. Without trimming, a configured issuer of -/// `https://sso.example.com/` produces `…//oauth/v2/token` which 404s. -pub(crate) fn build_token_url(oidc_issuer_url: &str) -> String { - format!("{}/oauth/v2/token", oidc_issuer_url.trim_end_matches('/')) -} - -// ---- helper types ---------------------------------------------------------- - -/// JSON keyfile content as Zitadel emits it for a `KEY_TYPE_JSON` -/// machine key. The `key` is a PEM-encoded RSA private key. -#[derive(Debug, Clone, Deserialize)] -pub struct MachineKeyFile { - #[serde(rename = "type")] - pub _type: String, - #[serde(rename = "keyId")] - pub key_id: String, - pub key: String, - #[serde(rename = "userId")] - pub user_id: String, -} - -#[derive(Debug, Clone)] -pub struct CachedToken { - pub(crate) access_token: String, - /// Unix seconds at which the token is no longer trusted by - /// `cached_if_fresh`. Computed from the OAuth response's `expires_in` - /// and the local clock at mint time. - pub(crate) expires_at_unix: i64, -} - -/// Refresh tokens this many seconds before their advertised expiry. -/// Five minutes leaves headroom for clock skew, slow networks, and -/// the round-trip cost of re-minting against Zitadel. -pub const TOKEN_REFRESH_LEEWAY_SECS: i64 = 5 * 60; - -/// Lifetime of the JWT *assertion* (the client-side bearer JWT we sign -/// to authenticate to Zitadel's token endpoint). Zitadel rejects -/// assertions with `exp - iat > 60s`; one minute is the safe ceiling. -pub const ASSERTION_LIFETIME_SECS: i64 = 60; - -// ---- factory --------------------------------------------------------------- - -/// Build the appropriate `CredentialSource` from the parsed config. -/// -/// For [`CredentialsSection::ZitadelJwt`] this reads the keyfile from -/// disk. Both the agent and the operator mount their key as a file -/// (Secret volume in the operator's Pod, dropped by -/// `FleetDeviceSetupScore` on the agent's VM); the path is just -/// configured differently. -pub fn credential_source_from_config(creds: &CredentialsSection) -> Result> { - match creds { - CredentialsSection::TomlShared { - nats_user, - nats_pass, - } => Ok(Arc::new(CredentialSource::TomlShared { - user: nats_user.clone(), - pass: nats_pass.clone(), - })), - CredentialsSection::ZitadelJwt { - key_path, - key_json, - oidc_issuer_url, - audience, - danger_accept_invalid_certs, - } => { - // `key_json` (inline) wins over `key_path` (file). The - // operator pod uses inline because OKD's restricted-v2 - // SCC + env-var-from-Secret deployment shape can't - // reliably mount Secret volumes; the agent uses the file - // path because it lives on a VM and a real file is the - // more natural rotation target. - let key = match key_json.as_deref().map(str::trim) { - Some(json) if !json.is_empty() => parse_machine_key(json)?, - _ => load_machine_key(key_path)?, - }; - Ok(Arc::new(CredentialSource::ZitadelJwt { - key, - oidc_issuer_url: oidc_issuer_url.clone(), - audience: audience.clone(), - http: reqwest::Client::builder() - .danger_accept_invalid_certs(*danger_accept_invalid_certs) - .timeout(Duration::from_secs(10)) - .build() - .context("building HTTP client for Zitadel token endpoint")?, - cache: Mutex::new(None), - })) + /// Build the appropriate `CredentialSource` from the parsed config. + /// + /// For [`CredentialsSection::ZitadelJwt`] this reads the keyfile from + /// disk. Both the agent and the operator mount their key as a file + /// (Secret volume in the operator's Pod, dropped by + /// `FleetDeviceSetupScore` on the agent's VM); the path is just + /// configured differently. + pub fn credential_source_from_config( + creds: &CredentialsSection, + ) -> Result> { + match creds { + CredentialsSection::TomlShared { + nats_user, + nats_pass, + } => Ok(Arc::new(CredentialSource::TomlShared { + user: nats_user.clone(), + pass: nats_pass.clone(), + })), + CredentialsSection::ZitadelJwt { + key_path, + key_json, + oidc_issuer_url, + audience, + danger_accept_invalid_certs, + } => { + // `key_json` (inline) wins over `key_path` (file). The + // operator pod uses inline because OKD's restricted-v2 + // SCC + env-var-from-Secret deployment shape can't + // reliably mount Secret volumes; the agent uses the file + // path because it lives on a VM and a real file is the + // more natural rotation target. + let key = match key_json.as_deref().map(str::trim) { + Some(json) if !json.is_empty() => parse_machine_key(json)?, + _ => load_machine_key(key_path)?, + }; + let bearer = ZitadelJwtBearer::new( + key, + oidc_issuer_url.clone(), + audience.clone(), + *danger_accept_invalid_certs, + )?; + Ok(Arc::new(CredentialSource::ZitadelJwt { + bearer: Arc::new(bearer), + })) + } } } } @@ -341,30 +133,6 @@ fn parse_machine_key(raw: &str) -> Result { mod tests { use super::*; - fn fake_key() -> MachineKeyFile { - MachineKeyFile { - _type: "serviceaccount".to_string(), - key_id: "kid-371358469099356247".to_string(), - // Real PEM not required for the pure-builder tests; the - // signing path that needs a parseable key is exercised - // end-to-end in the e2e harness. - key: "PEM-PLACEHOLDER".to_string(), - user_id: "uid-371358469065801815".to_string(), - } - } - - fn zjwt_source() -> CredentialSource { - CredentialSource::ZitadelJwt { - key: fake_key(), - oidc_issuer_url: "http://sso.fleet.local:8080".to_string(), - audience: "366378028009259037".to_string(), - http: reqwest::Client::new(), - cache: Mutex::new(None), - } - } - - // ---- next_credential / cache state ------------------------------------- - #[tokio::test] async fn toml_shared_returns_userpass_each_call() { let s = CredentialSource::TomlShared { @@ -380,174 +148,4 @@ mod tests { other => panic!("expected UserPass, got {other:?}"), } } - - #[test] - fn cached_token_within_leeway_is_treated_as_expired() { - // Sanity-check the comparison so refactors don't accidentally - // invert the leeway window. - let now = chrono::Utc::now().timestamp(); - let about_to_expire = CachedToken { - access_token: "x".to_string(), - expires_at_unix: now + TOKEN_REFRESH_LEEWAY_SECS - 1, - }; - assert!( - about_to_expire.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS <= now, - "tokens within the leeway window must be considered expired" - ); - - let comfortable = CachedToken { - access_token: "x".to_string(), - expires_at_unix: now + TOKEN_REFRESH_LEEWAY_SECS + 60, - }; - assert!( - comfortable.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS > now, - "tokens with comfortable headroom must be cache-hits" - ); - } - - #[test] - fn cached_if_fresh_returns_some_when_outside_leeway() { - let src = zjwt_source(); - let now = chrono::Utc::now().timestamp(); - if let CredentialSource::ZitadelJwt { cache, .. } = &src { - *cache.lock().unwrap() = Some(CachedToken { - access_token: "fresh".to_string(), - expires_at_unix: now + TOKEN_REFRESH_LEEWAY_SECS + 60, - }); - } - assert_eq!(src.cached_if_fresh(), Some("fresh".to_string())); - } - - #[test] - fn cached_if_fresh_returns_none_when_no_cache() { - // Brand-new ZitadelJwt source — no token has been minted yet. - // Forces the slow path on first connect. - let src = zjwt_source(); - assert_eq!(src.cached_if_fresh(), None); - } - - #[test] - fn cached_if_fresh_returns_none_for_toml_shared() { - // Defensive: cache_if_fresh is only meaningful for ZitadelJwt; - // TomlShared has no cache. A nonsensical call must return None, - // not panic, so the cold-path can degrade gracefully. - let src = CredentialSource::TomlShared { - user: "u".into(), - pass: "p".into(), - }; - assert_eq!(src.cached_if_fresh(), None); - } - - // ---- assertion claims / header (pure builders) ------------------------ - - #[test] - fn assertion_claims_carry_iss_sub_aud_exp_iat() { - let now = 1_700_000_000; - let claims = build_assertion_claims(&fake_key(), "http://sso.fleet.local:8080", now); - assert_eq!(claims["iss"], "uid-371358469065801815"); - assert_eq!(claims["sub"], "uid-371358469065801815"); - assert_eq!(claims["aud"], "http://sso.fleet.local:8080"); - assert_eq!(claims["iat"].as_i64(), Some(now)); - assert_eq!(claims["exp"].as_i64(), Some(now + ASSERTION_LIFETIME_SECS)); - } - - #[test] - fn assertion_lifetime_locked_at_60_seconds() { - // Zitadel rejects assertions where exp - iat > 60s. If anyone - // bumps ASSERTION_LIFETIME_SECS thinking "more is safer", the - // mints will silently start failing in prod with no helpful - // error. Lock the constant. - assert_eq!(ASSERTION_LIFETIME_SECS, 60); - } - - #[test] - fn assertion_header_carries_kid_and_rs256() { - let header = build_assertion_header(&fake_key()); - assert_eq!(header.alg, jsonwebtoken::Algorithm::RS256); - assert_eq!(header.kid.as_deref(), Some("kid-371358469099356247")); - } - - // ---- scope string ------------------------------------------------------ - - #[test] - fn scope_includes_plural_projects_roles() { - // The plural-projects URN is what tells Zitadel to emit the - // role claim. Day-one bug; lock it. - let s = build_scope("366378028009259037"); - assert!( - s.contains("urn:zitadel:iam:org:projects:roles"), - "scope must include the PLURAL projects-roles URN; got {s:?}" - ); - } - - #[test] - fn scope_audience_uses_singular_project_id_urn() { - // The singular-project URN tells Zitadel to put into the - // access token's aud claim. Different URN entirely from the - // plural one above; both required. - let s = build_scope("366378028009259037"); - assert!( - s.contains("urn:zitadel:iam:org:project:id:366378028009259037:aud"), - "scope must include the SINGULAR project:id::aud URN; got {s:?}" - ); - } - - #[test] - fn scope_includes_openid_base() { - let s = build_scope("any"); - assert!( - s.split_whitespace().any(|tok| tok == "openid"), - "scope must include `openid` as a standalone token; got {s:?}" - ); - } - - // ---- token URL --------------------------------------------------------- - - #[test] - fn token_url_appends_oauth_endpoint() { - assert_eq!( - build_token_url("http://sso.fleet.local:8080"), - "http://sso.fleet.local:8080/oauth/v2/token" - ); - } - - #[test] - fn token_url_strips_single_trailing_slash() { - // A trailing slash would yield `…//oauth/v2/token`, which 404s. - // Common configuration drift; the trim guards against it. - assert_eq!( - build_token_url("http://sso.fleet.local:8080/"), - "http://sso.fleet.local:8080/oauth/v2/token" - ); - } - - #[test] - fn token_url_strips_multiple_trailing_slashes() { - // Defensive — `trim_end_matches('/')` peels all of them, not - // just the first. Locks that semantics. - assert_eq!( - build_token_url("http://sso.fleet.local:8080///"), - "http://sso.fleet.local:8080/oauth/v2/token" - ); - } - - // ---- MachineKeyFile JSON parsing -------------------------------------- - - #[test] - fn machine_key_file_parses_zitadel_json_shape() { - // The serde renames (`type`, `keyId`, `userId`) are easy to - // break. This is the literal JSON shape Zitadel's - // /management/v1/users/.../keys endpoint emits. - let raw = r#"{ - "type": "serviceaccount", - "keyId": "371358469099356247", - "key": "-----BEGIN RSA PRIVATE KEY-----\nABC\n-----END RSA PRIVATE KEY-----\n", - "userId": "371358469065801815" - }"#; - let parsed: MachineKeyFile = serde_json::from_str(raw).expect("valid keyfile"); - assert_eq!(parsed._type, "serviceaccount"); - assert_eq!(parsed.key_id, "371358469099356247"); - assert_eq!(parsed.user_id, "371358469065801815"); - assert!(parsed.key.contains("BEGIN RSA PRIVATE KEY")); - } } diff --git a/fleet/harmony-fleet-auth/src/lib.rs b/fleet/harmony-fleet-auth/src/lib.rs index e020feeb..2d0b53e6 100644 --- a/fleet/harmony-fleet-auth/src/lib.rs +++ b/fleet/harmony-fleet-auth/src/lib.rs @@ -27,13 +27,17 @@ mod credentials; pub use agent_config::{AgentConfig, AgentSection, NatsSection, load_config}; pub use config::{CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR}; -pub use credentials::{ - ASSERTION_LIFETIME_SECS, CachedToken, CredentialSource, MachineKeyFile, NatsCredential, - TOKEN_REFRESH_LEEWAY_SECS, credential_source_from_config, -}; use std::sync::Arc; +pub use crate::credentials::{CredentialSource, NatsCredential}; + +pub fn credential_source_from_config( + creds: &CredentialsSection, +) -> anyhow::Result> { + CredentialSource::credential_source_from_config(creds) +} + /// Build `async_nats::ConnectOptions` wired with the auth callback /// that pulls fresh credentials from `creds` on every (re)connect. /// diff --git a/fleet/harmony-fleet-deploy/src/agent.rs b/fleet/harmony-fleet-deploy/src/agent.rs index 14d033aa..a8ac29d8 100644 --- a/fleet/harmony-fleet-deploy/src/agent.rs +++ b/fleet/harmony-fleet-deploy/src/agent.rs @@ -82,6 +82,7 @@ impl PodTarget { nats_pass: nats_pass.into(), }, labels: BTreeMap::new(), + openbao: None, }, image: image.into(), image_pull_policy: "IfNotPresent".to_string(), diff --git a/fleet/harmony-fleet-deploy/src/companion/agent.rs b/fleet/harmony-fleet-deploy/src/companion/agent.rs index b9b441e8..67cd852e 100644 --- a/fleet/harmony-fleet-deploy/src/companion/agent.rs +++ b/fleet/harmony-fleet-deploy/src/companion/agent.rs @@ -121,6 +121,7 @@ mod tests { nats_pass: "p".to_string(), }, labels: BTreeMap::new(), + openbao: None, } } diff --git a/fleet/harmony-fleet-deploy/src/operator/chart.rs b/fleet/harmony-fleet-deploy/src/operator/chart.rs index c7fb19ef..419b3976 100644 --- a/fleet/harmony-fleet-deploy/src/operator/chart.rs +++ b/fleet/harmony-fleet-deploy/src/operator/chart.rs @@ -457,6 +457,18 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { }); env.push(secret_env(ENV_WEB_AUTH_CONFIG)); env.push(secret_env(ENV_WEB_COOKIE_KEY)); + // Secret-grant sync (OpenBao) + the device-group scheduling gate + // (Zitadel role grants) — ADR-025. All optional: absent, the + // operator logs and runs ungated/without grant sync. + for name in [ + "OPENBAO_URL", + "OPENBAO_TOKEN", + "ZITADEL_URL", + "ZITADEL_PAT", + "ZITADEL_PROJECT_ID", + ] { + env.push(secret_env(name)); + } // Namespace deliberately omitted — same rationale as the // ServiceAccount: helm fills in the release namespace at install diff --git a/fleet/harmony-fleet-e2e/Cargo.toml b/fleet/harmony-fleet-e2e/Cargo.toml index f11246e0..c1c36666 100644 --- a/fleet/harmony-fleet-e2e/Cargo.toml +++ b/fleet/harmony-fleet-e2e/Cargo.toml @@ -18,6 +18,14 @@ path = "tests/ping.rs" name = "operator" path = "tests/operator.rs" +[[test]] +name = "openbao_policy" +path = "tests/openbao_policy.rs" + +[[test]] +name = "openbao_groups" +path = "tests/openbao_groups.rs" + [[test]] name = "vm_ping" path = "tests/vm_ping.rs" @@ -30,6 +38,10 @@ path = "tests/vm_isolation.rs" name = "vm_deploy_lifecycle" path = "tests/vm_deploy_lifecycle.rs" +[[test]] +name = "vm_secret_access" +path = "tests/vm_secret_access.rs" + [dependencies] # `kvm` for the VM-side harness (ProvisionVmScore + KvmVirtualMachineHost), # `podman` for `ReconcileScore`/`PodmanV0Score` the kv_admin helpers @@ -39,6 +51,7 @@ harmony-fleet-auth = { path = "../harmony-fleet-auth" } harmony-fleet-deploy = { path = "../harmony-fleet-deploy" } harmony-fleet-operator = { path = "../harmony-fleet-operator" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } +harmony-k8s = { path = "../../harmony-k8s" } harmony_types = { path = "../../harmony_types" } k3d-rs = { path = "../../k3d" } @@ -57,6 +70,9 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } directories = "6.0.0" uuid = { version = "1", features = ["v4"] } +jsonwebtoken = "9" +nkeys = "0.4" +reqwest = { workspace = true } [dev-dependencies] tokio = { workspace = true, features = ["full"] } diff --git a/fleet/harmony-fleet-e2e/src/callout.rs b/fleet/harmony-fleet-e2e/src/callout.rs new file mode 100644 index 00000000..d12f4357 --- /dev/null +++ b/fleet/harmony-fleet-e2e/src/callout.rs @@ -0,0 +1,569 @@ +//! Callout-mode auth fabric: Zitadel + NATS auth-callout + OpenBao, +//! composed from the same Scores `examples/fleet_staging_install` +//! deploys to staging. The e2e specifics are network plumbing only: +//! +//! - **Issuer URL** `http://sso.fleet-e2e.local:30424` is valid from +//! every vantage point through a single NodePort Service +//! (`zitadel-e2e`, port == nodePort == 30424): the host reaches it +//! via the k3d port mapping, in-cluster pods resolve the hostname +//! via a CoreDNS rewrite to that Service (whose in-cluster port is +//! the same number), VMs resolve it via an `/etc/hosts` entry +//! pointing at the libvirt gateway. Zitadel derives its issuer +//! from `Host`, so every consumer sees the same `iss`; host-side +//! helpers hit `127.0.0.1:30424` and carry the hostname in `Host`. +//! - **OpenBao** lives in the per-bring-up namespace like the other +//! workloads; a NodePort (mapped into k3d) makes it reachable from +//! VMs. +//! +//! Zitadel itself deploys into a stable `zitadel` namespace so the +//! multi-minute first install is paid once per cluster, not per run. + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use harmony::inventory::Inventory; +use harmony::modules::k8s::coredns::{CoreDNSRewrite, CoreDNSRewriteScore}; +use harmony::modules::k8s::resource::K8sResourceScore; +use harmony::modules::nats_auth_callout::NatsAuthCalloutScore; +use harmony::modules::openbao::{ + OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore, cached_root_token, +}; +use harmony::modules::zitadel::{ + MachineKeyType, ZitadelApiApp, ZitadelClientConfig, ZitadelMachineUser, ZitadelRole, + ZitadelScheme, ZitadelScore, ZitadelSetupScore, mint_device_credentials, +}; +use harmony::score::Score; +use harmony::topology::K8sAnywhereTopology; +use harmony_fleet_deploy::FleetNatsScore; +use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader, encode as jwt_encode}; +use k8s_openapi::api::core::v1::{Secret, Service, ServicePort, ServiceSpec}; +use kube::api::ObjectMeta; +use nkeys::KeyPair; +use serde::Deserialize; + +use crate::images::CALLOUT_IMAGE_TAG; + +pub const ZITADEL_HOST: &str = "sso.fleet-e2e.local"; +/// Host-mapped k3d port AND the `zitadel-e2e` Service's in-cluster +/// port — the same number on purpose, so one issuer URL resolves +/// everywhere (see module doc). Not 8080: dev hosts commonly have +/// something on it. +pub const ZITADEL_PORT: u16 = 30424; +pub const ZITADEL_NAMESPACE: &str = "zitadel"; +/// Host-mapped NodePort the OpenBao Service publishes for VM access. +pub const OPENBAO_NODE_PORT: u16 = 30425; +pub const OPENBAO_RELEASE: &str = "openbao"; +pub const KV_MOUNT: &str = "secret"; +/// KV prefix deployment secrets live under. Grant writer and agents +/// must agree on it; tests import this const for both sides. +pub const SECRET_PREFIX: &str = "fleet-e2e"; + +pub const PROJECT_NAME: &str = "fleet"; +pub const ADMIN_ROLE: &str = "fleet-admin"; +/// Role the callout maps to device-grade NATS permissions. Group +/// roles (`edge-*`) ride along in the same claim purely as OpenBao +/// groups (ADR-025 roles-as-groups interim). +pub const DEVICE_ROLE: &str = "device"; +pub const GROUP_ROLES: &[&str] = &["edge-a", "edge-b"]; +pub const ADMIN_USERNAME: &str = "e2e-admin"; + +const NATS_ACCOUNT: &str = "FLEET"; +const NATS_AUTH_USER: &str = "auth"; +// Dev-only static credential for the callout's own NATS user, same +// posture as the user-pass mode creds in `stack.rs`. +const NATS_AUTH_PASS: &str = "e2e-callout-auth"; +const CALLOUT_DEPLOYMENT: &str = "fleet-callout"; + +/// Handles the callout bring-up leaves behind for tests. +#[derive(Debug, Clone)] +pub struct CalloutHandles { + /// `http://sso.fleet-e2e.local:30424` — `iss` everywhere. + pub issuer_url: String, + pub project_id: String, + /// Machine key of the fleet-admin user the host-side NATS client + /// authenticates as. + pub admin_machine_key: String, + /// Zitadel IAM admin PAT (from the chart's `iam-admin-pat` + /// Secret) — used to enroll device machine users. + pub admin_pat: String, + pub openbao: OpenbaoInstance, + pub openbao_root_token: String, +} + +pub fn issuer_url() -> String { + format!("http://{ZITADEL_HOST}:{ZITADEL_PORT}") +} + +/// NodePort Service in front of Zitadel with in-cluster port == +/// nodePort == [`ZITADEL_PORT`] (targetPort = the chart's 8080), so +/// the issuer URL's port number is valid both inside and outside the +/// cluster. +fn zitadel_nodeport_service() -> Service { + let selector = std::collections::BTreeMap::from([ + ("app.kubernetes.io/name".to_string(), "zitadel".to_string()), + ( + "app.kubernetes.io/instance".to_string(), + "zitadel".to_string(), + ), + ]); + Service { + metadata: ObjectMeta { + name: Some("zitadel-e2e".to_string()), + namespace: Some(ZITADEL_NAMESPACE.to_string()), + ..Default::default() + }, + spec: Some(ServiceSpec { + type_: Some("NodePort".to_string()), + selector: Some(selector), + ports: Some(vec![ServicePort { + name: Some("http".to_string()), + port: ZITADEL_PORT as i32, + target_port: Some( + k8s_openapi::apimachinery::pkg::util::intstr::IntOrString::Int(8080), + ), + node_port: Some(ZITADEL_PORT as i32), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + } +} + +/// Deploy Zitadel + project/roles/admin-user, returning everything +/// except the OpenBao handles (filled by [`deploy_openbao`]). +pub async fn bring_up_zitadel( + topology: &K8sAnywhereTopology, + kube_client: &kube::Client, +) -> Result { + ZitadelScore { + host: ZITADEL_HOST.to_string(), + zitadel_version: "v4.12.1".to_string(), + external_secure: false, + external_port: Some(ZITADEL_PORT as u32), + namespace: ZITADEL_NAMESPACE.to_string(), + ..Default::default() + } + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("Zitadel deploy: {e}"))?; + + K8sResourceScore::single( + zitadel_nodeport_service(), + Some(ZITADEL_NAMESPACE.to_string()), + ) + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("Zitadel NodePort service: {e}"))?; + + CoreDNSRewriteScore { + rewrites: vec![CoreDNSRewrite { + hostname: ZITADEL_HOST.to_string(), + target: format!("zitadel-e2e.{ZITADEL_NAMESPACE}.svc.cluster.local"), + }], + } + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("CoreDNS rewrite: {e}"))?; + + wait_for_zitadel_ready().await?; + + let roles = [ADMIN_ROLE, DEVICE_ROLE] + .into_iter() + .chain(GROUP_ROLES.iter().copied()) + .map(|key| ZitadelRole { + project_name: PROJECT_NAME.to_string(), + key: key.to_string(), + display_name: key.to_string(), + group: None, + }) + .collect(); + + ZitadelSetupScore { + machine_users: vec![ZitadelMachineUser { + username: ADMIN_USERNAME.to_string(), + name: "E2E Admin".to_string(), + create_pat: false, + machine_key: Some(MachineKeyType::Json), + project_name: Some(PROJECT_NAME.to_string()), + grant_roles: vec![ADMIN_ROLE.to_string()], + }], + roles, + // The flattening Action turns project role keys into the + // `groups` array claim OpenBao binds against (ADR-025) — this + // bring-up is its live verification. + groups_claim_action: true, + applications: vec![], + api_apps: vec![ZitadelApiApp { + project_name: PROJECT_NAME.to_string(), + app_name: "nats".to_string(), + }], + ..zitadel_connection() + } + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("Zitadel setup: {e}"))?; + + let zcfg = ZitadelClientConfig::load() + .context("ZitadelSetupScore did not produce a client config cache")?; + let project_id = zcfg + .project_id_by_name(PROJECT_NAME) + .or(zcfg.project_id.as_ref()) + .context("project_id missing from cache")? + .clone(); + let admin_machine_key = zcfg + .machine_key(ADMIN_USERNAME) + .context("admin machine key missing from cache")? + .clone(); + let admin_pat = read_iam_admin_pat(kube_client).await?; + + Ok(CalloutHandles { + issuer_url: issuer_url(), + project_id, + admin_machine_key, + admin_pat, + // Placeholder until deploy_openbao fills them in. + openbao: OpenbaoInstance { + namespace: String::new(), + release: OPENBAO_RELEASE.to_string(), + }, + openbao_root_token: String::new(), + }) +} + +/// Network-shape-only ZitadelSetupScore: management API reached at +/// `127.0.0.1:8080` with the hostname carried in `Host` (k3d ingress +/// routes by it; no /etc/hosts entry needed on the host). +fn zitadel_connection() -> ZitadelSetupScore { + ZitadelSetupScore { + host: ZITADEL_HOST.to_string(), + scheme: ZitadelScheme::Http, + port: Some(ZITADEL_PORT), + skip_tls: true, + endpoint: Some(format!("http://127.0.0.1:{ZITADEL_PORT}")), + admin_org_id: None, + namespace: ZITADEL_NAMESPACE.to_string(), + applications: vec![], + api_apps: vec![], + roles: vec![], + machine_users: vec![], + groups_claim_action: false, + } +} + +/// Find-or-create the device's machine user with `roles` granted and +/// mint a fresh JSON machine key. Username == device id, so the +/// callout needs no prefix stripping and the operator-side group +/// source resolves grants by device id directly. +pub async fn enroll_device( + handles: &CalloutHandles, + device_id: &str, + roles: &[String], +) -> Result { + let minted = mint_device_credentials( + &zitadel_connection(), + &handles.admin_pat, + PROJECT_NAME, + device_id, + device_id, + roles, + ) + .await + .map_err(|e| anyhow::anyhow!("enrolling device '{device_id}': {e}"))?; + Ok(minted.key_json) +} + +/// Deploy NATS in callout mode + the callout service itself. +/// Returns the `FleetNatsScore` so the caller can derive the +/// in-cluster URL the same way the user-pass path does. +pub async fn deploy_nats_callout( + topology: &K8sAnywhereTopology, + namespace: &str, + release: &str, + node_port: u16, + handles: &CalloutHandles, +) -> Result { + let issuer_kp = KeyPair::new_account(); + let issuer_seed = issuer_kp + .seed() + .map_err(|e| anyhow::anyhow!("issuer NKey seed: {e}"))?; + + let nats = FleetNatsScore::callout( + namespace.to_string(), + node_port, + issuer_kp.public_key(), + NATS_ACCOUNT, + NATS_AUTH_USER, + NATS_AUTH_PASS, + ) + .release_name(release); + nats.interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("NATS (callout) deploy: {e}"))?; + + let mut callout = NatsAuthCalloutScore::new( + CALLOUT_DEPLOYMENT, + namespace, + nats.in_cluster_url(), + handles.issuer_url.clone(), + handles.project_id.clone(), + NATS_AUTH_USER, + NATS_AUTH_PASS, + issuer_seed, + ) + .image(CALLOUT_IMAGE_TAG) + .target_account(NATS_ACCOUNT) + .admin_role(ADMIN_ROLE) + .device_role(DEVICE_ROLE); + // Machine-user access tokens carry the userName in `client_id`; + // usernames ARE device ids here (see `enroll_device`). + callout.device_id_claim = "client_id".to_string(); + callout.device_id_prefix_strip = String::new(); + callout.roles_claim = format!("urn:zitadel:iam:org:project:{}:roles", handles.project_id); + callout + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("callout deploy: {e}"))?; + + Ok(nats) +} + +pub fn callout_deployment_name() -> &'static str { + CALLOUT_DEPLOYMENT +} + +/// Deploy OpenBao into the bring-up namespace: JWT auth bound to the +/// Zitadel issuer (batch tokens, 1h, `groups` claim — the staging +/// shape) plus a NodePort Service so VMs reach it through the libvirt +/// gateway. +pub async fn deploy_openbao( + topology: &K8sAnywhereTopology, + namespace: &str, + handles: &mut CalloutHandles, +) -> Result<()> { + let instance = OpenbaoInstance { + namespace: namespace.to_string(), + release: OPENBAO_RELEASE.to_string(), + }; + OpenbaoScore { + instance: instance.clone(), + host: "openbao-e2e.local".to_string(), + openshift: false, + tls_issuer: None, + } + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("OpenBao deploy: {e}"))?; + + OpenbaoSetupScore { + instance: instance.clone(), + kv_mount: KV_MOUNT.to_string(), + jwt_auth: Some(OpenbaoJwtAuth { + oidc_discovery_url: handles.issuer_url.clone(), + jwt_validation_pubkeys: String::new(), + bound_issuer: handles.issuer_url.clone(), + role_name: "fleet-device".to_string(), + bound_audiences: handles.project_id.clone(), + user_claim: "sub".to_string(), + groups_claim: "groups".to_string(), + token_type: "batch".to_string(), + ttl: "1h".to_string(), + max_ttl: "1h".to_string(), + }), + ..Default::default() + } + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("OpenBao setup: {e}"))?; + + // NodePort for VM access. Same selector shape as the chart's own + // ClusterIP Service. + let selector = std::collections::BTreeMap::from([ + ("app.kubernetes.io/name".to_string(), "openbao".to_string()), + ( + "app.kubernetes.io/instance".to_string(), + OPENBAO_RELEASE.to_string(), + ), + ("component".to_string(), "server".to_string()), + ]); + let svc = Service { + metadata: ObjectMeta { + name: Some("openbao-nodeport".to_string()), + namespace: Some(namespace.to_string()), + ..Default::default() + }, + spec: Some(ServiceSpec { + type_: Some("NodePort".to_string()), + selector: Some(selector), + ports: Some(vec![ServicePort { + name: Some("http".to_string()), + port: 8200, + node_port: Some(OPENBAO_NODE_PORT as i32), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + }; + K8sResourceScore::single(svc, Some(namespace.to_string())) + .interpret(&Inventory::autoload(), topology) + .await + .map_err(|e| anyhow::anyhow!("OpenBao NodePort service: {e}"))?; + + handles.openbao = instance.clone(); + handles.openbao_root_token = + cached_root_token(&instance).map_err(|e| anyhow::anyhow!("OpenBao root token: {e}"))?; + Ok(()) +} + +/// Host-side NATS admin client: authenticates through the callout +/// with a Zitadel access token minted from the fleet-admin machine +/// key. +pub async fn connect_nats_admin_sso( + nats_url: &str, + handles: &CalloutHandles, +) -> Result { + let key = Arc::new(handles.clone()); + // ConnectOptions isn't Clone — build a fresh one per attempt. + let opts = || { + let key = key.clone(); + async_nats::ConnectOptions::with_auth_callback(move |_nonce| { + let handles = key.clone(); + async move { + let token = mint_access_token(&handles.admin_machine_key, &handles.project_id) + .await + .map_err(|e| async_nats::AuthError::new(format!("minting admin token: {e}")))?; + let mut auth = async_nats::Auth::new(); + auth.token = Some(token); + Ok(auth) + } + }) + .ping_interval(Duration::from_secs(5)) + }; + + let mut last_err = None; + for attempt in 0..120 { + match opts().connect(nats_url).await { + Ok(c) => return Ok(c), + Err(e) => { + if attempt % 10 == 0 { + tracing::warn!(attempt, error = %e, "host NATS (callout) connect retry"); + } + last_err = Some(e); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + anyhow::bail!("connecting host NATS admin client via callout: {last_err:?}") +} + +/// RFC 7523 JWT-bearer mint against the e2e Zitadel. Transport goes +/// to `127.0.0.1:8080` with the hostname (incl. port — Zitadel +/// derives its issuer from it) in `Host`, mirroring +/// `example_fleet_auth_callout::mint_access_token`. Lives here +/// because `ZitadelJwtBearer` has no transport-endpoint override and +/// teaching it one for a test-only DNS quirk isn't worth the API. +pub async fn mint_access_token(machine_key_json: &str, project_id: &str) -> Result { + #[derive(Deserialize)] + struct MachineKey { + #[serde(rename = "keyId")] + key_id: String, + key: String, + #[serde(rename = "userId")] + user_id: String, + } + let key: MachineKey = + serde_json::from_str(machine_key_json).context("machine key JSON parse")?; + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_secs() as i64; + let claims = serde_json::json!({ + "iss": key.user_id, + "sub": key.user_id, + "aud": issuer_url(), + "exp": now + 60, + "iat": now, + }); + let mut header = JwtHeader::new(Algorithm::RS256); + header.kid = Some(key.key_id.clone()); + let assertion = jwt_encode( + &header, + &claims, + &EncodingKey::from_rsa_pem(key.key.as_bytes()).context("machine key RSA parse")?, + )?; + + let scope = format!( + "openid profile urn:zitadel:iam:org:projects:roles \ + urn:zitadel:iam:org:project:id:{project_id}:aud" + ); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build()?; + let resp = client + .post(format!("http://127.0.0.1:{ZITADEL_PORT}/oauth/v2/token")) + .header("Host", format!("{ZITADEL_HOST}:{ZITADEL_PORT}")) + .form(&[ + ( + "grant_type", + "urn:ietf:params:oauth:grant-type:jwt-bearer".to_string(), + ), + ("assertion", assertion), + ("scope", scope), + ]) + .send() + .await + .context("POST /oauth/v2/token")?; + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("token endpoint returned {status}: {body}"); + } + #[derive(Deserialize)] + struct TokenResponse { + access_token: String, + } + let tr: TokenResponse = resp.json().await.context("parse token response")?; + Ok(tr.access_token) +} + +async fn wait_for_zitadel_ready() -> Result<()> { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build()?; + for attempt in 1..=150 { + match client + .get(format!( + "http://127.0.0.1:{ZITADEL_PORT}/.well-known/openid-configuration" + )) + .header("Host", format!("{ZITADEL_HOST}:{ZITADEL_PORT}")) + .send() + .await + { + Ok(r) if r.status().is_success() => return Ok(()), + Ok(r) if attempt % 15 == 0 => { + tracing::info!("Zitadel HTTP {} (attempt {attempt}/150)", r.status()) + } + Err(e) if attempt % 15 == 0 => { + tracing::info!("Zitadel unreachable: {e} (attempt {attempt}/150)") + } + _ => {} + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + anyhow::bail!("timed out waiting for Zitadel on 127.0.0.1:{ZITADEL_PORT}") +} + +async fn read_iam_admin_pat(client: &kube::Client) -> Result { + let api: kube::Api = kube::Api::namespaced(client.clone(), ZITADEL_NAMESPACE); + let secret = api + .get("iam-admin-pat") + .await + .context("reading iam-admin-pat Secret (created by ZitadelScore)")?; + let pat = secret + .data + .and_then(|d| d.get("pat").map(|b| b.0.clone())) + .context("iam-admin-pat Secret has no 'pat' key")?; + Ok(String::from_utf8(pat)?.trim().to_string()) +} diff --git a/fleet/harmony-fleet-e2e/src/images.rs b/fleet/harmony-fleet-e2e/src/images.rs index 06bb2e06..3d52e962 100644 --- a/fleet/harmony-fleet-e2e/src/images.rs +++ b/fleet/harmony-fleet-e2e/src/images.rs @@ -1,8 +1,7 @@ //! Build + sideload the container images the e2e harness deploys. //! -//! v1: agent only (operator and callout get added when those layers -//! join the harness). Same pattern as -//! [`example_fleet_auth_callout::build_and_load_callout_image`]: +//! One generic path for every image (same pattern as +//! [`example_fleet_auth_callout::build_and_load_callout_image`]): //! //! 1. `cargo build --release -p ` on the host (uses the //! workspace cargo cache; incremental builds are fast). @@ -24,6 +23,7 @@ use k3d_rs::K3d; pub const AGENT_IMAGE_TAG: &str = "localhost/harmony-fleet-agent:e2e"; pub const OPERATOR_IMAGE_TAG: &str = "localhost/harmony-fleet-operator:e2e"; +pub const CALLOUT_IMAGE_TAG: &str = "localhost/harmony-nats-callout:e2e"; /// Tags of the images this harness build produces. Returned to the /// caller so manifests reference the exact same tag we just pushed @@ -38,157 +38,105 @@ pub struct Images { /// runs (the tag is fixed; cargo + podman caches make subsequent /// runs near-noop). pub async fn build_and_sideload(k3d: &K3d, cluster_name: &str) -> Result { - build_and_load_agent_image(k3d, cluster_name).await?; - build_and_load_operator_image(k3d, cluster_name).await?; + build_and_load_binary_image(k3d, cluster_name, "harmony-fleet-agent", AGENT_IMAGE_TAG).await?; + build_and_load_binary_image( + k3d, + cluster_name, + "harmony-fleet-operator", + OPERATOR_IMAGE_TAG, + ) + .await?; Ok(Images { agent: AGENT_IMAGE_TAG.to_string(), operator: OPERATOR_IMAGE_TAG.to_string(), }) } -async fn build_and_load_operator_image(k3d: &K3d, cluster_name: &str) -> Result<()> { - let workspace_root = workspace_root_from_env(); - tracing::info!("cargo build --release -p harmony-fleet-operator"); - let status = tokio::process::Command::new("cargo") - .args(["build", "--release", "-p", "harmony-fleet-operator"]) - .current_dir(&workspace_root) - .status() - .await - .context("spawning cargo build")?; - if !status.success() { - anyhow::bail!("cargo build harmony-fleet-operator failed"); - } - let ctx = tempfile::tempdir()?; - let bin_dst = ctx.path().join("target/release"); - std::fs::create_dir_all(&bin_dst)?; - std::fs::copy( - workspace_root.join("target/release/harmony-fleet-operator"), - bin_dst.join("harmony-fleet-operator"), - ) - .context("staging operator binary into build context")?; - std::fs::write( - ctx.path().join("Dockerfile"), - r#"FROM docker.io/library/archlinux:base -COPY target/release/harmony-fleet-operator /usr/local/bin/harmony-fleet-operator -USER 65532:65532 -ENTRYPOINT ["/usr/local/bin/harmony-fleet-operator"] -"#, - )?; - tracing::info!("podman build -t {OPERATOR_IMAGE_TAG}"); - let status = tokio::process::Command::new("podman") - .args(["build", "-q", "-t", OPERATOR_IMAGE_TAG, "."]) - .current_dir(ctx.path()) - .stderr(Stdio::inherit()) - .status() - .await - .context("spawning podman build")?; - if !status.success() { - anyhow::bail!("podman build for harmony-fleet-operator failed"); - } - let tar_path = std::env::temp_dir().join(format!( - "harmony-fleet-operator-image-{}.tar", - std::process::id() - )); - let _ = std::fs::remove_file(&tar_path); - let status = tokio::process::Command::new("podman") - .args(["save", "-o", tar_path.to_str().unwrap(), OPERATOR_IMAGE_TAG]) - .status() - .await - .context("podman save")?; - if !status.success() { - anyhow::bail!("podman save for harmony-fleet-operator failed"); - } - tracing::info!("k3d image import {OPERATOR_IMAGE_TAG} (cluster={cluster_name})"); - let tar_path_str = tar_path.to_str().unwrap().to_string(); - let cluster_for_blocking = cluster_name.to_string(); - let data_dir = k3d_data_dir(); - let result = tokio::task::spawn_blocking(move || { - K3d::new(data_dir, Some(cluster_for_blocking.clone())).run_k3d_command([ - "image", - "import", - tar_path_str.as_str(), - "-c", - cluster_for_blocking.as_str(), - ]) - }) - .await - .context("spawn_blocking k3d image import")?; - let _ = std::fs::remove_file(&tar_path); - let output = result.map_err(|e| anyhow::anyhow!("k3d image import failed: {e}"))?; - if !output.status.success() { - anyhow::bail!( - "k3d image import returned status {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr), - ); - } - let _ = k3d; - Ok(()) +/// Build + sideload the NATS auth-callout image (callout-mode stacks +/// only — keeps user-pass bring-ups from paying for a binary they +/// never deploy). +pub async fn build_and_load_callout_image(k3d: &K3d, cluster_name: &str) -> Result<()> { + build_and_load_binary_image(k3d, cluster_name, "harmony-nats-callout", CALLOUT_IMAGE_TAG).await } -async fn build_and_load_agent_image(k3d: &K3d, cluster_name: &str) -> Result<()> { +/// Where `cargo build --release` actually puts binaries — the child +/// cargo honors CARGO_TARGET_DIR, so staging must too or the copy +/// fails with "No such file or directory" on shared-target setups +/// (e.g. git worktrees). +fn release_dir(workspace_root: &std::path::Path) -> PathBuf { + std::env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| workspace_root.join("target")) + .join("release") +} + +async fn build_and_load_binary_image( + k3d: &K3d, + cluster_name: &str, + crate_name: &str, + image_tag: &str, +) -> Result<()> { let workspace_root = workspace_root_from_env(); - tracing::info!("cargo build --release -p harmony-fleet-agent"); + tracing::info!("cargo build --release -p {crate_name}"); let status = tokio::process::Command::new("cargo") - .args(["build", "--release", "-p", "harmony-fleet-agent"]) + .args(["build", "--release", "-p", crate_name]) .current_dir(&workspace_root) .status() .await .context("spawning cargo build")?; if !status.success() { - anyhow::bail!("cargo build harmony-fleet-agent failed"); + anyhow::bail!("cargo build {crate_name} failed"); } let ctx = tempfile::tempdir()?; let bin_dst = ctx.path().join("target/release"); std::fs::create_dir_all(&bin_dst)?; std::fs::copy( - workspace_root.join("target/release/harmony-fleet-agent"), - bin_dst.join("harmony-fleet-agent"), + release_dir(&workspace_root).join(crate_name), + bin_dst.join(crate_name), ) - .context("staging agent binary into build context")?; + .with_context(|| format!("staging {crate_name} binary into build context"))?; - // Single-stage Dockerfile inline (mirrors what the callout build - // does). archlinux:base for matched glibc against the host build — - // the canonical multi-stage Dockerfile in the agent crate is for - // production registries, not the e2e fast path. + // Single-stage Dockerfile inline. archlinux:base for matched glibc + // against the host build — the canonical multi-stage Dockerfiles in + // the crates are for production registries, not the e2e fast path. std::fs::write( ctx.path().join("Dockerfile"), - r#"FROM docker.io/library/archlinux:base -COPY target/release/harmony-fleet-agent /usr/local/bin/harmony-fleet-agent + format!( + r#"FROM docker.io/library/archlinux:base +COPY target/release/{crate_name} /usr/local/bin/{crate_name} USER 65532:65532 -ENTRYPOINT ["/usr/local/bin/harmony-fleet-agent"] -"#, +ENTRYPOINT ["/usr/local/bin/{crate_name}"] +"# + ), )?; - tracing::info!("podman build -t {AGENT_IMAGE_TAG}"); + tracing::info!("podman build -t {image_tag}"); let status = tokio::process::Command::new("podman") - .args(["build", "-q", "-t", AGENT_IMAGE_TAG, "."]) + .args(["build", "-q", "-t", image_tag, "."]) .current_dir(ctx.path()) .stderr(Stdio::inherit()) .status() .await .context("spawning podman build")?; if !status.success() { - anyhow::bail!("podman build for harmony-fleet-agent failed"); + anyhow::bail!("podman build for {crate_name} failed"); } - let tar_path = std::env::temp_dir().join(format!( - "harmony-fleet-agent-image-{}.tar", - std::process::id() - )); + let tar_path = + std::env::temp_dir().join(format!("{crate_name}-image-{}.tar", std::process::id())); let _ = std::fs::remove_file(&tar_path); let status = tokio::process::Command::new("podman") - .args(["save", "-o", tar_path.to_str().unwrap(), AGENT_IMAGE_TAG]) + .args(["save", "-o", tar_path.to_str().unwrap(), image_tag]) .status() .await .context("podman save")?; if !status.success() { - anyhow::bail!("podman save for harmony-fleet-agent failed"); + anyhow::bail!("podman save for {crate_name} failed"); } - tracing::info!("k3d image import {AGENT_IMAGE_TAG} (cluster={cluster_name})"); + tracing::info!("k3d image import {image_tag} (cluster={cluster_name})"); let tar_path_str = tar_path.to_str().unwrap().to_string(); let cluster_for_blocking = cluster_name.to_string(); let data_dir = k3d_data_dir(); @@ -199,6 +147,12 @@ ENTRYPOINT ["/usr/local/bin/harmony-fleet-agent"] tar_path_str.as_str(), "-c", cluster_for_blocking.as_str(), + // The default tools-node mode can fail to copy the tar + // (docker: "mkdirat var/run: file exists") while still + // exiting 0 — the image never lands and pods ErrImagePull. + // Direct mode imports through each node's containerd. + "--mode", + "direct", ]) }) .await @@ -212,10 +166,6 @@ ENTRYPOINT ["/usr/local/bin/harmony-fleet-agent"] String::from_utf8_lossy(&output.stderr), ); } - // Keep the k3d helper warm in scope so the borrow checker is - // happy (k3d is referenced by the caller's bring-up; this just - // ensures we used its cluster name even if the binary path went - // through the blocking task). let _ = k3d; Ok(()) } diff --git a/fleet/harmony-fleet-e2e/src/lib.rs b/fleet/harmony-fleet-e2e/src/lib.rs index a7c2bb96..76d25f6d 100644 --- a/fleet/harmony-fleet-e2e/src/lib.rs +++ b/fleet/harmony-fleet-e2e/src/lib.rs @@ -13,12 +13,14 @@ //! [`FleetNatsScore`]: harmony_fleet_deploy::FleetNatsScore //! [`FleetAgentScore`]: harmony_fleet_deploy::FleetAgentScore +pub mod callout; mod images; mod kv_admin; mod namespace; mod stack; pub mod vm; +pub use callout::CalloutHandles; pub use images::{AGENT_IMAGE_TAG, Images, build_and_sideload}; pub use kv_admin::{AdminKv, AdminKvError, PhaseExpectation}; pub use namespace::{ENV_KEEP_NAMESPACE, NamespaceGuard}; diff --git a/fleet/harmony-fleet-e2e/src/stack.rs b/fleet/harmony-fleet-e2e/src/stack.rs index 696d509d..a4fff157 100644 --- a/fleet/harmony-fleet-e2e/src/stack.rs +++ b/fleet/harmony-fleet-e2e/src/stack.rs @@ -60,10 +60,12 @@ const NATS_DEVICE_PASS: &str = "e2e-device"; const NATS_ADMIN_USER: &str = "admin"; const NATS_ADMIN_PASS: &str = "e2e-admin"; -/// Auth mode the harness brings NATS up with. v1 supports `UserPass` -/// only; the `Callout` variant is reserved on the public API for the -/// follow-up PR that wires `NatsAuthCalloutScore` + mock-OIDC behind -/// it. +/// Auth mode the harness brings NATS up with. +/// +/// `Callout` deploys the full production auth fabric: Zitadel (stable +/// `zitadel` namespace, reused across runs), the NATS auth-callout +/// service, and OpenBao with JWT auth bound to the Zitadel issuer. +/// See [`crate::callout`] for the network plumbing notes. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum AuthMode { #[default] @@ -136,8 +138,14 @@ pub enum BringUpError { name: String, detail: String, }, - #[error("AuthMode::Callout is reserved for PR 1.5 (mock OIDC fixture + NatsAuthCalloutScore)")] - CalloutNotYetImplemented, + #[error("callout auth fabric: {0}")] + Callout(#[source] anyhow::Error), + #[error( + "cluster '{CLUSTER_NAME}' exists without the port mappings callout mode needs \ + ({missing}); delete it (`k3d cluster delete {CLUSTER_NAME}`) and re-run so the \ + harness recreates it with the full mapping set" + )] + MissingPortMappings { missing: String }, #[error("connecting host-side NATS admin client to {url}: {detail}")] NatsConnect { url: String, detail: String }, } @@ -152,6 +160,9 @@ pub struct Stack { pub nats_client: async_nats::Client, pub admin_user: String, pub admin_pass: String, + /// Auth fabric handles — `Some` iff brought up with + /// [`AuthMode::Callout`]. + pub callout: Option, _guard: NamespaceGuard, } @@ -170,11 +181,22 @@ impl Stack { /// runs through the full deploy. For shared-across-tests bring-up /// use [`shared_stack`]. pub async fn bring_up(opts: StackOptions) -> Result { - if opts.auth_mode == AuthMode::Callout { - return Err(BringUpError::CalloutNotYetImplemented); + let callout_mode = opts.auth_mode == AuthMode::Callout; + // Static user-pass material is what the Pod agent + operator + // paths inject today; behind the callout they'd be rejected + // at connect. The VM harness brings callout-authenticated + // agents instead. + if callout_mode && (opts.num_devices > 0 || opts.deploy_operator) { + return Err(BringUpError::Callout(anyhow::anyhow!( + "callout mode supports infra-only bring-up (VM agents); \ + Pod agents / operator still authenticate with static creds" + ))); } let k3d = ensure_k3d_cluster().await?; + if callout_mode { + verify_callout_port_mappings().await?; + } let topology = K8sAnywhereTopology::from_env(); harmony::topology::Topology::ensure_ready(&topology) .await @@ -190,6 +212,11 @@ impl Stack { .await .map_err(BringUpError::ImageBuild)?; } + if callout_mode { + crate::images::build_and_load_callout_image(&k3d, CLUSTER_NAME) + .await + .map_err(BringUpError::ImageBuild)?; + } // Separate raw `kube::Client` for namespace lifecycle + the // readiness probes below. The topology's `K8sClient` wrapper @@ -209,6 +236,19 @@ impl Stack { prune_stale_operator_cluster_resources(&kube_client).await; } + // Zitadel + project/roles/admin identity. Sits outside the + // per-bring-up namespace (stable `zitadel` ns) so the + // multi-minute first install amortizes across runs. + let mut callout_handles = if callout_mode { + Some( + crate::callout::bring_up_zitadel(&topology, &kube_client) + .await + .map_err(BringUpError::Callout)?, + ) + } else { + None + }; + let namespace = unique_namespace(); let guard = NamespaceGuard::create(kube_client.clone(), namespace.clone()) .await @@ -226,32 +266,60 @@ impl Stack { // Dev credentials live here in the e2e crate, not in the // deploy crate — keeping them out of `harmony-fleet-deploy` // means production builds can't accidentally ship them. - let nats = FleetNatsScore::user_pass( - namespace.clone(), - NATS_NODE_PORT, - UserPassCredentials { - admin_user: NATS_ADMIN_USER.to_string(), - admin_pass: NATS_ADMIN_PASS.to_string(), - device_user: NATS_DEVICE_USER.to_string(), - device_pass: NATS_DEVICE_PASS.to_string(), - }, - ) - .release_name(NATS_RELEASE); - nats.interpret(&Inventory::autoload(), &topology) - .await - .map_err(BringUpError::NatsDeploy)?; - - // Wait for NATS to be reachable from the host BEFORE - // deploying the agent. Without this, the agent comes up, - // can't connect, hits CrashLoopBackOff for a while, and the - // agent Deployment-Ready wait spuriously times out. let nats_url = format!("nats://127.0.0.1:{NATS_NODE_PORT}"); - let nats_client = connect_nats_admin(&nats_url).await?; + let (nats_client, nats_in_cluster_url) = match callout_handles.as_mut() { + None => { + let nats = FleetNatsScore::user_pass( + namespace.clone(), + NATS_NODE_PORT, + UserPassCredentials { + admin_user: NATS_ADMIN_USER.to_string(), + admin_pass: NATS_ADMIN_PASS.to_string(), + device_user: NATS_DEVICE_USER.to_string(), + device_pass: NATS_DEVICE_PASS.to_string(), + }, + ) + .release_name(NATS_RELEASE); + nats.interpret(&Inventory::autoload(), &topology) + .await + .map_err(BringUpError::NatsDeploy)?; + // Wait for NATS to be reachable from the host BEFORE + // deploying the agent. Without this, the agent comes + // up, can't connect, hits CrashLoopBackOff for a + // while, and the agent Deployment-Ready wait + // spuriously times out. + (connect_nats_admin(&nats_url).await?, nats.in_cluster_url()) + } + Some(handles) => { + let nats = crate::callout::deploy_nats_callout( + &topology, + &namespace, + NATS_RELEASE, + NATS_NODE_PORT, + handles, + ) + .await + .map_err(BringUpError::Callout)?; + wait_for_deployment( + &kube_client, + &namespace, + crate::callout::callout_deployment_name(), + ) + .await?; + let client = crate::callout::connect_nats_admin_sso(&nats_url, handles) + .await + .map_err(BringUpError::Callout)?; + crate::callout::deploy_openbao(&topology, &namespace, handles) + .await + .map_err(BringUpError::Callout)?; + (client, nats.in_cluster_url()) + } + }; if opts.deploy_operator { let mut operator = FleetOperatorScore::new() .namespace(namespace.clone()) .image(opts.operator_image.clone()) - .nats_url(nats.in_cluster_url()) + .nats_url(nats_in_cluster_url.clone()) .log_level(opts.log_level.clone()); operator.credentials = Some(OperatorCredentials { @@ -288,7 +356,7 @@ impl Stack { PodTarget::user_pass( device_id.clone(), opts.agent_image.clone(), - nats.in_cluster_url(), + nats_in_cluster_url.clone(), NATS_DEVICE_USER, NATS_DEVICE_PASS, ) @@ -316,10 +384,26 @@ impl Stack { nats_client, admin_user: NATS_ADMIN_USER.to_string(), admin_pass: NATS_ADMIN_PASS.to_string(), + callout: callout_handles, _guard: guard, }) } + /// Dial a fresh admin NATS connection on the **caller's** runtime. + /// The shared `nats_client` rides the runtime of whichever + /// `#[tokio::test]` ran bring-up; when that test finishes, its + /// runtime — and the connection driver task — die with it. Any + /// test that can start after another test completed must connect + /// its own client. + pub async fn connect_admin(&self) -> Result { + match self.callout.as_ref() { + None => connect_nats_admin(&self.nats_url).await, + Some(handles) => crate::callout::connect_nats_admin_sso(&self.nats_url, handles) + .await + .map_err(BringUpError::Callout), + } + } + pub fn print_debug_info(&self) { eprintln!("[e2e] namespace: {}", self.namespace); eprintln!( @@ -364,8 +448,21 @@ async fn ensure_k3d_cluster() -> Result { .unwrap_or_else(|| std::path::PathBuf::from("/tmp/harmony")); std::fs::create_dir_all(&data_dir) .map_err(|e| BringUpError::K3dEnsure(format!("creating k3d data dir {data_dir:?}: {e}")))?; + // The callout-mode mappings ride along on every cluster create so + // one cluster serves both auth modes: the Zitadel and OpenBao + // NodePorts (see `callout.rs` for the issuer-port invariant). + // Mappings are create-time-only in k3d — + // `verify_callout_port_mappings` catches clusters predating them. let k3d = K3d::new(data_dir, Some(CLUSTER_NAME.to_string())).with_port_mappings(vec![ PortMapping::new(NATS_NODE_PORT as u32, NATS_NODE_PORT as u32), + PortMapping::new( + crate::callout::ZITADEL_PORT as u32, + crate::callout::ZITADEL_PORT as u32, + ), + PortMapping::new( + crate::callout::OPENBAO_NODE_PORT as u32, + crate::callout::OPENBAO_NODE_PORT as u32, + ), ]); k3d.ensure_installed() .await @@ -381,10 +478,59 @@ async fn ensure_k3d_cluster() -> Result { ); std::env::set_var("HARMONY_USE_LOCAL_K3D", "false"); std::env::set_var("HARMONY_AUTOINSTALL", "false"); + // Scores that persist credentials (ZitadelScore's admin + // password / masterkey) go through harmony_secret's global + // manager, which defaults to Infisical and panics unconfigured. + // The harness pins the local-file store + a namespace (the + // examples set these via env.sh); don't override a + // developer's explicit choice. + if std::env::var("HARMONY_SECRET_STORE").is_err() { + std::env::set_var("HARMONY_SECRET_STORE", "file"); + } + if std::env::var("HARMONY_SECRET_NAMESPACE").is_err() { + std::env::set_var("HARMONY_SECRET_NAMESPACE", "fleet-e2e"); + } } Ok(k3d) } +/// Callout mode needs host ports that only exist if the cluster was +/// created with the full mapping set (k3d mappings are create-time +/// fixed). The mappings live on the `serverlb` docker container — +/// probe it and fail with an actionable message instead of letting +/// Zitadel readiness time out 5 minutes later. +async fn verify_callout_port_mappings() -> Result<(), BringUpError> { + let output = tokio::process::Command::new("docker") + .args(["port", &format!("k3d-{CLUSTER_NAME}-serverlb")]) + .output() + .await; + let Ok(output) = output else { + // No docker CLI (or not the docker runtime) — skip the + // preflight; the Zitadel readiness wait will surface it. + tracing::warn!("docker CLI unavailable; skipping callout port-mapping preflight"); + return Ok(()); + }; + if !output.status.success() { + return Ok(()); // cluster not created yet — ensure_installed handles it + } + let ports = String::from_utf8_lossy(&output.stdout); + let missing: Vec = [ + crate::callout::ZITADEL_PORT, + crate::callout::OPENBAO_NODE_PORT, + ] + .into_iter() + .filter(|p| !ports.contains(&format!(":{p}"))) + .map(|p| p.to_string()) + .collect(); + if missing.is_empty() { + Ok(()) + } else { + Err(BringUpError::MissingPortMappings { + missing: missing.join(", "), + }) + } +} + /// Delete every namespace tagged `harmony.io/managed-by=fleet-e2e` /// and **wait** for them to fully terminate before returning. The /// wait is required: NATS uses a fixed NodePort (cluster-scoped diff --git a/fleet/harmony-fleet-e2e/src/vm/device.rs b/fleet/harmony-fleet-e2e/src/vm/device.rs index 94ca1c25..0cd0a690 100644 --- a/fleet/harmony-fleet-e2e/src/vm/device.rs +++ b/fleet/harmony-fleet-e2e/src/vm/device.rs @@ -23,8 +23,9 @@ use std::sync::Arc; use harmony::inventory::Inventory; use harmony::modules::fleet::{ - FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HARMONY_FLEET_POOL_NAME, - ProvisionVmScore, ensure_fleet_ssh_keypair, ensure_harmony_fleet_pool, read_public_key, + DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, + HARMONY_FLEET_POOL_NAME, HostsEntry, ProvisionVmScore, ensure_fleet_ssh_keypair, + ensure_harmony_fleet_pool, read_public_key, }; use harmony::modules::kvm::KvmVirtualMachineHost; use harmony::modules::kvm::config::init_executor; @@ -113,11 +114,14 @@ pub struct VmDeviceOptions { /// reachable from the VM (typically `nats:// /// :`). pub nats_url: String, - /// Static NATS creds the FleetNatsScore deploys (the device - /// user). Matches `nats::NATS_DEVICE_USER` / `NATS_DEVICE_PASS` - /// in `stack.rs` for the user-pass mode. - pub nats_user: String, - pub nats_pass: String, + /// On-device auth: static user-pass for the dev stack, a + /// pre-resolved Zitadel machine key in callout mode. + pub auth: FleetDeviceAuth, + /// `/etc/hosts` entries for names the VM can't otherwise resolve + /// (the Zitadel issuer in callout mode). + pub hosts_entries: Vec, + /// `[openbao]` section for the agent — callout mode only. + pub openbao: Option, /// Pre-built aarch64 agent binary path. The harness builds it /// once and passes the path into every device it provisions. pub agent_binary: PathBuf, @@ -219,12 +223,10 @@ impl VmDevice { device_id: Id::from(opts.device_id.clone()), labels: opts.labels.clone(), nats_urls: vec![opts.nats_url.clone()], - auth: FleetDeviceAuth::TomlShared { - nats_user: opts.nats_user.clone(), - nats_pass: opts.nats_pass.clone(), - }, + auth: opts.auth.clone(), agent_binary_path: opts.agent_binary.clone(), - hosts_entries: vec![], + hosts_entries: opts.hosts_entries.clone(), + openbao: opts.openbao.clone(), }); let interpret = Score::::create_interpret(&setup_score); interpret diff --git a/fleet/harmony-fleet-e2e/src/vm/stack.rs b/fleet/harmony-fleet-e2e/src/vm/stack.rs index c93e8938..033d179a 100644 --- a/fleet/harmony-fleet-e2e/src/vm/stack.rs +++ b/fleet/harmony-fleet-e2e/src/vm/stack.rs @@ -26,7 +26,9 @@ use tokio::process::Command; use tokio::sync::OnceCell; use uuid::Uuid; -use crate::stack::{BringUpError, NATS_NODE_PORT, Stack, StackOptions, shared_stack}; +use harmony::modules::fleet::{DeviceOpenbao, FleetDeviceAuth, HostsEntry}; + +use crate::stack::{AuthMode, BringUpError, NATS_NODE_PORT, Stack, StackOptions, shared_stack}; use crate::vm::agent_build::{AgentBuildError, build_agent_for}; use crate::vm::device::{VmDevice, VmDeviceError, VmDeviceOptions}; use crate::vm::network::{NetworkLookupError, libvirt_default_gateway_ip}; @@ -70,6 +72,16 @@ pub struct VmStackOptions { /// Empty by default; the FleetDeviceSetupScore is content with /// no labels (the device is targetable by `device-id` only). pub labels: BTreeMap, + /// NATS auth fabric for the whole stack. `Callout` enrolls each + /// VM as a Zitadel machine user (granted `device_roles`), points + /// the agent's `[openbao]` at the e2e OpenBao, and adds the + /// issuer `/etc/hosts` entry. + pub auth_mode: AuthMode, + /// Zitadel project roles granted to every enrolled device + /// (callout mode). Must include the callout's device role for + /// NATS permissions; `edge-*` roles double as OpenBao groups + /// (ADR-025 roles-as-groups). + pub device_roles: Vec, } impl Default for VmStackOptions { @@ -78,6 +90,8 @@ impl Default for VmStackOptions { num_devices: 1, arch: VmArchitecture::Aarch64, labels: BTreeMap::new(), + auth_mode: AuthMode::UserPass, + device_roles: vec![crate::callout::DEVICE_ROLE.to_string()], } } } @@ -119,6 +133,8 @@ pub enum VmBringUpError { #[source] source: VmDeviceError, }, + #[error("Zitadel device enrollment: {0}")] + Enroll(#[source] anyhow::Error), } #[derive(Debug, Error)] @@ -136,6 +152,8 @@ pub enum VmReadyError { #[source] source: CommandError, }, + #[error("admin NATS connect: {0}")] + Connect(#[from] BringUpError), } /// Brought-up fleet with NATS in k3d + N agents on VMs. @@ -175,7 +193,11 @@ impl VmStack { // 1. Infra (NATS, namespace, admin client). num_devices=0 // skips Pod-agent deploys — the VM agents take their // place. - let infra = shared_stack(StackOptions::infra_only()).await?; + let infra = shared_stack(StackOptions { + auth_mode: opts.auth_mode, + ..StackOptions::infra_only() + }) + .await?; // 2. Build the agent binary for the requested guest arch. // aarch64 cross-builds; x86_64 takes the host's native @@ -200,6 +222,43 @@ impl VmStack { // test never collides with a fresh provision. let vm_name = format!("fleet-e2e-vm-{run_id}-{i:02}"); let device_id = format!("vm-device-{run_id}-{i:02}"); + + let (auth, hosts_entries, openbao) = match infra.callout.as_ref() { + None => ( + FleetDeviceAuth::TomlShared { + nats_user: NATS_DEVICE_USER.to_string(), + nats_pass: NATS_DEVICE_PASS.to_string(), + }, + vec![], + None, + ), + Some(handles) => { + let machine_key_json = + crate::callout::enroll_device(handles, &device_id, &opts.device_roles) + .await + .map_err(VmBringUpError::Enroll)?; + ( + FleetDeviceAuth::ZitadelJwt { + machine_key_json, + oidc_issuer_url: handles.issuer_url.clone(), + audience: handles.project_id.clone(), + danger_accept_invalid_certs: false, + }, + // The issuer hostname resolves to the libvirt + // gateway, behind which the k3d port mapping + // serves the Zitadel ingress. + vec![HostsEntry { + ip: gateway.to_string(), + hostname: crate::callout::ZITADEL_HOST.to_string(), + }], + Some(DeviceOpenbao { + url: format!("http://{gateway}:{}", crate::callout::OPENBAO_NODE_PORT), + secret_prefix: crate::callout::SECRET_PREFIX.to_string(), + }), + ) + } + }; + let device = VmDevice::provision(VmDeviceOptions { device_id, vm_name: vm_name.clone(), @@ -208,8 +267,9 @@ impl VmStack { network: LIBVIRT_NETWORK.to_string(), labels: opts.labels.clone(), nats_url: vm_nats_url.clone(), - nats_user: NATS_DEVICE_USER.to_string(), - nats_pass: NATS_DEVICE_PASS.to_string(), + auth, + hosts_entries, + openbao, agent_binary: agent_binary.clone(), cpus: DEFAULT_VM_CPUS, memory_mib: DEFAULT_VM_MEMORY_MIB, @@ -249,7 +309,10 @@ impl VmStack { /// by retrying ping with a 3 s per-attempt timeout up to /// `per_device_budget`. pub async fn wait_until_ready(&self, per_device_budget: Duration) -> Result<(), VmReadyError> { - let client = FleetCommandsClient::new(self.infra.nats_client.clone()) + // Fresh connection on the caller's runtime — see + // `Stack::connect_admin` for why the shared client can't be + // trusted across tests. + let client = FleetCommandsClient::new(self.infra.connect_admin().await?) .with_timeout(Duration::from_secs(3)); for device in &self.devices { let id = device.device_id.to_string(); diff --git a/fleet/harmony-fleet-e2e/tests/openbao_groups.rs b/fleet/harmony-fleet-e2e/tests/openbao_groups.rs new file mode 100644 index 00000000..98b56602 --- /dev/null +++ b/fleet/harmony-fleet-e2e/tests/openbao_groups.rs @@ -0,0 +1,346 @@ +//! ADR-025 group enforcement, end to end against a real OpenBao: a +//! device's secret access is decided by the groups in its signed JWT +//! and the deployment policies attached to those groups. Self-contained +//! — the JWT auth mount validates against a test RSA key instead of a +//! live Zitadel, and tests mint their own device tokens. + +use harmony::inventory::Inventory; +use harmony::modules::fleet::secret_access::DeploymentSecretGrants; +use harmony::modules::openbao::{ + OpenBaoDeploymentSecretGrants, OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, + OpenbaoSetupScore, cached_root_token, +}; +use harmony::score::Score; +use harmony::topology::{K8sAnywhereTopology, K8sclient}; +use harmony_fleet_e2e::{StackOptions, shared_stack}; +use harmony_reconciler_contracts::DeploymentName; +use serde_json::json; + +const E2E_ENV: &str = "HARMONY_FLEET_E2E"; +const RELEASE: &str = "openbao-groups-test"; +const KV_MOUNT: &str = "secret"; +const FLEET_NS: &str = "fleet-e2e"; +const ISSUER: &str = "https://fleet-e2e.test"; +const AUDIENCE: &str = "fleet-e2e-project"; +const ROLE: &str = "fleet-device"; +const JWT_MOUNT: &str = "jwt"; + +// Test-only RSA keypair: lets the suite mint "Zitadel" tokens without a +// live IdP. Never used outside this test. +const TEST_JWT_PRIVATE_PEM: &str = r#"-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC0VwVrePTYWsym +J+MIOToktME/60/nYFhLi0awgpF5T6McF2TJ/v+Bo1KQQ5Fg53XRGZuFnSKr5WZ+ +KzJ7d9YpUSCc/9UMcd3ujhrp35X4rXOlCOqIklRXb/TTmxrl5LMr91zlHFOSxa+/ +N5yNYKQVsy9bHeJvOpX5ktWELVS2JoAbvvuVhsuWrFyAS9s3i4a2q1jkg9f+vyPc +BtExIw/Oe5L03x6ZQlPNPLh/c3uQlK7ikOB1877IAWK7/Nnj+L8FPmG9FmLp/CeZ +n6gsqeDq6KloyffCx0ZiyK3ijz2wIHIYo3SVe3jyDjRJLiBIJrcxynq4aid3OGDP +Tw13tshPAgMBAAECggEAFpEhK366XOyX2V76aQgIP5mi6SXAHPsQcXufP+1G2QBf +L9+5jBFt1cIjTcw2wAKW2rCypniwYwRU3Ar5yTuIwCjXxO26slD5baW4tnV1JeZG +p2tDNg3hogNvhp9NMWF6DlFo/X6a8q3sPShvXb2vKsv+bA+ES1jwdYerUlv90sEu +PzW4GcBPCQ19udZEIM1Y2bObKi6bAT3QAANkzQC7kRhV104l7rC9eHhfbaW1Ce9N +rPY2Ekq2ck3kuEz+YrZ0Lh3yvozslK8aPn05ZM1sYv+9Mxb0rwv4LqGdQfNvONbA +ZMLL++chHqTSr5tZ26POLjZ5ZTm/jlOK0ZJ5RE6ZQQKBgQD5JW3U2vtntGl6V2yj +zvDNpXeKNh2CBMB5cuub+G0J2XtAJXW4W3zsGytwNdmrc/IVVJI9CeJ/g+2PvI7d +fiUvSSDZyrvib6+kGj5qDLHEiLJ0LhqEP4/jzHcKoL1lID63C65Sw0ZhKgy/lPXn +elBcuzDIaI+9sre6A9TdnL1vrwKBgQC5TQkIh0aI9PFq184Ugd/Qaj8ckjAbu5ms +4c4cZIVvENFIjSwdk3/p88AP8UmRMTIUUViZ2rvZYJkVUzi15AFbti4sYB5IXTc6 +D00Ev+SYW/b1bf/Gu8o/c2UWrNY1823xwXjW0bRT0O9nJJhSDs7GCDOCL8pIDWNd +3xJWWea5YQKBgQDo6sjm+WGW7d699QUnjssoNcyvdS/Y4UBfScRiJ7koQB0tNtf6 +5ey/fhb57B32X4qtzkZ4sckBlQB+I7Ub9FYnAqKneRR0+BynubEny35VUcYjPwaj +JBkMyZVwZh0pcR+wWEc8MDqXk4wffs70/DAswQM2GCppGa8RQ4u5MlMXGQKBgC2J +2xB3fkQmPjZNtt2fVUu48K2q15vZLNkp3SdpJMm4zlYQK6lRGHx8ZNUGLVg7SN1p +aGbXNJclgDmXbDZ3REYvrYVGUerizbG9FOZ36kvvPe/vCo7ik7+oMk2ySCqWGozx +SSsx+iknDc7BVZXFAdf1KAN0jzfJ4oOtUKpcCbHBAoGBAJGyuQUswc7EQeYLn/H5 +Zl3kRQv/VU8+lW0tro//osD2Cl5sW1YRaU0+QsOHy/GDPRqcWEqGfyUmFvTK09vc +vSYCbIcS2U18wo7I/j/vicqou0vnNeZCOQHrYvHurGLNUQKqVmVuDArsHx9aDIH0 +lJT0SeBwt1iqJi1lIPtEQbOy +-----END PRIVATE KEY-----"#; + +const TEST_JWT_PUBLIC_PEM: &str = r#"-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtFcFa3j02FrMpifjCDk6 +JLTBP+tP52BYS4tGsIKReU+jHBdkyf7/gaNSkEORYOd10RmbhZ0iq+Vmfisye3fW +KVEgnP/VDHHd7o4a6d+V+K1zpQjqiJJUV2/005sa5eSzK/dc5RxTksWvvzecjWCk +FbMvWx3ibzqV+ZLVhC1UtiaAG777lYbLlqxcgEvbN4uGtqtY5IPX/r8j3AbRMSMP +znuS9N8emUJTzTy4f3N7kJSu4pDgdfO+yAFiu/zZ4/i/BT5hvRZi6fwnmZ+oLKng +6uipaMn3wsdGYsit4o89sCByGKN0lXt48g40SS4gSCa3Mcp6uGondzhgz08Nd7bI +TwIDAQAB +-----END PUBLIC KEY-----"#; + +fn e2e_enabled() -> bool { + matches!(std::env::var(E2E_ENV).as_deref(), Ok("1" | "true")) +} + +fn dn(s: &str) -> DeploymentName { + DeploymentName::try_new(s).expect("valid deployment name") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn group_grants_govern_device_secret_access() -> anyhow::Result<()> { + if !e2e_enabled() { + eprintln!("skipping {E2E_ENV}-gated OpenBao e2e (set {E2E_ENV}=1; requires k3d + helm)"); + return Ok(()); + } + + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .try_init(); + + let stack = shared_stack(StackOptions { + deploy_agent: false, + deploy_operator: false, + ..StackOptions::infra_only() + }) + .await?; + stack.print_debug_info(); + + let instance = OpenbaoInstance { + namespace: stack.namespace.clone(), + release: RELEASE.to_string(), + }; + let bao = deploy_openbao(&instance).await?; + + // The unit under test: deployment "web" readable by group edge-a. + let grants = OpenBaoDeploymentSecretGrants::new( + bao.base_url.clone(), + bao.root_token.clone(), + KV_MOUNT.to_string(), + FLEET_NS.to_string(), + ); + grants + .set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()])]) + .await?; + + // Device in edge-a: login binds the group, secret readable, batch token. + let login_a = bao.jwt_login(&mint_jwt("pi-01", &["edge-a"])).await?; + assert_eq!( + login_a.token_type, "batch", + "fleet role must issue batch tokens" + ); + bao.assert_can_read(&login_a.token, "web/config").await?; + + // Device in another group: valid identity, no grant. + let login_b = bao.jwt_login(&mint_jwt("pi-02", &["edge-b"])).await?; + bao.assert_denied(&login_b.token, "web/config").await?; + + // Detach binds at request time: the in-hand token loses access with + // no re-login involved. + grants.set_deployment_groups(&[(dn("web"), vec![])]).await?; + bao.assert_denied(&login_a.token, "web/config").await?; + + // Re-attach binds at request time too. + grants + .set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()])]) + .await?; + bao.assert_can_read(&login_a.token, "web/config").await?; + + // Group membership is read from the claim only at login: pi-02's old + // token keeps its old groups; a fresh login with edge-a membership + // gets access. + bao.assert_denied(&login_b.token, "web/config").await?; + let login_b2 = bao.jwt_login(&mint_jwt("pi-02", &["edge-a"])).await?; + bao.assert_can_read(&login_b2.token, "web/config").await?; + + // Per-device kill switch: disabling the entity blocks its in-flight + // batch token at request time (ADR-025-3 open item 1). + bao.set_entity_disabled("pi-01", true).await?; + bao.assert_denied(&login_a.token, "web/config").await?; + bao.set_entity_disabled("pi-01", false).await?; + bao.assert_can_read(&login_a.token, "web/config").await?; + + Ok(()) +} + +/// Deploys OpenBao + JWT auth configured for the test keypair, seeds the +/// deployment secret, and port-forwards 8200 for host-side HTTP. +async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result { + let topology = K8sAnywhereTopology::from_env(); + // Raw `Score::interpret` skips topology preparation; without this the + // k8s client is never initialized (`ensure_ready` is what the CLI / + // stack bring-up normally run). + harmony::topology::Topology::ensure_ready(&topology) + .await + .map_err(|e| anyhow::anyhow!("topology not ready: {e:?}"))?; + let scores: Vec>> = vec![ + Box::new(OpenbaoScore { + instance: instance.clone(), + host: "openbao-groups-test.local".to_string(), + openshift: false, + tls_issuer: None, + }), + Box::new(OpenbaoSetupScore { + instance: instance.clone(), + kv_mount: KV_MOUNT.to_string(), + policies: vec![], + users: vec![], + jwt_auth: Some(OpenbaoJwtAuth { + oidc_discovery_url: String::new(), + jwt_validation_pubkeys: TEST_JWT_PUBLIC_PEM.to_string(), + bound_issuer: ISSUER.to_string(), + role_name: ROLE.to_string(), + bound_audiences: AUDIENCE.to_string(), + user_claim: "sub".to_string(), + groups_claim: "groups".to_string(), + token_type: "batch".to_string(), + ttl: "1h".to_string(), + max_ttl: "1h".to_string(), + }), + }), + ]; + for score in scores { + score.interpret(&Inventory::autoload(), &topology).await?; + } + + let k8s = topology.k8s_client().await.map_err(anyhow::Error::msg)?; + let forward = k8s + .port_forward(&instance.pod(), &instance.namespace, 0, 8200) + .await?; + let bao = Bao { + base_url: format!("http://127.0.0.1:{}", forward.port()), + root_token: cached_root_token(instance).map_err(anyhow::Error::msg)?, + http: reqwest::Client::new(), + _forward: forward, + }; + + bao.http + .post(format!( + "{}/v1/{KV_MOUNT}/data/{FLEET_NS}/web/config", + bao.base_url + )) + .header("X-Vault-Token", &bao.root_token) + .json(&json!({ "data": { "value": "e2e" } })) + .send() + .await? + .error_for_status()?; + Ok(bao) +} + +struct Bao { + base_url: String, + root_token: String, + http: reqwest::Client, + _forward: harmony_k8s::PortForwardHandle, +} + +struct Login { + token: String, + token_type: String, +} + +impl Bao { + async fn jwt_login(&self, jwt: &str) -> anyhow::Result { + let body: serde_json::Value = self + .http + .post(format!("{}/v1/auth/{JWT_MOUNT}/login", self.base_url)) + .json(&json!({ "role": ROLE, "jwt": jwt })) + .send() + .await? + .error_for_status()? + .json() + .await?; + let auth = &body["auth"]; + Ok(Login { + token: auth["client_token"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("login response missing client_token: {body}"))? + .to_string(), + token_type: auth["token_type"].as_str().unwrap_or_default().to_string(), + }) + } + + async fn read_secret(&self, token: &str, rel: &str) -> anyhow::Result { + Ok(self + .http + .get(format!( + "{}/v1/{KV_MOUNT}/data/{FLEET_NS}/{rel}", + self.base_url + )) + .header("X-Vault-Token", token) + .send() + .await? + .status()) + } + + async fn assert_can_read(&self, token: &str, rel: &str) -> anyhow::Result<()> { + let status = self.read_secret(token, rel).await?; + anyhow::ensure!( + status.is_success(), + "expected read of {rel} to succeed, got {status}" + ); + Ok(()) + } + + async fn assert_denied(&self, token: &str, rel: &str) -> anyhow::Result<()> { + let status = self.read_secret(token, rel).await?; + anyhow::ensure!( + status == reqwest::StatusCode::FORBIDDEN, + "expected read of {rel} to be denied, got {status}" + ); + Ok(()) + } + + /// Resolve the device's entity through its JWT-mount alias and flip + /// `disabled` — the per-device kill switch. + async fn set_entity_disabled(&self, device: &str, disabled: bool) -> anyhow::Result<()> { + let accessor: serde_json::Value = self + .http + .get(format!("{}/v1/sys/auth", self.base_url)) + .header("X-Vault-Token", &self.root_token) + .send() + .await? + .error_for_status()? + .json() + .await?; + let accessor = accessor["data"][&format!("{JWT_MOUNT}/")]["accessor"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("jwt mount accessor not found"))? + .to_string(); + + let entity: serde_json::Value = self + .http + .post(format!("{}/v1/identity/lookup/entity", self.base_url)) + .header("X-Vault-Token", &self.root_token) + .json(&json!({ "alias_name": device, "alias_mount_accessor": accessor })) + .send() + .await? + .error_for_status()? + .json() + .await?; + let id = entity["data"]["id"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("entity for {device} not found: {entity}"))?; + + self.http + .post(format!("{}/v1/identity/entity/id/{id}", self.base_url)) + .header("X-Vault-Token", &self.root_token) + .json(&json!({ "disabled": disabled })) + .send() + .await? + .error_for_status()?; + Ok(()) + } +} + +fn mint_jwt(sub: &str, groups: &[&str]) -> String { + use jsonwebtoken::{Algorithm, EncodingKey, Header, encode}; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let claims = json!({ + "iss": ISSUER, + "aud": AUDIENCE, + "sub": sub, + "groups": groups, + "iat": now, + "exp": now + 300, + }); + encode( + &Header::new(Algorithm::RS256), + &claims, + &EncodingKey::from_rsa_pem(TEST_JWT_PRIVATE_PEM.as_bytes()).expect("test key parses"), + ) + .expect("test jwt signs") +} diff --git a/fleet/harmony-fleet-e2e/tests/openbao_policy.rs b/fleet/harmony-fleet-e2e/tests/openbao_policy.rs new file mode 100644 index 00000000..7b05102a --- /dev/null +++ b/fleet/harmony-fleet-e2e/tests/openbao_policy.rs @@ -0,0 +1,236 @@ +use std::sync::Arc; + +use harmony::inventory::Inventory; +use harmony::modules::openbao::{ + OpenbaoInstance, OpenbaoPolicy, OpenbaoScore, OpenbaoSetupScore, OpenbaoUser, cached_root_token, +}; +use harmony::score::Score; +use harmony::topology::{K8sAnywhereTopology, K8sclient}; +use harmony_fleet_e2e::{StackOptions, shared_stack}; + +const E2E_ENV: &str = "HARMONY_FLEET_E2E"; +const RELEASE: &str = "openbao-policy-test"; +const USER: &str = "policy-device"; +const PASSWORD: &str = "policy-device-pass"; +const POLICY: &str = "deployment-policy-test"; +const SECRET_PATH: &str = "fleet-staging/deployment-a"; + +fn e2e_enabled() -> bool { + matches!(std::env::var(E2E_ENV).as_deref(), Ok("1" | "true")) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn entity_policy_grants_existing_openbao_token_access() -> anyhow::Result<()> { + if !e2e_enabled() { + skip_e2e(); + return Ok(()); + } + + let stack = openbao_stack().await?; + stack.print_debug_info(); + + let instance = OpenbaoInstance { + namespace: stack.namespace.clone(), + release: RELEASE.to_string(), + }; + deploy_openbao(&instance).await?; + + let topology = K8sAnywhereTopology::from_env(); + harmony::topology::Topology::ensure_ready(&topology) + .await + .map_err(|e| anyhow::anyhow!("topology not ready: {e:?}"))?; + let k8s = topology.k8s_client().await.map_err(anyhow::Error::msg)?; + let root_token = cached_root_token(&instance).map_err(anyhow::Error::msg)?; + let user_token = login_user(&k8s, &instance).await?; + let entity_id = token_entity_id(&k8s, &instance, &user_token).await?; + + set_entity_policies(&k8s, &instance, &root_token, &entity_id, "").await?; + assert_secret_denied(&k8s, &instance, &user_token).await?; + + set_entity_policies(&k8s, &instance, &root_token, &entity_id, POLICY).await?; + assert_secret_allowed(&k8s, &instance, &user_token).await?; + + // FIXME this cleanup might not happen if the previous invocation fails + set_entity_policies(&k8s, &instance, &root_token, &entity_id, "").await?; + Ok(()) +} + +async fn openbao_stack() -> anyhow::Result> { + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .try_init(); + + Ok(shared_stack(StackOptions { + deploy_agent: false, + deploy_operator: false, + ..StackOptions::infra_only() + }) + .await?) +} + +async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result<()> { + let topology = K8sAnywhereTopology::from_env(); + // Raw `Score::interpret` skips topology preparation; without this the + // k8s client is never initialized (`ensure_ready` is what the CLI / + // stack bring-up normally run). + harmony::topology::Topology::ensure_ready(&topology) + .await + .map_err(|e| anyhow::anyhow!("topology not ready: {e:?}"))?; + let policy_hcl = format!( + r#"path "secret/data/{SECRET_PATH}" {{ capabilities = ["read"] }} +path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"# + ); + let scores: Vec>> = vec![ + Box::new(OpenbaoScore { + instance: instance.clone(), + host: "openbao-policy-test.local".to_string(), + openshift: false, + tls_issuer: None, + }), + Box::new(OpenbaoSetupScore { + instance: instance.clone(), + kv_mount: "secret".to_string(), + policies: vec![OpenbaoPolicy { + name: POLICY.to_string(), + hcl: policy_hcl, + }], + users: vec![OpenbaoUser { + username: USER.to_string(), + password: PASSWORD.to_string(), + policies: vec![], + }], + jwt_auth: None, + }), + ]; + + for score in scores { + score.interpret(&Inventory::autoload(), &topology).await?; + } + let k8s = topology.k8s_client().await.map_err(anyhow::Error::msg)?; + let root_token = cached_root_token(instance).map_err(anyhow::Error::msg)?; + bao( + &k8s, + instance, + &root_token, + "bao kv put secret/fleet-staging/deployment-a value=e2e", + ) + .await?; + Ok(()) +} + +async fn login_user( + k8s: &harmony_k8s::K8sClient, + instance: &OpenbaoInstance, +) -> anyhow::Result { + let out = exec( + k8s, + instance, + &format!("bao login -method=userpass -format=json username={USER} password={PASSWORD}"), + ) + .await?; + Ok( + serde_json::from_str::(&out)?["auth"]["client_token"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("login response missing auth.client_token: {out}"))? + .to_string(), + ) +} + +async fn token_entity_id( + k8s: &harmony_k8s::K8sClient, + instance: &OpenbaoInstance, + token: &str, +) -> anyhow::Result { + let out = bao(k8s, instance, token, "bao token lookup -format=json").await?; + Ok( + serde_json::from_str::(&out)?["data"]["entity_id"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("token lookup response missing data.entity_id: {out}"))? + .to_string(), + ) +} + +async fn set_entity_policies( + k8s: &harmony_k8s::K8sClient, + instance: &OpenbaoInstance, + root_token: &str, + entity_id: &str, + policies: &str, +) -> anyhow::Result<()> { + bao( + k8s, + instance, + root_token, + &format!("bao write identity/entity/id/{entity_id} policies={policies}"), + ) + .await?; + Ok(()) +} + +async fn assert_secret_denied( + k8s: &harmony_k8s::K8sClient, + instance: &OpenbaoInstance, + token: &str, +) -> anyhow::Result<()> { + let result = bao( + k8s, + instance, + token, + &format!("bao kv get secret/{SECRET_PATH}"), + ) + .await; + assert!(result.is_err(), "secret read unexpectedly succeeded"); + Ok(()) +} + +async fn assert_secret_allowed( + k8s: &harmony_k8s::K8sClient, + instance: &OpenbaoInstance, + token: &str, +) -> anyhow::Result<()> { + bao( + k8s, + instance, + token, + &format!("bao kv get secret/{SECRET_PATH}"), + ) + .await?; + Ok(()) +} + +async fn bao( + k8s: &harmony_k8s::K8sClient, + instance: &OpenbaoInstance, + token: &str, + command: &str, +) -> anyhow::Result { + exec( + k8s, + instance, + &format!("export VAULT_TOKEN={token} && {command}"), + ) + .await +} + +async fn exec( + k8s: &harmony_k8s::K8sClient, + instance: &OpenbaoInstance, + command: &str, +) -> anyhow::Result { + k8s.exec_pod_capture_output( + &instance.pod(), + Some(&instance.namespace), + vec!["sh", "-c", command], + ) + .await + .map_err(|e| anyhow::anyhow!(e)) +} + +fn skip_e2e() { + eprintln!( + "skipping {E2E_ENV}-gated OpenBao e2e test (set {E2E_ENV}=1 to run; requires k3d + helm)" + ); +} diff --git a/fleet/harmony-fleet-e2e/tests/operator.rs b/fleet/harmony-fleet-e2e/tests/operator.rs index 9d3c3c9e..2dcb6b22 100644 --- a/fleet/harmony-fleet-e2e/tests/operator.rs +++ b/fleet/harmony-fleet-e2e/tests/operator.rs @@ -127,6 +127,7 @@ async fn create_fleet_deployment(deployments: &Api, name: &str) -> a ..Default::default() }, spec: DeploymentSpec { + allowed_groups: None, target_selector: LabelSelector::default(), score: ReconcileScore::PodmanV0(PodmanV0Score { services: vec![PodmanService { @@ -134,6 +135,7 @@ async fn create_fleet_deployment(deployments: &Api, name: &str) -> a image: "docker.io/library/hello-world:latest".to_string(), ports: vec![], env: vec![], + secret_env: vec![], volumes: vec![], restart_policy: Default::default(), }], diff --git a/fleet/harmony-fleet-e2e/tests/vm_deploy_lifecycle.rs b/fleet/harmony-fleet-e2e/tests/vm_deploy_lifecycle.rs index c27fd552..a26e2fd7 100644 --- a/fleet/harmony-fleet-e2e/tests/vm_deploy_lifecycle.rs +++ b/fleet/harmony-fleet-e2e/tests/vm_deploy_lifecycle.rs @@ -182,6 +182,7 @@ fn podman_score(image_tag: &str) -> PodmanV0Score { image: format!("docker.io/library/{image_tag}"), ports: vec![], env: vec![], + secret_env: vec![], volumes: vec![], restart_policy: RestartPolicy::default(), }], diff --git a/fleet/harmony-fleet-e2e/tests/vm_isolation.rs b/fleet/harmony-fleet-e2e/tests/vm_isolation.rs index 88154fa1..ac546498 100644 --- a/fleet/harmony-fleet-e2e/tests/vm_isolation.rs +++ b/fleet/harmony-fleet-e2e/tests/vm_isolation.rs @@ -89,6 +89,7 @@ async fn agent_ignores_other_devices_keys() -> anyhow::Result<()> { image: "docker.io/library/nginx:alpine".to_string(), ports: vec![], env: vec![], + secret_env: vec![], volumes: vec![], restart_policy: RestartPolicy::default(), }], @@ -135,5 +136,6 @@ fn synthesize_agent_config(device_id: &str) -> AgentConfig { nats_pass: "e2e-device".to_string(), }, labels: BTreeMap::new(), + openbao: None, } } diff --git a/fleet/harmony-fleet-e2e/tests/vm_secret_access.rs b/fleet/harmony-fleet-e2e/tests/vm_secret_access.rs new file mode 100644 index 00000000..b5eabfb3 --- /dev/null +++ b/fleet/harmony-fleet-e2e/tests/vm_secret_access.rs @@ -0,0 +1,307 @@ +//! Full-stack device secret access (ADR-025), end to end with nothing +//! mocked: +//! +//! 1. the agent on a VM connects to NATS **through the auth callout** +//! with its Zitadel machine identity (the readiness ping proves the +//! whole SSO → callout → NATS chain); +//! 2. a deployment referencing a secret lands in desired state; +//! 3. the agent logs into OpenBao with the same Zitadel JWT — the +//! `groups` claim comes from the flattening Action, the group grant +//! from the same `OpenBaoDeploymentSecretGrants` the operator +//! composes — fetches the secret and injects it into the container +//! (SSH `podman inspect` is the ground truth); +//! 4. deleting the deployment's OpenBao policy makes the next fetch +//! fail, surfaced as `Phase::Failed` naming the denied secret. +//! +//! Gating: skipped unless `HARMONY_FLEET_VM_E2E=1`. + +use std::time::Duration; + +use harmony::modules::fleet::secret_access::DeploymentSecretGrants; +use harmony::modules::openbao::OpenBaoDeploymentSecretGrants; +use harmony::modules::podman::{EnvVar, PodmanService, PodmanV0Score, SecretEnvVar}; +use harmony::topology::{K8sAnywhereTopology, K8sclient, RestartPolicy}; +use harmony_fleet_e2e::callout::{DEVICE_ROLE, KV_MOUNT, SECRET_PREFIX}; +use harmony_fleet_e2e::{ + AdminKv, AuthMode, PhaseExpectation, VmStack, VmStackOptions, shared_vm_stack, +}; +use harmony_reconciler_contracts::{DeploymentName, Phase}; +use serde_json::json; + +const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E"; + +fn enabled() -> bool { + matches!(std::env::var(ENV_GATE).as_deref(), Ok("1" | "true")) +} + +fn dn(s: &str) -> DeploymentName { + DeploymentName::try_new(s).expect("test-static valid deployment name") +} + +/// First image pull on a TCG aarch64 guest can take minutes. +const RUN_BUDGET: Duration = Duration::from_secs(600); +const FAIL_BUDGET: Duration = Duration::from_secs(120); + +/// Both tests drive the same device + OpenBao; serialize so policy +/// surgery in one can't race the other's apply. +static SERIAL: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn callout_stack_options() -> VmStackOptions { + VmStackOptions { + auth_mode: AuthMode::Callout, + // `device` drives NATS permissions in the callout; `edge-a` + // is the OpenBao group the deployments below are granted to. + device_roles: vec![DEVICE_ROLE.to_string(), "edge-a".to_string()], + ..VmStackOptions::from_env() + } +} + +fn secret_score(service: &str, spec_marker: &str) -> PodmanV0Score { + PodmanV0Score { + services: vec![PodmanService { + name: service.to_string(), + image: "docker.io/library/nginx:alpine".to_string(), + ports: vec![], + // Changing the marker changes the serialized spec, which + // is how the tests force a re-apply (and with it a fresh + // secret resolution). + env: vec![EnvVar::new("SPEC_MARKER", spec_marker)], + secret_env: vec![SecretEnvVar { + name: "DB_PASSWORD".to_string(), + secret: "db_password".to_string(), + }], + volumes: vec![], + restart_policy: RestartPolicy::default(), + }], + } +} + +/// Host-side OpenBao admin surface: port-forward + root token. +struct BaoAdmin { + base_url: String, + root_token: String, + http: reqwest::Client, + _forward: harmony_k8s::PortForwardHandle, +} + +impl BaoAdmin { + async fn connect(stack: &VmStack) -> anyhow::Result { + let handles = stack + .infra + .callout + .as_ref() + .expect("callout stack carries OpenBao handles"); + let topology = K8sAnywhereTopology::from_env(); + harmony::topology::Topology::ensure_ready(&topology) + .await + .map_err(|e| anyhow::anyhow!("topology not ready: {e:?}"))?; + let k8s = topology.k8s_client().await.map_err(anyhow::Error::msg)?; + let forward = k8s + .port_forward(&handles.openbao.pod(), &handles.openbao.namespace, 0, 8200) + .await?; + Ok(Self { + base_url: format!("http://127.0.0.1:{}", forward.port()), + root_token: handles.openbao_root_token.clone(), + http: reqwest::Client::new(), + _forward: forward, + }) + } + + async fn put_secret( + &self, + deployment: &DeploymentName, + name: &str, + value: &str, + ) -> anyhow::Result<()> { + self.http + .post(format!( + "{}/v1/{KV_MOUNT}/data/{SECRET_PREFIX}/{deployment}/{name}", + self.base_url + )) + .header("X-Vault-Token", &self.root_token) + .json(&json!({ "data": { "value": value } })) + .send() + .await? + .error_for_status()?; + Ok(()) + } + + fn grants(&self) -> OpenBaoDeploymentSecretGrants { + OpenBaoDeploymentSecretGrants::new( + self.base_url.clone(), + self.root_token.clone(), + KV_MOUNT.to_string(), + SECRET_PREFIX.to_string(), + ) + } + + /// The negative test's act of sabotage: drop the deployment's + /// policy directly (operator drift / manual revocation), leaving + /// group attachments in place. + async fn delete_deployment_policy(&self, deployment: &DeploymentName) -> anyhow::Result<()> { + self.http + .delete(format!( + "{}/v1/sys/policies/acl/deployment-{deployment}", + self.base_url + )) + .header("X-Vault-Token", &self.root_token) + .send() + .await? + .error_for_status()?; + Ok(()) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn device_reads_deployment_secret_via_sso_groups() -> anyhow::Result<()> { + if !enabled() { + eprintln!("skipping {ENV_GATE}-gated VM e2e test (set {ENV_GATE}=1 to run)"); + return Ok(()); + } + let _serial = SERIAL.lock().await; + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .try_init(); + + let stack = shared_vm_stack(callout_stack_options()).await?; + stack.print_debug_info(); + // Ping over the callout-authenticated NATS connection — the agent + // is online only if its Zitadel JWT made it through the callout. + stack.wait_until_ready(Duration::from_secs(120)).await?; + + let device = stack.devices.first().expect("one VM device"); + let device_id = device.device_id.to_string(); + let deployment = dn("secret-web"); + let bao = BaoAdmin::connect(&stack).await?; + // Fresh per-test connection — the stack's shared client dies + // with the bring-up test's runtime (`Stack::connect_admin`). + let nats = stack.infra.connect_admin().await?; + let admin = AdminKv::connect(&nats).await?; + + let secret_value = format!("hunter2-{}", uuid::Uuid::new_v4().simple()); + bao.put_secret(&deployment, "db_password", &secret_value) + .await?; + bao.grants() + .set_deployment_groups(&[(deployment.clone(), vec!["edge-a".to_string()])]) + .await?; + + admin + .put_podman( + &device_id, + &deployment, + &secret_score("secret-web-svc", "v1"), + ) + .await?; + let state = admin + .wait_for_phase( + &device_id, + &deployment, + PhaseExpectation::running(), + RUN_BUDGET, + ) + .await? + .expect("Running phase implies a state entry"); + assert_eq!(state.phase, Phase::Running); + + // Ground truth: the secret VALUE made it into the container env. + let inspect = device + .ssh( + "sudo -iu fleet-agent podman inspect secret-web-svc \ + --format '{{json .Config.Env}}'", + ) + .await? + .into_successful() + .map_err(|e| anyhow::anyhow!("podman inspect failed: {e}"))?; + assert!( + inspect + .stdout + .contains(&format!("DB_PASSWORD={secret_value}")), + "container env must carry the fetched secret, got:\n{}", + inspect.stdout, + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn removed_policy_denies_secret_fetch() -> anyhow::Result<()> { + if !enabled() { + eprintln!("skipping {ENV_GATE}-gated VM e2e test (set {ENV_GATE}=1 to run)"); + return Ok(()); + } + let _serial = SERIAL.lock().await; + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .try_init(); + + let stack = shared_vm_stack(callout_stack_options()).await?; + stack.wait_until_ready(Duration::from_secs(120)).await?; + + let device = stack.devices.first().expect("one VM device"); + let device_id = device.device_id.to_string(); + let deployment = dn("secret-denied"); + let bao = BaoAdmin::connect(&stack).await?; + // Fresh per-test connection — the stack's shared client dies + // with the bring-up test's runtime (`Stack::connect_admin`). + let nats = stack.infra.connect_admin().await?; + let admin = AdminKv::connect(&nats).await?; + + bao.put_secret(&deployment, "db_password", "soon-revoked") + .await?; + bao.grants() + .set_deployment_groups(&[(deployment.clone(), vec!["edge-a".to_string()])]) + .await?; + + // Sanity: with the grant in place the deployment converges. + admin + .put_podman( + &device_id, + &deployment, + &secret_score("secret-denied-svc", "v1"), + ) + .await?; + admin + .wait_for_phase( + &device_id, + &deployment, + PhaseExpectation::running(), + RUN_BUDGET, + ) + .await?; + + // Remove the policy; the spec bump forces a re-apply and with it + // a fresh secret resolution (resolution binds at apply time — + // converged deployments never re-read, by design). + bao.delete_deployment_policy(&deployment).await?; + admin + .put_podman( + &device_id, + &deployment, + &secret_score("secret-denied-svc", "v2"), + ) + .await?; + let state = admin + .wait_for_phase( + &device_id, + &deployment, + PhaseExpectation::OneOf(vec![Phase::Failed]), + FAIL_BUDGET, + ) + .await? + .expect("Failed phase implies a state entry"); + assert_eq!(state.phase, Phase::Failed); + let err = state.last_error.unwrap_or_default(); + assert!( + err.contains("fetching secret") + && (err.contains("permission denied") || err.contains("403")), + "failure must name the denied secret fetch, got: {err}", + ); + + Ok(()) +} diff --git a/fleet/harmony-fleet-operator/src/fleet_aggregator.rs b/fleet/harmony-fleet-operator/src/fleet_aggregator.rs index 9eec10df..41c7cffb 100644 --- a/fleet/harmony-fleet-operator/src/fleet_aggregator.rs +++ b/fleet/harmony-fleet-operator/src/fleet_aggregator.rs @@ -33,12 +33,18 @@ use kube::{Client, ResourceExt}; use serde_json::json; use tokio::sync::Mutex; +use harmony::modules::fleet::groups::DeviceGroupSource; use harmony::modules::fleet::operator::{ AggregateLastError, Deployment, DeploymentAggregate, Device, }; +use harmony::modules::fleet::secret_access::DeploymentSecretGrants; use tracing::warn; const PATCH_TICK: Duration = Duration::from_secs(1); +/// How often the admin-managed group membership is re-fetched. Routine +/// convergence is bounded by the device token TTL anyway (ADR-025), so +/// a relaxed poll is fine. +const GROUP_POLL: Duration = Duration::from_secs(60); // --------------------------------------------------------------------------- // State @@ -74,6 +80,9 @@ pub struct DevicePair { pub struct CachedDeployment { key: DeploymentKey, deployment_name: DeploymentName, + /// `spec.allowedGroups` — the security gate. `None` = ungated (and + /// secretless); `Some` = only group members are schedulable. + allowed_groups: Option>, selector: LabelSelector, /// JSON-serialized score payload ready to `put` into /// desired-state. Cached because the same bytes are written to @@ -87,6 +96,11 @@ pub struct FleetState { deployments: HashMap, /// Cached Device labels, keyed by `metadata.name`. devices: HashMap>, + /// Admin-managed device → groups snapshot from the + /// [`DeviceGroupSource`]. Distinct from labels by design: labels + /// are device-reported placement vocabulary, groups are the + /// security boundary (ADR-025). + device_groups: HashMap>, /// Latest DeploymentState per (device, deployment) pair. states: HashMap, /// Which devices have we pushed desired-state for, per deployment? @@ -133,14 +147,40 @@ pub fn selector_matches(selector: &LabelSelector, labels: &BTreeMap>, -) -> HashSet { - devices +/// The security gate: is the device in one of the deployment's allowed +/// groups? `None` = ungated. `Some` requires intersection — including +/// `Some(vec![])`, which matches nothing (declared but empty is a +/// revocation, not a wildcard). +pub fn group_allows( + allowed: &Option>, + device_groups: Option<&HashSet>, +) -> bool { + match allowed { + None => true, + Some(allowed) => { + device_groups.is_some_and(|groups| allowed.iter().any(|g| groups.contains(g))) + } + } +} + +/// Is `deployment` allowed (groups) and placed (labels) on this device? +fn device_eligible( + deployment: &CachedDeployment, + labels: &BTreeMap, + device_groups: Option<&HashSet>, +) -> bool { + group_allows(&deployment.allowed_groups, device_groups) + && selector_matches(&deployment.selector, labels) +} + +/// Set of Device names the deployment is currently allowed and placed on. +fn matched_devices(deployment: &CachedDeployment, state: &FleetState) -> HashSet { + state + .devices .iter() - .filter(|(_, labels)| selector_matches(selector, labels)) + .filter(|(name, labels)| { + device_eligible(deployment, labels, state.device_groups.get(*name)) + }) .map(|(name, _)| name.clone()) .collect() } @@ -149,7 +189,12 @@ fn matched_devices( // Top-level run // --------------------------------------------------------------------------- -pub async fn run(client: Client, js: async_nats::jetstream::Context) -> anyhow::Result<()> { +pub async fn run( + client: Client, + js: async_nats::jetstream::Context, + secret_grants: Option>, + group_source: Option>, +) -> anyhow::Result<()> { let state_bucket = js .create_key_value(async_nats::jetstream::kv::Config { bucket: BUCKET_DEVICE_STATE.to_string(), @@ -197,8 +242,11 @@ pub async fn run(client: Client, js: async_nats::jetstream::Context) -> anyhow:: let deployment_watcher_handle = { let state = state.clone(); let desired = desired_bucket.clone(); + let grants = secret_grants.clone(); tokio::spawn(async move { - if let Err(e) = run_deployment_watcher(deployments_api.clone(), state, desired).await { + if let Err(e) = + run_deployment_watcher(deployments_api.clone(), state, desired, grants).await + { tracing::warn!(error = %e, "aggregator: deployment watcher exited"); } }) @@ -214,6 +262,31 @@ pub async fn run(client: Client, js: async_nats::jetstream::Context) -> anyhow:: }) }; + // Group membership poll: groups change rarely (admin action), and a + // device's secret access converges at its token TTL regardless — + // the poll only moves *scheduling* in line with membership. + let group_poll_handle = { + let state = state.clone(); + let desired = desired_bucket.clone(); + tokio::spawn(async move { + let Some(source) = group_source else { + // No source configured: park forever — returning would + // resolve the select! below and stop the aggregator. + std::future::pending::<()>().await; + unreachable!() + }; + let mut ticker = tokio::time::interval(GROUP_POLL); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + ticker.tick().await; + match source.device_groups().await { + Ok(snapshot) => apply_device_groups(&state, &desired, snapshot).await, + Err(e) => warn!(error = %e, "aggregator: group source fetch failed"), + } + } + }) + }; + let patch_state = state.clone(); let patch_loop = async move { let mut ticker = tokio::time::interval(PATCH_TICK); @@ -231,6 +304,7 @@ pub async fn run(client: Client, js: async_nats::jetstream::Context) -> anyhow:: _ = state_watcher_handle => Ok(()), _ = deployment_watcher_handle => Ok(()), _ = device_watcher_handle => Ok(()), + _ = group_poll_handle => Ok(()), } } @@ -281,6 +355,41 @@ async fn run_state_kv_watcher(bucket: Store, state: SharedFleetState) -> anyhow: Ok(()) } +/// Push the allowed-groups grant for `deployments` to the secrets +/// backend in one batch. A deployment absent from the cache (deleted, or +/// no `allowedGroups` declared) resolves to the empty set = revocation. +/// Non-fatal: a backend hiccup logs and is retried on the next +/// deployment change rather than tearing down reconciliation. Skipped +/// entirely when no backend is configured (dev without OpenBao). +async fn sync_deployment_grants( + state: &SharedFleetState, + grants: Option<&Arc>, + deployments: impl IntoIterator, +) { + let Some(grants) = grants else { return }; + let batch: Vec<(DeploymentName, Vec)> = { + let guard = state.lock().await; + deployments + .into_iter() + .map(|dn| { + let groups = guard + .deployments + .values() + .find(|c| c.deployment_name == dn) + .and_then(|c| c.allowed_groups.clone()) + .unwrap_or_default(); + (dn, groups) + }) + .collect() + }; + if batch.is_empty() { + return; + } + if let Err(e) = grants.set_deployment_groups(&batch).await { + warn!(error = %e, "aggregator: secret grant sync failed"); + } +} + /// Record a device's latest state, dedup against older timestamps, /// maintain last_error, mark the deployment dirty. pub fn apply_state(state: &mut FleetState, pair: DevicePair, ds: DeploymentState) { @@ -359,15 +468,16 @@ async fn run_deployment_watcher( api: Api, state: SharedFleetState, desired: Store, + grants: Option>, ) -> anyhow::Result<()> { let mut stream = watcher::watcher(api, WatcherConfig::default()).boxed(); while let Some(event) = stream.try_next().await? { match event { Event::Apply(cr) | Event::InitApply(cr) => { - on_deployment_upsert(&state, &desired, cr).await; + on_deployment_upsert(&state, &desired, grants.as_ref(), cr).await; } Event::Delete(cr) => { - on_deployment_delete(&state, &desired, cr).await; + on_deployment_delete(&state, &desired, grants.as_ref(), cr).await; } Event::Init | Event::InitDone => {} } @@ -375,7 +485,12 @@ async fn run_deployment_watcher( Ok(()) } -async fn on_deployment_upsert(state: &SharedFleetState, desired: &Store, cr: Deployment) { +async fn on_deployment_upsert( + state: &SharedFleetState, + desired: &Store, + grants: Option<&Arc>, + cr: Deployment, +) { let Some(key) = DeploymentKey::from_cr(&cr) else { return; }; @@ -383,7 +498,6 @@ async fn on_deployment_upsert(state: &SharedFleetState, desired: &Store, cr: Dep tracing::warn!(name = %key.name, "aggregator: CR name is not a valid DeploymentName, skipping"); return; }; - let selector = cr.spec.target_selector.clone(); let score_json = match serde_json::to_vec(&cr.spec.score) { Ok(v) => v, Err(e) => { @@ -391,19 +505,18 @@ async fn on_deployment_upsert(state: &SharedFleetState, desired: &Store, cr: Dep return; } }; + let cached = CachedDeployment { + key: key.clone(), + deployment_name: deployment_name.clone(), + allowed_groups: cr.spec.allowed_groups.clone(), + selector: cr.spec.target_selector.clone(), + score_json: score_json.clone(), + }; let (new_targets, previous_targets) = { let mut guard = state.lock().await; - let new_targets = matched_devices(&selector, &guard.devices); - guard.deployments.insert( - key.clone(), - CachedDeployment { - key: key.clone(), - deployment_name: deployment_name.clone(), - selector: selector.clone(), - score_json: score_json.clone(), - }, - ); + let new_targets = matched_devices(&cached, &guard); + guard.deployments.insert(key.clone(), cached); let previous = guard .owned_targets .remove(&deployment_name) @@ -415,6 +528,10 @@ async fn on_deployment_upsert(state: &SharedFleetState, desired: &Store, cr: Dep (new_targets, previous) }; + // Grant before workload signal: never tell a device to run a + // deployment whose secrets it cannot yet read. + sync_deployment_grants(state, grants, std::iter::once(deployment_name.clone())).await; + reconcile_kv( desired, &deployment_name, @@ -425,7 +542,12 @@ async fn on_deployment_upsert(state: &SharedFleetState, desired: &Store, cr: Dep .await; } -async fn on_deployment_delete(state: &SharedFleetState, desired: &Store, cr: Deployment) { +async fn on_deployment_delete( + state: &SharedFleetState, + desired: &Store, + grants: Option<&Arc>, + cr: Deployment, +) { let Some(key) = DeploymentKey::from_cr(&cr) else { return; }; @@ -453,6 +575,9 @@ async fn on_deployment_delete(state: &SharedFleetState, desired: &Store, cr: Dep tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete on CR delete failed"); } } + + // Gone from the cache, so the grant resolves to empty = revocation. + sync_deployment_grants(state, grants, std::iter::once(deployment_name)).await; } // --------------------------------------------------------------------------- @@ -483,63 +608,54 @@ async fn run_device_watcher( Ok(()) } -async fn on_device_upsert(state: &SharedFleetState, desired: &Store, dev: Device) { - let name = dev.name_any(); - let labels: BTreeMap = dev.metadata.labels.clone().unwrap_or_default(); +/// Re-evaluate every deployment's ownership of `name` from the current +/// labels + groups caches; mutates `owned_targets`/`dirty` and returns +/// the (deployment, was, now) transitions for KV reconciliation. +fn reevaluate_device(guard: &mut FleetState, name: &str) -> Vec<(CachedDeployment, bool, bool)> { + let labels = guard.devices.get(name).cloned().unwrap_or_default(); + let groups = guard.device_groups.get(name).cloned(); + let snapshot: Vec = guard.deployments.values().cloned().collect(); - // For every deployment, compute whether this single device now - // matches vs. previously matched; diff against owned_targets; - // collect the KV writes/deletes to perform after the lock is - // released. - let per_deployment: Vec<(CachedDeployment, bool, bool)> = { - let mut guard = state.lock().await; - let snapshot: Vec = guard.deployments.values().cloned().collect(); - let previously_matched: HashMap = snapshot - .iter() - .map(|d| { - let was = guard - .owned_targets - .get(&d.deployment_name) - .is_some_and(|set| set.contains(&name)); - (d.deployment_name.clone(), was) - }) - .collect(); - guard.devices.insert(name.clone(), labels.clone()); - - let mut out = Vec::with_capacity(snapshot.len()); - for d in snapshot { - let was = previously_matched - .get(&d.deployment_name) - .copied() - .unwrap_or(false); - let now = selector_matches(&d.selector, &labels); - if was != now { - let targets = guard - .owned_targets - .entry(d.deployment_name.clone()) - .or_default(); - if now { - targets.insert(name.clone()); - } else { - targets.remove(&name); - } - guard.dirty.insert(d.key.clone()); + let mut out = Vec::with_capacity(snapshot.len()); + for d in snapshot { + let was = guard + .owned_targets + .get(&d.deployment_name) + .is_some_and(|set| set.contains(name)); + let now = device_eligible(&d, &labels, groups.as_ref()); + if was != now { + let targets = guard + .owned_targets + .entry(d.deployment_name.clone()) + .or_default(); + if now { + targets.insert(name.to_string()); + } else { + targets.remove(name); } - out.push((d, was, now)); + guard.dirty.insert(d.key.clone()); } - out - }; + out.push((d, was, now)); + } + out +} - for (cached, was, now) in per_deployment { +/// Apply the KV writes/deletes a [`reevaluate_device`] diff calls for. +async fn reconcile_device_kv( + desired: &Store, + name: &str, + transitions: Vec<(CachedDeployment, bool, bool)>, +) { + for (cached, was, now) in transitions { match (was, now) { (false, true) => { - let k = desired_state_key(&name, &cached.deployment_name); + let k = desired_state_key(name, &cached.deployment_name); if let Err(e) = desired.put(&k, cached.score_json.clone().into()).await { tracing::debug!(key = %k, error = %e, "aggregator: desired-state put failed"); } } (true, false) => { - let k = desired_state_key(&name, &cached.deployment_name); + let k = desired_state_key(name, &cached.deployment_name); if let Err(e) = desired.delete(&k).await { tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete failed"); } @@ -549,6 +665,52 @@ async fn on_device_upsert(state: &SharedFleetState, desired: &Store, dev: Device } } +async fn on_device_upsert(state: &SharedFleetState, desired: &Store, dev: Device) { + let name = dev.name_any(); + let labels: BTreeMap = dev.metadata.labels.clone().unwrap_or_default(); + + let transitions = { + let mut guard = state.lock().await; + guard.devices.insert(name.clone(), labels); + reevaluate_device(&mut guard, &name) + }; + reconcile_device_kv(desired, &name, transitions).await; +} + +/// Swap in a fresh device → groups snapshot and re-evaluate scheduling +/// for every device whose membership changed. Secret grants are +/// untouched — they bind deployments to groups, not to devices. +async fn apply_device_groups( + state: &SharedFleetState, + desired: &Store, + snapshot: HashMap>, +) { + let changed: Vec<(String, Vec<(CachedDeployment, bool, bool)>)> = { + let mut guard = state.lock().await; + if guard.device_groups == snapshot { + return; + } + let affected: HashSet = guard + .device_groups + .keys() + .chain(snapshot.keys()) + .filter(|d| guard.device_groups.get(*d) != snapshot.get(*d)) + .cloned() + .collect(); + guard.device_groups = snapshot; + affected + .into_iter() + .map(|name| { + let transitions = reevaluate_device(&mut guard, &name); + (name, transitions) + }) + .collect() + }; + for (name, transitions) in changed { + reconcile_device_kv(desired, &name, transitions).await; + } +} + async fn on_device_delete(state: &SharedFleetState, desired: &Store, dev: Device) { let name = dev.name_any(); let removed_from: Vec = { @@ -738,6 +900,7 @@ mod tests { name: name.to_string(), }, deployment_name: dn(name), + allowed_groups: None, selector: LabelSelector { match_labels: Some(ml), match_expressions: None, @@ -746,6 +909,11 @@ mod tests { } } + fn with_groups(mut cached: CachedDeployment, groups: &[&str]) -> CachedDeployment { + cached.allowed_groups = Some(groups.iter().map(|s| s.to_string()).collect()); + cached + } + fn pair(device: &str, deployment: &str) -> DevicePair { DevicePair { device_id: device.to_string(), @@ -755,26 +923,76 @@ mod tests { #[test] fn selector_match_labels_only() { + // `zone` not `group`: labels are placement vocabulary; the + // authorization group is a distinct, admin-only concept. let mut ml = BTreeMap::new(); - ml.insert("group".to_string(), "edge-a".to_string()); + ml.insert("zone".to_string(), "lab".to_string()); let sel = LabelSelector { match_labels: Some(ml), match_expressions: None, }; let mut matching = BTreeMap::new(); - matching.insert("group".to_string(), "edge-a".to_string()); + matching.insert("zone".to_string(), "lab".to_string()); matching.insert("arch".to_string(), "aarch64".to_string()); assert!(selector_matches(&sel, &matching)); let mut non_matching = BTreeMap::new(); - non_matching.insert("group".to_string(), "edge-b".to_string()); + non_matching.insert("zone".to_string(), "warehouse".to_string()); assert!(!selector_matches(&sel, &non_matching)); let empty = BTreeMap::new(); assert!(!selector_matches(&sel, &empty)); } + #[test] + fn group_gate_semantics() { + let edge_a = HashSet::from(["edge-a".to_string()]); + + // Undeclared = ungated. + assert!(group_allows(&None, Some(&edge_a))); + assert!(group_allows(&None, None)); + + // Declared = member devices only; unknown membership fails closed. + let allowed = Some(vec!["edge-a".to_string(), "edge-b".to_string()]); + assert!(group_allows(&allowed, Some(&edge_a))); + assert!(!group_allows( + &allowed, + Some(&HashSet::from(["other".to_string()])) + )); + assert!(!group_allows(&allowed, None)); + + // Declared-but-empty is a revocation, not a wildcard. + assert!(!group_allows(&Some(vec![]), Some(&edge_a))); + } + + #[test] + fn groups_gate_and_labels_place() { + let mut s = FleetState::default(); + let mut lab = BTreeMap::new(); + lab.insert("zone".to_string(), "lab".to_string()); + // Three devices with the matching label; only pi-01 is in the + // allowed group, pi-03 has no known membership at all. + s.devices.insert("pi-01".to_string(), lab.clone()); + s.devices.insert("pi-02".to_string(), lab.clone()); + s.devices.insert("pi-03".to_string(), lab); + s.device_groups + .insert("pi-01".to_string(), HashSet::from(["edge-a".to_string()])); + s.device_groups + .insert("pi-02".to_string(), HashSet::from(["edge-b".to_string()])); + + let gated = with_groups(cached("ns", "web", "zone", "lab"), &["edge-a"]); + assert_eq!(matched_devices(&gated, &s), devs(&["pi-01"])); + + // A device-chosen label can never widen access: the label + // matches, the group doesn't, the device is not targeted. + let ungated = cached("ns", "web", "zone", "lab"); + assert_eq!( + matched_devices(&ungated, &s), + devs(&["pi-01", "pi-02", "pi-03"]) + ); + } + #[test] fn empty_selector_matches_everything() { let sel = LabelSelector::default(); @@ -786,7 +1004,7 @@ mod tests { #[test] fn compute_aggregate_counts_matched_devices() { - let cached = cached("fleet-demo", "hello", "group", "edge-a"); + let cached = cached("fleet-demo", "hello", "zone", "lab"); let key = cached.key.clone(); let mut s = FleetState::default(); @@ -817,25 +1035,73 @@ mod tests { assert_eq!(agg.pending, 1); } + fn devs(ids: &[&str]) -> HashSet { + ids.iter().map(|s| s.to_string()).collect() + } + + #[tokio::test] + async fn sync_pushes_declared_groups_per_deployment() { + use harmony::modules::fleet::secret_access::InMemoryDeploymentSecretGrants; + + let mut s = FleetState::default(); + let web = with_groups(cached("ns", "web", "zone", "lab"), &["edge-a", "edge-b"]); + let db = cached("ns", "db", "zone", "lab"); // no allowedGroups → no grant + s.deployments.insert(web.key.clone(), web); + s.deployments.insert(db.key.clone(), db); + let state: SharedFleetState = Arc::new(Mutex::new(s)); + + let inmem = Arc::new(InMemoryDeploymentSecretGrants::new()); + let grants: Arc = inmem.clone(); + + sync_deployment_grants(&state, Some(&grants), [dn("web"), dn("db")]).await; + + assert_eq!(inmem.groups_for(&dn("web")), vec!["edge-a", "edge-b"]); + assert!(inmem.groups_for(&dn("db")).is_empty()); + } + + #[tokio::test] + async fn sync_revokes_deleted_deployment() { + use harmony::modules::fleet::secret_access::InMemoryDeploymentSecretGrants; + + let inmem = Arc::new(InMemoryDeploymentSecretGrants::new()); + let grants: Arc = inmem.clone(); + grants + .set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()])]) + .await + .unwrap(); + + // Deployment no longer cached (CR deleted) → grant resolves empty. + let state: SharedFleetState = Arc::new(Mutex::new(FleetState::default())); + sync_deployment_grants(&state, Some(&grants), std::iter::once(dn("web"))).await; + + assert!(inmem.groups_for(&dn("web")).is_empty()); + } + #[test] - fn matched_devices_picks_by_label() { - let mut ml = BTreeMap::new(); - ml.insert("group".to_string(), "edge-a".to_string()); - let sel = LabelSelector { - match_labels: Some(ml), - match_expressions: None, - }; + fn membership_change_reschedules_device() { + let mut s = FleetState::default(); + let mut lab = BTreeMap::new(); + lab.insert("zone".to_string(), "lab".to_string()); + s.devices.insert("pi-01".to_string(), lab); + let web = with_groups(cached("ns", "web", "zone", "lab"), &["edge-a"]); + s.deployments.insert(web.key.clone(), web); - let mut devices: HashMap> = HashMap::new(); - let mut a = BTreeMap::new(); - a.insert("group".to_string(), "edge-a".to_string()); - devices.insert("pi-01".to_string(), a); - let mut b = BTreeMap::new(); - b.insert("group".to_string(), "edge-b".to_string()); - devices.insert("pi-02".to_string(), b); + // Not a group member: re-evaluation yields no ownership. + let transitions = reevaluate_device(&mut s, "pi-01"); + assert_eq!(transitions.len(), 1); + assert_eq!((transitions[0].1, transitions[0].2), (false, false)); - let matched = matched_devices(&sel, &devices); - assert_eq!(matched.len(), 1); - assert!(matched.contains("pi-01")); + // Membership lands (as the group poll would apply it) → owned. + s.device_groups + .insert("pi-01".to_string(), HashSet::from(["edge-a".to_string()])); + let transitions = reevaluate_device(&mut s, "pi-01"); + assert_eq!((transitions[0].1, transitions[0].2), (false, true)); + assert!(s.owned_targets[&dn("web")].contains("pi-01")); + + // Membership revoked → ownership withdrawn. + s.device_groups.remove("pi-01"); + let transitions = reevaluate_device(&mut s, "pi-01"); + assert_eq!((transitions[0].1, transitions[0].2), (true, false)); + assert!(!s.owned_targets[&dn("web")].contains("pi-01")); } } diff --git a/fleet/harmony-fleet-operator/src/main.rs b/fleet/harmony-fleet-operator/src/main.rs index e6ff215d..cca8608b 100644 --- a/fleet/harmony-fleet-operator/src/main.rs +++ b/fleet/harmony-fleet-operator/src/main.rs @@ -5,7 +5,11 @@ mod frontend; #[cfg(feature = "web-frontend")] mod service; -use harmony_fleet_operator::{device_reconciler, device_status, fleet_aggregator}; +use harmony::modules::fleet::groups::{DeviceGroupSource, StaticDeviceGroups}; +use harmony::modules::fleet::secret_access::DeploymentSecretGrants; +use harmony::modules::openbao::OpenBaoDeploymentSecretGrants; +use harmony::modules::zitadel::ZitadelDeviceGroups; +use harmony_fleet_operator::{device_reconciler, fleet_aggregator}; use anyhow::{Context, Result}; use async_nats::jetstream; @@ -18,6 +22,7 @@ use harmony_reconciler_contracts::BUCKET_DESIRED_STATE; use kube::Client; #[cfg(feature = "web-frontend")] use std::path::PathBuf; +use std::sync::Arc; use std::time::Duration; #[derive(Parser)] @@ -56,6 +61,12 @@ struct Cli { global = true )] credentials_toml: String, + + #[arg(long, env = "OPENBAO_URL", global = true)] + openbao_url: Option, + + #[arg(long, env = "OPENBAO_TOKEN", global = true)] + openbao_token: Option, } #[derive(Subcommand)] @@ -98,7 +109,16 @@ async fn main() -> Result<()> { let cli = Cli::parse(); match cli.command.unwrap_or(Command::Run) { - Command::Run => run(&cli.nats_url, &cli.kv_bucket, &cli.credentials_toml).await, + Command::Run => { + run( + &cli.nats_url, + &cli.kv_bucket, + &cli.credentials_toml, + &cli.openbao_url, + &cli.openbao_token, + ) + .await + } #[cfg(feature = "web-frontend")] Command::ServeWeb { mock, @@ -201,7 +221,13 @@ fn spawn_dashboard(client: Client) { }); } -async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()> { +async fn run( + nats_url: &str, + bucket: &str, + credentials_toml: &str, + openbao_url: &Option, + openbao_token: &Option, +) -> Result<()> { let nats = connect_with_retry(nats_url, credentials_toml).await?; tracing::info!(url = %nats_url, "connected to NATS"); let js = jetstream::new(nats); @@ -214,6 +240,53 @@ async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()> tracing::info!(bucket = %bucket, "KV bucket ready"); let client = Client::try_default().await?; + let secret_grants: Option> = match (openbao_url, openbao_token) + { + (Some(url), Some(token)) => { + // Secrets live under `/data///…` — + // the same namespace harmony_config stores under. + let kv_mount = + std::env::var("OPENBAO_KV_MOUNT").unwrap_or_else(|_| "secret".to_string()); + let prefix = std::env::var("HARMONY_CONFIG_NAMESPACE") + .unwrap_or_else(|_| "fleet-staging".to_string()); + tracing::info!(url = %url, %kv_mount, %prefix, "OpenBao secret-grant sync enabled"); + Some(Arc::new(OpenBaoDeploymentSecretGrants::new( + url.clone(), + token.clone(), + kv_mount, + prefix, + ))) + } + _ => { + tracing::warn!("OPENBAO_URL or OPENBAO_TOKEN not set; secret-grant sync disabled"); + None + } + }; + + // Group membership (the scheduling gate): Zitadel role grants in + // prod, a static map for dev/e2e, or ungated when neither is set. + let group_source: Option> = match ( + std::env::var("ZITADEL_URL"), + std::env::var("ZITADEL_PAT"), + std::env::var("ZITADEL_PROJECT_ID"), + std::env::var("FLEET_DEVICE_GROUPS"), + ) { + (Ok(url), Ok(pat), Ok(project), _) => { + tracing::info!(url = %url, %project, "Zitadel device-group source enabled"); + Some(Arc::new(ZitadelDeviceGroups::new(url, pat, project))) + } + (_, _, _, Ok(spec)) => { + tracing::info!("static device-group source enabled"); + Some(Arc::new(StaticDeviceGroups::parse(&spec))) + } + _ => { + tracing::warn!( + "no device-group source (ZITADEL_URL/PAT/PROJECT_ID or FLEET_DEVICE_GROUPS); \ + deployments with allowedGroups will match no devices" + ); + None + } + }; // Serve the read-only dashboard in the same process (best-effort; // it reads CRs only, no NATS). Built only with the web-frontend @@ -224,7 +297,6 @@ async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()> // Concurrent tasks: // controller — CR validation + finalizer-cleanup // device_reconciler — NATS device-info → Device CR - // device_status — NATS device-heartbeat → Device.status liveness // fleet_aggregator — watches Deployments + Devices + states, // writes desired-state KV, patches CR status // Any failing tears the process down; kube-rs Controller swallows @@ -232,13 +304,10 @@ async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()> let ctl_client = client.clone(); let dr_client = client.clone(); let dr_js = js.clone(); - let ds_client = client.clone(); - let ds_js = js.clone(); tokio::select! { r = controller::run(ctl_client, desired_state_kv) => r, r = device_reconciler::run(dr_client, dr_js) => r, - r = device_status::run(ds_client, ds_js) => r, - r = fleet_aggregator::run(client, js) => r, + r = fleet_aggregator::run(client, js, secret_grants, group_source) => r, } } diff --git a/fleet/harmony-fleet-operator/src/service/real.rs b/fleet/harmony-fleet-operator/src/service/real.rs index 5e48f59f..5e19aef4 100644 --- a/fleet/harmony-fleet-operator/src/service/real.rs +++ b/fleet/harmony-fleet-operator/src/service/real.rs @@ -424,6 +424,7 @@ mod tests { image: "nginx:1.25".into(), ports: vec![], env: vec![], + secret_env: vec![], volumes: vec![], restart_policy: Default::default(), }], diff --git a/harmony-k8s/src/apply.rs b/harmony-k8s/src/apply.rs index 9e4f2627..b8d70f32 100644 --- a/harmony-k8s/src/apply.rs +++ b/harmony-k8s/src/apply.rs @@ -223,7 +223,13 @@ impl K8sClient { return Ok(resource.clone()); } - let patch_params = PatchParams::apply(FIELD_MANAGER); + // Force conflicts: what harmony applies is declared in code — the + // source of truth. Without it, any field a different manager ever + // touched (e.g. a helm-installed copy of the same CRD) makes the + // re-apply 409 forever; taking ownership on conflict is the + // documented SSA pattern for controllers. + let mut patch_params = PatchParams::apply(FIELD_MANAGER); + patch_params.force = true; do_apply(&api, name, resource, &patch_params, &write_mode) .await .and_then(helper::dyn_to_typed) diff --git a/harmony/src/modules/fleet/groups.rs b/harmony/src/modules/fleet/groups.rs new file mode 100644 index 00000000..f5a03939 --- /dev/null +++ b/harmony/src/modules/fleet/groups.rs @@ -0,0 +1,81 @@ +//! Device group membership: the admin-managed identity input that gates +//! scheduling and secret access (ADR-025). +//! +//! Groups are the security boundary — they must come from identity the +//! device cannot self-assert (Zitadel role grants), never from +//! device-reported labels. The operator polls a [`DeviceGroupSource`] +//! and feeds the snapshot into scheduling. + +use std::collections::{HashMap, HashSet}; + +use async_trait::async_trait; + +#[derive(Debug, thiserror::Error)] +pub enum GroupSourceError { + #[error("group source error: {0}")] + Source(String), +} + +/// Complete device → groups view. Snapshot-based: membership changes +/// rarely and per-device, so a periodic full fetch beats per-device +/// queries. +#[async_trait] +pub trait DeviceGroupSource: Send + Sync { + async fn device_groups(&self) -> Result>, GroupSourceError>; +} + +/// Fixed membership parsed from a `dev=g1|g2;dev2=g3` string — dev +/// setups and e2e, where no Zitadel exists. +#[derive(Debug, Default)] +pub struct StaticDeviceGroups { + map: HashMap>, +} + +impl StaticDeviceGroups { + pub fn parse(spec: &str) -> Self { + let map = spec + .split(';') + .filter_map(|entry| { + let (device, groups) = entry.split_once('=')?; + let device = device.trim(); + if device.is_empty() { + return None; + } + Some(( + device.to_string(), + groups + .split('|') + .map(str::trim) + .filter(|g| !g.is_empty()) + .map(str::to_string) + .collect(), + )) + }) + .collect(); + Self { map } + } +} + +#[async_trait] +impl DeviceGroupSource for StaticDeviceGroups { + async fn device_groups(&self) -> Result>, GroupSourceError> { + Ok(self.map.clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn parses_devices_and_groups() { + let src = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b ;;junk"); + let map = src.device_groups().await.unwrap(); + assert_eq!(map.len(), 2); + assert_eq!( + map["pi-01"], + HashSet::from(["edge-a".to_string(), "edge-b".to_string()]) + ); + assert_eq!(map["pi-02"], HashSet::from(["edge-b".to_string()])); + } +} diff --git a/harmony/src/modules/fleet/mod.rs b/harmony/src/modules/fleet/mod.rs index e7c830b2..a0fd2a13 100644 --- a/harmony/src/modules/fleet/mod.rs +++ b/harmony/src/modules/fleet/mod.rs @@ -19,10 +19,12 @@ //! KVM VMs, etc. pub mod assets; +pub mod groups; #[cfg(all(feature = "kvm", unix))] pub mod libvirt_pool; pub mod operator; pub mod preflight; +pub mod secret_access; mod setup_score; #[cfg(all(feature = "kvm", unix))] mod vm_score; @@ -41,8 +43,8 @@ pub use libvirt_pool::{HARMONY_FLEET_POOL_NAME, HarmonyFleetPool, ensure_harmony // composite + per-component Scores. pub use preflight::{check_fleet_smoke_preflight, check_fleet_smoke_preflight_for_arch}; pub use setup_score::{ - AdminAuth, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HostsEntry, - merge_hosts_file, + AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, + HostsEntry, merge_hosts_file, }; #[cfg(all(feature = "kvm", unix))] pub use vm_score::ProvisionVmScore; diff --git a/harmony/src/modules/fleet/operator/crd.rs b/harmony/src/modules/fleet/operator/crd.rs index bdf027cd..badcf893 100644 --- a/harmony/src/modules/fleet/operator/crd.rs +++ b/harmony/src/modules/fleet/operator/crd.rs @@ -22,8 +22,16 @@ pub use crate::modules::podman::{PodmanService, PodmanV0Score, ReconcileScore}; )] #[serde(rename_all = "camelCase")] pub struct DeploymentSpec { - /// Which devices this deployment targets. matches against - /// `Device.metadata.labels`. + /// Device groups allowed to *view* this deployment — run it and + /// read its secrets (ADR-025). Membership is admin-managed identity + /// (Zitadel role grants), never device-reported labels: labels can + /// narrow placement below, but only groups grant. Absent = no group + /// gating and no secret access — the selector alone places a + /// secretless workload. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub allowed_groups: Option>, + /// Which devices this deployment targets, *within* the allowed + /// groups. Matches against `Device.metadata.labels`. pub target_selector: LabelSelector, pub score: ReconcileScore, pub rollout: Rollout, diff --git a/harmony/src/modules/fleet/secret_access.rs b/harmony/src/modules/fleet/secret_access.rs new file mode 100644 index 00000000..bf835c15 --- /dev/null +++ b/harmony/src/modules/fleet/secret_access.rs @@ -0,0 +1,132 @@ +//! Deployment → secret access control: the domain boundary the fleet +//! operator drives when a `Deployment`'s allowed groups change. +//! +//! Per ADR-025, groups are the security boundary: a deployment declares +//! which device groups may read its secrets, and devices prove group +//! membership through their signed identity at login. How a grant maps +//! onto a backend's primitive (an OpenBao policy attached to an external +//! group, …) is the *implementation's* concern; callers speak only +//! deployments and group names. That separation is what lets the +//! operator be unit-tested against [`InMemoryDeploymentSecretGrants`] +//! with no live secret store — the production backend is +//! [`crate::modules::openbao::OpenBaoDeploymentSecretGrants`]. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Mutex; + +use async_trait::async_trait; +use harmony_reconciler_contracts::DeploymentName; + +#[derive(Debug, thiserror::Error)] +pub enum SecretAccessError { + #[error("secret access backend error: {0}")] + Backend(String), +} + +/// Sets which device groups may read each deployment's secrets. +#[async_trait] +pub trait DeploymentSecretGrants: Send + Sync { + /// Set the *complete* set of groups allowed to read each listed + /// deployment's secrets. Absolute and idempotent: afterwards exactly + /// the given groups hold the grant — others are revoked, an empty + /// list revokes everywhere. Deployments not listed are untouched. + async fn set_deployment_groups( + &self, + grants: &[(DeploymentName, Vec)], + ) -> Result<(), SecretAccessError>; +} + +/// In-memory [`DeploymentSecretGrants`] for tests: records the latest +/// group set per deployment so a test can assert the operator synced the +/// right grants on each flow, with nothing deployed. +#[derive(Debug, Default)] +pub struct InMemoryDeploymentSecretGrants { + grants: Mutex>>, +} + +impl InMemoryDeploymentSecretGrants { + pub fn new() -> Self { + Self::default() + } + + /// Current groups for `deployment`, sorted. Empty when no group has + /// access (never granted, or revoked). + pub fn groups_for(&self, deployment: &DeploymentName) -> Vec { + self.grants + .lock() + .unwrap() + .get(deployment) + .map(|s| s.iter().cloned().collect()) + .unwrap_or_default() + } +} + +#[async_trait] +impl DeploymentSecretGrants for InMemoryDeploymentSecretGrants { + async fn set_deployment_groups( + &self, + grants: &[(DeploymentName, Vec)], + ) -> Result<(), SecretAccessError> { + let mut store = self.grants.lock().unwrap(); + for (deployment, groups) in grants { + if groups.is_empty() { + store.remove(deployment); + } else { + store.insert(deployment.clone(), groups.iter().cloned().collect()); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn dn(s: &str) -> DeploymentName { + DeploymentName::try_new(s).expect("valid test name") + } + + #[tokio::test] + async fn sets_overwrites_and_clears_grants() { + let grants = InMemoryDeploymentSecretGrants::new(); + let web = dn("web"); + + grants + .set_deployment_groups(&[(web.clone(), vec!["edge-a".into(), "edge-b".into()])]) + .await + .unwrap(); + assert_eq!(grants.groups_for(&web), vec!["edge-a", "edge-b"]); + + // Absolute, not additive: a smaller set revokes the rest. + grants + .set_deployment_groups(&[(web.clone(), vec!["edge-a".into()])]) + .await + .unwrap(); + assert_eq!(grants.groups_for(&web), vec!["edge-a"]); + + // Empty revokes everywhere. + grants + .set_deployment_groups(&[(web.clone(), vec![])]) + .await + .unwrap(); + assert!(grants.groups_for(&web).is_empty()); + } + + #[tokio::test] + async fn batch_touches_only_listed_deployments() { + let grants = InMemoryDeploymentSecretGrants::new(); + let web = dn("web"); + let db = dn("db"); + grants + .set_deployment_groups(&[(web.clone(), vec!["edge-a".into()])]) + .await + .unwrap(); + grants + .set_deployment_groups(&[(db.clone(), vec!["edge-b".into()])]) + .await + .unwrap(); + assert_eq!(grants.groups_for(&web), vec!["edge-a"]); + assert_eq!(grants.groups_for(&db), vec!["edge-b"]); + } +} diff --git a/harmony/src/modules/fleet/setup_score.rs b/harmony/src/modules/fleet/setup_score.rs index 1069614b..4ef1c9c4 100644 --- a/harmony/src/modules/fleet/setup_score.rs +++ b/harmony/src/modules/fleet/setup_score.rs @@ -74,6 +74,22 @@ pub struct FleetDeviceSetupConfig { /// real DNS instead. #[serde(default)] pub hosts_entries: Vec, + /// OpenBao endpoint the agent reads deployment secrets from + /// (ADR-025). Rendered as the agent's `[openbao]` section; mounts + /// and role use the agent-schema defaults (`jwt`, `fleet-device`, + /// `secret`). Requires `ZitadelJwt`/`ZitadelEnroll` auth — the + /// same machine identity logs into OpenBao. + #[serde(default)] + pub openbao: Option, +} + +/// Minimal `[openbao]` agent-config surface the setup score renders. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeviceOpenbao { + pub url: String, + /// KV prefix deployment subtrees live under — must match the + /// grant writer's prefix. + pub secret_prefix: String, } /// One line in `/etc/hosts`. Order doesn't matter (the file ends up @@ -246,6 +262,16 @@ impl FleetDeviceSetupConfig { danger_accept_invalid_certs, ), }; + let openbao = match &self.openbao { + None => String::new(), + Some(ob) => format!( + "\n[openbao]\n\ + url = \"{}\"\n\ + secret_prefix = \"{}\"\n", + toml_escape(&ob.url), + toml_escape(&ob.secret_prefix), + ), + }; format!( "[agent]\n\ device_id = \"{device_id}\"\n\ @@ -255,7 +281,8 @@ impl FleetDeviceSetupConfig { urls = [{urls}]\n\ \n\ [labels]\n\ - {labels}\n" + {labels}\n\ + {openbao}" ) } @@ -369,6 +396,7 @@ async fn resolve_zitadel_enroll( api_apps: vec![], roles: vec![], machine_users: vec![], + groups_claim_action: false, }; info!( @@ -862,6 +890,7 @@ mod tests { }, agent_binary_path: PathBuf::from("/dev/null"), hosts_entries: vec![], + openbao: None, } } @@ -879,6 +908,7 @@ mod tests { }, agent_binary_path: PathBuf::from("/dev/null"), hosts_entries: vec![], + openbao: None, } } @@ -936,6 +966,22 @@ mod tests { assert!(!toml.contains("nats_pass")); } + #[test] + fn render_toml_emits_openbao_section_only_when_set() { + let plain = base_config_zitadel(BTreeMap::new()).render_toml(); + assert!(!plain.contains("[openbao]"), "got:\n{plain}"); + + let mut cfg = base_config_zitadel(BTreeMap::new()); + cfg.openbao = Some(DeviceOpenbao { + url: "http://192.168.122.1:30425".to_string(), + secret_prefix: "fleet-e2e".to_string(), + }); + let toml = cfg.render_toml(); + assert!(toml.contains("[openbao]"), "got:\n{toml}"); + assert!(toml.contains(r#"url = "http://192.168.122.1:30425""#)); + assert!(toml.contains(r#"secret_prefix = "fleet-e2e""#)); + } + #[test] fn render_toml_zitadel_enroll_renders_same_as_zitadel_jwt() { // Pre-resolution (ZitadelEnroll) and post-resolution (ZitadelJwt) @@ -966,6 +1012,7 @@ mod tests { }, agent_binary_path: PathBuf::from("/dev/null"), hosts_entries: vec![], + openbao: None, }; assert_eq!(jwt, enroll_cfg.render_toml()); } diff --git a/harmony/src/modules/openbao/identity.rs b/harmony/src/modules/openbao/identity.rs new file mode 100644 index 00000000..641b9276 --- /dev/null +++ b/harmony/src/modules/openbao/identity.rs @@ -0,0 +1,342 @@ +//! OpenBao-backed [`DeploymentSecretGrants`]. +//! +//! A deployment's grant is one ACL policy `deployment-` (read on +//! its secret subtree) attached to the *external identity group* of each +//! allowed device group. Devices land in those groups at JWT login via +//! the role's `groups_claim`, so attaching/detaching the policy binds +//! for every member's existing tokens at request time — O(groups) +//! writes per deployment change, regardless of fleet size (ADR-025). +//! +//! External groups are matched to the token claim through a group alias +//! on the JWT mount; we create group + alias on first grant so the grant +//! can precede any device's first login. + +use std::collections::HashSet; + +use async_trait::async_trait; +use reqwest::StatusCode; +use serde_json::json; +use tokio::sync::OnceCell; + +use crate::modules::fleet::secret_access::{DeploymentSecretGrants, SecretAccessError}; +use harmony_reconciler_contracts::DeploymentName; + +const DEFAULT_JWT_MOUNT: &str = "jwt"; + +pub struct OpenBaoDeploymentSecretGrants { + client: reqwest::Client, + base_url: String, + token: String, + kv_mount: String, + /// Path prefix under the KV mount where the fleet's secrets live + /// (`/data///…`). + secret_prefix: String, + jwt_mount: String, + /// JWT auth mount accessor, resolved once and reused — needed to bind + /// a group alias to the login claim value. + jwt_accessor: OnceCell, +} + +impl OpenBaoDeploymentSecretGrants { + pub fn new(base_url: String, token: String, kv_mount: String, secret_prefix: String) -> Self { + Self::with_jwt_mount( + base_url, + token, + kv_mount, + secret_prefix, + DEFAULT_JWT_MOUNT.to_string(), + ) + } + + pub fn with_jwt_mount( + base_url: String, + token: String, + kv_mount: String, + secret_prefix: String, + jwt_mount: String, + ) -> Self { + Self { + client: reqwest::Client::new(), + base_url: base_url.trim_end_matches('/').to_string(), + token, + kv_mount, + secret_prefix, + jwt_mount, + jwt_accessor: OnceCell::new(), + } + } + + fn err(context: impl std::fmt::Display, e: impl std::fmt::Display) -> SecretAccessError { + SecretAccessError::Backend(format!("{context}: {e}")) + } + + fn policy_name(deployment: &DeploymentName) -> String { + format!("deployment-{}", deployment.as_str()) + } + + async fn request( + &self, + method: reqwest::Method, + path: &str, + body: Option, + ) -> Result { + let mut req = self + .client + .request(method.clone(), format!("{}/v1/{path}", self.base_url)) + .header("X-Vault-Token", &self.token); + if let Some(body) = body { + req = req.json(&body); + } + req.send() + .await + .map_err(|e| Self::err(format!("{method} {path}"), e)) + } + + /// JWT mount accessor from `sys/auth`, cached for the client's life. + async fn jwt_accessor(&self) -> Result<&str, SecretAccessError> { + self.jwt_accessor + .get_or_try_init(|| async { + let body: serde_json::Value = self + .request(reqwest::Method::GET, "sys/auth", None) + .await? + .error_for_status() + .map_err(|e| Self::err("GET sys/auth", e))? + .json() + .await + .map_err(|e| Self::err("parse sys/auth", e))?; + // sys/auth nests mounts under `data` over the HTTP API but + // emits them at the document root via the CLI; tolerate both. + let mount_key = format!("{}/", self.jwt_mount); + body.get("data") + .and_then(|d| d.get(&mount_key)) + .or_else(|| body.get(&mount_key)) + .and_then(|m| m.get("accessor")) + .and_then(|a| a.as_str()) + .map(str::to_string) + .ok_or_else(|| { + Self::err( + "resolve jwt accessor", + format!("mount '{mount_key}' not found in sys/auth"), + ) + }) + }) + .await + .map(String::as_str) + } + + async fn upsert_policy(&self, deployment: &DeploymentName) -> Result<(), SecretAccessError> { + let name = Self::policy_name(deployment); + let hcl = format!( + r#"path "{kv}/data/{prefix}/{dep}/*" {{ capabilities = ["read"] }} +path "{kv}/metadata/{prefix}/{dep}/*" {{ capabilities = ["read", "list"] }}"#, + kv = self.kv_mount, + prefix = self.secret_prefix, + dep = deployment.as_str(), + ); + self.request( + reqwest::Method::PUT, + &format!("sys/policies/acl/{name}"), + Some(json!({ "policy": hcl })), + ) + .await? + .error_for_status() + .map_err(|e| Self::err(format!("write policy {name}"), e))?; + Ok(()) + } + + async fn delete_policy(&self, deployment: &DeploymentName) -> Result<(), SecretAccessError> { + let name = Self::policy_name(deployment); + let resp = self + .request( + reqwest::Method::DELETE, + &format!("sys/policies/acl/{name}"), + None, + ) + .await?; + if !resp.status().is_success() && resp.status() != StatusCode::NOT_FOUND { + return Err(Self::err(format!("delete policy {name}"), resp.status())); + } + Ok(()) + } + + /// Group's current policy list, or `None` when the group doesn't + /// exist. + async fn read_group_policies( + &self, + group: &str, + ) -> Result>, SecretAccessError> { + let resp = self + .request( + reqwest::Method::GET, + &format!("identity/group/name/{group}"), + None, + ) + .await?; + if resp.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + let body: serde_json::Value = resp + .error_for_status() + .map_err(|e| Self::err(format!("read group {group}"), e))? + .json() + .await + .map_err(|e| Self::err(format!("parse group {group}"), e))?; + Ok(Some( + body["data"]["policies"] + .as_array() + .map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default(), + )) + } + + async fn write_group_policies( + &self, + group: &str, + policies: &[String], + ) -> Result<(), SecretAccessError> { + self.request( + reqwest::Method::POST, + &format!("identity/group/name/{group}"), + Some(json!({ "type": "external", "policies": policies })), + ) + .await? + .error_for_status() + .map_err(|e| Self::err(format!("write group {group}"), e))?; + Ok(()) + } + + /// Ensure the external group exists with its alias bound to the JWT + /// mount, and that it carries `policy`. + async fn attach(&self, group: &str, policy: &str) -> Result<(), SecretAccessError> { + match self.read_group_policies(group).await? { + Some(policies) if policies.iter().any(|p| p == policy) => return Ok(()), + Some(mut policies) => { + policies.push(policy.to_string()); + self.write_group_policies(group, &policies).await?; + } + None => { + self.write_group_policies(group, &[policy.to_string()]) + .await?; + self.ensure_group_alias(group).await?; + } + } + Ok(()) + } + + async fn detach(&self, group: &str, policy: &str) -> Result<(), SecretAccessError> { + if let Some(policies) = self.read_group_policies(group).await? { + if policies.iter().any(|p| p == policy) { + let remaining: Vec = policies.into_iter().filter(|p| p != policy).collect(); + self.write_group_policies(group, &remaining).await?; + } + } + Ok(()) + } + + async fn ensure_group_alias(&self, group: &str) -> Result<(), SecretAccessError> { + let accessor = self.jwt_accessor().await?.to_string(); + let body: serde_json::Value = self + .request( + reqwest::Method::GET, + &format!("identity/group/name/{group}"), + None, + ) + .await? + .error_for_status() + .map_err(|e| Self::err(format!("read group {group}"), e))? + .json() + .await + .map_err(|e| Self::err(format!("parse group {group}"), e))?; + if !body["data"]["alias"]["id"] + .as_str() + .unwrap_or("") + .is_empty() + { + return Ok(()); + } + let canonical_id = body["data"]["id"] + .as_str() + .ok_or_else(|| Self::err("read group", format!("{group} missing data.id")))?; + let resp = self + .request( + reqwest::Method::POST, + "identity/group-alias", + Some(json!({ + "name": group, + "mount_accessor": accessor, + "canonical_id": canonical_id, + })), + ) + .await?; + // A concurrent sync may have just created the alias; OpenBao + // answers 400 "alias already exists" — treat as success. + if !resp.status().is_success() && resp.status() != StatusCode::BAD_REQUEST { + return Err(Self::err( + format!("create group alias {group}"), + resp.status(), + )); + } + Ok(()) + } + + /// Names of all identity groups, empty when none exist. + async fn list_groups(&self) -> Result, SecretAccessError> { + let resp = self + .request(reqwest::Method::GET, "identity/group/name?list=true", None) + .await?; + if resp.status() == StatusCode::NOT_FOUND { + return Ok(vec![]); + } + let body: serde_json::Value = resp + .error_for_status() + .map_err(|e| Self::err("list groups", e))? + .json() + .await + .map_err(|e| Self::err("parse group list", e))?; + Ok(body["data"]["keys"] + .as_array() + .map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default()) + } +} + +#[async_trait] +impl DeploymentSecretGrants for OpenBaoDeploymentSecretGrants { + async fn set_deployment_groups( + &self, + grants: &[(DeploymentName, Vec)], + ) -> Result<(), SecretAccessError> { + for (deployment, groups) in grants { + let policy = Self::policy_name(deployment); + let allowed: HashSet<&str> = groups.iter().map(String::as_str).collect(); + + if allowed.is_empty() { + // Full revocation: detach everywhere, then drop the policy. + for group in self.list_groups().await? { + self.detach(&group, &policy).await?; + } + self.delete_policy(deployment).await?; + continue; + } + + self.upsert_policy(deployment).await?; + for group in groups { + self.attach(group, &policy).await?; + } + for group in self.list_groups().await? { + if !allowed.contains(group.as_str()) { + self.detach(&group, &policy).await?; + } + } + } + Ok(()) + } +} diff --git a/harmony/src/modules/openbao/mod.rs b/harmony/src/modules/openbao/mod.rs index 8b612d9f..d0a491ac 100644 --- a/harmony/src/modules/openbao/mod.rs +++ b/harmony/src/modules/openbao/mod.rs @@ -1,3 +1,4 @@ +pub mod identity; pub mod setup; use std::str::FromStr; @@ -13,6 +14,7 @@ use crate::{ topology::{HelmCommand, K8sclient, Topology}, }; +pub use identity::OpenBaoDeploymentSecretGrants; pub use setup::{OpenbaoJwtAuth, OpenbaoPolicy, OpenbaoSetupScore, OpenbaoUser, cached_root_token}; const DEFAULT_NAMESPACE: &str = "openbao"; @@ -83,7 +85,16 @@ impl OpenbaoScore { format!( r#"global: openshift: {openshift} +# No sidecar injection anywhere in the fleet (devices talk to OpenBao +# directly over JWT). The injector also installs cluster-scoped RBAC +# that outlives namespace-scoped cleanup and breaks reinstalls. +injector: + enabled: false server: + # No Kubernetes auth method either — the authDelegator + # ClusterRoleBinding is the same reinstall hazard. + authDelegator: + enabled: false standalone: enabled: true config: | diff --git a/harmony/src/modules/openbao/setup.rs b/harmony/src/modules/openbao/setup.rs index 5c6c62f4..147af974 100644 --- a/harmony/src/modules/openbao/setup.rs +++ b/harmony/src/modules/openbao/setup.rs @@ -35,12 +35,23 @@ pub struct OpenbaoUser { /// JWT auth method configuration for OpenBao. #[derive(Debug, Clone, Serialize)] pub struct OpenbaoJwtAuth { + /// Empty when `jwt_validation_pubkeys` supplies the keys instead + /// (e.g. e2e suites with no live IdP). pub oidc_discovery_url: String, + /// PEM public key(s) validating tokens directly, bypassing OIDC + /// discovery. Mutually exclusive with `oidc_discovery_url`. + pub jwt_validation_pubkeys: String, pub bound_issuer: String, pub role_name: String, pub bound_audiences: String, pub user_claim: String, - pub policies: Vec, + /// Token claim whose string-array value binds the login to external + /// identity groups of the same name. Empty = no group binding. + pub groups_claim: String, + /// `service` or `batch`. Fleet device roles use `batch`: no storage + /// cost per login, natural expiry is the group-membership sync + /// (ADR-025). + pub token_type: String, pub ttl: String, pub max_ttl: String, } @@ -446,9 +457,20 @@ impl OpenbaoSetupInterpret { .bao(k8s, root_token, &["bao", "auth", "enable", "jwt"]) .await; - // Configure JWT discovery. This may fail if the discovery URL is not + // Configure JWT validation: static public keys when provided, + // otherwise OIDC discovery. Discovery may fail if the URL is not // reachable from inside the cluster (e.g., Zitadel's ExternalDomain // isn't resolvable). Non-fatal — warn and continue. + // Single-quote the PEM: it carries newlines the pod-exec shell + // would otherwise split into separate "commands". + let validation_arg = if jwt.jwt_validation_pubkeys.is_empty() { + format!("oidc_discovery_url={}", jwt.oidc_discovery_url) + } else { + format!( + "jwt_validation_pubkeys='{}'", + jwt.jwt_validation_pubkeys.replace('\'', "'\\''") + ) + }; let config_result = self .bao( k8s, @@ -457,7 +479,7 @@ impl OpenbaoSetupInterpret { "bao", "write", "auth/jwt/config", - &format!("oidc_discovery_url={}", jwt.oidc_discovery_url), + &validation_arg, &format!("bound_issuer={}", jwt.bound_issuer), ], ) @@ -479,35 +501,28 @@ impl OpenbaoSetupInterpret { } } - let policies = jwt.policies.join(","); - self.bao( - k8s, - root_token, - &[ - "bao", - "write", - &format!("auth/jwt/role/{}", jwt.role_name), - "role_type=jwt", - &format!("bound_audiences={}", jwt.bound_audiences), - &format!("user_claim={}", jwt.user_claim), - &format!("policies={}", policies), - &format!("ttl={}", jwt.ttl), - &format!("max_ttl={}", jwt.max_ttl), - "token_type=service", - ], - ) - .await - .map_err(|e| { + let mut role_args = vec![ + "bao".to_string(), + "write".to_string(), + format!("auth/jwt/role/{}", jwt.role_name), + "role_type=jwt".to_string(), + format!("bound_audiences={}", jwt.bound_audiences), + format!("user_claim={}", jwt.user_claim), + format!("ttl={}", jwt.ttl), + format!("max_ttl={}", jwt.max_ttl), + format!("token_type={}", jwt.token_type), + ]; + if !jwt.groups_claim.is_empty() { + role_args.push(format!("groups_claim={}", jwt.groups_claim)); + } + let role_args: Vec<&str> = role_args.iter().map(String::as_str).collect(); + self.bao(k8s, root_token, &role_args).await.map_err(|e| { InterpretError::new(format!( "Failed to create JWT role '{}': {e}", jwt.role_name )) })?; - info!( - "[OpenbaoSetup] JWT role '{}' created (policies: {})", - jwt.role_name, policies - ); Ok(()) } } diff --git a/harmony/src/modules/podman/mod.rs b/harmony/src/modules/podman/mod.rs index 052faef6..e4425e86 100644 --- a/harmony/src/modules/podman/mod.rs +++ b/harmony/src/modules/podman/mod.rs @@ -5,6 +5,6 @@ mod topology; pub use crate::topology::EnvVar; pub use interpret::PodmanV0Interpret; -pub use score::{PodmanService, PodmanV0Score, ReconcileScore}; +pub use score::{PodmanService, PodmanV0Score, ReconcileScore, SecretEnvVar}; #[cfg(unix)] pub use topology::PodmanTopology; diff --git a/harmony/src/modules/podman/score.rs b/harmony/src/modules/podman/score.rs index 9994ebf3..65d135d5 100644 --- a/harmony/src/modules/podman/score.rs +++ b/harmony/src/modules/podman/score.rs @@ -18,6 +18,17 @@ use crate::{ use super::interpret::PodmanV0Interpret; +/// An env var whose value the device fetches from its secret store at +/// apply time (ADR-025). `secret` names the entry under the +/// deployment's KV subtree (`///`); +/// the agent injects the fetched value under `name`. Values never +/// transit NATS — only this reference does. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +pub struct SecretEnvVar { + pub name: String, + pub secret: String, +} + /// A single container managed by podman on the target host. /// /// Wire-compatible with prior releases: the new `env`, `volumes`, and @@ -32,6 +43,11 @@ pub struct PodmanService { /// `PartialEq` is deterministic for drift detection. #[serde(default)] pub env: Vec, + /// Env vars resolved from the device's secret store at apply time. + /// `skip_serializing_if` keeps payloads byte-identical for specs + /// that don't use secrets (string-compare idempotency). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub secret_env: Vec, /// Bind-mount volumes. Bind-only in v0; `host_path` is an absolute path /// on the device's filesystem. #[serde(default)] @@ -110,6 +126,7 @@ mod tests { image: image.to_string(), ports: vec![], env: vec![], + secret_env: vec![], volumes: vec![], restart_policy: RestartPolicy::default(), } @@ -168,6 +185,7 @@ mod tests { let ReconcileScore::PodmanV0(score) = parsed; assert_eq!(score.services.len(), 1); assert!(score.services[0].env.is_empty()); + assert!(score.services[0].secret_env.is_empty()); assert!(score.services[0].volumes.is_empty()); assert_eq!( score.services[0].restart_policy, @@ -175,6 +193,26 @@ mod tests { ); } + #[test] + fn secret_env_roundtrips_and_is_omitted_when_empty() { + // Omission when empty keeps payloads byte-identical to + // pre-secret specs — the agent's string-compare idempotency + // and old-agent compatibility both rely on it. + let plain = serde_json::to_string(&svc("web", "nginx")).unwrap(); + assert!(!plain.contains("secret_env"), "got: {plain}"); + + let with_secret = PodmanService { + secret_env: vec![SecretEnvVar { + name: "DB_PASSWORD".to_string(), + secret: "db_password".to_string(), + }], + ..svc("api", "myapp") + }; + let json = serde_json::to_string(&with_secret).unwrap(); + let back: PodmanService = serde_json::from_str(&json).unwrap(); + assert_eq!(back.secret_env, with_secret.secret_env); + } + #[test] fn restart_policy_serializes_kebab_case() { // docker-compose users expect `unless-stopped`, `on-failure` — diff --git a/harmony/src/modules/zitadel/device_groups.rs b/harmony/src/modules/zitadel/device_groups.rs new file mode 100644 index 00000000..698afaae --- /dev/null +++ b/harmony/src/modules/zitadel/device_groups.rs @@ -0,0 +1,104 @@ +//! Zitadel-backed [`DeviceGroupSource`]: project role grants are the +//! fleet's groups (roles-as-groups interim per ADR-025, swapped for +//! first-class Zitadel groups when they GA). The same role keys reach +//! OpenBao inside the device's token via the flattening Action, so the +//! operator's scheduling view and OpenBao's enforcement view share one +//! source of truth. + +use std::collections::{HashMap, HashSet}; + +use async_trait::async_trait; +use serde::Deserialize; + +use crate::modules::fleet::groups::{DeviceGroupSource, GroupSourceError}; + +pub struct ZitadelDeviceGroups { + client: reqwest::Client, + base_url: String, + pat: String, + project_id: String, +} + +impl ZitadelDeviceGroups { + pub fn new(base_url: String, pat: String, project_id: String) -> Self { + Self { + client: reqwest::Client::new(), + base_url: base_url.trim_end_matches('/').to_string(), + pat, + project_id, + } + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct GrantSearchResponse { + result: Option>, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct GrantEntry { + user_id: String, + #[serde(default)] + user_name: String, + #[serde(default)] + role_keys: Vec, +} + +const PAGE: u64 = 1000; + +#[async_trait] +impl DeviceGroupSource for ZitadelDeviceGroups { + async fn device_groups(&self) -> Result>, GroupSourceError> { + let mut map: HashMap> = HashMap::new(); + let mut offset = 0u64; + loop { + let resp = self + .client + .post(format!( + "{}/management/v1/users/grants/_search", + self.base_url + )) + .bearer_auth(&self.pat) + .json(&serde_json::json!({ + "query": { "offset": offset.to_string(), "limit": PAGE }, + "queries": [ + { "projectIdQuery": { "projectId": self.project_id } } + ] + })) + .send() + .await + .map_err(|e| GroupSourceError::Source(format!("grant search: {e}")))?; + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(GroupSourceError::Source(format!( + "grant search returned {status}: {body}" + ))); + } + let page: GrantSearchResponse = resp + .json() + .await + .map_err(|e| GroupSourceError::Source(format!("parse grant search: {e}")))?; + let entries = page.result.unwrap_or_default(); + let n = entries.len() as u64; + for grant in entries { + let roles: HashSet = grant.role_keys.into_iter().collect(); + // Fleets key devices by machine-user id or by user name + // depending on provisioning vintage; expose the grant under + // both so either convention resolves. + if !grant.user_name.is_empty() && grant.user_name != grant.user_id { + map.entry(grant.user_name) + .or_default() + .extend(roles.iter().cloned()); + } + map.entry(grant.user_id).or_default().extend(roles); + } + if n < PAGE { + return Ok(map); + } + offset += n; + } + } +} diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index 0be56877..db1a0d15 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -1,7 +1,9 @@ pub mod admin_auth; +pub mod device_groups; pub mod setup; pub use admin_auth::{ADMIN_API_SCOPES, DeviceCodeError, DeviceCodeFlowConfig, device_code_login}; +pub use device_groups::ZitadelDeviceGroups; pub use setup::{ MachineKeyType, MintedDeviceCredentials, ZitadelApiApp, ZitadelAppType, ZitadelApplication, ZitadelClientConfig, ZitadelMachineUser, ZitadelRole, ZitadelScheme, ZitadelSetupScore, @@ -865,7 +867,11 @@ login: namespace: Some(NonBlankString::from_str(&self.namespace).unwrap()), release_name: NonBlankString::from_str("zitadel").unwrap(), chart_name: NonBlankString::from_str("zitadel/zitadel").unwrap(), - chart_version: None, + // Pinned: newer charts ship a login UI that expects a newer + // server; with v4.12.x the login pod crashloops on + // Token.Invalid. 9.27.1 is the last chart matching this + // server line. + chart_version: Some(NonBlankString::from_str("9.27.1").unwrap()), values_overrides: None, values_yaml: Some(values_yaml), create_namespace: true, diff --git a/harmony/src/modules/zitadel/setup.rs b/harmony/src/modules/zitadel/setup.rs index 4fd88815..44b7b607 100644 --- a/harmony/src/modules/zitadel/setup.rs +++ b/harmony/src/modules/zitadel/setup.rs @@ -235,8 +235,30 @@ pub struct ZitadelSetupScore { /// Machine users to provision (with optional keys + role grants). #[serde(default)] pub machine_users: Vec, + /// Provision the token Action that flattens project role keys into + /// a `groups` string-array claim — the shape OpenBao's + /// `groups_claim` consumes (ADR-025) — and attach it to the org's + /// Complement Token flow. Zitadel's native roles claim is a map, + /// which OpenBao cannot bind groups from. + #[serde(default)] + pub groups_claim_action: bool, } +/// Function name doubles as the Action name — Zitadel requires the +/// script's entry function to match. +pub const GROUPS_CLAIM_ACTION_NAME: &str = "harmonyGroupsClaim"; + +pub const GROUPS_CLAIM_ACTION_SCRIPT: &str = r#"function harmonyGroupsClaim(ctx, api) { + if (ctx.v1.user.grants === undefined || ctx.v1.user.grants.count == 0) { + return; + } + let groups = []; + ctx.v1.user.grants.grants.forEach(grant => { + grant.roles.forEach(role => groups.push(role)); + }); + api.v1.claims.setClaim('groups', groups); +}"#; + /// Cached Zitadel provisioning results. #[derive(Debug, Default, Serialize, Deserialize)] pub struct ZitadelClientConfig { @@ -1339,6 +1361,156 @@ impl ZitadelSetupInterpret { String::from_utf8(bytes).map_err(|e| format!("keyDetails contained non-UTF8 bytes: {e}")) } + /// Upsert the roles→groups flattening Action and attach it to the + /// Complement Token flow's pre-userinfo and pre-access-token + /// triggers, merging with whatever actions the org already wires. + async fn ensure_groups_claim_action( + &self, + client: &reqwest::Client, + pat: &str, + ) -> Result<(), InterpretError> { + // Numeric flow/trigger ids from Zitadel's v1 management API: + // flow 2 = Complement Token; triggers 4 = Pre Userinfo + // creation, 5 = Pre access token creation. + const FLOW_COMPLEMENT_TOKEN: &str = "2"; + const TRIGGERS: [&str; 2] = ["4", "5"]; + + let action_id = match self + .find_action_id(client, pat, GROUPS_CLAIM_ACTION_NAME) + .await + .map_err(InterpretError::new)? + { + Some(id) => { + debug!("[ZitadelSetup] Action '{GROUPS_CLAIM_ACTION_NAME}' already exists"); + id + } + None => { + let resp = self + .post(client, "/management/v1/actions") + .bearer_auth(pat) + .json(&serde_json::json!({ + "name": GROUPS_CLAIM_ACTION_NAME, + "script": GROUPS_CLAIM_ACTION_SCRIPT, + "timeout": "10s", + "allowedToFail": false, + })) + .send() + .await + .map_err(|e| InterpretError::new(format!("Create action: {e}")))?; + if !resp.status().is_success() { + let body = resp.text().await.unwrap_or_default(); + return Err(InterpretError::new(format!("Create action failed: {body}"))); + } + let body: serde_json::Value = resp + .json() + .await + .map_err(|e| InterpretError::new(format!("Parse create action: {e}")))?; + info!("[ZitadelSetup] Action '{GROUPS_CLAIM_ACTION_NAME}' created"); + body["id"] + .as_str() + .ok_or_else(|| { + InterpretError::new("create action response missing id".to_string()) + })? + .to_string() + } + }; + + let flow: serde_json::Value = { + let resp = self + .request( + client, + reqwest::Method::GET, + &format!("/management/v1/flows/{FLOW_COMPLEMENT_TOKEN}"), + ) + .bearer_auth(pat) + .send() + .await + .map_err(|e| InterpretError::new(format!("Read flow: {e}")))?; + if resp.status().is_success() { + resp.json() + .await + .map_err(|e| InterpretError::new(format!("Parse flow: {e}")))? + } else { + // No flow configured yet — Zitadel answers 404. + serde_json::json!({}) + } + }; + + for trigger in TRIGGERS { + let mut action_ids: Vec = flow["flow"]["triggerActions"] + .as_array() + .into_iter() + .flatten() + .filter(|ta| { + ta["triggerType"]["id"].as_str() == Some(trigger) + || ta["triggerType"].as_str() == Some(trigger) + }) + .flat_map(|ta| { + ta["actions"] + .as_array() + .into_iter() + .flatten() + .filter_map(|a| a["id"].as_str().map(str::to_string)) + .collect::>() + }) + .collect(); + if action_ids.contains(&action_id) { + continue; + } + action_ids.push(action_id.clone()); + let resp = self + .post( + client, + &format!("/management/v1/flows/{FLOW_COMPLEMENT_TOKEN}/trigger/{trigger}"), + ) + .bearer_auth(pat) + .json(&serde_json::json!({ "actionIds": action_ids })) + .send() + .await + .map_err(|e| InterpretError::new(format!("Set trigger {trigger}: {e}")))?; + if !resp.status().is_success() { + let body = resp.text().await.unwrap_or_default(); + return Err(InterpretError::new(format!( + "Set trigger {trigger} failed: {body}" + ))); + } + info!("[ZitadelSetup] Action attached to Complement Token trigger {trigger}"); + } + Ok(()) + } + + async fn find_action_id( + &self, + client: &reqwest::Client, + pat: &str, + name: &str, + ) -> Result, String> { + let resp = self + .post(client, "/management/v1/actions/_search") + .bearer_auth(pat) + .json(&serde_json::json!({ + "queries": [ { "actionNameQuery": { "name": name } } ] + })) + .send() + .await + .map_err(|e| format!("Search actions: {e}"))?; + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(format!("action search returned {status}: {body}")); + } + let body: serde_json::Value = resp + .json() + .await + .map_err(|e| format!("Parse action search: {e}"))?; + Ok(body["result"] + .as_array() + .into_iter() + .flatten() + .find(|a| a["name"].as_str() == Some(name)) + .and_then(|a| a["id"].as_str().map(str::to_string))) + } + async fn find_user_grant( &self, client: &reqwest::Client, @@ -1666,6 +1838,11 @@ impl Interpret for ZitadelSetupInterpret { } } + if self.score.groups_claim_action { + self.ensure_groups_claim_action(&client, &pat).await?; + details.push(format!("action:{GROUPS_CLAIM_ACTION_NAME}")); + } + config.save().map_err(InterpretError::new)?; Ok(Outcome { @@ -1751,6 +1928,7 @@ mod tests { api_apps: vec![], roles: vec![], machine_users: vec![], + groups_claim_action: false, } } diff --git a/harmony_config/src/lib.rs b/harmony_config/src/lib.rs index dde30471..572e9b36 100644 --- a/harmony_config/src/lib.rs +++ b/harmony_config/src/lib.rs @@ -198,8 +198,7 @@ impl ConfigClient { /// Build an OpenBao-backed `StoreSource` from env vars, or `None` (with a /// `warn!`) when `OPENBAO_URL`/`VAULT_ADDR` is unset or the connection /// fails — so a missing OpenBao degrades the chain instead of breaking -/// startup. `jwt_role`/`jwt_auth_mount` are left to `OpenbaoSecretStore`'s -/// defaults, matching what `OpenbaoSetupScore` configures. +/// startup. async fn openbao_from_env(namespace: &str) -> Option> { let Some(url) = std::env::var("OPENBAO_URL") .or_else(|_| std::env::var("VAULT_ADDR")) @@ -210,19 +209,45 @@ async fn openbao_from_env(namespace: &str) -> Option> { }; let env = |k: &str| std::env::var(k).ok(); - let store = harmony_secret::OpenbaoSecretStore::new( - url, - env("OPENBAO_KV_MOUNT").unwrap_or_else(|| "secret".to_string()), - env("OPENBAO_AUTH_MOUNT").unwrap_or_else(|| "jwt".to_string()), - env("OPENBAO_SKIP_TLS").as_deref() == Some("true"), - env("OPENBAO_TOKEN"), - env("OPENBAO_USERNAME"), - env("OPENBAO_PASSWORD"), - env("HARMONY_SSO_URL"), - env("HARMONY_SSO_CLIENT_ID"), - None, - None, - ) + + // Headless Zitadel-machine → OpenBao (JWT-bearer) rung: needs a machine + // keyfile (path or inline JSON) plus the project-ID audience. The issuer + // is the same Zitadel base the OIDC rung uses (`HARMONY_SSO_URL`). Absent + // any piece, the rung stays off and the ladder falls through. + let zitadel_jwt_bearer = { + let key_path = env("HARMONY_ZITADEL_KEY_PATH").map(PathBuf::from); + let key_json = env("HARMONY_ZITADEL_KEY_JSON"); + match ( + key_path.is_some() || key_json.is_some(), + env("HARMONY_SSO_URL"), + env("HARMONY_ZITADEL_AUDIENCE"), + ) { + (true, Some(oidc_issuer_url), Some(audience)) => { + Some(harmony_secret::ZitadelJwtBearerConfig { + key_path, + key_json, + oidc_issuer_url, + audience, + }) + } + _ => None, + } + }; + + let store = harmony_secret::OpenbaoSecretStore::new(harmony_secret::OpenbaoStoreOptions { + base_url: url, + kv_mount: env("OPENBAO_KV_MOUNT").unwrap_or_else(|| "secret".to_string()), + auth_mount: env("OPENBAO_AUTH_MOUNT").unwrap_or_else(|| "jwt".to_string()), + skip_tls: env("OPENBAO_SKIP_TLS").as_deref() == Some("true"), + token: env("OPENBAO_TOKEN"), + username: env("OPENBAO_USERNAME"), + password: env("OPENBAO_PASSWORD"), + zitadel_sso_url: env("HARMONY_SSO_URL"), + zitadel_client_id: env("HARMONY_SSO_CLIENT_ID"), + jwt_role: env("OPENBAO_JWT_ROLE"), + jwt_auth_mount: env("OPENBAO_JWT_AUTH_MOUNT"), + zitadel_jwt_bearer, + }) .await; match store { diff --git a/harmony_secret/Cargo.toml b/harmony_secret/Cargo.toml index 8d8b9356..576dc819 100644 --- a/harmony_secret/Cargo.toml +++ b/harmony_secret/Cargo.toml @@ -12,6 +12,7 @@ license.workspace = true unexpected_cfgs = { level = "warn", check-cfg = ['cfg(secrete2etest)'] } [dependencies] +harmony_zitadel_jwt = { path = "../harmony_zitadel_jwt" } harmony_secret_derive = { version = "0.1.0", path = "../harmony_secret_derive" } serde = { version = "1.0.209", features = ["derive", "rc"] } serde_json = "1.0.127" diff --git a/harmony_secret/src/lib.rs b/harmony_secret/src/lib.rs index cda2786f..a66ebf78 100644 --- a/harmony_secret/src/lib.rs +++ b/harmony_secret/src/lib.rs @@ -26,7 +26,7 @@ use serde::{Serialize, de::DeserializeOwned}; use std::fmt; use store::InfisicalSecretStore; use store::LocalFileSecretStore; -pub use store::OpenbaoSecretStore; +pub use store::{OpenbaoSecretStore, OpenbaoStoreOptions, ZitadelJwtBearerConfig}; use thiserror::Error; use tokio::sync::OnceCell; @@ -85,19 +85,20 @@ async fn init_secret_manager() -> SecretManager { let store: Box = match store_type.as_str() { "file" => Box::new(LocalFileSecretStore), "openbao" | "vault" => { - let store = OpenbaoSecretStore::new( - OPENBAO_URL.clone().expect("Openbao/Vault URL must be set, see harmony_secret config for ways to provide it. You can try with OPENBAO_URL or VAULT_ADDR"), - OPENBAO_KV_MOUNT.clone(), - OPENBAO_AUTH_MOUNT.clone(), - *OPENBAO_SKIP_TLS, - OPENBAO_TOKEN.clone(), - OPENBAO_USERNAME.clone(), - OPENBAO_PASSWORD.clone(), - HARMONY_SSO_URL.clone(), - HARMONY_SSO_CLIENT_ID.clone(), - None, - None, - ) + let store = OpenbaoSecretStore::new(OpenbaoStoreOptions { + base_url: OPENBAO_URL.clone().expect("Openbao/Vault URL must be set, see harmony_secret config for ways to provide it. You can try with OPENBAO_URL or VAULT_ADDR"), + kv_mount: OPENBAO_KV_MOUNT.clone(), + auth_mount: OPENBAO_AUTH_MOUNT.clone(), + skip_tls: *OPENBAO_SKIP_TLS, + token: OPENBAO_TOKEN.clone(), + username: OPENBAO_USERNAME.clone(), + password: OPENBAO_PASSWORD.clone(), + zitadel_sso_url: HARMONY_SSO_URL.clone(), + zitadel_client_id: HARMONY_SSO_CLIENT_ID.clone(), + jwt_role: None, + jwt_auth_mount: None, + zitadel_jwt_bearer: None, + }) .await .expect("Failed to initialize Openbao/Vault secret store"); Box::new(store) diff --git a/harmony_secret/src/store/mod.rs b/harmony_secret/src/store/mod.rs index 8179291d..7b0ed18c 100644 --- a/harmony_secret/src/store/mod.rs +++ b/harmony_secret/src/store/mod.rs @@ -5,4 +5,4 @@ pub mod zitadel; pub use infisical::InfisicalSecretStore; pub use local_file::LocalFileSecretStore; -pub use openbao::OpenbaoSecretStore; +pub use openbao::{OpenbaoSecretStore, OpenbaoStoreOptions, ZitadelJwtBearerConfig}; diff --git a/harmony_secret/src/store/openbao.rs b/harmony_secret/src/store/openbao.rs index d0110cd3..b36702ed 100644 --- a/harmony_secret/src/store/openbao.rs +++ b/harmony_secret/src/store/openbao.rs @@ -1,10 +1,13 @@ use crate::{SecretStore, SecretStoreError}; use async_trait::async_trait; +use harmony_zitadel_jwt::{MachineKeyFile, ZitadelJwtBearer}; use log::{debug, info, warn}; use serde::{Deserialize, Serialize}; use std::fmt::Debug; use std::fs; use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::Mutex; use vaultrs::auth; use vaultrs::client::{VaultClient, VaultClientSettingsBuilder}; use vaultrs::kv2; @@ -30,21 +33,47 @@ impl From for AuthInfo { } pub struct OpenbaoSecretStore { - client: VaultClient, + client: Mutex, kv_mount: String, auth_mount: String, + jwt_bearer: Option>, + jwt_role: Option, + jwt_auth_mount: Option, + base_url: String, + skip_tls: bool, } impl Debug for OpenbaoSecretStore { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OpenbaoSecretStore") - .field("client", &self.client.settings) .field("kv_mount", &self.kv_mount) .field("auth_mount", &self.auth_mount) .finish() } } +pub struct OpenbaoStoreOptions { + pub base_url: String, + pub kv_mount: String, + pub auth_mount: String, + pub skip_tls: bool, + pub token: Option, + pub username: Option, + pub password: Option, + pub zitadel_sso_url: Option, + pub zitadel_client_id: Option, + pub zitadel_jwt_bearer: Option, + pub jwt_role: Option, + pub jwt_auth_mount: Option, +} + +pub struct ZitadelJwtBearerConfig { + pub key_path: Option, + pub key_json: Option, + pub oidc_issuer_url: String, + pub audience: String, +} + impl OpenbaoSecretStore { /// Creates a new Openbao/Vault secret store with authentication. /// @@ -52,34 +81,35 @@ impl OpenbaoSecretStore { /// - `auth_mount`: The auth method mount path (e.g., `"userpass"`). Used for userpass authentication. /// - `zitadel_sso_url`: Zitadel OIDC server URL (e.g., `"https://sso.example.com"`). If provided along with `zitadel_client_id`, Zitadel OIDC device flow will be attempted. /// - `zitadel_client_id`: OIDC client ID registered in Zitadel. - // TODO(clippy): refactor to a typed options struct - #[allow(clippy::too_many_arguments)] - pub async fn new( - base_url: String, - kv_mount: String, - auth_mount: String, - skip_tls: bool, - token: Option, - username: Option, - password: Option, - zitadel_sso_url: Option, - zitadel_client_id: Option, - jwt_role: Option, - jwt_auth_mount: Option, - ) -> Result { - info!("OPENBAO_STORE: Initializing client for URL: {base_url}"); + pub async fn new(options: OpenbaoStoreOptions) -> Result { + info!( + "OPENBAO_STORE: Initializing client for URL: {}", + options.base_url + ); // 1. If token is provided via env var, use it directly - if let Some(t) = token { + if let Some(t) = &options.token { debug!("OPENBAO_STORE: Using token from environment variable"); - return Self::with_token(&base_url, skip_tls, &t, &kv_mount, &auth_mount); + return Self::with_token( + &options.base_url, + options.skip_tls, + t, + &options.kv_mount, + &options.auth_mount, + ); } // 2. Try to load cached token - let cache_path = Self::get_token_cache_path(&base_url); + let cache_path = Self::get_token_cache_path(&options.base_url); if let Ok(mut cached_token) = Self::load_cached_token(&cache_path) { debug!("OPENBAO_STORE: Found cached token, validating..."); - if Self::validate_token(&base_url, skip_tls, &cached_token.client_token).await { + if Self::validate_token( + &options.base_url, + options.skip_tls, + &cached_token.client_token, + ) + .await + { info!("OPENBAO_STORE: Cached token is valid"); // Best-effort renewal: extend the server-side lease so @@ -89,7 +119,13 @@ impl OpenbaoSecretStore { // blip, etc.) doesn't block the caller — we just keep // the validated cached token as-is and let the next // invocation try again. - match Self::renew_self(&base_url, skip_tls, &cached_token.client_token).await { + match Self::renew_self( + &options.base_url, + options.skip_tls, + &cached_token.client_token, + ) + .await + { Ok(new_lease) => { info!("OPENBAO_STORE: cached token renewed, lease_duration={new_lease}s"); cached_token.lease_duration = Some(new_lease); @@ -106,26 +142,59 @@ impl OpenbaoSecretStore { } return Self::with_token( - &base_url, - skip_tls, + &options.base_url, + options.skip_tls, &cached_token.client_token, - &kv_mount, - &auth_mount, + &options.kv_mount, + &options.auth_mount, ); } warn!("OPENBAO_STORE: Cached token is invalid or expired"); } - // 3. Try Zitadel OIDC device flow if configured - if let (Some(sso_url), Some(client_id)) = (zitadel_sso_url, zitadel_client_id) { + // 3. Try Zitadel JWT-bearer for headless devices if configured + if let Some(jwt_bearer_config) = &options.zitadel_jwt_bearer { + info!("OPENBAO_STORE: Attempting Zitadel JWT-bearer authentication"); + match Self::zitadel_jwt_bearer(jwt_bearer_config, options.skip_tls) { + Ok(jwt_bearer) => { + let store = Self::with_token_and_jwt( + &options.base_url, + options.skip_tls, + "", + &options.kv_mount, + &options.auth_mount, + Some(Arc::new(jwt_bearer)), + options.jwt_role.clone(), + options.jwt_auth_mount.clone(), + )?; + match store.refresh_auth().await { + Ok(()) => { + info!("OPENBAO_STORE: Zitadel JWT-bearer authentication successful"); + return Ok(store); + } + Err(e) => warn!( + "OPENBAO_STORE: Zitadel JWT-bearer failed: {e}, trying other auth methods" + ), + } + } + Err(e) => warn!( + "OPENBAO_STORE: Zitadel JWT-bearer setup failed: {e}, trying other auth methods" + ), + } + } + + // 4. Try Zitadel OIDC device flow if configured + if let (Some(sso_url), Some(client_id)) = + (&options.zitadel_sso_url, &options.zitadel_client_id) + { info!("OPENBAO_STORE: Attempting Zitadel OIDC device flow"); match Self::authenticate_zitadel_oidc( - &base_url, - &sso_url, - &client_id, - skip_tls, - jwt_role.as_deref(), - jwt_auth_mount.as_deref(), + &options.base_url, + sso_url, + client_id, + options.skip_tls, + options.jwt_role.as_deref(), + options.jwt_auth_mount.as_deref(), ) .await { @@ -141,11 +210,11 @@ impl OpenbaoSecretStore { warn!("OPENBAO_STORE: Failed to cache OIDC token: {e}"); } return Self::with_token( - &base_url, - skip_tls, + &options.base_url, + options.skip_tls, &auth_info.client_token, - &kv_mount, - &auth_mount, + &options.kv_mount, + &options.auth_mount, ); } Err(e) => { @@ -154,8 +223,8 @@ impl OpenbaoSecretStore { } } - // 4. Authenticate with username/password - let (user, pass) = match (username, password) { + // 5. Authenticate with username/password + let (user, pass) = match (&options.username, &options.password) { (Some(u), Some(p)) => (u, p), _ => { return Err(SecretStoreError::Store( @@ -167,8 +236,14 @@ impl OpenbaoSecretStore { } }; - let token = - Self::authenticate_userpass(&base_url, &auth_mount, skip_tls, &user, &pass).await?; + let token = Self::authenticate_userpass( + &options.base_url, + &options.auth_mount, + options.skip_tls, + user, + pass, + ) + .await?; // Cache the token if let Err(e) = Self::cache_token(&cache_path, &token) { @@ -176,11 +251,11 @@ impl OpenbaoSecretStore { } Self::with_token( - &base_url, - skip_tls, + &options.base_url, + options.skip_tls, &token.client_token, - &kv_mount, - &auth_mount, + &options.kv_mount, + &options.auth_mount, ) } @@ -206,14 +281,139 @@ impl OpenbaoSecretStore { .map_err(|e| SecretStoreError::Store(e.into())) } - /// Create a client with an existing token - fn with_token( + /// Device-side constructor: authenticate exclusively through an + /// existing [`ZitadelJwtBearer`] (the same instance the NATS auth + /// path holds — one machine identity, two consumers). No token + /// file cache, no fallback chain: login happens lazily on the + /// first read via the permission-denied retry in `get_raw`, and a + /// later denial retries once through a fresh login because batch + /// tokens expire without renewal (ADR-025). + pub fn with_zitadel_bearer( + base_url: &str, + skip_tls: bool, + kv_mount: &str, + jwt_auth_mount: &str, + jwt_role: &str, + bearer: Arc, + ) -> Result { + Self::with_token_and_jwt( + base_url, + skip_tls, + "", + kv_mount, + jwt_auth_mount, + Some(bearer), + Some(jwt_role.to_string()), + Some(jwt_auth_mount.to_string()), + ) + } + + pub async fn refresh_auth(&self) -> Result<(), SecretStoreError> { + let bearer = self.jwt_bearer.as_ref().ok_or_else(|| { + SecretStoreError::Store("Zitadel JWT-bearer auth is not configured".into()) + })?; + let role = self + .jwt_role + .as_deref() + .ok_or_else(|| SecretStoreError::Store("OpenBao JWT role is not configured".into()))?; + let jwt_auth_mount = self.jwt_auth_mount.as_deref().unwrap_or("jwt"); + + bearer.invalidate_cache(); + let jwt = bearer + .bearer_token() + .await + .map_err(|e| SecretStoreError::Store(e.into()))?; + let token = + Self::jwt_login(&self.base_url, self.skip_tls, jwt_auth_mount, role, &jwt).await?; + let client = Self::build_vault_client(&self.base_url, self.skip_tls, &token)?; + + *self.client.lock().await = client; + Ok(()) + } + + fn zitadel_jwt_bearer( + config: &ZitadelJwtBearerConfig, + danger_accept_invalid_certs: bool, + ) -> Result { + let key = match config.key_json.as_deref().map(str::trim) { + Some(raw) if !raw.is_empty() => serde_json::from_str::(raw), + _ => { + let path = config.key_path.as_ref().ok_or_else(|| { + SecretStoreError::Store( + "Zitadel JWT-bearer requires key_json or key_path".into(), + ) + })?; + let raw = + fs::read_to_string(path).map_err(|e| SecretStoreError::Store(e.into()))?; + serde_json::from_str::(&raw) + } + } + .map_err(|e| SecretStoreError::Store(e.into()))?; + + ZitadelJwtBearer::new( + key, + config.oidc_issuer_url.clone(), + config.audience.clone(), + danger_accept_invalid_certs, + ) + .map_err(|e| SecretStoreError::Store(e.into())) + } + + async fn jwt_login( + base_url: &str, + skip_tls: bool, + jwt_auth_mount: &str, + role: &str, + jwt: &str, + ) -> Result { + let mut builder = reqwest::Client::builder(); + if skip_tls { + builder = builder.danger_accept_invalid_certs(true); + } + let client = builder + .build() + .map_err(|e| SecretStoreError::Store(e.into()))?; + let url = format!( + "{}/v1/auth/{}/login", + base_url.trim_end_matches('/'), + jwt_auth_mount.trim_matches('/') + ); + let response = client + .post(&url) + .json(&serde_json::json!({ "role": role, "jwt": jwt })) + .send() + .await + .map_err(|e| SecretStoreError::Store(e.into()))?; + let status = response.status(); + let body = response + .text() + .await + .map_err(|e| SecretStoreError::Store(e.into()))?; + if !status.is_success() { + return Err(SecretStoreError::Store( + format!("OpenBao JWT login returned {status}: {body}").into(), + )); + } + + let parsed: serde_json::Value = + serde_json::from_str(&body).map_err(|e| SecretStoreError::Store(e.into()))?; + parsed + .get("auth") + .and_then(|auth| auth.get("client_token")) + .and_then(|token| token.as_str()) + .map(str::to_string) + .ok_or_else(|| { + SecretStoreError::Store( + format!("OpenBao JWT login response missing auth.client_token: {body}").into(), + ) + }) + } + + fn build_vault_client( base_url: &str, skip_tls: bool, token: &str, - kv_mount: &str, - auth_mount: &str, - ) -> Result { + ) -> Result { let mut settings = VaultClientSettingsBuilder::default(); settings.address(base_url).token(token); @@ -222,17 +422,48 @@ impl OpenbaoSecretStore { settings.verify(false); } - let client = VaultClient::new( + VaultClient::new( settings .build() .map_err(|e| SecretStoreError::Store(Box::new(e)))?, ) - .map_err(|e| SecretStoreError::Store(Box::new(e)))?; + .map_err(|e| SecretStoreError::Store(Box::new(e))) + } + + /// Create a client with an existing token + fn with_token( + base_url: &str, + skip_tls: bool, + token: &str, + kv_mount: &str, + auth_mount: &str, + ) -> Result { + Self::with_token_and_jwt( + base_url, skip_tls, token, kv_mount, auth_mount, None, None, None, + ) + } + + fn with_token_and_jwt( + base_url: &str, + skip_tls: bool, + token: &str, + kv_mount: &str, + auth_mount: &str, + jwt_bearer: Option>, + jwt_role: Option, + jwt_auth_mount: Option, + ) -> Result { + let client = Self::build_vault_client(base_url, skip_tls, token)?; Ok(Self { - client, + client: Mutex::new(client), kv_mount: kv_mount.to_string(), auth_mount: auth_mount.to_string(), + base_url: base_url.to_string(), + skip_tls, + jwt_bearer, + jwt_role, + jwt_auth_mount, }) } @@ -404,25 +635,23 @@ impl OpenbaoSecretStore { } } -#[async_trait] -impl SecretStore for OpenbaoSecretStore { - async fn get_raw(&self, namespace: &str, key: &str) -> Result, SecretStoreError> { +impl OpenbaoSecretStore { + async fn read_value(&self, namespace: &str, key: &str) -> Result, SecretStoreError> { let path = format!("{}/{}", namespace, key); - info!("OPENBAO_STORE: Getting key '{key}' from namespace '{namespace}'"); - debug!("OPENBAO_STORE: Request path: {path}"); - - let data: serde_json::Value = kv2::read(&self.client, &self.kv_mount, &path) - .await - .map_err(|e| { - if e.to_string().contains("does not exist") || e.to_string().contains("404") { - SecretStoreError::NotFound { - namespace: namespace.to_string(), - key: key.to_string(), + let client = self.client.lock().await; + let data: serde_json::Value = + kv2::read(&*client, &self.kv_mount, &path) + .await + .map_err(|e| { + if e.to_string().contains("does not exist") || e.to_string().contains("404") { + SecretStoreError::NotFound { + namespace: namespace.to_string(), + key: key.to_string(), + } + } else { + SecretStoreError::Store(Box::new(e)) } - } else { - SecretStoreError::Store(Box::new(e)) - } - })?; + })?; let value = data.get("value").and_then(|v| v.as_str()).ok_or_else(|| { SecretStoreError::Store("Secret does not contain expected 'value' field".into()) @@ -430,6 +659,31 @@ impl SecretStore for OpenbaoSecretStore { Ok(value.as_bytes().to_vec()) } +} + +fn is_permission_denied(e: &SecretStoreError) -> bool { + let msg = e.to_string(); + msg.contains("permission denied") || msg.contains("403") +} + +#[async_trait] +impl SecretStore for OpenbaoSecretStore { + async fn get_raw(&self, namespace: &str, key: &str) -> Result, SecretStoreError> { + info!("OPENBAO_STORE: Getting key '{key}' from namespace '{namespace}'"); + + match self.read_value(namespace, key).await { + // Batch tokens expire without renewal; a fresh JWT login + // IS the refresh path (ADR-025). Exactly one retry: if a + // freshly-issued token is still denied, the policy really + // doesn't allow it and the error must surface. + Err(e) if self.jwt_bearer.is_some() && is_permission_denied(&e) => { + debug!("OPENBAO_STORE: read denied, re-logging in via JWT bearer: {e}"); + self.refresh_auth().await?; + self.read_value(namespace, key).await + } + r => r, + } + } async fn set_raw( &self, @@ -447,8 +701,9 @@ impl SecretStore for OpenbaoSecretStore { let data = serde_json::json!({ "value": value_str }); + let client = self.client.lock().await; - kv2::set(&self.client, &self.kv_mount, &path, &data) + kv2::set(&*client, &self.kv_mount, &path, &data) .await .map_err(|e| SecretStoreError::Store(Box::new(e)))?; diff --git a/harmony_zitadel_auth/Cargo.toml b/harmony_zitadel_auth/Cargo.toml index a0ee0c27..afeeba43 100644 --- a/harmony_zitadel_auth/Cargo.toml +++ b/harmony_zitadel_auth/Cargo.toml @@ -11,6 +11,7 @@ axum = ["dep:axum", "dep:axum-extra"] [dependencies] harmony_config = { path = "../harmony_config" } +harmony_zitadel_jwt = { path = "../harmony_zitadel_jwt" } anyhow.workspace = true base64.workspace = true chrono = { workspace = true, features = ["serde"] } diff --git a/harmony_zitadel_auth/src/axum_login_flow.rs b/harmony_zitadel_auth/src/axum_login_flow.rs index 96a9391f..0a61613e 100644 --- a/harmony_zitadel_auth/src/axum_login_flow.rs +++ b/harmony_zitadel_auth/src/axum_login_flow.rs @@ -9,8 +9,8 @@ use base64::engine::general_purpose::URL_SAFE_NO_PAD; use crate::config::ZitadelAuthConfig; use crate::jwks::JwksCache; use crate::login::{ - AuthCallbackQuery, RawAuthCallbackQuery, TokenResponse, build_login_attempt, build_logout_url, - exchange_code_for_token, jwt_exp, validate_callback_state, + AuthCallbackQuery, LoginQuery, RawAuthCallbackQuery, TokenResponse, build_login_attempt, + build_logout_url, exchange_code_for_token, jwt_exp, valid_next, validate_callback_state, }; use crate::session::LoginAttemptCookie; @@ -23,8 +23,9 @@ pub const HARMONY_SESSION_COOKIE: &str = "harmony_fleet_session"; pub async fn login_handler( jar: PrivateCookieJar, State(config): State, + Query(query): Query, ) -> Response { - match build_login_response(jar, &config) { + match build_login_response(jar, &config, query) { Ok(r) => r.into_response(), Err(e) => auth_error_response(e), } @@ -33,9 +34,16 @@ pub async fn login_handler( fn build_login_response( jar: PrivateCookieJar, config: &ZitadelAuthConfig, + query: LoginQuery, ) -> Result { let attempt = build_login_attempt(config)?; - let cookie_payload = LoginAttemptCookie::from(&attempt); + let mut cookie_payload = LoginAttemptCookie::from(&attempt); + cookie_payload.next = Some( + query + .next + .filter(|next| valid_next(next)) + .unwrap_or_else(|| "/".to_string()), + ); let cookie_value = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&cookie_payload)?); let mut builder = Cookie::build((LOGIN_ATTEMPT_COOKIE, cookie_value)) @@ -113,7 +121,12 @@ async fn build_callback_response( } let session_jar = session_jar.add(session_cookie(&tokens, config)); - Ok((jar, session_jar, Redirect::to("/")).into_response()) + let next = attempt + .next + .as_deref() + .filter(|next| valid_next(next)) + .unwrap_or("/"); + Ok((jar, session_jar, Redirect::to(next)).into_response()) } AuthCallbackQuery::Failure { error, diff --git a/harmony_zitadel_auth/src/lib.rs b/harmony_zitadel_auth/src/lib.rs index bb4f8af0..88a363ab 100644 --- a/harmony_zitadel_auth/src/lib.rs +++ b/harmony_zitadel_auth/src/lib.rs @@ -6,13 +6,14 @@ pub mod login; pub mod session; pub use config::{OperatorCookieKey, ZitadelAuthConfig}; +pub use harmony_zitadel_jwt::{MachineKeyFile, ZitadelJwtBearer}; pub use jwks::JwksCache; pub use login::{ - AuthCallbackQuery, LoginAttempt, RawAuthCallbackQuery, TokenResponse, ValidatedUser, - build_login_attempt, build_logout_url, exchange_code_for_token, jwt_exp, - validate_callback_state, validate_id_token, + AuthCallbackQuery, LoginAttempt, LoginQuery, RawAuthCallbackQuery, TokenResponse, + ValidatedUser, build_login_attempt, build_logout_url, exchange_code_for_token, jwt_exp, + valid_next, validate_callback_state, validate_id_token, }; pub use session::{LoginAttemptCookie, VerifiedSession}; diff --git a/harmony_zitadel_auth/src/login.rs b/harmony_zitadel_auth/src/login.rs index c92aae1b..a87c8578 100644 --- a/harmony_zitadel_auth/src/login.rs +++ b/harmony_zitadel_auth/src/login.rs @@ -37,6 +37,11 @@ pub struct TokenResponse { pub expires_in: Option, } +#[derive(Debug, Deserialize)] +pub struct LoginQuery { + pub next: Option, +} + #[derive(Debug, Deserialize)] pub struct RawAuthCallbackQuery { pub code: Option, @@ -63,10 +68,15 @@ impl From<&LoginAttempt> for LoginAttemptCookie { state: attempt.state.clone(), pkce_code_verifier: attempt.pkce_code_verifier.clone(), nonce: attempt.nonce.clone(), + next: None, } } } +pub fn valid_next(next: &str) -> bool { + next.starts_with('/') && !next.starts_with("//") && !next.chars().any(char::is_control) +} + impl TryFrom for AuthCallbackQuery { type Error = anyhow::Error; diff --git a/harmony_zitadel_auth/src/session.rs b/harmony_zitadel_auth/src/session.rs index 8e5f73c3..68f9dbf9 100644 --- a/harmony_zitadel_auth/src/session.rs +++ b/harmony_zitadel_auth/src/session.rs @@ -18,4 +18,5 @@ pub struct LoginAttemptCookie { pub state: String, pub pkce_code_verifier: String, pub nonce: String, + pub next: Option, } diff --git a/harmony_zitadel_jwt/Cargo.toml b/harmony_zitadel_jwt/Cargo.toml new file mode 100644 index 00000000..0ddbd2cc --- /dev/null +++ b/harmony_zitadel_jwt/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "harmony_zitadel_jwt" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +chrono = { workspace = true } +jsonwebtoken = "9" +reqwest = { workspace = true, features = ["json"] } +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true diff --git a/harmony_zitadel_jwt/src/lib.rs b/harmony_zitadel_jwt/src/lib.rs new file mode 100644 index 00000000..ceed32d2 --- /dev/null +++ b/harmony_zitadel_jwt/src/lib.rs @@ -0,0 +1,367 @@ +use serde::Deserialize; +use std::sync::Mutex; + +use anyhow::{Context, Result}; +use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader}; + +pub struct ZitadelJwtBearer { + key: MachineKeyFile, + oidc_issuer_url: String, + audience: String, + http: reqwest::Client, + cache: Mutex>, +} + +/// Refresh tokens this many seconds before their advertised expiry. +/// Five minutes leaves headroom for clock skew, slow networks, and +/// the round-trip cost of re-minting against Zitadel. +pub const TOKEN_REFRESH_LEEWAY_SECS: i64 = 5 * 60; + +/// Lifetime of the JWT *assertion* (the client-side bearer JWT we sign +/// to authenticate to Zitadel's token endpoint). Zitadel rejects +/// assertions with `exp - iat > 60s`; one minute is the safe ceiling. +pub const ASSERTION_LIFETIME_SECS: i64 = 60; + +/// JSON keyfile content as Zitadel emits it for a `KEY_TYPE_JSON` +/// machine key. The `key` is a PEM-encoded RSA private key. +#[derive(Debug, Clone, Deserialize)] +pub struct MachineKeyFile { + #[serde(rename = "type")] + pub _type: String, + #[serde(rename = "keyId")] + pub key_id: String, + pub key: String, + #[serde(rename = "userId")] + pub user_id: String, +} + +#[derive(Debug, Clone)] +pub struct CachedToken { + pub(crate) access_token: String, + /// Unix seconds at which the token is no longer trusted by + /// `cached_if_fresh`. Computed from the OAuth response's `expires_in` + /// and the local clock at mint time. + pub(crate) expires_at_unix: i64, +} + +#[derive(Deserialize)] +struct TokenResponse { + access_token: String, + #[serde(default)] + expires_in: Option, +} + +impl ZitadelJwtBearer { + pub fn new( + key: MachineKeyFile, + oidc_issuer_url: String, + audience: String, + danger_accept_invalid_certs: bool, + ) -> Result { + let http = reqwest::Client::builder() + .danger_accept_invalid_certs(danger_accept_invalid_certs) + .timeout(std::time::Duration::from_secs(10)) + .build() + .context("building HTTP client for Zitadel token endpoint")?; + + Ok(Self { + key, + oidc_issuer_url, + audience, + http, + cache: Mutex::new(None), + }) + } + + pub async fn bearer_token(&self) -> Result { + { + let now = chrono::Utc::now().timestamp(); + let guard = self.cache.lock().unwrap(); + if let Some(cached) = guard.as_ref() + && cached.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS > now + { + return Ok(cached.access_token.clone()); + } + } + + let now = chrono::Utc::now().timestamp(); + let assertion = build_assertion(&self.key, &self.oidc_issuer_url, now)?; + let scope = build_scope(&self.audience); + let token_url = build_token_url(&self.oidc_issuer_url); + + let resp = self + .http + .post(&token_url) + .form(&[ + ( + "grant_type", + "urn:ietf:params:oauth:grant-type:jwt-bearer".to_string(), + ), + ("assertion", assertion), + ("scope", scope), + ]) + .send() + .await + .with_context(|| format!("POST {token_url}"))?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("Zitadel token endpoint returned {status}: {body}"); + } + + let tr: TokenResponse = resp.json().await.context("parsing token response")?; + let expires_in = tr.expires_in.unwrap_or(3600); + let token = tr.access_token.clone(); + + *self.cache.lock().unwrap() = Some(CachedToken { + access_token: tr.access_token, + expires_at_unix: now + expires_in, + }); + + Ok(token) + } + + pub fn invalidate_cache(&self) { + *self.cache.lock().unwrap() = None; + } + + #[cfg(test)] + fn cached_if_fresh(&self) -> Option { + let now = chrono::Utc::now().timestamp(); + let guard = self.cache.lock().unwrap(); + let cached = guard.as_ref()?; + (cached.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS > now) + .then(|| cached.access_token.clone()) + } +} + +/// Build the JWT-bearer assertion. Split out from the network path so +/// the claims + header shape can be unit-tested without an HTTP server, +/// and split internally into the (pure) claim/header builders so they +/// can be unit-tested without an RSA private key fixture. +pub(crate) fn build_assertion( + key: &MachineKeyFile, + oidc_issuer_url: &str, + now: i64, +) -> Result { + let claims = build_assertion_claims(key, oidc_issuer_url, now); + let header = build_assertion_header(key); + let assertion = jsonwebtoken::encode( + &header, + &claims, + &EncodingKey::from_rsa_pem(key.key.as_bytes()) + .context("parsing RSA private key from machine key file")?, + ) + .context("signing JWT assertion")?; + Ok(assertion) +} + +/// Pure claim payload for the JWT-bearer assertion. `iss == sub == userId` +/// is a Zitadel requirement; `aud` is Zitadel itself (the token endpoint +/// is reached via `oidc_issuer_url`); `exp - iat` MUST be ≤ 60 s or +/// Zitadel rejects. +pub(crate) fn build_assertion_claims( + key: &MachineKeyFile, + oidc_issuer_url: &str, + now: i64, +) -> serde_json::Value { + serde_json::json!({ + "iss": key.user_id, + "sub": key.user_id, + "aud": oidc_issuer_url, + "exp": now + ASSERTION_LIFETIME_SECS, + "iat": now, + }) +} + +/// JWT header for the assertion. The `kid` tells Zitadel which of the +/// machine user's registered keys to verify the signature against. +pub(crate) fn build_assertion_header(key: &MachineKeyFile) -> JwtHeader { + let mut header = JwtHeader::new(Algorithm::RS256); + header.kid = Some(key.key_id.clone()); + header +} + +/// Build the OAuth `scope` string for the token-bearer request. +/// +/// Three scopes are needed for the access token to be useful here: +/// +/// * `openid` — base OIDC requirement. +/// * `urn:zitadel:iam:org:projects:roles` (PLURAL "projects") — +/// tells Zitadel to include the role-claim block in the access +/// token. Without this, the callout sees "no authorized role +/// in token" even when the user has a project role grant. +/// * `urn:zitadel:iam:org:project:id::aud` (SINGULAR +/// "project") — adds to the access token's `aud` claim +/// so the callout's audience validation accepts the project +/// ID we're using as the JWT-bearer audience. +/// +/// The plural-vs-singular distinction is a Zitadel convention, +/// not a typo. Both scopes are required. +pub(crate) fn build_scope(audience: &str) -> String { + format!( + "openid \ + urn:zitadel:iam:org:projects:roles \ + urn:zitadel:iam:org:project:id:{audience}:aud" + ) +} + +/// Resolve the token endpoint URL, tolerating a trailing slash on +/// `oidc_issuer_url`. Without trimming, a configured issuer of +/// `https://sso.example.com/` produces `…//oauth/v2/token` which 404s. +pub(crate) fn build_token_url(oidc_issuer_url: &str) -> String { + format!("{}/oauth/v2/token", oidc_issuer_url.trim_end_matches('/')) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fake_key() -> MachineKeyFile { + MachineKeyFile { + _type: "serviceaccount".to_string(), + key_id: "kid-371358469099356247".to_string(), + key: "PEM-PLACEHOLDER".to_string(), + user_id: "uid-371358469065801815".to_string(), + } + } + + fn bearer() -> ZitadelJwtBearer { + ZitadelJwtBearer::new( + fake_key(), + "http://sso.fleet.local:8080".to_string(), + "366378028009259037".to_string(), + false, + ) + .unwrap() + } + + #[test] + fn cached_token_within_leeway_is_treated_as_expired() { + let now = chrono::Utc::now().timestamp(); + let about_to_expire = CachedToken { + access_token: "x".to_string(), + expires_at_unix: now + TOKEN_REFRESH_LEEWAY_SECS - 1, + }; + assert!( + about_to_expire.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS <= now, + "tokens within the leeway window must be considered expired" + ); + + let comfortable = CachedToken { + access_token: "x".to_string(), + expires_at_unix: now + TOKEN_REFRESH_LEEWAY_SECS + 60, + }; + assert!( + comfortable.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS > now, + "tokens with comfortable headroom must be cache-hits" + ); + } + + #[test] + fn cached_if_fresh_returns_some_when_outside_leeway() { + let bearer = bearer(); + let now = chrono::Utc::now().timestamp(); + *bearer.cache.lock().unwrap() = Some(CachedToken { + access_token: "fresh".to_string(), + expires_at_unix: now + TOKEN_REFRESH_LEEWAY_SECS + 60, + }); + + assert_eq!(bearer.cached_if_fresh(), Some("fresh".to_string())); + } + + #[test] + fn cached_if_fresh_returns_none_when_no_cache() { + assert_eq!(bearer().cached_if_fresh(), None); + } + + #[test] + fn assertion_claims_carry_iss_sub_aud_exp_iat() { + let now = 1_700_000_000; + let claims = build_assertion_claims(&fake_key(), "http://sso.fleet.local:8080", now); + assert_eq!(claims["iss"], "uid-371358469065801815"); + assert_eq!(claims["sub"], "uid-371358469065801815"); + assert_eq!(claims["aud"], "http://sso.fleet.local:8080"); + assert_eq!(claims["iat"].as_i64(), Some(now)); + assert_eq!(claims["exp"].as_i64(), Some(now + ASSERTION_LIFETIME_SECS)); + } + + #[test] + fn assertion_lifetime_locked_at_60_seconds() { + assert_eq!(ASSERTION_LIFETIME_SECS, 60); + } + + #[test] + fn assertion_header_carries_kid_and_rs256() { + let header = build_assertion_header(&fake_key()); + assert_eq!(header.alg, jsonwebtoken::Algorithm::RS256); + assert_eq!(header.kid.as_deref(), Some("kid-371358469099356247")); + } + + #[test] + fn scope_includes_plural_projects_roles() { + let s = build_scope("366378028009259037"); + assert!( + s.contains("urn:zitadel:iam:org:projects:roles"), + "scope must include the PLURAL projects-roles URN; got {s:?}" + ); + } + + #[test] + fn scope_audience_uses_singular_project_id_urn() { + let s = build_scope("366378028009259037"); + assert!( + s.contains("urn:zitadel:iam:org:project:id:366378028009259037:aud"), + "scope must include the SINGULAR project:id::aud URN; got {s:?}" + ); + } + + #[test] + fn scope_includes_openid_base() { + let s = build_scope("any"); + assert!( + s.split_whitespace().any(|tok| tok == "openid"), + "scope must include `openid` as a standalone token; got {s:?}" + ); + } + + #[test] + fn token_url_appends_oauth_endpoint() { + assert_eq!( + build_token_url("http://sso.fleet.local:8080"), + "http://sso.fleet.local:8080/oauth/v2/token" + ); + } + + #[test] + fn token_url_strips_single_trailing_slash() { + assert_eq!( + build_token_url("http://sso.fleet.local:8080/"), + "http://sso.fleet.local:8080/oauth/v2/token" + ); + } + + #[test] + fn token_url_strips_multiple_trailing_slashes() { + assert_eq!( + build_token_url("http://sso.fleet.local:8080///"), + "http://sso.fleet.local:8080/oauth/v2/token" + ); + } + + #[test] + fn machine_key_file_parses_zitadel_json_shape() { + let raw = r#"{ + "type": "serviceaccount", + "keyId": "371358469099356247", + "key": "-----BEGIN RSA PRIVATE KEY-----\nABC\n-----END RSA PRIVATE KEY-----\n", + "userId": "371358469065801815" + }"#; + let parsed: MachineKeyFile = serde_json::from_str(raw).expect("valid keyfile"); + assert_eq!(parsed._type, "serviceaccount"); + assert_eq!(parsed.key_id, "371358469099356247"); + assert_eq!(parsed.user_id, "371358469065801815"); + assert!(parsed.key.contains("BEGIN RSA PRIVATE KEY")); + } +}