NationTech/harmony
3
0
Fork 2
Code Issues 39 Pull Requests 11 Actions Packages Projects Releases 1 Wiki Activity

feat/compose-import-example #335

Merged
johnride merged 34 commits from feat/compose-import-example into master 2026-06-19 14:44:25 +00:00
Conversation 0 Commits 34 Files Changed 103 +7984 -905
103 changed files with 7984 additions and 905 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.env.example
View File

@@ -1,7 +0,0 @@
FLEET_AUTH_ISSUER_URL=
FLEET_AUTH_AUTHORIZE_URL=
FLEET_AUTH_TOKEN_URL=
FLEET_AUTH_CLIENT_ID=
FLEET_AUTH_REDIRECT_URI=
FLEET_AUTH_SCOPE=
FLEET_AUTH_TRUSTED_AUDIENCES=

4
AGENTS.md
View File

@@ -92,3 +92,7 @@ mdbook build
## Conventions
Rust edition 2024, resolver v2. Conventional commits (`feat:`/`fix:`/`chore:`/`docs:`/`refactor:`). Small, single-purpose PRs (~200 lines, excluding generated code). License AGPL v3. Lean on the type system — compile-time guarantees over runtime checks; abstractions domain-level, not provider-specific.
## Dos and Don't
- Never tag "Co-Authored-By" with an agent name in commit messages.

55
Cargo.lock generated
View File

@@ -2492,6 +2492,18 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "docker-compose-types"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdfd601efc90644958510466e8904ca90da65a98fdfe4d21f940644a21c026b4"
dependencies = [
"derive_builder 0.20.2",
"indexmap 2.14.0",
"serde",
"serde_yaml",
]
[[package]]
name = "dockerfile_builder"
version = "0.1.6"
@@ -2818,6 +2830,29 @@ dependencies = [
"tokio",
]
[[package]]
name = "example-compose-java-react"
version = "0.1.0"
dependencies = [
"anyhow",
"async-trait",
"clap",
"docker-compose-types",
"env_logger",
"fqdn",
"harmony",
"harmony_cli",
"harmony_types",
"k8s-openapi",
"log",
"serde",
"serde_yaml",
"tempfile",
"thiserror 2.0.18",
"tokio",
"toml",
]
[[package]]
name = "example-fleet-auth-callout"
version = "0.1.0"
@@ -4063,6 +4098,7 @@ dependencies = [
"harmony",
"harmony-fleet-auth",
"harmony-reconciler-contracts",
"harmony_secret",
"serde",
"serde_json",
"thiserror 2.0.18",
@@ -4080,6 +4116,7 @@ dependencies = [
"async-nats",
"chrono",
"harmony-reconciler-contracts",
"harmony_zitadel_auth",
"jsonwebtoken",
"reqwest 0.12.28",
"serde",
@@ -4134,11 +4171,15 @@ dependencies = [
"harmony-fleet-auth",
"harmony-fleet-deploy",
"harmony-fleet-operator",
"harmony-k8s",
"harmony-reconciler-contracts",
"harmony_types",
"jsonwebtoken",
"k3d-rs",
"k8s-openapi",
"kube",
"nkeys",
"reqwest 0.12.28",
"serde",
"serde_json",
"tempfile",
@@ -4468,6 +4509,7 @@ dependencies = [
"async-trait",
"directories",
"harmony_secret_derive",
"harmony_zitadel_jwt",
"http 1.4.0",
"infisical",
"inquire 0.7.5",
@@ -4537,6 +4579,7 @@ dependencies = [
"base64 0.22.1",
"chrono",
"harmony_config",
"harmony_zitadel_jwt",
"jsonwebtoken",
"openidconnect",
"rand 0.9.4",
@@ -4551,6 +4594,18 @@ dependencies = [
"url",
]
[[package]]
name = "harmony_zitadel_jwt"
version = "0.1.0"
dependencies = [
"anyhow",
"chrono",
"jsonwebtoken",
"reqwest 0.12.28",
"serde",
"serde_json",
]
[[package]]
name = "hashbrown"
version = "0.12.3"

1
Cargo.toml
View File

@@ -5,6 +5,7 @@ members = [
"private_repos/*",
"harmony",
"harmony_zitadel_auth",
"harmony_zitadel_jwt",
"harmony_types",
"harmony_macros",
"harmony_tui",

95
ROADMAP/fleet_platform/agent_secret_access_e2e.md Normal file
View File

@@ -0,0 +1,95 @@
# Agent secret access: full-stack integration e2e
**Design:** [ADR-025](../../docs/adr/025-fleet-device-secret-access.md) ·
[user guide](../../docs/guides/fleet-device-secrets.md) ·
prior work: [groups rollout](./secret_access_groups_rollout.md)
**Branch:** `feat/fleet-device-secret-access`
**Written:** 2026-06-12
The groups rollout proved OpenBao-side enforcement against minted test
JWTs. This entry closes the loop with the *real* device path, in one VM
e2e:
- agent connects to NATS through the auth callout with its Zitadel
machine identity (first time callout runs in the e2e harness),
- a deployment in desired state references a secret,
- the agent logs into OpenBao with the same Zitadel JWT (groups claim
from the flattening Action — live verification of the deferred item)
and injects the secret into the container,
- removing the OpenBao policy makes the next fetch fail, visibly.
The operator is deliberately absent: CR→KV plumbing and grant sync are
already unit/e2e-covered; the test drives desired state via `AdminKv`
and grants via the same `OpenBaoDeploymentSecretGrants` the operator
composes. The axis under test is agent ↔ Zitadel ↔ callout/OpenBao.
## Phase 1 — agent secret read path (test-first)
- [x] `PodmanService.secret_env: Vec<SecretEnvVar { name, secret }>`
(serde default, wire-compat with old payloads). Secret lives at
`<kv>/<prefix>/<deployment>/<secret>`, field `value` —
harmony_secret's existing convention.
- [x] `OpenbaoSecretStore`: constructor taking the agent's existing
`ZitadelJwtBearer` (one identity for NATS + OpenBao, no token
cache files); `get_raw` retries once through `refresh_auth` on
permission-denied (batch tokens expire, re-login IS the sync —
ADR-025).
- [x] Agent `[openbao]` config section (url, role, mounts, prefix);
reconciler resolves `secret_env` → env before interpret; fetch
failure → `Phase::Failed` with the OpenBao error (the observable
the negative test asserts).
- [x] Unit tests: wire compat, config parse, resolution + failure
mapping via in-memory `SecretStore`.
## Phase 2 — e2e stack: callout mode
`AuthMode::Callout` in `fleet/harmony-fleet-e2e/src/stack.rs`, composed
from the same Scores as `fleet_staging_install` / `fleet_auth_callout`:
- [x] Zitadel + CoreDNS rewrite (`sso.fleet-e2e.local` → zitadel svc) +
`ZitadelSetupScore`: project, roles-as-groups (`edge-a`,
`edge-b`), `groups_claim_action: true`, machine users per device
(username = device id) + `e2e-admin` (fleet-admin role) for the
host-side NATS client.
- [x] NATS via `FleetNatsScore::callout` + `NatsAuthCalloutScore`
(callout image build/sideload joins images.rs).
- [x] OpenBao + JWT auth: `oidc_discovery_url` = in-cluster issuer,
`groups_claim=groups`, batch, 1h — same shape staging uses.
NodePort service so the VM reaches it via the libvirt gateway.
- [x] k3d cluster gains port mappings 30424 (Zitadel NodePort, with a
`zitadel-e2e` Service whose in-cluster port equals the nodePort
so the issuer `http://sso.fleet-e2e.local:30424` is valid from
host, VM and in-cluster; 8080 was rejected — dev hosts commonly
occupy it) and 30425 (OpenBao). Existing cluster without them →
clear bring-up error telling the user to delete `fleet-e2e`.
- [x] VM device setup: `FleetDeviceAuth` Zitadel machine key +
`hosts_entries` (sso → gateway IP) + `[openbao]` section.
- [x] Host-side admin NATS connect mints a Zitadel token via the
Host-header trick (no /etc/hosts edits on the host).
## Phase 3 — the tests (`tests/vm_secret_access.rs`)
Gated by `HARMONY_FLEET_VM_E2E=1`, shared VM stack:
- [x] **Grant + read**: device granted `edge-a` in Zitadel; secret
written with the root token; `set_deployment_groups([web →
[edge-a]])`; desired state with `secret_env` → `Phase::Running`
and SSH ground truth: `podman inspect` shows the env value.
- [x] **Policy removed**: delete `deployment-web` policy in OpenBao,
bump the spec to force re-apply → fetch denied →
`Phase::Failed`, error names the denial.
## Phase 4 — verification
- [x] `./build/check.sh` green
- [x] Existing fleet e2e suite green
- [x] New VM e2e green (x86_64; aarch64 before release)
- [x] Push
## Deferred (carried from groups rollout)
- Operator-pod env injection live test (grant sync in-cluster) — still
staging-milestone scope.
- TTL tuning, per-cell sizing — unchanged.
- aarch64 VM run of the new test if the manual x86_64 run is the gate
today.

463
ROADMAP/fleet_platform/device_secret_access_handoff.md Normal file
View File

@@ -0,0 +1,463 @@
# Fleet device secret access — implementation handoff
**Owner:** TBD
**Design:** [`docs/adr/025-fleet-device-secret-access.md`](../../docs/adr/025-fleet-device-secret-access.md)
**Status:** Superseded by implementation — see ADR-025 and its iteration
trail in [`docs/adr/025-device-secret-access/`](../../docs/adr/025-device-secret-access/)
(this plan targeted iteration [025-1](../../docs/adr/025-device-secret-access/025-1-jwt-deployments-claim.md);
what shipped is [025-2](../../docs/adr/025-device-secret-access/025-2-entity-by-policy.md))
**Written:** 2026-06-01
> **As-built note.** The implementation took a simpler scoping path than
> this plan (entity-by-policy instead of a JWT `deployments` claim +
> `groups_claim`). This six-PR breakdown is kept for history; the parts
> that actually shipped:
>
> | Plan | Shipped |
> |---|---|
> | PR-1 extract JWT-bearer minter | ✅ — now in the leaf crate `harmony_zitadel_jwt` (re-exported by `harmony_zitadel_auth`) |
> | PR-2 `refresh_auth` + `cached_scope` + `decode_deployments_claim` | ⚠️ `refresh_auth` only; scope decoding dropped (scope lives in the OpenBao entity, not the token) |
> | PR-3 `groups_claim`/`bound_claims_json` on `OpenbaoJwtAuth` | ❌ not needed |
> | PR-4 Zitadel deployments-claim Action Score | ❌ not needed |
> | PR-5 membership Score | ✅ as `DeviceSecretAccess` trait + `OpenBaoDeviceSecretAccess` (`harmony/src/modules/openbao/identity.rs`) + operator `sync_secret_access`; OpenBao entity writes only, no Zitadel-metadata write |
> | PR-6 agent before-reconcile refresh-check | ❌ not needed (entity policy changes are seen without re-auth) |
Read ADR-025 first. This document is the work plan, not the design.
## Summary
The fleet agent reads per-deployment secrets through the existing
`harmony_config` chain, which already wraps OpenBao via
`harmony_secret::OpenbaoSecretStore`. `OpenbaoSecretStore`'s auth
ladder is extended with one new rung — Zitadel **JWT-bearer**, keyed
off the device's existing machine keyfile (the same root of trust the
NATS callout uses). Per-deployment scope rides in the JWT itself as a
`deployments` array claim, populated by the operator through Zitadel
user metadata. OpenBao's `groups_claim` binds those values to
auto-created external groups, each with a small policy granting
`read` on `harmony-fleet/data/<deployment-id>/*`. The agent's NATS
watcher calls `refresh_auth()` on the store before reconciling a
deployment whose secrets are outside the cached scope.
## Architecture at a glance
```
Fleet operator (one per fleet, holds Zitadel admin scope)
│
├── on new Deployment CR:
│ 1. FleetDeviceDeploymentMembershipScore
│ ├─ Zitadel: append dep to device.metadata.deployments
│ └─ OpenBao: upsert external group + policy
│ fleet-deployment-<dep> → read on
│ harmony-fleet/data/<dep>/*
│ 2. existing NATS publish (data plane, unchanged)
│
▼
Fleet agent
│
├── NATS handler (main.rs):
│ if !secret_store.cached_scope().contains(&dep) {
│ secret_store.refresh_auth().await?;
│ }
│ reconciler.apply(…)
│
└── reads secrets via harmony_config (StoreSource wrapping
OpenbaoSecretStore with new JWT-bearer auth rung)
```
## Pre-flight
- [ ] Confirm staging Zitadel has a service account for the operator
with **write user metadata** scope. File a sub-task if not.
- [ ] Confirm the staging OpenBao KV mount `harmony-fleet` exists, or
add its creation to `OpenbaoSetupScore`.
- [ ] Validate `DeploymentName`'s character set is safe for KV paths
and Bao external-group names (`[a-zA-Z0-9_-]`).
## Work breakdown
Six PRs. Dependencies:
```
PR-1 harmony_zitadel_auth: extract JWT-bearer minter
└─ PR-2 harmony_secret: JWT-bearer auth + refresh_auth + cached_scope
└─ PR-6 agent main.rs: inline refresh-check
PR-3 harmony OpenbaoJwtAuth Score: bound_claims + groups_claim
PR-4 Zitadel Action Score: deployments claim
└─ PR-5 Operator: FleetDeviceDeploymentMembershipScore
PR-3 ─┘
```
PRs 1, 3, 4 are independent and can start in parallel.
---
### PR-1 — Extract Zitadel JWT-bearer minter into `harmony_zitadel_auth`
**Crate:** `harmony_zitadel_auth`
**Depends on:** nothing
**Blocks:** PR-2
Move `MachineKeyFile`, `CachedToken`, `build_assertion*`, `build_scope`,
`build_token_url`, and the mint+cache logic out of
`fleet/harmony-fleet-auth/src/credentials.rs` into a new
`harmony_zitadel_auth/src/jwt_bearer.rs`. Two real consumers (NATS
callout + OpenBao) cross Rule of Three at the second consumer.
`harmony_secret` cannot depend on `harmony-fleet-auth` (fleet-specific
crate; would invert the dependency graph). `harmony_zitadel_auth` is
neutral and already houses the human-OIDC counterpart.
**Shape.**
```rust
// harmony_zitadel_auth/src/jwt_bearer.rs
pub struct ZitadelJwtBearer {
key: MachineKeyFile,
oidc_issuer_url: String,
audience: String,
http: reqwest::Client,
cache: Mutex<Option<CachedToken>>,
}
impl ZitadelJwtBearer {
pub fn new(key: MachineKeyFile, oidc_issuer_url: String, audience: String,
danger_accept_invalid_certs: bool) -> Result<Self>;
/// Cached if comfortably valid, otherwise mints fresh.
pub async fn bearer_token(&self) -> Result<String>;
/// Force a re-mint on next call.
pub fn invalidate_cache(&self);
}
```
`CredentialSource::ZitadelJwt` in `harmony-fleet-auth` becomes a thin
wrapper around `Arc<ZitadelJwtBearer>`. Port the existing pure-builder
tests from `credentials.rs` into the new crate.
**Acceptance.** `cargo test -p harmony_zitadel_auth -p harmony-fleet-auth`
clean. Fleet e2e NATS auth still works.
---
### PR-2 — `OpenbaoSecretStore` JWT-bearer rung + `refresh_auth` + `cached_scope`
**Crate:** `harmony_secret` (`src/store/openbao.rs`)
**Depends on:** PR-1
**Blocks:** PR-6
Add the fifth rung to `OpenbaoSecretStore::new`'s auth ladder.
Position: between cached-token (rung 2) and Zitadel OIDC device flow
(rung 4). Triggered only when a machine keyfile is configured; if it
fails, fall through to the existing ladder.
**Constructor refactor.** Knock out the `too_many_arguments` clippy
TODO at `openbao.rs:55` while we're adding more args:
```rust
pub struct OpenbaoStoreOptions {
pub base_url: String,
pub kv_mount: String,
pub auth_mount: String,
pub skip_tls: bool,
pub token: Option<String>,
pub username: Option<String>, pub password: Option<String>,
pub zitadel_sso_url: Option<String>, pub zitadel_client_id: Option<String>,
pub zitadel_jwt_bearer: Option<ZitadelJwtBearerConfig>, // NEW
pub jwt_role: Option<String>, pub jwt_auth_mount: Option<String>,
}
pub struct ZitadelJwtBearerConfig {
pub key_path: Option<PathBuf>,
pub key_json: Option<String>,
pub oidc_issuer_url: String,
pub audience: String, // Zitadel project ID
}
```
**Refresh capability.** A single `Mutex<Inner>` over the refresh-affected
state. Refresh is rare and uncontended; `ArcSwap` would be ceremony.
```rust
pub struct OpenbaoSecretStore {
inner: Mutex<Inner>,
kv_mount: String,
auth_mount: String,
jwt_bearer: Option<Arc<ZitadelJwtBearer>>,
jwt_role: Option<String>,
jwt_auth_mount: Option<String>,
base_url: String,
skip_tls: bool,
}
struct Inner {
client: VaultClient,
scope: HashSet<String>, // deployment IDs from the Zitadel JWT
}
impl OpenbaoSecretStore {
pub async fn refresh_auth(&self) -> Result<(), SecretStoreError> {
let bearer = self.jwt_bearer.as_ref().ok_or_else(|| /* err */)?;
bearer.invalidate_cache();
let jwt = bearer.bearer_token().await?;
let scope = decode_deployments_claim(&jwt)?; // pure JWT decode
let session = jwt_login(&self.base_url, /* … */, &jwt).await?;
let client = build_vault_client(&self.base_url, self.skip_tls, &session.token)?;
*self.inner.lock().await = Inner { client, scope };
Ok(())
}
pub async fn cached_scope(&self) -> HashSet<String> {
self.inner.lock().await.scope.clone()
}
}
```
`decode_deployments_claim` is a small pure helper that base64-decodes
the JWT body and reads the `deployments` array. No Bao round-trip.
`get_raw` / `set_raw` lock `inner`, take a `&VaultClient`, perform the
call, release. The lock is uncontended on the hot path.
**On construction with JWT-bearer.** Wire `jwt_bearer` and call
`refresh_auth()` once before returning. If it fails, fall through to
the next ladder rung.
**Tests.**
- Ladder ordering: JWT-bearer present → tried before OIDC; absent →
unchanged.
- `refresh_auth` against a fake HTTP server updates `cached_scope`.
- `decode_deployments_claim` on a hand-crafted JWT returns the
expected set.
**Acceptance.** `cargo test -p harmony_secret` clean. `examples/openbao`
userpass path unaffected. Manual staging run authenticates via
JWT-bearer.
---
### PR-3 — Extend `OpenbaoJwtAuth` Score with `bound_claims_json` + `groups_claim`
**Crate:** `harmony` (`src/modules/openbao/setup.rs`)
**Depends on:** nothing
Two new fields, both defaulting to empty so existing callers are
unaffected:
```rust
pub struct OpenbaoJwtAuth {
// … existing fields …
#[serde(default)] pub bound_claims_json: String,
#[serde(default)] pub groups_claim: String,
}
```
Extend `configure_jwt` (line 435+) to pass each as a `bao write
auth/jwt/role/...` flag when non-empty.
**Acceptance.** `cargo check -p harmony --all-features` clean.
`examples/openbao` still works.
---
### PR-4 — Zitadel Action Score: surface `deployments` metadata as a claim
**Crate:** `harmony` (new module under `src/modules/zitadel/` or
wherever existing Zitadel deploy code lives)
**Depends on:** nothing
**Blocks:** PR-5
Declarative Score that upserts the post-access-token-creation Action
on the configured Zitadel project. Cannot be avoided: Zitadel emits
`urn:zitadel:iam:user:metadata` as base64-encoded values in a map,
and Bao's `groups_claim` can't consume that shape — the Action
decodes and re-emits as a flat string array.
**Action script** (canonical text):
```javascript
function addDeployments(ctx, api) {
const md = ctx.v1.user.getMetadata();
const entry = md.metadata.find(m => m.key === "deployments");
if (!entry) return;
try {
const deployments = JSON.parse(
Buffer.from(entry.value, "base64").toString("utf-8")
);
if (Array.isArray(deployments)) {
api.v1.claims.setClaim("deployments", deployments);
}
} catch (_) { /* malformed → no deployments */ }
}
```
**Score shape.**
```rust
pub struct ZitadelDeploymentsClaimActionScore {
pub project_id: String,
pub action_name: String, // default "fleet-deployments-claim"
}
```
Interpret upserts via Zitadel Management API, attaches to
`Complement Token / PreAccessTokenCreation`. Idempotent.
**Acceptance.** Manual decode of a staging token shows the
`deployments` claim.
---
### PR-5 — `FleetDeviceDeploymentMembershipScore` + operator wiring
**Crate:** `harmony` (`src/modules/fleet/` or wherever fleet Scores
live) for the Score; `fleet/harmony-fleet-operator` for wiring.
**Depends on:** PR-3 (Bao JWT role + accessor must exist), PR-4
(claim must surface)
One Score, two writes, declared in execution order. Per ADR-023 the
operator hands off ordering to the Score rather than sequencing the
external API calls by hand.
```rust
pub struct FleetDeviceDeploymentMembershipScore {
pub zitadel_project_id: String,
pub device_user_id: String, // Zitadel machine user ID
pub deployments: Vec<String>, // declarative full set
pub openbao_instance: OpenbaoInstance,
pub kv_mount: String, // "harmony-fleet"
pub jwt_auth_accessor: String, // resolved from PR-3
}
```
Interpret runs, in order:
1. **Zitadel metadata** — declarative replace on
`user.metadata.deployments`. Read current value, write only on
diff. Removal is "declare the new set without the removed entry."
2. **OpenBao external group + policy, per deployment in the set.** For
each `<dep>` in `deployments`:
- Policy `fleet-deployment-<dep>` granting `read` on
`harmony-fleet/data/<dep>/*` and
`read, list` on `harmony-fleet/metadata/<dep>/*`.
- External group `<dep>` (type=external) with that policy
attached, alias matching the JWT-auth accessor.
Idempotent throughout.
**Operator wiring.** In the existing per-`Deployment` reconciler,
compose this Score before the existing NATS desired-state publish.
If the Score errors, surface as a CR status condition and do not
publish — never tell a device to run a deployment it cannot
authenticate for.
**Note: NATS publish stays in operator code.** It's data-plane (the
"now do this" half of enrollment), already wired, and changing its
home isn't part of this work.
**Tests.**
- Unit on the Score: Zitadel diff logic (no-op on match, full write
on diff); policy text generation; group alias shape.
- Integration: against the real Bao in `examples/openbao` + a fake
Zitadel; second apply is a no-op.
**Acceptance.** A `Deployment` CR rolled in staging causes both
writes to land before the NATS publish; manual JWT-bearer login at
Bao returns a token with the expected per-deployment policies
attached via external-group bindings.
---
### PR-6 — Agent: inline refresh-check in `main.rs`
**Crate:** `fleet/harmony-fleet-agent`
**Depends on:** PR-2
Around the existing JetStream KV watcher in
`fleet/harmony-fleet-agent/src/main.rs:110-120`, gate
`reconciler.apply` on a scope check:
```rust
async_nats::jetstream::kv::Operation::Put => {
if let Some(dep) = deployment_from_key(&entry.key) {
if !secret_store.cached_scope().await.contains(dep.as_str()) {
tracing::info!(%dep, "deployment outside cached scope — refreshing");
if let Err(e) = secret_store.refresh_auth().await {
tracing::warn!(key = %entry.key, error = %e, "refresh failed");
continue;
}
}
}
if let Err(e) = reconciler.apply(&entry.key, &entry.value).await {
tracing::warn!(key = %entry.key, error = %e, "apply failed");
}
}
```
`secret_store` is the same `Arc<OpenbaoSecretStore>` held by the
agent's `ConfigManager` — share it through whatever construction
path `main.rs` already uses.
**No new module.** One consumer, one site, ~10 lines. Per CLAUDE.md
"Rule of Three: introduce an abstraction at the second real
instance." If/when a second pre-reconcile concern (image-pull creds,
monitoring registration) arrives, extract a layer then.
**No retry on "still missing after refresh."** The operator's
ordering guarantees Zitadel is consistent before the NATS publish.
If we observe the race in practice, add a single retry then.
**Tests.** None new — this is wiring between two tested components.
Confirm via the integration milestone below.
**Acceptance.** Rolling a new deployment to a staging device shows
one "outside cached scope — refreshing" log followed by a clean
reconcile.
---
## Integration milestone — staging dry run
After PRs 1-6 land:
- [ ] Apply the new `OpenbaoJwtAuth` config to staging Bao via
`OpenbaoSetupScore`.
- [ ] Apply `ZitadelDeploymentsClaimActionScore` to staging Zitadel.
- [ ] Hand-provision metadata + group/policy for one test deployment
+ one test device. Confirm the agent reads its secret.
- [ ] Roll a `Deployment` CR via the operator. Confirm
`FleetDeviceDeploymentMembershipScore` writes Zitadel and Bao
before NATS, and the agent reconciles cleanly.
- [ ] Negative: device not in deployment X gets a hard error reading
X's secret (not a silent fall-through).
- [ ] Negative: a JWT minted before the metadata update cannot read
the new deployment's secret until the agent's inline refresh
runs.
## Decisions deferred
1. **Vault token TTL.** ADR-025 recommends 15 min. Confirm in
staging; adjust `OpenbaoJwtAuth::ttl` if needed.
2. **Hard revocation on deployment removal.** Wait for TTL today.
Add a revoke companion only if a real fleet requires it.
3. **Bao down at agent startup.** `OpenbaoSecretStore::new` with
JWT-bearer must not panic; either fall through or surface a
retryable error. Confirm and document in the milestone.
## Required reading
- `docs/adr/025-fleet-device-secret-access.md` — design.
- `docs/adr/020-1-zitadel-openbao-secure-config-store.md` —
human-user counterpart of this auth flow.
- `docs/adr/023-deploy-architecture.md` — Score discipline for PRs 3-5.
- `harmony_secret/src/store/openbao.rs` — auth ladder being extended.
- `harmony_config/src/source/store.rs` — wrapper the agent uses.
- `fleet/harmony-fleet-auth/src/credentials.rs` — JWT-bearer mint
path being extracted in PR-1.
- `nats/callout/src/zitadel.rs` — JWT validation shape the Bao role
mirrors (`bound_issuer` + `bound_audiences`).
- `fleet/harmony-fleet-agent/src/main.rs` — site of the PR-6 inline
edit.

139
ROADMAP/fleet_platform/secret_access_groups_rollout.md Normal file
View File

@@ -0,0 +1,139 @@
# Secret access: groups implementation + branch consolidation
**Design:** [ADR-025](../../docs/adr/025-fleet-device-secret-access.md) ·
[user guide](../../docs/guides/fleet-device-secrets.md) ·
[analysis](../../docs/adr/025-device-secret-access/025-3-groups-as-security-boundary.md)
**Branch:** `feat/fleet-device-secret-access`
**Written:** 2026-06-11
Two efforts tracked here: (1) TDD implementation of the group-based
secret access design, (2) consolidation of the six stacked chapter
branches that will conflict with it.
## Part 1 — implementation (test-first)
Each phase: write the test, watch it fail, implement to green.
### Phase 1 — OpenBao group enforcement e2e (the security core)
New `fleet/harmony-fleet-e2e/tests/openbao_groups.rs`, self-contained
(no Zitadel: the JWT auth mount validates against a static test RSA
key via `jwt_validation_pubkeys`; tests mint their own JWTs). Proves
every row of ADR-025's convergence/revocation table:
- [x] JWT login with `groups: ["edge-a"]` claim binds the device to
the `edge-a` external group; batch token issued
- [x] Member device reads `deployment-web` secrets; non-member
(`groups: ["edge-b"]`) is denied
- [x] Attaching the deployment policy to a group grants **existing**
tokens immediately (request-time)
- [x] Detaching revokes existing tokens immediately
- [x] Membership change binds only at re-login: old token keeps old
groups, fresh login sees new groups
- [x] `disabled=true` on the entity blocks an in-flight **batch**
token (ADR-025-3 open item 1)
### Phase 2 — OpenBao group-grant backend
`DeviceSecretAccess` (device → deployments, retired with 025-2)
becomes deployment → allowed-groups:
- [x] Trait: `set_deployment_groups(&[(DeploymentName, Vec<GroupName>)])`
— absolute, idempotent; empty set detaches everywhere;
in-memory impl for operator tests
- [x] OpenBao impl: upsert policy `deployment-<name>` (read on its
subtree), upsert external group + JWT-mount alias per fleet
group, set group policy lists (add to allowed, remove from
formerly-allowed)
- [x] Phase 1 e2e drives this backend end to end
### Phase 3 — JWT auth role: groups, batch, TTL
`OpenbaoJwtAuth` (`harmony/src/modules/openbao/setup.rs`):
- [x] `groups_claim` (default empty → flag omitted)
- [x] `token_type` (default `service`; fleet role uses `batch`)
- [x] `jwt_validation_pubkeys` as alternative to `oidc_discovery_url`
(what makes Phase 1 self-contained)
- [x] Fleet defaults: `ttl=1h`, no renewal reliance (ADR-025)
### Phase 4 — operator: groups gate scheduling + drive grants
`harmony/src/modules/fleet/operator/crd.rs` +
`fleet/harmony-fleet-operator/src/fleet_aggregator.rs`:
- [x] `allowed_groups: Vec<String>` on `DeploymentSpec`
- [x] Membership source trait (device → groups) with in-memory test
impl; Zitadel impl reads role grants via the existing
Management API client (`users/grants/_search`,
`harmony/src/modules/zitadel/setup.rs`) — roles-as-groups
interim per ADR-025
- [x] Scheduling = members(allowed_groups) ∩ label selector; unit
tests: out-of-group device never targeted even on label match
- [x] On Deployment upsert/delete: sync deployment → groups to the
Phase 2 backend (replaces per-device `sync_secret_access`)
- [x] Rename the `group` *label* in test fixtures (placement vocab,
device-writable) so it can't be confused with authorization
groups
- [ ] Operator e2e: deployment with `allowedGroups` schedules only
onto in-group devices
### Phase 5 — Zitadel roles→groups Action
- [x] Score upserting the flattening Action (roles map → `groups`
string array claim) + flow trigger on the fleet project;
idempotent; unit-tested on request shape
- [ ] Staging verification deferred to the integration milestone
(needs a live Zitadel)
### Phase 6 — verification + push
- [x] `./build/check.sh` green
- [x] `HARMONY_FLEET_E2E=1` suite green (incl. existing
`openbao_policy.rs`, `ping.rs`, `operator.rs`)
- [x] Push `feat/fleet-device-secret-access`
## Part 2 — chapter branch consolidation
Six branches stack linearly from `97af299d` (pre-secret-access):
`fleet-device-exec-logs` ⊂ `ch1-role-gate` ⊂ `ch2-operator-recovery`
⊂ `ch3-log-streaming` ⊂ `ch4-agent-upgrade` ⊂
`ch5-graceful-deploy-upgrade` — except ch2's three follow-up fixes
never propagated to ch3–ch5. All will conflict with this branch
(`credentials.rs` was refactored into `harmony_zitadel_jwt`;
`fleet_aggregator.rs` changes in Phase 4).
Collapse to two feature-scoped branches, rebased onto
`feat/fleet-device-secret-access` after Part 1 lands:
| New branch | Absorbs | Scope |
|---|---|---|
| `feat/fleet-operator-recovery-and-logs` | exec-logs base fixes + ch1 + ch2 (fixes folded in) + ch3 | Operator reliability & observability: role gate via OIDC scope, aggregator recovery + orphan GC, log selector + live tail, agent auto-start replay |
| `feat/fleet-upgrade-path` | ch4 + ch5 | Agent self-upgrade + auto-rollback (ADR-022) and graceful roll-forward with container-ID identity |
- [x] Rebase/rewrite branch A; resolve `credentials.rs` →
`harmony_zitadel_jwt` conflicts; ch2 fixes squashed into the
recovery commit
- [x] Rebase branch B on top of A (it builds on ch3 state)
- [x] Both: `./build/check.sh` + fleet e2e green = working state
- [x] Push both; old six branches left untouched for reference until
the new ones are reviewed
## Deferred to staging integration milestone
- Flattening Action applied + token decode shows `groups` array
(flow/trigger ids `2`/`4`,`5` assumed from API docs — verify)
- Zitadel membership source verified against live role grants
- TTL = 1 h default revisited with measured login load
- Per-cell sizing guidance (identity-store memory measurement)
- Known gap: a failed grant sync only retries on the next Deployment
change (warn-and-continue). Add a periodic resync if staging shows
real transient failures.
- Operator e2e for the `allowedGroups` gate needs operator-pod env
injection in the harness (`FLEET_DEVICE_GROUPS`); gate currently
unit-tested, enforcement e2e-tested.
- Harness fixes that fell out of this work (on the PR): topology
`ensure_ready` before raw `Score::interpret`, `k3d image import
--mode direct` (tools-node copy fails silently), staging respects
`CARGO_TARGET_DIR`, typed SSA takes field ownership, recovery suite
serialized (in-process aggregators react to each other's CRs).

3
docs/SUMMARY.md
View File

@@ -25,6 +25,7 @@
- [Adding Capabilities](./guides/adding-capabilities.md)
- [Web Authentication and CSRF Security](./guides/web-auth-security.md)
- [Operator Dashboard SSO (Zitadel) — setup](./guides/operator-dashboard-sso.md)
- [Fleet Device Secrets](./guides/fleet-device-secrets.md)
## Configuration
@@ -54,3 +55,5 @@
- [019 · Network Bond Setup](./adr/019-Network-bond-setup.md)
- [020 · Interactive Configuration Crate](./adr/020-interactive-configuration-crate.md)
- [020-1 · Zitadel + OpenBao Secure Config Store](./adr/020-1-zitadel-openbao-secure-config-store.md)
- [025 · Fleet Device Secret Access](./adr/025-fleet-device-secret-access.md)
- [025-3 · Groups as the Security Boundary](./adr/025-device-secret-access/025-3-groups-as-security-boundary.md)

325
docs/adr/025-device-secret-access/025-1-jwt-deployments-claim.md Normal file
View File

@@ -0,0 +1,325 @@
# ADR 025-1: Fleet Device Secret Access via JWT `deployments` Claim (superseded)
Initial Author: Jean-Gabriel Gill-Couture
Initial Date: 2026-06-01
## Status
Superseded by [025-2](./025-2-entity-by-policy.md) during implementation.
The trust model, threat model, and JWT-bearer auth root survived as
designed and are now in the main
[ADR-025](../025-fleet-device-secret-access.md). What did not survive is
the scoping mechanism below: carrying per-deployment membership as a JWT
`deployments` claim (Zitadel user metadata + a token-customization
Action + OpenBao `groups_claim` + external groups). Preserved verbatim
as the first design iteration.
## Context
Fleet agents on devices need to read per-deployment secrets (image-pull
credentials, application secrets, etc.) from OpenBao. The agent already
holds one durable secret: a Zitadel machine-user JWT keyfile dropped by
`FleetDeviceSetupScore`. That key is the basis for the agent's existing
NATS authentication (`nats/callout` validates the Zitadel-minted access
token; `fleet/harmony-fleet-auth/src/credentials.rs` mints it via the
RFC 7523 JWT-bearer flow).
Three requirements shape the design:
1. **No new device-side secret.** The Zitadel machine key is already the
single root of trust on a device; the secret-access path must derive
from the same key, not introduce a second one.
2. **Per-deployment isolation, enforced cryptographically.** A device
enrolled in deployments A and B reads only `A`'s and `B`'s secrets.
A device that hosts no deployments reads nothing. The device cannot
widen its own scope — only the operator can change membership.
3. **Cross-project safety.** A second Zitadel project (a different
tenant, a different fleet, a malicious org) must not be able to
produce a token that OpenBao accepts. The trust boundary is the
project, not the deployment.
The kubelet analogy is the architectural north star: the agent is a
small runtime that learns its workload (and the credentials needed to
run it) from a control-plane authority. The agent never decides what it
is allowed to run or read; it presents a signed identity and the
infrastructure decides.
## Decision
Three coordinating pieces.
### 1. OpenBao JWT auth bound to the Zitadel project
OpenBao's JWT auth method validates incoming tokens against Zitadel's
OIDC discovery URL (JWKS). One auth role per fleet, configured against
**one** Zitadel project:
```
bound_issuer = <Zitadel issuer URL>
bound_audiences = <Zitadel project ID>
bound_claims = { "urn:zitadel:iam:org:project:roles": "fleet-device" }
user_claim = sub
groups_claim = deployments
```
`bound_audiences` is the project boundary. A token minted in any other
Zitadel project has a different `aud` claim and is rejected before any
membership claim is read. This is the same defense
`nats/callout/src/zitadel.rs` already applies via `set_audience`.
`groups_claim = deployments` instructs OpenBao to read the JWT's
`deployments` array and bind the resulting Vault token to one external
group per element. Each external group carries a per-deployment policy
granting `read` on `harmony-fleet/data/<deployment-id>/*`.
### 2. Operator-managed Zitadel metadata as the membership source of truth
The fleet operator is the only writer of `user.metadata.deployments`
on each device's Zitadel machine user. A Zitadel post-token-creation
**Action** copies that metadata into a top-level `deployments` claim on
the access token. The device never touches its own metadata.
When a new `Deployment` CR is observed in Kubernetes, the operator
executes three writes in a strict order:
1. **Zitadel metadata** — append the deployment ID to the device's
`deployments` array (per device targeted by the deployment).
2. **OpenBao external group + policy** — upsert
`identity/group/<deployment-id>` (`type=external`, alias matching the
JWT-auth accessor) and policy
`fleet-deployment-<deployment-id>` granting
`read` on `harmony-fleet/data/<deployment-id>/*`.
3. **NATS desired-state** — publish
`desired-state.<device-id>.<deployment-id>` with the workload score.
Reversed, the agent could see the desired-state, attempt a re-auth,
and find the deployment missing from its claims — a "permission denied
for a deployment I was told to run" race that is confusing to debug
and weakens the trust story. Trust state always precedes the workload
signal.
Removal runs in reverse: NATS delete → (optional) group/policy delete →
metadata removal. Currently-cached Vault tokens retain access until
their short TTL expires; explicit revocation is available via
`bao token revoke` on the device's accessor if hard revocation is
needed.
### 3. Client side: JWT-bearer in `harmony_secret`, refresh before reconcile in the agent
The agent does **not** grow a new secrets client. Per ADR-020,
`harmony_config` is the unified config+secret entry point and already
wraps OpenBao via `harmony_secret::OpenbaoSecretStore`. The missing
piece is auth: `OpenbaoSecretStore` supports env token, cached token,
Zitadel OIDC device flow (humans), and userpass — but not Zitadel
**JWT-bearer** for headless machine identity.
Three additions:
- A fifth rung on `OpenbaoSecretStore`'s auth ladder takes a Zitadel
machine keyfile + Bao JWT role + audience, mints via RFC 7523, and
POSTs to `/v1/auth/jwt/login`.
- The pure minting moves to `harmony_zitadel_auth` so NATS and
OpenBao auth share one implementation (Rule of Three: NATS callout
+ OpenBao auth = two real consumers).
- `OpenbaoSecretStore` gains `refresh_auth()` (re-mint + re-login,
guarded by an internal `Mutex`) and `cached_scope() ->
HashSet<String>` derived from decoding the in-hand Zitadel JWT —
no Bao round-trip needed since the `deployments` claim is already
in the token we just minted.
In the agent, the NATS KV watcher consults `cached_scope()` before
each `reconciler.apply()`. If the desired deployment isn't covered,
it calls `refresh_auth()` and proceeds. The check is inline in
`main.rs` — about ten lines around the existing watcher loop. No
new module: one consumer, one site, inlining is the right size.
### Secret path layout
```
harmony-fleet/data/<deployment-id>/<secret-name>
```
The Zitadel project ID does **not** appear in the path. Its job is
done at the JWT validation boundary (`bound_audiences`), not repeated
in every key.
## Rationale
**Why Zitadel project ID lives in `bound_audiences`, not the path.**
The same trust assertion in two places is duplication, not defense in
depth — both reduce to "the JWT signature is valid for this audience."
Concentrating it at the auth role:
- gives one source of truth ("which project owns this Bao instance");
- keeps secret paths readable and operator-friendly;
- decouples secret organization from Zitadel project identity (a
project ID rotation reconfigures one Bao role, not every path).
**Why user metadata over project roles for deployment membership.**
Project roles in Zitadel live in a flat namespace inside a project.
A handful of roles (`fleet-admin`, `fleet-device`) maps cleanly; one
role per deployment would not — role inventories at hundreds of
deployments per fleet become hard to audit and slow to mutate.
User metadata is a per-machine-user JSON store, naturally
multi-valued, and admin-only-writable. The Zitadel Action that copies
metadata to a claim is a one-time, fleet-wide piece of configuration.
**Why `groups_claim` over claim-templated paths.** Vault policy
templating (`{{identity.entity.aliases…metadata.<key>}}`) supports
single-value substitution but not iteration over an array. Multiple
deployments per device require either multiple JWT logins (one per
deployment) or one login that resolves to multiple policies.
`groups_claim` + external groups gives the latter cleanly: one login,
N policies attached automatically.
**Why `harmony_config` / `harmony_secret`, not a fleet-local secrets
client.** ADR-020 is explicit that `harmony_config` is the unified
config+secret entry point and `OpenbaoSecretStore` is the canonical
OpenBao client. Adding a parallel fleet-only client would duplicate
the auth ladder, cache-file layout, and `kv2` plumbing already in
`harmony_secret`. The fleet's need is an *additional auth branch*,
not a different store.
**Why scope is decoded from the Zitadel JWT, not asked of Bao.** The
agent already holds the JWT it's about to present at login; the
`deployments` claim is right there. A `/v1/auth/token/lookup-self`
round-trip after login would compute the same set from the other
direction, paying a network call to recover information already in
hand.
## Consequences
**Pros**
- One auth root on a device (the existing Zitadel machine key) covers
both NATS and OpenBao access. Rotation, revocation, and inventory
remain centralized.
- The operator owns membership; the agent owns identity. A compromised
device cannot widen its own access. A compromised operator's blast
radius is its own fleet (one Zitadel project, one Bao instance).
- Per-deployment policies are mechanical to generate. Bao policy text
is identical modulo the deployment ID, produced by a small templated
Score. New deployments add one external group + one policy; no
hand-written ACLs.
- The lifecycle layer is a reusable home for future
"before-reconcile" work without further architectural changes.
**Cons**
- **Two-token invalidation on membership change.** Both the cached
Zitadel access token and the cached Bao Vault token must be dropped
for new membership to take effect. This is encapsulated in the
`secrets.refresh()` call but is a real round-trip cost (one HTTPS to
Zitadel + one to Bao) on every membership change. Mitigated by the
fact that membership changes are rare relative to secret reads.
- **Removal latency = Vault token TTL.** Removing a device from a
deployment does not immediately revoke its currently-cached Vault
token; access ends at next renewal or TTL expiry. Short TTLs (15 min)
bound the worst case; explicit `bao token revoke -accessor` is
available if needed.
- **Operator gains Zitadel-admin scope.** The operator must hold
credentials that can write user metadata in the Zitadel project.
This is a high-privilege scope and concentrates trust in the
operator. The mitigation is a per-fleet Zitadel project: a
compromised operator can only mutate its own fleet's identities.
- **Zitadel Action required.** Surfacing user metadata as a JWT claim
needs a small Zitadel Action (server-side JavaScript). It is part of
the fleet's Zitadel setup and must be in version control / applied
by the fleet's bootstrap, not configured by hand. (See "Additional
Notes" for the script.)
## Alternatives considered
**Project roles for deployment membership.** Rejected: flat namespace
inside a project, no native multi-value semantics, role inventory
explodes at hundreds of deployments per fleet, mutations require
project-admin scope on a coarse-grained API. Kept for the coarse
`fleet-device` / `fleet-admin` distinction the NATS callout already
uses.
**Project ID embedded in the secret path
(`secrets/<project-id>/<deployment>/...`).** Rejected: the project
isolation is already enforced by `bound_audiences` at the JWT layer.
Encoding it in the path is duplication of the same assertion, couples
the secret tree to a Zitadel ID, and complicates project rotations.
Adds no security: a token that passes `bound_audiences` validation can
read the path regardless; one that fails cannot read anything.
**Claim-templated single policy
(`{{identity.…metadata.deployment_id}}`).** Rejected for the
multi-deployment case: Vault policy templating does not iterate over
arrays, so a single-policy template can only express "one deployment
per device." Acceptable for a single-deployment-per-device world; the
chosen kubelet-like architecture admits N deployments per device, and
collapsing the chosen `groups_claim` design to this would force
multiple JWT logins per refresh.
**Static per-device Bao token issued at provisioning.** Rejected:
introduces a second long-lived secret on the device, breaks rotation
(re-provisioning required), and provides no native per-deployment
scoping.
**OpenBao OIDC code flow.** Rejected: that flow is for human users
with a browser. Devices are headless and already hold a JWT-bearer
identity; using OIDC would re-invent the wheel and require a local
browser-equivalent.
**Lifecycle layer inside the NATS handler.** Rejected: conflates
transport with domain logic and makes the refresh-then-reconcile
ordering implicit. The dedicated module makes the contract testable
and lets future triggers reuse the same code path.
## Additional Notes
### Zitadel Action (token customization)
A single post-access-token-creation Action per fleet's Zitadel project
copies user metadata `deployments` into a top-level claim:
```javascript
// Trigger: pre-access-token-creation
function addDeployments(ctx, api) {
const md = ctx.v1.user.getMetadata();
const entry = md.metadata.find(m => m.key === "deployments");
if (!entry) return;
try {
const deployments = JSON.parse(
Buffer.from(entry.value, "base64").toString("utf-8")
);
if (Array.isArray(deployments)) {
api.v1.claims.setClaim("deployments", deployments);
}
} catch (_) { /* malformed metadata is treated as no deployments */ }
}
```
The Action lives in Zitadel's "Flows" configuration, attached to the
`Complement Token` flow on the relevant project. A Harmony Score
(`ZitadelTokenCustomizationScore` or similar) is the right home for
applying this declaratively; see plan document for status.
### Relationship to ADR-016 and ADR-020-1
ADR-016 (agent mesh on NATS JetStream) establishes the agent's
existing Zitadel-keyed identity for NATS. This ADR reuses that
identity unchanged.
ADR-020-1 establishes the human-developer authentication path to
OpenBao via Zitadel's Device Authorization Grant. This ADR is the
machine-user counterpart: same OpenBao, same Zitadel, different
auth-method binding (humans use device code; devices use
JWT-bearer-derived access tokens against `/auth/jwt/login`).
### Threat model summary
| Attacker | Capability | Defense |
|---|---|---|
| External (no Zitadel identity) | None | No valid JWT signature; rejected at JWKS validation. |
| Compromised device (key theft) | Full agent scope on its own deployments only | `groups_claim` restricts scope to the device's metadata; Zitadel admin can rotate the machine key and trigger immediate re-issuance. |
| Different Zitadel project (different tenant or malicious org) | Can mint valid Zitadel tokens for its own project | `bound_audiences` rejects at the JWT auth boundary before any claim is read. |
| Compromised operator | Can mutate Zitadel metadata + Bao policies for its fleet | One operator per fleet; operator credentials themselves stored in Bao under a separate auth path; compromise is contained to the operator's project. |
| Compromised Bao | Full access to all stored secrets | Out of scope — Bao is the root of secret trust by definition. ADR-006 covers Bao operational hardening. |

74
docs/adr/025-device-secret-access/025-2-entity-by-policy.md Normal file
View File

@@ -0,0 +1,74 @@
# ADR 025-2: Entity-by-Policy Scoping (as built)
Initial Author: Jean-Gabriel Gill-Couture
Initial Date: 2026-06-11
## Status
Retired. First implementation shipped on
`feat/fleet-device-secret-access`, then superseded by the group-based
design (main [ADR-025](../025-fleet-device-secret-access.md)) after
the scale analysis in
[025-3](./025-3-groups-as-security-boundary.md): per-device entity
writes are O(targeted devices) per deployment change, which does not
hold at fleet scale. The identity layer built alongside it
(`harmony_zitadel_jwt`, the JWT-bearer auth rung, `refresh_auth()`)
carries over unchanged.
## What changed vs 025-1
[025-1](./025-1-jwt-deployments-claim.md) carried per-deployment
membership *inside the token*: Zitadel user metadata → token Action →
JWT `deployments` claim → OpenBao `groups_claim` → external groups.
Implementation replaced that whole chain with one server-side write:
**A device's secret access is its OpenBao identity entity's policy
list.** The operator sets the entity's policies to exactly
`deployment-<name>` for each deployment currently scheduled on the
device. OpenBao resolves identity policies **at request time**, so a
grant or revocation takes effect on the device's very next read — the
device logs in once and keeps its token; no re-auth, no thundering herd
on bulk reassignment. Proven by the e2e test
`entity_policy_grants_existing_openbao_token_access`
(`fleet/harmony-fleet-e2e/tests/openbao_policy.rs`).
## Why the claim chain lost
- **Tokens snapshot, entities don't.** Claim-carried scope is frozen at
mint time; every membership change required invalidating two cached
tokens (Zitadel + Bao) per device. Entity policies change server-side
with zero device participation — strictly better for revocation and
for offline devices.
- **Less machinery.** Dropped: the Zitadel token Action (server-side JS
in the security path), `groups_claim` + external groups, the agent's
`cached_scope()` and before-reconcile refresh-check, and the
operator's Zitadel-metadata write scope. The operator now writes only
OpenBao.
- **One source of truth.** Membership derives from the operator's view
of the `Deployment` CRs in its reconcile loop, not from a second copy
in Zitadel user metadata.
## Shipped shape
| Piece | Home |
|---|---|
| `DeviceSecretAccess` trait (devices ↔ deployments, backend-agnostic) + in-memory test impl | `harmony/src/modules/fleet/secret_access.rs` |
| `OpenBaoDeviceSecretAccess` (entity resolution via JWT-mount alias, policy-list writes) | `harmony/src/modules/openbao/identity.rs` |
| Operator wiring: `sync_secret_access` after every selector/device change | `fleet/harmony-fleet-operator/src/fleet_aggregator.rs` |
| JWT-bearer minter (RFC 7523), extracted to a leaf crate to break a dependency cycle | `harmony_zitadel_jwt` (re-exported by `harmony_zitadel_auth`) |
| JWT-bearer rung on the auth ladder + `refresh_auth()` | `harmony_secret/src/store/openbao.rs` |
Entity resolution subtlety: OpenBao auto-provisions an entity (random
name) + alias (name = `user_claim`, i.e. the device id) on first JWT
login. The operator therefore resolves device → entity **via the
alias**, creating entity + alias itself when the device has never
logged in, so access can be granted ahead of first login.
## What this iteration deliberately does not solve
Scheduling — and therefore secret access — is gated only by the
deployment's label selector over **device-reported** labels. A device
that can choose its own labels can attract deployments and their
secrets. That gap is the subject of
[025-3](./025-3-groups-as-security-boundary.md).

177
docs/adr/025-device-secret-access/025-3-groups-as-security-boundary.md Normal file
View File

@@ -0,0 +1,177 @@
# ADR 025-3: Groups as the Security Boundary — Scale & Security Analysis
Initial Author: Jean-Gabriel Gill-Couture
Initial Date: 2026-06-11
## Status
Accepted — this analysis produced the design in the main
[ADR-025](../025-fleet-device-secret-access.md). Kept as the record of
the entity-vs-groups comparison, the verification work behind it, and
the state-of-the-art survey.
## Context
[025-2](./025-2-entity-by-policy.md) (entity-by-policy) shipped first
and is correct at small scale, but two concerns drove a re-design:
1. **Privilege escalation via labels.** Scheduling — and therefore
secret grants — keyed on a label selector over *device-reported*
labels. A compromised device relabels itself to match any
deployment's selector and receives its secrets. Kubernetes has the
same structural risk and closes it with the `NodeRestriction`
admission plugin; we had no equivalent, and no operator-facing way
to say "this deployment may only ever be seen by the lab fleet."
2. **Write amplification.** The operator rewrote one OpenBao identity
entity per targeted device per deployment change — O(targeted
devices) Raft-backed writes. The fleet ambition is millions of
devices.
The proposal: split authorization from placement, as Kubernetes splits
RBAC from scheduling. Groups (admin-managed, never self-asserted) gate
who may *view* a deployment; labels only place workloads within the
authorized set. Two enforcement variants were compared:
- **A — external groups:** Zitadel group membership rides in the token
(`groups_claim`); OpenBao maps it to external identity groups; each
deployment's policy attaches to its allowed groups.
- **B — entity-by-policy + group gate:** keep 025-2's per-device
entity policy lists; groups exist only in the control plane as a
scheduling filter.
An initial review recommended B (finer granularity, no re-login,
instant server-side revocation). Verification of the underlying
facts — plugin source, documented limits, at-scale precedent —
reversed that recommendation. **A wins at the stated scale.**
## Verified facts (with sources)
1. **Token renewal does NOT re-sync external-group membership for
JWT auth.** The plugin's renew handler only checks the role still
exists and extends TTLs; claims are never re-read, `GroupAliases`
never rebuilt. Membership converges **only at fresh login**.
HashiCorp's KB wording ("login or renewal") is true for methods
whose renew path re-queries the IdP (e.g. LDAP), not JWT.
Source: `vault-plugin-auth-jwt/path_login.go` and OpenBao's fork
`builtin/credential/jwt/path_login.go`.
2. **Identity policies resolve at request time.** "The evaluation of
policies applicable to the token through its identity will happen
at request time" (Vault/OpenBao identity concepts). Attaching or
detaching a policy on a group is visible to every member's existing
tokens immediately.
3. **Batch tokens carry identity.** OpenBao marshals `EntityID` into
the encrypted batch-token blob (`vault/token_store.go`), so
login-issued batch tokens resolve entity and external-group
policies per request — with **zero storage writes per login**.
Limits: no renewal, no accessor, not individually revocable.
(Token-backend-created batch tokens have no entity — the source of
conflicting community lore.)
4. **Entity disable is the per-device kill switch.**
`identity/entity` `disabled=true`: "associated tokens cannot be
used, but are not revoked" — request-time blocking, no device
cooperation, works for batch tokens.
5. **Identity store ceiling.** Entities pack into **256 storage
entries** (hard cap ≈ 0.5–1.25M entities at default limits,
~200 B/entity), the store is memory-resident (go-memdb), and each
entity upsert rewrites its whole shard — per-write cost grows with
fleet size. Community reports: degradation from ~50k identities;
5.7M entities → ~110 GB RAM and >10 min failover
(hashicorp/vault#8761). **JWT login unavoidably creates one
entity + alias per distinct `user_claim`** — under any design.
6. **Write throughput.** Reference-architecture benchmark ≈ 720
storage writes/s; >1k req/s is "high volume" guidance for
integrated storage. No bulk-entity-update API exists.
7. **Zitadel.** Machine-JWT token issuance benchmarked ≈ 437/s
(v2.66) to ≈ 852/s (v4-rc); roles claim
(`urn:zitadel:iam:org:project:roles`) is a **map**, while
`groups_claim` requires a **string array** → flattening Action
needed until first-class groups (epic zitadel#9702, in flight,
machine users included) ship a documented array claim.
8. **At-scale precedent — AWS IoT Core.** Policies attach to thing
groups; "policies are dynamically applied to things when they
change group membership"; Greengrass deploys secrets by targeting
thing groups. Group-attached policy *is* the canonical hyperscale
IoT authorization model. Azure scopes to per-device/module
identities; no per-workload grant materialization anywhere.
## The decisive arithmetic
Let a deployment target M devices in a fleet cell.
| | B: entity-by-policy | A: groups |
|---|---|---|
| Writes per deployment change | M entity upserts; each rewrites ~1/256 of all entities → at M = 10⁵, gigabytes of Raft churn, minutes–hours of convergence, partial-failure management | 1 policy attach per allowed group, instant for all existing tokens (fact 2) |
| Writes per membership change | 1 (instant) | 1 Zitadel write; converges at next login ≤ TTL (fact 1) |
| Steady-state auth load | ~0 (long-lived tokens viable) | N/TTL logins/s — 1M devices @ 1 h ≈ 280/s, stateless with batch tokens (fact 3); Zitadel-side token reused across logins (~23 mints/s @ 12 h validity) |
| Per-device emergency cutoff | strip entity policies | disable entity (fact 4) — equivalent |
| Compromised-device blast radius | deployments scheduled on it | all deployments its groups may view — by declaration; granularity = group sizing |
| IdP machinery | none | flattening Action until Zitadel groups GA (fact 7) |
B's write cost scales with *deployment churn × fleet size*; A's with
*membership churn*. Membership is the rare, per-device event —
deployments are the frequent, fleet-wide one. The data plane already
fans out O(M) per deployment (NATS desired-state KV), but NATS KV
sustains ~10⁵ writes/s where OpenBao identity writes run ~10²–10³/s
and degrade as the store grows: per-device *instructions* scale;
per-device *grants* don't. B's apparent security edge largely
dissolved under verification: per-device hard revocation exists in A
(fact 4), routine de-grouping ≤ TTL is an operational tolerance, and
finer isolation is available by defining finer groups. What remains is
A's group-wide secret visibility — accepted explicitly, since
`allowedGroups` *is* the declared audience.
A corollary of fact 1: don't renew tokens at all. Short-TTL batch
tokens whose natural expiry forces re-login make the login itself the
membership sync — deleting the re-login-signal mechanism the first
sketch of this proposal included.
Independent of A vs B, fact 5 caps any single OpenBao at ~10⁵–10⁶
device entities: "millions of devices" means **cell sharding** (one
Zitadel project + OpenBao pair per cell) under either design. Groups
shard naturally with cells.
## State of the art surveyed
- **Vault-ecosystem fleet patterns** — AppRole + trusted orchestrator,
TLS-cert + SCEP (Vault 1.20), SPIFFE auth (Vault 1.21, not in
OpenBao) — all solve device *identity*, which Zitadel already
provides; none address per-workload scoping. Nothing resembling
"a deployment declares which device groups may read its secrets"
exists in that ecosystem; the model is uniformly identity → policy
→ path.
- **Kubernetes `NodeAuthorizer`** — per-node grants derived from pod
placement, the inspiration for B — materializes nothing: it is an
in-memory request-time graph in the API server. The faithful OpenBao
equivalent would be a custom request-time authorization plugin,
rejected for coupling secret reads to control-plane liveness.
- **Templated policies** (`claim_mappings` → alias metadata → one
fleet-wide policy `…/{{identity.entity.aliases.<accessor>.metadata.group}}/*`)
achieve zero grant writes ever — AWS IoT's policy-variable trick —
but templating cannot iterate, forcing exactly one group per device
and group-scoped secret paths. Kept as a fallback simplification.
- **SPIFFE/SPIRE** — workload-level (not device-level) identity would
let each deployment's process authenticate as itself, shrinking the
blast radius below anything device-scoped and making grants
O(deployments). Deferred: SPIRE server + per-device agents, weak
node attestation on TPM-less hardware, and no OpenBao SPIFFE auth
method yet. The group/policy layer chosen here survives that
migration — only the auth mount changes.
## Resolved during review
- ~~Re-login signal on membership change~~ — unnecessary; TTL expiry
is the sync.
- ~~Where membership lives~~ — Zitadel (roles-as-groups interim);
Device CR fields would create a second authority.
- Vocabulary: the staging fleet's device *label* named `group` must be
renamed or clearly distinguished — labels are placement vocabulary
and device-writable; the authorization group is admin-only.
## Open items for implementation
1. Confirm in staging that a disabled entity blocks an in-flight
batch token at request time (fact 4 is documented; prove it e2e).
2. TTL default 1 h — revisit with measured login load.
3. Per-cell sizing guidance once identity-store memory is measured on
OpenBao (Vault-lineage numbers above are the planning basis).

196
docs/adr/025-fleet-device-secret-access.md Normal file
View File

@@ -0,0 +1,196 @@
# Architecture Decision Record: Fleet Device Secret Access
Initial Author: Jean-Gabriel Gill-Couture
Initial Date: 2026-06-01
Last Updated Date: 2026-06-11
## Status
Accepted. User-facing documentation:
[Fleet device secrets guide](../guides/fleet-device-secrets.md).
This ADR is the third design iteration; the trail is preserved in
[`025-device-secret-access/`](./025-device-secret-access/):
[025-1](./025-device-secret-access/025-1-jwt-deployments-claim.md)
(JWT claim scoping, superseded) →
[025-2](./025-device-secret-access/025-2-entity-by-policy.md)
(entity-by-policy, first implementation, retired) →
[025-3](./025-device-secret-access/025-3-groups-as-security-boundary.md)
(the scale and security analysis that produced this design).
## Context
Fleet agents must read per-deployment secrets (image-pull credentials,
application secrets) from OpenBao. Constraints:
1. **One root of trust per device.** The agent already holds a Zitadel
machine keyfile for NATS auth; secret access must derive from the
same key, never a second secret.
2. **Store-enforced isolation.** The secret store decides what a device
may read; a device cannot widen its own scope.
3. **Cross-project safety.** A token minted in another Zitadel project
must be rejected outright.
4. **Fleet scale.** The target is millions of devices. Control-plane
write cost must track *membership churn* (rare, per-device), never
*deployment churn × targeted devices* — a deployment rolling out to
a large group must not trigger a per-device write storm.
## Decision
**Groups are the security boundary; labels only place workloads.**
### Identity: one key, one login, short batch tokens
The device mints a Zitadel access token from its machine keyfile (RFC
7523 JWT-bearer, shared with NATS auth via `harmony_zitadel_jwt`) and
logs into OpenBao's JWT auth method through
`harmony_config`/`OpenbaoSecretStore`. The auth role pins
`bound_issuer` and `bound_audiences = <Zitadel project ID>` — the
project is the trust boundary — with `user_claim = sub` and
`groups_claim = groups`. The role issues **batch tokens** (no storage
writes per login) with **TTL ≈ 1 h, no renewal**. Expiry forces a fresh
login (`refresh_auth()`), and that login is the group-membership sync —
there is no separate re-login flow, signal, or membership watcher.
### Groups: admin-managed in Zitadel, mirrored as OpenBao external groups
A fleet group (e.g. `edge-a`, `customer-x`) is membership a device can
never self-assert: admins manage it in Zitadel, and it reaches OpenBao
inside the signed token's `groups` array claim. Each fleet group has an
OpenBao **external group** whose alias matches the JWT mount, so a
login binds the device's entity to its current groups. Interim claim
source until Zitadel's first-class groups are GA: one project role per
group plus the standard flattening Action (Zitadel's roles claim is a
map; `groups_claim` requires a string array).
### Authorization and scheduling: groups gate, labels place
A `Deployment` declares `allowedGroups` — the groups permitted to view
it (run it and read its secrets). The operator schedules onto
**members of the allowed groups ∩ the label selector**. Labels remain
device-reported placement vocabulary; they can narrow a deployment's
spread, never widen access. (This is Kubernetes' RBAC-vs-scheduling
split.)
### Grants: one policy per deployment, attached to groups
Secrets live under the deployment's subtree of the fleet KV mount; the
policy `deployment-<name>` grants read on exactly that subtree. When a
`Deployment` appears, the operator creates the policy and attaches it
to each allowed group's external group — **O(allowed groups) writes,
regardless of fleet size**. Identity policies resolve at request time,
so every already-issued token of every member device gains (or loses)
access immediately when a policy is attached (or detached).
### Convergence and revocation
| Change | Mechanism | Latency |
|---|---|---|
| Deployment created / `allowedGroups` edited | attach/detach policy on group | immediate (request time) |
| Device added to / removed from a group | next login (token expiry) | ≤ token TTL |
| Emergency: cut one device | disable its OpenBao entity | immediate |
| Emergency: cut one deployment or group | detach the policy | immediate |
| Device key compromised | rotate/revoke machine key in Zitadel + disable entity | immediate |
TTL is therefore an operational tolerance for *routine* de-grouping,
not a security boundary — emergencies never wait on it.
## Rationale
The full analysis with sources is
[025-3](./025-device-secret-access/025-3-groups-as-security-boundary.md);
the load-bearing points:
- **Write cost aligned with event granularity.** The retired
entity-by-policy design (025-2) wrote every targeted device's entity
per deployment change; each identity write rewrites 1/256 of all
entities, so cost grows with fleet size — gigabytes of Raft churn
and hours of convergence at 10⁵–10⁶ targets. Group attachment is one
write and binds instantly.
- **Groups close the label-escalation hole.** Labels are
device-reported; with labels gating secrets, a compromised device
relabels itself to attract any deployment (Kubernetes needs
`NodeRestriction` for exactly this). Group membership is signed by
Zitadel and admin-only.
- **Renewal does not re-sync groups.** Verified in the JWT auth plugin
source (Vault and OpenBao): renew only extends TTLs. Hence
short-lived, non-renewed tokens whose natural expiry is the sync —
batch tokens make this free (1M devices at 1 h TTL ≈ 280 stateless
logins/s; the Zitadel-side token is reused across many logins).
- **Precedent.** AWS IoT Core authorizes fleets exactly this way:
policies attach to thing groups and "are dynamically applied to
things when they change group membership"; Greengrass deploys
secrets by targeting thing groups. There is no Vault-ecosystem
pattern for per-device grant materialization at fleet scale.
- **Cell ceiling is independent of this choice.** OpenBao's identity
store is memory-resident and packs entities into 256 storage entries
(≈0.5–1.25M entities max), and JWT login creates one entity per
device under any design. Past ~10⁵ devices, fleets shard into cells
(one Zitadel project + OpenBao pair each); groups are scoped within
a cell.
## Consequences
**Pros**
- New deployment: no device session touched, no per-device writes, no
admin permission work beyond declaring `allowedGroups`.
- One auth root per device covers NATS and OpenBao; rotation,
inventory, and revocation stay centralized in Zitadel.
- The security boundary is legible: group rosters in Zitadel,
group→deployment grants on the OpenBao group object, both
human-auditable.
- Batch tokens keep steady-state load CPU-bound (JWKS verification),
with zero token-storage writes.
**Cons / accepted trade-offs**
- **Group-granular visibility.** Every device in an allowed group can
read the deployment's secrets, scheduled onto it or not. Isolation
granularity is group sizing — need finer isolation, define finer
groups. (Per-workload identity via SPIFFE is the future evolution
that lifts this; the policy layer survives that migration.)
- **De-grouping converges at TTL** (≤ 1 h). Emergencies use the
immediate levers above instead.
- **Zitadel Action required** until first-class groups GA — small,
version-controlled server-side JS in the token path.
- **Batch tokens are not individually revocable**; per-device cutoff
is entity disable, which blocks the token at request time.
- One entity + alias per device still accumulates in OpenBao (the
cell ceiling above).
## Threat model
| Attacker | Capability | Defense |
|---|---|---|
| External (no Zitadel identity) | None | JWKS signature validation. |
| Other Zitadel project | Mints valid tokens for its own project | `bound_audiences` rejects before any claim is read. |
| Compromised device (key theft) | Reads secrets of deployments its groups are allowed to view — nothing more. Cannot self-widen: labels don't grant, groups are Zitadel-signed. | Disable entity + rotate key; blast radius bounded by group sizing. |
| Compromised operator | Full control of its fleet's grants (policy attach, scheduling) | Blast radius = one cell (one project, one Bao); operator creds under a separate Bao auth path. |
| Compromised OpenBao | Everything in its cell | Out of scope — root of secret trust (ADR-006); cells bound the radius. |
## Alternatives considered
Full detail in the
[iteration trail](./025-device-secret-access/). In brief: **JWT
`deployments` claim** (025-1 — scope frozen at mint time, two-token
invalidation per change, Zitadel metadata as a second source of
truth); **entity-by-policy** (025-2, first implementation — instant
per-device grants but O(targeted devices) writes per deployment,
disqualifying at fleet scale); **one templated policy on alias
metadata** (zero writes ever, but policy templating cannot iterate, so
exactly one group per device — kept as a fallback simplification);
**static per-device Bao tokens** (second long-lived secret, no
scoping); **AppRole / TLS-cert / SPIFFE auth** (solve identity, which
Zitadel already provides; SPIFFE noted as the per-workload evolution
path); **request-time authorization plugin** (true NodeAuthorizer
equivalent, but couples secret reads to control-plane liveness).
## Relationship to other ADRs
ADR-016: the Zitadel machine identity reused here. ADR-020/020-1:
`harmony_config` as the single secret entry point; the human
counterpart of this machine auth path. ADR-023: all OpenBao/Zitadel
configuration lands via Scores.

4
docs/adr/README.md
View File

@@ -53,6 +53,10 @@ Every ADR follows this structure:
| [020-1](./020-1-zitadel-openbao-secure-config-store.md) | Zitadel + OpenBao Secure Config Store | Accepted |
| [020](./020-interactive-configuration-crate.md) | Interactive Configuration Crate | Proposed |
| [022](./022-fleet-agent-upgrade.md) | Fleet Agent Upgrade Procedure | Accepted |
| [023](./023-deploy-architecture.md) | Deploy Architecture | Accepted |
| [024](./024-fleet-platform-capability-decomposition.md) | Fleet Platform Capability Decomposition | Accepted |
| [025](./025-fleet-device-secret-access.md) | Fleet Device Secret Access | Accepted |
| [025-3](./025-device-secret-access/025-3-groups-as-security-boundary.md) | Groups as the Security Boundary — Scale & Security Analysis | Accepted |
## Contributing

113
docs/guides/fleet-device-secrets.md Normal file
View File

@@ -0,0 +1,113 @@
# Fleet device secrets
How fleet devices read their deployments' secrets — image-pull
credentials, application config, API keys — from OpenBao, and how you
control who reads what. Design reasoning lives in
[ADR-025](../adr/025-fleet-device-secret-access.md); this page is the
operating manual.
## The architecture in one minute
A device proves *who it is* with the Zitadel machine key it already
uses for NATS. *What it may read* is decided by **group membership**,
which only admins can change. *Where it runs* is decided by labels.
The three never mix:
```
Admin Zitadel OpenBao
│ assign device to groups ──▶ device ∈ {edge-a, …} │
│ write secret ──────────────────────────────────────────────▶ secret/<fleet>/<deployment>/…
│ │
Deployment CR │
│ allowedGroups: [edge-a] ──▶ operator attaches policy ──────▶ group edge-a may read
│ targetSelector: labels ──▶ operator schedules within │ deployment-<name> subtree
│ authorized devices │
│ │
Device │
└─ machine key ─▶ Zitadel token (groups: [edge-a]) ─▶ JWT login ─▶ reads its secrets
```
Three rules to remember:
| Concept | Who sets it | What it controls |
|---|---|---|
| **Group** | Admin, in Zitadel | Security boundary: which deployments' secrets a device may read |
| **Label** | Device config (self-reported) | Placement only: which authorized devices actually run the workload — narrows, never widens |
| **`allowedGroups`** | Deployment author | Which groups may view (run + read secrets of) the deployment |
## Storing a secret for a deployment
Write it under the deployment's subtree of the fleet KV mount:
```bash
bao kv put secret/<fleet-ns>/<deployment-name>/db-credentials \
username=app password=…
```
Nothing else to configure — access follows from the deployment's
`allowedGroups` below. Every device in an allowed group can read the
whole `<deployment-name>/` subtree, whether or not the workload landed
on it; size your groups to the audience the secret may have.
## Authorizing and placing a deployment
```yaml
apiVersion: fleet.nationtech.io/v1alpha1
kind: Deployment
metadata:
name: hello-web
spec:
allowedGroups: [edge-a] # security: who may view this deployment
targetSelector:
matchLabels:
hw: pi5 # placement: which authorized devices run it
score: …
```
The operator schedules onto devices that are **members of an allowed
group AND match the selector**, then attaches the deployment's read
policy to each allowed group. Existing device sessions pick the grant
up immediately — rolling out a new deployment never touches logins,
tokens, or per-device configuration.
## Managing device group membership
Until Zitadel's first-class groups are GA, a fleet group is a Zitadel
project role (one role per group, surfaced as a `groups` claim by the
fleet's token Action — applied by the fleet bootstrap, not by hand).
Grant or remove the role on the device's machine user to move it
between groups.
Membership changes take effect at the device's **next login, at most
one token TTL later (≈ 1 h)**. This is the only eventually-consistent
operation in the system — everything else binds immediately. If that
window matters, use the emergency levers below.
## Reading secrets on the device
Nothing to do: the agent reads through `harmony_config`, which
authenticates to OpenBao with the machine keyfile automatically
(JWT-bearer rung of `OpenbaoSecretStore`). Tokens are short-lived
batch tokens; the store re-logins transparently on expiry.
## Revocation runbook
| Situation | Action | Effect |
|---|---|---|
| Device leaves a group (routine) | Remove the role/group in Zitadel | Converges at next login, ≤ 1 TTL |
| Device compromised | Disable its OpenBao identity entity, then rotate/delete its Zitadel machine key | Token dead at next request; no new logins |
| Secret/deployment compromised | Detach `deployment-<name>` policy from its groups (or rotate the secret) | All devices lose read immediately |
| Whole group compromised | Detach the group's policies | Immediate, group-wide |
## Guarantees and limits
- A device can only ever read secrets of deployments whose
`allowedGroups` intersect its groups. Labels, CR edits, or a
compromised agent cannot widen that — membership is signed into the
Zitadel token and validated against the project (`bound_audiences`),
so other projects' tokens are rejected outright.
- Visibility is **group-granular**: being scheduled is not required to
read. Finer isolation ⇒ finer groups.
- One Zitadel project + OpenBao pair (a *cell*) comfortably serves
~10⁵ devices; beyond that, shard fleets into cells
([ADR-025](../adr/025-fleet-device-secret-access.md), Rationale).

41
examples/compose_java_react/Cargo.toml Normal file
View File

@@ -0,0 +1,41 @@
[package]
name = "example-compose-java-react"
edition = "2024"
version.workspace = true
readme.workspace = true
license.workspace = true
description = "Deploy a Java+React app to Kubernetes with Harmony by importing its existing docker-compose (publish + deploy, fleet pattern). See ADR-026."
[lib]
name = "example_compose_java_react"
path = "src/lib.rs"
# `deploy` — converge the app onto a cluster (helm upgrade --install).
[[bin]]
name = "compose-deploy"
path = "src/main.rs"
# `publish` — build + push the per-service images and the hydrated chart.
[[bin]]
name = "compose-publish"
path = "src/bin/publish.rs"
[dependencies]
harmony = { path = "../../harmony" }
harmony_cli = { path = "../../harmony_cli" }
harmony_types = { path = "../../harmony_types" }
docker-compose-types = "0.24"
anyhow = { workspace = true }
async-trait = { workspace = true }
clap = { workspace = true }
fqdn = "0.5.2"
k8s-openapi = { workspace = true }
log = { workspace = true }
env_logger = { workspace = true }
serde = { workspace = true }
serde_yaml = { workspace = true }
tempfile = "3"
thiserror = { workspace = true }
tokio = { workspace = true, features = ["full"] }
toml = { workspace = true }

6
examples/compose_java_react/Harmony.toml Normal file
View File

@@ -0,0 +1,6 @@
# Identity only (ADR-026 §3): the app *name*, nothing behavioral.
# Namespace, image, and selectors derive by convention from name × context;
# deploy knobs (replicas, storage class, ingress host, rolling strategy)
# live in typed Scores, never here.
[app]
name = "timesheet"

69
examples/compose_java_react/README.md Normal file
View File

@@ -0,0 +1,69 @@
# Deploy a Java + React app from its `docker-compose.yml`
Demonstrates deploying an existing containerized app to Kubernetes with
Harmony by **importing its `docker-compose.yml`** — no hand-written
manifests, no Argo. The compose file stays the source of truth for the
app's *base* shape; deploy-only concerns live in typed Rust.
```
app/ the "customer" project (their existing repo)
docker-compose.yml ← source of truth: images, ports, env, volumes
backend/ (Java + SQLite, Dockerfile)
frontend/ (React + nginx, Dockerfile)
Harmony.toml identity only — the app name (ADR-026 §3)
src/
compose.rs import docker-compose → typed model (loud on the unsupported)
chart.rs model + deploy knobs → a hydrated helm chart (typed k8s)
publish.rs docker build/push each service + helm package/push the chart
score.rs ComposeAppScore: helm upgrade --install + Ingress
main.rs `compose-deploy` bin
bin/publish `compose-publish` bin
```
## Design
- **compose = base, Score = deploy knobs.** The importer reads only
images, ports, env, and volumes. Replicas, RWX storage class, ingress
host, and rolling strategy are *not* in compose (or `Harmony.toml`) —
they are typed fields on the deploy `Score` (ADR-026 §6). A behavioral
knob in compose or `Harmony.toml` is a defect.
- **Build / publish / deploy split — the fleet pattern.** `publish`
builds + pushes the per-service images and a hydrated helm chart;
`deploy` is a `Score` that `helm upgrade --install`s it and layers the
public Ingress. No `ApplicationScore`, no ArgoCD.
- **Same Scores local → prod.** Only the topology changes
(`K8sAnywhereTopology::from_env`): local k3d to test, a tenant in prod.
- **Loud, never lossy.** Bind mounts and unparseable ports are hard
errors; `depends_on`/`command`/`restart` are warned, never silently
dropped.
## Run it
Local (build, don't push; render the chart from compose):
```sh
cargo run -p example-compose-java-react --bin compose-publish -- --no-push
cargo run -p example-compose-java-react --bin compose-deploy -- \
--host timesheet.local --storage-class <rwx-class> --yes
```
CD / prod (push to the registry, then install the published chart):
```sh
compose-publish --version 0.1.0
compose-deploy --published --version 0.1.0 \
--namespace <tenant>-timesheet --host timesheet.<tenant>.example \
--storage-class cephfs --cluster-issuer letsencrypt --yes
```
`compose-deploy --help` lists every deploy knob.
## ⚠️ SQLite + RollingUpdate + RWX
This demo ships `--rolling` (RollingUpdate) with a `ReadWriteMany` PVC, as
requested. During a rollout two pods briefly share the same SQLite file,
which SQLite does not make safe for concurrent writers over a networked
filesystem. It is tolerable only for an effectively single-writer,
low-traffic app. For true safe zero-downtime, either deploy with
`--recreate` (one writer at a time, brief downtime) or migrate the store
to Postgres.

15
examples/compose_java_react/app/README.md Normal file
View File

@@ -0,0 +1,15 @@
# Timesheet (demo compose app)
Minimal two-tier app used to demo Harmony deploying an existing `docker-compose`
stack. Add timesheet notes; they persist to SQLite.
- **backend** — JDK `HttpServer` (no framework) on `:8080`, SQLite via sqlite-jdbc.
REST: `GET /api/health`, `GET /api/entries`, `POST /api/entries`.
- **frontend** — React (Vite) served by nginx on `:80`; nginx proxies `/api/`
to the backend (`BACKEND_URL`), so the bundle stays env-agnostic.
- **volume** `timesheet-data` mounts at `/data`; the DB lives at `/data/timesheet.db`.
```sh
docker compose up --build
# frontend → http://localhost:8081 backend → http://localhost:8080/api/health
```

12
examples/compose_java_react/app/backend/Dockerfile Normal file
View File

@@ -0,0 +1,12 @@
FROM maven:3-eclipse-temurin-21 AS build
WORKDIR /src
COPY pom.xml .
COPY src ./src
RUN mvn -q -B package
FROM eclipse-temurin:21-jre
WORKDIR /app
COPY --from=build /src/target/timesheet-backend.jar app.jar
EXPOSE 8080
USER 1000
ENTRYPOINT ["java", "-jar", "app.jar"]

48
examples/compose_java_react/app/backend/pom.xml Normal file
View File

@@ -0,0 +1,48 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>io.nationtech.demo</groupId>
<artifactId>timesheet-backend</artifactId>
<version>1.0.0</version>
<packaging>jar</packaging>
<properties>
<maven.compiler.release>21</maven.compiler.release>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
<version>3.49.1.0</version>
</dependency>
</dependencies>
<build>
<finalName>timesheet-backend</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.6.0</version>
<executions>
<execution>
<phase>package</phase>
<goals><goal>shade</goal></goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>io.nationtech.demo.Main</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

125
examples/compose_java_react/app/backend/src/main/java/io/nationtech/demo/Main.java Normal file
View File

@@ -0,0 +1,125 @@
package io.nationtech.demo;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpServer;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;
import java.time.Instant;
public class Main {
static String dbUrl;
public static void main(String[] args) throws Exception {
String dbPath = System.getenv().getOrDefault("DB_PATH", "/data/timesheet.db");
dbUrl = "jdbc:sqlite:" + dbPath;
try (Connection c = DriverManager.getConnection(dbUrl); Statement s = c.createStatement()) {
s.execute("CREATE TABLE IF NOT EXISTS entries (id INTEGER PRIMARY KEY AUTOINCREMENT, note TEXT, created_at TEXT)");
}
HttpServer server = HttpServer.create(new InetSocketAddress("0.0.0.0", 8080), 0);
server.createContext("/api/health", ex -> send(ex, 200, "{\"status\":\"ok\"}"));
server.createContext("/api/entries", Main::entries);
server.setExecutor(null);
server.start();
System.out.println("listening on 0.0.0.0:8080, db=" + dbPath);
}
static void entries(HttpExchange ex) throws IOException {
try {
switch (ex.getRequestMethod()) {
case "GET" -> send(ex, 200, listEntries());
case "POST" -> {
String note = parseNote(new String(ex.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));
send(ex, 201, insertEntry(note));
}
default -> send(ex, 405, "{\"error\":\"method not allowed\"}");
}
} catch (Exception e) {
send(ex, 500, "{\"error\":" + jsonStr(e.getMessage()) + "}");
}
}
static String listEntries() throws Exception {
StringBuilder sb = new StringBuilder("[");
try (Connection c = DriverManager.getConnection(dbUrl);
Statement s = c.createStatement();
ResultSet rs = s.executeQuery("SELECT id, note, created_at FROM entries ORDER BY id")) {
boolean first = true;
while (rs.next()) {
if (!first) sb.append(",");
first = false;
sb.append(entryJson(rs.getLong("id"), rs.getString("note"), rs.getString("created_at")));
}
}
return sb.append("]").toString();
}
static String insertEntry(String note) throws Exception {
String createdAt = Instant.now().toString();
try (Connection c = DriverManager.getConnection(dbUrl);
PreparedStatement ps = c.prepareStatement(
"INSERT INTO entries (note, created_at) VALUES (?, ?)", Statement.RETURN_GENERATED_KEYS)) {
ps.setString(1, note);
ps.setString(2, createdAt);
ps.executeUpdate();
try (ResultSet keys = ps.getGeneratedKeys()) {
keys.next();
return entryJson(keys.getLong(1), note, createdAt);
}
}
}
static String entryJson(long id, String note, String createdAt) {
return "{\"id\":" + id + ",\"note\":" + jsonStr(note) + ",\"created_at\":" + jsonStr(createdAt) + "}";
}
// Tiny extractor for {"note":"..."} — sufficient for this demo's single-field body.
static String parseNote(String body) {
int k = body.indexOf("\"note\"");
if (k < 0) return "";
int colon = body.indexOf(':', k);
int open = body.indexOf('"', colon + 1);
if (open < 0) return "";
StringBuilder sb = new StringBuilder();
for (int i = open + 1; i < body.length(); i++) {
char ch = body.charAt(i);
if (ch == '\\' && i + 1 < body.length()) { sb.append(body.charAt(++i)); continue; }
if (ch == '"') break;
sb.append(ch);
}
return sb.toString();
}
static String jsonStr(String v) {
if (v == null) return "null";
StringBuilder sb = new StringBuilder("\"");
for (char ch : v.toCharArray()) {
switch (ch) {
case '"' -> sb.append("\\\"");
case '\\' -> sb.append("\\\\");
case '\n' -> sb.append("\\n");
case '\r' -> sb.append("\\r");
case '\t' -> sb.append("\\t");
default -> sb.append(ch);
}
}
return sb.append("\"").toString();
}
static void send(HttpExchange ex, int status, String body) throws IOException {
byte[] b = body.getBytes(StandardCharsets.UTF_8);
ex.getResponseHeaders().set("Content-Type", "application/json");
ex.sendResponseHeaders(status, b.length);
ex.getResponseBody().write(b);
ex.close();
}
}

27
examples/compose_java_react/app/docker-compose.yml Normal file
View File

@@ -0,0 +1,27 @@
# The customer's existing docker-compose — the single source of truth for
# the app's *base* shape (images, ports, env, volumes). Harmony imports
# this and derives the k8s deployment; deploy-only knobs (ingress host,
# storage class, replicas, rolling strategy) live in the deploy crate's
# Score, never here. See ADR-026.
name: timesheet
services:
backend:
build: ./backend
ports:
- "8080:8080"
environment:
- DB_PATH=/data/timesheet.db
volumes:
- timesheet-data:/data
frontend:
build: ./frontend
ports:
- "8081:80"
environment:
- BACKEND_URL=http://backend:8080
depends_on:
- backend
volumes:
timesheet-data:

2
examples/compose_java_react/app/frontend/.dockerignore Normal file
View File

@@ -0,0 +1,2 @@
node_modules
dist

14
examples/compose_java_react/app/frontend/Dockerfile Normal file
View File

@@ -0,0 +1,14 @@
FROM node:22-alpine AS build
WORKDIR /app
COPY package.json ./
RUN npm install
COPY . .
RUN npm run build
FROM nginx:alpine
COPY --from=build /app/dist /usr/share/nginx/html
COPY nginx.conf.template /etc/nginx/templates/default.conf.template
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
EXPOSE 80
ENTRYPOINT ["/entrypoint.sh"]

4
examples/compose_java_react/app/frontend/entrypoint.sh Normal file
View File

@@ -0,0 +1,4 @@
#!/bin/sh
set -e
envsubst '${BACKEND_URL}' < /etc/nginx/templates/default.conf.template > /etc/nginx/conf.d/default.conf
exec nginx -g 'daemon off;'

12
examples/compose_java_react/app/frontend/index.html Normal file
View File

@@ -0,0 +1,12 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Timesheet</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.jsx"></script>
</body>
</html>

13
examples/compose_java_react/app/frontend/nginx.conf.template Normal file
View File

@@ -0,0 +1,13 @@
server {
listen 80;
root /usr/share/nginx/html;
index index.html;
location /api/ {
proxy_pass ${BACKEND_URL};
}
location / {
try_files $uri /index.html;
}
}

18
examples/compose_java_react/app/frontend/package.json Normal file
View File

@@ -0,0 +1,18 @@
{
"name": "timesheet-frontend",
"private": true,
"version": "1.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "vite build"
},
"dependencies": {
"react": "^18.3.1",
"react-dom": "^18.3.1"
},
"devDependencies": {
"@vitejs/plugin-react": "^4.3.4",
"vite": "^5.4.11"
}
}

48
examples/compose_java_react/app/frontend/src/App.jsx Normal file
View File

@@ -0,0 +1,48 @@
import { useEffect, useState } from "react";
export default function App() {
const [entries, setEntries] = useState([]);
const [note, setNote] = useState("");
const load = () =>
fetch("/api/entries")
.then((r) => r.json())
.then(setEntries);
useEffect(() => {
load();
}, []);
const submit = async (e) => {
e.preventDefault();
if (!note.trim()) return;
await fetch("/api/entries", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ note }),
});
setNote("");
load();
};
return (
<main style={{ fontFamily: "sans-serif", maxWidth: 600, margin: "2rem auto" }}>
<h1>Timesheet</h1>
<form onSubmit={submit}>
<input
value={note}
onChange={(e) => setNote(e.target.value)}
placeholder="What did you work on?"
/>
<button type="submit">Add</button>
</form>
<ul>
{entries.map((e) => (
<li key={e.id}>
{e.note} <small>({e.created_at})</small>
</li>
))}
</ul>
</main>
);
}

5
examples/compose_java_react/app/frontend/src/main.jsx Normal file
View File

@@ -0,0 +1,5 @@
import React from "react";
import { createRoot } from "react-dom/client";
import App from "./App.jsx";
createRoot(document.getElementById("root")).render(<App />);

6
examples/compose_java_react/app/frontend/vite.config.js Normal file
View File

@@ -0,0 +1,6 @@
import { defineConfig } from "vite";
import react from "@vitejs/plugin-react";
export default defineConfig({
plugins: [react()],
});

56
examples/compose_java_react/src/bin/publish.rs Normal file
View File

@@ -0,0 +1,56 @@
//! `compose-publish` — build + push the per-service images and the
//! hydrated chart from the imported compose. `docker`/`helm` must be on
//! PATH and logged in to the registry.
use std::path::PathBuf;
use anyhow::{Context, Result};
use clap::Parser;
use example_compose_java_react::{ComposeApp, DeployConfig, HarmonyManifest, publish::publish};
#[derive(Parser, Debug)]
#[command(
name = "compose-publish",
about = "Build + push the app images and chart"
)]
struct Cli {
#[arg(long, default_value = env!("CARGO_MANIFEST_DIR"))]
project_dir: PathBuf,
#[arg(long)]
compose_dir: Option<PathBuf>,
#[arg(long, default_value = "hub.nationtech.io")]
registry: String,
#[arg(long, default_value = "harmony")]
project: String,
#[arg(long, default_value = "0.1.0")]
version: String,
/// Build + package only; skip both pushes (local k3d smoke-test).
#[arg(long)]
no_push: bool,
}
fn main() -> Result<()> {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
let cli = Cli::parse();
let manifest = HarmonyManifest::from_dir(&cli.project_dir)?;
let compose_dir = cli
.compose_dir
.unwrap_or_else(|| cli.project_dir.join("app"));
let app = ComposeApp::from_dir(&compose_dir).context("importing docker-compose")?;
let deploy = DeployConfig {
app_name: manifest.app.name,
chart_version: cli.version.clone(),
registry: cli.registry,
project: cli.project,
version: cli.version,
storage_class: None,
volume_size: "1Gi".to_string(),
replicas: 1,
rolling: true,
};
publish(&app, &deploy, !cli.no_push)
}

378
examples/compose_java_react/src/chart.rs Normal file
View File

@@ -0,0 +1,378 @@
//! Generate a helm chart from an imported [`ComposeApp`].
//!
//! Typed `k8s_openapi` resources serialized at build time (ADR-018, the
//! fleet operator chart pattern) — no `{{ .Values }}` templating. One
//! Deployment + Service per compose service, one RWX PVC per mounted
//! named volume. Ingress is layered at deploy time by the
//! [`Score`](crate::score), not baked here, so the host stays a deploy
//! knob.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use k8s_openapi::api::apps::v1::{
Deployment, DeploymentSpec, DeploymentStrategy, RollingUpdateDeployment,
};
use k8s_openapi::api::core::v1::{
Container, ContainerPort, EnvVar, PersistentVolumeClaim, PersistentVolumeClaimSpec,
PersistentVolumeClaimVolumeSource, PodSpec, PodTemplateSpec, Service, ServicePort, ServiceSpec,
Volume, VolumeMount, VolumeResourceRequirements,
};
use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta;
use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString;
use harmony::modules::application::helm::{HelmChart, HelmResourceKind};
use crate::compose::{ComposeApp, ComposeService};
/// Deploy-only knobs (ADR-026 §6) — the typed home for everything the
/// compose file deliberately does not carry.
#[derive(Debug, Clone, serde::Serialize)]
pub struct DeployConfig {
pub app_name: String,
pub chart_version: String,
/// Image coordinates for services that build from source.
pub registry: String,
pub project: String,
pub version: String,
/// RWX-capable storage class for PVCs (e.g. ceph `cephfs`). `None`
/// uses the cluster default — only correct if that default is RWX.
pub storage_class: Option<String>,
pub volume_size: String,
pub replicas: i32,
/// `true` → RollingUpdate (needs RWX so old+new pods share volumes);
/// `false` → Recreate (safe for single-writer stores like sqlite).
pub rolling: bool,
}
const ACCESS_MODE_RWX: &str = "ReadWriteMany";
/// The published image ref a service's Deployment should pull: built
/// services get our registry coordinates, prebuilt `image:` services are
/// used verbatim.
pub fn service_image(cfg: &DeployConfig, svc: &ComposeService) -> String {
if svc.build_context.is_some() {
format!(
"{}/{}/{}-{}:{}",
cfg.registry, cfg.project, cfg.app_name, svc.name, cfg.version
)
} else {
svc.image
.clone()
.expect("validated: image or build present")
}
}
/// Build + write the chart to `out_dir`; returns the chart directory
/// `helm install <path>` wants.
pub fn build_chart(
app: &ComposeApp,
cfg: &DeployConfig,
out_dir: &Path,
) -> Result<PathBuf, String> {
let mut chart = HelmChart::new(cfg.app_name.clone(), cfg.version.clone());
chart.version = cfg.chart_version.clone();
chart.description = format!("{} — imported from docker-compose by Harmony", cfg.app_name);
for svc in &app.services {
chart.add_resource(
HelmResourceKind::from_serializable(
format!("deployment-{}.yaml", svc.name),
&deployment(svc, cfg),
)
.map_err(|e| format!("serialize deployment {}: {e}", svc.name))?,
);
if let Some(service) = service(svc) {
chart.add_resource(
HelmResourceKind::from_serializable(format!("service-{}.yaml", svc.name), &service)
.map_err(|e| format!("serialize service {}: {e}", svc.name))?,
);
}
}
for vol in app.mounted_volumes() {
chart.add_resource(
HelmResourceKind::from_serializable(format!("pvc-{vol}.yaml"), &pvc(&vol, cfg))
.map_err(|e| format!("serialize pvc {vol}: {e}"))?,
);
}
chart
.write_to(out_dir)
.map_err(|e| format!("writing chart: {e}"))
}
fn labels(name: &str) -> BTreeMap<String, String> {
BTreeMap::from([
("app.kubernetes.io/name".to_string(), name.to_string()),
(
"app.kubernetes.io/managed-by".to_string(),
"harmony".to_string(),
),
])
}
fn deployment(svc: &ComposeService, cfg: &DeployConfig) -> Deployment {
let labels = labels(&svc.name);
let ports: Vec<ContainerPort> = svc
.ports
.iter()
.map(|p| ContainerPort {
container_port: i32::from(p.container),
protocol: Some(p.protocol.clone()),
..Default::default()
})
.collect();
let env: Vec<EnvVar> = svc
.env
.iter()
.map(|(k, v)| EnvVar {
name: k.clone(),
value: Some(v.clone()),
..Default::default()
})
.collect();
let volume_mounts: Vec<VolumeMount> = svc
.mounts
.iter()
.map(|m| VolumeMount {
name: m.volume.clone(),
mount_path: m.path.clone(),
..Default::default()
})
.collect();
let volumes: Vec<Volume> = svc
.mounts
.iter()
.map(|m| Volume {
name: m.volume.clone(),
persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
claim_name: m.volume.clone(),
..Default::default()
}),
..Default::default()
})
.collect();
let strategy = Some(if cfg.rolling {
DeploymentStrategy {
type_: Some("RollingUpdate".to_string()),
rolling_update: Some(RollingUpdateDeployment {
max_surge: Some(IntOrString::Int(1)),
max_unavailable: Some(IntOrString::Int(0)),
}),
}
} else {
DeploymentStrategy {
type_: Some("Recreate".to_string()),
rolling_update: None,
}
});
Deployment {
metadata: ObjectMeta {
name: Some(svc.name.clone()),
labels: Some(labels.clone()),
..Default::default()
},
spec: Some(DeploymentSpec {
replicas: Some(cfg.replicas),
strategy,
selector: LabelSelector {
match_labels: Some(labels.clone()),
..Default::default()
},
template: PodTemplateSpec {
metadata: Some(ObjectMeta {
labels: Some(labels),
..Default::default()
}),
spec: Some(PodSpec {
containers: vec![Container {
name: svc.name.clone(),
image: Some(service_image(cfg, svc)),
image_pull_policy: Some("IfNotPresent".to_string()),
ports: (!ports.is_empty()).then_some(ports),
env: (!env.is_empty()).then_some(env),
volume_mounts: (!volume_mounts.is_empty()).then_some(volume_mounts),
..Default::default()
}],
volumes: (!volumes.is_empty()).then_some(volumes),
..Default::default()
}),
},
..Default::default()
}),
..Default::default()
}
}
fn service(svc: &ComposeService) -> Option<Service> {
if svc.ports.is_empty() {
return None;
}
let labels = labels(&svc.name);
let ports = svc
.ports
.iter()
.map(|p| ServicePort {
name: Some(format!("port-{}", p.container)),
port: i32::from(p.container),
target_port: Some(IntOrString::Int(i32::from(p.container))),
protocol: Some(p.protocol.clone()),
..Default::default()
})
.collect();
Some(Service {
metadata: ObjectMeta {
name: Some(svc.name.clone()),
labels: Some(labels.clone()),
..Default::default()
},
spec: Some(ServiceSpec {
type_: Some("ClusterIP".to_string()),
selector: Some(labels),
ports: Some(ports),
..Default::default()
}),
..Default::default()
})
}
fn pvc(name: &str, cfg: &DeployConfig) -> PersistentVolumeClaim {
PersistentVolumeClaim {
metadata: ObjectMeta {
name: Some(name.to_string()),
labels: Some(labels(name)),
..Default::default()
},
spec: Some(PersistentVolumeClaimSpec {
access_modes: Some(vec![ACCESS_MODE_RWX.to_string()]),
storage_class_name: cfg.storage_class.clone(),
resources: Some(VolumeResourceRequirements {
requests: Some(BTreeMap::from([(
"storage".to_string(),
Quantity(cfg.volume_size.clone()),
)])),
..Default::default()
}),
..Default::default()
}),
..Default::default()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::Path;
fn fixture() -> (ComposeApp, DeployConfig, tempfile::TempDir) {
let yaml = "name: t\nservices:\n backend:\n build: ./backend\n ports: [\"8080:8080\"]\n volumes:\n - data:/data\n db:\n image: postgres:16\nvolumes:\n data:\n";
let app = ComposeApp::parse(yaml, Path::new("/p")).unwrap();
let cfg = DeployConfig {
app_name: "timesheet".to_string(),
chart_version: "0.1.0".to_string(),
registry: "hub.example".to_string(),
project: "harmony".to_string(),
version: "1.2.3".to_string(),
storage_class: Some("cephfs".to_string()),
volume_size: "2Gi".to_string(),
replicas: 2,
rolling: true,
};
let tmp = tempfile::tempdir().unwrap();
build_chart(&app, &cfg, tmp.path()).unwrap();
(app, cfg, tmp)
}
fn read(tmp: &tempfile::TempDir, file: &str) -> String {
std::fs::read_to_string(tmp.path().join("timesheet/templates").join(file)).unwrap()
}
#[test]
fn built_service_gets_registry_image_prebuilt_stays_verbatim() {
let cfg = fixture().1;
let app = ComposeApp::parse(
"services:\n backend:\n build: .\n db:\n image: postgres:16\n",
Path::new("/p"),
)
.unwrap();
assert_eq!(
service_image(&cfg, app.service("backend").unwrap()),
"hub.example/harmony/timesheet-backend:1.2.3"
);
assert_eq!(
service_image(&cfg, app.service("db").unwrap()),
"postgres:16"
);
}
#[test]
fn each_service_renders_its_own_deployment_file() {
let (_, _, tmp) = fixture();
// Per-service filenames — a shared "deployment.yaml" would collide.
assert!(
tmp.path()
.join("timesheet/templates/deployment-backend.yaml")
.exists()
);
assert!(
tmp.path()
.join("timesheet/templates/deployment-db.yaml")
.exists()
);
}
#[test]
fn portless_service_emits_no_service_object() {
let (_, _, tmp) = fixture();
assert!(
tmp.path()
.join("timesheet/templates/service-backend.yaml")
.exists()
);
assert!(
!tmp.path()
.join("timesheet/templates/service-db.yaml")
.exists()
);
}
#[test]
fn pvc_is_readwritemany_with_storage_class() {
let (_, _, tmp) = fixture();
let pvc = read(&tmp, "pvc-data.yaml");
assert!(pvc.contains("ReadWriteMany"), "{pvc}");
assert!(pvc.contains("cephfs"), "{pvc}");
assert!(pvc.contains("2Gi"), "{pvc}");
}
#[test]
fn rolling_strategy_and_replicas_render() {
let (_, _, tmp) = fixture();
let dep = read(&tmp, "deployment-backend.yaml");
assert!(dep.contains("RollingUpdate"), "{dep}");
assert!(dep.contains("replicas: 2"), "{dep}");
assert!(dep.contains("claimName: data"), "{dep}");
}
#[test]
fn recreate_strategy_when_not_rolling() {
let yaml = "services:\n a:\n build: .\n";
let app = ComposeApp::parse(yaml, Path::new("/p")).unwrap();
let mut cfg = fixture().1;
cfg.rolling = false;
let tmp = tempfile::tempdir().unwrap();
build_chart(&app, &cfg, tmp.path()).unwrap();
let dep = std::fs::read_to_string(tmp.path().join("timesheet/templates/deployment-a.yaml"))
.unwrap();
assert!(dep.contains("Recreate"), "{dep}");
}
}

388
examples/compose_java_react/src/compose.rs Normal file
View File

@@ -0,0 +1,388 @@
//! Import an existing `docker-compose.yml` into a typed model.
//!
//! The compose file is the source of truth for the app's *base* shape —
//! images, ports, env, volumes (ADR-026). We translate only that subset
//! into typed Rust at the boundary; deploy-only knobs (replicas, storage
//! class, ingress host, rolling strategy) are NOT read from here — they
//! live in the deploy [`Score`](crate::score). This is an adapter, like
//! reading a Dockerfile, not a YAML deployment interface.
//!
//! What we deliberately do NOT support is rejected loudly (bind mounts,
//! unparseable ports) or warned about (depends_on, networks, …) — a
//! deploy tool must never silently drop config it can't honor.
use std::path::{Path, PathBuf};
use docker_compose_types::{Compose, Environment, Ports, PublishedPort, Volumes};
use thiserror::Error;
#[derive(Debug, Error)]
pub enum ComposeError {
#[error("reading {path}: {source}")]
Read {
path: PathBuf,
source: std::io::Error,
},
#[error("parsing compose yaml: {0}")]
Parse(#[from] serde_yaml::Error),
#[error("service '{service}' has neither `image` nor `build` — nothing to deploy")]
NoImageOrBuild { service: String },
#[error("service '{service}': port '{spec}' is not parseable")]
BadPort { service: String, spec: String },
#[error(
"service '{service}': volume '{spec}' is a bind mount; only named volumes map to a k8s PVC"
)]
BindMount { service: String, spec: String },
#[error("service '{service}': volume source '{volume}' is not a named top-level volume")]
UnknownVolume { service: String, volume: String },
}
/// The imported app: an ordered list of services plus the named volumes
/// declared at the compose top level.
#[derive(Debug, Clone, serde::Serialize)]
pub struct ComposeApp {
pub services: Vec<ComposeService>,
pub named_volumes: Vec<String>,
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct ComposeService {
pub name: String,
/// Explicit prebuilt image (e.g. `postgres:16`) — used as-is, never
/// built. Mutually informative with `build_context`.
pub image: Option<String>,
/// Build context dir (resolved relative to the compose file) — when
/// set, `publish` builds + pushes the image under our registry.
pub build_context: Option<PathBuf>,
pub dockerfile: Option<String>,
pub ports: Vec<PortMap>,
pub env: Vec<(String, String)>,
pub mounts: Vec<VolumeMount>,
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct PortMap {
pub container: u16,
pub published: Option<u16>,
pub protocol: String,
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct VolumeMount {
pub volume: String,
pub path: String,
}
impl ComposeApp {
/// Parse `<dir>/docker-compose.yml` (falling back to `.yaml`).
pub fn from_dir(dir: &Path) -> Result<Self, ComposeError> {
let path = ["docker-compose.yml", "docker-compose.yaml", "compose.yaml"]
.iter()
.map(|f| dir.join(f))
.find(|p| p.exists())
.unwrap_or_else(|| dir.join("docker-compose.yml"));
let raw = std::fs::read_to_string(&path).map_err(|source| ComposeError::Read {
path: path.clone(),
source,
})?;
Self::parse(&raw, dir)
}
/// Parse compose YAML, resolving build contexts relative to `base_dir`.
pub fn parse(raw: &str, base_dir: &Path) -> Result<Self, ComposeError> {
let compose: Compose = serde_yaml::from_str(raw)?;
let named_volumes: Vec<String> = compose.volumes.0.keys().cloned().collect();
let mut services = Vec::new();
for (name, svc) in compose.services.0 {
let svc = svc.unwrap_or_default();
let build_context = match &svc.build_ {
Some(docker_compose_types::BuildStep::Simple(ctx)) => Some(base_dir.join(ctx)),
Some(docker_compose_types::BuildStep::Advanced(a)) => {
Some(base_dir.join(&a.context))
}
None => None,
};
let dockerfile = match &svc.build_ {
Some(docker_compose_types::BuildStep::Advanced(a)) => a.dockerfile.clone(),
_ => None,
};
if svc.image.is_none() && build_context.is_none() {
return Err(ComposeError::NoImageOrBuild { service: name });
}
warn_ignored(&name, &svc);
services.push(ComposeService {
ports: parse_ports(&name, &svc.ports)?,
env: parse_env(&svc.environment),
mounts: parse_volumes(&name, &svc.volumes, &named_volumes)?,
image: svc.image,
build_context,
dockerfile,
name,
});
}
Ok(Self {
services,
named_volumes,
})
}
pub fn service(&self, name: &str) -> Option<&ComposeService> {
self.services.iter().find(|s| s.name == name)
}
/// Named volumes actually referenced by a mount — the ones that need
/// a PVC. A declared-but-unmounted volume needs no k8s resource.
pub fn mounted_volumes(&self) -> Vec<String> {
self.named_volumes
.iter()
.filter(|v| {
self.services
.iter()
.any(|s| s.mounts.iter().any(|m| &&m.volume == v))
})
.cloned()
.collect()
}
}
fn parse_ports(service: &str, ports: &Ports) -> Result<Vec<PortMap>, ComposeError> {
match ports {
Ports::Short(specs) => specs.iter().map(|s| parse_short_port(service, s)).collect(),
Ports::Long(ports) => Ok(ports
.iter()
.map(|p| PortMap {
container: p.target,
published: match &p.published {
Some(PublishedPort::Single(n)) => Some(*n),
_ => None,
},
protocol: p.protocol.clone().unwrap_or_else(|| "TCP".to_string()),
})
.collect()),
}
}
/// `"8081:80"`, `"80"`, `"127.0.0.1:8080:80"`, `"80/udp"` → the last
/// colon-token is the container port, the previous (if any) the host
/// port, a host IP before that is ignored (k8s has no host binding).
fn parse_short_port(service: &str, spec: &str) -> Result<PortMap, ComposeError> {
let bad = || ComposeError::BadPort {
service: service.to_string(),
spec: spec.to_string(),
};
let (ports, protocol) = match spec.split_once('/') {
Some((p, proto)) => (p, proto.to_uppercase()),
None => (spec, "TCP".to_string()),
};
let tokens: Vec<&str> = ports.split(':').collect();
let container = tokens.last().ok_or_else(bad)?.parse().map_err(|_| bad())?;
let published = match tokens.len() {
1 => None,
n => tokens[n - 2].parse().ok(),
};
Ok(PortMap {
container,
published,
protocol,
})
}
fn parse_env(env: &Environment) -> Vec<(String, String)> {
match env {
Environment::List(items) => items
.iter()
.map(|item| match item.split_once('=') {
Some((k, v)) => (k.to_string(), v.to_string()),
None => (item.to_string(), String::new()),
})
.collect(),
Environment::KvPair(map) => map
.iter()
.map(|(k, v)| {
let v = v.as_ref().map(|s| s.to_string()).unwrap_or_default();
(k.clone(), v)
})
.collect(),
}
}
fn parse_volumes(
service: &str,
volumes: &[Volumes],
named: &[String],
) -> Result<Vec<VolumeMount>, ComposeError> {
let mut mounts = Vec::new();
for vol in volumes {
let (source, target) = match vol {
Volumes::Simple(spec) => {
let (source, target) =
spec.split_once(':')
.ok_or_else(|| ComposeError::BindMount {
service: service.to_string(),
spec: spec.clone(),
})?;
// strip a trailing `:ro`/`:rw` mode if present.
let target = target.split(':').next().unwrap_or(target);
(source.to_string(), target.to_string())
}
Volumes::Advanced(a) => (a.source.clone().unwrap_or_default(), a.target.clone()),
};
// A path-like source is a bind mount — not portable to a cluster.
if source.starts_with('.') || source.starts_with('/') {
return Err(ComposeError::BindMount {
service: service.to_string(),
spec: source,
});
}
if !named.contains(&source) {
return Err(ComposeError::UnknownVolume {
service: service.to_string(),
volume: source,
});
}
mounts.push(VolumeMount {
volume: source,
path: target,
});
}
Ok(mounts)
}
/// Compose keys we recognize but don't translate. A deploy tool that
/// drops these silently is a footgun, so we say so.
fn warn_ignored(name: &str, svc: &docker_compose_types::Service) {
if !svc.depends_on.is_empty() {
log::warn!(
"service '{name}': `depends_on` ignored — k8s ordering is via readiness, not startup order"
);
}
if svc.command.is_some() {
log::warn!("service '{name}': `command` override ignored — bake it into the image");
}
if svc.restart.is_some() {
log::warn!("service '{name}': `restart` ignored — k8s restarts via the Deployment policy");
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::Path;
const COMPOSE: &str = r#"
name: timesheet
services:
backend:
build: ./backend
ports: ["8080:8080"]
environment:
- DB_PATH=/data/timesheet.db
volumes:
- timesheet-data:/data
frontend:
build: ./frontend
ports: ["8081:80"]
environment:
- BACKEND_URL=http://backend:8080
depends_on:
- backend
volumes:
timesheet-data:
"#;
fn app() -> ComposeApp {
ComposeApp::parse(COMPOSE, Path::new("/proj")).expect("parse")
}
#[test]
fn imports_both_services_in_order() {
let app = app();
let names: Vec<&str> = app.services.iter().map(|s| s.name.as_str()).collect();
assert_eq!(names, ["backend", "frontend"]);
}
#[test]
fn build_context_resolves_against_base_dir() {
let backend = app().service("backend").unwrap().clone();
assert_eq!(
backend.build_context,
Some(PathBuf::from("/proj/./backend"))
);
assert!(backend.image.is_none());
}
#[test]
fn ports_split_published_and_container() {
let app = app();
let be = &app.service("backend").unwrap().ports[0];
assert_eq!((be.container, be.published), (8080, Some(8080)));
let fe = &app.service("frontend").unwrap().ports[0];
assert_eq!((fe.container, fe.published), (80, Some(8081)));
}
#[test]
fn env_parsed_as_key_value() {
let backend = app().service("backend").unwrap().clone();
assert_eq!(
backend.env,
vec![("DB_PATH".to_string(), "/data/timesheet.db".to_string())]
);
}
#[test]
fn named_volume_becomes_a_mount_and_only_mounted_volumes_need_pvcs() {
let app = app();
let m = &app.service("backend").unwrap().mounts[0];
assert_eq!(
(m.volume.as_str(), m.path.as_str()),
("timesheet-data", "/data")
);
assert_eq!(app.mounted_volumes(), vec!["timesheet-data".to_string()]);
}
#[test]
fn short_port_forms_parse() {
assert_eq!(parse_short_port("s", "80").unwrap().container, 80);
let hostip = parse_short_port("s", "127.0.0.1:8080:80").unwrap();
assert_eq!((hostip.container, hostip.published), (80, Some(8080)));
let udp = parse_short_port("s", "53:53/udp").unwrap();
assert_eq!(udp.protocol, "UDP");
assert!(parse_short_port("s", "notaport").is_err());
}
#[test]
fn bind_mounts_are_rejected() {
let yaml = "services:\n a:\n image: x\n volumes:\n - ./host:/data\n";
let err = ComposeApp::parse(yaml, Path::new("/p")).unwrap_err();
assert!(matches!(err, ComposeError::BindMount { .. }), "{err}");
}
#[test]
fn volume_not_declared_at_top_level_is_rejected() {
let yaml = "services:\n a:\n image: x\n volumes:\n - ghost:/data\n";
let err = ComposeApp::parse(yaml, Path::new("/p")).unwrap_err();
assert!(matches!(err, ComposeError::UnknownVolume { .. }), "{err}");
}
#[test]
fn service_without_image_or_build_is_rejected() {
let yaml = "services:\n a:\n ports: [\"80\"]\n";
let err = ComposeApp::parse(yaml, Path::new("/p")).unwrap_err();
assert!(matches!(err, ComposeError::NoImageOrBuild { .. }), "{err}");
}
/// The example's real compose must stay importable — it's the demo's
/// source of truth and a contract for the chart tests.
#[test]
fn the_examples_compose_file_imports() {
let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("app");
let app = ComposeApp::from_dir(&dir).expect("import example compose");
assert_eq!(app.services.len(), 2);
assert_eq!(app.mounted_volumes(), vec!["timesheet-data".to_string()]);
}
}

40
examples/compose_java_react/src/lib.rs Normal file
View File

@@ -0,0 +1,40 @@
//! Deploy a Java+React app to Kubernetes by importing its existing
//! `docker-compose.yml` (ADR-026). The compose file is the source of
//! truth for the app's base shape; deploy-only knobs live in
//! [`DeployConfig`]/[`ComposeAppScore`]. Build/publish/deploy follow the
//! fleet pattern — no `ApplicationScore`, no ArgoCD.
pub mod chart;
pub mod compose;
pub mod publish;
pub mod score;
pub use chart::DeployConfig;
pub use compose::ComposeApp;
pub use score::{ComposeAppScore, PublicEndpoint, PublishedChart};
use std::path::Path;
use anyhow::{Context, Result};
use serde::Deserialize;
/// Identity-only project manifest (ADR-026 §3): the app *name*, nothing
/// behavioral. A behavioral knob here is a defect — it belongs in a Score.
#[derive(Debug, Deserialize)]
pub struct HarmonyManifest {
pub app: AppIdentity,
}
#[derive(Debug, Deserialize)]
pub struct AppIdentity {
pub name: String,
}
impl HarmonyManifest {
pub fn from_dir(dir: &Path) -> Result<Self> {
let path = dir.join("Harmony.toml");
let raw = std::fs::read_to_string(&path)
.with_context(|| format!("reading {}", path.display()))?;
toml::from_str(&raw).with_context(|| format!("parsing {}", path.display()))
}
}

127
examples/compose_java_react/src/main.rs Normal file
View File

@@ -0,0 +1,127 @@
//! `compose-deploy` — converge the compose-imported app onto a cluster
//! (`helm upgrade --install`). The same Scores run against local k3d and
//! a remote tenant; only the topology (selected by env) changes.
use std::path::PathBuf;
use anyhow::{Context, Result};
use clap::Parser;
use harmony::inventory::Inventory;
use harmony::topology::K8sAnywhereTopology;
use harmony_cli::Args as HarmonyCliArgs;
use example_compose_java_react::{
ComposeApp, ComposeAppScore, DeployConfig, HarmonyManifest, PublicEndpoint, PublishedChart,
};
#[derive(Parser, Debug)]
#[command(
name = "compose-deploy",
about = "Deploy a docker-compose app to k8s via Harmony"
)]
struct Cli {
/// Project root holding `Harmony.toml` (defaults to this example).
#[arg(long, default_value = env!("CARGO_MANIFEST_DIR"))]
project_dir: PathBuf,
/// Directory holding `docker-compose.yml` (defaults to `<project>/app`).
#[arg(long)]
compose_dir: Option<PathBuf>,
#[arg(long, default_value = "timesheet")]
namespace: String,
#[arg(long, default_value = "hub.nationtech.io")]
registry: String,
#[arg(long, default_value = "harmony")]
project: String,
#[arg(long, default_value = "0.1.0")]
version: String,
/// RWX storage class (e.g. ceph `cephfs`). Omit for the cluster default.
#[arg(long)]
storage_class: Option<String>,
#[arg(long, default_value = "1Gi")]
volume_size: String,
#[arg(long, default_value_t = 2)]
replicas: i32,
/// Recreate instead of RollingUpdate — safe for single-writer sqlite.
#[arg(long)]
recreate: bool,
/// Install the published OCI chart instead of rendering from compose.
#[arg(long)]
published: bool,
/// Compose service to expose via Ingress.
#[arg(long, default_value = "frontend")]
public_service: String,
/// Ingress host; omit to keep the app cluster-internal.
#[arg(long)]
host: Option<String>,
/// cert-manager ClusterIssuer for TLS; omit for plain HTTP.
#[arg(long)]
cluster_issuer: Option<String>,
#[command(flatten)]
harmony_cli: HarmonyCliArgs,
}
#[tokio::main]
async fn main() -> Result<()> {
harmony_cli::cli_logger::init();
let cli = Cli::parse();
let manifest = HarmonyManifest::from_dir(&cli.project_dir)?;
let compose_dir = cli
.compose_dir
.unwrap_or_else(|| cli.project_dir.join("app"));
let app = ComposeApp::from_dir(&compose_dir).context("importing docker-compose")?;
let deploy = DeployConfig {
app_name: manifest.app.name.clone(),
chart_version: cli.version.clone(),
registry: cli.registry.clone(),
project: cli.project.clone(),
version: cli.version.clone(),
storage_class: cli.storage_class,
volume_size: cli.volume_size,
replicas: cli.replicas,
rolling: !cli.recreate,
};
// The host is a deploy-only knob; the port is read from the compose
// service so it stays the single source of truth for the app shape.
let public_endpoint = cli.host.map(|host| -> Result<PublicEndpoint> {
let svc = app
.service(&cli.public_service)
.with_context(|| format!("no compose service '{}'", cli.public_service))?;
let port = svc
.ports
.first()
.with_context(|| format!("service '{}' exposes no port", cli.public_service))?
.container;
Ok(PublicEndpoint {
service: cli.public_service.clone(),
port,
host,
cluster_issuer: cli.cluster_issuer,
})
});
let public_endpoint = public_endpoint.transpose()?;
let score = ComposeAppScore {
namespace: cli.namespace,
release_name: manifest.app.name,
published_chart: cli.published.then_some(PublishedChart {
registry: cli.registry,
project: cli.project,
version: cli.version,
}),
public_endpoint,
app,
deploy,
};
harmony_cli::run(
Inventory::autoload(),
K8sAnywhereTopology::from_env(),
vec![Box::new(score)],
Some(cli.harmony_cli),
)
.await
.map_err(|e| anyhow::anyhow!("{e}"))
}

96
examples/compose_java_react/src/publish.rs Normal file
View File

@@ -0,0 +1,96 @@
//! Build + push the per-service images and the hydrated chart.
//!
//! The fleet `release.rs` pattern, generalized over the compose
//! services: each service that builds from source is built then pushed
//! with docker; prebuilt `image:` services are left alone; then the
//! chart is hydrated, `helm package`d, and pushed to the OCI registry.
//! Plain `anyhow` — this is binary glue, not library API.
use std::path::{Path, PathBuf};
use std::process::Command;
use anyhow::{Context, Result, bail};
use crate::chart::{DeployConfig, build_chart, service_image};
use crate::compose::ComposeApp;
/// Build + push every built service image and the chart. With
/// `push = false`, builds + packages only (a local k3d smoke-test).
/// `docker` and `helm` must be on PATH and logged in to the registry.
pub fn publish(app: &ComposeApp, cfg: &DeployConfig, push: bool) -> Result<()> {
for svc in &app.services {
let Some(context) = &svc.build_context else {
log::info!("service '{}' uses prebuilt image, skipping build", svc.name);
continue;
};
let image = service_image(cfg, svc);
log::info!("docker build {image}");
let mut cmd = Command::new("docker");
cmd.arg("build").arg("-t").arg(&image);
if let Some(dockerfile) = &svc.dockerfile {
cmd.arg("-f").arg(context.join(dockerfile));
}
cmd.arg(context);
run(cmd)?;
if push {
log::info!("docker push {image}");
run({
let mut c = Command::new("docker");
c.arg("push").arg(&image);
c
})?;
}
}
let tmp = tempfile::tempdir().context("chart tempdir")?;
let chart_dir =
build_chart(app, cfg, tmp.path()).map_err(|e| anyhow::anyhow!("hydrate chart: {e}"))?;
let tgz = helm_package(&chart_dir, tmp.path())?;
if push {
let oci_repo = format!("oci://{}/{}", cfg.registry, cfg.project);
log::info!("helm push {} {oci_repo}", tgz.display());
run({
let mut c = Command::new("helm");
c.arg("push").arg(&tgz).arg(&oci_repo);
c
})?;
}
log::info!(
"published app={} version={} pushed={push}",
cfg.app_name,
cfg.version
);
Ok(())
}
fn run(mut cmd: Command) -> Result<()> {
let status = cmd
.status()
.with_context(|| format!("spawn {cmd:?} (is it installed and in PATH?)"))?;
if !status.success() {
bail!("`{cmd:?}` failed ({status})");
}
Ok(())
}
fn helm_package(chart_dir: &Path, out_dir: &Path) -> Result<PathBuf> {
let output = Command::new("helm")
.args(["package", path_str(chart_dir)?, "-d", path_str(out_dir)?])
.output()
.context("spawn helm package")?;
if !output.status.success() {
bail!("helm package failed ({})", output.status);
}
// helm prints "…saved it to: <path>"; the last token is the path.
String::from_utf8_lossy(&output.stdout)
.split_whitespace()
.last()
.map(PathBuf::from)
.context("helm package printed no path")
}
fn path_str(p: &Path) -> Result<&str> {
p.to_str()
.with_context(|| format!("path not utf-8: {}", p.display()))
}

197
examples/compose_java_react/src/score.rs Normal file
View File

@@ -0,0 +1,197 @@
//! [`ComposeAppScore`] — converge a compose-imported app onto a cluster.
//!
//! Mirrors `FleetOperatorScore`: render a self-contained helm chart from
//! the imported compose at interpret time (or install an already-published
//! OCI chart on the CD path), `helm upgrade --install` via
//! [`HelmChartScore`], then layer the public Ingress on top. No
//! `ApplicationScore`, no ArgoCD — the same build/publish/deploy split the
//! fleet stack uses.
use std::str::FromStr;
use async_trait::async_trait;
use harmony_types::id::Id;
use log::info;
use serde::Serialize;
use harmony::data::Version;
use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
use harmony::inventory::Inventory;
use harmony::modules::helm::chart::{HelmChartScore, NonBlankString};
use harmony::modules::k8s::ingress::K8sIngressScore;
use harmony::score::Score;
use harmony::topology::{HelmCommand, K8sclient, Topology};
use crate::chart::{DeployConfig, build_chart};
use crate::compose::ComposeApp;
/// The already-published OCI chart to install (the CD path). When set,
/// the chart is pulled from the registry instead of rendered locally.
#[derive(Debug, Clone, Serialize)]
pub struct PublishedChart {
pub registry: String,
pub project: String,
pub version: String,
}
/// Expose one service through an Ingress (a deploy-only knob — the host
/// is never read from compose).
#[derive(Debug, Clone, Serialize)]
pub struct PublicEndpoint {
/// Compose service name to route to (its Service + port).
pub service: String,
pub port: u16,
pub host: String,
/// cert-manager ClusterIssuer for TLS; `None` serves plain HTTP
/// (the right default on issuer-less k3d).
pub cluster_issuer: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
pub struct ComposeAppScore {
pub namespace: String,
pub release_name: String,
pub app: ComposeApp,
pub deploy: DeployConfig,
/// `None` renders + installs the chart from the imported compose
/// (dev/e2e); `Some` installs the published OCI chart (CD).
pub published_chart: Option<PublishedChart>,
pub public_endpoint: Option<PublicEndpoint>,
}
impl<T: Topology + HelmCommand + K8sclient> Score<T> for ComposeAppScore {
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
Box::new(ComposeAppInterpret {
score: self.clone(),
})
}
fn name(&self) -> String {
format!("ComposeAppScore({})", self.deploy.app_name)
}
}
#[derive(Debug)]
struct ComposeAppInterpret {
score: ComposeAppScore,
}
#[async_trait]
impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for ComposeAppInterpret {
async fn execute(
&self,
inventory: &Inventory,
topology: &T,
) -> Result<Outcome, InterpretError> {
let s = &self.score;
// Install from the published OCI chart (CD) or a chart rendered
// from the imported compose (dev/e2e). Each branch keeps its
// tempdir alive across the install.
let helm_outcome = if let Some(p) = &s.published_chart {
let chart_ref = format!("oci://{}/{}/{}", p.registry, p.project, s.deploy.app_name);
info!(
"Installing '{}' from published chart {chart_ref}:{}",
s.release_name, p.version
);
self.install(inventory, topology, &chart_ref, &p.version)
.await?
} else {
let tmp = tempfile::tempdir()
.map_err(|e| InterpretError::new(format!("chart tempdir: {e}")))?;
let chart_path = build_chart(&s.app, &s.deploy, tmp.path())
.map_err(|e| InterpretError::new(format!("render chart: {e}")))?;
let chart_path = chart_path
.to_str()
.ok_or_else(|| InterpretError::new("chart path not utf-8".to_string()))?;
info!(
"Installing '{}' from rendered chart {chart_path}",
s.release_name
);
self.install(inventory, topology, chart_path, &s.deploy.chart_version)
.await?
};
// Public Ingress, applied after the chart so the backing Service
// exists. Skipped when no endpoint is configured.
if let Some(ep) = &s.public_endpoint {
let to_fqdn = |v: &str| {
fqdn::FQDN::from_str(v)
.map_err(|e| InterpretError::new(format!("invalid fqdn '{v}': {e}")))
};
let scheme = if ep.cluster_issuer.is_some() {
"https"
} else {
"http"
};
info!(
"Exposing service '{}' at {scheme}://{}",
ep.service, ep.host
);
K8sIngressScore {
name: to_fqdn(&s.deploy.app_name)?,
host: to_fqdn(&ep.host)?,
backend_service: to_fqdn(&ep.service)?,
port: ep.port,
path: None,
path_type: None,
namespace: Some(to_fqdn(&s.namespace)?),
ingress_class_name: None,
cluster_issuer: ep.cluster_issuer.clone(),
}
.interpret(inventory, topology)
.await?;
}
Ok(Outcome::success_with_details(
helm_outcome.message,
vec![format!("app: {}", s.deploy.app_name)],
))
}
fn get_name(&self) -> InterpretName {
InterpretName::Custom("ComposeAppInterpret")
}
fn get_version(&self) -> Version {
Version::from("0.1.0").expect("static version literal")
}
fn get_status(&self) -> InterpretStatus {
InterpretStatus::QUEUED
}
fn get_children(&self) -> Vec<Id> {
vec![]
}
}
impl ComposeAppInterpret {
async fn install<T: Topology + HelmCommand + K8sclient>(
&self,
inventory: &Inventory,
topology: &T,
chart_name: &str,
version: &str,
) -> Result<Outcome, InterpretError> {
let s = &self.score;
HelmChartScore {
namespace: Some(non_blank(&s.namespace, "namespace")?),
release_name: non_blank(&s.release_name, "release_name")?,
chart_name: non_blank(chart_name, "chart_name")?,
chart_version: Some(non_blank(version, "chart_version")?),
values_overrides: None,
values_yaml: None,
create_namespace: true,
install_only: false,
repository: None,
}
.interpret(inventory, topology)
.await
}
}
fn non_blank(value: &str, field: &str) -> Result<NonBlankString, InterpretError> {
NonBlankString::from_str(value)
.map_err(|e| InterpretError::new(format!("{field} must be non-blank: {e}")))
}

69
examples/compose_java_react/tests/helm_render.rs Normal file
View File

@@ -0,0 +1,69 @@
//! Render the example app's chart from its real `docker-compose.yml` and
//! validate it with actual `helm` — the chart our importer emits must be
//! something helm accepts and that renders to valid k8s. Skips when helm
//! isn't on PATH so CI without helm stays green.
use std::path::Path;
use std::process::Command;
use example_compose_java_react::{ComposeApp, DeployConfig, chart::build_chart};
fn helm_present() -> bool {
Command::new("helm")
.arg("version")
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
#[test]
fn generated_chart_passes_helm_lint_and_template() {
if !helm_present() {
eprintln!("helm not on PATH — skipping chart validation");
return;
}
let app_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("app");
let app = ComposeApp::from_dir(&app_dir).expect("import compose");
let cfg = DeployConfig {
app_name: "timesheet".to_string(),
chart_version: "0.1.0".to_string(),
registry: "hub.example".to_string(),
project: "harmony".to_string(),
version: "0.1.0".to_string(),
storage_class: Some("cephfs".to_string()),
volume_size: "1Gi".to_string(),
replicas: 2,
rolling: true,
};
let tmp = tempfile::tempdir().unwrap();
let chart = build_chart(&app, &cfg, tmp.path()).expect("render chart");
let chart = chart.to_str().unwrap();
let lint = Command::new("helm").args(["lint", chart]).output().unwrap();
assert!(
lint.status.success(),
"helm lint failed:\n{}",
String::from_utf8_lossy(&lint.stdout)
);
let out = Command::new("helm")
.args(["template", "timesheet", chart])
.output()
.unwrap();
assert!(
out.status.success(),
"helm template failed:\n{}",
String::from_utf8_lossy(&out.stderr)
);
let rendered = String::from_utf8_lossy(&out.stdout);
// Both services + the RWX PVC must survive a real render.
assert!(rendered.contains("kind: Deployment"), "{rendered}");
assert!(rendered.contains("kind: Service"), "{rendered}");
assert!(
rendered.contains("kind: PersistentVolumeClaim"),
"{rendered}"
);
assert!(rendered.contains("ReadWriteMany"), "{rendered}");
}

3
examples/fleet_auth_callout/src/lib.rs
View File

@@ -267,6 +267,7 @@ pub async fn bring_up_stack() -> Result<StackHandles> {
group: None,
},
],
groups_claim_action: false,
machine_users: vec![
ZitadelMachineUser {
username: ADMIN_USERNAME.to_string(),
@@ -759,7 +760,7 @@ pub async fn mint_access_token(
access_token: String,
}
let tr: TokenResponse = resp.json().await.context("parse token response")?;
if std::env::var("FLEET_AUTH_CALLOUT_DEBUG_TOKENS").is_ok()
if std::env::var("HARMONY_SSO_CALLOUT_DEBUG_TOKENS").is_ok()
&& let Some(payload_b64) = tr.access_token.split('.').nth(1)
{
use base64::Engine;

1
examples/fleet_device_enroll/src/main.rs
View File

@@ -302,6 +302,7 @@ async fn main() -> Result<()> {
auth,
agent_binary_path: agent_binary,
hosts_entries: vec![],
openbao: None,
};
let setup_score = FleetDeviceSetupScore::new(setup_config);

3
examples/fleet_e2e_demo/src/lib.rs
View File

@@ -185,6 +185,7 @@ pub async fn bring_up_full_stack(opts: E2eDemoOpts) -> Result<E2eHandles> {
group: None,
},
],
groups_claim_action: false,
machine_users: vec![
ZitadelMachineUser {
username: ADMIN_USERNAME.to_string(),
@@ -354,6 +355,7 @@ async fn provision_device(
applications: vec![],
api_apps: vec![],
roles: vec![],
groups_claim_action: false,
machine_users: vec![ZitadelMachineUser {
username: username.clone(),
name: format!("Fleet Device {device_id}"),
@@ -418,6 +420,7 @@ async fn provision_device(
ip: opts.libvirt_host_ip.clone(),
hostname: ZITADEL_HOST.to_string(),
}],
openbao: None,
});
// Apply the score over SSH against the VM. Same pattern as

2
examples/fleet_load_test/src/main.rs
View File

@@ -406,6 +406,7 @@ async fn apply_one_cr(
let cr = Deployment::new(
&group.cr_name,
DeploymentSpec {
allowed_groups: None,
target_selector: LabelSelector {
match_labels: Some(match_labels),
match_expressions: None,
@@ -420,6 +421,7 @@ async fn apply_one_cr(
image: "docker.io/library/nginx:alpine".to_string(),
ports: vec!["8080:80".to_string()],
env: vec![],
secret_env: vec![],
volumes: vec![],
restart_policy: Default::default(),
}],

1
examples/fleet_rpi_setup/src/main.rs
View File

@@ -168,6 +168,7 @@ async fn main() -> Result<()> {
auth,
agent_binary_path: cli.agent_binary.clone(),
hosts_entries: vec![],
openbao: None,
});
// We have our own clap CLI, so harmony_cli must NOT call

1
examples/fleet_staging_deploy/src/lib.rs
View File

@@ -296,6 +296,7 @@ async fn provision_zitadel_project(
// No machine users provisioned here — `fleet_rpi_setup` mints
// them on demand per device, so the staging deploy stays
// device-count-agnostic.
groups_claim_action: false,
machine_users: vec![],
};
setup

51
examples/fleet_staging_install/src/main.rs
View File

@@ -22,7 +22,8 @@ use harmony::modules::nats::capability::NatsCluster;
use harmony::modules::nats::score_nats_k8s::{AuthCalloutCfg, NatsK8sScore, WebSocketRouteCfg};
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
use harmony::modules::openbao::{
OpenbaoInstance, OpenbaoPolicy, OpenbaoScore, OpenbaoSetupScore, cached_root_token,
OpenbaoInstance, OpenbaoJwtAuth, OpenbaoPolicy, OpenbaoScore, OpenbaoSetupScore,
cached_root_token,
};
use harmony::modules::zitadel::{
MachineKeyType, ZitadelApiApp, ZitadelAppType, ZitadelApplication, ZitadelClientConfig,
@@ -33,7 +34,7 @@ use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology};
use harmony_config::{Config, ConfigClient, StoreSource};
use harmony_fleet_deploy::{FleetDeploySecrets, FleetOperatorScore, OperatorCredentials};
use harmony_k8s::KubernetesDistribution;
use harmony_secret::OpenbaoSecretStore;
use harmony_secret::{OpenbaoSecretStore, OpenbaoStoreOptions};
use nkeys::KeyPair;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@@ -129,6 +130,10 @@ async fn main() -> Result<()> {
endpoint: None,
admin_org_id: None,
namespace: cfg.zitadel_namespace.clone(),
// Devices read per-deployment secrets gated by group membership;
// the Action surfaces role keys as the `groups` claim OpenBao
// binds against (ADR-025).
groups_claim_action: true,
// Device-code OIDC app for human admin login from
// `fleet_device_enroll`'s SSO flow. The numeric `client_id` Zitadel
// generates is read back below and printed for `--admin-oidc-client-id`.
@@ -224,6 +229,21 @@ async fn main() -> Result<()> {
path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"#
.to_string(),
}],
// Device login (ADR-025): batch tokens so re-login is free, 1h
// TTL as the group-membership convergence bound, groups bound
// from the Action-flattened `groups` claim.
jwt_auth: Some(OpenbaoJwtAuth {
oidc_discovery_url: format!("https://{zitadel_host}"),
jwt_validation_pubkeys: String::new(),
bound_issuer: format!("https://{zitadel_host}"),
role_name: "fleet-device".to_string(),
bound_audiences: project_id.clone(),
user_claim: "sub".to_string(),
groups_claim: "groups".to_string(),
token_type: "batch".to_string(),
ttl: "1h".to_string(),
max_ttl: "1h".to_string(),
}),
..Default::default()
}
.interpret(&Inventory::empty(), &topology)
@@ -325,19 +345,20 @@ path "secret/metadata/harmony/*" { capabilities = ["list","read"] }"#
.await
.context("port-forward to OpenBao")?;
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
let store = OpenbaoSecretStore::new(
format!("http://127.0.0.1:{}", pf.port()),
"secret".to_string(),
"token".to_string(),
true,
Some(cached_root_token(&openbao).map_err(|e| anyhow::anyhow!(e))?),
None,
None,
None,
None,
None,
None,
)
let store = OpenbaoSecretStore::new(OpenbaoStoreOptions {
base_url: format!("http://127.0.0.1:{}", pf.port()),
kv_mount: "secret".to_string(),
auth_mount: "token".to_string(),
skip_tls: true,
token: Some(cached_root_token(&openbao).map_err(|e| anyhow::anyhow!(e))?),
username: None,
password: None,
zitadel_sso_url: None,
zitadel_client_id: None,
jwt_role: None,
jwt_auth_mount: None,
zitadel_jwt_bearer: None,
})
.await
.context("OpenBao client")?;
ConfigClient::new(vec![

1
examples/fleet_vm_setup/src/main.rs
View File

@@ -219,6 +219,7 @@ async fn main() -> Result<()> {
},
agent_binary_path: agent_binary,
hosts_entries: vec![],
openbao: None,
});
run_setup_score(&setup_score, &linux_topology).await?;

2
examples/harmony_apply_deployment/src/main.rs
View File

@@ -204,6 +204,7 @@ fn build_cr(cli: &Cli) -> Deployment {
image: cli.image.clone(),
ports: vec![cli.port.clone()],
env,
secret_env: vec![],
volumes,
restart_policy: cli.restart.into(),
}],
@@ -226,6 +227,7 @@ fn build_cr(cli: &Cli) -> Deployment {
Deployment::new(
&cli.name,
DeploymentSpec {
allowed_groups: None,
target_selector: LabelSelector {
match_labels: Some(match_labels),
match_expressions: None,

32
examples/harmony_sso/src/main.rs
View File

@@ -311,6 +311,7 @@ async fn main() -> anyhow::Result<()> {
}],
api_apps: vec![],
roles: vec![],
groups_claim_action: false,
machine_users: vec![],
}
.interpret(&Inventory::autoload(), &topology)
@@ -337,11 +338,13 @@ async fn main() -> anyhow::Result<()> {
}],
jwt_auth: Some(OpenbaoJwtAuth {
oidc_discovery_url: format!("http://{}:{}", ZITADEL_HOST, HTTP_PORT),
jwt_validation_pubkeys: String::new(),
bound_issuer: format!("http://{}:{}", ZITADEL_HOST, HTTP_PORT),
role_name: "harmony-developer".to_string(),
bound_audiences: client_id.clone(),
user_claim: "email".to_string(),
policies: vec!["harmony-dev".to_string()],
groups_claim: String::new(),
token_type: "service".to_string(),
ttl: "4h".to_string(),
max_ttl: "24h".to_string(),
}),
@@ -366,19 +369,20 @@ async fn main() -> anyhow::Result<()> {
let openbao_url = format!("http://127.0.0.1:{}", _pf.port());
let sso_url = format!("http://{}:{}", ZITADEL_HOST, HTTP_PORT);
let store = OpenbaoSecretStore::new(
openbao_url,
"secret".to_string(),
"jwt".to_string(),
true,
None,
None,
None,
Some(sso_url),
Some(client_id),
Some("harmony-developer".to_string()),
Some("jwt".to_string()),
)
let store = OpenbaoSecretStore::new(harmony_secret::OpenbaoStoreOptions {
base_url: openbao_url,
kv_mount: "secret".to_string(),
auth_mount: "jwt".to_string(),
skip_tls: true,
token: None,
username: None,
password: None,
zitadel_sso_url: Some(sso_url),
zitadel_client_id: Some(client_id),
jwt_role: Some("harmony-developer".to_string()),
jwt_auth_mount: Some("jwt".to_string()),
zitadel_jwt_bearer: None,
})
.await
.context("SSO authentication failed")?;

4
examples/openbao/src/main.rs
View File

@@ -101,11 +101,13 @@ async fn main() -> Result<()> {
let jwt_auth =
(!cfg.oidc_issuer.is_empty() && !cfg.oidc_audience.is_empty()).then(|| OpenbaoJwtAuth {
oidc_discovery_url: cfg.oidc_issuer.clone(),
jwt_validation_pubkeys: String::new(),
bound_issuer: cfg.oidc_issuer.clone(),
role_name: "harmony".to_string(),
bound_audiences: cfg.oidc_audience.clone(),
user_claim: "sub".to_string(),
policies: vec![HARMONY_CONFIG_POLICY.to_string()],
groups_claim: String::new(),
token_type: "service".to_string(),
ttl: "1h".to_string(),
max_ttl: "8h".to_string(),
});

1
fleet/harmony-fleet-agent/Cargo.toml
View File

@@ -6,6 +6,7 @@ rust-version = "1.85"
[dependencies]
harmony-fleet-auth = { path = "../harmony-fleet-auth" }
harmony_secret = { path = "../../harmony_secret" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
harmony = { path = "../../harmony", default-features = false, features = ["podman"] }
async-nats = { workspace = true }

28
fleet/harmony-fleet-agent/src/main.rs
View File

@@ -227,6 +227,33 @@ async fn main() -> Result<()> {
let creds = credential_source_from_config(&cfg.credentials)
.context("building NATS credential source")?;
// Deployment secret store (ADR-025). The OpenBao login reuses the
// NATS path's ZitadelJwtBearer — one machine identity, two
// consumers — so `[openbao]` without `zitadel-jwt` credentials is
// a configuration error, not a fallback.
let secrets = match (&cfg.openbao, creds.as_ref()) {
(Some(ob), CredentialSource::ZitadelJwt { bearer }) => {
let store = harmony_secret::OpenbaoSecretStore::with_zitadel_bearer(
&ob.url,
ob.danger_accept_invalid_certs,
&ob.kv_mount,
&ob.jwt_auth_mount,
&ob.jwt_role,
bearer.clone(),
)
.map_err(|e| anyhow::anyhow!("building OpenBao secret store: {e}"))?;
tracing::info!(url = %ob.url, prefix = %ob.secret_prefix, "OpenBao secret store ready");
Some(reconciler::SecretSource {
store: Arc::new(store),
prefix: ob.secret_prefix.clone(),
})
}
(Some(_), _) => {
anyhow::bail!("[openbao] requires [credentials] type = \"zitadel-jwt\"");
}
(None, _) => None,
};
let client = connect_nats(&cfg, creds).await.map_err(|e| {
let msg = format!("Nats connection FAILED : {e}");
tracing::error!(msg);
@@ -266,6 +293,7 @@ async fn main() -> Result<()> {
t.clone(),
inventory.clone(),
Some(fleet.clone()),
secrets,
))
});

193
fleet/harmony-fleet-agent/src/reconciler.rs
View File

@@ -8,11 +8,21 @@ use harmony_reconciler_contracts::{DeploymentName, DeploymentState, Id, Phase};
use tokio::sync::Mutex;
use harmony::inventory::Inventory;
use harmony::modules::podman::{PodmanTopology, PodmanV0Score, ReconcileScore};
use harmony::modules::podman::{EnvVar, PodmanTopology, PodmanV0Score, ReconcileScore};
use harmony::score::Score;
use harmony_secret::SecretStore;
use crate::fleet_publisher::FleetPublisher;
/// Where deployment secrets come from (ADR-025). `prefix` scopes the
/// KV reads to `<prefix>/<deployment>/<secret>` — it must match the
/// prefix the grant writer scopes `deployment-<name>` policies to,
/// or every fetch is denied.
pub struct SecretSource {
pub store: Arc<dyn SecretStore>,
pub prefix: String,
}
/// Cache key → last-seen state, populated by `apply` and consulted by the
/// 30-second periodic tick and the delete path.
struct CachedEntry {
@@ -40,6 +50,9 @@ pub struct Reconciler {
/// Publish surface. Optional so unit tests without a live NATS
/// client still work; always populated in the real agent runtime.
fleet: Option<Arc<FleetPublisher>>,
/// Deployment secret store. `None` = any score referencing
/// `secret_env` fails to apply (loudly, via `Phase::Failed`).
secrets: Option<SecretSource>,
}
impl Reconciler {
@@ -48,6 +61,7 @@ impl Reconciler {
topology: Arc<PodmanTopology>,
inventory: Arc<Inventory>,
fleet: Option<Arc<FleetPublisher>>,
secrets: Option<SecretSource>,
) -> Self {
Self {
device_id,
@@ -56,9 +70,50 @@ impl Reconciler {
state: Mutex::new(HashMap::new()),
deployments: Mutex::new(HashMap::new()),
fleet,
secrets,
}
}
/// Materialize every `secret_env` reference into a plain env var.
/// Resolution happens once per spec change, never on the periodic
/// tick: secret values bind at apply time (rotation = spec bump or
/// re-apply), which keeps the secret store off the 30s hot path —
/// fleet-scale login load is the ADR-025 constraint.
async fn resolve_secrets(
&self,
deployment: Option<&DeploymentName>,
mut score: PodmanV0Score,
) -> Result<PodmanV0Score> {
for service in &mut score.services {
if service.secret_env.is_empty() {
continue;
}
let src = self.secrets.as_ref().ok_or_else(|| {
anyhow::anyhow!(
"service '{}' references secrets but this device has no secret store configured",
service.name
)
})?;
let dep = deployment
.ok_or_else(|| anyhow::anyhow!("secret_env requires a deployment-scoped key"))?;
let namespace = format!("{}/{}", src.prefix, dep);
for sref in &service.secret_env {
let bytes = src
.store
.get_raw(&namespace, &sref.secret)
.await
.map_err(|e| {
anyhow::anyhow!("fetching secret '{}/{}': {e}", namespace, sref.secret)
})?;
let value = String::from_utf8(bytes).map_err(|_| {
anyhow::anyhow!("secret '{}/{}' is not valid UTF-8", namespace, sref.secret)
})?;
service.env.push(EnvVar::new(&sref.name, &value));
}
}
Ok(score)
}
/// Record a new phase for a deployment and, if it changed, write
/// the updated [`DeploymentState`] to the KV. Same-phase
/// re-confirmations are no-ops so the periodic reconcile tick
@@ -139,7 +194,18 @@ impl Reconciler {
self.apply_phase(name, Phase::Pending, None).await;
}
match self.run_score(key, &incoming).await {
let resolved = match self.resolve_secrets(deployment.as_ref(), incoming).await {
Ok(score) => score,
Err(e) => {
if let Some(name) = &deployment {
self.apply_phase(name, Phase::Failed, Some(short(&e.to_string())))
.await;
}
return Err(e);
}
};
match self.run_score(key, &resolved).await {
Ok(()) => {
if let Some(name) = &deployment {
self.apply_phase(name, Phase::Running, None).await;
@@ -155,11 +221,14 @@ impl Reconciler {
}
let mut state = self.state.lock().await;
// Cache the *resolved* score: the periodic tick re-runs it
// as-is, so converged deployments never re-hit the secret
// store (see `resolve_secrets`).
state.insert(
key.to_string(),
CachedEntry {
serialized,
score: incoming,
score: resolved,
},
);
Ok(())
@@ -286,6 +355,10 @@ mod tests {
use std::path::PathBuf;
fn reconciler() -> Reconciler {
reconciler_with_secrets(None)
}
fn reconciler_with_secrets(secrets: Option<SecretSource>) -> Reconciler {
let topology = Arc::new(
PodmanTopology::from_unix_socket(PathBuf::from("/nonexistent/for-tests")).unwrap(),
);
@@ -295,6 +368,7 @@ mod tests {
topology,
inventory,
None,
secrets,
)
}
@@ -302,6 +376,119 @@ mod tests {
DeploymentName::try_new(s).expect("valid test name")
}
/// In-memory secret store keyed by `(namespace, key)`.
#[derive(Debug, Default)]
struct MapStore(HashMap<(String, String), String>);
#[async_trait::async_trait]
impl SecretStore for MapStore {
async fn get_raw(
&self,
namespace: &str,
key: &str,
) -> Result<Vec<u8>, harmony_secret::SecretStoreError> {
self.0
.get(&(namespace.to_string(), key.to_string()))
.map(|v| v.as_bytes().to_vec())
.ok_or_else(|| harmony_secret::SecretStoreError::Store("permission denied".into()))
}
async fn set_raw(
&self,
_namespace: &str,
_key: &str,
_value: &[u8],
) -> Result<(), harmony_secret::SecretStoreError> {
unimplemented!("tests never write")
}
}
fn secret_score(env_name: &str, secret: &str) -> PodmanV0Score {
use harmony::modules::podman::{PodmanService, SecretEnvVar};
PodmanV0Score {
services: vec![PodmanService {
name: "web-svc".to_string(),
image: "nginx".to_string(),
ports: vec![],
env: vec![],
secret_env: vec![SecretEnvVar {
name: env_name.to_string(),
secret: secret.to_string(),
}],
volumes: vec![],
restart_policy: Default::default(),
}],
}
}
fn map_source(entries: &[(&str, &str, &str)]) -> SecretSource {
let map = entries
.iter()
.map(|(ns, k, v)| ((ns.to_string(), k.to_string()), v.to_string()))
.collect();
SecretSource {
store: Arc::new(MapStore(map)),
prefix: "fleet-test".to_string(),
}
}
#[tokio::test]
async fn resolve_secrets_injects_env_value() {
let r = reconciler_with_secrets(Some(map_source(&[(
"fleet-test/web",
"db_password",
"hunter2",
)])));
let resolved = r
.resolve_secrets(Some(&dn("web")), secret_score("DB_PASSWORD", "db_password"))
.await
.expect("resolution succeeds");
let env = &resolved.services[0].env;
assert_eq!(env.len(), 1);
assert_eq!(env[0], EnvVar::new("DB_PASSWORD", "hunter2"));
}
#[tokio::test]
async fn resolve_secrets_without_store_fails() {
let r = reconciler();
let err = r
.resolve_secrets(Some(&dn("web")), secret_score("DB_PASSWORD", "db_password"))
.await
.expect_err("must fail without a store");
assert!(err.to_string().contains("no secret store"), "{err}");
}
#[tokio::test]
async fn apply_marks_failed_when_secret_denied() {
// Store has no entry → MapStore answers "permission denied",
// the same observable a revoked OpenBao policy produces. The
// failure must surface as Phase::Failed *before* any podman
// call (the test topology has no socket).
let r = reconciler_with_secrets(Some(map_source(&[])));
let payload = serde_json::to_vec(&ReconcileScore::PodmanV0(secret_score(
"DB_PASSWORD",
"db_password",
)))
.unwrap();
let err = r
.apply("test-device.web", &payload)
.await
.expect_err("denied fetch fails the apply");
assert!(err.to_string().contains("permission denied"), "{err}");
let phases = r.deployments.lock().await;
assert_eq!(phases.get(&dn("web")), Some(&Phase::Failed));
}
#[tokio::test]
async fn plain_scores_resolve_without_store() {
// No secret_env → no store needed; resolution is a no-op.
let r = reconciler();
let score = PodmanV0Score { services: vec![] };
r.resolve_secrets(Some(&dn("web")), score.clone())
.await
.expect("no-op resolution");
}
#[tokio::test]
async fn apply_phase_records_new_phase() {
let r = reconciler();

1
fleet/harmony-fleet-auth/Cargo.toml
View File

@@ -11,6 +11,7 @@ path = "src/lib.rs"
[dependencies]
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
harmony_zitadel_auth = { path = "../../harmony_zitadel_auth" }
async-nats = { workspace = true }
anyhow = { workspace = true }
chrono = { workspace = true }

26
fleet/harmony-fleet-auth/examples/mint_zitadel_token.rs Normal file
View File

@@ -0,0 +1,26 @@
use anyhow::{Context, Result};
use harmony_fleet_auth::{CredentialSource, CredentialsSection, NatsCredential};
#[tokio::main]
async fn main() -> Result<()> {
let key_path = std::env::var("ZITADEL_KEY_PATH").context("ZITADEL_KEY_PATH missing")?;
let issuer = std::env::var("ZITADEL_ISSUER").context("ZITADEL_ISSUER missing")?;
let audience = std::env::var("ZITADEL_PROJECT_ID").context("ZITADEL_PROJECT_ID missing")?;
let source =
CredentialSource::credential_source_from_config(&CredentialsSection::ZitadelJwt {
key_path: key_path.into(),
key_json: None,
oidc_issuer_url: issuer,
audience,
danger_accept_invalid_certs: false,
})?;
match source.next_credential().await? {
NatsCredential::BearerToken(token) => {
println!("{token}");
Ok(())
}
NatsCredential::UserPass { .. } => anyhow::bail!("expected bearer token"),
}
}

86
fleet/harmony-fleet-auth/src/agent_config.rs
View File

@@ -25,6 +25,42 @@ pub struct AgentConfig {
/// auto-published `device-id` label.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub labels: BTreeMap<String, String>,
/// Secret store the device reads deployment secrets from
/// (ADR-025). Absent = deployments referencing `secret_env` fail
/// to apply. Requires `zitadel-jwt` credentials — the same
/// machine identity logs into OpenBao's JWT mount.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub openbao: Option<OpenbaoSection>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct OpenbaoSection {
pub url: String,
/// KV prefix deployment subtrees live under
/// (`<kv_mount>/<secret_prefix>/<deployment>/<secret>`). Must
/// match the prefix the grant writer scopes policies to — no
/// default on purpose.
pub secret_prefix: String,
#[serde(default = "default_jwt_auth_mount")]
pub jwt_auth_mount: String,
#[serde(default = "default_jwt_role")]
pub jwt_role: String,
#[serde(default = "default_kv_mount")]
pub kv_mount: String,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub danger_accept_invalid_certs: bool,
}
fn default_jwt_auth_mount() -> String {
"jwt".to_string()
}
fn default_jwt_role() -> String {
"fleet-device".to_string()
}
fn default_kv_mount() -> String {
"secret".to_string()
}
#[derive(Debug, Clone, Deserialize, Serialize)]
@@ -93,6 +129,54 @@ arch = "aarch64"
}
}
#[test]
fn openbao_section_parses_with_defaults() {
let raw = r#"
[agent]
device_id = "pi-42"
[credentials]
type = "toml-shared"
nats_user = "u"
nats_pass = "p"
[nats]
urls = ["nats://nats:4222"]
[openbao]
url = "http://10.0.0.1:30425"
secret_prefix = "fleet-e2e"
"#;
let cfg: AgentConfig = toml::from_str(raw).expect("valid config");
let ob = cfg.openbao.expect("openbao section");
assert_eq!(ob.url, "http://10.0.0.1:30425");
assert_eq!(ob.secret_prefix, "fleet-e2e");
assert_eq!(ob.jwt_auth_mount, "jwt");
assert_eq!(ob.jwt_role, "fleet-device");
assert_eq!(ob.kv_mount, "secret");
assert!(!ob.danger_accept_invalid_certs);
}
#[test]
fn openbao_section_absent_is_none_and_omitted_on_serialize() {
let raw = r#"
[agent]
device_id = "pi-42"
[credentials]
type = "toml-shared"
nats_user = "u"
nats_pass = "p"
[nats]
urls = ["nats://nats:4222"]
"#;
let cfg: AgentConfig = toml::from_str(raw).expect("valid config");
assert!(cfg.openbao.is_none());
let rendered = toml::to_string(&cfg).expect("serialize");
assert!(!rendered.contains("[openbao]"), "got:\n{rendered}");
}
#[test]
fn labels_section_optional_defaults_empty() {
let raw = r#"
@@ -175,6 +259,7 @@ urls = ["nats://nats:4222"]
("group".to_string(), "site\"a".to_string()),
("arch".to_string(), "aarch64".to_string()),
]),
openbao: None,
};
let rendered = toml::to_string(&original).expect("serialize");
let parsed: AgentConfig = toml::from_str(&rendered).expect("deserialize");
@@ -215,6 +300,7 @@ urls = ["nats://nats:4222"]
nats_pass: "p".to_string(),
},
labels: BTreeMap::new(),
openbao: None,
};
let rendered = toml::to_string(&cfg).expect("serialize");
assert!(!rendered.contains("[labels]"), "got:\n{rendered}");

512
fleet/harmony-fleet-auth/src/credentials.rs
View File

@@ -23,14 +23,13 @@
//! cleaner than a Trait + factory.
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::sync::Arc;
use anyhow::{Context, Result};
use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader};
use serde::Deserialize;
use crate::config::CredentialsSection;
use harmony_zitadel_auth::{MachineKeyFile, ZitadelJwtBearer};
use crate::CredentialsSection;
/// Material the NATS connector needs to authenticate. Returned per
/// (re)connect attempt — the source decides whether to mint fresh.
@@ -44,17 +43,8 @@ pub enum NatsCredential {
/// from the parsed `[credentials]` section; cloned via Arc into the
/// async-nats auth callback.
pub enum CredentialSource {
TomlShared {
user: String,
pass: String,
},
ZitadelJwt {
key: MachineKeyFile,
oidc_issuer_url: String,
audience: String,
http: reqwest::Client,
cache: Mutex<Option<CachedToken>>,
},
TomlShared { user: String, pass: String },
ZitadelJwt { bearer: Arc<ZitadelJwtBearer> },
}
impl CredentialSource {
@@ -72,256 +62,58 @@ impl CredentialSource {
}
async fn zitadel_next(&self) -> Result<NatsCredential> {
// Fast path: lock the cache synchronously, copy out the token if
// it's comfortably valid, drop the lock. Holding a MutexGuard
// across `.await` would make this future !Sync, which
// async-nats's `with_auth_callback` rejects at compile time.
if let Some(token) = self.cached_if_fresh() {
return Ok(NatsCredential::BearerToken(token));
}
// Slow path: mint outside any lock. Two concurrent (re)connect
// attempts could both reach here and both mint; that's a wasted
// HTTP round-trip in a rare race, not a correctness issue —
// the second writer wins and replaces the first's value.
let fresh = self.zitadel_mint().await?;
let token = fresh.access_token.clone();
if let Self::ZitadelJwt {
cache, audience, ..
} = self
&& let Ok(mut guard) = cache.lock()
{
*guard = Some(fresh);
tracing::info!(audience = %audience, "minted fresh Zitadel access token");
}
let Self::ZitadelJwt { bearer } = self else {
anyhow::bail!("zitadel_next called on non-ZitadelJwt variant");
};
let token = bearer.bearer_token().await?;
Ok(NatsCredential::BearerToken(token))
}
fn cached_if_fresh(&self) -> Option<String> {
let Self::ZitadelJwt { cache, .. } = self else {
return None;
};
let now = chrono::Utc::now().timestamp();
let guard = cache.lock().ok()?;
let cached = guard.as_ref()?;
if cached.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS > now {
Some(cached.access_token.clone())
} else {
None
}
}
async fn zitadel_mint(&self) -> Result<CachedToken> {
let Self::ZitadelJwt {
key,
oidc_issuer_url,
audience,
http,
..
} = self
else {
anyhow::bail!("zitadel_mint called on non-ZitadelJwt variant");
};
let now = chrono::Utc::now().timestamp();
let assertion = build_assertion(key, oidc_issuer_url, now)?;
let scope = build_scope(audience);
let token_url = build_token_url(oidc_issuer_url);
let resp = http
.post(&token_url)
.form(&[
(
"grant_type",
"urn:ietf:params:oauth:grant-type:jwt-bearer".to_string(),
),
("assertion", assertion),
("scope", scope),
])
.send()
.await
.with_context(|| format!("POST {token_url}"))?;
if !resp.status().is_success() {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
anyhow::bail!("Zitadel token endpoint returned {status}: {body}");
}
#[derive(Deserialize)]
struct TokenResponse {
access_token: String,
#[serde(default)]
expires_in: Option<i64>,
}
let tr: TokenResponse = resp.json().await.context("parsing token response")?;
// Zitadel typically returns 12h (43200s); be defensive against
// a missing field by assuming a conservative 1h.
let expires_in = tr.expires_in.unwrap_or(3600);
Ok(CachedToken {
access_token: tr.access_token,
expires_at_unix: now + expires_in,
})
}
}
/// Build the JWT-bearer assertion. Split out from the network path so
/// the claims + header shape can be unit-tested without an HTTP server,
/// and split internally into the (pure) claim/header builders so they
/// can be unit-tested without an RSA private key fixture.
pub(crate) fn build_assertion(
key: &MachineKeyFile,
oidc_issuer_url: &str,
now: i64,
) -> Result<String> {
let claims = build_assertion_claims(key, oidc_issuer_url, now);
let header = build_assertion_header(key);
let assertion = jsonwebtoken::encode(
&header,
&claims,
&EncodingKey::from_rsa_pem(key.key.as_bytes())
.context("parsing RSA private key from machine key file")?,
)
.context("signing JWT assertion")?;
Ok(assertion)
}
/// Pure claim payload for the JWT-bearer assertion. `iss == sub == userId`
/// is a Zitadel requirement; `aud` is Zitadel itself (the token endpoint
/// is reached via `oidc_issuer_url`); `exp - iat` MUST be ≤ 60 s or
/// Zitadel rejects.
pub(crate) fn build_assertion_claims(
key: &MachineKeyFile,
oidc_issuer_url: &str,
now: i64,
) -> serde_json::Value {
serde_json::json!({
"iss": key.user_id,
"sub": key.user_id,
"aud": oidc_issuer_url,
"exp": now + ASSERTION_LIFETIME_SECS,
"iat": now,
})
}
/// JWT header for the assertion. The `kid` tells Zitadel which of the
/// machine user's registered keys to verify the signature against.
pub(crate) fn build_assertion_header(key: &MachineKeyFile) -> JwtHeader {
let mut header = JwtHeader::new(Algorithm::RS256);
header.kid = Some(key.key_id.clone());
header
}
/// Build the OAuth `scope` string for the token-bearer request.
///
/// Three scopes are needed for the access token to be useful here:
///
/// * `openid` — base OIDC requirement.
/// * `urn:zitadel:iam:org:projects:roles` (PLURAL "projects") —
/// tells Zitadel to include the role-claim block in the access
/// token. Without this, the callout sees "no authorized role
/// in token" even when the user has a project role grant.
/// * `urn:zitadel:iam:org:project:id:<aud>:aud` (SINGULAR
/// "project") — adds <aud> to the access token's `aud` claim
/// so the callout's audience validation accepts the project
/// ID we're using as the JWT-bearer audience.
///
/// The plural-vs-singular distinction is a Zitadel convention,
/// not a typo. Both scopes are required.
pub(crate) fn build_scope(audience: &str) -> String {
format!(
"openid \
urn:zitadel:iam:org:projects:roles \
urn:zitadel:iam:org:project:id:{audience}:aud"
)
}
/// Resolve the token endpoint URL, tolerating a trailing slash on
/// `oidc_issuer_url`. Without trimming, a configured issuer of
/// `https://sso.example.com/` produces `…//oauth/v2/token` which 404s.
pub(crate) fn build_token_url(oidc_issuer_url: &str) -> String {
format!("{}/oauth/v2/token", oidc_issuer_url.trim_end_matches('/'))
}
// ---- helper types ----------------------------------------------------------
/// JSON keyfile content as Zitadel emits it for a `KEY_TYPE_JSON`
/// machine key. The `key` is a PEM-encoded RSA private key.
#[derive(Debug, Clone, Deserialize)]
pub struct MachineKeyFile {
#[serde(rename = "type")]
pub _type: String,
#[serde(rename = "keyId")]
pub key_id: String,
pub key: String,
#[serde(rename = "userId")]
pub user_id: String,
}
#[derive(Debug, Clone)]
pub struct CachedToken {
pub(crate) access_token: String,
/// Unix seconds at which the token is no longer trusted by
/// `cached_if_fresh`. Computed from the OAuth response's `expires_in`
/// and the local clock at mint time.
pub(crate) expires_at_unix: i64,
}
/// Refresh tokens this many seconds before their advertised expiry.
/// Five minutes leaves headroom for clock skew, slow networks, and
/// the round-trip cost of re-minting against Zitadel.
pub const TOKEN_REFRESH_LEEWAY_SECS: i64 = 5 * 60;
/// Lifetime of the JWT *assertion* (the client-side bearer JWT we sign
/// to authenticate to Zitadel's token endpoint). Zitadel rejects
/// assertions with `exp - iat > 60s`; one minute is the safe ceiling.
pub const ASSERTION_LIFETIME_SECS: i64 = 60;
// ---- factory ---------------------------------------------------------------
/// Build the appropriate `CredentialSource` from the parsed config.
///
/// For [`CredentialsSection::ZitadelJwt`] this reads the keyfile from
/// disk. Both the agent and the operator mount their key as a file
/// (Secret volume in the operator's Pod, dropped by
/// `FleetDeviceSetupScore` on the agent's VM); the path is just
/// configured differently.
pub fn credential_source_from_config(creds: &CredentialsSection) -> Result<Arc<CredentialSource>> {
match creds {
CredentialsSection::TomlShared {
nats_user,
nats_pass,
} => Ok(Arc::new(CredentialSource::TomlShared {
user: nats_user.clone(),
pass: nats_pass.clone(),
})),
CredentialsSection::ZitadelJwt {
key_path,
key_json,
oidc_issuer_url,
audience,
danger_accept_invalid_certs,
} => {
// `key_json` (inline) wins over `key_path` (file). The
// operator pod uses inline because OKD's restricted-v2
// SCC + env-var-from-Secret deployment shape can't
// reliably mount Secret volumes; the agent uses the file
// path because it lives on a VM and a real file is the
// more natural rotation target.
let key = match key_json.as_deref().map(str::trim) {
Some(json) if !json.is_empty() => parse_machine_key(json)?,
_ => load_machine_key(key_path)?,
};
Ok(Arc::new(CredentialSource::ZitadelJwt {
key,
oidc_issuer_url: oidc_issuer_url.clone(),
audience: audience.clone(),
http: reqwest::Client::builder()
.danger_accept_invalid_certs(*danger_accept_invalid_certs)
.timeout(Duration::from_secs(10))
.build()
.context("building HTTP client for Zitadel token endpoint")?,
cache: Mutex::new(None),
}))
/// Build the appropriate `CredentialSource` from the parsed config.
///
/// For [`CredentialsSection::ZitadelJwt`] this reads the keyfile from
/// disk. Both the agent and the operator mount their key as a file
/// (Secret volume in the operator's Pod, dropped by
/// `FleetDeviceSetupScore` on the agent's VM); the path is just
/// configured differently.
pub fn credential_source_from_config(
creds: &CredentialsSection,
) -> Result<Arc<CredentialSource>> {
match creds {
CredentialsSection::TomlShared {
nats_user,
nats_pass,
} => Ok(Arc::new(CredentialSource::TomlShared {
user: nats_user.clone(),
pass: nats_pass.clone(),
})),
CredentialsSection::ZitadelJwt {
key_path,
key_json,
oidc_issuer_url,
audience,
danger_accept_invalid_certs,
} => {
// `key_json` (inline) wins over `key_path` (file). The
// operator pod uses inline because OKD's restricted-v2
// SCC + env-var-from-Secret deployment shape can't
// reliably mount Secret volumes; the agent uses the file
// path because it lives on a VM and a real file is the
// more natural rotation target.
let key = match key_json.as_deref().map(str::trim) {
Some(json) if !json.is_empty() => parse_machine_key(json)?,
_ => load_machine_key(key_path)?,
};
let bearer = ZitadelJwtBearer::new(
key,
oidc_issuer_url.clone(),
audience.clone(),
*danger_accept_invalid_certs,
)?;
Ok(Arc::new(CredentialSource::ZitadelJwt {
bearer: Arc::new(bearer),
}))
}
}
}
}
@@ -341,30 +133,6 @@ fn parse_machine_key(raw: &str) -> Result<MachineKeyFile> {
mod tests {
use super::*;
fn fake_key() -> MachineKeyFile {
MachineKeyFile {
_type: "serviceaccount".to_string(),
key_id: "kid-371358469099356247".to_string(),
// Real PEM not required for the pure-builder tests; the
// signing path that needs a parseable key is exercised
// end-to-end in the e2e harness.
key: "PEM-PLACEHOLDER".to_string(),
user_id: "uid-371358469065801815".to_string(),
}
}
fn zjwt_source() -> CredentialSource {
CredentialSource::ZitadelJwt {
key: fake_key(),
oidc_issuer_url: "http://sso.fleet.local:8080".to_string(),
audience: "366378028009259037".to_string(),
http: reqwest::Client::new(),
cache: Mutex::new(None),
}
}
// ---- next_credential / cache state -------------------------------------
#[tokio::test]
async fn toml_shared_returns_userpass_each_call() {
let s = CredentialSource::TomlShared {
@@ -380,174 +148,4 @@ mod tests {
other => panic!("expected UserPass, got {other:?}"),
}
}
#[test]
fn cached_token_within_leeway_is_treated_as_expired() {
// Sanity-check the comparison so refactors don't accidentally
// invert the leeway window.
let now = chrono::Utc::now().timestamp();
let about_to_expire = CachedToken {
access_token: "x".to_string(),
expires_at_unix: now + TOKEN_REFRESH_LEEWAY_SECS - 1,
};
assert!(
about_to_expire.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS <= now,
"tokens within the leeway window must be considered expired"
);
let comfortable = CachedToken {
access_token: "x".to_string(),
expires_at_unix: now + TOKEN_REFRESH_LEEWAY_SECS + 60,
};
assert!(
comfortable.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS > now,
"tokens with comfortable headroom must be cache-hits"
);
}
#[test]
fn cached_if_fresh_returns_some_when_outside_leeway() {
let src = zjwt_source();
let now = chrono::Utc::now().timestamp();
if let CredentialSource::ZitadelJwt { cache, .. } = &src {
*cache.lock().unwrap() = Some(CachedToken {
access_token: "fresh".to_string(),
expires_at_unix: now + TOKEN_REFRESH_LEEWAY_SECS + 60,
});
}
assert_eq!(src.cached_if_fresh(), Some("fresh".to_string()));
}
#[test]
fn cached_if_fresh_returns_none_when_no_cache() {
// Brand-new ZitadelJwt source — no token has been minted yet.
// Forces the slow path on first connect.
let src = zjwt_source();
assert_eq!(src.cached_if_fresh(), None);
}
#[test]
fn cached_if_fresh_returns_none_for_toml_shared() {
// Defensive: cache_if_fresh is only meaningful for ZitadelJwt;
// TomlShared has no cache. A nonsensical call must return None,
// not panic, so the cold-path can degrade gracefully.
let src = CredentialSource::TomlShared {
user: "u".into(),
pass: "p".into(),
};
assert_eq!(src.cached_if_fresh(), None);
}
// ---- assertion claims / header (pure builders) ------------------------
#[test]
fn assertion_claims_carry_iss_sub_aud_exp_iat() {
let now = 1_700_000_000;
let claims = build_assertion_claims(&fake_key(), "http://sso.fleet.local:8080", now);
assert_eq!(claims["iss"], "uid-371358469065801815");
assert_eq!(claims["sub"], "uid-371358469065801815");
assert_eq!(claims["aud"], "http://sso.fleet.local:8080");
assert_eq!(claims["iat"].as_i64(), Some(now));
assert_eq!(claims["exp"].as_i64(), Some(now + ASSERTION_LIFETIME_SECS));
}
#[test]
fn assertion_lifetime_locked_at_60_seconds() {
// Zitadel rejects assertions where exp - iat > 60s. If anyone
// bumps ASSERTION_LIFETIME_SECS thinking "more is safer", the
// mints will silently start failing in prod with no helpful
// error. Lock the constant.
assert_eq!(ASSERTION_LIFETIME_SECS, 60);
}
#[test]
fn assertion_header_carries_kid_and_rs256() {
let header = build_assertion_header(&fake_key());
assert_eq!(header.alg, jsonwebtoken::Algorithm::RS256);
assert_eq!(header.kid.as_deref(), Some("kid-371358469099356247"));
}
// ---- scope string ------------------------------------------------------
#[test]
fn scope_includes_plural_projects_roles() {
// The plural-projects URN is what tells Zitadel to emit the
// role claim. Day-one bug; lock it.
let s = build_scope("366378028009259037");
assert!(
s.contains("urn:zitadel:iam:org:projects:roles"),
"scope must include the PLURAL projects-roles URN; got {s:?}"
);
}
#[test]
fn scope_audience_uses_singular_project_id_urn() {
// The singular-project URN tells Zitadel to put <id> into the
// access token's aud claim. Different URN entirely from the
// plural one above; both required.
let s = build_scope("366378028009259037");
assert!(
s.contains("urn:zitadel:iam:org:project:id:366378028009259037:aud"),
"scope must include the SINGULAR project:id:<aud>:aud URN; got {s:?}"
);
}
#[test]
fn scope_includes_openid_base() {
let s = build_scope("any");
assert!(
s.split_whitespace().any(|tok| tok == "openid"),
"scope must include `openid` as a standalone token; got {s:?}"
);
}
// ---- token URL ---------------------------------------------------------
#[test]
fn token_url_appends_oauth_endpoint() {
assert_eq!(
build_token_url("http://sso.fleet.local:8080"),
"http://sso.fleet.local:8080/oauth/v2/token"
);
}
#[test]
fn token_url_strips_single_trailing_slash() {
// A trailing slash would yield `…//oauth/v2/token`, which 404s.
// Common configuration drift; the trim guards against it.
assert_eq!(
build_token_url("http://sso.fleet.local:8080/"),
"http://sso.fleet.local:8080/oauth/v2/token"
);
}
#[test]
fn token_url_strips_multiple_trailing_slashes() {
// Defensive — `trim_end_matches('/')` peels all of them, not
// just the first. Locks that semantics.
assert_eq!(
build_token_url("http://sso.fleet.local:8080///"),
"http://sso.fleet.local:8080/oauth/v2/token"
);
}
// ---- MachineKeyFile JSON parsing --------------------------------------
#[test]
fn machine_key_file_parses_zitadel_json_shape() {
// The serde renames (`type`, `keyId`, `userId`) are easy to
// break. This is the literal JSON shape Zitadel's
// /management/v1/users/.../keys endpoint emits.
let raw = r#"{
"type": "serviceaccount",
"keyId": "371358469099356247",
"key": "-----BEGIN RSA PRIVATE KEY-----\nABC\n-----END RSA PRIVATE KEY-----\n",
"userId": "371358469065801815"
}"#;
let parsed: MachineKeyFile = serde_json::from_str(raw).expect("valid keyfile");
assert_eq!(parsed._type, "serviceaccount");
assert_eq!(parsed.key_id, "371358469099356247");
assert_eq!(parsed.user_id, "371358469065801815");
assert!(parsed.key.contains("BEGIN RSA PRIVATE KEY"));
}
}

12
fleet/harmony-fleet-auth/src/lib.rs
View File

@@ -27,13 +27,17 @@ mod credentials;
pub use agent_config::{AgentConfig, AgentSection, NatsSection, load_config};
pub use config::{CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR};
pub use credentials::{
ASSERTION_LIFETIME_SECS, CachedToken, CredentialSource, MachineKeyFile, NatsCredential,
TOKEN_REFRESH_LEEWAY_SECS, credential_source_from_config,
};
use std::sync::Arc;
pub use crate::credentials::{CredentialSource, NatsCredential};
pub fn credential_source_from_config(
creds: &CredentialsSection,
) -> anyhow::Result<Arc<CredentialSource>> {
CredentialSource::credential_source_from_config(creds)
}
/// Build `async_nats::ConnectOptions` wired with the auth callback
/// that pulls fresh credentials from `creds` on every (re)connect.
///

1
fleet/harmony-fleet-deploy/src/agent.rs
View File

@@ -82,6 +82,7 @@ impl PodTarget {
nats_pass: nats_pass.into(),
},
labels: BTreeMap::new(),
openbao: None,
},
image: image.into(),
image_pull_policy: "IfNotPresent".to_string(),

1
fleet/harmony-fleet-deploy/src/companion/agent.rs
View File

@@ -121,6 +121,7 @@ mod tests {
nats_pass: "p".to_string(),
},
labels: BTreeMap::new(),
openbao: None,
}
}

12
fleet/harmony-fleet-deploy/src/operator/chart.rs
View File

@@ -457,6 +457,18 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
});
env.push(secret_env(ENV_WEB_AUTH_CONFIG));
env.push(secret_env(ENV_WEB_COOKIE_KEY));
// Secret-grant sync (OpenBao) + the device-group scheduling gate
// (Zitadel role grants) — ADR-025. All optional: absent, the
// operator logs and runs ungated/without grant sync.
for name in [
"OPENBAO_URL",
"OPENBAO_TOKEN",
"ZITADEL_URL",
"ZITADEL_PAT",
"ZITADEL_PROJECT_ID",
] {
env.push(secret_env(name));
}
// Namespace deliberately omitted — same rationale as the
// ServiceAccount: helm fills in the release namespace at install

16
fleet/harmony-fleet-e2e/Cargo.toml
View File

@@ -18,6 +18,14 @@ path = "tests/ping.rs"
name = "operator"
path = "tests/operator.rs"
[[test]]
name = "openbao_policy"
path = "tests/openbao_policy.rs"
[[test]]
name = "openbao_groups"
path = "tests/openbao_groups.rs"
[[test]]
name = "vm_ping"
path = "tests/vm_ping.rs"
@@ -30,6 +38,10 @@ path = "tests/vm_isolation.rs"
name = "vm_deploy_lifecycle"
path = "tests/vm_deploy_lifecycle.rs"
[[test]]
name = "vm_secret_access"
path = "tests/vm_secret_access.rs"
[dependencies]
# `kvm` for the VM-side harness (ProvisionVmScore + KvmVirtualMachineHost),
# `podman` for `ReconcileScore`/`PodmanV0Score` the kv_admin helpers
@@ -39,6 +51,7 @@ harmony-fleet-auth = { path = "../harmony-fleet-auth" }
harmony-fleet-deploy = { path = "../harmony-fleet-deploy" }
harmony-fleet-operator = { path = "../harmony-fleet-operator" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
harmony-k8s = { path = "../../harmony-k8s" }
harmony_types = { path = "../../harmony_types" }
k3d-rs = { path = "../../k3d" }
@@ -57,6 +70,9 @@ tracing = { workspace = true }
tracing-subscriber = { workspace = true }
directories = "6.0.0"
uuid = { version = "1", features = ["v4"] }
jsonwebtoken = "9"
nkeys = "0.4"
reqwest = { workspace = true }
[dev-dependencies]
tokio = { workspace = true, features = ["full"] }

569
fleet/harmony-fleet-e2e/src/callout.rs Normal file
View File

@@ -0,0 +1,569 @@
//! Callout-mode auth fabric: Zitadel + NATS auth-callout + OpenBao,
//! composed from the same Scores `examples/fleet_staging_install`
//! deploys to staging. The e2e specifics are network plumbing only:
//!
//! - **Issuer URL** `http://sso.fleet-e2e.local:30424` is valid from
//! every vantage point through a single NodePort Service
//! (`zitadel-e2e`, port == nodePort == 30424): the host reaches it
//! via the k3d port mapping, in-cluster pods resolve the hostname
//! via a CoreDNS rewrite to that Service (whose in-cluster port is
//! the same number), VMs resolve it via an `/etc/hosts` entry
//! pointing at the libvirt gateway. Zitadel derives its issuer
//! from `Host`, so every consumer sees the same `iss`; host-side
//! helpers hit `127.0.0.1:30424` and carry the hostname in `Host`.
//! - **OpenBao** lives in the per-bring-up namespace like the other
//! workloads; a NodePort (mapped into k3d) makes it reachable from
//! VMs.
//!
//! Zitadel itself deploys into a stable `zitadel` namespace so the
//! multi-minute first install is paid once per cluster, not per run.
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result};
use harmony::inventory::Inventory;
use harmony::modules::k8s::coredns::{CoreDNSRewrite, CoreDNSRewriteScore};
use harmony::modules::k8s::resource::K8sResourceScore;
use harmony::modules::nats_auth_callout::NatsAuthCalloutScore;
use harmony::modules::openbao::{
OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore, OpenbaoSetupScore, cached_root_token,
};
use harmony::modules::zitadel::{
MachineKeyType, ZitadelApiApp, ZitadelClientConfig, ZitadelMachineUser, ZitadelRole,
ZitadelScheme, ZitadelScore, ZitadelSetupScore, mint_device_credentials,
};
use harmony::score::Score;
use harmony::topology::K8sAnywhereTopology;
use harmony_fleet_deploy::FleetNatsScore;
use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader, encode as jwt_encode};
use k8s_openapi::api::core::v1::{Secret, Service, ServicePort, ServiceSpec};
use kube::api::ObjectMeta;
use nkeys::KeyPair;
use serde::Deserialize;
use crate::images::CALLOUT_IMAGE_TAG;
pub const ZITADEL_HOST: &str = "sso.fleet-e2e.local";
/// Host-mapped k3d port AND the `zitadel-e2e` Service's in-cluster
/// port — the same number on purpose, so one issuer URL resolves
/// everywhere (see module doc). Not 8080: dev hosts commonly have
/// something on it.
pub const ZITADEL_PORT: u16 = 30424;
pub const ZITADEL_NAMESPACE: &str = "zitadel";
/// Host-mapped NodePort the OpenBao Service publishes for VM access.
pub const OPENBAO_NODE_PORT: u16 = 30425;
pub const OPENBAO_RELEASE: &str = "openbao";
pub const KV_MOUNT: &str = "secret";
/// KV prefix deployment secrets live under. Grant writer and agents
/// must agree on it; tests import this const for both sides.
pub const SECRET_PREFIX: &str = "fleet-e2e";
pub const PROJECT_NAME: &str = "fleet";
pub const ADMIN_ROLE: &str = "fleet-admin";
/// Role the callout maps to device-grade NATS permissions. Group
/// roles (`edge-*`) ride along in the same claim purely as OpenBao
/// groups (ADR-025 roles-as-groups interim).
pub const DEVICE_ROLE: &str = "device";
pub const GROUP_ROLES: &[&str] = &["edge-a", "edge-b"];
pub const ADMIN_USERNAME: &str = "e2e-admin";
const NATS_ACCOUNT: &str = "FLEET";
const NATS_AUTH_USER: &str = "auth";
// Dev-only static credential for the callout's own NATS user, same
// posture as the user-pass mode creds in `stack.rs`.
const NATS_AUTH_PASS: &str = "e2e-callout-auth";
const CALLOUT_DEPLOYMENT: &str = "fleet-callout";
/// Handles the callout bring-up leaves behind for tests.
#[derive(Debug, Clone)]
pub struct CalloutHandles {
/// `http://sso.fleet-e2e.local:30424` — `iss` everywhere.
pub issuer_url: String,
pub project_id: String,
/// Machine key of the fleet-admin user the host-side NATS client
/// authenticates as.
pub admin_machine_key: String,
/// Zitadel IAM admin PAT (from the chart's `iam-admin-pat`
/// Secret) — used to enroll device machine users.
pub admin_pat: String,
pub openbao: OpenbaoInstance,
pub openbao_root_token: String,
}
pub fn issuer_url() -> String {
format!("http://{ZITADEL_HOST}:{ZITADEL_PORT}")
}
/// NodePort Service in front of Zitadel with in-cluster port ==
/// nodePort == [`ZITADEL_PORT`] (targetPort = the chart's 8080), so
/// the issuer URL's port number is valid both inside and outside the
/// cluster.
fn zitadel_nodeport_service() -> Service {
let selector = std::collections::BTreeMap::from([
("app.kubernetes.io/name".to_string(), "zitadel".to_string()),
(
"app.kubernetes.io/instance".to_string(),
"zitadel".to_string(),
),
]);
Service {
metadata: ObjectMeta {
name: Some("zitadel-e2e".to_string()),
namespace: Some(ZITADEL_NAMESPACE.to_string()),
..Default::default()
},
spec: Some(ServiceSpec {
type_: Some("NodePort".to_string()),
selector: Some(selector),
ports: Some(vec![ServicePort {
name: Some("http".to_string()),
port: ZITADEL_PORT as i32,
target_port: Some(
k8s_openapi::apimachinery::pkg::util::intstr::IntOrString::Int(8080),
),
node_port: Some(ZITADEL_PORT as i32),
..Default::default()
}]),
..Default::default()
}),
..Default::default()
}
}
/// Deploy Zitadel + project/roles/admin-user, returning everything
/// except the OpenBao handles (filled by [`deploy_openbao`]).
pub async fn bring_up_zitadel(
topology: &K8sAnywhereTopology,
kube_client: &kube::Client,
) -> Result<CalloutHandles> {
ZitadelScore {
host: ZITADEL_HOST.to_string(),
zitadel_version: "v4.12.1".to_string(),
external_secure: false,
external_port: Some(ZITADEL_PORT as u32),
namespace: ZITADEL_NAMESPACE.to_string(),
..Default::default()
}
.interpret(&Inventory::autoload(), topology)
.await
.map_err(|e| anyhow::anyhow!("Zitadel deploy: {e}"))?;
K8sResourceScore::single(
zitadel_nodeport_service(),
Some(ZITADEL_NAMESPACE.to_string()),
)
.interpret(&Inventory::autoload(), topology)
.await
.map_err(|e| anyhow::anyhow!("Zitadel NodePort service: {e}"))?;
CoreDNSRewriteScore {
rewrites: vec![CoreDNSRewrite {
hostname: ZITADEL_HOST.to_string(),
target: format!("zitadel-e2e.{ZITADEL_NAMESPACE}.svc.cluster.local"),
}],
}
.interpret(&Inventory::autoload(), topology)
.await
.map_err(|e| anyhow::anyhow!("CoreDNS rewrite: {e}"))?;
wait_for_zitadel_ready().await?;
let roles = [ADMIN_ROLE, DEVICE_ROLE]
.into_iter()
.chain(GROUP_ROLES.iter().copied())
.map(|key| ZitadelRole {
project_name: PROJECT_NAME.to_string(),
key: key.to_string(),
display_name: key.to_string(),
group: None,
})
.collect();
ZitadelSetupScore {
machine_users: vec![ZitadelMachineUser {
username: ADMIN_USERNAME.to_string(),
name: "E2E Admin".to_string(),
create_pat: false,
machine_key: Some(MachineKeyType::Json),
project_name: Some(PROJECT_NAME.to_string()),
grant_roles: vec![ADMIN_ROLE.to_string()],
}],
roles,
// The flattening Action turns project role keys into the
// `groups` array claim OpenBao binds against (ADR-025) — this
// bring-up is its live verification.
groups_claim_action: true,
applications: vec![],
api_apps: vec![ZitadelApiApp {
project_name: PROJECT_NAME.to_string(),
app_name: "nats".to_string(),
}],
..zitadel_connection()
}
.interpret(&Inventory::autoload(), topology)
.await
.map_err(|e| anyhow::anyhow!("Zitadel setup: {e}"))?;
let zcfg = ZitadelClientConfig::load()
.context("ZitadelSetupScore did not produce a client config cache")?;
let project_id = zcfg
.project_id_by_name(PROJECT_NAME)
.or(zcfg.project_id.as_ref())
.context("project_id missing from cache")?
.clone();
let admin_machine_key = zcfg
.machine_key(ADMIN_USERNAME)
.context("admin machine key missing from cache")?
.clone();
let admin_pat = read_iam_admin_pat(kube_client).await?;
Ok(CalloutHandles {
issuer_url: issuer_url(),
project_id,
admin_machine_key,
admin_pat,
// Placeholder until deploy_openbao fills them in.
openbao: OpenbaoInstance {
namespace: String::new(),
release: OPENBAO_RELEASE.to_string(),
},
openbao_root_token: String::new(),
})
}
/// Network-shape-only ZitadelSetupScore: management API reached at
/// `127.0.0.1:8080` with the hostname carried in `Host` (k3d ingress
/// routes by it; no /etc/hosts entry needed on the host).
fn zitadel_connection() -> ZitadelSetupScore {
ZitadelSetupScore {
host: ZITADEL_HOST.to_string(),
scheme: ZitadelScheme::Http,
port: Some(ZITADEL_PORT),
skip_tls: true,
endpoint: Some(format!("http://127.0.0.1:{ZITADEL_PORT}")),
admin_org_id: None,
namespace: ZITADEL_NAMESPACE.to_string(),
applications: vec![],
api_apps: vec![],
roles: vec![],
machine_users: vec![],
groups_claim_action: false,
}
}
/// Find-or-create the device's machine user with `roles` granted and
/// mint a fresh JSON machine key. Username == device id, so the
/// callout needs no prefix stripping and the operator-side group
/// source resolves grants by device id directly.
pub async fn enroll_device(
handles: &CalloutHandles,
device_id: &str,
roles: &[String],
) -> Result<String> {
let minted = mint_device_credentials(
&zitadel_connection(),
&handles.admin_pat,
PROJECT_NAME,
device_id,
device_id,
roles,
)
.await
.map_err(|e| anyhow::anyhow!("enrolling device '{device_id}': {e}"))?;
Ok(minted.key_json)
}
/// Deploy NATS in callout mode + the callout service itself.
/// Returns the `FleetNatsScore` so the caller can derive the
/// in-cluster URL the same way the user-pass path does.
pub async fn deploy_nats_callout(
topology: &K8sAnywhereTopology,
namespace: &str,
release: &str,
node_port: u16,
handles: &CalloutHandles,
) -> Result<FleetNatsScore> {
let issuer_kp = KeyPair::new_account();
let issuer_seed = issuer_kp
.seed()
.map_err(|e| anyhow::anyhow!("issuer NKey seed: {e}"))?;
let nats = FleetNatsScore::callout(
namespace.to_string(),
node_port,
issuer_kp.public_key(),
NATS_ACCOUNT,
NATS_AUTH_USER,
NATS_AUTH_PASS,
)
.release_name(release);
nats.interpret(&Inventory::autoload(), topology)
.await
.map_err(|e| anyhow::anyhow!("NATS (callout) deploy: {e}"))?;
let mut callout = NatsAuthCalloutScore::new(
CALLOUT_DEPLOYMENT,
namespace,
nats.in_cluster_url(),
handles.issuer_url.clone(),
handles.project_id.clone(),
NATS_AUTH_USER,
NATS_AUTH_PASS,
issuer_seed,
)
.image(CALLOUT_IMAGE_TAG)
.target_account(NATS_ACCOUNT)
.admin_role(ADMIN_ROLE)
.device_role(DEVICE_ROLE);
// Machine-user access tokens carry the userName in `client_id`;
// usernames ARE device ids here (see `enroll_device`).
callout.device_id_claim = "client_id".to_string();
callout.device_id_prefix_strip = String::new();
callout.roles_claim = format!("urn:zitadel:iam:org:project:{}:roles", handles.project_id);
callout
.interpret(&Inventory::autoload(), topology)
.await
.map_err(|e| anyhow::anyhow!("callout deploy: {e}"))?;
Ok(nats)
}
pub fn callout_deployment_name() -> &'static str {
CALLOUT_DEPLOYMENT
}
/// Deploy OpenBao into the bring-up namespace: JWT auth bound to the
/// Zitadel issuer (batch tokens, 1h, `groups` claim — the staging
/// shape) plus a NodePort Service so VMs reach it through the libvirt
/// gateway.
pub async fn deploy_openbao(
topology: &K8sAnywhereTopology,
namespace: &str,
handles: &mut CalloutHandles,
) -> Result<()> {
let instance = OpenbaoInstance {
namespace: namespace.to_string(),
release: OPENBAO_RELEASE.to_string(),
};
OpenbaoScore {
instance: instance.clone(),
host: "openbao-e2e.local".to_string(),
openshift: false,
tls_issuer: None,
}
.interpret(&Inventory::autoload(), topology)
.await
.map_err(|e| anyhow::anyhow!("OpenBao deploy: {e}"))?;
OpenbaoSetupScore {
instance: instance.clone(),
kv_mount: KV_MOUNT.to_string(),
jwt_auth: Some(OpenbaoJwtAuth {
oidc_discovery_url: handles.issuer_url.clone(),
jwt_validation_pubkeys: String::new(),
bound_issuer: handles.issuer_url.clone(),
role_name: "fleet-device".to_string(),
bound_audiences: handles.project_id.clone(),
user_claim: "sub".to_string(),
groups_claim: "groups".to_string(),
token_type: "batch".to_string(),
ttl: "1h".to_string(),
max_ttl: "1h".to_string(),
}),
..Default::default()
}
.interpret(&Inventory::autoload(), topology)
.await
.map_err(|e| anyhow::anyhow!("OpenBao setup: {e}"))?;
// NodePort for VM access. Same selector shape as the chart's own
// ClusterIP Service.
let selector = std::collections::BTreeMap::from([
("app.kubernetes.io/name".to_string(), "openbao".to_string()),
(
"app.kubernetes.io/instance".to_string(),
OPENBAO_RELEASE.to_string(),
),
("component".to_string(), "server".to_string()),
]);
let svc = Service {
metadata: ObjectMeta {
name: Some("openbao-nodeport".to_string()),
namespace: Some(namespace.to_string()),
..Default::default()
},
spec: Some(ServiceSpec {
type_: Some("NodePort".to_string()),
selector: Some(selector),
ports: Some(vec![ServicePort {
name: Some("http".to_string()),
port: 8200,
node_port: Some(OPENBAO_NODE_PORT as i32),
..Default::default()
}]),
..Default::default()
}),
..Default::default()
};
K8sResourceScore::single(svc, Some(namespace.to_string()))
.interpret(&Inventory::autoload(), topology)
.await
.map_err(|e| anyhow::anyhow!("OpenBao NodePort service: {e}"))?;
handles.openbao = instance.clone();
handles.openbao_root_token =
cached_root_token(&instance).map_err(|e| anyhow::anyhow!("OpenBao root token: {e}"))?;
Ok(())
}
/// Host-side NATS admin client: authenticates through the callout
/// with a Zitadel access token minted from the fleet-admin machine
/// key.
pub async fn connect_nats_admin_sso(
nats_url: &str,
handles: &CalloutHandles,
) -> Result<async_nats::Client> {
let key = Arc::new(handles.clone());
// ConnectOptions isn't Clone — build a fresh one per attempt.
let opts = || {
let key = key.clone();
async_nats::ConnectOptions::with_auth_callback(move |_nonce| {
let handles = key.clone();
async move {
let token = mint_access_token(&handles.admin_machine_key, &handles.project_id)
.await
.map_err(|e| async_nats::AuthError::new(format!("minting admin token: {e}")))?;
let mut auth = async_nats::Auth::new();
auth.token = Some(token);
Ok(auth)
}
})
.ping_interval(Duration::from_secs(5))
};
let mut last_err = None;
for attempt in 0..120 {
match opts().connect(nats_url).await {
Ok(c) => return Ok(c),
Err(e) => {
if attempt % 10 == 0 {
tracing::warn!(attempt, error = %e, "host NATS (callout) connect retry");
}
last_err = Some(e);
tokio::time::sleep(Duration::from_secs(1)).await;
}
}
}
anyhow::bail!("connecting host NATS admin client via callout: {last_err:?}")
}
/// RFC 7523 JWT-bearer mint against the e2e Zitadel. Transport goes
/// to `127.0.0.1:8080` with the hostname (incl. port — Zitadel
/// derives its issuer from it) in `Host`, mirroring
/// `example_fleet_auth_callout::mint_access_token`. Lives here
/// because `ZitadelJwtBearer` has no transport-endpoint override and
/// teaching it one for a test-only DNS quirk isn't worth the API.
pub async fn mint_access_token(machine_key_json: &str, project_id: &str) -> Result<String> {
#[derive(Deserialize)]
struct MachineKey {
#[serde(rename = "keyId")]
key_id: String,
key: String,
#[serde(rename = "userId")]
user_id: String,
}
let key: MachineKey =
serde_json::from_str(machine_key_json).context("machine key JSON parse")?;
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)?
.as_secs() as i64;
let claims = serde_json::json!({
"iss": key.user_id,
"sub": key.user_id,
"aud": issuer_url(),
"exp": now + 60,
"iat": now,
});
let mut header = JwtHeader::new(Algorithm::RS256);
header.kid = Some(key.key_id.clone());
let assertion = jwt_encode(
&header,
&claims,
&EncodingKey::from_rsa_pem(key.key.as_bytes()).context("machine key RSA parse")?,
)?;
let scope = format!(
"openid profile urn:zitadel:iam:org:projects:roles \
urn:zitadel:iam:org:project:id:{project_id}:aud"
);
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(10))
.build()?;
let resp = client
.post(format!("http://127.0.0.1:{ZITADEL_PORT}/oauth/v2/token"))
.header("Host", format!("{ZITADEL_HOST}:{ZITADEL_PORT}"))
.form(&[
(
"grant_type",
"urn:ietf:params:oauth:grant-type:jwt-bearer".to_string(),
),
("assertion", assertion),
("scope", scope),
])
.send()
.await
.context("POST /oauth/v2/token")?;
if !resp.status().is_success() {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
anyhow::bail!("token endpoint returned {status}: {body}");
}
#[derive(Deserialize)]
struct TokenResponse {
access_token: String,
}
let tr: TokenResponse = resp.json().await.context("parse token response")?;
Ok(tr.access_token)
}
async fn wait_for_zitadel_ready() -> Result<()> {
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(5))
.build()?;
for attempt in 1..=150 {
match client
.get(format!(
"http://127.0.0.1:{ZITADEL_PORT}/.well-known/openid-configuration"
))
.header("Host", format!("{ZITADEL_HOST}:{ZITADEL_PORT}"))
.send()
.await
{
Ok(r) if r.status().is_success() => return Ok(()),
Ok(r) if attempt % 15 == 0 => {
tracing::info!("Zitadel HTTP {} (attempt {attempt}/150)", r.status())
}
Err(e) if attempt % 15 == 0 => {
tracing::info!("Zitadel unreachable: {e} (attempt {attempt}/150)")
}
_ => {}
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
anyhow::bail!("timed out waiting for Zitadel on 127.0.0.1:{ZITADEL_PORT}")
}
async fn read_iam_admin_pat(client: &kube::Client) -> Result<String> {
let api: kube::Api<Secret> = kube::Api::namespaced(client.clone(), ZITADEL_NAMESPACE);
let secret = api
.get("iam-admin-pat")
.await
.context("reading iam-admin-pat Secret (created by ZitadelScore)")?;
let pat = secret
.data
.and_then(|d| d.get("pat").map(|b| b.0.clone()))
.context("iam-admin-pat Secret has no 'pat' key")?;
Ok(String::from_utf8(pat)?.trim().to_string())
}

174
fleet/harmony-fleet-e2e/src/images.rs
View File

@@ -1,8 +1,7 @@
//! Build + sideload the container images the e2e harness deploys.
//!
//! v1: agent only (operator and callout get added when those layers
//! join the harness). Same pattern as
//! [`example_fleet_auth_callout::build_and_load_callout_image`]:
//! One generic path for every image (same pattern as
//! [`example_fleet_auth_callout::build_and_load_callout_image`]):
//!
//! 1. `cargo build --release -p <crate>` on the host (uses the
//! workspace cargo cache; incremental builds are fast).
@@ -24,6 +23,7 @@ use k3d_rs::K3d;
pub const AGENT_IMAGE_TAG: &str = "localhost/harmony-fleet-agent:e2e";
pub const OPERATOR_IMAGE_TAG: &str = "localhost/harmony-fleet-operator:e2e";
pub const CALLOUT_IMAGE_TAG: &str = "localhost/harmony-nats-callout:e2e";
/// Tags of the images this harness build produces. Returned to the
/// caller so manifests reference the exact same tag we just pushed
@@ -38,157 +38,105 @@ pub struct Images {
/// runs (the tag is fixed; cargo + podman caches make subsequent
/// runs near-noop).
pub async fn build_and_sideload(k3d: &K3d, cluster_name: &str) -> Result<Images> {
build_and_load_agent_image(k3d, cluster_name).await?;
build_and_load_operator_image(k3d, cluster_name).await?;
build_and_load_binary_image(k3d, cluster_name, "harmony-fleet-agent", AGENT_IMAGE_TAG).await?;
build_and_load_binary_image(
k3d,
cluster_name,
"harmony-fleet-operator",
OPERATOR_IMAGE_TAG,
)
.await?;
Ok(Images {
agent: AGENT_IMAGE_TAG.to_string(),
operator: OPERATOR_IMAGE_TAG.to_string(),
})
}
async fn build_and_load_operator_image(k3d: &K3d, cluster_name: &str) -> Result<()> {
let workspace_root = workspace_root_from_env();
tracing::info!("cargo build --release -p harmony-fleet-operator");
let status = tokio::process::Command::new("cargo")
.args(["build", "--release", "-p", "harmony-fleet-operator"])
.current_dir(&workspace_root)
.status()
.await
.context("spawning cargo build")?;
if !status.success() {
anyhow::bail!("cargo build harmony-fleet-operator failed");
}
let ctx = tempfile::tempdir()?;
let bin_dst = ctx.path().join("target/release");
std::fs::create_dir_all(&bin_dst)?;
std::fs::copy(
workspace_root.join("target/release/harmony-fleet-operator"),
bin_dst.join("harmony-fleet-operator"),
)
.context("staging operator binary into build context")?;
std::fs::write(
ctx.path().join("Dockerfile"),
r#"FROM docker.io/library/archlinux:base
COPY target/release/harmony-fleet-operator /usr/local/bin/harmony-fleet-operator
USER 65532:65532
ENTRYPOINT ["/usr/local/bin/harmony-fleet-operator"]
"#,
)?;
tracing::info!("podman build -t {OPERATOR_IMAGE_TAG}");
let status = tokio::process::Command::new("podman")
.args(["build", "-q", "-t", OPERATOR_IMAGE_TAG, "."])
.current_dir(ctx.path())
.stderr(Stdio::inherit())
.status()
.await
.context("spawning podman build")?;
if !status.success() {
anyhow::bail!("podman build for harmony-fleet-operator failed");
}
let tar_path = std::env::temp_dir().join(format!(
"harmony-fleet-operator-image-{}.tar",
std::process::id()
));
let _ = std::fs::remove_file(&tar_path);
let status = tokio::process::Command::new("podman")
.args(["save", "-o", tar_path.to_str().unwrap(), OPERATOR_IMAGE_TAG])
.status()
.await
.context("podman save")?;
if !status.success() {
anyhow::bail!("podman save for harmony-fleet-operator failed");
}
tracing::info!("k3d image import {OPERATOR_IMAGE_TAG} (cluster={cluster_name})");
let tar_path_str = tar_path.to_str().unwrap().to_string();
let cluster_for_blocking = cluster_name.to_string();
let data_dir = k3d_data_dir();
let result = tokio::task::spawn_blocking(move || {
K3d::new(data_dir, Some(cluster_for_blocking.clone())).run_k3d_command([
"image",
"import",
tar_path_str.as_str(),
"-c",
cluster_for_blocking.as_str(),
])
})
.await
.context("spawn_blocking k3d image import")?;
let _ = std::fs::remove_file(&tar_path);
let output = result.map_err(|e| anyhow::anyhow!("k3d image import failed: {e}"))?;
if !output.status.success() {
anyhow::bail!(
"k3d image import returned status {}: {}",
output.status,
String::from_utf8_lossy(&output.stderr),
);
}
let _ = k3d;
Ok(())
/// Build + sideload the NATS auth-callout image (callout-mode stacks
/// only — keeps user-pass bring-ups from paying for a binary they
/// never deploy).
pub async fn build_and_load_callout_image(k3d: &K3d, cluster_name: &str) -> Result<()> {
build_and_load_binary_image(k3d, cluster_name, "harmony-nats-callout", CALLOUT_IMAGE_TAG).await
}
async fn build_and_load_agent_image(k3d: &K3d, cluster_name: &str) -> Result<()> {
/// Where `cargo build --release` actually puts binaries — the child
/// cargo honors CARGO_TARGET_DIR, so staging must too or the copy
/// fails with "No such file or directory" on shared-target setups
/// (e.g. git worktrees).
fn release_dir(workspace_root: &std::path::Path) -> PathBuf {
std::env::var_os("CARGO_TARGET_DIR")
.map(PathBuf::from)
.unwrap_or_else(|| workspace_root.join("target"))
.join("release")
}
async fn build_and_load_binary_image(
k3d: &K3d,
cluster_name: &str,
crate_name: &str,
image_tag: &str,
) -> Result<()> {
let workspace_root = workspace_root_from_env();
tracing::info!("cargo build --release -p harmony-fleet-agent");
tracing::info!("cargo build --release -p {crate_name}");
let status = tokio::process::Command::new("cargo")
.args(["build", "--release", "-p", "harmony-fleet-agent"])
.args(["build", "--release", "-p", crate_name])
.current_dir(&workspace_root)
.status()
.await
.context("spawning cargo build")?;
if !status.success() {
anyhow::bail!("cargo build harmony-fleet-agent failed");
anyhow::bail!("cargo build {crate_name} failed");
}
let ctx = tempfile::tempdir()?;
let bin_dst = ctx.path().join("target/release");
std::fs::create_dir_all(&bin_dst)?;
std::fs::copy(
workspace_root.join("target/release/harmony-fleet-agent"),
bin_dst.join("harmony-fleet-agent"),
release_dir(&workspace_root).join(crate_name),
bin_dst.join(crate_name),
)
.context("staging agent binary into build context")?;
.with_context(|| format!("staging {crate_name} binary into build context"))?;
// Single-stage Dockerfile inline (mirrors what the callout build
// does). archlinux:base for matched glibc against the host build —
// the canonical multi-stage Dockerfile in the agent crate is for
// production registries, not the e2e fast path.
// Single-stage Dockerfile inline. archlinux:base for matched glibc
// against the host build — the canonical multi-stage Dockerfiles in
// the crates are for production registries, not the e2e fast path.
std::fs::write(
ctx.path().join("Dockerfile"),
r#"FROM docker.io/library/archlinux:base
COPY target/release/harmony-fleet-agent /usr/local/bin/harmony-fleet-agent
format!(
r#"FROM docker.io/library/archlinux:base
COPY target/release/{crate_name} /usr/local/bin/{crate_name}
USER 65532:65532
ENTRYPOINT ["/usr/local/bin/harmony-fleet-agent"]
"#,
ENTRYPOINT ["/usr/local/bin/{crate_name}"]
"#
),
)?;
tracing::info!("podman build -t {AGENT_IMAGE_TAG}");
tracing::info!("podman build -t {image_tag}");
let status = tokio::process::Command::new("podman")
.args(["build", "-q", "-t", AGENT_IMAGE_TAG, "."])
.args(["build", "-q", "-t", image_tag, "."])
.current_dir(ctx.path())
.stderr(Stdio::inherit())
.status()
.await
.context("spawning podman build")?;
if !status.success() {
anyhow::bail!("podman build for harmony-fleet-agent failed");
anyhow::bail!("podman build for {crate_name} failed");
}
let tar_path = std::env::temp_dir().join(format!(
"harmony-fleet-agent-image-{}.tar",
std::process::id()
));
let tar_path =
std::env::temp_dir().join(format!("{crate_name}-image-{}.tar", std::process::id()));
let _ = std::fs::remove_file(&tar_path);
let status = tokio::process::Command::new("podman")
.args(["save", "-o", tar_path.to_str().unwrap(), AGENT_IMAGE_TAG])
.args(["save", "-o", tar_path.to_str().unwrap(), image_tag])
.status()
.await
.context("podman save")?;
if !status.success() {
anyhow::bail!("podman save for harmony-fleet-agent failed");
anyhow::bail!("podman save for {crate_name} failed");
}
tracing::info!("k3d image import {AGENT_IMAGE_TAG} (cluster={cluster_name})");
tracing::info!("k3d image import {image_tag} (cluster={cluster_name})");
let tar_path_str = tar_path.to_str().unwrap().to_string();
let cluster_for_blocking = cluster_name.to_string();
let data_dir = k3d_data_dir();
@@ -199,6 +147,12 @@ ENTRYPOINT ["/usr/local/bin/harmony-fleet-agent"]
tar_path_str.as_str(),
"-c",
cluster_for_blocking.as_str(),
// The default tools-node mode can fail to copy the tar
// (docker: "mkdirat var/run: file exists") while still
// exiting 0 — the image never lands and pods ErrImagePull.
// Direct mode imports through each node's containerd.
"--mode",
"direct",
])
})
.await
@@ -212,10 +166,6 @@ ENTRYPOINT ["/usr/local/bin/harmony-fleet-agent"]
String::from_utf8_lossy(&output.stderr),
);
}
// Keep the k3d helper warm in scope so the borrow checker is
// happy (k3d is referenced by the caller's bring-up; this just
// ensures we used its cluster name even if the binary path went
// through the blocking task).
let _ = k3d;
Ok(())
}

2
fleet/harmony-fleet-e2e/src/lib.rs
View File

@@ -13,12 +13,14 @@
//! [`FleetNatsScore`]: harmony_fleet_deploy::FleetNatsScore
//! [`FleetAgentScore`]: harmony_fleet_deploy::FleetAgentScore
pub mod callout;
mod images;
mod kv_admin;
mod namespace;
mod stack;
pub mod vm;
pub use callout::CalloutHandles;
pub use images::{AGENT_IMAGE_TAG, Images, build_and_sideload};
pub use kv_admin::{AdminKv, AdminKvError, PhaseExpectation};
pub use namespace::{ENV_KEEP_NAMESPACE, NamespaceGuard};

206
fleet/harmony-fleet-e2e/src/stack.rs
View File

@@ -60,10 +60,12 @@ const NATS_DEVICE_PASS: &str = "e2e-device";
const NATS_ADMIN_USER: &str = "admin";
const NATS_ADMIN_PASS: &str = "e2e-admin";
/// Auth mode the harness brings NATS up with. v1 supports `UserPass`
/// only; the `Callout` variant is reserved on the public API for the
/// follow-up PR that wires `NatsAuthCalloutScore` + mock-OIDC behind
/// it.
/// Auth mode the harness brings NATS up with.
///
/// `Callout` deploys the full production auth fabric: Zitadel (stable
/// `zitadel` namespace, reused across runs), the NATS auth-callout
/// service, and OpenBao with JWT auth bound to the Zitadel issuer.
/// See [`crate::callout`] for the network plumbing notes.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum AuthMode {
#[default]
@@ -136,8 +138,14 @@ pub enum BringUpError {
name: String,
detail: String,
},
#[error("AuthMode::Callout is reserved for PR 1.5 (mock OIDC fixture + NatsAuthCalloutScore)")]
CalloutNotYetImplemented,
#[error("callout auth fabric: {0}")]
Callout(#[source] anyhow::Error),
#[error(
"cluster '{CLUSTER_NAME}' exists without the port mappings callout mode needs \
({missing}); delete it (`k3d cluster delete {CLUSTER_NAME}`) and re-run so the \
harness recreates it with the full mapping set"
)]
MissingPortMappings { missing: String },
#[error("connecting host-side NATS admin client to {url}: {detail}")]
NatsConnect { url: String, detail: String },
}
@@ -152,6 +160,9 @@ pub struct Stack {
pub nats_client: async_nats::Client,
pub admin_user: String,
pub admin_pass: String,
/// Auth fabric handles — `Some` iff brought up with
/// [`AuthMode::Callout`].
pub callout: Option<crate::callout::CalloutHandles>,
_guard: NamespaceGuard,
}
@@ -170,11 +181,22 @@ impl Stack {
/// runs through the full deploy. For shared-across-tests bring-up
/// use [`shared_stack`].
pub async fn bring_up(opts: StackOptions) -> Result<Self, BringUpError> {
if opts.auth_mode == AuthMode::Callout {
return Err(BringUpError::CalloutNotYetImplemented);
let callout_mode = opts.auth_mode == AuthMode::Callout;
// Static user-pass material is what the Pod agent + operator
// paths inject today; behind the callout they'd be rejected
// at connect. The VM harness brings callout-authenticated
// agents instead.
if callout_mode && (opts.num_devices > 0 || opts.deploy_operator) {
return Err(BringUpError::Callout(anyhow::anyhow!(
"callout mode supports infra-only bring-up (VM agents); \
Pod agents / operator still authenticate with static creds"
)));
}
let k3d = ensure_k3d_cluster().await?;
if callout_mode {
verify_callout_port_mappings().await?;
}
let topology = K8sAnywhereTopology::from_env();
harmony::topology::Topology::ensure_ready(&topology)
.await
@@ -190,6 +212,11 @@ impl Stack {
.await
.map_err(BringUpError::ImageBuild)?;
}
if callout_mode {
crate::images::build_and_load_callout_image(&k3d, CLUSTER_NAME)
.await
.map_err(BringUpError::ImageBuild)?;
}
// Separate raw `kube::Client` for namespace lifecycle + the
// readiness probes below. The topology's `K8sClient` wrapper
@@ -209,6 +236,19 @@ impl Stack {
prune_stale_operator_cluster_resources(&kube_client).await;
}
// Zitadel + project/roles/admin identity. Sits outside the
// per-bring-up namespace (stable `zitadel` ns) so the
// multi-minute first install amortizes across runs.
let mut callout_handles = if callout_mode {
Some(
crate::callout::bring_up_zitadel(&topology, &kube_client)
.await
.map_err(BringUpError::Callout)?,
)
} else {
None
};
let namespace = unique_namespace();
let guard = NamespaceGuard::create(kube_client.clone(), namespace.clone())
.await
@@ -226,32 +266,60 @@ impl Stack {
// Dev credentials live here in the e2e crate, not in the
// deploy crate — keeping them out of `harmony-fleet-deploy`
// means production builds can't accidentally ship them.
let nats = FleetNatsScore::user_pass(
namespace.clone(),
NATS_NODE_PORT,
UserPassCredentials {
admin_user: NATS_ADMIN_USER.to_string(),
admin_pass: NATS_ADMIN_PASS.to_string(),
device_user: NATS_DEVICE_USER.to_string(),
device_pass: NATS_DEVICE_PASS.to_string(),
},
)
.release_name(NATS_RELEASE);
nats.interpret(&Inventory::autoload(), &topology)
.await
.map_err(BringUpError::NatsDeploy)?;
// Wait for NATS to be reachable from the host BEFORE
// deploying the agent. Without this, the agent comes up,
// can't connect, hits CrashLoopBackOff for a while, and the
// agent Deployment-Ready wait spuriously times out.
let nats_url = format!("nats://127.0.0.1:{NATS_NODE_PORT}");
let nats_client = connect_nats_admin(&nats_url).await?;
let (nats_client, nats_in_cluster_url) = match callout_handles.as_mut() {
None => {
let nats = FleetNatsScore::user_pass(
namespace.clone(),
NATS_NODE_PORT,
UserPassCredentials {
admin_user: NATS_ADMIN_USER.to_string(),
admin_pass: NATS_ADMIN_PASS.to_string(),
device_user: NATS_DEVICE_USER.to_string(),
device_pass: NATS_DEVICE_PASS.to_string(),
},
)
.release_name(NATS_RELEASE);
nats.interpret(&Inventory::autoload(), &topology)
.await
.map_err(BringUpError::NatsDeploy)?;
// Wait for NATS to be reachable from the host BEFORE
// deploying the agent. Without this, the agent comes
// up, can't connect, hits CrashLoopBackOff for a
// while, and the agent Deployment-Ready wait
// spuriously times out.
(connect_nats_admin(&nats_url).await?, nats.in_cluster_url())
}
Some(handles) => {
let nats = crate::callout::deploy_nats_callout(
&topology,
&namespace,
NATS_RELEASE,
NATS_NODE_PORT,
handles,
)
.await
.map_err(BringUpError::Callout)?;
wait_for_deployment(
&kube_client,
&namespace,
crate::callout::callout_deployment_name(),
)
.await?;
let client = crate::callout::connect_nats_admin_sso(&nats_url, handles)
.await
.map_err(BringUpError::Callout)?;
crate::callout::deploy_openbao(&topology, &namespace, handles)
.await
.map_err(BringUpError::Callout)?;
(client, nats.in_cluster_url())
}
};
if opts.deploy_operator {
let mut operator = FleetOperatorScore::new()
.namespace(namespace.clone())
.image(opts.operator_image.clone())
.nats_url(nats.in_cluster_url())
.nats_url(nats_in_cluster_url.clone())
.log_level(opts.log_level.clone());
operator.credentials = Some(OperatorCredentials {
@@ -288,7 +356,7 @@ impl Stack {
PodTarget::user_pass(
device_id.clone(),
opts.agent_image.clone(),
nats.in_cluster_url(),
nats_in_cluster_url.clone(),
NATS_DEVICE_USER,
NATS_DEVICE_PASS,
)
@@ -316,10 +384,26 @@ impl Stack {
nats_client,
admin_user: NATS_ADMIN_USER.to_string(),
admin_pass: NATS_ADMIN_PASS.to_string(),
callout: callout_handles,
_guard: guard,
})
}
/// Dial a fresh admin NATS connection on the **caller's** runtime.
/// The shared `nats_client` rides the runtime of whichever
/// `#[tokio::test]` ran bring-up; when that test finishes, its
/// runtime — and the connection driver task — die with it. Any
/// test that can start after another test completed must connect
/// its own client.
pub async fn connect_admin(&self) -> Result<async_nats::Client, BringUpError> {
match self.callout.as_ref() {
None => connect_nats_admin(&self.nats_url).await,
Some(handles) => crate::callout::connect_nats_admin_sso(&self.nats_url, handles)
.await
.map_err(BringUpError::Callout),
}
}
pub fn print_debug_info(&self) {
eprintln!("[e2e] namespace: {}", self.namespace);
eprintln!(
@@ -364,8 +448,21 @@ async fn ensure_k3d_cluster() -> Result<K3d, BringUpError> {
.unwrap_or_else(|| std::path::PathBuf::from("/tmp/harmony"));
std::fs::create_dir_all(&data_dir)
.map_err(|e| BringUpError::K3dEnsure(format!("creating k3d data dir {data_dir:?}: {e}")))?;
// The callout-mode mappings ride along on every cluster create so
// one cluster serves both auth modes: the Zitadel and OpenBao
// NodePorts (see `callout.rs` for the issuer-port invariant).
// Mappings are create-time-only in k3d —
// `verify_callout_port_mappings` catches clusters predating them.
let k3d = K3d::new(data_dir, Some(CLUSTER_NAME.to_string())).with_port_mappings(vec![
PortMapping::new(NATS_NODE_PORT as u32, NATS_NODE_PORT as u32),
PortMapping::new(
crate::callout::ZITADEL_PORT as u32,
crate::callout::ZITADEL_PORT as u32,
),
PortMapping::new(
crate::callout::OPENBAO_NODE_PORT as u32,
crate::callout::OPENBAO_NODE_PORT as u32,
),
]);
k3d.ensure_installed()
.await
@@ -381,10 +478,59 @@ async fn ensure_k3d_cluster() -> Result<K3d, BringUpError> {
);
std::env::set_var("HARMONY_USE_LOCAL_K3D", "false");
std::env::set_var("HARMONY_AUTOINSTALL", "false");
// Scores that persist credentials (ZitadelScore's admin
// password / masterkey) go through harmony_secret's global
// manager, which defaults to Infisical and panics unconfigured.
// The harness pins the local-file store + a namespace (the
// examples set these via env.sh); don't override a
// developer's explicit choice.
if std::env::var("HARMONY_SECRET_STORE").is_err() {
std::env::set_var("HARMONY_SECRET_STORE", "file");
}
if std::env::var("HARMONY_SECRET_NAMESPACE").is_err() {
std::env::set_var("HARMONY_SECRET_NAMESPACE", "fleet-e2e");
}
}
Ok(k3d)
}
/// Callout mode needs host ports that only exist if the cluster was
/// created with the full mapping set (k3d mappings are create-time
/// fixed). The mappings live on the `serverlb` docker container —
/// probe it and fail with an actionable message instead of letting
/// Zitadel readiness time out 5 minutes later.
async fn verify_callout_port_mappings() -> Result<(), BringUpError> {
let output = tokio::process::Command::new("docker")
.args(["port", &format!("k3d-{CLUSTER_NAME}-serverlb")])
.output()
.await;
let Ok(output) = output else {
// No docker CLI (or not the docker runtime) — skip the
// preflight; the Zitadel readiness wait will surface it.
tracing::warn!("docker CLI unavailable; skipping callout port-mapping preflight");
return Ok(());
};
if !output.status.success() {
return Ok(()); // cluster not created yet — ensure_installed handles it
}
let ports = String::from_utf8_lossy(&output.stdout);
let missing: Vec<String> = [
crate::callout::ZITADEL_PORT,
crate::callout::OPENBAO_NODE_PORT,
]
.into_iter()
.filter(|p| !ports.contains(&format!(":{p}")))
.map(|p| p.to_string())
.collect();
if missing.is_empty() {
Ok(())
} else {
Err(BringUpError::MissingPortMappings {
missing: missing.join(", "),
})
}
}
/// Delete every namespace tagged `harmony.io/managed-by=fleet-e2e`
/// and **wait** for them to fully terminate before returning. The
/// wait is required: NATS uses a fixed NodePort (cluster-scoped

26
fleet/harmony-fleet-e2e/src/vm/device.rs
View File

@@ -23,8 +23,9 @@ use std::sync::Arc;
use harmony::inventory::Inventory;
use harmony::modules::fleet::{
FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HARMONY_FLEET_POOL_NAME,
ProvisionVmScore, ensure_fleet_ssh_keypair, ensure_harmony_fleet_pool, read_public_key,
DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
HARMONY_FLEET_POOL_NAME, HostsEntry, ProvisionVmScore, ensure_fleet_ssh_keypair,
ensure_harmony_fleet_pool, read_public_key,
};
use harmony::modules::kvm::KvmVirtualMachineHost;
use harmony::modules::kvm::config::init_executor;
@@ -113,11 +114,14 @@ pub struct VmDeviceOptions {
/// reachable from the VM (typically `nats://<libvirt-gateway>
/// :<node_port>`).
pub nats_url: String,
/// Static NATS creds the FleetNatsScore deploys (the device
/// user). Matches `nats::NATS_DEVICE_USER` / `NATS_DEVICE_PASS`
/// in `stack.rs` for the user-pass mode.
pub nats_user: String,
pub nats_pass: String,
/// On-device auth: static user-pass for the dev stack, a
/// pre-resolved Zitadel machine key in callout mode.
pub auth: FleetDeviceAuth,
/// `/etc/hosts` entries for names the VM can't otherwise resolve
/// (the Zitadel issuer in callout mode).
pub hosts_entries: Vec<HostsEntry>,
/// `[openbao]` section for the agent — callout mode only.
pub openbao: Option<DeviceOpenbao>,
/// Pre-built aarch64 agent binary path. The harness builds it
/// once and passes the path into every device it provisions.
pub agent_binary: PathBuf,
@@ -219,12 +223,10 @@ impl VmDevice {
device_id: Id::from(opts.device_id.clone()),
labels: opts.labels.clone(),
nats_urls: vec![opts.nats_url.clone()],
auth: FleetDeviceAuth::TomlShared {
nats_user: opts.nats_user.clone(),
nats_pass: opts.nats_pass.clone(),
},
auth: opts.auth.clone(),
agent_binary_path: opts.agent_binary.clone(),
hosts_entries: vec![],
hosts_entries: opts.hosts_entries.clone(),
openbao: opts.openbao.clone(),
});
let interpret = Score::<LinuxHostTopology>::create_interpret(&setup_score);
interpret

73
fleet/harmony-fleet-e2e/src/vm/stack.rs
View File

@@ -26,7 +26,9 @@ use tokio::process::Command;
use tokio::sync::OnceCell;
use uuid::Uuid;
use crate::stack::{BringUpError, NATS_NODE_PORT, Stack, StackOptions, shared_stack};
use harmony::modules::fleet::{DeviceOpenbao, FleetDeviceAuth, HostsEntry};
use crate::stack::{AuthMode, BringUpError, NATS_NODE_PORT, Stack, StackOptions, shared_stack};
use crate::vm::agent_build::{AgentBuildError, build_agent_for};
use crate::vm::device::{VmDevice, VmDeviceError, VmDeviceOptions};
use crate::vm::network::{NetworkLookupError, libvirt_default_gateway_ip};
@@ -70,6 +72,16 @@ pub struct VmStackOptions {
/// Empty by default; the FleetDeviceSetupScore is content with
/// no labels (the device is targetable by `device-id` only).
pub labels: BTreeMap<String, String>,
/// NATS auth fabric for the whole stack. `Callout` enrolls each
/// VM as a Zitadel machine user (granted `device_roles`), points
/// the agent's `[openbao]` at the e2e OpenBao, and adds the
/// issuer `/etc/hosts` entry.
pub auth_mode: AuthMode,
/// Zitadel project roles granted to every enrolled device
/// (callout mode). Must include the callout's device role for
/// NATS permissions; `edge-*` roles double as OpenBao groups
/// (ADR-025 roles-as-groups).
pub device_roles: Vec<String>,
}
impl Default for VmStackOptions {
@@ -78,6 +90,8 @@ impl Default for VmStackOptions {
num_devices: 1,
arch: VmArchitecture::Aarch64,
labels: BTreeMap::new(),
auth_mode: AuthMode::UserPass,
device_roles: vec![crate::callout::DEVICE_ROLE.to_string()],
}
}
}
@@ -119,6 +133,8 @@ pub enum VmBringUpError {
#[source]
source: VmDeviceError,
},
#[error("Zitadel device enrollment: {0}")]
Enroll(#[source] anyhow::Error),
}
#[derive(Debug, Error)]
@@ -136,6 +152,8 @@ pub enum VmReadyError {
#[source]
source: CommandError,
},
#[error("admin NATS connect: {0}")]
Connect(#[from] BringUpError),
}
/// Brought-up fleet with NATS in k3d + N agents on VMs.
@@ -175,7 +193,11 @@ impl VmStack {
// 1. Infra (NATS, namespace, admin client). num_devices=0
// skips Pod-agent deploys — the VM agents take their
// place.
let infra = shared_stack(StackOptions::infra_only()).await?;
let infra = shared_stack(StackOptions {
auth_mode: opts.auth_mode,
..StackOptions::infra_only()
})
.await?;
// 2. Build the agent binary for the requested guest arch.
// aarch64 cross-builds; x86_64 takes the host's native
@@ -200,6 +222,43 @@ impl VmStack {
// test never collides with a fresh provision.
let vm_name = format!("fleet-e2e-vm-{run_id}-{i:02}");
let device_id = format!("vm-device-{run_id}-{i:02}");
let (auth, hosts_entries, openbao) = match infra.callout.as_ref() {
None => (
FleetDeviceAuth::TomlShared {
nats_user: NATS_DEVICE_USER.to_string(),
nats_pass: NATS_DEVICE_PASS.to_string(),
},
vec![],
None,
),
Some(handles) => {
let machine_key_json =
crate::callout::enroll_device(handles, &device_id, &opts.device_roles)
.await
.map_err(VmBringUpError::Enroll)?;
(
FleetDeviceAuth::ZitadelJwt {
machine_key_json,
oidc_issuer_url: handles.issuer_url.clone(),
audience: handles.project_id.clone(),
danger_accept_invalid_certs: false,
},
// The issuer hostname resolves to the libvirt
// gateway, behind which the k3d port mapping
// serves the Zitadel ingress.
vec![HostsEntry {
ip: gateway.to_string(),
hostname: crate::callout::ZITADEL_HOST.to_string(),
}],
Some(DeviceOpenbao {
url: format!("http://{gateway}:{}", crate::callout::OPENBAO_NODE_PORT),
secret_prefix: crate::callout::SECRET_PREFIX.to_string(),
}),
)
}
};
let device = VmDevice::provision(VmDeviceOptions {
device_id,
vm_name: vm_name.clone(),
@@ -208,8 +267,9 @@ impl VmStack {
network: LIBVIRT_NETWORK.to_string(),
labels: opts.labels.clone(),
nats_url: vm_nats_url.clone(),
nats_user: NATS_DEVICE_USER.to_string(),
nats_pass: NATS_DEVICE_PASS.to_string(),
auth,
hosts_entries,
openbao,
agent_binary: agent_binary.clone(),
cpus: DEFAULT_VM_CPUS,
memory_mib: DEFAULT_VM_MEMORY_MIB,
@@ -249,7 +309,10 @@ impl VmStack {
/// by retrying ping with a 3 s per-attempt timeout up to
/// `per_device_budget`.
pub async fn wait_until_ready(&self, per_device_budget: Duration) -> Result<(), VmReadyError> {
let client = FleetCommandsClient::new(self.infra.nats_client.clone())
// Fresh connection on the caller's runtime — see
// `Stack::connect_admin` for why the shared client can't be
// trusted across tests.
let client = FleetCommandsClient::new(self.infra.connect_admin().await?)
.with_timeout(Duration::from_secs(3));
for device in &self.devices {
let id = device.device_id.to_string();

346
fleet/harmony-fleet-e2e/tests/openbao_groups.rs Normal file
View File

@@ -0,0 +1,346 @@
//! ADR-025 group enforcement, end to end against a real OpenBao: a
//! device's secret access is decided by the groups in its signed JWT
//! and the deployment policies attached to those groups. Self-contained
//! — the JWT auth mount validates against a test RSA key instead of a
//! live Zitadel, and tests mint their own device tokens.
use harmony::inventory::Inventory;
use harmony::modules::fleet::secret_access::DeploymentSecretGrants;
use harmony::modules::openbao::{
OpenBaoDeploymentSecretGrants, OpenbaoInstance, OpenbaoJwtAuth, OpenbaoScore,
OpenbaoSetupScore, cached_root_token,
};
use harmony::score::Score;
use harmony::topology::{K8sAnywhereTopology, K8sclient};
use harmony_fleet_e2e::{StackOptions, shared_stack};
use harmony_reconciler_contracts::DeploymentName;
use serde_json::json;
const E2E_ENV: &str = "HARMONY_FLEET_E2E";
const RELEASE: &str = "openbao-groups-test";
const KV_MOUNT: &str = "secret";
const FLEET_NS: &str = "fleet-e2e";
const ISSUER: &str = "https://fleet-e2e.test";
const AUDIENCE: &str = "fleet-e2e-project";
const ROLE: &str = "fleet-device";
const JWT_MOUNT: &str = "jwt";
// Test-only RSA keypair: lets the suite mint "Zitadel" tokens without a
// live IdP. Never used outside this test.
const TEST_JWT_PRIVATE_PEM: &str = r#"-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC0VwVrePTYWsym
J+MIOToktME/60/nYFhLi0awgpF5T6McF2TJ/v+Bo1KQQ5Fg53XRGZuFnSKr5WZ+
KzJ7d9YpUSCc/9UMcd3ujhrp35X4rXOlCOqIklRXb/TTmxrl5LMr91zlHFOSxa+/
N5yNYKQVsy9bHeJvOpX5ktWELVS2JoAbvvuVhsuWrFyAS9s3i4a2q1jkg9f+vyPc
BtExIw/Oe5L03x6ZQlPNPLh/c3uQlK7ikOB1877IAWK7/Nnj+L8FPmG9FmLp/CeZ
n6gsqeDq6KloyffCx0ZiyK3ijz2wIHIYo3SVe3jyDjRJLiBIJrcxynq4aid3OGDP
Tw13tshPAgMBAAECggEAFpEhK366XOyX2V76aQgIP5mi6SXAHPsQcXufP+1G2QBf
L9+5jBFt1cIjTcw2wAKW2rCypniwYwRU3Ar5yTuIwCjXxO26slD5baW4tnV1JeZG
p2tDNg3hogNvhp9NMWF6DlFo/X6a8q3sPShvXb2vKsv+bA+ES1jwdYerUlv90sEu
PzW4GcBPCQ19udZEIM1Y2bObKi6bAT3QAANkzQC7kRhV104l7rC9eHhfbaW1Ce9N
rPY2Ekq2ck3kuEz+YrZ0Lh3yvozslK8aPn05ZM1sYv+9Mxb0rwv4LqGdQfNvONbA
ZMLL++chHqTSr5tZ26POLjZ5ZTm/jlOK0ZJ5RE6ZQQKBgQD5JW3U2vtntGl6V2yj
zvDNpXeKNh2CBMB5cuub+G0J2XtAJXW4W3zsGytwNdmrc/IVVJI9CeJ/g+2PvI7d
fiUvSSDZyrvib6+kGj5qDLHEiLJ0LhqEP4/jzHcKoL1lID63C65Sw0ZhKgy/lPXn
elBcuzDIaI+9sre6A9TdnL1vrwKBgQC5TQkIh0aI9PFq184Ugd/Qaj8ckjAbu5ms
4c4cZIVvENFIjSwdk3/p88AP8UmRMTIUUViZ2rvZYJkVUzi15AFbti4sYB5IXTc6
D00Ev+SYW/b1bf/Gu8o/c2UWrNY1823xwXjW0bRT0O9nJJhSDs7GCDOCL8pIDWNd
3xJWWea5YQKBgQDo6sjm+WGW7d699QUnjssoNcyvdS/Y4UBfScRiJ7koQB0tNtf6
5ey/fhb57B32X4qtzkZ4sckBlQB+I7Ub9FYnAqKneRR0+BynubEny35VUcYjPwaj
JBkMyZVwZh0pcR+wWEc8MDqXk4wffs70/DAswQM2GCppGa8RQ4u5MlMXGQKBgC2J
2xB3fkQmPjZNtt2fVUu48K2q15vZLNkp3SdpJMm4zlYQK6lRGHx8ZNUGLVg7SN1p
aGbXNJclgDmXbDZ3REYvrYVGUerizbG9FOZ36kvvPe/vCo7ik7+oMk2ySCqWGozx
SSsx+iknDc7BVZXFAdf1KAN0jzfJ4oOtUKpcCbHBAoGBAJGyuQUswc7EQeYLn/H5
Zl3kRQv/VU8+lW0tro//osD2Cl5sW1YRaU0+QsOHy/GDPRqcWEqGfyUmFvTK09vc
vSYCbIcS2U18wo7I/j/vicqou0vnNeZCOQHrYvHurGLNUQKqVmVuDArsHx9aDIH0
lJT0SeBwt1iqJi1lIPtEQbOy
-----END PRIVATE KEY-----"#;
const TEST_JWT_PUBLIC_PEM: &str = r#"-----BEGIN PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtFcFa3j02FrMpifjCDk6
JLTBP+tP52BYS4tGsIKReU+jHBdkyf7/gaNSkEORYOd10RmbhZ0iq+Vmfisye3fW
KVEgnP/VDHHd7o4a6d+V+K1zpQjqiJJUV2/005sa5eSzK/dc5RxTksWvvzecjWCk
FbMvWx3ibzqV+ZLVhC1UtiaAG777lYbLlqxcgEvbN4uGtqtY5IPX/r8j3AbRMSMP
znuS9N8emUJTzTy4f3N7kJSu4pDgdfO+yAFiu/zZ4/i/BT5hvRZi6fwnmZ+oLKng
6uipaMn3wsdGYsit4o89sCByGKN0lXt48g40SS4gSCa3Mcp6uGondzhgz08Nd7bI
TwIDAQAB
-----END PUBLIC KEY-----"#;
fn e2e_enabled() -> bool {
matches!(std::env::var(E2E_ENV).as_deref(), Ok("1" | "true"))
}
fn dn(s: &str) -> DeploymentName {
DeploymentName::try_new(s).expect("valid deployment name")
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn group_grants_govern_device_secret_access() -> anyhow::Result<()> {
if !e2e_enabled() {
eprintln!("skipping {E2E_ENV}-gated OpenBao e2e (set {E2E_ENV}=1; requires k3d + helm)");
return Ok(());
}
let _ = tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
)
.try_init();
let stack = shared_stack(StackOptions {
deploy_agent: false,
deploy_operator: false,
..StackOptions::infra_only()
})
.await?;
stack.print_debug_info();
let instance = OpenbaoInstance {
namespace: stack.namespace.clone(),
release: RELEASE.to_string(),
};
let bao = deploy_openbao(&instance).await?;
// The unit under test: deployment "web" readable by group edge-a.
let grants = OpenBaoDeploymentSecretGrants::new(
bao.base_url.clone(),
bao.root_token.clone(),
KV_MOUNT.to_string(),
FLEET_NS.to_string(),
);
grants
.set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()])])
.await?;
// Device in edge-a: login binds the group, secret readable, batch token.
let login_a = bao.jwt_login(&mint_jwt("pi-01", &["edge-a"])).await?;
assert_eq!(
login_a.token_type, "batch",
"fleet role must issue batch tokens"
);
bao.assert_can_read(&login_a.token, "web/config").await?;
// Device in another group: valid identity, no grant.
let login_b = bao.jwt_login(&mint_jwt("pi-02", &["edge-b"])).await?;
bao.assert_denied(&login_b.token, "web/config").await?;
// Detach binds at request time: the in-hand token loses access with
// no re-login involved.
grants.set_deployment_groups(&[(dn("web"), vec![])]).await?;
bao.assert_denied(&login_a.token, "web/config").await?;
// Re-attach binds at request time too.
grants
.set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()])])
.await?;
bao.assert_can_read(&login_a.token, "web/config").await?;
// Group membership is read from the claim only at login: pi-02's old
// token keeps its old groups; a fresh login with edge-a membership
// gets access.
bao.assert_denied(&login_b.token, "web/config").await?;
let login_b2 = bao.jwt_login(&mint_jwt("pi-02", &["edge-a"])).await?;
bao.assert_can_read(&login_b2.token, "web/config").await?;
// Per-device kill switch: disabling the entity blocks its in-flight
// batch token at request time (ADR-025-3 open item 1).
bao.set_entity_disabled("pi-01", true).await?;
bao.assert_denied(&login_a.token, "web/config").await?;
bao.set_entity_disabled("pi-01", false).await?;
bao.assert_can_read(&login_a.token, "web/config").await?;
Ok(())
}
/// Deploys OpenBao + JWT auth configured for the test keypair, seeds the
/// deployment secret, and port-forwards 8200 for host-side HTTP.
async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result<Bao> {
let topology = K8sAnywhereTopology::from_env();
// Raw `Score::interpret` skips topology preparation; without this the
// k8s client is never initialized (`ensure_ready` is what the CLI /
// stack bring-up normally run).
harmony::topology::Topology::ensure_ready(&topology)
.await
.map_err(|e| anyhow::anyhow!("topology not ready: {e:?}"))?;
let scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> = vec![
Box::new(OpenbaoScore {
instance: instance.clone(),
host: "openbao-groups-test.local".to_string(),
openshift: false,
tls_issuer: None,
}),
Box::new(OpenbaoSetupScore {
instance: instance.clone(),
kv_mount: KV_MOUNT.to_string(),
policies: vec![],
users: vec![],
jwt_auth: Some(OpenbaoJwtAuth {
oidc_discovery_url: String::new(),
jwt_validation_pubkeys: TEST_JWT_PUBLIC_PEM.to_string(),
bound_issuer: ISSUER.to_string(),
role_name: ROLE.to_string(),
bound_audiences: AUDIENCE.to_string(),
user_claim: "sub".to_string(),
groups_claim: "groups".to_string(),
token_type: "batch".to_string(),
ttl: "1h".to_string(),
max_ttl: "1h".to_string(),
}),
}),
];
for score in scores {
score.interpret(&Inventory::autoload(), &topology).await?;
}
let k8s = topology.k8s_client().await.map_err(anyhow::Error::msg)?;
let forward = k8s
.port_forward(&instance.pod(), &instance.namespace, 0, 8200)
.await?;
let bao = Bao {
base_url: format!("http://127.0.0.1:{}", forward.port()),
root_token: cached_root_token(instance).map_err(anyhow::Error::msg)?,
http: reqwest::Client::new(),
_forward: forward,
};
bao.http
.post(format!(
"{}/v1/{KV_MOUNT}/data/{FLEET_NS}/web/config",
bao.base_url
))
.header("X-Vault-Token", &bao.root_token)
.json(&json!({ "data": { "value": "e2e" } }))
.send()
.await?
.error_for_status()?;
Ok(bao)
}
struct Bao {
base_url: String,
root_token: String,
http: reqwest::Client,
_forward: harmony_k8s::PortForwardHandle,
}
struct Login {
token: String,
token_type: String,
}
impl Bao {
async fn jwt_login(&self, jwt: &str) -> anyhow::Result<Login> {
let body: serde_json::Value = self
.http
.post(format!("{}/v1/auth/{JWT_MOUNT}/login", self.base_url))
.json(&json!({ "role": ROLE, "jwt": jwt }))
.send()
.await?
.error_for_status()?
.json()
.await?;
let auth = &body["auth"];
Ok(Login {
token: auth["client_token"]
.as_str()
.ok_or_else(|| anyhow::anyhow!("login response missing client_token: {body}"))?
.to_string(),
token_type: auth["token_type"].as_str().unwrap_or_default().to_string(),
})
}
async fn read_secret(&self, token: &str, rel: &str) -> anyhow::Result<reqwest::StatusCode> {
Ok(self
.http
.get(format!(
"{}/v1/{KV_MOUNT}/data/{FLEET_NS}/{rel}",
self.base_url
))
.header("X-Vault-Token", token)
.send()
.await?
.status())
}
async fn assert_can_read(&self, token: &str, rel: &str) -> anyhow::Result<()> {
let status = self.read_secret(token, rel).await?;
anyhow::ensure!(
status.is_success(),
"expected read of {rel} to succeed, got {status}"
);
Ok(())
}
async fn assert_denied(&self, token: &str, rel: &str) -> anyhow::Result<()> {
let status = self.read_secret(token, rel).await?;
anyhow::ensure!(
status == reqwest::StatusCode::FORBIDDEN,
"expected read of {rel} to be denied, got {status}"
);
Ok(())
}
/// Resolve the device's entity through its JWT-mount alias and flip
/// `disabled` — the per-device kill switch.
async fn set_entity_disabled(&self, device: &str, disabled: bool) -> anyhow::Result<()> {
let accessor: serde_json::Value = self
.http
.get(format!("{}/v1/sys/auth", self.base_url))
.header("X-Vault-Token", &self.root_token)
.send()
.await?
.error_for_status()?
.json()
.await?;
let accessor = accessor["data"][&format!("{JWT_MOUNT}/")]["accessor"]
.as_str()
.ok_or_else(|| anyhow::anyhow!("jwt mount accessor not found"))?
.to_string();
let entity: serde_json::Value = self
.http
.post(format!("{}/v1/identity/lookup/entity", self.base_url))
.header("X-Vault-Token", &self.root_token)
.json(&json!({ "alias_name": device, "alias_mount_accessor": accessor }))
.send()
.await?
.error_for_status()?
.json()
.await?;
let id = entity["data"]["id"]
.as_str()
.ok_or_else(|| anyhow::anyhow!("entity for {device} not found: {entity}"))?;
self.http
.post(format!("{}/v1/identity/entity/id/{id}", self.base_url))
.header("X-Vault-Token", &self.root_token)
.json(&json!({ "disabled": disabled }))
.send()
.await?
.error_for_status()?;
Ok(())
}
}
fn mint_jwt(sub: &str, groups: &[&str]) -> String {
use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs();
let claims = json!({
"iss": ISSUER,
"aud": AUDIENCE,
"sub": sub,
"groups": groups,
"iat": now,
"exp": now + 300,
});
encode(
&Header::new(Algorithm::RS256),
&claims,
&EncodingKey::from_rsa_pem(TEST_JWT_PRIVATE_PEM.as_bytes()).expect("test key parses"),
)
.expect("test jwt signs")
}

236
fleet/harmony-fleet-e2e/tests/openbao_policy.rs Normal file
View File

@@ -0,0 +1,236 @@
use std::sync::Arc;
use harmony::inventory::Inventory;
use harmony::modules::openbao::{
OpenbaoInstance, OpenbaoPolicy, OpenbaoScore, OpenbaoSetupScore, OpenbaoUser, cached_root_token,
};
use harmony::score::Score;
use harmony::topology::{K8sAnywhereTopology, K8sclient};
use harmony_fleet_e2e::{StackOptions, shared_stack};
const E2E_ENV: &str = "HARMONY_FLEET_E2E";
const RELEASE: &str = "openbao-policy-test";
const USER: &str = "policy-device";
const PASSWORD: &str = "policy-device-pass";
const POLICY: &str = "deployment-policy-test";
const SECRET_PATH: &str = "fleet-staging/deployment-a";
fn e2e_enabled() -> bool {
matches!(std::env::var(E2E_ENV).as_deref(), Ok("1" | "true"))
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn entity_policy_grants_existing_openbao_token_access() -> anyhow::Result<()> {
if !e2e_enabled() {
skip_e2e();
return Ok(());
}
let stack = openbao_stack().await?;
stack.print_debug_info();
let instance = OpenbaoInstance {
namespace: stack.namespace.clone(),
release: RELEASE.to_string(),
};
deploy_openbao(&instance).await?;
let topology = K8sAnywhereTopology::from_env();
harmony::topology::Topology::ensure_ready(&topology)
.await
.map_err(|e| anyhow::anyhow!("topology not ready: {e:?}"))?;
let k8s = topology.k8s_client().await.map_err(anyhow::Error::msg)?;
let root_token = cached_root_token(&instance).map_err(anyhow::Error::msg)?;
let user_token = login_user(&k8s, &instance).await?;
let entity_id = token_entity_id(&k8s, &instance, &user_token).await?;
set_entity_policies(&k8s, &instance, &root_token, &entity_id, "").await?;
assert_secret_denied(&k8s, &instance, &user_token).await?;
set_entity_policies(&k8s, &instance, &root_token, &entity_id, POLICY).await?;
assert_secret_allowed(&k8s, &instance, &user_token).await?;
// FIXME this cleanup might not happen if the previous invocation fails
set_entity_policies(&k8s, &instance, &root_token, &entity_id, "").await?;
Ok(())
}
async fn openbao_stack() -> anyhow::Result<Arc<harmony_fleet_e2e::Stack>> {
let _ = tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
)
.try_init();
Ok(shared_stack(StackOptions {
deploy_agent: false,
deploy_operator: false,
..StackOptions::infra_only()
})
.await?)
}
async fn deploy_openbao(instance: &OpenbaoInstance) -> anyhow::Result<()> {
let topology = K8sAnywhereTopology::from_env();
// Raw `Score::interpret` skips topology preparation; without this the
// k8s client is never initialized (`ensure_ready` is what the CLI /
// stack bring-up normally run).
harmony::topology::Topology::ensure_ready(&topology)
.await
.map_err(|e| anyhow::anyhow!("topology not ready: {e:?}"))?;
let policy_hcl = format!(
r#"path "secret/data/{SECRET_PATH}" {{ capabilities = ["read"] }}
path "secret/metadata/{SECRET_PATH}" {{ capabilities = ["read"] }}"#
);
let scores: Vec<Box<dyn Score<K8sAnywhereTopology>>> = vec![
Box::new(OpenbaoScore {
instance: instance.clone(),
host: "openbao-policy-test.local".to_string(),
openshift: false,
tls_issuer: None,
}),
Box::new(OpenbaoSetupScore {
instance: instance.clone(),
kv_mount: "secret".to_string(),
policies: vec![OpenbaoPolicy {
name: POLICY.to_string(),
hcl: policy_hcl,
}],
users: vec![OpenbaoUser {
username: USER.to_string(),
password: PASSWORD.to_string(),
policies: vec![],
}],
jwt_auth: None,
}),
];
for score in scores {
score.interpret(&Inventory::autoload(), &topology).await?;
}
let k8s = topology.k8s_client().await.map_err(anyhow::Error::msg)?;
let root_token = cached_root_token(instance).map_err(anyhow::Error::msg)?;
bao(
&k8s,
instance,
&root_token,
"bao kv put secret/fleet-staging/deployment-a value=e2e",
)
.await?;
Ok(())
}
async fn login_user(
k8s: &harmony_k8s::K8sClient,
instance: &OpenbaoInstance,
) -> anyhow::Result<String> {
let out = exec(
k8s,
instance,
&format!("bao login -method=userpass -format=json username={USER} password={PASSWORD}"),
)
.await?;
Ok(
serde_json::from_str::<serde_json::Value>(&out)?["auth"]["client_token"]
.as_str()
.ok_or_else(|| anyhow::anyhow!("login response missing auth.client_token: {out}"))?
.to_string(),
)
}
async fn token_entity_id(
k8s: &harmony_k8s::K8sClient,
instance: &OpenbaoInstance,
token: &str,
) -> anyhow::Result<String> {
let out = bao(k8s, instance, token, "bao token lookup -format=json").await?;
Ok(
serde_json::from_str::<serde_json::Value>(&out)?["data"]["entity_id"]
.as_str()
.ok_or_else(|| anyhow::anyhow!("token lookup response missing data.entity_id: {out}"))?
.to_string(),
)
}
async fn set_entity_policies(
k8s: &harmony_k8s::K8sClient,
instance: &OpenbaoInstance,
root_token: &str,
entity_id: &str,
policies: &str,
) -> anyhow::Result<()> {
bao(
k8s,
instance,
root_token,
&format!("bao write identity/entity/id/{entity_id} policies={policies}"),
)
.await?;
Ok(())
}
async fn assert_secret_denied(
k8s: &harmony_k8s::K8sClient,
instance: &OpenbaoInstance,
token: &str,
) -> anyhow::Result<()> {
let result = bao(
k8s,
instance,
token,
&format!("bao kv get secret/{SECRET_PATH}"),
)
.await;
assert!(result.is_err(), "secret read unexpectedly succeeded");
Ok(())
}
async fn assert_secret_allowed(
k8s: &harmony_k8s::K8sClient,
instance: &OpenbaoInstance,
token: &str,
) -> anyhow::Result<()> {
bao(
k8s,
instance,
token,
&format!("bao kv get secret/{SECRET_PATH}"),
)
.await?;
Ok(())
}
async fn bao(
k8s: &harmony_k8s::K8sClient,
instance: &OpenbaoInstance,
token: &str,
command: &str,
) -> anyhow::Result<String> {
exec(
k8s,
instance,
&format!("export VAULT_TOKEN={token} && {command}"),
)
.await
}
async fn exec(
k8s: &harmony_k8s::K8sClient,
instance: &OpenbaoInstance,
command: &str,
) -> anyhow::Result<String> {
k8s.exec_pod_capture_output(
&instance.pod(),
Some(&instance.namespace),
vec!["sh", "-c", command],
)
.await
.map_err(|e| anyhow::anyhow!(e))
}
fn skip_e2e() {
eprintln!(
"skipping {E2E_ENV}-gated OpenBao e2e test (set {E2E_ENV}=1 to run; requires k3d + helm)"
);
}

2
fleet/harmony-fleet-e2e/tests/operator.rs
View File

@@ -127,6 +127,7 @@ async fn create_fleet_deployment(deployments: &Api<Deployment>, name: &str) -> a
..Default::default()
},
spec: DeploymentSpec {
allowed_groups: None,
target_selector: LabelSelector::default(),
score: ReconcileScore::PodmanV0(PodmanV0Score {
services: vec![PodmanService {
@@ -134,6 +135,7 @@ async fn create_fleet_deployment(deployments: &Api<Deployment>, name: &str) -> a
image: "docker.io/library/hello-world:latest".to_string(),
ports: vec![],
env: vec![],
secret_env: vec![],
volumes: vec![],
restart_policy: Default::default(),
}],

1
fleet/harmony-fleet-e2e/tests/vm_deploy_lifecycle.rs
View File

@@ -182,6 +182,7 @@ fn podman_score(image_tag: &str) -> PodmanV0Score {
image: format!("docker.io/library/{image_tag}"),
ports: vec![],
env: vec![],
secret_env: vec![],
volumes: vec![],
restart_policy: RestartPolicy::default(),
}],

2
fleet/harmony-fleet-e2e/tests/vm_isolation.rs
View File

@@ -89,6 +89,7 @@ async fn agent_ignores_other_devices_keys() -> anyhow::Result<()> {
image: "docker.io/library/nginx:alpine".to_string(),
ports: vec![],
env: vec![],
secret_env: vec![],
volumes: vec![],
restart_policy: RestartPolicy::default(),
}],
@@ -135,5 +136,6 @@ fn synthesize_agent_config(device_id: &str) -> AgentConfig {
nats_pass: "e2e-device".to_string(),
},
labels: BTreeMap::new(),
openbao: None,
}
}

307
fleet/harmony-fleet-e2e/tests/vm_secret_access.rs Normal file
View File

@@ -0,0 +1,307 @@
//! Full-stack device secret access (ADR-025), end to end with nothing
//! mocked:
//!
//! 1. the agent on a VM connects to NATS **through the auth callout**
//! with its Zitadel machine identity (the readiness ping proves the
//! whole SSO → callout → NATS chain);
//! 2. a deployment referencing a secret lands in desired state;
//! 3. the agent logs into OpenBao with the same Zitadel JWT — the
//! `groups` claim comes from the flattening Action, the group grant
//! from the same `OpenBaoDeploymentSecretGrants` the operator
//! composes — fetches the secret and injects it into the container
//! (SSH `podman inspect` is the ground truth);
//! 4. deleting the deployment's OpenBao policy makes the next fetch
//! fail, surfaced as `Phase::Failed` naming the denied secret.
//!
//! Gating: skipped unless `HARMONY_FLEET_VM_E2E=1`.
use std::time::Duration;
use harmony::modules::fleet::secret_access::DeploymentSecretGrants;
use harmony::modules::openbao::OpenBaoDeploymentSecretGrants;
use harmony::modules::podman::{EnvVar, PodmanService, PodmanV0Score, SecretEnvVar};
use harmony::topology::{K8sAnywhereTopology, K8sclient, RestartPolicy};
use harmony_fleet_e2e::callout::{DEVICE_ROLE, KV_MOUNT, SECRET_PREFIX};
use harmony_fleet_e2e::{
AdminKv, AuthMode, PhaseExpectation, VmStack, VmStackOptions, shared_vm_stack,
};
use harmony_reconciler_contracts::{DeploymentName, Phase};
use serde_json::json;
const ENV_GATE: &str = "HARMONY_FLEET_VM_E2E";
fn enabled() -> bool {
matches!(std::env::var(ENV_GATE).as_deref(), Ok("1" | "true"))
}
fn dn(s: &str) -> DeploymentName {
DeploymentName::try_new(s).expect("test-static valid deployment name")
}
/// First image pull on a TCG aarch64 guest can take minutes.
const RUN_BUDGET: Duration = Duration::from_secs(600);
const FAIL_BUDGET: Duration = Duration::from_secs(120);
/// Both tests drive the same device + OpenBao; serialize so policy
/// surgery in one can't race the other's apply.
static SERIAL: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
fn callout_stack_options() -> VmStackOptions {
VmStackOptions {
auth_mode: AuthMode::Callout,
// `device` drives NATS permissions in the callout; `edge-a`
// is the OpenBao group the deployments below are granted to.
device_roles: vec![DEVICE_ROLE.to_string(), "edge-a".to_string()],
..VmStackOptions::from_env()
}
}
fn secret_score(service: &str, spec_marker: &str) -> PodmanV0Score {
PodmanV0Score {
services: vec![PodmanService {
name: service.to_string(),
image: "docker.io/library/nginx:alpine".to_string(),
ports: vec![],
// Changing the marker changes the serialized spec, which
// is how the tests force a re-apply (and with it a fresh
// secret resolution).
env: vec![EnvVar::new("SPEC_MARKER", spec_marker)],
secret_env: vec![SecretEnvVar {
name: "DB_PASSWORD".to_string(),
secret: "db_password".to_string(),
}],
volumes: vec![],
restart_policy: RestartPolicy::default(),
}],
}
}
/// Host-side OpenBao admin surface: port-forward + root token.
struct BaoAdmin {
base_url: String,
root_token: String,
http: reqwest::Client,
_forward: harmony_k8s::PortForwardHandle,
}
impl BaoAdmin {
async fn connect(stack: &VmStack) -> anyhow::Result<Self> {
let handles = stack
.infra
.callout
.as_ref()
.expect("callout stack carries OpenBao handles");
let topology = K8sAnywhereTopology::from_env();
harmony::topology::Topology::ensure_ready(&topology)
.await
.map_err(|e| anyhow::anyhow!("topology not ready: {e:?}"))?;
let k8s = topology.k8s_client().await.map_err(anyhow::Error::msg)?;
let forward = k8s
.port_forward(&handles.openbao.pod(), &handles.openbao.namespace, 0, 8200)
.await?;
Ok(Self {
base_url: format!("http://127.0.0.1:{}", forward.port()),
root_token: handles.openbao_root_token.clone(),
http: reqwest::Client::new(),
_forward: forward,
})
}
async fn put_secret(
&self,
deployment: &DeploymentName,
name: &str,
value: &str,
) -> anyhow::Result<()> {
self.http
.post(format!(
"{}/v1/{KV_MOUNT}/data/{SECRET_PREFIX}/{deployment}/{name}",
self.base_url
))
.header("X-Vault-Token", &self.root_token)
.json(&json!({ "data": { "value": value } }))
.send()
.await?
.error_for_status()?;
Ok(())
}
fn grants(&self) -> OpenBaoDeploymentSecretGrants {
OpenBaoDeploymentSecretGrants::new(
self.base_url.clone(),
self.root_token.clone(),
KV_MOUNT.to_string(),
SECRET_PREFIX.to_string(),
)
}
/// The negative test's act of sabotage: drop the deployment's
/// policy directly (operator drift / manual revocation), leaving
/// group attachments in place.
async fn delete_deployment_policy(&self, deployment: &DeploymentName) -> anyhow::Result<()> {
self.http
.delete(format!(
"{}/v1/sys/policies/acl/deployment-{deployment}",
self.base_url
))
.header("X-Vault-Token", &self.root_token)
.send()
.await?
.error_for_status()?;
Ok(())
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn device_reads_deployment_secret_via_sso_groups() -> anyhow::Result<()> {
if !enabled() {
eprintln!("skipping {ENV_GATE}-gated VM e2e test (set {ENV_GATE}=1 to run)");
return Ok(());
}
let _serial = SERIAL.lock().await;
let _ = tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
)
.try_init();
let stack = shared_vm_stack(callout_stack_options()).await?;
stack.print_debug_info();
// Ping over the callout-authenticated NATS connection — the agent
// is online only if its Zitadel JWT made it through the callout.
stack.wait_until_ready(Duration::from_secs(120)).await?;
let device = stack.devices.first().expect("one VM device");
let device_id = device.device_id.to_string();
let deployment = dn("secret-web");
let bao = BaoAdmin::connect(&stack).await?;
// Fresh per-test connection — the stack's shared client dies
// with the bring-up test's runtime (`Stack::connect_admin`).
let nats = stack.infra.connect_admin().await?;
let admin = AdminKv::connect(&nats).await?;
let secret_value = format!("hunter2-{}", uuid::Uuid::new_v4().simple());
bao.put_secret(&deployment, "db_password", &secret_value)
.await?;
bao.grants()
.set_deployment_groups(&[(deployment.clone(), vec!["edge-a".to_string()])])
.await?;
admin
.put_podman(
&device_id,
&deployment,
&secret_score("secret-web-svc", "v1"),
)
.await?;
let state = admin
.wait_for_phase(
&device_id,
&deployment,
PhaseExpectation::running(),
RUN_BUDGET,
)
.await?
.expect("Running phase implies a state entry");
assert_eq!(state.phase, Phase::Running);
// Ground truth: the secret VALUE made it into the container env.
let inspect = device
.ssh(
"sudo -iu fleet-agent podman inspect secret-web-svc \
--format '{{json .Config.Env}}'",
)
.await?
.into_successful()
.map_err(|e| anyhow::anyhow!("podman inspect failed: {e}"))?;
assert!(
inspect
.stdout
.contains(&format!("DB_PASSWORD={secret_value}")),
"container env must carry the fetched secret, got:\n{}",
inspect.stdout,
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn removed_policy_denies_secret_fetch() -> anyhow::Result<()> {
if !enabled() {
eprintln!("skipping {ENV_GATE}-gated VM e2e test (set {ENV_GATE}=1 to run)");
return Ok(());
}
let _serial = SERIAL.lock().await;
let _ = tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
)
.try_init();
let stack = shared_vm_stack(callout_stack_options()).await?;
stack.wait_until_ready(Duration::from_secs(120)).await?;
let device = stack.devices.first().expect("one VM device");
let device_id = device.device_id.to_string();
let deployment = dn("secret-denied");
let bao = BaoAdmin::connect(&stack).await?;
// Fresh per-test connection — the stack's shared client dies
// with the bring-up test's runtime (`Stack::connect_admin`).
let nats = stack.infra.connect_admin().await?;
let admin = AdminKv::connect(&nats).await?;
bao.put_secret(&deployment, "db_password", "soon-revoked")
.await?;
bao.grants()
.set_deployment_groups(&[(deployment.clone(), vec!["edge-a".to_string()])])
.await?;
// Sanity: with the grant in place the deployment converges.
admin
.put_podman(
&device_id,
&deployment,
&secret_score("secret-denied-svc", "v1"),
)
.await?;
admin
.wait_for_phase(
&device_id,
&deployment,
PhaseExpectation::running(),
RUN_BUDGET,
)
.await?;
// Remove the policy; the spec bump forces a re-apply and with it
// a fresh secret resolution (resolution binds at apply time —
// converged deployments never re-read, by design).
bao.delete_deployment_policy(&deployment).await?;
admin
.put_podman(
&device_id,
&deployment,
&secret_score("secret-denied-svc", "v2"),
)
.await?;
let state = admin
.wait_for_phase(
&device_id,
&deployment,
PhaseExpectation::OneOf(vec![Phase::Failed]),
FAIL_BUDGET,
)
.await?
.expect("Failed phase implies a state entry");
assert_eq!(state.phase, Phase::Failed);
let err = state.last_error.unwrap_or_default();
assert!(
err.contains("fetching secret")
&& (err.contains("permission denied") || err.contains("403")),
"failure must name the denied secret fetch, got: {err}",
);
Ok(())
}

448
fleet/harmony-fleet-operator/src/fleet_aggregator.rs
View File

@@ -33,12 +33,18 @@ use kube::{Client, ResourceExt};
use serde_json::json;
use tokio::sync::Mutex;
use harmony::modules::fleet::groups::DeviceGroupSource;
use harmony::modules::fleet::operator::{
AggregateLastError, Deployment, DeploymentAggregate, Device,
};
use harmony::modules::fleet::secret_access::DeploymentSecretGrants;
use tracing::warn;
const PATCH_TICK: Duration = Duration::from_secs(1);
/// How often the admin-managed group membership is re-fetched. Routine
/// convergence is bounded by the device token TTL anyway (ADR-025), so
/// a relaxed poll is fine.
const GROUP_POLL: Duration = Duration::from_secs(60);
// ---------------------------------------------------------------------------
// State
@@ -74,6 +80,9 @@ pub struct DevicePair {
pub struct CachedDeployment {
key: DeploymentKey,
deployment_name: DeploymentName,
/// `spec.allowedGroups` — the security gate. `None` = ungated (and
/// secretless); `Some` = only group members are schedulable.
allowed_groups: Option<Vec<String>>,
selector: LabelSelector,
/// JSON-serialized score payload ready to `put` into
/// desired-state. Cached because the same bytes are written to
@@ -87,6 +96,11 @@ pub struct FleetState {
deployments: HashMap<DeploymentKey, CachedDeployment>,
/// Cached Device labels, keyed by `metadata.name`.
devices: HashMap<String, BTreeMap<String, String>>,
/// Admin-managed device → groups snapshot from the
/// [`DeviceGroupSource`]. Distinct from labels by design: labels
/// are device-reported placement vocabulary, groups are the
/// security boundary (ADR-025).
device_groups: HashMap<String, HashSet<String>>,
/// Latest DeploymentState per (device, deployment) pair.
states: HashMap<DevicePair, DeploymentState>,
/// Which devices have we pushed desired-state for, per deployment?
@@ -133,14 +147,40 @@ pub fn selector_matches(selector: &LabelSelector, labels: &BTreeMap<String, Stri
true
}
/// Set of Device names currently matching `selector`.
fn matched_devices(
selector: &LabelSelector,
devices: &HashMap<String, BTreeMap<String, String>>,
) -> HashSet<String> {
devices
/// The security gate: is the device in one of the deployment's allowed
/// groups? `None` = ungated. `Some` requires intersection — including
/// `Some(vec![])`, which matches nothing (declared but empty is a
/// revocation, not a wildcard).
pub fn group_allows(
allowed: &Option<Vec<String>>,
device_groups: Option<&HashSet<String>>,
) -> bool {
match allowed {
None => true,
Some(allowed) => {
device_groups.is_some_and(|groups| allowed.iter().any(|g| groups.contains(g)))
}
}
}
/// Is `deployment` allowed (groups) and placed (labels) on this device?
fn device_eligible(
deployment: &CachedDeployment,
labels: &BTreeMap<String, String>,
device_groups: Option<&HashSet<String>>,
) -> bool {
group_allows(&deployment.allowed_groups, device_groups)
&& selector_matches(&deployment.selector, labels)
}
/// Set of Device names the deployment is currently allowed and placed on.
fn matched_devices(deployment: &CachedDeployment, state: &FleetState) -> HashSet<String> {
state
.devices
.iter()
.filter(|(_, labels)| selector_matches(selector, labels))
.filter(|(name, labels)| {
device_eligible(deployment, labels, state.device_groups.get(*name))
})
.map(|(name, _)| name.clone())
.collect()
}
@@ -149,7 +189,12 @@ fn matched_devices(
// Top-level run
// ---------------------------------------------------------------------------
pub async fn run(client: Client, js: async_nats::jetstream::Context) -> anyhow::Result<()> {
pub async fn run(
client: Client,
js: async_nats::jetstream::Context,
secret_grants: Option<Arc<dyn DeploymentSecretGrants>>,
group_source: Option<Arc<dyn DeviceGroupSource>>,
) -> anyhow::Result<()> {
let state_bucket = js
.create_key_value(async_nats::jetstream::kv::Config {
bucket: BUCKET_DEVICE_STATE.to_string(),
@@ -197,8 +242,11 @@ pub async fn run(client: Client, js: async_nats::jetstream::Context) -> anyhow::
let deployment_watcher_handle = {
let state = state.clone();
let desired = desired_bucket.clone();
let grants = secret_grants.clone();
tokio::spawn(async move {
if let Err(e) = run_deployment_watcher(deployments_api.clone(), state, desired).await {
if let Err(e) =
run_deployment_watcher(deployments_api.clone(), state, desired, grants).await
{
tracing::warn!(error = %e, "aggregator: deployment watcher exited");
}
})
@@ -214,6 +262,31 @@ pub async fn run(client: Client, js: async_nats::jetstream::Context) -> anyhow::
})
};
// Group membership poll: groups change rarely (admin action), and a
// device's secret access converges at its token TTL regardless —
// the poll only moves *scheduling* in line with membership.
let group_poll_handle = {
let state = state.clone();
let desired = desired_bucket.clone();
tokio::spawn(async move {
let Some(source) = group_source else {
// No source configured: park forever — returning would
// resolve the select! below and stop the aggregator.
std::future::pending::<()>().await;
unreachable!()
};
let mut ticker = tokio::time::interval(GROUP_POLL);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
ticker.tick().await;
match source.device_groups().await {
Ok(snapshot) => apply_device_groups(&state, &desired, snapshot).await,
Err(e) => warn!(error = %e, "aggregator: group source fetch failed"),
}
}
})
};
let patch_state = state.clone();
let patch_loop = async move {
let mut ticker = tokio::time::interval(PATCH_TICK);
@@ -231,6 +304,7 @@ pub async fn run(client: Client, js: async_nats::jetstream::Context) -> anyhow::
_ = state_watcher_handle => Ok(()),
_ = deployment_watcher_handle => Ok(()),
_ = device_watcher_handle => Ok(()),
_ = group_poll_handle => Ok(()),
}
}
@@ -281,6 +355,41 @@ async fn run_state_kv_watcher(bucket: Store, state: SharedFleetState) -> anyhow:
Ok(())
}
/// Push the allowed-groups grant for `deployments` to the secrets
/// backend in one batch. A deployment absent from the cache (deleted, or
/// no `allowedGroups` declared) resolves to the empty set = revocation.
/// Non-fatal: a backend hiccup logs and is retried on the next
/// deployment change rather than tearing down reconciliation. Skipped
/// entirely when no backend is configured (dev without OpenBao).
async fn sync_deployment_grants(
state: &SharedFleetState,
grants: Option<&Arc<dyn DeploymentSecretGrants>>,
deployments: impl IntoIterator<Item = DeploymentName>,
) {
let Some(grants) = grants else { return };
let batch: Vec<(DeploymentName, Vec<String>)> = {
let guard = state.lock().await;
deployments
.into_iter()
.map(|dn| {
let groups = guard
.deployments
.values()
.find(|c| c.deployment_name == dn)
.and_then(|c| c.allowed_groups.clone())
.unwrap_or_default();
(dn, groups)
})
.collect()
};
if batch.is_empty() {
return;
}
if let Err(e) = grants.set_deployment_groups(&batch).await {
warn!(error = %e, "aggregator: secret grant sync failed");
}
}
/// Record a device's latest state, dedup against older timestamps,
/// maintain last_error, mark the deployment dirty.
pub fn apply_state(state: &mut FleetState, pair: DevicePair, ds: DeploymentState) {
@@ -359,15 +468,16 @@ async fn run_deployment_watcher(
api: Api<Deployment>,
state: SharedFleetState,
desired: Store,
grants: Option<Arc<dyn DeploymentSecretGrants>>,
) -> anyhow::Result<()> {
let mut stream = watcher::watcher(api, WatcherConfig::default()).boxed();
while let Some(event) = stream.try_next().await? {
match event {
Event::Apply(cr) | Event::InitApply(cr) => {
on_deployment_upsert(&state, &desired, cr).await;
on_deployment_upsert(&state, &desired, grants.as_ref(), cr).await;
}
Event::Delete(cr) => {
on_deployment_delete(&state, &desired, cr).await;
on_deployment_delete(&state, &desired, grants.as_ref(), cr).await;
}
Event::Init | Event::InitDone => {}
}
@@ -375,7 +485,12 @@ async fn run_deployment_watcher(
Ok(())
}
async fn on_deployment_upsert(state: &SharedFleetState, desired: &Store, cr: Deployment) {
async fn on_deployment_upsert(
state: &SharedFleetState,
desired: &Store,
grants: Option<&Arc<dyn DeploymentSecretGrants>>,
cr: Deployment,
) {
let Some(key) = DeploymentKey::from_cr(&cr) else {
return;
};
@@ -383,7 +498,6 @@ async fn on_deployment_upsert(state: &SharedFleetState, desired: &Store, cr: Dep
tracing::warn!(name = %key.name, "aggregator: CR name is not a valid DeploymentName, skipping");
return;
};
let selector = cr.spec.target_selector.clone();
let score_json = match serde_json::to_vec(&cr.spec.score) {
Ok(v) => v,
Err(e) => {
@@ -391,19 +505,18 @@ async fn on_deployment_upsert(state: &SharedFleetState, desired: &Store, cr: Dep
return;
}
};
let cached = CachedDeployment {
key: key.clone(),
deployment_name: deployment_name.clone(),
allowed_groups: cr.spec.allowed_groups.clone(),
selector: cr.spec.target_selector.clone(),
score_json: score_json.clone(),
};
let (new_targets, previous_targets) = {
let mut guard = state.lock().await;
let new_targets = matched_devices(&selector, &guard.devices);
guard.deployments.insert(
key.clone(),
CachedDeployment {
key: key.clone(),
deployment_name: deployment_name.clone(),
selector: selector.clone(),
score_json: score_json.clone(),
},
);
let new_targets = matched_devices(&cached, &guard);
guard.deployments.insert(key.clone(), cached);
let previous = guard
.owned_targets
.remove(&deployment_name)
@@ -415,6 +528,10 @@ async fn on_deployment_upsert(state: &SharedFleetState, desired: &Store, cr: Dep
(new_targets, previous)
};
// Grant before workload signal: never tell a device to run a
// deployment whose secrets it cannot yet read.
sync_deployment_grants(state, grants, std::iter::once(deployment_name.clone())).await;
reconcile_kv(
desired,
&deployment_name,
@@ -425,7 +542,12 @@ async fn on_deployment_upsert(state: &SharedFleetState, desired: &Store, cr: Dep
.await;
}
async fn on_deployment_delete(state: &SharedFleetState, desired: &Store, cr: Deployment) {
async fn on_deployment_delete(
state: &SharedFleetState,
desired: &Store,
grants: Option<&Arc<dyn DeploymentSecretGrants>>,
cr: Deployment,
) {
let Some(key) = DeploymentKey::from_cr(&cr) else {
return;
};
@@ -453,6 +575,9 @@ async fn on_deployment_delete(state: &SharedFleetState, desired: &Store, cr: Dep
tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete on CR delete failed");
}
}
// Gone from the cache, so the grant resolves to empty = revocation.
sync_deployment_grants(state, grants, std::iter::once(deployment_name)).await;
}
// ---------------------------------------------------------------------------
@@ -483,63 +608,54 @@ async fn run_device_watcher(
Ok(())
}
async fn on_device_upsert(state: &SharedFleetState, desired: &Store, dev: Device) {
let name = dev.name_any();
let labels: BTreeMap<String, String> = dev.metadata.labels.clone().unwrap_or_default();
/// Re-evaluate every deployment's ownership of `name` from the current
/// labels + groups caches; mutates `owned_targets`/`dirty` and returns
/// the (deployment, was, now) transitions for KV reconciliation.
fn reevaluate_device(guard: &mut FleetState, name: &str) -> Vec<(CachedDeployment, bool, bool)> {
let labels = guard.devices.get(name).cloned().unwrap_or_default();
let groups = guard.device_groups.get(name).cloned();
let snapshot: Vec<CachedDeployment> = guard.deployments.values().cloned().collect();
// For every deployment, compute whether this single device now
// matches vs. previously matched; diff against owned_targets;
// collect the KV writes/deletes to perform after the lock is
// released.
let per_deployment: Vec<(CachedDeployment, bool, bool)> = {
let mut guard = state.lock().await;
let snapshot: Vec<CachedDeployment> = guard.deployments.values().cloned().collect();
let previously_matched: HashMap<DeploymentName, bool> = snapshot
.iter()
.map(|d| {
let was = guard
.owned_targets
.get(&d.deployment_name)
.is_some_and(|set| set.contains(&name));
(d.deployment_name.clone(), was)
})
.collect();
guard.devices.insert(name.clone(), labels.clone());
let mut out = Vec::with_capacity(snapshot.len());
for d in snapshot {
let was = previously_matched
.get(&d.deployment_name)
.copied()
.unwrap_or(false);
let now = selector_matches(&d.selector, &labels);
if was != now {
let targets = guard
.owned_targets
.entry(d.deployment_name.clone())
.or_default();
if now {
targets.insert(name.clone());
} else {
targets.remove(&name);
}
guard.dirty.insert(d.key.clone());
let mut out = Vec::with_capacity(snapshot.len());
for d in snapshot {
let was = guard
.owned_targets
.get(&d.deployment_name)
.is_some_and(|set| set.contains(name));
let now = device_eligible(&d, &labels, groups.as_ref());
if was != now {
let targets = guard
.owned_targets
.entry(d.deployment_name.clone())
.or_default();
if now {
targets.insert(name.to_string());
} else {
targets.remove(name);
}
out.push((d, was, now));
guard.dirty.insert(d.key.clone());
}
out
};
out.push((d, was, now));
}
out
}
for (cached, was, now) in per_deployment {
/// Apply the KV writes/deletes a [`reevaluate_device`] diff calls for.
async fn reconcile_device_kv(
desired: &Store,
name: &str,
transitions: Vec<(CachedDeployment, bool, bool)>,
) {
for (cached, was, now) in transitions {
match (was, now) {
(false, true) => {
let k = desired_state_key(&name, &cached.deployment_name);
let k = desired_state_key(name, &cached.deployment_name);
if let Err(e) = desired.put(&k, cached.score_json.clone().into()).await {
tracing::debug!(key = %k, error = %e, "aggregator: desired-state put failed");
}
}
(true, false) => {
let k = desired_state_key(&name, &cached.deployment_name);
let k = desired_state_key(name, &cached.deployment_name);
if let Err(e) = desired.delete(&k).await {
tracing::debug!(key = %k, error = %e, "aggregator: desired-state delete failed");
}
@@ -549,6 +665,52 @@ async fn on_device_upsert(state: &SharedFleetState, desired: &Store, dev: Device
}
}
async fn on_device_upsert(state: &SharedFleetState, desired: &Store, dev: Device) {
let name = dev.name_any();
let labels: BTreeMap<String, String> = dev.metadata.labels.clone().unwrap_or_default();
let transitions = {
let mut guard = state.lock().await;
guard.devices.insert(name.clone(), labels);
reevaluate_device(&mut guard, &name)
};
reconcile_device_kv(desired, &name, transitions).await;
}
/// Swap in a fresh device → groups snapshot and re-evaluate scheduling
/// for every device whose membership changed. Secret grants are
/// untouched — they bind deployments to groups, not to devices.
async fn apply_device_groups(
state: &SharedFleetState,
desired: &Store,
snapshot: HashMap<String, HashSet<String>>,
) {
let changed: Vec<(String, Vec<(CachedDeployment, bool, bool)>)> = {
let mut guard = state.lock().await;
if guard.device_groups == snapshot {
return;
}
let affected: HashSet<String> = guard
.device_groups
.keys()
.chain(snapshot.keys())
.filter(|d| guard.device_groups.get(*d) != snapshot.get(*d))
.cloned()
.collect();
guard.device_groups = snapshot;
affected
.into_iter()
.map(|name| {
let transitions = reevaluate_device(&mut guard, &name);
(name, transitions)
})
.collect()
};
for (name, transitions) in changed {
reconcile_device_kv(desired, &name, transitions).await;
}
}
async fn on_device_delete(state: &SharedFleetState, desired: &Store, dev: Device) {
let name = dev.name_any();
let removed_from: Vec<DeploymentName> = {
@@ -738,6 +900,7 @@ mod tests {
name: name.to_string(),
},
deployment_name: dn(name),
allowed_groups: None,
selector: LabelSelector {
match_labels: Some(ml),
match_expressions: None,
@@ -746,6 +909,11 @@ mod tests {
}
}
fn with_groups(mut cached: CachedDeployment, groups: &[&str]) -> CachedDeployment {
cached.allowed_groups = Some(groups.iter().map(|s| s.to_string()).collect());
cached
}
fn pair(device: &str, deployment: &str) -> DevicePair {
DevicePair {
device_id: device.to_string(),
@@ -755,26 +923,76 @@ mod tests {
#[test]
fn selector_match_labels_only() {
// `zone` not `group`: labels are placement vocabulary; the
// authorization group is a distinct, admin-only concept.
let mut ml = BTreeMap::new();
ml.insert("group".to_string(), "edge-a".to_string());
ml.insert("zone".to_string(), "lab".to_string());
let sel = LabelSelector {
match_labels: Some(ml),
match_expressions: None,
};
let mut matching = BTreeMap::new();
matching.insert("group".to_string(), "edge-a".to_string());
matching.insert("zone".to_string(), "lab".to_string());
matching.insert("arch".to_string(), "aarch64".to_string());
assert!(selector_matches(&sel, &matching));
let mut non_matching = BTreeMap::new();
non_matching.insert("group".to_string(), "edge-b".to_string());
non_matching.insert("zone".to_string(), "warehouse".to_string());
assert!(!selector_matches(&sel, &non_matching));
let empty = BTreeMap::new();
assert!(!selector_matches(&sel, &empty));
}
#[test]
fn group_gate_semantics() {
let edge_a = HashSet::from(["edge-a".to_string()]);
// Undeclared = ungated.
assert!(group_allows(&None, Some(&edge_a)));
assert!(group_allows(&None, None));
// Declared = member devices only; unknown membership fails closed.
let allowed = Some(vec!["edge-a".to_string(), "edge-b".to_string()]);
assert!(group_allows(&allowed, Some(&edge_a)));
assert!(!group_allows(
&allowed,
Some(&HashSet::from(["other".to_string()]))
));
assert!(!group_allows(&allowed, None));
// Declared-but-empty is a revocation, not a wildcard.
assert!(!group_allows(&Some(vec![]), Some(&edge_a)));
}
#[test]
fn groups_gate_and_labels_place() {
let mut s = FleetState::default();
let mut lab = BTreeMap::new();
lab.insert("zone".to_string(), "lab".to_string());
// Three devices with the matching label; only pi-01 is in the
// allowed group, pi-03 has no known membership at all.
s.devices.insert("pi-01".to_string(), lab.clone());
s.devices.insert("pi-02".to_string(), lab.clone());
s.devices.insert("pi-03".to_string(), lab);
s.device_groups
.insert("pi-01".to_string(), HashSet::from(["edge-a".to_string()]));
s.device_groups
.insert("pi-02".to_string(), HashSet::from(["edge-b".to_string()]));
let gated = with_groups(cached("ns", "web", "zone", "lab"), &["edge-a"]);
assert_eq!(matched_devices(&gated, &s), devs(&["pi-01"]));
// A device-chosen label can never widen access: the label
// matches, the group doesn't, the device is not targeted.
let ungated = cached("ns", "web", "zone", "lab");
assert_eq!(
matched_devices(&ungated, &s),
devs(&["pi-01", "pi-02", "pi-03"])
);
}
#[test]
fn empty_selector_matches_everything() {
let sel = LabelSelector::default();
@@ -786,7 +1004,7 @@ mod tests {
#[test]
fn compute_aggregate_counts_matched_devices() {
let cached = cached("fleet-demo", "hello", "group", "edge-a");
let cached = cached("fleet-demo", "hello", "zone", "lab");
let key = cached.key.clone();
let mut s = FleetState::default();
@@ -817,25 +1035,73 @@ mod tests {
assert_eq!(agg.pending, 1);
}
fn devs(ids: &[&str]) -> HashSet<String> {
ids.iter().map(|s| s.to_string()).collect()
}
#[tokio::test]
async fn sync_pushes_declared_groups_per_deployment() {
use harmony::modules::fleet::secret_access::InMemoryDeploymentSecretGrants;
let mut s = FleetState::default();
let web = with_groups(cached("ns", "web", "zone", "lab"), &["edge-a", "edge-b"]);
let db = cached("ns", "db", "zone", "lab"); // no allowedGroups → no grant
s.deployments.insert(web.key.clone(), web);
s.deployments.insert(db.key.clone(), db);
let state: SharedFleetState = Arc::new(Mutex::new(s));
let inmem = Arc::new(InMemoryDeploymentSecretGrants::new());
let grants: Arc<dyn DeploymentSecretGrants> = inmem.clone();
sync_deployment_grants(&state, Some(&grants), [dn("web"), dn("db")]).await;
assert_eq!(inmem.groups_for(&dn("web")), vec!["edge-a", "edge-b"]);
assert!(inmem.groups_for(&dn("db")).is_empty());
}
#[tokio::test]
async fn sync_revokes_deleted_deployment() {
use harmony::modules::fleet::secret_access::InMemoryDeploymentSecretGrants;
let inmem = Arc::new(InMemoryDeploymentSecretGrants::new());
let grants: Arc<dyn DeploymentSecretGrants> = inmem.clone();
grants
.set_deployment_groups(&[(dn("web"), vec!["edge-a".to_string()])])
.await
.unwrap();
// Deployment no longer cached (CR deleted) → grant resolves empty.
let state: SharedFleetState = Arc::new(Mutex::new(FleetState::default()));
sync_deployment_grants(&state, Some(&grants), std::iter::once(dn("web"))).await;
assert!(inmem.groups_for(&dn("web")).is_empty());
}
#[test]
fn matched_devices_picks_by_label() {
let mut ml = BTreeMap::new();
ml.insert("group".to_string(), "edge-a".to_string());
let sel = LabelSelector {
match_labels: Some(ml),
match_expressions: None,
};
fn membership_change_reschedules_device() {
let mut s = FleetState::default();
let mut lab = BTreeMap::new();
lab.insert("zone".to_string(), "lab".to_string());
s.devices.insert("pi-01".to_string(), lab);
let web = with_groups(cached("ns", "web", "zone", "lab"), &["edge-a"]);
s.deployments.insert(web.key.clone(), web);
let mut devices: HashMap<String, BTreeMap<String, String>> = HashMap::new();
let mut a = BTreeMap::new();
a.insert("group".to_string(), "edge-a".to_string());
devices.insert("pi-01".to_string(), a);
let mut b = BTreeMap::new();
b.insert("group".to_string(), "edge-b".to_string());
devices.insert("pi-02".to_string(), b);
// Not a group member: re-evaluation yields no ownership.
let transitions = reevaluate_device(&mut s, "pi-01");
assert_eq!(transitions.len(), 1);
assert_eq!((transitions[0].1, transitions[0].2), (false, false));
let matched = matched_devices(&sel, &devices);
assert_eq!(matched.len(), 1);
assert!(matched.contains("pi-01"));
// Membership lands (as the group poll would apply it) → owned.
s.device_groups
.insert("pi-01".to_string(), HashSet::from(["edge-a".to_string()]));
let transitions = reevaluate_device(&mut s, "pi-01");
assert_eq!((transitions[0].1, transitions[0].2), (false, true));
assert!(s.owned_targets[&dn("web")].contains("pi-01"));
// Membership revoked → ownership withdrawn.
s.device_groups.remove("pi-01");
let transitions = reevaluate_device(&mut s, "pi-01");
assert_eq!((transitions[0].1, transitions[0].2), (true, false));
assert!(!s.owned_targets[&dn("web")].contains("pi-01"));
}
}

85
fleet/harmony-fleet-operator/src/main.rs
View File

@@ -5,7 +5,11 @@ mod frontend;
#[cfg(feature = "web-frontend")]
mod service;
use harmony_fleet_operator::{device_reconciler, device_status, fleet_aggregator};
use harmony::modules::fleet::groups::{DeviceGroupSource, StaticDeviceGroups};
use harmony::modules::fleet::secret_access::DeploymentSecretGrants;
use harmony::modules::openbao::OpenBaoDeploymentSecretGrants;
use harmony::modules::zitadel::ZitadelDeviceGroups;
use harmony_fleet_operator::{device_reconciler, fleet_aggregator};
use anyhow::{Context, Result};
use async_nats::jetstream;
@@ -18,6 +22,7 @@ use harmony_reconciler_contracts::BUCKET_DESIRED_STATE;
use kube::Client;
#[cfg(feature = "web-frontend")]
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
#[derive(Parser)]
@@ -56,6 +61,12 @@ struct Cli {
global = true
)]
credentials_toml: String,
#[arg(long, env = "OPENBAO_URL", global = true)]
openbao_url: Option<String>,
#[arg(long, env = "OPENBAO_TOKEN", global = true)]
openbao_token: Option<String>,
}
#[derive(Subcommand)]
@@ -98,7 +109,16 @@ async fn main() -> Result<()> {
let cli = Cli::parse();
match cli.command.unwrap_or(Command::Run) {
Command::Run => run(&cli.nats_url, &cli.kv_bucket, &cli.credentials_toml).await,
Command::Run => {
run(
&cli.nats_url,
&cli.kv_bucket,
&cli.credentials_toml,
&cli.openbao_url,
&cli.openbao_token,
)
.await
}
#[cfg(feature = "web-frontend")]
Command::ServeWeb {
mock,
@@ -201,7 +221,13 @@ fn spawn_dashboard(client: Client) {
});
}
async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()> {
async fn run(
nats_url: &str,
bucket: &str,
credentials_toml: &str,
openbao_url: &Option<String>,
openbao_token: &Option<String>,
) -> Result<()> {
let nats = connect_with_retry(nats_url, credentials_toml).await?;
tracing::info!(url = %nats_url, "connected to NATS");
let js = jetstream::new(nats);
@@ -214,6 +240,53 @@ async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()>
tracing::info!(bucket = %bucket, "KV bucket ready");
let client = Client::try_default().await?;
let secret_grants: Option<Arc<dyn DeploymentSecretGrants>> = match (openbao_url, openbao_token)
{
(Some(url), Some(token)) => {
// Secrets live under `<kv>/data/<namespace>/<deployment>/…` —
// the same namespace harmony_config stores under.
let kv_mount =
std::env::var("OPENBAO_KV_MOUNT").unwrap_or_else(|_| "secret".to_string());
let prefix = std::env::var("HARMONY_CONFIG_NAMESPACE")
.unwrap_or_else(|_| "fleet-staging".to_string());
tracing::info!(url = %url, %kv_mount, %prefix, "OpenBao secret-grant sync enabled");
Some(Arc::new(OpenBaoDeploymentSecretGrants::new(
url.clone(),
token.clone(),
kv_mount,
prefix,
)))
}
_ => {
tracing::warn!("OPENBAO_URL or OPENBAO_TOKEN not set; secret-grant sync disabled");
None
}
};
// Group membership (the scheduling gate): Zitadel role grants in
// prod, a static map for dev/e2e, or ungated when neither is set.
let group_source: Option<Arc<dyn DeviceGroupSource>> = match (
std::env::var("ZITADEL_URL"),
std::env::var("ZITADEL_PAT"),
std::env::var("ZITADEL_PROJECT_ID"),
std::env::var("FLEET_DEVICE_GROUPS"),
) {
(Ok(url), Ok(pat), Ok(project), _) => {
tracing::info!(url = %url, %project, "Zitadel device-group source enabled");
Some(Arc::new(ZitadelDeviceGroups::new(url, pat, project)))
}
(_, _, _, Ok(spec)) => {
tracing::info!("static device-group source enabled");
Some(Arc::new(StaticDeviceGroups::parse(&spec)))
}
_ => {
tracing::warn!(
"no device-group source (ZITADEL_URL/PAT/PROJECT_ID or FLEET_DEVICE_GROUPS); \
deployments with allowedGroups will match no devices"
);
None
}
};
// Serve the read-only dashboard in the same process (best-effort;
// it reads CRs only, no NATS). Built only with the web-frontend
@@ -224,7 +297,6 @@ async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()>
// Concurrent tasks:
// controller — CR validation + finalizer-cleanup
// device_reconciler — NATS device-info → Device CR
// device_status — NATS device-heartbeat → Device.status liveness
// fleet_aggregator — watches Deployments + Devices + states,
// writes desired-state KV, patches CR status
// Any failing tears the process down; kube-rs Controller swallows
@@ -232,13 +304,10 @@ async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()>
let ctl_client = client.clone();
let dr_client = client.clone();
let dr_js = js.clone();
let ds_client = client.clone();
let ds_js = js.clone();
tokio::select! {
r = controller::run(ctl_client, desired_state_kv) => r,
r = device_reconciler::run(dr_client, dr_js) => r,
r = device_status::run(ds_client, ds_js) => r,
r = fleet_aggregator::run(client, js) => r,
r = fleet_aggregator::run(client, js, secret_grants, group_source) => r,
}
}

1
fleet/harmony-fleet-operator/src/service/real.rs
View File

@@ -424,6 +424,7 @@ mod tests {
image: "nginx:1.25".into(),
ports: vec![],
env: vec![],
secret_env: vec![],
volumes: vec![],
restart_policy: Default::default(),
}],

8
harmony-k8s/src/apply.rs
View File

@@ -223,7 +223,13 @@ impl K8sClient {
return Ok(resource.clone());
}
let patch_params = PatchParams::apply(FIELD_MANAGER);
// Force conflicts: what harmony applies is declared in code — the
// source of truth. Without it, any field a different manager ever
// touched (e.g. a helm-installed copy of the same CRD) makes the
// re-apply 409 forever; taking ownership on conflict is the
// documented SSA pattern for controllers.
let mut patch_params = PatchParams::apply(FIELD_MANAGER);
patch_params.force = true;
do_apply(&api, name, resource, &patch_params, &write_mode)
.await
.and_then(helper::dyn_to_typed)

81
harmony/src/modules/fleet/groups.rs Normal file
View File

@@ -0,0 +1,81 @@
//! Device group membership: the admin-managed identity input that gates
//! scheduling and secret access (ADR-025).
//!
//! Groups are the security boundary — they must come from identity the
//! device cannot self-assert (Zitadel role grants), never from
//! device-reported labels. The operator polls a [`DeviceGroupSource`]
//! and feeds the snapshot into scheduling.
use std::collections::{HashMap, HashSet};
use async_trait::async_trait;
#[derive(Debug, thiserror::Error)]
pub enum GroupSourceError {
#[error("group source error: {0}")]
Source(String),
}
/// Complete device → groups view. Snapshot-based: membership changes
/// rarely and per-device, so a periodic full fetch beats per-device
/// queries.
#[async_trait]
pub trait DeviceGroupSource: Send + Sync {
async fn device_groups(&self) -> Result<HashMap<String, HashSet<String>>, GroupSourceError>;
}
/// Fixed membership parsed from a `dev=g1|g2;dev2=g3` string — dev
/// setups and e2e, where no Zitadel exists.
#[derive(Debug, Default)]
pub struct StaticDeviceGroups {
map: HashMap<String, HashSet<String>>,
}
impl StaticDeviceGroups {
pub fn parse(spec: &str) -> Self {
let map = spec
.split(';')
.filter_map(|entry| {
let (device, groups) = entry.split_once('=')?;
let device = device.trim();
if device.is_empty() {
return None;
}
Some((
device.to_string(),
groups
.split('|')
.map(str::trim)
.filter(|g| !g.is_empty())
.map(str::to_string)
.collect(),
))
})
.collect();
Self { map }
}
}
#[async_trait]
impl DeviceGroupSource for StaticDeviceGroups {
async fn device_groups(&self) -> Result<HashMap<String, HashSet<String>>, GroupSourceError> {
Ok(self.map.clone())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn parses_devices_and_groups() {
let src = StaticDeviceGroups::parse("pi-01=edge-a|edge-b; pi-02=edge-b ;;junk");
let map = src.device_groups().await.unwrap();
assert_eq!(map.len(), 2);
assert_eq!(
map["pi-01"],
HashSet::from(["edge-a".to_string(), "edge-b".to_string()])
);
assert_eq!(map["pi-02"], HashSet::from(["edge-b".to_string()]));
}
}

6
harmony/src/modules/fleet/mod.rs
View File

@@ -19,10 +19,12 @@
//! KVM VMs, etc.
pub mod assets;
pub mod groups;
#[cfg(all(feature = "kvm", unix))]
pub mod libvirt_pool;
pub mod operator;
pub mod preflight;
pub mod secret_access;
mod setup_score;
#[cfg(all(feature = "kvm", unix))]
mod vm_score;
@@ -41,8 +43,8 @@ pub use libvirt_pool::{HARMONY_FLEET_POOL_NAME, HarmonyFleetPool, ensure_harmony
// composite + per-component Scores.
pub use preflight::{check_fleet_smoke_preflight, check_fleet_smoke_preflight_for_arch};
pub use setup_score::{
AdminAuth, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HostsEntry,
merge_hosts_file,
AdminAuth, DeviceOpenbao, FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore,
HostsEntry, merge_hosts_file,
};
#[cfg(all(feature = "kvm", unix))]
pub use vm_score::ProvisionVmScore;

12
harmony/src/modules/fleet/operator/crd.rs
View File

@@ -22,8 +22,16 @@ pub use crate::modules::podman::{PodmanService, PodmanV0Score, ReconcileScore};
)]
#[serde(rename_all = "camelCase")]
pub struct DeploymentSpec {
/// Which devices this deployment targets. matches against
/// `Device.metadata.labels`.
/// Device groups allowed to *view* this deployment — run it and
/// read its secrets (ADR-025). Membership is admin-managed identity
/// (Zitadel role grants), never device-reported labels: labels can
/// narrow placement below, but only groups grant. Absent = no group
/// gating and no secret access — the selector alone places a
/// secretless workload.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub allowed_groups: Option<Vec<String>>,
/// Which devices this deployment targets, *within* the allowed
/// groups. Matches against `Device.metadata.labels`.
pub target_selector: LabelSelector,
pub score: ReconcileScore,
pub rollout: Rollout,

132
harmony/src/modules/fleet/secret_access.rs Normal file
View File

@@ -0,0 +1,132 @@
//! Deployment → secret access control: the domain boundary the fleet
//! operator drives when a `Deployment`'s allowed groups change.
//!
//! Per ADR-025, groups are the security boundary: a deployment declares
//! which device groups may read its secrets, and devices prove group
//! membership through their signed identity at login. How a grant maps
//! onto a backend's primitive (an OpenBao policy attached to an external
//! group, …) is the *implementation's* concern; callers speak only
//! deployments and group names. That separation is what lets the
//! operator be unit-tested against [`InMemoryDeploymentSecretGrants`]
//! with no live secret store — the production backend is
//! [`crate::modules::openbao::OpenBaoDeploymentSecretGrants`].
use std::collections::{BTreeMap, BTreeSet};
use std::sync::Mutex;
use async_trait::async_trait;
use harmony_reconciler_contracts::DeploymentName;
#[derive(Debug, thiserror::Error)]
pub enum SecretAccessError {
#[error("secret access backend error: {0}")]
Backend(String),
}
/// Sets which device groups may read each deployment's secrets.
#[async_trait]
pub trait DeploymentSecretGrants: Send + Sync {
/// Set the *complete* set of groups allowed to read each listed
/// deployment's secrets. Absolute and idempotent: afterwards exactly
/// the given groups hold the grant — others are revoked, an empty
/// list revokes everywhere. Deployments not listed are untouched.
async fn set_deployment_groups(
&self,
grants: &[(DeploymentName, Vec<String>)],
) -> Result<(), SecretAccessError>;
}
/// In-memory [`DeploymentSecretGrants`] for tests: records the latest
/// group set per deployment so a test can assert the operator synced the
/// right grants on each flow, with nothing deployed.
#[derive(Debug, Default)]
pub struct InMemoryDeploymentSecretGrants {
grants: Mutex<BTreeMap<DeploymentName, BTreeSet<String>>>,
}
impl InMemoryDeploymentSecretGrants {
pub fn new() -> Self {
Self::default()
}
/// Current groups for `deployment`, sorted. Empty when no group has
/// access (never granted, or revoked).
pub fn groups_for(&self, deployment: &DeploymentName) -> Vec<String> {
self.grants
.lock()
.unwrap()
.get(deployment)
.map(|s| s.iter().cloned().collect())
.unwrap_or_default()
}
}
#[async_trait]
impl DeploymentSecretGrants for InMemoryDeploymentSecretGrants {
async fn set_deployment_groups(
&self,
grants: &[(DeploymentName, Vec<String>)],
) -> Result<(), SecretAccessError> {
let mut store = self.grants.lock().unwrap();
for (deployment, groups) in grants {
if groups.is_empty() {
store.remove(deployment);
} else {
store.insert(deployment.clone(), groups.iter().cloned().collect());
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn dn(s: &str) -> DeploymentName {
DeploymentName::try_new(s).expect("valid test name")
}
#[tokio::test]
async fn sets_overwrites_and_clears_grants() {
let grants = InMemoryDeploymentSecretGrants::new();
let web = dn("web");
grants
.set_deployment_groups(&[(web.clone(), vec!["edge-a".into(), "edge-b".into()])])
.await
.unwrap();
assert_eq!(grants.groups_for(&web), vec!["edge-a", "edge-b"]);
// Absolute, not additive: a smaller set revokes the rest.
grants
.set_deployment_groups(&[(web.clone(), vec!["edge-a".into()])])
.await
.unwrap();
assert_eq!(grants.groups_for(&web), vec!["edge-a"]);
// Empty revokes everywhere.
grants
.set_deployment_groups(&[(web.clone(), vec![])])
.await
.unwrap();
assert!(grants.groups_for(&web).is_empty());
}
#[tokio::test]
async fn batch_touches_only_listed_deployments() {
let grants = InMemoryDeploymentSecretGrants::new();
let web = dn("web");
let db = dn("db");
grants
.set_deployment_groups(&[(web.clone(), vec!["edge-a".into()])])
.await
.unwrap();
grants
.set_deployment_groups(&[(db.clone(), vec!["edge-b".into()])])
.await
.unwrap();
assert_eq!(grants.groups_for(&web), vec!["edge-a"]);
assert_eq!(grants.groups_for(&db), vec!["edge-b"]);
}
}

49
harmony/src/modules/fleet/setup_score.rs
View File

@@ -74,6 +74,22 @@ pub struct FleetDeviceSetupConfig {
/// real DNS instead.
#[serde(default)]
pub hosts_entries: Vec<HostsEntry>,
/// OpenBao endpoint the agent reads deployment secrets from
/// (ADR-025). Rendered as the agent's `[openbao]` section; mounts
/// and role use the agent-schema defaults (`jwt`, `fleet-device`,
/// `secret`). Requires `ZitadelJwt`/`ZitadelEnroll` auth — the
/// same machine identity logs into OpenBao.
#[serde(default)]
pub openbao: Option<DeviceOpenbao>,
}
/// Minimal `[openbao]` agent-config surface the setup score renders.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeviceOpenbao {
pub url: String,
/// KV prefix deployment subtrees live under — must match the
/// grant writer's prefix.
pub secret_prefix: String,
}
/// One line in `/etc/hosts`. Order doesn't matter (the file ends up
@@ -246,6 +262,16 @@ impl FleetDeviceSetupConfig {
danger_accept_invalid_certs,
),
};
let openbao = match &self.openbao {
None => String::new(),
Some(ob) => format!(
"\n[openbao]\n\
url = \"{}\"\n\
secret_prefix = \"{}\"\n",
toml_escape(&ob.url),
toml_escape(&ob.secret_prefix),
),
};
format!(
"[agent]\n\
device_id = \"{device_id}\"\n\
@@ -255,7 +281,8 @@ impl FleetDeviceSetupConfig {
urls = [{urls}]\n\
\n\
[labels]\n\
{labels}\n"
{labels}\n\
{openbao}"
)
}
@@ -369,6 +396,7 @@ async fn resolve_zitadel_enroll(
api_apps: vec![],
roles: vec![],
machine_users: vec![],
groups_claim_action: false,
};
info!(
@@ -862,6 +890,7 @@ mod tests {
},
agent_binary_path: PathBuf::from("/dev/null"),
hosts_entries: vec![],
openbao: None,
}
}
@@ -879,6 +908,7 @@ mod tests {
},
agent_binary_path: PathBuf::from("/dev/null"),
hosts_entries: vec![],
openbao: None,
}
}
@@ -936,6 +966,22 @@ mod tests {
assert!(!toml.contains("nats_pass"));
}
#[test]
fn render_toml_emits_openbao_section_only_when_set() {
let plain = base_config_zitadel(BTreeMap::new()).render_toml();
assert!(!plain.contains("[openbao]"), "got:\n{plain}");
let mut cfg = base_config_zitadel(BTreeMap::new());
cfg.openbao = Some(DeviceOpenbao {
url: "http://192.168.122.1:30425".to_string(),
secret_prefix: "fleet-e2e".to_string(),
});
let toml = cfg.render_toml();
assert!(toml.contains("[openbao]"), "got:\n{toml}");
assert!(toml.contains(r#"url = "http://192.168.122.1:30425""#));
assert!(toml.contains(r#"secret_prefix = "fleet-e2e""#));
}
#[test]
fn render_toml_zitadel_enroll_renders_same_as_zitadel_jwt() {
// Pre-resolution (ZitadelEnroll) and post-resolution (ZitadelJwt)
@@ -966,6 +1012,7 @@ mod tests {
},
agent_binary_path: PathBuf::from("/dev/null"),
hosts_entries: vec![],
openbao: None,
};
assert_eq!(jwt, enroll_cfg.render_toml());
}

342
harmony/src/modules/openbao/identity.rs Normal file
View File

@@ -0,0 +1,342 @@
//! OpenBao-backed [`DeploymentSecretGrants`].
//!
//! A deployment's grant is one ACL policy `deployment-<name>` (read on
//! its secret subtree) attached to the *external identity group* of each
//! allowed device group. Devices land in those groups at JWT login via
//! the role's `groups_claim`, so attaching/detaching the policy binds
//! for every member's existing tokens at request time — O(groups)
//! writes per deployment change, regardless of fleet size (ADR-025).
//!
//! External groups are matched to the token claim through a group alias
//! on the JWT mount; we create group + alias on first grant so the grant
//! can precede any device's first login.
use std::collections::HashSet;
use async_trait::async_trait;
use reqwest::StatusCode;
use serde_json::json;
use tokio::sync::OnceCell;
use crate::modules::fleet::secret_access::{DeploymentSecretGrants, SecretAccessError};
use harmony_reconciler_contracts::DeploymentName;
const DEFAULT_JWT_MOUNT: &str = "jwt";
pub struct OpenBaoDeploymentSecretGrants {
client: reqwest::Client,
base_url: String,
token: String,
kv_mount: String,
/// Path prefix under the KV mount where the fleet's secrets live
/// (`<kv_mount>/data/<prefix>/<deployment>/…`).
secret_prefix: String,
jwt_mount: String,
/// JWT auth mount accessor, resolved once and reused — needed to bind
/// a group alias to the login claim value.
jwt_accessor: OnceCell<String>,
}
impl OpenBaoDeploymentSecretGrants {
pub fn new(base_url: String, token: String, kv_mount: String, secret_prefix: String) -> Self {
Self::with_jwt_mount(
base_url,
token,
kv_mount,
secret_prefix,
DEFAULT_JWT_MOUNT.to_string(),
)
}
pub fn with_jwt_mount(
base_url: String,
token: String,
kv_mount: String,
secret_prefix: String,
jwt_mount: String,
) -> Self {
Self {
client: reqwest::Client::new(),
base_url: base_url.trim_end_matches('/').to_string(),
token,
kv_mount,
secret_prefix,
jwt_mount,
jwt_accessor: OnceCell::new(),
}
}
fn err(context: impl std::fmt::Display, e: impl std::fmt::Display) -> SecretAccessError {
SecretAccessError::Backend(format!("{context}: {e}"))
}
fn policy_name(deployment: &DeploymentName) -> String {
format!("deployment-{}", deployment.as_str())
}
async fn request(
&self,
method: reqwest::Method,
path: &str,
body: Option<serde_json::Value>,
) -> Result<reqwest::Response, SecretAccessError> {
let mut req = self
.client
.request(method.clone(), format!("{}/v1/{path}", self.base_url))
.header("X-Vault-Token", &self.token);
if let Some(body) = body {
req = req.json(&body);
}
req.send()
.await
.map_err(|e| Self::err(format!("{method} {path}"), e))
}
/// JWT mount accessor from `sys/auth`, cached for the client's life.
async fn jwt_accessor(&self) -> Result<&str, SecretAccessError> {
self.jwt_accessor
.get_or_try_init(|| async {
let body: serde_json::Value = self
.request(reqwest::Method::GET, "sys/auth", None)
.await?
.error_for_status()
.map_err(|e| Self::err("GET sys/auth", e))?
.json()
.await
.map_err(|e| Self::err("parse sys/auth", e))?;
// sys/auth nests mounts under `data` over the HTTP API but
// emits them at the document root via the CLI; tolerate both.
let mount_key = format!("{}/", self.jwt_mount);
body.get("data")
.and_then(|d| d.get(&mount_key))
.or_else(|| body.get(&mount_key))
.and_then(|m| m.get("accessor"))
.and_then(|a| a.as_str())
.map(str::to_string)
.ok_or_else(|| {
Self::err(
"resolve jwt accessor",
format!("mount '{mount_key}' not found in sys/auth"),
)
})
})
.await
.map(String::as_str)
}
async fn upsert_policy(&self, deployment: &DeploymentName) -> Result<(), SecretAccessError> {
let name = Self::policy_name(deployment);
let hcl = format!(
r#"path "{kv}/data/{prefix}/{dep}/*" {{ capabilities = ["read"] }}
path "{kv}/metadata/{prefix}/{dep}/*" {{ capabilities = ["read", "list"] }}"#,
kv = self.kv_mount,
prefix = self.secret_prefix,
dep = deployment.as_str(),
);
self.request(
reqwest::Method::PUT,
&format!("sys/policies/acl/{name}"),
Some(json!({ "policy": hcl })),
)
.await?
.error_for_status()
.map_err(|e| Self::err(format!("write policy {name}"), e))?;
Ok(())
}
async fn delete_policy(&self, deployment: &DeploymentName) -> Result<(), SecretAccessError> {
let name = Self::policy_name(deployment);
let resp = self
.request(
reqwest::Method::DELETE,
&format!("sys/policies/acl/{name}"),
None,
)
.await?;
if !resp.status().is_success() && resp.status() != StatusCode::NOT_FOUND {
return Err(Self::err(format!("delete policy {name}"), resp.status()));
}
Ok(())
}
/// Group's current policy list, or `None` when the group doesn't
/// exist.
async fn read_group_policies(
&self,
group: &str,
) -> Result<Option<Vec<String>>, SecretAccessError> {
let resp = self
.request(
reqwest::Method::GET,
&format!("identity/group/name/{group}"),
None,
)
.await?;
if resp.status() == StatusCode::NOT_FOUND {
return Ok(None);
}
let body: serde_json::Value = resp
.error_for_status()
.map_err(|e| Self::err(format!("read group {group}"), e))?
.json()
.await
.map_err(|e| Self::err(format!("parse group {group}"), e))?;
Ok(Some(
body["data"]["policies"]
.as_array()
.map(|a| {
a.iter()
.filter_map(|v| v.as_str().map(str::to_string))
.collect()
})
.unwrap_or_default(),
))
}
async fn write_group_policies(
&self,
group: &str,
policies: &[String],
) -> Result<(), SecretAccessError> {
self.request(
reqwest::Method::POST,
&format!("identity/group/name/{group}"),
Some(json!({ "type": "external", "policies": policies })),
)
.await?
.error_for_status()
.map_err(|e| Self::err(format!("write group {group}"), e))?;
Ok(())
}
/// Ensure the external group exists with its alias bound to the JWT
/// mount, and that it carries `policy`.
async fn attach(&self, group: &str, policy: &str) -> Result<(), SecretAccessError> {
match self.read_group_policies(group).await? {
Some(policies) if policies.iter().any(|p| p == policy) => return Ok(()),
Some(mut policies) => {
policies.push(policy.to_string());
self.write_group_policies(group, &policies).await?;
}
None => {
self.write_group_policies(group, &[policy.to_string()])
.await?;
self.ensure_group_alias(group).await?;
}
}
Ok(())
}
async fn detach(&self, group: &str, policy: &str) -> Result<(), SecretAccessError> {
if let Some(policies) = self.read_group_policies(group).await? {
if policies.iter().any(|p| p == policy) {
let remaining: Vec<String> = policies.into_iter().filter(|p| p != policy).collect();
self.write_group_policies(group, &remaining).await?;
}
}
Ok(())
}
async fn ensure_group_alias(&self, group: &str) -> Result<(), SecretAccessError> {
let accessor = self.jwt_accessor().await?.to_string();
let body: serde_json::Value = self
.request(
reqwest::Method::GET,
&format!("identity/group/name/{group}"),
None,
)
.await?
.error_for_status()
.map_err(|e| Self::err(format!("read group {group}"), e))?
.json()
.await
.map_err(|e| Self::err(format!("parse group {group}"), e))?;
if !body["data"]["alias"]["id"]
.as_str()
.unwrap_or("")
.is_empty()
{
return Ok(());
}
let canonical_id = body["data"]["id"]
.as_str()
.ok_or_else(|| Self::err("read group", format!("{group} missing data.id")))?;
let resp = self
.request(
reqwest::Method::POST,
"identity/group-alias",
Some(json!({
"name": group,
"mount_accessor": accessor,
"canonical_id": canonical_id,
})),
)
.await?;
// A concurrent sync may have just created the alias; OpenBao
// answers 400 "alias already exists" — treat as success.
if !resp.status().is_success() && resp.status() != StatusCode::BAD_REQUEST {
return Err(Self::err(
format!("create group alias {group}"),
resp.status(),
));
}
Ok(())
}
/// Names of all identity groups, empty when none exist.
async fn list_groups(&self) -> Result<Vec<String>, SecretAccessError> {
let resp = self
.request(reqwest::Method::GET, "identity/group/name?list=true", None)
.await?;
if resp.status() == StatusCode::NOT_FOUND {
return Ok(vec![]);
}
let body: serde_json::Value = resp
.error_for_status()
.map_err(|e| Self::err("list groups", e))?
.json()
.await
.map_err(|e| Self::err("parse group list", e))?;
Ok(body["data"]["keys"]
.as_array()
.map(|a| {
a.iter()
.filter_map(|v| v.as_str().map(str::to_string))
.collect()
})
.unwrap_or_default())
}
}
#[async_trait]
impl DeploymentSecretGrants for OpenBaoDeploymentSecretGrants {
async fn set_deployment_groups(
&self,
grants: &[(DeploymentName, Vec<String>)],
) -> Result<(), SecretAccessError> {
for (deployment, groups) in grants {
let policy = Self::policy_name(deployment);
let allowed: HashSet<&str> = groups.iter().map(String::as_str).collect();
if allowed.is_empty() {
// Full revocation: detach everywhere, then drop the policy.
for group in self.list_groups().await? {
self.detach(&group, &policy).await?;
}
self.delete_policy(deployment).await?;
continue;
}
self.upsert_policy(deployment).await?;
for group in groups {
self.attach(group, &policy).await?;
}
for group in self.list_groups().await? {
if !allowed.contains(group.as_str()) {
self.detach(&group, &policy).await?;
}
}
}
Ok(())
}
}

11
harmony/src/modules/openbao/mod.rs
View File

@@ -1,3 +1,4 @@
pub mod identity;
pub mod setup;
use std::str::FromStr;
@@ -13,6 +14,7 @@ use crate::{
topology::{HelmCommand, K8sclient, Topology},
};
pub use identity::OpenBaoDeploymentSecretGrants;
pub use setup::{OpenbaoJwtAuth, OpenbaoPolicy, OpenbaoSetupScore, OpenbaoUser, cached_root_token};
const DEFAULT_NAMESPACE: &str = "openbao";
@@ -83,7 +85,16 @@ impl OpenbaoScore {
format!(
r#"global:
openshift: {openshift}
# No sidecar injection anywhere in the fleet (devices talk to OpenBao
# directly over JWT). The injector also installs cluster-scoped RBAC
# that outlives namespace-scoped cleanup and breaks reinstalls.
injector:
enabled: false
server:
# No Kubernetes auth method either — the authDelegator
# ClusterRoleBinding is the same reinstall hazard.
authDelegator:
enabled: false
standalone:
enabled: true
config: |

67
harmony/src/modules/openbao/setup.rs
View File

@@ -35,12 +35,23 @@ pub struct OpenbaoUser {
/// JWT auth method configuration for OpenBao.
#[derive(Debug, Clone, Serialize)]
pub struct OpenbaoJwtAuth {
/// Empty when `jwt_validation_pubkeys` supplies the keys instead
/// (e.g. e2e suites with no live IdP).
pub oidc_discovery_url: String,
/// PEM public key(s) validating tokens directly, bypassing OIDC
/// discovery. Mutually exclusive with `oidc_discovery_url`.
pub jwt_validation_pubkeys: String,
pub bound_issuer: String,
pub role_name: String,
pub bound_audiences: String,
pub user_claim: String,
pub policies: Vec<String>,
/// Token claim whose string-array value binds the login to external
/// identity groups of the same name. Empty = no group binding.
pub groups_claim: String,
/// `service` or `batch`. Fleet device roles use `batch`: no storage
/// cost per login, natural expiry is the group-membership sync
/// (ADR-025).
pub token_type: String,
pub ttl: String,
pub max_ttl: String,
}
@@ -446,9 +457,20 @@ impl OpenbaoSetupInterpret {
.bao(k8s, root_token, &["bao", "auth", "enable", "jwt"])
.await;
// Configure JWT discovery. This may fail if the discovery URL is not
// Configure JWT validation: static public keys when provided,
// otherwise OIDC discovery. Discovery may fail if the URL is not
// reachable from inside the cluster (e.g., Zitadel's ExternalDomain
// isn't resolvable). Non-fatal — warn and continue.
// Single-quote the PEM: it carries newlines the pod-exec shell
// would otherwise split into separate "commands".
let validation_arg = if jwt.jwt_validation_pubkeys.is_empty() {
format!("oidc_discovery_url={}", jwt.oidc_discovery_url)
} else {
format!(
"jwt_validation_pubkeys='{}'",
jwt.jwt_validation_pubkeys.replace('\'', "'\\''")
)
};
let config_result = self
.bao(
k8s,
@@ -457,7 +479,7 @@ impl OpenbaoSetupInterpret {
"bao",
"write",
"auth/jwt/config",
&format!("oidc_discovery_url={}", jwt.oidc_discovery_url),
&validation_arg,
&format!("bound_issuer={}", jwt.bound_issuer),
],
)
@@ -479,35 +501,28 @@ impl OpenbaoSetupInterpret {
}
}
let policies = jwt.policies.join(",");
self.bao(
k8s,
root_token,
&[
"bao",
"write",
&format!("auth/jwt/role/{}", jwt.role_name),
"role_type=jwt",
&format!("bound_audiences={}", jwt.bound_audiences),
&format!("user_claim={}", jwt.user_claim),
&format!("policies={}", policies),
&format!("ttl={}", jwt.ttl),
&format!("max_ttl={}", jwt.max_ttl),
"token_type=service",
],
)
.await
.map_err(|e| {
let mut role_args = vec![
"bao".to_string(),
"write".to_string(),
format!("auth/jwt/role/{}", jwt.role_name),
"role_type=jwt".to_string(),
format!("bound_audiences={}", jwt.bound_audiences),
format!("user_claim={}", jwt.user_claim),
format!("ttl={}", jwt.ttl),
format!("max_ttl={}", jwt.max_ttl),
format!("token_type={}", jwt.token_type),
];
if !jwt.groups_claim.is_empty() {
role_args.push(format!("groups_claim={}", jwt.groups_claim));
}
let role_args: Vec<&str> = role_args.iter().map(String::as_str).collect();
self.bao(k8s, root_token, &role_args).await.map_err(|e| {
InterpretError::new(format!(
"Failed to create JWT role '{}': {e}",
jwt.role_name
))
})?;
info!(
"[OpenbaoSetup] JWT role '{}' created (policies: {})",
jwt.role_name, policies
);
Ok(())
}
}

2
harmony/src/modules/podman/mod.rs
View File

@@ -5,6 +5,6 @@ mod topology;
pub use crate::topology::EnvVar;
pub use interpret::PodmanV0Interpret;
pub use score::{PodmanService, PodmanV0Score, ReconcileScore};
pub use score::{PodmanService, PodmanV0Score, ReconcileScore, SecretEnvVar};
#[cfg(unix)]
pub use topology::PodmanTopology;

38
harmony/src/modules/podman/score.rs
View File

@@ -18,6 +18,17 @@ use crate::{
use super::interpret::PodmanV0Interpret;
/// An env var whose value the device fetches from its secret store at
/// apply time (ADR-025). `secret` names the entry under the
/// deployment's KV subtree (`<kv>/<prefix>/<deployment>/<secret>`);
/// the agent injects the fetched value under `name`. Values never
/// transit NATS — only this reference does.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
pub struct SecretEnvVar {
pub name: String,
pub secret: String,
}
/// A single container managed by podman on the target host.
///
/// Wire-compatible with prior releases: the new `env`, `volumes`, and
@@ -32,6 +43,11 @@ pub struct PodmanService {
/// `PartialEq` is deterministic for drift detection.
#[serde(default)]
pub env: Vec<EnvVar>,
/// Env vars resolved from the device's secret store at apply time.
/// `skip_serializing_if` keeps payloads byte-identical for specs
/// that don't use secrets (string-compare idempotency).
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub secret_env: Vec<SecretEnvVar>,
/// Bind-mount volumes. Bind-only in v0; `host_path` is an absolute path
/// on the device's filesystem.
#[serde(default)]
@@ -110,6 +126,7 @@ mod tests {
image: image.to_string(),
ports: vec![],
env: vec![],
secret_env: vec![],
volumes: vec![],
restart_policy: RestartPolicy::default(),
}
@@ -168,6 +185,7 @@ mod tests {
let ReconcileScore::PodmanV0(score) = parsed;
assert_eq!(score.services.len(), 1);
assert!(score.services[0].env.is_empty());
assert!(score.services[0].secret_env.is_empty());
assert!(score.services[0].volumes.is_empty());
assert_eq!(
score.services[0].restart_policy,
@@ -175,6 +193,26 @@ mod tests {
);
}
#[test]
fn secret_env_roundtrips_and_is_omitted_when_empty() {
// Omission when empty keeps payloads byte-identical to
// pre-secret specs — the agent's string-compare idempotency
// and old-agent compatibility both rely on it.
let plain = serde_json::to_string(&svc("web", "nginx")).unwrap();
assert!(!plain.contains("secret_env"), "got: {plain}");
let with_secret = PodmanService {
secret_env: vec![SecretEnvVar {
name: "DB_PASSWORD".to_string(),
secret: "db_password".to_string(),
}],
..svc("api", "myapp")
};
let json = serde_json::to_string(&with_secret).unwrap();
let back: PodmanService = serde_json::from_str(&json).unwrap();
assert_eq!(back.secret_env, with_secret.secret_env);
}
#[test]
fn restart_policy_serializes_kebab_case() {
// docker-compose users expect `unless-stopped`, `on-failure` —

104
harmony/src/modules/zitadel/device_groups.rs Normal file
View File

@@ -0,0 +1,104 @@
//! Zitadel-backed [`DeviceGroupSource`]: project role grants are the
//! fleet's groups (roles-as-groups interim per ADR-025, swapped for
//! first-class Zitadel groups when they GA). The same role keys reach
//! OpenBao inside the device's token via the flattening Action, so the
//! operator's scheduling view and OpenBao's enforcement view share one
//! source of truth.
use std::collections::{HashMap, HashSet};
use async_trait::async_trait;
use serde::Deserialize;
use crate::modules::fleet::groups::{DeviceGroupSource, GroupSourceError};
pub struct ZitadelDeviceGroups {
client: reqwest::Client,
base_url: String,
pat: String,
project_id: String,
}
impl ZitadelDeviceGroups {
pub fn new(base_url: String, pat: String, project_id: String) -> Self {
Self {
client: reqwest::Client::new(),
base_url: base_url.trim_end_matches('/').to_string(),
pat,
project_id,
}
}
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct GrantSearchResponse {
result: Option<Vec<GrantEntry>>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct GrantEntry {
user_id: String,
#[serde(default)]
user_name: String,
#[serde(default)]
role_keys: Vec<String>,
}
const PAGE: u64 = 1000;
#[async_trait]
impl DeviceGroupSource for ZitadelDeviceGroups {
async fn device_groups(&self) -> Result<HashMap<String, HashSet<String>>, GroupSourceError> {
let mut map: HashMap<String, HashSet<String>> = HashMap::new();
let mut offset = 0u64;
loop {
let resp = self
.client
.post(format!(
"{}/management/v1/users/grants/_search",
self.base_url
))
.bearer_auth(&self.pat)
.json(&serde_json::json!({
"query": { "offset": offset.to_string(), "limit": PAGE },
"queries": [
{ "projectIdQuery": { "projectId": self.project_id } }
]
}))
.send()
.await
.map_err(|e| GroupSourceError::Source(format!("grant search: {e}")))?;
if !resp.status().is_success() {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
return Err(GroupSourceError::Source(format!(
"grant search returned {status}: {body}"
)));
}
let page: GrantSearchResponse = resp
.json()
.await
.map_err(|e| GroupSourceError::Source(format!("parse grant search: {e}")))?;
let entries = page.result.unwrap_or_default();
let n = entries.len() as u64;
for grant in entries {
let roles: HashSet<String> = grant.role_keys.into_iter().collect();
// Fleets key devices by machine-user id or by user name
// depending on provisioning vintage; expose the grant under
// both so either convention resolves.
if !grant.user_name.is_empty() && grant.user_name != grant.user_id {
map.entry(grant.user_name)
.or_default()
.extend(roles.iter().cloned());
}
map.entry(grant.user_id).or_default().extend(roles);
}
if n < PAGE {
return Ok(map);
}
offset += n;
}
}
}

8
harmony/src/modules/zitadel/mod.rs
View File

@@ -1,7 +1,9 @@
pub mod admin_auth;
pub mod device_groups;
pub mod setup;
pub use admin_auth::{ADMIN_API_SCOPES, DeviceCodeError, DeviceCodeFlowConfig, device_code_login};
pub use device_groups::ZitadelDeviceGroups;
pub use setup::{
MachineKeyType, MintedDeviceCredentials, ZitadelApiApp, ZitadelAppType, ZitadelApplication,
ZitadelClientConfig, ZitadelMachineUser, ZitadelRole, ZitadelScheme, ZitadelSetupScore,
@@ -865,7 +867,11 @@ login:
namespace: Some(NonBlankString::from_str(&self.namespace).unwrap()),
release_name: NonBlankString::from_str("zitadel").unwrap(),
chart_name: NonBlankString::from_str("zitadel/zitadel").unwrap(),
chart_version: None,
// Pinned: newer charts ship a login UI that expects a newer
// server; with v4.12.x the login pod crashloops on
// Token.Invalid. 9.27.1 is the last chart matching this
// server line.
chart_version: Some(NonBlankString::from_str("9.27.1").unwrap()),
values_overrides: None,
values_yaml: Some(values_yaml),
create_namespace: true,

178
harmony/src/modules/zitadel/setup.rs
View File

@@ -235,8 +235,30 @@ pub struct ZitadelSetupScore {
/// Machine users to provision (with optional keys + role grants).
#[serde(default)]
pub machine_users: Vec<ZitadelMachineUser>,
/// Provision the token Action that flattens project role keys into
/// a `groups` string-array claim — the shape OpenBao's
/// `groups_claim` consumes (ADR-025) — and attach it to the org's
/// Complement Token flow. Zitadel's native roles claim is a map,
/// which OpenBao cannot bind groups from.
#[serde(default)]
pub groups_claim_action: bool,
}
/// Function name doubles as the Action name — Zitadel requires the
/// script's entry function to match.
pub const GROUPS_CLAIM_ACTION_NAME: &str = "harmonyGroupsClaim";
pub const GROUPS_CLAIM_ACTION_SCRIPT: &str = r#"function harmonyGroupsClaim(ctx, api) {
if (ctx.v1.user.grants === undefined || ctx.v1.user.grants.count == 0) {
return;
}
let groups = [];
ctx.v1.user.grants.grants.forEach(grant => {
grant.roles.forEach(role => groups.push(role));
});
api.v1.claims.setClaim('groups', groups);
}"#;
/// Cached Zitadel provisioning results.
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct ZitadelClientConfig {
@@ -1339,6 +1361,156 @@ impl ZitadelSetupInterpret {
String::from_utf8(bytes).map_err(|e| format!("keyDetails contained non-UTF8 bytes: {e}"))
}
/// Upsert the roles→groups flattening Action and attach it to the
/// Complement Token flow's pre-userinfo and pre-access-token
/// triggers, merging with whatever actions the org already wires.
async fn ensure_groups_claim_action(
&self,
client: &reqwest::Client,
pat: &str,
) -> Result<(), InterpretError> {
// Numeric flow/trigger ids from Zitadel's v1 management API:
// flow 2 = Complement Token; triggers 4 = Pre Userinfo
// creation, 5 = Pre access token creation.
const FLOW_COMPLEMENT_TOKEN: &str = "2";
const TRIGGERS: [&str; 2] = ["4", "5"];
let action_id = match self
.find_action_id(client, pat, GROUPS_CLAIM_ACTION_NAME)
.await
.map_err(InterpretError::new)?
{
Some(id) => {
debug!("[ZitadelSetup] Action '{GROUPS_CLAIM_ACTION_NAME}' already exists");
id
}
None => {
let resp = self
.post(client, "/management/v1/actions")
.bearer_auth(pat)
.json(&serde_json::json!({
"name": GROUPS_CLAIM_ACTION_NAME,
"script": GROUPS_CLAIM_ACTION_SCRIPT,
"timeout": "10s",
"allowedToFail": false,
}))
.send()
.await
.map_err(|e| InterpretError::new(format!("Create action: {e}")))?;
if !resp.status().is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(InterpretError::new(format!("Create action failed: {body}")));
}
let body: serde_json::Value = resp
.json()
.await
.map_err(|e| InterpretError::new(format!("Parse create action: {e}")))?;
info!("[ZitadelSetup] Action '{GROUPS_CLAIM_ACTION_NAME}' created");
body["id"]
.as_str()
.ok_or_else(|| {
InterpretError::new("create action response missing id".to_string())
})?
.to_string()
}
};
let flow: serde_json::Value = {
let resp = self
.request(
client,
reqwest::Method::GET,
&format!("/management/v1/flows/{FLOW_COMPLEMENT_TOKEN}"),
)
.bearer_auth(pat)
.send()
.await
.map_err(|e| InterpretError::new(format!("Read flow: {e}")))?;
if resp.status().is_success() {
resp.json()
.await
.map_err(|e| InterpretError::new(format!("Parse flow: {e}")))?
} else {
// No flow configured yet — Zitadel answers 404.
serde_json::json!({})
}
};
for trigger in TRIGGERS {
let mut action_ids: Vec<String> = flow["flow"]["triggerActions"]
.as_array()
.into_iter()
.flatten()
.filter(|ta| {
ta["triggerType"]["id"].as_str() == Some(trigger)
|| ta["triggerType"].as_str() == Some(trigger)
})
.flat_map(|ta| {
ta["actions"]
.as_array()
.into_iter()
.flatten()
.filter_map(|a| a["id"].as_str().map(str::to_string))
.collect::<Vec<_>>()
})
.collect();
if action_ids.contains(&action_id) {
continue;
}
action_ids.push(action_id.clone());
let resp = self
.post(
client,
&format!("/management/v1/flows/{FLOW_COMPLEMENT_TOKEN}/trigger/{trigger}"),
)
.bearer_auth(pat)
.json(&serde_json::json!({ "actionIds": action_ids }))
.send()
.await
.map_err(|e| InterpretError::new(format!("Set trigger {trigger}: {e}")))?;
if !resp.status().is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(InterpretError::new(format!(
"Set trigger {trigger} failed: {body}"
)));
}
info!("[ZitadelSetup] Action attached to Complement Token trigger {trigger}");
}
Ok(())
}
async fn find_action_id(
&self,
client: &reqwest::Client,
pat: &str,
name: &str,
) -> Result<Option<String>, String> {
let resp = self
.post(client, "/management/v1/actions/_search")
.bearer_auth(pat)
.json(&serde_json::json!({
"queries": [ { "actionNameQuery": { "name": name } } ]
}))
.send()
.await
.map_err(|e| format!("Search actions: {e}"))?;
if !resp.status().is_success() {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
return Err(format!("action search returned {status}: {body}"));
}
let body: serde_json::Value = resp
.json()
.await
.map_err(|e| format!("Parse action search: {e}"))?;
Ok(body["result"]
.as_array()
.into_iter()
.flatten()
.find(|a| a["name"].as_str() == Some(name))
.and_then(|a| a["id"].as_str().map(str::to_string)))
}
async fn find_user_grant(
&self,
client: &reqwest::Client,
@@ -1666,6 +1838,11 @@ impl<T: Topology + K8sclient> Interpret<T> for ZitadelSetupInterpret {
}
}
if self.score.groups_claim_action {
self.ensure_groups_claim_action(&client, &pat).await?;
details.push(format!("action:{GROUPS_CLAIM_ACTION_NAME}"));
}
config.save().map_err(InterpretError::new)?;
Ok(Outcome {
@@ -1751,6 +1928,7 @@ mod tests {
api_apps: vec![],
roles: vec![],
machine_users: vec![],
groups_claim_action: false,
}
}

55
harmony_config/src/lib.rs
View File

@@ -198,8 +198,7 @@ impl ConfigClient {
/// Build an OpenBao-backed `StoreSource` from env vars, or `None` (with a
/// `warn!`) when `OPENBAO_URL`/`VAULT_ADDR` is unset or the connection
/// fails — so a missing OpenBao degrades the chain instead of breaking
/// startup. `jwt_role`/`jwt_auth_mount` are left to `OpenbaoSecretStore`'s
/// defaults, matching what `OpenbaoSetupScore` configures.
/// startup.
async fn openbao_from_env(namespace: &str) -> Option<Arc<dyn ConfigSource>> {
let Some(url) = std::env::var("OPENBAO_URL")
.or_else(|_| std::env::var("VAULT_ADDR"))
@@ -210,19 +209,45 @@ async fn openbao_from_env(namespace: &str) -> Option<Arc<dyn ConfigSource>> {
};
let env = |k: &str| std::env::var(k).ok();
let store = harmony_secret::OpenbaoSecretStore::new(
url,
env("OPENBAO_KV_MOUNT").unwrap_or_else(|| "secret".to_string()),
env("OPENBAO_AUTH_MOUNT").unwrap_or_else(|| "jwt".to_string()),
env("OPENBAO_SKIP_TLS").as_deref() == Some("true"),
env("OPENBAO_TOKEN"),
env("OPENBAO_USERNAME"),
env("OPENBAO_PASSWORD"),
env("HARMONY_SSO_URL"),
env("HARMONY_SSO_CLIENT_ID"),
None,
None,
)
// Headless Zitadel-machine → OpenBao (JWT-bearer) rung: needs a machine
// keyfile (path or inline JSON) plus the project-ID audience. The issuer
// is the same Zitadel base the OIDC rung uses (`HARMONY_SSO_URL`). Absent
// any piece, the rung stays off and the ladder falls through.
let zitadel_jwt_bearer = {
let key_path = env("HARMONY_ZITADEL_KEY_PATH").map(PathBuf::from);
let key_json = env("HARMONY_ZITADEL_KEY_JSON");
match (
key_path.is_some() || key_json.is_some(),
env("HARMONY_SSO_URL"),
env("HARMONY_ZITADEL_AUDIENCE"),
) {
(true, Some(oidc_issuer_url), Some(audience)) => {
Some(harmony_secret::ZitadelJwtBearerConfig {
key_path,
key_json,
oidc_issuer_url,
audience,
})
}
_ => None,
}
};
let store = harmony_secret::OpenbaoSecretStore::new(harmony_secret::OpenbaoStoreOptions {
base_url: url,
kv_mount: env("OPENBAO_KV_MOUNT").unwrap_or_else(|| "secret".to_string()),
auth_mount: env("OPENBAO_AUTH_MOUNT").unwrap_or_else(|| "jwt".to_string()),
skip_tls: env("OPENBAO_SKIP_TLS").as_deref() == Some("true"),
token: env("OPENBAO_TOKEN"),
username: env("OPENBAO_USERNAME"),
password: env("OPENBAO_PASSWORD"),
zitadel_sso_url: env("HARMONY_SSO_URL"),
zitadel_client_id: env("HARMONY_SSO_CLIENT_ID"),
jwt_role: env("OPENBAO_JWT_ROLE"),
jwt_auth_mount: env("OPENBAO_JWT_AUTH_MOUNT"),
zitadel_jwt_bearer,
})
.await;
match store {

1
harmony_secret/Cargo.toml
View File

@@ -12,6 +12,7 @@ license.workspace = true
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(secrete2etest)'] }
[dependencies]
harmony_zitadel_jwt = { path = "../harmony_zitadel_jwt" }
harmony_secret_derive = { version = "0.1.0", path = "../harmony_secret_derive" }
serde = { version = "1.0.209", features = ["derive", "rc"] }
serde_json = "1.0.127"

29
harmony_secret/src/lib.rs
View File

@@ -26,7 +26,7 @@ use serde::{Serialize, de::DeserializeOwned};
use std::fmt;
use store::InfisicalSecretStore;
use store::LocalFileSecretStore;
pub use store::OpenbaoSecretStore;
pub use store::{OpenbaoSecretStore, OpenbaoStoreOptions, ZitadelJwtBearerConfig};
use thiserror::Error;
use tokio::sync::OnceCell;
@@ -85,19 +85,20 @@ async fn init_secret_manager() -> SecretManager {
let store: Box<dyn SecretStore> = match store_type.as_str() {
"file" => Box::new(LocalFileSecretStore),
"openbao" | "vault" => {
let store = OpenbaoSecretStore::new(
OPENBAO_URL.clone().expect("Openbao/Vault URL must be set, see harmony_secret config for ways to provide it. You can try with OPENBAO_URL or VAULT_ADDR"),
OPENBAO_KV_MOUNT.clone(),
OPENBAO_AUTH_MOUNT.clone(),
*OPENBAO_SKIP_TLS,
OPENBAO_TOKEN.clone(),
OPENBAO_USERNAME.clone(),
OPENBAO_PASSWORD.clone(),
HARMONY_SSO_URL.clone(),
HARMONY_SSO_CLIENT_ID.clone(),
None,
None,
)
let store = OpenbaoSecretStore::new(OpenbaoStoreOptions {
base_url: OPENBAO_URL.clone().expect("Openbao/Vault URL must be set, see harmony_secret config for ways to provide it. You can try with OPENBAO_URL or VAULT_ADDR"),
kv_mount: OPENBAO_KV_MOUNT.clone(),
auth_mount: OPENBAO_AUTH_MOUNT.clone(),
skip_tls: *OPENBAO_SKIP_TLS,
token: OPENBAO_TOKEN.clone(),
username: OPENBAO_USERNAME.clone(),
password: OPENBAO_PASSWORD.clone(),
zitadel_sso_url: HARMONY_SSO_URL.clone(),
zitadel_client_id: HARMONY_SSO_CLIENT_ID.clone(),
jwt_role: None,
jwt_auth_mount: None,
zitadel_jwt_bearer: None,
})
.await
.expect("Failed to initialize Openbao/Vault secret store");
Box::new(store)

2
harmony_secret/src/store/mod.rs
View File

@@ -5,4 +5,4 @@ pub mod zitadel;
pub use infisical::InfisicalSecretStore;
pub use local_file::LocalFileSecretStore;
pub use openbao::OpenbaoSecretStore;
pub use openbao::{OpenbaoSecretStore, OpenbaoStoreOptions, ZitadelJwtBearerConfig};

401
harmony_secret/src/store/openbao.rs
View File

@@ -1,10 +1,13 @@
use crate::{SecretStore, SecretStoreError};
use async_trait::async_trait;
use harmony_zitadel_jwt::{MachineKeyFile, ZitadelJwtBearer};
use log::{debug, info, warn};
use serde::{Deserialize, Serialize};
use std::fmt::Debug;
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::Mutex;
use vaultrs::auth;
use vaultrs::client::{VaultClient, VaultClientSettingsBuilder};
use vaultrs::kv2;
@@ -30,21 +33,47 @@ impl From<vaultrs::api::AuthInfo> for AuthInfo {
}
pub struct OpenbaoSecretStore {
client: VaultClient,
client: Mutex<VaultClient>,
kv_mount: String,
auth_mount: String,
jwt_bearer: Option<Arc<ZitadelJwtBearer>>,
jwt_role: Option<String>,
jwt_auth_mount: Option<String>,
base_url: String,
skip_tls: bool,
}
impl Debug for OpenbaoSecretStore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("OpenbaoSecretStore")
.field("client", &self.client.settings)
.field("kv_mount", &self.kv_mount)
.field("auth_mount", &self.auth_mount)
.finish()
}
}
pub struct OpenbaoStoreOptions {
pub base_url: String,
pub kv_mount: String,
pub auth_mount: String,
pub skip_tls: bool,
pub token: Option<String>,
pub username: Option<String>,
pub password: Option<String>,
pub zitadel_sso_url: Option<String>,
pub zitadel_client_id: Option<String>,
pub zitadel_jwt_bearer: Option<ZitadelJwtBearerConfig>,
pub jwt_role: Option<String>,
pub jwt_auth_mount: Option<String>,
}
pub struct ZitadelJwtBearerConfig {
pub key_path: Option<PathBuf>,
pub key_json: Option<String>,
pub oidc_issuer_url: String,
pub audience: String,
}
impl OpenbaoSecretStore {
/// Creates a new Openbao/Vault secret store with authentication.
///
@@ -52,34 +81,35 @@ impl OpenbaoSecretStore {
/// - `auth_mount`: The auth method mount path (e.g., `"userpass"`). Used for userpass authentication.
/// - `zitadel_sso_url`: Zitadel OIDC server URL (e.g., `"https://sso.example.com"`). If provided along with `zitadel_client_id`, Zitadel OIDC device flow will be attempted.
/// - `zitadel_client_id`: OIDC client ID registered in Zitadel.
// TODO(clippy): refactor to a typed options struct
#[allow(clippy::too_many_arguments)]
pub async fn new(
base_url: String,
kv_mount: String,
auth_mount: String,
skip_tls: bool,
token: Option<String>,
username: Option<String>,
password: Option<String>,
zitadel_sso_url: Option<String>,
zitadel_client_id: Option<String>,
jwt_role: Option<String>,
jwt_auth_mount: Option<String>,
) -> Result<Self, SecretStoreError> {
info!("OPENBAO_STORE: Initializing client for URL: {base_url}");
pub async fn new(options: OpenbaoStoreOptions) -> Result<Self, SecretStoreError> {
info!(
"OPENBAO_STORE: Initializing client for URL: {}",
options.base_url
);
// 1. If token is provided via env var, use it directly
if let Some(t) = token {
if let Some(t) = &options.token {
debug!("OPENBAO_STORE: Using token from environment variable");
return Self::with_token(&base_url, skip_tls, &t, &kv_mount, &auth_mount);
return Self::with_token(
&options.base_url,
options.skip_tls,
t,
&options.kv_mount,
&options.auth_mount,
);
}
// 2. Try to load cached token
let cache_path = Self::get_token_cache_path(&base_url);
let cache_path = Self::get_token_cache_path(&options.base_url);
if let Ok(mut cached_token) = Self::load_cached_token(&cache_path) {
debug!("OPENBAO_STORE: Found cached token, validating...");
if Self::validate_token(&base_url, skip_tls, &cached_token.client_token).await {
if Self::validate_token(
&options.base_url,
options.skip_tls,
&cached_token.client_token,
)
.await
{
info!("OPENBAO_STORE: Cached token is valid");
// Best-effort renewal: extend the server-side lease so
@@ -89,7 +119,13 @@ impl OpenbaoSecretStore {
// blip, etc.) doesn't block the caller — we just keep
// the validated cached token as-is and let the next
// invocation try again.
match Self::renew_self(&base_url, skip_tls, &cached_token.client_token).await {
match Self::renew_self(
&options.base_url,
options.skip_tls,
&cached_token.client_token,
)
.await
{
Ok(new_lease) => {
info!("OPENBAO_STORE: cached token renewed, lease_duration={new_lease}s");
cached_token.lease_duration = Some(new_lease);
@@ -106,26 +142,59 @@ impl OpenbaoSecretStore {
}
return Self::with_token(
&base_url,
skip_tls,
&options.base_url,
options.skip_tls,
&cached_token.client_token,
&kv_mount,
&auth_mount,
&options.kv_mount,
&options.auth_mount,
);
}
warn!("OPENBAO_STORE: Cached token is invalid or expired");
}
// 3. Try Zitadel OIDC device flow if configured
if let (Some(sso_url), Some(client_id)) = (zitadel_sso_url, zitadel_client_id) {
// 3. Try Zitadel JWT-bearer for headless devices if configured
if let Some(jwt_bearer_config) = &options.zitadel_jwt_bearer {
info!("OPENBAO_STORE: Attempting Zitadel JWT-bearer authentication");
match Self::zitadel_jwt_bearer(jwt_bearer_config, options.skip_tls) {
Ok(jwt_bearer) => {
let store = Self::with_token_and_jwt(
&options.base_url,
options.skip_tls,
"",
&options.kv_mount,
&options.auth_mount,
Some(Arc::new(jwt_bearer)),
options.jwt_role.clone(),
options.jwt_auth_mount.clone(),
)?;
match store.refresh_auth().await {
Ok(()) => {
info!("OPENBAO_STORE: Zitadel JWT-bearer authentication successful");
return Ok(store);
}
Err(e) => warn!(
"OPENBAO_STORE: Zitadel JWT-bearer failed: {e}, trying other auth methods"
),
}
}
Err(e) => warn!(
"OPENBAO_STORE: Zitadel JWT-bearer setup failed: {e}, trying other auth methods"
),
}
}
// 4. Try Zitadel OIDC device flow if configured
if let (Some(sso_url), Some(client_id)) =
(&options.zitadel_sso_url, &options.zitadel_client_id)
{
info!("OPENBAO_STORE: Attempting Zitadel OIDC device flow");
match Self::authenticate_zitadel_oidc(
&base_url,
&sso_url,
&client_id,
skip_tls,
jwt_role.as_deref(),
jwt_auth_mount.as_deref(),
&options.base_url,
sso_url,
client_id,
options.skip_tls,
options.jwt_role.as_deref(),
options.jwt_auth_mount.as_deref(),
)
.await
{
@@ -141,11 +210,11 @@ impl OpenbaoSecretStore {
warn!("OPENBAO_STORE: Failed to cache OIDC token: {e}");
}
return Self::with_token(
&base_url,
skip_tls,
&options.base_url,
options.skip_tls,
&auth_info.client_token,
&kv_mount,
&auth_mount,
&options.kv_mount,
&options.auth_mount,
);
}
Err(e) => {
@@ -154,8 +223,8 @@ impl OpenbaoSecretStore {
}
}
// 4. Authenticate with username/password
let (user, pass) = match (username, password) {
// 5. Authenticate with username/password
let (user, pass) = match (&options.username, &options.password) {
(Some(u), Some(p)) => (u, p),
_ => {
return Err(SecretStoreError::Store(
@@ -167,8 +236,14 @@ impl OpenbaoSecretStore {
}
};
let token =
Self::authenticate_userpass(&base_url, &auth_mount, skip_tls, &user, &pass).await?;
let token = Self::authenticate_userpass(
&options.base_url,
&options.auth_mount,
options.skip_tls,
user,
pass,
)
.await?;
// Cache the token
if let Err(e) = Self::cache_token(&cache_path, &token) {
@@ -176,11 +251,11 @@ impl OpenbaoSecretStore {
}
Self::with_token(
&base_url,
skip_tls,
&options.base_url,
options.skip_tls,
&token.client_token,
&kv_mount,
&auth_mount,
&options.kv_mount,
&options.auth_mount,
)
}
@@ -206,14 +281,139 @@ impl OpenbaoSecretStore {
.map_err(|e| SecretStoreError::Store(e.into()))
}
/// Create a client with an existing token
fn with_token(
/// Device-side constructor: authenticate exclusively through an
/// existing [`ZitadelJwtBearer`] (the same instance the NATS auth
/// path holds — one machine identity, two consumers). No token
/// file cache, no fallback chain: login happens lazily on the
/// first read via the permission-denied retry in `get_raw`, and a
/// later denial retries once through a fresh login because batch
/// tokens expire without renewal (ADR-025).
pub fn with_zitadel_bearer(
base_url: &str,
skip_tls: bool,
kv_mount: &str,
jwt_auth_mount: &str,
jwt_role: &str,
bearer: Arc<ZitadelJwtBearer>,
) -> Result<Self, SecretStoreError> {
Self::with_token_and_jwt(
base_url,
skip_tls,
"",
kv_mount,
jwt_auth_mount,
Some(bearer),
Some(jwt_role.to_string()),
Some(jwt_auth_mount.to_string()),
)
}
pub async fn refresh_auth(&self) -> Result<(), SecretStoreError> {
let bearer = self.jwt_bearer.as_ref().ok_or_else(|| {
SecretStoreError::Store("Zitadel JWT-bearer auth is not configured".into())
})?;
let role = self
.jwt_role
.as_deref()
.ok_or_else(|| SecretStoreError::Store("OpenBao JWT role is not configured".into()))?;
let jwt_auth_mount = self.jwt_auth_mount.as_deref().unwrap_or("jwt");
bearer.invalidate_cache();
let jwt = bearer
.bearer_token()
.await
.map_err(|e| SecretStoreError::Store(e.into()))?;
let token =
Self::jwt_login(&self.base_url, self.skip_tls, jwt_auth_mount, role, &jwt).await?;
let client = Self::build_vault_client(&self.base_url, self.skip_tls, &token)?;
*self.client.lock().await = client;
Ok(())
}
fn zitadel_jwt_bearer(
config: &ZitadelJwtBearerConfig,
danger_accept_invalid_certs: bool,
) -> Result<ZitadelJwtBearer, SecretStoreError> {
let key = match config.key_json.as_deref().map(str::trim) {
Some(raw) if !raw.is_empty() => serde_json::from_str::<MachineKeyFile>(raw),
_ => {
let path = config.key_path.as_ref().ok_or_else(|| {
SecretStoreError::Store(
"Zitadel JWT-bearer requires key_json or key_path".into(),
)
})?;
let raw =
fs::read_to_string(path).map_err(|e| SecretStoreError::Store(e.into()))?;
serde_json::from_str::<MachineKeyFile>(&raw)
}
}
.map_err(|e| SecretStoreError::Store(e.into()))?;
ZitadelJwtBearer::new(
key,
config.oidc_issuer_url.clone(),
config.audience.clone(),
danger_accept_invalid_certs,
)
.map_err(|e| SecretStoreError::Store(e.into()))
}
async fn jwt_login(
base_url: &str,
skip_tls: bool,
jwt_auth_mount: &str,
role: &str,
jwt: &str,
) -> Result<String, SecretStoreError> {
let mut builder = reqwest::Client::builder();
if skip_tls {
builder = builder.danger_accept_invalid_certs(true);
}
let client = builder
.build()
.map_err(|e| SecretStoreError::Store(e.into()))?;
let url = format!(
"{}/v1/auth/{}/login",
base_url.trim_end_matches('/'),
jwt_auth_mount.trim_matches('/')
);
let response = client
.post(&url)
.json(&serde_json::json!({ "role": role, "jwt": jwt }))
.send()
.await
.map_err(|e| SecretStoreError::Store(e.into()))?;
let status = response.status();
let body = response
.text()
.await
.map_err(|e| SecretStoreError::Store(e.into()))?;
if !status.is_success() {
return Err(SecretStoreError::Store(
format!("OpenBao JWT login returned {status}: {body}").into(),
));
}
let parsed: serde_json::Value =
serde_json::from_str(&body).map_err(|e| SecretStoreError::Store(e.into()))?;
parsed
.get("auth")
.and_then(|auth| auth.get("client_token"))
.and_then(|token| token.as_str())
.map(str::to_string)
.ok_or_else(|| {
SecretStoreError::Store(
format!("OpenBao JWT login response missing auth.client_token: {body}").into(),
)
})
}
fn build_vault_client(
base_url: &str,
skip_tls: bool,
token: &str,
kv_mount: &str,
auth_mount: &str,
) -> Result<Self, SecretStoreError> {
) -> Result<VaultClient, SecretStoreError> {
let mut settings = VaultClientSettingsBuilder::default();
settings.address(base_url).token(token);
@@ -222,17 +422,48 @@ impl OpenbaoSecretStore {
settings.verify(false);
}
let client = VaultClient::new(
VaultClient::new(
settings
.build()
.map_err(|e| SecretStoreError::Store(Box::new(e)))?,
)
.map_err(|e| SecretStoreError::Store(Box::new(e)))?;
.map_err(|e| SecretStoreError::Store(Box::new(e)))
}
/// Create a client with an existing token
fn with_token(
base_url: &str,
skip_tls: bool,
token: &str,
kv_mount: &str,
auth_mount: &str,
) -> Result<Self, SecretStoreError> {
Self::with_token_and_jwt(
base_url, skip_tls, token, kv_mount, auth_mount, None, None, None,
)
}
fn with_token_and_jwt(
base_url: &str,
skip_tls: bool,
token: &str,
kv_mount: &str,
auth_mount: &str,
jwt_bearer: Option<Arc<ZitadelJwtBearer>>,
jwt_role: Option<String>,
jwt_auth_mount: Option<String>,
) -> Result<Self, SecretStoreError> {
let client = Self::build_vault_client(base_url, skip_tls, token)?;
Ok(Self {
client,
client: Mutex::new(client),
kv_mount: kv_mount.to_string(),
auth_mount: auth_mount.to_string(),
base_url: base_url.to_string(),
skip_tls,
jwt_bearer,
jwt_role,
jwt_auth_mount,
})
}
@@ -404,25 +635,23 @@ impl OpenbaoSecretStore {
}
}
#[async_trait]
impl SecretStore for OpenbaoSecretStore {
async fn get_raw(&self, namespace: &str, key: &str) -> Result<Vec<u8>, SecretStoreError> {
impl OpenbaoSecretStore {
async fn read_value(&self, namespace: &str, key: &str) -> Result<Vec<u8>, SecretStoreError> {
let path = format!("{}/{}", namespace, key);
info!("OPENBAO_STORE: Getting key '{key}' from namespace '{namespace}'");
debug!("OPENBAO_STORE: Request path: {path}");
let data: serde_json::Value = kv2::read(&self.client, &self.kv_mount, &path)
.await
.map_err(|e| {
if e.to_string().contains("does not exist") || e.to_string().contains("404") {
SecretStoreError::NotFound {
namespace: namespace.to_string(),
key: key.to_string(),
let client = self.client.lock().await;
let data: serde_json::Value =
kv2::read(&*client, &self.kv_mount, &path)
.await
.map_err(|e| {
if e.to_string().contains("does not exist") || e.to_string().contains("404") {
SecretStoreError::NotFound {
namespace: namespace.to_string(),
key: key.to_string(),
}
} else {
SecretStoreError::Store(Box::new(e))
}
} else {
SecretStoreError::Store(Box::new(e))
}
})?;
})?;
let value = data.get("value").and_then(|v| v.as_str()).ok_or_else(|| {
SecretStoreError::Store("Secret does not contain expected 'value' field".into())
@@ -430,6 +659,31 @@ impl SecretStore for OpenbaoSecretStore {
Ok(value.as_bytes().to_vec())
}
}
fn is_permission_denied(e: &SecretStoreError) -> bool {
let msg = e.to_string();
msg.contains("permission denied") || msg.contains("403")
}
#[async_trait]
impl SecretStore for OpenbaoSecretStore {
async fn get_raw(&self, namespace: &str, key: &str) -> Result<Vec<u8>, SecretStoreError> {
info!("OPENBAO_STORE: Getting key '{key}' from namespace '{namespace}'");
match self.read_value(namespace, key).await {
// Batch tokens expire without renewal; a fresh JWT login
// IS the refresh path (ADR-025). Exactly one retry: if a
// freshly-issued token is still denied, the policy really
// doesn't allow it and the error must surface.
Err(e) if self.jwt_bearer.is_some() && is_permission_denied(&e) => {
debug!("OPENBAO_STORE: read denied, re-logging in via JWT bearer: {e}");
self.refresh_auth().await?;
self.read_value(namespace, key).await
}
r => r,
}
}
async fn set_raw(
&self,
@@ -447,8 +701,9 @@ impl SecretStore for OpenbaoSecretStore {
let data = serde_json::json!({
"value": value_str
});
let client = self.client.lock().await;
kv2::set(&self.client, &self.kv_mount, &path, &data)
kv2::set(&*client, &self.kv_mount, &path, &data)
.await
.map_err(|e| SecretStoreError::Store(Box::new(e)))?;

1
harmony_zitadel_auth/Cargo.toml
View File

@@ -11,6 +11,7 @@ axum = ["dep:axum", "dep:axum-extra"]
[dependencies]
harmony_config = { path = "../harmony_config" }
harmony_zitadel_jwt = { path = "../harmony_zitadel_jwt" }
anyhow.workspace = true
base64.workspace = true
chrono = { workspace = true, features = ["serde"] }

23
harmony_zitadel_auth/src/axum_login_flow.rs
View File

@@ -9,8 +9,8 @@ use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use crate::config::ZitadelAuthConfig;
use crate::jwks::JwksCache;
use crate::login::{
AuthCallbackQuery, RawAuthCallbackQuery, TokenResponse, build_login_attempt, build_logout_url,
exchange_code_for_token, jwt_exp, validate_callback_state,
AuthCallbackQuery, LoginQuery, RawAuthCallbackQuery, TokenResponse, build_login_attempt,
build_logout_url, exchange_code_for_token, jwt_exp, valid_next, validate_callback_state,
};
use crate::session::LoginAttemptCookie;
@@ -23,8 +23,9 @@ pub const HARMONY_SESSION_COOKIE: &str = "harmony_fleet_session";
pub async fn login_handler(
jar: PrivateCookieJar,
State(config): State<ZitadelAuthConfig>,
Query(query): Query<LoginQuery>,
) -> Response {
match build_login_response(jar, &config) {
match build_login_response(jar, &config, query) {
Ok(r) => r.into_response(),
Err(e) => auth_error_response(e),
}
@@ -33,9 +34,16 @@ pub async fn login_handler(
fn build_login_response(
jar: PrivateCookieJar,
config: &ZitadelAuthConfig,
query: LoginQuery,
) -> Result<impl IntoResponse> {
let attempt = build_login_attempt(config)?;
let cookie_payload = LoginAttemptCookie::from(&attempt);
let mut cookie_payload = LoginAttemptCookie::from(&attempt);
cookie_payload.next = Some(
query
.next
.filter(|next| valid_next(next))
.unwrap_or_else(|| "/".to_string()),
);
let cookie_value = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&cookie_payload)?);
let mut builder = Cookie::build((LOGIN_ATTEMPT_COOKIE, cookie_value))
@@ -113,7 +121,12 @@ async fn build_callback_response(
}
let session_jar = session_jar.add(session_cookie(&tokens, config));
Ok((jar, session_jar, Redirect::to("/")).into_response())
let next = attempt
.next
.as_deref()
.filter(|next| valid_next(next))
.unwrap_or("/");
Ok((jar, session_jar, Redirect::to(next)).into_response())
}
AuthCallbackQuery::Failure {
error,

7
harmony_zitadel_auth/src/lib.rs
View File

@@ -6,13 +6,14 @@ pub mod login;
pub mod session;
pub use config::{OperatorCookieKey, ZitadelAuthConfig};
pub use harmony_zitadel_jwt::{MachineKeyFile, ZitadelJwtBearer};
pub use jwks::JwksCache;
pub use login::{
AuthCallbackQuery, LoginAttempt, RawAuthCallbackQuery, TokenResponse, ValidatedUser,
build_login_attempt, build_logout_url, exchange_code_for_token, jwt_exp,
validate_callback_state, validate_id_token,
AuthCallbackQuery, LoginAttempt, LoginQuery, RawAuthCallbackQuery, TokenResponse,
ValidatedUser, build_login_attempt, build_logout_url, exchange_code_for_token, jwt_exp,
valid_next, validate_callback_state, validate_id_token,
};
pub use session::{LoginAttemptCookie, VerifiedSession};

10
harmony_zitadel_auth/src/login.rs
View File

@@ -37,6 +37,11 @@ pub struct TokenResponse {
pub expires_in: Option<u64>,
}
#[derive(Debug, Deserialize)]
pub struct LoginQuery {
pub next: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct RawAuthCallbackQuery {
pub code: Option<String>,
@@ -63,10 +68,15 @@ impl From<&LoginAttempt> for LoginAttemptCookie {
state: attempt.state.clone(),
pkce_code_verifier: attempt.pkce_code_verifier.clone(),
nonce: attempt.nonce.clone(),
next: None,
}
}
}
pub fn valid_next(next: &str) -> bool {
next.starts_with('/') && !next.starts_with("//") && !next.chars().any(char::is_control)
}
impl TryFrom<RawAuthCallbackQuery> for AuthCallbackQuery {
type Error = anyhow::Error;

Some files were not shown because too many files have changed in this diff Show More