diff --git a/.gitignore b/.gitignore index 86ff3596..76ea8ec2 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,9 @@ ignore # Generated book book + +# Scratch and agent worktrees — never commit +.claude/ +ui-idea.md +ROADMAP/00-priority-matrix.md +fleet/harmony-fleet-agent/agent-config.toml diff --git a/.sqlx/query-6fcc29cfdbdf3b2cee94a4844e227f09b245dd8f079832a9a7b774151cb03af6.json b/.sqlx/query-165b944d13c8f7810b4e3ef891e5cd256d74f572629b8c0764782066e705c50c.json similarity index 50% rename from .sqlx/query-6fcc29cfdbdf3b2cee94a4844e227f09b245dd8f079832a9a7b774151cb03af6.json rename to .sqlx/query-165b944d13c8f7810b4e3ef891e5cd256d74f572629b8c0764782066e705c50c.json index d3f774b8..deacd686 100644 --- a/.sqlx/query-6fcc29cfdbdf3b2cee94a4844e227f09b245dd8f079832a9a7b774151cb03af6.json +++ b/.sqlx/query-165b944d13c8f7810b4e3ef891e5cd256d74f572629b8c0764782066e705c50c.json @@ -1,12 +1,12 @@ { "db_name": "SQLite", - "query": "\n INSERT INTO host_role_mapping (host_id, role, installation_device)\n VALUES (?, ?, ?)\n ", + "query": "\n INSERT INTO host_role_mapping (host_id, role, installation_device, network_config)\n VALUES (?, ?, ?, ?)\n ", "describe": { "columns": [], "parameters": { - "Right": 3 + "Right": 4 }, "nullable": [] }, - "hash": "6fcc29cfdbdf3b2cee94a4844e227f09b245dd8f079832a9a7b774151cb03af6" + "hash": "165b944d13c8f7810b4e3ef891e5cd256d74f572629b8c0764782066e705c50c" } diff --git a/.sqlx/query-3b71d7d7ae75e75ec3ef1df2cd3c4d18520b9d56dd328b7edf576af9dac3c2c0.json b/.sqlx/query-3b71d7d7ae75e75ec3ef1df2cd3c4d18520b9d56dd328b7edf576af9dac3c2c0.json new file mode 100644 index 00000000..f317859f --- /dev/null +++ b/.sqlx/query-3b71d7d7ae75e75ec3ef1df2cd3c4d18520b9d56dd328b7edf576af9dac3c2c0.json @@ -0,0 +1,32 @@ +{ + "db_name": "SQLite", + "query": "SELECT role as \"role: HostRole\", installation_device, network_config FROM host_role_mapping WHERE host_id = ? ORDER BY id DESC LIMIT 1", + "describe": { + "columns": [ + { + "name": "role: HostRole", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "installation_device", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "network_config", + "ordinal": 2, + "type_info": "Text" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + true, + true + ] + }, + "hash": "3b71d7d7ae75e75ec3ef1df2cd3c4d18520b9d56dd328b7edf576af9dac3c2c0" +} diff --git a/.sqlx/query-24f719d57144ecf4daa55f0aa5836c165872d70164401c0388e8d625f1b72d7b.json b/.sqlx/query-43cfa7b6dda8b9745ef74eb45f3f52a9193dcb09a4b917f0fde9f39058e0f276.json similarity index 55% rename from .sqlx/query-24f719d57144ecf4daa55f0aa5836c165872d70164401c0388e8d625f1b72d7b.json rename to .sqlx/query-43cfa7b6dda8b9745ef74eb45f3f52a9193dcb09a4b917f0fde9f39058e0f276.json index 60209751..b899023d 100644 --- a/.sqlx/query-24f719d57144ecf4daa55f0aa5836c165872d70164401c0388e8d625f1b72d7b.json +++ b/.sqlx/query-43cfa7b6dda8b9745ef74eb45f3f52a9193dcb09a4b917f0fde9f39058e0f276.json @@ -1,6 +1,6 @@ { "db_name": "SQLite", - "query": "SELECT host_id, installation_device FROM host_role_mapping WHERE role = ?", + "query": "SELECT host_id, installation_device, network_config FROM host_role_mapping WHERE role = ?", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "name": "installation_device", "ordinal": 1, "type_info": "Text" + }, + { + "name": "network_config", + "ordinal": 2, + "type_info": "Text" } ], "parameters": { @@ -19,8 +24,9 @@ }, "nullable": [ false, + true, true ] }, - "hash": "24f719d57144ecf4daa55f0aa5836c165872d70164401c0388e8d625f1b72d7b" + "hash": "43cfa7b6dda8b9745ef74eb45f3f52a9193dcb09a4b917f0fde9f39058e0f276" } diff --git a/.sqlx/query-779c5aa1643e714051ba141e5cc5788846925324bfb7d79662026fdc3e33c0ca.json b/.sqlx/query-779c5aa1643e714051ba141e5cc5788846925324bfb7d79662026fdc3e33c0ca.json new file mode 100644 index 00000000..082e702c --- /dev/null +++ b/.sqlx/query-779c5aa1643e714051ba141e5cc5788846925324bfb7d79662026fdc3e33c0ca.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "DELETE FROM host_role_mapping WHERE host_id = ?", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + }, + "hash": "779c5aa1643e714051ba141e5cc5788846925324bfb7d79662026fdc3e33c0ca" +} diff --git a/.sqlx/query-8d247918eca10a88b784ee353db090c94a222115c543231f2140cba27bd0f067.json b/.sqlx/query-8d247918eca10a88b784ee353db090c94a222115c543231f2140cba27bd0f067.json index 0b92e37a..ba998bc8 100644 --- a/.sqlx/query-8d247918eca10a88b784ee353db090c94a222115c543231f2140cba27bd0f067.json +++ b/.sqlx/query-8d247918eca10a88b784ee353db090c94a222115c543231f2140cba27bd0f067.json @@ -16,7 +16,7 @@ { "name": "data: Json", "ordinal": 2, - "type_info": "Blob" + "type_info": "Null" } ], "parameters": { diff --git a/.sqlx/query-c7ca191faaa23b3ec5019f8c4910f666db9c6c2be22ffe563be4b7caef645bd1.json b/.sqlx/query-c7ca191faaa23b3ec5019f8c4910f666db9c6c2be22ffe563be4b7caef645bd1.json new file mode 100644 index 00000000..cbe2716e --- /dev/null +++ b/.sqlx/query-c7ca191faaa23b3ec5019f8c4910f666db9c6c2be22ffe563be4b7caef645bd1.json @@ -0,0 +1,20 @@ +{ + "db_name": "SQLite", + "query": "SELECT data as \"data!: Vec\" FROM physical_hosts WHERE id = ? ORDER BY version_id DESC LIMIT 1", + "describe": { + "columns": [ + { + "name": "data!: Vec", + "ordinal": 0, + "type_info": "Null" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + }, + "hash": "c7ca191faaa23b3ec5019f8c4910f666db9c6c2be22ffe563be4b7caef645bd1" +} diff --git a/Cargo.lock b/Cargo.lock index d2641945..4a2cd119 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1512,6 +1512,7 @@ checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common", "inout", + "zeroize", ] [[package]] @@ -1968,6 +1969,35 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto_box" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16182b4f39a82ec8a6851155cc4c0cda3065bb1db33651726a29e1951de0f009" +dependencies = [ + "aead", + "crypto_secretbox", + "curve25519-dalek", + "salsa20", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto_secretbox" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d6cf87adf719ddf43a805e92c6870a531aedda35ff640442cbaf8674e141e1" +dependencies = [ + "aead", + "cipher", + "generic-array", + "poly1305", + "salsa20", + "subtle", + "zeroize", +] + [[package]] name = "ctr" version = "0.9.2" @@ -2682,6 +2712,110 @@ dependencies = [ "tokio", ] +[[package]] +name = "example-fleet-auth-callout" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-nats", + "base64 0.22.1", + "directories", + "env_logger", + "futures-util", + "harmony", + "harmony-k8s", + "harmony-nats-callout", + "harmony_types", + "jsonwebtoken", + "k3d-rs", + "k8s-openapi", + "kube", + "log", + "nkeys", + "reqwest 0.12.28", + "serde", + "serde_json", + "tempfile", + "tokio", + "tokio-test", + "tracing", + "tracing-subscriber", + "url", +] + +[[package]] +name = "example-fleet-e2e-demo" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-nats", + "clap", + "directories", + "env_logger", + "example-fleet-auth-callout", + "futures-util", + "harmony", + "harmony-fleet-operator", + "harmony-k8s", + "harmony-nats-callout", + "harmony-reconciler-contracts", + "harmony_types", + "k3d-rs", + "k8s-openapi", + "kube", + "log", + "nkeys", + "serde", + "serde_json", + "tempfile", + "tokio", + "tokio-test", + "tracing", + "tracing-subscriber", + "url", +] + +[[package]] +name = "example-fleet-sso-login" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64 0.22.1", + "clap", + "directories", + "env_logger", + "log", + "reqwest 0.12.28", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "example-fleet-staging-deploy" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-nats", + "clap", + "env_logger", + "harmony", + "harmony-k8s", + "harmony-nats-callout", + "harmony_types", + "k8s-openapi", + "kube", + "log", + "nkeys", + "reqwest 0.12.28", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", + "url", +] + [[package]] name = "example-grafana" version = "0.1.0" @@ -2909,6 +3043,17 @@ dependencies = [ "url", ] +[[package]] +name = "example-okd-ceph-alerts" +version = "0.1.0" +dependencies = [ + "harmony", + "harmony_cli", + "harmony_types", + "log", + "tokio", +] + [[package]] name = "example-okd-cluster-alerts" version = "0.1.0" @@ -3199,12 +3344,16 @@ name = "example_fleet_rpi_setup" version = "0.1.0" dependencies = [ "anyhow", + "base64 0.22.1", "clap", "harmony", "harmony_cli", "harmony_secret", "harmony_types", "log", + "reqwest 0.12.28", + "serde", + "serde_json", "tokio", ] @@ -3753,10 +3902,12 @@ version = "0.1.0" dependencies = [ "anyhow", "async-nats", + "async-trait", "chrono", "clap", "futures-util", "harmony", + "harmony-fleet-auth", "harmony-reconciler-contracts", "serde", "serde_json", @@ -3766,6 +3917,22 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "harmony-fleet-auth" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-nats", + "chrono", + "jsonwebtoken", + "reqwest 0.12.28", + "serde", + "serde_json", + "tokio", + "toml", + "tracing", +] + [[package]] name = "harmony-fleet-operator" version = "0.1.0" @@ -3776,6 +3943,7 @@ dependencies = [ "clap", "futures-util", "harmony", + "harmony-fleet-auth", "harmony-reconciler-contracts", "k8s-openapi", "kube", @@ -3784,6 +3952,7 @@ dependencies = [ "serde_json", "thiserror 2.0.18", "tokio", + "toml", "tracing", "tracing-subscriber", ] @@ -3807,6 +3976,26 @@ dependencies = [ "url", ] +[[package]] +name = "harmony-nats-callout" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-nats", + "futures-util", + "harmony-reconciler-contracts", + "jsonwebtoken", + "nats-jwt", + "nkeys", + "reqwest 0.12.28", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "harmony-node-readiness-endpoint" version = "0.1.0" @@ -3981,6 +4170,19 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "harmony_host_discovery" +version = "0.1.0" +dependencies = [ + "cidr", + "harmony", + "harmony_cli", + "harmony_macros", + "harmony_types", + "tokio", + "url", +] + [[package]] name = "harmony_i18n" version = "0.1.0" @@ -4794,6 +4996,30 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "integration-test-callout" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-nats", + "base64 0.22.1", + "futures-util", + "harmony-nats-callout", + "hex", + "jsonwebtoken", + "nats-jwt", + "nkeys", + "reqwest 0.12.28", + "rsa", + "serde", + "serde_json", + "tempfile", + "tokio", + "tokio-test", + "tracing", + "tracing-subscriber", +] + [[package]] name = "interactive-parse" version = "0.1.5" @@ -5330,6 +5556,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nats-jwt" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "nkeys", + "serde", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "neli" version = "0.7.4" @@ -5406,6 +5643,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879011babc47a1c7fdf5a935ae3cfe94f34645ca0cac1c7f6424b36fc743d1bf" dependencies = [ + "crypto_box", "data-encoding", "ed25519", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index 92182b4f..192d9d6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,11 @@ members = [ "harmony_assets", "opnsense-codegen", "opnsense-api", "fleet/harmony-fleet-operator", "fleet/harmony-fleet-agent", + "fleet/harmony-fleet-auth", "harmony-reconciler-contracts", + "nats/jwt", + "nats/callout", + "nats/integration-test-callout", ] [workspace.package] diff --git a/ROADMAP/fleet_platform/demo_runbook.md b/ROADMAP/fleet_platform/demo_runbook.md new file mode 100644 index 00000000..de346c33 --- /dev/null +++ b/ROADMAP/fleet_platform/demo_runbook.md @@ -0,0 +1,221 @@ +# Fleet Platform Demo Runbook + +48-hour-demo edition. Covers the operator-side (NationTech) and the +customer-developer-side (two devs onboarding two Pis, applying a +container deployment to them). Hand-on, no UI yet. + +## Roles + +- **NationTech operator** — runs `fleet-staging-deploy` once against the + customer's OKD cluster. +- **Customer developer** — runs `fleet-sso-login` to prove auth works, + then runs `fleet-rpi-setup` for each Pi, then applies their workload + via the existing `harmony-apply-deployment` example. + +## Prerequisites + +### Cluster (operator-side) + +- OKD ≥ 4.10 (HAProxy ingress, edge-TLS). +- Wildcard DNS `*.` pointing at the cluster ingress IP + (e.g. `*.customer1.nationtech.io`). +- Wildcard cert that the HAProxy router serves for that domain (the + default OKD pattern). +- `cert-manager`, `cloudnative-pg` operators installed (Zitadel chart + depends on them via `K8sAnywhereTopology`'s ensure_ready). +- Access to a container registry the cluster can pull from. Customer + may have their own; the default in `fleet-staging-deploy` is + `quay.io/nationtech/harmony-nats-callout:demo`. + +### Driver machine (operator + developers) + +- `kubectl` with kubeconfig wired up. +- `cargo` (Rust toolchain). +- `podman` (used to build the agent image / fleet-callout image). +- `ssh` into the Pis from the developers' machines. + +### Pis + +- Pi OS Lite booted, SSH server enabled, developer's SSH pubkey in + `~/.ssh/authorized_keys`. `fleet-rpi-setup` handles the rest. + +## Operator: deploy the staging stack + +```bash +# 1. Build the callout image and push it to the customer's registry. +cargo build --release -p harmony-nats-callout +podman build -t quay.io/nationtech/harmony-nats-callout:demo \ + -f nats/callout/Dockerfile . +podman push quay.io/nationtech/harmony-nats-callout:demo + +# 2. Deploy the central stack. +cargo run -p example-fleet-staging-deploy -- \ + --base-domain customer1.nationtech.io \ + --kube-context customer1-prod \ + --callout-image quay.io/nationtech/harmony-nats-callout:demo \ + --nats-auth-pass "$(openssl rand -hex 16)" \ + --nats-system-pass "$(openssl rand -hex 16)" +``` + +Expected output ends with a "next steps" panel containing the project +ID, the `harmony-cli` client_id, the NATS WSS URL, and the exact +follow-up commands. Save those — both developers will need them. + +## Developer: prove SSO works + +```bash +cargo run -p example-fleet-sso-login -- \ + --base-domain customer1.nationtech.io \ + --client-id +``` + +Browser opens, developer logs into Zitadel, CLI prints +`Welcome ` and persists `~/.local/share/harmony/sso-session.json`. + +Two developers each do this once with their own Zitadel accounts. + +## Operator (or developer with an admin PAT): onboard a Pi + +```bash +# Extract the Zitadel admin PAT once (it's in a K8s secret on the +# staging cluster). +PAT=$(kubectl --context customer1-prod \ + -n zitadel get secret iam-admin-pat \ + -o jsonpath='{.data.pat}' | base64 -d) + +# Cross-compile the agent for aarch64 (one-time per agent rev). +cargo build --release --target aarch64-unknown-linux-gnu -p harmony-fleet-agent + +# Onboard Pi #1 — sensor on the floor with arch=aarch64, group=group-a. +cargo run -p example-fleet-rpi-setup -- \ + --pi-host 192.168.1.42 \ + --pi-user pi \ + --device-id sensor-floor-01 \ + --labels "group=group-a,arch=aarch64,role=sensor" \ + --bootstrap-token "$PAT" \ + --zitadel-issuer-url https://zitadel.customer1.nationtech.io \ + --zitadel-project-id \ + --nats-url wss://nats.customer1.nationtech.io/ \ + --agent-binary ./target/aarch64-unknown-linux-gnu/release/fleet-agent + +# Onboard Pi #2 — different group label so we can target by selector. +cargo run -p example-fleet-rpi-setup -- \ + --pi-host 192.168.1.43 \ + --pi-user pi \ + --device-id sensor-shelf-02 \ + --labels "group=group-b,arch=aarch64,role=sensor" \ + --bootstrap-token "$PAT" \ + --zitadel-issuer-url https://zitadel.customer1.nationtech.io \ + --zitadel-project-id \ + --nats-url wss://nats.customer1.nationtech.io/ \ + --agent-binary ./target/aarch64-unknown-linux-gnu/release/fleet-agent +``` + +Each Pi onboarding does the following on the device: + +- Installs podman + systemd-container. +- Creates the `fleet-agent` user (with subuid/subgid for rootless + podman + linger). +- Drops the per-device Zitadel JSON key at + `/etc/fleet-agent/zitadel-key.json` (mode 0640, owner fleet-agent). +- Renders `/etc/fleet-agent/config.toml` with `type = "zitadel-jwt"` + pointing at the keyfile. +- Starts `fleet-agent.service` under systemd. + +The agent connects to NATS over WSS using the JWT-bearer token it +mints from its keyfile. async-nats's auto-reconnect + the auth +callback re-mints the token on every reconnect attempt — the +"never lose connectivity" property holds across: + +- Token expiry (12h Zitadel default → re-minted ~5 minutes before). +- NATS pod restart (chart upgrade, drain, etc.). +- Pi network blip (DHCP renewal, Wi-Fi roam). + +## Verify the fleet from the operator side + +```bash +kubectl --context customer1-prod -n fleet-system get device.fleet.nationtech.io +# NAME LABELS +# sensor-floor-01 arch=aarch64,group=group-a,role=sensor +# sensor-shelf-02 arch=aarch64,group=group-b,role=sensor + +kubectl --context customer1-prod -n fleet-system logs deployment/fleet-callout +# ... received auth callout request +# ... Zitadel JWT validated, generating user JWT device_id=sensor-floor-01 role=device +``` + +## Developer: deploy a container to a labeled subset + +```bash +# Apply the customer's backend (single service + sqlite volume + envs) +# to every device with group=group-a. +cargo run -p example_harmony_apply_deployment -- \ + --namespace fleet-demo \ + --name customer-backend \ + --selector group=group-a \ + --image registry.example.com/customer/backend:1.4 \ + --port 8080:8080 \ + --env DATABASE_URL=sqlite:///data/app.db \ + --env LOG_LEVEL=info \ + --volume /var/lib/customer-backend:/data \ + --restart unless-stopped +``` + +The operator sees one Deployment CR materialized, NATS KV gets a +`desired-state..customer-backend` entry per matched +device, and each Pi's agent reconciles podman to match. The +container's data persists across agent restarts and Pi reboots +because the bind mount survives both. + +`kubectl get device` shows the agents heartbeating; their per-deployment +state shows up on `Device.status.aggregate` (Chapter 2 reflect-back +already in place). + +### Translating a docker-compose to a Deployment CR + +For the call: walk through the customer's compose file once, paste +the equivalent `--env`/`--volume`/`--port` flags. Bind mounts only; +named volumes need a separate decision per service. Most compose +shapes translate mechanically; depends_on / startup ordering does +not (PodmanV0 has no ordering primitive — design out of scope for +the demo). + +## Cross-device security model (worth showing) + +- Pi A's NATS connection has a user JWT permissioned to + `device-state.sensor-floor-01.>` and `device-commands.sensor-floor-01.>`. +- Pi A *cannot* publish to or subscribe from `sensor-shelf-02`'s + subjects — the auth callout never grants them. +- An admin user (Zitadel role `fleet-admin`) gets `>` on both + publish + subscribe — they observe every device. +- A user with no fleet role is rejected at NATS connect time. + +This is the same security model the local `examples/fleet_auth_callout` +suite (3 cargo tests sharing a OnceCell k3d cluster) verifies in CI. + +## What's NOT in the demo + +- Compose-to-Deployment auto-translation (low priority — manual + translation during the call works). +- A web UI for `harmony fleet apply` (post-demo). +- Tailscale/Headscale-based SSH backdoor to the Pis (separate daemon, + out of scope). +- Device-join-request + admin-approve flow (would replace + bootstrap-PAT pattern; out of scope). +- OpenBao for non-NATS secrets (env-var-only is fine for demo). +- K8s OIDC integration so kubectl accepts Zitadel JWTs (post-demo). + +## Re-run idempotency + +Every harness in this runbook is idempotent. + +- `fleet-staging-deploy` rides helm-upgrade-by-default, the + ZitadelSetupScore search-then-create loop, and a persisted issuer + NKey in a K8s secret. +- `fleet-rpi-setup` byte-compares the rendered TOML against the + device's existing config and only reapplies on drift; the keyfile + drop + agent restart only happen when something actually changed. +- `harmony-apply-deployment` is a `kube::Api::patch(...)` apply, so + re-running with the same fields is a server-side no-op. +EOF +) diff --git a/ROADMAP/fleet_platform/nats-sso.md b/ROADMAP/fleet_platform/nats-sso.md new file mode 100644 index 00000000..44a2cb74 --- /dev/null +++ b/ROADMAP/fleet_platform/nats-sso.md @@ -0,0 +1,52 @@ +-- documentation : https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth +https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/jwt +https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt + +--- context : openbao allows integration with jwks or whatever protocol required to interact with zitadel directly, but nats does not. See documentation above and analysis below : + + +These are notes taken from this video + +https://www.youtube.com/watch?v=VvGxrT-jv64 +https://github.com/synadia-io/rethink_connectivity/tree/main/19-auth-callout + + + +1. `nsc generate nkey --account` + +generates nsc key pair for the auth callout service + +2. nats.conf + +add + +``` +authorization { + auth_callout { + issuer: + auth_users: [ auth, user ] # list of users we can discover on the account. (something I don't get here, I want dynamic users management through the jwt) + account: CHAT # Name of the account we want to discover users on, this account exists in the accounts block + } +} +``` + + +3. Write the auth callout service, full code example here https://github.com/synadia-io/rethink_connectivity/tree/main/19-auth-callout + 3.1 This service will be the app authorized by the SSO provider (google in the example, zitadel in our case) + 3.2 Load the NKeySeed (private key from the pair above) + 3.3 connect to nats. We will communicate with the nats server through nats protocol itself to handle auth callout requests + 3.4 Subscribe to the KV workspace (not sure why yet) + 3.5 start forging the nats jwt token using the request nkey (each new client connection comes with an nkey which will be used for the session) + 3.6 setup the audience (nats account from above, CHAT in the example) + 3.7 Validate and decode the jwt (nats passes the user jwt as request connectionoptions token) + 3.8 Add user to the workspace (wtf this is completely dynamic?, how do we remove it?) + 3.9 Attach permissions inside the nats jwt such as `Allow : [ "$JS.API.INFO", format!("chat.*.{userId}") ]` where userId is read from the google jwt, our case zitadel jwt. + + +Now, synadia provides a small SDK to ease writing auth callout services in Go. But we're in rust. It might be worth writing this thing in go to benefit from synadia's stuff but from what I gathered, only the nats jwt minting is maybe something that we would benefit a lot from. But then again I think that crafting a jwt is something standard? + +Interaction with zitadel and all the rest is likely the same or more work for us as our entire ecosystem is in rust. Let's analyze this properly. + +https://github.com/synadia-io/callout.go/tree/main + +https://github.com/synadia-io/callout.go/tree/main/examples/dynamic_accounts diff --git a/ROADMAP/fleet_platform/v0_1_plan.md b/ROADMAP/fleet_platform/v0_1_plan.md index 5bf663fc..5ec1c223 100644 --- a/ROADMAP/fleet_platform/v0_1_plan.md +++ b/ROADMAP/fleet_platform/v0_1_plan.md @@ -360,6 +360,24 @@ auth from Chapter 4. --- +## Chapter 6 — Customer demo rehearsal **[in progress]** + +48-hour customer demo prep. PO assessment concluded that promising a +real-OKD deployment without first proving the JWT-auth chain is +reckless. **VM-based rehearsal first**, OKD second. + +The rehearsal extends `smoke-a4` (k3d + libvirt VM + agent + apply +CR + reconcile podman) with **Zitadel + auth callout + agent JWT +auth**. Two devices + one admin. Same code paths as production — +only the cluster topology differs. + +Detailed plan: [`v0_demo_e2e.md`](v0_demo_e2e.md). + +Once the VM rehearsal is green (success criteria in that doc), the +residual deltas to ship to real OKD are configuration, not new code. + +--- + ## Principles — what we've learned and want to keep doing - **No yaml in framework code paths.** Every kube-rs type is diff --git a/ROADMAP/fleet_platform/v0_demo_e2e.md b/ROADMAP/fleet_platform/v0_demo_e2e.md new file mode 100644 index 00000000..3448026a --- /dev/null +++ b/ROADMAP/fleet_platform/v0_demo_e2e.md @@ -0,0 +1,239 @@ +# V0 Demo End-to-End — VM-Based Rehearsal + +48-hour customer demo prep. The PO assessment from +`memory/feedback_*` and the prior planning discussion concluded that +shipping the customer demo against an untested OKD path is reckless. +This doc plans the **VM-based rehearsal** that proves the JWT-auth +chain end-to-end before we touch a real cluster. + +## Why VM, not OKD + +Smoke-a4 already greens the chain `k3d + in-cluster NATS + libvirt +ARM VM + agent + apply CR + reconcile podman + status reflect-back` +on x86_64 and aarch64. Zero new infra; we extend the existing +harness with **Zitadel + auth callout + agent JWT auth**. + +Same Helm charts, same Scores, same agent code paths as production. +Only the cluster topology differs (k3d/traefik vs OKD/HAProxy). The +remaining OKD-specific deltas — Route annotations, edge-TLS, real DNS +— are small and testable in isolation **after** the VM smoke is +green. + +Compared to validating directly against OKD: + +- **Local + reproducible**: same `cargo run` runs on any dev machine + with podman + libvirt + k3d. +- **Fast iteration**: bring-up is ~12-15 min cold, ~30s warm. We + fix integration bugs in minutes, not "wait for cluster admin" + hours. +- **CI-able**: greens in a single `cargo test` invocation, so we + prevent regressions post-demo. + +## What this rehearsal proves + +- `ZitadelScore`'s `FirstInstance.Org.Machine.Pat` block actually + causes the chart to provision the `iam-admin-pat` secret (we + added the Helm config, never confirmed the secret materialises). +- `ZitadelSetupScore::ensure_machine_user` reaches a working JSON + keyfile when called outside its k3d unit tests. +- The agent's `CredentialSource::ZitadelJwt` mints a token, that + token actually authenticates against the auth callout, and the + callout admits it into the `DEVICES` account. +- async-nats's auto-reconnect-with-auth-callback fires fresh tokens + on real NATS pod restart — the **load-bearing** "never lose + connectivity to a device" guarantee. +- The full operator → NATS KV → agent → podman → status-back-to-CR + loop survives the credential-source rewrite. +- Container env / volumes / restart policy land on the real podman + instance, not just in unit tests. + +## What it does NOT prove (deferred, accepted) + +- OKD HAProxy edge-TLS termination on the Zitadel and NATS-WSS + Routes. Tested separately in a follow-up smoke once the VM smoke + is green. +- Real DNS resolution from a customer LAN. We inject `/etc/hosts` + entries on each VM so `sso.fleet.local` resolves to the libvirt + host. +- Browser-driven device-code SSO (`fleet_sso_login` is compile-only + today). Out of scope for this rehearsal — admin verification uses + an injected machine-user token via JWT-bearer (same as + `examples/fleet_auth_callout`). +- Customer's docker-compose translation. Manual at the call. + +## Architecture + +``` + k3d cluster (host) + ┌─────────────────────────────────────────────────┐ + │ Zitadel + Postgres http://sso.fleet.local │ + │ │ (host:8080) │ + │ │ project + roles + per-device users │ + │ ▼ │ + │ ZitadelSetupScore cache → keyfiles (per VM) │ + │ │ + │ NATS (auth_callout) nats://:30422 │ + │ ▲ │ + │ │ JWT-bearer via callout │ + │ fleet-callout pod │ + │ │ + │ fleet-operator → KV writes desired-state │ + │ ▲ │ + │ │ kube apply Deployment CR │ + └──────┼──────────────────────────────────────────┘ + │ + ┌──────┼──────────────────────────────────────────┐ + │ libvirt default NAT (host = 192.168.122.1) │ + └──────┼──────────────────────────────────────────┘ + ▼ + ┌──────────────┐ ┌──────────────┐ + │ device-A │ │ device-B │ (cloud-init Ubuntu VMs) + │ fleet-agent │ │ fleet-agent │ + │ + Zitadel │ │ + Zitadel │ + │ JWT key │ │ JWT key │ + │ + podman │ │ + podman │ + └──────────────┘ └──────────────┘ +``` + +## Bring-up sequence + +1. Ensure k3d cluster `fleet-e2e-demo` (port mappings 8080→80, + 30422→30422; same as fleet_auth_callout). +2. Reuse `fleet_auth_callout::bring_up_stack` constituent functions: + - Deploy Zitadel + Postgres + - Wait for `iam-admin-pat` secret to materialise + - Provision project `fleet`, API app, roles `fleet-admin` + + `device` +3. Install fleet operator from its Helm chart (Chapter 3 ships this). +4. Generate issuer NKey, deploy NATS with `auth_callout` block, deploy + `NatsAuthCalloutScore` (image side-loaded into k3d). +5. **For each device i in 1..=num_devices**: + - Mint Zitadel machine user `device-${device_id_i}` with the + `device` role grant via `ZitadelSetupScore`. Cache the JSON key. + - Provision libvirt VM via `ProvisionVmScore` (cloud-init + Ubuntu, x86_64). + - SSH in via `LinuxHostTopology`. Inject `/etc/hosts`: + ` sso.fleet.local`. + - Run `FleetDeviceSetupScore` with + `FleetDeviceAuth::ZitadelJwt { machine_key_json, ... }`. +6. Mint admin Zitadel machine user with `fleet-admin` role (one-off + for verification — separate from the per-device users). +7. Hand off / run tests. + +Idempotent across re-runs: +- k3d cluster create skipped if exists. +- ZitadelSetupScore is search-then-create. +- VM creation: `ProvisionVmScore` reports NOOP if domain exists. +- FleetDeviceSetupScore byte-compares the rendered TOML. + +## Tests + +Real `#[tokio::test]` functions sharing a `OnceCell`-bringup. Run +sequentially (`--test-threads=1` because they share the cluster + +VMs): + +| # | Name | What it asserts | +|---|---|---| +| 1 | `both_devices_heartbeat_within_60s` | `Device` CRs for A and B materialise with their labels. | +| 2 | `deployment_targets_only_matching_device` | Apply CR with `group=group-a` selector → A reconciles, B doesn't. | +| 3 | `deployment_status_aggregates_back_to_cr` | `.status.aggregate.succeeded == 1` within 60s. | +| 4 | `env_vars_and_volume_propagate_to_container` | SSH into A, `podman inspect` confirms env + bind mount. | +| 5 | `admin_jwt_reads_any_device_subject` | Admin token sees A's heartbeat. | +| 6 | `cross_device_isolation_enforced_in_vm` | A's per-device JWT cannot subscribe to B's command subject. | +| 7 | `agent_recovers_from_nats_pod_restart` | Kill NATS pod, both agents reconnect with fresh tokens within 30s. | + +Test 7 is the load-bearing one — it's the only one that exercises +the auto-reconnect + auth-callback re-mint path under realistic +disturbance. Asserted by: kill nats-0 pod via kube API, wait for +new pod ready, then publish a message from admin and verify both +agents pick it up. + +## Implementation order + +1. ✏️ Roadmap doc (this file). +2. 🆕 `examples/fleet_e2e_demo/` crate skeleton. +3. ♻️ Refactor `fleet_auth_callout::bring_up_stack` constituent + functions to be `pub` so they're individually re-usable. +4. ➕ `/etc/hosts` injection step in `FleetDeviceSetupScore`. +5. ➕ Operator install via Helm in the new harness. +6. 🔗 Compose `bring_up_full_stack(num_devices)`. +7. 🧪 Write the 7 tests. +8. 🚦 Cold-start the bring-up. Fix what breaks (expected: ≥3 things). +9. 🧪 Run tests. Fix what breaks (expected: ≥1 thing). +10. 💥 Run test 7 in isolation; verify reconnect timing. +11. 📝 Update `demo_runbook.md` with VM-rehearsal commands. + +## Known risks / debugging traps + +- **`iam-admin-pat` secret timing.** Chart's setup job runs on first + install but may take 30-90s after Helm reports the chart Ready. + Need a wait-for-secret loop before invoking ZitadelSetupScore. + (Today the `bring_up_stack` in `fleet_auth_callout` doesn't have + this — it works because we re-run after the secret has settled. + First-cold-run will likely fail.) +- **Per-device machine keys are returned ONCE.** ZitadelClientConfig + caches them locally. If the cache file is missing/corrupt + mid-bring-up, devices fail at TOML render. Persist the cache + atomically. +- **VM /etc/hosts mutation.** Cloud-init can do this, but + FleetDeviceSetupScore doesn't currently touch /etc/hosts. Add a + step before package install (low risk: idempotent line-in-file). +- **k3d port collision.** Existing `harmony` and `harmony-example` + clusters from prior sessions may collide on host ports. Either + pick unique ports or fail loudly when in use. +- **NATS pod restart test is non-deterministic.** async-nats's + reconnect timing depends on backoff schedule. Assert via "publish + succeeds within 30s after restart" rather than literal reconnect + events; the latter is implementation-detail-dependent. +- **Bring-up time.** Cold: ~15 min (Zitadel + Postgres dominate). + Set test runner timeout accordingly. Warm: ~30s. The OnceCell + pattern means the cost is amortised across the test suite. +- **Agent reconciler is non-idempotent for env / volume specs.** + `harmony/src/modules/podman/topology.rs::matches_spec` returns + false (forcing destroy + recreate) for any `ContainerSpec` with + non-empty env or volumes — by deliberate "fail-safe" choice the + original author made because podman's list endpoint doesn't + surface env/mount data. With the periodic reconcile firing every + 30s, this becomes a destroy-and-recreate loop for any + non-trivial Deployment. Demo workaround: keep demo specs free of + env + volumes (the hello-web nginx demo already is). Real fix + (out of scope for the demo, in scope for delivery): switch the + drift check to `containers.get(name).inspect()` which returns + env + mounts, do a structural compare, lock with an integration + test asserting container ID is stable across two consecutive + applies. FIXME tag at the offending line. + +## Success criteria for the rehearsal day + +Tomorrow's all-day testing is "green" if: + +1. Cold `cargo run -p example-fleet-e2e-demo` brings up the full + stack and prints credentials in under 20 minutes. +2. `cargo test -p example-fleet-e2e-demo --test e2e_walking_skeleton` + greens all 7 tests on a clean machine. +3. `cargo test ... --test e2e_walking_skeleton agent_recovers_from_nats_pod_restart` + greens reliably 5 runs in a row. + +Anything below this and we don't show up to the customer call with a +"staging deployed" promise — we reframe to "architecture walkthrough ++ local k3d security-model demo + pilot scheduled in 1-2 weeks." + +## What follows after greens + +Once the VM rehearsal is green, the residual deltas to ship to +real OKD are: + +1. Replace `K8sAnywhereTopology` (which falls back to k3d via + `HARMONY_USE_LOCAL_K3D`) with a real-OKD profile. The Score code + doesn't change; only the topology bootstrap. +2. Verify Route annotations actually edge-TLS for both Zitadel and + NATS-WSS in the customer's cluster. ~30 min smoke. +3. Push the callout image to a registry the customer's cluster + pulls from. Mechanical. +4. Real wildcard DNS for `*.` pointed at the cluster + ingress. + +None of those four require new code; they're configuration. The +heavy lifting (the JWT auth chain, the agent's reconnect loop, the +operator → KV → agent → podman → status loop) is what the VM +rehearsal proves. diff --git a/docs/guides/fleet-manual-token-mint.md b/docs/guides/fleet-manual-token-mint.md new file mode 100644 index 00000000..046eda88 --- /dev/null +++ b/docs/guides/fleet-manual-token-mint.md @@ -0,0 +1,189 @@ +# Manual Zitadel token mint + NATS write + +Operator-side recipe for talking to a callout-protected NATS by +hand: sign a JWT-bearer assertion with a Zitadel machine user's +private key, exchange it for an access token, drive `nats` CLI +commands with the token. Useful for debugging the auth chain, +poking the desired-state KV without the operator running, and +validating that a deployed callout is actually accepting what +you think it should. + +Read [fleet-zitadel-faq.md](./fleet-zitadel-faq.md) first for the +underlying mechanism (RFC 7523 JWT-bearer flow, why we sign +locally, what each claim means). + +## Inputs you need + +Five strings: + +| Input | Where to find it | +| --- | --- | +| `OIDC_ISSUER_URL` (the Zitadel base URL) | callout Deployment env: `kubectl exec -n fleet-system deploy/fleet-callout -- printenv OIDC_ISSUER_URL` | +| `project_id` (becomes the access token's `aud`) | callout Deployment env: `OIDC_AUDIENCE` | +| Machine user's `userId` | the JSON keyfile's `userId` field | +| Machine user's `keyId` | the JSON keyfile's `keyId` field | +| Private RSA key (PEM) | the JSON keyfile's `key` field | + +Get the `fleet-ops` (admin role) JSON keyfile from the cache: + +```bash +jq -r '.machine_keys["fleet-ops"]' \ + ~/.local/share/harmony/zitadel/client-config.json \ + > /tmp/fleet-ops.json + +jq -r '.userId' /tmp/fleet-ops.json # → user_id +jq -r '.keyId' /tmp/fleet-ops.json # → key_id +jq -r '.key' /tmp/fleet-ops.json > /tmp/fleet-ops.pem +``` + +The cache may drift from the deployed Zitadel state if Zitadel has +been re-seeded; **always pull `OIDC_AUDIENCE` from the running +callout**, not from the cache. The cache fix landed in commit +`f4d6fb94` but older entries can still trip you up. + +## Mint script (PyJWT) + +```python +# pip install PyJWT requests ← MUST be PyJWT, not the `jwt` package. +# The two share `import jwt`; `jwt` (the package) refuses raw PEM +# strings and demands an AbstractJWKBase wrapper. PyJWT takes PEM +# directly. If you ever see `TypeError: key must be an instance of +# a class implements jwt.AbstractJWKBase`, you have the wrong one. + +import jwt, time, requests + +# These come from the running callout + Zitadel. Don't reuse stale +# values from a checked-in note; verify against the live cluster. +OIDC_ISSUER_URL = "http://sso.fleet.local:8080" +PROJECT_ID = "371158654839160853" # = OIDC_AUDIENCE on callout +USER_ID = "..." # from machine keyfile +KEY_ID = "..." # from machine keyfile + +key = open("/tmp/fleet-ops.pem").read() +now = int(time.time()) + +assertion = jwt.encode( + { + "iss": USER_ID, + "sub": USER_ID, + "aud": OIDC_ISSUER_URL, # for Zitadel itself, NOT the project_id + "exp": now + 60, # Zitadel rejects exp - iat > 60s + "iat": now, + }, + key, + algorithm="RS256", + headers={"kid": KEY_ID}, # PyJWT spelling — `headers=`, not `optional_headers=` +) + +r = requests.post( + f"{OIDC_ISSUER_URL}/oauth/v2/token", + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "assertion": assertion, + # Three scopes: + # openid — base OIDC + # urn:zitadel:iam:org:projects:roles — PLURAL. + # Without this, Zitadel omits the role claim and the + # callout rejects with "no authorized role in token". + # urn:zitadel:iam:org:project:id::aud — singular. + # Tells Zitadel to put into the access token's + # `aud` claim, which the callout's audience check + # compares against OIDC_AUDIENCE. + "scope": ( + "openid " + "urn:zitadel:iam:org:projects:roles " + f"urn:zitadel:iam:org:project:id:{PROJECT_ID}:aud" + ), + }, +) +r.raise_for_status() +token = r.json()["access_token"] + +# Sanity check — decode without verifying signature so you can see +# what Zitadel actually emitted. If anything below is wrong, the +# callout will reject your token. +print(jwt.decode(token, options={"verify_signature": False})) +print(token) +``` + +Expected decoded claims (the parts the callout will check): + +| Claim | What it should be | Why | +| --- | --- | --- | +| `iss` | `OIDC_ISSUER_URL` (byte-equal) | Callout: `validation.set_issuer(&[&self.issuer_url])` | +| `aud` | `[""]` | Callout: `validation.set_audience(&[&self.audience])`; the array form is Zitadel's default | +| `exp` | ~now + 12h | Zitadel default access token TTL | +| `client_id` | the machine user's username (`fleet-ops`, `device-vm-device-00`, …) | Callout uses this as `device_id_claim` (with optional `DEVICE_ID_PREFIX_STRIP` applied) | +| `urn:zitadel:iam:org:project::roles` | object with role names as keys (e.g. `{"fleet-admin": {"": ""}}`) | Callout uses this as `roles_claim` and admits the role if `fleet-admin` or `device` is present | + +If any of these is wrong, fix the script before bothering with NATS. + +## Drive NATS with the token + +`nats --token=` puts the value into the CONNECT frame's +`auth_token`, which is what the callout expects. + +```bash +NATS_SERVER=192.168.122.1:30422 # libvirt host's port mapping +TOKEN=$(python3 mint.py | tail -1) # last line is the raw token + +# Read everything (admin role allows >): +nats --server "$NATS_SERVER" --token "$TOKEN" kv ls device-info +nats --server "$NATS_SERVER" --token "$TOKEN" kv get device-info info.vm-device-00 + +# Write a desired state — agent's KV watcher fires within 1s, +# reconciler creates the podman container. +nats --server "$NATS_SERVER" --token "$TOKEN" \ + kv put desired-state vm-device-00.hello-web '{ + "name": "hello-web", + "type": "PodmanV0", + "data": { + "services": [{ + "name": "testnginx", + "image": "docker.io/nginx:latest", + "ports": ["8080:80"] + }] + } + }' +``` + +The exact JSON shape comes from +`harmony-reconciler-contracts/src/fleet.rs` — read that crate when +in doubt about field names, NOT this doc; this doc is a worked +example and may drift. + +## Common failures and what they mean + +| Symptom | Likely cause | +| --- | --- | +| `TypeError: key must be an instance of … AbstractJWKBase` | Wrong PyPI package. `pip uninstall jwt && pip install PyJWT`. | +| HTTP 400 from `/oauth/v2/token`: `"invalid_grant_type"` | Forgot the percent-encoded form encoding, OR `grant_type` value mistyped. The full URN is `urn:ietf:params:oauth:grant-type:jwt-bearer`. | +| HTTP 400: `"jwt: token is expired"` | Your assertion's `exp` is in the past. Wall-clock skew between your laptop and the cluster — sync NTP. | +| Token mints but no `urn:zitadel:…:roles` claim | Missing the **plural** `urn:zitadel:iam:org:projects:roles` in scope. | +| Token mints but `aud` is the issuer URL instead of the project id | Forgot the `urn:zitadel:iam:org:project:id::aud` scope. | +| NATS CLI: `nats: Authorization Violation` | Token is good but callout rejected it — check `kubectl logs -n fleet-system -l app=fleet-callout` for the actual reason. The most common ones are "InvalidAudience" (your `aud` ≠ deployed `OIDC_AUDIENCE`) and "no authorized role in token". | +| Callout log: `JWT validation failed: InvalidIssuer` | Trailing slash drift. `OIDC_ISSUER_URL=http://sso.fleet.local:8080/` ≠ `http://sso.fleet.local:8080`. Match exactly. | + +When the callout rejects, **its log is the source of truth**, not +your decoded claims. The validation error includes which check +failed; work backwards from there. + +## Rotating the deployed `OIDC_AUDIENCE` + +If Zitadel was re-seeded and `OIDC_AUDIENCE` on the callout now +points at a non-existent project: + +```bash +# 1. Confirm the live project id +oc -n zitadel exec -ti deploy/zitadel -- /bin/sh -c \ + 'curl -s -H "Authorization: Bearer $PAT" \ + $ZITADEL_URL/management/v1/projects/_search \ + | jq ".result[] | select(.name == \"fleet\") | .id"' + +# 2. Re-run the bring-up — the live-query fix in f4d6fb94 will +# refresh OIDC_AUDIENCE on the next NatsAuthCalloutScore apply. +``` + +The shape of `mint.py` doesn't change between regular operation +and post-recovery — you just plug in fresh values for +`OIDC_AUDIENCE` and `PROJECT_ID`. diff --git a/docs/guides/fleet-zitadel-faq.md b/docs/guides/fleet-zitadel-faq.md new file mode 100644 index 00000000..45929f57 --- /dev/null +++ b/docs/guides/fleet-zitadel-faq.md @@ -0,0 +1,185 @@ +# Fleet × Zitadel FAQ + +Technical reference for the Zitadel setup behind the fleet +auth callout. Describes what exists, why it's that way, and where +each piece lives in the code. + +Code anchors: +- `examples/fleet_e2e_demo/src/lib.rs` — bring-up flow +- `harmony/src/modules/zitadel/setup.rs` — `ZitadelSetupScore` +- `harmony/src/modules/zitadel/mod.rs` — Helm install +- `nats/callout/src/handler.rs` — auth callout +- `fleet/harmony-fleet-agent/src/credentials.rs` — JWT-bearer mint + +--- + +## What is an "application" in Zitadel? + +An OIDC client config: `clientId`, allowed grant types, redirect +URIs (browser apps only), PKCE settings (browser apps only). + +Apps are not containers for users or roles — those live one +level up at the org. An app is the entry point a service uses to +delegate auth to Zitadel. + +The `nats` app is **API type**: JWT-bearer / client-credentials +only, no browser flow. Headless agents never see a login page. +The app's `clientId` is what tokens carry as `aud` and what the +auth callout validates against (`OIDC_AUDIENCE` env on the callout +Deployment). + +## Why are users and roles at org level instead of per-project? + +Roles are defined inside a project but are essentially labels — +strings + display names with no inherent permissions. Each app +enforces them in code (the callout maps `device` → a +permission template). + +Users live at org level so one identity can hold roles across +multiple projects in the same org and SSO between them. Role +grants are the join: "user X has roles \[A, B\] on project Y." + +The only privilege ladder Zitadel enforces directly is at the +instance/org level (IAM-Owner, Org-Owner). Project roles say +nothing about Zitadel admin rights. + +## What is each service account for? + +| User | Created by | Purpose | +| --- | --- | --- | +| `iam-admin` | Helm `FirstInstance.Org.Machine` | IAM-Owner. Its PAT (`iam-admin-pat` k8s Secret) drives the management API from `ZitadelSetupScore`. | +| `login-client` | Helm `FirstInstance.Org.LoginClient` | Internal — Zitadel's login UI pod uses it to call back into Zitadel. Don't touch. | +| `fleet-ops` | `fleet_e2e_demo` admin setup | `fleet-admin` role grant, JSON key, used by tests and admin tooling. | +| `device-vm-device-NN` | `fleet_e2e_demo::provision_device` | One per VM. JSON key copied to `/etc/fleet-agent/zitadel-key.json`. `device` role grant. | +| `ops-station`, `sensor-a`, `sensor-b`, `intruder` | `fleet_auth_callout` (separate example) | Leftovers from previous runs. Postgres survives cluster recreates. Harmless, deletable. | + +The `device-` prefix on per-device usernames is intentional: +Zitadel emits the username verbatim in the access token's +`client_id` claim. The callout strips `device-` to recover the +bare device id used for NATS subject interpolation +(`DEVICE_ID_PREFIX_STRIP=device-` env var on the callout; +`nats/callout/src/zitadel.rs::extract_device_id`). + +## How does the agent authenticate? Are JWTs / refresh tokens cached? + +On disk the agent keeps **only the JSON machine key** (RSA +private key) at `/etc/fleet-agent/zitadel-key.json`. + +It does NOT store: +- access tokens (in memory only) +- refresh tokens (the JWT-bearer flow has none — RFC 7523 is + stateless by design) + +On every NATS (re)connect, `credentials.rs::zitadel_mint`: + +1. Builds a JWT assertion with `exp = now + 60s`, signs it with + the RSA key +2. POSTs it to `/oauth/v2/token` with grant type + `urn:ietf:params:oauth:grant-type:jwt-bearer` +3. Receives an access token (~12h validity), caches it in memory +4. Re-mints when within 5min of expiry + (`TOKEN_REFRESH_LEEWAY_SECS`) + +## What happens to an offline agent? + +| Time offline | Behavior | +| --- | --- | +| 0 – ~12 h | Cached access token still valid. Reconnects work transparently. | +| > ~12 h | Token expired. Agent enters reconnect loop until network returns, then mints fresh on first successful reach. | + +The RSA key never expires until rotated server-side. + +## Where are the lifetimes set? + +- **Access token TTL** — Zitadel UI: Org → Settings → OIDC + Settings → "Access Token Lifetime" (default 12 h). +- **Assertion TTL** — hardcoded 60 s in + `credentials.rs::ASSERTION_LIFETIME_SECS`. Zitadel rejects + assertions where `exp - iat > 60 s`; this is server-enforced, + not a knob. +- **Machine key TTL** — set when the key is created in + `harmony/src/modules/zitadel/setup.rs::create_machine_key`. + +## Why is a JSON machine key more secure than a PAT? + +Both are "if stolen, full impersonation" — the same blast radius. +The difference is in leak surface: + +- **PAT**: a 60-char bearer string sent on every authenticated + request. Every log line, every env dump, every misrouted + request is a leak opportunity. +- **JSON key**: an RSA private key. Only ever signs short-lived + (60 s) assertions sent to one endpoint + (`/oauth/v2/token`). The bearer token NATS sees is + the access token — short-lived (12 h max), scoped, distinct + from the long-term secret. A full network capture of the + agent ↔ NATS traffic yields only access tokens that expire + within 12 h. + +Plus: Zitadel allows multiple keys per machine user, so rotation +is zero-downtime (mint new → push to device → delete old). PATs +rotate one-at-a-time and are disruptive. + +What this does not defend against: a fully compromised device +where the attacker reads the keyfile. That requires hardware +(TPM / secure element) and is out of scope. + +## The machine keys expire in year 9999. Isn't that effectively forever? + +Yes. Currently set in `ZitadelSetupScore::create_machine_key` as +a known-bad default chosen for demo convenience (re-running tests +shouldn't produce expired keys mid-run). Tracked as a known issue. + +## Why is the IAM-Owner PAT stored as a plain k8s Secret? + +K8s Secrets are base64-encoded, **not** encrypted at rest unless +etcd encryption-at-rest is explicitly enabled with a KMS provider. +Anyone with `get secrets` in the `zitadel` namespace effectively +has Zitadel admin. + +The PAT exists because `ZitadelSetupScore` calls Zitadel's +management API (create project, role, machine user, mint key), +which requires IAM-Owner privileges. A PAT is the simplest +credential that survives across applies. + +This is a known production-hardening gap. Harmony has the +`harmony_secret` crate (ADR-020) with OpenBao and local-encrypted-file +backends; the Score is currently wired against a k8s Secret only. + +## What lifetime is set for the human admin password — why does the ConfigMap show one that doesn't work? + +`ZitadelScore` regenerates a random admin password on every apply +and writes it to the rendered ConfigMap. Helm's `FirstInstance` +block only seeds Postgres on the **first** install against an +empty DB, so re-applies render a new ConfigMap password but leave +the original Postgres hash untouched. The displayed password is +stale on every apply after the first. + +To recover access: use the `iam-admin-pat` to call Zitadel's +management API and reset the human admin's password directly. +Tracked as a known bug. + +## Quick reference — tokens on the wire + +| Token | Lives where | Lifetime | Signed by | Purpose | +| --- | --- | --- | --- | --- | +| **Assertion** | Agent memory, in-flight | 60 s | Agent (RSA key) | "I'm machine user X — give me an access token" | +| **Access token** | Agent memory + on-the-wire to NATS | ~12 h | Zitadel | "Zitadel says I'm device X with role `device`" | +| **NATS user JWT** | NATS server connection state | callout-defined (~30 s) | Auth callout (NKey) | "I have these permissions on these subjects" | + +The agent only holds the RSA key on disk and the access token +in memory. The NATS user JWT is server-internal — agents don't +see it. + +## Code map + +| Topic | File | +| --- | --- | +| Helm install, masterkey, admin password | `harmony/src/modules/zitadel/mod.rs` | +| Project/role/machine user provisioning | `harmony/src/modules/zitadel/setup.rs` | +| Per-device machine user + key handoff | `examples/fleet_e2e_demo/src/lib.rs::provision_device` | +| JWT-bearer mint | `fleet/harmony-fleet-agent/src/credentials.rs::zitadel_mint` | +| Auth callout decision tree | `nats/callout/src/handler.rs::decide` | +| Per-device permission template | `nats/callout/src/permissions.rs::device_default` | +| End-to-end rehearsal runbook | `examples/fleet_e2e_demo/RUNBOOK.md` | +| Manual JWT-bearer mint + NATS write recipe | [`fleet-manual-token-mint.md`](./fleet-manual-token-mint.md) | diff --git a/examples/fleet_auth_callout/Cargo.toml b/examples/fleet_auth_callout/Cargo.toml new file mode 100644 index 00000000..4f1b99db --- /dev/null +++ b/examples/fleet_auth_callout/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "example-fleet-auth-callout" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true +description = "End-to-end fleet IoT security model: Zitadel + NATS + auth callout on k3d" + +[lib] +name = "example_fleet_auth_callout" +path = "src/lib.rs" + +[[bin]] +name = "fleet-auth-callout" +path = "src/main.rs" + +[[test]] +name = "security_model" +path = "tests/security_model.rs" + +[dependencies] +harmony = { path = "../../harmony" } +harmony-k8s = { path = "../../harmony-k8s" } +harmony_types = { path = "../../harmony_types" } +k3d-rs = { path = "../../k3d" } +harmony-nats-callout = { path = "../../nats/callout" } +async-nats.workspace = true +nkeys = "0.4" +jsonwebtoken = "9" +reqwest = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tokio-test.workspace = true +serde.workspace = true +serde_json.workspace = true +anyhow.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +log.workspace = true +env_logger.workspace = true +futures-util.workspace = true +k8s-openapi.workspace = true +kube.workspace = true +base64 = "0.22" +tempfile.workspace = true +url.workspace = true +directories = "6.0.0" diff --git a/examples/fleet_auth_callout/src/lib.rs b/examples/fleet_auth_callout/src/lib.rs new file mode 100644 index 00000000..83036090 --- /dev/null +++ b/examples/fleet_auth_callout/src/lib.rs @@ -0,0 +1,790 @@ +//! End-to-end fleet IoT security model harness. +//! +//! Brings up the full stack on a local k3d cluster: +//! 1. k3d cluster (creates if missing) with HTTP/NATS port mappings. +//! 2. Zitadel + Postgres (via the official Helm chart). +//! 3. Project + roles (`fleet-admin`, `device`) + 4 machine users + +//! JWT keys via ZitadelSetupScore. +//! 4. NATS server with `auth_callout` block referencing the issuer NKey. +//! 5. The harmony-nats-callout binary as a Deployment, sideloaded as a +//! container image into k3d. +//! +//! `main.rs` calls [`bring_up_stack`] then prints credentials and waits. +//! Tests under `tests/` share a single cluster via `OnceCell` and exercise +//! the security model through real `async_nats` clients using JWT-bearer +//! access tokens minted from the machine keys produced in step 3. +//! +//! ## Why this lives in an example, not under `harmony/src/modules/` +//! +//! Everything in this crate is a *composition* of reusable Scores plus +//! test fixtures (the JWT-bearer helper, image-build glue). The Scores +//! themselves are in `harmony/src/modules/{zitadel,nats_auth_callout}`. + +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::{Context, Result}; +use harmony::inventory::Inventory; +use harmony::modules::k8s::coredns::{CoreDNSRewrite, CoreDNSRewriteScore}; +use harmony::modules::nats::NatsHelmChartScore; +use harmony::modules::nats_auth_callout::{NatsAuthCalloutScore, render_auth_callout_block}; +use harmony::modules::zitadel::{ + MachineKeyType, ZitadelApiApp, ZitadelClientConfig, ZitadelMachineUser, ZitadelRole, + ZitadelScore, ZitadelSetupScore, +}; +use harmony::score::Score; +use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology}; +use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader, encode as jwt_encode}; +use k3d_rs::{K3d, PortMapping}; +use log::info; +use nkeys::KeyPair; +use serde::{Deserialize, Serialize}; + +pub const CLUSTER_NAME: &str = "fleet-auth-callout"; +pub const HTTP_PORT: u32 = 8080; +pub const NATS_NODE_PORT: i32 = 30422; +pub const ZITADEL_HOST: &str = "sso.fleet.local"; + +pub const FLEET_NAMESPACE: &str = "fleet-system"; +pub const NATS_NAMESPACE: &str = FLEET_NAMESPACE; +pub const NATS_RELEASE: &str = "fleet-nats"; +pub const CALLOUT_DEPLOYMENT_NAME: &str = "fleet-callout"; +/// `localhost/` prefix matches what podman tags images as internally — +/// `podman build -t foo:tag` produces `localhost/foo:tag`. After +/// `podman save → k3d image import`, the image lands in the k3d node's +/// containerd under that exact name. Without the prefix, K8s would +/// treat `foo:tag` as a Docker Hub reference and ImagePullBackOff. +pub const CALLOUT_IMAGE_TAG: &str = "localhost/harmony-nats-callout:dev"; + +pub const PROJECT_NAME: &str = "fleet"; +pub const API_APP_NAME: &str = "nats"; +pub const ADMIN_ROLE_KEY: &str = "fleet-admin"; +pub const DEVICE_ROLE_KEY: &str = "device"; + +pub const ADMIN_USERNAME: &str = "ops-station"; +pub const DEVICE_A_USERNAME: &str = "sensor-a"; +pub const DEVICE_B_USERNAME: &str = "sensor-b"; +pub const NO_ROLE_USERNAME: &str = "intruder"; + +/// Service-side NATS account user that the callout itself authenticates +/// with (listed in `auth_callout.auth_users` to bypass the callout). +pub const NATS_AUTH_USER: &str = "auth"; +pub const NATS_AUTH_PASS: &str = "auth-callout-pass"; +pub const NATS_ACCOUNT: &str = "DEVICES"; +pub const NATS_SYSTEM_USER: &str = "sys-admin"; +pub const NATS_SYSTEM_PASS: &str = "sys-admin-pass"; + +#[derive(Debug, Clone)] +pub struct StackHandles { + pub cluster_name: String, + pub nats_url_external: String, + pub zitadel_url: String, + pub project_id: String, + pub admin_machine_key: String, + pub device_a_machine_key: String, + pub device_b_machine_key: String, + pub intruder_machine_key: String, + pub issuer_pubkey: String, +} + +/// JSON keyfile content as Zitadel emits it for `KEY_TYPE_JSON` machine keys. +#[derive(Debug, Deserialize, Serialize)] +pub struct MachineKeyFile { + #[serde(rename = "type")] + pub r#type: String, + #[serde(rename = "keyId")] + pub key_id: String, + /// PEM-encoded RSA private key. + pub key: String, + #[serde(rename = "userId")] + pub user_id: String, +} + +fn data_dir() -> PathBuf { + directories::BaseDirs::new() + .map(|dirs| dirs.data_dir().join("harmony").join("k3d")) + .unwrap_or_else(|| PathBuf::from("/tmp/harmony")) +} + +pub fn create_k3d() -> K3d { + let base = data_dir(); + std::fs::create_dir_all(&base).expect("create k3d data dir"); + K3d::new(base, Some(CLUSTER_NAME.to_string())) + // HTTP_PORT:80 so /etc/hosts entries (or curl --resolve) hit ingress. + // NATS_NODE_PORT lets clients off-cluster talk to the NATS service. + .with_port_mappings(vec![ + PortMapping::new(HTTP_PORT, 80), + PortMapping::new(NATS_NODE_PORT as u32, NATS_NODE_PORT as u32), + ]) +} + +pub fn create_topology(k3d: &K3d) -> K8sAnywhereTopology { + let context = k3d + .context_name() + .unwrap_or_else(|| format!("k3d-{CLUSTER_NAME}")); + unsafe { + std::env::set_var("HARMONY_USE_LOCAL_K3D", "false"); + std::env::set_var("HARMONY_AUTOINSTALL", "false"); + std::env::set_var("HARMONY_K8S_CONTEXT", &context); + } + K8sAnywhereTopology::from_env() +} + +/// Build the NATS Helm values that wire `auth_callout` to a callout +/// service running in the same account, plus a NodePort for off-cluster +/// access from tests on the host. +/// +/// **Why the explicit `service.merge.spec.ports` list:** the upstream +/// chart's `service.ports..merge` field is *not* a strategic-merge +/// directive — it gets emitted as-is into the rendered Service (the +/// chart's `_helpers.tpl` does `merge (dict "name" $k) $v` which leaves +/// `merge: …` as a literal field on each port). K8s then rejects the +/// Service with "field not declared in schema". Only the top-level +/// `service.merge` is actually a `mergeOverwrite` patch; we use that +/// path and re-state the full ports list so `nats` gets our nodePort. +pub fn render_nats_values(issuer_pubkey: &str) -> String { + let auth_callout = render_auth_callout_block(issuer_pubkey, NATS_AUTH_USER, NATS_ACCOUNT); + format!( + r#"fullnameOverride: {nats_release} +config: + cluster: + enabled: false + jetstream: + enabled: true + fileStorage: + enabled: true + size: 2Gi + merge: + {auth_callout_indented} + accounts: + {nats_account}: + jetstream: enabled + users: + - user: "{auth_user}" + password: "{auth_pass}" + SYS: + users: + - user: "{sys_user}" + password: "{sys_pass}" + system_account: SYS +service: + merge: + spec: + type: NodePort + ports: + - appProtocol: tcp + name: nats + port: 4222 + targetPort: nats + nodePort: {node_port} + - appProtocol: http + name: monitor + port: 8222 + targetPort: monitor +"#, + nats_release = NATS_RELEASE, + auth_callout_indented = auth_callout + .lines() + .enumerate() + .map(|(i, l)| if i == 0 { + l.to_string() + } else { + format!(" {l}") + }) + .collect::>() + .join("\n"), + nats_account = NATS_ACCOUNT, + auth_user = NATS_AUTH_USER, + auth_pass = NATS_AUTH_PASS, + sys_user = NATS_SYSTEM_USER, + sys_pass = NATS_SYSTEM_PASS, + node_port = NATS_NODE_PORT, + ) +} + +/// Bring the entire stack up on a local k3d cluster. Idempotent — +/// re-running picks up existing resources. +/// +/// Returns handles + credentials. The machine key fields contain raw +/// JSON keyfile content (`MachineKeyFile`) and can be passed straight +/// to [`mint_access_token`] to authenticate as the corresponding user. +pub async fn bring_up_stack() -> Result { + let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) + .try_init(); + + let k3d = create_k3d(); + + info!("[1/8] ensuring k3d cluster '{CLUSTER_NAME}' is up"); + k3d.ensure_installed() + .await + .map_err(|e| anyhow::anyhow!("k3d ensure: {e}"))?; + + let topology = create_topology(&k3d); + topology.ensure_ready().await.context("topology init")?; + + info!("[2/8] deploying Zitadel (this takes several minutes the first time)"); + deploy_zitadel(&topology).await?; + + info!("[3/8] CoreDNS rewrite so in-cluster lookups for {ZITADEL_HOST} resolve"); + CoreDNSRewriteScore { + rewrites: vec![CoreDNSRewrite { + hostname: ZITADEL_HOST.to_string(), + target: "zitadel.zitadel.svc.cluster.local".to_string(), + }], + } + .interpret(&Inventory::autoload(), &topology) + .await + .context("CoreDNS rewrite")?; + + info!("[4/8] waiting for Zitadel HTTP to respond"); + wait_for_zitadel_ready().await?; + + info!("[5/8] provisioning project + roles + machine users in Zitadel"); + let setup = ZitadelSetupScore { + host: ZITADEL_HOST.to_string(), + port: HTTP_PORT as u16, + skip_tls: true, + applications: vec![], + api_apps: vec![ZitadelApiApp { + project_name: PROJECT_NAME.to_string(), + app_name: API_APP_NAME.to_string(), + }], + roles: vec![ + ZitadelRole { + project_name: PROJECT_NAME.to_string(), + key: ADMIN_ROLE_KEY.to_string(), + display_name: "Fleet Admin".to_string(), + group: None, + }, + ZitadelRole { + project_name: PROJECT_NAME.to_string(), + key: DEVICE_ROLE_KEY.to_string(), + display_name: "Device".to_string(), + group: None, + }, + ], + machine_users: vec![ + ZitadelMachineUser { + username: ADMIN_USERNAME.to_string(), + name: "Ops Station".to_string(), + create_pat: false, + machine_key: Some(MachineKeyType::Json), + project_name: Some(PROJECT_NAME.to_string()), + grant_roles: vec![ADMIN_ROLE_KEY.to_string()], + }, + ZitadelMachineUser { + username: DEVICE_A_USERNAME.to_string(), + name: "Sensor A".to_string(), + create_pat: false, + machine_key: Some(MachineKeyType::Json), + project_name: Some(PROJECT_NAME.to_string()), + grant_roles: vec![DEVICE_ROLE_KEY.to_string()], + }, + ZitadelMachineUser { + username: DEVICE_B_USERNAME.to_string(), + name: "Sensor B".to_string(), + create_pat: false, + machine_key: Some(MachineKeyType::Json), + project_name: Some(PROJECT_NAME.to_string()), + grant_roles: vec![DEVICE_ROLE_KEY.to_string()], + }, + ZitadelMachineUser { + username: NO_ROLE_USERNAME.to_string(), + name: "Intruder".to_string(), + create_pat: false, + machine_key: Some(MachineKeyType::Json), + project_name: None, + grant_roles: vec![], + }, + ], + }; + setup + .interpret(&Inventory::autoload(), &topology) + .await + .context("ZitadelSetupScore failed")?; + + let zcfg = ZitadelClientConfig::load() + .context("ZitadelSetupScore did not produce a client config cache")?; + let project_id = zcfg + .project_id_by_name(PROJECT_NAME) + .or(zcfg.project_id.as_ref()) + .context("project_id missing from cache")? + .clone(); + + info!("[6/8] generating callout issuer NKey + deploying NATS with auth_callout"); + // Re-use a deterministic seed across runs by stashing it in a + // K8s secret in the fleet namespace. Fall back to a fresh one + // and persist it. Keeping it stable lets us reuse the cached + // user JWTs Zitadel issued. + let issuer_seed = ensure_issuer_seed(&topology).await?; + let issuer_kp = KeyPair::from_seed(&issuer_seed) + .map_err(|e| anyhow::anyhow!("invalid persisted issuer seed: {e}"))?; + let issuer_pubkey = issuer_kp.public_key(); + + NatsHelmChartScore::new( + NATS_RELEASE.to_string(), + NATS_NAMESPACE.to_string(), + render_nats_values(&issuer_pubkey), + ) + .interpret(&Inventory::autoload(), &topology) + .await + .context("NATS deploy")?; + + info!("[7/8] building + sideloading callout image into k3d"); + build_and_load_callout_image(&k3d).await?; + + info!("[8/8] deploying NatsAuthCalloutScore"); + let mut callout = NatsAuthCalloutScore::new( + CALLOUT_DEPLOYMENT_NAME, + FLEET_NAMESPACE, + format!("nats://{NATS_RELEASE}.{NATS_NAMESPACE}.svc.cluster.local:4222"), + format!("http://{ZITADEL_HOST}:{HTTP_PORT}"), + // Zitadel emits aud = projectId for tokens issued via the + // `urn:zitadel:iam:org:project:id::aud` scope. + project_id.clone(), + NATS_AUTH_USER, + NATS_AUTH_PASS, + issuer_seed.clone(), + ) + .image(CALLOUT_IMAGE_TAG) + .target_account(NATS_ACCOUNT) + .admin_role(ADMIN_ROLE_KEY) + .device_role(DEVICE_ROLE_KEY) + .danger_accept_invalid_certs(true); + // Zitadel doesn't emit a custom `device_id` claim by default — that + // would require a Zitadel Action to map metadata into an extension + // claim. For this example we use `preferred_username`, which is + // populated with the machine user's username (`sensor-a`, + // `ops-station`, …). Production deployments that want a separate + // `device_id` claim should configure a Zitadel Action and override + // the device_id_claim path back to `device_id`. + // Zitadel access tokens for machine users: + // * Don't carry `preferred_username` (that's an OIDC ID-token claim); + // * Do carry `client_id` set to the machine user's userName — perfect + // for our device-id-from-username case. + // + // The project's role claim lives at a *project-scoped* path + // `urn:zitadel:iam:org:project::roles` (NOT the unqualified + // `urn:zitadel:iam:org:project:roles`) because we request the + // `urn:zitadel:iam:org:project:id::aud` scope. The latter + // forces Zitadel to scope role claims to the specific project, which + // is what we want for tenant isolation. + callout.device_id_claim = "client_id".to_string(); + // Zitadel's `client_id` for a machine user equals its userName, so + // a user created as `device-vm-device-00` (matching the + // `device_username()` convention used by both fleet_e2e_demo and + // fleet_rpi_setup) lands in the JWT verbatim. Strip the `device-` + // prefix so the callout interpolates permissions against the bare + // device id (`vm-device-00`) the agent uses for KV keys. + callout.device_id_prefix_strip = "device-".to_string(); + callout.roles_claim = format!("urn:zitadel:iam:org:project:{project_id}:roles"); + callout + .interpret(&Inventory::autoload(), &topology) + .await + .context("callout deploy")?; + + info!("waiting for callout pod to be Ready before handing the stack over"); + wait_for_callout_ready(&topology).await?; + + let admin_machine_key = zcfg + .machine_key(ADMIN_USERNAME) + .context("admin machine key missing from cache")? + .clone(); + let device_a_machine_key = zcfg + .machine_key(DEVICE_A_USERNAME) + .context("device A machine key missing from cache")? + .clone(); + let device_b_machine_key = zcfg + .machine_key(DEVICE_B_USERNAME) + .context("device B machine key missing from cache")? + .clone(); + let intruder_machine_key = zcfg + .machine_key(NO_ROLE_USERNAME) + .context("intruder machine key missing from cache")? + .clone(); + + Ok(StackHandles { + cluster_name: CLUSTER_NAME.to_string(), + nats_url_external: format!("nats://127.0.0.1:{NATS_NODE_PORT}"), + zitadel_url: format!("http://{ZITADEL_HOST}:{HTTP_PORT}"), + project_id, + admin_machine_key, + device_a_machine_key, + device_b_machine_key, + intruder_machine_key, + issuer_pubkey, + }) +} + +pub async fn deploy_zitadel(topology: &K8sAnywhereTopology) -> Result<()> { + let zitadel = ZitadelScore { + host: ZITADEL_HOST.to_string(), + zitadel_version: "v4.12.1".to_string(), + external_secure: false, + // Match the host-side k3d port mapping so Zitadel's emitted + // issuer is `http://sso.fleet.local:8080`. Without this, JWT-bearer + // audience validation fails with `Errors.Internal` (the assertion + // `aud` doesn't match the chart-default issuer at port 80). + external_port: Some(HTTP_PORT), + }; + zitadel + .interpret(&Inventory::autoload(), topology) + .await + .context("ZitadelScore deploy")?; + Ok(()) +} + +pub async fn wait_for_callout_ready(topology: &K8sAnywhereTopology) -> Result<()> { + let _ = topology; + // `kubectl rollout status deployment` is the canonical "is the new + // ReplicaSet's pod up?" check — it handles observed-generation + // tracking, terminating-old-replica edge cases, and pod-readiness in + // one call. Reproducing that in the kube client is doable but error- + // prone; shelling out keeps it short and obviously-correct. + let status = tokio::process::Command::new("kubectl") + .args([ + "--context", + "k3d-fleet-auth-callout", + "rollout", + "status", + "-n", + FLEET_NAMESPACE, + &format!("deployment/{CALLOUT_DEPLOYMENT_NAME}"), + "--timeout=60s", + ]) + .status() + .await + .context("invoke kubectl rollout status")?; + if !status.success() { + anyhow::bail!("kubectl rollout status timed out / failed"); + } + Ok(()) +} + +pub async fn wait_for_zitadel_ready() -> Result<()> { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build()?; + for attempt in 1..=120 { + match client + .get(format!( + "http://127.0.0.1:{HTTP_PORT}/.well-known/openid-configuration" + )) + // Include the port in Host so Zitadel emits a matching issuer URL + // — see `mint_access_token` for the underlying mechanism. + .header("Host", format!("{ZITADEL_HOST}:{HTTP_PORT}")) + .send() + .await + { + Ok(r) if r.status().is_success() => return Ok(()), + Ok(r) if attempt % 15 == 0 => { + info!("Zitadel HTTP {} (attempt {attempt}/120)", r.status()) + } + Err(e) if attempt % 15 == 0 => { + info!("Zitadel unreachable: {e} (attempt {attempt}/120)") + } + _ => {} + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + anyhow::bail!("timed out waiting for Zitadel") +} + +/// Persist the callout's issuer NKey seed in a K8s secret so re-runs of +/// the example don't invalidate previously issued user JWTs in NATS. +pub async fn ensure_issuer_seed(topology: &K8sAnywhereTopology) -> Result { + use k8s_openapi::ByteString; + use k8s_openapi::api::core::v1::{Namespace, Secret}; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + use std::collections::BTreeMap; + + let k8s = topology + .k8s_client() + .await + .map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?; + + // Ensure namespace exists first — secret creation requires it. + if k8s + .get_resource::(FLEET_NAMESPACE, None) + .await? + .is_none() + { + let ns = Namespace { + metadata: ObjectMeta { + name: Some(FLEET_NAMESPACE.to_string()), + ..Default::default() + }, + ..Default::default() + }; + k8s.create(&ns, None).await.ok(); + } + + let secret_name = "callout-issuer-seed"; + + if let Some(existing) = k8s + .get_resource::(secret_name, Some(FLEET_NAMESPACE)) + .await? + && let Some(data) = existing.data + && let Some(seed_bytes) = data.get("seed") + { + let seed = String::from_utf8(seed_bytes.0.clone())?; + return Ok(seed.trim().to_string()); + } + + let seed = KeyPair::new_account() + .seed() + .map_err(|e| anyhow::anyhow!("nkey seed: {e}"))?; + let mut data = BTreeMap::new(); + data.insert("seed".to_string(), ByteString(seed.as_bytes().to_vec())); + let secret = Secret { + metadata: ObjectMeta { + name: Some(secret_name.to_string()), + namespace: Some(FLEET_NAMESPACE.to_string()), + ..Default::default() + }, + data: Some(data), + type_: Some("Opaque".to_string()), + ..Default::default() + }; + k8s.create(&secret, Some(FLEET_NAMESPACE)).await.ok(); + Ok(seed) +} + +/// Build the callout binary, package the container image, and import it +/// into the running k3d cluster. Mirrors `fleet/scripts/load-test.sh`'s +/// staging-context pattern (the workspace `.dockerignore` excludes +/// `target/`). +pub async fn build_and_load_callout_image(k3d: &K3d) -> Result<()> { + let workspace_root = std::env::var("CARGO_MANIFEST_DIR") + .map(|d| PathBuf::from(d).join("..").join("..")) + .unwrap_or_else(|_| PathBuf::from(".")); + let workspace_root = workspace_root.canonicalize().unwrap_or(workspace_root); + + info!("cargo build --release -p harmony-nats-callout"); + let status = tokio::process::Command::new("cargo") + .args(["build", "--release", "-p", "harmony-nats-callout"]) + .current_dir(&workspace_root) + .status() + .await?; + if !status.success() { + anyhow::bail!("cargo build failed"); + } + + let ctx = tempfile::tempdir()?; + let bin_dst = ctx.path().join("target/release"); + std::fs::create_dir_all(&bin_dst)?; + std::fs::copy( + workspace_root.join("target/release/harmony-nats-callout"), + bin_dst.join("harmony-nats-callout"), + )?; + std::fs::copy( + workspace_root.join("nats/callout/Dockerfile"), + ctx.path().join("Dockerfile"), + )?; + + info!("podman build → {CALLOUT_IMAGE_TAG}"); + let status = tokio::process::Command::new("podman") + .args(["build", "-q", "-t", CALLOUT_IMAGE_TAG, "."]) + .current_dir(ctx.path()) + .status() + .await?; + if !status.success() { + anyhow::bail!("podman build failed"); + } + + info!("k3d image import {CALLOUT_IMAGE_TAG}"); + let cluster = k3d.cluster_name().unwrap_or(CLUSTER_NAME).to_string(); + // Deterministic .tar path with a per-process suffix so concurrent + // test crates don't trample each other. + let tar_path = + std::env::temp_dir().join(format!("harmony-callout-image-{}.tar", std::process::id())); + // `podman save` (docker-archive format) refuses to overwrite an + // existing archive — wipe any leftover from a prior failed run. + let _ = std::fs::remove_file(&tar_path); + let status = tokio::process::Command::new("podman") + .args(["save", "-o", tar_path.to_str().unwrap(), CALLOUT_IMAGE_TAG]) + .status() + .await?; + if !status.success() { + anyhow::bail!("podman save failed"); + } + // The k3d binary lives in `~/.local/share/harmony/k3d/k3d` — it's + // managed by k3d-rs, not on the system PATH (the user's interactive + // shell typically has it as an alias, but child processes don't + // inherit aliases). Run it via k3d-rs's accessor. + let tar_path_str = tar_path.to_str().unwrap().to_string(); + let cluster_for_blocking = cluster.clone(); + let tar_path_clone = tar_path.clone(); + let result = tokio::task::spawn_blocking(move || { + k3d_rs::K3d::new(data_dir(), Some(cluster_for_blocking.clone())).run_k3d_command([ + "image", + "import", + tar_path_str.as_str(), + "-c", + cluster_for_blocking.as_str(), + ]) + }) + .await + .context("spawn_blocking k3d image import")?; + let _ = std::fs::remove_file(&tar_path_clone); + let output = result.map_err(|e| anyhow::anyhow!("k3d image import failed: {e}"))?; + if !output.status.success() { + anyhow::bail!( + "k3d image import returned {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(()) +} + +/// RFC 7523 JWT-bearer client for Zitadel. +/// +/// `issuer_url` should be the externally-visible Zitadel URL +/// (e.g. `http://sso.fleet.local:8080`) — it's used as the JWT +/// assertion's `aud` claim. The actual HTTP transport hits +/// `127.0.0.1:HTTP_PORT` and forwards the hostname via the `Host` +/// header, which is how the k3d ingress routes without requiring a +/// host-side `/etc/hosts` entry. +/// +/// `machine_key_json` is the raw keyfile content Zitadel emits +/// (decoded from `keyDetails`). `scopes` are appended to the standard +/// set; pass `[format!("urn:zitadel:iam:org:project:id:{project_id}:aud")]` +/// to make the resulting access token's `aud` include the project ID. +pub async fn mint_access_token( + issuer_url: &str, + machine_key_json: &str, + scopes: &[String], +) -> Result { + let key: MachineKeyFile = + serde_json::from_str(machine_key_json).context("machine key JSON parse")?; + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_secs() as i64; + + let claims = serde_json::json!({ + "iss": key.user_id, + "sub": key.user_id, + "aud": issuer_url, + "exp": now + 60, + "iat": now, + }); + + let mut header = JwtHeader::new(Algorithm::RS256); + header.kid = Some(key.key_id.clone()); + let assertion = jwt_encode( + &header, + &claims, + &EncodingKey::from_rsa_pem(key.key.as_bytes()) + .context("parse RSA private key from machine key file")?, + )?; + + let scope = { + let mut s = vec![ + "openid".to_string(), + "profile".to_string(), + "urn:zitadel:iam:org:projects:roles".to_string(), + ]; + s.extend(scopes.iter().cloned()); + s.join(" ") + }; + + let client = reqwest::Client::builder() + .danger_accept_invalid_certs(true) + .timeout(Duration::from_secs(10)) + .build()?; + // The Zitadel chart's ingress routes by Host header. Hitting + // 127.0.0.1:HTTP_PORT bypasses the need for an /etc/hosts entry + // on the host running the tests (k3d's loadbalancer maps the + // port; the ingress controller dispatches by Host header). + // + // The Host MUST include the port: Zitadel derives the OIDC issuer + // string from the request's Host header. With `Host: sso.fleet.local` + // it emits `iss: http://sso.fleet.local`; with `Host: sso.fleet.local:8080` + // it emits `iss: http://sso.fleet.local:8080`. Our JWT assertion's `aud` + // must match Zitadel's issuer exactly, so we always send the port. + let host = url::Url::parse(issuer_url) + .ok() + .and_then(|u| { + let h = u.host_str()?; + let p = u.port_or_known_default(); + Some(match p { + Some(p) => format!("{h}:{p}"), + None => h.to_string(), + }) + }) + .unwrap_or_else(|| format!("{ZITADEL_HOST}:{HTTP_PORT}")); + let token_url = format!("http://127.0.0.1:{HTTP_PORT}/oauth/v2/token"); + + let resp = client + .post(&token_url) + .header("Host", host) + .form(&[ + ( + "grant_type", + "urn:ietf:params:oauth:grant-type:jwt-bearer".to_string(), + ), + ("assertion", assertion), + ("scope", scope), + ]) + .send() + .await + .context("POST /oauth/v2/token")?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("token endpoint returned {status}: {body}"); + } + + #[derive(Deserialize)] + struct TokenResponse { + access_token: String, + } + let tr: TokenResponse = resp.json().await.context("parse token response")?; + if std::env::var("FLEET_AUTH_CALLOUT_DEBUG_TOKENS").is_ok() + && let Some(payload_b64) = tr.access_token.split('.').nth(1) + { + use base64::Engine; + let pad = "=".repeat((4 - payload_b64.len() % 4) % 4); + if let Ok(bytes) = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(format!("{payload_b64}{pad}").trim_end_matches('=')) + && let Ok(claims) = serde_json::from_slice::(&bytes) + { + log::info!( + "[debug] access token claims: {}", + serde_json::to_string_pretty(&claims).unwrap_or_default() + ); + } + } + Ok(tr.access_token) +} + +/// Build the standard scope list for our project: standard claims + a +/// project-id audience scope so the access token's `aud` matches what the +/// callout's `oidc_audience` expects. +pub fn scopes_for_project(project_id: &str) -> Vec { + vec![format!("urn:zitadel:iam:org:project:id:{project_id}:aud")] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn render_nats_values_inlines_auth_callout_block() { + let yaml = render_nats_values("ABCDEF"); + assert!(yaml.contains("issuer: ABCDEF")); + assert!(yaml.contains("auth_users: [ auth ]")); + assert!(yaml.contains("account: DEVICES")); + assert!(yaml.contains("system_account: SYS")); + assert!(yaml.contains("nodePort: 30422")); + } + + #[test] + fn scopes_for_project_emits_audience_scope() { + let s = scopes_for_project("12345"); + assert_eq!(s, vec!["urn:zitadel:iam:org:project:id:12345:aud"]); + } +} diff --git a/examples/fleet_auth_callout/src/main.rs b/examples/fleet_auth_callout/src/main.rs new file mode 100644 index 00000000..dbeebc56 --- /dev/null +++ b/examples/fleet_auth_callout/src/main.rs @@ -0,0 +1,55 @@ +//! `cargo run -p example-fleet-auth-callout` brings the full Zitadel + +//! NATS + auth callout stack up on a local k3d cluster, prints the URLs +//! and credentials, and waits for Ctrl-C. +//! +//! Tests under `tests/` exercise the security model. They do NOT run +//! unless explicitly requested with `cargo test -p example-fleet-auth-callout` +//! since they bring up the same heavy stack. + +use anyhow::Result; +use example_fleet_auth_callout::{ + ADMIN_USERNAME, DEVICE_A_USERNAME, DEVICE_B_USERNAME, NO_ROLE_USERNAME, bring_up_stack, +}; + +#[tokio::main] +async fn main() -> Result<()> { + let handles = bring_up_stack().await?; + + println!("\n========================================================="); + println!(" Fleet Auth Callout — STACK READY"); + println!("========================================================="); + println!(" k3d cluster: {}", handles.cluster_name); + println!(" Zitadel: {}", handles.zitadel_url); + println!( + " admin login: admin / (see Zitadel ConfigMap 'zitadel-config-yaml' for password)" + ); + println!(" NATS (external): {}", handles.nats_url_external); + println!(" account: DEVICES"); + println!(" Project ID: {}", handles.project_id); + println!(" Issuer pubkey: {}", handles.issuer_pubkey); + println!(); + println!(" Machine keys provisioned (admin / sensor-a / sensor-b / intruder):"); + for (name, key_json) in [ + (ADMIN_USERNAME, &handles.admin_machine_key), + (DEVICE_A_USERNAME, &handles.device_a_machine_key), + (DEVICE_B_USERNAME, &handles.device_b_machine_key), + (NO_ROLE_USERNAME, &handles.intruder_machine_key), + ] { + // Print only the keyId so the output is tidy; the full keyfile is + // cached at ~/.local/share/harmony/zitadel/client-config.json + let key_id = serde_json::from_str::(key_json) + .ok() + .and_then(|v| { + v.get("keyId") + .and_then(|k| k.as_str().map(|s| s.to_string())) + }) + .unwrap_or_else(|| "".to_string()); + println!(" {name:14} keyId={key_id}"); + } + println!(); + println!(" Stack is running. Press Ctrl-C to exit (cluster keeps running)."); + println!("========================================================="); + + tokio::signal::ctrl_c().await?; + Ok(()) +} diff --git a/examples/fleet_auth_callout/tests/security_model.rs b/examples/fleet_auth_callout/tests/security_model.rs new file mode 100644 index 00000000..9b1d05c8 --- /dev/null +++ b/examples/fleet_auth_callout/tests/security_model.rs @@ -0,0 +1,131 @@ +//! Real cargo tests proving the IoT fleet security model. +//! +//! All tests share a single bringup of the stack via [`OnceCell`]. The +//! cluster keeps running across the suite, with each test using the +//! cached machine keys to mint Zitadel JWTs and exercise NATS through +//! the auth callout. Three invariants: +//! +//! 1. `admin_can_read_any_device_subject` — fleet-admin sees other devices' state. +//! 2. `device_can_only_access_own_subjects` — sensor-a is denied access to sensor-b's commands. +//! 3. `unknown_role_is_rejected` — a Zitadel-authenticated user with no +//! fleet role cannot connect to NATS. +//! +//! ## Why these tests are real-stack +//! +//! Mocking the OIDC issuer or NATS would only re-prove the unit tests +//! already cover. The point of this suite is to confirm — in CI, in +//! cargo — that the **deployed** stack on k3d enforces the security +//! model end-to-end. Hidden cluster-level misconfiguration (an unset +//! `auth_callout` block, a wrong issuer pubkey, a CoreDNS rewrite drift, +//! a permissions YAML typo) only shows up here. + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use async_nats::ConnectOptions; +use example_fleet_auth_callout::{ + StackHandles, bring_up_stack, mint_access_token, scopes_for_project, +}; +use futures_util::StreamExt; +use tokio::sync::OnceCell; + +static STACK: OnceCell> = OnceCell::const_new(); + +async fn shared_stack() -> Result> { + let cell = STACK + .get_or_try_init(|| async { + let handles = bring_up_stack().await?; + anyhow::Ok(Arc::new(handles)) + }) + .await?; + Ok(cell.clone()) +} + +async fn connect_with_role(stack: &StackHandles, key_json: &str) -> Result { + let token = mint_access_token( + &stack.zitadel_url, + key_json, + &scopes_for_project(&stack.project_id), + ) + .await + .context("mint Zitadel access token")?; + + ConnectOptions::with_token(token) + .connection_timeout(Duration::from_secs(5)) + .connect(&stack.nats_url_external) + .await + .map_err(|e| anyhow::anyhow!("NATS connect: {e}")) +} + +#[tokio::test] +async fn admin_can_read_any_device_subject() -> Result<()> { + let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); + let stack = shared_stack().await?; + + let admin = connect_with_role(&stack, &stack.admin_machine_key).await?; + let device = connect_with_role(&stack, &stack.device_a_machine_key).await?; + + let mut admin_sub = admin.subscribe("device-state.>").await?; + admin.flush().await?; + + device + .publish("device-state.sensor-a", "telemetry-payload".into()) + .await?; + device.flush().await?; + + let msg = tokio::time::timeout(Duration::from_secs(5), admin_sub.next()) + .await + .context("admin sub timeout")? + .context("admin sub closed")?; + assert_eq!(msg.payload.as_ref(), b"telemetry-payload"); + + Ok(()) +} + +#[tokio::test] +async fn device_can_only_access_own_subjects() -> Result<()> { + let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); + let stack = shared_stack().await?; + + let device_a = connect_with_role(&stack, &stack.device_a_machine_key).await?; + let device_b = connect_with_role(&stack, &stack.device_b_machine_key).await?; + + let _b_sub = device_b.subscribe("device-commands.sensor-b").await?; + let mut a_wrong = device_a.subscribe("device-commands.sensor-b").await?; + device_a.flush().await?; + device_b.flush().await?; + + // We only care that A's subscription does NOT receive B's traffic; + // pushing through B-side traffic would be a no-op since A's + // subscription was rejected by NATS at SUB time. + device_b + .publish("device-commands.sensor-b", "should-not-leak".into()) + .await?; + device_b.flush().await?; + + let result = tokio::time::timeout(Duration::from_millis(750), a_wrong.next()).await; + assert!( + result.is_err(), + "device A must not observe device B's commands" + ); + + Ok(()) +} + +#[tokio::test] +async fn unknown_role_is_rejected() -> Result<()> { + let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); + let stack = shared_stack().await?; + + // The intruder has a valid Zitadel JWT but no fleet-admin/device role + // grant. The callout must reject the connection — NATS surfaces that + // as `authorization violation` at connect time. + let result = connect_with_role(&stack, &stack.intruder_machine_key).await; + assert!( + result.is_err(), + "JWT without fleet role must not be admitted to NATS" + ); + + Ok(()) +} diff --git a/examples/fleet_e2e_demo/Cargo.toml b/examples/fleet_e2e_demo/Cargo.toml new file mode 100644 index 00000000..9de767c0 --- /dev/null +++ b/examples/fleet_e2e_demo/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "example-fleet-e2e-demo" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true +description = "VM-based end-to-end rehearsal: k3d + Zitadel + NATS auth callout + libvirt VM agents + operator → CR → podman → status" + +[lib] +name = "example_fleet_e2e_demo" +path = "src/lib.rs" + +[[bin]] +name = "fleet-e2e-demo" +path = "src/main.rs" + +[[test]] +name = "e2e_walking_skeleton" +path = "tests/e2e_walking_skeleton.rs" + +[dependencies] +harmony = { path = "../../harmony", features = ["kvm"] } +harmony-k8s = { path = "../../harmony-k8s" } +harmony_types = { path = "../../harmony_types" } +example-fleet-auth-callout = { path = "../fleet_auth_callout" } +harmony-nats-callout = { path = "../../nats/callout" } +harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } +harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" } +k3d-rs = { path = "../../k3d" } +async-nats.workspace = true +nkeys = "0.4" +tokio = { workspace = true, features = ["full"] } +tokio-test.workspace = true +serde.workspace = true +serde_json.workspace = true +anyhow.workspace = true +log.workspace = true +env_logger.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +futures-util.workspace = true +k8s-openapi.workspace = true +kube.workspace = true +clap = { version = "4", features = ["derive", "env"] } +directories = "6.0.0" +tempfile = "3" +url.workspace = true diff --git a/examples/fleet_e2e_demo/src/lib.rs b/examples/fleet_e2e_demo/src/lib.rs new file mode 100644 index 00000000..433e701b --- /dev/null +++ b/examples/fleet_e2e_demo/src/lib.rs @@ -0,0 +1,815 @@ +//! VM-based end-to-end rehearsal of the customer demo flow. +//! +//! Goal: prove the JWT-auth chain works on a real-system agent +//! before pointing the demo at OKD. See +//! `ROADMAP/fleet_platform/v0_demo_e2e.md` for the full plan. +//! +//! Bring-up sequence: +//! 1. k3d cluster with HTTP + NATS port mappings (re-uses +//! fleet_auth_callout's k3d helpers — same cluster name so +//! re-runs of either example reuse the same cluster). +//! 2. Zitadel + Postgres via ZitadelScore. +//! 3. Wait for Zitadel HTTP and the chart-provisioned `iam-admin-pat` +//! secret (the chart's setup job is async). +//! 4. ZitadelSetupScore for the project + API app + roles + admin +//! machine user (no per-device users yet). +//! 5. NATS with auth_callout block + the callout pod. +//! 6. For each device i: +//! - ZitadelSetupScore minting a per-device machine user with +//! the `device` role grant. The JSON keyfile is cached in +//! `ZitadelClientConfig` and read back here for the agent. +//! - libvirt VM via `ProvisionVmScore`. +//! - SSH-inject `/etc/hosts` so the VM resolves +//! `sso.fleet.local` to the libvirt host. +//! - `FleetDeviceSetupScore` with `FleetDeviceAuth::ZitadelJwt` +//! pointing at the dropped keyfile. +//! +//! Tests in `tests/e2e_walking_skeleton.rs` share a single bring-up +//! via `OnceCell` and exercise: heartbeats, label-selector targeting, +//! status reflect-back, env+volume propagation, admin cross-device +//! read, per-device isolation, NATS-pod-restart reconnect. + +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::{Context, Result}; +use example_fleet_auth_callout::{ + ADMIN_ROLE_KEY, API_APP_NAME, CALLOUT_DEPLOYMENT_NAME, CALLOUT_IMAGE_TAG, DEVICE_ROLE_KEY, + FLEET_NAMESPACE, HTTP_PORT, NATS_ACCOUNT, NATS_AUTH_PASS, NATS_AUTH_USER, NATS_NAMESPACE, + NATS_NODE_PORT, NATS_RELEASE, PROJECT_NAME, ZITADEL_HOST, build_and_load_callout_image, + create_k3d, create_topology, deploy_zitadel, ensure_issuer_seed, render_nats_values, + wait_for_callout_ready, wait_for_zitadel_ready, +}; +use harmony::inventory::Inventory; +use harmony::modules::fleet::{ + FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HostsEntry, + ensure_fleet_ssh_keypair, +}; +use harmony::modules::k8s::coredns::{CoreDNSRewrite, CoreDNSRewriteScore}; +use harmony::modules::linux::{LinuxHostTopology, SshCredentials, ensure_ansible_venv}; +use harmony::modules::nats::NatsHelmChartScore; +use harmony::modules::nats_auth_callout::NatsAuthCalloutScore; +use harmony::modules::zitadel::{ + MachineKeyType, ZitadelApiApp, ZitadelClientConfig, ZitadelMachineUser, ZitadelRole, + ZitadelSetupScore, +}; +use harmony::score::Score; +use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology}; +use harmony_types::id::Id; +use log::{info, warn}; +use nkeys::KeyPair; + +// ---- constants ------------------------------------------------------------- + +/// Libvirt's default NAT gateway. The host's IP from inside any VM +/// attached to the `default` libvirt network. We bake this in because +/// every smoke-a* harness assumes it; if a customer runs their own +/// libvirt with a different bridge they can override via env. +pub const DEFAULT_LIBVIRT_HOST_IP: &str = "192.168.122.1"; + +pub const ADMIN_USERNAME: &str = "fleet-ops"; +/// Separate machine user for the in-cluster operator. Distinct from +/// `fleet-ops` (manual admin tooling) so the audit trail can tell +/// operator-driven actions apart from human operator actions. Same +/// `fleet-admin` role grant — only the identity differs. +pub const OPERATOR_USERNAME: &str = "fleet-operator"; +pub const OPERATOR_IMAGE_TAG: &str = "localhost/harmony-fleet-operator:dev"; + +/// Per-device username convention: `device-${device_id}`. Matches what +/// `fleet_rpi_setup` produces, so callout's `device_id_claim = +/// "client_id"` extracts the device id verbatim from the `client_id` +/// claim Zitadel emits in machine-user access tokens. +pub fn device_username(device_id: &str) -> String { + format!("device-{device_id}") +} + +// ---- options + handles ----------------------------------------------------- + +#[derive(Debug, Clone)] +pub struct E2eDemoOpts { + /// Number of VM-as-device agents to provision. + pub num_devices: usize, + /// Path to the cross-compiled `fleet-agent` binary uploaded to + /// each VM. Defaults to `target/release/fleet-agent` (the same + /// path that smoke-a4 produces). + pub agent_binary: PathBuf, + /// Override for the libvirt host IP (the address VMs see as the + /// gateway). Defaults to [`DEFAULT_LIBVIRT_HOST_IP`]. + pub libvirt_host_ip: String, +} + +impl Default for E2eDemoOpts { + fn default() -> Self { + Self { + num_devices: 2, + agent_binary: workspace_target_path("release/harmony-fleet-agent"), + libvirt_host_ip: DEFAULT_LIBVIRT_HOST_IP.to_string(), + } + } +} + +#[derive(Debug, Clone)] +pub struct DeviceHandle { + pub index: usize, + pub device_id: String, + pub vm_ip: String, + pub labels: std::collections::BTreeMap, +} + +#[derive(Debug, Clone)] +pub struct E2eHandles { + pub cluster_name: String, + pub nats_url_external: String, + pub zitadel_url: String, + pub project_id: String, + pub issuer_pubkey: String, + pub admin_machine_key: String, + pub devices: Vec, +} + +// ---- bring up -------------------------------------------------------------- + +pub async fn bring_up_full_stack(opts: E2eDemoOpts) -> Result { + let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) + .try_init(); + + info!("[e2e-demo 1/9] ensuring k3d cluster"); + let k3d = create_k3d(); + k3d.ensure_installed() + .await + .map_err(|e| anyhow::anyhow!("k3d ensure: {e}"))?; + let topology = create_topology(&k3d); + topology.ensure_ready().await.context("topology init")?; + + info!("[e2e-demo 2/9] deploying Zitadel (cold start: ~5 min)"); + deploy_zitadel(&topology).await?; + + info!("[e2e-demo 3/9] CoreDNS rewrite + waiting for Zitadel HTTP + iam-admin-pat secret"); + CoreDNSRewriteScore { + rewrites: vec![CoreDNSRewrite { + hostname: ZITADEL_HOST.to_string(), + target: "zitadel.zitadel.svc.cluster.local".to_string(), + }], + } + .interpret(&Inventory::autoload(), &topology) + .await + .context("CoreDNSRewriteScore")?; + wait_for_zitadel_ready().await?; + wait_for_iam_admin_pat_secret(&topology).await?; + + info!("[e2e-demo 4/9] provisioning project, API app, roles, admin machine user"); + let admin_setup = ZitadelSetupScore { + host: ZITADEL_HOST.to_string(), + port: HTTP_PORT as u16, + skip_tls: true, + applications: vec![], + api_apps: vec![ZitadelApiApp { + project_name: PROJECT_NAME.to_string(), + app_name: API_APP_NAME.to_string(), + }], + roles: vec![ + ZitadelRole { + project_name: PROJECT_NAME.to_string(), + key: ADMIN_ROLE_KEY.to_string(), + display_name: "Fleet Admin".to_string(), + group: None, + }, + ZitadelRole { + project_name: PROJECT_NAME.to_string(), + key: DEVICE_ROLE_KEY.to_string(), + display_name: "Device".to_string(), + group: None, + }, + ], + machine_users: vec![ + ZitadelMachineUser { + username: ADMIN_USERNAME.to_string(), + name: "Fleet Operations".to_string(), + create_pat: false, + machine_key: Some(MachineKeyType::Json), + project_name: Some(PROJECT_NAME.to_string()), + grant_roles: vec![ADMIN_ROLE_KEY.to_string()], + }, + // Separate machine user for the in-cluster operator pod. + // Same `fleet-admin` role grant as the manual admin + // identity, but distinct username so JWT `client_id` lets + // log analysis tell operator-driven actions apart from + // human operator actions. + ZitadelMachineUser { + username: OPERATOR_USERNAME.to_string(), + name: "Fleet Operator (in-cluster)".to_string(), + create_pat: false, + machine_key: Some(MachineKeyType::Json), + project_name: Some(PROJECT_NAME.to_string()), + grant_roles: vec![ADMIN_ROLE_KEY.to_string()], + }, + ], + }; + admin_setup + .interpret(&Inventory::autoload(), &topology) + .await + .context("admin ZitadelSetupScore")?; + + let zcfg = ZitadelClientConfig::load() + .context("ZitadelSetupScore did not produce a client config cache")?; + let project_id = zcfg + .project_id_by_name(PROJECT_NAME) + .or(zcfg.project_id.as_ref()) + .context("project_id missing from cache")? + .clone(); + let admin_machine_key = zcfg + .machine_key(ADMIN_USERNAME) + .context("admin machine key missing from cache")? + .clone(); + + info!("[e2e-demo 5/9] generating issuer NKey, deploying NATS with auth_callout"); + let issuer_seed = ensure_issuer_seed(&topology).await?; + let issuer_kp = KeyPair::from_seed(&issuer_seed) + .map_err(|e| anyhow::anyhow!("invalid persisted issuer seed: {e}"))?; + let issuer_pubkey = issuer_kp.public_key(); + + NatsHelmChartScore::new( + NATS_RELEASE.to_string(), + NATS_NAMESPACE.to_string(), + render_nats_values(&issuer_pubkey), + ) + .interpret(&Inventory::autoload(), &topology) + .await + .context("NATS deploy")?; + + info!("[e2e-demo 6/9] building + sideloading callout image into k3d"); + build_and_load_callout_image(&k3d).await?; + + info!("[e2e-demo 7/9] deploying NatsAuthCalloutScore"); + let mut callout = NatsAuthCalloutScore::new( + CALLOUT_DEPLOYMENT_NAME, + FLEET_NAMESPACE, + format!("nats://{NATS_RELEASE}.{NATS_NAMESPACE}.svc.cluster.local:4222"), + format!("http://{ZITADEL_HOST}:{HTTP_PORT}"), + project_id.clone(), + NATS_AUTH_USER, + NATS_AUTH_PASS, + issuer_seed.clone(), + ) + .image(CALLOUT_IMAGE_TAG) + .target_account(NATS_ACCOUNT) + .admin_role(ADMIN_ROLE_KEY) + .device_role(DEVICE_ROLE_KEY) + .danger_accept_invalid_certs(true); + // Same convention as fleet_auth_callout: the username is in the + // access token's `client_id` claim. The role claim path is + // project-scoped because the JWT-bearer flow requests project + // audience scope. + callout.device_id_claim = "client_id".to_string(); + // Zitadel's `client_id` for a machine user equals its userName, so a + // user created as `device-vm-device-00` (the convention shared with + // fleet_rpi_setup and fleet_auth_callout) lands in the JWT verbatim. + // Strip the `device-` prefix so the callout interpolates permissions + // against the bare device id (`vm-device-00`) the agent uses for KV + // keys + direct subjects. + callout.device_id_prefix_strip = "device-".to_string(); + callout.roles_claim = format!("urn:zitadel:iam:org:project:{project_id}:roles"); + callout + .interpret(&Inventory::autoload(), &topology) + .await + .context("callout deploy")?; + wait_for_callout_ready(&topology).await?; + + info!("[e2e-demo 8/10] building + sideloading operator image into k3d"); + build_and_load_operator_image(&k3d).await?; + + info!("[e2e-demo 9/10] deploying fleet operator with Zitadel JWT auth"); + let operator_machine_key = zcfg + .machine_key(OPERATOR_USERNAME) + .with_context(|| format!("machine key for {OPERATOR_USERNAME} missing from cache"))? + .clone(); + deploy_operator(&topology, &project_id, &operator_machine_key).await?; + wait_for_operator_ready(&topology).await?; + + info!( + "[e2e-demo 10/10] provisioning {} VM(s) and onboarding agent(s)", + opts.num_devices + ); + let mut devices = Vec::with_capacity(opts.num_devices); + for i in 0..opts.num_devices { + let handle = provision_device(i, &opts, &topology, &project_id).await?; + devices.push(handle); + } + + info!( + "full stack ready: {} device(s), operator + admin role configured", + devices.len() + ); + + Ok(E2eHandles { + cluster_name: example_fleet_auth_callout::CLUSTER_NAME.to_string(), + nats_url_external: format!("nats://127.0.0.1:{NATS_NODE_PORT}"), + zitadel_url: format!("http://{ZITADEL_HOST}:{HTTP_PORT}"), + project_id, + issuer_pubkey, + admin_machine_key, + devices, + }) +} + +// ---- per-device provisioning ---------------------------------------------- + +async fn provision_device( + index: usize, + opts: &E2eDemoOpts, + topology: &K8sAnywhereTopology, + project_id: &str, +) -> Result { + let device_id = format!("vm-device-{index:02}"); + let username = device_username(&device_id); + info!("[device {index}] minting Zitadel machine user {username}"); + + // Per-device ZitadelSetupScore (search-then-create — running this + // for an existing user is a NOOP that just refreshes the cache + // entry pointing at the persisted machine key). The keyfile is + // re-minted because Zitadel doesn't expose the private half of + // an existing key — accept that any prior key drifts to "stale + // until expiry" on the previous device installation. + let device_setup = ZitadelSetupScore { + host: ZITADEL_HOST.to_string(), + port: HTTP_PORT as u16, + skip_tls: true, + applications: vec![], + api_apps: vec![], + roles: vec![], + machine_users: vec![ZitadelMachineUser { + username: username.clone(), + name: format!("Fleet Device {device_id}"), + create_pat: false, + machine_key: Some(MachineKeyType::Json), + project_name: Some(PROJECT_NAME.to_string()), + grant_roles: vec![DEVICE_ROLE_KEY.to_string()], + }], + }; + device_setup + .interpret(&Inventory::autoload(), topology) + .await + .with_context(|| format!("ZitadelSetupScore for {username}"))?; + + let zcfg = ZitadelClientConfig::load() + .context("ZitadelClientConfig disappeared between admin and device setup")?; + let machine_key_json = zcfg + .machine_key(&username) + .with_context(|| format!("machine key for {username} missing from cache"))? + .clone(); + + // -- VM provisioning would go here. Deferred to keep the harness + // cold-start observable in pieces — the kvm bits (ProvisionVmScore) + // require root + libvirtd + the cloud image. Today the harness + // expects the operator to have provisioned VMs out-of-band (e.g. + // via fleet_vm_setup, or a pre-existing libvirt domain). We read + // the IP from a convention path (see `discover_vm_ip`) so the + // test driver can iterate on the agent path without re-paying VM + // boot every test cycle. + // + // Follow-up: fold ProvisionVmScore::ensure_vm here once the + // bring-up has been demonstrated end-to-end at least once. + let vm_ip = discover_vm_ip(index) + .with_context(|| format!("could not resolve IP for device {index}"))?; + + info!("[device {index}] {device_id} at {vm_ip} — installing agent with Zitadel JWT auth"); + let labels = build_device_labels(&device_id, index); + let agent_score = FleetDeviceSetupScore::new(FleetDeviceSetupConfig { + device_id: Id::from(device_id.clone()), + labels: labels.clone(), + // Agent connects to NATS at the libvirt host's IP via the + // NodePort. The libvirt default network NATs the VM through + // the host so the host's port mapping is reachable. + nats_urls: vec![format!("nats://{}:{NATS_NODE_PORT}", opts.libvirt_host_ip)], + auth: FleetDeviceAuth::ZitadelJwt { + machine_key_json, + // Issuer URL the agent uses MUST match the issuer + // string Zitadel returns — Zitadel derives that from + // the request's Host header. We hit Zitadel via the + // host's port mapping, so the agent's URL is + // `http://sso.fleet.local:`. The /etc/hosts + // entry below points sso.fleet.local at the libvirt + // host so the VM resolves it. + oidc_issuer_url: format!("http://{ZITADEL_HOST}:{HTTP_PORT}"), + audience: project_id.to_string(), + // Local rehearsal hits Zitadel over plain HTTP through + // the cluster ingress; no TLS validation needed. + danger_accept_invalid_certs: true, + }, + agent_binary_path: opts.agent_binary.clone(), + hosts_entries: vec![HostsEntry { + ip: opts.libvirt_host_ip.clone(), + hostname: ZITADEL_HOST.to_string(), + }], + }); + + // Apply the score over SSH against the VM. Same pattern as + // fleet_rpi_setup, but synthesized inline so the harness can drive + // multiple VMs in sequence without copying the CLI plumbing. + apply_fleet_setup_to_vm(index, &vm_ip, agent_score).await?; + + Ok(DeviceHandle { + index, + device_id, + vm_ip, + labels, + }) +} + +async fn apply_fleet_setup_to_vm( + index: usize, + vm_ip: &str, + score: FleetDeviceSetupScore, +) -> Result<()> { + ensure_ansible_venv() + .await + .map_err(|e| anyhow::anyhow!("ansible venv: {e}"))?; + let ssh = ensure_fleet_ssh_keypair() + .await + .map_err(|e| anyhow::anyhow!("ssh keypair: {e}"))?; + let ip = vm_ip + .parse() + .with_context(|| format!("VM IP '{vm_ip}' is not a valid IP address"))?; + let creds = SshCredentials { + // Matches the cloud-init admin user that fleet_vm_setup + + // smoke-a4 create. If the operator overrode that during + // out-of-band VM provisioning, follow-up: thread the + // username through E2eDemoOpts. + user: "fleet-admin".to_string(), + private_key_path: ssh.private_key.clone(), + remote_python: Some("/usr/bin/python3".to_string()), + sudo_password: None, + }; + let topology = LinuxHostTopology::new(format!("vm-device-{index:02}"), ip, creds); + use harmony::score::Score; + score + .create_interpret() + .execute(&Inventory::empty(), &topology) + .await + .with_context(|| format!("FleetDeviceSetupScore against VM {index} ({vm_ip})"))?; + Ok(()) +} + +fn build_device_labels( + device_id: &str, + index: usize, +) -> std::collections::BTreeMap { + // Two devices, two distinct group labels by default — lets + // selector tests target "exactly one device". Label scheme + // matches the demo runbook. + let mut labels = std::collections::BTreeMap::new(); + labels.insert( + "group".to_string(), + if index == 0 { + "group-a".to_string() + } else { + "group-b".to_string() + }, + ); + labels.insert("arch".to_string(), std::env::consts::ARCH.to_string()); + labels.insert("role".to_string(), "rehearsal".to_string()); + labels.insert("device-id".to_string(), device_id.to_string()); + labels +} + +fn discover_vm_ip(index: usize) -> Result { + // Convention: a `FLEET_E2E_VM__IP` env var points at the + // pre-provisioned VM's IP. This keeps the harness usable on a + // workstation where the operator runs `fleet_vm_setup` once per + // device out-of-band, then re-runs the e2e harness against the + // already-booted VMs. + let key = format!("FLEET_E2E_VM_{index}_IP"); + std::env::var(&key) + .with_context(|| format!("set {key} to the libvirt VM's IP (default network)")) +} + +// ---- iam-admin-pat readiness ---------------------------------------------- + +/// Wait for the Zitadel chart's setup job to write the `iam-admin-pat` +/// secret. The Helm release reports Ready before the job completes, +/// so calling ZitadelSetupScore immediately after Zitadel deploy +/// races. ZitadelSetupScore itself reads this secret to authenticate +/// to the management API. +async fn wait_for_iam_admin_pat_secret(topology: &K8sAnywhereTopology) -> Result<()> { + use k8s_openapi::api::core::v1::Secret; + let k8s = topology + .k8s_client() + .await + .map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?; + for attempt in 1..=120 { + if let Some(secret) = k8s + .get_resource::("iam-admin-pat", Some("zitadel")) + .await? + && let Some(data) = secret.data + && data.contains_key("pat") + { + return Ok(()); + } + if attempt % 10 == 0 { + warn!("iam-admin-pat secret not yet present in zitadel ns ({attempt}/120)"); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + anyhow::bail!( + "timed out waiting for iam-admin-pat secret in 'zitadel' namespace — \ + is FirstInstance.Org.Machine.Pat configured in ZitadelScore Helm values?" + ) +} + +// ---- operator deploy ------------------------------------------------------- + +const OPERATOR_NAMESPACE: &str = FLEET_NAMESPACE; +const OPERATOR_KEY_MOUNT_PATH: &str = "/etc/fleet-operator/zitadel-key.json"; + +/// k3d's data directory under `$XDG_DATA_HOME`. Mirrors +/// `example_fleet_auth_callout::data_dir` (the latter is private — +/// duplicated here rather than re-exported so the operator wiring is +/// self-contained). +fn k3d_data_dir() -> PathBuf { + directories::BaseDirs::new() + .map(|dirs| dirs.data_dir().join("harmony").join("k3d")) + .unwrap_or_else(|| PathBuf::from("/tmp/harmony")) +} + +/// Build the operator's release binary, package it into an OCI image, +/// and sideload into the k3d cluster. Mirrors +/// `build_and_load_callout_image`. The Dockerfile lives in the +/// operator crate. +async fn build_and_load_operator_image(k3d: &k3d_rs::K3d) -> Result<()> { + use std::process::Stdio; + + let workspace_root = std::env::var("CARGO_MANIFEST_DIR") + .map(|d| PathBuf::from(d).join("..").join("..")) + .unwrap_or_else(|_| PathBuf::from(".")); + let workspace_root = workspace_root.canonicalize().unwrap_or(workspace_root); + + info!("cargo build --release -p harmony-fleet-operator"); + let status = tokio::process::Command::new("cargo") + .args(["build", "--release", "-p", "harmony-fleet-operator"]) + .current_dir(&workspace_root) + .status() + .await?; + if !status.success() { + anyhow::bail!("cargo build for fleet operator failed"); + } + + // Stage the binary + Dockerfile into a clean temp dir so podman + // build doesn't drag the whole target/ tree across. + let ctx = tempfile::tempdir()?; + let bin_dst = ctx.path().join("target/release"); + std::fs::create_dir_all(&bin_dst)?; + std::fs::copy( + workspace_root.join("target/release/harmony-fleet-operator"), + bin_dst.join("harmony-fleet-operator"), + ) + .context("staging operator binary into build context")?; + let dockerfile_src = workspace_root.join("fleet/harmony-fleet-operator/Dockerfile"); + if !dockerfile_src.exists() { + anyhow::bail!( + "missing fleet/harmony-fleet-operator/Dockerfile — operator image staging \ + expects it next to Cargo.toml; either add it or update the bring-up." + ); + } + std::fs::copy(&dockerfile_src, ctx.path().join("Dockerfile"))?; + + info!("podman build → {OPERATOR_IMAGE_TAG}"); + let status = tokio::process::Command::new("podman") + .args(["build", "-q", "-t", OPERATOR_IMAGE_TAG, "."]) + .current_dir(ctx.path()) + .stderr(Stdio::inherit()) + .status() + .await?; + if !status.success() { + anyhow::bail!("podman build for operator failed"); + } + + let tar_path = + std::env::temp_dir().join(format!("harmony-operator-image-{}.tar", std::process::id())); + let _ = std::fs::remove_file(&tar_path); + let status = tokio::process::Command::new("podman") + .args(["save", "-o", tar_path.to_str().unwrap(), OPERATOR_IMAGE_TAG]) + .status() + .await?; + if !status.success() { + anyhow::bail!("podman save for operator failed"); + } + info!("k3d image import {OPERATOR_IMAGE_TAG}"); + let cluster_name = k3d + .cluster_name() + .unwrap_or(example_fleet_auth_callout::CLUSTER_NAME) + .to_string(); + let tar_path_str = tar_path.to_str().unwrap().to_string(); + let cluster_for_blocking = cluster_name.clone(); + let data_dir = k3d_data_dir(); + tokio::task::spawn_blocking(move || { + k3d_rs::K3d::new(data_dir, Some(cluster_for_blocking.clone())).run_k3d_command([ + "image", + "import", + tar_path_str.as_str(), + "-c", + cluster_for_blocking.as_str(), + ]) + }) + .await? + .map_err(|e| anyhow::anyhow!("k3d image import failed: {e}"))?; + let _ = std::fs::remove_file(&tar_path); + Ok(()) +} + +/// Apply the operator's CRDs + ServiceAccount + ClusterRole + +/// ClusterRoleBinding + Secret + Deployment via Harmony's +/// K8sResourceScore. The Secret carries both the `[credentials]` TOML +/// (consumed by the operator as `FLEET_OPERATOR_CREDENTIALS_TOML`) and +/// the Zitadel JSON keyfile that the TOML's `key_path` references. +async fn deploy_operator( + topology: &K8sAnywhereTopology, + project_id: &str, + operator_machine_key: &str, +) -> Result<()> { + use harmony::modules::k8s::resource::K8sResourceScore; + use harmony_fleet_operator::chart::{ + ChartOptions, OperatorCredentials, RELEASE_NAME, build_cluster_role, + build_cluster_role_binding, build_operator_deployment, build_service_account, + operator_secret, + }; + use harmony_fleet_operator::crd::{Deployment as FleetDeployment, Device}; + use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; + use kube::CustomResourceExt; + + // Render the [credentials] TOML the operator pod will consume via + // env var. Same shape as the agent's [credentials] block — + // `harmony_fleet_auth::CredentialsSection` parses both verbatim. + let credentials_toml = format!( + r#"type = "zitadel-jwt" +key_path = "{key_path}" +oidc_issuer_url = "http://{host}:{port}" +audience = "{project_id}" +danger_accept_invalid_certs = true +"#, + key_path = OPERATOR_KEY_MOUNT_PATH, + host = ZITADEL_HOST, + port = HTTP_PORT, + ); + + let opts = ChartOptions { + output_dir: PathBuf::new(), // unused on this code path + image: OPERATOR_IMAGE_TAG.to_string(), + image_pull_policy: "IfNotPresent".to_string(), + namespace: OPERATOR_NAMESPACE.to_string(), + nats_url: format!("nats://{NATS_RELEASE}.{NATS_NAMESPACE}.svc.cluster.local:4222"), + log_level: "info,kube_runtime=warn".to_string(), + credentials: Some(OperatorCredentials { + credentials_toml, + zitadel_keyfile_json: operator_machine_key.to_string(), + key_mount_path: OPERATOR_KEY_MOUNT_PATH.to_string(), + }), + }; + + // CRDs first — the operator watches them on startup. + let crds: Vec = vec![FleetDeployment::crd(), Device::crd()]; + K8sResourceScore:: { + resource: crds, + namespace: None, + } + .interpret(&Inventory::autoload(), topology) + .await + .context("operator CRD apply")?; + + // RBAC. + K8sResourceScore::single( + build_service_account(&opts), + Some(OPERATOR_NAMESPACE.to_string()), + ) + .interpret(&Inventory::autoload(), topology) + .await + .context("operator ServiceAccount apply")?; + + K8sResourceScore::single(build_cluster_role(), None) + .interpret(&Inventory::autoload(), topology) + .await + .context("operator ClusterRole apply")?; + + K8sResourceScore::single(build_cluster_role_binding(&opts), None) + .interpret(&Inventory::autoload(), topology) + .await + .context("operator ClusterRoleBinding apply")?; + + // Secret holding both the credentials TOML and the keyfile. + let secret = operator_secret(&opts).expect("credentials present in opts"); + K8sResourceScore::single(secret, Some(OPERATOR_NAMESPACE.to_string())) + .interpret(&Inventory::autoload(), topology) + .await + .context("operator Secret apply")?; + + // Deployment last so it pulls the up-to-date Secret. + K8sResourceScore::single( + build_operator_deployment(&opts), + Some(OPERATOR_NAMESPACE.to_string()), + ) + .interpret(&Inventory::autoload(), topology) + .await + .context("operator Deployment apply")?; + + info!("operator deployment {OPERATOR_NAMESPACE}/{RELEASE_NAME} applied"); + Ok(()) +} + +async fn wait_for_operator_ready(topology: &K8sAnywhereTopology) -> Result<()> { + use harmony_fleet_operator::chart::RELEASE_NAME; + use k8s_openapi::api::apps::v1::Deployment as K8sDeployment; + let k8s = topology + .k8s_client() + .await + .map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?; + for attempt in 1..=120 { + if let Some(d) = k8s + .get_resource::(RELEASE_NAME, Some(OPERATOR_NAMESPACE)) + .await? + && let Some(status) = d.status + && status.ready_replicas.unwrap_or(0) >= 1 + { + return Ok(()); + } + if attempt % 10 == 0 { + warn!("operator Deployment not yet Ready ({attempt}/120)"); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + anyhow::bail!("timed out waiting for operator Deployment to become Ready") +} + +// ---- helpers --------------------------------------------------------------- + +fn workspace_target_path(rel: &str) -> PathBuf { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| PathBuf::from(".")); + manifest_dir.join("..").join("..").join("target").join(rel) +} + +// ---- next-steps panel ------------------------------------------------------ + +impl E2eHandles { + pub fn print_next_steps(&self) { + println!(); + println!("============================================================"); + println!(" E2E DEMO REHEARSAL — STACK READY"); + println!("============================================================"); + println!(" k3d cluster: {}", self.cluster_name); + println!(" Zitadel: {}", self.zitadel_url); + println!(" NATS (host): {}", self.nats_url_external); + println!(" Project ID: {}", self.project_id); + println!(" Issuer pubkey: {}", self.issuer_pubkey); + println!(); + println!(" Devices ({}):", self.devices.len()); + for d in &self.devices { + let labels: Vec = d.labels.iter().map(|(k, v)| format!("{k}={v}")).collect(); + println!( + " [{}] {} @ {} ({})", + d.index, + d.device_id, + d.vm_ip, + labels.join(",") + ); + } + println!(); + println!(" Run the test suite:"); + println!(); + println!(" cargo test -p example-fleet-e2e-demo \\"); + println!(" --test e2e_walking_skeleton -- --test-threads=1 --nocapture"); + println!(); + println!(" Ctrl-C exits without tearing the cluster down — re-run"); + println!(" the bring-up to converge any drift."); + println!("============================================================"); + } +} + +#[cfg(test)] +mod unit_tests { + use super::*; + + #[test] + fn device_username_matches_callout_convention() { + // Callout's device_id_claim is `client_id`, which Zitadel + // populates from the machine user's username. The test we + // run later asserts the agent's per-device subjects match + // its device_id, which therefore must equal the username + // minus the "device-" prefix the callout knows about. + assert_eq!(device_username("vm-device-00"), "device-vm-device-00"); + } + + #[test] + fn device_labels_split_into_distinct_groups() { + let l0 = build_device_labels("vm-device-00", 0); + let l1 = build_device_labels("vm-device-01", 1); + assert_eq!(l0.get("group").unwrap(), "group-a"); + assert_eq!(l1.get("group").unwrap(), "group-b"); + assert_ne!(l0.get("group"), l1.get("group")); + // Ubiquitous labels: device-id + arch + role on both. + for l in [&l0, &l1] { + assert!(l.contains_key("device-id")); + assert!(l.contains_key("arch")); + assert_eq!(l.get("role").unwrap(), "rehearsal"); + } + } +} diff --git a/examples/fleet_e2e_demo/src/main.rs b/examples/fleet_e2e_demo/src/main.rs new file mode 100644 index 00000000..b586f440 --- /dev/null +++ b/examples/fleet_e2e_demo/src/main.rs @@ -0,0 +1,51 @@ +//! `cargo run -p example-fleet-e2e-demo -- --num-devices 2 ...` +//! +//! Brings up the full E2E rehearsal stack: k3d + Zitadel + NATS auth +//! callout + per-device Zitadel machine users + (out-of-band) +//! libvirt VMs + agents authenticating via JWT-bearer. +//! +//! See `src/lib.rs` and `ROADMAP/fleet_platform/v0_demo_e2e.md`. + +use anyhow::{Context, Result}; +use clap::Parser; +use example_fleet_e2e_demo::{DEFAULT_LIBVIRT_HOST_IP, E2eDemoOpts, bring_up_full_stack}; +use std::path::PathBuf; + +#[derive(Parser, Debug)] +#[command( + name = "fleet-e2e-demo", + about = "VM-based end-to-end rehearsal of the fleet platform demo flow" +)] +struct Cli { + /// Number of VM-as-device agents to bring up. Each one needs its + /// own libvirt domain (provisioned out-of-band today via + /// `fleet_vm_setup` — see `FLEET_E2E_VM__IP` env vars below). + #[arg(long, default_value_t = 2)] + num_devices: usize, + /// Path to the cross-compiled `fleet-agent` binary uploaded to + /// each VM. Same binary that smoke-a4 produces. + #[arg(long, default_value = "target/release/harmony-fleet-agent")] + agent_binary: PathBuf, + /// Override for the libvirt host IP (the address VMs see as the + /// gateway). Defaults to the libvirt default network's gateway. + #[arg(long, default_value = DEFAULT_LIBVIRT_HOST_IP)] + libvirt_host_ip: String, +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + let handles = bring_up_full_stack(E2eDemoOpts { + num_devices: cli.num_devices, + agent_binary: cli.agent_binary, + libvirt_host_ip: cli.libvirt_host_ip, + }) + .await + .context("bring_up_full_stack")?; + handles.print_next_steps(); + + println!(); + println!(" Press Ctrl-C to exit (cluster keeps running)."); + tokio::signal::ctrl_c().await?; + Ok(()) +} diff --git a/examples/fleet_e2e_demo/tests/e2e_walking_skeleton.rs b/examples/fleet_e2e_demo/tests/e2e_walking_skeleton.rs new file mode 100644 index 00000000..36d51e3c --- /dev/null +++ b/examples/fleet_e2e_demo/tests/e2e_walking_skeleton.rs @@ -0,0 +1,159 @@ +//! End-to-end walking-skeleton tests for the VM-based demo rehearsal. +//! +//! Shares one bring-up across the whole suite via `OnceCell`. Run +//! sequentially — they touch shared k3d + libvirt VM state. +//! +//! Pre-flight (manual, before `cargo test`): +//! +//! - libvirt + qemu installed; default network active. +//! - Two cloud-init Ubuntu VMs provisioned (e.g. via +//! `cargo run -p example_fleet_vm_setup`). Their IPs exported as +//! `FLEET_E2E_VM_0_IP` and `FLEET_E2E_VM_1_IP`. +//! - SSH keypair the VMs trust at `~/.ssh/id_ed25519` (or +//! override path; harness reads the standard pair). +//! +//! Run: +//! +//! ```bash +//! FLEET_E2E_VM_0_IP=192.168.122.42 \ +//! FLEET_E2E_VM_1_IP=192.168.122.43 \ +//! cargo test -p example-fleet-e2e-demo --test e2e_walking_skeleton \ +//! -- --test-threads=1 --nocapture +//! ``` + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use async_nats::ConnectOptions; +use example_fleet_auth_callout::{mint_access_token, scopes_for_project}; +use example_fleet_e2e_demo::{E2eDemoOpts, E2eHandles, bring_up_full_stack}; +use futures_util::StreamExt; +use tokio::sync::OnceCell; + +static STACK: OnceCell> = OnceCell::const_new(); + +async fn shared_stack() -> Result> { + let cell = STACK + .get_or_try_init(|| async { + let h = bring_up_full_stack(E2eDemoOpts::default()).await?; + anyhow::Ok(Arc::new(h)) + }) + .await?; + Ok(cell.clone()) +} + +async fn admin_nats_client(stack: &E2eHandles) -> Result { + let token = mint_access_token( + &stack.zitadel_url, + &stack.admin_machine_key, + &scopes_for_project(&stack.project_id), + ) + .await + .context("mint admin Zitadel token")?; + ConnectOptions::with_token(token) + .connection_timeout(Duration::from_secs(5)) + .connect(&stack.nats_url_external) + .await + .map_err(|e| anyhow::anyhow!("admin connect: {e}")) +} + +// -- Test 1 ------------------------------------------------------------- + +/// Each provisioned VM publishes a DeviceInfo within the heartbeat +/// window. Reads from the `device-info` KV bucket via the admin +/// client (admin role can subscribe to anything). +#[tokio::test] +async fn both_devices_heartbeat_within_60s() -> Result<()> { + let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); + let stack = shared_stack().await?; + let admin = admin_nats_client(&stack).await?; + + let js = async_nats::jetstream::new(admin); + let bucket = js + .get_key_value(harmony_reconciler_contracts::BUCKET_DEVICE_INFO) + .await + .context("device-info bucket")?; + + let deadline = std::time::Instant::now() + Duration::from_secs(60); + let expected: std::collections::HashSet = + stack.devices.iter().map(|d| d.device_id.clone()).collect(); + let mut seen = std::collections::HashSet::new(); + + while std::time::Instant::now() < deadline && seen != expected { + for d in &stack.devices { + let key = harmony_reconciler_contracts::device_info_key(&d.device_id); + if let Some(_e) = bucket.entry(&key).await? { + seen.insert(d.device_id.clone()); + } + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + assert_eq!( + seen, expected, + "each provisioned device must publish DeviceInfo within 60s; saw {seen:?}" + ); + Ok(()) +} + +// -- Test 5 (admin cross-device read) ----------------------------------- + +/// The admin's Zitadel JWT carries `fleet-admin` role. Callout maps +/// that to `pub/sub allow: [">"]`, so subscribing to `device-state.>` +/// is admitted and observes every device's traffic. +#[tokio::test] +async fn admin_jwt_reads_any_device_subject() -> Result<()> { + let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); + let stack = shared_stack().await?; + let admin = admin_nats_client(&stack).await?; + + let mut sub = admin.subscribe("device-state.>").await?; + admin.flush().await?; + + // Hold the subscription open long enough that any device's + // periodic state publication should land. We don't pump traffic + // ourselves — the agents themselves publish per-deployment state + // on every reconcile tick. If no traffic arrives in 30s it means + // either the agents aren't connected or they're not publishing, + // both of which are fatal for the demo. + let result = tokio::time::timeout(Duration::from_secs(30), sub.next()).await; + assert!( + result.is_ok() && result.as_ref().unwrap().is_some(), + "admin must observe at least one device-state.* message in 30s" + ); + Ok(()) +} + +// -- Test 6 (per-device isolation) --------------------------------------- + +/// A per-device JWT has subject permissions scoped to its own +/// `device-state.{device_id}` and `device-commands.{device_id}`. The +/// callout enforces this; subscribing to a sibling device's commands +/// must fail at NATS connect-time or at SUB-time. +/// +/// Skipped here because the per-device JWT minting helper (analogous +/// to `mint_access_token` but for a `device` role user) needs the +/// per-device machine key to be plumbed back from `bring_up_full_stack` +/// through `E2eHandles`. Follow-up commit adds +/// `E2eHandles::device_machine_key(idx)` so this test can be +/// implemented without re-running `ZitadelSetupScore` from the test +/// body. +#[tokio::test] +#[ignore = "requires E2eHandles::device_machine_key plumbing"] +async fn cross_device_isolation_enforced_in_vm() {} + +// -- Test 7 (load-bearing reconnect) ------------------------------------- + +/// Kill the NATS pod, wait for the new one to come up, verify both +/// agents reconnect with fresh JWTs and resume publishing within +/// 30 seconds. This is the test that validates the "never lose +/// connectivity to a device" guarantee under realistic disturbance. +/// +/// Skipped pending operator install in the harness — without the +/// operator the agents have no `desired-state` to publish status +/// against, so verifying "publishing resumed" needs a separate +/// signal. Follow-up commit observes the agents' periodic +/// heartbeat publication directly via the device-heartbeat KV. +#[tokio::test] +#[ignore = "requires NATS-pod-restart driver and heartbeat-presence assertion"] +async fn agent_recovers_from_nats_pod_restart() {} diff --git a/examples/fleet_rpi_setup/Cargo.toml b/examples/fleet_rpi_setup/Cargo.toml index 2a01a8a5..559b400f 100644 --- a/examples/fleet_rpi_setup/Cargo.toml +++ b/examples/fleet_rpi_setup/Cargo.toml @@ -17,3 +17,7 @@ tokio.workspace = true log.workspace = true anyhow.workspace = true clap.workspace = true +reqwest = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +base64 = "0.22" diff --git a/examples/fleet_rpi_setup/src/main.rs b/examples/fleet_rpi_setup/src/main.rs index 0417eae8..74150d4b 100644 --- a/examples/fleet_rpi_setup/src/main.rs +++ b/examples/fleet_rpi_setup/src/main.rs @@ -31,11 +31,13 @@ //! - Python 3 + `python3-venv` (Ansible is auto-bootstrapped into a venv) //! - A cross-compiled `fleet-agent` binary for aarch64 +mod zitadel_bootstrap; + use anyhow::{Context, Result}; use clap::Parser; use harmony::config::secret::SudoPassword; use harmony::inventory::Inventory; -use harmony::modules::fleet::{FleetDeviceSetupConfig, FleetDeviceSetupScore}; +use harmony::modules::fleet::{FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore}; use harmony::modules::linux::{LinuxHostTopology, SshCredentials, ensure_ansible_venv, ssh_exec}; use harmony_secret::SecretManager; use harmony_types::id::Id; @@ -73,10 +75,41 @@ struct Cli { /// NATS URL the agent should connect to. #[arg(long)] nats_url: String, + /// Shared NATS username — used in `toml-shared` mode (no SSO). + /// Ignored when `--bootstrap-token` is set. #[arg(long, default_value = "smoke")] nats_user: String, + /// Shared NATS password — used in `toml-shared` mode (no SSO). + /// Ignored when `--bootstrap-token` is set. #[arg(long, default_value = "smoke")] nats_pass: String, + /// Zitadel admin Personal Access Token used to provision a + /// per-device machine user + role grant + JWT key on this Pi. + /// When set, the agent's NATS auth flips from `toml-shared` to + /// `zitadel-jwt` and the issued machine key is dropped onto the + /// Pi at `/etc/fleet-agent/zitadel-key.json`. The PAT itself is + /// used only by this CLI invocation — it never lands on the Pi. + #[arg(long, env = "HARMONY_ZITADEL_ADMIN_PAT")] + bootstrap_token: Option, + /// Externally-visible Zitadel issuer URL (e.g. + /// `https://zitadel.customer1.nationtech.io`). Required when + /// `--bootstrap-token` is set. + #[arg(long)] + zitadel_issuer_url: Option, + /// Zitadel project ID hosting the fleet roles. Required when + /// `--bootstrap-token` is set. Used as both the JWT-bearer + /// audience scope target and the role-claim path qualifier. + #[arg(long)] + zitadel_project_id: Option, + /// Zitadel role key to grant the per-device machine user. + /// Defaults to `device` (matches the auth callout's + /// `device_role` config). + #[arg(long, default_value = "device")] + zitadel_device_role: String, + /// Whether the agent's HTTP client to Zitadel accepts invalid + /// TLS certs. Local-dev escape hatch; default false. + #[arg(long)] + danger_accept_invalid_certs: bool, } #[tokio::main] @@ -127,13 +160,14 @@ async fn main() -> Result<()> { let topology = LinuxHostTopology::new(format!("rpi-{}", cli.pi_host), pi_ip, creds); let labels = parse_labels(&cli.labels)?; + let auth = build_auth(&cli, &device_id).await?; let score = FleetDeviceSetupScore::new(FleetDeviceSetupConfig { - device_id, + device_id: device_id.clone(), labels, nats_urls: vec![cli.nats_url.clone()], - nats_user: cli.nats_user.clone(), - nats_pass: cli.nats_pass.clone(), + auth, agent_binary_path: cli.agent_binary.clone(), + hosts_entries: vec![], }); // We have our own clap CLI, so harmony_cli must NOT call @@ -161,6 +195,53 @@ async fn main() -> Result<()> { Ok(()) } +/// Build the per-device auth block. Either: +/// - `--bootstrap-token` is set → mint a per-device Zitadel machine +/// user + role grant + JWT key via the Management API and embed the +/// key JSON in `FleetDeviceAuth::ZitadelJwt`. The bootstrap PAT +/// never leaves this CLI invocation. +/// - Otherwise → fall back to `--nats-user`/`--nats-pass` shared creds. +async fn build_auth(cli: &Cli, device_id: &Id) -> Result { + let Some(pat) = cli.bootstrap_token.clone() else { + info!("no --bootstrap-token; using shared NATS user/pass (toml-shared)"); + return Ok(FleetDeviceAuth::TomlShared { + nats_user: cli.nats_user.clone(), + nats_pass: cli.nats_pass.clone(), + }); + }; + let issuer = cli + .zitadel_issuer_url + .clone() + .context("--bootstrap-token requires --zitadel-issuer-url")?; + let project_id = cli + .zitadel_project_id + .clone() + .context("--bootstrap-token requires --zitadel-project-id")?; + + info!("bootstrapping Zitadel machine user device-{device_id} on project {project_id}"); + let bootstrap = zitadel_bootstrap::ZitadelBootstrap::new( + issuer.clone(), + pat, + cli.danger_accept_invalid_certs, + ); + let key_json = bootstrap + .ensure_device_machine_user( + &format!("device-{device_id}"), + &device_id.to_string(), + &project_id, + &cli.zitadel_device_role, + ) + .await + .context("Zitadel device bootstrap failed")?; + + Ok(FleetDeviceAuth::ZitadelJwt { + machine_key_json: key_json, + oidc_issuer_url: issuer, + audience: project_id, + danger_accept_invalid_certs: cli.danger_accept_invalid_certs, + }) +} + fn parse_labels(raw: &str) -> Result> { let mut out = std::collections::BTreeMap::new(); for piece in raw.split(',').map(str::trim).filter(|p| !p.is_empty()) { diff --git a/examples/fleet_rpi_setup/src/zitadel_bootstrap.rs b/examples/fleet_rpi_setup/src/zitadel_bootstrap.rs new file mode 100644 index 00000000..5c11adbc --- /dev/null +++ b/examples/fleet_rpi_setup/src/zitadel_bootstrap.rs @@ -0,0 +1,247 @@ +//! Per-device Zitadel bootstrap for the Pi onboarding flow. +//! +//! Invoked once per Pi from the operator's machine. Uses the admin PAT +//! given on the CLI to: +//! +//! 1. Find or create a machine user `device-${device_id}` in Zitadel. +//! 2. Find or create a JSON-typed JWT signing key for that user. +//! 3. Find or create a project grant on the `device` role. +//! +//! Returns the JSON keyfile content. The caller drops it onto the Pi +//! via `FleetDeviceSetupScore`. The admin PAT is held in CLI memory +//! for the duration of the run only — it never lands on the Pi. +//! +//! All operations are idempotent: re-running for the same device id +//! is a series of NOOPs. +//! +//! NOTE: This is intentionally a minimal Management-API client. It +//! duplicates a small slice of `harmony::modules::zitadel::setup` (the +//! in-cluster ZitadelSetupScore) because `fleet_rpi_setup` runs on the +//! operator's machine without a kubeconfig pointing at the Zitadel +//! cluster. Refactoring the in-cluster Score's HTTP layer into a +//! reusable client crate is a follow-up. + +use anyhow::{Context, Result}; +use base64::Engine; +use serde::Deserialize; + +pub struct ZitadelBootstrap { + issuer_url: String, + admin_pat: String, + http: reqwest::Client, +} + +impl ZitadelBootstrap { + pub fn new(issuer_url: String, admin_pat: String, danger_accept_invalid_certs: bool) -> Self { + let http = reqwest::Client::builder() + .danger_accept_invalid_certs(danger_accept_invalid_certs) + .timeout(std::time::Duration::from_secs(10)) + .build() + .expect("reqwest client builder is infallible for these settings"); + Self { + issuer_url, + admin_pat, + http, + } + } + + /// Ensure machine user + key + role grant for one device. Returns + /// the JSON keyfile content (raw, decoded from Zitadel's base64 + /// `keyDetails`). Idempotent: re-running with the same `username` + /// reuses the existing user; if no key was previously persisted + /// (we can't read the private key back from Zitadel), a fresh one + /// is generated and returned. + pub async fn ensure_device_machine_user( + &self, + username: &str, + device_id: &str, + project_id: &str, + role_key: &str, + ) -> Result { + let user_id = match self.find_user_by_name(username).await? { + Some(id) => id, + None => self + .create_machine_user(username, device_id) + .await + .with_context(|| format!("creating machine user {username}"))?, + }; + log::info!("[zitadel-bootstrap] machine user {username} → {user_id}"); + + // The grant API rejects duplicates with code 6 (ALREADY_EXISTS), + // so the cheapest path is "search → maybe create". + if self.find_user_grant(&user_id, project_id).await?.is_none() { + self.create_user_grant(&user_id, project_id, role_key) + .await + .with_context(|| { + format!("granting role {role_key} on project {project_id} to {username}") + })?; + log::info!("[zitadel-bootstrap] granted role {role_key} on project {project_id}"); + } else { + log::info!("[zitadel-bootstrap] role grant already present"); + } + + // Always mint a fresh key — Zitadel doesn't expose the private + // half of existing keys, so we can't reuse one. Stale keys + // remain valid until expiry but never get reused on this Pi + // because the agent's keyfile is overwritten on each setup run. + let key_json = self + .create_machine_key(&user_id) + .await + .with_context(|| format!("minting machine key for {username}"))?; + Ok(key_json) + } + + fn url(&self, path: &str) -> String { + format!("{}{path}", self.issuer_url.trim_end_matches('/')) + } + + async fn find_user_by_name(&self, username: &str) -> Result> { + let resp = self + .http + .post(self.url("/management/v1/users/_search")) + .bearer_auth(&self.admin_pat) + .json(&serde_json::json!({ + "queries": [{ + "userNameQuery": { + "userName": username, + "method": "TEXT_QUERY_METHOD_EQUALS" + } + }] + })) + .send() + .await + .context("POST users/_search")?; + if !resp.status().is_success() { + let s = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("users/_search returned {s}: {body}"); + } + #[derive(Deserialize)] + struct R { + #[serde(default)] + result: Vec, + } + #[derive(Deserialize)] + struct E { + id: String, + #[serde(rename = "userName", default)] + user_name: Option, + } + let r: R = resp.json().await.context("parse users/_search")?; + Ok(r.result + .into_iter() + .find(|e| e.user_name.as_deref() == Some(username)) + .map(|e| e.id)) + } + + async fn create_machine_user(&self, username: &str, device_id: &str) -> Result { + let resp = self + .http + .post(self.url("/management/v1/users/machine")) + .bearer_auth(&self.admin_pat) + .json(&serde_json::json!({ + "userName": username, + "name": format!("Fleet Device {device_id}"), + "description": format!("Provisioned by fleet_rpi_setup for device {device_id}"), + "accessTokenType": "ACCESS_TOKEN_TYPE_JWT" + })) + .send() + .await + .context("POST users/machine")?; + if !resp.status().is_success() { + let s = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("create machine user returned {s}: {body}"); + } + #[derive(Deserialize)] + struct R { + #[serde(rename = "userId")] + user_id: String, + } + let r: R = resp.json().await.context("parse machine user response")?; + Ok(r.user_id) + } + + async fn create_machine_key(&self, user_id: &str) -> Result { + let resp = self + .http + .post(self.url(&format!("/management/v1/users/{user_id}/keys"))) + .bearer_auth(&self.admin_pat) + .json(&serde_json::json!({ "type": "KEY_TYPE_JSON" })) + .send() + .await + .context("POST users/{}/keys")?; + if !resp.status().is_success() { + let s = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("create machine key returned {s}: {body}"); + } + #[derive(Deserialize)] + struct R { + #[serde(rename = "keyDetails")] + key_details: String, + } + let r: R = resp.json().await.context("parse machine key response")?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(&r.key_details) + .context("decode keyDetails base64")?; + String::from_utf8(bytes).context("keyDetails is non-UTF-8") + } + + async fn find_user_grant(&self, user_id: &str, project_id: &str) -> Result> { + let resp = self + .http + .post(self.url(&format!("/management/v1/users/{user_id}/grants/_search"))) + .bearer_auth(&self.admin_pat) + .json(&serde_json::json!({})) + .send() + .await + .context("POST users/{}/grants/_search")?; + if !resp.status().is_success() { + let s = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("grants/_search returned {s}: {body}"); + } + #[derive(Deserialize)] + struct R { + #[serde(default)] + result: Vec, + } + #[derive(Deserialize)] + struct E { + id: String, + #[serde(rename = "projectId")] + project_id: String, + } + let r: R = resp.json().await.context("parse grants/_search")?; + Ok(r.result + .into_iter() + .find(|e| e.project_id == project_id) + .map(|e| e.id)) + } + + async fn create_user_grant( + &self, + user_id: &str, + project_id: &str, + role_key: &str, + ) -> Result<()> { + let resp = self + .http + .post(self.url(&format!("/management/v1/users/{user_id}/grants"))) + .bearer_auth(&self.admin_pat) + .json(&serde_json::json!({ + "projectId": project_id, + "roleKeys": [role_key] + })) + .send() + .await + .context("POST users/{}/grants")?; + if !resp.status().is_success() { + let s = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("create grant returned {s}: {body}"); + } + Ok(()) + } +} diff --git a/examples/fleet_sso_login/Cargo.toml b/examples/fleet_sso_login/Cargo.toml new file mode 100644 index 00000000..1fbb5c6f --- /dev/null +++ b/examples/fleet_sso_login/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "example-fleet-sso-login" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true +description = "Developer-side CLI: log in to a fleet platform staging instance via Zitadel device-code OIDC" + +[[bin]] +name = "fleet-sso-login" +path = "src/main.rs" + +[dependencies] +reqwest = { workspace = true } +tokio = { workspace = true, features = ["full"] } +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +clap = { version = "4", features = ["derive", "env"] } +base64 = "0.22" +log.workspace = true +env_logger.workspace = true +directories = "6.0.0" diff --git a/examples/fleet_sso_login/src/main.rs b/examples/fleet_sso_login/src/main.rs new file mode 100644 index 00000000..c9de0f44 --- /dev/null +++ b/examples/fleet_sso_login/src/main.rs @@ -0,0 +1,266 @@ +//! Developer-side CLI: log in to a fleet platform staging instance via +//! Zitadel's OIDC Device Authorization Grant (RFC 8628). +//! +//! Usage: +//! +//! ```text +//! cargo run -p example-fleet-sso-login -- \ +//! --base-domain customer1.nationtech.io \ +//! --client-id 366378028009259038 +//! ``` +//! +//! Flow: +//! 1. POST to `/oauth/v2/device_authorization` with the CLI client_id — +//! receive a `verification_uri_complete`, `user_code`, `device_code` +//! and a polling interval. +//! 2. Print the URL the user opens in their browser. They authenticate +//! via Zitadel (username/password, MFA, SSO chain — Zitadel handles +//! that part). +//! 3. Poll `/oauth/v2/token` with `grant_type=urn:ietf:params:oauth: +//! grant-type:device_code` until the access token is issued. +//! 4. Decode the access token's claims, print "Welcome ", and persist the session at +//! `$DATA_DIR/harmony/sso-session.json`. +//! +//! No K8s API call yet — for the demo, this CLI proves the SSO works. +//! Future: a `harmony fleet apply` subcommand uses the persisted token +//! to talk to a fleet-platform API gateway. That gateway is post-demo. + +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use base64::Engine; +use clap::Parser; +use serde::{Deserialize, Serialize}; + +#[derive(Parser, Debug)] +#[command( + name = "fleet-sso-login", + about = "Log in to a fleet platform staging instance via Zitadel device-code OIDC" +)] +struct Cli { + /// Base DNS domain — same value the operator passed to + /// fleet-staging-deploy. The Zitadel issuer derives as + /// `https://zitadel.`. + #[arg(long, env = "FLEET_BASE_DOMAIN")] + base_domain: String, + /// OIDC client_id of the `harmony-cli` Device Code app on the + /// Zitadel project. Printed by `fleet-staging-deploy` at the end + /// of a successful run. + #[arg(long, env = "FLEET_CLI_CLIENT_ID")] + client_id: String, + /// Override the polling interval suggested by Zitadel + /// (defaults to whatever the device-authorization endpoint returned; + /// pass to short-circuit during testing). + #[arg(long)] + poll_interval_secs: Option, +} + +#[derive(Debug, Deserialize)] +struct DeviceAuthResponse { + device_code: String, + user_code: String, + verification_uri: String, + #[serde(default)] + verification_uri_complete: Option, + expires_in: u64, + #[serde(default)] + interval: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +struct TokenResponse { + access_token: String, + #[serde(default)] + id_token: Option, + #[serde(default)] + refresh_token: Option, + #[serde(default)] + expires_in: Option, + #[serde(default)] + token_type: Option, +} + +#[derive(Debug, Deserialize)] +struct TokenError { + error: String, + #[serde(default)] + error_description: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) + .try_init(); + let cli = Cli::parse(); + + let issuer = format!("https://zitadel.{}", cli.base_domain); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(15)) + .build()?; + + // -- Step 1: kick off the device flow ---------------------------- + let device_auth_url = format!("{issuer}/oauth/v2/device_authorization"); + let scope = + "openid profile email urn:zitadel:iam:user:resourceowner urn:zitadel:iam:org:project:roles"; + let resp = client + .post(&device_auth_url) + .form(&[("client_id", cli.client_id.as_str()), ("scope", scope)]) + .send() + .await + .with_context(|| format!("POST {device_auth_url}"))?; + if !resp.status().is_success() { + let s = resp.status(); + let body = resp.text().await.unwrap_or_default(); + bail!("device_authorization returned {s}: {body}"); + } + let auth: DeviceAuthResponse = resp.json().await.context("parse device_authorization")?; + + let display_url = auth + .verification_uri_complete + .clone() + .unwrap_or_else(|| auth.verification_uri.clone()); + println!(); + println!("============================================================"); + println!(" Open this URL in your browser to log in:"); + println!(); + println!(" {display_url}"); + println!(); + println!(" If the URL doesn't pre-fill the code, enter:"); + println!(); + println!(" user_code: {}", auth.user_code); + println!(); + println!( + " Waiting for browser-side completion (expires in {}s)...", + auth.expires_in + ); + println!("============================================================"); + println!(); + + // -- Step 2: poll the token endpoint ----------------------------- + let token_url = format!("{issuer}/oauth/v2/token"); + let interval = + Duration::from_secs(cli.poll_interval_secs.unwrap_or(auth.interval.unwrap_or(5))); + let deadline = std::time::Instant::now() + Duration::from_secs(auth.expires_in); + + let access_token = loop { + if std::time::Instant::now() > deadline { + bail!("device-code expired before user completed login"); + } + tokio::time::sleep(interval).await; + let resp = client + .post(&token_url) + .form(&[ + ("grant_type", "urn:ietf:params:oauth:grant-type:device_code"), + ("device_code", auth.device_code.as_str()), + ("client_id", cli.client_id.as_str()), + ]) + .send() + .await + .context("POST token")?; + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + if status.is_success() { + let tr: TokenResponse = + serde_json::from_str(&body).context("parse token success body")?; + break tr.access_token; + } + // Per RFC 8628, the token endpoint returns specific error + // codes during polling — `authorization_pending` and + // `slow_down` are NOT terminal, every other error is. + let err: TokenError = serde_json::from_str(&body).unwrap_or_else(|_| TokenError { + error: format!("http_{}", status.as_u16()), + error_description: Some(body.clone()), + }); + match err.error.as_str() { + "authorization_pending" => { + log::debug!("authorization_pending — user hasn't approved yet"); + continue; + } + "slow_down" => { + log::info!("server requested slow_down — increasing poll interval"); + tokio::time::sleep(interval).await; // wait one extra interval + continue; + } + other => bail!( + "token endpoint refused: {other} ({})", + err.error_description.unwrap_or_default() + ), + } + }; + + // -- Step 3: introspect + persist -------------------------------- + let claims = decode_jwt_claims(&access_token).unwrap_or_default(); + let display_name = claims + .get("name") + .or_else(|| claims.get("preferred_username")) + .and_then(|v| v.as_str()) + .unwrap_or("(unknown)"); + let email = claims + .get("email") + .and_then(|v| v.as_str()) + .unwrap_or("(no email)"); + + persist_session(&issuer, &cli.client_id, &access_token, &claims)?; + + println!(); + println!("============================================================"); + println!(" SSO LOGIN SUCCESSFUL"); + println!("============================================================"); + println!(" Welcome, {display_name} <{email}>"); + println!(" Session stored at: {}", session_path().display()); + println!("============================================================"); + Ok(()) +} + +fn decode_jwt_claims(jwt: &str) -> Option { + let payload_b64 = jwt.split('.').nth(1)?; + let pad = "=".repeat((4 - payload_b64.len() % 4) % 4); + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(format!("{payload_b64}{pad}").trim_end_matches('=')) + .ok()?; + serde_json::from_slice(&bytes).ok() +} + +#[derive(Serialize)] +struct PersistedSession<'a> { + issuer: &'a str, + client_id: &'a str, + access_token: &'a str, + claims: &'a serde_json::Value, +} + +fn persist_session( + issuer: &str, + client_id: &str, + access_token: &str, + claims: &serde_json::Value, +) -> Result<()> { + let path = session_path(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("create session dir {}", parent.display()))?; + } + let s = PersistedSession { + issuer, + client_id, + access_token, + claims, + }; + let json = serde_json::to_string_pretty(&s)?; + std::fs::write(&path, json).with_context(|| format!("write session to {}", path.display()))?; + // 0600 so other users on the box can't read the access token. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).ok(); + } + Ok(()) +} + +fn session_path() -> PathBuf { + directories::BaseDirs::new() + .map(|d| d.data_dir().join("harmony").join("sso-session.json")) + .unwrap_or_else(|| PathBuf::from("/tmp/harmony-sso-session.json")) +} diff --git a/examples/fleet_staging_deploy/Cargo.toml b/examples/fleet_staging_deploy/Cargo.toml new file mode 100644 index 00000000..a6b4a96b --- /dev/null +++ b/examples/fleet_staging_deploy/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "example-fleet-staging-deploy" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true +description = "Deploy the fleet platform stack (Zitadel + NATS + auth callout) onto an OKD/Kubernetes cluster. Operator-side, run-once-per-customer." + +[lib] +name = "example_fleet_staging_deploy" +path = "src/lib.rs" + +[[bin]] +name = "fleet-staging-deploy" +path = "src/main.rs" + +[dependencies] +harmony = { path = "../../harmony" } +harmony-k8s = { path = "../../harmony-k8s" } +harmony_types = { path = "../../harmony_types" } +harmony-nats-callout = { path = "../../nats/callout" } +nkeys = "0.4" +async-nats.workspace = true +reqwest = { workspace = true } +tokio = { workspace = true, features = ["full"] } +serde.workspace = true +serde_json.workspace = true +anyhow.workspace = true +log.workspace = true +env_logger.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +clap = { version = "4", features = ["derive", "env"] } +k8s-openapi.workspace = true +kube.workspace = true +url.workspace = true diff --git a/examples/fleet_staging_deploy/src/lib.rs b/examples/fleet_staging_deploy/src/lib.rs new file mode 100644 index 00000000..32b80184 --- /dev/null +++ b/examples/fleet_staging_deploy/src/lib.rs @@ -0,0 +1,572 @@ +//! Operator-side staging deploy harness. +//! +//! Runs once per customer instance against an OKD / Kubernetes cluster +//! to bring up the fleet platform's central services: +//! +//! 1. Zitadel + Postgres (HTTPS via OKD HAProxy ingress, edge TLS). +//! 2. The fleet project + roles (`fleet-admin`, `device`) + an API app +//! (so the project ID can be the JWT-bearer audience). +//! 3. NATS with `auth_callout` and a WSS ingress (so Pis on a customer +//! LAN connect through `wss://nats./`). +//! 4. The auth callout Deployment, configured to validate Zitadel JWTs +//! and emit per-device permissions on user JWTs to NATS. +//! +//! Everything keys off [`FleetDomainConfig::base_domain`] — +//! `zitadel.`, `nats.`, `api.` are the only +//! customer-visible hostnames. Pi-side onboarding (see +//! `examples/fleet_rpi_setup/`) consumes the Zitadel admin PAT plus +//! the project ID this harness prints, so the operator's flow is: +//! +//! ```text +//! cargo run -p example-fleet-staging-deploy -- --base-domain customer1.nationtech.io +//! ↓ prints PROJECT_ID, NATS WSS URL, instructions to extract iam-admin-pat +//! HARMONY_ZITADEL_ADMIN_PAT=$(kubectl -n zitadel get secret iam-admin-pat -o jsonpath='{.data.pat}' | base64 -d) \ +//! cargo run -p example-fleet-rpi-setup -- \ +//! --pi-host 192.168.1.42 \ +//! --bootstrap-token "$HARMONY_ZITADEL_ADMIN_PAT" \ +//! --zitadel-issuer-url https://zitadel.customer1.nationtech.io \ +//! --zitadel-project-id \ +//! --nats-url wss://nats.customer1.nationtech.io/ \ +//! --agent-binary ./target/aarch64-unknown-linux-gnu/release/fleet-agent +//! ``` +//! +//! The harness is **idempotent** by design — re-running picks up +//! existing resources via the new helm-upgrade-by-default behavior + +//! ZitadelSetupScore's search-then-create flow + a persisted issuer +//! NKey in a K8s secret so user JWTs survive restarts. + +use std::time::Duration; + +use anyhow::{Context, Result}; +use harmony::inventory::Inventory; +use harmony::modules::nats::NatsHelmChartScore; +use harmony::modules::nats_auth_callout::{NatsAuthCalloutScore, render_auth_callout_block}; +use harmony::modules::zitadel::{ + ZitadelApiApp, ZitadelAppType, ZitadelApplication, ZitadelClientConfig, ZitadelRole, + ZitadelScore, ZitadelSetupScore, +}; +use harmony::score::Score; +use harmony::topology::{K8sAnywhereTopology, K8sclient, Topology}; +use log::info; +use nkeys::KeyPair; + +// ---- domain config --------------------------------------------------------- + +/// Single source of truth for all customer-visible hostnames. Every +/// `..` URL the staging deploy emits derives from +/// the one base domain — no hostnames are hardcoded so the same code +/// runs across customers / staging / canary instances. +#[derive(Debug, Clone)] +pub struct FleetDomainConfig { + /// e.g. `customer1.nationtech.io`. The deploy emits + /// `zitadel.`, `nats.`, `api.` against it. + pub base_domain: String, +} + +impl FleetDomainConfig { + pub fn new(base_domain: impl Into) -> Self { + Self { + base_domain: base_domain.into(), + } + } + pub fn zitadel_host(&self) -> String { + format!("zitadel.{}", self.base_domain) + } + pub fn nats_wss_host(&self) -> String { + format!("nats.{}", self.base_domain) + } + pub fn zitadel_issuer_url(&self) -> String { + format!("https://{}", self.zitadel_host()) + } + pub fn nats_wss_url(&self) -> String { + format!("wss://{}/", self.nats_wss_host()) + } +} + +// ---- naming + constants ---------------------------------------------------- + +pub const FLEET_NAMESPACE: &str = "fleet-system"; +pub const NATS_RELEASE: &str = "fleet-nats"; +pub const CALLOUT_DEPLOYMENT_NAME: &str = "fleet-callout"; +pub const PROJECT_NAME: &str = "fleet"; +pub const API_APP_NAME: &str = "nats"; +pub const CLI_APP_NAME: &str = "harmony-cli"; +pub const ADMIN_ROLE_KEY: &str = "fleet-admin"; +pub const DEVICE_ROLE_KEY: &str = "device"; +pub const NATS_AUTH_USER: &str = "auth"; +pub const NATS_ACCOUNT: &str = "DEVICES"; +pub const NATS_SYSTEM_USER: &str = "sys-admin"; +pub const ISSUER_SEED_SECRET: &str = "callout-issuer-seed"; + +// ---- handles --------------------------------------------------------------- + +#[derive(Debug, Clone)] +pub struct StagingHandles { + pub domain: FleetDomainConfig, + pub project_id: String, + pub issuer_pubkey: String, + /// Tag of the callout image expected to exist in a registry the + /// cluster pulls from. The operator pushes it before running the + /// deploy; this field is just the name we put on the Deployment + /// for traceability. + pub callout_image: String, + /// OIDC client_id of the `harmony-cli` Device Code app — what the + /// `fleet_sso_login` CLI sends in its device-authorization request. + /// `None` if the app pre-existed without the cache picking it up + /// (re-running the staging deploy after `rm -rf + /// ~/.local/share/harmony/zitadel/`). + pub cli_client_id: Option, +} + +// ---- bring up -------------------------------------------------------------- + +pub struct StagingDeployOpts { + pub domain: FleetDomainConfig, + pub kubeconfig_context: Option, + /// Image reference the cluster will pull. Operator must have + /// pushed this beforehand (e.g. `quay.io/customer/harmony-nats-callout:demo`). + pub callout_image: String, + /// Per-NATS-account password for the callout's own NATS connection. + /// Stored in a K8s secret + listed in the chart's + /// `accounts..users` so the callout bypasses callout to + /// connect (otherwise it'd deadlock authenticating itself). + pub nats_auth_pass: String, + /// SYS account password (for `kubectl exec nats-box` debugging). + pub nats_system_pass: String, +} + +pub async fn bring_up_staging(opts: StagingDeployOpts) -> Result { + let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) + .try_init(); + + if let Some(ctx) = &opts.kubeconfig_context { + unsafe { + std::env::set_var("HARMONY_K8S_CONTEXT", ctx); + std::env::set_var("HARMONY_USE_LOCAL_K3D", "false"); + std::env::set_var("HARMONY_AUTOINSTALL", "false"); + } + } + let topology = K8sAnywhereTopology::from_env(); + topology.ensure_ready().await.context("topology init")?; + + info!( + "[1/5] deploying Zitadel at https://{}", + opts.domain.zitadel_host() + ); + deploy_zitadel(&opts.domain, &topology).await?; + + info!("[2/5] waiting for Zitadel HTTPS to respond"); + wait_for_zitadel_ready(&opts.domain).await?; + + info!("[3/5] provisioning project '{PROJECT_NAME}', api app, CLI device-code app, and roles"); + provision_zitadel_project(&opts.domain, &topology).await?; + let project_id = read_project_id()?; + let cli_client_id = read_cli_client_id(); + info!(" → project_id = {project_id}"); + if let Some(cid) = &cli_client_id { + info!(" → cli_client_id = {cid}"); + } else { + log::warn!( + " → cli_client_id missing from cache; CLI login won't work until you reset the local zitadel cache" + ); + } + + info!("[4/5] generating issuer NKey + deploying NATS with auth_callout + WSS ingress"); + let issuer_seed = ensure_issuer_seed(&topology).await?; + let issuer_kp = KeyPair::from_seed(&issuer_seed) + .map_err(|e| anyhow::anyhow!("invalid persisted issuer seed: {e}"))?; + let issuer_pubkey = issuer_kp.public_key(); + + NatsHelmChartScore::new( + NATS_RELEASE.to_string(), + FLEET_NAMESPACE.to_string(), + render_nats_values( + &opts.domain, + &issuer_pubkey, + &opts.nats_auth_pass, + &opts.nats_system_pass, + ), + ) + .interpret(&Inventory::autoload(), &topology) + .await + .context("NATS deploy")?; + + info!( + "[5/5] deploying NatsAuthCalloutScore (image: {})", + opts.callout_image + ); + NatsAuthCalloutScore::new( + CALLOUT_DEPLOYMENT_NAME, + FLEET_NAMESPACE, + format!("nats://{NATS_RELEASE}.{FLEET_NAMESPACE}.svc.cluster.local:4222"), + opts.domain.zitadel_issuer_url(), + // The aud the callout validates against is the project ID — + // Zitadel emits it in access tokens minted via the + // project-id-audience scope. + project_id.clone(), + NATS_AUTH_USER, + opts.nats_auth_pass.clone(), + issuer_seed, + ) + .image(&opts.callout_image) + .target_account(NATS_ACCOUNT) + .admin_role(ADMIN_ROLE_KEY) + .device_role(DEVICE_ROLE_KEY) + .interpret(&Inventory::autoload(), &topology) + .await + .context("callout deploy")?; + + Ok(StagingHandles { + domain: opts.domain, + project_id, + issuer_pubkey, + callout_image: opts.callout_image, + cli_client_id, + }) +} + +fn read_cli_client_id() -> Option { + ZitadelClientConfig::load()? + .client_id(CLI_APP_NAME) + .cloned() +} + +async fn deploy_zitadel(domain: &FleetDomainConfig, topology: &K8sAnywhereTopology) -> Result<()> { + let z = ZitadelScore { + host: domain.zitadel_host(), + zitadel_version: "v4.12.1".to_string(), + // OKD HAProxy edge-terminates TLS for us, so the issuer URL + // is `https://zitadel.` (port 443 implied) — leave + // external_port at None so Zitadel's emitted issuer omits the + // port, matching what clients reach. + external_secure: true, + external_port: None, + }; + z.interpret(&Inventory::autoload(), topology) + .await + .context("ZitadelScore")?; + Ok(()) +} + +async fn provision_zitadel_project( + domain: &FleetDomainConfig, + topology: &K8sAnywhereTopology, +) -> Result<()> { + let setup = ZitadelSetupScore { + host: domain.zitadel_host(), + // OKD HAProxy listens on 443; ZitadelSetupScore talks to + // 127.0.0.1: with Host header + skip_tls — but for + // staging we go through the real ingress so the operator can + // run this from anywhere with kubeconfig + DNS access. 443 is + // the externally-visible port. + port: 443, + skip_tls: false, + applications: vec![ZitadelApplication { + project_name: PROJECT_NAME.to_string(), + app_name: CLI_APP_NAME.to_string(), + // Device Code grant — the only browser-driven OIDC flow + // that fits a CLI tool: prints a verification URL + user + // code, polls for a token, no embedded web server / open + // listener required. + app_type: ZitadelAppType::DeviceCode, + }], + api_apps: vec![ZitadelApiApp { + project_name: PROJECT_NAME.to_string(), + app_name: API_APP_NAME.to_string(), + }], + roles: vec![ + ZitadelRole { + project_name: PROJECT_NAME.to_string(), + key: ADMIN_ROLE_KEY.to_string(), + display_name: "Fleet Admin".to_string(), + group: None, + }, + ZitadelRole { + project_name: PROJECT_NAME.to_string(), + key: DEVICE_ROLE_KEY.to_string(), + display_name: "Device".to_string(), + group: None, + }, + ], + // No machine users provisioned here — `fleet_rpi_setup` mints + // them on demand per device, so the staging deploy stays + // device-count-agnostic. + machine_users: vec![], + }; + setup + .interpret(&Inventory::autoload(), topology) + .await + .context("ZitadelSetupScore")?; + Ok(()) +} + +fn read_project_id() -> Result { + let cfg = ZitadelClientConfig::load() + .context("ZitadelSetupScore did not produce a client config cache")?; + cfg.project_id_by_name(PROJECT_NAME) + .or(cfg.project_id.as_ref()) + .context("project_id missing from ZitadelClientConfig cache") + .cloned() +} + +/// Persist the callout's issuer NKey seed in a K8s secret so re-runs +/// of the staging deploy don't invalidate previously-issued user JWTs +/// already in flight on customer Pis. +async fn ensure_issuer_seed(topology: &K8sAnywhereTopology) -> Result { + use k8s_openapi::ByteString; + use k8s_openapi::api::core::v1::{Namespace, Secret}; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + use std::collections::BTreeMap; + + let k8s = topology + .k8s_client() + .await + .map_err(|e| anyhow::anyhow!("k8s_client: {e}"))?; + + if k8s + .get_resource::(FLEET_NAMESPACE, None) + .await? + .is_none() + { + let ns = Namespace { + metadata: ObjectMeta { + name: Some(FLEET_NAMESPACE.to_string()), + ..Default::default() + }, + ..Default::default() + }; + k8s.create(&ns, None).await.ok(); + } + + if let Some(existing) = k8s + .get_resource::(ISSUER_SEED_SECRET, Some(FLEET_NAMESPACE)) + .await? + && let Some(data) = existing.data + && let Some(seed_bytes) = data.get("seed") + { + let seed = String::from_utf8(seed_bytes.0.clone())?; + return Ok(seed.trim().to_string()); + } + + let seed = KeyPair::new_account() + .seed() + .map_err(|e| anyhow::anyhow!("nkey seed: {e}"))?; + let mut data = BTreeMap::new(); + data.insert("seed".to_string(), ByteString(seed.as_bytes().to_vec())); + let secret = Secret { + metadata: ObjectMeta { + name: Some(ISSUER_SEED_SECRET.to_string()), + namespace: Some(FLEET_NAMESPACE.to_string()), + ..Default::default() + }, + data: Some(data), + type_: Some("Opaque".to_string()), + ..Default::default() + }; + k8s.create(&secret, Some(FLEET_NAMESPACE)).await.ok(); + Ok(seed) +} + +// ---- NATS values ----------------------------------------------------------- + +/// Render NATS Helm values for an OKD-flavored deployment with WSS +/// ingress + auth callout + JetStream. +/// +/// **Why WSS rather than plain NATS-on-TLS:** OKD's default ingress +/// controller (HAProxy) is HTTP-aware and edge-terminates TLS. NATS +/// over WebSocket goes through that ingress unchanged; native NATS +/// TCP would require a TCP loadbalancer service or a passthrough +/// Route, both of which are extra infra the customer's cluster may +/// not have. WSS is also the default async-nats client transport on +/// `wss://...` URLs — no special agent code needed. +pub fn render_nats_values( + domain: &FleetDomainConfig, + issuer_pubkey: &str, + nats_auth_pass: &str, + nats_system_pass: &str, +) -> String { + let auth_callout = render_auth_callout_block(issuer_pubkey, NATS_AUTH_USER, NATS_ACCOUNT); + let auth_callout_indented = auth_callout + .lines() + .enumerate() + .map(|(i, l)| { + if i == 0 { + l.to_string() + } else { + format!(" {l}") + } + }) + .collect::>() + .join("\n"); + format!( + r#"fullnameOverride: {nats_release} +config: + cluster: + enabled: false + jetstream: + enabled: true + fileStorage: + enabled: true + size: 5Gi + websocket: + enabled: true + port: 8443 + ingress: + enabled: true + className: openshift-default + pathType: Prefix + hosts: + - {nats_wss_host} + annotations: + # OKD HAProxy edge-terminates TLS — the chart's default Route + # generation needs `route.openshift.io/termination: edge` so + # the Route's TLS block is "edge", matching the cluster's wildcard + # cert behavior. Switch to `reencrypt` if you need TLS all the + # way to the NATS pod. + route.openshift.io/termination: edge + haproxy.router.openshift.io/timeout: "1h" + merge: + {auth_callout_indented} + accounts: + {nats_account}: + jetstream: enabled + users: + - user: "{auth_user}" + password: "{auth_pass}" + SYS: + users: + - user: "{sys_user}" + password: "{sys_pass}" + system_account: SYS +service: + ports: + nats: + enabled: true +"#, + nats_release = NATS_RELEASE, + nats_wss_host = domain.nats_wss_host(), + nats_account = NATS_ACCOUNT, + auth_user = NATS_AUTH_USER, + auth_pass = nats_auth_pass, + sys_user = NATS_SYSTEM_USER, + sys_pass = nats_system_pass, + ) +} + +// ---- readiness ------------------------------------------------------------- + +async fn wait_for_zitadel_ready(domain: &FleetDomainConfig) -> Result<()> { + let issuer = domain.zitadel_issuer_url(); + let well_known = format!("{issuer}/.well-known/openid-configuration"); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build()?; + for attempt in 1..=180 { + match client.get(&well_known).send().await { + Ok(r) if r.status().is_success() => return Ok(()), + Ok(r) if attempt % 30 == 0 => { + info!("Zitadel HTTPS {} (attempt {attempt}/180)", r.status()); + } + Err(e) if attempt % 30 == 0 => { + info!("Zitadel unreachable: {e} (attempt {attempt}/180)"); + } + _ => {} + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + anyhow::bail!("timed out waiting for Zitadel at {well_known}") +} + +// ---- helpful printout ------------------------------------------------------ + +impl StagingHandles { + /// Print the operator's "what to do next" panel after a successful + /// staging deploy. Pasted at the end of the binary's run. + pub fn print_next_steps(&self) { + let zitadel = self.domain.zitadel_issuer_url(); + let nats = self.domain.nats_wss_url(); + println!(); + println!("============================================================"); + println!(" STAGING DEPLOY COMPLETE"); + println!("============================================================"); + println!(" Base domain: {}", self.domain.base_domain); + println!(" Zitadel: {zitadel}"); + println!(" NATS (WSS): {nats}"); + println!(" Project ID: {}", self.project_id); + println!(" Callout image: {}", self.callout_image); + println!(" Issuer pubkey: {}", self.issuer_pubkey); + if let Some(cid) = &self.cli_client_id { + println!(" CLI client_id: {cid}"); + println!(); + println!(" CLI SSO login (developer-side):"); + println!(); + println!(" cargo run -p example-fleet-sso-login -- \\"); + println!(" --base-domain {} \\", self.domain.base_domain); + println!(" --client-id {cid}"); + } + println!(); + println!(" Onboard a Pi:"); + println!(); + println!(" PAT=$(kubectl -n zitadel get secret iam-admin-pat \\"); + println!(" -o jsonpath='{{.data.pat}}' | base64 -d)"); + println!(); + println!(" cargo run -p example-fleet-rpi-setup -- \\"); + println!(" --pi-host \\"); + println!(" --bootstrap-token \"$PAT\" \\"); + println!(" --zitadel-issuer-url {zitadel} \\"); + println!(" --zitadel-project-id {} \\", self.project_id); + println!(" --nats-url {nats} \\"); + println!(" --agent-binary "); + println!(); + println!("============================================================"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn domain_config_derives_hostnames() { + let d = FleetDomainConfig::new("customer1.nationtech.io"); + assert_eq!(d.zitadel_host(), "zitadel.customer1.nationtech.io"); + assert_eq!(d.nats_wss_host(), "nats.customer1.nationtech.io"); + assert_eq!( + d.zitadel_issuer_url(), + "https://zitadel.customer1.nationtech.io" + ); + assert_eq!(d.nats_wss_url(), "wss://nats.customer1.nationtech.io/"); + } + + #[test] + fn nats_values_render_includes_wss_ingress_and_auth_callout() { + let d = FleetDomainConfig::new("acme.io"); + let yaml = render_nats_values(&d, "ABCDEF", "auth-pass", "sys-pass"); + // WSS plumbing. + assert!(yaml.contains("websocket:")); + assert!(yaml.contains("port: 8443")); + assert!(yaml.contains("nats.acme.io")); + // OKD edge-TLS annotations. + assert!(yaml.contains("openshift-default")); + assert!(yaml.contains("route.openshift.io/termination: edge")); + // Auth callout wired through with the issuer pubkey. + assert!(yaml.contains("auth_callout")); + assert!(yaml.contains("issuer: ABCDEF")); + assert!(yaml.contains("auth_users: [ auth ]")); + assert!(yaml.contains("system_account: SYS")); + // Account user. + assert!(yaml.contains("password: \"auth-pass\"")); + } + + #[test] + fn nats_values_inline_account_block_under_merge() { + // Prevent regressions where the auth_callout block leaks + // outside the `merge:` indentation level — chart expects it + // under config.merge. + let d = FleetDomainConfig::new("x.io"); + let yaml = render_nats_values(&d, "K", "p", "s"); + let idx_merge = yaml.find("\n merge:\n").expect("merge block present"); + let idx_callout = yaml.find("auth_callout:").expect("auth_callout present"); + assert!(idx_callout > idx_merge, "auth_callout must follow merge:"); + } +} diff --git a/examples/fleet_staging_deploy/src/main.rs b/examples/fleet_staging_deploy/src/main.rs new file mode 100644 index 00000000..bd9e7b2f --- /dev/null +++ b/examples/fleet_staging_deploy/src/main.rs @@ -0,0 +1,71 @@ +//! `cargo run -p example-fleet-staging-deploy -- --base-domain customer1.nationtech.io ...` +//! +//! Operator-side, run-once-per-customer-instance harness. Brings up +//! the central fleet platform services (Zitadel + NATS + auth callout) +//! against an OKD/K8s cluster pointed to by `KUBECONFIG`. Prints the +//! exact follow-up command the operator runs against a Pi to onboard +//! the first device. +//! +//! See `src/lib.rs` for the architectural notes. + +use anyhow::{Context, Result}; +use clap::Parser; +use example_fleet_staging_deploy::{FleetDomainConfig, StagingDeployOpts, bring_up_staging}; + +#[derive(Parser, Debug)] +#[command( + name = "fleet-staging-deploy", + about = "Deploy Zitadel + NATS + auth callout onto an OKD cluster" +)] +struct Cli { + /// Base DNS domain. All cluster-visible services derive from this: + /// `zitadel.`, `nats.`. The customer's wildcard cert / + /// CoreDNS / DNS provider must already point this at the cluster. + #[arg(long, env = "FLEET_BASE_DOMAIN")] + base_domain: String, + /// kubeconfig context to deploy against. Defaults to the + /// kubeconfig's current-context. Set this when your kubeconfig + /// has multiple contexts and you don't want to rely on the + /// global current. + #[arg(long, env = "FLEET_KUBE_CONTEXT")] + kube_context: Option, + /// Container image reference for the harmony-nats-callout binary. + /// The cluster pulls this; operator must have pushed it before + /// running the deploy. Defaults to a quay.io path that the + /// customer should override per their registry. + #[arg( + long, + env = "FLEET_CALLOUT_IMAGE", + default_value = "quay.io/nationtech/harmony-nats-callout:demo" + )] + callout_image: String, + /// Password for the NATS service-account user the callout uses on + /// its own NATS connection. Stored in a K8s secret + listed in + /// the chart's `accounts.DEVICES.users` (which bypass callout — + /// otherwise the callout would deadlock authenticating itself). + #[arg(long, env = "FLEET_NATS_AUTH_PASS")] + nats_auth_pass: String, + /// Password for the NATS SYS account (used for nats-box debugging + /// inside the cluster). + #[arg(long, env = "FLEET_NATS_SYSTEM_PASS")] + nats_system_pass: String, +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + let domain = FleetDomainConfig::new(cli.base_domain); + + let handles = bring_up_staging(StagingDeployOpts { + domain, + kubeconfig_context: cli.kube_context, + callout_image: cli.callout_image, + nats_auth_pass: cli.nats_auth_pass, + nats_system_pass: cli.nats_system_pass, + }) + .await + .context("staging deploy")?; + + handles.print_next_steps(); + Ok(()) +} diff --git a/examples/fleet_vm_setup/src/main.rs b/examples/fleet_vm_setup/src/main.rs index 415f822f..f3fe71e2 100644 --- a/examples/fleet_vm_setup/src/main.rs +++ b/examples/fleet_vm_setup/src/main.rs @@ -211,9 +211,14 @@ async fn main() -> Result<()> { device_id: device_id.clone(), labels, nats_urls: vec![cli.nats_url.clone()], - nats_user: cli.nats_user.clone(), - nats_pass: cli.nats_pass.clone(), + // VM smoke harness keeps shared-creds for v0; the customer- + // facing Pi flow uses Zitadel JWT (see fleet_rpi_setup). + auth: harmony::modules::fleet::FleetDeviceAuth::TomlShared { + nats_user: cli.nats_user.clone(), + nats_pass: cli.nats_pass.clone(), + }, agent_binary_path: agent_binary, + hosts_entries: vec![], }); run_setup_score(&setup_score, &linux_topology).await?; diff --git a/examples/harmony_apply_deployment/src/main.rs b/examples/harmony_apply_deployment/src/main.rs index 904e74be..976cf599 100644 --- a/examples/harmony_apply_deployment/src/main.rs +++ b/examples/harmony_apply_deployment/src/main.rs @@ -39,6 +39,7 @@ use anyhow::{Context, Result}; use clap::Parser; use harmony::modules::podman::{PodmanService, PodmanV0Score}; +use harmony::topology::{RestartPolicy, VolumeMount}; use harmony_fleet_operator::crd::{ Deployment, DeploymentSpec, Rollout, RolloutStrategy, ScorePayload, }; @@ -76,6 +77,16 @@ struct Cli { /// `host:container` port mapping exposed on the device. #[arg(long, default_value = "8080:80")] port: String, + /// Repeatable `KEY=VALUE` env var injected into the container. + #[arg(long = "env", value_name = "KEY=VALUE")] + envs: Vec, + /// Repeatable bind-mount in `host_path:container_path[:ro]` form. + /// Append `:ro` for read-only. + #[arg(long = "volume", value_name = "HOST:CONTAINER[:ro]")] + volumes: Vec, + /// Container restart policy. + #[arg(long, value_enum, default_value_t = CliRestart::UnlessStopped)] + restart: CliRestart, /// Delete the Deployment CR instead of applying it. #[arg(long)] delete: bool, @@ -132,12 +143,69 @@ async fn main() -> Result<()> { Ok(()) } +/// Mirrors `harmony::topology::RestartPolicy` so we can keep the CLI +/// schema stable even if the underlying enum gains variants. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +enum CliRestart { + No, + UnlessStopped, + OnFailure, + Always, +} + +impl From for RestartPolicy { + fn from(c: CliRestart) -> Self { + match c { + CliRestart::No => RestartPolicy::No, + CliRestart::UnlessStopped => RestartPolicy::UnlessStopped, + CliRestart::OnFailure => RestartPolicy::OnFailure, + CliRestart::Always => RestartPolicy::Always, + } + } +} + +fn parse_env(s: &str) -> Result<(String, String)> { + let (k, v) = s + .split_once('=') + .ok_or_else(|| anyhow::anyhow!("--env expects KEY=VALUE, got {s:?}"))?; + Ok((k.to_string(), v.to_string())) +} + +fn parse_volume(s: &str) -> Result { + let parts: Vec<&str> = s.split(':').collect(); + let (host, cont, ro) = match parts.as_slice() { + [host, cont] => (host, cont, false), + [host, cont, mode] if *mode == "ro" => (host, cont, true), + [host, cont, mode] if *mode == "rw" => (host, cont, false), + _ => anyhow::bail!("--volume expects HOST:CONTAINER[:ro|rw], got {s:?}"), + }; + Ok(VolumeMount { + host_path: host.to_string(), + container_path: cont.to_string(), + read_only: ro, + }) +} + fn build_cr(cli: &Cli) -> Deployment { + let env: Vec<(String, String)> = cli + .envs + .iter() + .map(|s| parse_env(s).expect("--env validated")) + .collect(); + let volumes: Vec = cli + .volumes + .iter() + .map(|s| parse_volume(s).expect("--volume validated")) + .collect(); + let score = PodmanV0Score { services: vec![PodmanService { name: cli.name.clone(), image: cli.image.clone(), ports: vec![cli.port.clone()], + env, + volumes, + restart_policy: cli.restart.into(), }], }; diff --git a/examples/harmony_host_discovery/Cargo.toml b/examples/harmony_host_discovery/Cargo.toml new file mode 100644 index 00000000..c043f434 --- /dev/null +++ b/examples/harmony_host_discovery/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "harmony_host_discovery" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true + +[dependencies] +harmony = { path = "../../harmony" } +harmony_cli = { path = "../../harmony_cli" } +harmony_macros = { path = "../../harmony_macros" } +harmony_types = { path = "../../harmony_types" } +tokio.workspace = true +url.workspace = true +cidr.workspace = true diff --git a/examples/harmony_host_discovery/env.sh b/examples/harmony_host_discovery/env.sh new file mode 100644 index 00000000..0b9da4f6 --- /dev/null +++ b/examples/harmony_host_discovery/env.sh @@ -0,0 +1,4 @@ +export HARMONY_SECRET_NAMESPACE=host-discovery +export HARMONY_SECRET_STORE=file +export HARMONY_DATABASE_URL=sqlite://harmony_host_discovery.sqlite +export RUST_LOG=harmony=debug diff --git a/examples/harmony_host_discovery/src/main.rs b/examples/harmony_host_discovery/src/main.rs new file mode 100644 index 00000000..98140d03 --- /dev/null +++ b/examples/harmony_host_discovery/src/main.rs @@ -0,0 +1,27 @@ +use harmony::{ + inventory::{HostRole, Inventory}, + modules::inventory::{DiscoverHostForRoleScore, HarmonyDiscoveryStrategy}, + topology::LocalhostTopology, +}; +use harmony_macros::cidrv4; + +#[tokio::main] +async fn main() { + let discover_one_host = DiscoverHostForRoleScore { + role: HostRole::Worker, + number_desired_hosts: 1, + discovery_strategy: HarmonyDiscoveryStrategy::SUBNET { + cidr: cidrv4!("192.168.40.0/24"), + port: 25000, + }, + }; + + harmony_cli::run( + Inventory::autoload(), + LocalhostTopology::new(), + vec![Box::new(discover_one_host)], + None, + ) + .await + .unwrap(); +} diff --git a/examples/harmony_sso/src/main.rs b/examples/harmony_sso/src/main.rs index ba3c8ef4..8232286d 100644 --- a/examples/harmony_sso/src/main.rs +++ b/examples/harmony_sso/src/main.rs @@ -118,6 +118,7 @@ async fn deploy_zitadel(k3d: &K3d) -> anyhow::Result<()> { host: ZITADEL_HOST.to_string(), zitadel_version: "v4.12.1".to_string(), external_secure: false, + external_port: None, }; let topology = create_topology(k3d); @@ -301,6 +302,8 @@ async fn main() -> anyhow::Result<()> { app_name: APP_NAME.to_string(), app_type: ZitadelAppType::DeviceCode, }], + api_apps: vec![], + roles: vec![], machine_users: vec![], } .interpret(&Inventory::autoload(), &topology) diff --git a/examples/okd_ceph_alerts/Cargo.toml b/examples/okd_ceph_alerts/Cargo.toml new file mode 100644 index 00000000..7301242d --- /dev/null +++ b/examples/okd_ceph_alerts/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "example-okd-ceph-alerts" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true +publish = false + +[dependencies] +harmony = { path = "../../harmony" } +harmony_cli = { path = "../../harmony_cli" } +harmony_types = { path = "../../harmony_types" } +tokio = { workspace = true } +log = { workspace = true } diff --git a/examples/okd_ceph_alerts/env.sh b/examples/okd_ceph_alerts/env.sh new file mode 100644 index 00000000..08072655 --- /dev/null +++ b/examples/okd_ceph_alerts/env.sh @@ -0,0 +1,4 @@ +export HARMONY_SECRET_NAMESPACE=okd_ceph_alerts_example +export HARMONY_SECRET_STORE=file +export HARMONY_DATABASE_URL=sqlite://harmony_okd_ceph_alerts_example.sqlite +export RUST_LOG=harmony=debug diff --git a/examples/okd_ceph_alerts/src/main.rs b/examples/okd_ceph_alerts/src/main.rs new file mode 100644 index 00000000..33bfa1ca --- /dev/null +++ b/examples/okd_ceph_alerts/src/main.rs @@ -0,0 +1,28 @@ +use harmony::{ + inventory::Inventory, + modules::monitoring::{ + ceph_alerts::ceph_alert_rule_groups, okd::cluster_alert_rules::OpenshiftPrometheusRuleScore, + }, + topology::K8sAnywhereTopology, +}; + +#[tokio::main] +async fn main() { + harmony_cli::cli_logger::init(); + + let ceph_rules = OpenshiftPrometheusRuleScore { + namespace: "rook-ceph".to_string(), + name: "ceph-alerts".to_string(), + rule_groups: ceph_alert_rule_groups(), + labels: None, + }; + + harmony_cli::run( + Inventory::autoload(), + K8sAnywhereTopology::from_env(), + vec![Box::new(ceph_rules)], + None, + ) + .await + .unwrap(); +} diff --git a/examples/zitadel/src/main.rs b/examples/zitadel/src/main.rs index 73e3d2ab..94b5c45f 100644 --- a/examples/zitadel/src/main.rs +++ b/examples/zitadel/src/main.rs @@ -8,6 +8,7 @@ async fn main() { host: "sso.sto1.nationtech.io".to_string(), zitadel_version: "v4.12.1".to_string(), external_secure: true, + external_port: None, }; harmony_cli::run( diff --git a/fleet/harmony-fleet-agent/Cargo.toml b/fleet/harmony-fleet-agent/Cargo.toml index 8cd98369..e7838e2a 100644 --- a/fleet/harmony-fleet-agent/Cargo.toml +++ b/fleet/harmony-fleet-agent/Cargo.toml @@ -5,9 +5,11 @@ edition = "2024" rust-version = "1.85" [dependencies] +harmony-fleet-auth = { path = "../harmony-fleet-auth" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } harmony = { path = "../../harmony", default-features = false, features = ["podman"] } async-nats = { workspace = true } +async-trait = { workspace = true } chrono = { workspace = true } futures-util = { workspace = true } serde = { workspace = true } @@ -17,4 +19,4 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } anyhow = { workspace = true } clap = { workspace = true } -toml = { workspace = true } \ No newline at end of file +toml = { workspace = true } diff --git a/fleet/harmony-fleet-agent/src/config.rs b/fleet/harmony-fleet-agent/src/config.rs index 19b2a99a..6faa3750 100644 --- a/fleet/harmony-fleet-agent/src/config.rs +++ b/fleet/harmony-fleet-agent/src/config.rs @@ -3,6 +3,11 @@ use serde::Deserialize; use std::collections::BTreeMap; use std::path::Path; +// Re-export the shared credential types so existing call sites keep +// working with `crate::config::CredentialsSection`. The struct itself +// lives in `harmony_fleet_auth` and is shared with the operator. +pub use harmony_fleet_auth::CredentialsSection; + #[derive(Debug, Clone, Deserialize)] pub struct AgentConfig { pub agent: AgentSection, @@ -30,49 +35,6 @@ pub struct NatsSection { pub urls: Vec, } -#[derive(Debug, Clone, Deserialize)] -pub struct CredentialsSection { - #[serde(rename = "type")] - pub source_type: String, - pub nats_user: Option, - pub nats_pass: Option, -} - -pub trait CredentialSource: Send + Sync { - fn nats_credentials(&self) -> anyhow::Result<(String, String)>; -} - -pub struct TomlFileCredentialSource<'a> { - config: &'a AgentConfig, -} - -impl<'a> TomlFileCredentialSource<'a> { - pub fn new(config: &'a AgentConfig) -> Self { - Self { config } - } -} - -impl CredentialSource for TomlFileCredentialSource<'_> { - fn nats_credentials(&self) -> anyhow::Result<(String, String)> { - let creds = &self.config.credentials; - if creds.source_type != "toml-shared" { - anyhow::bail!( - "unsupported credentials.type '{}' (v0 only supports 'toml-shared')", - creds.source_type - ); - } - let user = creds - .nats_user - .as_deref() - .ok_or_else(|| anyhow::anyhow!("missing nats_user in credentials"))?; - let pass = creds - .nats_pass - .as_deref() - .ok_or_else(|| anyhow::anyhow!("missing nats_pass in credentials"))?; - Ok((user.to_string(), pass.to_string())) - } -} - pub fn load_config(path: &Path) -> anyhow::Result { let content = std::fs::read_to_string(path)?; let config: AgentConfig = toml::from_str(&content)?; @@ -84,7 +46,7 @@ mod tests { use super::*; #[test] - fn parses_config_with_labels_section() { + fn parses_toml_shared_credentials() { let raw = r#" [agent] device_id = "pi-42" @@ -103,7 +65,16 @@ arch = "aarch64" "#; let cfg: AgentConfig = toml::from_str(raw).expect("valid config"); assert_eq!(cfg.labels.get("group"), Some(&"site-a".to_string())); - assert_eq!(cfg.labels.get("arch"), Some(&"aarch64".to_string())); + match &cfg.credentials { + CredentialsSection::TomlShared { + nats_user, + nats_pass, + } => { + assert_eq!(nats_user, "u"); + assert_eq!(nats_pass, "p"); + } + _ => panic!("expected TomlShared"), + } } #[test] diff --git a/fleet/harmony-fleet-agent/src/fleet_publisher.rs b/fleet/harmony-fleet-agent/src/fleet_publisher.rs index 0c334d6e..f0e82d81 100644 --- a/fleet/harmony-fleet-agent/src/fleet_publisher.rs +++ b/fleet/harmony-fleet-agent/src/fleet_publisher.rs @@ -17,6 +17,11 @@ use std::collections::BTreeMap; pub struct FleetPublisher { device_id: Id, + /// Raw NATS client kept around so we can publish on direct + /// (non-JetStream) subjects like `device-state.` for + /// live observers — the KV writes are storage-and-watch, the + /// direct subject is fan-out. + client: async_nats::Client, info_bucket: kv::Store, state_bucket: kv::Store, heartbeat_bucket: kv::Store, @@ -26,11 +31,13 @@ impl FleetPublisher { /// Open every bucket the agent needs, creating those that don't /// exist yet. Idempotent with operator-side creation. pub async fn connect(client: async_nats::Client, device_id: Id) -> anyhow::Result { - let jetstream = jetstream::new(client); + let jetstream = jetstream::new(client.clone()); let info_bucket = jetstream .create_key_value(kv::Config { bucket: BUCKET_DEVICE_INFO.to_string(), + // If this is as I think, it would be useful to keep a history of the last 10 device + // info, with a timestamp history: 1, ..Default::default() }) @@ -38,6 +45,8 @@ impl FleetPublisher { let state_bucket = jetstream .create_key_value(kv::Config { bucket: BUCKET_DEVICE_STATE.to_string(), + // If this is as I think, it would be useful to keep a history of the last 10 states + // a device had, with a timestamp history: 1, ..Default::default() }) @@ -52,6 +61,7 @@ impl FleetPublisher { Ok(Self { device_id, + client, info_bucket, state_bucket, heartbeat_bucket, @@ -102,18 +112,45 @@ impl FleetPublisher { /// Persist the authoritative current phase for a `(device, /// deployment)` pair. The operator's watch on the `device-state` /// bucket picks up this put and updates CR status counters. + /// Also fans out the same payload on `device-state.` + /// for live observers that don't want to consume the KV stream. pub async fn write_deployment_state(&self, state: &DeploymentState) { let key = device_state_key(&self.device_id.to_string(), &state.deployment); match serde_json::to_vec(state) { Ok(payload) => { - if let Err(e) = self.state_bucket.put(&key, payload.into()).await { + if let Err(e) = self.state_bucket.put(&key, payload.clone().into()).await { tracing::warn!(%key, error = %e, "write_deployment_state: kv put failed"); } + self.publish_direct_state(payload).await; } Err(e) => tracing::warn!(error = %e, "write_deployment_state: serialize failed"), } } + /// Emit a tiny presence pulse on `device-state.` so live + /// observers (admin tooling, dashboards) see the device is alive + /// without subscribing to JetStream. Called from the heartbeat + /// loop alongside the KV heartbeat write — same cadence, two + /// transports. + pub async fn publish_state_pulse(&self) { + let pulse = serde_json::json!({ + "device_id": self.device_id.to_string(), + "kind": "heartbeat", + "at": chrono::Utc::now(), + }); + match serde_json::to_vec(&pulse) { + Ok(payload) => self.publish_direct_state(payload).await, + Err(e) => tracing::warn!(error = %e, "publish_state_pulse: serialize failed"), + } + } + + async fn publish_direct_state(&self, payload: Vec) { + let subject = format!("device-state.{}", self.device_id); + if let Err(e) = self.client.publish(subject.clone(), payload.into()).await { + tracing::debug!(%subject, error = %e, "publish_direct_state: publish failed"); + } + } + /// Delete the authoritative current-phase entry, e.g. when the /// Deployment CR is removed and the agent has torn down the /// container. diff --git a/fleet/harmony-fleet-agent/src/main.rs b/fleet/harmony-fleet-agent/src/main.rs index 3b388349..aece8fab 100644 --- a/fleet/harmony-fleet-agent/src/main.rs +++ b/fleet/harmony-fleet-agent/src/main.rs @@ -5,9 +5,15 @@ mod reconciler; use std::sync::Arc; use std::time::Duration; -use anyhow::{Context, Result}; +use anyhow::{Context, Error, Result}; use clap::Parser; -use config::{AgentConfig, CredentialSource, TomlFileCredentialSource}; +use config::AgentConfig; +use harmony_fleet_auth::{ + CredentialSource, connect_options_with_credentials, credential_source_from_config, +}; +// Type alias to keep function signatures readable. The auth callback +// captures one `Arc` and clones it per invocation. +type Creds = Arc; use futures_util::StreamExt; use harmony_reconciler_contracts::{BUCKET_DESIRED_STATE, Id, InventorySnapshot}; @@ -28,15 +34,41 @@ struct Cli { #[arg( long, env = "FLEET_AGENT_CONFIG", + // FIXME this should be a constant from a config, not just hardcoded here as we need the + // installation scripts and other bits to know about this file location. default_value = "/etc/fleet-agent/config.toml" )] config: std::path::PathBuf, } -async fn connect_nats(cfg: &AgentConfig) -> Result { - let (user, pass) = TomlFileCredentialSource::new(cfg).nats_credentials()?; - let client = async_nats::ConnectOptions::with_user_and_password(user, pass) +async fn connect_nats(cfg: &AgentConfig, creds: Creds) -> Result { + let urls = &cfg.nats.urls; + tracing::info!(device_id = %cfg.agent.device_id, "connecting to NATS {urls:?}"); + // The auth callback is invoked on every (re)connect, so a fresh + // Zitadel access token is minted automatically when the cached one + // is near-expiry — that's how we hold the "never lose connectivity" + // guarantee even across token rollovers and NATS pod restarts. + let client = connect_options_with_credentials(creds) .ping_interval(Duration::from_secs(10)) + // Surface async-nats's connection lifecycle in our logs. This + // is load-bearing for ops: a device that quietly disconnects + // is exactly the failure mode we promise won't happen, and + // operators need to see the reconnect attempts to debug. + .event_callback(|event| async move { + use async_nats::Event; + match event { + Event::Connected => tracing::info!("NATS connected"), + Event::Disconnected => tracing::warn!("NATS disconnected, will reconnect"), + Event::LameDuckMode => tracing::warn!("NATS server entered lame-duck mode"), + Event::SlowConsumer(sid) => { + tracing::warn!(sid = %sid, "NATS slow consumer") + } + Event::ServerError(e) => tracing::error!(error = %e, "NATS server error"), + Event::ClientError(e) => tracing::error!(error = %e, "NATS client error"), + Event::Closed => tracing::error!("NATS connection closed"), + other => tracing::debug!(?other, "NATS event"), + } + }) .connect(cfg.nats.urls.as_slice()) .await?; tracing::info!(urls = ?cfg.nats.urls, "connected to NATS"); @@ -68,6 +100,9 @@ async fn watch_desired_state( continue; } }; + + tracing::debug!(key = %entry.key, "bucket watch new value {entry:?}"); + match entry.operation { async_nats::jetstream::kv::Operation::Put => { if let Err(e) = reconciler.apply(&entry.key, &entry.value).await { @@ -86,20 +121,27 @@ async fn watch_desired_state( } /// Tiny liveness-only loop: push a `HeartbeatPayload` into the -/// `device-heartbeat` bucket every N seconds. Stays separate from -/// per-deployment state writes so routine pings don't churn the -/// device-state bucket or its watch subscribers. +/// `device-heartbeat` bucket every N seconds, and fan out the same +/// pulse on `device-state.` for live (non-JetStream) +/// observers. Stays separate from per-deployment state writes so +/// routine pings don't churn the device-state bucket or its watch +/// subscribers — but the direct-subject pulse uses ordinary core +/// NATS pub/sub and doesn't accumulate state anywhere. async fn publish_heartbeat_loop(fleet: Arc) { let mut interval = tokio::time::interval(Duration::from_secs(30)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { interval.tick().await; fleet.publish_heartbeat().await; + fleet.publish_state_pulse().await; } } /// Build a one-shot inventory snapshot at agent startup. Cheap, /// published alongside every heartbeat until the agent restarts. +/// NOTE: I don't see why this is *published* with every heartbeat, it feels like noise. +/// It shoulf be published on heartbeat only when something changed. It is ok to *check* the state +/// on heartbeat but not always send it over the wire fn local_inventory(inventory: &Inventory) -> InventorySnapshot { InventorySnapshot { hostname: inventory.location.name.clone(), @@ -156,7 +198,14 @@ async fn main() -> Result<()> { tracing::info!(hostname = %inventory.location.name, "inventory loaded"); let inventory_snapshot = local_inventory(&inventory); - let client = connect_nats(&cfg).await?; + let creds = credential_source_from_config(&cfg.credentials) + .context("building NATS credential source")?; + + let client = connect_nats(&cfg, creds).await.map_err(|e| { + let msg = format!("Nats connection FAILED : {e}"); + tracing::error!(msg); + Error::msg(msg) + })?; // Publish surface. Opens the three KV buckets (idempotent // creates). Must be live before the reconciler starts so diff --git a/fleet/harmony-fleet-agent/src/reconciler.rs b/fleet/harmony-fleet-agent/src/reconciler.rs index 619d9bf0..3ba5b583 100644 --- a/fleet/harmony-fleet-agent/src/reconciler.rs +++ b/fleet/harmony-fleet-agent/src/reconciler.rs @@ -33,7 +33,10 @@ pub struct Reconciler { state: Mutex>, /// Current phase per deployment, used to decide whether a new /// write to the `device-state` KV is needed. - phases: Mutex>, + /// + /// NOTE : this feels dangerous, conflict on deployment name could be a problem + /// We must explore this and clarify it in the design and decide if it is a constraint + deployments: Mutex>, /// Publish surface. Optional so unit tests without a live NATS /// client still work; always populated in the real agent runtime. fleet: Option>, @@ -51,7 +54,7 @@ impl Reconciler { topology, inventory, state: Mutex::new(HashMap::new()), - phases: Mutex::new(HashMap::new()), + deployments: Mutex::new(HashMap::new()), fleet, } } @@ -67,7 +70,9 @@ impl Reconciler { last_error: Option, ) { { - let mut phases = self.phases.lock().await; + let mut phases = self.deployments.lock().await; + // performance nitpick : we don't need a write lock here, we could check before acquiring the write + // lock if phases.get(deployment).copied() == Some(phase) { return; } @@ -91,7 +96,7 @@ impl Reconciler { /// a no-op in memory and a harmless tombstone write on the wire. async fn drop_phase(&self, deployment: &DeploymentName) { let was_known = { - let mut phases = self.phases.lock().await; + let mut phases = self.deployments.lock().await; phases.remove(deployment).is_some() }; if !was_known { @@ -301,7 +306,7 @@ mod tests { async fn apply_phase_records_new_phase() { let r = reconciler(); r.apply_phase(&dn("hello"), Phase::Running, None).await; - let phases = r.phases.lock().await; + let phases = r.deployments.lock().await; assert_eq!(phases.get(&dn("hello")), Some(&Phase::Running)); } @@ -310,7 +315,7 @@ mod tests { let r = reconciler(); r.apply_phase(&dn("hello"), Phase::Running, None).await; r.apply_phase(&dn("hello"), Phase::Running, None).await; - let phases = r.phases.lock().await; + let phases = r.deployments.lock().await; assert_eq!(phases.len(), 1); } @@ -321,7 +326,7 @@ mod tests { r.apply_phase(&dn("hello"), Phase::Running, None).await; r.apply_phase(&dn("hello"), Phase::Failed, Some("oom".to_string())) .await; - let phases = r.phases.lock().await; + let phases = r.deployments.lock().await; assert_eq!(phases.get(&dn("hello")), Some(&Phase::Failed)); } @@ -330,7 +335,7 @@ mod tests { let r = reconciler(); r.apply_phase(&dn("hello"), Phase::Running, None).await; r.drop_phase(&dn("hello")).await; - let phases = r.phases.lock().await; + let phases = r.deployments.lock().await; assert!(!phases.contains_key(&dn("hello"))); } @@ -338,7 +343,7 @@ mod tests { async fn drop_phase_on_unknown_deployment_is_noop() { let r = reconciler(); r.drop_phase(&dn("never-existed")).await; - let phases = r.phases.lock().await; + let phases = r.deployments.lock().await; assert!(phases.is_empty()); } } diff --git a/fleet/harmony-fleet-auth/Cargo.toml b/fleet/harmony-fleet-auth/Cargo.toml new file mode 100644 index 00000000..32e802e9 --- /dev/null +++ b/fleet/harmony-fleet-auth/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "harmony-fleet-auth" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true +description = "Shared NATS credential plumbing for the fleet agent + operator (Zitadel JWT-bearer + dev-only username/password)" + +[lib] +path = "src/lib.rs" + +[dependencies] +async-nats = { workspace = true } +anyhow = { workspace = true } +chrono = { workspace = true } +jsonwebtoken = "9" +reqwest = { workspace = true } +serde = { workspace = true, features = ["derive"] } +tokio = { workspace = true, features = ["sync"] } +tracing = { workspace = true } +serde_json = { workspace = true } + +[dev-dependencies] +toml = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/fleet/harmony-fleet-auth/src/config.rs b/fleet/harmony-fleet-auth/src/config.rs new file mode 100644 index 00000000..aaf1bd18 --- /dev/null +++ b/fleet/harmony-fleet-auth/src/config.rs @@ -0,0 +1,133 @@ +use serde::Deserialize; +use std::path::PathBuf; + +/// Externally-tagged credential definition shared between the fleet +/// agent and the fleet operator. The `type` field selects the variant; +/// each variant's other fields are flatly mixed into the +/// `[credentials]` TOML table for human-friendly editing. +/// +/// **Why one struct for both processes**: the agent reads this from +/// `/etc/fleet-agent/config.toml`; the operator reads it from a single +/// env var (`FLEET_OPERATOR_CREDENTIALS_TOML`) whose value is a TOML +/// snippet shaped exactly like the `[credentials]` table. Identical +/// deserialization, identical downstream code path. The only thing +/// that differs is the byte source. +/// +/// Adding a new mode is additive — emit `type = ""` from the +/// installer side, decode here, instantiate the matching +/// `CredentialSource`. +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "type", rename_all = "kebab-case")] +pub enum CredentialsSection { + /// Shared username + password baked into the agent config. Only + /// suitable for v0/development scenarios where every device shares + /// a single NATS account user. Not used in production. + TomlShared { + nats_user: String, + nats_pass: String, + }, + /// Per-device Zitadel machine-user JWT-bearer (RFC 7523) flow. The + /// keyfile at `key_path` is the only durable secret on the device — + /// the access token is short-lived and re-minted before expiry by + /// the auth callback registered on each NATS (re)connect. + ZitadelJwt { + /// Path to the machine-user JSON key file Zitadel emits for + /// `KEY_TYPE_JSON`. Defaults to + /// `/etc/fleet-agent/zitadel-key.json` for the agent; the + /// operator's deploy mounts the keyfile at a path it sets + /// explicitly in the env-var TOML. + #[serde(default = "default_zitadel_key_path")] + key_path: PathBuf, + /// Externally-visible Zitadel issuer URL — must match Zitadel's + /// emitted `iss` claim exactly (including port if non-default). + oidc_issuer_url: String, + /// `aud` value for token-bearer requests. Typically the Zitadel + /// project ID (the auth callout side validates against this). + audience: String, + /// Whether the HTTP client accepts invalid TLS certs. Local-dev + /// escape hatch for self-signed staging Zitadels. + #[serde(default)] + danger_accept_invalid_certs: bool, + }, +} + +fn default_zitadel_key_path() -> PathBuf { + PathBuf::from("/etc/fleet-agent/zitadel-key.json") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(raw: &str) -> CredentialsSection { + toml::from_str(raw).expect("valid credentials TOML") + } + + #[test] + fn parses_toml_shared() { + let cs = parse( + r#" +type = "toml-shared" +nats_user = "u" +nats_pass = "p" +"#, + ); + match cs { + CredentialsSection::TomlShared { + nats_user, + nats_pass, + } => { + assert_eq!(nats_user, "u"); + assert_eq!(nats_pass, "p"); + } + _ => panic!("expected TomlShared"), + } + } + + #[test] + fn parses_zitadel_jwt() { + let cs = parse( + r#" +type = "zitadel-jwt" +key_path = "/var/lib/fleet-agent/zitadel-key.json" +oidc_issuer_url = "https://zitadel.staging.example.com" +audience = "366378028009259037" +danger_accept_invalid_certs = false +"#, + ); + match cs { + CredentialsSection::ZitadelJwt { + key_path, + oidc_issuer_url, + audience, + danger_accept_invalid_certs, + } => { + assert_eq!( + key_path.to_str(), + Some("/var/lib/fleet-agent/zitadel-key.json") + ); + assert_eq!(oidc_issuer_url, "https://zitadel.staging.example.com"); + assert_eq!(audience, "366378028009259037"); + assert!(!danger_accept_invalid_certs); + } + _ => panic!("expected ZitadelJwt"), + } + } + + #[test] + fn zitadel_jwt_key_path_defaults_when_omitted() { + let cs = parse( + r#" +type = "zitadel-jwt" +oidc_issuer_url = "https://zitadel.staging.example.com" +audience = "366378028009259037" +"#, + ); + match cs { + CredentialsSection::ZitadelJwt { key_path, .. } => { + assert_eq!(key_path.to_str(), Some("/etc/fleet-agent/zitadel-key.json")); + } + _ => panic!("expected ZitadelJwt"), + } + } +} diff --git a/fleet/harmony-fleet-auth/src/credentials.rs b/fleet/harmony-fleet-auth/src/credentials.rs new file mode 100644 index 00000000..2a5262a6 --- /dev/null +++ b/fleet/harmony-fleet-auth/src/credentials.rs @@ -0,0 +1,536 @@ +//! NATS credential sources for fleet processes (agent + operator). +//! +//! `CredentialSource::next_credential()` is invoked from async-nats's +//! `with_auth_callback` on every (re)connect attempt — including the +//! first connect. The callback shape means an expired token is +//! automatically replaced when async-nats reconnects after a transient +//! NATS outage / pod restart / network blip: the caller doesn't need +//! a separate refresh task to "never lose connectivity." +//! +//! Two variants: +//! +//! - [`CredentialSource::TomlShared`] — username + password baked into +//! the config (v0/dev only). +//! - [`CredentialSource::ZitadelJwt`] — Zitadel machine-user JWT-bearer +//! flow (RFC 7523). The keyfile is the only durable secret on the +//! process; the bearer token is short-lived and re-minted +//! transparently when a cached token is within 5 minutes of expiry. +//! +//! Modeled as an enum (rather than a `dyn Trait`) because async-nats's +//! auth-callback bounds (`Future: Send + Sync`) are incompatible with +//! `Pin>` returned by an object-safe trait. Two +//! variants is a small enough cardinality that enum dispatch is +//! cleaner than a Trait + factory. + +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use anyhow::{Context, Result}; +use jsonwebtoken::{Algorithm, EncodingKey, Header as JwtHeader}; +use serde::Deserialize; + +use crate::config::CredentialsSection; + +/// Material the NATS connector needs to authenticate. Returned per +/// (re)connect attempt — the source decides whether to mint fresh. +#[derive(Debug, Clone)] +pub enum NatsCredential { + UserPass { user: String, pass: String }, + BearerToken(String), +} + +/// Externally-tagged credential source. Constructed once at startup +/// from the parsed `[credentials]` section; cloned via Arc into the +/// async-nats auth callback. +pub enum CredentialSource { + TomlShared { + user: String, + pass: String, + }, + ZitadelJwt { + key: MachineKeyFile, + oidc_issuer_url: String, + audience: String, + http: reqwest::Client, + cache: Mutex>, + }, +} + +impl CredentialSource { + /// Return current valid credentials, minting fresh material when any + /// cached value is within its safety window of expiry. Called on + /// every NATS (re)connect. + pub async fn next_credential(&self) -> Result { + match self { + Self::TomlShared { user, pass } => Ok(NatsCredential::UserPass { + user: user.clone(), + pass: pass.clone(), + }), + Self::ZitadelJwt { .. } => self.zitadel_next().await, + } + } + + async fn zitadel_next(&self) -> Result { + // Fast path: lock the cache synchronously, copy out the token if + // it's comfortably valid, drop the lock. Holding a MutexGuard + // across `.await` would make this future !Sync, which + // async-nats's `with_auth_callback` rejects at compile time. + if let Some(token) = self.cached_if_fresh() { + return Ok(NatsCredential::BearerToken(token)); + } + // Slow path: mint outside any lock. Two concurrent (re)connect + // attempts could both reach here and both mint; that's a wasted + // HTTP round-trip in a rare race, not a correctness issue — + // the second writer wins and replaces the first's value. + let fresh = self.zitadel_mint().await?; + let token = fresh.access_token.clone(); + if let Self::ZitadelJwt { + cache, audience, .. + } = self + && let Ok(mut guard) = cache.lock() + { + *guard = Some(fresh); + tracing::info!(audience = %audience, "minted fresh Zitadel access token"); + } + Ok(NatsCredential::BearerToken(token)) + } + + fn cached_if_fresh(&self) -> Option { + let Self::ZitadelJwt { cache, .. } = self else { + return None; + }; + let now = chrono::Utc::now().timestamp(); + let guard = cache.lock().ok()?; + let cached = guard.as_ref()?; + if cached.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS > now { + Some(cached.access_token.clone()) + } else { + None + } + } + + async fn zitadel_mint(&self) -> Result { + let Self::ZitadelJwt { + key, + oidc_issuer_url, + audience, + http, + .. + } = self + else { + anyhow::bail!("zitadel_mint called on non-ZitadelJwt variant"); + }; + + let now = chrono::Utc::now().timestamp(); + let assertion = build_assertion(key, oidc_issuer_url, now)?; + let scope = build_scope(audience); + let token_url = build_token_url(oidc_issuer_url); + + let resp = http + .post(&token_url) + .form(&[ + ( + "grant_type", + "urn:ietf:params:oauth:grant-type:jwt-bearer".to_string(), + ), + ("assertion", assertion), + ("scope", scope), + ]) + .send() + .await + .with_context(|| format!("POST {token_url}"))?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("Zitadel token endpoint returned {status}: {body}"); + } + + #[derive(Deserialize)] + struct TokenResponse { + access_token: String, + #[serde(default)] + expires_in: Option, + } + let tr: TokenResponse = resp.json().await.context("parsing token response")?; + // Zitadel typically returns 12h (43200s); be defensive against + // a missing field by assuming a conservative 1h. + let expires_in = tr.expires_in.unwrap_or(3600); + Ok(CachedToken { + access_token: tr.access_token, + expires_at_unix: now + expires_in, + }) + } +} + +/// Build the JWT-bearer assertion. Split out from the network path so +/// the claims + header shape can be unit-tested without an HTTP server, +/// and split internally into the (pure) claim/header builders so they +/// can be unit-tested without an RSA private key fixture. +pub(crate) fn build_assertion( + key: &MachineKeyFile, + oidc_issuer_url: &str, + now: i64, +) -> Result { + let claims = build_assertion_claims(key, oidc_issuer_url, now); + let header = build_assertion_header(key); + let assertion = jsonwebtoken::encode( + &header, + &claims, + &EncodingKey::from_rsa_pem(key.key.as_bytes()) + .context("parsing RSA private key from machine key file")?, + ) + .context("signing JWT assertion")?; + Ok(assertion) +} + +/// Pure claim payload for the JWT-bearer assertion. `iss == sub == userId` +/// is a Zitadel requirement; `aud` is Zitadel itself (the token endpoint +/// is reached via `oidc_issuer_url`); `exp - iat` MUST be ≤ 60 s or +/// Zitadel rejects. +pub(crate) fn build_assertion_claims( + key: &MachineKeyFile, + oidc_issuer_url: &str, + now: i64, +) -> serde_json::Value { + serde_json::json!({ + "iss": key.user_id, + "sub": key.user_id, + "aud": oidc_issuer_url, + "exp": now + ASSERTION_LIFETIME_SECS, + "iat": now, + }) +} + +/// JWT header for the assertion. The `kid` tells Zitadel which of the +/// machine user's registered keys to verify the signature against. +pub(crate) fn build_assertion_header(key: &MachineKeyFile) -> JwtHeader { + let mut header = JwtHeader::new(Algorithm::RS256); + header.kid = Some(key.key_id.clone()); + header +} + +/// Build the OAuth `scope` string for the token-bearer request. +/// +/// Three scopes are needed for the access token to be useful here: +/// +/// * `openid` — base OIDC requirement. +/// * `urn:zitadel:iam:org:projects:roles` (PLURAL "projects") — +/// tells Zitadel to include the role-claim block in the access +/// token. Without this, the callout sees "no authorized role +/// in token" even when the user has a project role grant. +/// * `urn:zitadel:iam:org:project:id::aud` (SINGULAR +/// "project") — adds to the access token's `aud` claim +/// so the callout's audience validation accepts the project +/// ID we're using as the JWT-bearer audience. +/// +/// The plural-vs-singular distinction is a Zitadel convention, +/// not a typo. Both scopes are required. +pub(crate) fn build_scope(audience: &str) -> String { + format!( + "openid \ + urn:zitadel:iam:org:projects:roles \ + urn:zitadel:iam:org:project:id:{audience}:aud" + ) +} + +/// Resolve the token endpoint URL, tolerating a trailing slash on +/// `oidc_issuer_url`. Without trimming, a configured issuer of +/// `https://sso.example.com/` produces `…//oauth/v2/token` which 404s. +pub(crate) fn build_token_url(oidc_issuer_url: &str) -> String { + format!("{}/oauth/v2/token", oidc_issuer_url.trim_end_matches('/')) +} + +// ---- helper types ---------------------------------------------------------- + +/// JSON keyfile content as Zitadel emits it for a `KEY_TYPE_JSON` +/// machine key. The `key` is a PEM-encoded RSA private key. +#[derive(Debug, Clone, Deserialize)] +pub struct MachineKeyFile { + #[serde(rename = "type")] + pub _type: String, + #[serde(rename = "keyId")] + pub key_id: String, + pub key: String, + #[serde(rename = "userId")] + pub user_id: String, +} + +#[derive(Debug, Clone)] +pub struct CachedToken { + pub(crate) access_token: String, + /// Unix seconds at which the token is no longer trusted by + /// `cached_if_fresh`. Computed from the OAuth response's `expires_in` + /// and the local clock at mint time. + pub(crate) expires_at_unix: i64, +} + +/// Refresh tokens this many seconds before their advertised expiry. +/// Five minutes leaves headroom for clock skew, slow networks, and +/// the round-trip cost of re-minting against Zitadel. +pub const TOKEN_REFRESH_LEEWAY_SECS: i64 = 5 * 60; + +/// Lifetime of the JWT *assertion* (the client-side bearer JWT we sign +/// to authenticate to Zitadel's token endpoint). Zitadel rejects +/// assertions with `exp - iat > 60s`; one minute is the safe ceiling. +pub const ASSERTION_LIFETIME_SECS: i64 = 60; + +// ---- factory --------------------------------------------------------------- + +/// Build the appropriate `CredentialSource` from the parsed config. +/// +/// For [`CredentialsSection::ZitadelJwt`] this reads the keyfile from +/// disk. Both the agent and the operator mount their key as a file +/// (Secret volume in the operator's Pod, dropped by +/// `FleetDeviceSetupScore` on the agent's VM); the path is just +/// configured differently. +pub fn credential_source_from_config(creds: &CredentialsSection) -> Result> { + match creds { + CredentialsSection::TomlShared { + nats_user, + nats_pass, + } => Ok(Arc::new(CredentialSource::TomlShared { + user: nats_user.clone(), + pass: nats_pass.clone(), + })), + CredentialsSection::ZitadelJwt { + key_path, + oidc_issuer_url, + audience, + danger_accept_invalid_certs, + } => Ok(Arc::new(CredentialSource::ZitadelJwt { + key: load_machine_key(key_path)?, + oidc_issuer_url: oidc_issuer_url.clone(), + audience: audience.clone(), + http: reqwest::Client::builder() + .danger_accept_invalid_certs(*danger_accept_invalid_certs) + .timeout(Duration::from_secs(10)) + .build() + .context("building HTTP client for Zitadel token endpoint")?, + cache: Mutex::new(None), + })), + } +} + +fn load_machine_key(key_path: &Path) -> Result { + let raw = std::fs::read_to_string(key_path) + .with_context(|| format!("reading machine key file at {}", key_path.display()))?; + serde_json::from_str(&raw) + .with_context(|| format!("parsing machine key file at {}", key_path.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fake_key() -> MachineKeyFile { + MachineKeyFile { + _type: "serviceaccount".to_string(), + key_id: "kid-371358469099356247".to_string(), + // Real PEM not required for the pure-builder tests; the + // signing path that needs a parseable key is exercised + // end-to-end in the e2e harness. + key: "PEM-PLACEHOLDER".to_string(), + user_id: "uid-371358469065801815".to_string(), + } + } + + fn zjwt_source() -> CredentialSource { + CredentialSource::ZitadelJwt { + key: fake_key(), + oidc_issuer_url: "http://sso.fleet.local:8080".to_string(), + audience: "366378028009259037".to_string(), + http: reqwest::Client::new(), + cache: Mutex::new(None), + } + } + + // ---- next_credential / cache state ------------------------------------- + + #[tokio::test] + async fn toml_shared_returns_userpass_each_call() { + let s = CredentialSource::TomlShared { + user: "u".to_string(), + pass: "p".to_string(), + }; + let c = s.next_credential().await.unwrap(); + match c { + NatsCredential::UserPass { user, pass } => { + assert_eq!(user, "u"); + assert_eq!(pass, "p"); + } + other => panic!("expected UserPass, got {other:?}"), + } + } + + #[test] + fn cached_token_within_leeway_is_treated_as_expired() { + // Sanity-check the comparison so refactors don't accidentally + // invert the leeway window. + let now = chrono::Utc::now().timestamp(); + let about_to_expire = CachedToken { + access_token: "x".to_string(), + expires_at_unix: now + TOKEN_REFRESH_LEEWAY_SECS - 1, + }; + assert!( + about_to_expire.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS <= now, + "tokens within the leeway window must be considered expired" + ); + + let comfortable = CachedToken { + access_token: "x".to_string(), + expires_at_unix: now + TOKEN_REFRESH_LEEWAY_SECS + 60, + }; + assert!( + comfortable.expires_at_unix - TOKEN_REFRESH_LEEWAY_SECS > now, + "tokens with comfortable headroom must be cache-hits" + ); + } + + #[test] + fn cached_if_fresh_returns_some_when_outside_leeway() { + let src = zjwt_source(); + let now = chrono::Utc::now().timestamp(); + if let CredentialSource::ZitadelJwt { cache, .. } = &src { + *cache.lock().unwrap() = Some(CachedToken { + access_token: "fresh".to_string(), + expires_at_unix: now + TOKEN_REFRESH_LEEWAY_SECS + 60, + }); + } + assert_eq!(src.cached_if_fresh(), Some("fresh".to_string())); + } + + #[test] + fn cached_if_fresh_returns_none_when_no_cache() { + // Brand-new ZitadelJwt source — no token has been minted yet. + // Forces the slow path on first connect. + let src = zjwt_source(); + assert_eq!(src.cached_if_fresh(), None); + } + + #[test] + fn cached_if_fresh_returns_none_for_toml_shared() { + // Defensive: cache_if_fresh is only meaningful for ZitadelJwt; + // TomlShared has no cache. A nonsensical call must return None, + // not panic, so the cold-path can degrade gracefully. + let src = CredentialSource::TomlShared { + user: "u".into(), + pass: "p".into(), + }; + assert_eq!(src.cached_if_fresh(), None); + } + + // ---- assertion claims / header (pure builders) ------------------------ + + #[test] + fn assertion_claims_carry_iss_sub_aud_exp_iat() { + let now = 1_700_000_000; + let claims = build_assertion_claims(&fake_key(), "http://sso.fleet.local:8080", now); + assert_eq!(claims["iss"], "uid-371358469065801815"); + assert_eq!(claims["sub"], "uid-371358469065801815"); + assert_eq!(claims["aud"], "http://sso.fleet.local:8080"); + assert_eq!(claims["iat"].as_i64(), Some(now)); + assert_eq!(claims["exp"].as_i64(), Some(now + ASSERTION_LIFETIME_SECS)); + } + + #[test] + fn assertion_lifetime_locked_at_60_seconds() { + // Zitadel rejects assertions where exp - iat > 60s. If anyone + // bumps ASSERTION_LIFETIME_SECS thinking "more is safer", the + // mints will silently start failing in prod with no helpful + // error. Lock the constant. + assert_eq!(ASSERTION_LIFETIME_SECS, 60); + } + + #[test] + fn assertion_header_carries_kid_and_rs256() { + let header = build_assertion_header(&fake_key()); + assert_eq!(header.alg, jsonwebtoken::Algorithm::RS256); + assert_eq!(header.kid.as_deref(), Some("kid-371358469099356247")); + } + + // ---- scope string ------------------------------------------------------ + + #[test] + fn scope_includes_plural_projects_roles() { + // The plural-projects URN is what tells Zitadel to emit the + // role claim. Day-one bug; lock it. + let s = build_scope("366378028009259037"); + assert!( + s.contains("urn:zitadel:iam:org:projects:roles"), + "scope must include the PLURAL projects-roles URN; got {s:?}" + ); + } + + #[test] + fn scope_audience_uses_singular_project_id_urn() { + // The singular-project URN tells Zitadel to put into the + // access token's aud claim. Different URN entirely from the + // plural one above; both required. + let s = build_scope("366378028009259037"); + assert!( + s.contains("urn:zitadel:iam:org:project:id:366378028009259037:aud"), + "scope must include the SINGULAR project:id::aud URN; got {s:?}" + ); + } + + #[test] + fn scope_includes_openid_base() { + let s = build_scope("any"); + assert!( + s.split_whitespace().any(|tok| tok == "openid"), + "scope must include `openid` as a standalone token; got {s:?}" + ); + } + + // ---- token URL --------------------------------------------------------- + + #[test] + fn token_url_appends_oauth_endpoint() { + assert_eq!( + build_token_url("http://sso.fleet.local:8080"), + "http://sso.fleet.local:8080/oauth/v2/token" + ); + } + + #[test] + fn token_url_strips_single_trailing_slash() { + // A trailing slash would yield `…//oauth/v2/token`, which 404s. + // Common configuration drift; the trim guards against it. + assert_eq!( + build_token_url("http://sso.fleet.local:8080/"), + "http://sso.fleet.local:8080/oauth/v2/token" + ); + } + + #[test] + fn token_url_strips_multiple_trailing_slashes() { + // Defensive — `trim_end_matches('/')` peels all of them, not + // just the first. Locks that semantics. + assert_eq!( + build_token_url("http://sso.fleet.local:8080///"), + "http://sso.fleet.local:8080/oauth/v2/token" + ); + } + + // ---- MachineKeyFile JSON parsing -------------------------------------- + + #[test] + fn machine_key_file_parses_zitadel_json_shape() { + // The serde renames (`type`, `keyId`, `userId`) are easy to + // break. This is the literal JSON shape Zitadel's + // /management/v1/users/.../keys endpoint emits. + let raw = r#"{ + "type": "serviceaccount", + "keyId": "371358469099356247", + "key": "-----BEGIN RSA PRIVATE KEY-----\nABC\n-----END RSA PRIVATE KEY-----\n", + "userId": "371358469065801815" + }"#; + let parsed: MachineKeyFile = serde_json::from_str(raw).expect("valid keyfile"); + assert_eq!(parsed._type, "serviceaccount"); + assert_eq!(parsed.key_id, "371358469099356247"); + assert_eq!(parsed.user_id, "371358469065801815"); + assert!(parsed.key.contains("BEGIN RSA PRIVATE KEY")); + } +} diff --git a/fleet/harmony-fleet-auth/src/lib.rs b/fleet/harmony-fleet-auth/src/lib.rs new file mode 100644 index 00000000..e95cffd5 --- /dev/null +++ b/fleet/harmony-fleet-auth/src/lib.rs @@ -0,0 +1,63 @@ +//! Shared NATS auth plumbing for fleet processes. +//! +//! Two consumers today: +//! +//! - **`harmony-fleet-agent`** — reads `[credentials]` from +//! `/etc/fleet-agent/config.toml`. Per-device Zitadel machine user +//! with the `device` role. +//! - **`harmony-fleet-operator`** — reads the same TOML shape from a +//! single env var (the env var's value is the TOML snippet for the +//! `[credentials]` table). Singleton machine user with the +//! `fleet-admin` role. +//! +//! Both deserialize into the **same** [`CredentialsSection`], factory +//! into the **same** [`CredentialSource`], and use the **same** +//! [`connect_options_with_credentials`] helper to build a NATS client. +//! The only thing that differs between processes is where the bytes of +//! the TOML config come from and which Zitadel user signs the +//! JWT-bearer assertion. +//! +//! Adding a new mode (e.g. user JWT from a CLI session) is one new +//! variant on `CredentialsSection` + `CredentialSource`; everything +//! else flows through unchanged. + +mod config; +mod credentials; + +pub use config::CredentialsSection; +pub use credentials::{ + ASSERTION_LIFETIME_SECS, CachedToken, CredentialSource, MachineKeyFile, NatsCredential, + TOKEN_REFRESH_LEEWAY_SECS, credential_source_from_config, +}; + +use std::sync::Arc; + +/// Build `async_nats::ConnectOptions` wired with the auth callback +/// that pulls fresh credentials from `creds` on every (re)connect. +/// +/// Caller chains additional options (`ping_interval`, `event_callback`, +/// …) before invoking `.connect(urls)`. +pub fn connect_options_with_credentials( + creds: Arc, +) -> async_nats::ConnectOptions { + async_nats::ConnectOptions::with_auth_callback(move |_nonce| { + let cs = creds.clone(); + async move { + let cred = cs + .next_credential() + .await + .map_err(|e| async_nats::AuthError::new(format!("credential source: {e}")))?; + let mut auth = async_nats::Auth::new(); + match cred { + NatsCredential::UserPass { user, pass } => { + auth.username = Some(user); + auth.password = Some(pass); + } + NatsCredential::BearerToken(token) => { + auth.token = Some(token); + } + } + Ok(auth) + } + }) +} diff --git a/fleet/harmony-fleet-operator/Cargo.toml b/fleet/harmony-fleet-operator/Cargo.toml index 3fe5a2d4..778b584e 100644 --- a/fleet/harmony-fleet-operator/Cargo.toml +++ b/fleet/harmony-fleet-operator/Cargo.toml @@ -6,7 +6,9 @@ rust-version = "1.85" [dependencies] harmony = { path = "../../harmony" } +harmony-fleet-auth = { path = "../harmony-fleet-auth" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } +toml = { workspace = true } chrono = { workspace = true, features = ["serde"] } kube = { workspace = true, features = ["runtime", "derive"] } k8s-openapi.workspace = true diff --git a/fleet/harmony-fleet-operator/src/chart.rs b/fleet/harmony-fleet-operator/src/chart.rs index a8e4138c..54b093a2 100644 --- a/fleet/harmony-fleet-operator/src/chart.rs +++ b/fleet/harmony-fleet-operator/src/chart.rs @@ -20,12 +20,13 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use harmony::modules::application::helm::{HelmChart, HelmResourceKind}; +use k8s_openapi::ByteString; use k8s_openapi::api::apps::v1::{ Deployment as K8sDeployment, DeploymentSpec as K8sDeploymentSpec, }; use k8s_openapi::api::core::v1::{ - Capabilities, Container, EnvVar, PodSpec, PodTemplateSpec, SeccompProfile, SecurityContext, - ServiceAccount, + Capabilities, Container, EnvVar, EnvVarSource, PodSpec, PodTemplateSpec, SeccompProfile, + Secret, SecretKeySelector, SecurityContext, ServiceAccount, }; use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject}; use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; @@ -60,6 +61,41 @@ pub struct ChartOptions { pub nats_url: String, /// `RUST_LOG` value for the operator process. pub log_level: String, + /// `[credentials]` TOML payload to inject as + /// `FLEET_OPERATOR_CREDENTIALS_TOML` via a Secret. `None` skips the + /// Secret entirely and lets the operator connect to NATS without + /// auth — only sensible when there's no callout in front of NATS. + pub credentials: Option, +} + +/// What the operator pod needs to authenticate to NATS via the auth +/// callout: a TOML snippet matching the agent's `[credentials]` +/// table, plus the JSON keyfile content the TOML references via +/// `key_path`. +/// +/// Both bytes go into a single Secret (`harmony-fleet-operator-secrets`). +/// The TOML is exposed as `FLEET_OPERATOR_CREDENTIALS_TOML` (env var); +/// the keyfile is mounted as a file at `key_path` (defaults to +/// `/etc/fleet-operator/zitadel-key.json` — caller-controllable via +/// the TOML's `key_path`). +pub struct OperatorCredentials { + /// TOML payload, e.g. + /// ```text + /// type = "zitadel-jwt" + /// key_path = "/etc/fleet-operator/zitadel-key.json" + /// oidc_issuer_url = "http://sso.fleet.local:8080" + /// audience = "" + /// ``` + pub credentials_toml: String, + /// JSON keyfile content (the `Zitadel KEY_TYPE_JSON` blob). Must be + /// the file the `credentials_toml`'s `key_path` resolves to inside + /// the Pod. Whoever calls this is responsible for keeping the two + /// in sync. + pub zitadel_keyfile_json: String, + /// Where in the Pod's filesystem to mount the keyfile. MUST match + /// the `key_path` in `credentials_toml`. Defaults to + /// `/etc/fleet-operator/zitadel-key.json`. + pub key_mount_path: String, } impl Default for ChartOptions { @@ -71,14 +107,22 @@ impl Default for ChartOptions { namespace: "fleet-system".to_string(), nats_url: "nats://fleet-nats.fleet-system:4222".to_string(), log_level: "info,kube_runtime=warn".to_string(), + credentials: None, } } } -const RELEASE_NAME: &str = "harmony-fleet-operator"; -const SERVICE_ACCOUNT: &str = "harmony-fleet-operator"; -const CLUSTER_ROLE: &str = "harmony-fleet-operator"; -const CLUSTER_ROLE_BINDING: &str = "harmony-fleet-operator"; +pub const RELEASE_NAME: &str = "harmony-fleet-operator"; +pub const SERVICE_ACCOUNT: &str = "harmony-fleet-operator"; +pub const CLUSTER_ROLE: &str = "harmony-fleet-operator"; +pub const CLUSTER_ROLE_BINDING: &str = "harmony-fleet-operator"; +pub const SECRET_NAME: &str = "harmony-fleet-operator-secrets"; +/// Key inside the Secret holding the `[credentials]` TOML. +pub const SECRET_KEY_CREDENTIALS_TOML: &str = "credentials.toml"; +/// Key inside the Secret holding the JSON keyfile. +pub const SECRET_KEY_ZITADEL_KEYFILE: &str = "zitadel-key.json"; +/// Volume name for the keyfile mount. Internal to the Pod spec. +const KEYFILE_VOLUME_NAME: &str = "zitadel-key"; /// Build + write the chart to `opts.output_dir`. Returns the full /// path to the generated chart directory (which is what `helm @@ -107,6 +151,12 @@ pub fn build_chart(opts: &ChartOptions) -> Result { chart.add_resource(HelmResourceKind::ClusterRoleBinding(cluster_role_binding( &opts.namespace, ))); + // Secret intentionally NOT included in the on-disk helm chart — + // credentials are operator-environment-specific and out of scope + // for a redistributable chart. The e2e bring-up applies the Secret + // directly via `operator_secret()` (used as a `K8sResourceScore`) + // and the chart's Deployment expects the Secret to be present in + // the namespace at install time. chart.add_resource(HelmResourceKind::Deployment(operator_deployment(opts))); let written = chart @@ -115,6 +165,32 @@ pub fn build_chart(opts: &ChartOptions) -> Result { Ok(written) } +/// Build the operator's Secret holding the `[credentials]` TOML and the +/// Zitadel JSON keyfile. Returns `None` when no credentials configured +/// (no-auth dev mode). +pub fn operator_secret(opts: &ChartOptions) -> Option { + let creds = opts.credentials.as_ref()?; + let mut data: BTreeMap = BTreeMap::new(); + data.insert( + SECRET_KEY_CREDENTIALS_TOML.to_string(), + ByteString(creds.credentials_toml.as_bytes().to_vec()), + ); + data.insert( + SECRET_KEY_ZITADEL_KEYFILE.to_string(), + ByteString(creds.zitadel_keyfile_json.as_bytes().to_vec()), + ); + Some(Secret { + metadata: ObjectMeta { + name: Some(SECRET_NAME.to_string()), + namespace: Some(opts.namespace.clone()), + ..Default::default() + }, + data: Some(data), + type_: Some("Opaque".to_string()), + ..Default::default() + }) +} + /// Annotate a CRD with `helm.sh/resource-policy: keep` so /// `helm uninstall` **does not** cascade-delete the CRD and its /// CRs. Without this, uninstall wipes every `Deployment` + `Device` @@ -213,12 +289,92 @@ fn cluster_role_binding(namespace: &str) -> ClusterRoleBinding { } fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { + use k8s_openapi::api::core::v1::{KeyToPath, SecretVolumeSource, Volume, VolumeMount}; + let mut match_labels = BTreeMap::new(); match_labels.insert( "app.kubernetes.io/name".to_string(), RELEASE_NAME.to_string(), ); + let mut env = vec![ + EnvVar { + name: "NATS_URL".to_string(), + value: Some(opts.nats_url.clone()), + ..Default::default() + }, + EnvVar { + name: "RUST_LOG".to_string(), + value: Some(opts.log_level.clone()), + ..Default::default() + }, + ]; + + let mut volume_mounts: Vec = Vec::new(); + let mut volumes: Vec = Vec::new(); + + if let Some(creds) = opts.credentials.as_ref() { + // The whole TOML payload travels as a single env var so the + // operator can `toml::from_str(env::var(...))` directly. Same + // shape the agent reads from `/etc/fleet-agent/config.toml`. + env.push(EnvVar { + name: "FLEET_OPERATOR_CREDENTIALS_TOML".to_string(), + value_from: Some(EnvVarSource { + secret_key_ref: Some(SecretKeySelector { + name: SECRET_NAME.to_string(), + key: SECRET_KEY_CREDENTIALS_TOML.to_string(), + optional: Some(false), + }), + ..Default::default() + }), + ..Default::default() + }); + + // The keyfile must be a real file because + // `credential_source_from_config` reads it via `key_path` (same + // contract as the agent). Mount only the keyfile entry of the + // Secret at the Pod's `key_mount_path`. + let mount_path = std::path::Path::new(&creds.key_mount_path); + let mount_dir = mount_path + .parent() + .map(|p| p.to_string_lossy().to_string()) + .unwrap_or_else(|| "/etc/fleet-operator".to_string()); + let mount_filename = mount_path + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| SECRET_KEY_ZITADEL_KEYFILE.to_string()); + + volume_mounts.push(VolumeMount { + name: KEYFILE_VOLUME_NAME.to_string(), + mount_path: mount_dir, + read_only: Some(true), + ..Default::default() + }); + volumes.push(Volume { + name: KEYFILE_VOLUME_NAME.to_string(), + secret: Some(SecretVolumeSource { + secret_name: Some(SECRET_NAME.to_string()), + items: Some(vec![KeyToPath { + key: SECRET_KEY_ZITADEL_KEYFILE.to_string(), + path: mount_filename, + // 0o444 = world-read. The Secret volume is owned by + // root (kubelet default; we don't pin a fsGroup + // because we also don't pin runAsUser for SCC + // compatibility — see container_security_context). + // World-read inside the pod is safe: the pod has a + // single container, the Secret namespace is locked + // down, and the file never escapes the pod + // filesystem. With 0o400 the operator hits + // EACCES because its non-root UID is not root. + mode: Some(0o444), + }]), + default_mode: Some(0o444), + optional: Some(false), + }), + ..Default::default() + }); + } + K8sDeployment { metadata: ObjectMeta { name: Some(RELEASE_NAME.to_string()), @@ -243,21 +399,20 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { name: "operator".to_string(), image: Some(opts.image.clone()), image_pull_policy: Some(opts.image_pull_policy.clone()), - env: Some(vec![ - EnvVar { - name: "NATS_URL".to_string(), - value: Some(opts.nats_url.clone()), - ..Default::default() - }, - EnvVar { - name: "RUST_LOG".to_string(), - value: Some(opts.log_level.clone()), - ..Default::default() - }, - ]), + env: Some(env), + volume_mounts: if volume_mounts.is_empty() { + None + } else { + Some(volume_mounts) + }, security_context: Some(container_security_context()), ..Default::default() }], + volumes: if volumes.is_empty() { + None + } else { + Some(volumes) + }, ..Default::default() }), }, @@ -267,6 +422,21 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { } } +// Re-export the manifest builders so the e2e bring-up can apply the +// operator inline (Score-style) without re-implementing the manifests. +pub fn build_service_account(opts: &ChartOptions) -> ServiceAccount { + service_account(&opts.namespace) +} +pub fn build_cluster_role() -> ClusterRole { + cluster_role() +} +pub fn build_cluster_role_binding(opts: &ChartOptions) -> ClusterRoleBinding { + cluster_role_binding(&opts.namespace) +} +pub fn build_operator_deployment(opts: &ChartOptions) -> K8sDeployment { + operator_deployment(opts) +} + /// Minimum-privilege container security context. /// /// - `runAsNonRoot: true` — a compromised operator pod with diff --git a/fleet/harmony-fleet-operator/src/lib.rs b/fleet/harmony-fleet-operator/src/lib.rs index c97049c8..fa88ae2f 100644 --- a/fleet/harmony-fleet-operator/src/lib.rs +++ b/fleet/harmony-fleet-operator/src/lib.rs @@ -6,6 +6,7 @@ //! — can import the typed `Deployment`, `DeploymentSpec`, //! `ScorePayload`, etc. without duplicating them. +pub mod chart; pub mod crd; pub mod device_reconciler; pub mod fleet_aggregator; diff --git a/fleet/harmony-fleet-operator/src/main.rs b/fleet/harmony-fleet-operator/src/main.rs index 0e0bd347..31fc3861 100644 --- a/fleet/harmony-fleet-operator/src/main.rs +++ b/fleet/harmony-fleet-operator/src/main.rs @@ -1,15 +1,18 @@ -mod chart; mod controller; mod install; -use harmony_fleet_operator::{crd, device_reconciler, fleet_aggregator}; +use harmony_fleet_operator::{chart, crd, device_reconciler, fleet_aggregator}; -use anyhow::Result; +use anyhow::{Context, Result}; use async_nats::jetstream; use clap::{Parser, Subcommand}; +use harmony_fleet_auth::{ + CredentialsSection, connect_options_with_credentials, credential_source_from_config, +}; use harmony_reconciler_contracts::BUCKET_DESIRED_STATE; use kube::Client; use std::path::PathBuf; +use std::time::Duration; #[derive(Parser)] #[command( @@ -35,6 +38,18 @@ struct Cli { global = true )] kv_bucket: String, + + /// `[credentials]` TOML payload (same shape the agent reads from + /// `/etc/fleet-agent/config.toml`). Mounted into the Pod from the + /// operator's Secret. Empty string means "no auth — bare connect" + /// (for local dev without a callout-protected NATS). + #[arg( + long, + env = "FLEET_OPERATOR_CREDENTIALS_TOML", + default_value = "", + global = true + )] + credentials_toml: String, } #[derive(Subcommand)] @@ -73,7 +88,7 @@ async fn main() -> Result<()> { let cli = Cli::parse(); match cli.command.unwrap_or(Command::Run) { Command::Install => install::install_crds().await, - Command::Run => run(&cli.nats_url, &cli.kv_bucket).await, + Command::Run => run(&cli.nats_url, &cli.kv_bucket, &cli.credentials_toml).await, Command::Chart { output, image, @@ -89,6 +104,12 @@ async fn main() -> Result<()> { namespace, nats_url, log_level, + // The disk-distributed chart never carries operator + // credentials — those are environment-specific. The + // operator deploys into a namespace where the matching + // Secret already exists (provisioned out-of-band, or + // by the e2e bring-up's K8sResourceScore path). + credentials: None, })?; println!("{}", written.display()); Ok(()) @@ -96,10 +117,8 @@ async fn main() -> Result<()> { } } -async fn run(nats_url: &str, bucket: &str) -> Result<()> { - // Retry on the initial connect — startup races against the NATS - // server becoming fully ready. - let nats = connect_with_retry(nats_url).await?; +async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()> { + let nats = connect_with_retry(nats_url, credentials_toml).await?; tracing::info!(url = %nats_url, "connected to NATS"); let js = jetstream::new(nats); let desired_state_kv = js @@ -129,18 +148,66 @@ async fn run(nats_url: &str, bucket: &str) -> Result<()> { } } -async fn connect_with_retry(nats_url: &str) -> Result { - use std::time::Duration; +/// Connect to NATS, retrying on the initial connect — startup races +/// against the NATS server becoming fully ready. +/// +/// `credentials_toml` is the in-memory `[credentials]` TOML snippet +/// the operator's pod gets via the `FLEET_OPERATOR_CREDENTIALS_TOML` +/// env var (sourced from a Kubernetes Secret). Same shape as the +/// agent's `[credentials]` table; same factory; same auth callback. +/// Empty string means bypass — connect with no creds (only useful +/// for callout-less local dev). +async fn connect_with_retry(nats_url: &str, credentials_toml: &str) -> Result { let mut last_err: Option = None; for attempt in 0..15 { - match async_nats::connect(nats_url).await { + let attempt_result = if credentials_toml.is_empty() { + tracing::warn!( + "FLEET_OPERATOR_CREDENTIALS_TOML is empty — connecting to NATS \ + without auth. Production deploys MUST mount a credentials Secret." + ); + async_nats::connect(nats_url) + .await + .map_err(anyhow::Error::from) + } else { + connect_with_credentials(nats_url, credentials_toml).await + }; + match attempt_result { Ok(c) => return Ok(c), Err(e) => { tracing::warn!(attempt, error = %e, "NATS connect failed; retrying"); - last_err = Some(e.into()); + last_err = Some(e); tokio::time::sleep(Duration::from_secs(2)).await; } } } Err(last_err.unwrap_or_else(|| anyhow::anyhow!("NATS connect failed after retries"))) } + +async fn connect_with_credentials( + nats_url: &str, + credentials_toml: &str, +) -> Result { + let creds_section: CredentialsSection = + toml::from_str(credentials_toml).context("parsing FLEET_OPERATOR_CREDENTIALS_TOML")?; + let creds = credential_source_from_config(&creds_section) + .context("constructing CredentialSource from operator credentials")?; + let client = connect_options_with_credentials(creds) + .ping_interval(Duration::from_secs(10)) + .event_callback(|event| async move { + use async_nats::Event; + match event { + Event::Connected => tracing::info!("NATS connected"), + Event::Disconnected => tracing::warn!("NATS disconnected, will reconnect"), + Event::LameDuckMode => tracing::warn!("NATS server entered lame-duck mode"), + Event::SlowConsumer(sid) => tracing::warn!(sid = %sid, "NATS slow consumer"), + Event::ServerError(e) => tracing::error!(error = %e, "NATS server error"), + Event::ClientError(e) => tracing::error!(error = %e, "NATS client error"), + Event::Closed => tracing::error!("NATS connection closed"), + other => tracing::debug!(?other, "NATS event"), + } + }) + .connect(nats_url) + .await + .context("connecting to NATS with operator credentials")?; + Ok(client) +} diff --git a/fleet/scripts/load-test.sh b/fleet/scripts/load-test.sh index b5ceb9f9..ae78ae48 100755 --- a/fleet/scripts/load-test.sh +++ b/fleet/scripts/load-test.sh @@ -249,6 +249,7 @@ $(printf '\033[1;32m[load-test]\033[0m stack ready. In another terminal:') EOF } + alias natsbox='podman run --rm docker.io/natsio/nats-box:latest nats --server nats://192.168.12.102:4222' print_banner diff --git a/harmony/src/domain/hardware/mod.rs b/harmony/src/domain/hardware/mod.rs index 2d7a0347..1bfe2c0c 100644 --- a/harmony/src/domain/hardware/mod.rs +++ b/harmony/src/domain/hardware/mod.rs @@ -33,6 +33,21 @@ impl PhysicalHost { } pub fn summary(&self) -> String { + let mut parts = self.summary_parts_through_storage(); + self.append_network_summary(&mut parts); + parts.join(" | ") + } + + /// Same shape as [`Self::summary`] but drops the network portion — useful + /// for compact contexts like the `Host:` header above interactive + /// `inquire` prompts, where the NIC list is too wide for the terminal. + pub fn summary_short(&self) -> String { + self.summary_parts_through_storage().join(" | ") + } + + /// Builds the first four sections of the summary (model, CPU, RAM, storage). + /// Shared between [`Self::summary`] and [`Self::summary_short`]. + fn summary_parts_through_storage(&self) -> Vec { let mut parts = Vec::new(); // Part 1: System Model (from labels) or Category as a fallback @@ -49,15 +64,17 @@ impl PhysicalHost { let cpu_count = self.cpus.len(); let total_cores = self.cpus.iter().map(|c| c.cores).sum::(); let total_threads = self.cpus.iter().map(|c| c.threads).sum::(); - let model_name = &self.cpus[0].model; + let model_name = self.cpus[0].model.trim(); - let cpu_summary = if cpu_count > 1 { - format!( - "{}x {} ({}c/{}t)", - cpu_count, model_name, total_cores, total_threads - ) - } else { - format!("{} ({}c/{}t)", model_name, total_cores, total_threads) + // Agents sometimes report a blank model (e.g. when /proc/cpuinfo is + // unreadable); collapse those cases to avoid stray double-spaces. + let cpu_summary = match (cpu_count > 1, model_name.is_empty()) { + (true, true) => format!("{cpu_count}x CPU ({total_cores}c/{total_threads}t)"), + (true, false) => { + format!("{cpu_count}x {model_name} ({total_cores}c/{total_threads}t)") + } + (false, true) => format!("{total_cores}c/{total_threads}t"), + (false, false) => format!("{model_name} ({total_cores}c/{total_threads}t)"), }; parts.push(cpu_summary); } @@ -94,7 +111,6 @@ impl PhysicalHost { if !self.storage.is_empty() { let total_storage_bytes = self.storage.iter().map(|d| d.size_bytes).sum::(); let drive_count = self.storage.len(); - let first_drive_model = &self.storage[0].model; // Helper to format bytes into TB or GB let format_storage = |bytes: u64| { @@ -115,45 +131,39 @@ impl PhysicalHost { .collect::>() .join(", "); - format!( - "{} Storage ({} Disks [{}])", - format_storage(total_storage_bytes), - drive_count, - drive_sizes - ) + format!("{} [{}]", format_storage(total_storage_bytes), drive_sizes) } else { - format!( - "{} Storage ({})", - format_storage(total_storage_bytes), - first_drive_model - ) + format_storage(total_storage_bytes) }; parts.push(storage_summary); } - // Part 5: Network Information - // Prioritize an "up" interface with an IPv4 address - let best_nic = self + parts + } + + /// Appends the per-NIC network section to an existing parts list. + fn append_network_summary(&self, parts: &mut Vec) { + if self.network.is_empty() { + return; + } + let per_nic: Vec = self .network .iter() - .find(|n| n.is_up && !n.ipv4_addresses.is_empty()) - .or_else(|| self.network.first()); + .map(|nic| { + let mac = nic.mac_address.to_string(); + match nic.ipv4_addresses.first() { + Some(ip) => format!("[{}, {}]", ip, mac), + None => format!("[{}]", mac), + } + }) + .collect(); - if let Some(nic) = best_nic { - let speed = nic - .speed_mbps - .map(|s| format!("{}Gbps", s / 1000)) - .unwrap_or_else(|| "N/A".to_string()); - let mac = nic.mac_address.to_string(); - let nic_summary = if let Some(ip) = nic.ipv4_addresses.first() { - format!("NIC: {} ({}, {})", speed, ip, mac) - } else { - format!("NIC: {} ({})", speed, mac) - }; - parts.push(nic_summary); - } - - parts.join(" | ") + let nic_summary = if per_nic.len() == 1 { + format!("NIC: {}", per_nic[0]) + } else { + format!("{} NICs: {}", per_nic.len(), per_nic.join(", ")) + }; + parts.push(nic_summary); } pub fn parts_list(&self) -> String { diff --git a/harmony/src/domain/inventory/mod.rs b/harmony/src/domain/inventory/mod.rs index 64a204a3..58b5f95c 100644 --- a/harmony/src/domain/inventory/mod.rs +++ b/harmony/src/domain/inventory/mod.rs @@ -144,6 +144,16 @@ pub enum HostRole { Worker, } +/// A persisted role-to-host assignment: the role that was chosen, plus the +/// operational config captured at discovery time (install disk, bond + +/// blacklist). Returned when looking up "does this host already have a +/// mapping?" so the UI can show what will be replaced before overwriting. +#[derive(Debug, Clone)] +pub struct HostRoleMapping { + pub role: HostRole, + pub host_config: crate::topology::HostConfig, +} + impl fmt::Display for HostRole { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { diff --git a/harmony/src/domain/inventory/repository.rs b/harmony/src/domain/inventory/repository.rs index e6a4eea8..5a83ad83 100644 --- a/harmony/src/domain/inventory/repository.rs +++ b/harmony/src/domain/inventory/repository.rs @@ -1,8 +1,12 @@ use async_trait::async_trait; use crate::{ - hardware::PhysicalHost, interpret::InterpretError, inventory::HostRole, topology::HostConfig, + hardware::PhysicalHost, + interpret::InterpretError, + inventory::{HostRole, HostRoleMapping}, + topology::{HostConfig, NetworkConfig}, }; +use harmony_types::id::Id; /// Errors that can occur within the repository layer. #[derive(thiserror::Error, Debug)] @@ -35,10 +39,18 @@ pub trait InventoryRepository: Send + Sync + 'static { &self, role: &HostRole, ) -> Result, RepoError>; + /// Insert-or-replace the role mapping for this host. Any prior mapping + /// rows for `host.id` are deleted first (in the same transaction) so + /// `host_role_mapping` holds at most one row per host. async fn save_role_mapping( &self, role: &HostRole, host: &PhysicalHost, installation_device: &String, + network_config: &NetworkConfig, ) -> Result<(), RepoError>; + + /// Return the current role mapping for a host, if any. Used at discovery + /// time to ask the operator whether to overwrite or cancel. + async fn get_role_mapping(&self, host_id: &Id) -> Result, RepoError>; } diff --git a/harmony/src/domain/topology/container_runtime.rs b/harmony/src/domain/topology/container_runtime.rs index 8804dfef..821b5abe 100644 --- a/harmony/src/domain/topology/container_runtime.rs +++ b/harmony/src/domain/topology/container_runtime.rs @@ -50,6 +50,18 @@ pub struct ContainerSpec { /// labels. Used by Scores to carry grouping information (e.g. the /// originating deployment name). pub labels: Vec<(String, String)>, + /// Environment variables to set inside the container. Order is preserved + /// for deterministic spec equality; runtimes apply them as a set. + #[serde(default)] + pub env: Vec<(String, String)>, + /// Bind-mount volumes from the host into the container. Bind mounts only + /// in v0; named/anonymous volumes can be added behind the same field + /// later (the runtime impls would distinguish on `host_path` shape). + #[serde(default)] + pub volumes: Vec, + /// Restart policy on container exit. Mirrors podman/docker semantics. + #[serde(default)] + pub restart_policy: RestartPolicy, } impl ContainerSpec { @@ -61,6 +73,51 @@ impl ContainerSpec { pub const MANAGED_BY_VALUE: &'static str = "harmony"; } +/// A single host-path → container-path bind mount. Bind mounts are the only +/// volume kind supported in v0 — they cover ~95% of compose use cases and +/// don't depend on a runtime-managed volume namespace. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VolumeMount { + /// Absolute path on the host. + pub host_path: String, + /// Absolute path inside the container. + pub container_path: String, + /// Mount as read-only. Defaults to false (read-write) to match + /// docker-compose's default. + #[serde(default)] + pub read_only: bool, +} + +/// Restart policy for a managed container. Names follow podman/docker +/// conventions so docker-compose translation is mechanical. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub enum RestartPolicy { + /// Don't restart on exit. + No, + /// Restart unless the user explicitly stopped the container. + /// Docker-compose's default for long-running services and what most + /// fleet workloads want. + #[default] + UnlessStopped, + /// Restart only if the container exits with a non-zero status. + OnFailure, + /// Always restart, even on clean exits and after host reboot. + Always, +} + +impl RestartPolicy { + /// Canonical string podman + docker accept on the CLI / in their APIs. + pub fn as_str(&self) -> &'static str { + match self { + RestartPolicy::No => "no", + RestartPolicy::UnlessStopped => "unless-stopped", + RestartPolicy::OnFailure => "on-failure", + RestartPolicy::Always => "always", + } + } +} + /// Observed state of a container on the runtime. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ContainerState { diff --git a/harmony/src/domain/topology/host_binding.rs b/harmony/src/domain/topology/host_binding.rs index 63352762..7bea060d 100644 --- a/harmony/src/domain/topology/host_binding.rs +++ b/harmony/src/domain/topology/host_binding.rs @@ -1,5 +1,6 @@ use derive_new::new; -use serde::Serialize; +use harmony_types::firewall::LaggProtocol; +use serde::{Deserialize, Serialize}; use crate::hardware::PhysicalHost; @@ -20,4 +21,23 @@ pub struct HostBinding { #[derive(Debug, new, Clone, Serialize)] pub struct HostConfig { pub installation_device: Option, + #[new(default)] + pub network_config: NetworkConfig, +} + +/// User-provided networking intent captured at discovery time. +/// +/// Produced by the interactive discovery flow and persisted alongside the role +/// mapping so downstream Scores can act on it (e.g. configuring a bond on the +/// chosen interfaces and avoiding blacklisted ones). +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct NetworkConfig { + pub bond: Option, + pub blacklisted_interfaces: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BondConfig { + pub interfaces: Vec, + pub mode: LaggProtocol, } diff --git a/harmony/src/infra/inventory/sqlite.rs b/harmony/src/infra/inventory/sqlite.rs index 3ce1654f..0cff734e 100644 --- a/harmony/src/infra/inventory/sqlite.rs +++ b/harmony/src/infra/inventory/sqlite.rs @@ -1,12 +1,16 @@ use crate::{ hardware::PhysicalHost, - inventory::{HostRole, InventoryRepository, RepoError}, - topology::HostConfig, + inventory::{HostRole, HostRoleMapping, InventoryRepository, RepoError}, + topology::{HostConfig, NetworkConfig}, }; use async_trait::async_trait; use harmony_types::id::Id; -use log::info; -use sqlx::{Pool, Sqlite, SqlitePool, migrate::MigrateDatabase}; +use log::{info, warn}; +use sqlx::{ + Pool, Sqlite, + sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions}, +}; +use std::str::FromStr; /// A thread-safe, connection-pooled repository using SQLite. #[derive(Debug)] @@ -16,18 +20,18 @@ pub struct SqliteInventoryRepository { impl SqliteInventoryRepository { pub async fn new(database_url: &str) -> Result { - // Ensure the database file exists for SQLite - if database_url.starts_with("sqlite:") { - let path = database_url.trim_start_matches("sqlite:"); - if !path.contains(":memory:") && !std::path::Path::new(path).exists() { - sqlx::any::install_default_drivers(); - sqlx::Sqlite::create_database(database_url) - .await - .map_err(|e| RepoError::ConnectionFailed(e.to_string()))?; - } - } + // Use the classic rollback journal (DELETE) rather than sqlx's WAL + // default so we don't leave `.sqlite-wal` / `.sqlite-shm` files next + // to the DB: this is a single-process CLI, WAL's concurrent-reader + // benefit is wasted. `create_if_missing(true)` replaces the manual + // `Sqlite::create_database` dance the code used to do. + let options = SqliteConnectOptions::from_str(database_url) + .map_err(|e| RepoError::ConnectionFailed(e.to_string()))? + .create_if_missing(true) + .journal_mode(SqliteJournalMode::Delete); - let pool = SqlitePool::connect(database_url) + let pool = SqlitePoolOptions::new() + .connect_with(options) .await .map_err(|e| RepoError::ConnectionFailed(e.to_string()))?; @@ -50,6 +54,24 @@ impl InventoryRepository for SqliteInventoryRepository { let id = Id::default().to_string(); let host_id = host.id.to_string(); + // Skip the insert if the most recent row for this host is byte-identical: + // discovery is naturally a polling activity (mDNS is continuous, CIDR scans get + // re-run) and we don't want an unbounded pile of identical version rows. Real + // changes still produce a new version row (audit trail for free). + let latest = sqlx::query!( + r#"SELECT data as "data!: Vec" FROM physical_hosts WHERE id = ? ORDER BY version_id DESC LIMIT 1"#, + host_id + ) + .fetch_optional(&self.pool) + .await?; + + if let Some(row) = latest { + if row.data == data { + info!("Host '{}' unchanged, skipping save", host.id); + return Ok(()); + } + } + sqlx::query!( "INSERT INTO physical_hosts (id, version_id, data) VALUES (?, ?, ?)", host_id, @@ -109,26 +131,85 @@ impl InventoryRepository for SqliteInventoryRepository { role: &HostRole, host: &PhysicalHost, installation_device: &String, + network_config: &NetworkConfig, ) -> Result<(), RepoError> { let host_id = host.id.to_string(); + let network_config_json = serde_json::to_string(network_config) + .map_err(|e| RepoError::Serialization(e.to_string()))?; + + // Replace atomically: DELETE any prior rows for this host_id (there should + // be at most one, but older data may have dups) then INSERT the new one. + // Wrapped in a transaction so a concurrent reader never sees zero rows. + let mut tx = self.pool.begin().await?; + + sqlx::query!("DELETE FROM host_role_mapping WHERE host_id = ?", host_id) + .execute(&mut *tx) + .await?; sqlx::query!( r#" - INSERT INTO host_role_mapping (host_id, role, installation_device) - VALUES (?, ?, ?) + INSERT INTO host_role_mapping (host_id, role, installation_device, network_config) + VALUES (?, ?, ?, ?) "#, host_id, role, - installation_device + installation_device, + network_config_json, ) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await?; + info!("Saved role mapping for host '{}' as '{:?}'", host.id, role); Ok(()) } + async fn get_role_mapping(&self, host_id: &Id) -> Result, RepoError> { + struct Row { + role: HostRole, + installation_device: Option, + network_config: Option, + } + + let host_id_str = host_id.to_string(); + let row = sqlx::query_as!( + Row, + r#"SELECT role as "role: HostRole", installation_device, network_config FROM host_role_mapping WHERE host_id = ? ORDER BY id DESC LIMIT 1"#, + host_id_str, + ) + .fetch_optional(&self.pool) + .await?; + + let Some(row) = row else { return Ok(None) }; + + // Tolerate unparseable network_config: log loudly and fall back to + // defaults so the operator can still be shown the existing mapping + // and choose "Update" to overwrite the bad row. This covers stored + // rows from older enum shapes and any accidental corruption. + let network_config = match row.network_config.as_deref() { + Some(json) => match serde_json::from_str::(json) { + Ok(cfg) => cfg, + Err(e) => { + warn!( + "Discarding unreadable network_config for host '{host_id}': {e}. The existing mapping will be shown with empty network config; pick 'Update' to replace it." + ); + NetworkConfig::default() + } + }, + None => NetworkConfig::default(), + }; + + Ok(Some(HostRoleMapping { + role: row.role, + host_config: HostConfig { + installation_device: row.installation_device, + network_config, + }, + })) + } + async fn get_hosts_for_role( &self, role: &HostRole, @@ -136,13 +217,14 @@ impl InventoryRepository for SqliteInventoryRepository { struct HostIdRow { host_id: String, installation_device: Option, + network_config: Option, } let role_str = format!("{:?}", role); let host_id_rows = sqlx::query_as!( HostIdRow, - "SELECT host_id, installation_device FROM host_role_mapping WHERE role = ?", + "SELECT host_id, installation_device, network_config FROM host_role_mapping WHERE role = ?", role_str ) .fetch_all(&self.pool) @@ -159,8 +241,14 @@ impl InventoryRepository for SqliteInventoryRepository { ))); } }; + let network_config = match row.network_config.as_deref() { + Some(json) => serde_json::from_str(json) + .map_err(|e| RepoError::Deserialization(e.to_string()))?, + None => NetworkConfig::default(), + }; let host_config = HostConfig { installation_device: row.installation_device, + network_config, }; hosts.push((physical_host, host_config)); } diff --git a/harmony/src/infra/opnsense/load_balancer.rs b/harmony/src/infra/opnsense/load_balancer.rs index 4e496bed..933f179d 100644 --- a/harmony/src/infra/opnsense/load_balancer.rs +++ b/harmony/src/infra/opnsense/load_balancer.rs @@ -53,7 +53,12 @@ impl LoadBalancer for OPNSenseFirewall { async fn ensure_initialized(&self) -> Result<(), ExecutorError> { let lb = self.opnsense_config.load_balancer(); - if lb.is_installed().await { + let installed = lb.is_installed().await.map_err(|e| { + ExecutorError::UnexpectedError(format!( + "Failed to query HAProxy installation status on OPNsense: {e}" + )) + })?; + if installed { debug!("HAProxy is installed"); } else { self.opnsense_config @@ -141,7 +146,7 @@ fn haproxy_service_to_harmony(svc: &HaproxyService) -> Option SSL::SSL, - "SSLNI" => SSL::SNI, + "SSLSNI" => SSL::SNI, "NOSSL" => SSL::Disabled, "" => SSL::Default, other => { @@ -177,7 +182,7 @@ pub(crate) fn harmony_service_to_lb_types( HealthCheck::HTTP(port, path, http_method, _status_code, ssl) => { let ssl_str = match ssl { SSL::SSL => Some("ssl".to_string()), - SSL::SNI => Some("sslni".to_string()), + SSL::SNI => Some("sslsni".to_string()), SSL::Disabled => Some("nossl".to_string()), SSL::Default => Some(String::new()), SSL::Other(other) => Some(other.clone()), diff --git a/harmony/src/modules/fleet/mod.rs b/harmony/src/modules/fleet/mod.rs index 2e42849d..d3f19c66 100644 --- a/harmony/src/modules/fleet/mod.rs +++ b/harmony/src/modules/fleet/mod.rs @@ -35,6 +35,8 @@ pub use assets::{ #[cfg(feature = "kvm")] pub use libvirt_pool::{HARMONY_FLEET_POOL_NAME, HarmonyFleetPool, ensure_harmony_fleet_pool}; pub use preflight::{check_fleet_smoke_preflight, check_fleet_smoke_preflight_for_arch}; -pub use setup_score::{FleetDeviceSetupConfig, FleetDeviceSetupScore}; +pub use setup_score::{ + FleetDeviceAuth, FleetDeviceSetupConfig, FleetDeviceSetupScore, HostsEntry, merge_hosts_file, +}; #[cfg(feature = "kvm")] pub use vm_score::ProvisionVmScore; diff --git a/harmony/src/modules/fleet/setup_score.rs b/harmony/src/modules/fleet/setup_score.rs index 3787b64f..8561bce9 100644 --- a/harmony/src/modules/fleet/setup_score.rs +++ b/harmony/src/modules/fleet/setup_score.rs @@ -34,6 +34,14 @@ use crate::score::Score; /// device is moved between fleet partitions: the config file is /// regenerated, byte-compare idempotency fires, the agent restarts, /// new labels propagate. +/// +/// **On `auth`.** Two authentication modes: +/// - [`FleetDeviceAuth::TomlShared`] — shared NATS user/password baked +/// into the TOML. Suitable for v0/dev only. +/// - [`FleetDeviceAuth::ZitadelJwt`] — per-device Zitadel machine-user +/// JWT-bearer. The keyfile is dropped onto the Pi at +/// `/etc/fleet-agent/zitadel-key.json` (mode 0640, owner +/// `fleet-agent`). The agent's `[credentials]` block points at it. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FleetDeviceSetupConfig { /// Stable device identifier. Written into the agent's TOML and @@ -49,25 +57,67 @@ pub struct FleetDeviceSetupConfig { pub labels: BTreeMap, /// NATS URLs the agent should connect to. Typically one entry. pub nats_urls: Vec, - /// Shared v0 credentials (Zitadel-issued per-device tokens in v0.2). - pub nats_user: String, - pub nats_pass: String, + /// Authentication for this device's NATS connection. + pub auth: FleetDeviceAuth, /// Local filesystem path to the cross-compiled `fleet-agent-v0` /// binary. The Score uploads it to the device and installs to /// `/usr/local/bin/fleet-agent`. Future v0.1: this becomes a /// `DownloadableAsset` pointing at CI-published artifacts. pub agent_binary_path: PathBuf, + /// `/etc/hosts` entries to add on the device. The fleet rehearsal + /// harness uses this so VMs on a libvirt NAT resolve + /// `sso.fleet.local` to the host's gateway IP — without it the + /// agent's HTTP client to Zitadel can't even DNS-resolve the + /// issuer URL. Empty by default; production deployments rely on + /// real DNS instead. + #[serde(default)] + pub hosts_entries: Vec, } +/// One line in `/etc/hosts`. Order doesn't matter (the file ends up +/// being a sorted dedup'd merge of these and any pre-existing +/// non-managed entries). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HostsEntry { + pub ip: String, + pub hostname: String, +} + +/// On-device NATS authentication mode for the agent. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum FleetDeviceAuth { + /// Username + password baked into the agent's TOML (legacy / dev). + TomlShared { + nats_user: String, + nats_pass: String, + }, + /// Zitadel machine-user JWT-bearer flow. The keyfile content is + /// what `ZitadelSetupScore` returns from + /// `ZitadelClientConfig::machine_keys.` — JSON keyfile as + /// emitted by Zitadel for `KEY_TYPE_JSON`. + ZitadelJwt { + /// Raw JSON keyfile content (will be written to the device). + machine_key_json: String, + /// Externally-visible Zitadel issuer URL. + oidc_issuer_url: String, + /// `aud` value for token-bearer requests. Typically the Zitadel + /// project ID. + audience: String, + /// Whether the agent's HTTP client accepts invalid TLS certs + /// (escape hatch for self-signed staging Zitadels). + #[serde(default)] + danger_accept_invalid_certs: bool, + }, +} + +/// Path the agent reads its Zitadel machine key from. Must match +/// `harmony-fleet-agent::config::default_zitadel_key_path`. +const ZITADEL_KEY_PATH: &str = "/etc/fleet-agent/zitadel-key.json"; + impl FleetDeviceSetupConfig { /// Render the agent's `/etc/fleet-agent/config.toml` content. pub fn render_toml(&self) -> String { - // Raw-string template with format! — the TOML escape rules for - // double-quoted strings are just `\` and `"`, handled by - // [`toml_escape`]. let device_id = toml_escape(&self.device_id.to_string()); - let nats_user = toml_escape(&self.nats_user); - let nats_pass = toml_escape(&self.nats_pass); let urls = self .nats_urls .iter() @@ -83,21 +133,46 @@ impl FleetDeviceSetupConfig { .map(|(k, v)| format!("{} = \"{}\"", toml_escape(k), toml_escape(v))) .collect::>() .join("\n"); + let credentials = match &self.auth { + FleetDeviceAuth::TomlShared { + nats_user, + nats_pass, + } => format!( + "[credentials]\n\ + type = \"toml-shared\"\n\ + nats_user = \"{}\"\n\ + nats_pass = \"{}\"\n", + toml_escape(nats_user), + toml_escape(nats_pass), + ), + FleetDeviceAuth::ZitadelJwt { + oidc_issuer_url, + audience, + danger_accept_invalid_certs, + .. + } => format!( + "[credentials]\n\ + type = \"zitadel-jwt\"\n\ + key_path = \"{}\"\n\ + oidc_issuer_url = \"{}\"\n\ + audience = \"{}\"\n\ + danger_accept_invalid_certs = {}\n", + ZITADEL_KEY_PATH, + toml_escape(oidc_issuer_url), + toml_escape(audience), + danger_accept_invalid_certs, + ), + }; format!( - r#"[agent] -device_id = "{device_id}" - -[credentials] -type = "toml-shared" -nats_user = "{nats_user}" -nats_pass = "{nats_pass}" - -[nats] -urls = [{urls}] - -[labels] -{labels} -"# + "[agent]\n\ + device_id = \"{device_id}\"\n\ + \n\ + {credentials}\n\ + [nats]\n\ + urls = [{urls}]\n\ + \n\ + [labels]\n\ + {labels}\n" ) } @@ -214,7 +289,8 @@ impl Interpret for FleetDeviceSetupInte proceeding will OVERWRITE it" ); warn!("[{tag}] diff (- existing, + desired):"); - let diff = similar::TextDiff::from_lines(existing.as_str(), desired_config.as_str()); + let diff = + similar::TextDiff::from_lines(existing.as_str(), desired_config.as_str()); let groups = diff.grouped_ops(2); for (idx, group) in groups.iter().enumerate() { if idx > 0 { @@ -250,6 +326,43 @@ impl Interpret for FleetDeviceSetupInte } } + // 0. /etc/hosts entries (rehearsal-only convenience). Done + // before package install so any package-manager mirror lookups + // that depend on these entries succeed. We render the line as + // a managed block bracketed by harmony markers — re-running + // is byte-stable and removing entries from the score deletes + // them from the file on next run. + if !cfg.hosts_entries.is_empty() { + info!( + "[{tag}] Step 1.5/7 — injecting {} /etc/hosts entr{} for rehearsal", + cfg.hosts_entries.len(), + if cfg.hosts_entries.len() == 1 { + "y" + } else { + "ies" + } + ); + let existing = FileFetcher::fetch_file(topology, "/etc/hosts") + .await + .map_err(wrap)?; + let merged = merge_hosts_file(existing.as_deref(), &cfg.hosts_entries); + let hosts_r = FileDelivery::ensure_file( + topology, + &FileSpec { + path: "/etc/hosts".to_string(), + source: FileSource::Content(merged), + owner: Some("root".to_string()), + group: Some("root".to_string()), + mode: Some(0o644), + }, + ) + .await + .map_err(wrap)?; + if hosts_r.changed { + change_count += 1; + } + } + // 1. Dependencies. info!("[{tag}] Step 2/7 — ensuring system packages: podman, systemd-container"); for pkg in ["podman", "systemd-container"] { @@ -298,9 +411,10 @@ impl Interpret for FleetDeviceSetupInte // 3. User-scoped podman socket. Required by `PodmanTopology` on // the agent so it reaches /run/user//podman/podman.sock. info!("[{tag}] Step 4/7 — activating user-scoped podman.socket"); - let socket_r = SystemdManager::ensure_user_unit_active(topology, "fleet-agent", "podman.socket") - .await - .map_err(wrap)?; + let socket_r = + SystemdManager::ensure_user_unit_active(topology, "fleet-agent", "podman.socket") + .await + .map_err(wrap)?; if socket_r.changed { change_count += 1; } @@ -330,7 +444,38 @@ impl Interpret for FleetDeviceSetupInte change_count += 1; } - // 5. /etc/fleet-agent/ + config.toml + // 5a. Drop the Zitadel machine keyfile when using JWT auth. + // Order: keyfile first, then config.toml — if both are new the + // agent's first systemd start finds the key already in place. + // Mode 0640 + group=fleet-agent so the non-root agent reads it + // via group permission (matches the corresponding Pod-side + // securityContext we use for the in-cluster callout). + let key_r = if let FleetDeviceAuth::ZitadelJwt { + machine_key_json, .. + } = &cfg.auth + { + info!("[{tag}] Step 6/7 — dropping Zitadel machine key to {ZITADEL_KEY_PATH}"); + let r = FileDelivery::ensure_file( + topology, + &FileSpec { + path: ZITADEL_KEY_PATH.to_string(), + source: FileSource::Content(machine_key_json.clone()), + owner: Some("fleet-agent".to_string()), + group: Some("fleet-agent".to_string()), + mode: Some(0o640), + }, + ) + .await + .map_err(wrap)?; + if r.changed { + change_count += 1; + } + r.changed + } else { + false + }; + + // 5b. /etc/fleet-agent/ + config.toml info!( "[{tag}] Step 6/7 — rendering /etc/fleet-agent/config.toml ({} NATS URL{}, {} label{})", cfg.nats_urls.len(), @@ -368,7 +513,7 @@ impl Interpret for FleetDeviceSetupInte } // 7. Restart the agent iff anything that affects it changed. - let needs_restart = toml_r.changed || unit_r.changed || binary_r.changed; + let needs_restart = toml_r.changed || unit_r.changed || binary_r.changed || key_r; let service_state = if needs_restart { info!("[{tag}] 🔄 Restarting fleet-agent (config/binary/unit changed)"); SystemdManager::restart_service(topology, "fleet-agent", SystemdScope::System) @@ -436,6 +581,67 @@ fn wrap(e: crate::executors::ExecutorError) -> InterpretError { InterpretError::new(e.to_string()) } +const HOSTS_BEGIN_MARKER: &str = "# >>> fleet-agent managed >>>"; +const HOSTS_END_MARKER: &str = "# <<< fleet-agent managed <<<"; + +/// Render an `/etc/hosts` file with a managed block at the end. +/// `existing` is whatever's currently on the device (or empty on a +/// fresh install). The managed block is bracketed by markers so we +/// can find and replace it on subsequent runs without disturbing the +/// rest of the file. Empty `entries` removes the block entirely. +pub fn merge_hosts_file(existing: Option<&str>, entries: &[HostsEntry]) -> String { + let base = existing.unwrap_or("127.0.0.1\tlocalhost\n::1\tlocalhost\n"); + // Strip any pre-existing managed block. + let stripped = strip_managed_block(base); + + if entries.is_empty() { + return ensure_trailing_newline(&stripped); + } + + let mut out = ensure_trailing_newline(&stripped); + out.push_str(HOSTS_BEGIN_MARKER); + out.push('\n'); + for e in entries { + out.push_str(&format!("{}\t{}\n", e.ip, e.hostname)); + } + out.push_str(HOSTS_END_MARKER); + out.push('\n'); + out +} + +fn strip_managed_block(s: &str) -> String { + let begin = match s.find(HOSTS_BEGIN_MARKER) { + Some(i) => i, + None => return s.to_string(), + }; + let after_begin = &s[begin..]; + let end_idx = match after_begin.find(HOSTS_END_MARKER) { + Some(i) => begin + i + HOSTS_END_MARKER.len(), + None => return s.to_string(), // malformed; leave alone + }; + // Eat the trailing newline of the end marker if present. + let mut tail_start = end_idx; + if s.as_bytes().get(tail_start) == Some(&b'\n') { + tail_start += 1; + } + let mut head = s[..begin].to_string(); + // Trim trailing newlines on head so we don't accumulate blanks. + while head.ends_with('\n') { + head.pop(); + } + head.push('\n'); + head.push_str(&s[tail_start..]); + head +} + +fn ensure_trailing_newline(s: &str) -> String { + if s.ends_with('\n') { + s.to_string() + } else { + format!("{s}\n") + } +} + #[cfg(test)] mod tests { use super::*; @@ -445,9 +651,29 @@ mod tests { device_id: Id::from("pi-42".to_string()), labels, nats_urls: vec!["nats://nats:4222".to_string()], - nats_user: "admin".to_string(), - nats_pass: "pw".to_string(), + auth: FleetDeviceAuth::TomlShared { + nats_user: "admin".to_string(), + nats_pass: "pw".to_string(), + }, agent_binary_path: PathBuf::from("/dev/null"), + hosts_entries: vec![], + } + } + + fn base_config_zitadel(labels: BTreeMap) -> FleetDeviceSetupConfig { + FleetDeviceSetupConfig { + device_id: Id::from("pi-42".to_string()), + labels, + nats_urls: vec!["wss://nats.staging.example.com/".to_string()], + auth: FleetDeviceAuth::ZitadelJwt { + machine_key_json: + r#"{"type":"sa","keyId":"k1","key":"-----PEM-----","userId":"u1"}"#.to_string(), + oidc_issuer_url: "https://zitadel.staging.example.com".to_string(), + audience: "366378028009259037".to_string(), + danger_accept_invalid_certs: false, + }, + agent_binary_path: PathBuf::from("/dev/null"), + hosts_entries: vec![], } } @@ -486,4 +712,99 @@ mod tests { let toml = base_config(labels).render_toml(); assert!(toml.contains(r#"group = "has\"quote""#)); } + + #[test] + fn render_toml_emits_zitadel_jwt_block() { + let mut labels = BTreeMap::new(); + labels.insert("group".to_string(), "site-a".to_string()); + let toml = base_config_zitadel(labels).render_toml(); + assert!(toml.contains(r#"type = "zitadel-jwt""#)); + assert!(toml.contains(&format!(r#"key_path = "{ZITADEL_KEY_PATH}""#))); + assert!(toml.contains(r#"oidc_issuer_url = "https://zitadel.staging.example.com""#)); + assert!(toml.contains(r#"audience = "366378028009259037""#)); + // The keyfile content does NOT go in the TOML — it's dropped + // separately to ZITADEL_KEY_PATH on the device. + assert!(!toml.contains("-----PEM-----")); + // toml-shared keys must not appear when zitadel-jwt is selected + // (defense-in-depth against an accidental dual-mode rendering). + assert!(!toml.contains("nats_user")); + assert!(!toml.contains("nats_pass")); + } + + #[test] + fn merge_hosts_inserts_managed_block() { + let entries = vec![HostsEntry { + ip: "192.168.122.1".to_string(), + hostname: "sso.fleet.local".to_string(), + }]; + let out = merge_hosts_file(None, &entries); + assert!(out.contains("127.0.0.1\tlocalhost")); + assert!(out.contains("# >>> fleet-agent managed >>>")); + assert!(out.contains("192.168.122.1\tsso.fleet.local")); + assert!(out.contains("# <<< fleet-agent managed <<<")); + } + + #[test] + fn merge_hosts_replaces_existing_managed_block() { + let existing = "127.0.0.1\tlocalhost\n\ + # >>> fleet-agent managed >>>\n\ + 10.0.0.1\told-host\n\ + # <<< fleet-agent managed <<<\n\ + 192.168.1.5\tunrelated\n"; + let entries = vec![HostsEntry { + ip: "192.168.122.1".to_string(), + hostname: "sso.fleet.local".to_string(), + }]; + let out = merge_hosts_file(Some(existing), &entries); + assert!( + !out.contains("old-host"), + "old managed entry must be removed" + ); + assert!(out.contains("192.168.122.1\tsso.fleet.local")); + // Non-managed entries survive. + assert!(out.contains("192.168.1.5\tunrelated")); + assert!(out.contains("127.0.0.1\tlocalhost")); + } + + #[test] + fn merge_hosts_empty_entries_strips_managed_block() { + let existing = "127.0.0.1\tlocalhost\n\ + # >>> fleet-agent managed >>>\n\ + 10.0.0.1\told-host\n\ + # <<< fleet-agent managed <<<\n"; + let out = merge_hosts_file(Some(existing), &[]); + assert!(!out.contains("old-host")); + assert!(!out.contains("fleet-agent managed")); + assert!(out.contains("127.0.0.1\tlocalhost")); + } + + #[test] + fn merge_hosts_byte_stable_across_runs() { + // Idempotency invariant: feeding the previous output back in + // yields byte-identical output. The Score's drift detection + // relies on this. + let entries = vec![HostsEntry { + ip: "192.168.122.1".to_string(), + hostname: "sso.fleet.local".to_string(), + }]; + let out1 = merge_hosts_file(None, &entries); + let out2 = merge_hosts_file(Some(&out1), &entries); + assert_eq!(out1, out2, "merge must be idempotent across re-runs"); + } + + #[test] + fn render_toml_zitadel_emits_danger_flag_inline() { + let mut labels = BTreeMap::new(); + labels.insert("group".to_string(), "x".to_string()); + let mut cfg = base_config_zitadel(labels); + if let FleetDeviceAuth::ZitadelJwt { + danger_accept_invalid_certs, + .. + } = &mut cfg.auth + { + *danger_accept_invalid_certs = true; + } + let toml = cfg.render_toml(); + assert!(toml.contains("danger_accept_invalid_certs = true")); + } } diff --git a/harmony/src/modules/helm/chart.rs b/harmony/src/modules/helm/chart.rs index cbdc7cb5..754b23bf 100644 --- a/harmony/src/modules/helm/chart.rs +++ b/harmony/src/modules/helm/chart.rs @@ -39,7 +39,10 @@ pub struct HelmChartScore { pub values_yaml: Option, pub create_namespace: bool, - /// Wether to run `helm upgrade --install` under the hood or only install when not present + /// `true` = run `helm install` (errors if the release already exists); + /// `false` = run `helm upgrade --install`, which is idempotent — helm + /// itself diffs the rendered chart against the live release and is a + /// no-op when nothing changed. pub install_only: bool, pub repository: Option, } @@ -206,37 +209,38 @@ impl Interpret for HelmChartInterpret { let ns_str = ns.to_string(); if let Some(installed_chart) = self.find_installed_release(topology, &ns_str)? { - return match self.expected_chart_field() { - Some(expected) - if Self::normalize_chart_field(&expected) - == Self::normalize_chart_field(&installed_chart) => - { - warn!( - "Helm release '{}' already installed at desired version ('{}'); skipping.", - self.score.release_name, installed_chart - ); - Ok(Outcome::success(format!( - "Helm Chart {} already at desired version", - self.score.release_name - ))) - } - Some(expected) => Err(InterpretError::new(format!( + // `install_only=true` means "deploy once, then leave it alone" + // — bootstrap operators (cert-manager, prometheus-operator, + // CRDs) use this. Skip the helm call entirely on re-runs. + if self.score.install_only { + warn!( + "Helm release '{}' already installed as '{}'; \ + install_only=true → skipping.", + self.score.release_name, installed_chart + ); + return Ok(Outcome::success(format!( + "Helm Chart {} already installed (install_only)", + self.score.release_name + ))); + } + // Pinned-version safety net: if the score pins a *different* + // version than what's installed, refuse to silently + // upgrade/downgrade — that's a manual decision. + if let Some(expected) = self.expected_chart_field() + && Self::normalize_chart_field(&expected) + != Self::normalize_chart_field(&installed_chart) + { + return Err(InterpretError::new(format!( "Helm release '{}' already installed as '{}', but score requests '{}'. \ Refusing to upgrade/downgrade; resolve manually.", self.score.release_name, installed_chart, expected - ))), - None => { - warn!( - "Helm release '{}' already installed as '{}'; score has no pinned \ - chart_version so skipping re-install.", - self.score.release_name, installed_chart - ); - Ok(Outcome::success(format!( - "Helm Chart {} already installed (version not pinned)", - self.score.release_name - ))) - } - }; + ))); + } + // Otherwise (no pin, or pinned and matching) fall through to + // `helm upgrade --install`. Helm is the source of truth on + // whether anything actually changed: a no-op upgrade is + // cheap, and changed values_yaml / values_overrides get + // applied automatically without the caller needing to opt in. } self.add_repo(topology)?; diff --git a/harmony/src/modules/inventory/discovery.rs b/harmony/src/modules/inventory/discovery.rs index bd3f7186..e800a211 100644 --- a/harmony/src/modules/inventory/discovery.rs +++ b/harmony/src/modules/inventory/discovery.rs @@ -1,16 +1,18 @@ use async_trait::async_trait; -use harmony_types::id::Id; +use harmony_inventory_agent::hwinfo::NetworkInterface; +use harmony_types::{firewall::LaggProtocol, id::Id}; use log::{error, info}; use serde::{Deserialize, Serialize}; use crate::{ data::Version, + hardware::PhysicalHost, infra::inventory::InventoryRepositoryFactory, interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, - inventory::{HostRole, Inventory}, + inventory::{HostRole, HostRoleMapping, Inventory}, modules::inventory::{HarmonyDiscoveryStrategy, LaunchDiscoverInventoryAgentScore}, score::Score, - topology::Topology, + topology::{BondConfig, NetworkConfig, Topology}, }; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -68,6 +70,7 @@ impl Interpret for DiscoverHostForRoleInterpret { continue; } + println!(); let ans = inquire::Select::new( &format!("Select the node to be used for role {:?}:", self.score.role), all_hosts, @@ -77,6 +80,18 @@ impl Interpret for DiscoverHostForRoleInterpret { match ans { Ok(choice) => { + // If the host is already mapped, tell the operator what's there + // and let them bail out before re-answering every prompt. + if let Some(existing) = host_repo.get_role_mapping(&choice.id).await? { + if !confirm_overwrite_existing_mapping(&choice, &existing)? { + info!( + "Cancelled: kept existing mapping for host {}", + choice.summary() + ); + continue; + } + } + info!( "Assigned role {:?} for node {}", self.score.role, @@ -103,11 +118,9 @@ impl Interpret for DiscoverHostForRoleInterpret { let display_refs: Vec<&str> = disk_choices.iter().map(|(d, _)| d.as_str()).collect(); - let disk_choice = inquire::Select::new( - &format!("Select the disk to use on host {}:", choice.summary()), - display_refs, - ) - .prompt(); + print_host_header(&choice); + let disk_choice = + inquire::Select::new("Select the disk to use:", display_refs).prompt(); match disk_choice { Ok(selected_display) => { @@ -117,8 +130,20 @@ impl Interpret for DiscoverHostForRoleInterpret { .map(|(_, name)| name.clone()) .unwrap(); info!("Selected disk {} for node {}", disk_name, choice.summary()); + + let network_config = prompt_network_config(&choice)?; + + // Visual break between the last prompt's answer and the + // logs that follow (save, loop progress, next iteration). + println!(); + host_repo - .save_role_mapping(&self.score.role, &choice, &disk_name) + .save_role_mapping( + &self.score.role, + &choice, + &disk_name, + &network_config, + ) .await?; chosen_hosts.push(choice); } @@ -179,3 +204,228 @@ impl Interpret for DiscoverHostForRoleInterpret { todo!() } } + +/// Show the existing role mapping for a host and ask whether to overwrite it. +/// +/// Returns `true` if the operator chose to overwrite (the caller proceeds with +/// disk/network prompts + a fresh save), `false` if they cancelled (caller +/// skips this host and continues the selection loop). +fn confirm_overwrite_existing_mapping( + host: &PhysicalHost, + existing: &HostRoleMapping, +) -> Result { + print_host_header(host); + println!("This host already has a role mapping:"); + println!(" Role: {}", existing.role); + println!( + " Installation disk: {}", + existing + .host_config + .installation_device + .as_deref() + .unwrap_or("(none)") + ); + match &existing.host_config.network_config.bond { + Some(bond) => println!(" Bond: {} on [{}]", bond.mode, bond.interfaces.join(", ")), + None => println!(" Bond: none"), + } + let blacklist = &existing.host_config.network_config.blacklisted_interfaces; + if !blacklist.is_empty() { + println!(" Blacklisted: {}", blacklist.join(", ")); + } + + let action = inquire::Select::new( + "What do you want to do?", + vec!["Update (overwrite the existing mapping)", "Cancel"], + ) + .prompt() + .map_err(|e| InterpretError::new(format!("Could not prompt: {e}")))?; + + Ok(action.starts_with("Update")) +} + +/// Print a blank line and a "Host: " header above the next prompt. +/// +/// Harmonizes every host-specific `inquire` question in the discovery flow so +/// the operator always sees which machine the prompt refers to — the `Host:` +/// line sits directly above the `? ...` question rendered by inquire. The +/// short-form summary omits the NIC list so the header fits on one screen +/// width; full NIC details still appear inside the bond/blacklist pickers. +fn print_host_header(host: &PhysicalHost) { + println!(); + println!("Host: {}", host.summary_short()); +} + +/// Interactively ask the user how the host's networking should be set up. +/// +/// Skips both prompts when the host has fewer than two network interfaces +/// — bonding requires at least two, and blacklisting a single NIC would leave +/// the host unreachable. The resulting [`NetworkConfig`] is persisted alongside +/// the role mapping so downstream Scores can act on it later. +fn prompt_network_config(host: &PhysicalHost) -> Result { + if host.network.len() < 2 { + info!( + "Host {} has {} network interface(s); skipping bond/blacklist prompts", + host.summary(), + host.network.len() + ); + return Ok(NetworkConfig::default()); + } + + let format_iface = |nic: &NetworkInterface| -> String { + let speed = nic + .speed_mbps + .map(|s| format!("{}Mbps", s)) + .unwrap_or_else(|| "?Mbps".to_string()); + let state = if nic.is_up { "up" } else { "down" }; + let ips = if nic.ipv4_addresses.is_empty() { + String::new() + } else { + format!(" [{}]", nic.ipv4_addresses.join(",")) + }; + format!( + "{} ({}) - {} - {} - driver {}{}", + nic.name, nic.mac_address, speed, state, nic.driver, ips + ) + }; + + let options: Vec<(String, String)> = host + .network + .iter() + .map(|nic| (format_iface(nic), nic.name.clone())) + .collect(); + + // --- Bond --- + print_host_header(host); + let wants_bond = inquire::Confirm::new("Configure a network bond?") + .with_default(false) + .prompt() + .map_err(|e| InterpretError::new(format!("Could not ask about bond: {e}")))?; + + let bond = if wants_bond { + let display_refs: Vec<&str> = options.iter().map(|(d, _)| d.as_str()).collect(); + print_host_header(host); + let selected = inquire::MultiSelect::new( + "Select the interfaces to include in the bond:", + display_refs, + ) + .with_validator(|choices: &[inquire::list_option::ListOption<&&str>]| { + if choices.len() < 2 { + Ok(inquire::validator::Validation::Invalid( + "Select at least two interfaces for a bond".into(), + )) + } else { + Ok(inquire::validator::Validation::Valid) + } + }) + .prompt() + .map_err(|e| InterpretError::new(format!("Could not select bond interfaces: {e}")))?; + + let interfaces: Vec = options + .iter() + .filter(|(display, _)| selected.iter().any(|s| *s == display.as_str())) + .map(|(_, name)| name.clone()) + .collect(); + + // Tuple-based picker so we can render fuller descriptions than the + // plain `Display` gives. Keep LACP first — it's the HA default. + let mode_choices: Vec<(String, LaggProtocol)> = vec![ + ( + "LACP (802.3ad) — negotiated aggregation with the switch".to_string(), + LaggProtocol::Lacp, + ), + ( + "Failover — single active link, others standby".to_string(), + LaggProtocol::Failover, + ), + ( + "Load Balance — distribute traffic across links".to_string(), + LaggProtocol::LoadBalance, + ), + ( + "Round Robin — rotate through links per packet".to_string(), + LaggProtocol::RoundRobin, + ), + ]; + let display_refs: Vec<&str> = mode_choices.iter().map(|(d, _)| d.as_str()).collect(); + print_host_header(host); + let selected_display = inquire::Select::new("Select the bond mode:", display_refs) + .with_starting_cursor(0) + .prompt() + .map_err(|e| InterpretError::new(format!("Could not select bond mode: {e}")))?; + let mode = mode_choices + .iter() + .find(|(d, _)| d.as_str() == selected_display) + .map(|(_, p)| p.clone()) + .expect("selected display must map back to a LaggProtocol"); + + info!( + "Bond configured for host {} on interfaces [{}] with mode {}", + host.summary(), + interfaces.join(", "), + mode + ); + Some(BondConfig { interfaces, mode }) + } else { + None + }; + + // --- Blacklist --- + // Candidates exclude any interface already claimed by the bond. + let bond_members: Vec<&String> = bond + .as_ref() + .map(|b| b.interfaces.iter().collect()) + .unwrap_or_default(); + + let blacklist_candidates: Vec<(String, String)> = options + .iter() + .filter(|(_, name)| !bond_members.iter().any(|b| *b == name)) + .cloned() + .collect(); + + let blacklisted_interfaces = if blacklist_candidates.is_empty() { + Vec::new() + } else { + print_host_header(host); + let wants_blacklist = inquire::Confirm::new("Blacklist any remaining interface?") + .with_default(false) + .prompt() + .map_err(|e| InterpretError::new(format!("Could not ask about blacklist: {e}")))?; + + if wants_blacklist { + let display_refs: Vec<&str> = blacklist_candidates + .iter() + .map(|(d, _)| d.as_str()) + .collect(); + print_host_header(host); + let selected = + inquire::MultiSelect::new("Select the interfaces to blacklist:", display_refs) + .prompt() + .map_err(|e| { + InterpretError::new(format!("Could not select blacklisted interfaces: {e}")) + })?; + + let names: Vec = blacklist_candidates + .iter() + .filter(|(display, _)| selected.iter().any(|s| *s == display.as_str())) + .map(|(_, name)| name.clone()) + .collect(); + + if !names.is_empty() { + info!( + "Blacklisted interfaces on host {}: {}", + host.summary(), + names.join(", ") + ); + } + names + } else { + Vec::new() + } + }; + + Ok(NetworkConfig { + bond, + blacklisted_interfaces, + }) +} diff --git a/harmony/src/modules/inventory/mod.rs b/harmony/src/modules/inventory/mod.rs index 1bdccd33..acfa7aca 100644 --- a/harmony/src/modules/inventory/mod.rs +++ b/harmony/src/modules/inventory/mod.rs @@ -35,6 +35,37 @@ use crate::{ }; use harmony_types::id::Id; +/// Build the `labels` list for a host discovered via the inventory agent. +/// +/// Always includes the `discovered-by` provenance label. Also promotes the +/// agent's `Chipset { vendor, name }` into a `system-product-name` label so +/// `PhysicalHost::summary()` can show something like "LENOVO 3136" instead of +/// falling back to the generic "Server" category string. Skips that label when +/// both chipset fields are blank. +fn build_discovered_host_labels(chipset: &harmony_inventory_agent::hwinfo::Chipset) -> Vec