From 8de61859b4a18a6ea0086ad75a3aecd9c78aedeb Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Mon, 22 Jun 2026 22:02:20 -0400 Subject: [PATCH 1/2] feat(zitadel): full-fidelity provisioning + app deploy ergonomics Extend the zitadel module and harmony_app so an app deploy crate can stand up and fully provision a live Zitadel, and deploy a compose app, on autoprovisioned k3d or a remote tenant. - ZitadelSetupScore: human users, org/instance memberships (ORG_OWNER, IAM_LOGIN_CLIENT), machine client-secret minting, and a self-established port-forward so the management API is reachable on k3d (no ingress). - ZitadelCredentialsExportScore: publish provisioned project id / client ids / client secret / machine key into a ConfigMap + Secret for apps to reference. - harmony_app: AppContext autoprovision mode + K8sAnywhereConfig::local_k3d(); import_to_k3d ensures the cluster exists (fixes ship ordering); SecretFileMount chart support to mount a secret as a file. --- examples/fleet_auth_callout/src/lib.rs | 5 + examples/fleet_e2e_demo/src/lib.rs | 10 + examples/fleet_staging_deploy/src/lib.rs | 1 + examples/fleet_staging_install/src/main.rs | 5 + examples/harmony_sso/src/main.rs | 1 + fleet/harmony-fleet-e2e/src/callout.rs | 6 + .../topology/k8s_anywhere/k8s_anywhere.rs | 16 + harmony/src/modules/fleet/setup_score.rs | 1 + harmony/src/modules/zitadel/mod.rs | 5 +- harmony/src/modules/zitadel/setup.rs | 607 +++++++++++++++++- harmony_app/src/capabilities.rs | 1 + harmony_app/src/chart.rs | 49 +- harmony_app/src/context.rs | 76 ++- harmony_app/src/lib.rs | 2 +- harmony_app/src/publish.rs | 18 + harmony_app/tests/helm_render.rs | 1 + 16 files changed, 771 insertions(+), 33 deletions(-) diff --git a/examples/fleet_auth_callout/src/lib.rs b/examples/fleet_auth_callout/src/lib.rs index b993d0bd..de4a0237 100644 --- a/examples/fleet_auth_callout/src/lib.rs +++ b/examples/fleet_auth_callout/src/lib.rs @@ -268,6 +268,11 @@ pub async fn bring_up_stack() -> Result { }, ], groups_claim_action: false, + human_users: vec![], + org_members: vec![], + instance_members: vec![], + machine_secrets: vec![], + port_forward_service: None, machine_users: vec![ ZitadelMachineUser { username: ADMIN_USERNAME.to_string(), diff --git a/examples/fleet_e2e_demo/src/lib.rs b/examples/fleet_e2e_demo/src/lib.rs index 40b8cd19..60c09725 100644 --- a/examples/fleet_e2e_demo/src/lib.rs +++ b/examples/fleet_e2e_demo/src/lib.rs @@ -186,6 +186,11 @@ pub async fn bring_up_full_stack(opts: E2eDemoOpts) -> Result { }, ], groups_claim_action: false, + human_users: vec![], + org_members: vec![], + instance_members: vec![], + machine_secrets: vec![], + port_forward_service: None, machine_users: vec![ ZitadelMachineUser { username: ADMIN_USERNAME.to_string(), @@ -356,6 +361,11 @@ async fn provision_device( api_apps: vec![], roles: vec![], groups_claim_action: false, + human_users: vec![], + org_members: vec![], + instance_members: vec![], + machine_secrets: vec![], + port_forward_service: None, machine_users: vec![ZitadelMachineUser { username: username.clone(), name: format!("Fleet Device {device_id}"), diff --git a/examples/fleet_staging_deploy/src/lib.rs b/examples/fleet_staging_deploy/src/lib.rs index 47b2f923..59069767 100644 --- a/examples/fleet_staging_deploy/src/lib.rs +++ b/examples/fleet_staging_deploy/src/lib.rs @@ -298,6 +298,7 @@ async fn provision_zitadel_project( // device-count-agnostic. groups_claim_action: false, machine_users: vec![], + ..Default::default() }; setup .interpret(&Inventory::autoload(), topology) diff --git a/examples/fleet_staging_install/src/main.rs b/examples/fleet_staging_install/src/main.rs index c13e4f62..3215b3f9 100644 --- a/examples/fleet_staging_install/src/main.rs +++ b/examples/fleet_staging_install/src/main.rs @@ -134,6 +134,11 @@ async fn main() -> Result<()> { // the Action surfaces role keys as the `groups` claim OpenBao // binds against (ADR-025). groups_claim_action: true, + human_users: vec![], + org_members: vec![], + instance_members: vec![], + machine_secrets: vec![], + port_forward_service: None, // Device-code OIDC app for human admin login from // `fleet_device_enroll`'s SSO flow. The numeric `client_id` Zitadel // generates is read back below and printed for `--admin-oidc-client-id`. diff --git a/examples/harmony_sso/src/main.rs b/examples/harmony_sso/src/main.rs index c15650ef..71318940 100644 --- a/examples/harmony_sso/src/main.rs +++ b/examples/harmony_sso/src/main.rs @@ -313,6 +313,7 @@ async fn main() -> anyhow::Result<()> { roles: vec![], groups_claim_action: false, machine_users: vec![], + ..Default::default() } .interpret(&Inventory::autoload(), &topology) .await diff --git a/fleet/harmony-fleet-e2e/src/callout.rs b/fleet/harmony-fleet-e2e/src/callout.rs index d12f4357..a841f4e5 100644 --- a/fleet/harmony-fleet-e2e/src/callout.rs +++ b/fleet/harmony-fleet-e2e/src/callout.rs @@ -194,6 +194,11 @@ pub async fn bring_up_zitadel( // `groups` array claim OpenBao binds against (ADR-025) — this // bring-up is its live verification. groups_claim_action: true, + human_users: vec![], + org_members: vec![], + instance_members: vec![], + machine_secrets: vec![], + port_forward_service: None, applications: vec![], api_apps: vec![ZitadelApiApp { project_name: PROJECT_NAME.to_string(), @@ -249,6 +254,7 @@ fn zitadel_connection() -> ZitadelSetupScore { roles: vec![], machine_users: vec![], groups_claim_action: false, + ..Default::default() } } diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index eaaf6d26..73184b7a 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -1155,6 +1155,22 @@ impl K8sAnywhereConfig { } } + /// Autoprovision a local k3d cluster (the offline dev/CI path): install + /// k3d if absent, never touch a remote cluster. The env-free sibling of + /// [`from_env`](K8sAnywhereTopology::from_env) for programmatic callers + /// (the app/context layer) that select the local target explicitly. + pub fn local_k3d() -> Self { + Self { + kubeconfig: None, + use_system_kubeconfig: false, + autoinstall: true, + use_local_k3d: true, + harmony_profile: "dev".to_string(), + k8s_context: None, + public_domain: None, + } + } + /// Reads an environment variable `env_var` and parses its content : /// Comma-separated `key=value` pairs, e.g., /// `kubeconfig=/path/to/primary.kubeconfig,context=primary-ctx` diff --git a/harmony/src/modules/fleet/setup_score.rs b/harmony/src/modules/fleet/setup_score.rs index 4ef1c9c4..ab689343 100644 --- a/harmony/src/modules/fleet/setup_score.rs +++ b/harmony/src/modules/fleet/setup_score.rs @@ -397,6 +397,7 @@ async fn resolve_zitadel_enroll( roles: vec![], machine_users: vec![], groups_claim_action: false, + ..Default::default() }; info!( diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index fb34375e..0076ab4c 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -6,8 +6,9 @@ pub use admin_auth::{ADMIN_API_SCOPES, DeviceCodeError, DeviceCodeFlowConfig, de pub use device_groups::ZitadelDeviceGroups; pub use setup::{ MachineKeyType, MintedDeviceCredentials, ZitadelApiApp, ZitadelAppType, ZitadelApplication, - ZitadelClientConfig, ZitadelClientIdExportScore, ZitadelMachineUser, ZitadelRole, - ZitadelScheme, ZitadelSetupScore, mint_device_credentials, + ZitadelClientConfig, ZitadelClientIdExportScore, ZitadelCredentialsExportScore, + ZitadelHumanUser, ZitadelMachineUser, ZitadelMember, ZitadelRole, ZitadelScheme, + ZitadelSetupScore, mint_device_credentials, }; use harmony_k8s::KubernetesDistribution; diff --git a/harmony/src/modules/zitadel/setup.rs b/harmony/src/modules/zitadel/setup.rs index 75f902a3..8f23897f 100644 --- a/harmony/src/modules/zitadel/setup.rs +++ b/harmony/src/modules/zitadel/setup.rs @@ -165,6 +165,34 @@ pub struct ZitadelMachineUser { pub grant_roles: Vec, } +/// A human (password) user — the people who log in to the app. Created via +/// `users/human/_import` (email pre-verified, no first-login password change), +/// then granted `grant_roles` on `project_name`. Idempotent: an existing user +/// has its password reset to the declared value so re-provisioning is safe. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ZitadelHumanUser { + /// Username + email (the app keys identities by email). + pub email: String, + pub first_name: String, + pub last_name: String, + pub password: String, + /// Project to grant `grant_roles` on (required when `grant_roles` is set). + #[serde(default)] + pub project_name: Option, + #[serde(default)] + pub grant_roles: Vec, +} + +/// A membership grant: give a user `roles` at the org or instance level. +/// Used for `ORG_OWNER` (org-scoped management) and `IAM_LOGIN_CLIENT` +/// (instance-scoped, authorizes the v2 Session/OIDC auth-request APIs). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ZitadelMember { + /// Username of an already-provisioned (machine or human) user. + pub username: String, + pub roles: Vec, +} + /// Score that provisions identity resources in a deployed Zitadel instance. /// /// This is the "day two" counterpart to [`super::ZitadelScore`] (which @@ -241,6 +269,26 @@ pub struct ZitadelSetupScore { /// Machine users to provision (with optional keys + role grants). #[serde(default)] pub machine_users: Vec, + /// Human (password) users to provision + grant. + #[serde(default)] + pub human_users: Vec, + /// Org-level memberships to grant (e.g. `ORG_OWNER` to a backend SA). + #[serde(default)] + pub org_members: Vec, + /// Instance-level memberships to grant (e.g. `IAM_LOGIN_CLIENT`). + #[serde(default)] + pub instance_members: Vec, + /// Usernames of machine users to mint an OIDC client secret for + /// (client_credentials grant). The id + secret land in + /// [`ZitadelClientConfig::machine_client_ids`] / `machine_secrets`. + #[serde(default)] + pub machine_secrets: Vec, + /// When set, the interpret opens a port-forward to a pod of this Service + /// (by `app.kubernetes.io/name=` in `namespace`) and talks to the + /// management API through it — so a host-side run can provision an + /// in-cluster Zitadel with no ingress (the k3d path). Overrides `endpoint`. + #[serde(default)] + pub port_forward_service: Option, /// Provision the token Action that flattens project role keys into /// a `groups` string-array claim — the shape OpenBao's /// `groups_claim` consumes (ADR-025) — and attach it to the org's @@ -250,6 +298,30 @@ pub struct ZitadelSetupScore { pub groups_claim_action: bool, } +impl Default for ZitadelSetupScore { + fn default() -> Self { + Self { + host: String::new(), + scheme: ZitadelScheme::default(), + port: None, + skip_tls: false, + endpoint: None, + namespace: default_zitadel_namespace(), + admin_org_id: None, + applications: Vec::new(), + api_apps: Vec::new(), + roles: Vec::new(), + machine_users: Vec::new(), + human_users: Vec::new(), + org_members: Vec::new(), + instance_members: Vec::new(), + machine_secrets: Vec::new(), + port_forward_service: None, + groups_claim_action: false, + } + } +} + /// Function name doubles as the Action name — Zitadel requires the /// script's entry function to match. pub const GROUPS_CLAIM_ACTION_NAME: &str = "harmonyGroupsClaim"; @@ -285,6 +357,16 @@ pub struct ZitadelClientConfig { /// `::` for serde simplicity. #[serde(default)] pub user_grants: HashMap, + /// `username` → OIDC client_id minted with its client secret. + #[serde(default)] + pub machine_client_ids: HashMap, + /// `username` → OIDC client secret (client_credentials grant). Secret + /// material: this cache file must be treated as a secret. + #[serde(default)] + pub machine_secrets: HashMap, + /// `email` → human `userId`. + #[serde(default)] + pub human_user_ids: HashMap, } impl ZitadelClientConfig { @@ -336,6 +418,16 @@ impl ZitadelClientConfig { fn user_grant_key(username: &str, project_name: &str) -> String { format!("{username}::{project_name}") } + + /// OIDC client_id minted alongside a machine user's client secret. + pub fn machine_client_id(&self, username: &str) -> Option<&String> { + self.machine_client_ids.get(username) + } + + /// OIDC client secret for a machine user (client_credentials grant). + pub fn machine_secret(&self, username: &str) -> Option<&String> { + self.machine_secrets.get(username) + } } impl Score for ZitadelSetupScore { @@ -469,6 +561,14 @@ struct UserGrantCreateResponse { user_grant_id: String, } +#[derive(Deserialize)] +struct MachineSecretResponse { + #[serde(rename = "clientId")] + client_id: String, + #[serde(rename = "clientSecret")] + client_secret: String, +} + impl ZitadelSetupInterpret { /// Build the request URL for `path`. When `endpoint` is set, returns /// ``; otherwise `://[:]`, @@ -1769,6 +1869,266 @@ impl ZitadelSetupInterpret { Ok(()) } + + /// Resolve a user id by username, live-querying then falling back to the + /// cache (covers machine + human users). + async fn resolve_user_id( + &self, + client: &reqwest::Client, + pat: &str, + username: &str, + config: &ZitadelClientConfig, + ) -> Result { + if let Some(id) = self + .find_machine_user(client, pat, username) + .await + .map_err(InterpretError::new)? + { + return Ok(id); + } + config + .machine_user_ids + .get(username) + .or_else(|| config.human_user_ids.get(username)) + .cloned() + .ok_or_else(|| { + InterpretError::new(format!( + "member references unknown user '{username}' — provision it first" + )) + }) + } + + /// Treat a membership response as success when the user is already a member + /// (409 / "already") — the only non-idempotent shape of these endpoints. + async fn ok_or_already_member( + &self, + resp: reqwest::Response, + who: &str, + scope: &str, + ) -> Result<(), InterpretError> { + if resp.status().is_success() { + return Ok(()); + } + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + if status == reqwest::StatusCode::CONFLICT || body.to_lowercase().contains("already") { + debug!("[ZitadelSetup] '{who}' is already a {scope} member"); + return Ok(()); + } + Err(InterpretError::new(format!( + "Add {scope} member '{who}' failed: {body}" + ))) + } + + /// Create a human (password) user via `users/human/_import`. + async fn create_human_user( + &self, + client: &reqwest::Client, + pat: &str, + user: &ZitadelHumanUser, + ) -> Result { + let display_name = format!("{} {}", user.first_name, user.last_name); + let resp = self + .post(client, "/management/v1/users/human/_import") + .bearer_auth(pat) + .json(&serde_json::json!({ + "userName": user.email, + "profile": { + "firstName": user.first_name, + "lastName": user.last_name, + "displayName": display_name, + }, + "email": { "email": user.email, "isEmailVerified": true }, + "password": user.password, + "passwordChangeRequired": false, + })) + .send() + .await + .map_err(|e| format!("Failed to create human user: {e}"))?; + if !resp.status().is_success() { + let body = resp.text().await.unwrap_or_default(); + return Err(format!("Create human user '{}' failed: {body}", user.email)); + } + let parsed: UserCreateResponse = + serde_json::from_str(&resp.text().await.map_err(|e| format!("Read body: {e}"))?) + .map_err(|e| format!("Parse human user response: {e}"))?; + Ok(parsed.user_id) + } + + /// Ensure a human user exists (create or reset its password) and holds its + /// `grant_roles` on `project_name`. + async fn ensure_human_user( + &self, + client: &reqwest::Client, + pat: &str, + user: &ZitadelHumanUser, + config: &mut ZitadelClientConfig, + ) -> Result<(), InterpretError> { + let user_id = match self + .find_machine_user(client, pat, &user.email) + .await + .map_err(InterpretError::new)? + { + Some(id) => { + // Reset to the declared password so a re-provision is deterministic. + let resp = self + .request( + client, + reqwest::Method::POST, + &format!("/v2/users/{id}/password"), + ) + .bearer_auth(pat) + .json(&serde_json::json!({ + "newPassword": { "password": user.password, "changeRequired": false } + })) + .send() + .await + .map_err(|e| InterpretError::new(format!("Reset password: {e}")))?; + if !resp.status().is_success() { + let body = resp.text().await.unwrap_or_default(); + warn!( + "[ZitadelSetup] Could not reset password for '{}': {body}", + user.email + ); + } + id + } + None => self + .create_human_user(client, pat, user) + .await + .map_err(InterpretError::new)?, + }; + config + .human_user_ids + .insert(user.email.clone(), user_id.clone()); + info!( + "[ZitadelSetup] Human user '{}' resolved: {user_id}", + user.email + ); + + if !user.grant_roles.is_empty() { + let project_name = user.project_name.as_ref().ok_or_else(|| { + InterpretError::new(format!( + "human user '{}' has grant_roles but no project_name", + user.email + )) + })?; + let project_id = self + .ensure_project(client, pat, project_name, config) + .await?; + let grant_key = ZitadelClientConfig::user_grant_key(&user.email, project_name); + let grant_id = if let Some(id) = self + .find_user_grant(client, pat, &user_id, &project_id) + .await + .map_err(InterpretError::new)? + { + id + } else { + self.create_user_grant(client, pat, &user_id, &project_id, &user.grant_roles) + .await + .map_err(InterpretError::new)? + }; + config.user_grants.insert(grant_key, grant_id); + } + Ok(()) + } + + /// Mint (and cache) an OIDC client secret for a machine user, enabling the + /// client_credentials grant. Zitadel returns the secret once only, so an + /// existing cache entry is reused rather than rotated. + async fn ensure_machine_secret( + &self, + client: &reqwest::Client, + pat: &str, + username: &str, + config: &mut ZitadelClientConfig, + ) -> Result<(), InterpretError> { + if config.machine_secrets.contains_key(username) { + debug!("[ZitadelSetup] Reusing cached client secret for '{username}'"); + return Ok(()); + } + let user_id = self + .find_machine_user(client, pat, username) + .await + .map_err(InterpretError::new)? + .ok_or_else(|| { + InterpretError::new(format!( + "machine_secrets references unknown user '{username}' — declare it in machine_users" + )) + })?; + let resp = self + .put(client, &format!("/management/v1/users/{user_id}/secret")) + .bearer_auth(pat) + .json(&serde_json::json!({})) + .send() + .await + .map_err(|e| InterpretError::new(format!("Generate client secret: {e}")))?; + if !resp.status().is_success() { + let body = resp.text().await.unwrap_or_default(); + return Err(InterpretError::new(format!( + "Generate client secret for '{username}' failed: {body}" + ))); + } + let parsed: MachineSecretResponse = serde_json::from_str( + &resp + .text() + .await + .map_err(|e| InterpretError::new(format!("Read body: {e}")))?, + ) + .map_err(|e| InterpretError::new(format!("Parse secret response: {e}")))?; + config + .machine_client_ids + .insert(username.to_string(), parsed.client_id); + config + .machine_secrets + .insert(username.to_string(), parsed.client_secret); + info!("[ZitadelSetup] Client secret minted for '{username}'"); + Ok(()) + } + + /// Grant a user org-level `roles` (e.g. `ORG_OWNER`). + async fn ensure_org_member( + &self, + client: &reqwest::Client, + pat: &str, + member: &ZitadelMember, + config: &ZitadelClientConfig, + ) -> Result<(), InterpretError> { + let user_id = self + .resolve_user_id(client, pat, &member.username, config) + .await?; + let resp = self + .post(client, "/management/v1/orgs/me/members") + .bearer_auth(pat) + .json(&serde_json::json!({ "userId": user_id, "roles": member.roles })) + .send() + .await + .map_err(|e| InterpretError::new(format!("Add org member: {e}")))?; + self.ok_or_already_member(resp, &member.username, "org") + .await + } + + /// Grant a user instance-level `roles` (e.g. `IAM_LOGIN_CLIENT`). + async fn ensure_instance_member( + &self, + client: &reqwest::Client, + pat: &str, + member: &ZitadelMember, + config: &ZitadelClientConfig, + ) -> Result<(), InterpretError> { + let user_id = self + .resolve_user_id(client, pat, &member.username, config) + .await?; + let resp = self + .post(client, "/admin/v1/members") + .bearer_auth(pat) + .json(&serde_json::json!({ "userId": user_id, "roles": member.roles })) + .send() + .await + .map_err(|e| InterpretError::new(format!("Add instance member: {e}")))?; + self.ok_or_already_member(resp, &member.username, "instance") + .await + } } /// Result of [`mint_device_credentials`]. @@ -1889,7 +2249,51 @@ impl Interpret for ZitadelSetupInterpret { let pat = self.read_admin_pat(&k8s).await?; debug!("[ZitadelSetup] Admin PAT loaded from secret"); - let client = self.http_client().map_err(InterpretError::new)?; + // The k3d path: no ingress reaches the in-cluster Zitadel from this + // host-side run, so open a port-forward to one of its pods and talk to + // the management API through 127.0.0.1. `_pf` must outlive every call + // below. `me` is `self` with the resolved endpoint patched in. + let _pf; + let me: std::borrow::Cow<'_, ZitadelSetupInterpret> = + if let Some(svc) = &self.score.port_forward_service { + let label = format!("app.kubernetes.io/name={svc}"); + let pods = k8s + .list_resources::( + Some(&self.score.namespace), + Some(kube::api::ListParams::default().labels(&label)), + ) + .await + .map_err(|e| InterpretError::new(format!("listing '{svc}' pods: {e}")))?; + // The chart's init/setup Job pods share this label; only the + // serving Deployment pod is Running, so filter on phase. + let pod = pods + .items + .into_iter() + .find(|p| p.status.as_ref().and_then(|s| s.phase.as_deref()) == Some("Running")) + .and_then(|p| p.metadata.name) + .ok_or_else(|| { + InterpretError::new(format!( + "no Running pod for service '{svc}' in namespace '{}'", + self.score.namespace + )) + })?; + // Zitadel serves http on 8080 in-cluster (chart default). + let handle = k8s + .port_forward(&pod, &self.score.namespace, 0, 8080) + .await + .map_err(|e| InterpretError::new(format!("port-forward to '{pod}': {e}")))?; + let endpoint = format!("http://127.0.0.1:{}", handle.port()); + info!("[ZitadelSetup] Provisioning via port-forward {endpoint} -> {svc}"); + _pf = Some(handle); + let mut score = self.score.clone(); + score.endpoint = Some(endpoint); + std::borrow::Cow::Owned(ZitadelSetupInterpret { score }) + } else { + _pf = None; + std::borrow::Cow::Borrowed(self) + }; + + let client = me.http_client().map_err(InterpretError::new)?; // Block on /debug/ready AND on the management API actually // responding before issuing real work. Two distinct races to @@ -1900,19 +2304,19 @@ impl Interpret for ZitadelSetupInterpret { // flips to 200, so the first POST to /management/... can race // and get a 503 with `code:14` + `transport: Error while // dialing`. Both are now handled in `wait_until_ready`. - self.wait_until_ready(&client, &pat).await?; + me.wait_until_ready(&client, &pat).await?; let mut config = ZitadelClientConfig::load().unwrap_or_default(); let mut details = Vec::new(); - for app in &self.score.applications { - let client_id = self.ensure_app(&client, &pat, app, &mut config).await?; + for app in &me.score.applications { + let client_id = me.ensure_app(&client, &pat, app, &mut config).await?; details.push(format!("oidc_app:{}={}", app.app_name, client_id)); } - for api_app in &self.score.api_apps { - self.ensure_api_app(&client, &pat, api_app, &mut config) + for api_app in &me.score.api_apps { + me.ensure_api_app(&client, &pat, api_app, &mut config) .await?; details.push(format!( "api_app:{}@{}", @@ -1920,13 +2324,13 @@ impl Interpret for ZitadelSetupInterpret { )); } - for role in &self.score.roles { - self.ensure_role(&client, &pat, role, &mut config).await?; + for role in &me.score.roles { + me.ensure_role(&client, &pat, role, &mut config).await?; details.push(format!("role:{}@{}", role.key, role.project_name)); } - for user in &self.score.machine_users { - self.ensure_machine_user(&client, &pat, user, &mut config) + for user in &me.score.machine_users { + me.ensure_machine_user(&client, &pat, user, &mut config) .await?; details.push(format!("machine_user:{}", user.username)); if user.create_pat { @@ -1934,8 +2338,32 @@ impl Interpret for ZitadelSetupInterpret { } } - if self.score.groups_claim_action { - self.ensure_groups_claim_action(&client, &pat).await?; + for user in &me.score.human_users { + me.ensure_human_user(&client, &pat, user, &mut config) + .await?; + details.push(format!("human_user:{}", user.email)); + } + + // Secrets first (the user must exist), then org/instance memberships. + for username in &me.score.machine_secrets { + me.ensure_machine_secret(&client, &pat, username, &mut config) + .await?; + details.push(format!("machine_secret:{username}")); + } + + for member in &me.score.org_members { + me.ensure_org_member(&client, &pat, member, &config).await?; + details.push(format!("org_member:{}", member.username)); + } + + for member in &me.score.instance_members { + me.ensure_instance_member(&client, &pat, member, &config) + .await?; + details.push(format!("instance_member:{}", member.username)); + } + + if me.score.groups_claim_action { + me.ensure_groups_claim_action(&client, &pat).await?; details.push(format!("action:{GROUPS_CLAIM_ACTION_NAME}")); } @@ -2060,6 +2488,159 @@ impl Interpret for ZitadelClientIdExportInterpret { } } +// --------------------------------------------------------------------------- +// ZitadelCredentialsExportScore — publish a bundle of provisioned values +// --------------------------------------------------------------------------- + +/// Publish a set of values provisioned by [`ZitadelSetupScore`] (read from its +/// cache) into a ConfigMap (non-secret: project id, client ids) and a Secret +/// (client secrets), so a deployed app can consume them by reference +/// (`configMapKeyRef` / `secretKeyRef`). The generalized sibling of +/// [`ZitadelClientIdExportScore`] for apps that need the whole login bundle, +/// not just one client_id. Must run after `ZitadelSetupScore` in the same run. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ZitadelCredentialsExportScore { + pub namespace: String, + pub configmap_name: String, + pub secret_name: String, + /// Project whose id is exported to the ConfigMap under `project_id_key`. + #[serde(default)] + pub project_name: Option, + pub project_id_key: String, + /// `(configmap_key, app_name)` — each OIDC app's client_id. + #[serde(default)] + pub app_client_ids: Vec<(String, String)>, + /// `(configmap_key, username)` — each machine user's minted client_id. + #[serde(default)] + pub machine_client_ids: Vec<(String, String)>, + /// `(secret_key, username)` — each machine user's minted client secret. + #[serde(default)] + pub machine_secrets: Vec<(String, String)>, + /// `(secret_key, username)` — each machine user's JSON key file content, + /// for consumers that authenticate via the JWT-profile (private key) grant. + #[serde(default)] + pub machine_keys: Vec<(String, String)>, +} + +impl Score for ZitadelCredentialsExportScore { + fn name(&self) -> String { + format!("ZitadelCredentialsExport({})", self.configmap_name) + } + fn create_interpret(&self) -> Box> { + Box::new(ZitadelCredentialsExportInterpret { + score: self.clone(), + }) + } +} + +#[derive(Debug, Clone)] +struct ZitadelCredentialsExportInterpret { + score: ZitadelCredentialsExportScore, +} + +#[async_trait] +impl Interpret for ZitadelCredentialsExportInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + use crate::modules::k8s::resource::K8sResourceScore; + use k8s_openapi::api::core::v1::{ConfigMap, Secret}; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + use std::collections::BTreeMap; + + let s = &self.score; + let cache = ZitadelClientConfig::load().ok_or_else(|| { + InterpretError::new("Zitadel cache not found — run ZitadelSetupScore first".to_string()) + })?; + let missing = |what: &str, key: &str| { + InterpretError::new(format!( + "{what} '{key}' not in Zitadel cache — was it provisioned?" + )) + }; + + let mut cm_data = BTreeMap::new(); + if let Some(project) = &s.project_name { + let id = cache + .project_id_by_name(project) + .ok_or_else(|| missing("project", project))?; + cm_data.insert(s.project_id_key.clone(), id.clone()); + } + for (key, app) in &s.app_client_ids { + let id = cache + .client_id(app) + .ok_or_else(|| missing("app client_id", app))?; + cm_data.insert(key.clone(), id.clone()); + } + for (key, user) in &s.machine_client_ids { + let id = cache + .machine_client_id(user) + .ok_or_else(|| missing("machine client_id", user))?; + cm_data.insert(key.clone(), id.clone()); + } + + let mut secret_data = BTreeMap::new(); + for (key, user) in &s.machine_secrets { + let secret = cache + .machine_secret(user) + .ok_or_else(|| missing("machine secret", user))?; + secret_data.insert(key.clone(), secret.clone()); + } + for (key, user) in &s.machine_keys { + let json = cache + .machine_key(user) + .ok_or_else(|| missing("machine key", user))?; + secret_data.insert(key.clone(), json.clone()); + } + + let cm = ConfigMap { + metadata: ObjectMeta { + name: Some(s.configmap_name.clone()), + namespace: Some(s.namespace.clone()), + ..Default::default() + }, + data: Some(cm_data), + ..Default::default() + }; + K8sResourceScore::single(cm, Some(s.namespace.clone())) + .interpret(inventory, topology) + .await?; + + if !secret_data.is_empty() { + let secret = Secret { + metadata: ObjectMeta { + name: Some(s.secret_name.clone()), + namespace: Some(s.namespace.clone()), + ..Default::default() + }, + string_data: Some(secret_data), + ..Default::default() + }; + K8sResourceScore::single(secret, Some(s.namespace.clone())) + .interpret(inventory, topology) + .await?; + } + + Ok(Outcome::success(format!( + "Exported Zitadel credentials to {}/{} (+secret {})", + s.namespace, s.configmap_name, s.secret_name + ))) + } + fn get_name(&self) -> InterpretName { + InterpretName::Custom("ZitadelCredentialsExport") + } + fn get_version(&self) -> Version { + Version::from("0.1.0").expect("static version") + } + fn get_status(&self) -> InterpretStatus { + InterpretStatus::QUEUED + } + fn get_children(&self) -> Vec { + vec![] + } +} + #[cfg(test)] mod tests { use super::*; @@ -2102,6 +2683,7 @@ mod tests { roles: vec![], machine_users: vec![], groups_claim_action: false, + ..Default::default() }; setup .interpret(&inv, &topo) @@ -2176,6 +2758,7 @@ mod tests { roles: vec![], machine_users: vec![], groups_claim_action: false, + ..Default::default() } } diff --git a/harmony_app/src/capabilities.rs b/harmony_app/src/capabilities.rs index c87b2f92..58d0530e 100644 --- a/harmony_app/src/capabilities.rs +++ b/harmony_app/src/capabilities.rs @@ -243,6 +243,7 @@ impl Capability for ZitadelAuth { roles: vec![], machine_users: vec![], groups_claim_action: false, + ..Default::default() }; let export = ZitadelClientIdExportScore { app_name: app.name.to_string(), diff --git a/harmony_app/src/chart.rs b/harmony_app/src/chart.rs index 48dc022f..7fa2ce09 100644 --- a/harmony_app/src/chart.rs +++ b/harmony_app/src/chart.rs @@ -15,9 +15,9 @@ use k8s_openapi::api::apps::v1::{ Deployment, DeploymentSpec, DeploymentStrategy, RollingUpdateDeployment, }; use k8s_openapi::api::core::v1::{ - Container, ContainerPort, EnvFromSource, EnvVar, PersistentVolumeClaim, + Container, ContainerPort, EnvFromSource, EnvVar, KeyToPath, PersistentVolumeClaim, PersistentVolumeClaimSpec, PersistentVolumeClaimVolumeSource, PodSpec, PodTemplateSpec, - SecretEnvSource, Service, ServicePort, ServiceSpec, Volume, VolumeMount, + SecretEnvSource, SecretVolumeSource, Service, ServicePort, ServiceSpec, Volume, VolumeMount, VolumeResourceRequirements, }; use k8s_openapi::apimachinery::pkg::api::resource::Quantity; @@ -52,6 +52,19 @@ pub struct DeployConfig { /// wired by reference (e.g. a DB URL via `secretKeyRef`), never by value, /// so the chart stays publishable. pub extra_env: Vec, + /// Secret values projected into every container as files (e.g. a service + /// account key whose consumer wants a path, not an env var). By reference, + /// like `extra_env`, so the chart carries no secret material. + pub secret_file_mounts: Vec, +} + +/// Mount one key of a Secret as a file at `path` (its parent dir is the mount +/// point; the basename is the projected filename). +#[derive(Debug, Clone, serde::Serialize)] +pub struct SecretFileMount { + pub secret: String, + pub key: String, + pub path: String, } impl DeployConfig { @@ -78,6 +91,7 @@ impl DeployConfig { replicas: if prod { 2 } else { 1 }, rolling: prod, extra_env: Vec::new(), + secret_file_mounts: Vec::new(), } } } @@ -189,7 +203,7 @@ fn deployment(svc: &ComposeService, cfg: &DeployConfig) -> Deployment { // Capability-contributed env (e.g. a DB connection by `secretKeyRef`). env.extend(cfg.extra_env.iter().cloned()); - let volume_mounts: Vec = svc + let mut volume_mounts: Vec = svc .mounts .iter() .map(|m| VolumeMount { @@ -199,7 +213,7 @@ fn deployment(svc: &ComposeService, cfg: &DeployConfig) -> Deployment { }) .collect(); - let volumes: Vec = svc + let mut volumes: Vec = svc .mounts .iter() .map(|m| Volume { @@ -212,6 +226,32 @@ fn deployment(svc: &ComposeService, cfg: &DeployConfig) -> Deployment { }) .collect(); + // Secret-as-file mounts (by reference): one volume per mount, projecting + // `key` to the file basename under the parent dir of `path`. + for (i, m) in cfg.secret_file_mounts.iter().enumerate() { + let name = format!("secret-file-{i}"); + let (dir, file) = m.path.rsplit_once('/').unwrap_or((".", m.path.as_str())); + volumes.push(Volume { + name: name.clone(), + secret: Some(SecretVolumeSource { + secret_name: Some(m.secret.clone()), + items: Some(vec![KeyToPath { + key: m.key.clone(), + path: file.to_string(), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + }); + volume_mounts.push(VolumeMount { + name, + mount_path: dir.to_string(), + read_only: Some(true), + ..Default::default() + }); + } + let strategy = Some(if cfg.rolling { DeploymentStrategy { type_: Some("RollingUpdate".to_string()), @@ -360,6 +400,7 @@ mod tests { replicas: 2, rolling: true, extra_env: vec![], + secret_file_mounts: vec![], }; let tmp = tempfile::tempdir().unwrap(); build_chart(&app, &cfg, tmp.path()).unwrap(); diff --git a/harmony_app/src/context.rs b/harmony_app/src/context.rs index 349f8666..00e88792 100644 --- a/harmony_app/src/context.rs +++ b/harmony_app/src/context.rs @@ -27,15 +27,25 @@ struct ContextsFile { contexts: std::collections::HashMap, } -/// One named context. Cluster access is exactly one of `k3d` (local) or -/// `openbao_namespace` (kubeconfig from OpenBao, CI/prod). +/// One named context. Cluster access is exactly one of: `autoprovision` +/// (K8sAnywhere installs a local k3d), `k3d` (an existing local cluster), or +/// `openbao_namespace` (kubeconfig brokered from OpenBao, the CI/prod path). #[derive(Debug, Deserialize)] struct ContextDef { profile: Profile, + /// Let K8sAnywhereTopology autoprovision a local k3d cluster (offline; no + /// pre-created cluster, no kubeconfig). The hands-off local/CI path. + #[serde(default)] + autoprovision: bool, k3d: Option, openbao_namespace: Option, } +/// The cluster K8sAnywhereTopology autoprovisions (its K3DInstallationScore +/// default). Named so operational verbs and `publish` (k3d image import) find +/// the same cluster the deploy created. +const AUTOPROVISION_CLUSTER: &str = "harmony"; + /// Cluster credential brokered from OpenBao — its own secret /// (`secret//ClusterAccess`), separate from the app's secrets, so /// "how to reach the cluster" and "the app's config" stay distinct. @@ -51,9 +61,14 @@ pub struct AppContext { version: String, local_config_dir: Option, k3d_cluster: Option, - kubeconfig: PathBuf, + /// True when K8sAnywhere autoprovisions the cluster: there is no kubeconfig + /// at resolve time (the cluster is created during deploy), so the topology + /// is built env-free and operational verbs resolve the k3d kubeconfig lazily. + autoprovision: bool, + /// The brokered kubeconfig (k3d/OpenBao paths). `None` under autoprovision. + kubeconfig: Option, /// The kubeconfig file must outlive every client/topology built from it. - _kubeconfig_guard: NamedTempFile, + _kubeconfig_guard: Option, } impl AppContext { @@ -85,9 +100,10 @@ impl AppContext { ) })?; - let guard = match (&def.k3d, &def.openbao_namespace) { - (Some(cluster), None) => k3d_kubeconfig(cluster)?, - (None, Some(ns)) => { + let guard = match (def.autoprovision, &def.k3d, &def.openbao_namespace) { + (true, None, None) => None, + (false, Some(cluster), None) => Some(k3d_kubeconfig(cluster)?), + (false, None, Some(ns)) => { let access: ClusterAccess = ConfigClient::for_namespace(ns) .await .get() @@ -98,9 +114,18 @@ impl AppContext { — are the Zitadel key + OPENBAO_* env vars set?)" ) })?; - write_kubeconfig(access.kubeconfig.as_bytes())? + Some(write_kubeconfig(access.kubeconfig.as_bytes())?) } - _ => bail!("context '{name}' must set exactly one of `k3d` or `openbao_namespace`"), + _ => bail!( + "context '{name}' must set exactly one of `autoprovision = true`, \ + `k3d`, or `openbao_namespace`" + ), + }; + + let k3d_cluster = if def.autoprovision { + Some(AUTOPROVISION_CLUSTER.to_string()) + } else { + def.k3d.clone() }; Ok(Self { @@ -108,8 +133,9 @@ impl AppContext { profile: def.profile, version: version.into(), local_config_dir, - k3d_cluster: def.k3d.clone(), - kubeconfig: guard.path().to_path_buf(), + k3d_cluster, + autoprovision: def.autoprovision, + kubeconfig: guard.as_ref().map(|g| g.path().to_path_buf()), _kubeconfig_guard: guard, }) } @@ -130,17 +156,39 @@ impl AppContext { self.k3d_cluster.as_deref() } - /// The converge target — built from the context's kubeconfig, no env. + /// The converge target. Under autoprovision K8sAnywhere installs + manages + /// a local k3d cluster; otherwise it's pinned to the brokered kubeconfig + /// (no env, so we never deploy to the wrong cluster). pub fn topology(&self) -> K8sAnywhereTopology { + if self.autoprovision { + return K8sAnywhereTopology::with_config(K8sAnywhereConfig::local_k3d()); + } + let kubeconfig = self + .kubeconfig + .as_ref() + .expect("non-autoprovision context resolves a kubeconfig"); K8sAnywhereTopology::with_config(K8sAnywhereConfig::kubeconfig( - self.kubeconfig.to_string_lossy().to_string(), + kubeconfig.to_string_lossy().to_string(), None, )) } /// A read client for operational verbs (status/logs) — no topology prep. + /// Under autoprovision the cluster exists only after a deploy, so its + /// kubeconfig is resolved lazily here from the managed k3d. pub async fn k8s_client(&self) -> Result { - K8sClient::from_kubeconfig(&self.kubeconfig.to_string_lossy()) + let _guard; + let path = match &self.kubeconfig { + Some(p) => p.clone(), + None => { + let cluster = self.k3d_cluster.as_deref().unwrap_or(AUTOPROVISION_CLUSTER); + let guard = k3d_kubeconfig(cluster)?; + let path = guard.path().to_path_buf(); + _guard = guard; + path + } + }; + K8sClient::from_kubeconfig(&path.to_string_lossy()) .await .context("building k8s client from the context kubeconfig") } diff --git a/harmony_app/src/lib.rs b/harmony_app/src/lib.rs index db2117b9..541e4685 100644 --- a/harmony_app/src/lib.rs +++ b/harmony_app/src/lib.rs @@ -29,7 +29,7 @@ pub use app::{ deploy, logs, ship, status, }; pub use capabilities::{AppRef, Capability, Monitoring, Postgres, ZitadelAuth}; -pub use chart::{DeployConfig, cluster_issuer_for, service_image}; +pub use chart::{DeployConfig, SecretFileMount, cluster_issuer_for, service_image}; pub use compose::ComposeApp; pub use context::{AppContext, ClusterAccess}; pub use deploy::ComposeDeploy; diff --git a/harmony_app/src/publish.rs b/harmony_app/src/publish.rs index caf80929..357397f6 100644 --- a/harmony_app/src/publish.rs +++ b/harmony_app/src/publish.rs @@ -104,6 +104,24 @@ pub fn import_to_k3d(app: &ComposeApp, cfg: &DeployConfig, cluster: &str) -> Res } else { "k3d".as_ref() }; + // `ship` imports before it converges, but on the autoprovision path the + // cluster is created during converge — so ensure it exists first. K8sAnywhere + // then reuses this cluster (its installer starts an existing one). + let exists = Command::new(k3d) + .args(["cluster", "list", cluster, "--no-headers"]) + .output() + .map(|o| o.status.success() && !o.stdout.is_empty()) + .unwrap_or(false); + if !exists { + log::info!("k3d cluster create {cluster} (autoprovision)"); + let status = Command::new(k3d) + .args(["cluster", "create", cluster, "--wait"]) + .status() + .context("spawn k3d cluster create")?; + if !status.success() { + bail!("k3d cluster create {cluster} failed ({status})"); + } + } log::info!("k3d image import {} -> {cluster}", images.join(" ")); let status = Command::new(k3d) .args(["image", "import", "-c", cluster]) diff --git a/harmony_app/tests/helm_render.rs b/harmony_app/tests/helm_render.rs index 69c4246d..459a8bb3 100644 --- a/harmony_app/tests/helm_render.rs +++ b/harmony_app/tests/helm_render.rs @@ -36,6 +36,7 @@ fn generated_chart_passes_helm_lint_and_template() { replicas: 2, rolling: true, extra_env: vec![], + secret_file_mounts: vec![], }; let tmp = tempfile::tempdir().unwrap(); -- 2.39.5 From 1e895d7c83d2f19a7862cb19a383ece77f83765a Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Mon, 22 Jun 2026 22:18:32 -0400 Subject: [PATCH 2/2] docs(roadmap): defer tenant application monitoring; capture design tension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lightweight, cluster-config-aware app monitoring: attach to a cluster stack vs. provision a tenant-local one, so tenants don't each duplicate cluster-level Prometheus data while staying self-contained. Audits the four overlapping backends (none deprecated yet — pending the model decision). --- ROADMAP/14-tenant-application-monitoring.md | 86 +++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 ROADMAP/14-tenant-application-monitoring.md diff --git a/ROADMAP/14-tenant-application-monitoring.md b/ROADMAP/14-tenant-application-monitoring.md new file mode 100644 index 00000000..55b48f79 --- /dev/null +++ b/ROADMAP/14-tenant-application-monitoring.md @@ -0,0 +1,86 @@ +# Phase 14: Tenant Application Monitoring & Alerting (deferred — design first) + +## Status + +**Deferred (2026-06-22).** Captured while baking Harmony deploys into a client +app (compose app → tenant + Postgres + live Zitadel on K8sAnywhere). +Monitoring/alerting is the next brick for that deploy, but the existing modules +don't yet fit; decide the model before implementing. **No code deprecated yet** +— that waits on the decision below. + +## Goal + +One lightweight application monitoring + alerting capability a deploy crate adds +with `.with(Monitoring::…)`, that: + +- can be **installed or not in dev** (cheap to omit; no heavy stack for a local + k3d inner loop), +- is **installed correctly in prod as a function of the cluster's configuration** + — not a fixed backend. It must answer, per target cluster: + - is there already a **prometheus-operator** / cluster monitoring stack to + attach to (OKD user-workload-monitoring, kube-prometheus-stack, COO)? + - are **in-cluster Prometheus metrics** already scraped at the cluster level + (kube-state-metrics, node/cadvisor) so the tenant needs only *rules*, not a + *scraper*? + - how are **Prometheus permissions scoped for a tenant** (namespaced + PrometheusRule/ServiceMonitor + RBAC vs. a tenant-owned Prometheus)? + +## The tension to resolve (the actual reason this is deferred) + +- **Anti-pattern risk:** if every tenant rolls its own Prometheus, we duplicate + data already monitored at the cluster level — N Prometheis re-scraping the same + kube-state-metrics, N Alertmanagers, N Grafanas. Wasteful and operationally + noisy. +- **Counter-value:** a **self-contained tenant** is high customer value — a + client can look at *their* namespace and understand their app's health without + cluster-admin context. Easy to reason about, easy to hand off. +- The resolution is probably **not** "one backend" but a capability that picks: + attach-to-cluster-stack (rules + a scoped Grafana view) when one exists, vs. + provision a minimal tenant-scoped stack when it doesn't — driven by a topology + capability/probe, the same way `K8sAnywhereTopology` already detects distro and + `TenantManager` already owns namespace isolation (ADR-011) and identity + (ADR-027). Cluster-level data stays shared; the *tenant view* + *app alert + rules* are what's per-tenant. + +## Current state (audited 2026-06-22) + +Four overlapping, none-satisfying backends (all `AlertSender` impls): + +| Module | Sender | Author / created | Note | +|---|---|---|---| +| `monitoring/kube_prometheus/helm_prometheus_alert_score.rs` + `helm/` | `KubePrometheus` | Willem, 2025-06 | Installs full kube-prometheus-stack via Helm. Works on k3d. Used by `harmony_app::capabilities::Monitoring`. Heaviest; the "tenant rolls its own" path. | +| `monitoring/application_monitoring/application_monitoring_score.rs` | `CRDPrometheus` | Willem, 2025-08 | Prometheus-operator CRDs directly. | +| `monitoring/application_monitoring/rhobs_application_monitoring_score.rs` | `RHOBObservability` | Willem, 2025-09 | Cluster Observability Operator `MonitoringStack`. OKD-oriented. | +| `monitoring/prometheus/` | `Prometheus` | Willem, 2025-07 | Standalone Prometheus. | + +Adjacent (keep — different concern, the cluster/ops view, not per-app alerting): + +- `monitoring/cluster_dashboards/` — Grafana-operator dashboards + datasource + + route (Sylvain Tremblay, 2026-03). Cluster-level dashboards on an existing + Prometheus; OKD-route oriented. +- `monitoring/ceph_alerts.rs`, `monitoring/okd/cluster_alert_rules.rs` — alerts + on OKD's native stack (Sylvain Tremblay, 2026-04). + +`harmony_app::capabilities::Monitoring` (the `.with(Monitoring::new().alert(r))` +DX) currently emits `HelmPrometheusAlertingScore` (the Helm/KubePrometheus path) ++ a namespace-scoped "app has no available replicas for 5m" rule via +kube-state-metrics. The DX shape is right; the **backend choice underneath it is +the open question**. + +## Decision criteria (to settle next) + +1. Capability detects cluster monitoring posture (operator present? cluster + metrics scraped?) and **attaches** vs. **provisions** accordingly — ideally a + `Topology`/`TenantManager` capability, not per-app branching. +2. Default to **rules + tenant-scoped view on shared cluster data**; provision a + tenant-local stack only when no cluster stack exists (e.g. bare k3d dev). +3. Keep the deploy-crate DX a one-liner (`.with(Monitoring::…)`); the + environment, not the app, decides install vs. attach. +4. Once chosen: collapse the four backends to the canonical one and + `#[deprecated]` (not delete) the overlaps; keep the workspace compiling. + +## First consumer waiting on this + +The first client app deploy crate using this paradigm (frontend + backend + +Zitadel; its external SaaS dependency is not deployed). It deliberately ships +**without** a monitoring Score today — add it here once the model lands. -- 2.39.5