From cbf3570f123e693cc86a1118bf9874fc6d14c156 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Wed, 24 Jun 2026 08:53:51 -0400 Subject: [PATCH 01/15] feat(app): forward compose build.args; wait for Zitadel pod before setup build.args support (harmony_app): - Parse `build.args` (advanced build form) into ComposeService.build_args, mirroring parse_env (Simple/List/KvPair). - build_images() forwards them as `docker build --build-arg K=V`, so an app can bake per-environment build-time config (e.g. Quarkus profile, a Vite API URL). Zitadel readiness (harmony): - ZitadelSetupScore failed at 0.0s on a cold cluster ("no Running pod for service 'zitadel'") because it checked once, before the Zitadel Deployment pod was Running. Poll for a Running pod (300s/3s) so a one-shot `ship` converges instead of needing a re-run. --- harmony/src/modules/zitadel/setup.rs | 63 +++++++++++++++++++--------- harmony_app/src/compose.rs | 45 ++++++++++++++++++++ harmony_app/src/publish.rs | 3 ++ 3 files changed, 91 insertions(+), 20 deletions(-) diff --git a/harmony/src/modules/zitadel/setup.rs b/harmony/src/modules/zitadel/setup.rs index 8f23897f..7e33d1cf 100644 --- a/harmony/src/modules/zitadel/setup.rs +++ b/harmony/src/modules/zitadel/setup.rs @@ -2257,26 +2257,49 @@ impl Interpret for ZitadelSetupInterpret { let me: std::borrow::Cow<'_, ZitadelSetupInterpret> = if let Some(svc) = &self.score.port_forward_service { let label = format!("app.kubernetes.io/name={svc}"); - let pods = k8s - .list_resources::( - Some(&self.score.namespace), - Some(kube::api::ListParams::default().labels(&label)), - ) - .await - .map_err(|e| InterpretError::new(format!("listing '{svc}' pods: {e}")))?; - // The chart's init/setup Job pods share this label; only the - // serving Deployment pod is Running, so filter on phase. - let pod = pods - .items - .into_iter() - .find(|p| p.status.as_ref().and_then(|s| s.phase.as_deref()) == Some("Running")) - .and_then(|p| p.metadata.name) - .ok_or_else(|| { - InterpretError::new(format!( - "no Running pod for service '{svc}' in namespace '{}'", - self.score.namespace - )) - })?; + // On a cold cluster the Zitadel Deployment pod is not Running yet + // (image pull + DB migration take a minute or more), so poll for + // it instead of failing on the first miss — otherwise a one-shot + // `ship` aborts here and only a re-run (pod now up) succeeds. + let pod = { + use std::time::{Duration, Instant}; + let timeout = Duration::from_secs(300); + let interval = Duration::from_secs(3); + let deadline = Instant::now() + timeout; + loop { + let pods = k8s + .list_resources::( + Some(&self.score.namespace), + Some(kube::api::ListParams::default().labels(&label)), + ) + .await + .map_err(|e| { + InterpretError::new(format!("listing '{svc}' pods: {e}")) + })?; + // The chart's init/setup Job pods share this label; only + // the serving Deployment pod is Running, so filter on phase. + let running = pods + .items + .into_iter() + .find(|p| { + p.status.as_ref().and_then(|s| s.phase.as_deref()) + == Some("Running") + }) + .and_then(|p| p.metadata.name); + if let Some(name) = running { + break name; + } + if Instant::now() >= deadline { + return Err(InterpretError::new(format!( + "no Running pod for service '{svc}' in namespace '{}' within {}s", + self.score.namespace, + timeout.as_secs(), + ))); + } + debug!("[ZitadelSetup] waiting for a Running '{svc}' pod..."); + tokio::time::sleep(interval).await; + } + }; // Zitadel serves http on 8080 in-cluster (chart default). let handle = k8s .port_forward(&pod, &self.score.namespace, 0, 8080) diff --git a/harmony_app/src/compose.rs b/harmony_app/src/compose.rs index e7d34e74..c6c0483a 100644 --- a/harmony_app/src/compose.rs +++ b/harmony_app/src/compose.rs @@ -55,6 +55,9 @@ pub struct ComposeService { /// set, `publish` builds + pushes the image under our registry. pub build_context: Option, pub dockerfile: Option, + /// `build.args` — baked into the image at `docker build` time (forwarded as + /// `--build-arg`). Only the advanced build form carries them. + pub build_args: Vec<(String, String)>, pub ports: Vec, pub env: Vec<(String, String)>, pub mounts: Vec, @@ -108,6 +111,10 @@ impl ComposeApp { Some(docker_compose_types::BuildStep::Advanced(a)) => a.dockerfile.clone(), _ => None, }; + let build_args = match &svc.build_ { + Some(docker_compose_types::BuildStep::Advanced(a)) => parse_build_args(&a.args), + _ => Vec::new(), + }; if svc.image.is_none() && build_context.is_none() { return Err(ComposeError::NoImageOrBuild { service: name }); } @@ -121,6 +128,7 @@ impl ComposeApp { image: svc.image, build_context, dockerfile, + build_args, name, }); } @@ -192,6 +200,26 @@ fn parse_short_port(service: &str, spec: &str) -> Result }) } +/// Flatten compose `build.args` into ordered key/value pairs, mirroring +/// `parse_env`. List items without `=` get an empty value (docker then resolves +/// them from the build environment). +fn parse_build_args(args: &Option) -> Vec<(String, String)> { + use docker_compose_types::BuildArgs; + match args { + None | Some(BuildArgs::Simple(_)) => Vec::new(), + Some(BuildArgs::List(items)) => items + .iter() + .map(|item| match item.split_once('=') { + Some((k, v)) => (k.to_string(), v.to_string()), + None => (item.to_string(), String::new()), + }) + .collect(), + Some(BuildArgs::KvPair(map)) => { + map.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + } + } +} + fn parse_env(env: &Environment) -> Vec<(String, String)> { match env { Environment::List(items) => items @@ -334,6 +362,23 @@ volumes: ); } + #[test] + fn build_args_parsed_from_advanced_build() { + let yaml = "services:\n api:\n build:\n context: .\n args:\n QUARKUS_BUILD_PROFILE: dev\n"; + let app = ComposeApp::parse(yaml, Path::new(".")).unwrap(); + assert_eq!( + app.service("api").unwrap().build_args, + vec![("QUARKUS_BUILD_PROFILE".to_string(), "dev".to_string())] + ); + } + + #[test] + fn build_args_empty_for_simple_build() { + let yaml = "services:\n api:\n build: .\n"; + let app = ComposeApp::parse(yaml, Path::new(".")).unwrap(); + assert!(app.service("api").unwrap().build_args.is_empty()); + } + #[test] fn named_volume_becomes_a_mount_and_only_mounted_volumes_need_pvcs() { let app = app(); diff --git a/harmony_app/src/publish.rs b/harmony_app/src/publish.rs index 357397f6..34b533fc 100644 --- a/harmony_app/src/publish.rs +++ b/harmony_app/src/publish.rs @@ -45,6 +45,9 @@ pub fn build_images(app: &ComposeApp, cfg: &DeployConfig) -> Result<()> { if let Some(dockerfile) = &svc.dockerfile { cmd.arg("-f").arg(context.join(dockerfile)); } + for (key, value) in &svc.build_args { + cmd.arg("--build-arg").arg(format!("{key}={value}")); + } cmd.arg(context); run(cmd)?; } -- 2.39.5 From 0090c9e23f3226786d230b0fcab710d37205d74a Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 25 Jun 2026 11:37:08 -0400 Subject: [PATCH 02/15] fix(zitadel-jwt): exchange the id_token for OpenBao, not the opaque access_token MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ZitadelJwtBearer posted Zitadel's access_token to OpenBao's jwt/login. Zitadel issues opaque access tokens by default, so OpenBao rejected it with "compact JWS format must have three parts". The token endpoint already returns an id_token (a JWT) because build_scope requests `openid`; use it. It carries the service user in `sub` and the project id in `aud` (the latter via the project:id:aud scope) — exactly what the OpenBao JWT role binds on. No Zitadel-side change needed (the manual id_token exchange already proved this works). --- harmony_zitadel_jwt/src/lib.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/harmony_zitadel_jwt/src/lib.rs b/harmony_zitadel_jwt/src/lib.rs index ceed32d2..a07abbe1 100644 --- a/harmony_zitadel_jwt/src/lib.rs +++ b/harmony_zitadel_jwt/src/lib.rs @@ -46,7 +46,14 @@ pub struct CachedToken { #[derive(Deserialize)] struct TokenResponse { - access_token: String, + /// The verifiable JWT we exchange for an OpenBao token (requested via the + /// `openid` scope). Zitadel's `access_token` is **opaque** by default — not + /// a usable OpenBao JWT — so we deliberately use the id_token, which carries + /// the service user in `sub` and the project id in `aud` (the latter via the + /// `urn:zitadel:iam:org:project:id::aud` scope), exactly what the + /// OpenBao JWT role binds on. + #[serde(default)] + id_token: Option, #[serde(default)] expires_in: Option, } @@ -112,10 +119,16 @@ impl ZitadelJwtBearer { let tr: TokenResponse = resp.json().await.context("parsing token response")?; let expires_in = tr.expires_in.unwrap_or(3600); - let token = tr.access_token.clone(); + // Send the id_token (a JWT), NOT the opaque access_token — OpenBao's JWT + // auth must be able to verify it against Zitadel's JWKS. + let token = tr.id_token.context( + "Zitadel token response had no id_token — the `openid` scope is \ + required (it is in build_scope) and the service user must be \ + allowed to receive an id_token", + )?; *self.cache.lock().unwrap() = Some(CachedToken { - access_token: tr.access_token, + access_token: token.clone(), expires_at_unix: now + expires_in, }); -- 2.39.5 From 0094a966a77e7e1d861ab0d8f98a31c6ebc69eab Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 25 Jun 2026 11:37:08 -0400 Subject: [PATCH 03/15] feat(github-runner): install GitHub self-hosted runners (ARC) via HelmChartScore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New modules::github_runner. GithubRunnerScore yields two HelmChartScores — the ARC controller (gha-runner-scale-set-controller) and a runner scale set (gha-runner-scale-set) — as OCI charts. dind enabled (so docker build works), configurable resource limits applied to both the runner and dind containers, and the GitHub credential referenced by a pre-created Secret name (kept out of code). The scale-set name becomes the workflow runs-on label. --- harmony/src/modules/github_runner.rs | 208 +++++++++++++++++++++++++++ harmony/src/modules/mod.rs | 1 + 2 files changed, 209 insertions(+) create mode 100644 harmony/src/modules/github_runner.rs diff --git a/harmony/src/modules/github_runner.rs b/harmony/src/modules/github_runner.rs new file mode 100644 index 00000000..3839f6e8 --- /dev/null +++ b/harmony/src/modules/github_runner.rs @@ -0,0 +1,208 @@ +//! Install GitHub self-hosted runners on Kubernetes via **Actions Runner +//! Controller (ARC)**, built on [`HelmChartScore`]. +//! +//! ARC is two Helm releases: +//! 1. the **controller** (`gha-runner-scale-set-controller`, cluster-scoped) — +//! install once per cluster; +//! 2. one or more **runner scale sets** (`gha-runner-scale-set`) — each +//! registers a set of ephemeral runners under a name that becomes the +//! workflow `runs-on:` label. +//! +//! [`GithubRunnerScore`] bundles the config and yields both as [`HelmChartScore`]s +//! (controller first). The GitHub credential is referenced by the name of a +//! pre-created Kubernetes Secret (holding `github_token`) — it never appears in +//! code or values, so no secret is baked into the deploy. +//! +//! ```ignore +//! let runner = GithubRunnerScore { +//! github_config_url: "https://github.com/Devsights/folk-timesheets".into(), +//! github_config_secret: "okd-cb1-github".into(), // pre-created Secret +//! runner_scale_set_name: "okd-cb1".into(), // = runs-on label +//! ..GithubRunnerScore::defaults() +//! }; +//! // add both to your scores(), controller first: +//! vec![Box::new(runner.controller()), Box::new(runner.runner_set())] +//! ``` +//! +//! On OpenShift/OKD, `containerMode: dind` needs a privileged container — grant +//! the runner ServiceAccount the `privileged` SCC. The runner needs **no** +//! cluster RBAC: jobs reach the cluster through their own brokered kubeconfig. + +use std::str::FromStr; + +use non_blank_string_rs::NonBlankString; + +use crate::modules::helm::chart::HelmChartScore; + +const CONTROLLER_CHART: &str = + "oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller"; +const RUNNER_SET_CHART: &str = + "oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set"; +/// Pin a known-good ARC chart version (controller + scale set share the line). +const DEFAULT_CHART_VERSION: &str = "0.9.3"; +const DEFAULT_RUNNER_IMAGE: &str = "ghcr.io/actions/actions-runner:latest"; + +/// One GitHub self-hosted runner deployment (ARC controller + a runner set). +pub struct GithubRunnerScore { + /// Repo or org URL the runners register against, e.g. + /// `https://github.com/Devsights/folk-timesheets`. + pub github_config_url: String, + /// Name of a **pre-created** Kubernetes Secret in `runners_namespace` holding + /// the `github_token` key (a fine-grained PAT or GitHub App credentials). + /// Kept out of code/values — create it separately (e.g. from OpenBao). + pub github_config_secret: String, + /// Scale-set name — this becomes the workflow `runs-on:` label (e.g. `okd-cb1`). + pub runner_scale_set_name: String, + /// Namespace for the runner pods (default `arc-runners`). + pub runners_namespace: String, + /// Namespace for the controller (default `arc-systems`). + pub controller_namespace: String, + pub min_runners: u32, + pub max_runners: u32, + /// CPU/memory limits applied to BOTH the runner and the dind container + /// (the build runs in dind, so it needs the headroom too). + pub cpu_limit: String, + pub memory_limit: String, + /// Runner container image. + pub runner_image: String, + /// ARC chart version (controller + scale set). + pub chart_version: String, +} + +impl GithubRunnerScore { + /// Sensible defaults; override the four required fields (`github_config_url`, + /// `github_config_secret`, `runner_scale_set_name`, and resources as needed). + pub fn defaults() -> Self { + Self { + github_config_url: String::new(), + github_config_secret: String::new(), + runner_scale_set_name: String::new(), + runners_namespace: "arc-runners".to_string(), + controller_namespace: "arc-systems".to_string(), + min_runners: 1, + max_runners: 2, + cpu_limit: "8".to_string(), + memory_limit: "16Gi".to_string(), + runner_image: DEFAULT_RUNNER_IMAGE.to_string(), + chart_version: DEFAULT_CHART_VERSION.to_string(), + } + } + + /// The cluster-scoped ARC controller. Install once per cluster (idempotent). + pub fn controller(&self) -> HelmChartScore { + HelmChartScore { + namespace: Some(nbs(&self.controller_namespace)), + release_name: nbs("arc"), + chart_name: nbs(CONTROLLER_CHART), + chart_version: Some(nbs(&self.chart_version)), + values_overrides: None, + values_yaml: None, + create_namespace: true, + install_only: false, + repository: None, // OCI chart — referenced directly, no `helm repo add` + } + } + + /// The runner scale set — registers ephemeral runners labelled + /// `runner_scale_set_name`, with dind (so `docker build` works) and the + /// configured resource limits. + pub fn runner_set(&self) -> HelmChartScore { + HelmChartScore { + namespace: Some(nbs(&self.runners_namespace)), + release_name: nbs(&self.runner_scale_set_name), + chart_name: nbs(RUNNER_SET_CHART), + chart_version: Some(nbs(&self.chart_version)), + values_overrides: None, + values_yaml: Some(self.runner_set_values()), + create_namespace: true, + install_only: false, + repository: None, + } + } + + fn runner_set_values(&self) -> String { + let Self { + github_config_url, + github_config_secret, + runner_scale_set_name, + min_runners, + max_runners, + cpu_limit, + memory_limit, + runner_image, + .. + } = self; + // dind is where `docker build` actually runs, so it gets the same limits. + format!( + "githubConfigUrl: {github_config_url}\n\ + githubConfigSecret: {github_config_secret}\n\ + runnerScaleSetName: {runner_scale_set_name}\n\ + minRunners: {min_runners}\n\ + maxRunners: {max_runners}\n\ + containerMode:\n\ + \x20 type: dind\n\ + template:\n\ + \x20 spec:\n\ + \x20 containers:\n\ + \x20 - name: runner\n\ + \x20 image: {runner_image}\n\ + \x20 resources:\n\ + \x20 limits:\n\ + \x20 cpu: \"{cpu_limit}\"\n\ + \x20 memory: \"{memory_limit}\"\n\ + \x20 - name: dind\n\ + \x20 resources:\n\ + \x20 limits:\n\ + \x20 cpu: \"{cpu_limit}\"\n\ + \x20 memory: \"{memory_limit}\"\n" + ) + } +} + +/// `NonBlankString` from a known-non-blank `&str` (panics on blank, like the +/// rest of the helm modules — these are constants or required config). +fn nbs(s: &str) -> NonBlankString { + NonBlankString::from_str(s).expect("non-blank string") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> GithubRunnerScore { + GithubRunnerScore { + github_config_url: "https://github.com/Devsights/folk-timesheets".to_string(), + github_config_secret: "okd-cb1-github".to_string(), + runner_scale_set_name: "okd-cb1".to_string(), + cpu_limit: "46".to_string(), + memory_limit: "32Gi".to_string(), + ..GithubRunnerScore::defaults() + } + } + + #[test] + fn controller_is_an_oci_chart_with_no_repo() { + let c = sample().controller(); + assert_eq!(c.chart_name.to_string(), CONTROLLER_CHART); + assert!(c.repository.is_none()); + assert_eq!(c.namespace.unwrap().to_string(), "arc-systems"); + } + + #[test] + fn runner_set_values_carry_name_dind_and_limits() { + let v = sample().runner_set_values(); + assert!(v.contains("runnerScaleSetName: okd-cb1")); + assert!(v.contains("type: dind")); + assert!(v.contains("cpu: \"46\"")); + assert!(v.contains("memory: \"32Gi\"")); + assert!(v.contains("githubConfigSecret: okd-cb1-github")); + } + + #[test] + fn runner_set_values_are_valid_yaml() { + let v = sample().runner_set_values(); + let parsed: serde_yaml::Value = serde_yaml::from_str(&v).expect("valid yaml"); + assert_eq!(parsed["runnerScaleSetName"], "okd-cb1"); + assert_eq!(parsed["containerMode"]["type"], "dind"); + } +} diff --git a/harmony/src/modules/mod.rs b/harmony/src/modules/mod.rs index 545997de..5ad4a788 100644 --- a/harmony/src/modules/mod.rs +++ b/harmony/src/modules/mod.rs @@ -6,6 +6,7 @@ pub mod dhcp; pub mod dns; pub mod dummy; pub mod fleet; +pub mod github_runner; pub mod helm; pub mod http; pub mod inventory; -- 2.39.5 From 09ed0d81dba78598d461380186043f6738995ee2 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 25 Jun 2026 21:50:20 -0400 Subject: [PATCH 04/15] docs(github-runner): platform guide for hosting client runners (org-scoped App) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NationTech-hosted model: one NationTech-owned GitHub App (org permissions Self-hosted runners: R/W + Metadata: RO), installed once per client org — that single authorization covers all the org's repos. Per client: a Secret with that org's installation ID + a runner scale set whose githubConfigUrl is the org URL. Cited from the official GitHub ARC + Apps docs. Point the module's example/docs at the org URL. --- docs/guides/github-self-hosted-runners.md | 160 ++++++++++++++++++++++ harmony/src/modules/github_runner.rs | 18 ++- 2 files changed, 171 insertions(+), 7 deletions(-) create mode 100644 docs/guides/github-self-hosted-runners.md diff --git a/docs/guides/github-self-hosted-runners.md b/docs/guides/github-self-hosted-runners.md new file mode 100644 index 00000000..c0cd277a --- /dev/null +++ b/docs/guides/github-self-hosted-runners.md @@ -0,0 +1,160 @@ +# Hosting GitHub self-hosted runners for client orgs (ARC) + +NationTech hosts CI runners for client projects on its managed OKD clusters, +using **Actions Runner Controller (ARC)** and the +[`github_runner`](../../harmony/src/modules/github_runner.rs) module. This guide +covers the **GitHub authentication** side. + +## Model + +- **One GitHub App, owned by NationTech**, holds the credentials (App ID + + private key). It is registered **once**, ever. +- Authentication is a **GitHub App** (not a PAT): not tied to a person, the token + is short-lived and auto-refreshed by ARC, and scoped to runner registration. +- Runners are **organization-scoped**: a runner scale set whose `githubConfigUrl` + is a *client org URL* registers runners usable by **every repository in that + org** — current and future.[^deploy] Repository-level `Administration` rights are + **not** used.[^arc-auth] +- The **only manipulation the client does** is authorizing the App's install on + their org **once** — that covers all their projects. Everything else + (cluster, runner scale set, Secret) is NationTech-side. + +``` +NationTech GitHub App (registered once) + │ installed once on… + ├── client org A ──→ scale set A (githubConfigUrl = org A) on the OKD cluster + ├── client org B ──→ scale set B + └── … (one install per client org = all their repos) +``` + +--- + +## Part A — Register the App (NationTech, once for all clients) + +In the **NationTech** org: profile → **Your organizations** → **Settings** → +**Developer settings** → **GitHub Apps** → **New GitHub App**.[^register] + +- **GitHub App name** — e.g. `NationTech CI Runners` (unique, ≤34 chars).[^register] +- **Homepage URL** — any valid URL (required).[^register] +- **Webhook → Active** — **deselect**; the App is auth-only.[^register] +- **Where can this GitHub App be installed?** — **Any account** (so it can be + installed on client orgs). +- **Organization permissions** — set exactly (these are what org-scoped runner + registration needs):[^arc-auth] + + | Permission | Access | + |---|---| + | **Self-hosted runners** | **Read and write** | + | **Metadata** | **Read-only** (selected automatically) | + + > Repository **Administration** is *only* needed for repo-scoped runners; we + > use org scope, so leave it at No access.[^arc-auth] + +Click **Create GitHub App**, then: + +1. Note the **App ID** (shown in the App's "About" section on its Edit page).[^keys][^arc-auth] +2. Under **Private keys** → **Generate a private key** → save the downloaded + `.pem` (PEM, `PKCS#1`; GitHub keeps only the public half — store it in the + platform vault).[^keys] + +The App ID + private key are shared across **all** client installs. Only the +installation ID (Part B) differs per org. + +--- + +## Part B — Onboard a client org (the client's one authorization + NationTech wiring) + +### B.1 — Client authorizes the install (their only step) + +A **client org owner** installs the App on their org once: + +- Open the App's public install page (App → **Install App**, or share the App's + `https://github.com/apps/` URL), choose the **client org**, and + select **All repositories** → **Install**. + +This single authorization covers every repo in that org, now and future. + +Then read the **Installation ID** — the trailing number of the installation +settings URL on the client org:[^arc-auth] + +``` +https://github.com/organizations//settings/installations/INSTALLATION_ID +``` + +(For the current client this is ` = Devsights`.) + +### B.2 — NationTech creates the Secret (on the OKD cluster) + +ARC expects exactly these keys, in the **same namespace** as the runner scale +set (`arc-runners`).[^arc-auth] App ID + key are the shared ones from Part A; the +installation ID is this org's: + +```sh +oc create secret generic -github \ + --namespace arc-runners \ + --from-literal=github_app_id= \ + --from-literal=github_app_installation_id= \ + --from-file=github_app_private_key=/path/to/nationtech-ci-runners.private-key.pem +``` + +> `--from-file` avoids escaping the multi-line PEM; the official example uses +> `--from-literal=github_app_private_key='-----BEGIN RSA PRIVATE KEY-----…'` — +> both are equivalent. (`kubectl` == `oc` here.)[^arc-auth] + +### B.3 — NationTech deploys the runner scale set + +Via the `github_runner` module, with the **org** URL and the per-org Secret: + +```rust +use harmony::modules::github_runner::GithubRunnerScore; + +let runner = GithubRunnerScore { + github_config_url: "https://github.com/Devsights".into(), // the ORG → all its repos + github_config_secret: "devsights-github".into(), // the Secret from B.2 + runner_scale_set_name: "okd-cb1".into(), // == the runs-on label + cpu_limit: "46".into(), + memory_limit: "32Gi".into(), + ..GithubRunnerScore::defaults() +}; +// controller once per cluster, then the runner set: +vec![Box::new(runner.controller()), Box::new(runner.runner_set())] +``` + +OKD: grant the runner ServiceAccount the `privileged` SCC (dind needs it): +`oc adm policy add-scc-to-user privileged -z -n arc-runners`. The +runner has **no** cluster RBAC — jobs reach their target cluster via their own +brokered kubeconfig. + +--- + +## Verify + +```sh +oc get autoscalingrunnerset okd-cb1 -n arc-runners # stops erroring once the Secret exists +oc get pods -n arc-runners -w # listener settles; a runner pod appears +``` + +In the client org: **Settings → Actions → Runners** shows the runner group; any +repo's workflow can then target it with `runs-on: okd-cb1`. + +## Rotation / revocation + +- **Rotate the key:** generate a new private key, update every per-org Secret, + then delete the old key under the App's **Private keys**. GitHub Apps keep + multiple active keys for zero-downtime rotation.[^keys] +- **Off-board a client:** the client uninstalls the App from their org; NationTech + deletes that org's scale set + Secret. + +--- + +## Sources + +- [Authenticating ARC to the GitHub API — GitHub Docs][^arc-auth] — repo vs org permissions, App ID, installation ID, Secret keys + same-namespace rule. +- [Deploying runner scale sets with ARC — GitHub Docs][^deploy] — `githubConfigUrl` at repo/org/enterprise level; `githubConfigSecret`. +- [Registering a GitHub App — GitHub Docs][^register] — registration flow, fields, disabling the webhook, "Any account" installability, permissions. +- [Managing private keys for GitHub Apps — GitHub Docs][^keys] — App ID location, generating the `.pem`, rotation. + +[^arc-auth]: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/authenticating-to-the-github-api +[^deploy]: https://docs.github.com/en/actions/how-tos/manage-runners/use-actions-runner-controller/deploy-runner-scale-sets +[^register]: https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/registering-a-github-app +[^keys]: https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/managing-private-keys-for-github-apps diff --git a/harmony/src/modules/github_runner.rs b/harmony/src/modules/github_runner.rs index 3839f6e8..bbb655a7 100644 --- a/harmony/src/modules/github_runner.rs +++ b/harmony/src/modules/github_runner.rs @@ -15,15 +15,18 @@ //! //! ```ignore //! let runner = GithubRunnerScore { -//! github_config_url: "https://github.com/Devsights/folk-timesheets".into(), -//! github_config_secret: "okd-cb1-github".into(), // pre-created Secret -//! runner_scale_set_name: "okd-cb1".into(), // = runs-on label +//! github_config_url: "https://github.com/Devsights".into(), // an ORG → all its repos +//! github_config_secret: "devsights-github".into(), // pre-created Secret +//! runner_scale_set_name: "okd-cb1".into(), // = runs-on label //! ..GithubRunnerScore::defaults() //! }; //! // add both to your scores(), controller first: //! vec![Box::new(runner.controller()), Box::new(runner.runner_set())] //! ``` //! +//! See `docs/guides/github-self-hosted-runners.md` for the GitHub-App auth setup +//! (one App, one org install per client, hosted by the platform). +//! //! On OpenShift/OKD, `containerMode: dind` needs a privileged container — grant //! the runner ServiceAccount the `privileged` SCC. The runner needs **no** //! cluster RBAC: jobs reach the cluster through their own brokered kubeconfig. @@ -44,8 +47,9 @@ const DEFAULT_RUNNER_IMAGE: &str = "ghcr.io/actions/actions-runner:latest"; /// One GitHub self-hosted runner deployment (ARC controller + a runner set). pub struct GithubRunnerScore { - /// Repo or org URL the runners register against, e.g. - /// `https://github.com/Devsights/folk-timesheets`. + /// URL the runners register against. Prefer an **org** URL + /// (`https://github.com/`) so one scale set serves all of that org's + /// repos; a single repo URL also works for a repo-scoped runner. pub github_config_url: String, /// Name of a **pre-created** Kubernetes Secret in `runners_namespace` holding /// the `github_token` key (a fine-grained PAT or GitHub App credentials). @@ -92,7 +96,7 @@ impl GithubRunnerScore { pub fn controller(&self) -> HelmChartScore { HelmChartScore { namespace: Some(nbs(&self.controller_namespace)), - release_name: nbs("arc"), + release_name: nbs("github-runner-arc"), chart_name: nbs(CONTROLLER_CHART), chart_version: Some(nbs(&self.chart_version)), values_overrides: None, @@ -109,7 +113,7 @@ impl GithubRunnerScore { pub fn runner_set(&self) -> HelmChartScore { HelmChartScore { namespace: Some(nbs(&self.runners_namespace)), - release_name: nbs(&self.runner_scale_set_name), + release_name: nbs(format!("github-runner-{}", &self.runner_scale_set_name).as_str()), chart_name: nbs(RUNNER_SET_CHART), chart_version: Some(nbs(&self.chart_version)), values_overrides: None, -- 2.39.5 From 3d2fd728e0f4d485a4f3284e1291a2ccbebf37d3 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 25 Jun 2026 21:52:05 -0400 Subject: [PATCH 05/15] docs(github-runner): drop the runner setup guide from harmony Hosting client CI runners + the GitHub-App auth is platform/operations, to live in a NationTech private repo (TBD by the operator), not in harmony. Keep the reusable module; reference the auth setup generically instead of by repo path. --- docs/guides/github-self-hosted-runners.md | 160 ---------------------- harmony/src/modules/github_runner.rs | 4 +- 2 files changed, 2 insertions(+), 162 deletions(-) delete mode 100644 docs/guides/github-self-hosted-runners.md diff --git a/docs/guides/github-self-hosted-runners.md b/docs/guides/github-self-hosted-runners.md deleted file mode 100644 index c0cd277a..00000000 --- a/docs/guides/github-self-hosted-runners.md +++ /dev/null @@ -1,160 +0,0 @@ -# Hosting GitHub self-hosted runners for client orgs (ARC) - -NationTech hosts CI runners for client projects on its managed OKD clusters, -using **Actions Runner Controller (ARC)** and the -[`github_runner`](../../harmony/src/modules/github_runner.rs) module. This guide -covers the **GitHub authentication** side. - -## Model - -- **One GitHub App, owned by NationTech**, holds the credentials (App ID + - private key). It is registered **once**, ever. -- Authentication is a **GitHub App** (not a PAT): not tied to a person, the token - is short-lived and auto-refreshed by ARC, and scoped to runner registration. -- Runners are **organization-scoped**: a runner scale set whose `githubConfigUrl` - is a *client org URL* registers runners usable by **every repository in that - org** — current and future.[^deploy] Repository-level `Administration` rights are - **not** used.[^arc-auth] -- The **only manipulation the client does** is authorizing the App's install on - their org **once** — that covers all their projects. Everything else - (cluster, runner scale set, Secret) is NationTech-side. - -``` -NationTech GitHub App (registered once) - │ installed once on… - ├── client org A ──→ scale set A (githubConfigUrl = org A) on the OKD cluster - ├── client org B ──→ scale set B - └── … (one install per client org = all their repos) -``` - ---- - -## Part A — Register the App (NationTech, once for all clients) - -In the **NationTech** org: profile → **Your organizations** → **Settings** → -**Developer settings** → **GitHub Apps** → **New GitHub App**.[^register] - -- **GitHub App name** — e.g. `NationTech CI Runners` (unique, ≤34 chars).[^register] -- **Homepage URL** — any valid URL (required).[^register] -- **Webhook → Active** — **deselect**; the App is auth-only.[^register] -- **Where can this GitHub App be installed?** — **Any account** (so it can be - installed on client orgs). -- **Organization permissions** — set exactly (these are what org-scoped runner - registration needs):[^arc-auth] - - | Permission | Access | - |---|---| - | **Self-hosted runners** | **Read and write** | - | **Metadata** | **Read-only** (selected automatically) | - - > Repository **Administration** is *only* needed for repo-scoped runners; we - > use org scope, so leave it at No access.[^arc-auth] - -Click **Create GitHub App**, then: - -1. Note the **App ID** (shown in the App's "About" section on its Edit page).[^keys][^arc-auth] -2. Under **Private keys** → **Generate a private key** → save the downloaded - `.pem` (PEM, `PKCS#1`; GitHub keeps only the public half — store it in the - platform vault).[^keys] - -The App ID + private key are shared across **all** client installs. Only the -installation ID (Part B) differs per org. - ---- - -## Part B — Onboard a client org (the client's one authorization + NationTech wiring) - -### B.1 — Client authorizes the install (their only step) - -A **client org owner** installs the App on their org once: - -- Open the App's public install page (App → **Install App**, or share the App's - `https://github.com/apps/` URL), choose the **client org**, and - select **All repositories** → **Install**. - -This single authorization covers every repo in that org, now and future. - -Then read the **Installation ID** — the trailing number of the installation -settings URL on the client org:[^arc-auth] - -``` -https://github.com/organizations//settings/installations/INSTALLATION_ID -``` - -(For the current client this is ` = Devsights`.) - -### B.2 — NationTech creates the Secret (on the OKD cluster) - -ARC expects exactly these keys, in the **same namespace** as the runner scale -set (`arc-runners`).[^arc-auth] App ID + key are the shared ones from Part A; the -installation ID is this org's: - -```sh -oc create secret generic -github \ - --namespace arc-runners \ - --from-literal=github_app_id= \ - --from-literal=github_app_installation_id= \ - --from-file=github_app_private_key=/path/to/nationtech-ci-runners.private-key.pem -``` - -> `--from-file` avoids escaping the multi-line PEM; the official example uses -> `--from-literal=github_app_private_key='-----BEGIN RSA PRIVATE KEY-----…'` — -> both are equivalent. (`kubectl` == `oc` here.)[^arc-auth] - -### B.3 — NationTech deploys the runner scale set - -Via the `github_runner` module, with the **org** URL and the per-org Secret: - -```rust -use harmony::modules::github_runner::GithubRunnerScore; - -let runner = GithubRunnerScore { - github_config_url: "https://github.com/Devsights".into(), // the ORG → all its repos - github_config_secret: "devsights-github".into(), // the Secret from B.2 - runner_scale_set_name: "okd-cb1".into(), // == the runs-on label - cpu_limit: "46".into(), - memory_limit: "32Gi".into(), - ..GithubRunnerScore::defaults() -}; -// controller once per cluster, then the runner set: -vec![Box::new(runner.controller()), Box::new(runner.runner_set())] -``` - -OKD: grant the runner ServiceAccount the `privileged` SCC (dind needs it): -`oc adm policy add-scc-to-user privileged -z -n arc-runners`. The -runner has **no** cluster RBAC — jobs reach their target cluster via their own -brokered kubeconfig. - ---- - -## Verify - -```sh -oc get autoscalingrunnerset okd-cb1 -n arc-runners # stops erroring once the Secret exists -oc get pods -n arc-runners -w # listener settles; a runner pod appears -``` - -In the client org: **Settings → Actions → Runners** shows the runner group; any -repo's workflow can then target it with `runs-on: okd-cb1`. - -## Rotation / revocation - -- **Rotate the key:** generate a new private key, update every per-org Secret, - then delete the old key under the App's **Private keys**. GitHub Apps keep - multiple active keys for zero-downtime rotation.[^keys] -- **Off-board a client:** the client uninstalls the App from their org; NationTech - deletes that org's scale set + Secret. - ---- - -## Sources - -- [Authenticating ARC to the GitHub API — GitHub Docs][^arc-auth] — repo vs org permissions, App ID, installation ID, Secret keys + same-namespace rule. -- [Deploying runner scale sets with ARC — GitHub Docs][^deploy] — `githubConfigUrl` at repo/org/enterprise level; `githubConfigSecret`. -- [Registering a GitHub App — GitHub Docs][^register] — registration flow, fields, disabling the webhook, "Any account" installability, permissions. -- [Managing private keys for GitHub Apps — GitHub Docs][^keys] — App ID location, generating the `.pem`, rotation. - -[^arc-auth]: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/authenticating-to-the-github-api -[^deploy]: https://docs.github.com/en/actions/how-tos/manage-runners/use-actions-runner-controller/deploy-runner-scale-sets -[^register]: https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/registering-a-github-app -[^keys]: https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/managing-private-keys-for-github-apps diff --git a/harmony/src/modules/github_runner.rs b/harmony/src/modules/github_runner.rs index bbb655a7..7f7db9e0 100644 --- a/harmony/src/modules/github_runner.rs +++ b/harmony/src/modules/github_runner.rs @@ -24,8 +24,8 @@ //! vec![Box::new(runner.controller()), Box::new(runner.runner_set())] //! ``` //! -//! See `docs/guides/github-self-hosted-runners.md` for the GitHub-App auth setup -//! (one App, one org install per client, hosted by the platform). +//! GitHub-App auth (one org-scoped App, installed once per client org) is a +//! platform/operations concern documented separately, not in this repo. //! //! On OpenShift/OKD, `containerMode: dind` needs a privileged container — grant //! the runner ServiceAccount the `privileged` SCC. The runner needs **no** -- 2.39.5 From e92e263a42e6e44ba206fdbf543e503acae28a82 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 26 Jun 2026 14:59:06 -0400 Subject: [PATCH 06/15] fix(k8s): skip cluster-scope operator checks when access is restricted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A namespace-scoped deployer (e.g. a tenant CI service account) cannot list pods cluster-wide, so the cert-manager and CNPG operator readiness checks 403'd and failed the whole deploy — pushing operators to grant the tenant cluster-admin, which is a security anti-pattern. Both checks now treat a 403 Forbidden as "this cluster-wide platform operator is managed out-of-band": log a clean info ("Skipped … due to restricted access") and continue, assuming the operator is present. Non-403 errors still propagate. --- .../topology/k8s_anywhere/k8s_anywhere.rs | 28 ++++++++++++++-- harmony/src/modules/postgresql/score_k8s.rs | 32 ++++++++++++++++--- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 73184b7a..06abd99a 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -390,6 +390,14 @@ impl Serialize for K8sAnywhereTopology { } } +/// True when a Kubernetes API error is an RBAC denial (403 Forbidden). Used to +/// skip cluster-scope checks a namespace-scoped deployer isn't allowed to run, +/// rather than failing the whole deploy. +fn is_forbidden(e: &E) -> bool { + let msg = e.to_string().to_lowercase(); + msg.contains("forbidden") || msg.contains("403") +} + #[async_trait] impl CertificateManagement for K8sAnywhereTopology { async fn install(&self) -> Result { @@ -408,13 +416,29 @@ impl CertificateManagement for K8sAnywhereTopology { async fn ensure_certificate_management_ready(&self) -> Result { let k8s_client = self.k8s_client().await.unwrap(); - let pods = k8s_client + let pods = match k8s_client .list_all_resources_with_labels::( "app.kubernetes.io/component=controller,\ app.kubernetes.io/name=cert-manager", ) .await - .map_err(|e| ExecutorError::UnexpectedError(format!("{}", e)))?; + { + Ok(pods) => pods, + // A namespace-scoped deployer (e.g. a tenant CI service account) + // cannot list pods cluster-wide. cert-manager is a cluster-wide + // platform component managed out-of-band, so don't fail the deploy — + // skip the readiness check and assume it is present. + Err(e) if is_forbidden(&e) => { + info!( + "Skipped cert-manager setup due to restricted (namespace-scoped) \ + access — assuming cert-manager is managed cluster-wide by the platform" + ); + return Ok(Outcome::noop( + "cert-manager readiness skipped (restricted access)".to_string(), + )); + } + Err(e) => return Err(ExecutorError::UnexpectedError(format!("{}", e))), + }; if pods.is_empty() { info!("cert-manager not installed (no controller pods found)"); diff --git a/harmony/src/modules/postgresql/score_k8s.rs b/harmony/src/modules/postgresql/score_k8s.rs index 89f1e430..50f70b1e 100644 --- a/harmony/src/modules/postgresql/score_k8s.rs +++ b/harmony/src/modules/postgresql/score_k8s.rs @@ -22,6 +22,13 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use log::{debug, info, warn}; use serde::Serialize; +/// True when a Kubernetes API error is an RBAC denial (403 Forbidden) — lets a +/// namespace-scoped deployer skip cluster-scope checks it isn't allowed to run. +fn is_forbidden(e: &E) -> bool { + let msg = e.to_string().to_lowercase(); + msg.contains("forbidden") || msg.contains("403") +} + /// Deploys an opinionated, highly available PostgreSQL cluster managed by CNPG. /// /// This score automatically ensures the CloudNativePG (CNPG) operator is installed @@ -129,12 +136,29 @@ impl K8sPostgreSQLInterpret { .await .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {}", e)))?; - let pods = k8s_client + let pods = match k8s_client .list_all_resources_with_labels::("app.kubernetes.io/name=cloudnative-pg") .await - .map_err(|e| { - InterpretError::new(format!("Failed to list CNPG operator pods: {}", e)) - })?; + { + Ok(pods) => pods, + // A namespace-scoped deployer (e.g. a tenant CI service account) + // cannot list pods cluster-wide. CNPG is a cluster-wide platform + // operator managed out-of-band, so skip the install/readiness check + // rather than fail the deploy — assume it is present. + Err(e) if is_forbidden(&e) => { + info!( + "Skipped CNPG operator setup due to restricted (namespace-scoped) \ + access — assuming the CNPG operator is managed cluster-wide by the platform" + ); + return Ok(()); + } + Err(e) => { + return Err(InterpretError::new(format!( + "Failed to list CNPG operator pods: {}", + e + ))); + } + }; if !pods.is_empty() { info!("CNPG operator is already installed"); -- 2.39.5 From a8809c0838b8ff405bf62a0a87a84b012487f6c7 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 26 Jun 2026 15:05:05 -0400 Subject: [PATCH 07/15] fix(zitadel): don't patch the namespace if it already exists ZitadelScore applied the (cluster-scoped) Namespace object unconditionally, so a namespace-scoped tenant deployer 403'd on "patch namespaces" even though the namespace was already provisioned by platform onboarding. Mirror the Postgres module: check namespace_exists (a GET, which the deployer is allowed) and only create when genuinely missing. --- harmony/src/modules/zitadel/mod.rs | 37 +++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index 0076ab4c..6d77436b 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -230,19 +230,34 @@ impl Interpret for Zitade self.host ); - info!("Creating namespace {ns} if it does not exist"); - K8sResourceScore::single( - Namespace { - metadata: ObjectMeta { - name: Some(self.namespace.to_string()), + // The tenant namespace is provisioned by platform onboarding, and the + // Namespace object is cluster-scoped — a namespace-scoped deployer can't + // patch it (and shouldn't). Only create it if it's genuinely missing. + let k8s_client = topology + .k8s_client() + .await + .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {e}")))?; + if k8s_client + .namespace_exists(ns) + .await + .map_err(|e| InterpretError::new(format!("Failed to check namespace '{ns}': {e}")))? + { + info!("Namespace '{ns}' already exists; skipping create"); + } else { + info!("Creating namespace {ns}"); + K8sResourceScore::single( + Namespace { + metadata: ObjectMeta { + name: Some(self.namespace.to_string()), + ..Default::default() + }, ..Default::default() }, - ..Default::default() - }, - None, - ) - .interpret(inventory, topology) - .await?; + None, + ) + .interpret(inventory, topology) + .await?; + } // --- Step 1: PostgreSQL ------------------------------------------- -- 2.39.5 From c18aa5d178a1b5e5ceb983bd05b68a157e7c2599 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 26 Jun 2026 15:14:27 -0400 Subject: [PATCH 08/15] refactor(k8s): centralize namespace creation in K8sClient::ensure_namespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add K8sClient::ensure_namespace(ns): a GET decides — no-op if the namespace already exists (e.g. provisioned by platform onboarding), create only when genuinely missing. Never patches the cluster-scoped Namespace object, so a namespace-scoped tenant deployer doesn't need cluster RBAC. Route the tenant-deploy path through it and stop helm from patching the namespace: - postgresql K8sPostgreSQLScore: use ensure_namespace (was inline). - zitadel: use ensure_namespace; HelmChartScore create_namespace=false. - app ComposeAppScore: ensure_namespace before install; create_namespace=false. Fixes the prod deploy 403 "cannot patch resource namespaces" under a namespace-scoped service account. --- harmony-k8s/src/resources.rs | 23 ++++++++++++- harmony/src/modules/postgresql/score_k8s.rs | 37 +++------------------ harmony/src/modules/zitadel/mod.rs | 35 ++++++------------- harmony_app/src/score.rs | 12 ++++++- 4 files changed, 48 insertions(+), 59 deletions(-) diff --git a/harmony-k8s/src/resources.rs b/harmony-k8s/src/resources.rs index d0d5562c..de377bc7 100644 --- a/harmony-k8s/src/resources.rs +++ b/harmony-k8s/src/resources.rs @@ -13,7 +13,7 @@ use kube::{ runtime::conditions, runtime::wait::await_condition, }; -use log::debug; +use log::{debug, info}; use serde::de::DeserializeOwned; use serde_json::Value; use std::time::Duration; @@ -373,6 +373,27 @@ impl K8sClient { .await } + /// Ensure the namespace exists, **without ever patching it**. A GET (which a + /// namespace-scoped deployer is allowed to do) decides: if the namespace is + /// already present — e.g. provisioned by platform onboarding — this is a + /// no-op; only a genuinely missing namespace is created. This keeps tenant + /// deployers from needing cluster-scoped permission to patch the + /// (cluster-scoped) Namespace object, which a namespaced RBAC role can't do. + /// + /// Use this instead of applying a `Namespace` resource or passing + /// `helm --create-namespace` when the target namespace may be pre-created + /// and the deployer is namespace-scoped. + pub async fn ensure_namespace(&self, name: &str) -> Result<(), Error> { + if self.namespace_exists(name).await? { + info!("Namespace '{name}' already exists"); + return Ok(()); + } + info!("Creating namespace '{name}'"); + self.create_namespace(name).await?; + self.wait_for_namespace(name, Some(Duration::from_secs(30))) + .await + } + pub async fn wait_for_namespace( &self, name: &str, diff --git a/harmony/src/modules/postgresql/score_k8s.rs b/harmony/src/modules/postgresql/score_k8s.rs index 50f70b1e..231b2d07 100644 --- a/harmony/src/modules/postgresql/score_k8s.rs +++ b/harmony/src/modules/postgresql/score_k8s.rs @@ -89,42 +89,15 @@ impl K8sPostgreSQLInterpret { .await .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {}", e)))?; - let namespace_name = &self.config.namespace; - - if k8s_client - .namespace_exists(namespace_name) + k8s_client + .ensure_namespace(&self.config.namespace) .await .map_err(|e| { InterpretError::new(format!( - "Failed to check namespace '{}': {}", - namespace_name, e + "Failed to ensure namespace '{}': {}", + self.config.namespace, e )) - })? - { - info!("Namespace '{}' already exists", namespace_name); - return Ok(()); - } - - info!("Creating namespace '{}'", namespace_name); - k8s_client - .create_namespace(namespace_name) - .await - .map_err(|e| { - InterpretError::new(format!( - "Failed to create namespace '{}': {}", - namespace_name, e - )) - })?; - - k8s_client - .wait_for_namespace(namespace_name, Some(std::time::Duration::from_secs(30))) - .await - .map_err(|e| { - InterpretError::new(format!("Namespace '{}' not ready: {}", namespace_name, e)) - })?; - - info!("Namespace '{}' is ready", namespace_name); - Ok(()) + }) } async fn ensure_cnpg_operator( diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index 6d77436b..c073ef58 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -36,7 +36,6 @@ use crate::{ interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, modules::helm::chart::{HelmChartScore, HelmRepository}, - modules::k8s::resource::K8sResourceScore, modules::postgresql::capability::{PostgreSQL, PostgreSQLClusterRole, PostgreSQLConfig}, score::Score, topology::{HelmCommand, K8sclient, Topology}, @@ -230,34 +229,17 @@ impl Interpret for Zitade self.host ); - // The tenant namespace is provisioned by platform onboarding, and the - // Namespace object is cluster-scoped — a namespace-scoped deployer can't - // patch it (and shouldn't). Only create it if it's genuinely missing. + // Ensure the tenant namespace exists without patching it (it is + // provisioned by platform onboarding; a namespace-scoped deployer can't + // patch the cluster-scoped Namespace object). let k8s_client = topology .k8s_client() .await .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {e}")))?; - if k8s_client - .namespace_exists(ns) + k8s_client + .ensure_namespace(ns) .await - .map_err(|e| InterpretError::new(format!("Failed to check namespace '{ns}': {e}")))? - { - info!("Namespace '{ns}' already exists; skipping create"); - } else { - info!("Creating namespace {ns}"); - K8sResourceScore::single( - Namespace { - metadata: ObjectMeta { - name: Some(self.namespace.to_string()), - ..Default::default() - }, - ..Default::default() - }, - None, - ) - .interpret(inventory, topology) - .await?; - } + .map_err(|e| InterpretError::new(format!("Failed to ensure namespace '{ns}': {e}")))?; // --- Step 1: PostgreSQL ------------------------------------------- @@ -890,7 +872,10 @@ login: chart_version: Some(NonBlankString::from_str("9.27.1").unwrap()), values_overrides: None, values_yaml: Some(values_yaml), - create_namespace: true, + // The namespace is ensured up front via `ensure_namespace`; don't let + // helm `--create-namespace` patch the (cluster-scoped) Namespace + // object, which a namespace-scoped deployer isn't allowed to do. + create_namespace: false, install_only: false, repository: Some(HelmRepository::new( "zitadel".to_string(), diff --git a/harmony_app/src/score.rs b/harmony_app/src/score.rs index 420e13cf..9ef05a48 100644 --- a/harmony_app/src/score.rs +++ b/harmony_app/src/score.rs @@ -229,6 +229,16 @@ impl ComposeAppInterpret { version: &str, ) -> Result { let s = &self.score; + // Ensure the namespace exists without patching it (a namespace-scoped + // deployer can't patch the cluster-scoped Namespace object), then install + // without helm `--create-namespace`. + topology + .k8s_client() + .await + .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {e}")))? + .ensure_namespace(&s.namespace) + .await + .map_err(|e| InterpretError::new(format!("Failed to ensure namespace '{}': {e}", s.namespace)))?; HelmChartScore { namespace: Some(non_blank(&s.namespace, "namespace")?), release_name: non_blank(&s.release_name, "release_name")?, @@ -236,7 +246,7 @@ impl ComposeAppInterpret { chart_version: Some(non_blank(version, "chart_version")?), values_overrides: None, values_yaml: None, - create_namespace: true, + create_namespace: false, install_only: false, repository: None, } -- 2.39.5 From 4ba5c46bc64c40c606a44b2e09a46bc47e70200b Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 26 Jun 2026 17:37:51 -0400 Subject: [PATCH 09/15] =?UTF-8?q?fix(zitadel):=20idempotent=20install=20?= =?UTF-8?q?=E2=80=94=20durable=20masterkey=20+=20install-once?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two re-run hazards under the install/converge model: 1. Masterkey was generated fresh every run and only survived via the 409 on the k8s Secret — a deleted Secret would mint a new key and orphan the existing Zitadel DB. Now resolve it durably: prefer the in-cluster Secret (matches the running DB), else the value persisted in harmony_secret, else generate; then mirror into harmony_secret so a namespace reset reuses it. (New ZitadelMasterkey persisted secret, mirroring ZitadelAdmin.) 2. The chart's `zitadel-setup` init Job is a pre-upgrade hook that fails when re-run on `helm upgrade`. Set install_only=true: once the release exists, skip the helm op entirely. Zitadel is install-once + stateful; re-deploys converge via ZitadelSetupScore, not by re-running the chart. --- harmony/src/modules/zitadel/mod.rs | 62 ++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index c073ef58..65bbc544 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -399,12 +399,47 @@ impl Interpret for Zitade MASTERKEY_SECRET_NAME, self.namespace ); - // Masterkey for symmetric encryption — must be exactly 32 ASCII bytes (alphanumeric only). - let masterkey = rng() - .sample_iter(&rand::distr::Alphanumeric) - .take(32) - .map(char::from) - .collect::(); + // Masterkey for symmetric encryption — exactly 32 ASCII alphanumeric + // bytes, and it MUST be stable: regenerating it makes the existing + // Zitadel DB undecryptable. Resolve in priority order: + // 1. the in-cluster Secret, if present (authoritative — it's what the + // running instance encrypted its data with), + // 2. the value persisted in harmony_secret (survives a namespace reset), + // 3. a freshly generated one. + // Then mirror the resolved value into harmony_secret so a deleted/ + // recreated namespace reuses it rather than minting a new (broken) key. + let existing_masterkey = k8s_client + .get_resource::(MASTERKEY_SECRET_NAME, Some(&self.namespace)) + .await + .ok() + .flatten() + .and_then(|s| s.data?.get("masterkey").cloned()) + .and_then(|bs| String::from_utf8(bs.0).ok()); + + let masterkey = match existing_masterkey { + Some(k) => k, + None => match SecretManager::get::().await { + Ok(m) => m.masterkey, + Err(e) => { + debug!("[Zitadel] No persisted masterkey yet ({e}); generating"); + rng() + .sample_iter(&rand::distr::Alphanumeric) + .take(32) + .map(char::from) + .collect::() + } + }, + }; + + if SecretManager::get::().await.is_err() { + SecretManager::set(&ZitadelMasterkey { + masterkey: masterkey.clone(), + }) + .await + .map_err(|e| { + InterpretError::new(format!("Failed to persist Zitadel masterkey: {e}")) + })?; + } debug!( "[Zitadel] Created masterkey secret '{}' in namespace '{}'", @@ -876,7 +911,13 @@ login: // helm `--create-namespace` patch the (cluster-scoped) Namespace // object, which a namespace-scoped deployer isn't allowed to do. create_namespace: false, - install_only: false, + // Install-once: Zitadel is stateful (DB + masterkey persist), and its + // chart's `zitadel-setup` init Job is a pre-upgrade hook that isn't + // safely re-runnable. Once the release exists we skip the helm op + // entirely — re-deploys converge via ZitadelSetupScore, not by + // re-running the chart. (Deliberate version upgrades are a separate, + // explicit action.) + install_only: true, repository: Some(HelmRepository::new( "zitadel".to_string(), hurl!("https://charts.zitadel.com"), @@ -940,3 +981,10 @@ struct ZitadelAdmin { username: String, password: String, } + +/// Persisted Zitadel masterkey (32 ASCII alphanumeric bytes). Stored so it +/// survives a namespace/Secret reset — a new key would orphan the existing DB. +#[derive(Secret, Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] +struct ZitadelMasterkey { + masterkey: String, +} -- 2.39.5 From 75aed3677f89191beb34027ea8473d6e851ebe2a Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 26 Jun 2026 17:46:42 -0400 Subject: [PATCH 10/15] fix(zitadel): default the Zitadel CNPG cluster to a single instance Storage durability is provided by the backing layer (Ceph); CNPG-level replication added ops/cost (and a stuck second replica on single-node clusters) without buying availability. Default instances 2 -> 1. --- harmony/src/modules/zitadel/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index 65bbc544..f7d86dc9 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -245,7 +245,10 @@ impl Interpret for Zitade let pg_config = PostgreSQLConfig { cluster_name: PG_CLUSTER_NAME.to_string(), - instances: 2, + // Single instance by default: storage durability comes from the + // backing layer (Ceph), so CNPG-level replication adds cost/ops + // without buying availability here. + instances: 1, storage_size: StorageSize::gi(10), role: PostgreSQLClusterRole::Primary, namespace: self.namespace.to_string(), -- 2.39.5 From 0127558468c84a4f758629a1987ba0f1f849de85 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 26 Jun 2026 18:29:22 -0400 Subject: [PATCH 11/15] feat(registry-pull): pull-secret Score + app imagePullSecrets + config fill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets a namespace-scoped deploy pull private images declaratively, derived from context (no procedural step): - harmony: RegistryPullSecretScore — builds a kubernetes.io/dockerconfigjson Secret in the namespace from pull-only registry creds (idempotent, namespaced). - harmony_app: DeployConfig.image_pull_secrets renders into each pod's spec.template.spec.imagePullSecrets (mirrors extra_env / secret_file_mounts). - harmony_config: get_or_prompt now fills only the MISSING fields of a partial stored value (e.g. when a Secret struct gains a field) and persists the merge, instead of failing or re-prompting everything. Prompting is behind a FieldPrompter seam so the flow is unit-tested (no terminal). --- harmony/src/modules/mod.rs | 1 + harmony/src/modules/registry_pull_secret.rs | 126 ++++++++++++ harmony_app/src/chart.rs | 39 +++- harmony_config/src/lib.rs | 115 ++++++++++- harmony_config/src/source/prompt.rs | 216 +++++++++++++------- 5 files changed, 409 insertions(+), 88 deletions(-) create mode 100644 harmony/src/modules/registry_pull_secret.rs diff --git a/harmony/src/modules/mod.rs b/harmony/src/modules/mod.rs index 5ad4a788..b0600667 100644 --- a/harmony/src/modules/mod.rs +++ b/harmony/src/modules/mod.rs @@ -12,6 +12,7 @@ pub mod http; pub mod inventory; pub mod k3d; pub mod k8s; +pub mod registry_pull_secret; #[cfg(all(feature = "kvm", unix))] pub mod kvm; pub mod lamp; diff --git a/harmony/src/modules/registry_pull_secret.rs b/harmony/src/modules/registry_pull_secret.rs new file mode 100644 index 00000000..df1c7c2b --- /dev/null +++ b/harmony/src/modules/registry_pull_secret.rs @@ -0,0 +1,126 @@ +//! A reusable Score that materializes a `kubernetes.io/dockerconfigjson` Secret +//! so the kubelet can pull an app's images from a private registry. +//! +//! The Secret is referenced by name from each pod's `imagePullSecrets` (see +//! `harmony_app`'s `DeployConfig::image_pull_secrets`). Keep the credentials +//! **pull-only** and load them from a vault (e.g. OpenBao `DeploySecrets`) — they +//! never belong in code or chart values. The Secret is namespaced, so a +//! namespace-scoped deployer can apply it without any cluster RBAC. +//! +//! ```ignore +//! let pull = RegistryPullSecretScore { +//! namespace: "devsights-folk".into(), +//! name: "registry-pull".into(), +//! registry: "hub.nationtech.io".into(), +//! username: pull_user, +//! token: pull_token, +//! }; +//! // add `pull` to scores(), and set deploy.image_pull_secrets = vec!["registry-pull"] +//! ``` + +use std::collections::BTreeMap; + +use base64::Engine; +use k8s_openapi::ByteString; +use k8s_openapi::api::core::v1::Secret as K8sSecret; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use serde::Serialize; + +use crate::interpret::Interpret; +use crate::modules::k8s::resource::K8sResourceScore; +use crate::score::Score; +use crate::topology::{K8sclient, Topology}; + +/// Creates a `kubernetes.io/dockerconfigjson` Secret in `namespace` from +/// pull-only registry credentials. Idempotent — a namespaced Secret apply. +#[derive(Debug, Clone, Serialize)] +pub struct RegistryPullSecretScore { + pub namespace: String, + /// Secret name — the same string goes in each pod's `imagePullSecrets`. + pub name: String, + /// Registry host the creds authenticate to, e.g. `hub.nationtech.io`. + pub registry: String, + pub username: String, + /// Pull-only robot token. Serialized into the Secret, never logged. + pub token: String, +} + +impl RegistryPullSecretScore { + /// Build the `.dockerconfigjson` Secret. The inner `auth` field is + /// `base64(user:token)` per Docker's config format; the kubelet reads it to + /// authenticate image pulls. + fn secret(&self) -> K8sSecret { + let auth = base64::engine::general_purpose::STANDARD + .encode(format!("{}:{}", self.username, self.token)); + let dockerconfig = serde_json::json!({ + "auths": { + &self.registry: { + "username": self.username, + "password": self.token, + "auth": auth, + } + } + }); + let mut data = BTreeMap::new(); + data.insert( + ".dockerconfigjson".to_string(), + ByteString(dockerconfig.to_string().into_bytes()), + ); + K8sSecret { + metadata: ObjectMeta { + name: Some(self.name.clone()), + namespace: Some(self.namespace.clone()), + ..Default::default() + }, + type_: Some("kubernetes.io/dockerconfigjson".to_string()), + data: Some(data), + ..Default::default() + } + } +} + +impl Score for RegistryPullSecretScore { + fn name(&self) -> String { + format!("RegistryPullSecretScore({}/{})", self.namespace, self.name) + } + + fn create_interpret(&self) -> Box> { + K8sResourceScore::single(self.secret(), Some(self.namespace.clone())).create_interpret() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> RegistryPullSecretScore { + RegistryPullSecretScore { + namespace: "devsights-folk".into(), + name: "registry-pull".into(), + registry: "hub.nationtech.io".into(), + username: "robot$pull".into(), + token: "s3cr3t".into(), + } + } + + #[test] + fn builds_a_typed_dockerconfigjson_secret() { + let s = sample().secret(); + assert_eq!(s.type_.as_deref(), Some("kubernetes.io/dockerconfigjson")); + assert_eq!(s.metadata.name.as_deref(), Some("registry-pull")); + assert_eq!(s.metadata.namespace.as_deref(), Some("devsights-folk")); + } + + #[test] + fn dockerconfig_carries_creds_and_base64_auth() { + let s = sample().secret(); + let blob = s.data.unwrap().remove(".dockerconfigjson").unwrap(); + let json: serde_json::Value = serde_json::from_slice(&blob.0).unwrap(); + let entry = &json["auths"]["hub.nationtech.io"]; + assert_eq!(entry["username"], "robot$pull"); + assert_eq!(entry["password"], "s3cr3t"); + let expected_auth = + base64::engine::general_purpose::STANDARD.encode("robot$pull:s3cr3t"); + assert_eq!(entry["auth"], expected_auth); + } +} diff --git a/harmony_app/src/chart.rs b/harmony_app/src/chart.rs index 7fa2ce09..fd344fe9 100644 --- a/harmony_app/src/chart.rs +++ b/harmony_app/src/chart.rs @@ -15,10 +15,10 @@ use k8s_openapi::api::apps::v1::{ Deployment, DeploymentSpec, DeploymentStrategy, RollingUpdateDeployment, }; use k8s_openapi::api::core::v1::{ - Container, ContainerPort, EnvFromSource, EnvVar, KeyToPath, PersistentVolumeClaim, - PersistentVolumeClaimSpec, PersistentVolumeClaimVolumeSource, PodSpec, PodTemplateSpec, - SecretEnvSource, SecretVolumeSource, Service, ServicePort, ServiceSpec, Volume, VolumeMount, - VolumeResourceRequirements, + Container, ContainerPort, EnvFromSource, EnvVar, KeyToPath, LocalObjectReference, + PersistentVolumeClaim, PersistentVolumeClaimSpec, PersistentVolumeClaimVolumeSource, PodSpec, + PodTemplateSpec, SecretEnvSource, SecretVolumeSource, Service, ServicePort, ServiceSpec, Volume, + VolumeMount, VolumeResourceRequirements, }; use k8s_openapi::apimachinery::pkg::api::resource::Quantity; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; @@ -56,6 +56,10 @@ pub struct DeployConfig { /// account key whose consumer wants a path, not an env var). By reference, /// like `extra_env`, so the chart carries no secret material. pub secret_file_mounts: Vec, + /// Names of `kubernetes.io/dockerconfigjson` Secrets the kubelet uses to + /// pull this app's images from a private registry, set on every pod's + /// `imagePullSecrets`. By reference — the Secret is applied separately. + pub image_pull_secrets: Vec, } /// Mount one key of a Secret as a file at `path` (its parent dir is the mount @@ -92,6 +96,7 @@ impl DeployConfig { rolling: prod, extra_env: Vec::new(), secret_file_mounts: Vec::new(), + image_pull_secrets: Vec::new(), } } } @@ -305,6 +310,12 @@ fn deployment(svc: &ComposeService, cfg: &DeployConfig) -> Deployment { volume_mounts: (!volume_mounts.is_empty()).then_some(volume_mounts), ..Default::default() }], + image_pull_secrets: (!cfg.image_pull_secrets.is_empty()).then(|| { + cfg.image_pull_secrets + .iter() + .map(|name| LocalObjectReference { name: name.clone() }) + .collect() + }), volumes: (!volumes.is_empty()).then_some(volumes), ..Default::default() }), @@ -401,6 +412,7 @@ mod tests { rolling: true, extra_env: vec![], secret_file_mounts: vec![], + image_pull_secrets: vec![], }; let tmp = tempfile::tempdir().unwrap(); build_chart(&app, &cfg, tmp.path()).unwrap(); @@ -445,6 +457,25 @@ mod tests { ); } + #[test] + fn image_pull_secrets_render_on_pods_when_set() { + let (app, mut cfg, _t) = fixture(); + cfg.image_pull_secrets = vec!["registry-pull".to_string()]; + let tmp = tempfile::tempdir().unwrap(); + build_chart(&app, &cfg, tmp.path()).unwrap(); + let dep = read(&tmp, "deployment-backend.yaml"); + assert!(dep.contains("imagePullSecrets"), "{dep}"); + assert!(dep.contains("registry-pull"), "{dep}"); + } + + #[test] + fn no_image_pull_secrets_block_when_empty() { + // `fixture()` leaves image_pull_secrets empty → no block rendered. + let (_, _, tmp) = fixture(); + let dep = read(&tmp, "deployment-backend.yaml"); + assert!(!dep.contains("imagePullSecrets"), "{dep}"); + } + #[test] fn portless_service_emits_no_service_object() { let (_, _, tmp) = fixture(); diff --git a/harmony_config/src/lib.rs b/harmony_config/src/lib.rs index ae2a4cd9..26d4d9f7 100644 --- a/harmony_config/src/lib.rs +++ b/harmony_config/src/lib.rs @@ -19,6 +19,7 @@ pub use harmony_config_derive::Config; pub use source::env::EnvSource; pub use source::local_file::LocalFileSource; pub use source::prompt::PromptSource; +pub use source::prompt::{FieldPrompter, FieldType, InquirePrompter}; pub use source::sqlite::SqliteSource; pub use source::store::StoreSource; pub use source::toml_file::TomlFileSource; @@ -171,15 +172,45 @@ impl ConfigClient { } pub async fn get_or_prompt(&self) -> Result { - match self.get::().await { - Ok(config) => Ok(config), - Err(ConfigError::NotFound { .. }) => { - let config = PromptSource::new().prompt_for::().await?; - self.set(&config).await?; - Ok(config) + self.get_or_fill_missing::(&InquirePrompter).await + } + + /// Like [`get`](Self::get), but when no source holds a *complete* `T`, + /// prompt for the fields that are missing — keeping any partial values + /// already stored (e.g. after a `Secret` struct gains a field) — then + /// persist the merged result. Prompting is delegated to `prompter` so the + /// flow is testable without a terminal. + async fn get_or_fill_missing( + &self, + prompter: &dyn FieldPrompter, + ) -> Result { + let mut partial: Option> = None; + for source in &self.sources { + if let Some(value) = source.get(T::CLASS, T::KEY).await? { + match serde_json::from_value::(value.clone()) { + Ok(config) => return Ok(config), + Err(e) => { + warn!( + "Incomplete or stale value for {} in a source ({e}); \ + will prompt for any missing fields", + T::KEY + ); + // Remember the first partial object so we fill its gaps + // instead of re-prompting fields we already have. + if partial.is_none() + && let serde_json::Value::Object(map) = value + { + partial = Some(map); + } + } + } } - Err(e) => Err(e), } + let config = PromptSource::new() + .prompt_filling::(partial.unwrap_or_default(), prompter) + .await?; + self.set(&config).await?; + Ok(config) } pub async fn set(&self, config: &T) -> Result<(), ConfigError> { @@ -432,6 +463,76 @@ mod tests { } } + /// A `FieldPrompter` double: returns canned answers and records which + /// fields it was asked for, so a test can assert only the missing fields + /// were prompted. + struct FakePrompter { + answers: std::collections::HashMap<&'static str, serde_json::Value>, + asked: std::sync::Mutex>, + } + + impl FieldPrompter for FakePrompter { + fn prompt( + &self, + _key: &str, + field: &str, + _ty: FieldType, + _is_secret: bool, + ) -> Result { + self.asked.lock().unwrap().push(field.to_string()); + self.answers + .get(field) + .cloned() + .ok_or_else(|| ConfigError::PromptError(format!("no canned answer for `{field}`"))) + } + } + + #[tokio::test] + async fn get_or_prompt_fills_only_missing_secret_fields_and_persists() { + // A Secret struct that has GAINED the `token` field since the stored + // value was written. + #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Config)] + #[config(secret)] + struct EvolvingSecret { + username: String, + token: String, + } + + // Store the OLD shape — only `username` existed back then. + let mut data = std::collections::HashMap::new(); + data.insert( + "EvolvingSecret".to_string(), + serde_json::json!({ "username": "alice" }), + ); + let mock = Arc::new(MockSource::with_data(data)); + let client = ConfigClient::new(vec![mock.clone()]); + + let fake = FakePrompter { + answers: std::collections::HashMap::from([("token", serde_json::json!("s3cr3t"))]), + asked: std::sync::Mutex::new(Vec::new()), + }; + + let got: EvolvingSecret = client.get_or_fill_missing(&fake).await.unwrap(); + + // Only the missing field was prompted; the existing value was kept. + assert_eq!(*fake.asked.lock().unwrap(), vec!["token".to_string()]); + assert_eq!(got.username, "alice"); + assert_eq!(got.token, "s3cr3t"); + + // The merged value was written back to the store. + let stored = mock + .data + .lock() + .unwrap() + .get("EvolvingSecret") + .cloned() + .unwrap(); + assert_eq!( + stored, + serde_json::json!({ "username": "alice", "token": "s3cr3t" }) + ); + } + #[tokio::test] async fn test_get_returns_value_when_found_in_first_source() { let config = TestConfig { diff --git a/harmony_config/src/source/prompt.rs b/harmony_config/src/source/prompt.rs index f25ea13b..dcd6a87f 100644 --- a/harmony_config/src/source/prompt.rs +++ b/harmony_config/src/source/prompt.rs @@ -9,6 +9,93 @@ use crate::{Config, ConfigClass, ConfigError, ConfigSource}; // terminal ownership, so concurrent prompts would corrupt the terminal. static PROMPT_MUTEX: Mutex<()> = Mutex::const_new(()); +/// The primitive shape of a flat config field, resolved from its JsonSchema. +/// Decouples schema-walking from the actual input widget so the latter can be +/// swapped out (e.g. for tests) — see [`FieldPrompter`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FieldType { + String, + Integer, + Number, + Boolean, +} + +/// Produces a value for a single config field. The real implementation +/// ([`InquirePrompter`]) reads it interactively; tests inject a double so the +/// fill-missing flow can be exercised without a terminal. +pub trait FieldPrompter: Send + Sync { + fn prompt( + &self, + key: &str, + field: &str, + ty: FieldType, + is_secret: bool, + ) -> Result; +} + +/// Reads each field interactively via `inquire`: secret strings are masked +/// (`Password`), other primitives use type-appropriate input. +pub struct InquirePrompter; + +impl FieldPrompter for InquirePrompter { + fn prompt( + &self, + key: &str, + field: &str, + ty: FieldType, + is_secret: bool, + ) -> Result { + let label = format!("{field}:"); + match ty { + FieldType::String if is_secret => { + // Masked mode echoes `*` per keystroke (inquire's default + // Hidden mode shows nothing until enter). + let raw = Password::new(&label) + .with_display_mode(PasswordDisplayMode::Masked) + .without_confirmation() + .prompt() + .map_err(|e| { + ConfigError::PromptError(format!( + "Password prompt for `{key}::{field}` failed: {e}" + )) + })?; + Ok(serde_json::Value::String(raw)) + } + FieldType::String => { + let raw = Text::new(&label).prompt().map_err(|e| { + ConfigError::PromptError(format!("Text prompt for `{key}::{field}` failed: {e}")) + })?; + Ok(serde_json::Value::String(raw)) + } + FieldType::Integer => { + // i64 covers Config integer fields; serde narrows on deserialize. + let n = CustomType::::new(&label).prompt().map_err(|e| { + ConfigError::PromptError(format!( + "Integer prompt for `{key}::{field}` failed: {e}" + )) + })?; + Ok(serde_json::json!(n)) + } + FieldType::Number => { + let n = CustomType::::new(&label).prompt().map_err(|e| { + ConfigError::PromptError(format!( + "Number prompt for `{key}::{field}` failed: {e}" + )) + })?; + Ok(serde_json::json!(n)) + } + FieldType::Boolean => { + let b = CustomType::::new(&label).prompt().map_err(|e| { + ConfigError::PromptError(format!( + "Boolean prompt for `{key}::{field}` failed: {e}" + )) + })?; + Ok(serde_json::Value::Bool(b)) + } + } + } +} + pub struct PromptSource; impl PromptSource { @@ -16,16 +103,38 @@ impl PromptSource { Self } - /// Prompt for a `T`. `Standard` structs go through `interactive_parse` - /// (full nested support); `Secret` structs go through a flat-field - /// walker that reads `#[config(secret)]` fields via `inquire::Password`. + /// Prompt for a full `T` (all fields). Equivalent to filling an empty value. pub async fn prompt_for(&self) -> Result { + self.prompt_filling::(serde_json::Map::new(), &InquirePrompter) + .await + } + + /// Prompt for the fields of `T` that are **absent** from `existing`, keeping + /// the values already present, then deserialize the merged object. This is + /// what lets a `Secret` struct gain a field without re-prompting the rest. + /// + /// `Secret` structs go through a flat-field walker (only the missing fields + /// are prompted, via `prompter`); `Standard` structs always go through the + /// full interactive parser (partial fill of nested structs is unsupported). + pub async fn prompt_filling( + &self, + existing: serde_json::Map, + prompter: &dyn FieldPrompter, + ) -> Result { let _guard = PROMPT_MUTEX.lock().await; - let banner = format!( - "── Configuring `{}` ({:?}) — please fill the fields below ──", - T::KEY, - T::CLASS, - ); + let banner = if existing.is_empty() { + format!( + "── Configuring `{}` ({:?}) — please fill the fields below ──", + T::KEY, + T::CLASS, + ) + } else { + format!( + "── `{}` ({:?}) is missing fields — fill them in (existing values are kept) ──", + T::KEY, + T::CLASS, + ) + }; // inquire renders on stderr; match that channel and pad with blank // lines so the banner stays separate from preceding log output. eprintln!(); @@ -35,7 +144,7 @@ impl PromptSource { ConfigClass::Standard => { T::parse_to_obj().map_err(|e| ConfigError::PromptError(e.to_string())) } - ConfigClass::Secret => prompt_secret_struct::(), + ConfigClass::Secret => prompt_secret_struct_filling::(existing, prompter), } } } @@ -66,10 +175,13 @@ impl ConfigSource for PromptSource { } } -// Walks a Secret struct's schema, reading `T::SECRET_FIELDS` via Password -// and other fields via type-appropriate prompts. Only flat primitive -// fields are supported; anything else returns a clear PromptError. -fn prompt_secret_struct() -> Result { +// Walks a Secret struct's schema and prompts (via `prompter`) for every field +// NOT already present in `json`, leaving existing values untouched. Only flat +// primitive fields are supported; anything else returns a clear PromptError. +fn prompt_secret_struct_filling( + mut json: serde_json::Map, + prompter: &dyn FieldPrompter, +) -> Result { let root: RootSchema = schemars::schema_for!(T); let object = root.schema.object.as_deref().ok_or_else(|| { ConfigError::PromptError(format!( @@ -79,11 +191,15 @@ fn prompt_secret_struct() -> Result { )) })?; - let mut json = serde_json::Map::with_capacity(object.properties.len()); for (field_name, field_schema) in &object.properties { + // Already stored (e.g. from a previous version of the struct) — keep it. + if json.contains_key(field_name) { + continue; + } let field_object = schema_object(field_schema, T::KEY, field_name)?; + let ty = field_type::(field_name, field_object)?; let is_secret = T::SECRET_FIELDS.contains(&field_name.as_str()); - let value = prompt_field::(field_name, field_object, is_secret)?; + let value = prompter.prompt(T::KEY, field_name, ty, is_secret)?; json.insert(field_name.clone(), value); } @@ -109,14 +225,10 @@ fn schema_object<'a>( } } -// Picks the inquire widget for one field from its JsonSchema. Secret -// strings use Password; other primitives use type-appropriate input. -// Returns Err for unsupported shapes — never a silent unmasked fallback. -fn prompt_field( - field_name: &str, - schema: &SchemaObject, - is_secret: bool, -) -> Result { +// Resolves one field's primitive `FieldType` from its JsonSchema. Returns Err +// for unsupported shapes (multi-type/nullable, nested, arrays) — never a silent +// fallback. +fn field_type(field_name: &str, schema: &SchemaObject) -> Result { let instance_type = schema.instance_type.as_ref().ok_or_else(|| { ConfigError::PromptError(format!( "Secret struct `{}` field `{field_name}` is not a flat primitive \ @@ -136,61 +248,11 @@ fn prompt_field( } }; - let label = format!("{field_name}:"); match single { - InstanceType::String => { - if is_secret { - // Masked mode echoes `*` per keystroke (inquire's default - // Hidden mode shows nothing until enter). - let raw = Password::new(&label) - .with_display_mode(PasswordDisplayMode::Masked) - .without_confirmation() - .prompt() - .map_err(|e| { - ConfigError::PromptError(format!( - "Password prompt for `{}::{field_name}` failed: {e}", - T::KEY - )) - })?; - Ok(serde_json::Value::String(raw)) - } else { - let raw = Text::new(&label).prompt().map_err(|e| { - ConfigError::PromptError(format!( - "Text prompt for `{}::{field_name}` failed: {e}", - T::KEY - )) - })?; - Ok(serde_json::Value::String(raw)) - } - } - InstanceType::Integer => { - // i64 covers Config integer fields; serde narrows on deserialize. - let n = CustomType::::new(&label).prompt().map_err(|e| { - ConfigError::PromptError(format!( - "Integer prompt for `{}::{field_name}` failed: {e}", - T::KEY - )) - })?; - Ok(serde_json::json!(n)) - } - InstanceType::Number => { - let n = CustomType::::new(&label).prompt().map_err(|e| { - ConfigError::PromptError(format!( - "Number prompt for `{}::{field_name}` failed: {e}", - T::KEY - )) - })?; - Ok(serde_json::json!(n)) - } - InstanceType::Boolean => { - let b = CustomType::::new(&label).prompt().map_err(|e| { - ConfigError::PromptError(format!( - "Boolean prompt for `{}::{field_name}` failed: {e}", - T::KEY - )) - })?; - Ok(serde_json::Value::Bool(b)) - } + InstanceType::String => Ok(FieldType::String), + InstanceType::Integer => Ok(FieldType::Integer), + InstanceType::Number => Ok(FieldType::Number), + InstanceType::Boolean => Ok(FieldType::Boolean), other => Err(ConfigError::PromptError(format!( "Secret struct `{}` field `{field_name}` has unsupported type \ `{other:?}`; the walker only handles flat primitives.", -- 2.39.5 From 49ffea99a567262f129b35709fb33320ec7034fc Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 3 Jul 2026 18:01:13 -0400 Subject: [PATCH 12/15] feat(app): add --force-conflicts argument to app cli that will be passed down to helm upgrade --install --- examples/nats-supercluster/src/main.rs | 1 + examples/nats/src/main.rs | 1 + .../src/operator/score.rs | 2 + .../application/features/helm_argocd_score.rs | 1 + harmony/src/modules/cert_manager/helm.rs | 1 + harmony/src/modules/cert_manager/operator.rs | 1 + harmony/src/modules/github_runner.rs | 2 + harmony/src/modules/helm/chart.rs | 7 +- harmony/src/modules/lamp.rs | 1 + harmony/src/modules/mod.rs | 2 +- .../monitoring/grafana/helm/helm_grafana.rs | 1 + .../kube_prometheus/crd/grafana_operator.rs | 1 + .../crd/prometheus_operator.rs | 1 + .../rhob_cluster_observability_operator.rs | 1 + .../helm/kube_prometheus_helm_chart.rs | 1 + .../monitoring/ntfy/helm/ntfy_helm_chart.rs | 1 + .../prometheus/helm/prometheus_helm.rs | 1 + harmony/src/modules/nats/helm_chart.rs | 1 + harmony/src/modules/openbao/mod.rs | 1 + harmony/src/modules/postgresql/operator.rs | 1 + harmony/src/modules/registry_pull_secret.rs | 3 +- harmony/src/modules/zitadel/mod.rs | 1 + harmony/src/modules/zitadel/setup.rs | 116 +++++++++--------- harmony_app/src/app.rs | 35 +++++- harmony_app/src/chart.rs | 10 +- harmony_app/src/compose.rs | 4 +- harmony_app/src/deploy.rs | 33 +++-- harmony_app/src/lib.rs | 4 +- harmony_app/src/score.rs | 6 +- harmony_app/tests/helm_render.rs | 1 + harmony_cli/src/app.rs | 32 ++++- harmony_config/src/source/prompt.rs | 9 +- 32 files changed, 192 insertions(+), 91 deletions(-) diff --git a/examples/nats-supercluster/src/main.rs b/examples/nats-supercluster/src/main.rs index 17338330..a6d1bc89 100644 --- a/examples/nats-supercluster/src/main.rs +++ b/examples/nats-supercluster/src/main.rs @@ -469,6 +469,7 @@ natsBox: values_yaml, create_namespace: true, install_only: false, + force_conflicts: false, repository: Some(HelmRepository::new( "nats".to_string(), hurl!("https://nats-io.github.io/k8s/helm/charts/"), diff --git a/examples/nats/src/main.rs b/examples/nats/src/main.rs index 75955841..ca1fcc9c 100644 --- a/examples/nats/src/main.rs +++ b/examples/nats/src/main.rs @@ -103,6 +103,7 @@ natsBox: values_yaml, create_namespace: true, install_only: false, + force_conflicts: false, repository: Some(HelmRepository::new( "nats".to_string(), hurl!("https://nats-io.github.io/k8s/helm/charts/"), diff --git a/fleet/harmony-fleet-deploy/src/operator/score.rs b/fleet/harmony-fleet-deploy/src/operator/score.rs index 305fb253..c60eaee0 100644 --- a/fleet/harmony-fleet-deploy/src/operator/score.rs +++ b/fleet/harmony-fleet-deploy/src/operator/score.rs @@ -273,6 +273,7 @@ impl Interpret for FleetOperatorInterp values_yaml: None, create_namespace: true, install_only: false, + force_conflicts: false, repository: None, } .interpret(inventory, topology) @@ -310,6 +311,7 @@ impl Interpret for FleetOperatorInterp values_yaml: None, create_namespace: true, install_only: false, + force_conflicts: false, repository: None, } .interpret(inventory, topology) diff --git a/harmony/src/modules/application/features/helm_argocd_score.rs b/harmony/src/modules/application/features/helm_argocd_score.rs index 6f02a221..748ddb91 100644 --- a/harmony/src/modules/application/features/helm_argocd_score.rs +++ b/harmony/src/modules/application/features/helm_argocd_score.rs @@ -1087,6 +1087,7 @@ commitServer: values_yaml: Some(values.to_string()), create_namespace: true, install_only: false, + force_conflicts: false, repository: Some(HelmRepository::new( "argo".to_string(), hurl!("https://argoproj.github.io/argo-helm"), diff --git a/harmony/src/modules/cert_manager/helm.rs b/harmony/src/modules/cert_manager/helm.rs index 4d8467a1..439d7d96 100644 --- a/harmony/src/modules/cert_manager/helm.rs +++ b/harmony/src/modules/cert_manager/helm.rs @@ -31,6 +31,7 @@ impl Score for CertManagerHelmScore { values_yaml: None, create_namespace: true, install_only: true, + force_conflicts: false, repository: Some(HelmRepository::new( "jetstack".to_string(), hurl!("https://charts.jetstack.io"), diff --git a/harmony/src/modules/cert_manager/operator.rs b/harmony/src/modules/cert_manager/operator.rs index d6a1040e..04c10876 100644 --- a/harmony/src/modules/cert_manager/operator.rs +++ b/harmony/src/modules/cert_manager/operator.rs @@ -53,6 +53,7 @@ impl Score for CertManagerOp values_yaml: None, create_namespace: true, install_only: true, + force_conflicts: false, repository: None, }; diff --git a/harmony/src/modules/github_runner.rs b/harmony/src/modules/github_runner.rs index 7f7db9e0..7af642bc 100644 --- a/harmony/src/modules/github_runner.rs +++ b/harmony/src/modules/github_runner.rs @@ -103,6 +103,7 @@ impl GithubRunnerScore { values_yaml: None, create_namespace: true, install_only: false, + force_conflicts: false, repository: None, // OCI chart — referenced directly, no `helm repo add` } } @@ -120,6 +121,7 @@ impl GithubRunnerScore { values_yaml: Some(self.runner_set_values()), create_namespace: true, install_only: false, + force_conflicts: false, repository: None, } } diff --git a/harmony/src/modules/helm/chart.rs b/harmony/src/modules/helm/chart.rs index 273960d1..cc2bc551 100644 --- a/harmony/src/modules/helm/chart.rs +++ b/harmony/src/modules/helm/chart.rs @@ -44,6 +44,7 @@ pub struct HelmChartScore { /// itself diffs the rendered chart against the live release and is a /// no-op when nothing changed. pub install_only: bool, + pub force_conflicts: bool, pub repository: Option, } @@ -248,7 +249,11 @@ impl Interpret for HelmChartInterpret { let mut args = if self.score.install_only { vec!["install"] } else { - vec!["upgrade", "--install"] + let mut v = vec!["upgrade", "--install"]; + if self.score.force_conflicts { + v.push("--force-conflicts"); + } + v }; args.extend(vec![ diff --git a/harmony/src/modules/lamp.rs b/harmony/src/modules/lamp.rs index 7a45e96b..470d782b 100644 --- a/harmony/src/modules/lamp.rs +++ b/harmony/src/modules/lamp.rs @@ -209,6 +209,7 @@ impl LAMPInterpret { values_overrides: Some(values_overrides), create_namespace: true, install_only: false, + force_conflicts: false, values_yaml: None, repository: None, }; diff --git a/harmony/src/modules/mod.rs b/harmony/src/modules/mod.rs index b0600667..493f9997 100644 --- a/harmony/src/modules/mod.rs +++ b/harmony/src/modules/mod.rs @@ -12,7 +12,6 @@ pub mod http; pub mod inventory; pub mod k3d; pub mod k8s; -pub mod registry_pull_secret; #[cfg(all(feature = "kvm", unix))] pub mod kvm; pub mod lamp; @@ -30,6 +29,7 @@ pub mod opnsense; pub mod podman; pub mod postgresql; pub mod prometheus; +pub mod registry_pull_secret; pub mod storage; pub mod tenant; pub mod tftp; diff --git a/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs b/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs index 4c268514..e799bd31 100644 --- a/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs +++ b/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs @@ -38,6 +38,7 @@ pub fn grafana_helm_chart_score( values_yaml: None, create_namespace: true, install_only: true, + force_conflicts: false, repository: Some(HelmRepository::new( "grafana".to_string(), hurl!("https://grafana.github.io/helm-charts"), diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/grafana_operator.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/grafana_operator.rs index ac7c9f55..4c53833a 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/grafana_operator.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/grafana_operator.rs @@ -15,6 +15,7 @@ pub fn grafana_operator_helm_chart_score(ns: String) -> HelmChartScore { values_yaml: None, create_namespace: true, install_only: true, + force_conflicts: false, repository: None, } } diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/prometheus_operator.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/prometheus_operator.rs index 413c2544..e877b9f0 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/prometheus_operator.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/prometheus_operator.rs @@ -17,6 +17,7 @@ pub fn prometheus_operator_helm_chart_score(ns: String) -> HelmChartScore { values_yaml: None, create_namespace: true, install_only: true, + force_conflicts: false, repository: None, } } diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_cluster_observability_operator.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_cluster_observability_operator.rs index bc7ad9fe..145a5c6c 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_cluster_observability_operator.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_cluster_observability_operator.rs @@ -17,6 +17,7 @@ pub fn rhob_cluster_observability_operator() -> HelmChartScore { values_yaml: None, create_namespace: true, install_only: true, + force_conflicts: false, repository: None, } } diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs b/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs index 7d6aec89..096e2d40 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs @@ -418,6 +418,7 @@ prometheusOperator: values_yaml: Some(values.to_string()), create_namespace: true, install_only: true, + force_conflicts: false, repository: None, } } diff --git a/harmony/src/modules/monitoring/ntfy/helm/ntfy_helm_chart.rs b/harmony/src/modules/monitoring/ntfy/helm/ntfy_helm_chart.rs index 57fffabb..74fa1fea 100644 --- a/harmony/src/modules/monitoring/ntfy/helm/ntfy_helm_chart.rs +++ b/harmony/src/modules/monitoring/ntfy/helm/ntfy_helm_chart.rs @@ -89,6 +89,7 @@ persistence: values_yaml: Some(values.to_string()), create_namespace: true, install_only: false, + force_conflicts: false, repository: None, } } diff --git a/harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs b/harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs index 611c500f..03db55a3 100644 --- a/harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs +++ b/harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs @@ -42,6 +42,7 @@ fullnameOverride: prometheus-{ns} values_yaml: Some(values.to_string()), create_namespace: true, install_only: true, + force_conflicts: false, repository: None, } } diff --git a/harmony/src/modules/nats/helm_chart.rs b/harmony/src/modules/nats/helm_chart.rs index 5a1f17b5..29624f76 100644 --- a/harmony/src/modules/nats/helm_chart.rs +++ b/harmony/src/modules/nats/helm_chart.rs @@ -96,6 +96,7 @@ impl NatsHelmChartScore { values_yaml: Some(self.values_yaml), create_namespace: self.create_namespace, install_only: self.install_only, + force_conflicts: false, repository: Some(HelmRepository::new( REPO_NAME.to_string(), hurl!("https://nats-io.github.io/k8s/helm/charts/"), diff --git a/harmony/src/modules/openbao/mod.rs b/harmony/src/modules/openbao/mod.rs index d0a491ac..c6062bfc 100644 --- a/harmony/src/modules/openbao/mod.rs +++ b/harmony/src/modules/openbao/mod.rs @@ -156,6 +156,7 @@ impl Score for OpenbaoScore { values_yaml: Some(self.values()), create_namespace: true, install_only: false, + force_conflicts: false, repository: Some(HelmRepository::new( "openbao".to_string(), hurl!("https://openbao.github.io/openbao-helm"), diff --git a/harmony/src/modules/postgresql/operator.rs b/harmony/src/modules/postgresql/operator.rs index 11386c34..817679cf 100644 --- a/harmony/src/modules/postgresql/operator.rs +++ b/harmony/src/modules/postgresql/operator.rs @@ -136,6 +136,7 @@ impl Score for CloudNativePg values_yaml: None, create_namespace: true, install_only: true, + force_conflicts: false, repository: None, }; diff --git a/harmony/src/modules/registry_pull_secret.rs b/harmony/src/modules/registry_pull_secret.rs index df1c7c2b..7771b012 100644 --- a/harmony/src/modules/registry_pull_secret.rs +++ b/harmony/src/modules/registry_pull_secret.rs @@ -119,8 +119,7 @@ mod tests { let entry = &json["auths"]["hub.nationtech.io"]; assert_eq!(entry["username"], "robot$pull"); assert_eq!(entry["password"], "s3cr3t"); - let expected_auth = - base64::engine::general_purpose::STANDARD.encode("robot$pull:s3cr3t"); + let expected_auth = base64::engine::general_purpose::STANDARD.encode("robot$pull:s3cr3t"); assert_eq!(entry["auth"], expected_auth); } } diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index f7d86dc9..4481691c 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -921,6 +921,7 @@ login: // re-running the chart. (Deliberate version upgrades are a separate, // explicit action.) install_only: true, + force_conflicts: false, repository: Some(HelmRepository::new( "zitadel".to_string(), hurl!("https://charts.zitadel.com"), diff --git a/harmony/src/modules/zitadel/setup.rs b/harmony/src/modules/zitadel/setup.rs index 7e33d1cf..0d6318ec 100644 --- a/harmony/src/modules/zitadel/setup.rs +++ b/harmony/src/modules/zitadel/setup.rs @@ -2254,67 +2254,65 @@ impl Interpret for ZitadelSetupInterpret { // the management API through 127.0.0.1. `_pf` must outlive every call // below. `me` is `self` with the resolved endpoint patched in. let _pf; - let me: std::borrow::Cow<'_, ZitadelSetupInterpret> = - if let Some(svc) = &self.score.port_forward_service { - let label = format!("app.kubernetes.io/name={svc}"); - // On a cold cluster the Zitadel Deployment pod is not Running yet - // (image pull + DB migration take a minute or more), so poll for - // it instead of failing on the first miss — otherwise a one-shot - // `ship` aborts here and only a re-run (pod now up) succeeds. - let pod = { - use std::time::{Duration, Instant}; - let timeout = Duration::from_secs(300); - let interval = Duration::from_secs(3); - let deadline = Instant::now() + timeout; - loop { - let pods = k8s - .list_resources::( - Some(&self.score.namespace), - Some(kube::api::ListParams::default().labels(&label)), - ) - .await - .map_err(|e| { - InterpretError::new(format!("listing '{svc}' pods: {e}")) - })?; - // The chart's init/setup Job pods share this label; only - // the serving Deployment pod is Running, so filter on phase. - let running = pods - .items - .into_iter() - .find(|p| { - p.status.as_ref().and_then(|s| s.phase.as_deref()) - == Some("Running") - }) - .and_then(|p| p.metadata.name); - if let Some(name) = running { - break name; - } - if Instant::now() >= deadline { - return Err(InterpretError::new(format!( - "no Running pod for service '{svc}' in namespace '{}' within {}s", - self.score.namespace, - timeout.as_secs(), - ))); - } - debug!("[ZitadelSetup] waiting for a Running '{svc}' pod..."); - tokio::time::sleep(interval).await; + let me: std::borrow::Cow<'_, ZitadelSetupInterpret> = if let Some(svc) = + &self.score.port_forward_service + { + let label = format!("app.kubernetes.io/name={svc}"); + // On a cold cluster the Zitadel Deployment pod is not Running yet + // (image pull + DB migration take a minute or more), so poll for + // it instead of failing on the first miss — otherwise a one-shot + // `ship` aborts here and only a re-run (pod now up) succeeds. + let pod = { + use std::time::{Duration, Instant}; + let timeout = Duration::from_secs(300); + let interval = Duration::from_secs(3); + let deadline = Instant::now() + timeout; + loop { + let pods = k8s + .list_resources::( + Some(&self.score.namespace), + Some(kube::api::ListParams::default().labels(&label)), + ) + .await + .map_err(|e| InterpretError::new(format!("listing '{svc}' pods: {e}")))?; + // The chart's init/setup Job pods share this label; only + // the serving Deployment pod is Running, so filter on phase. + let running = pods + .items + .into_iter() + .find(|p| { + p.status.as_ref().and_then(|s| s.phase.as_deref()) == Some("Running") + }) + .and_then(|p| p.metadata.name); + if let Some(name) = running { + break name; } - }; - // Zitadel serves http on 8080 in-cluster (chart default). - let handle = k8s - .port_forward(&pod, &self.score.namespace, 0, 8080) - .await - .map_err(|e| InterpretError::new(format!("port-forward to '{pod}': {e}")))?; - let endpoint = format!("http://127.0.0.1:{}", handle.port()); - info!("[ZitadelSetup] Provisioning via port-forward {endpoint} -> {svc}"); - _pf = Some(handle); - let mut score = self.score.clone(); - score.endpoint = Some(endpoint); - std::borrow::Cow::Owned(ZitadelSetupInterpret { score }) - } else { - _pf = None; - std::borrow::Cow::Borrowed(self) + if Instant::now() >= deadline { + return Err(InterpretError::new(format!( + "no Running pod for service '{svc}' in namespace '{}' within {}s", + self.score.namespace, + timeout.as_secs(), + ))); + } + debug!("[ZitadelSetup] waiting for a Running '{svc}' pod..."); + tokio::time::sleep(interval).await; + } }; + // Zitadel serves http on 8080 in-cluster (chart default). + let handle = k8s + .port_forward(&pod, &self.score.namespace, 0, 8080) + .await + .map_err(|e| InterpretError::new(format!("port-forward to '{pod}': {e}")))?; + let endpoint = format!("http://127.0.0.1:{}", handle.port()); + info!("[ZitadelSetup] Provisioning via port-forward {endpoint} -> {svc}"); + _pf = Some(handle); + let mut score = self.score.clone(); + score.endpoint = Some(endpoint); + std::borrow::Cow::Owned(ZitadelSetupInterpret { score }) + } else { + _pf = None; + std::borrow::Cow::Borrowed(self) + }; let client = me.http_client().map_err(InterpretError::new)?; diff --git a/harmony_app/src/app.rs b/harmony_app/src/app.rs index f71978fe..cc86726b 100644 --- a/harmony_app/src/app.rs +++ b/harmony_app/src/app.rs @@ -34,6 +34,14 @@ pub trait HarmonyApp: Send + Sync { /// branching on `ctx.profile()`. Called by `deploy`/`ship`. async fn scores(&self, ctx: &AppContext) -> Result>>>; + async fn scores_with_options( + &self, + ctx: &AppContext, + _options: DeployOptions, + ) -> Result>>> { + self.scores(ctx).await + } + /// Build + distribute images for `ctx` (push to a registry, or import to /// k3d). Default: nothing to build. Called by `ship`. async fn publish(&self, _ctx: &AppContext) -> Result<()> { @@ -54,6 +62,11 @@ pub struct DeployReport { pub steps: Vec, } +#[derive(Debug, Clone, Copy, Default)] +pub struct DeployOptions { + pub force_conflicts: bool, +} + #[derive(Debug, Clone, serde::Serialize)] pub struct WorkloadStatus { pub name: String, @@ -81,7 +94,16 @@ pub async fn deploy( topology: T, ctx: &AppContext, ) -> Result { - let scores = app.scores(ctx).await?; + deploy_with_options(app, topology, ctx, DeployOptions::default()).await +} + +pub async fn deploy_with_options( + app: &dyn HarmonyApp, + topology: T, + ctx: &AppContext, + options: DeployOptions, +) -> Result { + let scores = app.scores_with_options(ctx, options).await?; let to_run: Vec>> = scores.iter().map(|s| s.clone_box()).collect(); let mut maestro = Maestro::new_without_initialization(Inventory::autoload(), topology); @@ -111,9 +133,18 @@ pub async fn ship( app: &dyn HarmonyApp, topology: T, ctx: &AppContext, +) -> Result { + ship_with_options(app, topology, ctx, DeployOptions::default()).await +} + +pub async fn ship_with_options( + app: &dyn HarmonyApp, + topology: T, + ctx: &AppContext, + options: DeployOptions, ) -> Result { app.publish(ctx).await?; - deploy(app, topology, ctx).await + deploy_with_options(app, topology, ctx, options).await } /// Workload readiness in the app's namespace (operational, read-only). diff --git a/harmony_app/src/chart.rs b/harmony_app/src/chart.rs index fd344fe9..6b0300c4 100644 --- a/harmony_app/src/chart.rs +++ b/harmony_app/src/chart.rs @@ -17,8 +17,8 @@ use k8s_openapi::api::apps::v1::{ use k8s_openapi::api::core::v1::{ Container, ContainerPort, EnvFromSource, EnvVar, KeyToPath, LocalObjectReference, PersistentVolumeClaim, PersistentVolumeClaimSpec, PersistentVolumeClaimVolumeSource, PodSpec, - PodTemplateSpec, SecretEnvSource, SecretVolumeSource, Service, ServicePort, ServiceSpec, Volume, - VolumeMount, VolumeResourceRequirements, + PodTemplateSpec, SecretEnvSource, SecretVolumeSource, Service, ServicePort, ServiceSpec, + Volume, VolumeMount, VolumeResourceRequirements, }; use k8s_openapi::apimachinery::pkg::api::resource::Quantity; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; @@ -45,7 +45,7 @@ pub struct DeployConfig { pub storage_class: Option, pub volume_size: String, pub replicas: i32, - /// `true` → RollingUpdate (needs RWX so old+new pods share volumes); + /// `true` → RollingUpdate (when persistence enabled needs RWX so old+new pods share volumes); /// `false` → Recreate (safe for single-writer stores like sqlite). pub rolling: bool, /// Extra env injected into every service container by capabilities — @@ -90,9 +90,9 @@ impl DeployConfig { registry: registry.into(), project: project.into(), version, - storage_class: Some(if prod { "cephfs" } else { "local-path" }.to_string()), + storage_class: None, volume_size: "1Gi".to_string(), - replicas: if prod { 2 } else { 1 }, + replicas: 1, rolling: prod, extra_env: Vec::new(), secret_file_mounts: Vec::new(), diff --git a/harmony_app/src/compose.rs b/harmony_app/src/compose.rs index c6c0483a..2e1e9e7a 100644 --- a/harmony_app/src/compose.rs +++ b/harmony_app/src/compose.rs @@ -214,9 +214,7 @@ fn parse_build_args(args: &Option) -> Vec<(Stri None => (item.to_string(), String::new()), }) .collect(), - Some(BuildArgs::KvPair(map)) => { - map.iter().map(|(k, v)| (k.clone(), v.clone())).collect() - } + Some(BuildArgs::KvPair(map)) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), } } diff --git a/harmony_app/src/deploy.rs b/harmony_app/src/deploy.rs index 607cac2e..5bc9c5a7 100644 --- a/harmony_app/src/deploy.rs +++ b/harmony_app/src/deploy.rs @@ -7,7 +7,7 @@ use std::collections::BTreeMap; use std::path::Path; -use crate::{AppContext, AppIdentity, AppRef, Capability, HarmonyApp, Profile}; +use crate::{AppContext, AppIdentity, AppRef, Capability, DeployOptions, HarmonyApp, Profile}; use anyhow::{Result, anyhow}; use async_trait::async_trait; use harmony::score::Score; @@ -104,7 +104,12 @@ impl ComposeDeploy { } /// Derive the deploy Score for a profile (pure — the testable core). - pub fn score(&self, profile: Profile, version: &str) -> Result { + pub fn score( + &self, + profile: Profile, + version: &str, + force_conflicts: bool, + ) -> Result { let public_endpoint = match &self.expose { Some(e) => Some(PublicEndpoint::from_compose( &self.app, @@ -133,6 +138,7 @@ impl ComposeDeploy { app: self.app.clone(), deploy, app_secrets: self.app_secrets.clone(), + force_conflicts, }) } } @@ -147,8 +153,17 @@ impl HarmonyApp for ComposeDeploy { } async fn scores(&self, ctx: &AppContext) -> Result>>> { + self.scores_with_options(ctx, DeployOptions::default()) + .await + } + + async fn scores_with_options( + &self, + ctx: &AppContext, + options: DeployOptions, + ) -> Result>>> { let app_score = self - .score(ctx.profile(), ctx.version()) + .score(ctx.profile(), ctx.version(), options.force_conflicts) .map_err(|e| anyhow!(e))?; let mut scores: Vec>> = vec![Box::new(app_score)]; let app_ref = self.app_ref(ctx.profile()); @@ -201,7 +216,7 @@ mod tests { #[test] fn namespace_and_project_default_to_name() { let s = CD::from_compose("timesheet", fixture()) - .score(Profile::Local, "0.1.0") + .score(Profile::Local, "0.1.0", false) .unwrap(); assert_eq!(s.namespace, "timesheet"); assert_eq!(s.release_name, "timesheet"); @@ -211,7 +226,7 @@ mod tests { fn local_profile_renders_locally_rwo_http() { let s = CD::from_compose("ts", fixture()) .expose("frontend", "ts.local") - .score(Profile::Local, "1.0.0") + .score(Profile::Local, "1.0.0", false) .unwrap(); assert!(s.published_chart.is_none(), "local renders from compose"); assert_eq!(s.deploy.replicas, 1); @@ -229,13 +244,13 @@ mod tests { .registry("hub.x") .project("p") .expose("frontend", "ts.x") - .score(Profile::Prod, "2.0.0") + .score(Profile::Prod, "2.0.0", false) .unwrap(); let pc = s.published_chart.expect("prod installs published chart"); assert_eq!(pc.registry, "hub.x"); assert_eq!(pc.project, "p"); assert_eq!(pc.version, "2.0.0"); - assert_eq!(s.deploy.replicas, 2); + assert_eq!(s.deploy.replicas, 1); assert!(s.deploy.rolling); assert_eq!( s.public_endpoint.unwrap().cluster_issuer.as_deref(), @@ -247,7 +262,7 @@ mod tests { fn secrets_are_carried_to_the_score() { let s = CD::from_compose("ts", fixture()) .secret("DB_PASSWORD", "dev") - .score(Profile::Local, "0.1.0") + .score(Profile::Local, "0.1.0", false) .unwrap(); assert_eq!(s.app_secrets.get("DB_PASSWORD").unwrap(), "dev"); } @@ -256,7 +271,7 @@ mod tests { fn postgres_capability_wires_database_url_by_reference() { let s = CD::from_compose("ts", fixture()) .with(Postgres::managed()) - .score(Profile::Local, "0.1.0") + .score(Profile::Local, "0.1.0", false) .unwrap(); let db = s .deploy diff --git a/harmony_app/src/lib.rs b/harmony_app/src/lib.rs index 541e4685..87800026 100644 --- a/harmony_app/src/lib.rs +++ b/harmony_app/src/lib.rs @@ -25,8 +25,8 @@ pub mod publish; pub mod score; pub use app::{ - AppIdentity, DeployReport, HarmonyApp, PodLogs, StatusReport, StepOutcome, WorkloadStatus, - deploy, logs, ship, status, + AppIdentity, DeployOptions, DeployReport, HarmonyApp, PodLogs, StatusReport, StepOutcome, + WorkloadStatus, deploy, deploy_with_options, logs, ship, ship_with_options, status, }; pub use capabilities::{AppRef, Capability, Monitoring, Postgres, ZitadelAuth}; pub use chart::{DeployConfig, SecretFileMount, cluster_issuer_for, service_image}; diff --git a/harmony_app/src/score.rs b/harmony_app/src/score.rs index 9ef05a48..24f6c4ac 100644 --- a/harmony_app/src/score.rs +++ b/harmony_app/src/score.rs @@ -92,6 +92,7 @@ pub struct ComposeAppScore { /// values must never reach Score display/logs. #[serde(skip)] pub app_secrets: BTreeMap, + pub force_conflicts: bool, } impl Score for ComposeAppScore { @@ -238,7 +239,9 @@ impl ComposeAppInterpret { .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {e}")))? .ensure_namespace(&s.namespace) .await - .map_err(|e| InterpretError::new(format!("Failed to ensure namespace '{}': {e}", s.namespace)))?; + .map_err(|e| { + InterpretError::new(format!("Failed to ensure namespace '{}': {e}", s.namespace)) + })?; HelmChartScore { namespace: Some(non_blank(&s.namespace, "namespace")?), release_name: non_blank(&s.release_name, "release_name")?, @@ -248,6 +251,7 @@ impl ComposeAppInterpret { values_yaml: None, create_namespace: false, install_only: false, + force_conflicts: s.force_conflicts, repository: None, } .interpret(inventory, topology) diff --git a/harmony_app/tests/helm_render.rs b/harmony_app/tests/helm_render.rs index 459a8bb3..8ee2a07a 100644 --- a/harmony_app/tests/helm_render.rs +++ b/harmony_app/tests/helm_render.rs @@ -37,6 +37,7 @@ fn generated_chart_passes_helm_lint_and_template() { rolling: true, extra_env: vec![], secret_file_mounts: vec![], + image_pull_secrets: vec![], }; let tmp = tempfile::tempdir().unwrap(); diff --git a/harmony_cli/src/app.rs b/harmony_cli/src/app.rs index aafb2faf..eb0eec9b 100644 --- a/harmony_cli/src/app.rs +++ b/harmony_cli/src/app.rs @@ -40,9 +40,17 @@ struct AppCli { #[derive(Subcommand, Debug)] enum Verb { /// Build + publish + deploy. - Ship, + Ship { + /// Pass `--force-conflicts` to the app chart's helm upgrade. + #[arg(long)] + force_conflicts: bool, + }, /// Converge the app's Scores (no build). - Deploy, + Deploy { + /// Pass `--force-conflicts` to the app chart's helm upgrade. + #[arg(long)] + force_conflicts: bool, + }, /// Workload readiness in the app's namespace. Status, /// Recent logs from the app's pods. @@ -66,8 +74,24 @@ pub async fn app_main + 'static>(app: A) -> R let ctx = AppContext::resolve(&context, cli.tag, cli.local_config, cli.config).await?; match cli.verb { - Verb::Ship => render_deploy(harmony_app::ship(&app, ctx.topology(), &ctx).await?), - Verb::Deploy => render_deploy(harmony_app::deploy(&app, ctx.topology(), &ctx).await?), + Verb::Ship { force_conflicts } => render_deploy( + harmony_app::ship_with_options( + &app, + ctx.topology(), + &ctx, + harmony_app::DeployOptions { force_conflicts }, + ) + .await?, + ), + Verb::Deploy { force_conflicts } => render_deploy( + harmony_app::deploy_with_options( + &app, + ctx.topology(), + &ctx, + harmony_app::DeployOptions { force_conflicts }, + ) + .await?, + ), Verb::Status => render_status(harmony_app::status(&app, &ctx).await?), Verb::Logs { tail } => render_logs(harmony_app::logs(&app, &ctx, Some(tail)).await?), } diff --git a/harmony_config/src/source/prompt.rs b/harmony_config/src/source/prompt.rs index dcd6a87f..da128249 100644 --- a/harmony_config/src/source/prompt.rs +++ b/harmony_config/src/source/prompt.rs @@ -63,7 +63,9 @@ impl FieldPrompter for InquirePrompter { } FieldType::String => { let raw = Text::new(&label).prompt().map_err(|e| { - ConfigError::PromptError(format!("Text prompt for `{key}::{field}` failed: {e}")) + ConfigError::PromptError(format!( + "Text prompt for `{key}::{field}` failed: {e}" + )) })?; Ok(serde_json::Value::String(raw)) } @@ -228,7 +230,10 @@ fn schema_object<'a>( // Resolves one field's primitive `FieldType` from its JsonSchema. Returns Err // for unsupported shapes (multi-type/nullable, nested, arrays) — never a silent // fallback. -fn field_type(field_name: &str, schema: &SchemaObject) -> Result { +fn field_type( + field_name: &str, + schema: &SchemaObject, +) -> Result { let instance_type = schema.instance_type.as_ref().ok_or_else(|| { ConfigError::PromptError(format!( "Secret struct `{}` field `{field_name}` is not a flat primitive \ -- 2.39.5 From 2880850fd13dd10427b1b432a8ebe7cd0a7fa68f Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sun, 5 Jul 2026 01:37:07 -0400 Subject: [PATCH 13/15] fix(app): load OpenBao auth from deploy context --- harmony_app/src/context.rs | 80 +++++++++++++++++++++++--------------- harmony_config/src/lib.rs | 19 ++++----- 2 files changed, 59 insertions(+), 40 deletions(-) diff --git a/harmony_app/src/context.rs b/harmony_app/src/context.rs index 651ff733..ed051de5 100644 --- a/harmony_app/src/context.rs +++ b/harmony_app/src/context.rs @@ -10,10 +10,11 @@ use std::ffi::OsString; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Command; +use std::sync::Arc; use anyhow::{Context, Result, bail}; use harmony::topology::{K8sAnywhereConfig, K8sAnywhereTopology}; -use harmony_config::{Config, ConfigClient}; +use harmony_config::{Config, ConfigClient, ConfigSource, LocalFileSource, PromptSource}; use harmony_k8s::K8sClient; use log::{debug, info}; use schemars::JsonSchema; @@ -42,6 +43,8 @@ struct ContextDef { openbao_namespace: Option, openbao_url: Option, zitadel_url: Option, + openbao_role: Option, + zitadel_audience: Option, } /// The cluster K8sAnywhereTopology autoprovisions (its K3DInstallationScore @@ -72,6 +75,7 @@ pub struct AppContext { kubeconfig: Option, /// The kubeconfig file must outlive every client/topology built from it. _kubeconfig_guard: Option, + config_client: Arc, } impl AppContext { @@ -108,6 +112,11 @@ impl AppContext { def.profile ); debug!("Context '{name}' definition: {def:?}"); + let config_sources = build_config_sources(def, local_config_dir.clone()) + .await + .with_context(|| format!("building config sources for context '{name}'"))?; + harmony_config::init(config_sources.clone()).await; + let config_client = Arc::new(ConfigClient::new(config_sources)); let guard = match (def.autoprovision, &def.k3d, &def.openbao_namespace) { (true, None, None) => { info!("Cluster access: autoprovision local k3d ('{AUTOPROVISION_CLUSTER}')"); @@ -119,21 +128,12 @@ impl AppContext { } (false, None, Some(ns)) => { info!("Cluster access: kubeconfig from OpenBao (namespace '{ns}')"); - // The kubeconfig is an OpenBao secret, so read it from an - // OpenBao-only client built with the context's URLs (each - // falling through to env when unset). - let access: ClusterAccess = - openbao_client(ns, def.openbao_url.clone(), def.zitadel_url.clone()) - .await - .with_context(|| format!("reaching OpenBao for context '{name}'"))? - .get() - .await - .with_context(|| { - format!( - "loading cluster kubeconfig from OpenBao ({ns}/ClusterAccess) \ - — are the Zitadel key + OpenBao URL/role set?" - ) - })?; + let access: ClusterAccess = config_client.get().await.with_context(|| { + format!( + "loading cluster kubeconfig from OpenBao ({ns}/ClusterAccess) \ + — is HARMONY_ZITADEL_KEY_JSON set for this context?" + ) + })?; Some(write_kubeconfig(access.kubeconfig.as_bytes())?) } _ => bail!( @@ -157,6 +157,7 @@ impl AppContext { autoprovision: def.autoprovision, kubeconfig: guard.as_ref().map(|g| g.path().to_path_buf()), _kubeconfig_guard: guard, + config_client, }) } @@ -172,6 +173,9 @@ impl AppContext { pub fn local_config_dir(&self) -> Option<&Path> { self.local_config_dir.as_deref() } + pub fn config_client(&self) -> &ConfigClient { + &self.config_client + } pub fn k3d_cluster(&self) -> Option<&str> { self.k3d_cluster.as_deref() } @@ -231,22 +235,36 @@ fn find_contexts_file() -> Result { } } -/// An OpenBao-only `ConfigClient` for `namespace`, using the given URLs (each -/// falling through to env). Errors when no OpenBao endpoint is resolvable. -async fn openbao_client( - namespace: &str, - openbao_url: Option, - zitadel_url: Option, -) -> Result { - let source = harmony_config::openbao_source(namespace, openbao_url, zitadel_url) +async fn build_config_sources( + def: &ContextDef, + local_config_dir: Option, +) -> Result>> { + let mut sources: Vec> = Vec::new(); + + if let Some(namespace) = &def.openbao_namespace { + let source = harmony_config::openbao_source( + namespace, + def.openbao_url.clone(), + def.zitadel_url.clone(), + def.zitadel_audience.clone(), + Some( + def.openbao_role + .clone() + .unwrap_or_else(|| format!("{namespace}-cd")), + ), + ) .await - .with_context(|| { - format!( - "no OpenBao source for '{namespace}' — set `openbao_url`/OPENBAO_URL \ - and the Zitadel auth env (HARMONY_ZITADEL_KEY_*, AUDIENCE, OPENBAO_JWT_ROLE)" - ) - })?; - Ok(ConfigClient::new(vec![source])) + .with_context(|| format!("reaching OpenBao for namespace '{namespace}'"))?; + sources.push(source); + } else { + let dir = local_config_dir + .or_else(LocalFileSource::default_path) + .context("local contexts need a config directory")?; + sources.push(Arc::new(LocalFileSource::new(dir))); + } + + sources.push(Arc::new(PromptSource::new())); + Ok(sources) } fn write_kubeconfig(contents: &[u8]) -> Result { diff --git a/harmony_config/src/lib.rs b/harmony_config/src/lib.rs index 26d4d9f7..5a9ca15d 100644 --- a/harmony_config/src/lib.rs +++ b/harmony_config/src/lib.rs @@ -229,19 +229,20 @@ impl ConfigClient { /// Build an OpenBao-backed `StoreSource` purely from env — the default chain. async fn openbao_from_env(namespace: &str) -> Option> { - openbao_source(namespace, None, None).await + openbao_source(namespace, None, None, None, None).await } -/// Build an OpenBao-backed `StoreSource`. `openbao_url` and `zitadel_sso_url` -/// override their env vars (`OPENBAO_URL`/`VAULT_ADDR`, `HARMONY_SSO_URL`) when -/// `Some`, else fall through to the env — so a caller that knows the endpoints -/// (e.g. a deploy context) supplies them without mutating process env. Returns -/// `None` (with a `warn!`) when no OpenBao URL is resolvable or the connection -/// fails, so a missing OpenBao degrades the chain instead of breaking startup. +/// Build an OpenBao-backed `StoreSource`. Explicit arguments override their env +/// vars, so deploy contexts can provide OpenBao/Zitadel coordinates without +/// mutating process env. Returns `None` (with a `warn!`) when no OpenBao URL is +/// resolvable or the connection fails, so a missing OpenBao degrades the chain +/// instead of breaking startup. pub async fn openbao_source( namespace: &str, openbao_url: Option, zitadel_sso_url: Option, + zitadel_audience: Option, + openbao_jwt_role: Option, ) -> Option> { let env = |k: &str| std::env::var(k).ok(); @@ -264,7 +265,7 @@ pub async fn openbao_source( match ( key_path.is_some() || key_json.is_some(), sso_url.clone(), - env("HARMONY_ZITADEL_AUDIENCE"), + zitadel_audience.or_else(|| env("HARMONY_ZITADEL_AUDIENCE")), ) { (true, Some(oidc_issuer_url), Some(audience)) => { Some(harmony_secret::ZitadelJwtBearerConfig { @@ -288,7 +289,7 @@ pub async fn openbao_source( password: env("OPENBAO_PASSWORD"), zitadel_sso_url: sso_url, zitadel_client_id: env("HARMONY_SSO_CLIENT_ID"), - jwt_role: env("OPENBAO_JWT_ROLE"), + jwt_role: openbao_jwt_role.or_else(|| env("OPENBAO_JWT_ROLE")), jwt_auth_mount: env("OPENBAO_JWT_AUTH_MOUNT"), zitadel_jwt_bearer, }) -- 2.39.5 From 7995c982d22886ecfeb04ee6f813bbc00d9bc7e4 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sun, 5 Jul 2026 02:01:32 -0400 Subject: [PATCH 14/15] fix(zitadel): persist install secrets through harmony_config --- Cargo.lock | 1 + harmony/Cargo.toml | 1 + harmony/src/modules/zitadel/mod.rs | 26 ++++++++++++++------------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 74ba26cf..4d57dd76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4011,6 +4011,7 @@ dependencies = [ "futures-util", "harmony-k8s", "harmony-reconciler-contracts", + "harmony_config", "harmony_execution", "harmony_inventory_agent", "harmony_macros", diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index 6c6a0e9e..7ad95cbb 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -43,6 +43,7 @@ opnsense-api = { path = "../opnsense-api" } opnsense-config = { path = "../opnsense-config" } opnsense-config-xml = { path = "../opnsense-config-xml" } harmony_macros = { path = "../harmony_macros" } +harmony_config = { path = "../harmony_config" } harmony_types = { path = "../harmony_types" } harmony_execution = { path = "../harmony_execution" } harmony-k8s = { path = "../harmony-k8s" } diff --git a/harmony/src/modules/zitadel/mod.rs b/harmony/src/modules/zitadel/mod.rs index 4481691c..a3522f76 100644 --- a/harmony/src/modules/zitadel/mod.rs +++ b/harmony/src/modules/zitadel/mod.rs @@ -22,8 +22,8 @@ use std::collections::BTreeMap; use std::str::FromStr; use async_trait::async_trait; +use harmony_config::Config; use harmony_macros::hurl; -use harmony_secret::{Secret, SecretManager}; use harmony_types::id::Id; use harmony_types::storage::StorageSize; use log::{debug, error, info, trace, warn}; @@ -356,10 +356,10 @@ impl Interpret for Zitade // password every time (Zitadel's chart only honors FirstInstance.* // on the very first install; on re-runs the live password is // whatever's already in the DB, but our printed banner used to - // emit a fresh random — misleading the operator). harmony_secret + // emit a fresh random — misleading the operator). harmony_config // namespaces by install context (config-resolved), so two // installs in the same context share credentials. - let admin = match SecretManager::get::().await { + let admin = match harmony_config::get::().await { Ok(a) => a, Err(e) => { debug!("[Zitadel] No persisted admin credentials yet ({e}); generating"); @@ -367,7 +367,7 @@ impl Interpret for Zitade username: "admin".to_string(), password: generate_secure_password(16), }; - SecretManager::set(&a).await.map_err(|err| { + harmony_config::set(&a).await.map_err(|err| { InterpretError::new(format!("Failed to persist Zitadel admin password: {err}")) })?; a @@ -407,9 +407,9 @@ impl Interpret for Zitade // Zitadel DB undecryptable. Resolve in priority order: // 1. the in-cluster Secret, if present (authoritative — it's what the // running instance encrypted its data with), - // 2. the value persisted in harmony_secret (survives a namespace reset), + // 2. the value persisted in harmony_config (survives a namespace reset), // 3. a freshly generated one. - // Then mirror the resolved value into harmony_secret so a deleted/ + // Then mirror the resolved value into harmony_config so a deleted/ // recreated namespace reuses it rather than minting a new (broken) key. let existing_masterkey = k8s_client .get_resource::(MASTERKEY_SECRET_NAME, Some(&self.namespace)) @@ -421,7 +421,7 @@ impl Interpret for Zitade let masterkey = match existing_masterkey { Some(k) => k, - None => match SecretManager::get::().await { + None => match harmony_config::get::().await { Ok(m) => m.masterkey, Err(e) => { debug!("[Zitadel] No persisted masterkey yet ({e}); generating"); @@ -434,8 +434,8 @@ impl Interpret for Zitade }, }; - if SecretManager::get::().await.is_err() { - SecretManager::set(&ZitadelMasterkey { + if harmony_config::get::().await.is_err() { + harmony_config::set(&ZitadelMasterkey { masterkey: masterkey.clone(), }) .await @@ -979,16 +979,18 @@ login: /// Persisted admin credentials for the Zitadel instance — written on /// first install, reused on subsequent runs so the printed banner /// matches what's actually live in the DB. Stored under whichever -/// `harmony_secret` backend is configured (LocalFile by default). -#[derive(Secret, Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] +/// `harmony_config` backend is configured (LocalFile by default). +#[derive(Config, Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] struct ZitadelAdmin { username: String, + #[config(secret)] password: String, } /// Persisted Zitadel masterkey (32 ASCII alphanumeric bytes). Stored so it /// survives a namespace/Secret reset — a new key would orphan the existing DB. -#[derive(Secret, Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] +#[derive(Config, Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)] struct ZitadelMasterkey { + #[config(secret)] masterkey: String, } -- 2.39.5 From ae8d144816a109a5cfcae1763b4202425ae66086 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sun, 5 Jul 2026 07:56:29 -0400 Subject: [PATCH 15/15] docs(github-runner): note OKD DinD follow-up --- harmony/src/modules/github_runner.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/harmony/src/modules/github_runner.rs b/harmony/src/modules/github_runner.rs index 7af642bc..a25a6792 100644 --- a/harmony/src/modules/github_runner.rs +++ b/harmony/src/modules/github_runner.rs @@ -30,6 +30,11 @@ //! On OpenShift/OKD, `containerMode: dind` needs a privileged container — grant //! the runner ServiceAccount the `privileged` SCC. The runner needs **no** //! cluster RBAC: jobs reach the cluster through their own brokered kubeconfig. +//! +//! TODO(okd): encode the OKD-safe runner pod template instead of patching ARC +//! post-install. The working deployment needed the runner container to execute +//! `/home/runner/run.sh` as UID 1001/GID 123, the dind sidecar privileged as +//! root, and `dockerd --storage-driver=vfs`; nested overlay fails on OKD/CRI-O. use std::str::FromStr; -- 2.39.5