Cargo.lock

@@ -4011,6 +4011,7 @@ dependencies = [
  "futures-util",
  "harmony-k8s",
  "harmony-reconciler-contracts",
+ "harmony_config",
  "harmony_execution",
  "harmony_inventory_agent",
  "harmony_macros",

examples/nats-supercluster/src/main.rs

@@ -469,6 +469,7 @@ natsBox:
         values_yaml,
         create_namespace: true,
         install_only: false,
+        force_conflicts: false,
         repository: Some(HelmRepository::new(
             "nats".to_string(),
             hurl!("https://nats-io.github.io/k8s/helm/charts/"),

examples/nats/src/main.rs

@@ -103,6 +103,7 @@ natsBox:
         values_yaml,
         create_namespace: true,
         install_only: false,
+        force_conflicts: false,
         repository: Some(HelmRepository::new(
             "nats".to_string(),
             hurl!("https://nats-io.github.io/k8s/helm/charts/"),

fleet/harmony-fleet-deploy/src/operator/score.rs

@@ -273,6 +273,7 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
                 values_yaml: None,
                 create_namespace: true,
                 install_only: false,
+                force_conflicts: false,
                 repository: None,
             }
             .interpret(inventory, topology)
@@ -310,6 +311,7 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
                 values_yaml: None,
                 create_namespace: true,
                 install_only: false,
+                force_conflicts: false,
                 repository: None,
             }
             .interpret(inventory, topology)

harmony-k8s/src/resources.rs

@@ -13,7 +13,7 @@ use kube::{
     runtime::conditions,
     runtime::wait::await_condition,
 };
-use log::debug;
+use log::{debug, info};
 use serde::de::DeserializeOwned;
 use serde_json::Value;
 use std::time::Duration;
@@ -373,6 +373,27 @@ impl K8sClient {
             .await
     }
 
+    /// Ensure the namespace exists, **without ever patching it**. A GET (which a
+    /// namespace-scoped deployer is allowed to do) decides: if the namespace is
+    /// already present — e.g. provisioned by platform onboarding — this is a
+    /// no-op; only a genuinely missing namespace is created. This keeps tenant
+    /// deployers from needing cluster-scoped permission to patch the
+    /// (cluster-scoped) Namespace object, which a namespaced RBAC role can't do.
+    ///
+    /// Use this instead of applying a `Namespace` resource or passing
+    /// `helm --create-namespace` when the target namespace may be pre-created
+    /// and the deployer is namespace-scoped.
+    pub async fn ensure_namespace(&self, name: &str) -> Result<(), Error> {
+        if self.namespace_exists(name).await? {
+            info!("Namespace '{name}' already exists");
+            return Ok(());
+        }
+        info!("Creating namespace '{name}'");
+        self.create_namespace(name).await?;
+        self.wait_for_namespace(name, Some(Duration::from_secs(30)))
+            .await
+    }
+
     pub async fn wait_for_namespace(
         &self,
         name: &str,

harmony/Cargo.toml

@@ -43,6 +43,7 @@ opnsense-api = { path = "../opnsense-api" }
 opnsense-config = { path = "../opnsense-config" }
 opnsense-config-xml = { path = "../opnsense-config-xml" }
 harmony_macros = { path = "../harmony_macros" }
+harmony_config = { path = "../harmony_config" }
 harmony_types = { path = "../harmony_types" }
 harmony_execution = { path = "../harmony_execution" }
 harmony-k8s = { path = "../harmony-k8s" }

harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs

@@ -390,6 +390,14 @@ impl Serialize for K8sAnywhereTopology {
     }
 }
 
+/// True when a Kubernetes API error is an RBAC denial (403 Forbidden). Used to
+/// skip cluster-scope checks a namespace-scoped deployer isn't allowed to run,
+/// rather than failing the whole deploy.
+fn is_forbidden<E: std::fmt::Display>(e: &E) -> bool {
+    let msg = e.to_string().to_lowercase();
+    msg.contains("forbidden") || msg.contains("403")
+}
+
 #[async_trait]
 impl CertificateManagement for K8sAnywhereTopology {
     async fn install(&self) -> Result<Outcome, ExecutorError> {
@@ -408,13 +416,29 @@ impl CertificateManagement for K8sAnywhereTopology {
 
     async fn ensure_certificate_management_ready(&self) -> Result<Outcome, ExecutorError> {
         let k8s_client = self.k8s_client().await.unwrap();
-        let pods = k8s_client
+        let pods = match k8s_client
             .list_all_resources_with_labels::<Pod>(
                 "app.kubernetes.io/component=controller,\
          app.kubernetes.io/name=cert-manager",
             )
             .await
-            .map_err(|e| ExecutorError::UnexpectedError(format!("{}", e)))?;
+        {
+            Ok(pods) => pods,
+            // A namespace-scoped deployer (e.g. a tenant CI service account)
+            // cannot list pods cluster-wide. cert-manager is a cluster-wide
+            // platform component managed out-of-band, so don't fail the deploy —
+            // skip the readiness check and assume it is present.
+            Err(e) if is_forbidden(&e) => {
+                info!(
+                    "Skipped cert-manager setup due to restricted (namespace-scoped) \
+                     access — assuming cert-manager is managed cluster-wide by the platform"
+                );
+                return Ok(Outcome::noop(
+                    "cert-manager readiness skipped (restricted access)".to_string(),
+                ));
+            }
+            Err(e) => return Err(ExecutorError::UnexpectedError(format!("{}", e))),
+        };
 
         if pods.is_empty() {
             info!("cert-manager not installed (no controller pods found)");

harmony/src/modules/application/features/helm_argocd_score.rs

@@ -1087,6 +1087,7 @@ commitServer:
         values_yaml: Some(values.to_string()),
         create_namespace: true,
         install_only: false,
+        force_conflicts: false,
         repository: Some(HelmRepository::new(
             "argo".to_string(),
             hurl!("https://argoproj.github.io/argo-helm"),

harmony/src/modules/cert_manager/helm.rs

@@ -31,6 +31,7 @@ impl<T: Topology + HelmCommand> Score<T> for CertManagerHelmScore {
             values_yaml: None,
             create_namespace: true,
             install_only: true,
+            force_conflicts: false,
             repository: Some(HelmRepository::new(
                 "jetstack".to_string(),
                 hurl!("https://charts.jetstack.io"),

harmony/src/modules/cert_manager/operator.rs

@@ -53,6 +53,7 @@ impl<T: Topology + K8sclient + HelmCommand + 'static> Score<T> for CertManagerOp
             values_yaml: None,
             create_namespace: true,
             install_only: true,
+            force_conflicts: false,
             repository: None,
         };
 

harmony/src/modules/github_runner.rs

@@ -0,0 +1,219 @@
+//! Install GitHub self-hosted runners on Kubernetes via **Actions Runner
+//! Controller (ARC)**, built on [`HelmChartScore`].
+//!
+//! ARC is two Helm releases:
+//!   1. the **controller** (`gha-runner-scale-set-controller`, cluster-scoped) —
+//!      install once per cluster;
+//!   2. one or more **runner scale sets** (`gha-runner-scale-set`) — each
+//!      registers a set of ephemeral runners under a name that becomes the
+//!      workflow `runs-on:` label.
+//!
+//! [`GithubRunnerScore`] bundles the config and yields both as [`HelmChartScore`]s
+//! (controller first). The GitHub credential is referenced by the name of a
+//! pre-created Kubernetes Secret (holding `github_token`) — it never appears in
+//! code or values, so no secret is baked into the deploy.
+//!
+//! ```ignore
+//! let runner = GithubRunnerScore {
+//!     github_config_url: "https://github.com/Devsights".into(), // an ORG → all its repos
+//!     github_config_secret: "devsights-github".into(),          // pre-created Secret
+//!     runner_scale_set_name: "okd-cb1".into(),                  // = runs-on label
+//!     ..GithubRunnerScore::defaults()
+//! };
+//! // add both to your scores(), controller first:
+//! vec![Box::new(runner.controller()), Box::new(runner.runner_set())]
+//! ```
+//!
+//! GitHub-App auth (one org-scoped App, installed once per client org) is a
+//! platform/operations concern documented separately, not in this repo.
+//!
+//! On OpenShift/OKD, `containerMode: dind` needs a privileged container — grant
+//! the runner ServiceAccount the `privileged` SCC. The runner needs **no**
+//! cluster RBAC: jobs reach the cluster through their own brokered kubeconfig.
+//!
+//! TODO(okd): encode the OKD-safe runner pod template instead of patching ARC
+//! post-install. The working deployment needed the runner container to execute
+//! `/home/runner/run.sh` as UID 1001/GID 123, the dind sidecar privileged as
+//! root, and `dockerd --storage-driver=vfs`; nested overlay fails on OKD/CRI-O.
+
+use std::str::FromStr;
+
+use non_blank_string_rs::NonBlankString;
+
+use crate::modules::helm::chart::HelmChartScore;
+
+const CONTROLLER_CHART: &str =
+    "oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller";
+const RUNNER_SET_CHART: &str =
+    "oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set";
+/// Pin a known-good ARC chart version (controller + scale set share the line).
+const DEFAULT_CHART_VERSION: &str = "0.9.3";
+const DEFAULT_RUNNER_IMAGE: &str = "ghcr.io/actions/actions-runner:latest";
+
+/// One GitHub self-hosted runner deployment (ARC controller + a runner set).
+pub struct GithubRunnerScore {
+    /// URL the runners register against. Prefer an **org** URL
+    /// (`https://github.com/<org>`) so one scale set serves all of that org's
+    /// repos; a single repo URL also works for a repo-scoped runner.
+    pub github_config_url: String,
+    /// Name of a **pre-created** Kubernetes Secret in `runners_namespace` holding
+    /// the `github_token` key (a fine-grained PAT or GitHub App credentials).
+    /// Kept out of code/values — create it separately (e.g. from OpenBao).
+    pub github_config_secret: String,
+    /// Scale-set name — this becomes the workflow `runs-on:` label (e.g. `okd-cb1`).
+    pub runner_scale_set_name: String,
+    /// Namespace for the runner pods (default `arc-runners`).
+    pub runners_namespace: String,
+    /// Namespace for the controller (default `arc-systems`).
+    pub controller_namespace: String,
+    pub min_runners: u32,
+    pub max_runners: u32,
+    /// CPU/memory limits applied to BOTH the runner and the dind container
+    /// (the build runs in dind, so it needs the headroom too).
+    pub cpu_limit: String,
+    pub memory_limit: String,
+    /// Runner container image.
+    pub runner_image: String,
+    /// ARC chart version (controller + scale set).
+    pub chart_version: String,
+}
+
+impl GithubRunnerScore {
+    /// Sensible defaults; override the four required fields (`github_config_url`,
+    /// `github_config_secret`, `runner_scale_set_name`, and resources as needed).
+    pub fn defaults() -> Self {
+        Self {
+            github_config_url: String::new(),
+            github_config_secret: String::new(),
+            runner_scale_set_name: String::new(),
+            runners_namespace: "arc-runners".to_string(),
+            controller_namespace: "arc-systems".to_string(),
+            min_runners: 1,
+            max_runners: 2,
+            cpu_limit: "8".to_string(),
+            memory_limit: "16Gi".to_string(),
+            runner_image: DEFAULT_RUNNER_IMAGE.to_string(),
+            chart_version: DEFAULT_CHART_VERSION.to_string(),
+        }
+    }
+
+    /// The cluster-scoped ARC controller. Install once per cluster (idempotent).
+    pub fn controller(&self) -> HelmChartScore {
+        HelmChartScore {
+            namespace: Some(nbs(&self.controller_namespace)),
+            release_name: nbs("github-runner-arc"),
+            chart_name: nbs(CONTROLLER_CHART),
+            chart_version: Some(nbs(&self.chart_version)),
+            values_overrides: None,
+            values_yaml: None,
+            create_namespace: true,
+            install_only: false,
+            force_conflicts: false,
+            repository: None, // OCI chart — referenced directly, no `helm repo add`
+        }
+    }
+
+    /// The runner scale set — registers ephemeral runners labelled
+    /// `runner_scale_set_name`, with dind (so `docker build` works) and the
+    /// configured resource limits.
+    pub fn runner_set(&self) -> HelmChartScore {
+        HelmChartScore {
+            namespace: Some(nbs(&self.runners_namespace)),
+            release_name: nbs(format!("github-runner-{}", &self.runner_scale_set_name).as_str()),
+            chart_name: nbs(RUNNER_SET_CHART),
+            chart_version: Some(nbs(&self.chart_version)),
+            values_overrides: None,
+            values_yaml: Some(self.runner_set_values()),
+            create_namespace: true,
+            install_only: false,
+            force_conflicts: false,
+            repository: None,
+        }
+    }
+
+    fn runner_set_values(&self) -> String {
+        let Self {
+            github_config_url,
+            github_config_secret,
+            runner_scale_set_name,
+            min_runners,
+            max_runners,
+            cpu_limit,
+            memory_limit,
+            runner_image,
+            ..
+        } = self;
+        // dind is where `docker build` actually runs, so it gets the same limits.
+        format!(
+            "githubConfigUrl: {github_config_url}\n\
+             githubConfigSecret: {github_config_secret}\n\
+             runnerScaleSetName: {runner_scale_set_name}\n\
+             minRunners: {min_runners}\n\
+             maxRunners: {max_runners}\n\
+             containerMode:\n\
+            \x20 type: dind\n\
+             template:\n\
+            \x20 spec:\n\
+            \x20   containers:\n\
+            \x20     - name: runner\n\
+            \x20       image: {runner_image}\n\
+            \x20       resources:\n\
+            \x20         limits:\n\
+            \x20           cpu: \"{cpu_limit}\"\n\
+            \x20           memory: \"{memory_limit}\"\n\
+            \x20     - name: dind\n\
+            \x20       resources:\n\
+            \x20         limits:\n\
+            \x20           cpu: \"{cpu_limit}\"\n\
+            \x20           memory: \"{memory_limit}\"\n"
+        )
+    }
+}
+
+/// `NonBlankString` from a known-non-blank `&str` (panics on blank, like the
+/// rest of the helm modules — these are constants or required config).
+fn nbs(s: &str) -> NonBlankString {
+    NonBlankString::from_str(s).expect("non-blank string")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn sample() -> GithubRunnerScore {
+        GithubRunnerScore {
+            github_config_url: "https://github.com/Devsights/folk-timesheets".to_string(),
+            github_config_secret: "okd-cb1-github".to_string(),
+            runner_scale_set_name: "okd-cb1".to_string(),
+            cpu_limit: "46".to_string(),
+            memory_limit: "32Gi".to_string(),
+            ..GithubRunnerScore::defaults()
+        }
+    }
+
+    #[test]
+    fn controller_is_an_oci_chart_with_no_repo() {
+        let c = sample().controller();
+        assert_eq!(c.chart_name.to_string(), CONTROLLER_CHART);
+        assert!(c.repository.is_none());
+        assert_eq!(c.namespace.unwrap().to_string(), "arc-systems");
+    }
+
+    #[test]
+    fn runner_set_values_carry_name_dind_and_limits() {
+        let v = sample().runner_set_values();
+        assert!(v.contains("runnerScaleSetName: okd-cb1"));
+        assert!(v.contains("type: dind"));
+        assert!(v.contains("cpu: \"46\""));
+        assert!(v.contains("memory: \"32Gi\""));
+        assert!(v.contains("githubConfigSecret: okd-cb1-github"));
+    }
+
+    #[test]
+    fn runner_set_values_are_valid_yaml() {
+        let v = sample().runner_set_values();
+        let parsed: serde_yaml::Value = serde_yaml::from_str(&v).expect("valid yaml");
+        assert_eq!(parsed["runnerScaleSetName"], "okd-cb1");
+        assert_eq!(parsed["containerMode"]["type"], "dind");
+    }
+}

harmony/src/modules/helm/chart.rs

@@ -44,6 +44,7 @@ pub struct HelmChartScore {
     /// itself diffs the rendered chart against the live release and is a
     /// no-op when nothing changed.
     pub install_only: bool,
+    pub force_conflicts: bool,
     pub repository: Option<HelmRepository>,
 }
 
@@ -248,7 +249,11 @@ impl<T: Topology + HelmCommand> Interpret<T> for HelmChartInterpret {
         let mut args = if self.score.install_only {
             vec!["install"]
         } else {
-            vec!["upgrade", "--install"]
+            let mut v = vec!["upgrade", "--install"];
+            if self.score.force_conflicts {
+                v.push("--force-conflicts");
+            }
+            v
         };
 
         args.extend(vec![

harmony/src/modules/lamp.rs

@@ -209,6 +209,7 @@ impl LAMPInterpret {
             values_overrides: Some(values_overrides),
             create_namespace: true,
             install_only: false,
+            force_conflicts: false,
             values_yaml: None,
             repository: None,
         };

harmony/src/modules/mod.rs

@@ -6,6 +6,7 @@ pub mod dhcp;
 pub mod dns;
 pub mod dummy;
 pub mod fleet;
+pub mod github_runner;
 pub mod helm;
 pub mod http;
 pub mod inventory;
@@ -28,6 +29,7 @@ pub mod opnsense;
 pub mod podman;
 pub mod postgresql;
 pub mod prometheus;
+pub mod registry_pull_secret;
 pub mod storage;
 pub mod tenant;
 pub mod tftp;

harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs

@@ -38,6 +38,7 @@ pub fn grafana_helm_chart_score(
         values_yaml: None,
         create_namespace: true,
         install_only: true,
+        force_conflicts: false,
         repository: Some(HelmRepository::new(
             "grafana".to_string(),
             hurl!("https://grafana.github.io/helm-charts"),

harmony/src/modules/monitoring/kube_prometheus/crd/grafana_operator.rs

@@ -15,6 +15,7 @@ pub fn grafana_operator_helm_chart_score(ns: String) -> HelmChartScore {
         values_yaml: None,
         create_namespace: true,
         install_only: true,
+        force_conflicts: false,
         repository: None,
     }
 }

harmony/src/modules/monitoring/kube_prometheus/crd/prometheus_operator.rs

@@ -17,6 +17,7 @@ pub fn prometheus_operator_helm_chart_score(ns: String) -> HelmChartScore {
         values_yaml: None,
         create_namespace: true,
         install_only: true,
+        force_conflicts: false,
         repository: None,
     }
 }

harmony/src/modules/monitoring/kube_prometheus/crd/rhob_cluster_observability_operator.rs

@@ -17,6 +17,7 @@ pub fn rhob_cluster_observability_operator() -> HelmChartScore {
         values_yaml: None,
         create_namespace: true,
         install_only: true,
+        force_conflicts: false,
         repository: None,
     }
 }

harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs

@@ -418,6 +418,7 @@ prometheusOperator:
         values_yaml: Some(values.to_string()),
         create_namespace: true,
         install_only: true,
+        force_conflicts: false,
         repository: None,
     }
 }

harmony/src/modules/monitoring/ntfy/helm/ntfy_helm_chart.rs

@@ -89,6 +89,7 @@ persistence:
         values_yaml: Some(values.to_string()),
         create_namespace: true,
         install_only: false,
+        force_conflicts: false,
         repository: None,
     }
 }

harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs

@@ -42,6 +42,7 @@ fullnameOverride: prometheus-{ns}
         values_yaml: Some(values.to_string()),
         create_namespace: true,
         install_only: true,
+        force_conflicts: false,
         repository: None,
     }
 }

harmony/src/modules/nats/helm_chart.rs

@@ -96,6 +96,7 @@ impl NatsHelmChartScore {
             values_yaml: Some(self.values_yaml),
             create_namespace: self.create_namespace,
             install_only: self.install_only,
+            force_conflicts: false,
             repository: Some(HelmRepository::new(
                 REPO_NAME.to_string(),
                 hurl!("https://nats-io.github.io/k8s/helm/charts/"),

harmony/src/modules/openbao/mod.rs

@@ -156,6 +156,7 @@ impl<T: Topology + K8sclient + HelmCommand> Score<T> for OpenbaoScore {
             values_yaml: Some(self.values()),
             create_namespace: true,
             install_only: false,
+            force_conflicts: false,
             repository: Some(HelmRepository::new(
                 "openbao".to_string(),
                 hurl!("https://openbao.github.io/openbao-helm"),

harmony/src/modules/postgresql/operator.rs

@@ -136,6 +136,7 @@ impl<T: Topology + K8sclient + HelmCommand + 'static> Score<T> for CloudNativePg
             values_yaml: None,
             create_namespace: true,
             install_only: true,
+            force_conflicts: false,
             repository: None,
         };
 

harmony/src/modules/postgresql/score_k8s.rs

@@ -22,6 +22,13 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta;
 use log::{debug, info, warn};
 use serde::Serialize;
 
+/// True when a Kubernetes API error is an RBAC denial (403 Forbidden) — lets a
+/// namespace-scoped deployer skip cluster-scope checks it isn't allowed to run.
+fn is_forbidden<E: std::fmt::Display>(e: &E) -> bool {
+    let msg = e.to_string().to_lowercase();
+    msg.contains("forbidden") || msg.contains("403")
+}
+
 /// Deploys an opinionated, highly available PostgreSQL cluster managed by CNPG.
 ///
 /// This score automatically ensures the CloudNativePG (CNPG) operator is installed
@@ -82,42 +89,15 @@ impl K8sPostgreSQLInterpret {
             .await
             .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {}", e)))?;
 
-        let namespace_name = &self.config.namespace;
-
-        if k8s_client
-            .namespace_exists(namespace_name)
+        k8s_client
+            .ensure_namespace(&self.config.namespace)
             .await
             .map_err(|e| {
                 InterpretError::new(format!(
-                    "Failed to check namespace '{}': {}",
-                    namespace_name, e
+                    "Failed to ensure namespace '{}': {}",
+                    self.config.namespace, e
                 ))
-            })?
-        {
-            info!("Namespace '{}' already exists", namespace_name);
-            return Ok(());
-        }
-
-        info!("Creating namespace '{}'", namespace_name);
-        k8s_client
-            .create_namespace(namespace_name)
-            .await
-            .map_err(|e| {
-                InterpretError::new(format!(
-                    "Failed to create namespace '{}': {}",
-                    namespace_name, e
-                ))
-            })?;
-
-        k8s_client
-            .wait_for_namespace(namespace_name, Some(std::time::Duration::from_secs(30)))
-            .await
-            .map_err(|e| {
-                InterpretError::new(format!("Namespace '{}' not ready: {}", namespace_name, e))
-            })?;
-
-        info!("Namespace '{}' is ready", namespace_name);
-        Ok(())
+            })
     }
 
     async fn ensure_cnpg_operator<T: Topology + K8sclient + HelmCommand + 'static>(
@@ -129,12 +109,29 @@ impl K8sPostgreSQLInterpret {
             .await
             .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {}", e)))?;
 
-        let pods = k8s_client
+        let pods = match k8s_client
             .list_all_resources_with_labels::<Pod>("app.kubernetes.io/name=cloudnative-pg")
             .await
-            .map_err(|e| {
-                InterpretError::new(format!("Failed to list CNPG operator pods: {}", e))
-            })?;
+        {
+            Ok(pods) => pods,
+            // A namespace-scoped deployer (e.g. a tenant CI service account)
+            // cannot list pods cluster-wide. CNPG is a cluster-wide platform
+            // operator managed out-of-band, so skip the install/readiness check
+            // rather than fail the deploy — assume it is present.
+            Err(e) if is_forbidden(&e) => {
+                info!(
+                    "Skipped CNPG operator setup due to restricted (namespace-scoped) \
+                     access — assuming the CNPG operator is managed cluster-wide by the platform"
+                );
+                return Ok(());
+            }
+            Err(e) => {
+                return Err(InterpretError::new(format!(
+                    "Failed to list CNPG operator pods: {}",
+                    e
+                )));
+            }
+        };
 
         if !pods.is_empty() {
             info!("CNPG operator is already installed");

harmony/src/modules/registry_pull_secret.rs

@@ -0,0 +1,125 @@
+//! A reusable Score that materializes a `kubernetes.io/dockerconfigjson` Secret
+//! so the kubelet can pull an app's images from a private registry.
+//!
+//! The Secret is referenced by name from each pod's `imagePullSecrets` (see
+//! `harmony_app`'s `DeployConfig::image_pull_secrets`). Keep the credentials
+//! **pull-only** and load them from a vault (e.g. OpenBao `DeploySecrets`) — they
+//! never belong in code or chart values. The Secret is namespaced, so a
+//! namespace-scoped deployer can apply it without any cluster RBAC.
+//!
+//! ```ignore
+//! let pull = RegistryPullSecretScore {
+//!     namespace: "devsights-folk".into(),
+//!     name: "registry-pull".into(),
+//!     registry: "hub.nationtech.io".into(),
+//!     username: pull_user,
+//!     token: pull_token,
+//! };
+//! // add `pull` to scores(), and set deploy.image_pull_secrets = vec!["registry-pull"]
+//! ```
+
+use std::collections::BTreeMap;
+
+use base64::Engine;
+use k8s_openapi::ByteString;
+use k8s_openapi::api::core::v1::Secret as K8sSecret;
+use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta;
+use serde::Serialize;
+
+use crate::interpret::Interpret;
+use crate::modules::k8s::resource::K8sResourceScore;
+use crate::score::Score;
+use crate::topology::{K8sclient, Topology};
+
+/// Creates a `kubernetes.io/dockerconfigjson` Secret in `namespace` from
+/// pull-only registry credentials. Idempotent — a namespaced Secret apply.
+#[derive(Debug, Clone, Serialize)]
+pub struct RegistryPullSecretScore {
+    pub namespace: String,
+    /// Secret name — the same string goes in each pod's `imagePullSecrets`.
+    pub name: String,
+    /// Registry host the creds authenticate to, e.g. `hub.nationtech.io`.
+    pub registry: String,
+    pub username: String,
+    /// Pull-only robot token. Serialized into the Secret, never logged.
+    pub token: String,
+}
+
+impl RegistryPullSecretScore {
+    /// Build the `.dockerconfigjson` Secret. The inner `auth` field is
+    /// `base64(user:token)` per Docker's config format; the kubelet reads it to
+    /// authenticate image pulls.
+    fn secret(&self) -> K8sSecret {
+        let auth = base64::engine::general_purpose::STANDARD
+            .encode(format!("{}:{}", self.username, self.token));
+        let dockerconfig = serde_json::json!({
+            "auths": {
+                &self.registry: {
+                    "username": self.username,
+                    "password": self.token,
+                    "auth": auth,
+                }
+            }
+        });
+        let mut data = BTreeMap::new();
+        data.insert(
+            ".dockerconfigjson".to_string(),
+            ByteString(dockerconfig.to_string().into_bytes()),
+        );
+        K8sSecret {
+            metadata: ObjectMeta {
+                name: Some(self.name.clone()),
+                namespace: Some(self.namespace.clone()),
+                ..Default::default()
+            },
+            type_: Some("kubernetes.io/dockerconfigjson".to_string()),
+            data: Some(data),
+            ..Default::default()
+        }
+    }
+}
+
+impl<T: Topology + K8sclient + 'static> Score<T> for RegistryPullSecretScore {
+    fn name(&self) -> String {
+        format!("RegistryPullSecretScore({}/{})", self.namespace, self.name)
+    }
+
+    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
+        K8sResourceScore::single(self.secret(), Some(self.namespace.clone())).create_interpret()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn sample() -> RegistryPullSecretScore {
+        RegistryPullSecretScore {
+            namespace: "devsights-folk".into(),
+            name: "registry-pull".into(),
+            registry: "hub.nationtech.io".into(),
+            username: "robot$pull".into(),
+            token: "s3cr3t".into(),
+        }
+    }
+
+    #[test]
+    fn builds_a_typed_dockerconfigjson_secret() {
+        let s = sample().secret();
+        assert_eq!(s.type_.as_deref(), Some("kubernetes.io/dockerconfigjson"));
+        assert_eq!(s.metadata.name.as_deref(), Some("registry-pull"));
+        assert_eq!(s.metadata.namespace.as_deref(), Some("devsights-folk"));
+    }
+
+    #[test]
+    fn dockerconfig_carries_creds_and_base64_auth() {
+        let s = sample().secret();
+        let blob = s.data.unwrap().remove(".dockerconfigjson").unwrap();
+        let json: serde_json::Value = serde_json::from_slice(&blob.0).unwrap();
+        let entry = &json["auths"]["hub.nationtech.io"];
+        assert_eq!(entry["username"], "robot$pull");
+        assert_eq!(entry["password"], "s3cr3t");
+        let expected_auth = base64::engine::general_purpose::STANDARD.encode("robot$pull:s3cr3t");
+        assert_eq!(entry["auth"], expected_auth);
+    }
+}

harmony/src/modules/zitadel/mod.rs

@@ -22,8 +22,8 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 
 use async_trait::async_trait;
+use harmony_config::Config;
 use harmony_macros::hurl;
-use harmony_secret::{Secret, SecretManager};
 use harmony_types::id::Id;
 use harmony_types::storage::StorageSize;
 use log::{debug, error, info, trace, warn};
@@ -36,7 +36,6 @@ use crate::{
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::Inventory,
     modules::helm::chart::{HelmChartScore, HelmRepository},
-    modules::k8s::resource::K8sResourceScore,
     modules::postgresql::capability::{PostgreSQL, PostgreSQLClusterRole, PostgreSQLConfig},
     score::Score,
     topology::{HelmCommand, K8sclient, Topology},
@@ -230,25 +229,26 @@ impl<T: Topology + K8sclient + HelmCommand + PostgreSQL> Interpret<T> for Zitade
             self.host
         );
 
-        info!("Creating namespace {ns} if it does not exist");
-        K8sResourceScore::single(
-            Namespace {
-                metadata: ObjectMeta {
-                    name: Some(self.namespace.to_string()),
-                    ..Default::default()
-                },
-                ..Default::default()
-            },
-            None,
-        )
-        .interpret(inventory, topology)
-        .await?;
+        // Ensure the tenant namespace exists without patching it (it is
+        // provisioned by platform onboarding; a namespace-scoped deployer can't
+        // patch the cluster-scoped Namespace object).
+        let k8s_client = topology
+            .k8s_client()
+            .await
+            .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {e}")))?;
+        k8s_client
+            .ensure_namespace(ns)
+            .await
+            .map_err(|e| InterpretError::new(format!("Failed to ensure namespace '{ns}': {e}")))?;
 
         // --- Step 1: PostgreSQL -------------------------------------------
 
         let pg_config = PostgreSQLConfig {
             cluster_name: PG_CLUSTER_NAME.to_string(),
-            instances: 2,
+            // Single instance by default: storage durability comes from the
+            // backing layer (Ceph), so CNPG-level replication adds cost/ops
+            // without buying availability here.
+            instances: 1,
             storage_size: StorageSize::gi(10),
             role: PostgreSQLClusterRole::Primary,
             namespace: self.namespace.to_string(),
@@ -356,10 +356,10 @@ impl<T: Topology + K8sclient + HelmCommand + PostgreSQL> Interpret<T> for Zitade
         // password every time (Zitadel's chart only honors FirstInstance.*
         // on the very first install; on re-runs the live password is
         // whatever's already in the DB, but our printed banner used to
-        // emit a fresh random — misleading the operator). harmony_secret
+        // emit a fresh random — misleading the operator). harmony_config
         // namespaces by install context (config-resolved), so two
         // installs in the same context share credentials.
-        let admin = match SecretManager::get::<ZitadelAdmin>().await {
+        let admin = match harmony_config::get::<ZitadelAdmin>().await {
             Ok(a) => a,
             Err(e) => {
                 debug!("[Zitadel] No persisted admin credentials yet ({e}); generating");
@@ -367,7 +367,7 @@ impl<T: Topology + K8sclient + HelmCommand + PostgreSQL> Interpret<T> for Zitade
                     username: "admin".to_string(),
                     password: generate_secure_password(16),
                 };
-                SecretManager::set(&a).await.map_err(|err| {
+                harmony_config::set(&a).await.map_err(|err| {
                     InterpretError::new(format!("Failed to persist Zitadel admin password: {err}"))
                 })?;
                 a
@@ -402,12 +402,47 @@ impl<T: Topology + K8sclient + HelmCommand + PostgreSQL> Interpret<T> for Zitade
             MASTERKEY_SECRET_NAME, self.namespace
         );
 
-        // Masterkey for symmetric encryption — must be exactly 32 ASCII bytes (alphanumeric only).
-        let masterkey = rng()
+        // Masterkey for symmetric encryption — exactly 32 ASCII alphanumeric
+        // bytes, and it MUST be stable: regenerating it makes the existing
+        // Zitadel DB undecryptable. Resolve in priority order:
+        //   1. the in-cluster Secret, if present (authoritative — it's what the
+        //      running instance encrypted its data with),
+        //   2. the value persisted in harmony_config (survives a namespace reset),
+        //   3. a freshly generated one.
+        // Then mirror the resolved value into harmony_config so a deleted/
+        // recreated namespace reuses it rather than minting a new (broken) key.
+        let existing_masterkey = k8s_client
+            .get_resource::<K8sSecret>(MASTERKEY_SECRET_NAME, Some(&self.namespace))
+            .await
+            .ok()
+            .flatten()
+            .and_then(|s| s.data?.get("masterkey").cloned())
+            .and_then(|bs| String::from_utf8(bs.0).ok());
+
+        let masterkey = match existing_masterkey {
+            Some(k) => k,
+            None => match harmony_config::get::<ZitadelMasterkey>().await {
+                Ok(m) => m.masterkey,
+                Err(e) => {
+                    debug!("[Zitadel] No persisted masterkey yet ({e}); generating");
+                    rng()
                         .sample_iter(&rand::distr::Alphanumeric)
                         .take(32)
                         .map(char::from)
-            .collect::<String>();
+                        .collect::<String>()
+                }
+            },
+        };
+
+        if harmony_config::get::<ZitadelMasterkey>().await.is_err() {
+            harmony_config::set(&ZitadelMasterkey {
+                masterkey: masterkey.clone(),
+            })
+            .await
+            .map_err(|e| {
+                InterpretError::new(format!("Failed to persist Zitadel masterkey: {e}"))
+            })?;
+        }
 
         debug!(
             "[Zitadel] Created masterkey secret '{}' in namespace '{}'",
@@ -875,8 +910,18 @@ login:
             chart_version: Some(NonBlankString::from_str("9.27.1").unwrap()),
             values_overrides: None,
             values_yaml: Some(values_yaml),
-            create_namespace: true,
-            install_only: false,
+            // The namespace is ensured up front via `ensure_namespace`; don't let
+            // helm `--create-namespace` patch the (cluster-scoped) Namespace
+            // object, which a namespace-scoped deployer isn't allowed to do.
+            create_namespace: false,
+            // Install-once: Zitadel is stateful (DB + masterkey persist), and its
+            // chart's `zitadel-setup` init Job is a pre-upgrade hook that isn't
+            // safely re-runnable. Once the release exists we skip the helm op
+            // entirely — re-deploys converge via ZitadelSetupScore, not by
+            // re-running the chart. (Deliberate version upgrades are a separate,
+            // explicit action.)
+            install_only: true,
+            force_conflicts: false,
             repository: Some(HelmRepository::new(
                 "zitadel".to_string(),
                 hurl!("https://charts.zitadel.com"),
@@ -934,9 +979,18 @@ login:
 /// Persisted admin credentials for the Zitadel instance — written on
 /// first install, reused on subsequent runs so the printed banner
 /// matches what's actually live in the DB. Stored under whichever
-/// `harmony_secret` backend is configured (LocalFile by default).
-#[derive(Secret, Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)]
+/// `harmony_config` backend is configured (LocalFile by default).
+#[derive(Config, Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)]
 struct ZitadelAdmin {
     username: String,
+    #[config(secret)]
     password: String,
 }
+
+/// Persisted Zitadel masterkey (32 ASCII alphanumeric bytes). Stored so it
+/// survives a namespace/Secret reset — a new key would orphan the existing DB.
+#[derive(Config, Serialize, Deserialize, JsonSchema, Debug, PartialEq, Clone)]
+struct ZitadelMasterkey {
+    #[config(secret)]
+    masterkey: String,
+}

harmony/src/modules/zitadel/setup.rs

@@ -2254,9 +2254,20 @@ impl<T: Topology + K8sclient> Interpret<T> for ZitadelSetupInterpret {
         // the management API through 127.0.0.1. `_pf` must outlive every call
         // below. `me` is `self` with the resolved endpoint patched in.
         let _pf;
-        let me: std::borrow::Cow<'_, ZitadelSetupInterpret> =
-            if let Some(svc) = &self.score.port_forward_service {
+        let me: std::borrow::Cow<'_, ZitadelSetupInterpret> = if let Some(svc) =
+            &self.score.port_forward_service
+        {
             let label = format!("app.kubernetes.io/name={svc}");
+            // On a cold cluster the Zitadel Deployment pod is not Running yet
+            // (image pull + DB migration take a minute or more), so poll for
+            // it instead of failing on the first miss — otherwise a one-shot
+            // `ship` aborts here and only a re-run (pod now up) succeeds.
+            let pod = {
+                use std::time::{Duration, Instant};
+                let timeout = Duration::from_secs(300);
+                let interval = Duration::from_secs(3);
+                let deadline = Instant::now() + timeout;
+                loop {
                     let pods = k8s
                         .list_resources::<k8s_openapi::api::core::v1::Pod>(
                             Some(&self.score.namespace),
@@ -2264,19 +2275,29 @@ impl<T: Topology + K8sclient> Interpret<T> for ZitadelSetupInterpret {
                         )
                         .await
                         .map_err(|e| InterpretError::new(format!("listing '{svc}' pods: {e}")))?;
-                // The chart's init/setup Job pods share this label; only the
-                // serving Deployment pod is Running, so filter on phase.
-                let pod = pods
+                    // The chart's init/setup Job pods share this label; only
+                    // the serving Deployment pod is Running, so filter on phase.
+                    let running = pods
                         .items
                         .into_iter()
-                    .find(|p| p.status.as_ref().and_then(|s| s.phase.as_deref()) == Some("Running"))
-                    .and_then(|p| p.metadata.name)
-                    .ok_or_else(|| {
-                        InterpretError::new(format!(
-                            "no Running pod for service '{svc}' in namespace '{}'",
-                            self.score.namespace
-                        ))
-                    })?;
+                        .find(|p| {
+                            p.status.as_ref().and_then(|s| s.phase.as_deref()) == Some("Running")
+                        })
+                        .and_then(|p| p.metadata.name);
+                    if let Some(name) = running {
+                        break name;
+                    }
+                    if Instant::now() >= deadline {
+                        return Err(InterpretError::new(format!(
+                            "no Running pod for service '{svc}' in namespace '{}' within {}s",
+                            self.score.namespace,
+                            timeout.as_secs(),
+                        )));
+                    }
+                    debug!("[ZitadelSetup] waiting for a Running '{svc}' pod...");
+                    tokio::time::sleep(interval).await;
+                }
+            };
             // Zitadel serves http on 8080 in-cluster (chart default).
             let handle = k8s
                 .port_forward(&pod, &self.score.namespace, 0, 8080)

harmony_app/src/app.rs

@@ -34,6 +34,14 @@ pub trait HarmonyApp<T: Topology>: Send + Sync {
     /// branching on `ctx.profile()`. Called by `deploy`/`ship`.
     async fn scores(&self, ctx: &AppContext) -> Result<Vec<Box<dyn Score<T>>>>;
 
+    async fn scores_with_options(
+        &self,
+        ctx: &AppContext,
+        _options: DeployOptions,
+    ) -> Result<Vec<Box<dyn Score<T>>>> {
+        self.scores(ctx).await
+    }
+
     /// Build + distribute images for `ctx` (push to a registry, or import to
     /// k3d). Default: nothing to build. Called by `ship`.
     async fn publish(&self, _ctx: &AppContext) -> Result<()> {
@@ -54,6 +62,11 @@ pub struct DeployReport {
     pub steps: Vec<StepOutcome>,
 }
 
+#[derive(Debug, Clone, Copy, Default)]
+pub struct DeployOptions {
+    pub force_conflicts: bool,
+}
+
 #[derive(Debug, Clone, serde::Serialize)]
 pub struct WorkloadStatus {
     pub name: String,
@@ -81,7 +94,16 @@ pub async fn deploy<T: Topology + Send + Sync + 'static>(
     topology: T,
     ctx: &AppContext,
 ) -> Result<DeployReport> {
-    let scores = app.scores(ctx).await?;
+    deploy_with_options(app, topology, ctx, DeployOptions::default()).await
+}
+
+pub async fn deploy_with_options<T: Topology + Send + Sync + 'static>(
+    app: &dyn HarmonyApp<T>,
+    topology: T,
+    ctx: &AppContext,
+    options: DeployOptions,
+) -> Result<DeployReport> {
+    let scores = app.scores_with_options(ctx, options).await?;
     let to_run: Vec<Box<dyn Score<T>>> = scores.iter().map(|s| s.clone_box()).collect();
 
     let mut maestro = Maestro::new_without_initialization(Inventory::autoload(), topology);
@@ -111,9 +133,18 @@ pub async fn ship<T: Topology + Send + Sync + 'static>(
     app: &dyn HarmonyApp<T>,
     topology: T,
     ctx: &AppContext,
+) -> Result<DeployReport> {
+    ship_with_options(app, topology, ctx, DeployOptions::default()).await
+}
+
+pub async fn ship_with_options<T: Topology + Send + Sync + 'static>(
+    app: &dyn HarmonyApp<T>,
+    topology: T,
+    ctx: &AppContext,
+    options: DeployOptions,
 ) -> Result<DeployReport> {
     app.publish(ctx).await?;
-    deploy(app, topology, ctx).await
+    deploy_with_options(app, topology, ctx, options).await
 }
 
 /// Workload readiness in the app's namespace (operational, read-only).

harmony_app/src/chart.rs

@@ -15,10 +15,10 @@ use k8s_openapi::api::apps::v1::{
     Deployment, DeploymentSpec, DeploymentStrategy, RollingUpdateDeployment,
 };
 use k8s_openapi::api::core::v1::{
-    Container, ContainerPort, EnvFromSource, EnvVar, KeyToPath, PersistentVolumeClaim,
-    PersistentVolumeClaimSpec, PersistentVolumeClaimVolumeSource, PodSpec, PodTemplateSpec,
-    SecretEnvSource, SecretVolumeSource, Service, ServicePort, ServiceSpec, Volume, VolumeMount,
-    VolumeResourceRequirements,
+    Container, ContainerPort, EnvFromSource, EnvVar, KeyToPath, LocalObjectReference,
+    PersistentVolumeClaim, PersistentVolumeClaimSpec, PersistentVolumeClaimVolumeSource, PodSpec,
+    PodTemplateSpec, SecretEnvSource, SecretVolumeSource, Service, ServicePort, ServiceSpec,
+    Volume, VolumeMount, VolumeResourceRequirements,
 };
 use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
@@ -45,7 +45,7 @@ pub struct DeployConfig {
     pub storage_class: Option<String>,
     pub volume_size: String,
     pub replicas: i32,
-    /// `true` → RollingUpdate (needs RWX so old+new pods share volumes);
+    /// `true` → RollingUpdate (when persistence enabled needs RWX so old+new pods share volumes);
     /// `false` → Recreate (safe for single-writer stores like sqlite).
     pub rolling: bool,
     /// Extra env injected into every service container by capabilities —
@@ -56,6 +56,10 @@ pub struct DeployConfig {
     /// account key whose consumer wants a path, not an env var). By reference,
     /// like `extra_env`, so the chart carries no secret material.
     pub secret_file_mounts: Vec<SecretFileMount>,
+    /// Names of `kubernetes.io/dockerconfigjson` Secrets the kubelet uses to
+    /// pull this app's images from a private registry, set on every pod's
+    /// `imagePullSecrets`. By reference — the Secret is applied separately.
+    pub image_pull_secrets: Vec<String>,
 }
 
 /// Mount one key of a Secret as a file at `path` (its parent dir is the mount
@@ -86,12 +90,13 @@ impl DeployConfig {
             registry: registry.into(),
             project: project.into(),
             version,
-            storage_class: Some(if prod { "cephfs" } else { "local-path" }.to_string()),
+            storage_class: None,
             volume_size: "1Gi".to_string(),
-            replicas: if prod { 2 } else { 1 },
+            replicas: 1,
             rolling: prod,
             extra_env: Vec::new(),
             secret_file_mounts: Vec::new(),
+            image_pull_secrets: Vec::new(),
         }
     }
 }
@@ -305,6 +310,12 @@ fn deployment(svc: &ComposeService, cfg: &DeployConfig) -> Deployment {
                         volume_mounts: (!volume_mounts.is_empty()).then_some(volume_mounts),
                         ..Default::default()
                     }],
+                    image_pull_secrets: (!cfg.image_pull_secrets.is_empty()).then(|| {
+                        cfg.image_pull_secrets
+                            .iter()
+                            .map(|name| LocalObjectReference { name: name.clone() })
+                            .collect()
+                    }),
                     volumes: (!volumes.is_empty()).then_some(volumes),
                     ..Default::default()
                 }),
@@ -401,6 +412,7 @@ mod tests {
             rolling: true,
             extra_env: vec![],
             secret_file_mounts: vec![],
+            image_pull_secrets: vec![],
         };
         let tmp = tempfile::tempdir().unwrap();
         build_chart(&app, &cfg, tmp.path()).unwrap();
@@ -445,6 +457,25 @@ mod tests {
         );
     }
 
+    #[test]
+    fn image_pull_secrets_render_on_pods_when_set() {
+        let (app, mut cfg, _t) = fixture();
+        cfg.image_pull_secrets = vec!["registry-pull".to_string()];
+        let tmp = tempfile::tempdir().unwrap();
+        build_chart(&app, &cfg, tmp.path()).unwrap();
+        let dep = read(&tmp, "deployment-backend.yaml");
+        assert!(dep.contains("imagePullSecrets"), "{dep}");
+        assert!(dep.contains("registry-pull"), "{dep}");
+    }
+
+    #[test]
+    fn no_image_pull_secrets_block_when_empty() {
+        // `fixture()` leaves image_pull_secrets empty → no block rendered.
+        let (_, _, tmp) = fixture();
+        let dep = read(&tmp, "deployment-backend.yaml");
+        assert!(!dep.contains("imagePullSecrets"), "{dep}");
+    }
+
     #[test]
     fn portless_service_emits_no_service_object() {
         let (_, _, tmp) = fixture();

harmony_app/src/compose.rs

@@ -55,6 +55,9 @@ pub struct ComposeService {
     /// set, `publish` builds + pushes the image under our registry.
     pub build_context: Option<PathBuf>,
     pub dockerfile: Option<String>,
+    /// `build.args` — baked into the image at `docker build` time (forwarded as
+    /// `--build-arg`). Only the advanced build form carries them.
+    pub build_args: Vec<(String, String)>,
     pub ports: Vec<PortMap>,
     pub env: Vec<(String, String)>,
     pub mounts: Vec<VolumeMount>,
@@ -108,6 +111,10 @@ impl ComposeApp {
                 Some(docker_compose_types::BuildStep::Advanced(a)) => a.dockerfile.clone(),
                 _ => None,
             };
+            let build_args = match &svc.build_ {
+                Some(docker_compose_types::BuildStep::Advanced(a)) => parse_build_args(&a.args),
+                _ => Vec::new(),
+            };
             if svc.image.is_none() && build_context.is_none() {
                 return Err(ComposeError::NoImageOrBuild { service: name });
             }
@@ -121,6 +128,7 @@ impl ComposeApp {
                 image: svc.image,
                 build_context,
                 dockerfile,
+                build_args,
                 name,
             });
         }
@@ -192,6 +200,24 @@ fn parse_short_port(service: &str, spec: &str) -> Result<PortMap, ComposeError>
     })
 }
 
+/// Flatten compose `build.args` into ordered key/value pairs, mirroring
+/// `parse_env`. List items without `=` get an empty value (docker then resolves
+/// them from the build environment).
+fn parse_build_args(args: &Option<docker_compose_types::BuildArgs>) -> Vec<(String, String)> {
+    use docker_compose_types::BuildArgs;
+    match args {
+        None | Some(BuildArgs::Simple(_)) => Vec::new(),
+        Some(BuildArgs::List(items)) => items
+            .iter()
+            .map(|item| match item.split_once('=') {
+                Some((k, v)) => (k.to_string(), v.to_string()),
+                None => (item.to_string(), String::new()),
+            })
+            .collect(),
+        Some(BuildArgs::KvPair(map)) => map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
+    }
+}
+
 fn parse_env(env: &Environment) -> Vec<(String, String)> {
     match env {
         Environment::List(items) => items
@@ -334,6 +360,23 @@ volumes:
         );
     }
 
+    #[test]
+    fn build_args_parsed_from_advanced_build() {
+        let yaml = "services:\n  api:\n    build:\n      context: .\n      args:\n        QUARKUS_BUILD_PROFILE: dev\n";
+        let app = ComposeApp::parse(yaml, Path::new(".")).unwrap();
+        assert_eq!(
+            app.service("api").unwrap().build_args,
+            vec![("QUARKUS_BUILD_PROFILE".to_string(), "dev".to_string())]
+        );
+    }
+
+    #[test]
+    fn build_args_empty_for_simple_build() {
+        let yaml = "services:\n  api:\n    build: .\n";
+        let app = ComposeApp::parse(yaml, Path::new(".")).unwrap();
+        assert!(app.service("api").unwrap().build_args.is_empty());
+    }
+
     #[test]
     fn named_volume_becomes_a_mount_and_only_mounted_volumes_need_pvcs() {
         let app = app();

harmony_app/src/context.rs

@@ -10,10 +10,11 @@ use std::ffi::OsString;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::process::Command;
+use std::sync::Arc;
 
 use anyhow::{Context, Result, bail};
 use harmony::topology::{K8sAnywhereConfig, K8sAnywhereTopology};
-use harmony_config::{Config, ConfigClient};
+use harmony_config::{Config, ConfigClient, ConfigSource, LocalFileSource, PromptSource};
 use harmony_k8s::K8sClient;
 use log::{debug, info};
 use schemars::JsonSchema;
@@ -42,6 +43,8 @@ struct ContextDef {
     openbao_namespace: Option<String>,
     openbao_url: Option<String>,
     zitadel_url: Option<String>,
+    openbao_role: Option<String>,
+    zitadel_audience: Option<String>,
 }
 
 /// The cluster K8sAnywhereTopology autoprovisions (its K3DInstallationScore
@@ -72,6 +75,7 @@ pub struct AppContext {
     kubeconfig: Option<PathBuf>,
     /// The kubeconfig file must outlive every client/topology built from it.
     _kubeconfig_guard: Option<NamedTempFile>,
+    config_client: Arc<ConfigClient>,
 }
 
 impl AppContext {
@@ -108,6 +112,11 @@ impl AppContext {
             def.profile
         );
         debug!("Context '{name}' definition: {def:?}");
+        let config_sources = build_config_sources(def, local_config_dir.clone())
+            .await
+            .with_context(|| format!("building config sources for context '{name}'"))?;
+        harmony_config::init(config_sources.clone()).await;
+        let config_client = Arc::new(ConfigClient::new(config_sources));
         let guard = match (def.autoprovision, &def.k3d, &def.openbao_namespace) {
             (true, None, None) => {
                 info!("Cluster access: autoprovision local k3d ('{AUTOPROVISION_CLUSTER}')");
@@ -119,19 +128,10 @@ impl AppContext {
             }
             (false, None, Some(ns)) => {
                 info!("Cluster access: kubeconfig from OpenBao (namespace '{ns}')");
-                // The kubeconfig is an OpenBao secret, so read it from an
-                // OpenBao-only client built with the context's URLs (each
-                // falling through to env when unset).
-                let access: ClusterAccess =
-                    openbao_client(ns, def.openbao_url.clone(), def.zitadel_url.clone())
-                        .await
-                        .with_context(|| format!("reaching OpenBao for context '{name}'"))?
-                        .get()
-                        .await
-                        .with_context(|| {
+                let access: ClusterAccess = config_client.get().await.with_context(|| {
                     format!(
                         "loading cluster kubeconfig from OpenBao ({ns}/ClusterAccess) \
-                         — are the Zitadel key + OpenBao URL/role set?"
+                         — is HARMONY_ZITADEL_KEY_JSON set for this context?"
                     )
                 })?;
                 Some(write_kubeconfig(access.kubeconfig.as_bytes())?)
@@ -157,6 +157,7 @@ impl AppContext {
             autoprovision: def.autoprovision,
             kubeconfig: guard.as_ref().map(|g| g.path().to_path_buf()),
             _kubeconfig_guard: guard,
+            config_client,
         })
     }
 
@@ -172,6 +173,9 @@ impl AppContext {
     pub fn local_config_dir(&self) -> Option<&Path> {
         self.local_config_dir.as_deref()
     }
+    pub fn config_client(&self) -> &ConfigClient {
+        &self.config_client
+    }
     pub fn k3d_cluster(&self) -> Option<&str> {
         self.k3d_cluster.as_deref()
     }
@@ -231,22 +235,36 @@ fn find_contexts_file() -> Result<PathBuf> {
     }
 }
 
-/// An OpenBao-only `ConfigClient` for `namespace`, using the given URLs (each
-/// falling through to env). Errors when no OpenBao endpoint is resolvable.
-async fn openbao_client(
-    namespace: &str,
-    openbao_url: Option<String>,
-    zitadel_url: Option<String>,
-) -> Result<ConfigClient> {
-    let source = harmony_config::openbao_source(namespace, openbao_url, zitadel_url)
-        .await
-        .with_context(|| {
-            format!(
-                "no OpenBao source for '{namespace}' — set `openbao_url`/OPENBAO_URL \
-                 and the Zitadel auth env (HARMONY_ZITADEL_KEY_*, AUDIENCE, OPENBAO_JWT_ROLE)"
+async fn build_config_sources(
+    def: &ContextDef,
+    local_config_dir: Option<PathBuf>,
+) -> Result<Vec<Arc<dyn ConfigSource>>> {
+    let mut sources: Vec<Arc<dyn ConfigSource>> = Vec::new();
+
+    if let Some(namespace) = &def.openbao_namespace {
+        let source = harmony_config::openbao_source(
+            namespace,
+            def.openbao_url.clone(),
+            def.zitadel_url.clone(),
+            def.zitadel_audience.clone(),
+            Some(
+                def.openbao_role
+                    .clone()
+                    .unwrap_or_else(|| format!("{namespace}-cd")),
+            ),
         )
-        })?;
-    Ok(ConfigClient::new(vec![source]))
+        .await
+        .with_context(|| format!("reaching OpenBao for namespace '{namespace}'"))?;
+        sources.push(source);
+    } else {
+        let dir = local_config_dir
+            .or_else(LocalFileSource::default_path)
+            .context("local contexts need a config directory")?;
+        sources.push(Arc::new(LocalFileSource::new(dir)));
+    }
+
+    sources.push(Arc::new(PromptSource::new()));
+    Ok(sources)
 }
 
 fn write_kubeconfig(contents: &[u8]) -> Result<NamedTempFile> {

harmony_app/src/deploy.rs

@@ -7,7 +7,7 @@
 use std::collections::BTreeMap;
 use std::path::Path;
 
-use crate::{AppContext, AppIdentity, AppRef, Capability, HarmonyApp, Profile};
+use crate::{AppContext, AppIdentity, AppRef, Capability, DeployOptions, HarmonyApp, Profile};
 use anyhow::{Result, anyhow};
 use async_trait::async_trait;
 use harmony::score::Score;
@@ -104,7 +104,12 @@ impl<T: Topology> ComposeDeploy<T> {
     }
 
     /// Derive the deploy Score for a profile (pure — the testable core).
-    pub fn score(&self, profile: Profile, version: &str) -> Result<ComposeAppScore, String> {
+    pub fn score(
+        &self,
+        profile: Profile,
+        version: &str,
+        force_conflicts: bool,
+    ) -> Result<ComposeAppScore, String> {
         let public_endpoint = match &self.expose {
             Some(e) => Some(PublicEndpoint::from_compose(
                 &self.app,
@@ -133,6 +138,7 @@ impl<T: Topology> ComposeDeploy<T> {
             app: self.app.clone(),
             deploy,
             app_secrets: self.app_secrets.clone(),
+            force_conflicts,
         })
     }
 }
@@ -147,8 +153,17 @@ impl<T: Topology + HelmCommand + K8sclient> HarmonyApp<T> for ComposeDeploy<T> {
     }
 
     async fn scores(&self, ctx: &AppContext) -> Result<Vec<Box<dyn Score<T>>>> {
+        self.scores_with_options(ctx, DeployOptions::default())
+            .await
+    }
+
+    async fn scores_with_options(
+        &self,
+        ctx: &AppContext,
+        options: DeployOptions,
+    ) -> Result<Vec<Box<dyn Score<T>>>> {
         let app_score = self
-            .score(ctx.profile(), ctx.version())
+            .score(ctx.profile(), ctx.version(), options.force_conflicts)
             .map_err(|e| anyhow!(e))?;
         let mut scores: Vec<Box<dyn Score<T>>> = vec![Box::new(app_score)];
         let app_ref = self.app_ref(ctx.profile());
@@ -201,7 +216,7 @@ mod tests {
     #[test]
     fn namespace_and_project_default_to_name() {
         let s = CD::from_compose("timesheet", fixture())
-            .score(Profile::Local, "0.1.0")
+            .score(Profile::Local, "0.1.0", false)
             .unwrap();
         assert_eq!(s.namespace, "timesheet");
         assert_eq!(s.release_name, "timesheet");
@@ -211,7 +226,7 @@ mod tests {
     fn local_profile_renders_locally_rwo_http() {
         let s = CD::from_compose("ts", fixture())
             .expose("frontend", "ts.local")
-            .score(Profile::Local, "1.0.0")
+            .score(Profile::Local, "1.0.0", false)
             .unwrap();
         assert!(s.published_chart.is_none(), "local renders from compose");
         assert_eq!(s.deploy.replicas, 1);
@@ -229,13 +244,13 @@ mod tests {
             .registry("hub.x")
             .project("p")
             .expose("frontend", "ts.x")
-            .score(Profile::Prod, "2.0.0")
+            .score(Profile::Prod, "2.0.0", false)
             .unwrap();
         let pc = s.published_chart.expect("prod installs published chart");
         assert_eq!(pc.registry, "hub.x");
         assert_eq!(pc.project, "p");
         assert_eq!(pc.version, "2.0.0");
-        assert_eq!(s.deploy.replicas, 2);
+        assert_eq!(s.deploy.replicas, 1);
         assert!(s.deploy.rolling);
         assert_eq!(
             s.public_endpoint.unwrap().cluster_issuer.as_deref(),
@@ -247,7 +262,7 @@ mod tests {
     fn secrets_are_carried_to_the_score() {
         let s = CD::from_compose("ts", fixture())
             .secret("DB_PASSWORD", "dev")
-            .score(Profile::Local, "0.1.0")
+            .score(Profile::Local, "0.1.0", false)
             .unwrap();
         assert_eq!(s.app_secrets.get("DB_PASSWORD").unwrap(), "dev");
     }
@@ -256,7 +271,7 @@ mod tests {
     fn postgres_capability_wires_database_url_by_reference() {
         let s = CD::from_compose("ts", fixture())
             .with(Postgres::managed())
-            .score(Profile::Local, "0.1.0")
+            .score(Profile::Local, "0.1.0", false)
             .unwrap();
         let db = s
             .deploy

harmony_app/src/lib.rs

@@ -25,8 +25,8 @@ pub mod publish;
 pub mod score;
 
 pub use app::{
-    AppIdentity, DeployReport, HarmonyApp, PodLogs, StatusReport, StepOutcome, WorkloadStatus,
-    deploy, logs, ship, status,
+    AppIdentity, DeployOptions, DeployReport, HarmonyApp, PodLogs, StatusReport, StepOutcome,
+    WorkloadStatus, deploy, deploy_with_options, logs, ship, ship_with_options, status,
 };
 pub use capabilities::{AppRef, Capability, Monitoring, Postgres, ZitadelAuth};
 pub use chart::{DeployConfig, SecretFileMount, cluster_issuer_for, service_image};

harmony_app/src/publish.rs

@@ -45,6 +45,9 @@ pub fn build_images(app: &ComposeApp, cfg: &DeployConfig) -> Result<()> {
         if let Some(dockerfile) = &svc.dockerfile {
             cmd.arg("-f").arg(context.join(dockerfile));
         }
+        for (key, value) in &svc.build_args {
+            cmd.arg("--build-arg").arg(format!("{key}={value}"));
+        }
         cmd.arg(context);
         run(cmd)?;
     }

harmony_app/src/score.rs

@@ -92,6 +92,7 @@ pub struct ComposeAppScore {
     /// values must never reach Score display/logs.
     #[serde(skip)]
     pub app_secrets: BTreeMap<String, String>,
+    pub force_conflicts: bool,
 }
 
 impl<T: Topology + HelmCommand + K8sclient> Score<T> for ComposeAppScore {
@@ -229,6 +230,18 @@ impl ComposeAppInterpret {
         version: &str,
     ) -> Result<Outcome, InterpretError> {
         let s = &self.score;
+        // Ensure the namespace exists without patching it (a namespace-scoped
+        // deployer can't patch the cluster-scoped Namespace object), then install
+        // without helm `--create-namespace`.
+        topology
+            .k8s_client()
+            .await
+            .map_err(|e| InterpretError::new(format!("Failed to get k8s client: {e}")))?
+            .ensure_namespace(&s.namespace)
+            .await
+            .map_err(|e| {
+                InterpretError::new(format!("Failed to ensure namespace '{}': {e}", s.namespace))
+            })?;
         HelmChartScore {
             namespace: Some(non_blank(&s.namespace, "namespace")?),
             release_name: non_blank(&s.release_name, "release_name")?,
@@ -236,8 +249,9 @@ impl ComposeAppInterpret {
             chart_version: Some(non_blank(version, "chart_version")?),
             values_overrides: None,
             values_yaml: None,
-            create_namespace: true,
+            create_namespace: false,
             install_only: false,
+            force_conflicts: s.force_conflicts,
             repository: None,
         }
         .interpret(inventory, topology)

harmony_app/tests/helm_render.rs

@@ -37,6 +37,7 @@ fn generated_chart_passes_helm_lint_and_template() {
         rolling: true,
         extra_env: vec![],
         secret_file_mounts: vec![],
+        image_pull_secrets: vec![],
     };
 
     let tmp = tempfile::tempdir().unwrap();

harmony_cli/src/app.rs

@@ -40,9 +40,17 @@ struct AppCli {
 #[derive(Subcommand, Debug)]
 enum Verb {
     /// Build + publish + deploy.
-    Ship,
+    Ship {
+        /// Pass `--force-conflicts` to the app chart's helm upgrade.
+        #[arg(long)]
+        force_conflicts: bool,
+    },
     /// Converge the app's Scores (no build).
-    Deploy,
+    Deploy {
+        /// Pass `--force-conflicts` to the app chart's helm upgrade.
+        #[arg(long)]
+        force_conflicts: bool,
+    },
     /// Workload readiness in the app's namespace.
     Status,
     /// Recent logs from the app's pods.
@@ -66,8 +74,24 @@ pub async fn app_main<A: HarmonyApp<K8sAnywhereTopology> + 'static>(app: A) -> R
     let ctx = AppContext::resolve(&context, cli.tag, cli.local_config, cli.config).await?;
 
     match cli.verb {
-        Verb::Ship => render_deploy(harmony_app::ship(&app, ctx.topology(), &ctx).await?),
-        Verb::Deploy => render_deploy(harmony_app::deploy(&app, ctx.topology(), &ctx).await?),
+        Verb::Ship { force_conflicts } => render_deploy(
+            harmony_app::ship_with_options(
+                &app,
+                ctx.topology(),
+                &ctx,
+                harmony_app::DeployOptions { force_conflicts },
+            )
+            .await?,
+        ),
+        Verb::Deploy { force_conflicts } => render_deploy(
+            harmony_app::deploy_with_options(
+                &app,
+                ctx.topology(),
+                &ctx,
+                harmony_app::DeployOptions { force_conflicts },
+            )
+            .await?,
+        ),
         Verb::Status => render_status(harmony_app::status(&app, &ctx).await?),
         Verb::Logs { tail } => render_logs(harmony_app::logs(&app, &ctx, Some(tail)).await?),
     }

harmony_config/src/lib.rs

@@ -19,6 +19,7 @@ pub use harmony_config_derive::Config;
 pub use source::env::EnvSource;
 pub use source::local_file::LocalFileSource;
 pub use source::prompt::PromptSource;
+pub use source::prompt::{FieldPrompter, FieldType, InquirePrompter};
 pub use source::sqlite::SqliteSource;
 pub use source::store::StoreSource;
 pub use source::toml_file::TomlFileSource;
@@ -171,16 +172,46 @@ impl ConfigClient {
     }
 
     pub async fn get_or_prompt<T: Config>(&self) -> Result<T, ConfigError> {
-        match self.get::<T>().await {
-            Ok(config) => Ok(config),
-            Err(ConfigError::NotFound { .. }) => {
-                let config = PromptSource::new().prompt_for::<T>().await?;
+        self.get_or_fill_missing::<T>(&InquirePrompter).await
+    }
+
+    /// Like [`get`](Self::get), but when no source holds a *complete* `T`,
+    /// prompt for the fields that are missing — keeping any partial values
+    /// already stored (e.g. after a `Secret` struct gains a field) — then
+    /// persist the merged result. Prompting is delegated to `prompter` so the
+    /// flow is testable without a terminal.
+    async fn get_or_fill_missing<T: Config>(
+        &self,
+        prompter: &dyn FieldPrompter,
+    ) -> Result<T, ConfigError> {
+        let mut partial: Option<serde_json::Map<String, serde_json::Value>> = None;
+        for source in &self.sources {
+            if let Some(value) = source.get(T::CLASS, T::KEY).await? {
+                match serde_json::from_value::<T>(value.clone()) {
+                    Ok(config) => return Ok(config),
+                    Err(e) => {
+                        warn!(
+                            "Incomplete or stale value for {} in a source ({e}); \
+                             will prompt for any missing fields",
+                            T::KEY
+                        );
+                        // Remember the first partial object so we fill its gaps
+                        // instead of re-prompting fields we already have.
+                        if partial.is_none()
+                            && let serde_json::Value::Object(map) = value
+                        {
+                            partial = Some(map);
+                        }
+                    }
+                }
+            }
+        }
+        let config = PromptSource::new()
+            .prompt_filling::<T>(partial.unwrap_or_default(), prompter)
+            .await?;
         self.set(&config).await?;
         Ok(config)
     }
-            Err(e) => Err(e),
-        }
-    }
 
     pub async fn set<T: Config>(&self, config: &T) -> Result<(), ConfigError> {
         let value = serde_json::to_value(config).map_err(|e| ConfigError::Serialization {
@@ -198,19 +229,20 @@ impl ConfigClient {
 
 /// Build an OpenBao-backed `StoreSource` purely from env — the default chain.
 async fn openbao_from_env(namespace: &str) -> Option<Arc<dyn ConfigSource>> {
-    openbao_source(namespace, None, None).await
+    openbao_source(namespace, None, None, None, None).await
 }
 
-/// Build an OpenBao-backed `StoreSource`. `openbao_url` and `zitadel_sso_url`
-/// override their env vars (`OPENBAO_URL`/`VAULT_ADDR`, `HARMONY_SSO_URL`) when
-/// `Some`, else fall through to the env — so a caller that knows the endpoints
-/// (e.g. a deploy context) supplies them without mutating process env. Returns
-/// `None` (with a `warn!`) when no OpenBao URL is resolvable or the connection
-/// fails, so a missing OpenBao degrades the chain instead of breaking startup.
+/// Build an OpenBao-backed `StoreSource`. Explicit arguments override their env
+/// vars, so deploy contexts can provide OpenBao/Zitadel coordinates without
+/// mutating process env. Returns `None` (with a `warn!`) when no OpenBao URL is
+/// resolvable or the connection fails, so a missing OpenBao degrades the chain
+/// instead of breaking startup.
 pub async fn openbao_source(
     namespace: &str,
     openbao_url: Option<String>,
     zitadel_sso_url: Option<String>,
+    zitadel_audience: Option<String>,
+    openbao_jwt_role: Option<String>,
 ) -> Option<Arc<dyn ConfigSource>> {
     let env = |k: &str| std::env::var(k).ok();
 
@@ -233,7 +265,7 @@ pub async fn openbao_source(
         match (
             key_path.is_some() || key_json.is_some(),
             sso_url.clone(),
-            env("HARMONY_ZITADEL_AUDIENCE"),
+            zitadel_audience.or_else(|| env("HARMONY_ZITADEL_AUDIENCE")),
         ) {
             (true, Some(oidc_issuer_url), Some(audience)) => {
                 Some(harmony_secret::ZitadelJwtBearerConfig {
@@ -257,7 +289,7 @@ pub async fn openbao_source(
         password: env("OPENBAO_PASSWORD"),
         zitadel_sso_url: sso_url,
         zitadel_client_id: env("HARMONY_SSO_CLIENT_ID"),
-        jwt_role: env("OPENBAO_JWT_ROLE"),
+        jwt_role: openbao_jwt_role.or_else(|| env("OPENBAO_JWT_ROLE")),
         jwt_auth_mount: env("OPENBAO_JWT_AUTH_MOUNT"),
         zitadel_jwt_bearer,
     })
@@ -432,6 +464,76 @@ mod tests {
         }
     }
 
+    /// A `FieldPrompter` double: returns canned answers and records which
+    /// fields it was asked for, so a test can assert only the missing fields
+    /// were prompted.
+    struct FakePrompter {
+        answers: std::collections::HashMap<&'static str, serde_json::Value>,
+        asked: std::sync::Mutex<Vec<String>>,
+    }
+
+    impl FieldPrompter for FakePrompter {
+        fn prompt(
+            &self,
+            _key: &str,
+            field: &str,
+            _ty: FieldType,
+            _is_secret: bool,
+        ) -> Result<serde_json::Value, ConfigError> {
+            self.asked.lock().unwrap().push(field.to_string());
+            self.answers
+                .get(field)
+                .cloned()
+                .ok_or_else(|| ConfigError::PromptError(format!("no canned answer for `{field}`")))
+        }
+    }
+
+    #[tokio::test]
+    async fn get_or_prompt_fills_only_missing_secret_fields_and_persists() {
+        // A Secret struct that has GAINED the `token` field since the stored
+        // value was written.
+        #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Config)]
+        #[config(secret)]
+        struct EvolvingSecret {
+            username: String,
+            token: String,
+        }
+
+        // Store the OLD shape — only `username` existed back then.
+        let mut data = std::collections::HashMap::new();
+        data.insert(
+            "EvolvingSecret".to_string(),
+            serde_json::json!({ "username": "alice" }),
+        );
+        let mock = Arc::new(MockSource::with_data(data));
+        let client = ConfigClient::new(vec![mock.clone()]);
+
+        let fake = FakePrompter {
+            answers: std::collections::HashMap::from([("token", serde_json::json!("s3cr3t"))]),
+            asked: std::sync::Mutex::new(Vec::new()),
+        };
+
+        let got: EvolvingSecret = client.get_or_fill_missing(&fake).await.unwrap();
+
+        // Only the missing field was prompted; the existing value was kept.
+        assert_eq!(*fake.asked.lock().unwrap(), vec!["token".to_string()]);
+        assert_eq!(got.username, "alice");
+        assert_eq!(got.token, "s3cr3t");
+
+        // The merged value was written back to the store.
+        let stored = mock
+            .data
+            .lock()
+            .unwrap()
+            .get("EvolvingSecret")
+            .cloned()
+            .unwrap();
+        assert_eq!(
+            stored,
+            serde_json::json!({ "username": "alice", "token": "s3cr3t" })
+        );
+    }
+
     #[tokio::test]
     async fn test_get_returns_value_when_found_in_first_source() {
         let config = TestConfig {

harmony_config/src/source/prompt.rs

@@ -9,6 +9,95 @@ use crate::{Config, ConfigClass, ConfigError, ConfigSource};
 // terminal ownership, so concurrent prompts would corrupt the terminal.
 static PROMPT_MUTEX: Mutex<()> = Mutex::const_new(());
 
+/// The primitive shape of a flat config field, resolved from its JsonSchema.
+/// Decouples schema-walking from the actual input widget so the latter can be
+/// swapped out (e.g. for tests) — see [`FieldPrompter`].
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FieldType {
+    String,
+    Integer,
+    Number,
+    Boolean,
+}
+
+/// Produces a value for a single config field. The real implementation
+/// ([`InquirePrompter`]) reads it interactively; tests inject a double so the
+/// fill-missing flow can be exercised without a terminal.
+pub trait FieldPrompter: Send + Sync {
+    fn prompt(
+        &self,
+        key: &str,
+        field: &str,
+        ty: FieldType,
+        is_secret: bool,
+    ) -> Result<serde_json::Value, ConfigError>;
+}
+
+/// Reads each field interactively via `inquire`: secret strings are masked
+/// (`Password`), other primitives use type-appropriate input.
+pub struct InquirePrompter;
+
+impl FieldPrompter for InquirePrompter {
+    fn prompt(
+        &self,
+        key: &str,
+        field: &str,
+        ty: FieldType,
+        is_secret: bool,
+    ) -> Result<serde_json::Value, ConfigError> {
+        let label = format!("{field}:");
+        match ty {
+            FieldType::String if is_secret => {
+                // Masked mode echoes `*` per keystroke (inquire's default
+                // Hidden mode shows nothing until enter).
+                let raw = Password::new(&label)
+                    .with_display_mode(PasswordDisplayMode::Masked)
+                    .without_confirmation()
+                    .prompt()
+                    .map_err(|e| {
+                        ConfigError::PromptError(format!(
+                            "Password prompt for `{key}::{field}` failed: {e}"
+                        ))
+                    })?;
+                Ok(serde_json::Value::String(raw))
+            }
+            FieldType::String => {
+                let raw = Text::new(&label).prompt().map_err(|e| {
+                    ConfigError::PromptError(format!(
+                        "Text prompt for `{key}::{field}` failed: {e}"
+                    ))
+                })?;
+                Ok(serde_json::Value::String(raw))
+            }
+            FieldType::Integer => {
+                // i64 covers Config integer fields; serde narrows on deserialize.
+                let n = CustomType::<i64>::new(&label).prompt().map_err(|e| {
+                    ConfigError::PromptError(format!(
+                        "Integer prompt for `{key}::{field}` failed: {e}"
+                    ))
+                })?;
+                Ok(serde_json::json!(n))
+            }
+            FieldType::Number => {
+                let n = CustomType::<f64>::new(&label).prompt().map_err(|e| {
+                    ConfigError::PromptError(format!(
+                        "Number prompt for `{key}::{field}` failed: {e}"
+                    ))
+                })?;
+                Ok(serde_json::json!(n))
+            }
+            FieldType::Boolean => {
+                let b = CustomType::<bool>::new(&label).prompt().map_err(|e| {
+                    ConfigError::PromptError(format!(
+                        "Boolean prompt for `{key}::{field}` failed: {e}"
+                    ))
+                })?;
+                Ok(serde_json::Value::Bool(b))
+            }
+        }
+    }
+}
+
 pub struct PromptSource;
 
 impl PromptSource {
@@ -16,16 +105,38 @@ impl PromptSource {
         Self
     }
 
-    /// Prompt for a `T`. `Standard` structs go through `interactive_parse`
-    /// (full nested support); `Secret` structs go through a flat-field
-    /// walker that reads `#[config(secret)]` fields via `inquire::Password`.
+    /// Prompt for a full `T` (all fields). Equivalent to filling an empty value.
     pub async fn prompt_for<T: Config>(&self) -> Result<T, ConfigError> {
+        self.prompt_filling::<T>(serde_json::Map::new(), &InquirePrompter)
+            .await
+    }
+
+    /// Prompt for the fields of `T` that are **absent** from `existing`, keeping
+    /// the values already present, then deserialize the merged object. This is
+    /// what lets a `Secret` struct gain a field without re-prompting the rest.
+    ///
+    /// `Secret` structs go through a flat-field walker (only the missing fields
+    /// are prompted, via `prompter`); `Standard` structs always go through the
+    /// full interactive parser (partial fill of nested structs is unsupported).
+    pub async fn prompt_filling<T: Config>(
+        &self,
+        existing: serde_json::Map<String, serde_json::Value>,
+        prompter: &dyn FieldPrompter,
+    ) -> Result<T, ConfigError> {
         let _guard = PROMPT_MUTEX.lock().await;
-        let banner = format!(
+        let banner = if existing.is_empty() {
+            format!(
                 "── Configuring `{}` ({:?}) — please fill the fields below ──",
                 T::KEY,
                 T::CLASS,
-        );
+            )
+        } else {
+            format!(
+                "── `{}` ({:?}) is missing fields — fill them in (existing values are kept) ──",
+                T::KEY,
+                T::CLASS,
+            )
+        };
         // inquire renders on stderr; match that channel and pad with blank
         // lines so the banner stays separate from preceding log output.
         eprintln!();
@@ -35,7 +146,7 @@ impl PromptSource {
             ConfigClass::Standard => {
                 T::parse_to_obj().map_err(|e| ConfigError::PromptError(e.to_string()))
             }
-            ConfigClass::Secret => prompt_secret_struct::<T>(),
+            ConfigClass::Secret => prompt_secret_struct_filling::<T>(existing, prompter),
         }
     }
 }
@@ -66,10 +177,13 @@ impl ConfigSource for PromptSource {
     }
 }
 
-// Walks a Secret struct's schema, reading `T::SECRET_FIELDS` via Password
-// and other fields via type-appropriate prompts. Only flat primitive
-// fields are supported; anything else returns a clear PromptError.
-fn prompt_secret_struct<T: Config>() -> Result<T, ConfigError> {
+// Walks a Secret struct's schema and prompts (via `prompter`) for every field
+// NOT already present in `json`, leaving existing values untouched. Only flat
+// primitive fields are supported; anything else returns a clear PromptError.
+fn prompt_secret_struct_filling<T: Config>(
+    mut json: serde_json::Map<String, serde_json::Value>,
+    prompter: &dyn FieldPrompter,
+) -> Result<T, ConfigError> {
     let root: RootSchema = schemars::schema_for!(T);
     let object = root.schema.object.as_deref().ok_or_else(|| {
         ConfigError::PromptError(format!(
@@ -79,11 +193,15 @@ fn prompt_secret_struct<T: Config>() -> Result<T, ConfigError> {
         ))
     })?;
 
-    let mut json = serde_json::Map::with_capacity(object.properties.len());
     for (field_name, field_schema) in &object.properties {
+        // Already stored (e.g. from a previous version of the struct) — keep it.
+        if json.contains_key(field_name) {
+            continue;
+        }
         let field_object = schema_object(field_schema, T::KEY, field_name)?;
+        let ty = field_type::<T>(field_name, field_object)?;
         let is_secret = T::SECRET_FIELDS.contains(&field_name.as_str());
-        let value = prompt_field::<T>(field_name, field_object, is_secret)?;
+        let value = prompter.prompt(T::KEY, field_name, ty, is_secret)?;
         json.insert(field_name.clone(), value);
     }
 
@@ -109,14 +227,13 @@ fn schema_object<'a>(
     }
 }
 
-// Picks the inquire widget for one field from its JsonSchema. Secret
-// strings use Password; other primitives use type-appropriate input.
-// Returns Err for unsupported shapes — never a silent unmasked fallback.
-fn prompt_field<T: Config>(
+// Resolves one field's primitive `FieldType` from its JsonSchema. Returns Err
+// for unsupported shapes (multi-type/nullable, nested, arrays) — never a silent
+// fallback.
+fn field_type<T: Config>(
     field_name: &str,
     schema: &SchemaObject,
-    is_secret: bool,
-) -> Result<serde_json::Value, ConfigError> {
+) -> Result<FieldType, ConfigError> {
     let instance_type = schema.instance_type.as_ref().ok_or_else(|| {
         ConfigError::PromptError(format!(
             "Secret struct `{}` field `{field_name}` is not a flat primitive \
@@ -136,61 +253,11 @@ fn prompt_field<T: Config>(
         }
     };
 
-    let label = format!("{field_name}:");
     match single {
-        InstanceType::String => {
-            if is_secret {
-                // Masked mode echoes `*` per keystroke (inquire's default
-                // Hidden mode shows nothing until enter).
-                let raw = Password::new(&label)
-                    .with_display_mode(PasswordDisplayMode::Masked)
-                    .without_confirmation()
-                    .prompt()
-                    .map_err(|e| {
-                        ConfigError::PromptError(format!(
-                            "Password prompt for `{}::{field_name}` failed: {e}",
-                            T::KEY
-                        ))
-                    })?;
-                Ok(serde_json::Value::String(raw))
-            } else {
-                let raw = Text::new(&label).prompt().map_err(|e| {
-                    ConfigError::PromptError(format!(
-                        "Text prompt for `{}::{field_name}` failed: {e}",
-                        T::KEY
-                    ))
-                })?;
-                Ok(serde_json::Value::String(raw))
-            }
-        }
-        InstanceType::Integer => {
-            // i64 covers Config integer fields; serde narrows on deserialize.
-            let n = CustomType::<i64>::new(&label).prompt().map_err(|e| {
-                ConfigError::PromptError(format!(
-                    "Integer prompt for `{}::{field_name}` failed: {e}",
-                    T::KEY
-                ))
-            })?;
-            Ok(serde_json::json!(n))
-        }
-        InstanceType::Number => {
-            let n = CustomType::<f64>::new(&label).prompt().map_err(|e| {
-                ConfigError::PromptError(format!(
-                    "Number prompt for `{}::{field_name}` failed: {e}",
-                    T::KEY
-                ))
-            })?;
-            Ok(serde_json::json!(n))
-        }
-        InstanceType::Boolean => {
-            let b = CustomType::<bool>::new(&label).prompt().map_err(|e| {
-                ConfigError::PromptError(format!(
-                    "Boolean prompt for `{}::{field_name}` failed: {e}",
-                    T::KEY
-                ))
-            })?;
-            Ok(serde_json::Value::Bool(b))
-        }
+        InstanceType::String => Ok(FieldType::String),
+        InstanceType::Integer => Ok(FieldType::Integer),
+        InstanceType::Number => Ok(FieldType::Number),
+        InstanceType::Boolean => Ok(FieldType::Boolean),
         other => Err(ConfigError::PromptError(format!(
             "Secret struct `{}` field `{field_name}` has unsupported type \
              `{other:?}`; the walker only handles flat primitives.",

harmony_zitadel_jwt/src/lib.rs

@@ -46,7 +46,14 @@ pub struct CachedToken {
 
 #[derive(Deserialize)]
 struct TokenResponse {
-    access_token: String,
+    /// The verifiable JWT we exchange for an OpenBao token (requested via the
+    /// `openid` scope). Zitadel's `access_token` is **opaque** by default — not
+    /// a usable OpenBao JWT — so we deliberately use the id_token, which carries
+    /// the service user in `sub` and the project id in `aud` (the latter via the
+    /// `urn:zitadel:iam:org:project:id:<aud>:aud` scope), exactly what the
+    /// OpenBao JWT role binds on.
+    #[serde(default)]
+    id_token: Option<String>,
     #[serde(default)]
     expires_in: Option<i64>,
 }
@@ -112,10 +119,16 @@ impl ZitadelJwtBearer {
 
         let tr: TokenResponse = resp.json().await.context("parsing token response")?;
         let expires_in = tr.expires_in.unwrap_or(3600);
-        let token = tr.access_token.clone();
+        // Send the id_token (a JWT), NOT the opaque access_token — OpenBao's JWT
+        // auth must be able to verify it against Zitadel's JWKS.
+        let token = tr.id_token.context(
+            "Zitadel token response had no id_token — the `openid` scope is \
+             required (it is in build_scope) and the service user must be \
+             allowed to receive an id_token",
+        )?;
 
         *self.cache.lock().unwrap() = Some(CachedToken {
-            access_token: tr.access_token,
+            access_token: token.clone(),
             expires_at_unix: now + expires_in,
         });