From 2190e60c2a569b6833a55243e475e3da5afb31ad Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Mon, 1 Jun 2026 15:05:15 -0400 Subject: [PATCH] chore: Refactor FLEET_OPERATOR_CREDENTIALS string that appeared in multiple places into a constant shared across all consumers --- Cargo.lock | 1 + examples/fleet_e2e_demo/Cargo.toml | 1 + examples/fleet_e2e_demo/src/lib.rs | 5 +- fleet/harmony-fleet-auth/src/config.rs | 7 +- fleet/harmony-fleet-auth/src/lib.rs | 2 +- .../src/operator/chart.rs | 82 +++++++++++++------ fleet/harmony-fleet-operator/src/main.rs | 13 +-- 7 files changed, 75 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 52da3fe0..e3fc0a4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2861,6 +2861,7 @@ dependencies = [ "example-fleet-auth-callout", "futures-util", "harmony", + "harmony-fleet-auth", "harmony-fleet-deploy", "harmony-fleet-operator", "harmony-k8s", diff --git a/examples/fleet_e2e_demo/Cargo.toml b/examples/fleet_e2e_demo/Cargo.toml index 8f90bc52..bca29efa 100644 --- a/examples/fleet_e2e_demo/Cargo.toml +++ b/examples/fleet_e2e_demo/Cargo.toml @@ -25,6 +25,7 @@ harmony_types = { path = "../../harmony_types" } example-fleet-auth-callout = { path = "../fleet_auth_callout" } harmony-nats-callout = { path = "../../nats/callout" } harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } +harmony-fleet-auth = { path = "../../fleet/harmony-fleet-auth" } harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" } harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" } k3d-rs = { path = "../../k3d" } diff --git a/examples/fleet_e2e_demo/src/lib.rs b/examples/fleet_e2e_demo/src/lib.rs index 71790221..2f0e78b4 100644 --- a/examples/fleet_e2e_demo/src/lib.rs +++ b/examples/fleet_e2e_demo/src/lib.rs @@ -635,7 +635,7 @@ async fn build_and_load_operator_image(k3d: &k3d_rs::K3d) -> Result<()> { /// Apply the operator's CRDs + ServiceAccount + ClusterRole + /// ClusterRoleBinding + Secret + Deployment via Harmony's /// K8sResourceScore. The Secret carries both the `[credentials]` TOML -/// (consumed by the operator as `FLEET_OPERATOR_CREDENTIALS_TOML`) and +/// (consumed by the operator as [`OPERATOR_CREDENTIALS_ENV_VAR`]) and /// the Zitadel JSON keyfile that the TOML's `key_path` references. async fn deploy_operator( topology: &K8sAnywhereTopology, @@ -644,6 +644,7 @@ async fn deploy_operator( ) -> Result<()> { use harmony::modules::fleet::operator::crd::{Deployment as FleetDeployment, Device}; use harmony::modules::k8s::resource::K8sResourceScore; + use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR; use harmony_fleet_deploy::operator::chart::{ ChartOptions, OperatorCredentials, RELEASE_NAME, build_cluster_role, build_cluster_role_binding, build_operator_deployment, build_service_account, @@ -653,7 +654,7 @@ async fn deploy_operator( use kube::CustomResourceExt; // Render the [credentials] TOML the operator pod consumes via the - // FLEET_OPERATOR_CREDENTIALS_TOML env var (sourced from a Secret + // OPERATOR_CREDENTIALS_ENV_VAR env var (sourced from a Secret // key). The Zitadel JSON keyfile is embedded inline under // `key_json`; the operator never sees a file. Triple-quoted TOML // string keeps the JSON's `"`s untouched. diff --git a/fleet/harmony-fleet-auth/src/config.rs b/fleet/harmony-fleet-auth/src/config.rs index b01edb6d..ce8768d6 100644 --- a/fleet/harmony-fleet-auth/src/config.rs +++ b/fleet/harmony-fleet-auth/src/config.rs @@ -1,6 +1,11 @@ use serde::{Deserialize, Serialize}; use std::path::PathBuf; +/// Env var the operator reads its `[credentials]` TOML from. The +/// deploy crate's chart wires this as a `secretKeyRef`; the operator +/// runtime reads it at startup. Single source of truth for both sides. +pub const OPERATOR_CREDENTIALS_ENV_VAR: &str = "FLEET_OPERATOR_CREDENTIALS_TOML"; + /// Externally-tagged credential definition shared between the fleet /// agent and the fleet operator. The `type` field selects the variant; /// each variant's other fields are flatly mixed into the @@ -8,7 +13,7 @@ use std::path::PathBuf; /// /// **Why one struct for both processes**: the agent reads this from /// `/etc/fleet-agent/config.toml`; the operator reads it from a single -/// env var (`FLEET_OPERATOR_CREDENTIALS_TOML`) whose value is a TOML +/// env var ([`OPERATOR_CREDENTIALS_ENV_VAR`]) whose value is a TOML /// snippet shaped exactly like the `[credentials]` table. Identical /// deserialization, identical downstream code path. The only thing /// that differs is the byte source. diff --git a/fleet/harmony-fleet-auth/src/lib.rs b/fleet/harmony-fleet-auth/src/lib.rs index 3cce33e1..e020feeb 100644 --- a/fleet/harmony-fleet-auth/src/lib.rs +++ b/fleet/harmony-fleet-auth/src/lib.rs @@ -26,7 +26,7 @@ mod config; mod credentials; pub use agent_config::{AgentConfig, AgentSection, NatsSection, load_config}; -pub use config::CredentialsSection; +pub use config::{CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR}; pub use credentials::{ ASSERTION_LIFETIME_SECS, CachedToken, CredentialSource, MachineKeyFile, NatsCredential, TOKEN_REFRESH_LEEWAY_SECS, credential_source_from_config, diff --git a/fleet/harmony-fleet-deploy/src/operator/chart.rs b/fleet/harmony-fleet-deploy/src/operator/chart.rs index 1befead0..92496699 100644 --- a/fleet/harmony-fleet-deploy/src/operator/chart.rs +++ b/fleet/harmony-fleet-deploy/src/operator/chart.rs @@ -36,6 +36,7 @@ use serde::Serialize; use harmony::modules::application::helm::{HelmChart, HelmResourceKind}; use harmony::modules::fleet::operator::crd::{Deployment, Device}; +use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR; /// Inputs for chart generation. Default values are aimed at a /// local-dev k3d install; override via the `chart` subcommand flags. @@ -58,7 +59,7 @@ pub struct ChartOptions { /// `RUST_LOG` value for the operator process. pub log_level: String, /// `[credentials]` TOML payload to inject as - /// `FLEET_OPERATOR_CREDENTIALS_TOML` via a Secret. `None` skips the + /// [`OPERATOR_CREDENTIALS_ENV_VAR`] via a Secret. `None` skips the /// Secret entirely and lets the operator connect to NATS without /// auth — only sensible when there's no callout in front of NATS. pub credentials: Option, @@ -78,7 +79,7 @@ pub struct ChartOptions { /// The Zitadel JSON keyfile content is embedded inline inside the /// TOML as the `key_json` field (multi-line triple-quoted string). /// One env-var-from-Secret on the operator Pod -/// (`FLEET_OPERATOR_CREDENTIALS_TOML`) and we're done — no volume +/// ([`OPERATOR_CREDENTIALS_ENV_VAR`]) and we're done — no volume /// mounts, OKD restricted-v2 SCC compatible. #[derive(Debug, Clone, Serialize)] pub struct OperatorCredentials { @@ -186,12 +187,10 @@ pub fn build_chart(opts: &ChartOptions) -> Result { chart.add_resource(HelmResourceKind::ClusterRoleBinding(cluster_role_binding( "{{ .Release.Namespace }}", ))); - // Secret intentionally NOT included in the on-disk helm chart — - // credentials are operator-environment-specific and out of scope - // for a redistributable chart. The e2e bring-up applies the Secret - // directly via `operator_secret()` (used as a `K8sResourceScore`) - // and the chart's Deployment expects the Secret to be present in - // the namespace at install time. + chart.add_resource( + HelmResourceKind::from_serializable("secret-credentials.yaml", &empty_credentials_secret()) + .context("serializing credentials Secret")?, + ); chart.add_resource(HelmResourceKind::Deployment(operator_deployment(opts))); let written = chart @@ -200,6 +199,19 @@ pub fn build_chart(opts: &ChartOptions) -> Result { Ok(written) } +/// Empty Secret resource for the chart — helm creates it, the deploy +/// script (or user) provisions the actual `credentials.toml` data. +fn empty_credentials_secret() -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(SECRET_NAME.to_string()), + ..Default::default() + }, + type_: Some("Opaque".to_string()), + ..Default::default() + } +} + /// Build the operator's Secret holding the `[credentials]` TOML /// (with the JSON keyfile inlined under `key_json`). Returns `None` /// when no credentials are configured (no-auth dev mode). @@ -344,26 +356,24 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { }, ]; - if opts.credentials.is_some() { - // The whole `[credentials]` TOML payload — including the JSON - // keyfile inlined under `key_json` — travels as a single env - // var sourced from a Secret key. No volume mounts: keeps the - // pod compatible with OKD's restricted-v2 SCC and the - // `harmony_fleet_auth::CredentialsSection` deserializer - // handles inline-vs-file from the same TOML shape. - env.push(EnvVar { - name: "FLEET_OPERATOR_CREDENTIALS_TOML".to_string(), - value_from: Some(EnvVarSource { - secret_key_ref: Some(SecretKeySelector { - name: SECRET_NAME.to_string(), - key: SECRET_KEY_CREDENTIALS_TOML.to_string(), - optional: Some(false), - }), - ..Default::default() + // The whole `[credentials]` TOML payload — including the JSON + // keyfile inlined under `key_json` — travels as a single env + // var sourced from a Secret key. No volume mounts: keeps the + // pod compatible with OKD's restricted-v2 SCC and the + // `harmony_fleet_auth::CredentialsSection` deserializer + // handles inline-vs-file from the same TOML shape. + env.push(EnvVar { + name: OPERATOR_CREDENTIALS_ENV_VAR.to_string(), + value_from: Some(EnvVarSource { + secret_key_ref: Some(SecretKeySelector { + name: SECRET_NAME.to_string(), + key: SECRET_KEY_CREDENTIALS_TOML.to_string(), + optional: Some(true), }), ..Default::default() - }); - } + }), + ..Default::default() + }); // Namespace deliberately omitted — same rationale as the // ServiceAccount: helm fills in the release namespace at install @@ -499,4 +509,24 @@ mod tests { // image/SCC negotiate. assert!(sc.run_as_user.is_none()); } + + #[test] + fn chart_includes_credentials_secret_and_env_var() { + let tmp = tempfile::tempdir().unwrap(); + let chart_path = build_chart(&ChartOptions { + output_dir: tmp.path().to_path_buf(), + ..Default::default() + }) + .unwrap(); + + let secret_yaml = + std::fs::read_to_string(chart_path.join("templates/secret-credentials.yaml")) + .expect("secret-credentials.yaml must exist in chart"); + assert!(secret_yaml.contains(SECRET_NAME)); + + let deployment_yaml = std::fs::read_to_string(chart_path.join("templates/deployment.yaml")) + .expect("deployment.yaml must exist in chart"); + assert!(deployment_yaml.contains(OPERATOR_CREDENTIALS_ENV_VAR)); + assert!(deployment_yaml.contains(SECRET_NAME)); + } } diff --git a/fleet/harmony-fleet-operator/src/main.rs b/fleet/harmony-fleet-operator/src/main.rs index d4ff8e4b..6c8a0ed1 100644 --- a/fleet/harmony-fleet-operator/src/main.rs +++ b/fleet/harmony-fleet-operator/src/main.rs @@ -9,7 +9,8 @@ use anyhow::{Context, Result}; use async_nats::jetstream; use clap::{Parser, Subcommand}; use harmony_fleet_auth::{ - CredentialsSection, connect_options_with_credentials, credential_source_from_config, + CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR, connect_options_with_credentials, + credential_source_from_config, }; use harmony_reconciler_contracts::BUCKET_DESIRED_STATE; use kube::Client; @@ -48,7 +49,7 @@ struct Cli { /// (for local dev without a callout-protected NATS). #[arg( long, - env = "FLEET_OPERATOR_CREDENTIALS_TOML", + env = OPERATOR_CREDENTIALS_ENV_VAR, default_value = "", global = true )] @@ -187,7 +188,7 @@ async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()> /// against the NATS server becoming fully ready. /// /// `credentials_toml` is the in-memory `[credentials]` TOML snippet -/// the operator's pod gets via the `FLEET_OPERATOR_CREDENTIALS_TOML` +/// the operator's pod gets via the [`OPERATOR_CREDENTIALS_ENV_VAR`] /// env var (sourced from a Kubernetes Secret). Same shape as the /// agent's `[credentials]` table; same factory; same auth callback. /// Empty string means bypass — connect with no creds (only useful @@ -197,7 +198,7 @@ async fn connect_with_retry(nats_url: &str, credentials_toml: &str) -> Result Result { - let creds_section: CredentialsSection = - toml::from_str(credentials_toml).context("parsing FLEET_OPERATOR_CREDENTIALS_TOML")?; + let creds_section: CredentialsSection = toml::from_str(credentials_toml) + .with_context(|| format!("parsing {OPERATOR_CREDENTIALS_ENV_VAR}"))?; let creds = credential_source_from_config(&creds_section) .context("constructing CredentialSource from operator credentials")?; let client = connect_options_with_credentials(creds) -- 2.39.5