chore: Refactor FLEET_OPERATOR_CREDENTIALS string that appeared in multiple places into a constant shared across all consumers #320

Merged
johnride merged 1 commits from chore/fleet_operator_creds_constant into master 2026-06-01 19:09:24 +00:00
7 changed files with 75 additions and 36 deletions

1
Cargo.lock generated
View File

@@ -2861,6 +2861,7 @@ dependencies = [
"example-fleet-auth-callout",
"futures-util",
"harmony",
"harmony-fleet-auth",
"harmony-fleet-deploy",
"harmony-fleet-operator",
"harmony-k8s",

View File

@@ -25,6 +25,7 @@ harmony_types = { path = "../../harmony_types" }
example-fleet-auth-callout = { path = "../fleet_auth_callout" }
harmony-nats-callout = { path = "../../nats/callout" }
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
harmony-fleet-auth = { path = "../../fleet/harmony-fleet-auth" }
harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
k3d-rs = { path = "../../k3d" }

View File

@@ -635,7 +635,7 @@ async fn build_and_load_operator_image(k3d: &k3d_rs::K3d) -> Result<()> {
/// Apply the operator's CRDs + ServiceAccount + ClusterRole +
/// ClusterRoleBinding + Secret + Deployment via Harmony's
/// K8sResourceScore. The Secret carries both the `[credentials]` TOML
/// (consumed by the operator as `FLEET_OPERATOR_CREDENTIALS_TOML`) and
/// (consumed by the operator as [`OPERATOR_CREDENTIALS_ENV_VAR`]) and
/// the Zitadel JSON keyfile that the TOML's `key_path` references.
async fn deploy_operator(
topology: &K8sAnywhereTopology,
@@ -644,6 +644,7 @@ async fn deploy_operator(
) -> Result<()> {
use harmony::modules::fleet::operator::crd::{Deployment as FleetDeployment, Device};
use harmony::modules::k8s::resource::K8sResourceScore;
use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR;
use harmony_fleet_deploy::operator::chart::{
ChartOptions, OperatorCredentials, RELEASE_NAME, build_cluster_role,
build_cluster_role_binding, build_operator_deployment, build_service_account,
@@ -653,7 +654,7 @@ async fn deploy_operator(
use kube::CustomResourceExt;
// Render the [credentials] TOML the operator pod consumes via the
// FLEET_OPERATOR_CREDENTIALS_TOML env var (sourced from a Secret
// OPERATOR_CREDENTIALS_ENV_VAR env var (sourced from a Secret
// key). The Zitadel JSON keyfile is embedded inline under
// `key_json`; the operator never sees a file. Triple-quoted TOML
// string keeps the JSON's `"`s untouched.

View File

@@ -1,6 +1,11 @@
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
/// Env var the operator reads its `[credentials]` TOML from. The
/// deploy crate's chart wires this as a `secretKeyRef`; the operator
/// runtime reads it at startup. Single source of truth for both sides.
pub const OPERATOR_CREDENTIALS_ENV_VAR: &str = "FLEET_OPERATOR_CREDENTIALS_TOML";
/// Externally-tagged credential definition shared between the fleet
/// agent and the fleet operator. The `type` field selects the variant;
/// each variant's other fields are flatly mixed into the
@@ -8,7 +13,7 @@ use std::path::PathBuf;
///
/// **Why one struct for both processes**: the agent reads this from
/// `/etc/fleet-agent/config.toml`; the operator reads it from a single
/// env var (`FLEET_OPERATOR_CREDENTIALS_TOML`) whose value is a TOML
/// env var ([`OPERATOR_CREDENTIALS_ENV_VAR`]) whose value is a TOML
/// snippet shaped exactly like the `[credentials]` table. Identical
/// deserialization, identical downstream code path. The only thing
/// that differs is the byte source.

View File

@@ -26,7 +26,7 @@ mod config;
mod credentials;
pub use agent_config::{AgentConfig, AgentSection, NatsSection, load_config};
pub use config::CredentialsSection;
pub use config::{CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR};
pub use credentials::{
ASSERTION_LIFETIME_SECS, CachedToken, CredentialSource, MachineKeyFile, NatsCredential,
TOKEN_REFRESH_LEEWAY_SECS, credential_source_from_config,

View File

@@ -36,6 +36,7 @@ use serde::Serialize;
use harmony::modules::application::helm::{HelmChart, HelmResourceKind};
use harmony::modules::fleet::operator::crd::{Deployment, Device};
use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR;
/// Inputs for chart generation. Default values are aimed at a
/// local-dev k3d install; override via the `chart` subcommand flags.
@@ -58,7 +59,7 @@ pub struct ChartOptions {
/// `RUST_LOG` value for the operator process.
pub log_level: String,
/// `[credentials]` TOML payload to inject as
/// `FLEET_OPERATOR_CREDENTIALS_TOML` via a Secret. `None` skips the
/// [`OPERATOR_CREDENTIALS_ENV_VAR`] via a Secret. `None` skips the
/// Secret entirely and lets the operator connect to NATS without
/// auth — only sensible when there's no callout in front of NATS.
pub credentials: Option<OperatorCredentials>,
@@ -78,7 +79,7 @@ pub struct ChartOptions {
/// The Zitadel JSON keyfile content is embedded inline inside the
/// TOML as the `key_json` field (multi-line triple-quoted string).
/// One env-var-from-Secret on the operator Pod
/// (`FLEET_OPERATOR_CREDENTIALS_TOML`) and we're done — no volume
/// ([`OPERATOR_CREDENTIALS_ENV_VAR`]) and we're done — no volume
/// mounts, OKD restricted-v2 SCC compatible.
#[derive(Debug, Clone, Serialize)]
pub struct OperatorCredentials {
@@ -186,12 +187,10 @@ pub fn build_chart(opts: &ChartOptions) -> Result<PathBuf> {
chart.add_resource(HelmResourceKind::ClusterRoleBinding(cluster_role_binding(
"{{ .Release.Namespace }}",
)));
// Secret intentionally NOT included in the on-disk helm chart —
// credentials are operator-environment-specific and out of scope
// for a redistributable chart. The e2e bring-up applies the Secret
// directly via `operator_secret()` (used as a `K8sResourceScore`)
// and the chart's Deployment expects the Secret to be present in
// the namespace at install time.
chart.add_resource(
HelmResourceKind::from_serializable("secret-credentials.yaml", &empty_credentials_secret())
.context("serializing credentials Secret")?,
);
chart.add_resource(HelmResourceKind::Deployment(operator_deployment(opts)));
let written = chart
@@ -200,6 +199,19 @@ pub fn build_chart(opts: &ChartOptions) -> Result<PathBuf> {
Ok(written)
}
/// Empty Secret resource for the chart — helm creates it, the deploy
/// script (or user) provisions the actual `credentials.toml` data.
fn empty_credentials_secret() -> Secret {
Secret {
metadata: ObjectMeta {
name: Some(SECRET_NAME.to_string()),
..Default::default()
},
type_: Some("Opaque".to_string()),
..Default::default()
}
}
/// Build the operator's Secret holding the `[credentials]` TOML
/// (with the JSON keyfile inlined under `key_json`). Returns `None`
/// when no credentials are configured (no-auth dev mode).
@@ -344,26 +356,24 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
},
];
if opts.credentials.is_some() {
// The whole `[credentials]` TOML payload — including the JSON
// keyfile inlined under `key_json` — travels as a single env
// var sourced from a Secret key. No volume mounts: keeps the
// pod compatible with OKD's restricted-v2 SCC and the
// `harmony_fleet_auth::CredentialsSection` deserializer
// handles inline-vs-file from the same TOML shape.
env.push(EnvVar {
name: "FLEET_OPERATOR_CREDENTIALS_TOML".to_string(),
value_from: Some(EnvVarSource {
secret_key_ref: Some(SecretKeySelector {
name: SECRET_NAME.to_string(),
key: SECRET_KEY_CREDENTIALS_TOML.to_string(),
optional: Some(false),
}),
..Default::default()
// The whole `[credentials]` TOML payload — including the JSON
// keyfile inlined under `key_json` — travels as a single env
// var sourced from a Secret key. No volume mounts: keeps the
// pod compatible with OKD's restricted-v2 SCC and the
// `harmony_fleet_auth::CredentialsSection` deserializer
// handles inline-vs-file from the same TOML shape.
env.push(EnvVar {
name: OPERATOR_CREDENTIALS_ENV_VAR.to_string(),
value_from: Some(EnvVarSource {
secret_key_ref: Some(SecretKeySelector {
name: SECRET_NAME.to_string(),
key: SECRET_KEY_CREDENTIALS_TOML.to_string(),
optional: Some(true),
}),
..Default::default()
});
}
}),
..Default::default()
});
// Namespace deliberately omitted — same rationale as the
// ServiceAccount: helm fills in the release namespace at install
@@ -499,4 +509,24 @@ mod tests {
// image/SCC negotiate.
assert!(sc.run_as_user.is_none());
}
#[test]
fn chart_includes_credentials_secret_and_env_var() {
let tmp = tempfile::tempdir().unwrap();
let chart_path = build_chart(&ChartOptions {
output_dir: tmp.path().to_path_buf(),
..Default::default()
})
.unwrap();
let secret_yaml =
std::fs::read_to_string(chart_path.join("templates/secret-credentials.yaml"))
.expect("secret-credentials.yaml must exist in chart");
assert!(secret_yaml.contains(SECRET_NAME));
let deployment_yaml = std::fs::read_to_string(chart_path.join("templates/deployment.yaml"))
.expect("deployment.yaml must exist in chart");
assert!(deployment_yaml.contains(OPERATOR_CREDENTIALS_ENV_VAR));
assert!(deployment_yaml.contains(SECRET_NAME));
}
}

View File

@@ -9,7 +9,8 @@ use anyhow::{Context, Result};
use async_nats::jetstream;
use clap::{Parser, Subcommand};
use harmony_fleet_auth::{
CredentialsSection, connect_options_with_credentials, credential_source_from_config,
CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR, connect_options_with_credentials,
credential_source_from_config,
};
use harmony_reconciler_contracts::BUCKET_DESIRED_STATE;
use kube::Client;
@@ -48,7 +49,7 @@ struct Cli {
/// (for local dev without a callout-protected NATS).
#[arg(
long,
env = "FLEET_OPERATOR_CREDENTIALS_TOML",
env = OPERATOR_CREDENTIALS_ENV_VAR,
default_value = "",
global = true
)]
@@ -187,7 +188,7 @@ async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()>
/// against the NATS server becoming fully ready.
///
/// `credentials_toml` is the in-memory `[credentials]` TOML snippet
/// the operator's pod gets via the `FLEET_OPERATOR_CREDENTIALS_TOML`
/// the operator's pod gets via the [`OPERATOR_CREDENTIALS_ENV_VAR`]
/// env var (sourced from a Kubernetes Secret). Same shape as the
/// agent's `[credentials]` table; same factory; same auth callback.
/// Empty string means bypass — connect with no creds (only useful
@@ -197,7 +198,7 @@ async fn connect_with_retry(nats_url: &str, credentials_toml: &str) -> Result<as
for attempt in 0..15 {
let attempt_result = if credentials_toml.is_empty() {
tracing::warn!(
"FLEET_OPERATOR_CREDENTIALS_TOML is empty — connecting to NATS \
"{OPERATOR_CREDENTIALS_ENV_VAR} is empty — connecting to NATS \
without auth. Production deploys MUST mount a credentials Secret."
);
async_nats::connect(nats_url)
@@ -222,8 +223,8 @@ async fn connect_with_credentials(
nats_url: &str,
credentials_toml: &str,
) -> Result<async_nats::Client> {
let creds_section: CredentialsSection =
toml::from_str(credentials_toml).context("parsing FLEET_OPERATOR_CREDENTIALS_TOML")?;
let creds_section: CredentialsSection = toml::from_str(credentials_toml)
.with_context(|| format!("parsing {OPERATOR_CREDENTIALS_ENV_VAR}"))?;
let creds = credential_source_from_config(&creds_section)
.context("constructing CredentialSource from operator credentials")?;
let client = connect_options_with_credentials(creds)