chore: Refactor FLEET_OPERATOR_CREDENTIALS string that appeared in multiple places into a constant shared across all consumers #320
1
Cargo.lock
generated
1
Cargo.lock
generated
@@ -2861,6 +2861,7 @@ dependencies = [
|
||||
"example-fleet-auth-callout",
|
||||
"futures-util",
|
||||
"harmony",
|
||||
"harmony-fleet-auth",
|
||||
"harmony-fleet-deploy",
|
||||
"harmony-fleet-operator",
|
||||
"harmony-k8s",
|
||||
|
||||
@@ -25,6 +25,7 @@ harmony_types = { path = "../../harmony_types" }
|
||||
example-fleet-auth-callout = { path = "../fleet_auth_callout" }
|
||||
harmony-nats-callout = { path = "../../nats/callout" }
|
||||
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
||||
harmony-fleet-auth = { path = "../../fleet/harmony-fleet-auth" }
|
||||
harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
|
||||
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
|
||||
k3d-rs = { path = "../../k3d" }
|
||||
|
||||
@@ -635,7 +635,7 @@ async fn build_and_load_operator_image(k3d: &k3d_rs::K3d) -> Result<()> {
|
||||
/// Apply the operator's CRDs + ServiceAccount + ClusterRole +
|
||||
/// ClusterRoleBinding + Secret + Deployment via Harmony's
|
||||
/// K8sResourceScore. The Secret carries both the `[credentials]` TOML
|
||||
/// (consumed by the operator as `FLEET_OPERATOR_CREDENTIALS_TOML`) and
|
||||
/// (consumed by the operator as [`OPERATOR_CREDENTIALS_ENV_VAR`]) and
|
||||
/// the Zitadel JSON keyfile that the TOML's `key_path` references.
|
||||
async fn deploy_operator(
|
||||
topology: &K8sAnywhereTopology,
|
||||
@@ -644,6 +644,7 @@ async fn deploy_operator(
|
||||
) -> Result<()> {
|
||||
use harmony::modules::fleet::operator::crd::{Deployment as FleetDeployment, Device};
|
||||
use harmony::modules::k8s::resource::K8sResourceScore;
|
||||
use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR;
|
||||
use harmony_fleet_deploy::operator::chart::{
|
||||
ChartOptions, OperatorCredentials, RELEASE_NAME, build_cluster_role,
|
||||
build_cluster_role_binding, build_operator_deployment, build_service_account,
|
||||
@@ -653,7 +654,7 @@ async fn deploy_operator(
|
||||
use kube::CustomResourceExt;
|
||||
|
||||
// Render the [credentials] TOML the operator pod consumes via the
|
||||
// FLEET_OPERATOR_CREDENTIALS_TOML env var (sourced from a Secret
|
||||
// OPERATOR_CREDENTIALS_ENV_VAR env var (sourced from a Secret
|
||||
// key). The Zitadel JSON keyfile is embedded inline under
|
||||
// `key_json`; the operator never sees a file. Triple-quoted TOML
|
||||
// string keeps the JSON's `"`s untouched.
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Env var the operator reads its `[credentials]` TOML from. The
|
||||
/// deploy crate's chart wires this as a `secretKeyRef`; the operator
|
||||
/// runtime reads it at startup. Single source of truth for both sides.
|
||||
pub const OPERATOR_CREDENTIALS_ENV_VAR: &str = "FLEET_OPERATOR_CREDENTIALS_TOML";
|
||||
|
||||
/// Externally-tagged credential definition shared between the fleet
|
||||
/// agent and the fleet operator. The `type` field selects the variant;
|
||||
/// each variant's other fields are flatly mixed into the
|
||||
@@ -8,7 +13,7 @@ use std::path::PathBuf;
|
||||
///
|
||||
/// **Why one struct for both processes**: the agent reads this from
|
||||
/// `/etc/fleet-agent/config.toml`; the operator reads it from a single
|
||||
/// env var (`FLEET_OPERATOR_CREDENTIALS_TOML`) whose value is a TOML
|
||||
/// env var ([`OPERATOR_CREDENTIALS_ENV_VAR`]) whose value is a TOML
|
||||
/// snippet shaped exactly like the `[credentials]` table. Identical
|
||||
/// deserialization, identical downstream code path. The only thing
|
||||
/// that differs is the byte source.
|
||||
|
||||
@@ -26,7 +26,7 @@ mod config;
|
||||
mod credentials;
|
||||
|
||||
pub use agent_config::{AgentConfig, AgentSection, NatsSection, load_config};
|
||||
pub use config::CredentialsSection;
|
||||
pub use config::{CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR};
|
||||
pub use credentials::{
|
||||
ASSERTION_LIFETIME_SECS, CachedToken, CredentialSource, MachineKeyFile, NatsCredential,
|
||||
TOKEN_REFRESH_LEEWAY_SECS, credential_source_from_config,
|
||||
|
||||
@@ -36,6 +36,7 @@ use serde::Serialize;
|
||||
|
||||
use harmony::modules::application::helm::{HelmChart, HelmResourceKind};
|
||||
use harmony::modules::fleet::operator::crd::{Deployment, Device};
|
||||
use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR;
|
||||
|
||||
/// Inputs for chart generation. Default values are aimed at a
|
||||
/// local-dev k3d install; override via the `chart` subcommand flags.
|
||||
@@ -58,7 +59,7 @@ pub struct ChartOptions {
|
||||
/// `RUST_LOG` value for the operator process.
|
||||
pub log_level: String,
|
||||
/// `[credentials]` TOML payload to inject as
|
||||
/// `FLEET_OPERATOR_CREDENTIALS_TOML` via a Secret. `None` skips the
|
||||
/// [`OPERATOR_CREDENTIALS_ENV_VAR`] via a Secret. `None` skips the
|
||||
/// Secret entirely and lets the operator connect to NATS without
|
||||
/// auth — only sensible when there's no callout in front of NATS.
|
||||
pub credentials: Option<OperatorCredentials>,
|
||||
@@ -78,7 +79,7 @@ pub struct ChartOptions {
|
||||
/// The Zitadel JSON keyfile content is embedded inline inside the
|
||||
/// TOML as the `key_json` field (multi-line triple-quoted string).
|
||||
/// One env-var-from-Secret on the operator Pod
|
||||
/// (`FLEET_OPERATOR_CREDENTIALS_TOML`) and we're done — no volume
|
||||
/// ([`OPERATOR_CREDENTIALS_ENV_VAR`]) and we're done — no volume
|
||||
/// mounts, OKD restricted-v2 SCC compatible.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct OperatorCredentials {
|
||||
@@ -186,12 +187,10 @@ pub fn build_chart(opts: &ChartOptions) -> Result<PathBuf> {
|
||||
chart.add_resource(HelmResourceKind::ClusterRoleBinding(cluster_role_binding(
|
||||
"{{ .Release.Namespace }}",
|
||||
)));
|
||||
// Secret intentionally NOT included in the on-disk helm chart —
|
||||
// credentials are operator-environment-specific and out of scope
|
||||
// for a redistributable chart. The e2e bring-up applies the Secret
|
||||
// directly via `operator_secret()` (used as a `K8sResourceScore`)
|
||||
// and the chart's Deployment expects the Secret to be present in
|
||||
// the namespace at install time.
|
||||
chart.add_resource(
|
||||
HelmResourceKind::from_serializable("secret-credentials.yaml", &empty_credentials_secret())
|
||||
.context("serializing credentials Secret")?,
|
||||
);
|
||||
chart.add_resource(HelmResourceKind::Deployment(operator_deployment(opts)));
|
||||
|
||||
let written = chart
|
||||
@@ -200,6 +199,19 @@ pub fn build_chart(opts: &ChartOptions) -> Result<PathBuf> {
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
/// Empty Secret resource for the chart — helm creates it, the deploy
|
||||
/// script (or user) provisions the actual `credentials.toml` data.
|
||||
fn empty_credentials_secret() -> Secret {
|
||||
Secret {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(SECRET_NAME.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
type_: Some("Opaque".to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the operator's Secret holding the `[credentials]` TOML
|
||||
/// (with the JSON keyfile inlined under `key_json`). Returns `None`
|
||||
/// when no credentials are configured (no-auth dev mode).
|
||||
@@ -344,26 +356,24 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
|
||||
},
|
||||
];
|
||||
|
||||
if opts.credentials.is_some() {
|
||||
// The whole `[credentials]` TOML payload — including the JSON
|
||||
// keyfile inlined under `key_json` — travels as a single env
|
||||
// var sourced from a Secret key. No volume mounts: keeps the
|
||||
// pod compatible with OKD's restricted-v2 SCC and the
|
||||
// `harmony_fleet_auth::CredentialsSection` deserializer
|
||||
// handles inline-vs-file from the same TOML shape.
|
||||
env.push(EnvVar {
|
||||
name: "FLEET_OPERATOR_CREDENTIALS_TOML".to_string(),
|
||||
value_from: Some(EnvVarSource {
|
||||
secret_key_ref: Some(SecretKeySelector {
|
||||
name: SECRET_NAME.to_string(),
|
||||
key: SECRET_KEY_CREDENTIALS_TOML.to_string(),
|
||||
optional: Some(false),
|
||||
}),
|
||||
..Default::default()
|
||||
// The whole `[credentials]` TOML payload — including the JSON
|
||||
// keyfile inlined under `key_json` — travels as a single env
|
||||
// var sourced from a Secret key. No volume mounts: keeps the
|
||||
// pod compatible with OKD's restricted-v2 SCC and the
|
||||
// `harmony_fleet_auth::CredentialsSection` deserializer
|
||||
// handles inline-vs-file from the same TOML shape.
|
||||
env.push(EnvVar {
|
||||
name: OPERATOR_CREDENTIALS_ENV_VAR.to_string(),
|
||||
value_from: Some(EnvVarSource {
|
||||
secret_key_ref: Some(SecretKeySelector {
|
||||
name: SECRET_NAME.to_string(),
|
||||
key: SECRET_KEY_CREDENTIALS_TOML.to_string(),
|
||||
optional: Some(true),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
// Namespace deliberately omitted — same rationale as the
|
||||
// ServiceAccount: helm fills in the release namespace at install
|
||||
@@ -499,4 +509,24 @@ mod tests {
|
||||
// image/SCC negotiate.
|
||||
assert!(sc.run_as_user.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chart_includes_credentials_secret_and_env_var() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let chart_path = build_chart(&ChartOptions {
|
||||
output_dir: tmp.path().to_path_buf(),
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let secret_yaml =
|
||||
std::fs::read_to_string(chart_path.join("templates/secret-credentials.yaml"))
|
||||
.expect("secret-credentials.yaml must exist in chart");
|
||||
assert!(secret_yaml.contains(SECRET_NAME));
|
||||
|
||||
let deployment_yaml = std::fs::read_to_string(chart_path.join("templates/deployment.yaml"))
|
||||
.expect("deployment.yaml must exist in chart");
|
||||
assert!(deployment_yaml.contains(OPERATOR_CREDENTIALS_ENV_VAR));
|
||||
assert!(deployment_yaml.contains(SECRET_NAME));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,8 @@ use anyhow::{Context, Result};
|
||||
use async_nats::jetstream;
|
||||
use clap::{Parser, Subcommand};
|
||||
use harmony_fleet_auth::{
|
||||
CredentialsSection, connect_options_with_credentials, credential_source_from_config,
|
||||
CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR, connect_options_with_credentials,
|
||||
credential_source_from_config,
|
||||
};
|
||||
use harmony_reconciler_contracts::BUCKET_DESIRED_STATE;
|
||||
use kube::Client;
|
||||
@@ -48,7 +49,7 @@ struct Cli {
|
||||
/// (for local dev without a callout-protected NATS).
|
||||
#[arg(
|
||||
long,
|
||||
env = "FLEET_OPERATOR_CREDENTIALS_TOML",
|
||||
env = OPERATOR_CREDENTIALS_ENV_VAR,
|
||||
default_value = "",
|
||||
global = true
|
||||
)]
|
||||
@@ -187,7 +188,7 @@ async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()>
|
||||
/// against the NATS server becoming fully ready.
|
||||
///
|
||||
/// `credentials_toml` is the in-memory `[credentials]` TOML snippet
|
||||
/// the operator's pod gets via the `FLEET_OPERATOR_CREDENTIALS_TOML`
|
||||
/// the operator's pod gets via the [`OPERATOR_CREDENTIALS_ENV_VAR`]
|
||||
/// env var (sourced from a Kubernetes Secret). Same shape as the
|
||||
/// agent's `[credentials]` table; same factory; same auth callback.
|
||||
/// Empty string means bypass — connect with no creds (only useful
|
||||
@@ -197,7 +198,7 @@ async fn connect_with_retry(nats_url: &str, credentials_toml: &str) -> Result<as
|
||||
for attempt in 0..15 {
|
||||
let attempt_result = if credentials_toml.is_empty() {
|
||||
tracing::warn!(
|
||||
"FLEET_OPERATOR_CREDENTIALS_TOML is empty — connecting to NATS \
|
||||
"{OPERATOR_CREDENTIALS_ENV_VAR} is empty — connecting to NATS \
|
||||
without auth. Production deploys MUST mount a credentials Secret."
|
||||
);
|
||||
async_nats::connect(nats_url)
|
||||
@@ -222,8 +223,8 @@ async fn connect_with_credentials(
|
||||
nats_url: &str,
|
||||
credentials_toml: &str,
|
||||
) -> Result<async_nats::Client> {
|
||||
let creds_section: CredentialsSection =
|
||||
toml::from_str(credentials_toml).context("parsing FLEET_OPERATOR_CREDENTIALS_TOML")?;
|
||||
let creds_section: CredentialsSection = toml::from_str(credentials_toml)
|
||||
.with_context(|| format!("parsing {OPERATOR_CREDENTIALS_ENV_VAR}"))?;
|
||||
let creds = credential_source_from_config(&creds_section)
|
||||
.context("constructing CredentialSource from operator credentials")?;
|
||||
let client = connect_options_with_credentials(creds)
|
||||
|
||||
Reference in New Issue
Block a user