chore: Refactor FLEET_OPERATOR_CREDENTIALS string that appeared in multiple places into a constant shared across all consumers #320
1
Cargo.lock
generated
1
Cargo.lock
generated
@@ -2861,6 +2861,7 @@ dependencies = [
|
|||||||
"example-fleet-auth-callout",
|
"example-fleet-auth-callout",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"harmony",
|
"harmony",
|
||||||
|
"harmony-fleet-auth",
|
||||||
"harmony-fleet-deploy",
|
"harmony-fleet-deploy",
|
||||||
"harmony-fleet-operator",
|
"harmony-fleet-operator",
|
||||||
"harmony-k8s",
|
"harmony-k8s",
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ harmony_types = { path = "../../harmony_types" }
|
|||||||
example-fleet-auth-callout = { path = "../fleet_auth_callout" }
|
example-fleet-auth-callout = { path = "../fleet_auth_callout" }
|
||||||
harmony-nats-callout = { path = "../../nats/callout" }
|
harmony-nats-callout = { path = "../../nats/callout" }
|
||||||
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
||||||
|
harmony-fleet-auth = { path = "../../fleet/harmony-fleet-auth" }
|
||||||
harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
|
harmony-fleet-operator = { path = "../../fleet/harmony-fleet-operator" }
|
||||||
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
|
harmony-fleet-deploy = { path = "../../fleet/harmony-fleet-deploy" }
|
||||||
k3d-rs = { path = "../../k3d" }
|
k3d-rs = { path = "../../k3d" }
|
||||||
|
|||||||
@@ -635,7 +635,7 @@ async fn build_and_load_operator_image(k3d: &k3d_rs::K3d) -> Result<()> {
|
|||||||
/// Apply the operator's CRDs + ServiceAccount + ClusterRole +
|
/// Apply the operator's CRDs + ServiceAccount + ClusterRole +
|
||||||
/// ClusterRoleBinding + Secret + Deployment via Harmony's
|
/// ClusterRoleBinding + Secret + Deployment via Harmony's
|
||||||
/// K8sResourceScore. The Secret carries both the `[credentials]` TOML
|
/// K8sResourceScore. The Secret carries both the `[credentials]` TOML
|
||||||
/// (consumed by the operator as `FLEET_OPERATOR_CREDENTIALS_TOML`) and
|
/// (consumed by the operator as [`OPERATOR_CREDENTIALS_ENV_VAR`]) and
|
||||||
/// the Zitadel JSON keyfile that the TOML's `key_path` references.
|
/// the Zitadel JSON keyfile that the TOML's `key_path` references.
|
||||||
async fn deploy_operator(
|
async fn deploy_operator(
|
||||||
topology: &K8sAnywhereTopology,
|
topology: &K8sAnywhereTopology,
|
||||||
@@ -644,6 +644,7 @@ async fn deploy_operator(
|
|||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
use harmony::modules::fleet::operator::crd::{Deployment as FleetDeployment, Device};
|
use harmony::modules::fleet::operator::crd::{Deployment as FleetDeployment, Device};
|
||||||
use harmony::modules::k8s::resource::K8sResourceScore;
|
use harmony::modules::k8s::resource::K8sResourceScore;
|
||||||
|
use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR;
|
||||||
use harmony_fleet_deploy::operator::chart::{
|
use harmony_fleet_deploy::operator::chart::{
|
||||||
ChartOptions, OperatorCredentials, RELEASE_NAME, build_cluster_role,
|
ChartOptions, OperatorCredentials, RELEASE_NAME, build_cluster_role,
|
||||||
build_cluster_role_binding, build_operator_deployment, build_service_account,
|
build_cluster_role_binding, build_operator_deployment, build_service_account,
|
||||||
@@ -653,7 +654,7 @@ async fn deploy_operator(
|
|||||||
use kube::CustomResourceExt;
|
use kube::CustomResourceExt;
|
||||||
|
|
||||||
// Render the [credentials] TOML the operator pod consumes via the
|
// Render the [credentials] TOML the operator pod consumes via the
|
||||||
// FLEET_OPERATOR_CREDENTIALS_TOML env var (sourced from a Secret
|
// OPERATOR_CREDENTIALS_ENV_VAR env var (sourced from a Secret
|
||||||
// key). The Zitadel JSON keyfile is embedded inline under
|
// key). The Zitadel JSON keyfile is embedded inline under
|
||||||
// `key_json`; the operator never sees a file. Triple-quoted TOML
|
// `key_json`; the operator never sees a file. Triple-quoted TOML
|
||||||
// string keeps the JSON's `"`s untouched.
|
// string keeps the JSON's `"`s untouched.
|
||||||
|
|||||||
@@ -1,6 +1,11 @@
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
/// Env var the operator reads its `[credentials]` TOML from. The
|
||||||
|
/// deploy crate's chart wires this as a `secretKeyRef`; the operator
|
||||||
|
/// runtime reads it at startup. Single source of truth for both sides.
|
||||||
|
pub const OPERATOR_CREDENTIALS_ENV_VAR: &str = "FLEET_OPERATOR_CREDENTIALS_TOML";
|
||||||
|
|
||||||
/// Externally-tagged credential definition shared between the fleet
|
/// Externally-tagged credential definition shared between the fleet
|
||||||
/// agent and the fleet operator. The `type` field selects the variant;
|
/// agent and the fleet operator. The `type` field selects the variant;
|
||||||
/// each variant's other fields are flatly mixed into the
|
/// each variant's other fields are flatly mixed into the
|
||||||
@@ -8,7 +13,7 @@ use std::path::PathBuf;
|
|||||||
///
|
///
|
||||||
/// **Why one struct for both processes**: the agent reads this from
|
/// **Why one struct for both processes**: the agent reads this from
|
||||||
/// `/etc/fleet-agent/config.toml`; the operator reads it from a single
|
/// `/etc/fleet-agent/config.toml`; the operator reads it from a single
|
||||||
/// env var (`FLEET_OPERATOR_CREDENTIALS_TOML`) whose value is a TOML
|
/// env var ([`OPERATOR_CREDENTIALS_ENV_VAR`]) whose value is a TOML
|
||||||
/// snippet shaped exactly like the `[credentials]` table. Identical
|
/// snippet shaped exactly like the `[credentials]` table. Identical
|
||||||
/// deserialization, identical downstream code path. The only thing
|
/// deserialization, identical downstream code path. The only thing
|
||||||
/// that differs is the byte source.
|
/// that differs is the byte source.
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ mod config;
|
|||||||
mod credentials;
|
mod credentials;
|
||||||
|
|
||||||
pub use agent_config::{AgentConfig, AgentSection, NatsSection, load_config};
|
pub use agent_config::{AgentConfig, AgentSection, NatsSection, load_config};
|
||||||
pub use config::CredentialsSection;
|
pub use config::{CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR};
|
||||||
pub use credentials::{
|
pub use credentials::{
|
||||||
ASSERTION_LIFETIME_SECS, CachedToken, CredentialSource, MachineKeyFile, NatsCredential,
|
ASSERTION_LIFETIME_SECS, CachedToken, CredentialSource, MachineKeyFile, NatsCredential,
|
||||||
TOKEN_REFRESH_LEEWAY_SECS, credential_source_from_config,
|
TOKEN_REFRESH_LEEWAY_SECS, credential_source_from_config,
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ use serde::Serialize;
|
|||||||
|
|
||||||
use harmony::modules::application::helm::{HelmChart, HelmResourceKind};
|
use harmony::modules::application::helm::{HelmChart, HelmResourceKind};
|
||||||
use harmony::modules::fleet::operator::crd::{Deployment, Device};
|
use harmony::modules::fleet::operator::crd::{Deployment, Device};
|
||||||
|
use harmony_fleet_auth::OPERATOR_CREDENTIALS_ENV_VAR;
|
||||||
|
|
||||||
/// Inputs for chart generation. Default values are aimed at a
|
/// Inputs for chart generation. Default values are aimed at a
|
||||||
/// local-dev k3d install; override via the `chart` subcommand flags.
|
/// local-dev k3d install; override via the `chart` subcommand flags.
|
||||||
@@ -58,7 +59,7 @@ pub struct ChartOptions {
|
|||||||
/// `RUST_LOG` value for the operator process.
|
/// `RUST_LOG` value for the operator process.
|
||||||
pub log_level: String,
|
pub log_level: String,
|
||||||
/// `[credentials]` TOML payload to inject as
|
/// `[credentials]` TOML payload to inject as
|
||||||
/// `FLEET_OPERATOR_CREDENTIALS_TOML` via a Secret. `None` skips the
|
/// [`OPERATOR_CREDENTIALS_ENV_VAR`] via a Secret. `None` skips the
|
||||||
/// Secret entirely and lets the operator connect to NATS without
|
/// Secret entirely and lets the operator connect to NATS without
|
||||||
/// auth — only sensible when there's no callout in front of NATS.
|
/// auth — only sensible when there's no callout in front of NATS.
|
||||||
pub credentials: Option<OperatorCredentials>,
|
pub credentials: Option<OperatorCredentials>,
|
||||||
@@ -78,7 +79,7 @@ pub struct ChartOptions {
|
|||||||
/// The Zitadel JSON keyfile content is embedded inline inside the
|
/// The Zitadel JSON keyfile content is embedded inline inside the
|
||||||
/// TOML as the `key_json` field (multi-line triple-quoted string).
|
/// TOML as the `key_json` field (multi-line triple-quoted string).
|
||||||
/// One env-var-from-Secret on the operator Pod
|
/// One env-var-from-Secret on the operator Pod
|
||||||
/// (`FLEET_OPERATOR_CREDENTIALS_TOML`) and we're done — no volume
|
/// ([`OPERATOR_CREDENTIALS_ENV_VAR`]) and we're done — no volume
|
||||||
/// mounts, OKD restricted-v2 SCC compatible.
|
/// mounts, OKD restricted-v2 SCC compatible.
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub struct OperatorCredentials {
|
pub struct OperatorCredentials {
|
||||||
@@ -186,12 +187,10 @@ pub fn build_chart(opts: &ChartOptions) -> Result<PathBuf> {
|
|||||||
chart.add_resource(HelmResourceKind::ClusterRoleBinding(cluster_role_binding(
|
chart.add_resource(HelmResourceKind::ClusterRoleBinding(cluster_role_binding(
|
||||||
"{{ .Release.Namespace }}",
|
"{{ .Release.Namespace }}",
|
||||||
)));
|
)));
|
||||||
// Secret intentionally NOT included in the on-disk helm chart —
|
chart.add_resource(
|
||||||
// credentials are operator-environment-specific and out of scope
|
HelmResourceKind::from_serializable("secret-credentials.yaml", &empty_credentials_secret())
|
||||||
// for a redistributable chart. The e2e bring-up applies the Secret
|
.context("serializing credentials Secret")?,
|
||||||
// directly via `operator_secret()` (used as a `K8sResourceScore`)
|
);
|
||||||
// and the chart's Deployment expects the Secret to be present in
|
|
||||||
// the namespace at install time.
|
|
||||||
chart.add_resource(HelmResourceKind::Deployment(operator_deployment(opts)));
|
chart.add_resource(HelmResourceKind::Deployment(operator_deployment(opts)));
|
||||||
|
|
||||||
let written = chart
|
let written = chart
|
||||||
@@ -200,6 +199,19 @@ pub fn build_chart(opts: &ChartOptions) -> Result<PathBuf> {
|
|||||||
Ok(written)
|
Ok(written)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Empty Secret resource for the chart — helm creates it, the deploy
|
||||||
|
/// script (or user) provisions the actual `credentials.toml` data.
|
||||||
|
fn empty_credentials_secret() -> Secret {
|
||||||
|
Secret {
|
||||||
|
metadata: ObjectMeta {
|
||||||
|
name: Some(SECRET_NAME.to_string()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
type_: Some("Opaque".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Build the operator's Secret holding the `[credentials]` TOML
|
/// Build the operator's Secret holding the `[credentials]` TOML
|
||||||
/// (with the JSON keyfile inlined under `key_json`). Returns `None`
|
/// (with the JSON keyfile inlined under `key_json`). Returns `None`
|
||||||
/// when no credentials are configured (no-auth dev mode).
|
/// when no credentials are configured (no-auth dev mode).
|
||||||
@@ -344,26 +356,24 @@ fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
|
|||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
if opts.credentials.is_some() {
|
// The whole `[credentials]` TOML payload — including the JSON
|
||||||
// The whole `[credentials]` TOML payload — including the JSON
|
// keyfile inlined under `key_json` — travels as a single env
|
||||||
// keyfile inlined under `key_json` — travels as a single env
|
// var sourced from a Secret key. No volume mounts: keeps the
|
||||||
// var sourced from a Secret key. No volume mounts: keeps the
|
// pod compatible with OKD's restricted-v2 SCC and the
|
||||||
// pod compatible with OKD's restricted-v2 SCC and the
|
// `harmony_fleet_auth::CredentialsSection` deserializer
|
||||||
// `harmony_fleet_auth::CredentialsSection` deserializer
|
// handles inline-vs-file from the same TOML shape.
|
||||||
// handles inline-vs-file from the same TOML shape.
|
env.push(EnvVar {
|
||||||
env.push(EnvVar {
|
name: OPERATOR_CREDENTIALS_ENV_VAR.to_string(),
|
||||||
name: "FLEET_OPERATOR_CREDENTIALS_TOML".to_string(),
|
value_from: Some(EnvVarSource {
|
||||||
value_from: Some(EnvVarSource {
|
secret_key_ref: Some(SecretKeySelector {
|
||||||
secret_key_ref: Some(SecretKeySelector {
|
name: SECRET_NAME.to_string(),
|
||||||
name: SECRET_NAME.to_string(),
|
key: SECRET_KEY_CREDENTIALS_TOML.to_string(),
|
||||||
key: SECRET_KEY_CREDENTIALS_TOML.to_string(),
|
optional: Some(true),
|
||||||
optional: Some(false),
|
|
||||||
}),
|
|
||||||
..Default::default()
|
|
||||||
}),
|
}),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
});
|
}),
|
||||||
}
|
..Default::default()
|
||||||
|
});
|
||||||
|
|
||||||
// Namespace deliberately omitted — same rationale as the
|
// Namespace deliberately omitted — same rationale as the
|
||||||
// ServiceAccount: helm fills in the release namespace at install
|
// ServiceAccount: helm fills in the release namespace at install
|
||||||
@@ -499,4 +509,24 @@ mod tests {
|
|||||||
// image/SCC negotiate.
|
// image/SCC negotiate.
|
||||||
assert!(sc.run_as_user.is_none());
|
assert!(sc.run_as_user.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chart_includes_credentials_secret_and_env_var() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let chart_path = build_chart(&ChartOptions {
|
||||||
|
output_dir: tmp.path().to_path_buf(),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let secret_yaml =
|
||||||
|
std::fs::read_to_string(chart_path.join("templates/secret-credentials.yaml"))
|
||||||
|
.expect("secret-credentials.yaml must exist in chart");
|
||||||
|
assert!(secret_yaml.contains(SECRET_NAME));
|
||||||
|
|
||||||
|
let deployment_yaml = std::fs::read_to_string(chart_path.join("templates/deployment.yaml"))
|
||||||
|
.expect("deployment.yaml must exist in chart");
|
||||||
|
assert!(deployment_yaml.contains(OPERATOR_CREDENTIALS_ENV_VAR));
|
||||||
|
assert!(deployment_yaml.contains(SECRET_NAME));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,8 @@ use anyhow::{Context, Result};
|
|||||||
use async_nats::jetstream;
|
use async_nats::jetstream;
|
||||||
use clap::{Parser, Subcommand};
|
use clap::{Parser, Subcommand};
|
||||||
use harmony_fleet_auth::{
|
use harmony_fleet_auth::{
|
||||||
CredentialsSection, connect_options_with_credentials, credential_source_from_config,
|
CredentialsSection, OPERATOR_CREDENTIALS_ENV_VAR, connect_options_with_credentials,
|
||||||
|
credential_source_from_config,
|
||||||
};
|
};
|
||||||
use harmony_reconciler_contracts::BUCKET_DESIRED_STATE;
|
use harmony_reconciler_contracts::BUCKET_DESIRED_STATE;
|
||||||
use kube::Client;
|
use kube::Client;
|
||||||
@@ -48,7 +49,7 @@ struct Cli {
|
|||||||
/// (for local dev without a callout-protected NATS).
|
/// (for local dev without a callout-protected NATS).
|
||||||
#[arg(
|
#[arg(
|
||||||
long,
|
long,
|
||||||
env = "FLEET_OPERATOR_CREDENTIALS_TOML",
|
env = OPERATOR_CREDENTIALS_ENV_VAR,
|
||||||
default_value = "",
|
default_value = "",
|
||||||
global = true
|
global = true
|
||||||
)]
|
)]
|
||||||
@@ -187,7 +188,7 @@ async fn run(nats_url: &str, bucket: &str, credentials_toml: &str) -> Result<()>
|
|||||||
/// against the NATS server becoming fully ready.
|
/// against the NATS server becoming fully ready.
|
||||||
///
|
///
|
||||||
/// `credentials_toml` is the in-memory `[credentials]` TOML snippet
|
/// `credentials_toml` is the in-memory `[credentials]` TOML snippet
|
||||||
/// the operator's pod gets via the `FLEET_OPERATOR_CREDENTIALS_TOML`
|
/// the operator's pod gets via the [`OPERATOR_CREDENTIALS_ENV_VAR`]
|
||||||
/// env var (sourced from a Kubernetes Secret). Same shape as the
|
/// env var (sourced from a Kubernetes Secret). Same shape as the
|
||||||
/// agent's `[credentials]` table; same factory; same auth callback.
|
/// agent's `[credentials]` table; same factory; same auth callback.
|
||||||
/// Empty string means bypass — connect with no creds (only useful
|
/// Empty string means bypass — connect with no creds (only useful
|
||||||
@@ -197,7 +198,7 @@ async fn connect_with_retry(nats_url: &str, credentials_toml: &str) -> Result<as
|
|||||||
for attempt in 0..15 {
|
for attempt in 0..15 {
|
||||||
let attempt_result = if credentials_toml.is_empty() {
|
let attempt_result = if credentials_toml.is_empty() {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
"FLEET_OPERATOR_CREDENTIALS_TOML is empty — connecting to NATS \
|
"{OPERATOR_CREDENTIALS_ENV_VAR} is empty — connecting to NATS \
|
||||||
without auth. Production deploys MUST mount a credentials Secret."
|
without auth. Production deploys MUST mount a credentials Secret."
|
||||||
);
|
);
|
||||||
async_nats::connect(nats_url)
|
async_nats::connect(nats_url)
|
||||||
@@ -222,8 +223,8 @@ async fn connect_with_credentials(
|
|||||||
nats_url: &str,
|
nats_url: &str,
|
||||||
credentials_toml: &str,
|
credentials_toml: &str,
|
||||||
) -> Result<async_nats::Client> {
|
) -> Result<async_nats::Client> {
|
||||||
let creds_section: CredentialsSection =
|
let creds_section: CredentialsSection = toml::from_str(credentials_toml)
|
||||||
toml::from_str(credentials_toml).context("parsing FLEET_OPERATOR_CREDENTIALS_TOML")?;
|
.with_context(|| format!("parsing {OPERATOR_CREDENTIALS_ENV_VAR}"))?;
|
||||||
let creds = credential_source_from_config(&creds_section)
|
let creds = credential_source_from_config(&creds_section)
|
||||||
.context("constructing CredentialSource from operator credentials")?;
|
.context("constructing CredentialSource from operator credentials")?;
|
||||||
let client = connect_options_with_credentials(creds)
|
let client = connect_options_with_credentials(creds)
|
||||||
|
|||||||
Reference in New Issue
Block a user