feat(fleet): expose operator UI via cert-manager TLS ingress #321
1
Cargo.lock
generated
1
Cargo.lock
generated
@@ -4096,6 +4096,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"clap",
|
||||
"env_logger",
|
||||
"fqdn",
|
||||
"harmony",
|
||||
"harmony-fleet-auth",
|
||||
"harmony-reconciler-contracts",
|
||||
|
||||
@@ -33,6 +33,7 @@ harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
|
||||
anyhow = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
fqdn = "0.5.2"
|
||||
k8s-openapi = { workspace = true }
|
||||
kube = { workspace = true, features = ["runtime", "derive"] }
|
||||
log = { workspace = true }
|
||||
|
||||
@@ -111,11 +111,15 @@ async fn main() -> Result<()> {
|
||||
.operator_chart_project
|
||||
.unwrap_or(config.operator_chart_project);
|
||||
|
||||
// Coherent with the other staging hosts (sso-stg., secrets-stg.).
|
||||
let ui_host = format!("fleet-stg.{}", config.base_domain);
|
||||
|
||||
let operator = FleetOperatorScore::new()
|
||||
.namespace(namespace)
|
||||
.nats_url(config.nats_url)
|
||||
.credentials(secrets.operator_credentials_toml)
|
||||
.published_chart(registry, project, version);
|
||||
.published_chart(registry, project, version)
|
||||
.ingress(ui_host, Some(config.cluster_issuer));
|
||||
|
||||
harmony_cli::run(
|
||||
Inventory::autoload(),
|
||||
|
||||
@@ -25,11 +25,12 @@ use k8s_openapi::api::apps::v1::{
|
||||
};
|
||||
use k8s_openapi::api::core::v1::{
|
||||
Capabilities, Container, EnvVar, EnvVarSource, PodSpec, PodTemplateSpec, SeccompProfile,
|
||||
Secret, SecretKeySelector, SecurityContext, ServiceAccount,
|
||||
Secret, SecretKeySelector, SecurityContext, Service, ServiceAccount, ServicePort, ServiceSpec,
|
||||
};
|
||||
use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject};
|
||||
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
|
||||
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
|
||||
use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString;
|
||||
use kube::CustomResourceExt;
|
||||
use kube::api::ObjectMeta;
|
||||
use serde::Serialize;
|
||||
@@ -143,6 +144,11 @@ pub const SERVICE_ACCOUNT: &str = "harmony-fleet-operator";
|
||||
pub const CLUSTER_ROLE: &str = "harmony-fleet-operator";
|
||||
pub const CLUSTER_ROLE_BINDING: &str = "harmony-fleet-operator";
|
||||
pub const SECRET_NAME: &str = "harmony-fleet-operator-secrets";
|
||||
/// Port the operator UI listens on and the Service exposes. Mirrors the
|
||||
/// operator binary's bind default (`harmony-fleet-operator`'s
|
||||
/// `DEFAULT_PORT`); the operator is consumed here as a container image,
|
||||
/// not a crate dependency, so the value is restated at this boundary.
|
||||
pub const OPERATOR_HTTP_PORT: u16 = 18080;
|
||||
/// Single Secret key holding the entire `[credentials]` TOML —
|
||||
/// including the inline JSON keyfile under `key_json`.
|
||||
pub const SECRET_KEY_CREDENTIALS_TOML: &str = "credentials.toml";
|
||||
@@ -192,6 +198,10 @@ pub fn build_chart(opts: &ChartOptions) -> Result<PathBuf> {
|
||||
.context("serializing credentials Secret")?,
|
||||
);
|
||||
chart.add_resource(HelmResourceKind::Deployment(operator_deployment(opts)));
|
||||
chart.add_resource(
|
||||
HelmResourceKind::from_serializable("service.yaml", &operator_service())
|
||||
.context("serializing operator Service")?,
|
||||
);
|
||||
|
||||
let written = chart
|
||||
.write_to(Path::new(&opts.output_dir))
|
||||
@@ -336,6 +346,34 @@ fn cluster_role_binding(namespace: &str) -> ClusterRoleBinding {
|
||||
}
|
||||
}
|
||||
|
||||
/// ClusterIP Service fronting the operator UI — the Ingress backend.
|
||||
/// Namespace-neutral like the Deployment (helm/direct-apply assigns it).
|
||||
fn operator_service() -> Service {
|
||||
let mut match_labels = BTreeMap::new();
|
||||
match_labels.insert(
|
||||
"app.kubernetes.io/name".to_string(),
|
||||
RELEASE_NAME.to_string(),
|
||||
);
|
||||
Service {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(RELEASE_NAME.to_string()),
|
||||
labels: Some(match_labels.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
spec: Some(ServiceSpec {
|
||||
selector: Some(match_labels),
|
||||
ports: Some(vec![ServicePort {
|
||||
name: Some("http".to_string()),
|
||||
port: OPERATOR_HTTP_PORT as i32,
|
||||
target_port: Some(IntOrString::Int(OPERATOR_HTTP_PORT as i32)),
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
|
||||
let mut match_labels = BTreeMap::new();
|
||||
match_labels.insert(
|
||||
|
||||
@@ -34,6 +34,7 @@ use harmony::data::Version;
|
||||
use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
|
||||
use harmony::inventory::Inventory;
|
||||
use harmony::modules::helm::chart::{HelmChartScore, NonBlankString};
|
||||
use harmony::modules::k8s::ingress::K8sIngressScore;
|
||||
use harmony::modules::k8s::resource::K8sResourceScore;
|
||||
use harmony::score::Score;
|
||||
use harmony::topology::{HelmCommand, K8sclient, Topology};
|
||||
@@ -71,6 +72,12 @@ pub struct FleetOperatorScore {
|
||||
/// `None` renders + installs the chart from local source (dev/e2e);
|
||||
/// `Some` installs the published OCI chart at a pinned version (CD).
|
||||
pub published_chart: Option<PublishedChart>,
|
||||
/// `Some(host)` exposes the UI via an Ingress after install (CD);
|
||||
/// `None` leaves it cluster-internal (dev/e2e harnesses).
|
||||
pub operator_ui_host: Option<String>,
|
||||
/// cert-manager `ClusterIssuer` for the UI Ingress. `None` (or no
|
||||
/// host) serves plain HTTP — the right default on issuer-less k3d.
|
||||
pub cluster_issuer: Option<String>,
|
||||
}
|
||||
|
||||
impl FleetOperatorScore {
|
||||
@@ -89,9 +96,20 @@ impl FleetOperatorScore {
|
||||
log_level: defaults.log_level,
|
||||
credentials: defaults.credentials,
|
||||
published_chart: None,
|
||||
operator_ui_host: None,
|
||||
cluster_issuer: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Expose the operator UI via an Ingress at `host` after install.
|
||||
/// `cluster_issuer` `Some` requests cert-manager TLS; `None` serves
|
||||
/// plain HTTP. Not calling this (dev/e2e) leaves the UI internal.
|
||||
pub fn ingress(mut self, host: impl Into<String>, cluster_issuer: Option<String>) -> Self {
|
||||
self.operator_ui_host = Some(host.into());
|
||||
self.cluster_issuer = cluster_issuer;
|
||||
self
|
||||
}
|
||||
|
||||
/// Install the published OCI chart at `version` instead of rendering
|
||||
/// one from local source (the CD `harmony apply` path).
|
||||
pub fn published_chart(
|
||||
@@ -258,14 +276,41 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
|
||||
.await?
|
||||
};
|
||||
|
||||
Ok(Outcome::success_with_details(
|
||||
helm_outcome.message,
|
||||
vec![
|
||||
format!("operator namespace: {}", self.score.namespace),
|
||||
format!("operator release: {}", self.score.release_name),
|
||||
format!("operator NATS URL: {}", self.score.nats_url),
|
||||
],
|
||||
))
|
||||
// Expose the UI. Applied after the chart so the backing Service
|
||||
// (shipped in the chart) exists. Skipped when no host is set —
|
||||
// dev/e2e harnesses keep the operator cluster-internal.
|
||||
let mut details = vec![
|
||||
format!("operator namespace: {}", self.score.namespace),
|
||||
format!("operator release: {}", self.score.release_name),
|
||||
format!("operator NATS URL: {}", self.score.nats_url),
|
||||
];
|
||||
if let Some(host) = &self.score.operator_ui_host {
|
||||
let to_fqdn = |s: &str| {
|
||||
fqdn::FQDN::from_str(s)
|
||||
.map_err(|e| InterpretError::new(format!("invalid ingress fqdn '{s}': {e}")))
|
||||
};
|
||||
K8sIngressScore {
|
||||
name: to_fqdn(chart::RELEASE_NAME)?,
|
||||
host: to_fqdn(host)?,
|
||||
backend_service: to_fqdn(chart::RELEASE_NAME)?,
|
||||
port: chart::OPERATOR_HTTP_PORT,
|
||||
path: None,
|
||||
path_type: None,
|
||||
namespace: Some(to_fqdn(&self.score.namespace)?),
|
||||
ingress_class_name: None,
|
||||
cluster_issuer: self.score.cluster_issuer.clone(),
|
||||
}
|
||||
.interpret(inventory, topology)
|
||||
.await?;
|
||||
let scheme = if self.score.cluster_issuer.is_some() {
|
||||
"https"
|
||||
} else {
|
||||
"http"
|
||||
};
|
||||
details.push(format!("operator UI: {scheme}://{host}"));
|
||||
}
|
||||
|
||||
Ok(Outcome::success_with_details(helm_outcome.message, details))
|
||||
}
|
||||
|
||||
fn get_name(&self) -> InterpretName {
|
||||
|
||||
@@ -45,6 +45,13 @@ pub struct FleetDeployConfig {
|
||||
|
||||
/// OCI chart project (e.g. `harmony`).
|
||||
pub operator_chart_project: String,
|
||||
|
||||
/// Public base domain. The operator UI host is derived as
|
||||
/// `fleet-stg.{base_domain}`, coherent with `sso-stg.`/`secrets-stg.`.
|
||||
pub base_domain: String,
|
||||
|
||||
/// cert-manager `ClusterIssuer` for the operator UI's TLS cert.
|
||||
pub cluster_issuer: String,
|
||||
}
|
||||
|
||||
impl Default for FleetDeployConfig {
|
||||
@@ -56,6 +63,8 @@ impl Default for FleetDeployConfig {
|
||||
openbao_namespace: "openbao-staging".to_string(),
|
||||
operator_chart_registry: "hub.nationtech.io".to_string(),
|
||||
operator_chart_project: "harmony".to_string(),
|
||||
base_domain: "cb1.nationtech.io".to_string(),
|
||||
|
|
||||
cluster_issuer: "letsencrypt-prod".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -85,5 +94,7 @@ mod tests {
|
||||
assert_eq!(c.namespace, "fleet-staging");
|
||||
assert_eq!(c.zitadel_namespace, "zitadel-staging");
|
||||
assert_eq!(c.openbao_namespace, "openbao-staging");
|
||||
assert_eq!(c.base_domain, "cb1.nationtech.io");
|
||||
assert_eq!(c.cluster_issuer, "letsencrypt-prod");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -763,6 +763,7 @@ impl K8sAnywhereTopology {
|
||||
path_type: Some(PathType::Prefix),
|
||||
namespace: Some(fqdn::fqdn!(&ns)),
|
||||
ingress_class_name: Some("openshift-default".to_string()),
|
||||
cluster_issuer: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -45,10 +45,20 @@ pub struct K8sIngressScore {
|
||||
pub path_type: Option<PathType>,
|
||||
pub namespace: Option<fqdn::FQDN>,
|
||||
pub ingress_class_name: Option<String>,
|
||||
/// `Some(issuer)` requests cert-manager TLS: the
|
||||
/// `cert-manager.io/cluster-issuer` annotation plus a `tls` block
|
||||
/// with a `secretName` (both are required for a portable Ingress —
|
||||
/// see `docs/guides/kubernetes-ingress.md`). `None` serves plain
|
||||
/// HTTP. The OKD `route.openshift.io/termination: edge` annotation
|
||||
/// is added alongside and is a no-op on non-OpenShift clusters.
|
||||
pub cluster_issuer: Option<String>,
|
||||
}
|
||||
|
||||
impl<T: Topology + K8sclient> Score<T> for K8sIngressScore {
|
||||
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
|
||||
impl K8sIngressScore {
|
||||
/// Render the `networking.k8s.io/v1` Ingress this score describes.
|
||||
/// Extracted from `create_interpret` so the TLS/annotation shape is
|
||||
/// unit-testable without a topology.
|
||||
fn render_ingress(&self) -> Ingress {
|
||||
let path = match self.path.clone() {
|
||||
Some(p) => p,
|
||||
None => ingress_path!("/"),
|
||||
@@ -64,7 +74,7 @@ impl<T: Topology + K8sclient> Score<T> for K8sIngressScore {
|
||||
None => "\"default\"".to_string(),
|
||||
};
|
||||
|
||||
let ingress = json!(
|
||||
let mut ingress = json!(
|
||||
{
|
||||
"metadata": {
|
||||
"name": self.name.to_string(),
|
||||
@@ -95,15 +105,36 @@ impl<T: Topology + K8sclient> Score<T> for K8sIngressScore {
|
||||
}
|
||||
);
|
||||
|
||||
// cert-manager TLS: the annotation issues the cert into
|
||||
// `secretName`, and the `tls` block must name that same Secret
|
||||
// for the Ingress to be portable (a bare `tls` host without a
|
||||
// secretName is rejected by OKD's ingress-to-route translation).
|
||||
if let Some(issuer) = &self.cluster_issuer {
|
||||
let secret_name = format!("{}-tls", self.host.to_string().replace('.', "-"));
|
||||
ingress["metadata"]["annotations"] = json!({
|
||||
"cert-manager.io/cluster-issuer": issuer,
|
||||
"route.openshift.io/termination": "edge",
|
||||
});
|
||||
ingress["spec"]["tls"] = json!([{
|
||||
"hosts": [self.host.to_string()],
|
||||
"secretName": secret_name,
|
||||
}]);
|
||||
}
|
||||
|
||||
trace!("Building ingresss object from Value {ingress:#}");
|
||||
let ingress: Ingress = serde_json::from_value(ingress).unwrap();
|
||||
debug!(
|
||||
"Successfully built Ingress for host {:?}",
|
||||
ingress.metadata.name
|
||||
);
|
||||
ingress
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Topology + K8sclient> Score<T> for K8sIngressScore {
|
||||
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
|
||||
Box::new(K8sIngressInterpret {
|
||||
ingress,
|
||||
ingress: self.render_ingress(),
|
||||
service: self.name.to_string(),
|
||||
namespace: self.namespace.clone().map(|f| f.to_string()),
|
||||
host: self.host.clone(),
|
||||
@@ -173,3 +204,59 @@ impl<T: Topology + K8sclient> Interpret<T> for K8sIngressInterpret {
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use fqdn::fqdn;
|
||||
|
||||
fn score(cluster_issuer: Option<String>) -> K8sIngressScore {
|
||||
K8sIngressScore {
|
||||
name: fqdn!("op"),
|
||||
host: fqdn!("fleet-stg.cb1.nationtech.io"),
|
||||
backend_service: fqdn!("op"),
|
||||
port: 18080,
|
||||
path: None,
|
||||
path_type: None,
|
||||
namespace: None,
|
||||
ingress_class_name: None,
|
||||
cluster_issuer,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_http_has_no_tls_or_cert_manager() {
|
||||
let ing = score(None).render_ingress();
|
||||
assert!(ing.metadata.annotations.is_none());
|
||||
assert!(ing.spec.unwrap().tls.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_issuer_renders_cert_manager_and_tls() {
|
||||
let ing = score(Some("letsencrypt-prod".into())).render_ingress();
|
||||
let ann = ing.metadata.annotations.expect("annotations");
|
||||
assert_eq!(
|
||||
ann.get("cert-manager.io/cluster-issuer")
|
||||
.map(String::as_str),
|
||||
Some("letsencrypt-prod")
|
||||
);
|
||||
// OKD edge termination — no-op off OpenShift, safe to emit always.
|
||||
assert_eq!(
|
||||
ann.get("route.openshift.io/termination")
|
||||
.map(String::as_str),
|
||||
Some("edge")
|
||||
);
|
||||
let tls = ing.spec.unwrap().tls.expect("tls block");
|
||||
let entry = &tls[0];
|
||||
assert_eq!(
|
||||
entry.hosts.as_deref(),
|
||||
Some(&["fleet-stg.cb1.nationtech.io".to_string()][..])
|
||||
);
|
||||
// secretName must be present and match the host — a bare tls host
|
||||
// without it is rejected by OKD's ingress-to-route translation.
|
||||
assert_eq!(
|
||||
entry.secret_name.as_deref(),
|
||||
Some("fleet-stg-cb1-nationtech-io-tls")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,6 +151,7 @@ impl<T: Topology + K8sclient + HelmCommand> Interpret<T> for LAMPInterpret {
|
||||
namespace: self
|
||||
.get_namespace()
|
||||
.map(|nbs| fqdn!(nbs.to_string().as_str())),
|
||||
cluster_issuer: None,
|
||||
};
|
||||
|
||||
lamp_ingress.interpret(inventory, topology).await?;
|
||||
|
||||
@@ -246,6 +246,7 @@ impl NatsK8sInterpret {
|
||||
path_type: todo!(),
|
||||
namespace: todo!(),
|
||||
ingress_class_name: todo!(),
|
||||
cluster_issuer: todo!(),
|
||||
}
|
||||
.interpret(inventory, topology)
|
||||
.await
|
||||
|
||||
@@ -294,6 +294,7 @@ impl RHOBAlertingInterpret {
|
||||
path_type: Some(PathType::Prefix),
|
||||
namespace: Some(fqdn!(&namespace)),
|
||||
ingress_class_name: Some("openshift-default".to_string()),
|
||||
cluster_issuer: None,
|
||||
};
|
||||
|
||||
let prometheus_domain = topology
|
||||
@@ -310,6 +311,7 @@ impl RHOBAlertingInterpret {
|
||||
path_type: Some(PathType::Prefix),
|
||||
namespace: Some(fqdn!(&namespace)),
|
||||
ingress_class_name: Some("openshift-default".to_string()),
|
||||
cluster_issuer: None,
|
||||
};
|
||||
|
||||
alert_manager_ingress.interpret(inventory, topology).await?;
|
||||
@@ -508,6 +510,7 @@ impl RHOBAlertingInterpret {
|
||||
path_type: Some(PathType::Prefix),
|
||||
namespace: Some(fqdn!(&namespace)),
|
||||
ingress_class_name: Some("openshift-default".to_string()),
|
||||
cluster_issuer: None,
|
||||
};
|
||||
|
||||
grafana_ingress.interpret(inventory, topology).await?;
|
||||
|
||||
Reference in New Issue
Block a user
We need to figure out a better (more secure and less vendor-locked) way to provide sane defaults for public hostnames.
We do plan on providing a free tier for harmony users but that is not done yet.