feat(fleet): expose operator UI via cert-manager TLS ingress #321

Merged
johnride merged 1 commits from feat/fleet-operator-ui-ingress into master 2026-06-01 20:26:50 +00:00
11 changed files with 207 additions and 14 deletions

1
Cargo.lock generated
View File

@@ -4096,6 +4096,7 @@ dependencies = [
"async-trait",
"clap",
"env_logger",
"fqdn",
"harmony",
"harmony-fleet-auth",
"harmony-reconciler-contracts",

View File

@@ -33,6 +33,7 @@ harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" }
anyhow = { workspace = true }
async-trait = { workspace = true }
clap = { workspace = true }
fqdn = "0.5.2"
k8s-openapi = { workspace = true }
kube = { workspace = true, features = ["runtime", "derive"] }
log = { workspace = true }

View File

@@ -111,11 +111,15 @@ async fn main() -> Result<()> {
.operator_chart_project
.unwrap_or(config.operator_chart_project);
// Coherent with the other staging hosts (sso-stg., secrets-stg.).
let ui_host = format!("fleet-stg.{}", config.base_domain);
let operator = FleetOperatorScore::new()
.namespace(namespace)
.nats_url(config.nats_url)
.credentials(secrets.operator_credentials_toml)
.published_chart(registry, project, version);
.published_chart(registry, project, version)
.ingress(ui_host, Some(config.cluster_issuer));
harmony_cli::run(
Inventory::autoload(),

View File

@@ -25,11 +25,12 @@ use k8s_openapi::api::apps::v1::{
};
use k8s_openapi::api::core::v1::{
Capabilities, Container, EnvVar, EnvVarSource, PodSpec, PodTemplateSpec, SeccompProfile,
Secret, SecretKeySelector, SecurityContext, ServiceAccount,
Secret, SecretKeySelector, SecurityContext, Service, ServiceAccount, ServicePort, ServiceSpec,
};
use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject};
use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString;
use kube::CustomResourceExt;
use kube::api::ObjectMeta;
use serde::Serialize;
@@ -143,6 +144,11 @@ pub const SERVICE_ACCOUNT: &str = "harmony-fleet-operator";
pub const CLUSTER_ROLE: &str = "harmony-fleet-operator";
pub const CLUSTER_ROLE_BINDING: &str = "harmony-fleet-operator";
pub const SECRET_NAME: &str = "harmony-fleet-operator-secrets";
/// Port the operator UI listens on and the Service exposes. Mirrors the
/// operator binary's bind default (`harmony-fleet-operator`'s
/// `DEFAULT_PORT`); the operator is consumed here as a container image,
/// not a crate dependency, so the value is restated at this boundary.
pub const OPERATOR_HTTP_PORT: u16 = 18080;
/// Single Secret key holding the entire `[credentials]` TOML —
/// including the inline JSON keyfile under `key_json`.
pub const SECRET_KEY_CREDENTIALS_TOML: &str = "credentials.toml";
@@ -192,6 +198,10 @@ pub fn build_chart(opts: &ChartOptions) -> Result<PathBuf> {
.context("serializing credentials Secret")?,
);
chart.add_resource(HelmResourceKind::Deployment(operator_deployment(opts)));
chart.add_resource(
HelmResourceKind::from_serializable("service.yaml", &operator_service())
.context("serializing operator Service")?,
);
let written = chart
.write_to(Path::new(&opts.output_dir))
@@ -336,6 +346,34 @@ fn cluster_role_binding(namespace: &str) -> ClusterRoleBinding {
}
}
/// ClusterIP Service fronting the operator UI — the Ingress backend.
/// Namespace-neutral like the Deployment (helm/direct-apply assigns it).
fn operator_service() -> Service {
let mut match_labels = BTreeMap::new();
match_labels.insert(
"app.kubernetes.io/name".to_string(),
RELEASE_NAME.to_string(),
);
Service {
metadata: ObjectMeta {
name: Some(RELEASE_NAME.to_string()),
labels: Some(match_labels.clone()),
..Default::default()
},
spec: Some(ServiceSpec {
selector: Some(match_labels),
ports: Some(vec![ServicePort {
name: Some("http".to_string()),
port: OPERATOR_HTTP_PORT as i32,
target_port: Some(IntOrString::Int(OPERATOR_HTTP_PORT as i32)),
..Default::default()
}]),
..Default::default()
}),
..Default::default()
}
}
fn operator_deployment(opts: &ChartOptions) -> K8sDeployment {
let mut match_labels = BTreeMap::new();
match_labels.insert(

View File

@@ -34,6 +34,7 @@ use harmony::data::Version;
use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome};
use harmony::inventory::Inventory;
use harmony::modules::helm::chart::{HelmChartScore, NonBlankString};
use harmony::modules::k8s::ingress::K8sIngressScore;
use harmony::modules::k8s::resource::K8sResourceScore;
use harmony::score::Score;
use harmony::topology::{HelmCommand, K8sclient, Topology};
@@ -71,6 +72,12 @@ pub struct FleetOperatorScore {
/// `None` renders + installs the chart from local source (dev/e2e);
/// `Some` installs the published OCI chart at a pinned version (CD).
pub published_chart: Option<PublishedChart>,
/// `Some(host)` exposes the UI via an Ingress after install (CD);
/// `None` leaves it cluster-internal (dev/e2e harnesses).
pub operator_ui_host: Option<String>,
/// cert-manager `ClusterIssuer` for the UI Ingress. `None` (or no
/// host) serves plain HTTP — the right default on issuer-less k3d.
pub cluster_issuer: Option<String>,
}
impl FleetOperatorScore {
@@ -89,9 +96,20 @@ impl FleetOperatorScore {
log_level: defaults.log_level,
credentials: defaults.credentials,
published_chart: None,
operator_ui_host: None,
cluster_issuer: None,
}
}
/// Expose the operator UI via an Ingress at `host` after install.
/// `cluster_issuer` `Some` requests cert-manager TLS; `None` serves
/// plain HTTP. Not calling this (dev/e2e) leaves the UI internal.
pub fn ingress(mut self, host: impl Into<String>, cluster_issuer: Option<String>) -> Self {
self.operator_ui_host = Some(host.into());
self.cluster_issuer = cluster_issuer;
self
}
/// Install the published OCI chart at `version` instead of rendering
/// one from local source (the CD `harmony apply` path).
pub fn published_chart(
@@ -258,14 +276,41 @@ impl<T: Topology + HelmCommand + K8sclient> Interpret<T> for FleetOperatorInterp
.await?
};
Ok(Outcome::success_with_details(
helm_outcome.message,
vec![
format!("operator namespace: {}", self.score.namespace),
format!("operator release: {}", self.score.release_name),
format!("operator NATS URL: {}", self.score.nats_url),
],
))
// Expose the UI. Applied after the chart so the backing Service
// (shipped in the chart) exists. Skipped when no host is set —
// dev/e2e harnesses keep the operator cluster-internal.
let mut details = vec![
format!("operator namespace: {}", self.score.namespace),
format!("operator release: {}", self.score.release_name),
format!("operator NATS URL: {}", self.score.nats_url),
];
if let Some(host) = &self.score.operator_ui_host {
let to_fqdn = |s: &str| {
fqdn::FQDN::from_str(s)
.map_err(|e| InterpretError::new(format!("invalid ingress fqdn '{s}': {e}")))
};
K8sIngressScore {
name: to_fqdn(chart::RELEASE_NAME)?,
host: to_fqdn(host)?,
backend_service: to_fqdn(chart::RELEASE_NAME)?,
port: chart::OPERATOR_HTTP_PORT,
path: None,
path_type: None,
namespace: Some(to_fqdn(&self.score.namespace)?),
ingress_class_name: None,
cluster_issuer: self.score.cluster_issuer.clone(),
}
.interpret(inventory, topology)
.await?;
let scheme = if self.score.cluster_issuer.is_some() {
"https"
} else {
"http"
};
details.push(format!("operator UI: {scheme}://{host}"));
}
Ok(Outcome::success_with_details(helm_outcome.message, details))
}
fn get_name(&self) -> InterpretName {

View File

@@ -45,6 +45,13 @@ pub struct FleetDeployConfig {
/// OCI chart project (e.g. `harmony`).
pub operator_chart_project: String,
/// Public base domain. The operator UI host is derived as
/// `fleet-stg.{base_domain}`, coherent with `sso-stg.`/`secrets-stg.`.
pub base_domain: String,
/// cert-manager `ClusterIssuer` for the operator UI's TLS cert.
pub cluster_issuer: String,
}
impl Default for FleetDeployConfig {
@@ -56,6 +63,8 @@ impl Default for FleetDeployConfig {
openbao_namespace: "openbao-staging".to_string(),
operator_chart_registry: "hub.nationtech.io".to_string(),
operator_chart_project: "harmony".to_string(),
base_domain: "cb1.nationtech.io".to_string(),
Review

We need to figure out a better (more secure and less vendor-locked) way to provide sane defaults for public hostnames.

We do plan on providing a free tier for harmony users but that is not done yet.

We need to figure out a better (more secure and less vendor-locked) way to provide sane defaults for public hostnames. We do plan on providing a free tier for harmony users but that is not done yet.
cluster_issuer: "letsencrypt-prod".to_string(),
}
}
}
@@ -85,5 +94,7 @@ mod tests {
assert_eq!(c.namespace, "fleet-staging");
assert_eq!(c.zitadel_namespace, "zitadel-staging");
assert_eq!(c.openbao_namespace, "openbao-staging");
assert_eq!(c.base_domain, "cb1.nationtech.io");
assert_eq!(c.cluster_issuer, "letsencrypt-prod");
}
}

View File

@@ -763,6 +763,7 @@ impl K8sAnywhereTopology {
path_type: Some(PathType::Prefix),
namespace: Some(fqdn::fqdn!(&ns)),
ingress_class_name: Some("openshift-default".to_string()),
cluster_issuer: None,
}
}

View File

@@ -45,10 +45,20 @@ pub struct K8sIngressScore {
pub path_type: Option<PathType>,
pub namespace: Option<fqdn::FQDN>,
pub ingress_class_name: Option<String>,
/// `Some(issuer)` requests cert-manager TLS: the
/// `cert-manager.io/cluster-issuer` annotation plus a `tls` block
/// with a `secretName` (both are required for a portable Ingress —
/// see `docs/guides/kubernetes-ingress.md`). `None` serves plain
/// HTTP. The OKD `route.openshift.io/termination: edge` annotation
/// is added alongside and is a no-op on non-OpenShift clusters.
pub cluster_issuer: Option<String>,
}
impl<T: Topology + K8sclient> Score<T> for K8sIngressScore {
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
impl K8sIngressScore {
/// Render the `networking.k8s.io/v1` Ingress this score describes.
/// Extracted from `create_interpret` so the TLS/annotation shape is
/// unit-testable without a topology.
fn render_ingress(&self) -> Ingress {
let path = match self.path.clone() {
Some(p) => p,
None => ingress_path!("/"),
@@ -64,7 +74,7 @@ impl<T: Topology + K8sclient> Score<T> for K8sIngressScore {
None => "\"default\"".to_string(),
};
let ingress = json!(
let mut ingress = json!(
{
"metadata": {
"name": self.name.to_string(),
@@ -95,15 +105,36 @@ impl<T: Topology + K8sclient> Score<T> for K8sIngressScore {
}
);
// cert-manager TLS: the annotation issues the cert into
// `secretName`, and the `tls` block must name that same Secret
// for the Ingress to be portable (a bare `tls` host without a
// secretName is rejected by OKD's ingress-to-route translation).
if let Some(issuer) = &self.cluster_issuer {
let secret_name = format!("{}-tls", self.host.to_string().replace('.', "-"));
ingress["metadata"]["annotations"] = json!({
"cert-manager.io/cluster-issuer": issuer,
"route.openshift.io/termination": "edge",
});
ingress["spec"]["tls"] = json!([{
"hosts": [self.host.to_string()],
"secretName": secret_name,
}]);
}
trace!("Building ingresss object from Value {ingress:#}");
let ingress: Ingress = serde_json::from_value(ingress).unwrap();
debug!(
"Successfully built Ingress for host {:?}",
ingress.metadata.name
);
ingress
}
}
impl<T: Topology + K8sclient> Score<T> for K8sIngressScore {
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
Box::new(K8sIngressInterpret {
ingress,
ingress: self.render_ingress(),
service: self.name.to_string(),
namespace: self.namespace.clone().map(|f| f.to_string()),
host: self.host.clone(),
@@ -173,3 +204,59 @@ impl<T: Topology + K8sclient> Interpret<T> for K8sIngressInterpret {
vec![]
}
}
#[cfg(test)]
mod tests {
use super::*;
use fqdn::fqdn;
fn score(cluster_issuer: Option<String>) -> K8sIngressScore {
K8sIngressScore {
name: fqdn!("op"),
host: fqdn!("fleet-stg.cb1.nationtech.io"),
backend_service: fqdn!("op"),
port: 18080,
path: None,
path_type: None,
namespace: None,
ingress_class_name: None,
cluster_issuer,
}
}
#[test]
fn plain_http_has_no_tls_or_cert_manager() {
let ing = score(None).render_ingress();
assert!(ing.metadata.annotations.is_none());
assert!(ing.spec.unwrap().tls.is_none());
}
#[test]
fn cluster_issuer_renders_cert_manager_and_tls() {
let ing = score(Some("letsencrypt-prod".into())).render_ingress();
let ann = ing.metadata.annotations.expect("annotations");
assert_eq!(
ann.get("cert-manager.io/cluster-issuer")
.map(String::as_str),
Some("letsencrypt-prod")
);
// OKD edge termination — no-op off OpenShift, safe to emit always.
assert_eq!(
ann.get("route.openshift.io/termination")
.map(String::as_str),
Some("edge")
);
let tls = ing.spec.unwrap().tls.expect("tls block");
let entry = &tls[0];
assert_eq!(
entry.hosts.as_deref(),
Some(&["fleet-stg.cb1.nationtech.io".to_string()][..])
);
// secretName must be present and match the host — a bare tls host
// without it is rejected by OKD's ingress-to-route translation.
assert_eq!(
entry.secret_name.as_deref(),
Some("fleet-stg-cb1-nationtech-io-tls")
);
}
}

View File

@@ -151,6 +151,7 @@ impl<T: Topology + K8sclient + HelmCommand> Interpret<T> for LAMPInterpret {
namespace: self
.get_namespace()
.map(|nbs| fqdn!(nbs.to_string().as_str())),
cluster_issuer: None,
};
lamp_ingress.interpret(inventory, topology).await?;

View File

@@ -246,6 +246,7 @@ impl NatsK8sInterpret {
path_type: todo!(),
namespace: todo!(),
ingress_class_name: todo!(),
cluster_issuer: todo!(),
}
.interpret(inventory, topology)
.await

View File

@@ -294,6 +294,7 @@ impl RHOBAlertingInterpret {
path_type: Some(PathType::Prefix),
namespace: Some(fqdn!(&namespace)),
ingress_class_name: Some("openshift-default".to_string()),
cluster_issuer: None,
};
let prometheus_domain = topology
@@ -310,6 +311,7 @@ impl RHOBAlertingInterpret {
path_type: Some(PathType::Prefix),
namespace: Some(fqdn!(&namespace)),
ingress_class_name: Some("openshift-default".to_string()),
cluster_issuer: None,
};
alert_manager_ingress.interpret(inventory, topology).await?;
@@ -508,6 +510,7 @@ impl RHOBAlertingInterpret {
path_type: Some(PathType::Prefix),
namespace: Some(fqdn!(&namespace)),
ingress_class_name: Some("openshift-default".to_string()),
cluster_issuer: None,
};
grafana_ingress.interpret(inventory, topology).await?;