diff --git a/Cargo.lock b/Cargo.lock index e3fc0a4e..45330c58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4096,6 +4096,7 @@ dependencies = [ "async-trait", "clap", "env_logger", + "fqdn", "harmony", "harmony-fleet-auth", "harmony-reconciler-contracts", diff --git a/fleet/harmony-fleet-deploy/Cargo.toml b/fleet/harmony-fleet-deploy/Cargo.toml index 1c740e33..5b760da4 100644 --- a/fleet/harmony-fleet-deploy/Cargo.toml +++ b/fleet/harmony-fleet-deploy/Cargo.toml @@ -33,6 +33,7 @@ harmony-reconciler-contracts = { path = "../../harmony-reconciler-contracts" } anyhow = { workspace = true } async-trait = { workspace = true } clap = { workspace = true } +fqdn = "0.5.2" k8s-openapi = { workspace = true } kube = { workspace = true, features = ["runtime", "derive"] } log = { workspace = true } diff --git a/fleet/harmony-fleet-deploy/src/main.rs b/fleet/harmony-fleet-deploy/src/main.rs index ed69d401..97dfa828 100644 --- a/fleet/harmony-fleet-deploy/src/main.rs +++ b/fleet/harmony-fleet-deploy/src/main.rs @@ -111,11 +111,15 @@ async fn main() -> Result<()> { .operator_chart_project .unwrap_or(config.operator_chart_project); + // Coherent with the other staging hosts (sso-stg., secrets-stg.). + let ui_host = format!("fleet-stg.{}", config.base_domain); + let operator = FleetOperatorScore::new() .namespace(namespace) .nats_url(config.nats_url) .credentials(secrets.operator_credentials_toml) - .published_chart(registry, project, version); + .published_chart(registry, project, version) + .ingress(ui_host, Some(config.cluster_issuer)); harmony_cli::run( Inventory::autoload(), diff --git a/fleet/harmony-fleet-deploy/src/operator/chart.rs b/fleet/harmony-fleet-deploy/src/operator/chart.rs index 92496699..1b56749a 100644 --- a/fleet/harmony-fleet-deploy/src/operator/chart.rs +++ b/fleet/harmony-fleet-deploy/src/operator/chart.rs @@ -25,11 +25,12 @@ use k8s_openapi::api::apps::v1::{ }; use k8s_openapi::api::core::v1::{ Capabilities, Container, EnvVar, EnvVarSource, PodSpec, PodTemplateSpec, SeccompProfile, - Secret, SecretKeySelector, SecurityContext, ServiceAccount, + Secret, SecretKeySelector, SecurityContext, Service, ServiceAccount, ServicePort, ServiceSpec, }; use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, RoleRef, Subject}; use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; use kube::CustomResourceExt; use kube::api::ObjectMeta; use serde::Serialize; @@ -143,6 +144,11 @@ pub const SERVICE_ACCOUNT: &str = "harmony-fleet-operator"; pub const CLUSTER_ROLE: &str = "harmony-fleet-operator"; pub const CLUSTER_ROLE_BINDING: &str = "harmony-fleet-operator"; pub const SECRET_NAME: &str = "harmony-fleet-operator-secrets"; +/// Port the operator UI listens on and the Service exposes. Mirrors the +/// operator binary's bind default (`harmony-fleet-operator`'s +/// `DEFAULT_PORT`); the operator is consumed here as a container image, +/// not a crate dependency, so the value is restated at this boundary. +pub const OPERATOR_HTTP_PORT: u16 = 18080; /// Single Secret key holding the entire `[credentials]` TOML — /// including the inline JSON keyfile under `key_json`. pub const SECRET_KEY_CREDENTIALS_TOML: &str = "credentials.toml"; @@ -192,6 +198,10 @@ pub fn build_chart(opts: &ChartOptions) -> Result { .context("serializing credentials Secret")?, ); chart.add_resource(HelmResourceKind::Deployment(operator_deployment(opts))); + chart.add_resource( + HelmResourceKind::from_serializable("service.yaml", &operator_service()) + .context("serializing operator Service")?, + ); let written = chart .write_to(Path::new(&opts.output_dir)) @@ -336,6 +346,34 @@ fn cluster_role_binding(namespace: &str) -> ClusterRoleBinding { } } +/// ClusterIP Service fronting the operator UI — the Ingress backend. +/// Namespace-neutral like the Deployment (helm/direct-apply assigns it). +fn operator_service() -> Service { + let mut match_labels = BTreeMap::new(); + match_labels.insert( + "app.kubernetes.io/name".to_string(), + RELEASE_NAME.to_string(), + ); + Service { + metadata: ObjectMeta { + name: Some(RELEASE_NAME.to_string()), + labels: Some(match_labels.clone()), + ..Default::default() + }, + spec: Some(ServiceSpec { + selector: Some(match_labels), + ports: Some(vec![ServicePort { + name: Some("http".to_string()), + port: OPERATOR_HTTP_PORT as i32, + target_port: Some(IntOrString::Int(OPERATOR_HTTP_PORT as i32)), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + } +} + fn operator_deployment(opts: &ChartOptions) -> K8sDeployment { let mut match_labels = BTreeMap::new(); match_labels.insert( diff --git a/fleet/harmony-fleet-deploy/src/operator/score.rs b/fleet/harmony-fleet-deploy/src/operator/score.rs index bef47eef..94376016 100644 --- a/fleet/harmony-fleet-deploy/src/operator/score.rs +++ b/fleet/harmony-fleet-deploy/src/operator/score.rs @@ -34,6 +34,7 @@ use harmony::data::Version; use harmony::interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}; use harmony::inventory::Inventory; use harmony::modules::helm::chart::{HelmChartScore, NonBlankString}; +use harmony::modules::k8s::ingress::K8sIngressScore; use harmony::modules::k8s::resource::K8sResourceScore; use harmony::score::Score; use harmony::topology::{HelmCommand, K8sclient, Topology}; @@ -71,6 +72,12 @@ pub struct FleetOperatorScore { /// `None` renders + installs the chart from local source (dev/e2e); /// `Some` installs the published OCI chart at a pinned version (CD). pub published_chart: Option, + /// `Some(host)` exposes the UI via an Ingress after install (CD); + /// `None` leaves it cluster-internal (dev/e2e harnesses). + pub operator_ui_host: Option, + /// cert-manager `ClusterIssuer` for the UI Ingress. `None` (or no + /// host) serves plain HTTP — the right default on issuer-less k3d. + pub cluster_issuer: Option, } impl FleetOperatorScore { @@ -89,9 +96,20 @@ impl FleetOperatorScore { log_level: defaults.log_level, credentials: defaults.credentials, published_chart: None, + operator_ui_host: None, + cluster_issuer: None, } } + /// Expose the operator UI via an Ingress at `host` after install. + /// `cluster_issuer` `Some` requests cert-manager TLS; `None` serves + /// plain HTTP. Not calling this (dev/e2e) leaves the UI internal. + pub fn ingress(mut self, host: impl Into, cluster_issuer: Option) -> Self { + self.operator_ui_host = Some(host.into()); + self.cluster_issuer = cluster_issuer; + self + } + /// Install the published OCI chart at `version` instead of rendering /// one from local source (the CD `harmony apply` path). pub fn published_chart( @@ -258,14 +276,41 @@ impl Interpret for FleetOperatorInterp .await? }; - Ok(Outcome::success_with_details( - helm_outcome.message, - vec![ - format!("operator namespace: {}", self.score.namespace), - format!("operator release: {}", self.score.release_name), - format!("operator NATS URL: {}", self.score.nats_url), - ], - )) + // Expose the UI. Applied after the chart so the backing Service + // (shipped in the chart) exists. Skipped when no host is set — + // dev/e2e harnesses keep the operator cluster-internal. + let mut details = vec![ + format!("operator namespace: {}", self.score.namespace), + format!("operator release: {}", self.score.release_name), + format!("operator NATS URL: {}", self.score.nats_url), + ]; + if let Some(host) = &self.score.operator_ui_host { + let to_fqdn = |s: &str| { + fqdn::FQDN::from_str(s) + .map_err(|e| InterpretError::new(format!("invalid ingress fqdn '{s}': {e}"))) + }; + K8sIngressScore { + name: to_fqdn(chart::RELEASE_NAME)?, + host: to_fqdn(host)?, + backend_service: to_fqdn(chart::RELEASE_NAME)?, + port: chart::OPERATOR_HTTP_PORT, + path: None, + path_type: None, + namespace: Some(to_fqdn(&self.score.namespace)?), + ingress_class_name: None, + cluster_issuer: self.score.cluster_issuer.clone(), + } + .interpret(inventory, topology) + .await?; + let scheme = if self.score.cluster_issuer.is_some() { + "https" + } else { + "http" + }; + details.push(format!("operator UI: {scheme}://{host}")); + } + + Ok(Outcome::success_with_details(helm_outcome.message, details)) } fn get_name(&self) -> InterpretName { diff --git a/fleet/harmony-fleet-deploy/src/secrets.rs b/fleet/harmony-fleet-deploy/src/secrets.rs index ec67147e..d86dad5c 100644 --- a/fleet/harmony-fleet-deploy/src/secrets.rs +++ b/fleet/harmony-fleet-deploy/src/secrets.rs @@ -45,6 +45,13 @@ pub struct FleetDeployConfig { /// OCI chart project (e.g. `harmony`). pub operator_chart_project: String, + + /// Public base domain. The operator UI host is derived as + /// `fleet-stg.{base_domain}`, coherent with `sso-stg.`/`secrets-stg.`. + pub base_domain: String, + + /// cert-manager `ClusterIssuer` for the operator UI's TLS cert. + pub cluster_issuer: String, } impl Default for FleetDeployConfig { @@ -56,6 +63,8 @@ impl Default for FleetDeployConfig { openbao_namespace: "openbao-staging".to_string(), operator_chart_registry: "hub.nationtech.io".to_string(), operator_chart_project: "harmony".to_string(), + base_domain: "cb1.nationtech.io".to_string(), + cluster_issuer: "letsencrypt-prod".to_string(), } } } @@ -85,5 +94,7 @@ mod tests { assert_eq!(c.namespace, "fleet-staging"); assert_eq!(c.zitadel_namespace, "zitadel-staging"); assert_eq!(c.openbao_namespace, "openbao-staging"); + assert_eq!(c.base_domain, "cb1.nationtech.io"); + assert_eq!(c.cluster_issuer, "letsencrypt-prod"); } } diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 2dbadcf6..1f74b426 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -763,6 +763,7 @@ impl K8sAnywhereTopology { path_type: Some(PathType::Prefix), namespace: Some(fqdn::fqdn!(&ns)), ingress_class_name: Some("openshift-default".to_string()), + cluster_issuer: None, } } diff --git a/harmony/src/modules/k8s/ingress.rs b/harmony/src/modules/k8s/ingress.rs index 045f1f8d..48ae9d07 100644 --- a/harmony/src/modules/k8s/ingress.rs +++ b/harmony/src/modules/k8s/ingress.rs @@ -45,10 +45,20 @@ pub struct K8sIngressScore { pub path_type: Option, pub namespace: Option, pub ingress_class_name: Option, + /// `Some(issuer)` requests cert-manager TLS: the + /// `cert-manager.io/cluster-issuer` annotation plus a `tls` block + /// with a `secretName` (both are required for a portable Ingress — + /// see `docs/guides/kubernetes-ingress.md`). `None` serves plain + /// HTTP. The OKD `route.openshift.io/termination: edge` annotation + /// is added alongside and is a no-op on non-OpenShift clusters. + pub cluster_issuer: Option, } -impl Score for K8sIngressScore { - fn create_interpret(&self) -> Box> { +impl K8sIngressScore { + /// Render the `networking.k8s.io/v1` Ingress this score describes. + /// Extracted from `create_interpret` so the TLS/annotation shape is + /// unit-testable without a topology. + fn render_ingress(&self) -> Ingress { let path = match self.path.clone() { Some(p) => p, None => ingress_path!("/"), @@ -64,7 +74,7 @@ impl Score for K8sIngressScore { None => "\"default\"".to_string(), }; - let ingress = json!( + let mut ingress = json!( { "metadata": { "name": self.name.to_string(), @@ -95,15 +105,36 @@ impl Score for K8sIngressScore { } ); + // cert-manager TLS: the annotation issues the cert into + // `secretName`, and the `tls` block must name that same Secret + // for the Ingress to be portable (a bare `tls` host without a + // secretName is rejected by OKD's ingress-to-route translation). + if let Some(issuer) = &self.cluster_issuer { + let secret_name = format!("{}-tls", self.host.to_string().replace('.', "-")); + ingress["metadata"]["annotations"] = json!({ + "cert-manager.io/cluster-issuer": issuer, + "route.openshift.io/termination": "edge", + }); + ingress["spec"]["tls"] = json!([{ + "hosts": [self.host.to_string()], + "secretName": secret_name, + }]); + } + trace!("Building ingresss object from Value {ingress:#}"); let ingress: Ingress = serde_json::from_value(ingress).unwrap(); debug!( "Successfully built Ingress for host {:?}", ingress.metadata.name ); + ingress + } +} +impl Score for K8sIngressScore { + fn create_interpret(&self) -> Box> { Box::new(K8sIngressInterpret { - ingress, + ingress: self.render_ingress(), service: self.name.to_string(), namespace: self.namespace.clone().map(|f| f.to_string()), host: self.host.clone(), @@ -173,3 +204,59 @@ impl Interpret for K8sIngressInterpret { vec![] } } + +#[cfg(test)] +mod tests { + use super::*; + use fqdn::fqdn; + + fn score(cluster_issuer: Option) -> K8sIngressScore { + K8sIngressScore { + name: fqdn!("op"), + host: fqdn!("fleet-stg.cb1.nationtech.io"), + backend_service: fqdn!("op"), + port: 18080, + path: None, + path_type: None, + namespace: None, + ingress_class_name: None, + cluster_issuer, + } + } + + #[test] + fn plain_http_has_no_tls_or_cert_manager() { + let ing = score(None).render_ingress(); + assert!(ing.metadata.annotations.is_none()); + assert!(ing.spec.unwrap().tls.is_none()); + } + + #[test] + fn cluster_issuer_renders_cert_manager_and_tls() { + let ing = score(Some("letsencrypt-prod".into())).render_ingress(); + let ann = ing.metadata.annotations.expect("annotations"); + assert_eq!( + ann.get("cert-manager.io/cluster-issuer") + .map(String::as_str), + Some("letsencrypt-prod") + ); + // OKD edge termination — no-op off OpenShift, safe to emit always. + assert_eq!( + ann.get("route.openshift.io/termination") + .map(String::as_str), + Some("edge") + ); + let tls = ing.spec.unwrap().tls.expect("tls block"); + let entry = &tls[0]; + assert_eq!( + entry.hosts.as_deref(), + Some(&["fleet-stg.cb1.nationtech.io".to_string()][..]) + ); + // secretName must be present and match the host — a bare tls host + // without it is rejected by OKD's ingress-to-route translation. + assert_eq!( + entry.secret_name.as_deref(), + Some("fleet-stg-cb1-nationtech-io-tls") + ); + } +} diff --git a/harmony/src/modules/lamp.rs b/harmony/src/modules/lamp.rs index a33fa1d3..7a45e96b 100644 --- a/harmony/src/modules/lamp.rs +++ b/harmony/src/modules/lamp.rs @@ -151,6 +151,7 @@ impl Interpret for LAMPInterpret { namespace: self .get_namespace() .map(|nbs| fqdn!(nbs.to_string().as_str())), + cluster_issuer: None, }; lamp_ingress.interpret(inventory, topology).await?; diff --git a/harmony/src/modules/nats/score_nats_k8s.rs b/harmony/src/modules/nats/score_nats_k8s.rs index a5d80515..387ee887 100644 --- a/harmony/src/modules/nats/score_nats_k8s.rs +++ b/harmony/src/modules/nats/score_nats_k8s.rs @@ -246,6 +246,7 @@ impl NatsK8sInterpret { path_type: todo!(), namespace: todo!(), ingress_class_name: todo!(), + cluster_issuer: todo!(), } .interpret(inventory, topology) .await diff --git a/harmony/src/modules/prometheus/rhob_alerting_score.rs b/harmony/src/modules/prometheus/rhob_alerting_score.rs index 772aeda3..892c3980 100644 --- a/harmony/src/modules/prometheus/rhob_alerting_score.rs +++ b/harmony/src/modules/prometheus/rhob_alerting_score.rs @@ -294,6 +294,7 @@ impl RHOBAlertingInterpret { path_type: Some(PathType::Prefix), namespace: Some(fqdn!(&namespace)), ingress_class_name: Some("openshift-default".to_string()), + cluster_issuer: None, }; let prometheus_domain = topology @@ -310,6 +311,7 @@ impl RHOBAlertingInterpret { path_type: Some(PathType::Prefix), namespace: Some(fqdn!(&namespace)), ingress_class_name: Some("openshift-default".to_string()), + cluster_issuer: None, }; alert_manager_ingress.interpret(inventory, topology).await?; @@ -508,6 +510,7 @@ impl RHOBAlertingInterpret { path_type: Some(PathType::Prefix), namespace: Some(fqdn!(&namespace)), ingress_class_name: Some("openshift-default".to_string()), + cluster_issuer: None, }; grafana_ingress.interpret(inventory, topology).await?;