From f7404bed367c90135fab8c1e34bb199a97a8108a Mon Sep 17 00:00:00 2001 From: wjro Date: Wed, 7 Jan 2026 16:14:58 -0500 Subject: [PATCH 01/21] wip: initial setup for installing nats helm chart score --- examples/nats/src/main.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/nats/src/main.rs b/examples/nats/src/main.rs index 8c391d5e..e986bb4f 100644 --- a/examples/nats/src/main.rs +++ b/examples/nats/src/main.rs @@ -25,6 +25,12 @@ async fn main() { leafnodes: enabled: false # port: 7422 + websocket: + enabled: true + ingress: + enabled: true + hosts: + - nats-demo.sto1.nationtech.io gateway: enabled: false # name: my-gateway -- 2.39.5 From 77583a1ad114e93e04d333380b5c527fbb91c14c Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 8 Jan 2026 16:03:15 -0500 Subject: [PATCH 02/21] wip: nats multi cluster, fixing helm command to follow multiple k8s config by providing the helm command from the topology itself, fix cli_logger that can now be initialized multiple times, some more stuff --- examples/nats/src/main.rs | 45 ++-- harmony/src/domain/topology/helm_command.rs | 6 +- .../topology/k8s_anywhere/k8s_anywhere.rs | 29 ++- harmony/src/domain/topology/router.rs | 11 +- harmony/src/modules/network/failover.rs | 2 +- .../src/modules/okd/crd/ingresses_config.rs | 214 ++++++++++++++++++ harmony/src/modules/okd/crd/mod.rs | 1 + harmony/src/modules/okd/crd/route.rs | 1 - harmony_cli/src/cli_logger.rs | 9 +- 9 files changed, 287 insertions(+), 31 deletions(-) create mode 100644 harmony/src/modules/okd/crd/ingresses_config.rs diff --git a/examples/nats/src/main.rs b/examples/nats/src/main.rs index e986bb4f..7cdb3eb9 100644 --- a/examples/nats/src/main.rs +++ b/examples/nats/src/main.rs @@ -3,15 +3,27 @@ use std::str::FromStr; use harmony::{ inventory::Inventory, modules::helm::chart::{HelmChartScore, HelmRepository, NonBlankString}, - topology::K8sAnywhereTopology, + topology::{HelmCommand, K8sAnywhereConfig, K8sAnywhereTopology, TlsRouter, Topology}, }; use harmony_macros::hurl; use log::info; #[tokio::main] async fn main() { - // env_logger::init(); - let values_yaml = Some( + deploy_nats(K8sAnywhereTopology::with_config( + K8sAnywhereConfig::remote_k8s_from_env_var("HARMONY_NATS_SITE_1"), + )) + .await; + deploy_nats(K8sAnywhereTopology::with_config( + K8sAnywhereConfig::remote_k8s_from_env_var("HARMONY_NATS_SITE_2"), + )) + .await; +} + +async fn deploy_nats(topology: T) { + topology.ensure_ready().await.unwrap(); + + let values_yaml = Some(format!( r#"config: cluster: enabled: true @@ -27,10 +39,12 @@ async fn main() { # port: 7422 websocket: enabled: true - ingress: - enabled: true - hosts: - - nats-demo.sto1.nationtech.io + ingress: + enabled: true + className: openshift-default + pathType: Prefix + hosts: + - nats-ws.{} gateway: enabled: false # name: my-gateway @@ -38,9 +52,9 @@ async fn main() { natsBox: container: image: - tag: nonroot"# - .to_string(), - ); + tag: nonroot"#, + topology.get_internal_domain().await.unwrap().unwrap(), + )); let namespace = "nats"; let nats = HelmChartScore { namespace: Some(NonBlankString::from_str(namespace).unwrap()), @@ -58,14 +72,9 @@ natsBox: )), }; - harmony_cli::run( - Inventory::autoload(), - K8sAnywhereTopology::from_env(), - vec![Box::new(nats)], - None, - ) - .await - .unwrap(); + harmony_cli::run(Inventory::autoload(), topology, vec![Box::new(nats)], None) + .await + .unwrap(); info!( "Enjoy! You can test your nats cluster by running : `kubectl exec -n {namespace} -it deployment/nats-box -- nats pub test hi`" diff --git a/harmony/src/domain/topology/helm_command.rs b/harmony/src/domain/topology/helm_command.rs index f3dd697b..1d5cb19c 100644 --- a/harmony/src/domain/topology/helm_command.rs +++ b/harmony/src/domain/topology/helm_command.rs @@ -1 +1,5 @@ -pub trait HelmCommand {} +use std::process::Command; + +pub trait HelmCommand { + fn get_helm_command(&self) -> Command; +} diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 22dfaad1..72fa819f 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -35,6 +35,7 @@ use crate::{ service_monitor::ServiceMonitor, }, }, + okd::crd::ingresses_config::Ingress as IngressResource, okd::route::OKDTlsPassthroughScore, prometheus::{ k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore, @@ -107,8 +108,32 @@ impl K8sclient for K8sAnywhereTopology { #[async_trait] impl TlsRouter for K8sAnywhereTopology { - async fn get_wildcard_domain(&self) -> Result, String> { - todo!() + async fn get_internal_domain(&self) -> Result, String> { + match self.get_k8s_distribution().await.map_err(|e| { + format!( + "Could not get internal domain, error getting k8s distribution : {}", + e.to_string() + ) + })? { + KubernetesDistribution::OpenshiftFamily => { + let client = self.k8s_client().await?; + if let Some(ingress_config) = client + .get_resource::("cluster", None) + .await + .map_err(|e| { + format!("Error attempting to get ingress config : {}", e.to_string()) + })? + { + debug!("Found ingress config {:?}", ingress_config.spec); + Ok(ingress_config.spec.domain.clone()) + } else { + warn!("Could not find a domain configured in this cluster"); + Ok(None) + } + } + KubernetesDistribution::K3sFamily => todo!(), + KubernetesDistribution::Default => todo!(), + } } /// Returns the port that this router exposes externally. diff --git a/harmony/src/domain/topology/router.rs b/harmony/src/domain/topology/router.rs index 5217c825..30d5b46d 100644 --- a/harmony/src/domain/topology/router.rs +++ b/harmony/src/domain/topology/router.rs @@ -112,12 +112,13 @@ pub trait TlsRouter: Send + Sync { /// HAProxy frontend→backend \"postgres-upstream\". async fn install_route(&self, config: TlsRoute) -> Result<(), String>; - /// Gets the base domain that can be used to deploy applications that will be automatically - /// routed to this cluster. + /// Gets the base domain of this cluster. On openshift family clusters, this is the domain + /// used by default for all components, including the default ingress controller that + /// transforms ingress to routes. /// - /// For example, if we have *.apps.nationtech.io pointing to a public load balancer, then this - /// function would install route apps.nationtech.io - async fn get_wildcard_domain(&self) -> Result, String>; + /// For example, get_internal_domain on a cluster that has `console-openshift-console.apps.mycluster.something` + /// will return `apps.mycluster.something` + async fn get_internal_domain(&self) -> Result, String>; /// Returns the port that this router exposes externally. async fn get_router_port(&self) -> u16; diff --git a/harmony/src/modules/network/failover.rs b/harmony/src/modules/network/failover.rs index 3880c02b..d5fd8c09 100644 --- a/harmony/src/modules/network/failover.rs +++ b/harmony/src/modules/network/failover.rs @@ -5,7 +5,7 @@ use crate::topology::{FailoverTopology, TlsRoute, TlsRouter}; #[async_trait] impl TlsRouter for FailoverTopology { - async fn get_wildcard_domain(&self) -> Result, String> { + async fn get_internal_domain(&self) -> Result, String> { todo!() } diff --git a/harmony/src/modules/okd/crd/ingresses_config.rs b/harmony/src/modules/okd/crd/ingresses_config.rs new file mode 100644 index 00000000..4c901565 --- /dev/null +++ b/harmony/src/modules/okd/crd/ingresses_config.rs @@ -0,0 +1,214 @@ +use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ListMeta, ObjectMeta}; +use k8s_openapi::{ClusterResourceScope, Resource}; +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Ingress { + #[serde(skip_serializing_if = "Option::is_none")] + pub api_version: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub kind: Option, + pub metadata: ObjectMeta, + + pub spec: IngressSpec, + + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, +} + +impl Resource for Ingress { + const API_VERSION: &'static str = "config.openshift.io/v1"; + const GROUP: &'static str = "config.openshift.io"; + const VERSION: &'static str = "v1"; + const KIND: &'static str = "Ingress"; + const URL_PATH_SEGMENT: &'static str = "ingresses"; + type Scope = ClusterResourceScope; +} + +impl k8s_openapi::Metadata for Ingress { + type Ty = ObjectMeta; + + fn metadata(&self) -> &Self::Ty { + &self.metadata + } + + fn metadata_mut(&mut self) -> &mut Self::Ty { + &mut self.metadata + } +} + +impl Default for Ingress { + fn default() -> Self { + Ingress { + api_version: Some("config.openshift.io/v1".to_string()), + kind: Some("Ingress".to_string()), + metadata: ObjectMeta::default(), + spec: IngressSpec::default(), + status: None, + } + } +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct IngressList { + pub metadata: ListMeta, + pub items: Vec, +} + +impl Default for IngressList { + fn default() -> Self { + Self { + metadata: ListMeta::default(), + items: Vec::new(), + } + } +} + +impl Resource for IngressList { + const API_VERSION: &'static str = "config.openshift.io/v1"; + const GROUP: &'static str = "config.openshift.io"; + const VERSION: &'static str = "v1"; + const KIND: &'static str = "IngressList"; + const URL_PATH_SEGMENT: &'static str = "ingresses"; + type Scope = ClusterResourceScope; +} + +impl k8s_openapi::Metadata for IngressList { + type Ty = ListMeta; + + fn metadata(&self) -> &Self::Ty { + &self.metadata + } + + fn metadata_mut(&mut self) -> &mut Self::Ty { + &mut self.metadata + } +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct IngressSpec { + #[serde(skip_serializing_if = "Option::is_none")] + pub apps_domain: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub component_routes: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub domain: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub load_balancer: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub required_hsts_policies: Option>, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct ComponentRouteSpec { + pub hostname: String, + pub name: String, + pub namespace: String, + + #[serde(skip_serializing_if = "Option::is_none")] + pub serving_cert_key_pair_secret: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct SecretNameReference { + pub name: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct LoadBalancer { + #[serde(skip_serializing_if = "Option::is_none")] + pub platform: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct IngressPlatform { + #[serde(skip_serializing_if = "Option::is_none")] + pub aws: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub r#type: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct AWSPlatformLoadBalancer { + pub r#type: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct RequiredHSTSPolicy { + pub domain_patterns: Vec, + + #[serde(skip_serializing_if = "Option::is_none")] + pub include_sub_domains_policy: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub max_age: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub namespace_selector: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub preload_policy: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct MaxAgePolicy { + #[serde(skip_serializing_if = "Option::is_none")] + pub largest_max_age: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub smallest_max_age: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct IngressStatus { + #[serde(skip_serializing_if = "Option::is_none")] + pub component_routes: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub default_placement: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct ComponentRouteStatus { + #[serde(skip_serializing_if = "Option::is_none")] + pub conditions: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub consuming_users: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub current_hostnames: Option>, + + pub default_hostname: String, + pub name: String, + pub namespace: String, + pub related_objects: Vec, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct ObjectReference { + pub group: String, + pub name: String, + pub namespace: String, + pub resource: String, +} + diff --git a/harmony/src/modules/okd/crd/mod.rs b/harmony/src/modules/okd/crd/mod.rs index 71c4d0a1..c073458f 100644 --- a/harmony/src/modules/okd/crd/mod.rs +++ b/harmony/src/modules/okd/crd/mod.rs @@ -1,2 +1,3 @@ pub mod nmstate; pub mod route; +pub mod ingresses_config; diff --git a/harmony/src/modules/okd/crd/route.rs b/harmony/src/modules/okd/crd/route.rs index 4c396d02..7c9f1568 100644 --- a/harmony/src/modules/okd/crd/route.rs +++ b/harmony/src/modules/okd/crd/route.rs @@ -1,5 +1,4 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ListMeta, ObjectMeta, Time}; -use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; use k8s_openapi::{NamespaceResourceScope, Resource}; use serde::{Deserialize, Serialize}; diff --git a/harmony_cli/src/cli_logger.rs b/harmony_cli/src/cli_logger.rs index 2cb2a939..03501ef8 100644 --- a/harmony_cli/src/cli_logger.rs +++ b/harmony_cli/src/cli_logger.rs @@ -7,11 +7,14 @@ use harmony::{ }; use log::{error, info, log_enabled}; use std::io::Write; -use std::sync::Mutex; +use std::sync::{Mutex, OnceLock}; pub fn init() { - configure_logger(); - handle_events(); + static INITIALIZED: OnceLock<()> = OnceLock::new(); + INITIALIZED.get_or_init(|| { + configure_logger(); + handle_events(); + }); } fn configure_logger() { -- 2.39.5 From 69332805756b6a2774d92b92f019f9c9a4f13396 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Thu, 8 Jan 2026 23:42:54 -0500 Subject: [PATCH 03/21] feat(helm): refactor helm execution to use topology-specific commands Refactors the `HelmChartInterpret` to move away from the `helm-wrapper-rs` crate in favor of a custom command builder pattern. This allows the `HelmCommand` trait to provide topology-specific configurations, such as `kubeconfig` and `kube-context`, directly to the `helm` CLI. - Implements `get_helm_command` for `K8sAnywhereTopology` to inject configuration flags. - Replaces `DefaultHelmExecutor` with a manual `Command` construction in `run_helm_command`. - Updates `HelmChartInterpret` to pass the topology through to repository and installation logic. - Cleans up unused imports and removes the temporary `HelmCommand` implementation for `LocalhostTopology`. --- .../topology/k8s_anywhere/k8s_anywhere.rs | 18 +- harmony/src/domain/topology/localhost.rs | 5 +- harmony/src/modules/helm/chart.rs | 170 +++++++----------- 3 files changed, 80 insertions(+), 113 deletions(-) diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 72fa819f..c3a6baa9 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -1112,7 +1112,21 @@ impl MultiTargetTopology for K8sAnywhereTopology { } } -impl HelmCommand for K8sAnywhereTopology {} +impl HelmCommand for K8sAnywhereTopology { + fn get_helm_command(&self) -> Command { + let mut cmd = Command::new("helm"); + if let Some(k) = &self.config.kubeconfig { + cmd.args(["--kubeconfig", k]); + } + + if let Some(c) = &self.config.k8s_context { + cmd.args(["--kube-context", c]); + } + + info!("Using helm command {cmd:?}"); + cmd + } +} #[async_trait] impl TenantManager for K8sAnywhereTopology { @@ -1133,7 +1147,7 @@ impl TenantManager for K8sAnywhereTopology { #[async_trait] impl Ingress for K8sAnywhereTopology { async fn get_domain(&self, service: &str) -> Result { - use log::{debug, trace, warn}; + use log::{trace, warn}; let client = self.k8s_client().await?; diff --git a/harmony/src/domain/topology/localhost.rs b/harmony/src/domain/topology/localhost.rs index 667b3f88..3af4d629 100644 --- a/harmony/src/domain/topology/localhost.rs +++ b/harmony/src/domain/topology/localhost.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use derive_new::new; use serde::{Deserialize, Serialize}; -use super::{HelmCommand, PreparationError, PreparationOutcome, Topology}; +use super::{PreparationError, PreparationOutcome, Topology}; #[derive(new, Clone, Debug, Serialize, Deserialize)] pub struct LocalhostTopology; @@ -19,6 +19,3 @@ impl Topology for LocalhostTopology { }) } } - -// TODO: Delete this, temp for test -impl HelmCommand for LocalhostTopology {} diff --git a/harmony/src/modules/helm/chart.rs b/harmony/src/modules/helm/chart.rs index 4b678f10..d4471261 100644 --- a/harmony/src/modules/helm/chart.rs +++ b/harmony/src/modules/helm/chart.rs @@ -6,15 +6,11 @@ use crate::topology::{HelmCommand, Topology}; use async_trait::async_trait; use harmony_types::id::Id; use harmony_types::net::Url; -use helm_wrapper_rs; -use helm_wrapper_rs::blocking::{DefaultHelmExecutor, HelmExecutor}; use log::{debug, info, warn}; pub use non_blank_string_rs::NonBlankString; use serde::Serialize; use std::collections::HashMap; -use std::path::Path; -use std::process::{Command, Output, Stdio}; -use std::str::FromStr; +use std::process::{Output, Stdio}; use temp_file::TempFile; #[derive(Debug, Clone, Serialize)] @@ -65,7 +61,7 @@ pub struct HelmChartInterpret { pub score: HelmChartScore, } impl HelmChartInterpret { - fn add_repo(&self) -> Result<(), InterpretError> { + fn add_repo(&self, topology: &T) -> Result<(), InterpretError> { let repo = match &self.score.repository { Some(repo) => repo, None => { @@ -84,7 +80,7 @@ impl HelmChartInterpret { add_args.push("--force-update"); } - let add_output = run_helm_command(&add_args)?; + let add_output = run_helm_command(topology, &add_args)?; let full_output = format!( "{}\n{}", String::from_utf8_lossy(&add_output.stdout), @@ -100,23 +96,19 @@ impl HelmChartInterpret { } } -fn run_helm_command(args: &[&str]) -> Result { - let command_str = format!("helm {}", args.join(" ")); - debug!( - "Got KUBECONFIG: `{}`", - std::env::var("KUBECONFIG").unwrap_or("".to_string()) - ); - debug!("Running Helm command: `{}`", command_str); +fn run_helm_command(topology: &T, args: &[&str]) -> Result { + let mut helm_cmd = topology.get_helm_command(); + helm_cmd.args(args); - let output = Command::new("helm") - .args(args) + debug!("Running Helm command: `{:?}`", helm_cmd); + + let output = helm_cmd .stdout(Stdio::piped()) .stderr(Stdio::piped()) .output() .map_err(|e| { InterpretError::new(format!( - "Failed to execute helm command '{}': {}. Is helm installed and in PATH?", - command_str, e + "Failed to execute helm command '{helm_cmd:?}': {e}. Is helm installed and in PATH?", )) })?; @@ -124,13 +116,13 @@ fn run_helm_command(args: &[&str]) -> Result { let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); warn!( - "Helm command `{}` failed with status: {}\nStdout:\n{}\nStderr:\n{}", - command_str, output.status, stdout, stderr + "Helm command `{helm_cmd:?}` failed with status: {}\nStdout:\n{stdout}\nStderr:\n{stderr}", + output.status ); } else { debug!( - "Helm command `{}` finished successfully. Status: {}", - command_str, output.status + "Helm command `{helm_cmd:?}` finished successfully. Status: {}", + output.status ); } @@ -142,7 +134,7 @@ impl Interpret for HelmChartInterpret { async fn execute( &self, _inventory: &Inventory, - _topology: &T, + topology: &T, ) -> Result { let ns = self .score @@ -150,98 +142,62 @@ impl Interpret for HelmChartInterpret { .as_ref() .unwrap_or_else(|| todo!("Get namespace from active kubernetes cluster")); - let tf: TempFile; - let yaml_path: Option<&Path> = match self.score.values_yaml.as_ref() { - Some(yaml_str) => { - tf = temp_file::with_contents(yaml_str.as_bytes()); - debug!( - "values yaml string for chart {} :\n {yaml_str}", - self.score.chart_name - ); - Some(tf.path()) - } - None => None, + self.add_repo(topology)?; + + let mut args = if self.score.install_only { + vec!["install"] + } else { + vec!["upgrade", "--install"] }; - self.add_repo()?; - - let helm_executor = DefaultHelmExecutor::new_with_opts( - &NonBlankString::from_str("helm").unwrap(), - None, - 900, - false, - false, - ); - - let mut helm_options = Vec::new(); - if self.score.create_namespace { - helm_options.push(NonBlankString::from_str("--create-namespace").unwrap()); - } - - if self.score.install_only { - let chart_list = match helm_executor.list(Some(ns)) { - Ok(charts) => charts, - Err(e) => { - return Err(InterpretError::new(format!( - "Failed to list scores in namespace {:?} because of error : {}", - self.score.namespace, e - ))); - } - }; - - if chart_list - .iter() - .any(|item| item.name == self.score.release_name.to_string()) - { - info!( - "Release '{}' already exists in namespace '{}'. Skipping installation as install_only is true.", - self.score.release_name, ns - ); - - return Ok(Outcome::success(format!( - "Helm Chart '{}' already installed to namespace {ns} and install_only=true", - self.score.release_name - ))); - } else { - info!( - "Release '{}' not found in namespace '{}'. Proceeding with installation.", - self.score.release_name, ns - ); - } - } - - let res = helm_executor.install_or_upgrade( - ns, + args.extend(vec![ &self.score.release_name, &self.score.chart_name, - self.score.chart_version.as_ref(), - self.score.values_overrides.as_ref(), - yaml_path, - Some(&helm_options), - ); + "--namespace", + &ns, + ]); - let status = match res { - Ok(status) => status, - Err(err) => return Err(InterpretError::new(err.to_string())), - }; + if self.score.create_namespace { + args.push("--create-namespace"); + } - match status { - helm_wrapper_rs::HelmDeployStatus::Deployed => Ok(Outcome::success(format!( + if let Some(version) = &self.score.chart_version { + args.push("--version"); + args.push(&version); + } + + let tf: TempFile; + if let Some(yaml_str) = &self.score.values_yaml { + tf = temp_file::with_contents(yaml_str.as_bytes()); + args.push("--values"); + args.push(tf.path().to_str().unwrap()); + } + + let overrides_strings: Vec; + if let Some(overrides) = &self.score.values_overrides { + overrides_strings = overrides + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect(); + for o in overrides_strings.iter() { + args.push("--set"); + args.push(&o); + } + } + + let output = run_helm_command(topology, &args)?; + + if output.status.success() { + Ok(Outcome::success(format!( "Helm Chart {} deployed", self.score.release_name - ))), - helm_wrapper_rs::HelmDeployStatus::PendingInstall => Ok(Outcome::running(format!( - "Helm Chart {} pending install...", - self.score.release_name - ))), - helm_wrapper_rs::HelmDeployStatus::PendingUpgrade => Ok(Outcome::running(format!( - "Helm Chart {} pending upgrade...", - self.score.release_name - ))), - helm_wrapper_rs::HelmDeployStatus::Failed => Err(InterpretError::new(format!( - "Helm Chart {} installation failed", - self.score.release_name - ))), + ))) + } else { + Err(InterpretError::new(format!( + "Helm Chart {} installation failed: {}", + self.score.release_name, + String::from_utf8_lossy(&output.stderr) + ))) } } -- 2.39.5 From 270b6b87df7de838996d2ded6ca6b83f99c90ff4 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 9 Jan 2026 17:30:51 -0500 Subject: [PATCH 04/21] wip nats supercluster --- examples/nats/src/main.rs | 62 +++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/examples/nats/src/main.rs b/examples/nats/src/main.rs index 7cdb3eb9..75955841 100644 --- a/examples/nats/src/main.rs +++ b/examples/nats/src/main.rs @@ -10,19 +10,50 @@ use log::info; #[tokio::main] async fn main() { - deploy_nats(K8sAnywhereTopology::with_config( - K8sAnywhereConfig::remote_k8s_from_env_var("HARMONY_NATS_SITE_1"), - )) - .await; - deploy_nats(K8sAnywhereTopology::with_config( - K8sAnywhereConfig::remote_k8s_from_env_var("HARMONY_NATS_SITE_2"), - )) - .await; + let site1_topo = K8sAnywhereTopology::with_config(K8sAnywhereConfig::remote_k8s_from_env_var( + "HARMONY_NATS_SITE_1", + )); + let site2_topo = K8sAnywhereTopology::with_config(K8sAnywhereConfig::remote_k8s_from_env_var( + "HARMONY_NATS_SITE_2", + )); + + let site1_domain = site1_topo.get_internal_domain().await.unwrap().unwrap(); + let site2_domain = site2_topo.get_internal_domain().await.unwrap().unwrap(); + + let site1_gateway = format!("nats-gateway.{}", site1_domain); + let site2_gateway = format!("nats-gateway.{}", site2_domain); + + tokio::join!( + deploy_nats( + site1_topo, + "site-1", + vec![("site-2".to_string(), site2_gateway)] + ), + deploy_nats( + site2_topo, + "site-2", + vec![("site-1".to_string(), site1_gateway)] + ), + ); } -async fn deploy_nats(topology: T) { +async fn deploy_nats( + topology: T, + cluster_name: &str, + remote_gateways: Vec<(String, String)>, +) { topology.ensure_ready().await.unwrap(); + let mut gateway_gateways = String::new(); + for (name, url) in remote_gateways { + gateway_gateways.push_str(&format!( + r#" + - name: {name} + urls: + - nats://{url}:7222"# + )); + } + let values_yaml = Some(format!( r#"config: cluster: @@ -46,14 +77,21 @@ async fn deploy_nats(topology: hosts: - nats-ws.{} gateway: - enabled: false - # name: my-gateway - # port: 7522 + enabled: true + name: {} + port: 7222 + gateways: {} +service: + ports: + gateway: + enabled: true natsBox: container: image: tag: nonroot"#, topology.get_internal_domain().await.unwrap().unwrap(), + cluster_name, + gateway_gateways, )); let namespace = "nats"; let nats = HelmChartScore { -- 2.39.5 From 1837623394e4178410cf02309b1a0dc05b93e4d8 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Tue, 13 Jan 2026 10:40:30 -0500 Subject: [PATCH 05/21] adr: 17-1 nats clusters interconnection using islands of trust. mTLS via shared ca-bundle with each cluster distributing its own CA. --- ...-Nats-Clusters-Interconnection-Topology.md | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 adr/017-1-Nats-Clusters-Interconnection-Topology.md diff --git a/adr/017-1-Nats-Clusters-Interconnection-Topology.md b/adr/017-1-Nats-Clusters-Interconnection-Topology.md new file mode 100644 index 00000000..ac1e9c87 --- /dev/null +++ b/adr/017-1-Nats-Clusters-Interconnection-Topology.md @@ -0,0 +1,189 @@ +### 1. ADR 017-1: NATS Cluster Interconnection & Trust Topology + +# Architecture Decision Record: NATS Cluster Interconnection & Trust Topology + +**Status:** Proposed +**Date:** 2026-01-12 +**Precedes:** [017-Staleness-Detection-for-Failover.md] + +## Context + +In ADR 017, we defined the failover mechanisms for the Harmony mesh. However, for a Primary (Site A) and a Replica (Site B) to communicate securely—or for the Global Mesh to function across disparate locations—we must establish a robust Transport Layer Security (TLS) strategy. + +Our primary deployment platform is OKD (Kubernetes). While OKD provides an internal `service-ca`, it is designed primarily for intra-cluster service-to-service communication. It lacks the flexibility required for: +1. **Public/External Gateway Identities:** NATS Gateways need to identify themselves via public DNS names or external IPs, not just internal `.svc` cluster domains. +2. **Cross-Cluster Trust:** We need a mechanism to allow Cluster A to trust Cluster B without sharing a single private root key. + +## Decision + +We will implement an **"Islands of Trust"** topology using **cert-manager** on OKD. + +### 1. Per-Cluster Certificate Authorities (CA) + +* We explicitly **reject** the use of a single "Supercluster CA" shared across all sites. + * Instead, every Harmony Cluster (Site A, Site B, etc.) will generate its own unique Self-Signed Root CA managed by `cert-manager` inside that cluster. +* **Lifecycle:** Root CAs will have a long duration (e.g., 10 years) to minimize rotation friction, while Leaf Certificates (NATS servers) will remain short-lived (e.g., 90 days) and rotate automatically. + +> Note : The decision to have a single CA for various workloads managed by Harmony on each deployment, or to have multiple CA for each service that requires interconnection is not made yet. This ADR leans towards one CA per service. This allows for maximum flexibility. But the direction might change and no clear decision has been made yet. The alternative of establishing that each cluster/harmony deployment has a single identity could make mTLS very simple between tenants. + +### 2. Trust Federation via Bundle Exchange + +To enable secure communication (mTLS) between clusters (e.g., for NATS Gateways or Leaf Nodes): + +* **No Private Keys are shared.** +* We will aggregate the **Public CA Certificates** of all trusted clusters into a shared `ca-bundle.pem`. +* This bundle is distributed to the NATS configuration of every node. +* **Verification Logic:** When Site A connects to Site B, Site A verifies Site B's certificate against the bundle. Since Site B's CA public key is in the bundle, the connection is accepted. + +### 3. Tooling + +* We will use **cert-manager** (deployed via Operator on OKD) rather than OKD's built-in `service-ca`. This provides us with standard CRDs (`Issuer`, `Certificate`) to manage the lifecycle, rotation, and complex SANs (Subject Alternative Names) required for external connectivity. +* Harmony will manage installation, configuration and bundle creation across all sites + +## Rationale + +**Security Blast Radius (The "Key Leak" Scenario)** +If we used a single global CA and the private key for Site A was compromised (e.g., physical theft of a server from a basement), the attacker could impersonate *any* site in the global mesh. +By using Per-Cluster CAs: +* If Site A is compromised, only Site A's identity is stolen. +* We can "evict" Site A from the mesh simply by removing Site A's Public CA from the `ca-bundle.pem` on the remaining healthy clusters and reloading. The attacker can no longer authenticate. + +**Decentralized Autonomy** +This aligns with the "Humane Computing" vision. A local cluster owns its identity. It does not depend on a central authority to issue its certificates. It can function in isolation (offline) indefinitely without needing to "phone home" to renew credentials. + +## Consequences + +**Positive** +* **High Security:** Compromise of one node does not compromise the global mesh. +* **Flexibility:** Easier to integrate with third-party clusters or partners by simply adding their public CA to the bundle. +* **Standardization:** `cert-manager` is the industry standard, making the configuration portable to non-OKD K8s clusters if needed. + +**Negative** +* **Configuration Complexity:** We must manage a mechanism to distribute the `ca-bundle.pem` containing public keys to all sites. This should be automated (e.g., via a Harmony Agent) to ensure timely updates and revocation. +* **Revocation Latency:** Revoking a compromised cluster requires updating and reloading the bundle on all other clusters. This is slower than OCSP/CRL but acceptable for infrastructure-level trust if automation is in place. + +--- + +# 2. Concrete overview of the process, how it can be implemented manually across multiple OKD clusters + +All of this will be automated via Harmony, but to understand correctly the process it is outlined in details here : + +## 1. Deploying and Configuring cert-manager on OKD + +While OKD has a built-in `service-ca` controller, it is "opinionated" and primarily signs certs for internal services (like `my-svc.my-namespace.svc`). It is **not suitable** for the Harmony Global Mesh because you cannot easily control the Subject Alternative Names (SANs) for external routes (e.g., `nats.site-a.nationtech.io`), nor can you easily export its CA to other clusters. + +**The Solution:** Use the **cert-manager Operator for Red Hat OpenShift**. + +### Step 1: Install the Operator +1. Log in to the OKD Web Console. +2. Navigate to **Operators** -> **OperatorHub**. +3. Search for **"cert-manager"**. +4. Choose the **"cert-manager Operator for Red Hat OpenShift"** (Red Hat provided) or the community version. +5. Click **Install**. Use the default settings (Namespace: `cert-manager-operator`). + +### Step 2: Create the "Island" CA (The Issuer) +Once installed, you define your cluster's unique identity. Apply this YAML to your NATS namespace. + +```yaml +# filepath: k8s/01-issuer.yaml +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: harmony-selfsigned-issuer + namespace: harmony-nats +spec: + selfSigned: {} +--- +# This generates the unique Root CA for THIS specific cluster +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: harmony-root-ca + namespace: harmony-nats +spec: + isCA: true + commonName: "harmony-site-a-ca" # CHANGE THIS per cluster (e.g., site-b-ca) + duration: 87600h # 10 years + renewBefore: 2160h # 3 months before expiry + secretName: harmony-root-ca-secret + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: harmony-selfsigned-issuer + kind: Issuer + group: cert-manager.io +--- +# This Issuer uses the Root CA generated above to sign NATS certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: harmony-ca-issuer + namespace: harmony-nats +spec: + ca: + secretName: harmony-root-ca-secret +``` + +### Step 3: Generate the NATS Server Certificate +This certificate will be used by the NATS server. It includes both internal DNS names (for local clients) and external DNS names (for the global mesh). + +```yaml +# filepath: k8s/02-nats-cert.yaml +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: nats-server-cert + namespace: harmony-nats +spec: + secretName: nats-server-tls + duration: 2160h # 90 days + renewBefore: 360h # 15 days + issuerRef: + name: harmony-ca-issuer + kind: Issuer + # CRITICAL: Define all names this server can be reached by + dnsNames: + - "nats" + - "nats.harmony-nats.svc" + - "nats.harmony-nats.svc.cluster.local" + - "*.nats.harmony-nats.svc.cluster.local" + - "nats-gateway.site-a.nationtech.io" # External Route for Mesh +``` + +## 2. Implementing the "Islands of Trust" (Trust Bundle) + +To make Site A and Site B talk, you need to exchange **Public Keys**. + +1. **Extract Public CA from Site A:** + ```bash + oc get secret harmony-root-ca-secret -n harmony-nats -o jsonpath='{.data.ca\.crt}' | base64 -d > site-a.crt + ``` +2. **Extract Public CA from Site B:** + ```bash + oc get secret harmony-root-ca-secret -n harmony-nats -o jsonpath='{.data.ca\.crt}' | base64 -d > site-b.crt + ``` +3. **Create the Bundle:** + Combine them into one file. + ```bash + cat site-a.crt site-b.crt > ca-bundle.crt + ``` +4. **Upload Bundle to Both Clusters:** + Create a ConfigMap or Secret in *both* clusters containing this combined bundle. + ```bash + oc create configmap nats-trust-bundle --from-file=ca.crt=ca-bundle.crt -n harmony-nats + ``` +5. **Configure NATS:** + Mount this ConfigMap and point NATS to it. + + ```conf + # nats.conf snippet + tls { + cert_file: "/etc/nats-certs/tls.crt" + key_file: "/etc/nats-certs/tls.key" + # Point to the bundle containing BOTH Site A and Site B public CAs + ca_file: "/etc/nats-trust/ca.crt" + } + ``` + +This setup ensures that Site A can verify Site B's certificate (signed by `harmony-site-b-ca`) because Site B's CA is in Site A's trust store, and vice versa, without ever sharing the private keys that generated them. -- 2.39.5 From ced371ca4347229ef8e280a07fe22a56bf808ce8 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Fri, 16 Jan 2026 09:45:59 -0500 Subject: [PATCH 06/21] feat: Nats supercluster example working --- examples/nats-supercluster/Cargo.toml | 18 ++ examples/nats-supercluster/env_example.sh | 6 + examples/nats-supercluster/src/main.rs | 196 ++++++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 examples/nats-supercluster/Cargo.toml create mode 100644 examples/nats-supercluster/env_example.sh create mode 100644 examples/nats-supercluster/src/main.rs diff --git a/examples/nats-supercluster/Cargo.toml b/examples/nats-supercluster/Cargo.toml new file mode 100644 index 00000000..fd1591a5 --- /dev/null +++ b/examples/nats-supercluster/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "example-nats-supercluster" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true +publish = false + +[dependencies] +harmony = { path = "../../harmony" } +harmony_cli = { path = "../../harmony_cli" } +harmony_types = { path = "../../harmony_types" } +cidr = { workspace = true } +tokio = { workspace = true } +harmony_macros = { path = "../../harmony_macros" } +log = { workspace = true } +env_logger = { workspace = true } +url = { workspace = true } diff --git a/examples/nats-supercluster/env_example.sh b/examples/nats-supercluster/env_example.sh new file mode 100644 index 00000000..cb236473 --- /dev/null +++ b/examples/nats-supercluster/env_example.sh @@ -0,0 +1,6 @@ +# Cluster 1 +export HARMONY_NATS_SITE_1="kubeconfig=$HOME/.config/nt/kube/config,context=your_cluster_1_kube_context_name" +export HARMONY_NATS_SITE_1_DOMAIN="your_cluster_1_public_domain" +# Cluster 2 +export HARMONY_NATS_SITE_2="kubeconfig=$HOME/.config/nt/kube/config,context=your_cluster_2_kube_context_name" +export HARMONY_NATS_SITE_2_DOMAIN="your_cluster_2_public_domain" diff --git a/examples/nats-supercluster/src/main.rs b/examples/nats-supercluster/src/main.rs new file mode 100644 index 00000000..9af3948a --- /dev/null +++ b/examples/nats-supercluster/src/main.rs @@ -0,0 +1,196 @@ +use std::str::FromStr; + +use harmony::{ + inventory::Inventory, + modules::helm::chart::{HelmChartScore, HelmRepository, NonBlankString}, + topology::{HelmCommand, K8sAnywhereConfig, K8sAnywhereTopology, TlsRouter, Topology}, +}; +use harmony_macros::hurl; +use log::{debug, info}; + +#[tokio::main] +async fn main() { + let site1_topo = K8sAnywhereTopology::with_config(K8sAnywhereConfig::remote_k8s_from_env_var( + "HARMONY_NATS_SITE_1", + )); + let site2_topo = K8sAnywhereTopology::with_config(K8sAnywhereConfig::remote_k8s_from_env_var( + "HARMONY_NATS_SITE_2", + )); + let (t1, t2) = tokio::join!(site1_topo.ensure_ready(), site2_topo.ensure_ready(),); + + t1.unwrap(); + t2.unwrap(); + + let site1_domain = std::env::var("HARMONY_NATS_SITE_1_DOMAIN") + .expect("HARMONY_NATS_SITE_1_DOMAIN env var not found"); + let site2_domain = std::env::var("HARMONY_NATS_SITE_2_DOMAIN") + .expect("HARMONY_NATS_SITE_2_DOMAIN env var not found"); + + // TODO automate creation of this ca bundle + // It is simply a secret that contains one key ca.crt + // And the value is the base64 with each clusters ca.crt concatenated + let supercluster_ca_secret_name = "nats-supercluster-ca-bundle"; + + let nats_site_1 = NatsCluster { + replicas: 1, + name: "nats-site1", + gateway_advertise: format!("nats-site1-gw.{site1_domain}:443"), + supercluster_ca_secret_name, + tls_secret_name: "nats-gateway-tls", + jetstream_enabled: "false", + }; + + let nats_site_2 = NatsCluster { + replicas: 1, + name: "nats-site2", + gateway_advertise: format!("nats-site2-gw.{site2_domain}:443"), + supercluster_ca_secret_name, + tls_secret_name: "nats-gateway-tls", + jetstream_enabled: "false", + }; + + tokio::join!( + deploy_nats( + site1_topo, + &nats_site_1, + vec![&nats_site_2] + ), + deploy_nats( + site2_topo, + &nats_site_2, + vec![&nats_site_1] + ), + ); +} + +struct NatsCluster { + replicas: usize, + name: &'static str, + gateway_advertise: String, + supercluster_ca_secret_name: &'static str, + tls_secret_name: &'static str, + jetstream_enabled: &'static str, +} + +async fn deploy_nats( + topology: T, + cluster: &NatsCluster, + peers: Vec<&NatsCluster>, +) { + let mut gateway_gateways = String::new(); + for peer in peers { + // Construct wss:// URLs on port 443 for the remote gateways + gateway_gateways.push_str(&format!( + r#" + - name: {} + urls: + - nats://{}"#, + peer.name, peer.gateway_advertise + )); + } + let domain = topology.get_internal_domain().await.unwrap().unwrap(); + + // Inject gateway config into the 'merge' block to comply with chart structure + let values_yaml = Some(format!( + r#"config: + merge: + authorization: + default_permissions: + publish: ["TEST.*"] + subscribe: ["PUBLIC.>"] + users: + # - user: "admin" + # password: "admin_1" + # permissions: + # publish: ">" + # subscribe: ">" + - password: "enGk0cgZUabM6bN6FXHT" + user: "testUser" + accounts: + system: + users: + - user: "admin" + password: "admin_2" + logtime: true + debug: true + trace: true + system_account: system + cluster: + name: {cluster_name} + enabled: true + replicas: {replicas} + jetstream: + enabled: {jetstream_enabled} + fileStorage: + enabled: true + size: 10Gi + storageDirectory: /data/jetstream + leafnodes: + enabled: false + websocket: + enabled: false + ingress: + enabled: true + className: openshift-default + pathType: Prefix + hosts: + - nats-ws.{domain} + gateway: + enabled: true + port: 7222 + name: {cluster_name} + merge: + advertise: {gateway_advertise} + gateways: {gateway_gateways} + tls: + enabled: true + secretName: {tls_secret_name} + # merge: + # ca_file: "/etc/nats-certs/gateway/ca.crt" +service: + ports: + gateway: + enabled: true +tlsCA: + enabled: true + secretName: {supercluster_ca_secret_name} +natsBox: + container: + image: + tag: nonroot"#, + cluster_name = cluster.name, + replicas = cluster.replicas, + domain = domain, + gateway_gateways = gateway_gateways, + gateway_advertise = cluster.gateway_advertise, + tls_secret_name = cluster.tls_secret_name, + jetstream_enabled = cluster.jetstream_enabled, +supercluster_ca_secret_name = cluster.supercluster_ca_secret_name, + )); + let namespace = "harmony-nats"; + + debug!("Prepared Helm Chart values : \n{values_yaml:#?}"); + let nats = HelmChartScore { + namespace: Some(NonBlankString::from_str(namespace).unwrap()), + release_name: NonBlankString::from_str(&cluster.name).unwrap(), + chart_name: NonBlankString::from_str("nats/nats").unwrap(), + chart_version: None, + values_overrides: None, + values_yaml, + create_namespace: true, + install_only: false, + repository: Some(HelmRepository::new( + "nats".to_string(), + hurl!("https://nats-io.github.io/k8s/helm/charts/"), + true, + )), + }; + + harmony_cli::run(Inventory::autoload(), topology, vec![Box::new(nats)], None) + .await + .unwrap(); + + info!( + "Enjoy! You can test your nats cluster by running : `kubectl exec -n {namespace} -it deployment/nats-box -- nats pub test hi`" + ); +} -- 2.39.5 From 043cd561e9062247b44954c283cf480ce522d6f5 Mon Sep 17 00:00:00 2001 From: wjro Date: Tue, 13 Jan 2026 12:09:56 -0500 Subject: [PATCH 07/21] feat: added cert manager capability as well as scores to install openshift subscription to community cert-manager operator --- examples/cert_manager/Cargo.toml | 19 ++++++ examples/cert_manager/src/main.rs | 26 ++++++++ .../topology/k8s_anywhere/k8s_anywhere.rs | 25 +++++++ .../src/modules/cert_manager/capability.rs | 18 +++++ harmony/src/modules/cert_manager/mod.rs | 3 + harmony/src/modules/cert_manager/operator.rs | 64 ++++++++++++++++++ harmony/src/modules/cert_manager/score_k8s.rs | 66 +++++++++++++++++++ 7 files changed, 221 insertions(+) create mode 100644 examples/cert_manager/Cargo.toml create mode 100644 examples/cert_manager/src/main.rs create mode 100644 harmony/src/modules/cert_manager/capability.rs create mode 100644 harmony/src/modules/cert_manager/operator.rs create mode 100644 harmony/src/modules/cert_manager/score_k8s.rs diff --git a/examples/cert_manager/Cargo.toml b/examples/cert_manager/Cargo.toml new file mode 100644 index 00000000..a67fcf83 --- /dev/null +++ b/examples/cert_manager/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "cert_manager" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true +publish = false + +[dependencies] +harmony = { path = "../../harmony" } +harmony_cli = { path = "../../harmony_cli" } +harmony_types = { path = "../../harmony_types" } +cidr = { workspace = true } +tokio = { workspace = true } +harmony_macros = { path = "../../harmony_macros" } +log = { workspace = true } +env_logger = { workspace = true } +url = { workspace = true } +assert_cmd = "2.0.16" diff --git a/examples/cert_manager/src/main.rs b/examples/cert_manager/src/main.rs new file mode 100644 index 00000000..ee0a2033 --- /dev/null +++ b/examples/cert_manager/src/main.rs @@ -0,0 +1,26 @@ +use harmony::{ + inventory::Inventory, + modules::{ + cert_manager::{ + capability::CertificateManagementConfig, score_k8s::CertificateManagementScore, + }, + postgresql::{PostgreSQLScore, capability::PostgreSQLConfig}, + }, + topology::K8sAnywhereTopology, +}; + +#[tokio::main] +async fn main() { + let cert_manager = CertificateManagementScore { + config: CertificateManagementConfig {}, + }; + + harmony_cli::run( + Inventory::autoload(), + K8sAnywhereTopology::from_env(), + vec![Box::new(cert_manager)], + None, + ) + .await + .unwrap(); +} diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index c3a6baa9..9946f435 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -17,6 +17,10 @@ use crate::{ interpret::InterpretStatus, inventory::Inventory, modules::{ + cert_manager::{ + capability::{CertificateManagement, CertificateManagementConfig}, + operator::CertManagerOperatorScore, + }, k3d::K3DInstallationScore, k8s::ingress::{K8sIngressScore, PathType}, monitoring::{ @@ -384,6 +388,27 @@ impl Serialize for K8sAnywhereTopology { } } +#[async_trait] +impl CertificateManagement for K8sAnywhereTopology { + async fn install( + &self, + config: &CertificateManagementConfig, + ) -> Result { + let cert_management_operator = CertManagerOperatorScore::default(); + + cert_management_operator + .interpret(&Inventory::empty(), self) + .await + .map_err(|e| PreparationError { msg: e.to_string() })?; + Ok(PreparationOutcome::Success { + details: format!( + "Installed cert-manager into ns: {}", + cert_management_operator.namespace + ), + }) + } +} + impl K8sAnywhereTopology { pub fn from_env() -> Self { Self { diff --git a/harmony/src/modules/cert_manager/capability.rs b/harmony/src/modules/cert_manager/capability.rs new file mode 100644 index 00000000..fffe5373 --- /dev/null +++ b/harmony/src/modules/cert_manager/capability.rs @@ -0,0 +1,18 @@ +use async_trait::async_trait; +use serde::Serialize; + +use crate::{ + interpret::Outcome, + topology::{PreparationError, PreparationOutcome}, +}; + +#[async_trait] +pub trait CertificateManagement: Send + Sync { + async fn install( + &self, + config: &CertificateManagementConfig, + ) -> Result; +} + +#[derive(Debug, Clone, Serialize)] +pub struct CertificateManagementConfig {} diff --git a/harmony/src/modules/cert_manager/mod.rs b/harmony/src/modules/cert_manager/mod.rs index 032439ea..3e4da4f3 100644 --- a/harmony/src/modules/cert_manager/mod.rs +++ b/harmony/src/modules/cert_manager/mod.rs @@ -1,3 +1,6 @@ +pub mod capability; pub mod cluster_issuer; mod helm; +pub mod operator; +pub mod score_k8s; pub use helm::*; diff --git a/harmony/src/modules/cert_manager/operator.rs b/harmony/src/modules/cert_manager/operator.rs new file mode 100644 index 00000000..e4112db2 --- /dev/null +++ b/harmony/src/modules/cert_manager/operator.rs @@ -0,0 +1,64 @@ +use kube::api::ObjectMeta; +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::k8s::{ + apps::crd::{Subscription, SubscriptionSpec}, + resource::K8sResourceScore, + }, + score::Score, + topology::{K8sclient, Topology, k8s::K8sClient}, +}; + +/// Install the Cert-Manager Operator via RedHat Community Operators registry.redhat.io/redhat/community-operator-index:v4.19 +/// This Score creates a Subscription CR in the specified namespace + +#[derive(Debug, Clone, Serialize)] +pub struct CertManagerOperatorScore { + pub namespace: String, + pub channel: String, + pub install_plan_approval: String, + pub source: String, + pub source_namespace: String, +} + +impl Default for CertManagerOperatorScore { + fn default() -> Self { + Self { + namespace: "openshift-operators".to_string(), + channel: "stable".to_string(), + install_plan_approval: "Automatic".to_string(), + source: "community-operators".to_string(), + source_namespace: "openshift-marketplace".to_string(), + } + } +} + +impl Score for CertManagerOperatorScore { + fn name(&self) -> String { + "CertManagerOperatorScore".to_string() + } + + fn create_interpret(&self) -> Box> { + let metadata = ObjectMeta { + name: Some("cert-manager".to_string()), + namespace: Some(self.namespace.clone()), + ..ObjectMeta::default() + }; + + let spec = SubscriptionSpec { + channel: Some(self.channel.clone()), + config: None, + install_plan_approval: Some(self.install_plan_approval.clone()), + name: "cert-manager".to_string(), + source: self.source.clone(), + source_namespace: self.source_namespace.clone(), + starting_csv: None, + }; + + let subscription = Subscription { metadata, spec }; + + K8sResourceScore::single(subscription, Some(self.namespace.clone())).create_interpret() + } +} diff --git a/harmony/src/modules/cert_manager/score_k8s.rs b/harmony/src/modules/cert_manager/score_k8s.rs new file mode 100644 index 00000000..18e08264 --- /dev/null +++ b/harmony/src/modules/cert_manager/score_k8s.rs @@ -0,0 +1,66 @@ +use async_trait::async_trait; +use harmony_types::id::Id; +use serde::Serialize; + +use crate::{ + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + modules::cert_manager::capability::{CertificateManagement, CertificateManagementConfig}, + score::Score, + topology::Topology, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct CertificateManagementScore { + pub config: CertificateManagementConfig, +} + +impl Score for CertificateManagementScore { + fn name(&self) -> String { + "CertificateManagementScore".to_string() + } + + fn create_interpret(&self) -> Box> { + Box::new(CertificateManagementInterpret { + config: self.config.clone(), + }) + } +} + +#[derive(Debug)] +struct CertificateManagementInterpret { + config: CertificateManagementConfig, +} + +#[async_trait] +impl Interpret for CertificateManagementInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + let cert_management = topology + .install(&self.config) + .await + .map_err(|e| InterpretError::new(e.to_string()))?; + + Ok(Outcome::success(format!("Installed CertificateManagement"))) + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("CertificateManagementInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} -- 2.39.5 From d3a8171e3cefa7bf73a4b2cee9fafc1aa9f0b9d5 Mon Sep 17 00:00:00 2001 From: wjro Date: Tue, 13 Jan 2026 14:05:10 -0500 Subject: [PATCH 08/21] feat(cert-manager): added crds for cert-manager --- .../modules/cert_manager/crd/certificate.rs | 113 ++++++++++++++++++ .../cert_manager/crd/cluster_issuer.rs | 45 +++++++ .../src/modules/cert_manager/crd/issuer.rs | 44 +++++++ harmony/src/modules/cert_manager/crd/mod.rs | 63 ++++++++++ harmony/src/modules/cert_manager/mod.rs | 1 + 5 files changed, 266 insertions(+) create mode 100644 harmony/src/modules/cert_manager/crd/certificate.rs create mode 100644 harmony/src/modules/cert_manager/crd/cluster_issuer.rs create mode 100644 harmony/src/modules/cert_manager/crd/issuer.rs create mode 100644 harmony/src/modules/cert_manager/crd/mod.rs diff --git a/harmony/src/modules/cert_manager/crd/certificate.rs b/harmony/src/modules/cert_manager/crd/certificate.rs new file mode 100644 index 00000000..7c0866b7 --- /dev/null +++ b/harmony/src/modules/cert_manager/crd/certificate.rs @@ -0,0 +1,113 @@ +use kube::{CustomResource, api::ObjectMeta}; +use serde::{Deserialize, Serialize}; + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug)] +#[kube( + group = "cert-manager.io", + version = "v1", + kind = "Certificate", + plural = "certificates", + namespaced = true, + schema = "disabled" +)] +#[serde(rename_all = "camelCase")] +pub struct CertificateSpec { + /// Name of the Secret where the certificate will be stored + pub secret_name: String, + + /// Common Name (optional but often discouraged in favor of SANs) + #[serde(skip_serializing_if = "Option::is_none")] + pub common_name: Option, + + /// DNS Subject Alternative Names + #[serde(skip_serializing_if = "Option::is_none")] + pub dns_names: Option>, + + /// IP Subject Alternative Names + #[serde(skip_serializing_if = "Option::is_none")] + pub ip_addresses: Option>, + + /// Certificate duration (e.g. "2160h") + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, + + /// How long before expiry cert-manager should renew + #[serde(skip_serializing_if = "Option::is_none")] + pub renew_before: Option, + + /// Reference to the Issuer or ClusterIssuer + pub issuer_ref: IssuerRef, + + /// Is this a CA certificate + #[serde(skip_serializing_if = "Option::is_none")] + pub is_ca: Option, + + /// Private key configuration + #[serde(skip_serializing_if = "Option::is_none")] + pub private_key: Option, +} + +impl Default for Certificate { + fn default() -> Self { + Certificate { + metadata: ObjectMeta::default(), + spec: CertificateSpec::default(), + } + } +} + +impl Default for CertificateSpec { + fn default() -> Self { + Self { + secret_name: String::new(), + common_name: None, + dns_names: None, + ip_addresses: None, + duration: None, + renew_before: None, + issuer_ref: IssuerRef::default(), + is_ca: None, + private_key: None, + } + } +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct IssuerRef { + pub name: String, + + /// Either "Issuer" or "ClusterIssuer" + #[serde(skip_serializing_if = "Option::is_none")] + pub kind: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub group: Option, +} + +impl Default for IssuerRef { + fn default() -> Self { + Self { + name: String::new(), + kind: None, + group: None, + } + } +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct PrivateKey { + /// RSA or ECDSA + #[serde(skip_serializing_if = "Option::is_none")] + pub algorithm: Option, + + /// Key size (e.g. 2048, 4096) + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, + + /// Rotation policy: "Never" or "Always" + #[serde(skip_serializing_if = "Option::is_none")] + pub rotation_policy: Option, +} + diff --git a/harmony/src/modules/cert_manager/crd/cluster_issuer.rs b/harmony/src/modules/cert_manager/crd/cluster_issuer.rs new file mode 100644 index 00000000..49395489 --- /dev/null +++ b/harmony/src/modules/cert_manager/crd/cluster_issuer.rs @@ -0,0 +1,45 @@ +use kube::{CustomResource, api::ObjectMeta}; +use serde::{Deserialize, Serialize}; + +use crate::modules::cert_manager::crd::{AcmeIssuer, CaIssuer, SelfSignedIssuer}; + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug)] +#[kube( + group = "cert-manager.io", + version = "v1", + kind = "ClusterIssuer", + plural = "clusterissuers", + namespaced = false, + schema = "disabled" +)] +#[serde(rename_all = "camelCase")] +pub struct ClusterIssuerSpec { + #[serde(skip_serializing_if = "Option::is_none")] + pub self_signed: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub ca: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub acme: Option, +} + +impl Default for ClusterIssuer { + fn default() -> Self { + ClusterIssuer { + metadata: ObjectMeta::default(), + spec: ClusterIssuerSpec::default(), + } + } +} + +impl Default for ClusterIssuerSpec { + fn default() -> Self { + Self { + self_signed: None, + ca: None, + acme: None, + } + } +} + diff --git a/harmony/src/modules/cert_manager/crd/issuer.rs b/harmony/src/modules/cert_manager/crd/issuer.rs new file mode 100644 index 00000000..e4d57d33 --- /dev/null +++ b/harmony/src/modules/cert_manager/crd/issuer.rs @@ -0,0 +1,44 @@ +use kube::{CustomResource, api::ObjectMeta}; +use serde::{Deserialize, Serialize}; + +use crate::modules::cert_manager::crd::{AcmeIssuer, CaIssuer, SelfSignedIssuer}; + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug)] +#[kube( + group = "cert-manager.io", + version = "v1", + kind = "Issuer", + plural = "issuers", + namespaced = true, + schema = "disabled" +)] +#[serde(rename_all = "camelCase")] +pub struct IssuerSpec { + #[serde(skip_serializing_if = "Option::is_none")] + pub self_signed: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub ca: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub acme: Option, +} + +impl Default for Issuer { + fn default() -> Self { + Issuer { + metadata: ObjectMeta::default(), + spec: IssuerSpec::default(), + } + } +} + +impl Default for IssuerSpec { + fn default() -> Self { + Self { + self_signed: None, + ca: None, + acme: None, + } + } +} diff --git a/harmony/src/modules/cert_manager/crd/mod.rs b/harmony/src/modules/cert_manager/crd/mod.rs new file mode 100644 index 00000000..c9050b7d --- /dev/null +++ b/harmony/src/modules/cert_manager/crd/mod.rs @@ -0,0 +1,63 @@ +use serde::{Deserialize, Serialize}; + + +pub mod certificate; +pub mod issuer; +pub mod cluster_issuer; + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct CaIssuer { + /// Secret containing `tls.crt` and `tls.key` + pub secret_name: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct SelfSignedIssuer {} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct AcmeIssuer { + pub server: String, + pub email: String, + + /// Secret used to store the ACME account private key + pub private_key_secret_ref: SecretKeySelector, + + pub solvers: Vec, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct SecretKeySelector { + pub name: String, + pub key: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct AcmeSolver { + #[serde(skip_serializing_if = "Option::is_none")] + pub http01: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub dns01: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Dns01Solver {} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Http01Solver { + pub ingress: IngressSolver, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct IngressSolver { + #[serde(skip_serializing_if = "Option::is_none")] + pub class: Option, +} diff --git a/harmony/src/modules/cert_manager/mod.rs b/harmony/src/modules/cert_manager/mod.rs index 3e4da4f3..430252d2 100644 --- a/harmony/src/modules/cert_manager/mod.rs +++ b/harmony/src/modules/cert_manager/mod.rs @@ -3,4 +3,5 @@ pub mod cluster_issuer; mod helm; pub mod operator; pub mod score_k8s; +pub mod crd; pub use helm::*; -- 2.39.5 From 947733b2400fbce445692015be009a3c32eb727a Mon Sep 17 00:00:00 2001 From: wjro Date: Tue, 13 Jan 2026 15:43:58 -0500 Subject: [PATCH 09/21] wip: added scores and basic implentation to create certs and issuers --- examples/cert_manager/src/main.rs | 8 ++- .../topology/k8s_anywhere/k8s_anywhere.rs | 58 ++++++++++++++++++- .../src/modules/cert_manager/capability.rs | 40 +++++++++++-- harmony/src/modules/cert_manager/crd/mod.rs | 3 + .../cert_manager/crd/score_certificate.rs | 47 +++++++++++++++ .../cert_manager/crd/score_cluster_issuer.rs | 51 ++++++++++++++++ .../modules/cert_manager/crd/score_issuer.rs | 48 +++++++++++++++ 7 files changed, 248 insertions(+), 7 deletions(-) create mode 100644 harmony/src/modules/cert_manager/crd/score_certificate.rs create mode 100644 harmony/src/modules/cert_manager/crd/score_cluster_issuer.rs create mode 100644 harmony/src/modules/cert_manager/crd/score_issuer.rs diff --git a/examples/cert_manager/src/main.rs b/examples/cert_manager/src/main.rs index ee0a2033..df6484e2 100644 --- a/examples/cert_manager/src/main.rs +++ b/examples/cert_manager/src/main.rs @@ -12,7 +12,13 @@ use harmony::{ #[tokio::main] async fn main() { let cert_manager = CertificateManagementScore { - config: CertificateManagementConfig {}, + config: CertificateManagementConfig { + name: todo!(), + namespace: todo!(), + acme_issuer: todo!(), + ca_issuer: todo!(), + self_signed: todo!(), + }, }; harmony_cli::run( diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 9946f435..20705ed0 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -18,7 +18,8 @@ use crate::{ inventory::Inventory, modules::{ cert_manager::{ - capability::{CertificateManagement, CertificateManagementConfig}, + capability::{CertificateManagement, CertificateManagementConfig, Issuer}, + crd::{score_certificate::CertificateScore, score_issuer::IssuerScore}, operator::CertManagerOperatorScore, }, k3d::K3DInstallationScore, @@ -407,6 +408,38 @@ impl CertificateManagement for K8sAnywhereTopology { ), }) } + + async fn ensure_ready( + &self, + config: &CertificateManagementConfig, + ) -> Result { + self.certificate_issuer_ready(Issuer::Issuer, config) + .await?; + Ok(PreparationOutcome::Success { + details: "issuer ready".to_string(), + }) + } + + async fn create_certificate( + &self, + cert_name: String, + config: &CertificateManagementConfig, + ) -> Result { + let cert = CertificateScore { + name: cert_name, + config: config.clone(), + }; + cert.interpret(&Inventory::empty(), self) + .await + .map_err(|e| PreparationError { msg: e.to_string() })?; + + Ok(PreparationOutcome::Success { + details: format!( + "Created cert into ns: {:#?}", + config.namespace.clone() + ), + }) + } } impl K8sAnywhereTopology { @@ -960,6 +993,29 @@ impl K8sAnywhereTopology { ), }) } + + async fn certificate_issuer_ready( + &self, + issuer: Issuer, + config: &CertificateManagementConfig, + ) -> Result { + match issuer { + Issuer::ClusterIssuer => todo!(), + + Issuer::Issuer => { + let issuer_score = IssuerScore { + config: config.clone(), + }; + issuer_score + .interpret(&Inventory::empty(), self) + .await + .map_err(|e| PreparationError { msg: e.to_string() })?; + Ok(PreparationOutcome::Success { + details: format!("issuer of kind {} is ready", issuer.to_string()), + }) + } + } + } } #[derive(Clone, Debug)] diff --git a/harmony/src/modules/cert_manager/capability.rs b/harmony/src/modules/cert_manager/capability.rs index fffe5373..bd0c921f 100644 --- a/harmony/src/modules/cert_manager/capability.rs +++ b/harmony/src/modules/cert_manager/capability.rs @@ -1,10 +1,7 @@ use async_trait::async_trait; use serde::Serialize; -use crate::{ - interpret::Outcome, - topology::{PreparationError, PreparationOutcome}, -}; +use crate::{modules::cert_manager::crd::{AcmeIssuer, CaIssuer}, topology::{PreparationError, PreparationOutcome}}; #[async_trait] pub trait CertificateManagement: Send + Sync { @@ -12,7 +9,40 @@ pub trait CertificateManagement: Send + Sync { &self, config: &CertificateManagementConfig, ) -> Result; + + async fn ensure_ready( + &self, + config: &CertificateManagementConfig, + ) -> Result; + + async fn create_certificate( + &self, + cert_name: String, + config: &CertificateManagementConfig, + ) -> Result; } #[derive(Debug, Clone, Serialize)] -pub struct CertificateManagementConfig {} +pub struct CertificateManagementConfig { + pub name: String, + pub namespace: Option, + pub acme_issuer: Option, + pub ca_issuer: Option, + pub self_signed: bool, +} + +#[derive(Serialize)] +pub enum Issuer { + ClusterIssuer, + Issuer, +} + +impl std::fmt::Display for Issuer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Issuer::Issuer => f.write_str("Issuer"), + Issuer::ClusterIssuer => f.write_str("ClusterIssuer"), + } + } +} + diff --git a/harmony/src/modules/cert_manager/crd/mod.rs b/harmony/src/modules/cert_manager/crd/mod.rs index c9050b7d..70ea4693 100644 --- a/harmony/src/modules/cert_manager/crd/mod.rs +++ b/harmony/src/modules/cert_manager/crd/mod.rs @@ -4,6 +4,9 @@ use serde::{Deserialize, Serialize}; pub mod certificate; pub mod issuer; pub mod cluster_issuer; +//pub mod score_cluster_issuer; +pub mod score_issuer; +pub mod score_certificate; #[derive(Deserialize, Serialize, Clone, Debug)] #[serde(rename_all = "camelCase")] diff --git a/harmony/src/modules/cert_manager/crd/score_certificate.rs b/harmony/src/modules/cert_manager/crd/score_certificate.rs new file mode 100644 index 00000000..53c2a346 --- /dev/null +++ b/harmony/src/modules/cert_manager/crd/score_certificate.rs @@ -0,0 +1,47 @@ +use kube::api::ObjectMeta; +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::{ + cert_manager::{ + capability::CertificateManagementConfig, + crd::certificate::{Certificate, CertificateSpec, IssuerRef}, + }, + k8s::resource::K8sResourceScore, + }, + score::Score, + topology::{K8sclient, Topology}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct CertificateScore { + pub name: String, + pub config: CertificateManagementConfig, +} + +impl Score for CertificateScore { + fn name(&self) -> String { + "CertificateScore".to_string() + } + + fn create_interpret(&self) -> Box> { + let cert = Certificate { + metadata: ObjectMeta { + name: Some(self.name.clone()), + namespace: self.config.namespace.clone(), + ..Default::default() + }, + spec: CertificateSpec { + secret_name: format!("{}-tls", self.name.clone()), + issuer_ref: IssuerRef { + name: self.config.name.clone(), + kind: Some("Issuer".into()), + group: Some("cert-manager.io".into()), + }, + ..Default::default() + }, + }; + K8sResourceScore::single(cert, self.config.namespace.clone()).create_interpret() + } +} diff --git a/harmony/src/modules/cert_manager/crd/score_cluster_issuer.rs b/harmony/src/modules/cert_manager/crd/score_cluster_issuer.rs new file mode 100644 index 00000000..5437fc7c --- /dev/null +++ b/harmony/src/modules/cert_manager/crd/score_cluster_issuer.rs @@ -0,0 +1,51 @@ +use kube::api::ObjectMeta; +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::{ + cert_manager::crd::{ + AcmeIssuer, CaIssuer, SelfSignedIssuer, + cluster_issuer::{ClusterIssuer, ClusterIssuerSpec}, + }, + k8s::resource::K8sResourceScore, + }, + score::Score, + topology::{K8sclient, Topology}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct ClusterIssuerScore { + name: String, + acme_issuer: Option, + ca_issuer: Option, + self_signed: bool, +} + +impl Score for ClusterIssuerScore { + fn name(&self) -> String { + "ClusterIssuerScore".to_string() + } + + fn create_interpret(&self) -> Box> { + let metadata = ObjectMeta { + name: Some(self.name.clone()), + namespace: None, + ..ObjectMeta::default() + }; + + let spec = ClusterIssuerSpec { + acme: self.acme_issuer.clone(), + ca: self.ca_issuer.clone(), + self_signed: if self.self_signed { + Some(SelfSignedIssuer::default()) + } else { + None + }, + }; + + let cluster_issuer = ClusterIssuer { metadata, spec }; + + K8sResourceScore::single(cluster_issuer, None).create_interpret() + } +} diff --git a/harmony/src/modules/cert_manager/crd/score_issuer.rs b/harmony/src/modules/cert_manager/crd/score_issuer.rs new file mode 100644 index 00000000..05af62dd --- /dev/null +++ b/harmony/src/modules/cert_manager/crd/score_issuer.rs @@ -0,0 +1,48 @@ +use kube::api::ObjectMeta; +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::{ + cert_manager::{capability::CertificateManagementConfig, crd::{ + AcmeIssuer, CaIssuer, SelfSignedIssuer, + issuer::{Issuer, IssuerSpec}, + }}, + k8s::resource::K8sResourceScore, + }, + score::Score, + topology::{K8sclient, Topology}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct IssuerScore { + pub config: CertificateManagementConfig, +} + +impl Score for IssuerScore { + fn name(&self) -> String { + "IssuerScore".to_string() + } + + fn create_interpret(&self) -> Box> { + let metadata = ObjectMeta { + name: Some(self.config.name.clone()), + namespace: self.config.namespace.clone(), + ..ObjectMeta::default() + }; + + let spec = IssuerSpec { + acme: self.config.acme_issuer.clone(), + ca: self.config.ca_issuer.clone(), + self_signed: if self.config.self_signed { + Some(SelfSignedIssuer::default()) + } else { + None + }, + }; + + let issuer = Issuer { metadata, spec }; + + K8sResourceScore::single(issuer, self.config.namespace.clone()).create_interpret() + } +} -- 2.39.5 From 26256d9945631a671196716815f8b10d1e99324e Mon Sep 17 00:00:00 2001 From: wjro Date: Wed, 14 Jan 2026 14:39:05 -0500 Subject: [PATCH 10/21] fix: added create_issuer fn to trait and its implementation is k8sanywhere --- .../topology/k8s_anywhere/k8s_anywhere.rs | 92 ++++++++++++------- .../src/modules/cert_manager/capability.rs | 29 +++--- .../modules/cert_manager/crd/certificate.rs | 1 - .../cert_manager/crd/cluster_issuer.rs | 1 - harmony/src/modules/cert_manager/crd/mod.rs | 5 +- .../cert_manager/crd/score_certificate.rs | 9 +- .../modules/cert_manager/crd/score_issuer.rs | 13 ++- harmony/src/modules/cert_manager/mod.rs | 2 +- 8 files changed, 88 insertions(+), 64 deletions(-) diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 20705ed0..0a2746f9 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -18,7 +18,7 @@ use crate::{ inventory::Inventory, modules::{ cert_manager::{ - capability::{CertificateManagement, CertificateManagementConfig, Issuer}, + capability::{CertificateManagement, CertificateManagementConfig}, crd::{score_certificate::CertificateScore, score_issuer::IssuerScore}, operator::CertManagerOperatorScore, }, @@ -401,6 +401,7 @@ impl CertificateManagement for K8sAnywhereTopology { .interpret(&Inventory::empty(), self) .await .map_err(|e| PreparationError { msg: e.to_string() })?; + Ok(PreparationOutcome::Success { details: format!( "Installed cert-manager into ns: {}", @@ -413,31 +414,52 @@ impl CertificateManagement for K8sAnywhereTopology { &self, config: &CertificateManagementConfig, ) -> Result { - self.certificate_issuer_ready(Issuer::Issuer, config) - .await?; + todo!() + } + + async fn create_issuer( + &self, + issuer_name: String, + config: &CertificateManagementConfig, + ) -> Result { + let issuer_score = IssuerScore { + config: config.clone(), + }; + + issuer_score + .interpret(&Inventory::empty(), self) + .await + .map_err(|e| PreparationError { msg: e.to_string() })?; + Ok(PreparationOutcome::Success { - details: "issuer ready".to_string(), + details: format!("issuer of kind {} is ready", issuer_name), }) } async fn create_certificate( &self, cert_name: String, + issuer_name: String, config: &CertificateManagementConfig, ) -> Result { + self.certificate_issuer_ready( + issuer_name.clone(), + self.k8s_client().await.unwrap(), + config, + ) + .await?; + let cert = CertificateScore { - name: cert_name, + cert_name: cert_name, config: config.clone(), + issuer_name, }; cert.interpret(&Inventory::empty(), self) .await .map_err(|e| PreparationError { msg: e.to_string() })?; Ok(PreparationOutcome::Success { - details: format!( - "Created cert into ns: {:#?}", - config.namespace.clone() - ), + details: format!("Created cert into ns: {:#?}", config.namespace.clone()), }) } } @@ -461,6 +483,35 @@ impl K8sAnywhereTopology { } } + pub async fn certificate_issuer_ready( + &self, + issuer_name: String, + k8s_client: Arc, + config: &CertificateManagementConfig, + ) -> Result { + let ns = config.namespace.clone().ok_or_else(|| PreparationError { + msg: "namespace is required".to_string(), + })?; + + let gvk = GroupVersionKind { + group: "cert-manager.io".to_string(), + version: "v1".to_string(), + kind: "Issuer".to_string(), + }; + + match k8s_client + .get_resource_json_value(&issuer_name, Some(&ns), &gvk) + .await + { + Ok(_cert_issuer) => Ok(PreparationOutcome::Success { + details: format!("issuer of kind {} is ready", issuer_name), + }), + Err(e) => Err(PreparationError { + msg: format!("{} issuer {} not present", e.to_string(), issuer_name), + }), + } + } + pub async fn get_k8s_distribution(&self) -> Result<&KubernetesDistribution, PreparationError> { self.k8s_distribution .get_or_try_init(async || { @@ -993,29 +1044,6 @@ impl K8sAnywhereTopology { ), }) } - - async fn certificate_issuer_ready( - &self, - issuer: Issuer, - config: &CertificateManagementConfig, - ) -> Result { - match issuer { - Issuer::ClusterIssuer => todo!(), - - Issuer::Issuer => { - let issuer_score = IssuerScore { - config: config.clone(), - }; - issuer_score - .interpret(&Inventory::empty(), self) - .await - .map_err(|e| PreparationError { msg: e.to_string() })?; - Ok(PreparationOutcome::Success { - details: format!("issuer of kind {} is ready", issuer.to_string()), - }) - } - } - } } #[derive(Clone, Debug)] diff --git a/harmony/src/modules/cert_manager/capability.rs b/harmony/src/modules/cert_manager/capability.rs index bd0c921f..0c6940da 100644 --- a/harmony/src/modules/cert_manager/capability.rs +++ b/harmony/src/modules/cert_manager/capability.rs @@ -1,8 +1,12 @@ use async_trait::async_trait; use serde::Serialize; -use crate::{modules::cert_manager::crd::{AcmeIssuer, CaIssuer}, topology::{PreparationError, PreparationOutcome}}; +use crate::{ + modules::cert_manager::crd::{AcmeIssuer, CaIssuer}, + topology::{PreparationError, PreparationOutcome}, +}; +///TODO rust doc explaining issuer, certificate etc #[async_trait] pub trait CertificateManagement: Send + Sync { async fn install( @@ -15,9 +19,16 @@ pub trait CertificateManagement: Send + Sync { config: &CertificateManagementConfig, ) -> Result; + async fn create_issuer( + &self, + issuer_name: String, + config: &CertificateManagementConfig, + ) -> Result; + async fn create_certificate( &self, cert_name: String, + issuer_name: String, config: &CertificateManagementConfig, ) -> Result; } @@ -30,19 +41,3 @@ pub struct CertificateManagementConfig { pub ca_issuer: Option, pub self_signed: bool, } - -#[derive(Serialize)] -pub enum Issuer { - ClusterIssuer, - Issuer, -} - -impl std::fmt::Display for Issuer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Issuer::Issuer => f.write_str("Issuer"), - Issuer::ClusterIssuer => f.write_str("ClusterIssuer"), - } - } -} - diff --git a/harmony/src/modules/cert_manager/crd/certificate.rs b/harmony/src/modules/cert_manager/crd/certificate.rs index 7c0866b7..2375b0b2 100644 --- a/harmony/src/modules/cert_manager/crd/certificate.rs +++ b/harmony/src/modules/cert_manager/crd/certificate.rs @@ -110,4 +110,3 @@ pub struct PrivateKey { #[serde(skip_serializing_if = "Option::is_none")] pub rotation_policy: Option, } - diff --git a/harmony/src/modules/cert_manager/crd/cluster_issuer.rs b/harmony/src/modules/cert_manager/crd/cluster_issuer.rs index 49395489..0348972c 100644 --- a/harmony/src/modules/cert_manager/crd/cluster_issuer.rs +++ b/harmony/src/modules/cert_manager/crd/cluster_issuer.rs @@ -42,4 +42,3 @@ impl Default for ClusterIssuerSpec { } } } - diff --git a/harmony/src/modules/cert_manager/crd/mod.rs b/harmony/src/modules/cert_manager/crd/mod.rs index 70ea4693..7f8e6d1c 100644 --- a/harmony/src/modules/cert_manager/crd/mod.rs +++ b/harmony/src/modules/cert_manager/crd/mod.rs @@ -1,12 +1,11 @@ use serde::{Deserialize, Serialize}; - pub mod certificate; -pub mod issuer; pub mod cluster_issuer; +pub mod issuer; //pub mod score_cluster_issuer; -pub mod score_issuer; pub mod score_certificate; +pub mod score_issuer; #[derive(Deserialize, Serialize, Clone, Debug)] #[serde(rename_all = "camelCase")] diff --git a/harmony/src/modules/cert_manager/crd/score_certificate.rs b/harmony/src/modules/cert_manager/crd/score_certificate.rs index 53c2a346..9e512350 100644 --- a/harmony/src/modules/cert_manager/crd/score_certificate.rs +++ b/harmony/src/modules/cert_manager/crd/score_certificate.rs @@ -16,7 +16,8 @@ use crate::{ #[derive(Debug, Clone, Serialize)] pub struct CertificateScore { - pub name: String, + pub cert_name: String, + pub issuer_name: String, pub config: CertificateManagementConfig, } @@ -28,14 +29,14 @@ impl Score for CertificateScore { fn create_interpret(&self) -> Box> { let cert = Certificate { metadata: ObjectMeta { - name: Some(self.name.clone()), + name: Some(self.cert_name.clone()), namespace: self.config.namespace.clone(), ..Default::default() }, spec: CertificateSpec { - secret_name: format!("{}-tls", self.name.clone()), + secret_name: format!("{}-tls", self.cert_name.clone()), issuer_ref: IssuerRef { - name: self.config.name.clone(), + name: self.issuer_name.clone(), kind: Some("Issuer".into()), group: Some("cert-manager.io".into()), }, diff --git a/harmony/src/modules/cert_manager/crd/score_issuer.rs b/harmony/src/modules/cert_manager/crd/score_issuer.rs index 05af62dd..f30f155e 100644 --- a/harmony/src/modules/cert_manager/crd/score_issuer.rs +++ b/harmony/src/modules/cert_manager/crd/score_issuer.rs @@ -4,10 +4,13 @@ use serde::Serialize; use crate::{ interpret::Interpret, modules::{ - cert_manager::{capability::CertificateManagementConfig, crd::{ - AcmeIssuer, CaIssuer, SelfSignedIssuer, - issuer::{Issuer, IssuerSpec}, - }}, + cert_manager::{ + capability::CertificateManagementConfig, + crd::{ + SelfSignedIssuer, + issuer::{Issuer, IssuerSpec}, + }, + }, k8s::resource::K8sResourceScore, }, score::Score, @@ -26,7 +29,7 @@ impl Score for IssuerScore { fn create_interpret(&self) -> Box> { let metadata = ObjectMeta { - name: Some(self.config.name.clone()), + name: Some(format!("{}-issuer", self.config.namespace.clone().unwrap())), namespace: self.config.namespace.clone(), ..ObjectMeta::default() }; diff --git a/harmony/src/modules/cert_manager/mod.rs b/harmony/src/modules/cert_manager/mod.rs index 430252d2..9de3b9af 100644 --- a/harmony/src/modules/cert_manager/mod.rs +++ b/harmony/src/modules/cert_manager/mod.rs @@ -1,7 +1,7 @@ pub mod capability; pub mod cluster_issuer; +pub mod crd; mod helm; pub mod operator; pub mod score_k8s; -pub mod crd; pub use helm::*; -- 2.39.5 From 4f2a7050f5f3084ad706b1b464abe7acfc519247 Mon Sep 17 00:00:00 2001 From: wjro Date: Wed, 14 Jan 2026 16:18:59 -0500 Subject: [PATCH 11/21] feat: added working examples to add self signed issuer and self signed certificate. modified get_resource_json_value to be able to get cluster scoped operators --- examples/cert_manager/src/main.rs | 32 ++++++--- harmony/src/domain/topology/k8s.rs | 26 +++++-- .../topology/k8s_anywhere/k8s_anywhere.rs | 32 +++++++-- .../src/modules/cert_manager/capability.rs | 4 +- .../cert_manager/crd/score_certificate.rs | 1 + .../modules/cert_manager/crd/score_issuer.rs | 3 +- harmony/src/modules/cert_manager/mod.rs | 4 +- .../modules/cert_manager/score_create_cert.rs | 72 +++++++++++++++++++ .../cert_manager/score_create_issuer.rs | 66 +++++++++++++++++ .../{score_k8s.rs => score_operator.rs} | 7 +- 10 files changed, 217 insertions(+), 30 deletions(-) create mode 100644 harmony/src/modules/cert_manager/score_create_cert.rs create mode 100644 harmony/src/modules/cert_manager/score_create_issuer.rs rename harmony/src/modules/cert_manager/{score_k8s.rs => score_operator.rs} (90%) diff --git a/examples/cert_manager/src/main.rs b/examples/cert_manager/src/main.rs index df6484e2..03855940 100644 --- a/examples/cert_manager/src/main.rs +++ b/examples/cert_manager/src/main.rs @@ -2,7 +2,9 @@ use harmony::{ inventory::Inventory, modules::{ cert_manager::{ - capability::CertificateManagementConfig, score_k8s::CertificateManagementScore, + capability::CertificateManagementConfig, score_create_cert::CertificateCreationScore, + score_create_issuer::CertificateIssuerScore, + score_operator::CertificateManagementScore, }, postgresql::{PostgreSQLScore, capability::PostgreSQLConfig}, }, @@ -11,20 +13,32 @@ use harmony::{ #[tokio::main] async fn main() { + let config = CertificateManagementConfig { + namespace: Some("test".to_string()), + acme_issuer: None, + ca_issuer: None, + self_signed: true, + }; + let cert_manager = CertificateManagementScore { - config: CertificateManagementConfig { - name: todo!(), - namespace: todo!(), - acme_issuer: todo!(), - ca_issuer: todo!(), - self_signed: todo!(), - }, + config: config.clone(), + }; + + let issuer = CertificateIssuerScore { + config: config.clone(), + issuer_name: "test-self-signed-issuer".to_string(), + }; + + let cert = CertificateCreationScore { + config: config.clone(), + cert_name: "test-self-signed-cert".to_string(), + issuer_name: "test-self-signed-issuer".to_string(), }; harmony_cli::run( Inventory::autoload(), K8sAnywhereTopology::from_env(), - vec![Box::new(cert_manager)], + vec![Box::new(cert_manager), Box::new(issuer), Box::new(cert)], None, ) .await diff --git a/harmony/src/domain/topology/k8s.rs b/harmony/src/domain/topology/k8s.rs index ef34f3cf..bb786eb9 100644 --- a/harmony/src/domain/topology/k8s.rs +++ b/harmony/src/domain/topology/k8s.rs @@ -230,14 +230,26 @@ impl K8sClient { namespace: Option<&str>, gvk: &GroupVersionKind, ) -> Result { - let gvk = ApiResource::from_gvk(gvk); - let resource: Api = if let Some(ns) = namespace { - Api::namespaced_with(self.client.clone(), ns, &gvk) - } else { - Api::default_namespaced_with(self.client.clone(), &gvk) - }; + let api_resource = ApiResource::from_gvk(gvk); - resource.get(name).await + // 1. Try namespaced first (if a namespace was provided) + if let Some(ns) = namespace { + let api: Api = + Api::namespaced_with(self.client.clone(), ns, &api_resource); + + match api.get(name).await { + Ok(obj) => return Ok(obj), + Err(Error::Api(ae)) if ae.code == 404 => { + // fall through and try cluster-scoped + } + Err(e) => return Err(e), + } + } + + // 2. Fallback to cluster-scoped + let api: Api = Api::all_with(self.client.clone(), &api_resource); + + api.get(name).await } pub async fn get_secret_json_value( diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 0a2746f9..656c0fd0 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -391,10 +391,7 @@ impl Serialize for K8sAnywhereTopology { #[async_trait] impl CertificateManagement for K8sAnywhereTopology { - async fn install( - &self, - config: &CertificateManagementConfig, - ) -> Result { + async fn install(&self) -> Result { let cert_management_operator = CertManagerOperatorScore::default(); cert_management_operator @@ -410,11 +407,33 @@ impl CertificateManagement for K8sAnywhereTopology { }) } - async fn ensure_ready( + async fn ensure_certificate_management_ready( &self, config: &CertificateManagementConfig, ) -> Result { - todo!() + let k8s_client = self.k8s_client().await.unwrap(); + let gvk = GroupVersionKind { + group: "operators.coreos.com".to_string(), + version: "v1".to_string(), + kind: "Operator".to_string(), + }; + //TODO make this generic across k8s distributions using k8s family + match k8s_client + .get_resource_json_value( + "cert-manager.openshift-operators", + None, + &gvk, + ) + .await + { + Ok(_ready) => Ok(PreparationOutcome::Success { + details: "Certificate Management Ready".to_string(), + }), + Err(e) => { + debug!("{} operator not found", e.to_string()); + self.install().await + } + } } async fn create_issuer( @@ -423,6 +442,7 @@ impl CertificateManagement for K8sAnywhereTopology { config: &CertificateManagementConfig, ) -> Result { let issuer_score = IssuerScore { + issuer_name: issuer_name.clone(), config: config.clone(), }; diff --git a/harmony/src/modules/cert_manager/capability.rs b/harmony/src/modules/cert_manager/capability.rs index 0c6940da..0cad041d 100644 --- a/harmony/src/modules/cert_manager/capability.rs +++ b/harmony/src/modules/cert_manager/capability.rs @@ -11,10 +11,9 @@ use crate::{ pub trait CertificateManagement: Send + Sync { async fn install( &self, - config: &CertificateManagementConfig, ) -> Result; - async fn ensure_ready( + async fn ensure_certificate_management_ready( &self, config: &CertificateManagementConfig, ) -> Result; @@ -35,7 +34,6 @@ pub trait CertificateManagement: Send + Sync { #[derive(Debug, Clone, Serialize)] pub struct CertificateManagementConfig { - pub name: String, pub namespace: Option, pub acme_issuer: Option, pub ca_issuer: Option, diff --git a/harmony/src/modules/cert_manager/crd/score_certificate.rs b/harmony/src/modules/cert_manager/crd/score_certificate.rs index 9e512350..fb9a145a 100644 --- a/harmony/src/modules/cert_manager/crd/score_certificate.rs +++ b/harmony/src/modules/cert_manager/crd/score_certificate.rs @@ -40,6 +40,7 @@ impl Score for CertificateScore { kind: Some("Issuer".into()), group: Some("cert-manager.io".into()), }, + dns_names: Some(vec!["test.example.local".to_string()]), ..Default::default() }, }; diff --git a/harmony/src/modules/cert_manager/crd/score_issuer.rs b/harmony/src/modules/cert_manager/crd/score_issuer.rs index f30f155e..c858d6ef 100644 --- a/harmony/src/modules/cert_manager/crd/score_issuer.rs +++ b/harmony/src/modules/cert_manager/crd/score_issuer.rs @@ -19,6 +19,7 @@ use crate::{ #[derive(Debug, Clone, Serialize)] pub struct IssuerScore { + pub issuer_name: String, pub config: CertificateManagementConfig, } @@ -29,7 +30,7 @@ impl Score for IssuerScore { fn create_interpret(&self) -> Box> { let metadata = ObjectMeta { - name: Some(format!("{}-issuer", self.config.namespace.clone().unwrap())), + name: Some(self.issuer_name.clone()), namespace: self.config.namespace.clone(), ..ObjectMeta::default() }; diff --git a/harmony/src/modules/cert_manager/mod.rs b/harmony/src/modules/cert_manager/mod.rs index 9de3b9af..9b09876f 100644 --- a/harmony/src/modules/cert_manager/mod.rs +++ b/harmony/src/modules/cert_manager/mod.rs @@ -3,5 +3,7 @@ pub mod cluster_issuer; pub mod crd; mod helm; pub mod operator; -pub mod score_k8s; +pub mod score_operator; +pub mod score_create_cert; +pub mod score_create_issuer; pub use helm::*; diff --git a/harmony/src/modules/cert_manager/score_create_cert.rs b/harmony/src/modules/cert_manager/score_create_cert.rs new file mode 100644 index 00000000..ceb109d2 --- /dev/null +++ b/harmony/src/modules/cert_manager/score_create_cert.rs @@ -0,0 +1,72 @@ +use async_trait::async_trait; +use harmony_types::id::Id; +use serde::Serialize; + +use crate::{ + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + modules::cert_manager::capability::{CertificateManagement, CertificateManagementConfig}, + score::Score, + topology::Topology, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct CertificateCreationScore { + pub cert_name: String, + pub issuer_name: String, + pub config: CertificateManagementConfig, +} + +impl Score for CertificateCreationScore { + fn name(&self) -> String { + "CertificateCreationScore".to_string() + } + + fn create_interpret(&self) -> Box> { + Box::new(CertificateCreationInterpret { + cert_name: self.cert_name.clone(), + issuer_name: self.issuer_name.clone(), + config: self.config.clone(), + }) + } +} + +#[derive(Debug)] +struct CertificateCreationInterpret { + cert_name: String, + issuer_name: String, + config: CertificateManagementConfig, +} + +#[async_trait] +impl Interpret for CertificateCreationInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + let _certificate = topology + .create_certificate(self.cert_name.clone(), self.issuer_name.clone(), &self.config) + .await + .map_err(|e| InterpretError::new(e.to_string()))?; + + Ok(Outcome::success(format!("Installed CertificateManagement"))) + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("CertificateManagementInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} diff --git a/harmony/src/modules/cert_manager/score_create_issuer.rs b/harmony/src/modules/cert_manager/score_create_issuer.rs new file mode 100644 index 00000000..7ccc5482 --- /dev/null +++ b/harmony/src/modules/cert_manager/score_create_issuer.rs @@ -0,0 +1,66 @@ +use async_trait::async_trait; +use harmony_types::id::Id; +use log::debug; +use serde::Serialize; + +use crate::{data::Version, interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, modules::cert_manager::capability::{CertificateManagement, CertificateManagementConfig}, score::Score, topology::Topology}; + + + +#[derive(Debug, Clone, Serialize)] +pub struct CertificateIssuerScore { + pub issuer_name: String, + pub config: CertificateManagementConfig, +} + +impl Score for CertificateIssuerScore { + fn name(&self) -> String { + "CertificateIssuerScore".to_string() + } + + fn create_interpret(&self) -> Box> { + Box::new(CertificateIssuerInterpret { + config: self.config.clone(), + issuer_name: self.issuer_name.clone(), + }) + } +} + +#[derive(Debug)] +struct CertificateIssuerInterpret { + config: CertificateManagementConfig, + issuer_name: String, +} + +#[async_trait] +impl Interpret for CertificateIssuerInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + debug!("issuer name: {}", self.issuer_name.clone()); + let _cert_issuer = topology + .create_issuer(self.issuer_name.clone(), &self.config) + .await + .map_err(|e| InterpretError::new(e.to_string()))?; + + Ok(Outcome::success(format!("Installed CertificateManagement"))) + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("CertificateManagementInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} diff --git a/harmony/src/modules/cert_manager/score_k8s.rs b/harmony/src/modules/cert_manager/score_operator.rs similarity index 90% rename from harmony/src/modules/cert_manager/score_k8s.rs rename to harmony/src/modules/cert_manager/score_operator.rs index 18e08264..61c24560 100644 --- a/harmony/src/modules/cert_manager/score_k8s.rs +++ b/harmony/src/modules/cert_manager/score_operator.rs @@ -40,12 +40,13 @@ impl Interpret for CertificateManagement inventory: &Inventory, topology: &T, ) -> Result { - let cert_management = topology - .install(&self.config) + + topology + .ensure_certificate_management_ready(&self.config) .await .map_err(|e| InterpretError::new(e.to_string()))?; - Ok(Outcome::success(format!("Installed CertificateManagement"))) + Ok(Outcome::success(format!("CertificateManagement is ready"))) } fn get_name(&self) -> InterpretName { -- 2.39.5 From 502e544cd3de36f4b50b32a3c5136f8c5a9a2fce Mon Sep 17 00:00:00 2001 From: wjro Date: Wed, 14 Jan 2026 16:19:56 -0500 Subject: [PATCH 12/21] cargo fmt --- .../src/domain/topology/k8s_anywhere/k8s_anywhere.rs | 6 +----- harmony/src/modules/cert_manager/capability.rs | 4 +--- harmony/src/modules/cert_manager/mod.rs | 2 +- harmony/src/modules/cert_manager/score_create_cert.rs | 6 +++++- .../src/modules/cert_manager/score_create_issuer.rs | 11 ++++++++--- harmony/src/modules/cert_manager/score_operator.rs | 1 - 6 files changed, 16 insertions(+), 14 deletions(-) diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 656c0fd0..4a411702 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -419,11 +419,7 @@ impl CertificateManagement for K8sAnywhereTopology { }; //TODO make this generic across k8s distributions using k8s family match k8s_client - .get_resource_json_value( - "cert-manager.openshift-operators", - None, - &gvk, - ) + .get_resource_json_value("cert-manager.openshift-operators", None, &gvk) .await { Ok(_ready) => Ok(PreparationOutcome::Success { diff --git a/harmony/src/modules/cert_manager/capability.rs b/harmony/src/modules/cert_manager/capability.rs index 0cad041d..a2806601 100644 --- a/harmony/src/modules/cert_manager/capability.rs +++ b/harmony/src/modules/cert_manager/capability.rs @@ -9,9 +9,7 @@ use crate::{ ///TODO rust doc explaining issuer, certificate etc #[async_trait] pub trait CertificateManagement: Send + Sync { - async fn install( - &self, - ) -> Result; + async fn install(&self) -> Result; async fn ensure_certificate_management_ready( &self, diff --git a/harmony/src/modules/cert_manager/mod.rs b/harmony/src/modules/cert_manager/mod.rs index 9b09876f..d1ff660e 100644 --- a/harmony/src/modules/cert_manager/mod.rs +++ b/harmony/src/modules/cert_manager/mod.rs @@ -3,7 +3,7 @@ pub mod cluster_issuer; pub mod crd; mod helm; pub mod operator; -pub mod score_operator; pub mod score_create_cert; pub mod score_create_issuer; +pub mod score_operator; pub use helm::*; diff --git a/harmony/src/modules/cert_manager/score_create_cert.rs b/harmony/src/modules/cert_manager/score_create_cert.rs index ceb109d2..23e761b5 100644 --- a/harmony/src/modules/cert_manager/score_create_cert.rs +++ b/harmony/src/modules/cert_manager/score_create_cert.rs @@ -47,7 +47,11 @@ impl Interpret for CertificateCreationIn topology: &T, ) -> Result { let _certificate = topology - .create_certificate(self.cert_name.clone(), self.issuer_name.clone(), &self.config) + .create_certificate( + self.cert_name.clone(), + self.issuer_name.clone(), + &self.config, + ) .await .map_err(|e| InterpretError::new(e.to_string()))?; diff --git a/harmony/src/modules/cert_manager/score_create_issuer.rs b/harmony/src/modules/cert_manager/score_create_issuer.rs index 7ccc5482..447ddf67 100644 --- a/harmony/src/modules/cert_manager/score_create_issuer.rs +++ b/harmony/src/modules/cert_manager/score_create_issuer.rs @@ -3,9 +3,14 @@ use harmony_types::id::Id; use log::debug; use serde::Serialize; -use crate::{data::Version, interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, modules::cert_manager::capability::{CertificateManagement, CertificateManagementConfig}, score::Score, topology::Topology}; - - +use crate::{ + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + modules::cert_manager::capability::{CertificateManagement, CertificateManagementConfig}, + score::Score, + topology::Topology, +}; #[derive(Debug, Clone, Serialize)] pub struct CertificateIssuerScore { diff --git a/harmony/src/modules/cert_manager/score_operator.rs b/harmony/src/modules/cert_manager/score_operator.rs index 61c24560..b13e0854 100644 --- a/harmony/src/modules/cert_manager/score_operator.rs +++ b/harmony/src/modules/cert_manager/score_operator.rs @@ -40,7 +40,6 @@ impl Interpret for CertificateManagement inventory: &Inventory, topology: &T, ) -> Result { - topology .ensure_certificate_management_ready(&self.config) .await -- 2.39.5 From 865dab2fc14b794c76c8af9099bd3fd614422bcd Mon Sep 17 00:00:00 2001 From: wjro Date: Fri, 16 Jan 2026 13:16:06 -0500 Subject: [PATCH 13/21] feat: added fn get_ca_cert to trait certificateManagement --- .../topology/k8s_anywhere/k8s_anywhere.rs | 33 +++++++++++++++++++ .../src/modules/cert_manager/capability.rs | 6 ++++ 2 files changed, 39 insertions(+) diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 4a411702..aa2adb55 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -478,6 +478,39 @@ impl CertificateManagement for K8sAnywhereTopology { details: format!("Created cert into ns: {:#?}", config.namespace.clone()), }) } + + async fn get_ca_certificate( + &self, + cert_name: String, + config: &CertificateManagementConfig, + ) -> Result { + let namespace = config.namespace.clone().unwrap(); + + let client = self.k8s_client().await.unwrap(); + + let secret = client + .get_secret_json_value(&cert_name, Some(&namespace)) + .await? + .data; + + let ca_cert = secret + .get("data") + .ok_or_else(|| PreparationError { + msg: format!("failed to get data from secret {}", cert_name), + })? + .get("ca.crt") + .ok_or_else(|| PreparationError { + msg: format!("failed to get ca.crt from secret {}", cert_name), + })?; + + trace!("{:#?}", ca_cert.clone()); + + let cert: String = serde_json::from_value(ca_cert.clone()) + .map_err(|e| PreparationError { msg: e.to_string() })?; + + trace!("{:#?}", cert.clone()); + Ok(cert) + } } impl K8sAnywhereTopology { diff --git a/harmony/src/modules/cert_manager/capability.rs b/harmony/src/modules/cert_manager/capability.rs index a2806601..f39f6e6d 100644 --- a/harmony/src/modules/cert_manager/capability.rs +++ b/harmony/src/modules/cert_manager/capability.rs @@ -28,6 +28,12 @@ pub trait CertificateManagement: Send + Sync { issuer_name: String, config: &CertificateManagementConfig, ) -> Result; + + async fn get_ca_certificate( + &self, + cert_name: String, + config: &CertificateManagementConfig, + ) -> Result; } #[derive(Debug, Clone, Serialize)] -- 2.39.5 From 779444699ff23c44dbadc797859683fb52eb6554 Mon Sep 17 00:00:00 2001 From: wjro Date: Fri, 16 Jan 2026 13:39:10 -0500 Subject: [PATCH 14/21] fix: modified k8sanywhere implentation of get_ca_cert to use the kubernetes certificate name to find its respective secret and ca.crt --- .../topology/k8s_anywhere/k8s_anywhere.rs | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index aa2adb55..ae70a268 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -485,31 +485,56 @@ impl CertificateManagement for K8sAnywhereTopology { config: &CertificateManagementConfig, ) -> Result { let namespace = config.namespace.clone().unwrap(); - + let certificate_gvk = GroupVersionKind { + group: "cert-manager.io".to_string(), + version: "v1".to_string(), + kind: "Certificate".to_string(), + }; let client = self.k8s_client().await.unwrap(); + let certificate_data = client + .get_resource_json_value(&cert_name, Some(&namespace), &certificate_gvk) + .await? + .data; + + trace!("Certificate Data {:#?}", certificate_data); + + let secret_name = certificate_data + .get("spec") + .ok_or_else(|| PreparationError { + msg: format!("failed to get spec from Certificate {}", cert_name), + })? + .get("secretName") + .ok_or_else(|| PreparationError { + msg: format!("failed to get secretName from Certificate {}", cert_name), + })?; + + trace!("Secret Name {:#?}", secret_name); + + let secret_name: String = serde_json::from_value(secret_name.clone()) + .map_err(|e| PreparationError { msg: e.to_string() })?; let secret = client - .get_secret_json_value(&cert_name, Some(&namespace)) + .get_secret_json_value(&secret_name, Some(&namespace)) .await? .data; let ca_cert = secret .get("data") .ok_or_else(|| PreparationError { - msg: format!("failed to get data from secret {}", cert_name), + msg: format!("failed to get data from secret {}", secret_name), })? .get("ca.crt") .ok_or_else(|| PreparationError { - msg: format!("failed to get ca.crt from secret {}", cert_name), + msg: format!("failed to get ca.crt from secret {}", secret_name), })?; - trace!("{:#?}", ca_cert.clone()); + trace!("ca.crt {:#?}", ca_cert.clone()); - let cert: String = serde_json::from_value(ca_cert.clone()) + let ca_cert: String = serde_json::from_value(ca_cert.clone()) .map_err(|e| PreparationError { msg: e.to_string() })?; - trace!("{:#?}", cert.clone()); - Ok(cert) + trace!("ca.crt string {:#?}", ca_cert.clone()); + Ok(ca_cert) } } -- 2.39.5 From 2b324d7962b7c3f6632c17303513f7465e332372 Mon Sep 17 00:00:00 2001 From: wjro Date: Mon, 19 Jan 2026 11:37:47 -0500 Subject: [PATCH 15/21] fix: modified trait to use other return types, modified trait function name to be ensure ready, use rust CRD definitions rather than constructing gvk for certificateManagement trait function in k8sanywhere --- examples/cert_manager/src/main.rs | 10 +- .../topology/k8s_anywhere/k8s_anywhere.rs | 199 +++++++++--------- .../src/modules/cert_manager/capability.rs | 15 +- .../modules/cert_manager/score_create_cert.rs | 6 + .../modules/cert_manager/score_operator.rs | 3 +- 5 files changed, 118 insertions(+), 115 deletions(-) diff --git a/examples/cert_manager/src/main.rs b/examples/cert_manager/src/main.rs index 03855940..1d0c3acd 100644 --- a/examples/cert_manager/src/main.rs +++ b/examples/cert_manager/src/main.rs @@ -1,12 +1,8 @@ use harmony::{ inventory::Inventory, - modules::{ - cert_manager::{ - capability::CertificateManagementConfig, score_create_cert::CertificateCreationScore, - score_create_issuer::CertificateIssuerScore, - score_operator::CertificateManagementScore, - }, - postgresql::{PostgreSQLScore, capability::PostgreSQLConfig}, + modules::cert_manager::{ + capability::CertificateManagementConfig, score_create_cert::CertificateCreationScore, + score_create_issuer::CertificateIssuerScore, score_operator::CertificateManagementScore, }, topology::K8sAnywhereTopology, }; diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index ae70a268..d0ef5a32 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -14,16 +14,24 @@ use tokio::sync::OnceCell; use crate::{ executors::ExecutorError, - interpret::InterpretStatus, + interpret::{InterpretStatus, Outcome}, inventory::Inventory, modules::{ cert_manager::{ capability::{CertificateManagement, CertificateManagementConfig}, - crd::{score_certificate::CertificateScore, score_issuer::IssuerScore}, + crd::{ + certificate::Certificate, + issuer::{Issuer, IssuerSpec}, + score_certificate::CertificateScore, + score_issuer::IssuerScore, + }, operator::CertManagerOperatorScore, }, k3d::K3DInstallationScore, - k8s::ingress::{K8sIngressScore, PathType}, + k8s::{ + apps::crd::Subscription, + ingress::{K8sIngressScore, PathType}, + }, monitoring::{ grafana::{grafana::Grafana, helm::helm_grafana::grafana_helm_chart_score}, kube_prometheus::crd::{ @@ -391,44 +399,36 @@ impl Serialize for K8sAnywhereTopology { #[async_trait] impl CertificateManagement for K8sAnywhereTopology { - async fn install(&self) -> Result { + async fn install(&self) -> Result { let cert_management_operator = CertManagerOperatorScore::default(); cert_management_operator .interpret(&Inventory::empty(), self) .await - .map_err(|e| PreparationError { msg: e.to_string() })?; + .map_err(|e| ExecutorError::UnexpectedError(e.to_string()))?; - Ok(PreparationOutcome::Success { - details: format!( - "Installed cert-manager into ns: {}", - cert_management_operator.namespace - ), - }) + Ok(Outcome::success(format!( + "Installed cert-manager into ns: {}", + cert_management_operator.namespace + ))) } - async fn ensure_certificate_management_ready( + async fn ensure_ready( &self, config: &CertificateManagementConfig, - ) -> Result { + ) -> Result { let k8s_client = self.k8s_client().await.unwrap(); - let gvk = GroupVersionKind { - group: "operators.coreos.com".to_string(), - version: "v1".to_string(), - kind: "Operator".to_string(), - }; - //TODO make this generic across k8s distributions using k8s family + match k8s_client - .get_resource_json_value("cert-manager.openshift-operators", None, &gvk) + .get_resource::("cert-manager", Some("openshift-operators")) .await + .map_err(|e| ExecutorError::UnexpectedError(format!("{}", e)))? { - Ok(_ready) => Ok(PreparationOutcome::Success { - details: "Certificate Management Ready".to_string(), - }), - Err(e) => { - debug!("{} operator not found", e.to_string()); - self.install().await + Some(subscription) => { + trace!("subscription {:#?}", subscription,); + Ok(Outcome::success(format!("Certificate Management Ready",))) } + None => self.install().await, } } @@ -436,7 +436,7 @@ impl CertificateManagement for K8sAnywhereTopology { &self, issuer_name: String, config: &CertificateManagementConfig, - ) -> Result { + ) -> Result { let issuer_score = IssuerScore { issuer_name: issuer_name.clone(), config: config.clone(), @@ -445,11 +445,12 @@ impl CertificateManagement for K8sAnywhereTopology { issuer_score .interpret(&Inventory::empty(), self) .await - .map_err(|e| PreparationError { msg: e.to_string() })?; + .map_err(|e| ExecutorError::UnexpectedError(e.to_string()))?; - Ok(PreparationOutcome::Success { - details: format!("issuer of kind {} is ready", issuer_name), - }) + Ok(Outcome::success(format!( + "issuer of kind {} is ready", + issuer_name + ))) } async fn create_certificate( @@ -457,7 +458,7 @@ impl CertificateManagement for K8sAnywhereTopology { cert_name: String, issuer_name: String, config: &CertificateManagementConfig, - ) -> Result { + ) -> Result { self.certificate_issuer_ready( issuer_name.clone(), self.k8s_client().await.unwrap(), @@ -472,69 +473,66 @@ impl CertificateManagement for K8sAnywhereTopology { }; cert.interpret(&Inventory::empty(), self) .await - .map_err(|e| PreparationError { msg: e.to_string() })?; + .map_err(|e| ExecutorError::UnexpectedError(e.to_string()))?; - Ok(PreparationOutcome::Success { - details: format!("Created cert into ns: {:#?}", config.namespace.clone()), - }) + Ok(Outcome::success(format!( + "Created cert into ns: {:#?}", + config.namespace.clone() + ))) } async fn get_ca_certificate( &self, cert_name: String, config: &CertificateManagementConfig, - ) -> Result { + ) -> Result { let namespace = config.namespace.clone().unwrap(); - let certificate_gvk = GroupVersionKind { - group: "cert-manager.io".to_string(), - version: "v1".to_string(), - kind: "Certificate".to_string(), - }; + let client = self.k8s_client().await.unwrap(); - let certificate_data = client - .get_resource_json_value(&cert_name, Some(&namespace), &certificate_gvk) - .await? - .data; - trace!("Certificate Data {:#?}", certificate_data); + if let Some(certificate) = client + .get_resource::(&cert_name, Some(&namespace)) + .await + .map_err(|e| ExecutorError::UnexpectedError(format!("{}", e)))? + { + let secret_name = certificate.spec.secret_name.clone(); - let secret_name = certificate_data - .get("spec") - .ok_or_else(|| PreparationError { - msg: format!("failed to get spec from Certificate {}", cert_name), - })? - .get("secretName") - .ok_or_else(|| PreparationError { - msg: format!("failed to get secretName from Certificate {}", cert_name), - })?; + trace!("Secret Name {:#?}", secret_name); + if let Some(secret) = client + .get_resource::(&secret_name, Some(&namespace)) + .await + .map_err(|e| { + ExecutorError::UnexpectedError(format!( + "secret {} not found in namespace {}: {}", + secret_name, namespace, e + )) + })? + { + let ca_cert = secret + .data + .as_ref() + .and_then(|d| d.get("ca.crt")) + .ok_or_else(|| { + ExecutorError::UnexpectedError("Secret missing key 'ca.crt'".into()) + })?; - trace!("Secret Name {:#?}", secret_name); + let ca_cert = String::from_utf8(ca_cert.0.clone()).map_err(|_| { + ExecutorError::UnexpectedError("ca.crt is not valid UTF-8".into()) + })?; - let secret_name: String = serde_json::from_value(secret_name.clone()) - .map_err(|e| PreparationError { msg: e.to_string() })?; - - let secret = client - .get_secret_json_value(&secret_name, Some(&namespace)) - .await? - .data; - - let ca_cert = secret - .get("data") - .ok_or_else(|| PreparationError { - msg: format!("failed to get data from secret {}", secret_name), - })? - .get("ca.crt") - .ok_or_else(|| PreparationError { - msg: format!("failed to get ca.crt from secret {}", secret_name), - })?; - - trace!("ca.crt {:#?}", ca_cert.clone()); - - let ca_cert: String = serde_json::from_value(ca_cert.clone()) - .map_err(|e| PreparationError { msg: e.to_string() })?; - - trace!("ca.crt string {:#?}", ca_cert.clone()); - Ok(ca_cert) + return Ok(ca_cert); + } else { + Err(ExecutorError::UnexpectedError(format!( + "Error getting secret associated with cert_name: {}", + cert_name + ))) + } + } else { + return Err(ExecutorError::UnexpectedError(format!( + "Certificate {} not found in namespace {}", + cert_name, namespace + ))); + } } } @@ -562,27 +560,30 @@ impl K8sAnywhereTopology { issuer_name: String, k8s_client: Arc, config: &CertificateManagementConfig, - ) -> Result { - let ns = config.namespace.clone().ok_or_else(|| PreparationError { - msg: "namespace is required".to_string(), - })?; - - let gvk = GroupVersionKind { - group: "cert-manager.io".to_string(), - version: "v1".to_string(), - kind: "Issuer".to_string(), - }; + ) -> Result { + let ns = config + .namespace + .clone() + .ok_or_else(|| ExecutorError::UnexpectedError("namespace is required".to_string()))?; match k8s_client - .get_resource_json_value(&issuer_name, Some(&ns), &gvk) + .get_resource::(&issuer_name, Some(&ns)) .await { - Ok(_cert_issuer) => Ok(PreparationOutcome::Success { - details: format!("issuer of kind {} is ready", issuer_name), - }), - Err(e) => Err(PreparationError { - msg: format!("{} issuer {} not present", e.to_string(), issuer_name), - }), + Ok(Some(_cert_issuer)) => Ok(Outcome::success(format!( + "issuer of kind {} is ready", + issuer_name + ))), + + Ok(None) => Err(ExecutorError::UnexpectedError(format!( + "Issuer {} not present in namespace {}", + issuer_name, ns + ))), + + Err(e) => Err(ExecutorError::UnexpectedError(format!( + "Failed to fetch Issuer {}: {}", + issuer_name, e + ))), } } diff --git a/harmony/src/modules/cert_manager/capability.rs b/harmony/src/modules/cert_manager/capability.rs index f39f6e6d..0610a726 100644 --- a/harmony/src/modules/cert_manager/capability.rs +++ b/harmony/src/modules/cert_manager/capability.rs @@ -2,38 +2,39 @@ use async_trait::async_trait; use serde::Serialize; use crate::{ + executors::ExecutorError, + interpret::Outcome, modules::cert_manager::crd::{AcmeIssuer, CaIssuer}, - topology::{PreparationError, PreparationOutcome}, }; ///TODO rust doc explaining issuer, certificate etc #[async_trait] pub trait CertificateManagement: Send + Sync { - async fn install(&self) -> Result; + async fn install(&self) -> Result; - async fn ensure_certificate_management_ready( + async fn ensure_ready( &self, config: &CertificateManagementConfig, - ) -> Result; + ) -> Result; async fn create_issuer( &self, issuer_name: String, config: &CertificateManagementConfig, - ) -> Result; + ) -> Result; async fn create_certificate( &self, cert_name: String, issuer_name: String, config: &CertificateManagementConfig, - ) -> Result; + ) -> Result; async fn get_ca_certificate( &self, cert_name: String, config: &CertificateManagementConfig, - ) -> Result; + ) -> Result; } #[derive(Debug, Clone, Serialize)] diff --git a/harmony/src/modules/cert_manager/score_create_cert.rs b/harmony/src/modules/cert_manager/score_create_cert.rs index 23e761b5..8f3a21b6 100644 --- a/harmony/src/modules/cert_manager/score_create_cert.rs +++ b/harmony/src/modules/cert_manager/score_create_cert.rs @@ -1,5 +1,6 @@ use async_trait::async_trait; use harmony_types::id::Id; +use log::trace; use serde::Serialize; use crate::{ @@ -55,6 +56,11 @@ impl Interpret for CertificateCreationIn .await .map_err(|e| InterpretError::new(e.to_string()))?; + let ca_cert = topology + .get_ca_certificate("test-self-signed-cert".to_string(), &self.config) + .await?; + trace!("cacert: {}", ca_cert); + Ok(Outcome::success(format!("Installed CertificateManagement"))) } diff --git a/harmony/src/modules/cert_manager/score_operator.rs b/harmony/src/modules/cert_manager/score_operator.rs index b13e0854..ec783266 100644 --- a/harmony/src/modules/cert_manager/score_operator.rs +++ b/harmony/src/modules/cert_manager/score_operator.rs @@ -40,8 +40,7 @@ impl Interpret for CertificateManagement inventory: &Inventory, topology: &T, ) -> Result { - topology - .ensure_certificate_management_ready(&self.config) + let _cert_management = &CertificateManagement::ensure_ready(topology, &self.config) .await .map_err(|e| InterpretError::new(e.to_string()))?; -- 2.39.5 From a4515d34ae7c28aa88ab7676b765a7dff5fa70a3 Mon Sep 17 00:00:00 2001 From: wjro Date: Mon, 19 Jan 2026 12:48:47 -0500 Subject: [PATCH 16/21] fix: modified score names for better clarity --- examples/cert_manager/src/main.rs | 6 +++--- .../domain/topology/k8s_anywhere/k8s_anywhere.rs | 10 ++++------ harmony/src/modules/cert_manager/crd/mod.rs | 4 ++-- ...re_certificate.rs => score_k8s_certificate.rs} | 4 ++-- ...ster_issuer.rs => score_k8s_cluster_issuer.rs} | 0 .../crd/{score_issuer.rs => score_k8s_issuer.rs} | 4 ++-- harmony/src/modules/cert_manager/mod.rs | 6 +++--- ...score_operator.rs => score_cert_management.rs} | 0 ...{score_create_cert.rs => score_certificate.rs} | 15 +++++---------- .../{score_create_issuer.rs => score_issuer.rs} | 0 10 files changed, 21 insertions(+), 28 deletions(-) rename harmony/src/modules/cert_manager/crd/{score_certificate.rs => score_k8s_certificate.rs} (93%) rename harmony/src/modules/cert_manager/crd/{score_cluster_issuer.rs => score_k8s_cluster_issuer.rs} (100%) rename harmony/src/modules/cert_manager/crd/{score_issuer.rs => score_k8s_issuer.rs} (93%) rename harmony/src/modules/cert_manager/{score_operator.rs => score_cert_management.rs} (100%) rename harmony/src/modules/cert_manager/{score_create_cert.rs => score_certificate.rs} (84%) rename harmony/src/modules/cert_manager/{score_create_issuer.rs => score_issuer.rs} (100%) diff --git a/examples/cert_manager/src/main.rs b/examples/cert_manager/src/main.rs index 1d0c3acd..7ab3ce18 100644 --- a/examples/cert_manager/src/main.rs +++ b/examples/cert_manager/src/main.rs @@ -1,8 +1,8 @@ use harmony::{ inventory::Inventory, modules::cert_manager::{ - capability::CertificateManagementConfig, score_create_cert::CertificateCreationScore, - score_create_issuer::CertificateIssuerScore, score_operator::CertificateManagementScore, + capability::CertificateManagementConfig, score_cert_management::CertificateManagementScore, + score_certificate::CertificateScore, score_issuer::CertificateIssuerScore, }, topology::K8sAnywhereTopology, }; @@ -25,7 +25,7 @@ async fn main() { issuer_name: "test-self-signed-issuer".to_string(), }; - let cert = CertificateCreationScore { + let cert = CertificateScore { config: config.clone(), cert_name: "test-self-signed-cert".to_string(), issuer_name: "test-self-signed-issuer".to_string(), diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index d0ef5a32..ec9420d6 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -20,10 +20,8 @@ use crate::{ cert_manager::{ capability::{CertificateManagement, CertificateManagementConfig}, crd::{ - certificate::Certificate, - issuer::{Issuer, IssuerSpec}, - score_certificate::CertificateScore, - score_issuer::IssuerScore, + certificate::Certificate, issuer::Issuer, + score_k8s_certificate::K8sCertificateScore, score_k8s_issuer::K8sIssuerScore, }, operator::CertManagerOperatorScore, }, @@ -437,7 +435,7 @@ impl CertificateManagement for K8sAnywhereTopology { issuer_name: String, config: &CertificateManagementConfig, ) -> Result { - let issuer_score = IssuerScore { + let issuer_score = K8sIssuerScore { issuer_name: issuer_name.clone(), config: config.clone(), }; @@ -466,7 +464,7 @@ impl CertificateManagement for K8sAnywhereTopology { ) .await?; - let cert = CertificateScore { + let cert = K8sCertificateScore { cert_name: cert_name, config: config.clone(), issuer_name, diff --git a/harmony/src/modules/cert_manager/crd/mod.rs b/harmony/src/modules/cert_manager/crd/mod.rs index 7f8e6d1c..7725c1ab 100644 --- a/harmony/src/modules/cert_manager/crd/mod.rs +++ b/harmony/src/modules/cert_manager/crd/mod.rs @@ -4,8 +4,8 @@ pub mod certificate; pub mod cluster_issuer; pub mod issuer; //pub mod score_cluster_issuer; -pub mod score_certificate; -pub mod score_issuer; +pub mod score_k8s_certificate; +pub mod score_k8s_issuer; #[derive(Deserialize, Serialize, Clone, Debug)] #[serde(rename_all = "camelCase")] diff --git a/harmony/src/modules/cert_manager/crd/score_certificate.rs b/harmony/src/modules/cert_manager/crd/score_k8s_certificate.rs similarity index 93% rename from harmony/src/modules/cert_manager/crd/score_certificate.rs rename to harmony/src/modules/cert_manager/crd/score_k8s_certificate.rs index fb9a145a..adc7cc48 100644 --- a/harmony/src/modules/cert_manager/crd/score_certificate.rs +++ b/harmony/src/modules/cert_manager/crd/score_k8s_certificate.rs @@ -15,13 +15,13 @@ use crate::{ }; #[derive(Debug, Clone, Serialize)] -pub struct CertificateScore { +pub struct K8sCertificateScore { pub cert_name: String, pub issuer_name: String, pub config: CertificateManagementConfig, } -impl Score for CertificateScore { +impl Score for K8sCertificateScore { fn name(&self) -> String { "CertificateScore".to_string() } diff --git a/harmony/src/modules/cert_manager/crd/score_cluster_issuer.rs b/harmony/src/modules/cert_manager/crd/score_k8s_cluster_issuer.rs similarity index 100% rename from harmony/src/modules/cert_manager/crd/score_cluster_issuer.rs rename to harmony/src/modules/cert_manager/crd/score_k8s_cluster_issuer.rs diff --git a/harmony/src/modules/cert_manager/crd/score_issuer.rs b/harmony/src/modules/cert_manager/crd/score_k8s_issuer.rs similarity index 93% rename from harmony/src/modules/cert_manager/crd/score_issuer.rs rename to harmony/src/modules/cert_manager/crd/score_k8s_issuer.rs index c858d6ef..213da588 100644 --- a/harmony/src/modules/cert_manager/crd/score_issuer.rs +++ b/harmony/src/modules/cert_manager/crd/score_k8s_issuer.rs @@ -18,12 +18,12 @@ use crate::{ }; #[derive(Debug, Clone, Serialize)] -pub struct IssuerScore { +pub struct K8sIssuerScore { pub issuer_name: String, pub config: CertificateManagementConfig, } -impl Score for IssuerScore { +impl Score for K8sIssuerScore { fn name(&self) -> String { "IssuerScore".to_string() } diff --git a/harmony/src/modules/cert_manager/mod.rs b/harmony/src/modules/cert_manager/mod.rs index d1ff660e..46c1cb46 100644 --- a/harmony/src/modules/cert_manager/mod.rs +++ b/harmony/src/modules/cert_manager/mod.rs @@ -3,7 +3,7 @@ pub mod cluster_issuer; pub mod crd; mod helm; pub mod operator; -pub mod score_create_cert; -pub mod score_create_issuer; -pub mod score_operator; +pub mod score_cert_management; +pub mod score_certificate; +pub mod score_issuer; pub use helm::*; diff --git a/harmony/src/modules/cert_manager/score_operator.rs b/harmony/src/modules/cert_manager/score_cert_management.rs similarity index 100% rename from harmony/src/modules/cert_manager/score_operator.rs rename to harmony/src/modules/cert_manager/score_cert_management.rs diff --git a/harmony/src/modules/cert_manager/score_create_cert.rs b/harmony/src/modules/cert_manager/score_certificate.rs similarity index 84% rename from harmony/src/modules/cert_manager/score_create_cert.rs rename to harmony/src/modules/cert_manager/score_certificate.rs index 8f3a21b6..19cf7f13 100644 --- a/harmony/src/modules/cert_manager/score_create_cert.rs +++ b/harmony/src/modules/cert_manager/score_certificate.rs @@ -13,19 +13,19 @@ use crate::{ }; #[derive(Debug, Clone, Serialize)] -pub struct CertificateCreationScore { +pub struct CertificateScore { pub cert_name: String, pub issuer_name: String, pub config: CertificateManagementConfig, } -impl Score for CertificateCreationScore { +impl Score for CertificateScore { fn name(&self) -> String { "CertificateCreationScore".to_string() } fn create_interpret(&self) -> Box> { - Box::new(CertificateCreationInterpret { + Box::new(CertificateInterpret { cert_name: self.cert_name.clone(), issuer_name: self.issuer_name.clone(), config: self.config.clone(), @@ -34,14 +34,14 @@ impl Score for CertificateCreationScore } #[derive(Debug)] -struct CertificateCreationInterpret { +struct CertificateInterpret { cert_name: String, issuer_name: String, config: CertificateManagementConfig, } #[async_trait] -impl Interpret for CertificateCreationInterpret { +impl Interpret for CertificateInterpret { async fn execute( &self, inventory: &Inventory, @@ -56,11 +56,6 @@ impl Interpret for CertificateCreationIn .await .map_err(|e| InterpretError::new(e.to_string()))?; - let ca_cert = topology - .get_ca_certificate("test-self-signed-cert".to_string(), &self.config) - .await?; - trace!("cacert: {}", ca_cert); - Ok(Outcome::success(format!("Installed CertificateManagement"))) } diff --git a/harmony/src/modules/cert_manager/score_create_issuer.rs b/harmony/src/modules/cert_manager/score_issuer.rs similarity index 100% rename from harmony/src/modules/cert_manager/score_create_issuer.rs rename to harmony/src/modules/cert_manager/score_issuer.rs -- 2.39.5 From f6a20832cf64817e52b3053eb5601dbb208062ba Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Tue, 20 Jan 2026 12:11:48 -0500 Subject: [PATCH 17/21] lint: Remove useless variable assignment --- harmony/src/modules/cert_manager/score_cert_management.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/harmony/src/modules/cert_manager/score_cert_management.rs b/harmony/src/modules/cert_manager/score_cert_management.rs index ec783266..23d30df0 100644 --- a/harmony/src/modules/cert_manager/score_cert_management.rs +++ b/harmony/src/modules/cert_manager/score_cert_management.rs @@ -37,10 +37,10 @@ struct CertificateManagementInterpret { impl Interpret for CertificateManagementInterpret { async fn execute( &self, - inventory: &Inventory, + _inventory: &Inventory, topology: &T, ) -> Result { - let _cert_management = &CertificateManagement::ensure_ready(topology, &self.config) + CertificateManagement::ensure_ready(topology, &self.config) .await .map_err(|e| InterpretError::new(e.to_string()))?; -- 2.39.5 From bc962be31f85234104ba4208923c25d37e65773f Mon Sep 17 00:00:00 2001 From: wjro Date: Tue, 20 Jan 2026 13:21:24 -0500 Subject: [PATCH 18/21] fix: moved cert management ensure ready to k8sanywhere --- examples/cert_manager/src/main.rs | 6 +---- harmony/src/domain/topology/k8s.rs | 26 +++++-------------- .../topology/k8s_anywhere/k8s_anywhere.rs | 19 +++++++++----- .../src/modules/cert_manager/capability.rs | 5 +--- .../cert_manager/crd/score_k8s_certificate.rs | 4 +-- .../cert_manager/crd/score_k8s_issuer.rs | 4 +-- .../cert_manager/score_cert_management.rs | 14 +++------- harmony/src/modules/mod.rs | 1 + 8 files changed, 30 insertions(+), 49 deletions(-) diff --git a/examples/cert_manager/src/main.rs b/examples/cert_manager/src/main.rs index 7ab3ce18..3403bfb1 100644 --- a/examples/cert_manager/src/main.rs +++ b/examples/cert_manager/src/main.rs @@ -16,10 +16,6 @@ async fn main() { self_signed: true, }; - let cert_manager = CertificateManagementScore { - config: config.clone(), - }; - let issuer = CertificateIssuerScore { config: config.clone(), issuer_name: "test-self-signed-issuer".to_string(), @@ -34,7 +30,7 @@ async fn main() { harmony_cli::run( Inventory::autoload(), K8sAnywhereTopology::from_env(), - vec![Box::new(cert_manager), Box::new(issuer), Box::new(cert)], + vec![Box::new(issuer), Box::new(cert)], None, ) .await diff --git a/harmony/src/domain/topology/k8s.rs b/harmony/src/domain/topology/k8s.rs index bb786eb9..ef34f3cf 100644 --- a/harmony/src/domain/topology/k8s.rs +++ b/harmony/src/domain/topology/k8s.rs @@ -230,26 +230,14 @@ impl K8sClient { namespace: Option<&str>, gvk: &GroupVersionKind, ) -> Result { - let api_resource = ApiResource::from_gvk(gvk); + let gvk = ApiResource::from_gvk(gvk); + let resource: Api = if let Some(ns) = namespace { + Api::namespaced_with(self.client.clone(), ns, &gvk) + } else { + Api::default_namespaced_with(self.client.clone(), &gvk) + }; - // 1. Try namespaced first (if a namespace was provided) - if let Some(ns) = namespace { - let api: Api = - Api::namespaced_with(self.client.clone(), ns, &api_resource); - - match api.get(name).await { - Ok(obj) => return Ok(obj), - Err(Error::Api(ae)) if ae.code == 404 => { - // fall through and try cluster-scoped - } - Err(e) => return Err(e), - } - } - - // 2. Fallback to cluster-scoped - let api: Api = Api::all_with(self.client.clone(), &api_resource); - - api.get(name).await + resource.get(name).await } pub async fn get_secret_json_value( diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index ec9420d6..15003a83 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -21,9 +21,11 @@ use crate::{ capability::{CertificateManagement, CertificateManagementConfig}, crd::{ certificate::Certificate, issuer::Issuer, - score_k8s_certificate::K8sCertificateScore, score_k8s_issuer::K8sIssuerScore, + score_k8s_certificate::K8sCertManagerCertificateScore, + score_k8s_issuer::K8sCertManagerIssuerScore, }, operator::CertManagerOperatorScore, + score_cert_management::CertificateManagementScore, }, k3d::K3DInstallationScore, k8s::{ @@ -411,10 +413,7 @@ impl CertificateManagement for K8sAnywhereTopology { ))) } - async fn ensure_ready( - &self, - config: &CertificateManagementConfig, - ) -> Result { + async fn ensure_ready(&self) -> Result { let k8s_client = self.k8s_client().await.unwrap(); match k8s_client @@ -435,7 +434,7 @@ impl CertificateManagement for K8sAnywhereTopology { issuer_name: String, config: &CertificateManagementConfig, ) -> Result { - let issuer_score = K8sIssuerScore { + let issuer_score = K8sCertManagerIssuerScore { issuer_name: issuer_name.clone(), config: config.clone(), }; @@ -464,7 +463,7 @@ impl CertificateManagement for K8sAnywhereTopology { ) .await?; - let cert = K8sCertificateScore { + let cert = K8sCertManagerCertificateScore { cert_name: cert_name, config: config.clone(), issuer_name, @@ -1271,6 +1270,12 @@ impl Topology for K8sAnywhereTopology { .await .map_err(PreparationError::new)?; + let cert_mgmt = CertificateManagementScore {}; + cert_mgmt + .interpret(&Inventory::empty(), self) + .await + .map_err(|e| PreparationError::new(format!("{}", e)))?; + match self.is_helm_available() { Ok(()) => Ok(PreparationOutcome::Success { details: format!("{} + helm available", k8s_state.message.clone()), diff --git a/harmony/src/modules/cert_manager/capability.rs b/harmony/src/modules/cert_manager/capability.rs index 0610a726..979b27f7 100644 --- a/harmony/src/modules/cert_manager/capability.rs +++ b/harmony/src/modules/cert_manager/capability.rs @@ -12,10 +12,7 @@ use crate::{ pub trait CertificateManagement: Send + Sync { async fn install(&self) -> Result; - async fn ensure_ready( - &self, - config: &CertificateManagementConfig, - ) -> Result; + async fn ensure_ready(&self) -> Result; async fn create_issuer( &self, diff --git a/harmony/src/modules/cert_manager/crd/score_k8s_certificate.rs b/harmony/src/modules/cert_manager/crd/score_k8s_certificate.rs index adc7cc48..a1600b6e 100644 --- a/harmony/src/modules/cert_manager/crd/score_k8s_certificate.rs +++ b/harmony/src/modules/cert_manager/crd/score_k8s_certificate.rs @@ -15,13 +15,13 @@ use crate::{ }; #[derive(Debug, Clone, Serialize)] -pub struct K8sCertificateScore { +pub struct K8sCertManagerCertificateScore { pub cert_name: String, pub issuer_name: String, pub config: CertificateManagementConfig, } -impl Score for K8sCertificateScore { +impl Score for K8sCertManagerCertificateScore { fn name(&self) -> String { "CertificateScore".to_string() } diff --git a/harmony/src/modules/cert_manager/crd/score_k8s_issuer.rs b/harmony/src/modules/cert_manager/crd/score_k8s_issuer.rs index 213da588..750a017f 100644 --- a/harmony/src/modules/cert_manager/crd/score_k8s_issuer.rs +++ b/harmony/src/modules/cert_manager/crd/score_k8s_issuer.rs @@ -18,12 +18,12 @@ use crate::{ }; #[derive(Debug, Clone, Serialize)] -pub struct K8sIssuerScore { +pub struct K8sCertManagerIssuerScore { pub issuer_name: String, pub config: CertificateManagementConfig, } -impl Score for K8sIssuerScore { +impl Score for K8sCertManagerIssuerScore { fn name(&self) -> String { "IssuerScore".to_string() } diff --git a/harmony/src/modules/cert_manager/score_cert_management.rs b/harmony/src/modules/cert_manager/score_cert_management.rs index 23d30df0..fefd46cd 100644 --- a/harmony/src/modules/cert_manager/score_cert_management.rs +++ b/harmony/src/modules/cert_manager/score_cert_management.rs @@ -12,9 +12,7 @@ use crate::{ }; #[derive(Debug, Clone, Serialize)] -pub struct CertificateManagementScore { - pub config: CertificateManagementConfig, -} +pub struct CertificateManagementScore {} impl Score for CertificateManagementScore { fn name(&self) -> String { @@ -22,16 +20,12 @@ impl Score for CertificateManagementScor } fn create_interpret(&self) -> Box> { - Box::new(CertificateManagementInterpret { - config: self.config.clone(), - }) + Box::new(CertificateManagementInterpret {}) } } #[derive(Debug)] -struct CertificateManagementInterpret { - config: CertificateManagementConfig, -} +struct CertificateManagementInterpret {} #[async_trait] impl Interpret for CertificateManagementInterpret { @@ -40,7 +34,7 @@ impl Interpret for CertificateManagement _inventory: &Inventory, topology: &T, ) -> Result { - CertificateManagement::ensure_ready(topology, &self.config) + CertificateManagement::ensure_ready(topology) .await .map_err(|e| InterpretError::new(e.to_string()))?; diff --git a/harmony/src/modules/mod.rs b/harmony/src/modules/mod.rs index c845fb10..3fa69469 100644 --- a/harmony/src/modules/mod.rs +++ b/harmony/src/modules/mod.rs @@ -13,6 +13,7 @@ pub mod k8s; pub mod lamp; pub mod load_balancer; pub mod monitoring; +pub mod nats; pub mod network; pub mod okd; pub mod opnsense; -- 2.39.5 From 52bff9b6be36f406ca24b61fa6f9b446f9772064 Mon Sep 17 00:00:00 2001 From: wjro Date: Tue, 20 Jan 2026 13:43:52 -0500 Subject: [PATCH 19/21] fix: mod.rs --- harmony/src/modules/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/harmony/src/modules/mod.rs b/harmony/src/modules/mod.rs index 3fa69469..c845fb10 100644 --- a/harmony/src/modules/mod.rs +++ b/harmony/src/modules/mod.rs @@ -13,7 +13,6 @@ pub mod k8s; pub mod lamp; pub mod load_balancer; pub mod monitoring; -pub mod nats; pub mod network; pub mod okd; pub mod opnsense; -- 2.39.5 From 740b5500f2618c532423763079747abb26ef0459 Mon Sep 17 00:00:00 2001 From: wjro Date: Thu, 22 Jan 2026 11:35:51 -0500 Subject: [PATCH 20/21] feat: added poc for deploying nats supercluster with certificates, issuers, and okd routes --- examples/cert_manager/src/main.rs | 3 + examples/nats-supercluster/Cargo.toml | 1 + examples/nats-supercluster/src/main.rs | 410 +++++++++++++++--- .../topology/k8s_anywhere/k8s_anywhere.rs | 8 +- .../src/modules/cert_manager/capability.rs | 5 +- .../modules/cert_manager/crd/certificate.rs | 1 + .../cert_manager/crd/score_k8s_certificate.rs | 7 +- .../cert_manager/score_cert_management.rs | 2 +- .../modules/cert_manager/score_certificate.rs | 12 + .../src/modules/okd/crd/ingresses_config.rs | 1 - harmony/src/modules/okd/crd/mod.rs | 2 +- 11 files changed, 381 insertions(+), 71 deletions(-) diff --git a/examples/cert_manager/src/main.rs b/examples/cert_manager/src/main.rs index 3403bfb1..8ddf9e0b 100644 --- a/examples/cert_manager/src/main.rs +++ b/examples/cert_manager/src/main.rs @@ -25,6 +25,9 @@ async fn main() { config: config.clone(), cert_name: "test-self-signed-cert".to_string(), issuer_name: "test-self-signed-issuer".to_string(), + common_name: None, + dns_names: Some(vec!["test.dns.name".to_string()]), + is_ca: Some(false), }; harmony_cli::run( diff --git a/examples/nats-supercluster/Cargo.toml b/examples/nats-supercluster/Cargo.toml index fd1591a5..1744c01c 100644 --- a/examples/nats-supercluster/Cargo.toml +++ b/examples/nats-supercluster/Cargo.toml @@ -16,3 +16,4 @@ harmony_macros = { path = "../../harmony_macros" } log = { workspace = true } env_logger = { workspace = true } url = { workspace = true } +k8s-openapi.workspace = true diff --git a/examples/nats-supercluster/src/main.rs b/examples/nats-supercluster/src/main.rs index 9af3948a..8df41607 100644 --- a/examples/nats-supercluster/src/main.rs +++ b/examples/nats-supercluster/src/main.rs @@ -1,82 +1,369 @@ -use std::str::FromStr; +use std::{collections::BTreeMap, str::FromStr}; use harmony::{ + interpret::{InterpretError, Outcome}, inventory::Inventory, - modules::helm::chart::{HelmChartScore, HelmRepository, NonBlankString}, - topology::{HelmCommand, K8sAnywhereConfig, K8sAnywhereTopology, TlsRouter, Topology}, + modules::{ + cert_manager::{ + capability::{CertificateManagement, CertificateManagementConfig}, + crd::CaIssuer, + }, + helm::chart::{HelmChartScore, HelmRepository, NonBlankString}, + k8s::resource::K8sResourceScore, + okd::{ + crd::route::{RoutePort, RouteSpec, RouteTargetReference, TLSConfig}, + route::OKDRouteScore, + }, + }, + score::Score, + topology::{ + HelmCommand, K8sAnywhereConfig, K8sAnywhereTopology, K8sclient, TlsRouter, Topology, + }, }; use harmony_macros::hurl; +use k8s_openapi::{ + ByteString, api::core::v1::Secret, apimachinery::pkg::apis::meta::v1::ObjectMeta, +}; use log::{debug, info}; #[tokio::main] -async fn main() { - let site1_topo = K8sAnywhereTopology::with_config(K8sAnywhereConfig::remote_k8s_from_env_var( +async fn main() -> Result<(), InterpretError> { + run().await?; + Ok(()) +} + +async fn apply_scores( + topology: T, + scores: Vec>>, +) -> Result<(), InterpretError> { + info!("applying {} scores", scores.len()); + + harmony_cli::run(Inventory::autoload(), topology, scores, None) + .await + .map_err(|e| InterpretError::new(e.to_string()))?; + + Ok(()) +} + +async fn run() -> Result { + let namespace = "nats-supercluster-test"; + + log::info!("starting nats supercluster bootstrap"); + + // -------------------------------------------------- + // 1. Build site contexts + // -------------------------------------------------- + + let site1 = site( + "site1", "HARMONY_NATS_SITE_1", - )); - let site2_topo = K8sAnywhereTopology::with_config(K8sAnywhereConfig::remote_k8s_from_env_var( - "HARMONY_NATS_SITE_2", - )); - let (t1, t2) = tokio::join!(site1_topo.ensure_ready(), site2_topo.ensure_ready(),); - - t1.unwrap(); - t2.unwrap(); - - let site1_domain = std::env::var("HARMONY_NATS_SITE_1_DOMAIN") - .expect("HARMONY_NATS_SITE_1_DOMAIN env var not found"); - let site2_domain = std::env::var("HARMONY_NATS_SITE_2_DOMAIN") - .expect("HARMONY_NATS_SITE_2_DOMAIN env var not found"); - - // TODO automate creation of this ca bundle - // It is simply a secret that contains one key ca.crt - // And the value is the base64 with each clusters ca.crt concatenated - let supercluster_ca_secret_name = "nats-supercluster-ca-bundle"; - - let nats_site_1 = NatsCluster { - replicas: 1, - name: "nats-site1", - gateway_advertise: format!("nats-site1-gw.{site1_domain}:443"), - supercluster_ca_secret_name, - tls_secret_name: "nats-gateway-tls", - jetstream_enabled: "false", - }; - - let nats_site_2 = NatsCluster { - replicas: 1, - name: "nats-site2", - gateway_advertise: format!("nats-site2-gw.{site2_domain}:443"), - supercluster_ca_secret_name, - tls_secret_name: "nats-gateway-tls", - jetstream_enabled: "false", - }; - - tokio::join!( - deploy_nats( - site1_topo, - &nats_site_1, - vec![&nats_site_2] - ), - deploy_nats( - site2_topo, - &nats_site_2, - vec![&nats_site_1] - ), + "HARMONY_NATS_SITE_1_DOMAIN", + "nats-cb1-cert-test1", ); + + let site2 = site( + "site2", + "HARMONY_NATS_SITE_2", + "HARMONY_NATS_SITE_2_DOMAIN", + "nats-sto1-cert-test2", + ); + + // -------------------------------------------------- + // 2. Ensure clusters are reachable + // -------------------------------------------------- + + log::info!("ensuring both topologies are ready"); + + tokio::try_join!(site1.topology.ensure_ready(), site2.topology.ensure_ready(),)?; + + // -------------------------------------------------- + // 3. Create certificates + // -------------------------------------------------- + + log::info!("creating certificates"); + + tokio::try_join!( + create_nats_certs(site1.topology.clone(), &site1.cluster, namespace.into()), + create_nats_certs(site2.topology.clone(), &site2.cluster, namespace.into()), + )?; + + // -------------------------------------------------- + // 4. Build CA bundle + // -------------------------------------------------- + + log::info!("building supercluster CA bundle"); + + let ca_cfg = CertificateManagementConfig { + namespace: Some(namespace.into()), + acme_issuer: None, + ca_issuer: Some(CaIssuer { + secret_name: "harmony-root-ca-tls".into(), + }), + self_signed: true, + }; + + let mut ca_bundle = Vec::new(); + + add_ca_cert_to_ca_bundle(site1.topology.clone(), &mut ca_bundle, &ca_cfg).await?; + add_ca_cert_to_ca_bundle(site2.topology.clone(), &mut ca_bundle, &ca_cfg).await?; + + let ca_bundle = CaBundle { + name: "nats-supercluster-ca-bundle".into(), + ca_crt: ca_bundle, + }; + + let ca_secret = build_ca_bundle_secret(namespace, &ca_bundle).await; + + // -------------------------------------------------- + // 5. Build Scores + // -------------------------------------------------- + + log::info!("building scores"); + + let site1_scores = vec![ + build_ca_bundle_secret_score(site1.topology.clone(), ca_secret.clone(), namespace.into()) + .await, + build_route_score(site1.topology.clone(), &site1.cluster, namespace.into()).await, + build_deploy_nats_score( + site1.topology.clone(), + &site1.cluster, + vec![&site2.cluster], + namespace.into(), + ) + .await, + ]; + + let site2_scores = vec![ + build_ca_bundle_secret_score(site2.topology.clone(), ca_secret, namespace.into()).await, + build_route_score(site2.topology.clone(), &site2.cluster, namespace.into()).await, + build_deploy_nats_score( + site2.topology.clone(), + &site2.cluster, + vec![&site1.cluster], + namespace.into(), + ) + .await, + ]; + + // -------------------------------------------------- + // 6. Apply Scores + // -------------------------------------------------- + + log::info!("applying scores"); + + tokio::try_join!( + apply_scores(site1.topology.clone(), site1_scores), + apply_scores(site2.topology.clone(), site2_scores), + )?; + + log::info!("supercluster bootstrap complete"); + + Ok(Outcome::success( + "Enjoy! You can test your nats cluster by running : `kubectl exec -n {namespace} -it deployment/nats-box -- nats pub test hi`".to_string(), + )) +} + +fn site( + name: &'static str, + topo_env: &str, + domain_env: &str, + cluster_name: &'static str, +) -> SiteContext { + let domain = std::env::var(domain_env).expect("missing domain env"); + + let topology = + K8sAnywhereTopology::with_config(K8sAnywhereConfig::remote_k8s_from_env_var(topo_env)); + + SiteContext { + name, + topology, + cluster: NatsCluster { + replicas: 1, + name: cluster_name, + gateway_advertise: format!("{cluster_name}-gw.{domain}:443"), + dns_name: format!("{cluster_name}-gw.{domain}"), + supercluster_ca_secret_name: "nats-supercluster-ca-bundle", + tls_secret_name: "nats-gateway-tls", + jetstream_enabled: "false", + }, + } +} + +struct SiteContext { + name: &'static str, + topology: T, + cluster: NatsCluster, } struct NatsCluster { replicas: usize, name: &'static str, gateway_advertise: String, + dns_name: String, supercluster_ca_secret_name: &'static str, tls_secret_name: &'static str, jetstream_enabled: &'static str, } -async fn deploy_nats( +async fn create_nats_certs( + topology: T, + cluster: &NatsCluster, + namespace: String, +) -> Result { + //the order is pretty important + debug!("Applying certs to ns {}", namespace); + let ca_cert_config = CertificateManagementConfig { + namespace: Some(namespace.to_string().clone()), + acme_issuer: None, + ca_issuer: Some(CaIssuer { + secret_name: "harmony-root-ca-tls".to_string(), + }), + self_signed: false, + }; + + let self_signed_cert_config = CertificateManagementConfig { + namespace: Some(namespace.to_string().clone()), + acme_issuer: None, + ca_issuer: None, + self_signed: true, + }; + + debug!("creating issuer 'harmony-selfsigned-issuer'"); + topology + .create_issuer( + "harmony-selfsigned-issuer".to_string(), + &self_signed_cert_config, + ) + .await?; + + debug!("creating certificate harmony-root-ca"); + topology + .create_certificate( + "harmony-root-ca".to_string(), + "harmony-selfsigned-issuer".to_string(), + Some(format!("harmony-{}-ca", cluster.name)), + None, + Some(true), + &ca_cert_config, + ) + .await?; + + debug!("creating issuer 'harmony-ca-issuer'"); + topology + .create_issuer("harmony-ca-issuer".to_string(), &ca_cert_config) + .await?; + + debug!("creating certificate nats-gateway"); + topology + .create_certificate( + "nats-gateway".to_string(), + "harmony-ca-issuer".to_string(), + None, + Some(vec![cluster.dns_name.clone()]), + Some(true), + &ca_cert_config, + ) + .await?; + + Ok(Outcome::success("success".to_string())) +} + +async fn add_ca_cert_to_ca_bundle( + topology: T, + bundle: &mut Vec, + config: &CertificateManagementConfig, +) -> Result +where + T: Topology + CertificateManagement, +{ + debug!("getting ca cert"); + let ca = topology + .get_ca_certificate("harmony-root-ca".to_string(), config) + .await?; + + debug!("pushing ca cert to bundle vec: {:#?}", ca); + bundle.push(ca); + Ok(Outcome::success("pushed to bundle vec".to_string())) +} + +struct CaBundle { + name: String, + ca_crt: Vec, +} + +async fn build_ca_bundle_secret(namespace: &str, bundle: &CaBundle) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(bundle.name.clone()), + namespace: Some(namespace.to_string()), + ..Default::default() + }, + data: Some(build_secret_data(bundle).await), + immutable: Some(false), + type_: Some("Opaque".to_string()), + string_data: None, + } +} + +async fn build_secret_data(bundle: &CaBundle) -> BTreeMap { + let mut data = BTreeMap::new(); + + data.insert("ca.crt".to_string(), ByteString(build_ca_pem(bundle).await)); + + data +} + +async fn build_ca_pem(bundle: &CaBundle) -> Vec { + bundle.ca_crt.join("\n").into_bytes() +} + +async fn build_ca_bundle_secret_score( + topology: T, + secret: Secret, + namespace: String, +) -> Box> { + debug!( + "deploying secret to ns: {} \nsecret: {:#?}", + namespace, secret + ); + let k8ssecret = K8sResourceScore::single(secret, Some(namespace)); + Box::new(k8ssecret) +} + +async fn build_route_score( + topology: T, + cluster: &NatsCluster, + namespace: String, +) -> Box> { + let route = OKDRouteScore { + name: cluster.name.to_string(), + namespace, + spec: RouteSpec { + to: RouteTargetReference { + kind: "Service".to_string(), + name: cluster.name.to_string(), + weight: Some(100), + }, + host: Some(cluster.dns_name.clone()), + port: Some(RoutePort { target_port: 7222 }), + tls: Some(TLSConfig { + insecure_edge_termination_policy: None, + termination: "passthrough".to_string(), + ..Default::default() + }), + wildcard_policy: None, + ..Default::default() + }, + }; + Box::new(route) +} + +async fn build_deploy_nats_score( topology: T, cluster: &NatsCluster, peers: Vec<&NatsCluster>, -) { + namespace: String, +) -> Box> { let mut gateway_gateways = String::new(); for peer in peers { // Construct wss:// URLs on port 443 for the remote gateways @@ -165,13 +452,12 @@ natsBox: gateway_advertise = cluster.gateway_advertise, tls_secret_name = cluster.tls_secret_name, jetstream_enabled = cluster.jetstream_enabled, -supercluster_ca_secret_name = cluster.supercluster_ca_secret_name, + supercluster_ca_secret_name = cluster.supercluster_ca_secret_name, )); - let namespace = "harmony-nats"; debug!("Prepared Helm Chart values : \n{values_yaml:#?}"); let nats = HelmChartScore { - namespace: Some(NonBlankString::from_str(namespace).unwrap()), + namespace: Some(NonBlankString::from_str(&namespace).unwrap()), release_name: NonBlankString::from_str(&cluster.name).unwrap(), chart_name: NonBlankString::from_str("nats/nats").unwrap(), chart_version: None, @@ -186,11 +472,5 @@ supercluster_ca_secret_name = cluster.supercluster_ca_secret_name, )), }; - harmony_cli::run(Inventory::autoload(), topology, vec![Box::new(nats)], None) - .await - .unwrap(); - - info!( - "Enjoy! You can test your nats cluster by running : `kubectl exec -n {namespace} -it deployment/nats-box -- nats pub test hi`" - ); + Box::new(nats) } diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 15003a83..09898034 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -413,7 +413,7 @@ impl CertificateManagement for K8sAnywhereTopology { ))) } - async fn ensure_ready(&self) -> Result { + async fn ensure_certificate_management_ready(&self) -> Result { let k8s_client = self.k8s_client().await.unwrap(); match k8s_client @@ -454,6 +454,9 @@ impl CertificateManagement for K8sAnywhereTopology { &self, cert_name: String, issuer_name: String, + common_name: Option, + dns_names: Option>, + is_ca: Option, config: &CertificateManagementConfig, ) -> Result { self.certificate_issuer_ready( @@ -467,6 +470,9 @@ impl CertificateManagement for K8sAnywhereTopology { cert_name: cert_name, config: config.clone(), issuer_name, + common_name, + is_ca, + dns_names, }; cert.interpret(&Inventory::empty(), self) .await diff --git a/harmony/src/modules/cert_manager/capability.rs b/harmony/src/modules/cert_manager/capability.rs index 979b27f7..10aedff6 100644 --- a/harmony/src/modules/cert_manager/capability.rs +++ b/harmony/src/modules/cert_manager/capability.rs @@ -12,7 +12,7 @@ use crate::{ pub trait CertificateManagement: Send + Sync { async fn install(&self) -> Result; - async fn ensure_ready(&self) -> Result; + async fn ensure_certificate_management_ready(&self) -> Result; async fn create_issuer( &self, @@ -24,6 +24,9 @@ pub trait CertificateManagement: Send + Sync { &self, cert_name: String, issuer_name: String, + common_name: Option, + dns_names: Option>, + is_ca: Option, config: &CertificateManagementConfig, ) -> Result; diff --git a/harmony/src/modules/cert_manager/crd/certificate.rs b/harmony/src/modules/cert_manager/crd/certificate.rs index 2375b0b2..3dc98e83 100644 --- a/harmony/src/modules/cert_manager/crd/certificate.rs +++ b/harmony/src/modules/cert_manager/crd/certificate.rs @@ -40,6 +40,7 @@ pub struct CertificateSpec { /// Is this a CA certificate #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "isCA")] pub is_ca: Option, /// Private key configuration diff --git a/harmony/src/modules/cert_manager/crd/score_k8s_certificate.rs b/harmony/src/modules/cert_manager/crd/score_k8s_certificate.rs index a1600b6e..b150218b 100644 --- a/harmony/src/modules/cert_manager/crd/score_k8s_certificate.rs +++ b/harmony/src/modules/cert_manager/crd/score_k8s_certificate.rs @@ -18,6 +18,9 @@ use crate::{ pub struct K8sCertManagerCertificateScore { pub cert_name: String, pub issuer_name: String, + pub common_name: Option, + pub dns_names: Option>, + pub is_ca: Option, pub config: CertificateManagementConfig, } @@ -40,7 +43,9 @@ impl Score for K8sCertManagerCertificateScore { kind: Some("Issuer".into()), group: Some("cert-manager.io".into()), }, - dns_names: Some(vec!["test.example.local".to_string()]), + common_name: self.common_name.clone(), + is_ca: self.is_ca.clone(), + dns_names: self.dns_names.clone(), ..Default::default() }, }; diff --git a/harmony/src/modules/cert_manager/score_cert_management.rs b/harmony/src/modules/cert_manager/score_cert_management.rs index fefd46cd..8f194bac 100644 --- a/harmony/src/modules/cert_manager/score_cert_management.rs +++ b/harmony/src/modules/cert_manager/score_cert_management.rs @@ -34,7 +34,7 @@ impl Interpret for CertificateManagement _inventory: &Inventory, topology: &T, ) -> Result { - CertificateManagement::ensure_ready(topology) + CertificateManagement::ensure_certificate_management_ready(topology) .await .map_err(|e| InterpretError::new(e.to_string()))?; diff --git a/harmony/src/modules/cert_manager/score_certificate.rs b/harmony/src/modules/cert_manager/score_certificate.rs index 19cf7f13..35dbf702 100644 --- a/harmony/src/modules/cert_manager/score_certificate.rs +++ b/harmony/src/modules/cert_manager/score_certificate.rs @@ -16,6 +16,9 @@ use crate::{ pub struct CertificateScore { pub cert_name: String, pub issuer_name: String, + pub common_name: Option, + pub dns_names: Option>, + pub is_ca: Option, pub config: CertificateManagementConfig, } @@ -28,6 +31,9 @@ impl Score for CertificateScore { Box::new(CertificateInterpret { cert_name: self.cert_name.clone(), issuer_name: self.issuer_name.clone(), + common_name: self.common_name.clone(), + dns_names: self.dns_names.clone(), + is_ca: self.is_ca.clone(), config: self.config.clone(), }) } @@ -37,6 +43,9 @@ impl Score for CertificateScore { struct CertificateInterpret { cert_name: String, issuer_name: String, + common_name: Option, + dns_names: Option>, + is_ca: Option, config: CertificateManagementConfig, } @@ -51,6 +60,9 @@ impl Interpret for CertificateInterpret .create_certificate( self.cert_name.clone(), self.issuer_name.clone(), + self.common_name.clone(), + self.dns_names.clone(), + self.is_ca.clone(), &self.config, ) .await diff --git a/harmony/src/modules/okd/crd/ingresses_config.rs b/harmony/src/modules/okd/crd/ingresses_config.rs index 4c901565..801662f5 100644 --- a/harmony/src/modules/okd/crd/ingresses_config.rs +++ b/harmony/src/modules/okd/crd/ingresses_config.rs @@ -211,4 +211,3 @@ pub struct ObjectReference { pub namespace: String, pub resource: String, } - diff --git a/harmony/src/modules/okd/crd/mod.rs b/harmony/src/modules/okd/crd/mod.rs index c073458f..ba1d1a41 100644 --- a/harmony/src/modules/okd/crd/mod.rs +++ b/harmony/src/modules/okd/crd/mod.rs @@ -1,3 +1,3 @@ +pub mod ingresses_config; pub mod nmstate; pub mod route; -pub mod ingresses_config; -- 2.39.5 From 86572613426dbfd7fdb901bc1e5aff4e4c935dc4 Mon Sep 17 00:00:00 2001 From: wjro Date: Fri, 23 Jan 2026 15:03:03 -0500 Subject: [PATCH 21/21] fix: extracted variables, removed uncool side effect --- examples/cert_manager/src/main.rs | 5 +- examples/nats-supercluster/src/main.rs | 241 +++++++++--------- .../topology/k8s_anywhere/k8s_anywhere.rs | 2 +- .../src/modules/cert_manager/score_issuer.rs | 4 +- 4 files changed, 129 insertions(+), 123 deletions(-) diff --git a/examples/cert_manager/src/main.rs b/examples/cert_manager/src/main.rs index 8ddf9e0b..e024725f 100644 --- a/examples/cert_manager/src/main.rs +++ b/examples/cert_manager/src/main.rs @@ -16,15 +16,16 @@ async fn main() { self_signed: true, }; + let issuer_name = "test-self-signed-issuer".to_string(); let issuer = CertificateIssuerScore { + issuer_name: issuer_name.clone(), config: config.clone(), - issuer_name: "test-self-signed-issuer".to_string(), }; let cert = CertificateScore { config: config.clone(), + issuer_name, cert_name: "test-self-signed-cert".to_string(), - issuer_name: "test-self-signed-issuer".to_string(), common_name: None, dns_names: Some(vec!["test.dns.name".to_string()]), is_ca: Some(false), diff --git a/examples/nats-supercluster/src/main.rs b/examples/nats-supercluster/src/main.rs index 8df41607..8f4efdcf 100644 --- a/examples/nats-supercluster/src/main.rs +++ b/examples/nats-supercluster/src/main.rs @@ -28,25 +28,10 @@ use log::{debug, info}; #[tokio::main] async fn main() -> Result<(), InterpretError> { - run().await?; - Ok(()) -} - -async fn apply_scores( - topology: T, - scores: Vec>>, -) -> Result<(), InterpretError> { - info!("applying {} scores", scores.len()); - - harmony_cli::run(Inventory::autoload(), topology, scores, None) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - - Ok(()) -} - -async fn run() -> Result { let namespace = "nats-supercluster-test"; + let self_signed_issuer_name = "harmony-self-signed-issuer"; + let ca_issuer_name = "harmony-ca-issuer"; + let root_ca_cert_name = "harmony-root-ca"; log::info!("starting nats supercluster bootstrap"); @@ -55,17 +40,15 @@ async fn run() -> Result { // -------------------------------------------------- let site1 = site( - "site1", "HARMONY_NATS_SITE_1", "HARMONY_NATS_SITE_1_DOMAIN", - "nats-cb1-cert-test1", + "nats-sto1-cert-test1", ); let site2 = site( - "site2", "HARMONY_NATS_SITE_2", "HARMONY_NATS_SITE_2_DOMAIN", - "nats-sto1-cert-test2", + "nats-cb1-cert-test2", ); // -------------------------------------------------- @@ -82,9 +65,41 @@ async fn run() -> Result { log::info!("creating certificates"); + let root_ca_config = CertificateManagementConfig { + namespace: Some(namespace.into()), + acme_issuer: None, + ca_issuer: Some(CaIssuer { + secret_name: format!("{}-tls", root_ca_cert_name), + }), + self_signed: false, + }; + + let self_signed_config = CertificateManagementConfig { + namespace: Some(namespace.to_string().clone()), + acme_issuer: None, + ca_issuer: None, + self_signed: true, + }; + tokio::try_join!( - create_nats_certs(site1.topology.clone(), &site1.cluster, namespace.into()), - create_nats_certs(site2.topology.clone(), &site2.cluster, namespace.into()), + create_nats_certs( + site1.topology.clone(), + &site1.cluster, + ca_issuer_name, + &root_ca_config, + self_signed_issuer_name, + &self_signed_config, + root_ca_cert_name + ), + create_nats_certs( + site2.topology.clone(), + &site2.cluster, + ca_issuer_name, + &root_ca_config, + self_signed_issuer_name, + &self_signed_config, + root_ca_cert_name + ), )?; // -------------------------------------------------- @@ -93,26 +108,20 @@ async fn run() -> Result { log::info!("building supercluster CA bundle"); - let ca_cfg = CertificateManagementConfig { - namespace: Some(namespace.into()), - acme_issuer: None, - ca_issuer: Some(CaIssuer { - secret_name: "harmony-root-ca-tls".into(), - }), - self_signed: true, - }; - let mut ca_bundle = Vec::new(); - add_ca_cert_to_ca_bundle(site1.topology.clone(), &mut ca_bundle, &ca_cfg).await?; - add_ca_cert_to_ca_bundle(site2.topology.clone(), &mut ca_bundle, &ca_cfg).await?; - - let ca_bundle = CaBundle { - name: "nats-supercluster-ca-bundle".into(), - ca_crt: ca_bundle, - }; - - let ca_secret = build_ca_bundle_secret(namespace, &ca_bundle).await; + ca_bundle.push( + site1 + .topology + .get_ca_certificate(root_ca_cert_name.to_string(), &root_ca_config) + .await?, + ); + ca_bundle.push( + site2 + .topology + .get_ca_certificate(root_ca_cert_name.to_string(), &root_ca_config) + .await?, + ); // -------------------------------------------------- // 5. Build Scores @@ -121,8 +130,13 @@ async fn run() -> Result { log::info!("building scores"); let site1_scores = vec![ - build_ca_bundle_secret_score(site1.topology.clone(), ca_secret.clone(), namespace.into()) - .await, + build_ca_bundle_secret_score( + site1.topology.clone(), + &site1.cluster, + &ca_bundle, + namespace.into(), + ) + .await, build_route_score(site1.topology.clone(), &site1.cluster, namespace.into()).await, build_deploy_nats_score( site1.topology.clone(), @@ -134,7 +148,13 @@ async fn run() -> Result { ]; let site2_scores = vec![ - build_ca_bundle_secret_score(site2.topology.clone(), ca_secret, namespace.into()).await, + build_ca_bundle_secret_score( + site2.topology.clone(), + &site2.cluster, + &ca_bundle, + namespace.into(), + ) + .await, build_route_score(site2.topology.clone(), &site2.cluster, namespace.into()).await, build_deploy_nats_score( site2.topology.clone(), @@ -157,14 +177,26 @@ async fn run() -> Result { )?; log::info!("supercluster bootstrap complete"); + log::info!( + "Enjoy! You can test your nats cluster by running : `kubectl exec -n {namespace} -it deployment/nats-box -- nats pub test hi`" + ); + Ok(()) +} - Ok(Outcome::success( - "Enjoy! You can test your nats cluster by running : `kubectl exec -n {namespace} -it deployment/nats-box -- nats pub test hi`".to_string(), - )) +async fn apply_scores( + topology: T, + scores: Vec>>, +) -> Result<(), InterpretError> { + info!("applying {} scores", scores.len()); + + harmony_cli::run(Inventory::autoload(), topology, scores, None) + .await + .map_err(|e| InterpretError::new(e.to_string()))?; + + Ok(()) } fn site( - name: &'static str, topo_env: &str, domain_env: &str, cluster_name: &'static str, @@ -175,7 +207,6 @@ fn site( K8sAnywhereTopology::with_config(K8sAnywhereConfig::remote_k8s_from_env_var(topo_env)); SiteContext { - name, topology, cluster: NatsCluster { replicas: 1, @@ -183,14 +214,13 @@ fn site( gateway_advertise: format!("{cluster_name}-gw.{domain}:443"), dns_name: format!("{cluster_name}-gw.{domain}"), supercluster_ca_secret_name: "nats-supercluster-ca-bundle", - tls_secret_name: "nats-gateway-tls", + tls_cert_name: "nats-gateway", jetstream_enabled: "false", }, } } struct SiteContext { - name: &'static str, topology: T, cluster: NatsCluster, } @@ -201,100 +231,74 @@ struct NatsCluster { gateway_advertise: String, dns_name: String, supercluster_ca_secret_name: &'static str, - tls_secret_name: &'static str, + tls_cert_name: &'static str, jetstream_enabled: &'static str, } async fn create_nats_certs( topology: T, cluster: &NatsCluster, - namespace: String, + ca_issuer_name: &str, + ca_cert_mgmt_config: &CertificateManagementConfig, + self_signed_issuer_name: &str, + self_signed_cert_config: &CertificateManagementConfig, + root_ca_cert_name: &str, ) -> Result { //the order is pretty important - debug!("Applying certs to ns {}", namespace); - let ca_cert_config = CertificateManagementConfig { - namespace: Some(namespace.to_string().clone()), - acme_issuer: None, - ca_issuer: Some(CaIssuer { - secret_name: "harmony-root-ca-tls".to_string(), - }), - self_signed: false, - }; - let self_signed_cert_config = CertificateManagementConfig { - namespace: Some(namespace.to_string().clone()), - acme_issuer: None, - ca_issuer: None, - self_signed: true, - }; + debug!( + "Applying certs to ns {:#?}", + ca_cert_mgmt_config.namespace.clone() + ); - debug!("creating issuer 'harmony-selfsigned-issuer'"); + debug!("creating issuer '{}'", self_signed_issuer_name); topology .create_issuer( - "harmony-selfsigned-issuer".to_string(), + self_signed_issuer_name.to_string(), &self_signed_cert_config, ) .await?; - debug!("creating certificate harmony-root-ca"); + debug!("creating certificate {root_ca_cert_name}"); topology .create_certificate( - "harmony-root-ca".to_string(), - "harmony-selfsigned-issuer".to_string(), + root_ca_cert_name.to_string(), + self_signed_issuer_name.to_string(), Some(format!("harmony-{}-ca", cluster.name)), None, Some(true), - &ca_cert_config, + ca_cert_mgmt_config, ) .await?; - debug!("creating issuer 'harmony-ca-issuer'"); + debug!("creating issuer '{}'", ca_issuer_name); topology - .create_issuer("harmony-ca-issuer".to_string(), &ca_cert_config) + .create_issuer(ca_issuer_name.to_string(), ca_cert_mgmt_config) .await?; - debug!("creating certificate nats-gateway"); + debug!("creating certificate {}", cluster.tls_cert_name); topology .create_certificate( - "nats-gateway".to_string(), - "harmony-ca-issuer".to_string(), + cluster.tls_cert_name.to_string(), + ca_issuer_name.to_string(), None, Some(vec![cluster.dns_name.clone()]), Some(true), - &ca_cert_config, + ca_cert_mgmt_config, ) .await?; Ok(Outcome::success("success".to_string())) } -async fn add_ca_cert_to_ca_bundle( - topology: T, - bundle: &mut Vec, - config: &CertificateManagementConfig, -) -> Result -where - T: Topology + CertificateManagement, -{ - debug!("getting ca cert"); - let ca = topology - .get_ca_certificate("harmony-root-ca".to_string(), config) - .await?; - - debug!("pushing ca cert to bundle vec: {:#?}", ca); - bundle.push(ca); - Ok(Outcome::success("pushed to bundle vec".to_string())) -} - -struct CaBundle { - name: String, - ca_crt: Vec, -} - -async fn build_ca_bundle_secret(namespace: &str, bundle: &CaBundle) -> Secret { +async fn build_ca_bundle_secret( + namespace: &str, + nats_cluster: &NatsCluster, + bundle: &Vec, +) -> Secret { Secret { metadata: ObjectMeta { - name: Some(bundle.name.clone()), + name: Some(nats_cluster.supercluster_ca_secret_name.to_string()), namespace: Some(namespace.to_string()), ..Default::default() }, @@ -305,33 +309,34 @@ async fn build_ca_bundle_secret(namespace: &str, bundle: &CaBundle) -> Secret { } } -async fn build_secret_data(bundle: &CaBundle) -> BTreeMap { +async fn build_secret_data(bundle: &Vec) -> BTreeMap { let mut data = BTreeMap::new(); - data.insert("ca.crt".to_string(), ByteString(build_ca_pem(bundle).await)); + data.insert( + "ca.crt".to_string(), + ByteString(bundle.join("\n").into_bytes()), + ); data } -async fn build_ca_pem(bundle: &CaBundle) -> Vec { - bundle.ca_crt.join("\n").into_bytes() -} - async fn build_ca_bundle_secret_score( - topology: T, - secret: Secret, + _topology: T, + nats_cluster: &NatsCluster, + ca_bundle: &Vec, namespace: String, ) -> Box> { + let bundle_secret = build_ca_bundle_secret(&namespace, nats_cluster, ca_bundle).await; debug!( "deploying secret to ns: {} \nsecret: {:#?}", - namespace, secret + namespace, bundle_secret ); - let k8ssecret = K8sResourceScore::single(secret, Some(namespace)); + let k8ssecret = K8sResourceScore::single(bundle_secret, Some(namespace)); Box::new(k8ssecret) } async fn build_route_score( - topology: T, + _topology: T, cluster: &NatsCluster, namespace: String, ) -> Box> { @@ -450,7 +455,7 @@ natsBox: domain = domain, gateway_gateways = gateway_gateways, gateway_advertise = cluster.gateway_advertise, - tls_secret_name = cluster.tls_secret_name, + tls_secret_name = format!("{}-tls", cluster.tls_cert_name), jetstream_enabled = cluster.jetstream_enabled, supercluster_ca_secret_name = cluster.supercluster_ca_secret_name, )); diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 09898034..877a53c9 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -468,8 +468,8 @@ impl CertificateManagement for K8sAnywhereTopology { let cert = K8sCertManagerCertificateScore { cert_name: cert_name, - config: config.clone(), issuer_name, + config: config.clone(), common_name, is_ca, dns_names, diff --git a/harmony/src/modules/cert_manager/score_issuer.rs b/harmony/src/modules/cert_manager/score_issuer.rs index 447ddf67..285d994f 100644 --- a/harmony/src/modules/cert_manager/score_issuer.rs +++ b/harmony/src/modules/cert_manager/score_issuer.rs @@ -25,16 +25,16 @@ impl Score for CertificateIssuerScore { fn create_interpret(&self) -> Box> { Box::new(CertificateIssuerInterpret { - config: self.config.clone(), issuer_name: self.issuer_name.clone(), + config: self.config.clone(), }) } } #[derive(Debug)] struct CertificateIssuerInterpret { - config: CertificateManagementConfig, issuer_name: String, + config: CertificateManagementConfig, } #[async_trait] -- 2.39.5