From 11481b16cdf661a42433398f0185b5f2e38f01be Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Tue, 9 Sep 2025 16:41:53 -0400 Subject: [PATCH] fix: Multiple ingress fixes for localk3d, it works nicely now for Application and ntfy at least. Also fix k3d kubeconfig context by force switching to it every time. Not perfect but better and more intuitive for the user to view his resources. --- harmony/src/domain/topology/ingress.rs | 2 +- harmony/src/domain/topology/k8s_anywhere.rs | 6 +- .../features/continuous_delivery.rs | 2 +- .../application/features/helm_argocd_score.rs | 2 +- .../application/features/monitoring.rs | 2 +- .../application/features/rhob_monitoring.rs | 7 ++- harmony/src/modules/application/rust.rs | 2 +- harmony/src/modules/helm/chart.rs | 4 ++ .../monitoring/ntfy/helm/ntfy_helm_chart.rs | 6 ++ .../src/modules/prometheus/alerts/k8s/pod.rs | 4 +- .../modules/prometheus/rhob_alerting_score.rs | 6 +- k3d/src/lib.rs | 61 +++++++++++++++---- 12 files changed, 80 insertions(+), 24 deletions(-) diff --git a/harmony/src/domain/topology/ingress.rs b/harmony/src/domain/topology/ingress.rs index 6d2a5d6..69c9382 100644 --- a/harmony/src/domain/topology/ingress.rs +++ b/harmony/src/domain/topology/ingress.rs @@ -3,5 +3,5 @@ use async_trait::async_trait; #[async_trait] pub trait Ingress { - async fn get_domain(&self, service: String) -> Result; + async fn get_domain(&self, service: &str) -> Result; } diff --git a/harmony/src/domain/topology/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere.rs index 582e0f7..bb7dc6a 100644 --- a/harmony/src/domain/topology/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere.rs @@ -372,6 +372,8 @@ impl K8sAnywhereTopology { if let Some(Some(k8s_state)) = self.k8s_state.get() { match k8s_state.source { K8sSource::LocalK3d => { + warn!("Installing observability operator is not supported on LocalK3d source"); + return Ok(PreparationOutcome::Noop); debug!("installing cluster observability operator"); todo!(); let op_score = @@ -576,12 +578,12 @@ impl TenantManager for K8sAnywhereTopology { #[async_trait] impl Ingress for K8sAnywhereTopology { //TODO this is specifically for openshift/okd which violates the k8sanywhere idea - async fn get_domain(&self, service: String) -> Result { + async fn get_domain(&self, service: &str) -> Result { let client = self.k8s_client().await?; if let Some(Some(k8s_state)) = self.k8s_state.get() { match k8s_state.source { - K8sSource::LocalK3d => Ok("localhost".to_string()), + K8sSource::LocalK3d => Ok(format!("{service}.local.k3d")), K8sSource::Kubeconfig => { self.openshift_ingress_operator_available().await?; diff --git a/harmony/src/modules/application/features/continuous_delivery.rs b/harmony/src/modules/application/features/continuous_delivery.rs index 27b64f3..63e34a6 100644 --- a/harmony/src/modules/application/features/continuous_delivery.rs +++ b/harmony/src/modules/application/features/continuous_delivery.rs @@ -144,7 +144,7 @@ impl< async fn ensure_installed(&self, topology: &T) -> Result<(), String> { let image = self.application.image_name(); let domain = topology - .get_domain(self.application.name()) + .get_domain(&self.application.name()) .await .map_err(|e| e.to_string())?; diff --git a/harmony/src/modules/application/features/helm_argocd_score.rs b/harmony/src/modules/application/features/helm_argocd_score.rs index 7dcc2f2..84aff9a 100644 --- a/harmony/src/modules/application/features/helm_argocd_score.rs +++ b/harmony/src/modules/application/features/helm_argocd_score.rs @@ -55,7 +55,7 @@ impl Interpret for ArgoInter topology: &T, ) -> Result { let k8s_client = topology.k8s_client().await?; - let domain = topology.get_domain("argo".into()).await?; + let domain = topology.get_domain("argo").await?; let helm_score = argo_helm_chart_score(&self.score.namespace, self.score.openshift, &domain); diff --git a/harmony/src/modules/application/features/monitoring.rs b/harmony/src/modules/application/features/monitoring.rs index 80b4a04..b8531fe 100644 --- a/harmony/src/modules/application/features/monitoring.rs +++ b/harmony/src/modules/application/features/monitoring.rs @@ -50,7 +50,7 @@ impl< .await .map(|ns| ns.name.clone()) .unwrap_or_else(|| self.application.name()); - let domain = topology.get_domain("ntfy".into()).await.unwrap(); + let domain = topology.get_domain("ntfy").await.unwrap(); let mut alerting_score = ApplicationMonitoringScore { sender: CRDPrometheus { diff --git a/harmony/src/modules/application/features/rhob_monitoring.rs b/harmony/src/modules/application/features/rhob_monitoring.rs index 62a5323..e6f51a4 100644 --- a/harmony/src/modules/application/features/rhob_monitoring.rs +++ b/harmony/src/modules/application/features/rhob_monitoring.rs @@ -6,6 +6,7 @@ use crate::modules::monitoring::application_monitoring::rhobs_application_monito use crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::RHOBObservability; use crate::topology::MultiTargetTopology; +use crate::topology::ingress::Ingress; use crate::{ inventory::Inventory, modules::monitoring::{ @@ -37,6 +38,7 @@ impl< + TenantManager + K8sclient + MultiTargetTopology + + Ingress + std::fmt::Debug + PrometheusApplicationMonitoring, > ApplicationFeature for RHOBMonitoring @@ -59,7 +61,10 @@ impl< }; let ntfy = NtfyScore { namespace: namespace.clone(), - host: "ntfy.harmonydemo.apps.ncd0.harmony.mcd".to_string(), + host: topology + .get_domain("ntfy") + .await + .map_err(|e| format!("Could not get domain {e}"))?, }; ntfy.interpret(&Inventory::empty(), topology) .await diff --git a/harmony/src/modules/application/rust.rs b/harmony/src/modules/application/rust.rs index 3205682..a49b3db 100644 --- a/harmony/src/modules/application/rust.rs +++ b/harmony/src/modules/application/rust.rs @@ -222,6 +222,7 @@ impl RustWebapp { ".git", ".github", ".harmony_generated", + "harmony", "node_modules", ]; let mut entries: Vec<_> = WalkDir::new(project_root) @@ -425,7 +426,6 @@ impl RustWebapp { fs::create_dir_all(&templates_dir)?; let (image_repo, image_tag) = image_url.rsplit_once(':').unwrap_or((image_url, "latest")); - let domain = format!("{}.{domain}", self.name); // Create Chart.yaml let chart_yaml = format!( diff --git a/harmony/src/modules/helm/chart.rs b/harmony/src/modules/helm/chart.rs index 9048ce6..37769d7 100644 --- a/harmony/src/modules/helm/chart.rs +++ b/harmony/src/modules/helm/chart.rs @@ -153,6 +153,10 @@ impl Interpret for HelmChartInterpret { let yaml_path: Option<&Path> = match self.score.values_yaml.as_ref() { Some(yaml_str) => { tf = temp_file::with_contents(yaml_str.as_bytes()); + debug!( + "values yaml string for chart {} :\n {yaml_str}", + self.score.chart_name + ); Some(tf.path()) } None => None, diff --git a/harmony/src/modules/monitoring/ntfy/helm/ntfy_helm_chart.rs b/harmony/src/modules/monitoring/ntfy/helm/ntfy_helm_chart.rs index ecd9e01..57fffab 100644 --- a/harmony/src/modules/monitoring/ntfy/helm/ntfy_helm_chart.rs +++ b/harmony/src/modules/monitoring/ntfy/helm/ntfy_helm_chart.rs @@ -45,6 +45,12 @@ service: ingress: enabled: {ingress_enabled} + hosts: + - host: {host} + paths: + - path: / + pathType: ImplementationSpecific + route: enabled: {route_enabled} diff --git a/harmony/src/modules/prometheus/alerts/k8s/pod.rs b/harmony/src/modules/prometheus/alerts/k8s/pod.rs index 152ec2f..d116403 100644 --- a/harmony/src/modules/prometheus/alerts/k8s/pod.rs +++ b/harmony/src/modules/prometheus/alerts/k8s/pod.rs @@ -21,8 +21,8 @@ pub fn pod_failed() -> PrometheusAlertRule { pub fn alert_container_restarting() -> PrometheusAlertRule { PrometheusAlertRule { alert: "ContainerRestarting".into(), - expr: "increase(kube_pod_container_status_restarts_total[5m]) > 3".into(), - r#for: Some("5m".into()), + expr: "increase(kube_pod_container_status_restarts_total[30s]) > 3".into(), + r#for: Some("30s".into()), labels: HashMap::from([("severity".into(), "warning".into())]), annotations: HashMap::from([ ( diff --git a/harmony/src/modules/prometheus/rhob_alerting_score.rs b/harmony/src/modules/prometheus/rhob_alerting_score.rs index b547031..788c58e 100644 --- a/harmony/src/modules/prometheus/rhob_alerting_score.rs +++ b/harmony/src/modules/prometheus/rhob_alerting_score.rs @@ -282,7 +282,7 @@ impl RHOBAlertingInterpret { .map_err(|e| InterpretError::new(e.to_string()))?; let alert_manager_domain = topology - .get_domain(format!("alert-manager-{}", self.sender.namespace.clone())) + .get_domain(&format!("alert-manager-{}", self.sender.namespace.clone())) .await?; let name = format!("{}-alert-manager", self.sender.namespace.clone()); let backend_service = format!("alertmanager-operated"); @@ -299,7 +299,7 @@ impl RHOBAlertingInterpret { }; let prometheus_domain = topology - .get_domain(format!("prometheus-{}", self.sender.namespace.clone())) + .get_domain(&format!("prometheus-{}", self.sender.namespace.clone())) .await?; let name = format!("{}-prometheus", self.sender.namespace.clone()); let backend_service = format!("prometheus-operated"); @@ -497,7 +497,7 @@ impl RHOBAlertingInterpret { .await .map_err(|e| InterpretError::new(e.to_string()))?; let domain = topology - .get_domain(format!("grafana-{}", self.sender.namespace.clone())) + .get_domain(&format!("grafana-{}", self.sender.namespace.clone())) .await?; let name = format!("{}-grafana", self.sender.namespace.clone()); let backend_service = format!("grafana-{}-service", self.sender.namespace.clone()); diff --git a/k3d/src/lib.rs b/k3d/src/lib.rs index 7117d72..63611f4 100644 --- a/k3d/src/lib.rs +++ b/k3d/src/lib.rs @@ -2,8 +2,8 @@ mod downloadable_asset; use downloadable_asset::*; use kube::Client; -use log::debug; -use std::path::PathBuf; +use log::{debug, info}; +use std::{ffi::OsStr, path::PathBuf}; const K3D_BIN_FILE_NAME: &str = "k3d"; @@ -213,15 +213,19 @@ impl K3d { } } + let client; if !self.is_cluster_initialized() { debug!("Cluster is not initialized, initializing now"); - return self.initialize_cluster().await; + client = self.initialize_cluster().await?; + } else { + self.start_cluster().await?; + + debug!("K3d and cluster are already properly set up"); + client = self.create_kubernetes_client().await?; } - self.start_cluster().await?; - - debug!("K3d and cluster are already properly set up"); - self.create_kubernetes_client().await + self.ensure_k3d_config_is_default(self.get_cluster_name()?)?; + Ok(client) } // Private helper methods @@ -302,7 +306,16 @@ impl K3d { S: AsRef, { let binary_path = self.get_k3d_binary()?; - let output = std::process::Command::new(binary_path).args(args).output(); + self.run_command(binary_path, args) + } + + pub fn run_command(&self, cmd: C, args: I) -> Result + where + I: IntoIterator, + S: AsRef, + C: AsRef, + { + let output = std::process::Command::new(cmd).args(args).output(); match output { Ok(output) => { let stderr = String::from_utf8_lossy(&output.stderr); @@ -311,7 +324,7 @@ impl K3d { debug!("stdout : {}", stdout); Ok(output) } - Err(e) => Err(format!("Failed to execute k3d command: {}", e)), + Err(e) => Err(format!("Failed to execute command: {}", e)), } } @@ -323,12 +336,38 @@ impl K3d { return Err(format!("Failed to create cluster: {}", stderr)); } - debug!("Successfully created k3d cluster '{}'", cluster_name); + info!("Successfully created k3d cluster '{}'", cluster_name); + Ok(()) + } + + fn ensure_k3d_config_is_default(&self, cluster_name: &str) -> Result<(), String> { + let output = self.run_k3d_command(["kubeconfig", "merge", "-d", cluster_name])?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("Failed to setup k3d kubeconfig : {}", stderr)); + } + + let output = self.run_command( + "kubectl", + ["config", "use-context", &format!("k3d-{cluster_name}")], + )?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!( + "Failed to switch kubectl context to k3d : {}", + stderr + )); + } + info!( + "kubectl is now using 'k3d-{}' as default context", + cluster_name + ); Ok(()) } async fn create_kubernetes_client(&self) -> Result { - // TODO: Connect the client to the right k3d cluster (see https://git.nationtech.io/NationTech/harmony/issues/92) Client::try_default() .await .map_err(|e| format!("Failed to create Kubernetes client: {}", e))