From 8d446ec2e42231d5b6dc02e338afd775b1aa070e Mon Sep 17 00:00:00 2001 From: wjro Date: Thu, 19 Feb 2026 16:25:59 -0500 Subject: [PATCH 01/15] wip: refactoring monitoring --- .../topology/k8s_anywhere/k8s_anywhere.rs | 14 +- .../src/domain/topology/k8s_anywhere/mod.rs | 1 + .../k8s_anywhere/openshift_monitoring.rs | 73 +++++++++ .../topology/oberservability/monitoring.rs | 135 ++++++++++------ .../application/features/monitoring.rs | 4 +- .../application/features/rhob_monitoring.rs | 4 +- .../application_monitoring_score.rs | 4 +- .../rhobs_application_monitoring_score.rs | 8 +- .../crd/crd_alertmanager_config.rs | 8 +- .../src/modules/monitoring/okd/installable.rs | 40 +++++ harmony/src/modules/monitoring/okd/mod.rs | 5 + harmony/src/modules/monitoring/okd/old.md | 31 ++++ .../okd/score_cluster_monitoring.rs | 46 ++++++ .../okd/score_openshift_receiver.rs | 149 ++++++++++++++++++ .../monitoring/okd/score_user_workload.rs | 45 ++++++ .../score_verify_user_workload_monitoring.rs | 68 ++++++++ .../k8s_prometheus_alerting_score.rs | 6 +- harmony/src/modules/prometheus/prometheus.rs | 50 +++--- .../modules/prometheus/rhob_alerting_score.rs | 6 +- 19 files changed, 599 insertions(+), 98 deletions(-) create mode 100644 harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs create mode 100644 harmony/src/modules/monitoring/okd/installable.rs create mode 100644 harmony/src/modules/monitoring/okd/old.md create mode 100644 harmony/src/modules/monitoring/okd/score_cluster_monitoring.rs create mode 100644 harmony/src/modules/monitoring/okd/score_openshift_receiver.rs create mode 100644 harmony/src/modules/monitoring/okd/score_user_workload.rs create mode 100644 harmony/src/modules/monitoring/okd/score_verify_user_workload_monitoring.rs diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 55091d23..5462d67a 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -48,7 +48,7 @@ use crate::{ okd::{crd::ingresses_config::Ingress as IngressResource, route::OKDTlsPassthroughScore}, prometheus::{ k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore, - prometheus::PrometheusMonitoring, rhob_alerting_score::RHOBAlertingScore, + prometheus::Monitor, rhob_alerting_score::RHOBAlertingScore, }, }, score::Score, @@ -298,8 +298,8 @@ impl Grafana for K8sAnywhereTopology { } #[async_trait] -impl PrometheusMonitoring for K8sAnywhereTopology { - async fn install_prometheus( +impl Monitor for K8sAnywhereTopology { + async fn install_montoring( &self, sender: &CRDPrometheus, _inventory: &Inventory, @@ -318,7 +318,7 @@ impl PrometheusMonitoring for K8sAnywhereTopology { }) } - async fn ensure_prometheus_operator( + async fn ensure_monitoring( &self, sender: &CRDPrometheus, _inventory: &Inventory, @@ -339,8 +339,8 @@ impl PrometheusMonitoring for K8sAnywhereTopology { } #[async_trait] -impl PrometheusMonitoring for K8sAnywhereTopology { - async fn install_prometheus( +impl Monitor for K8sAnywhereTopology { + async fn install_montoring( &self, sender: &RHOBObservability, inventory: &Inventory, @@ -374,7 +374,7 @@ impl PrometheusMonitoring for K8sAnywhereTopology { } } - async fn ensure_prometheus_operator( + async fn ensure_monitoring( &self, sender: &RHOBObservability, inventory: &Inventory, diff --git a/harmony/src/domain/topology/k8s_anywhere/mod.rs b/harmony/src/domain/topology/k8s_anywhere/mod.rs index d14c28ba..c9f634e9 100644 --- a/harmony/src/domain/topology/k8s_anywhere/mod.rs +++ b/harmony/src/domain/topology/k8s_anywhere/mod.rs @@ -1,4 +1,5 @@ mod k8s_anywhere; pub mod nats; mod postgres; +pub mod openshift_monitoring; pub use k8s_anywhere::*; diff --git a/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs b/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs new file mode 100644 index 00000000..6695ffb4 --- /dev/null +++ b/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs @@ -0,0 +1,73 @@ +use async_trait::async_trait; + +use crate::{ + inventory::Inventory, + modules::monitoring::okd::{ + OpenshiftClusterAlertSender, score_openshift_receiver::OpenshiftReceiverScore, + }, + topology::{ + K8sAnywhereTopology, PreparationError, PreparationOutcome, + oberservability::monitoring::{AlertReceiver, AlertRule, Monitor, ScrapeTarget}, + }, +}; + +#[async_trait] +impl Monitor for K8sAnywhereTopology { + async fn install_receivers( + &self, + sender: &OpenshiftClusterAlertSender, + inventory: &Inventory, + receivers: Option>>>, + ) -> Result { + if let Some(receivers) = receivers { + for receiver in receivers { + let receiver_score = OpenshiftReceiverScore { + receiver}; + receiver_score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(e))?; + } + Ok(PreparationOutcome::Success { + details: "Success".to_string(), + }) + } else { + Ok(PreparationOutcome::Noop) + } + } + + async fn install_rules( + &self, + sender: &OpenshiftClusterAlertSender, + inventory: &Inventory, + rules: Option>>>, + ) -> Result { + if let Some(rules) = rules { + todo!() + } else { + Ok(PreparationOutcome::Noop) + } + } + + async fn add_scrape_targets( + &self, + sender: &OpenshiftClusterAlertSender, + inventory: &Inventory, + scrape_targets: Option>>>, + ) -> Result { + if let Some(scrape_targets) = scrape_targets { + todo!() + } else { + Ok(PreparationOutcome::Noop) + } + } + + async fn ensure_monitoring( + &self, + sender: &OpenshiftClusterAlertSender, + inventory: &Inventory, + ) -> Result { + todo!() + } +} diff --git a/harmony/src/domain/topology/oberservability/monitoring.rs b/harmony/src/domain/topology/oberservability/monitoring.rs index 78bb1419..f5d81d4b 100644 --- a/harmony/src/domain/topology/oberservability/monitoring.rs +++ b/harmony/src/domain/topology/oberservability/monitoring.rs @@ -2,21 +2,88 @@ use std::{any::Any, collections::HashMap}; use async_trait::async_trait; use kube::api::DynamicObject; -use log::debug; +use log::{debug, info}; use crate::{ data::Version, interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, - topology::{Topology, installable::Installable}, + topology::{PreparationError, PreparationOutcome, Topology, installable::Installable}, }; use harmony_types::id::Id; +/// Defines the application that sends the alerts to a receivers +/// for example prometheus #[async_trait] pub trait AlertSender: Send + Sync + std::fmt::Debug { fn name(&self) -> String; } +/// Defines the entity that receives the alerts from a sender. For example Discord, Slack, etc +#[async_trait] +pub trait AlertReceiver: std::fmt::Debug + Send + Sync { + async fn install(&self, sender: &S) -> Result; + fn name(&self) -> String; + fn clone_box(&self) -> Box>; + fn as_any(&self) -> &dyn Any; + fn as_alertmanager_receiver(&self) -> Result; +} + +#[derive(Debug)] +pub struct AlertManagerReceiver { + pub receiver_config: serde_json::Value, + // FIXME we should not leak k8s here. DynamicObject is k8s specific + pub additional_ressources: Vec, + pub route_config: serde_json::Value, +} + +/// Defines a generic rule that can be applied to a sender, such as aprometheus alert rule +#[async_trait] +pub trait AlertRule: std::fmt::Debug + Send + Sync { + async fn install(&self, sender: &S) -> Result; + fn clone_box(&self) -> Box>; +} + +/// A generic scrape target that can be added to a sender to scrape metrics from, for example a +/// server outside of the cluster +#[async_trait] +pub trait ScrapeTarget: std::fmt::Debug + Send + Sync { + async fn install(&self, sender: &S) -> Result; + fn clone_box(&self) -> Box>; +} + +/// Trait which defines how an alert sender is impleneted for a specific topology +#[async_trait] +pub trait Monitor { + async fn install_receivers( + &self, + sender: &S, + inventory: &Inventory, + receivers: Option>>>, + ) -> Result; + + async fn install_rules( + &self, + sender: &S, + inventory: &Inventory, + rules: Option>>>, + ) -> Result; + + async fn add_scrape_targets( + &self, + sender: &S, + inventory: &Inventory, + scrape_targets: Option>>>, + ) -> Result; + + async fn ensure_monitoring( + &self, + sender: &S, + inventory: &Inventory, + ) -> Result; +} + +/// Alerting interpret to install an alert sender on a given topology #[derive(Debug)] pub struct AlertingInterpret { pub sender: S, @@ -26,28 +93,33 @@ pub struct AlertingInterpret { } #[async_trait] -impl, T: Topology> Interpret for AlertingInterpret { +impl, T: Topology + Monitor> Interpret + for AlertingInterpret +{ async fn execute( &self, inventory: &Inventory, topology: &T, ) -> Result { - debug!("hit sender configure for AlertingInterpret"); + info!("Configuring alert sender {}", self.sender.name()); self.sender.configure(inventory, topology).await?; - for receiver in self.receivers.iter() { - receiver.install(&self.sender).await?; - } - for rule in self.rules.iter() { - debug!("installing rule: {:#?}", rule); - rule.install(&self.sender).await?; - } - if let Some(targets) = &self.scrape_targets { - for target in targets.iter() { - debug!("installing scrape_target: {:#?}", target); - target.install(&self.sender).await?; - } - } + + info!("Installing receivers"); + topology + .install_receivers(&self.sender, inventory, Some(self.receivers)) + .await?; + + info!("Installing rules"); + topology + .install_rules(&self.sender, inventory, Some(self.rules)) + .await?; + + info!("Adding extra scrape targets"); + topology.add_scrape_targets(&self.sender, inventory, self.scrape_targets).await?; + + info!("Ensuring alert sender {} is ready", self.sender.name()); self.sender.ensure_installed(inventory, topology).await?; + Ok(Outcome::success(format!( "successfully installed alert sender {}", self.sender.name() @@ -70,32 +142,3 @@ impl, T: Topology> Interpret for AlertingInte todo!() } } - -#[async_trait] -pub trait AlertReceiver: std::fmt::Debug + Send + Sync { - async fn install(&self, sender: &S) -> Result; - fn name(&self) -> String; - fn clone_box(&self) -> Box>; - fn as_any(&self) -> &dyn Any; - fn as_alertmanager_receiver(&self) -> Result; -} - -#[derive(Debug)] -pub struct AlertManagerReceiver { - pub receiver_config: serde_json::Value, - // FIXME we should not leak k8s here. DynamicObject is k8s specific - pub additional_ressources: Vec, - pub route_config: serde_json::Value, -} - -#[async_trait] -pub trait AlertRule: std::fmt::Debug + Send + Sync { - async fn install(&self, sender: &S) -> Result; - fn clone_box(&self) -> Box>; -} - -#[async_trait] -pub trait ScrapeTarget: std::fmt::Debug + Send + Sync { - async fn install(&self, sender: &S) -> Result; - fn clone_box(&self) -> Box>; -} diff --git a/harmony/src/modules/application/features/monitoring.rs b/harmony/src/modules/application/features/monitoring.rs index 6f5bb214..3ef5d5ad 100644 --- a/harmony/src/modules/application/features/monitoring.rs +++ b/harmony/src/modules/application/features/monitoring.rs @@ -9,6 +9,7 @@ use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{ }; use crate::topology::MultiTargetTopology; use crate::topology::ingress::Ingress; +use crate::topology::oberservability::monitoring::Monitor; use crate::{ inventory::Inventory, modules::monitoring::{ @@ -18,7 +19,6 @@ use crate::{ topology::{HelmCommand, K8sclient, Topology, tenant::TenantManager}, }; use crate::{ - modules::prometheus::prometheus::PrometheusMonitoring, topology::oberservability::monitoring::AlertReceiver, }; use async_trait::async_trait; @@ -46,7 +46,7 @@ impl< + TenantManager + K8sclient + MultiTargetTopology - + PrometheusMonitoring + + Monitor + Grafana + Ingress + std::fmt::Debug, diff --git a/harmony/src/modules/application/features/rhob_monitoring.rs b/harmony/src/modules/application/features/rhob_monitoring.rs index 60c7aca9..792603b7 100644 --- a/harmony/src/modules/application/features/rhob_monitoring.rs +++ b/harmony/src/modules/application/features/rhob_monitoring.rs @@ -8,6 +8,7 @@ use crate::modules::monitoring::application_monitoring::rhobs_application_monito use crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::RHOBObservability; use crate::topology::MultiTargetTopology; use crate::topology::ingress::Ingress; +use crate::topology::oberservability::monitoring::Monitor; use crate::{ inventory::Inventory, modules::monitoring::{ @@ -17,7 +18,6 @@ use crate::{ topology::{HelmCommand, K8sclient, Topology, tenant::TenantManager}, }; use crate::{ - modules::prometheus::prometheus::PrometheusMonitoring, topology::oberservability::monitoring::AlertReceiver, }; use async_trait::async_trait; @@ -41,7 +41,7 @@ impl< + MultiTargetTopology + Ingress + std::fmt::Debug - + PrometheusMonitoring, + + Monitor, > ApplicationFeature for Monitoring { async fn ensure_installed( diff --git a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs index 8f6b6245..ef554bc9 100644 --- a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs +++ b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs @@ -10,7 +10,7 @@ use crate::{ monitoring::{ grafana::grafana::Grafana, kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus, }, - prometheus::prometheus::PrometheusMonitoring, + prometheus::prometheus::Monitor, }, score::Score, topology::{ @@ -26,7 +26,7 @@ pub struct ApplicationMonitoringScore { pub receivers: Vec>>, } -impl + K8sclient + Grafana> Score +impl + K8sclient + Grafana> Score for ApplicationMonitoringScore { fn create_interpret(&self) -> Box> { diff --git a/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs index 6f45c883..83bed109 100644 --- a/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs +++ b/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs @@ -12,7 +12,7 @@ use crate::{ monitoring::kube_prometheus::crd::{ crd_alertmanager_config::CRDPrometheus, rhob_alertmanager_config::RHOBObservability, }, - prometheus::prometheus::PrometheusMonitoring, + prometheus::prometheus::Monitor, }, score::Score, topology::{PreparationOutcome, Topology, oberservability::monitoring::AlertReceiver}, @@ -26,7 +26,7 @@ pub struct ApplicationRHOBMonitoringScore { pub receivers: Vec>>, } -impl> Score +impl> Score for ApplicationRHOBMonitoringScore { fn create_interpret(&self) -> Box> { @@ -49,7 +49,7 @@ pub struct ApplicationRHOBMonitoringInterpret { } #[async_trait] -impl> Interpret +impl> Interpret for ApplicationRHOBMonitoringInterpret { async fn execute( @@ -58,7 +58,7 @@ impl> Interpret topology: &T, ) -> Result { let result = topology - .install_prometheus( + .install_montoring( &self.score.sender, inventory, Some(self.score.receivers.clone()), diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs index 88ec7454..6894d615 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs @@ -12,7 +12,7 @@ use crate::{ monitoring::{ grafana::grafana::Grafana, kube_prometheus::crd::service_monitor::ServiceMonitor, }, - prometheus::prometheus::PrometheusMonitoring, + prometheus::prometheus::Monitor, }, topology::{ K8sclient, Topology, @@ -70,12 +70,12 @@ impl Serialize for Box> { } #[async_trait] -impl + Grafana> Installable +impl + Grafana> Installable for CRDPrometheus { async fn configure(&self, inventory: &Inventory, topology: &T) -> Result<(), InterpretError> { topology.ensure_grafana_operator(inventory).await?; - topology.ensure_prometheus_operator(self, inventory).await?; + topology.ensure_monitoring(self, inventory).await?; Ok(()) } @@ -85,7 +85,7 @@ impl + Grafana> In topology: &T, ) -> Result<(), InterpretError> { topology.install_grafana().await?; - topology.install_prometheus(&self, inventory, None).await?; + topology.install_montoring(&self, inventory, None).await?; Ok(()) } } diff --git a/harmony/src/modules/monitoring/okd/installable.rs b/harmony/src/modules/monitoring/okd/installable.rs new file mode 100644 index 00000000..5b31a640 --- /dev/null +++ b/harmony/src/modules/monitoring/okd/installable.rs @@ -0,0 +1,40 @@ +use async_trait::async_trait; +use log::info; + +use crate::{ + interpret::InterpretError, inventory::Inventory, modules::monitoring::okd::{OpenshiftClusterAlertSender, enable_user_workload::OpenshiftUserWorkloadMonitoring, score_cluster_monitoring::OpenshiftClusterMonitoringScore, score_verify_user_workload_monitoring::VerifyUserWorkload}, score::Score, topology::{K8sclient, Topology, installable::Installable} +}; + +#[async_trait] +impl Installable for OpenshiftClusterAlertSender { + async fn configure(&self, inventory: &Inventory, topology: &T) -> Result<(), InterpretError> { + info!("enabling cluster monitoring"); + let cluster_monitoring_score = OpenshiftClusterMonitoringScore {}; + cluster_monitoring_score + .create_interpret() + .execute(inventory, topology) + .await?; + + info!("enabling user workload monitoring"); + let user_workload_score = OpenshiftUserWorkloadMonitoring {}; + user_workload_score + .create_interpret() + .execute(inventory, topology) + .await?; + Ok(()) + } + + async fn ensure_installed( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result<(), InterpretError> { + let verify_monitoring_score = VerifyUserWorkload {}; + info!("Verifying user workload and cluster monitoring installed"); + verify_monitoring_score + .create_interpret() + .execute(inventory, topology) + .await?; + Ok(()) + } +} diff --git a/harmony/src/modules/monitoring/okd/mod.rs b/harmony/src/modules/monitoring/okd/mod.rs index ac246c5f..067394bf 100644 --- a/harmony/src/modules/monitoring/okd/mod.rs +++ b/harmony/src/modules/monitoring/okd/mod.rs @@ -3,6 +3,11 @@ use crate::topology::oberservability::monitoring::AlertSender; pub mod cluster_monitoring; pub(crate) mod config; pub mod enable_user_workload; +pub mod installable; +pub mod score_cluster_monitoring; +pub mod score_user_workload; +pub mod score_verify_user_workload_monitoring; +pub mod score_openshift_receiver; #[derive(Debug)] pub struct OpenshiftClusterAlertSender; diff --git a/harmony/src/modules/monitoring/okd/old.md b/harmony/src/modules/monitoring/okd/old.md new file mode 100644 index 00000000..ef36f118 --- /dev/null +++ b/harmony/src/modules/monitoring/okd/old.md @@ -0,0 +1,31 @@ + +#[derive(Debug)] +pub struct OpenshiftClusterAlertInterpret { + receivers: Vec>>, +} + +#[async_trait] +impl Interpret for OpenshiftClusterAlertInterpret { + async fn execute( + &self, + _inventory: &Inventory, + topology: &T, + ) -> Result { + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("OpenshiftClusterAlertInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/okd/score_cluster_monitoring.rs b/harmony/src/modules/monitoring/okd/score_cluster_monitoring.rs new file mode 100644 index 00000000..66c09458 --- /dev/null +++ b/harmony/src/modules/monitoring/okd/score_cluster_monitoring.rs @@ -0,0 +1,46 @@ +use std::collections::BTreeMap; + +use k8s_openapi::api::core::v1::ConfigMap; +use kube::api::ObjectMeta; +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::k8s::resource::K8sResourceScore, + score::Score, + topology::{K8sclient, Topology}, +}; + +#[derive(Clone, Debug, Serialize)] +pub struct OpenshiftClusterMonitoringScore {} + +impl Score for OpenshiftClusterMonitoringScore { + fn name(&self) -> String { + "OpenshiftClusterMonitoringScore".to_string() + } + + fn create_interpret(&self) -> Box> { + let namespace = "openshift-monitoring".to_string(); + let mut data = BTreeMap::new(); + data.insert( + "config.yaml".to_string(), + r#" +enableUserWorkload: true +alertmanagerMain: + enableUserAlertmanagerConfig: true +"# + .to_string(), + ); + + let cm = ConfigMap { + metadata: ObjectMeta { + name: Some("cluster-monitoring-config".to_string()), + namespace: Some(namespace.clone()), + ..Default::default() + }, + data: Some(data), + ..Default::default() + }; + K8sResourceScore::single(cm, Some(namespace)).create_interpret() + } +} diff --git a/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs b/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs new file mode 100644 index 00000000..f182fb92 --- /dev/null +++ b/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs @@ -0,0 +1,149 @@ +use async_trait::async_trait; +use base64::prelude::BASE64_STANDARD; +use harmony_types::id::Id; +use kube::api::DynamicObject; +use log::{debug, info, trace}; +use serde::Serialize; + +use crate::{ + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + modules::monitoring::okd::OpenshiftClusterAlertSender, + score::Score, + topology::{K8sclient, Topology, oberservability::monitoring::AlertReceiver}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct OpenshiftReceiverScore { + pub receiver: Box>, +} + +impl Score for OpenshiftReceiverScore { + fn name(&self) -> String { + "OpenshiftAlertReceiverScore".to_string() + } + + fn create_interpret(&self) -> Box> { + Box::new(OpenshiftReceiverInterpret { + receiver: self.receiver, + }) + } +} + +#[derive(Debug)] +pub struct OpenshiftReceiverInterpret { + receiver: Box>, +} + +#[async_trait] +impl Interpret for OpenshiftReceiverInterpret { + async fn execute( + &self, + _inventory: &Inventory, + topology: &T, + ) -> Result { + let client = topology.k8s_client().await?; + let ns = "openshift-monitoring"; + + // 1️⃣ Get the alertmanager-main secret + let mut am_secret: DynamicObject = client + .get_secret_json_value("alertmanager-main", Some(ns)) + .await?; + let data = am_secret + .data + .get_mut("data") + .ok_or_else(|| InterpretError::new("Missing 'data' field in alertmanager-main secret".into()))? + .as_object_mut() + .ok_or_else(|| InterpretError::new("'data' field must be a JSON object".into()))?; + + // 2️⃣ Decode alertmanager.yaml + let config_b64 = data + .get("alertmanager.yaml") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + let config_bytes = BASE64_STANDARD.decode(config_b64).unwrap_or_default(); + let mut am_config: serde_yaml::Value = + serde_yaml::from_slice(&config_bytes).unwrap_or_else(|_| serde_yaml::Value::Mapping(serde_yaml::Mapping::new())); + + // Ensure "receivers" exists + let receivers_seq = am_config + .get_mut("receivers") + .and_then(|r| r.as_sequence_mut()) + .unwrap_or_else(|| { + am_config["receivers"] = serde_yaml::Value::Sequence(vec![]); + am_config["receivers"].as_sequence_mut().unwrap() + }); + + // Ensure "route/routes" exists + let route_seq = am_config + .pointer_mut("/route/routes") + .and_then(|r| r.as_sequence_mut()) + .unwrap_or_else(|| { + am_config["route"] = serde_yaml::Value::Mapping(serde_yaml::Mapping::new()); + am_config["route"]["routes"] = serde_yaml::Value::Sequence(vec![]); + am_config.pointer_mut("/route/routes").unwrap().as_sequence_mut().unwrap() + }); + + // 3️⃣ Add/update the single receiver + let name = self.receiver.name(); + let am_receiver = self.receiver.as_alertmanager_receiver()?; + + // Receiver + let receiver_yaml: serde_yaml::Value = + serde_yaml::from_str(&serde_json::to_string(&am_receiver.receiver_config)?) + .map_err(|e| InterpretError::new(format!("Failed to convert receiver config to YAML: {e}")))?; + + if let Some(idx) = receivers_seq.iter().position(|r| r.get("name").and_then(|n| n.as_str()) == Some(&name)) { + receivers_seq[idx] = receiver_yaml; + } else { + receivers_seq.push(receiver_yaml); + } + + // Route + let route_yaml: serde_yaml::Value = + serde_yaml::from_str(&serde_json::to_string(&am_receiver.route_config)?) + .map_err(|e| InterpretError::new(format!("Failed to convert route config to YAML: {e}")))?; + + if let Some(idx) = route_seq.iter().position(|r| r.get("receiver").and_then(|n| n.as_str()) == Some(name)) { + route_seq[idx] = route_yaml; + } else { + route_seq.push(route_yaml); + } + + // 4️⃣ Apply additional resources + client + .apply_dynamic_many(&am_receiver.additional_ressources, Some(ns), true) + .await?; + + // 5️⃣ Save updated alertmanager.yaml + let yaml_str = serde_yaml::to_string(&am_config)?; + let mut yaml_b64 = String::new(); + BASE64_STANDARD.encode_string(yaml_str, &mut yaml_b64); + data.insert("alertmanager.yaml".to_string(), serde_json::Value::String(yaml_b64)); + am_secret.metadata.managed_fields = None; + + client.apply_dynamic(&am_secret, Some(ns), true).await?; + + Ok(Outcome::success(format!( + "Configured OpenShift cluster alert receiver: {}", + name + ))) + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("OpenshiftAlertReceiverInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/okd/score_user_workload.rs b/harmony/src/modules/monitoring/okd/score_user_workload.rs new file mode 100644 index 00000000..fdea01ff --- /dev/null +++ b/harmony/src/modules/monitoring/okd/score_user_workload.rs @@ -0,0 +1,45 @@ +use std::collections::BTreeMap; + +use crate::{ + interpret::Interpret, + modules::k8s::resource::K8sResourceScore, + score::Score, + topology::{K8sclient, Topology}, +}; +use k8s_openapi::api::core::v1::ConfigMap; +use kube::api::ObjectMeta; +use serde::Serialize; + +#[derive(Clone, Debug, Serialize)] +pub struct OpenshiftUserWorkloadMonitoring {} + +impl Score for OpenshiftUserWorkloadMonitoring { + fn name(&self) -> String { + "OpenshiftUserWorkloadMonitoringScore".to_string() + } + + fn create_interpret(&self) -> Box> { + let namespace = "openshift-user-workload-monitoring".to_string(); + let mut data = BTreeMap::new(); + data.insert( + "config.yaml".to_string(), + r#" +alertmanager: + enabled: true + enableAlertmanagerConfig: true +"# + .to_string(), + ); + let cm = ConfigMap { + metadata: ObjectMeta { + name: Some("user-workload-monitoring-config".to_string()), + namespace: Some(namespace.clone()), + ..Default::default() + }, + data: Some(data), + ..Default::default() + }; + + K8sResourceScore::single(cm, Some(namespace)).create_interpret() + } +} diff --git a/harmony/src/modules/monitoring/okd/score_verify_user_workload_monitoring.rs b/harmony/src/modules/monitoring/okd/score_verify_user_workload_monitoring.rs new file mode 100644 index 00000000..1d41d358 --- /dev/null +++ b/harmony/src/modules/monitoring/okd/score_verify_user_workload_monitoring.rs @@ -0,0 +1,68 @@ +use async_trait::async_trait; +use harmony_types::id::Id; +use serde::Serialize; + +use crate::{ + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + score::Score, + topology::{K8sclient, Topology}, +}; + +#[derive(Clone, Debug, Serialize)] +pub struct VerifyUserWorkload {} + +impl Score for VerifyUserWorkload { + fn name(&self) -> String { + "VerifyUserWorkload".to_string() + } + + fn create_interpret(&self) -> Box> { + Box::new(VerifyUserWorkloadInterpret {}) + } +} + +#[derive(Clone, Debug, Serialize)] +pub struct VerifyUserWorkloadInterpret {} + +#[async_trait] +impl Interpret for VerifyUserWorkloadInterpret { + async fn execute( + &self, + _inventory: &Inventory, + topology: &T, + ) -> Result { + let client = topology.k8s_client().await?; + let namespace = "openshift-user-workload-monitoring"; + let alertmanager_name = "alertmanager-user-workload-0"; + let prometheus_name = "prometheus-user-workload-0"; + client + .wait_for_pod_ready(alertmanager_name, Some(namespace)) + .await?; + client + .wait_for_pod_ready(prometheus_name, Some(namespace)) + .await?; + + Ok(Outcome::success(format!( + "pods: {}, {} ready in ns: {}", + alertmanager_name, prometheus_name, namespace + ))) + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("VerifyUserWorkloadInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} diff --git a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs index 7093ee8b..7796d255 100644 --- a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs +++ b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs @@ -40,7 +40,7 @@ use crate::{ }; use harmony_types::id::Id; -use super::prometheus::PrometheusMonitoring; +use super::prometheus::Monitor; #[derive(Clone, Debug, Serialize)] pub struct K8sPrometheusCRDAlertingScore { @@ -50,7 +50,7 @@ pub struct K8sPrometheusCRDAlertingScore { pub prometheus_rules: Vec, } -impl> Score +impl> Score for K8sPrometheusCRDAlertingScore { fn create_interpret(&self) -> Box> { @@ -76,7 +76,7 @@ pub struct K8sPrometheusCRDAlertingInterpret { } #[async_trait] -impl> Interpret +impl> Interpret for K8sPrometheusCRDAlertingInterpret { async fn execute( diff --git a/harmony/src/modules/prometheus/prometheus.rs b/harmony/src/modules/prometheus/prometheus.rs index efb89da8..2e2d0b94 100644 --- a/harmony/src/modules/prometheus/prometheus.rs +++ b/harmony/src/modules/prometheus/prometheus.rs @@ -1,25 +1,25 @@ -use async_trait::async_trait; - -use crate::{ - inventory::Inventory, - topology::{ - PreparationError, PreparationOutcome, - oberservability::monitoring::{AlertReceiver, AlertSender}, - }, -}; - -#[async_trait] -pub trait PrometheusMonitoring { - async fn install_prometheus( - &self, - sender: &S, - inventory: &Inventory, - receivers: Option>>>, - ) -> Result; - - async fn ensure_prometheus_operator( - &self, - sender: &S, - inventory: &Inventory, - ) -> Result; -} +// use async_trait::async_trait; +// +// use crate::{ +// inventory::Inventory, +// topology::{ +// PreparationError, PreparationOutcome, +// oberservability::monitoring::{AlertReceiver, AlertSender}, +// }, +// }; +// +// #[async_trait] +// pub trait Monitor { +// async fn install_montoring( +// &self, +// sender: &S, +// inventory: &Inventory, +// receivers: Option>>>, +// ) -> Result; +// +// async fn ensure_monitoring( +// &self, +// sender: &S, +// inventory: &Inventory, +// ) -> Result; +// } diff --git a/harmony/src/modules/prometheus/rhob_alerting_score.rs b/harmony/src/modules/prometheus/rhob_alerting_score.rs index 644e6f95..f44bf8cd 100644 --- a/harmony/src/modules/prometheus/rhob_alerting_score.rs +++ b/harmony/src/modules/prometheus/rhob_alerting_score.rs @@ -38,7 +38,7 @@ use crate::{ }; use harmony_types::id::Id; -use super::prometheus::PrometheusMonitoring; +use super::prometheus::Monitor; #[derive(Clone, Debug, Serialize)] pub struct RHOBAlertingScore { @@ -48,7 +48,7 @@ pub struct RHOBAlertingScore { pub prometheus_rules: Vec, } -impl> Score +impl> Score for RHOBAlertingScore { fn create_interpret(&self) -> Box> { @@ -74,7 +74,7 @@ pub struct RHOBAlertingInterpret { } #[async_trait] -impl> Interpret +impl> Interpret for RHOBAlertingInterpret { async fn execute( -- 2.39.5 From bc6a41d40cca48965433d18ead1848f8c6379fc7 Mon Sep 17 00:00:00 2001 From: wjro Date: Fri, 20 Feb 2026 12:49:55 -0500 Subject: [PATCH 02/15] wip: removed use of installable trait, added all installation and ensure ready functions to the trait monitor, first impl of AlertReceiver for OpenshiftClusterAlertSender --- .../k8s_anywhere/openshift_monitoring.rs | 71 +- .../topology/oberservability/monitoring.rs | 70 +- .../alert_channel/discord_alert_channel.rs | 665 +++++++++--------- .../alert_channel/webhook_receiver.rs | 8 +- .../monitoring/okd/cluster_monitoring.rs | 270 ------- harmony/src/modules/monitoring/okd/config.rs | 90 --- .../monitoring/okd/enable_user_workload.rs | 60 -- .../src/modules/monitoring/okd/installable.rs | 40 -- harmony/src/modules/monitoring/okd/mod.rs | 6 +- harmony/src/modules/monitoring/okd/old.md | 31 - .../okd/score_openshift_alert_rule.rs | 16 + .../okd/score_openshift_receiver.rs | 52 +- .../okd/score_openshift_scrape_target.rs | 16 + .../k8s_prometheus_alerting_score.rs | 2 +- .../modules/prometheus/rhob_alerting_score.rs | 2 +- 15 files changed, 540 insertions(+), 859 deletions(-) delete mode 100644 harmony/src/modules/monitoring/okd/cluster_monitoring.rs delete mode 100644 harmony/src/modules/monitoring/okd/config.rs delete mode 100644 harmony/src/modules/monitoring/okd/enable_user_workload.rs delete mode 100644 harmony/src/modules/monitoring/okd/installable.rs delete mode 100644 harmony/src/modules/monitoring/okd/old.md create mode 100644 harmony/src/modules/monitoring/okd/score_openshift_alert_rule.rs create mode 100644 harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs diff --git a/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs b/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs index 6695ffb4..4cc31320 100644 --- a/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs +++ b/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs @@ -1,9 +1,10 @@ use async_trait::async_trait; +use log::info; use crate::{ inventory::Inventory, modules::monitoring::okd::{ - OpenshiftClusterAlertSender, score_openshift_receiver::OpenshiftReceiverScore, + OpenshiftClusterAlertSender, score_cluster_monitoring::OpenshiftClusterMonitoringScore, score_openshift_alert_rule::OpenshiftAlertRuleScore, score_openshift_receiver::OpenshiftReceiverScore, score_openshift_scrape_target::OpenshiftScrapeTargetScore, score_user_workload::OpenshiftUserWorkloadMonitoring, score_verify_user_workload_monitoring::VerifyUserWorkload }, topology::{ K8sAnywhereTopology, PreparationError, PreparationOutcome, @@ -13,6 +14,30 @@ use crate::{ #[async_trait] impl Monitor for K8sAnywhereTopology { + async fn install_alert_sender( + &self, + sender: &OpenshiftClusterAlertSender, + inventory: &Inventory, + ) -> Result { + info!("enabling cluster monitoring"); + let cluster_monitoring_score = OpenshiftClusterMonitoringScore {}; + cluster_monitoring_score + .create_interpret() + .execute(inventory, self) + .await?; + + info!("enabling user workload monitoring"); + let user_workload_score = OpenshiftUserWorkloadMonitoring {}; + user_workload_score + .create_interpret() + .execute(inventory, self) + .await?; + + Ok(PreparationOutcome::Success { + details: "Successfully configured cluster monitoring".to_string(), + }) + } + async fn install_receivers( &self, sender: &OpenshiftClusterAlertSender, @@ -21,16 +46,16 @@ impl Monitor for K8sAnywhereTopology { ) -> Result { if let Some(receivers) = receivers { for receiver in receivers { - let receiver_score = OpenshiftReceiverScore { - receiver}; + info!("Installing receiver {}", receiver.name()); + let receiver_score = OpenshiftReceiverScore { receiver }; receiver_score .create_interpret() .execute(inventory, self) - .await - .map_err(|e| PreparationError::new(e))?; + .await?; } Ok(PreparationOutcome::Success { - details: "Success".to_string(), + details: "Successfully installed receivers for OpenshiftClusterMonitoring" + .to_string(), }) } else { Ok(PreparationOutcome::Noop) @@ -44,7 +69,17 @@ impl Monitor for K8sAnywhereTopology { rules: Option>>>, ) -> Result { if let Some(rules) = rules { - todo!() + for rule in rules { + info!("Installing rule "); + let rule_score = OpenshiftAlertRuleScore {}; + rule_score + .create_interpret() + .execute(inventory, self) + .await?; + } + Ok(PreparationOutcome::Success { + details: "Successfully installed rules for OpenshiftClusterMonitoring".to_string(), + }) } else { Ok(PreparationOutcome::Noop) } @@ -57,17 +92,33 @@ impl Monitor for K8sAnywhereTopology { scrape_targets: Option>>>, ) -> Result { if let Some(scrape_targets) = scrape_targets { - todo!() + for scrape_target in scrape_targets { + !info!("Installing scrape target"); + let scrape_target_score = OpenshiftScrapeTargetScore {}; + scrape_target_score + .create_interpret() + .execute(inventory, self) + .await?; + } + Ok(PreparationOutcome::Success { + details: "Successfully added scrape targets for OpenshiftClusterMonitoring" + .to_string(), + }) } else { Ok(PreparationOutcome::Noop) } } - async fn ensure_monitoring( + async fn ensure_monitoring_installed( &self, sender: &OpenshiftClusterAlertSender, inventory: &Inventory, ) -> Result { - todo!() + let verify_monitoring_score = VerifyUserWorkload {}; + info!("Verifying user workload and cluster monitoring installed"); + verify_monitoring_score + .create_interpret() + .execute(inventory, self) + .await? } } diff --git a/harmony/src/domain/topology/oberservability/monitoring.rs b/harmony/src/domain/topology/oberservability/monitoring.rs index f5d81d4b..c9acc77e 100644 --- a/harmony/src/domain/topology/oberservability/monitoring.rs +++ b/harmony/src/domain/topology/oberservability/monitoring.rs @@ -20,13 +20,49 @@ pub trait AlertSender: Send + Sync + std::fmt::Debug { } /// Defines the entity that receives the alerts from a sender. For example Discord, Slack, etc -#[async_trait] +/// pub trait AlertReceiver: std::fmt::Debug + Send + Sync { - async fn install(&self, sender: &S) -> Result; + fn build_route(&self) -> Result; + fn build_receiver(&self) -> Result; fn name(&self) -> String; fn clone_box(&self) -> Box>; - fn as_any(&self) -> &dyn Any; - fn as_alertmanager_receiver(&self) -> Result; +} + +pub struct ReceiverEndpoint {} + +///Generic routing that can map to various alert sender backends +pub struct AlertRoute { + pub receiver: String, + pub matchers: Vec, + pub group_by: Vec, + pub repeat_interval: Option, + pub continue_matching: bool, + pub children: Vec, +} + +impl AlertRoute { + fn default(name: String) -> Self { + Self { + receiver: name, + matchers: vec![], + group_by: vec![], + repeat_interval: Some(30), + continue_matching: true, + children: vec![], + } + } +} + +pub struct AlertMatcher { + pub label: String, + pub operator: MatchOp, + pub value: String, +} + +pub enum MatchOp { + Eq, + NotEq, + Regex, } #[derive(Debug)] @@ -55,6 +91,12 @@ pub trait ScrapeTarget: std::fmt::Debug + Send + Sync { /// Trait which defines how an alert sender is impleneted for a specific topology #[async_trait] pub trait Monitor { + async fn install_alert_sender( + &self, + sender: &S, + inventory: &Inventory, + ) -> Result; + async fn install_receivers( &self, sender: &S, @@ -76,7 +118,7 @@ pub trait Monitor { scrape_targets: Option>>>, ) -> Result; - async fn ensure_monitoring( + async fn ensure_monitoring_installed( &self, sender: &S, inventory: &Inventory, @@ -93,16 +135,16 @@ pub struct AlertingInterpret { } #[async_trait] -impl, T: Topology + Monitor> Interpret - for AlertingInterpret -{ +impl> Interpret for AlertingInterpret { async fn execute( &self, inventory: &Inventory, topology: &T, ) -> Result { info!("Configuring alert sender {}", self.sender.name()); - self.sender.configure(inventory, topology).await?; + topology + .install_alert_sender(&self.sender, inventory) + .await?; info!("Installing receivers"); topology @@ -115,11 +157,15 @@ impl, T: Topology + Monitor> Interpret .await?; info!("Adding extra scrape targets"); - topology.add_scrape_targets(&self.sender, inventory, self.scrape_targets).await?; + topology + .add_scrape_targets(&self.sender, inventory, self.scrape_targets) + .await?; info!("Ensuring alert sender {} is ready", self.sender.name()); - self.sender.ensure_installed(inventory, topology).await?; - + topology + .ensure_monitoring_installed(&self.sender, inventory) + .await?; + Ok(Outcome::success(format!( "successfully installed alert sender {}", self.sender.name() diff --git a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs index 94627896..d5fdc8da 100644 --- a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs +++ b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs @@ -17,7 +17,9 @@ use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{ }; use crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::RHOBObservability; use crate::modules::monitoring::okd::OpenshiftClusterAlertSender; -use crate::topology::oberservability::monitoring::AlertManagerReceiver; +use crate::topology::oberservability::monitoring::{ + AlertManagerReceiver, AlertRoute, MatchOp, ReceiverEndpoint +}; use crate::{ interpret::{InterpretError, Outcome}, modules::monitoring::{ @@ -32,10 +34,17 @@ use crate::{ use harmony_types::net::Url; #[derive(Debug, Clone, Serialize)] -pub struct DiscordWebhook { - pub name: K8sName, +pub struct DiscordReceiver { + pub name: String, pub url: Url, - pub selectors: Vec>, + pub route: AlertRoute, +} + +#[derive(Debug, Clone, Serialize)] +pub struct DiscordWebhook { + pub name: String, + pub url: Url, + pub route: Vec>, } impl DiscordWebhook { @@ -86,13 +95,41 @@ impl DiscordWebhook { } } -#[async_trait] -impl AlertReceiver for DiscordWebhook { - async fn install( +impl AlertReceiver for DiscordReceiver { + fn build_route( &self, - sender: &OpenshiftClusterAlertSender, - ) -> Result { - todo!() + ) -> Result { + let matchers: Vec = self + .route + .matchers + .iter() + .map(|m| match m.operator { + MatchOp::Eq => format!("{} = {}", m.label, m.value), + MatchOp::NotEq => format!("{} != {}", m.label, m.value), + MatchOp::Regex => format!("{} =~ {}", m.label, m.value), + }) + .collect(); + + let route_block = serde_yaml::to_value(json!({ + "receiver": self.name, + "matchers": matchers, + }))?; + Ok(route_block) + } + + fn build_receiver( + &self, + ) -> Result { + + let receiver_block = serde_yaml::to_value(json!({ + "name": self.name, + "discord_configs": [{ + "webhook_url": format!("{{{{ webhook-url \"{}\" }}}}", self.url), + "title": "{{ template \"discord.default.title\" . }}", + "message": "{{ template \"discord.default.message\" . }}" + }] + }))?; + Ok(receiver_block) } fn name(&self) -> String { @@ -103,308 +140,308 @@ impl AlertReceiver for DiscordWebhook { Box::new(self.clone()) } - fn as_any(&self) -> &dyn Any { - todo!() - } - - fn as_alertmanager_receiver(&self) -> Result { - self.get_receiver_config() - } -} - -#[async_trait] -impl AlertReceiver for DiscordWebhook { - fn as_alertmanager_receiver(&self) -> Result { - todo!() - } - - async fn install(&self, sender: &RHOBObservability) -> Result { - let ns = sender.namespace.clone(); - - let config = self.get_receiver_config()?; - for resource in config.additional_ressources.iter() { - todo!("can I apply a dynamicresource"); - // sender.client.apply(resource, Some(&ns)).await; - } - - let spec = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfigSpec { - data: json!({ - "route": { - "receiver": self.name, - }, - "receivers": [ - config.receiver_config - ] - }), - }; - - let alertmanager_configs = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfig { - metadata: ObjectMeta { - name: Some(self.name.clone().to_string()), - labels: Some(std::collections::BTreeMap::from([( - "alertmanagerConfig".to_string(), - "enabled".to_string(), - )])), - namespace: Some(sender.namespace.clone()), - ..Default::default() - }, - spec, - }; - debug!( - "alertmanager_configs yaml:\n{:#?}", - serde_yaml::to_string(&alertmanager_configs) - ); - debug!( - "alert manager configs: \n{:#?}", - alertmanager_configs.clone() - ); - - sender - .client - .apply(&alertmanager_configs, Some(&sender.namespace)) - .await?; - Ok(Outcome::success(format!( - "installed rhob-alertmanagerconfigs for {}", - self.name - ))) - } - - fn name(&self) -> String { - "webhook-receiver".to_string() - } - - fn clone_box(&self) -> Box> { - Box::new(self.clone()) - } - - fn as_any(&self) -> &dyn Any { - self - } -} - -#[async_trait] -impl AlertReceiver for DiscordWebhook { - fn as_alertmanager_receiver(&self) -> Result { - todo!() - } - async fn install(&self, sender: &CRDPrometheus) -> Result { - let ns = sender.namespace.clone(); - let secret_name = format!("{}-secret", self.name.clone()); - let webhook_key = format!("{}", self.url.clone()); - - let mut string_data = BTreeMap::new(); - string_data.insert("webhook-url".to_string(), webhook_key.clone()); - - let secret = Secret { - metadata: kube::core::ObjectMeta { - name: Some(secret_name.clone()), - ..Default::default() - }, - string_data: Some(string_data), - type_: Some("Opaque".to_string()), - ..Default::default() - }; - - let _ = sender.client.apply(&secret, Some(&ns)).await; - - let spec = AlertmanagerConfigSpec { - data: json!({ - "route": { - "receiver": self.name, - }, - "receivers": [ - { - "name": self.name, - "discordConfigs": [ - { - "apiURL": { - "name": secret_name, - "key": "webhook-url", - }, - "title": "{{ template \"discord.default.title\" . }}", - "message": "{{ template \"discord.default.message\" . }}" - } - ] - } - ] - }), - }; - - let alertmanager_configs = AlertmanagerConfig { - metadata: ObjectMeta { - name: Some(self.name.clone().to_string()), - labels: Some(std::collections::BTreeMap::from([( - "alertmanagerConfig".to_string(), - "enabled".to_string(), - )])), - namespace: Some(ns), - ..Default::default() - }, - spec, - }; - - sender - .client - .apply(&alertmanager_configs, Some(&sender.namespace)) - .await?; - Ok(Outcome::success(format!( - "installed crd-alertmanagerconfigs for {}", - self.name - ))) - } - fn name(&self) -> String { - "discord-webhook".to_string() - } - fn clone_box(&self) -> Box> { - Box::new(self.clone()) - } - fn as_any(&self) -> &dyn Any { - self - } -} - -#[async_trait] -impl AlertReceiver for DiscordWebhook { - fn as_alertmanager_receiver(&self) -> Result { - todo!() - } - async fn install(&self, sender: &Prometheus) -> Result { - sender.install_receiver(self).await - } - fn name(&self) -> String { - "discord-webhook".to_string() - } - fn clone_box(&self) -> Box> { - Box::new(self.clone()) - } - fn as_any(&self) -> &dyn Any { - self - } -} - -#[async_trait] -impl PrometheusReceiver for DiscordWebhook { - fn name(&self) -> String { - self.name.clone().to_string() - } - async fn configure_receiver(&self) -> AlertManagerChannelConfig { - self.get_config().await - } -} - -#[async_trait] -impl AlertReceiver for DiscordWebhook { - fn as_alertmanager_receiver(&self) -> Result { - todo!() - } - async fn install(&self, sender: &KubePrometheus) -> Result { - sender.install_receiver(self).await - } - fn clone_box(&self) -> Box> { - Box::new(self.clone()) - } - fn name(&self) -> String { - "discord-webhook".to_string() - } - fn as_any(&self) -> &dyn Any { - self - } -} - -#[async_trait] -impl KubePrometheusReceiver for DiscordWebhook { - fn name(&self) -> String { - self.name.clone().to_string() - } - async fn configure_receiver(&self) -> AlertManagerChannelConfig { - self.get_config().await - } -} - -#[async_trait] -impl AlertChannelConfig for DiscordWebhook { - async fn get_config(&self) -> AlertManagerChannelConfig { - let channel_global_config = None; - let channel_receiver = self.alert_channel_receiver().await; - let channel_route = self.alert_channel_route().await; - - AlertManagerChannelConfig { - channel_global_config, - channel_receiver, - channel_route, - } - } -} - -impl DiscordWebhook { - async fn alert_channel_route(&self) -> serde_yaml::Value { - let mut route = Mapping::new(); - route.insert( - Value::String("receiver".to_string()), - Value::String(self.name.clone().to_string()), - ); - route.insert( - Value::String("matchers".to_string()), - Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]), - ); - route.insert(Value::String("continue".to_string()), Value::Bool(true)); - Value::Mapping(route) - } - - async fn alert_channel_receiver(&self) -> serde_yaml::Value { - let mut receiver = Mapping::new(); - receiver.insert( - Value::String("name".to_string()), - Value::String(self.name.clone().to_string()), - ); - - let mut discord_config = Mapping::new(); - discord_config.insert( - Value::String("webhook_url".to_string()), - Value::String(self.url.to_string()), - ); - - receiver.insert( - Value::String("discord_configs".to_string()), - Value::Sequence(vec![Value::Mapping(discord_config)]), - ); - - Value::Mapping(receiver) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn discord_serialize_should_match() { - let discord_receiver = DiscordWebhook { - name: K8sName("test-discord".to_string()), - url: Url::Url(url::Url::parse("https://discord.i.dont.exist.com").unwrap()), - selectors: vec![], - }; - - let discord_receiver_receiver = - serde_yaml::to_string(&discord_receiver.alert_channel_receiver().await).unwrap(); - println!("receiver \n{:#}", discord_receiver_receiver); - let discord_receiver_receiver_yaml = r#"name: test-discord -discord_configs: -- webhook_url: https://discord.i.dont.exist.com/ -"# - .to_string(); - - let discord_receiver_route = - serde_yaml::to_string(&discord_receiver.alert_channel_route().await).unwrap(); - println!("route \n{:#}", discord_receiver_route); - let discord_receiver_route_yaml = r#"receiver: test-discord -matchers: -- alertname!=Watchdog -continue: true -"# - .to_string(); - - assert_eq!(discord_receiver_receiver, discord_receiver_receiver_yaml); - assert_eq!(discord_receiver_route, discord_receiver_route_yaml); - } + // fn as_any(&self) -> &dyn Any { + // todo!() + // } + // + // fn as_alertmanager_receiver(&self) -> Result { + // self.get_receiver_config() + // } } +// +// #[async_trait] +// impl AlertReceiver for DiscordWebhook { +// fn as_alertmanager_receiver(&self) -> Result { +// todo!() +// } +// +// async fn install(&self, sender: &RHOBObservability) -> Result { +// let ns = sender.namespace.clone(); +// +// let config = self.get_receiver_config()?; +// for resource in config.additional_ressources.iter() { +// todo!("can I apply a dynamicresource"); +// // sender.client.apply(resource, Some(&ns)).await; +// } +// +// let spec = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfigSpec { +// data: json!({ +// "route": { +// "receiver": self.name, +// }, +// "receivers": [ +// config.receiver_config +// ] +// }), +// }; +// +// let alertmanager_configs = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfig { +// metadata: ObjectMeta { +// name: Some(self.name.clone().to_string()), +// labels: Some(std::collections::BTreeMap::from([( +// "alertmanagerConfig".to_string(), +// "enabled".to_string(), +// )])), +// namespace: Some(sender.namespace.clone()), +// ..Default::default() +// }, +// spec, +// }; +// debug!( +// "alertmanager_configs yaml:\n{:#?}", +// serde_yaml::to_string(&alertmanager_configs) +// ); +// debug!( +// "alert manager configs: \n{:#?}", +// alertmanager_configs.clone() +// ); +// +// sender +// .client +// .apply(&alertmanager_configs, Some(&sender.namespace)) +// .await?; +// Ok(Outcome::success(format!( +// "installed rhob-alertmanagerconfigs for {}", +// self.name +// ))) +// } +// +// fn name(&self) -> String { +// "webhook-receiver".to_string() +// } +// +// fn clone_box(&self) -> Box> { +// Box::new(self.clone()) +// } +// +// fn as_any(&self) -> &dyn Any { +// self +// } +// } +// +// #[async_trait] +// impl AlertReceiver for DiscordWebhook { +// fn as_alertmanager_receiver(&self) -> Result { +// todo!() +// } +// async fn install(&self, sender: &CRDPrometheus) -> Result { +// let ns = sender.namespace.clone(); +// let secret_name = format!("{}-secret", self.name.clone()); +// let webhook_key = format!("{}", self.url.clone()); +// +// let mut string_data = BTreeMap::new(); +// string_data.insert("webhook-url".to_string(), webhook_key.clone()); +// +// let secret = Secret { +// metadata: kube::core::ObjectMeta { +// name: Some(secret_name.clone()), +// ..Default::default() +// }, +// string_data: Some(string_data), +// type_: Some("Opaque".to_string()), +// ..Default::default() +// }; +// +// let _ = sender.client.apply(&secret, Some(&ns)).await; +// +// let spec = AlertmanagerConfigSpec { +// data: json!({ +// "route": { +// "receiver": self.name, +// }, +// "receivers": [ +// { +// "name": self.name, +// "discordConfigs": [ +// { +// "apiURL": { +// "name": secret_name, +// "key": "webhook-url", +// }, +// "title": "{{ template \"discord.default.title\" . }}", +// "message": "{{ template \"discord.default.message\" . }}" +// } +// ] +// } +// ] +// }), +// }; +// +// let alertmanager_configs = AlertmanagerConfig { +// metadata: ObjectMeta { +// name: Some(self.name.clone().to_string()), +// labels: Some(std::collections::BTreeMap::from([( +// "alertmanagerConfig".to_string(), +// "enabled".to_string(), +// )])), +// namespace: Some(ns), +// ..Default::default() +// }, +// spec, +// }; +// +// sender +// .client +// .apply(&alertmanager_configs, Some(&sender.namespace)) +// .await?; +// Ok(Outcome::success(format!( +// "installed crd-alertmanagerconfigs for {}", +// self.name +// ))) +// } +// fn name(&self) -> String { +// "discord-webhook".to_string() +// } +// fn clone_box(&self) -> Box> { +// Box::new(self.clone()) +// } +// fn as_any(&self) -> &dyn Any { +// self +// } +// } +// +// #[async_trait] +// impl AlertReceiver for DiscordWebhook { +// fn as_alertmanager_receiver(&self) -> Result { +// todo!() +// } +// async fn install(&self, sender: &Prometheus) -> Result { +// sender.install_receiver(self).await +// } +// fn name(&self) -> String { +// "discord-webhook".to_string() +// } +// fn clone_box(&self) -> Box> { +// Box::new(self.clone()) +// } +// fn as_any(&self) -> &dyn Any { +// self +// } +// } +// +// #[async_trait] +// impl PrometheusReceiver for DiscordWebhook { +// fn name(&self) -> String { +// self.name.clone().to_string() +// } +// async fn configure_receiver(&self) -> AlertManagerChannelConfig { +// self.get_config().await +// } +// } +// +// #[async_trait] +// impl AlertReceiver for DiscordWebhook { +// fn as_alertmanager_receiver(&self) -> Result { +// todo!() +// } +// async fn install(&self, sender: &KubePrometheus) -> Result { +// sender.install_receiver(self).await +// } +// fn clone_box(&self) -> Box> { +// Box::new(self.clone()) +// } +// fn name(&self) -> String { +// "discord-webhook".to_string() +// } +// fn as_any(&self) -> &dyn Any { +// self +// } +// } +// +// #[async_trait] +// impl KubePrometheusReceiver for DiscordWebhook { +// fn name(&self) -> String { +// self.name.clone().to_string() +// } +// async fn configure_receiver(&self) -> AlertManagerChannelConfig { +// self.get_config().await +// } +// } +// +// #[async_trait] +// impl AlertChannelConfig for DiscordWebhook { +// async fn get_config(&self) -> AlertManagerChannelConfig { +// let channel_global_config = None; +// let channel_receiver = self.alert_channel_receiver().await; +// let channel_route = self.alert_channel_route().await; +// +// AlertManagerChannelConfig { +// channel_global_config, +// channel_receiver, +// channel_route, +// } +// } +// } +// +// impl DiscordWebhook { +// async fn alert_channel_route(&self) -> serde_yaml::Value { +// let mut route = Mapping::new(); +// route.insert( +// Value::String("receiver".to_string()), +// Value::String(self.name.clone().to_string()), +// ); +// route.insert( +// Value::String("matchers".to_string()), +// Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]), +// ); +// route.insert(Value::String("continue".to_string()), Value::Bool(true)); +// Value::Mapping(route) +// } +// +// async fn alert_channel_receiver(&self) -> serde_yaml::Value { +// let mut receiver = Mapping::new(); +// receiver.insert( +// Value::String("name".to_string()), +// Value::String(self.name.clone().to_string()), +// ); +// +// let mut discord_config = Mapping::new(); +// discord_config.insert( +// Value::String("webhook_url".to_string()), +// Value::String(self.url.to_string()), +// ); +// +// receiver.insert( +// Value::String("discord_configs".to_string()), +// Value::Sequence(vec![Value::Mapping(discord_config)]), +// ); +// +// Value::Mapping(receiver) +// } +// } +// +// #[cfg(test)] +// mod tests { +// use super::*; +// +// #[tokio::test] +// async fn discord_serialize_should_match() { +// let discord_receiver = DiscordWebhook { +// name: K8sName("test-discord".to_string()), +// url: Url::Url(url::Url::parse("https://discord.i.dont.exist.com").unwrap()), +// selectors: vec![], +// }; +// +// let discord_receiver_receiver = +// serde_yaml::to_string(&discord_receiver.alert_channel_receiver().await).unwrap(); +// println!("receiver \n{:#}", discord_receiver_receiver); +// let discord_receiver_receiver_yaml = r#"name: test-discord +// discord_configs: +// - webhook_url: https://discord.i.dont.exist.com/ +// "# +// .to_string(); +// +// let discord_receiver_route = +// serde_yaml::to_string(&discord_receiver.alert_channel_route().await).unwrap(); +// println!("route \n{:#}", discord_receiver_route); +// let discord_receiver_route_yaml = r#"receiver: test-discord +// matchers: +// - alertname!=Watchdog +// continue: true +// "# +// .to_string(); +// +// assert_eq!(discord_receiver_receiver, discord_receiver_receiver_yaml); +// assert_eq!(discord_receiver_route, discord_receiver_route_yaml); +// } +// } diff --git a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs index a141df02..9548bf89 100644 --- a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs +++ b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs @@ -34,7 +34,7 @@ impl AlertReceiver for WebhookReceiver { fn as_alertmanager_receiver(&self) -> Result { todo!() } - async fn install(&self, sender: &RHOBObservability) -> Result { + async fn build_route(&self, sender: &RHOBObservability) -> Result { let spec = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfigSpec { data: json!({ "route": { @@ -103,7 +103,7 @@ impl AlertReceiver for WebhookReceiver { fn as_alertmanager_receiver(&self) -> Result { todo!() } - async fn install(&self, sender: &CRDPrometheus) -> Result { + async fn build_route(&self, sender: &CRDPrometheus) -> Result { let spec = crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::AlertmanagerConfigSpec { data: json!({ "route": { @@ -167,7 +167,7 @@ impl AlertReceiver for WebhookReceiver { fn as_alertmanager_receiver(&self) -> Result { todo!() } - async fn install(&self, sender: &Prometheus) -> Result { + async fn build_route(&self, sender: &Prometheus) -> Result { sender.install_receiver(self).await } fn name(&self) -> String { @@ -196,7 +196,7 @@ impl AlertReceiver for WebhookReceiver { fn as_alertmanager_receiver(&self) -> Result { todo!() } - async fn install(&self, sender: &KubePrometheus) -> Result { + async fn build_route(&self, sender: &KubePrometheus) -> Result { sender.install_receiver(self).await } fn name(&self) -> String { diff --git a/harmony/src/modules/monitoring/okd/cluster_monitoring.rs b/harmony/src/modules/monitoring/okd/cluster_monitoring.rs deleted file mode 100644 index 56fb59e0..00000000 --- a/harmony/src/modules/monitoring/okd/cluster_monitoring.rs +++ /dev/null @@ -1,270 +0,0 @@ -use base64::prelude::*; - -use async_trait::async_trait; -use harmony_types::id::Id; -use kube::api::DynamicObject; -use log::{debug, info, trace}; -use serde::Serialize; - -use crate::{ - data::Version, - interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, - inventory::Inventory, - modules::monitoring::okd::OpenshiftClusterAlertSender, - score::Score, - topology::{K8sclient, Topology, oberservability::monitoring::AlertReceiver}, -}; - -impl Clone for Box> { - fn clone(&self) -> Self { - self.clone_box() - } -} - -impl Serialize for Box> { - fn serialize(&self, _serializer: S) -> Result - where - S: serde::Serializer, - { - todo!() - } -} - -#[derive(Debug, Clone, Serialize)] -pub struct OpenshiftClusterAlertScore { - pub receivers: Vec>>, -} - -impl Score for OpenshiftClusterAlertScore { - fn name(&self) -> String { - "ClusterAlertScore".to_string() - } - - #[doc(hidden)] - fn create_interpret(&self) -> Box> { - Box::new(OpenshiftClusterAlertInterpret { - receivers: self.receivers.clone(), - }) - } -} - -#[derive(Debug)] -pub struct OpenshiftClusterAlertInterpret { - receivers: Vec>>, -} - -#[async_trait] -impl Interpret for OpenshiftClusterAlertInterpret { - async fn execute( - &self, - _inventory: &Inventory, - topology: &T, - ) -> Result { - let client = topology.k8s_client().await?; - let openshift_monitoring_namespace = "openshift-monitoring"; - - let mut alertmanager_main_secret: DynamicObject = client - .get_secret_json_value("alertmanager-main", Some(openshift_monitoring_namespace)) - .await?; - trace!("Got secret {alertmanager_main_secret:#?}"); - - let data: &mut serde_json::Value = &mut alertmanager_main_secret.data; - trace!("Alertmanager-main secret data {data:#?}"); - let data_obj = data - .get_mut("data") - .ok_or(InterpretError::new( - "Missing 'data' field in alertmanager-main secret.".to_string(), - ))? - .as_object_mut() - .ok_or(InterpretError::new( - "'data' field in alertmanager-main secret is expected to be an object ." - .to_string(), - ))?; - - let config_b64 = data_obj - .get("alertmanager.yaml") - .ok_or(InterpretError::new( - "Missing 'alertmanager.yaml' in alertmanager-main secret data".to_string(), - ))? - .as_str() - .unwrap_or(""); - trace!("Config base64 {config_b64}"); - - let config_bytes = BASE64_STANDARD.decode(config_b64).unwrap_or_default(); - - let mut am_config: serde_yaml::Value = - serde_yaml::from_str(&String::from_utf8(config_bytes).unwrap_or_default()) - .unwrap_or_default(); - - debug!("Current alertmanager config {am_config:#?}"); - - let existing_receivers_sequence = if let Some(receivers) = am_config.get_mut("receivers") { - match receivers.as_sequence_mut() { - Some(seq) => seq, - None => { - return Err(InterpretError::new(format!( - "Expected alertmanager config receivers to be a sequence, got {:?}", - receivers - ))); - } - } - } else { - &mut serde_yaml::Sequence::default() - }; - - let mut additional_resources = vec![]; - - for custom_receiver in &self.receivers { - let name = custom_receiver.name(); - let alertmanager_receiver = custom_receiver.as_alertmanager_receiver()?; - - let receiver_json_value = alertmanager_receiver.receiver_config; - - let receiver_yaml_string = - serde_json::to_string(&receiver_json_value).map_err(|e| { - InterpretError::new(format!("Failed to serialize receiver config: {}", e)) - })?; - - let receiver_yaml_value: serde_yaml::Value = - serde_yaml::from_str(&receiver_yaml_string).map_err(|e| { - InterpretError::new(format!("Failed to parse receiver config as YAML: {}", e)) - })?; - - if let Some(idx) = existing_receivers_sequence.iter().position(|r| { - r.get("name") - .and_then(|n| n.as_str()) - .map_or(false, |n| n == name) - }) { - info!("Replacing existing AlertManager receiver: {}", name); - existing_receivers_sequence[idx] = receiver_yaml_value; - } else { - debug!("Adding new AlertManager receiver: {}", name); - existing_receivers_sequence.push(receiver_yaml_value); - } - - additional_resources.push(alertmanager_receiver.additional_ressources); - } - - let existing_route_mapping = if let Some(route) = am_config.get_mut("route") { - match route.as_mapping_mut() { - Some(map) => map, - None => { - return Err(InterpretError::new(format!( - "Expected alertmanager config route to be a mapping, got {:?}", - route - ))); - } - } - } else { - &mut serde_yaml::Mapping::default() - }; - - let existing_route_sequence = if let Some(routes) = existing_route_mapping.get_mut("routes") - { - match routes.as_sequence_mut() { - Some(seq) => seq, - None => { - return Err(InterpretError::new(format!( - "Expected alertmanager config routes to be a sequence, got {:?}", - routes - ))); - } - } - } else { - &mut serde_yaml::Sequence::default() - }; - - for custom_receiver in &self.receivers { - let name = custom_receiver.name(); - let alertmanager_receiver = custom_receiver.as_alertmanager_receiver()?; - - let route_json_value = alertmanager_receiver.route_config; - let route_yaml_string = serde_json::to_string(&route_json_value).map_err(|e| { - InterpretError::new(format!("Failed to serialize route config: {}", e)) - })?; - - let route_yaml_value: serde_yaml::Value = serde_yaml::from_str(&route_yaml_string) - .map_err(|e| { - InterpretError::new(format!("Failed to parse route config as YAML: {}", e)) - })?; - - if let Some(idy) = existing_route_sequence.iter().position(|r| { - r.get("receiver") - .and_then(|n| n.as_str()) - .map_or(false, |n| n == name) - }) { - info!("Replacing existing AlertManager receiver: {}", name); - existing_route_sequence[idy] = route_yaml_value; - } else { - debug!("Adding new AlertManager receiver: {}", name); - existing_route_sequence.push(route_yaml_value); - } - } - debug!("Current alertmanager config {am_config:#?}"); - // TODO - // - save new version of alertmanager config - // - write additional ressources to the cluster - let am_config = serde_yaml::to_string(&am_config).map_err(|e| { - InterpretError::new(format!( - "Failed to serialize new alertmanager config to string : {e}" - )) - })?; - - let mut am_config_b64 = String::new(); - BASE64_STANDARD.encode_string(am_config, &mut am_config_b64); - - // TODO put update configmap value and save new value - data_obj.insert( - "alertmanager.yaml".to_string(), - serde_json::Value::String(am_config_b64), - ); - - // https://kubernetes.io/docs/reference/using-api/server-side-apply/#field-management - alertmanager_main_secret.metadata.managed_fields = None; - - trace!("Applying new alertmanager_main_secret {alertmanager_main_secret:#?}"); - client - .apply_dynamic( - &alertmanager_main_secret, - Some(openshift_monitoring_namespace), - true, - ) - .await?; - - let additional_resources = additional_resources.concat(); - trace!("Applying additional ressources for alert receivers {additional_resources:#?}"); - client - .apply_dynamic_many( - &additional_resources, - Some(openshift_monitoring_namespace), - true, - ) - .await?; - - Ok(Outcome::success(format!( - "Successfully configured {} cluster alert receivers: {}", - self.receivers.len(), - self.receivers - .iter() - .map(|r| r.name()) - .collect::>() - .join(", ") - ))) - } - - fn get_name(&self) -> InterpretName { - InterpretName::Custom("OpenshiftClusterAlertInterpret") - } - - fn get_version(&self) -> Version { - todo!() - } - - fn get_status(&self) -> InterpretStatus { - todo!() - } - - fn get_children(&self) -> Vec { - todo!() - } -} diff --git a/harmony/src/modules/monitoring/okd/config.rs b/harmony/src/modules/monitoring/okd/config.rs deleted file mode 100644 index b86c5f02..00000000 --- a/harmony/src/modules/monitoring/okd/config.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::{collections::BTreeMap, sync::Arc}; - -use crate::{ - interpret::{InterpretError, Outcome}, - topology::k8s::K8sClient, -}; -use k8s_openapi::api::core::v1::ConfigMap; -use kube::api::ObjectMeta; - -pub(crate) struct Config; - -impl Config { - pub async fn create_cluster_monitoring_config_cm( - client: &Arc, - ) -> Result { - let mut data = BTreeMap::new(); - data.insert( - "config.yaml".to_string(), - r#" -enableUserWorkload: true -alertmanagerMain: - enableUserAlertmanagerConfig: true -"# - .to_string(), - ); - - let cm = ConfigMap { - metadata: ObjectMeta { - name: Some("cluster-monitoring-config".to_string()), - namespace: Some("openshift-monitoring".to_string()), - ..Default::default() - }, - data: Some(data), - ..Default::default() - }; - client.apply(&cm, Some("openshift-monitoring")).await?; - - Ok(Outcome::success( - "updated cluster-monitoring-config-map".to_string(), - )) - } - - pub async fn create_user_workload_monitoring_config_cm( - client: &Arc, - ) -> Result { - let mut data = BTreeMap::new(); - data.insert( - "config.yaml".to_string(), - r#" -alertmanager: - enabled: true - enableAlertmanagerConfig: true -"# - .to_string(), - ); - let cm = ConfigMap { - metadata: ObjectMeta { - name: Some("user-workload-monitoring-config".to_string()), - namespace: Some("openshift-user-workload-monitoring".to_string()), - ..Default::default() - }, - data: Some(data), - ..Default::default() - }; - client - .apply(&cm, Some("openshift-user-workload-monitoring")) - .await?; - - Ok(Outcome::success( - "updated openshift-user-monitoring-config-map".to_string(), - )) - } - - pub async fn verify_user_workload(client: &Arc) -> Result { - let namespace = "openshift-user-workload-monitoring"; - let alertmanager_name = "alertmanager-user-workload-0"; - let prometheus_name = "prometheus-user-workload-0"; - client - .wait_for_pod_ready(alertmanager_name, Some(namespace)) - .await?; - client - .wait_for_pod_ready(prometheus_name, Some(namespace)) - .await?; - - Ok(Outcome::success(format!( - "pods: {}, {} ready in ns: {}", - alertmanager_name, prometheus_name, namespace - ))) - } -} diff --git a/harmony/src/modules/monitoring/okd/enable_user_workload.rs b/harmony/src/modules/monitoring/okd/enable_user_workload.rs deleted file mode 100644 index 2ed0fa41..00000000 --- a/harmony/src/modules/monitoring/okd/enable_user_workload.rs +++ /dev/null @@ -1,60 +0,0 @@ -use crate::{ - data::Version, - interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, - inventory::Inventory, - modules::monitoring::okd::config::Config, - score::Score, - topology::{K8sclient, Topology}, -}; -use async_trait::async_trait; -use harmony_types::id::Id; -use serde::Serialize; - -#[derive(Clone, Debug, Serialize)] -pub struct OpenshiftUserWorkloadMonitoring {} - -impl Score for OpenshiftUserWorkloadMonitoring { - fn name(&self) -> String { - "OpenshiftUserWorkloadMonitoringScore".to_string() - } - - fn create_interpret(&self) -> Box> { - Box::new(OpenshiftUserWorkloadMonitoringInterpret {}) - } -} - -#[derive(Clone, Debug, Serialize)] -pub struct OpenshiftUserWorkloadMonitoringInterpret {} - -#[async_trait] -impl Interpret for OpenshiftUserWorkloadMonitoringInterpret { - async fn execute( - &self, - _inventory: &Inventory, - topology: &T, - ) -> Result { - let client = topology.k8s_client().await.unwrap(); - Config::create_cluster_monitoring_config_cm(&client).await?; - Config::create_user_workload_monitoring_config_cm(&client).await?; - Config::verify_user_workload(&client).await?; - Ok(Outcome::success( - "successfully enabled user-workload-monitoring".to_string(), - )) - } - - fn get_name(&self) -> InterpretName { - InterpretName::Custom("OpenshiftUserWorkloadMonitoring") - } - - fn get_version(&self) -> Version { - todo!() - } - - fn get_status(&self) -> InterpretStatus { - todo!() - } - - fn get_children(&self) -> Vec { - todo!() - } -} diff --git a/harmony/src/modules/monitoring/okd/installable.rs b/harmony/src/modules/monitoring/okd/installable.rs deleted file mode 100644 index 5b31a640..00000000 --- a/harmony/src/modules/monitoring/okd/installable.rs +++ /dev/null @@ -1,40 +0,0 @@ -use async_trait::async_trait; -use log::info; - -use crate::{ - interpret::InterpretError, inventory::Inventory, modules::monitoring::okd::{OpenshiftClusterAlertSender, enable_user_workload::OpenshiftUserWorkloadMonitoring, score_cluster_monitoring::OpenshiftClusterMonitoringScore, score_verify_user_workload_monitoring::VerifyUserWorkload}, score::Score, topology::{K8sclient, Topology, installable::Installable} -}; - -#[async_trait] -impl Installable for OpenshiftClusterAlertSender { - async fn configure(&self, inventory: &Inventory, topology: &T) -> Result<(), InterpretError> { - info!("enabling cluster monitoring"); - let cluster_monitoring_score = OpenshiftClusterMonitoringScore {}; - cluster_monitoring_score - .create_interpret() - .execute(inventory, topology) - .await?; - - info!("enabling user workload monitoring"); - let user_workload_score = OpenshiftUserWorkloadMonitoring {}; - user_workload_score - .create_interpret() - .execute(inventory, topology) - .await?; - Ok(()) - } - - async fn ensure_installed( - &self, - inventory: &Inventory, - topology: &T, - ) -> Result<(), InterpretError> { - let verify_monitoring_score = VerifyUserWorkload {}; - info!("Verifying user workload and cluster monitoring installed"); - verify_monitoring_score - .create_interpret() - .execute(inventory, topology) - .await?; - Ok(()) - } -} diff --git a/harmony/src/modules/monitoring/okd/mod.rs b/harmony/src/modules/monitoring/okd/mod.rs index 067394bf..c573c56b 100644 --- a/harmony/src/modules/monitoring/okd/mod.rs +++ b/harmony/src/modules/monitoring/okd/mod.rs @@ -1,13 +1,11 @@ use crate::topology::oberservability::monitoring::AlertSender; -pub mod cluster_monitoring; -pub(crate) mod config; -pub mod enable_user_workload; -pub mod installable; pub mod score_cluster_monitoring; pub mod score_user_workload; pub mod score_verify_user_workload_monitoring; pub mod score_openshift_receiver; +pub mod score_openshift_alert_rule; +pub mod score_openshift_scrape_target; #[derive(Debug)] pub struct OpenshiftClusterAlertSender; diff --git a/harmony/src/modules/monitoring/okd/old.md b/harmony/src/modules/monitoring/okd/old.md deleted file mode 100644 index ef36f118..00000000 --- a/harmony/src/modules/monitoring/okd/old.md +++ /dev/null @@ -1,31 +0,0 @@ - -#[derive(Debug)] -pub struct OpenshiftClusterAlertInterpret { - receivers: Vec>>, -} - -#[async_trait] -impl Interpret for OpenshiftClusterAlertInterpret { - async fn execute( - &self, - _inventory: &Inventory, - topology: &T, - ) -> Result { - } - - fn get_name(&self) -> InterpretName { - InterpretName::Custom("OpenshiftClusterAlertInterpret") - } - - fn get_version(&self) -> Version { - todo!() - } - - fn get_status(&self) -> InterpretStatus { - todo!() - } - - fn get_children(&self) -> Vec { - todo!() - } -} diff --git a/harmony/src/modules/monitoring/okd/score_openshift_alert_rule.rs b/harmony/src/modules/monitoring/okd/score_openshift_alert_rule.rs new file mode 100644 index 00000000..74c091e3 --- /dev/null +++ b/harmony/src/modules/monitoring/okd/score_openshift_alert_rule.rs @@ -0,0 +1,16 @@ +use serde::Serialize; + +use crate::{interpret::Interpret, score::Score, topology::Topology}; + +#[derive(Debug, Clone, Serialize)] +pub struct OpenshiftAlertRuleScore {} + +impl Score for OpenshiftAlertRuleScore { + fn name(&self) -> String { + "OpenshiftAlertingRuleScore".to_string() + } + + fn create_interpret(&self) -> Box> { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs b/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs index f182fb92..afb5ae06 100644 --- a/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs +++ b/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs @@ -53,7 +53,9 @@ impl Interpret for OpenshiftReceiverInterpret { let data = am_secret .data .get_mut("data") - .ok_or_else(|| InterpretError::new("Missing 'data' field in alertmanager-main secret".into()))? + .ok_or_else(|| { + InterpretError::new("Missing 'data' field in alertmanager-main secret".into()) + })? .as_object_mut() .ok_or_else(|| InterpretError::new("'data' field must be a JSON object".into()))?; @@ -63,8 +65,8 @@ impl Interpret for OpenshiftReceiverInterpret { .and_then(|v| v.as_str()) .unwrap_or_default(); let config_bytes = BASE64_STANDARD.decode(config_b64).unwrap_or_default(); - let mut am_config: serde_yaml::Value = - serde_yaml::from_slice(&config_bytes).unwrap_or_else(|_| serde_yaml::Value::Mapping(serde_yaml::Mapping::new())); + let mut am_config: serde_yaml::Value = serde_yaml::from_slice(&config_bytes) + .unwrap_or_else(|_| serde_yaml::Value::Mapping(serde_yaml::Mapping::new())); // Ensure "receivers" exists let receivers_seq = am_config @@ -82,45 +84,51 @@ impl Interpret for OpenshiftReceiverInterpret { .unwrap_or_else(|| { am_config["route"] = serde_yaml::Value::Mapping(serde_yaml::Mapping::new()); am_config["route"]["routes"] = serde_yaml::Value::Sequence(vec![]); - am_config.pointer_mut("/route/routes").unwrap().as_sequence_mut().unwrap() + am_config + .pointer_mut("/route/routes") + .unwrap() + .as_sequence_mut() + .unwrap() }); // 3️⃣ Add/update the single receiver let name = self.receiver.name(); - let am_receiver = self.receiver.as_alertmanager_receiver()?; + // Receiver - let receiver_yaml: serde_yaml::Value = - serde_yaml::from_str(&serde_json::to_string(&am_receiver.receiver_config)?) - .map_err(|e| InterpretError::new(format!("Failed to convert receiver config to YAML: {e}")))?; + let receiver = self.receiver.build_receiver()?; - if let Some(idx) = receivers_seq.iter().position(|r| r.get("name").and_then(|n| n.as_str()) == Some(&name)) { - receivers_seq[idx] = receiver_yaml; + if let Some(idx) = receivers_seq + .iter() + .position(|r| r.get("name").and_then(|n| n.as_str()) == Some(&name)) + { + receivers_seq[idx] = receiver; } else { - receivers_seq.push(receiver_yaml); + receivers_seq.push(receiver); } // Route - let route_yaml: serde_yaml::Value = - serde_yaml::from_str(&serde_json::to_string(&am_receiver.route_config)?) - .map_err(|e| InterpretError::new(format!("Failed to convert route config to YAML: {e}")))?; + // + let route = self.receiver.build_route(); - if let Some(idx) = route_seq.iter().position(|r| r.get("receiver").and_then(|n| n.as_str()) == Some(name)) { - route_seq[idx] = route_yaml; + if let Some(idx) = route_seq + .iter() + .position(|r| r.get("receiver").and_then(|n| n.as_str()) == Some(name)) + { + route_seq[idx] = route; } else { - route_seq.push(route_yaml); + route_seq.push(route); } - // 4️⃣ Apply additional resources - client - .apply_dynamic_many(&am_receiver.additional_ressources, Some(ns), true) - .await?; // 5️⃣ Save updated alertmanager.yaml let yaml_str = serde_yaml::to_string(&am_config)?; let mut yaml_b64 = String::new(); BASE64_STANDARD.encode_string(yaml_str, &mut yaml_b64); - data.insert("alertmanager.yaml".to_string(), serde_json::Value::String(yaml_b64)); + data.insert( + "alertmanager.yaml".to_string(), + serde_json::Value::String(yaml_b64), + ); am_secret.metadata.managed_fields = None; client.apply_dynamic(&am_secret, Some(ns), true).await?; diff --git a/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs b/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs new file mode 100644 index 00000000..a30dfb69 --- /dev/null +++ b/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs @@ -0,0 +1,16 @@ +use serde::Serialize; + +use crate::{interpret::Interpret, score::Score, topology::Topology}; + +#[derive(Debug, Clone, Serialize)] +pub struct OpenshiftScrapeTargetScore {} + +impl Score for OpenshiftScrapeTargetScore{ + fn name(&self) -> String { + "OpenshiftAlertingRuleScore".to_string() + } + + fn create_interpret(&self) -> Box> { + todo!() + } +} diff --git a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs index 7796d255..e382cfdc 100644 --- a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs +++ b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs @@ -567,7 +567,7 @@ impl K8sPrometheusCRDAlertingInterpret { receivers: &Vec>>, ) -> Result { for receiver in receivers.iter() { - receiver.install(sender).await.map_err(|err| { + receiver.build_route(sender).await.map_err(|err| { InterpretError::new(format!("failed to install receiver: {}", err)) })?; } diff --git a/harmony/src/modules/prometheus/rhob_alerting_score.rs b/harmony/src/modules/prometheus/rhob_alerting_score.rs index f44bf8cd..5522e001 100644 --- a/harmony/src/modules/prometheus/rhob_alerting_score.rs +++ b/harmony/src/modules/prometheus/rhob_alerting_score.rs @@ -518,7 +518,7 @@ impl RHOBAlertingInterpret { receivers: &Vec>>, ) -> Result { for receiver in receivers.iter() { - receiver.install(sender).await.map_err(|err| { + receiver.build_route(sender).await.map_err(|err| { InterpretError::new(format!("failed to install receiver: {}", err)) })?; } -- 2.39.5 From 836db9e6b1bc3c77fafab1391e2eb54a4fa73712 Mon Sep 17 00:00:00 2001 From: wjro Date: Mon, 23 Feb 2026 13:18:40 -0500 Subject: [PATCH 03/15] wip: refactored redhat cluster observability operator --- examples/monitoring/src/main.rs | 2 +- examples/monitoring_with_tenant/src/main.rs | 2 +- examples/okd_cluster_alerts/src/main.rs | 2 +- .../rhob_application_monitoring/src/main.rs | 10 +- examples/rust/src/main.rs | 10 +- examples/try_rust_webapp/src/main.rs | 16 +- .../topology/k8s_anywhere/k8s_anywhere.rs | 321 +++++----- .../src/domain/topology/k8s_anywhere/mod.rs | 2 +- .../k8s_anywhere/openshift_monitoring.rs | 35 +- .../topology/oberservability/monitoring.rs | 52 +- .../application/features/monitoring.rs | 19 +- .../application/features/rhob_monitoring.rs | 24 +- .../alert_channel/discord_alert_channel.rs | 84 ++- .../alert_channel/webhook_receiver.rs | 582 +++++++++--------- .../application_monitoring_score.rs | 5 +- .../monitoring/application_monitoring/mod.rs | 2 +- .../rhobs_application_monitoring_score.rs | 52 +- .../crd/crd_grafana.rs | 3 +- .../crd/grafana_default_dashboard.rs | 0 .../crd/grafana_operator.rs | 0 .../src/modules/monitoring/grafana/crd/mod.rs | 4 + .../crd/rhob_grafana.rs | 0 harmony/src/modules/monitoring/grafana/mod.rs | 1 + .../crd/crd_alertmanager_config.rs | 29 +- .../crd/crd_prometheus_rules.rs | 23 +- .../monitoring/kube_prometheus/crd/mod.rs | 13 - .../helm_prometheus_alert_score.rs | 6 +- .../modules/monitoring/kube_prometheus/mod.rs | 4 +- .../monitoring/kube_prometheus/prometheus.rs | 13 - .../monitoring/kube_prometheus/types.rs | 1 - harmony/src/modules/monitoring/mod.rs | 5 +- harmony/src/modules/monitoring/okd/mod.rs | 43 +- .../okd/score_openshift_receiver.rs | 82 +-- .../okd/score_openshift_scrape_target.rs | 2 +- .../monitoring/prometheus/prometheus.rs | 12 - .../red_hat_cluster_observability/crd/mod.rs | 11 + .../crd/rhob_alertmanager_config.rs | 11 +- .../crd/rhob_alertmanagers.rs | 3 +- .../rhob_cluster_observability_operator.rs | 0 .../crd/rhob_default_rules.rs | 5 +- .../crd/rhob_monitoring_stack.rs | 6 +- .../crd/rhob_prometheus_rules.rs | 23 +- .../crd/rhob_prometheuses.rs | 9 +- .../crd/rhob_role.rs | 0 .../crd/rhob_service_monitor.rs | 0 .../red_hat_cluster_observability/mod.rs | 63 ++ .../redhat_cluster_observability.rs | 36 ++ .../score_alert_receiver.rs | 100 +++ .../score_coo_monitoring_stack.rs | 45 ++ ...e_redhat_cluster_observability_operator.rs | 113 ++++ .../trait_redhat_cluster_observability.rs | 114 ++++ .../k8s_prometheus_alerting_score.rs | 12 +- harmony/src/modules/prometheus/mod.rs | 2 +- .../modules/prometheus/rhob_alerting_score.rs | 10 +- 54 files changed, 1293 insertions(+), 731 deletions(-) rename harmony/src/modules/monitoring/{kube_prometheus => grafana}/crd/crd_grafana.rs (98%) rename harmony/src/modules/monitoring/{kube_prometheus => grafana}/crd/grafana_default_dashboard.rs (100%) rename harmony/src/modules/monitoring/{kube_prometheus => grafana}/crd/grafana_operator.rs (100%) create mode 100644 harmony/src/modules/monitoring/grafana/crd/mod.rs rename harmony/src/modules/monitoring/{kube_prometheus => grafana}/crd/rhob_grafana.rs (100%) create mode 100644 harmony/src/modules/monitoring/red_hat_cluster_observability/crd/mod.rs rename harmony/src/modules/monitoring/{kube_prometheus => red_hat_cluster_observability}/crd/rhob_alertmanager_config.rs (86%) rename harmony/src/modules/monitoring/{kube_prometheus => red_hat_cluster_observability}/crd/rhob_alertmanagers.rs (93%) rename harmony/src/modules/monitoring/{kube_prometheus => red_hat_cluster_observability}/crd/rhob_cluster_observability_operator.rs (100%) rename harmony/src/modules/monitoring/{kube_prometheus => red_hat_cluster_observability}/crd/rhob_default_rules.rs (88%) rename harmony/src/modules/monitoring/{kube_prometheus => red_hat_cluster_observability}/crd/rhob_monitoring_stack.rs (89%) rename harmony/src/modules/monitoring/{kube_prometheus => red_hat_cluster_observability}/crd/rhob_prometheus_rules.rs (71%) rename harmony/src/modules/monitoring/{kube_prometheus => red_hat_cluster_observability}/crd/rhob_prometheuses.rs (96%) rename harmony/src/modules/monitoring/{kube_prometheus => red_hat_cluster_observability}/crd/rhob_role.rs (100%) rename harmony/src/modules/monitoring/{kube_prometheus => red_hat_cluster_observability}/crd/rhob_service_monitor.rs (100%) create mode 100644 harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs create mode 100644 harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs create mode 100644 harmony/src/modules/monitoring/red_hat_cluster_observability/score_alert_receiver.rs create mode 100644 harmony/src/modules/monitoring/red_hat_cluster_observability/score_coo_monitoring_stack.rs create mode 100644 harmony/src/modules/monitoring/red_hat_cluster_observability/score_redhat_cluster_observability_operator.rs create mode 100644 harmony/src/modules/monitoring/red_hat_cluster_observability/trait_redhat_cluster_observability.rs diff --git a/examples/monitoring/src/main.rs b/examples/monitoring/src/main.rs index 7037094d..fca2728f 100644 --- a/examples/monitoring/src/main.rs +++ b/examples/monitoring/src/main.rs @@ -29,7 +29,7 @@ use harmony_types::{k8s_name::K8sName, net::Url}; #[tokio::main] async fn main() { let discord_receiver = DiscordWebhook { - name: K8sName("test-discord".to_string()), + name: "test-discord".to_string(), url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()), selectors: vec![], }; diff --git a/examples/monitoring_with_tenant/src/main.rs b/examples/monitoring_with_tenant/src/main.rs index f67f9d8a..bc2b8b99 100644 --- a/examples/monitoring_with_tenant/src/main.rs +++ b/examples/monitoring_with_tenant/src/main.rs @@ -43,7 +43,7 @@ async fn main() { }; let discord_receiver = DiscordWebhook { - name: K8sName("test-discord".to_string()), + name: "test-discord".to_string(), url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()), selectors: vec![], }; diff --git a/examples/okd_cluster_alerts/src/main.rs b/examples/okd_cluster_alerts/src/main.rs index 93dac3b4..5f5d8dc2 100644 --- a/examples/okd_cluster_alerts/src/main.rs +++ b/examples/okd_cluster_alerts/src/main.rs @@ -26,7 +26,7 @@ async fn main() { K8sAnywhereTopology::from_env(), vec![Box::new(OpenshiftClusterAlertScore { receivers: vec![Box::new(DiscordWebhook { - name: K8sName("wills-discord-webhook-example".to_string()), + name: "wills-discord-webhook-example".to_string(), url: hurl!("https://something.io"), selectors: selectors, })], diff --git a/examples/rhob_application_monitoring/src/main.rs b/examples/rhob_application_monitoring/src/main.rs index 6eeaea2c..e865029d 100644 --- a/examples/rhob_application_monitoring/src/main.rs +++ b/examples/rhob_application_monitoring/src/main.rs @@ -23,17 +23,17 @@ async fn main() { }); let discord_receiver = DiscordWebhook { - name: K8sName("test-discord".to_string()), + name: "test-discord".to_string(), url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()), selectors: vec![], }; let app = ApplicationScore { features: vec![ - Box::new(Monitoring { - application: application.clone(), - alert_receiver: vec![Box::new(discord_receiver)], - }), + // Box::new(Monitoring { + // application: application.clone(), + // alert_receiver: vec![Box::new(discord_receiver)], + // }), // TODO add backups, multisite ha, etc ], application, diff --git a/examples/rust/src/main.rs b/examples/rust/src/main.rs index a1006068..60252141 100644 --- a/examples/rust/src/main.rs +++ b/examples/rust/src/main.rs @@ -27,7 +27,7 @@ async fn main() { }); let discord_receiver = DiscordWebhook { - name: K8sName("test-discord".to_string()), + name: "test-discord".to_string(), url: hurl!("https://discord.doesnt.exist.com"), selectors: vec![], }; @@ -42,10 +42,10 @@ async fn main() { Box::new(PackagingDeployment { application: application.clone(), }), - Box::new(Monitoring { - application: application.clone(), - alert_receiver: vec![Box::new(discord_receiver), Box::new(webhook_receiver)], - }), + // Box::new(Monitoring { + // application: application.clone(), + // alert_receiver: vec![Box::new(discord_receiver), Box::new(webhook_receiver)], + // }), // TODO add backups, multisite ha, etc ], application, diff --git a/examples/try_rust_webapp/src/main.rs b/examples/try_rust_webapp/src/main.rs index d3c88b32..2dbb3de3 100644 --- a/examples/try_rust_webapp/src/main.rs +++ b/examples/try_rust_webapp/src/main.rs @@ -30,14 +30,14 @@ async fn main() { Box::new(PackagingDeployment { application: application.clone(), }), - Box::new(Monitoring { - application: application.clone(), - alert_receiver: vec![Box::new(DiscordWebhook { - name: K8sName("test-discord".to_string()), - url: hurl!("https://discord.doesnt.exist.com"), - selectors: vec![], - })], - }), + // Box::new(Monitoring { + // application: application.clone(), + // alert_receiver: vec![Box::new(DiscordWebhook { + // name: K8sName("test-discord".to_string()), + // url: hurl!("https://discord.doesnt.exist.com"), + // selectors: vec![], + // })], + // }), ], application, }; diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 5462d67a..6ba26187 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -30,29 +30,22 @@ use crate::{ k3d::K3DInstallationScore, k8s::ingress::{K8sIngressScore, PathType}, monitoring::{ - grafana::{grafana::Grafana, helm::helm_grafana::grafana_helm_chart_score}, + grafana::{crd::crd_grafana::{Grafana as GrafanaCRD,GrafanaCom, GrafanaDashboard, GrafanaDashboardDatasource, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig, GrafanaDatasourceJsonData, GrafanaDatasourceSecureJsonData, GrafanaDatasourceSpec, GrafanaSpec}, grafana::Grafana, helm::helm_grafana::grafana_helm_chart_score}, kube_prometheus::crd::{ crd_alertmanager_config::CRDPrometheus, - crd_grafana::{ - Grafana as GrafanaCRD, GrafanaCom, GrafanaDashboard, - GrafanaDashboardDatasource, GrafanaDashboardSpec, GrafanaDatasource, - GrafanaDatasourceConfig, GrafanaDatasourceJsonData, - GrafanaDatasourceSecureJsonData, GrafanaDatasourceSpec, GrafanaSpec, - }, crd_prometheuses::LabelSelector, prometheus_operator::prometheus_operator_helm_chart_score, - rhob_alertmanager_config::RHOBObservability, service_monitor::ServiceMonitor, }, }, okd::{crd::ingresses_config::Ingress as IngressResource, route::OKDTlsPassthroughScore}, prometheus::{ k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore, - prometheus::Monitor, rhob_alerting_score::RHOBAlertingScore, + rhob_alerting_score::RHOBAlertingScore, }, }, score::Score, - topology::{TlsRoute, TlsRouter, ingress::Ingress}, + topology::{TlsRoute, TlsRouter, ingress::Ingress, oberservability::monitoring::Observability}, }; use super::super::{ @@ -297,91 +290,91 @@ impl Grafana for K8sAnywhereTopology { } } -#[async_trait] -impl Monitor for K8sAnywhereTopology { - async fn install_montoring( - &self, - sender: &CRDPrometheus, - _inventory: &Inventory, - _receivers: Option>>>, - ) -> Result { - let client = self.k8s_client().await?; - - for monitor in sender.service_monitor.iter() { - client - .apply(monitor, Some(&sender.namespace)) - .await - .map_err(|e| PreparationError::new(e.to_string()))?; - } - Ok(PreparationOutcome::Success { - details: "successfuly installed prometheus components".to_string(), - }) - } - - async fn ensure_monitoring( - &self, - sender: &CRDPrometheus, - _inventory: &Inventory, - ) -> Result { - let po_result = self.ensure_prometheus_operator(sender).await?; - - match po_result { - PreparationOutcome::Success { details: _ } => { - debug!("Detected prometheus crds operator present in cluster."); - return Ok(po_result); - } - PreparationOutcome::Noop => { - debug!("Skipping Prometheus CR installation due to missing operator."); - return Ok(po_result); - } - } - } -} - -#[async_trait] -impl Monitor for K8sAnywhereTopology { - async fn install_montoring( - &self, - sender: &RHOBObservability, - inventory: &Inventory, - receivers: Option>>>, - ) -> Result { - let po_result = self.ensure_cluster_observability_operator(sender).await?; - - if po_result == PreparationOutcome::Noop { - debug!("Skipping Prometheus CR installation due to missing operator."); - return Ok(po_result); - } - - let result = self - .get_cluster_observability_operator_prometheus_application_score( - sender.clone(), - receivers, - ) - .await - .interpret(inventory, self) - .await; - - match result { - Ok(outcome) => match outcome.status { - InterpretStatus::SUCCESS => Ok(PreparationOutcome::Success { - details: outcome.message, - }), - InterpretStatus::NOOP => Ok(PreparationOutcome::Noop), - _ => Err(PreparationError::new(outcome.message)), - }, - Err(err) => Err(PreparationError::new(err.to_string())), - } - } - - async fn ensure_monitoring( - &self, - sender: &RHOBObservability, - inventory: &Inventory, - ) -> Result { - todo!() - } -} +// #[async_trait] +// impl Monitor for K8sAnywhereTopology { +// async fn install_montoring( +// &self, +// sender: &CRDPrometheus, +// _inventory: &Inventory, +// _receivers: Option>>>, +// ) -> Result { +// let client = self.k8s_client().await?; +// +// for monitor in sender.service_monitor.iter() { +// client +// .apply(monitor, Some(&sender.namespace)) +// .await +// .map_err(|e| PreparationError::new(e.to_string()))?; +// } +// Ok(PreparationOutcome::Success { +// details: "successfuly installed prometheus components".to_string(), +// }) +// } +// +// async fn ensure_monitoring( +// &self, +// sender: &CRDPrometheus, +// _inventory: &Inventory, +// ) -> Result { +// let po_result = self.ensure_prometheus_operator(sender).await?; +// +// match po_result { +// PreparationOutcome::Success { details: _ } => { +// debug!("Detected prometheus crds operator present in cluster."); +// return Ok(po_result); +// } +// PreparationOutcome::Noop => { +// debug!("Skipping Prometheus CR installation due to missing operator."); +// return Ok(po_result); +// } +// } +// } +// } +// +// #[async_trait] +// impl Monitor for K8sAnywhereTopology { +// async fn install_montoring( +// &self, +// sender: &RHOBObservability, +// inventory: &Inventory, +// receivers: Option>>>, +// ) -> Result { +// let po_result = self.ensure_cluster_observability_operator(sender).await?; +// +// if po_result == PreparationOutcome::Noop { +// debug!("Skipping Prometheus CR installation due to missing operator."); +// return Ok(po_result); +// } +// +// let result = self +// .get_cluster_observability_operator_prometheus_application_score( +// sender.clone(), +// receivers, +// ) +// .await +// .interpret(inventory, self) +// .await; +// +// match result { +// Ok(outcome) => match outcome.status { +// InterpretStatus::SUCCESS => Ok(PreparationOutcome::Success { +// details: outcome.message, +// }), +// InterpretStatus::NOOP => Ok(PreparationOutcome::Noop), +// _ => Err(PreparationError::new(outcome.message)), +// }, +// Err(err) => Err(PreparationError::new(err.to_string())), +// } +// } +// +// async fn ensure_monitoring( +// &self, +// sender: &RHOBObservability, +// inventory: &Inventory, +// ) -> Result { +// todo!() +// } +// } impl Serialize for K8sAnywhereTopology { fn serialize(&self, _serializer: S) -> Result @@ -771,18 +764,18 @@ impl K8sAnywhereTopology { } } - async fn get_cluster_observability_operator_prometheus_application_score( - &self, - sender: RHOBObservability, - receivers: Option>>>, - ) -> RHOBAlertingScore { - RHOBAlertingScore { - sender, - receivers: receivers.unwrap_or_default(), - service_monitors: vec![], - prometheus_rules: vec![], - } - } + // async fn get_cluster_observability_operator_prometheus_application_score( + // &self, + // sender: RHOBObservability, + // receivers: Option>>>, + // ) -> RHOBAlertingScore { + // RHOBAlertingScore { + // sender, + // receivers: receivers.unwrap_or_default(), + // service_monitors: vec![], + // prometheus_rules: vec![], + // } + // } async fn get_k8s_prometheus_application_score( &self, @@ -964,63 +957,63 @@ impl K8sAnywhereTopology { } } - async fn ensure_cluster_observability_operator( - &self, - sender: &RHOBObservability, - ) -> Result { - let status = Command::new("sh") - .args(["-c", "kubectl get crd -A | grep -i rhobs"]) - .status() - .map_err(|e| PreparationError::new(format!("could not connect to cluster: {}", e)))?; - - if !status.success() { - if let Some(Some(k8s_state)) = self.k8s_state.get() { - match k8s_state.source { - K8sSource::LocalK3d => { - warn!( - "Installing observability operator is not supported on LocalK3d source" - ); - return Ok(PreparationOutcome::Noop); - debug!("installing cluster observability operator"); - todo!(); - let op_score = - prometheus_operator_helm_chart_score(sender.namespace.clone()); - let result = op_score.interpret(&Inventory::empty(), self).await; - - return match result { - Ok(outcome) => match outcome.status { - InterpretStatus::SUCCESS => Ok(PreparationOutcome::Success { - details: "installed cluster observability operator".into(), - }), - InterpretStatus::NOOP => Ok(PreparationOutcome::Noop), - _ => Err(PreparationError::new( - "failed to install cluster observability operator (unknown error)".into(), - )), - }, - Err(err) => Err(PreparationError::new(err.to_string())), - }; - } - K8sSource::Kubeconfig => { - debug!( - "unable to install cluster observability operator, contact cluster admin" - ); - return Ok(PreparationOutcome::Noop); - } - } - } else { - warn!( - "Unable to detect k8s_state. Skipping Cluster Observability Operator install." - ); - return Ok(PreparationOutcome::Noop); - } - } - - debug!("Cluster Observability Operator is already present, skipping install"); - - Ok(PreparationOutcome::Success { - details: "cluster observability operator present in cluster".into(), - }) - } + // async fn ensure_cluster_observability_operator( + // &self, + // sender: &RHOBObservability, + // ) -> Result { + // let status = Command::new("sh") + // .args(["-c", "kubectl get crd -A | grep -i rhobs"]) + // .status() + // .map_err(|e| PreparationError::new(format!("could not connect to cluster: {}", e)))?; + // + // if !status.success() { + // if let Some(Some(k8s_state)) = self.k8s_state.get() { + // match k8s_state.source { + // K8sSource::LocalK3d => { + // warn!( + // "Installing observability operator is not supported on LocalK3d source" + // ); + // return Ok(PreparationOutcome::Noop); + // debug!("installing cluster observability operator"); + // todo!(); + // let op_score = + // prometheus_operator_helm_chart_score(sender.namespace.clone()); + // let result = op_score.interpret(&Inventory::empty(), self).await; + // + // return match result { + // Ok(outcome) => match outcome.status { + // InterpretStatus::SUCCESS => Ok(PreparationOutcome::Success { + // details: "installed cluster observability operator".into(), + // }), + // InterpretStatus::NOOP => Ok(PreparationOutcome::Noop), + // _ => Err(PreparationError::new( + // "failed to install cluster observability operator (unknown error)".into(), + // )), + // }, + // Err(err) => Err(PreparationError::new(err.to_string())), + // }; + // } + // K8sSource::Kubeconfig => { + // debug!( + // "unable to install cluster observability operator, contact cluster admin" + // ); + // return Ok(PreparationOutcome::Noop); + // } + // } + // } else { + // warn!( + // "Unable to detect k8s_state. Skipping Cluster Observability Operator install." + // ); + // return Ok(PreparationOutcome::Noop); + // } + // } + // + // debug!("Cluster Observability Operator is already present, skipping install"); + // + // Ok(PreparationOutcome::Success { + // details: "cluster observability operator present in cluster".into(), + // }) + // } async fn ensure_prometheus_operator( &self, diff --git a/harmony/src/domain/topology/k8s_anywhere/mod.rs b/harmony/src/domain/topology/k8s_anywhere/mod.rs index c9f634e9..144f3c1c 100644 --- a/harmony/src/domain/topology/k8s_anywhere/mod.rs +++ b/harmony/src/domain/topology/k8s_anywhere/mod.rs @@ -1,5 +1,5 @@ mod k8s_anywhere; pub mod nats; -mod postgres; pub mod openshift_monitoring; +mod postgres; pub use k8s_anywhere::*; diff --git a/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs b/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs index 4cc31320..890decf2 100644 --- a/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs +++ b/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs @@ -1,19 +1,25 @@ use async_trait::async_trait; use log::info; +use crate::score::Score; use crate::{ inventory::Inventory, modules::monitoring::okd::{ - OpenshiftClusterAlertSender, score_cluster_monitoring::OpenshiftClusterMonitoringScore, score_openshift_alert_rule::OpenshiftAlertRuleScore, score_openshift_receiver::OpenshiftReceiverScore, score_openshift_scrape_target::OpenshiftScrapeTargetScore, score_user_workload::OpenshiftUserWorkloadMonitoring, score_verify_user_workload_monitoring::VerifyUserWorkload + OpenshiftClusterAlertSender, score_cluster_monitoring::OpenshiftClusterMonitoringScore, + score_openshift_alert_rule::OpenshiftAlertRuleScore, + score_openshift_receiver::OpenshiftReceiverScore, + score_openshift_scrape_target::OpenshiftScrapeTargetScore, + score_user_workload::OpenshiftUserWorkloadMonitoring, + score_verify_user_workload_monitoring::VerifyUserWorkload, }, topology::{ K8sAnywhereTopology, PreparationError, PreparationOutcome, - oberservability::monitoring::{AlertReceiver, AlertRule, Monitor, ScrapeTarget}, + oberservability::monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, }, }; #[async_trait] -impl Monitor for K8sAnywhereTopology { +impl Observability for K8sAnywhereTopology { async fn install_alert_sender( &self, sender: &OpenshiftClusterAlertSender, @@ -24,14 +30,16 @@ impl Monitor for K8sAnywhereTopology { cluster_monitoring_score .create_interpret() .execute(inventory, self) - .await?; + .await + .map_err(|e| PreparationError { msg: e.to_string() })?; info!("enabling user workload monitoring"); let user_workload_score = OpenshiftUserWorkloadMonitoring {}; user_workload_score .create_interpret() .execute(inventory, self) - .await?; + .await + .map_err(|e| PreparationError { msg: e.to_string() })?; Ok(PreparationOutcome::Success { details: "Successfully configured cluster monitoring".to_string(), @@ -51,7 +59,8 @@ impl Monitor for K8sAnywhereTopology { receiver_score .create_interpret() .execute(inventory, self) - .await?; + .await + .map_err(|e| PreparationError { msg: e.to_string() })?; } Ok(PreparationOutcome::Success { details: "Successfully installed receivers for OpenshiftClusterMonitoring" @@ -75,7 +84,8 @@ impl Monitor for K8sAnywhereTopology { rule_score .create_interpret() .execute(inventory, self) - .await?; + .await + .map_err(|e| PreparationError { msg: e.to_string() })?; } Ok(PreparationOutcome::Success { details: "Successfully installed rules for OpenshiftClusterMonitoring".to_string(), @@ -93,12 +103,13 @@ impl Monitor for K8sAnywhereTopology { ) -> Result { if let Some(scrape_targets) = scrape_targets { for scrape_target in scrape_targets { - !info!("Installing scrape target"); + info!("Installing scrape target"); let scrape_target_score = OpenshiftScrapeTargetScore {}; scrape_target_score .create_interpret() .execute(inventory, self) - .await?; + .await + .map_err(|e| PreparationError { msg: e.to_string() })?; } Ok(PreparationOutcome::Success { details: "Successfully added scrape targets for OpenshiftClusterMonitoring" @@ -119,6 +130,10 @@ impl Monitor for K8sAnywhereTopology { verify_monitoring_score .create_interpret() .execute(inventory, self) - .await? + .await + .map_err(|e| PreparationError { msg: e.to_string() })?; + Ok(PreparationOutcome::Success { + details: "OpenshiftClusterMonitoring ready".to_string(), + }) } } diff --git a/harmony/src/domain/topology/oberservability/monitoring.rs b/harmony/src/domain/topology/oberservability/monitoring.rs index c9acc77e..45a4a4a7 100644 --- a/harmony/src/domain/topology/oberservability/monitoring.rs +++ b/harmony/src/domain/topology/oberservability/monitoring.rs @@ -3,6 +3,7 @@ use std::{any::Any, collections::HashMap}; use async_trait::async_trait; use kube::api::DynamicObject; use log::{debug, info}; +use serde::Serialize; use crate::{ data::Version, @@ -31,12 +32,17 @@ pub trait AlertReceiver: std::fmt::Debug + Send + Sync { pub struct ReceiverEndpoint {} ///Generic routing that can map to various alert sender backends +#[derive(Debug, Clone, Serialize)] pub struct AlertRoute { pub receiver: String, pub matchers: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] pub group_by: Vec, - pub repeat_interval: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub repeat_interval: Option, + #[serde(rename = "continue")] pub continue_matching: bool, + #[serde(skip_serializing_if = "Vec::is_empty")] pub children: Vec, } @@ -46,25 +52,39 @@ impl AlertRoute { receiver: name, matchers: vec![], group_by: vec![], - repeat_interval: Some(30), + repeat_interval: Some("30s".to_string()), continue_matching: true, children: vec![], } } } +#[derive(Debug, Clone, Serialize)] pub struct AlertMatcher { pub label: String, pub operator: MatchOp, pub value: String, } +#[derive(Debug, Clone)] pub enum MatchOp { Eq, NotEq, Regex, } +impl Serialize for MatchOp { + fn serialize(&self, serializer: S) -> Result + where S: serde::Serializer { + let op = match self { + MatchOp::Eq => "=", + MatchOp::NotEq => "!=", + MatchOp::Regex => "=~", + }; + serializer.serialize_str(op) + } +} + #[derive(Debug)] pub struct AlertManagerReceiver { pub receiver_config: serde_json::Value, @@ -90,7 +110,7 @@ pub trait ScrapeTarget: std::fmt::Debug + Send + Sync { /// Trait which defines how an alert sender is impleneted for a specific topology #[async_trait] -pub trait Monitor { +pub trait Observability { async fn install_alert_sender( &self, sender: &S, @@ -135,7 +155,7 @@ pub struct AlertingInterpret { } #[async_trait] -impl> Interpret for AlertingInterpret { +impl> Interpret for AlertingInterpret { async fn execute( &self, inventory: &Inventory, @@ -148,17 +168,17 @@ impl> Interpret for AlertingInterpre info!("Installing receivers"); topology - .install_receivers(&self.sender, inventory, Some(self.receivers)) + .install_receivers(&self.sender, inventory, Some(self.receivers.clone())) .await?; info!("Installing rules"); topology - .install_rules(&self.sender, inventory, Some(self.rules)) + .install_rules(&self.sender, inventory, Some(self.rules.clone())) .await?; info!("Adding extra scrape targets"); topology - .add_scrape_targets(&self.sender, inventory, self.scrape_targets) + .add_scrape_targets(&self.sender, inventory, self.scrape_targets.clone()) .await?; info!("Ensuring alert sender {} is ready", self.sender.name()); @@ -188,3 +208,21 @@ impl> Interpret for AlertingInterpre todo!() } } + +impl Clone for Box> { + fn clone(&self) -> Self { + self.clone_box() + } +} + +impl Clone for Box> { + fn clone(&self) -> Self { + self.clone_box() + } +} + +impl Clone for Box> { + fn clone(&self) -> Self { + self.clone_box() + } +} diff --git a/harmony/src/modules/application/features/monitoring.rs b/harmony/src/modules/application/features/monitoring.rs index 3ef5d5ad..8ae74072 100644 --- a/harmony/src/modules/application/features/monitoring.rs +++ b/harmony/src/modules/application/features/monitoring.rs @@ -9,7 +9,8 @@ use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{ }; use crate::topology::MultiTargetTopology; use crate::topology::ingress::Ingress; -use crate::topology::oberservability::monitoring::Monitor; +use crate::topology::oberservability::monitoring::AlertReceiver; +use crate::topology::oberservability::monitoring::Observability; use crate::{ inventory::Inventory, modules::monitoring::{ @@ -18,9 +19,6 @@ use crate::{ score::Score, topology::{HelmCommand, K8sclient, Topology, tenant::TenantManager}, }; -use crate::{ - topology::oberservability::monitoring::AlertReceiver, -}; use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose}; use harmony_secret::SecretManager; @@ -46,7 +44,7 @@ impl< + TenantManager + K8sclient + MultiTargetTopology - + Monitor + + Observability + Grafana + Ingress + std::fmt::Debug, @@ -119,11 +117,12 @@ impl< ), }; - alerting_score.receivers.push(Box::new(ntfy_receiver)); - alerting_score - .interpret(&Inventory::empty(), topology) - .await - .map_err(|e| e.to_string())?; + todo!(); + // alerting_score.receivers.push(Box::new(ntfy_receiver)); + // alerting_score + // .interpret(&Inventory::empty(), topology) + // .await + // .map_err(|e| e.to_string())?; Ok(InstallationOutcome::success()) } diff --git a/harmony/src/modules/application/features/rhob_monitoring.rs b/harmony/src/modules/application/features/rhob_monitoring.rs index 792603b7..643a5c63 100644 --- a/harmony/src/modules/application/features/rhob_monitoring.rs +++ b/harmony/src/modules/application/features/rhob_monitoring.rs @@ -3,12 +3,13 @@ use std::sync::Arc; use crate::modules::application::{ Application, ApplicationFeature, InstallationError, InstallationOutcome, }; -use crate::modules::monitoring::application_monitoring::rhobs_application_monitoring_score::ApplicationRHOBMonitoringScore; -use crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::RHOBObservability; +use crate::modules::monitoring::red_hat_cluster_observability::RedHatClusterObservability; +use crate::modules::monitoring::red_hat_cluster_observability::redhat_cluster_observability::RedHatClusterObservabilityScore; use crate::topology::MultiTargetTopology; use crate::topology::ingress::Ingress; -use crate::topology::oberservability::monitoring::Monitor; +use crate::topology::oberservability::monitoring::AlertReceiver; +use crate::topology::oberservability::monitoring::Observability; use crate::{ inventory::Inventory, modules::monitoring::{ @@ -17,9 +18,6 @@ use crate::{ score::Score, topology::{HelmCommand, K8sclient, Topology, tenant::TenantManager}, }; -use crate::{ - topology::oberservability::monitoring::AlertReceiver, -}; use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose}; use harmony_types::net::Url; @@ -28,9 +26,10 @@ use log::{debug, info}; #[derive(Debug, Clone)] pub struct Monitoring { pub application: Arc, - pub alert_receiver: Vec>>, + pub alert_receiver: Vec>>, } +///TODO TEST this #[async_trait] impl< T: Topology @@ -41,7 +40,7 @@ impl< + MultiTargetTopology + Ingress + std::fmt::Debug - + Monitor, + + Observability, > ApplicationFeature for Monitoring { async fn ensure_installed( @@ -55,13 +54,14 @@ impl< .map(|ns| ns.name.clone()) .unwrap_or_else(|| self.application.name()); - let mut alerting_score = ApplicationRHOBMonitoringScore { - sender: RHOBObservability { + let mut alerting_score = RedHatClusterObservabilityScore { + sender: RedHatClusterObservability { namespace: namespace.clone(), - client: topology.k8s_client().await.unwrap(), + resource_selector: todo!(), }, - application: self.application.clone(), receivers: self.alert_receiver.clone(), + rules: vec![], + scrape_targets: None, }; let domain = topology .get_domain("ntfy") diff --git a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs index d5fdc8da..5fa3570d 100644 --- a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs +++ b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs @@ -15,19 +15,14 @@ use crate::infra::kube::kube_resource_to_dynamic; use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{ AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus, }; -use crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::RHOBObservability; use crate::modules::monitoring::okd::OpenshiftClusterAlertSender; +use crate::modules::monitoring::red_hat_cluster_observability::RedHatClusterObservability; use crate::topology::oberservability::monitoring::{ - AlertManagerReceiver, AlertRoute, MatchOp, ReceiverEndpoint + AlertManagerReceiver, AlertRoute, MatchOp, ReceiverEndpoint, }; use crate::{ interpret::{InterpretError, Outcome}, modules::monitoring::{ - kube_prometheus::{ - prometheus::{KubePrometheus, KubePrometheusReceiver}, - types::{AlertChannelConfig, AlertManagerChannelConfig}, - }, - prometheus::prometheus::{Prometheus, PrometheusReceiver}, }, topology::oberservability::monitoring::AlertReceiver, }; @@ -44,7 +39,7 @@ pub struct DiscordReceiver { pub struct DiscordWebhook { pub name: String, pub url: Url, - pub route: Vec>, + pub selectors: Vec>, } impl DiscordWebhook { @@ -96,9 +91,7 @@ impl DiscordWebhook { } impl AlertReceiver for DiscordReceiver { - fn build_route( - &self, - ) -> Result { + fn build_route(&self) -> Result { let matchers: Vec = self .route .matchers @@ -113,14 +106,12 @@ impl AlertReceiver for DiscordReceiver { let route_block = serde_yaml::to_value(json!({ "receiver": self.name, "matchers": matchers, - }))?; + })) + .unwrap(); Ok(route_block) } - fn build_receiver( - &self, - ) -> Result { - + fn build_receiver(&self) -> Result { let receiver_block = serde_yaml::to_value(json!({ "name": self.name, "discord_configs": [{ @@ -128,7 +119,8 @@ impl AlertReceiver for DiscordReceiver { "title": "{{ template \"discord.default.title\" . }}", "message": "{{ template \"discord.default.message\" . }}" }] - }))?; + })) + .map_err(|e| InterpretError::new(e.to_string()))?; Ok(receiver_block) } @@ -139,14 +131,58 @@ impl AlertReceiver for DiscordReceiver { fn clone_box(&self) -> Box> { Box::new(self.clone()) } +} - // fn as_any(&self) -> &dyn Any { - // todo!() - // } - // - // fn as_alertmanager_receiver(&self) -> Result { - // self.get_receiver_config() - // } +impl AlertReceiver for DiscordReceiver { + fn build_route(&self) -> Result { + serde_yaml::to_value(&self.route).map_err(|e| InterpretError::new(e.to_string())) + + } + + fn build_receiver(&self) -> Result { + + //FIXME this secret needs to be applied so that the discord Configs for RedHatCO + //CRD AlertmanagerConfigs can access the URL + let secret_name = format!("{}-secret", self.name.clone()); + let webhook_key = format!("{}", self.url.clone()); + + let mut string_data = BTreeMap::new(); + string_data.insert("webhook-url".to_string(), webhook_key.clone()); + + let secret = Secret { + metadata: kube::core::ObjectMeta { + name: Some(secret_name.clone()), + ..Default::default() + }, + string_data: Some(string_data), + type_: Some("Opaque".to_string()), + ..Default::default() + }; + + let receiver_config = json!({ + "name": self.name, + "discordConfigs": [ + { + "apiURL": { + "key": "webhook-url", + "name": format!("{}-secret", self.name) + }, + "title": "{{ template \"discord.default.title\" . }}", + "message": "{{ template \"discord.default.message\" . }}" + } + ] + }); + serde_yaml::to_value(receiver_config) + .map_err(|e| InterpretError::new(e.to_string())) + } + + fn name(&self) -> String { + self.name.clone() + } + + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } } // // #[async_trait] diff --git a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs index 9548bf89..5e0602f8 100644 --- a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs +++ b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs @@ -10,14 +10,7 @@ use serde_yaml::{Mapping, Value}; use crate::{ interpret::{InterpretError, Outcome}, modules::monitoring::{ - kube_prometheus::{ - crd::{ - crd_alertmanager_config::CRDPrometheus, rhob_alertmanager_config::RHOBObservability, - }, - prometheus::{KubePrometheus, KubePrometheusReceiver}, - types::{AlertChannelConfig, AlertManagerChannelConfig}, - }, - prometheus::prometheus::{Prometheus, PrometheusReceiver}, + okd::OpenshiftClusterAlertSender, red_hat_cluster_observability::RedHatClusterObservability, }, topology::oberservability::monitoring::{AlertManagerReceiver, AlertReceiver}, }; @@ -29,279 +22,320 @@ pub struct WebhookReceiver { pub url: Url, } -#[async_trait] -impl AlertReceiver for WebhookReceiver { - fn as_alertmanager_receiver(&self) -> Result { - todo!() - } - async fn build_route(&self, sender: &RHOBObservability) -> Result { - let spec = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfigSpec { - data: json!({ - "route": { - "receiver": self.name, - }, - "receivers": [ - { - "name": self.name, - "webhookConfigs": [ - { - "url": self.url, - "httpConfig": { - "tlsConfig": { - "insecureSkipVerify": true - } - } - } - ] - } - ] - }), - }; - - let alertmanager_configs = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfig { - metadata: ObjectMeta { - name: Some(self.name.clone()), - labels: Some(std::collections::BTreeMap::from([( - "alertmanagerConfig".to_string(), - "enabled".to_string(), - )])), - namespace: Some(sender.namespace.clone()), - ..Default::default() - }, - spec, - }; - debug!( - "alert manager configs: \n{:#?}", - alertmanager_configs.clone() - ); - - sender - .client - .apply(&alertmanager_configs, Some(&sender.namespace)) - .await?; - Ok(Outcome::success(format!( - "installed rhob-alertmanagerconfigs for {}", - self.name - ))) - } - - fn name(&self) -> String { - "webhook-receiver".to_string() - } - - fn clone_box(&self) -> Box> { - Box::new(self.clone()) - } - - fn as_any(&self) -> &dyn Any { - self - } -} - -#[async_trait] -impl AlertReceiver for WebhookReceiver { - fn as_alertmanager_receiver(&self) -> Result { - todo!() - } - async fn build_route(&self, sender: &CRDPrometheus) -> Result { - let spec = crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::AlertmanagerConfigSpec { - data: json!({ - "route": { - "receiver": self.name, - }, - "receivers": [ - { - "name": self.name, - "webhookConfigs": [ - { - "url": self.url, - } - ] - } - ] - }), - }; - - let alertmanager_configs = crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::AlertmanagerConfig { - metadata: ObjectMeta { - name: Some(self.name.clone()), - labels: Some(std::collections::BTreeMap::from([( - "alertmanagerConfig".to_string(), - "enabled".to_string(), - )])), - namespace: Some(sender.namespace.clone()), - ..Default::default() - }, - spec, - }; - debug!( - "alert manager configs: \n{:#?}", - alertmanager_configs.clone() - ); - - sender - .client - .apply(&alertmanager_configs, Some(&sender.namespace)) - .await?; - Ok(Outcome::success(format!( - "installed crd-alertmanagerconfigs for {}", - self.name - ))) - } - - fn name(&self) -> String { - "webhook-receiver".to_string() - } - - fn clone_box(&self) -> Box> { - Box::new(self.clone()) - } - - fn as_any(&self) -> &dyn Any { - self - } -} - -#[async_trait] -impl AlertReceiver for WebhookReceiver { - fn as_alertmanager_receiver(&self) -> Result { - todo!() - } - async fn build_route(&self, sender: &Prometheus) -> Result { - sender.install_receiver(self).await - } - fn name(&self) -> String { - "webhook-receiver".to_string() - } - fn clone_box(&self) -> Box> { - Box::new(self.clone()) - } - fn as_any(&self) -> &dyn Any { - self - } -} - -#[async_trait] -impl PrometheusReceiver for WebhookReceiver { - fn name(&self) -> String { - self.name.clone() - } - async fn configure_receiver(&self) -> AlertManagerChannelConfig { - self.get_config().await - } -} - -#[async_trait] -impl AlertReceiver for WebhookReceiver { - fn as_alertmanager_receiver(&self) -> Result { - todo!() - } - async fn build_route(&self, sender: &KubePrometheus) -> Result { - sender.install_receiver(self).await - } - fn name(&self) -> String { - "webhook-receiver".to_string() - } - fn clone_box(&self) -> Box> { - Box::new(self.clone()) - } - fn as_any(&self) -> &dyn Any { - self - } -} - -#[async_trait] -impl KubePrometheusReceiver for WebhookReceiver { - fn name(&self) -> String { - self.name.clone() - } - async fn configure_receiver(&self) -> AlertManagerChannelConfig { - self.get_config().await - } -} - -#[async_trait] -impl AlertChannelConfig for WebhookReceiver { - async fn get_config(&self) -> AlertManagerChannelConfig { - let channel_global_config = None; - let channel_receiver = self.alert_channel_receiver().await; - let channel_route = self.alert_channel_route().await; - - AlertManagerChannelConfig { - channel_global_config, - channel_receiver, - channel_route, - } - } -} - impl WebhookReceiver { - async fn alert_channel_route(&self) -> serde_yaml::Value { - let mut route = Mapping::new(); - route.insert( - Value::String("receiver".to_string()), - Value::String(self.name.clone()), - ); - route.insert( - Value::String("matchers".to_string()), - Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]), - ); - route.insert(Value::String("continue".to_string()), Value::Bool(true)); - Value::Mapping(route) + fn build_receiver(&self) -> serde_json::Value { + let receiver = json!({ + "name": self.name, + "webhookConfigs": [ + { + "url": self.url, + "httpConfig": { + "tlsConfig": { + "insecureSkipVerify": true + } + } + } + ]}); + receiver } - - async fn alert_channel_receiver(&self) -> serde_yaml::Value { - let mut receiver = Mapping::new(); - receiver.insert( - Value::String("name".to_string()), - Value::String(self.name.clone()), - ); - - let mut webhook_config = Mapping::new(); - webhook_config.insert( - Value::String("url".to_string()), - Value::String(self.url.to_string()), - ); - - receiver.insert( - Value::String("webhook_configs".to_string()), - Value::Sequence(vec![Value::Mapping(webhook_config)]), - ); - - Value::Mapping(receiver) + fn build_route(&self) -> serde_json::Value { + let route = json!({ + "name": self.name}); + route } } -#[cfg(test)] -mod tests { - use super::*; - #[tokio::test] - async fn webhook_serialize_should_match() { - let webhook_receiver = WebhookReceiver { - name: "test-webhook".to_string(), - url: Url::Url(url::Url::parse("https://webhook.i.dont.exist.com").unwrap()), - }; +impl AlertReceiver for WebhookReceiver { + fn build_receiver(&self) -> Result { + let receiver = self.build_receiver(); + serde_yaml::to_value(receiver).map_err(|e| InterpretError::new(e.to_string())) + } - let webhook_receiver_receiver = - serde_yaml::to_string(&webhook_receiver.alert_channel_receiver().await).unwrap(); - println!("receiver \n{:#}", webhook_receiver_receiver); - let webhook_receiver_receiver_yaml = r#"name: test-webhook -webhook_configs: -- url: https://webhook.i.dont.exist.com/ -"# - .to_string(); + fn build_route(&self) -> Result { + let route = self.build_route(); + serde_yaml::to_value(route).map_err(|e| InterpretError::new(e.to_string())) + } - let webhook_receiver_route = - serde_yaml::to_string(&webhook_receiver.alert_channel_route().await).unwrap(); - println!("route \n{:#}", webhook_receiver_route); - let webhook_receiver_route_yaml = r#"receiver: test-webhook -matchers: -- alertname!=Watchdog -continue: true -"# - .to_string(); + fn name(&self) -> String { + self.name.clone() + } - assert_eq!(webhook_receiver_receiver, webhook_receiver_receiver_yaml); - assert_eq!(webhook_receiver_route, webhook_receiver_route_yaml); + fn clone_box(&self) -> Box> { + Box::new(self.clone()) } } + +impl AlertReceiver for WebhookReceiver { + fn build_receiver(&self) -> Result { + let receiver = self.build_receiver(); + serde_yaml::to_value(receiver).map_err(|e| InterpretError::new(e.to_string())) + } + + fn build_route(&self) -> Result { + let route = self.build_route(); + serde_yaml::to_value(route).map_err(|e| InterpretError::new(e.to_string())) + // let spec = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfigSpec { + // data: json!({ + // "route": { + // "receiver": self.name, + // }, + // "receivers": [ + // { + // "name": self.name, + // "webhookConfigs": [ + // { + // "url": self.url, + // "httpConfig": { + // "tlsConfig": { + // "insecureSkipVerify": true + // } + // } + // } + // ] + // } + // ] + // }), + // }; + // + // let alertmanager_configs = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfig { + // metadata: ObjectMeta { + // name: Some(self.name.clone()), + // labels: Some(std::collections::BTreeMap::from([( + // "alertmanagerConfig".to_string(), + // "enabled".to_string(), + // )])), + // namespace: Some(sender.namespace.clone()), + // ..Default::default() + // }, + // spec, + // }; + // debug!( + // "alert manager configs: \n{:#?}", + // alertmanager_configs.clone() + // ); + // + // sender + // .client + // .apply(&alertmanager_configs, Some(&sender.namespace)); + // Ok(Outcome::success(format!( + // "installed rhob-alertmanagerconfigs for {}", + // self.name + // ))) + } + + fn name(&self) -> String { + self.name.clone() + } + + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } +} + +// #[async_trait] +// impl AlertReceiver for WebhookReceiver { +// fn as_alertmanager_receiver(&self) -> Result { +// todo!() +// } +// async fn build_route(&self, sender: &CRDPrometheus) -> Result { +// let spec = crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::AlertmanagerConfigSpec { +// data: json!({ +// "route": { +// "receiver": self.name, +// }, +// "receivers": [ +// { +// "name": self.name, +// "webhookConfigs": [ +// { +// "url": self.url, +// } +// ] +// } +// ] +// }), +// }; +// +// let alertmanager_configs = crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::AlertmanagerConfig { +// metadata: ObjectMeta { +// name: Some(self.name.clone()), +// labels: Some(std::collections::BTreeMap::from([( +// "alertmanagerConfig".to_string(), +// "enabled".to_string(), +// )])), +// namespace: Some(sender.namespace.clone()), +// ..Default::default() +// }, +// spec, +// }; +// debug!( +// "alert manager configs: \n{:#?}", +// alertmanager_configs.clone() +// ); +// +// sender +// .client +// .apply(&alertmanager_configs, Some(&sender.namespace)) +// .await?; +// Ok(Outcome::success(format!( +// "installed crd-alertmanagerconfigs for {}", +// self.name +// ))) +// } +// +// fn name(&self) -> String { +// "webhook-receiver".to_string() +// } +// +// fn clone_box(&self) -> Box> { +// Box::new(self.clone()) +// } +// +// fn as_any(&self) -> &dyn Any { +// self +// } +// } +// +// #[async_trait] +// impl AlertReceiver for WebhookReceiver { +// fn as_alertmanager_receiver(&self) -> Result { +// todo!() +// } +// async fn build_route(&self, sender: &Prometheus) -> Result { +// sender.install_receiver(self).await +// } +// fn name(&self) -> String { +// "webhook-receiver".to_string() +// } +// fn clone_box(&self) -> Box> { +// Box::new(self.clone()) +// } +// fn as_any(&self) -> &dyn Any { +// self +// } +// } +// +// #[async_trait] +// impl PrometheusReceiver for WebhookReceiver { +// fn name(&self) -> String { +// self.name.clone() +// } +// async fn configure_receiver(&self) -> AlertManagerChannelConfig { +// self.get_config().await +// } +// } +// +// #[async_trait] +// impl AlertReceiver for WebhookReceiver { +// fn as_alertmanager_receiver(&self) -> Result { +// todo!() +// } +// async fn build_route(&self, sender: &KubePrometheus) -> Result { +// sender.install_receiver(self).await +// } +// fn name(&self) -> String { +// "webhook-receiver".to_string() +// } +// fn clone_box(&self) -> Box> { +// Box::new(self.clone()) +// } +// fn as_any(&self) -> &dyn Any { +// self +// } +// } +// +// #[async_trait] +// impl KubePrometheusReceiver for WebhookReceiver { +// fn name(&self) -> String { +// self.name.clone() +// } +// async fn configure_receiver(&self) -> AlertManagerChannelConfig { +// self.get_config().await +// } +// } +// +// #[async_trait] +// impl AlertChannelConfig for WebhookReceiver { +// async fn get_config(&self) -> AlertManagerChannelConfig { +// let channel_global_config = None; +// let channel_receiver = self.alert_channel_receiver().await; +// let channel_route = self.alert_channel_route().await; +// +// AlertManagerChannelConfig { +// channel_global_config, +// channel_receiver, +// channel_route, +// } +// } +// } +// +// impl WebhookReceiver { +// async fn alert_channel_route(&self) -> serde_yaml::Value { +// let mut route = Mapping::new(); +// route.insert( +// Value::String("receiver".to_string()), +// Value::String(self.name.clone()), +// ); +// route.insert( +// Value::String("matchers".to_string()), +// Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]), +// ); +// route.insert(Value::String("continue".to_string()), Value::Bool(true)); +// Value::Mapping(route) +// } +// +// async fn alert_channel_receiver(&self) -> serde_yaml::Value { +// let mut receiver = Mapping::new(); +// receiver.insert( +// Value::String("name".to_string()), +// Value::String(self.name.clone()), +// ); +// +// let mut webhook_config = Mapping::new(); +// webhook_config.insert( +// Value::String("url".to_string()), +// Value::String(self.url.to_string()), +// ); +// +// receiver.insert( +// Value::String("webhook_configs".to_string()), +// Value::Sequence(vec![Value::Mapping(webhook_config)]), +// ); +// +// Value::Mapping(receiver) +// } +// } +// +// #[cfg(test)] +// mod tests { +// use super::*; +// #[tokio::test] +// async fn webhook_serialize_should_match() { +// let webhook_receiver = WebhookReceiver { +// name: "test-webhook".to_string(), +// url: Url::Url(url::Url::parse("https://webhook.i.dont.exist.com").unwrap()), +// }; +// +// let webhook_receiver_receiver = +// serde_yaml::to_string(&webhook_receiver.alert_channel_receiver().await).unwrap(); +// println!("receiver \n{:#}", webhook_receiver_receiver); +// let webhook_receiver_receiver_yaml = r#"name: test-webhook +// webhook_configs: +// - url: https://webhook.i.dont.exist.com/ +// "# +// .to_string(); +// +// let webhook_receiver_route = +// serde_yaml::to_string(&webhook_receiver.alert_channel_route().await).unwrap(); +// println!("route \n{:#}", webhook_receiver_route); +// let webhook_receiver_route_yaml = r#"receiver: test-webhook +// matchers: +// - alertname!=Watchdog +// continue: true +// "# +// .to_string(); +// +// assert_eq!(webhook_receiver_receiver, webhook_receiver_receiver_yaml); +// assert_eq!(webhook_receiver_route, webhook_receiver_route_yaml); +// } +// } diff --git a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs index ef554bc9..b5b2aba8 100644 --- a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs +++ b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs @@ -10,12 +10,11 @@ use crate::{ monitoring::{ grafana::grafana::Grafana, kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus, }, - prometheus::prometheus::Monitor, }, score::Score, topology::{ K8sclient, Topology, - oberservability::monitoring::{AlertReceiver, AlertingInterpret, ScrapeTarget}, + oberservability::monitoring::{AlertReceiver, AlertingInterpret, Observability, ScrapeTarget}, }, }; @@ -26,7 +25,7 @@ pub struct ApplicationMonitoringScore { pub receivers: Vec>>, } -impl + K8sclient + Grafana> Score +impl + K8sclient + Grafana> Score for ApplicationMonitoringScore { fn create_interpret(&self) -> Box> { diff --git a/harmony/src/modules/monitoring/application_monitoring/mod.rs b/harmony/src/modules/monitoring/application_monitoring/mod.rs index 5d12f78a..02815703 100644 --- a/harmony/src/modules/monitoring/application_monitoring/mod.rs +++ b/harmony/src/modules/monitoring/application_monitoring/mod.rs @@ -1,2 +1,2 @@ pub mod application_monitoring_score; -pub mod rhobs_application_monitoring_score; +// pub mod rhobs_application_monitoring_score; diff --git a/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs index 83bed109..488eb8a9 100644 --- a/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs +++ b/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs @@ -12,10 +12,12 @@ use crate::{ monitoring::kube_prometheus::crd::{ crd_alertmanager_config::CRDPrometheus, rhob_alertmanager_config::RHOBObservability, }, - prometheus::prometheus::Monitor, }, score::Score, - topology::{PreparationOutcome, Topology, oberservability::monitoring::AlertReceiver}, + topology::{ + PreparationOutcome, Topology, + oberservability::monitoring::{AlertReceiver, Observability}, + }, }; use harmony_types::id::Id; @@ -26,9 +28,7 @@ pub struct ApplicationRHOBMonitoringScore { pub receivers: Vec>>, } -impl> Score - for ApplicationRHOBMonitoringScore -{ +impl> Score for ApplicationRHOBMonitoringScore { fn create_interpret(&self) -> Box> { Box::new(ApplicationRHOBMonitoringInterpret { score: self.clone(), @@ -49,33 +49,31 @@ pub struct ApplicationRHOBMonitoringInterpret { } #[async_trait] -impl> Interpret - for ApplicationRHOBMonitoringInterpret -{ +impl> Interpret for ApplicationRHOBMonitoringInterpret { async fn execute( &self, inventory: &Inventory, topology: &T, ) -> Result { - let result = topology - .install_montoring( - &self.score.sender, - inventory, - Some(self.score.receivers.clone()), - ) - .await; - - match result { - Ok(outcome) => match outcome { - PreparationOutcome::Success { details: _ } => { - Ok(Outcome::success("Prometheus installed".into())) - } - PreparationOutcome::Noop => { - Ok(Outcome::noop("Prometheus installation skipped".into())) - } - }, - Err(err) => Err(InterpretError::from(err)), - } + todo!() + // let result = topology + // .install_montoring( + // &self.score.sender, + // inventory, + // Some(self.score.receivers.clone()), + // ) + // .await; + // match result { + // Ok(outcome) => match outcome { + // PreparationOutcome::Success { details: _ } => { + // Ok(Outcome::success("Prometheus installed".into())) + // } + // PreparationOutcome::Noop => { + // Ok(Outcome::noop("Prometheus installation skipped".into())) + // } + // }, + // Err(err) => Err(InterpretError::from(err)), + // } } fn get_name(&self) -> InterpretName { diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_grafana.rs b/harmony/src/modules/monitoring/grafana/crd/crd_grafana.rs similarity index 98% rename from harmony/src/modules/monitoring/kube_prometheus/crd/crd_grafana.rs rename to harmony/src/modules/monitoring/grafana/crd/crd_grafana.rs index 386890e0..e72aa0ec 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_grafana.rs +++ b/harmony/src/modules/monitoring/grafana/crd/crd_grafana.rs @@ -4,7 +4,8 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use super::crd_prometheuses::LabelSelector; +use crate::modules::monitoring::kube_prometheus::crd::crd_prometheuses::LabelSelector; + #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] #[kube( diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/grafana_default_dashboard.rs b/harmony/src/modules/monitoring/grafana/crd/grafana_default_dashboard.rs similarity index 100% rename from harmony/src/modules/monitoring/kube_prometheus/crd/grafana_default_dashboard.rs rename to harmony/src/modules/monitoring/grafana/crd/grafana_default_dashboard.rs diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/grafana_operator.rs b/harmony/src/modules/monitoring/grafana/crd/grafana_operator.rs similarity index 100% rename from harmony/src/modules/monitoring/kube_prometheus/crd/grafana_operator.rs rename to harmony/src/modules/monitoring/grafana/crd/grafana_operator.rs diff --git a/harmony/src/modules/monitoring/grafana/crd/mod.rs b/harmony/src/modules/monitoring/grafana/crd/mod.rs new file mode 100644 index 00000000..1a91d94d --- /dev/null +++ b/harmony/src/modules/monitoring/grafana/crd/mod.rs @@ -0,0 +1,4 @@ +pub mod crd_grafana; +pub mod grafana_default_dashboard; +pub mod grafana_operator; +pub mod rhob_grafana; diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_grafana.rs b/harmony/src/modules/monitoring/grafana/crd/rhob_grafana.rs similarity index 100% rename from harmony/src/modules/monitoring/kube_prometheus/crd/rhob_grafana.rs rename to harmony/src/modules/monitoring/grafana/crd/rhob_grafana.rs diff --git a/harmony/src/modules/monitoring/grafana/mod.rs b/harmony/src/modules/monitoring/grafana/mod.rs index 8dccab1d..552ed66c 100644 --- a/harmony/src/modules/monitoring/grafana/mod.rs +++ b/harmony/src/modules/monitoring/grafana/mod.rs @@ -1,2 +1,3 @@ pub mod grafana; pub mod helm; +pub mod crd; diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs index 6894d615..61ad21fe 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs @@ -8,17 +8,14 @@ use serde::{Deserialize, Serialize}; use crate::{ interpret::{InterpretError, Outcome}, inventory::Inventory, - modules::{ - monitoring::{ - grafana::grafana::Grafana, kube_prometheus::crd::service_monitor::ServiceMonitor, - }, - prometheus::prometheus::Monitor, + modules::monitoring::{ + grafana::grafana::Grafana, kube_prometheus::crd::service_monitor::ServiceMonitor, }, topology::{ K8sclient, Topology, installable::Installable, k8s::K8sClient, - oberservability::monitoring::{AlertReceiver, AlertSender, ScrapeTarget}, + oberservability::monitoring::{AlertReceiver, AlertSender, Observability, ScrapeTarget}, }, }; @@ -48,18 +45,6 @@ impl AlertSender for CRDPrometheus { } } -impl Clone for Box> { - fn clone(&self) -> Self { - self.clone_box() - } -} - -impl Clone for Box> { - fn clone(&self) -> Self { - self.clone_box() - } -} - impl Serialize for Box> { fn serialize(&self, _serializer: S) -> Result where @@ -70,12 +55,10 @@ impl Serialize for Box> { } #[async_trait] -impl + Grafana> Installable - for CRDPrometheus -{ +impl + Grafana> Installable for CRDPrometheus { async fn configure(&self, inventory: &Inventory, topology: &T) -> Result<(), InterpretError> { topology.ensure_grafana_operator(inventory).await?; - topology.ensure_monitoring(self, inventory).await?; + // topology.ensure_monitoring(self, inventory).await?; Ok(()) } @@ -85,7 +68,7 @@ impl + Grafana> Installable topology: &T, ) -> Result<(), InterpretError> { topology.install_grafana().await?; - topology.install_montoring(&self, inventory, None).await?; + // topology.install_montoring(&self, inventory, None).await?; Ok(()) } } diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs index 6d08456b..3cb4d7ff 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs @@ -4,7 +4,6 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule; #[derive(CustomResource, Debug, Serialize, Deserialize, Clone, JsonSchema)] #[kube( @@ -44,14 +43,14 @@ pub struct Rule { pub annotations: Option>, } -impl From for Rule { - fn from(value: PrometheusAlertRule) -> Self { - Rule { - alert: Some(value.alert), - expr: Some(value.expr), - for_: value.r#for, - labels: Some(value.labels.into_iter().collect::>()), - annotations: Some(value.annotations.into_iter().collect::>()), - } - } -} +// impl From for Rule { +// fn from(value: PrometheusAlertRule) -> Self { +// Rule { +// alert: Some(value.alert), +// expr: Some(value.expr), +// for_: value.r#for, +// labels: Some(value.labels.into_iter().collect::>()), +// annotations: Some(value.annotations.into_iter().collect::>()), +// } +// } +// } diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/mod.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/mod.rs index c8cb8546..ff6b2ee8 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/mod.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/mod.rs @@ -1,22 +1,9 @@ pub mod crd_alertmanager_config; pub mod crd_alertmanagers; pub mod crd_default_rules; -pub mod crd_grafana; pub mod crd_prometheus_rules; pub mod crd_prometheuses; pub mod crd_scrape_config; -pub mod grafana_default_dashboard; -pub mod grafana_operator; pub mod prometheus_operator; -pub mod rhob_alertmanager_config; -pub mod rhob_alertmanagers; -pub mod rhob_cluster_observability_operator; -pub mod rhob_default_rules; -pub mod rhob_grafana; -pub mod rhob_monitoring_stack; -pub mod rhob_prometheus_rules; -pub mod rhob_prometheuses; -pub mod rhob_role; -pub mod rhob_service_monitor; pub mod role; pub mod service_monitor; diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs b/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs index 468d3088..fdc0ed2b 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs @@ -8,7 +8,7 @@ use crate::{ score::Score, topology::{ HelmCommand, Topology, - oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret}, + oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Monitor}, tenant::TenantManager, }, }; @@ -20,7 +20,9 @@ pub struct HelmPrometheusAlertingScore { pub service_monitors: Vec, } -impl Score for HelmPrometheusAlertingScore { +impl> Score + for HelmPrometheusAlertingScore +{ fn create_interpret(&self) -> Box> { let config = Arc::new(Mutex::new(KubePrometheusConfig::new())); config diff --git a/harmony/src/modules/monitoring/kube_prometheus/mod.rs b/harmony/src/modules/monitoring/kube_prometheus/mod.rs index 122e9396..1fac351a 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/mod.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/mod.rs @@ -1,5 +1,5 @@ pub mod crd; pub mod helm; -pub mod helm_prometheus_alert_score; -pub mod prometheus; +// pub mod helm_prometheus_alert_score; +// pub mod prometheus; pub mod types; diff --git a/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs b/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs index 98970e6c..0e01b0f4 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs @@ -7,7 +7,6 @@ use serde::Serialize; use crate::{ interpret::{InterpretError, Outcome}, inventory::Inventory, - modules::monitoring::alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, score, topology::{ HelmCommand, Topology, @@ -139,12 +138,6 @@ impl Serialize for Box> { } } -impl Clone for Box> { - fn clone(&self) -> Self { - self.clone_box() - } -} - #[async_trait] pub trait KubePrometheusRule: Send + Sync + std::fmt::Debug { fn name(&self) -> String; @@ -159,9 +152,3 @@ impl Serialize for Box> { todo!() } } - -impl Clone for Box> { - fn clone(&self) -> Self { - self.clone_box() - } -} diff --git a/harmony/src/modules/monitoring/kube_prometheus/types.rs b/harmony/src/modules/monitoring/kube_prometheus/types.rs index abe5896a..5e626bb3 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/types.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/types.rs @@ -5,7 +5,6 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_yaml::{Mapping, Sequence, Value}; -use crate::modules::monitoring::alert_rule::prometheus_alert_rule::AlertManagerRuleGroup; #[async_trait] pub trait AlertChannelConfig { diff --git a/harmony/src/modules/monitoring/mod.rs b/harmony/src/modules/monitoring/mod.rs index 7f07d5af..99b412d7 100644 --- a/harmony/src/modules/monitoring/mod.rs +++ b/harmony/src/modules/monitoring/mod.rs @@ -1,9 +1,10 @@ pub mod alert_channel; -pub mod alert_rule; +// pub mod alert_rule; pub mod application_monitoring; pub mod grafana; pub mod kube_prometheus; pub mod ntfy; pub mod okd; -pub mod prometheus; +// pub mod prometheus; pub mod scrape_target; +pub mod red_hat_cluster_observability; diff --git a/harmony/src/modules/monitoring/okd/mod.rs b/harmony/src/modules/monitoring/okd/mod.rs index c573c56b..c30067b8 100644 --- a/harmony/src/modules/monitoring/okd/mod.rs +++ b/harmony/src/modules/monitoring/okd/mod.rs @@ -1,11 +1,13 @@ -use crate::topology::oberservability::monitoring::AlertSender; +use serde::Serialize; + +use crate::topology::oberservability::monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}; pub mod score_cluster_monitoring; +pub mod score_openshift_alert_rule; +pub mod score_openshift_receiver; +pub mod score_openshift_scrape_target; pub mod score_user_workload; pub mod score_verify_user_workload_monitoring; -pub mod score_openshift_receiver; -pub mod score_openshift_alert_rule; -pub mod score_openshift_scrape_target; #[derive(Debug)] pub struct OpenshiftClusterAlertSender; @@ -15,3 +17,36 @@ impl AlertSender for OpenshiftClusterAlertSender { "OpenshiftClusterAlertSender".to_string() } } + +// impl Clone for Box> { +// fn clone(&self) -> Self { +// self.clone_box() +// } +// } + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs b/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs index afb5ae06..d5743822 100644 --- a/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs +++ b/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs @@ -1,5 +1,5 @@ use async_trait::async_trait; -use base64::prelude::BASE64_STANDARD; +use base64::{Engine as _, prelude::BASE64_STANDARD}; use harmony_types::id::Id; use kube::api::DynamicObject; use log::{debug, info, trace}; @@ -26,7 +26,7 @@ impl Score for OpenshiftReceiverScore { fn create_interpret(&self) -> Box> { Box::new(OpenshiftReceiverInterpret { - receiver: self.receiver, + receiver: self.receiver.clone(), }) } } @@ -59,7 +59,6 @@ impl Interpret for OpenshiftReceiverInterpret { .as_object_mut() .ok_or_else(|| InterpretError::new("'data' field must be a JSON object".into()))?; - // 2️⃣ Decode alertmanager.yaml let config_b64 = data .get("alertmanager.yaml") .and_then(|v| v.as_str()) @@ -68,61 +67,46 @@ impl Interpret for OpenshiftReceiverInterpret { let mut am_config: serde_yaml::Value = serde_yaml::from_slice(&config_bytes) .unwrap_or_else(|_| serde_yaml::Value::Mapping(serde_yaml::Mapping::new())); - // Ensure "receivers" exists - let receivers_seq = am_config - .get_mut("receivers") - .and_then(|r| r.as_sequence_mut()) - .unwrap_or_else(|| { - am_config["receivers"] = serde_yaml::Value::Sequence(vec![]); - am_config["receivers"].as_sequence_mut().unwrap() - }); - - // Ensure "route/routes" exists - let route_seq = am_config - .pointer_mut("/route/routes") - .and_then(|r| r.as_sequence_mut()) - .unwrap_or_else(|| { - am_config["route"] = serde_yaml::Value::Mapping(serde_yaml::Mapping::new()); - am_config["route"]["routes"] = serde_yaml::Value::Sequence(vec![]); - am_config - .pointer_mut("/route/routes") - .unwrap() - .as_sequence_mut() - .unwrap() - }); - - // 3️⃣ Add/update the single receiver let name = self.receiver.name(); - - - // Receiver let receiver = self.receiver.build_receiver()?; + let route = self.receiver.build_route().unwrap(); - if let Some(idx) = receivers_seq - .iter() - .position(|r| r.get("name").and_then(|n| n.as_str()) == Some(&name)) - { - receivers_seq[idx] = receiver; - } else { - receivers_seq.push(receiver); + if am_config.get("receivers").is_none() { + am_config["receivers"] = serde_yaml::Value::Sequence(vec![]); + } + if am_config.get("route").is_none() { + am_config["route"] = serde_yaml::Value::Mapping(serde_yaml::Mapping::new()); + } + if am_config["route"].get("routes").is_none() { + am_config["route"]["routes"] = serde_yaml::Value::Sequence(vec![]); } - // Route - // - let route = self.receiver.build_route(); - - if let Some(idx) = route_seq - .iter() - .position(|r| r.get("receiver").and_then(|n| n.as_str()) == Some(name)) { - route_seq[idx] = route; - } else { - route_seq.push(route); + let receivers_seq = am_config["receivers"].as_sequence_mut().unwrap(); + if let Some(idx) = receivers_seq + .iter() + .position(|r| r.get("name").and_then(|n| n.as_str()) == Some(&name)) + { + receivers_seq[idx] = receiver; + } else { + receivers_seq.push(receiver); + } } + { + let route_seq = am_config["route"]["routes"].as_sequence_mut().unwrap(); + if let Some(idx) = route_seq + .iter() + .position(|r| r.get("receiver").and_then(|n| n.as_str()) == Some(&name)) + { + route_seq[idx] = route; + } else { + route_seq.push(route); + } + } - // 5️⃣ Save updated alertmanager.yaml - let yaml_str = serde_yaml::to_string(&am_config)?; + let yaml_str = + serde_yaml::to_string(&am_config).map_err(|e| InterpretError::new(e.to_string()))?; let mut yaml_b64 = String::new(); BASE64_STANDARD.encode_string(yaml_str, &mut yaml_b64); data.insert( diff --git a/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs b/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs index a30dfb69..c4ff6cb6 100644 --- a/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs +++ b/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs @@ -5,7 +5,7 @@ use crate::{interpret::Interpret, score::Score, topology::Topology}; #[derive(Debug, Clone, Serialize)] pub struct OpenshiftScrapeTargetScore {} -impl Score for OpenshiftScrapeTargetScore{ +impl Score for OpenshiftScrapeTargetScore { fn name(&self) -> String { "OpenshiftAlertingRuleScore".to_string() } diff --git a/harmony/src/modules/monitoring/prometheus/prometheus.rs b/harmony/src/modules/monitoring/prometheus/prometheus.rs index 2fe0d06c..23d74b1d 100644 --- a/harmony/src/modules/monitoring/prometheus/prometheus.rs +++ b/harmony/src/modules/monitoring/prometheus/prometheus.rs @@ -166,12 +166,6 @@ impl Serialize for Box> { } } -impl Clone for Box> { - fn clone(&self) -> Self { - self.clone_box() - } -} - #[async_trait] pub trait PrometheusRule: Send + Sync + std::fmt::Debug { fn name(&self) -> String; @@ -186,9 +180,3 @@ impl Serialize for Box> { todo!() } } - -impl Clone for Box> { - fn clone(&self) -> Self { - self.clone_box() - } -} diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/mod.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/mod.rs new file mode 100644 index 00000000..218f1316 --- /dev/null +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/mod.rs @@ -0,0 +1,11 @@ + +pub mod rhob_alertmanager_config; +pub mod rhob_alertmanagers; +pub mod rhob_cluster_observability_operator; +pub mod rhob_default_rules; +pub mod rhob_monitoring_stack; +pub mod rhob_prometheus_rules; +pub mod rhob_prometheuses; +pub mod rhob_role; +pub mod rhob_service_monitor; + diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_alertmanager_config.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs similarity index 86% rename from harmony/src/modules/monitoring/kube_prometheus/crd/rhob_alertmanager_config.rs rename to harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs index a53b24e3..c6b0a775 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs @@ -9,13 +9,14 @@ use crate::topology::{ oberservability::monitoring::{AlertReceiver, AlertSender}, }; -#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema, Default)] #[kube( group = "monitoring.rhobs", version = "v1alpha1", kind = "AlertmanagerConfig", plural = "alertmanagerconfigs", - namespaced + namespaced, + derive = "Default", )] pub struct AlertmanagerConfigSpec { #[serde(flatten)] @@ -34,12 +35,6 @@ impl AlertSender for RHOBObservability { } } -impl Clone for Box> { - fn clone(&self) -> Self { - self.clone_box() - } -} - impl Serialize for Box> { fn serialize(&self, _serializer: S) -> Result where diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_alertmanagers.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanagers.rs similarity index 93% rename from harmony/src/modules/monitoring/kube_prometheus/crd/rhob_alertmanagers.rs rename to harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanagers.rs index 44354673..a687858f 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_alertmanagers.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanagers.rs @@ -2,7 +2,8 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use super::crd_prometheuses::LabelSelector; +use crate::modules::monitoring::red_hat_cluster_observability::crd::rhob_prometheuses::LabelSelector; + /// Rust CRD for `Alertmanager` from Prometheus Operator #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_cluster_observability_operator.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_cluster_observability_operator.rs similarity index 100% rename from harmony/src/modules/monitoring/kube_prometheus/crd/rhob_cluster_observability_operator.rs rename to harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_cluster_observability_operator.rs diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_default_rules.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_default_rules.rs similarity index 88% rename from harmony/src/modules/monitoring/kube_prometheus/crd/rhob_default_rules.rs rename to harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_default_rules.rs index 459bd3fb..8cb481bd 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_default_rules.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_default_rules.rs @@ -1,11 +1,10 @@ use crate::modules::{ - monitoring::kube_prometheus::crd::rhob_prometheus_rules::Rule, - prometheus::alerts::k8s::{ + monitoring::red_hat_cluster_observability::crd::rhob_prometheus_rules::Rule, prometheus::alerts::k8s::{ deployment::alert_deployment_unavailable, pod::{alert_container_restarting, alert_pod_not_ready, pod_failed}, pvc::high_pvc_fill_rate_over_two_days, service::alert_service_down, - }, + } }; pub fn build_default_application_rules() -> Vec { diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_monitoring_stack.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_monitoring_stack.rs similarity index 89% rename from harmony/src/modules/monitoring/kube_prometheus/crd/rhob_monitoring_stack.rs rename to harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_monitoring_stack.rs index be9ccc0f..43d640ce 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_monitoring_stack.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_monitoring_stack.rs @@ -2,7 +2,8 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::modules::monitoring::kube_prometheus::crd::rhob_prometheuses::LabelSelector; +use crate::modules::monitoring::red_hat_cluster_observability::crd::rhob_prometheuses::LabelSelector; + /// MonitoringStack CRD for monitoring.rhobs/v1alpha1 #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] @@ -11,7 +12,8 @@ use crate::modules::monitoring::kube_prometheus::crd::rhob_prometheuses::LabelSe version = "v1alpha1", kind = "MonitoringStack", plural = "monitoringstacks", - namespaced + namespaced, + derive = "Default" )] #[serde(rename_all = "camelCase")] pub struct MonitoringStackSpec { diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_prometheus_rules.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheus_rules.rs similarity index 71% rename from harmony/src/modules/monitoring/kube_prometheus/crd/rhob_prometheus_rules.rs rename to harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheus_rules.rs index e2b5b608..fef14e52 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_prometheus_rules.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheus_rules.rs @@ -4,7 +4,6 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule; #[derive(CustomResource, Debug, Serialize, Deserialize, Clone, JsonSchema)] #[kube( @@ -44,14 +43,14 @@ pub struct Rule { pub annotations: Option>, } -impl From for Rule { - fn from(value: PrometheusAlertRule) -> Self { - Rule { - alert: Some(value.alert), - expr: Some(value.expr), - for_: value.r#for, - labels: Some(value.labels.into_iter().collect::>()), - annotations: Some(value.annotations.into_iter().collect::>()), - } - } -} +// impl From for Rule { +// fn from(value: PrometheusAlertRule) -> Self { +// Rule { +// alert: Some(value.alert), +// expr: Some(value.expr), +// for_: value.r#for, +// labels: Some(value.labels.into_iter().collect::>()), +// annotations: Some(value.annotations.into_iter().collect::>()), +// } +// } +// } diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_prometheuses.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheuses.rs similarity index 96% rename from harmony/src/modules/monitoring/kube_prometheus/crd/rhob_prometheuses.rs rename to harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheuses.rs index 18d3f57f..d368a728 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_prometheuses.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheuses.rs @@ -4,7 +4,6 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::modules::monitoring::kube_prometheus::types::Operator; #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] #[kube( @@ -93,6 +92,14 @@ pub struct LabelSelectorRequirement { pub values: Vec, } +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub enum Operator { + In, + NotIn, + Exists, + DoesNotExist, +} + impl Default for PrometheusSpec { fn default() -> Self { PrometheusSpec { diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_role.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_role.rs similarity index 100% rename from harmony/src/modules/monitoring/kube_prometheus/crd/rhob_role.rs rename to harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_role.rs diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/rhob_service_monitor.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_service_monitor.rs similarity index 100% rename from harmony/src/modules/monitoring/kube_prometheus/crd/rhob_service_monitor.rs rename to harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_service_monitor.rs diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs new file mode 100644 index 00000000..ed413d0f --- /dev/null +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs @@ -0,0 +1,63 @@ +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::monitoring::red_hat_cluster_observability::crd::rhob_prometheuses::LabelSelector, + score::Score, + topology::{ + Topology, + oberservability::monitoring::{ + AlertReceiver, AlertRule, AlertSender, AlertingInterpret, Observability, ScrapeTarget + }, + }, +}; + +pub mod crd; +pub mod redhat_cluster_observability; +pub mod score_alert_receiver; +pub mod score_coo_monitoring_stack; +pub mod score_redhat_cluster_observability_operator; +pub mod trait_redhat_cluster_observability; + +/// RedHat Cluster Observability Operator allows fine grained control of monitoring stack +/// solutions. This can be run in parallel with the default openshift-monitoring stack. +/// https://www.redhat.com/en/blog/introducing-cluster-observability-operator +#[derive(Debug, Clone, Serialize)] +pub struct RedHatClusterObservability { + ///namespace where the MonitoringStack CRD will be deployed + pub namespace: String, + /// Labels used to match resources the stack will monitor + pub resource_selector: Option, +} + +impl AlertSender for RedHatClusterObservability { + fn name(&self) -> String { + "RedHatClusterObs".to_string() + } +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs new file mode 100644 index 00000000..4aa9e12b --- /dev/null +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs @@ -0,0 +1,36 @@ +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::monitoring::red_hat_cluster_observability::RedHatClusterObservability, + score::Score, + topology::{ + Topology, + oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget}, + }, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct RedHatClusterObservabilityScore { + pub sender: RedHatClusterObservability, + pub receivers: Vec>>, + pub rules: Vec>>, + pub scrape_targets: Option>>>, +} + +impl> Score + for RedHatClusterObservabilityScore +{ + fn name(&self) -> String { + "RedHatClusterObservabilityScore".to_string() + } + + fn create_interpret(&self) -> Box> { + Box::new(AlertingInterpret { + sender: self.sender.clone(), + receivers: self.receivers.clone(), + rules: self.rules.clone(), + scrape_targets: self.scrape_targets.clone(), + }) + } +} diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/score_alert_receiver.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/score_alert_receiver.rs new file mode 100644 index 00000000..51da5179 --- /dev/null +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/score_alert_receiver.rs @@ -0,0 +1,100 @@ +use std::collections::BTreeMap; + +use async_trait::async_trait; +use harmony_types::id::Id; +use k8s_openapi::api::core::v1::Secret; +use kube::api::ObjectMeta; +use serde::Serialize; + +use crate::{ + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + modules::{ + k8s::resource::K8sResourceScore, + monitoring::red_hat_cluster_observability::{ + RedHatClusterObservability, + crd::rhob_alertmanager_config::{AlertmanagerConfig, AlertmanagerConfigSpec}, + }, + }, + score::Score, + topology::{K8sclient, Topology, oberservability::monitoring::AlertReceiver}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct RedHatClusterObservabilityReceiverScore { + pub sender: RedHatClusterObservability, + pub receiver: Box>, +} + +impl Score for RedHatClusterObservabilityReceiverScore { + fn name(&self) -> String { + "RedHatClusterObsReceiverScore".to_string() + } + + fn create_interpret(&self) -> Box> { + Box::new(RedHatClusterObsReceiverInterpret { + sender: self.sender.clone(), + receiver: self.receiver.clone(), + }) + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct RedHatClusterObsReceiverInterpret { + sender: RedHatClusterObservability, + pub receiver: Box>, +} + +#[async_trait] +impl Interpret for RedHatClusterObsReceiverInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + let name = self.receiver.name(); + let route = self.receiver.build_route()?; + let receiver = self.receiver.build_receiver()?; + + let data = serde_json::json!({ + "route": route, + "receivers": [receiver] + }); + + let spec = AlertmanagerConfigSpec { data }; + + let metadata = ObjectMeta { + name: Some(name), + namespace: Some(self.sender.namespace.clone()), + labels: Some(std::collections::BTreeMap::from([( + "alertmanagerConfig".to_string(), + "enabled".to_string(), + )])), + ..ObjectMeta::default() + }; + + let alert_manager_config = AlertmanagerConfig { metadata, spec }; + + K8sResourceScore::single(alert_manager_config, Some(self.sender.namespace.clone())) + .create_interpret() + .execute(inventory, topology) + .await + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("RedHatClusterObsReceiverInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/score_coo_monitoring_stack.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/score_coo_monitoring_stack.rs new file mode 100644 index 00000000..45caaf53 --- /dev/null +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/score_coo_monitoring_stack.rs @@ -0,0 +1,45 @@ +use kube::api::ObjectMeta; +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::{ + k8s::resource::K8sResourceScore, + monitoring::red_hat_cluster_observability::crd::{ + rhob_monitoring_stack::{MonitoringStack, MonitoringStackSpec}, + rhob_prometheuses::LabelSelector, + }, + }, + score::Score, + topology::{K8sclient, Topology}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct RedHatClusterObservabilityMonitoringStackScore { + pub namespace: String, + pub resource_selector: Option, +} + +impl Score for RedHatClusterObservabilityMonitoringStackScore { + fn name(&self) -> String { + "RedHatClusterObservabilityMonitoringStackScore".to_string() + } + + fn create_interpret(&self) -> Box> { + let metadata = ObjectMeta { + name: Some(format!("{}-monitoring", self.namespace.clone()).into()), + namespace: Some(self.namespace.clone()), + labels: Some([("monitoring-stack".into(), "true".into())].into()), + ..Default::default() + }; + let spec = MonitoringStackSpec { + log_level: Some("debug".into()), + retention: Some("1d".into()), + resource_selector: self.resource_selector.clone(), + }; + + let monitoring_stack = MonitoringStack { metadata, spec }; + + K8sResourceScore::single(monitoring_stack, Some(self.namespace.clone())).create_interpret() + } +} diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/score_redhat_cluster_observability_operator.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/score_redhat_cluster_observability_operator.rs new file mode 100644 index 00000000..2c529e78 --- /dev/null +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/score_redhat_cluster_observability_operator.rs @@ -0,0 +1,113 @@ +use async_trait::async_trait; +use harmony_types::id::Id; +use kube::api::ObjectMeta; +use serde::Serialize; + +use crate::{ + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + modules::k8s::{ + apps::crd::{Subscription, SubscriptionSpec}, + resource::K8sResourceScore, + }, + score::Score, + topology::{K8sclient, Topology}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct RedHatClusterObservabilityOperatorScore { + pub namespace: String, + pub channel: String, + pub install_plan_approval: String, + pub source: String, + pub source_namespace: String, +} + +impl Default for RedHatClusterObservabilityOperatorScore { + fn default() -> Self { + Self { + namespace: "openshift-operators".to_string(), + channel: "stable".to_string(), + install_plan_approval: "Automatic".to_string(), + source: "community-operators".to_string(), + source_namespace: "openshift-marketplace".to_string(), + } + } +} + +impl RedHatClusterObservabilityOperatorScore { + pub fn new() -> Self { + Self { + ..Default::default() + } + } +} + +impl Score for RedHatClusterObservabilityOperatorScore { + fn name(&self) -> String { + "RedHatClusterObservabilityScore".to_string() + } + + fn create_interpret(&self) -> Box> { + Box::new(RedHatClusterObservabilityOperatorInterpret { + score: self.clone(), + }) + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct RedHatClusterObservabilityOperatorInterpret { + score: RedHatClusterObservabilityOperatorScore, +} + +#[async_trait] +impl Interpret for RedHatClusterObservabilityOperatorInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + let metadata = ObjectMeta { + name: Some("observabiliy-operator".to_string()), + namespace: Some(self.score.namespace.clone()), + ..ObjectMeta::default() + }; + + let spec = SubscriptionSpec { + channel: Some("stable".to_string()), + config: None, + install_plan_approval: Some(self.score.install_plan_approval.clone()), + name: "observability-operator".to_string(), + source: self.score.source.clone(), + source_namespace: self.score.source_namespace.clone(), + starting_csv: None, + }; + + let subscription = Subscription { metadata, spec }; + K8sResourceScore::single(subscription, Some(self.score.namespace.clone())) + .create_interpret() + .execute(inventory, topology) + .await?; + + Ok(Outcome::success( + "Installed RedhatClusterObservabilityOperator".to_string(), + )) + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("RedHatClusterObservabilityInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/trait_redhat_cluster_observability.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/trait_redhat_cluster_observability.rs new file mode 100644 index 00000000..087ed6dc --- /dev/null +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/trait_redhat_cluster_observability.rs @@ -0,0 +1,114 @@ +use crate::{ + modules::monitoring::red_hat_cluster_observability::{ + score_alert_receiver::RedHatClusterObservabilityReceiverScore, + score_coo_monitoring_stack::RedHatClusterObservabilityMonitoringStackScore, + }, + score::Score, +}; +use async_trait::async_trait; +use log::info; + +use crate::{ + inventory::Inventory, + modules::monitoring::red_hat_cluster_observability::{ + RedHatClusterObservability, + score_redhat_cluster_observability_operator::RedHatClusterObservabilityOperatorScore, + }, + topology::{ + K8sAnywhereTopology, PreparationError, PreparationOutcome, + oberservability::monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, + }, +}; + +#[async_trait] +impl Observability for K8sAnywhereTopology { + async fn install_alert_sender( + &self, + sender: &RedHatClusterObservability, + inventory: &Inventory, + ) -> Result { + info!("Verifying Redhat Cluster Observability Operator"); + + let coo_score = RedHatClusterObservabilityOperatorScore::default(); + + coo_score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(e.to_string()))?; + + info!( + "Installing Cluster Observability Operator Monitoring Stack in ns {}", + sender.namespace.clone() + ); + + let coo_monitoring_stack_score = RedHatClusterObservabilityMonitoringStackScore { + namespace: sender.namespace.clone(), + resource_selector: sender.resource_selector.clone(), + }; + + coo_monitoring_stack_score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(e.to_string()))?; + + Ok(PreparationOutcome::Success { + details: "Successfully installed RedHatClusterObservability Operator".to_string(), + }) + } + + async fn install_receivers( + &self, + sender: &RedHatClusterObservability, + inventory: &Inventory, + receivers: Option>>>, + ) -> Result { + if let Some(receivers) = receivers { + for receiver in receivers { + info!("Installing receiver {}", receiver.name()); + let receiver_score = RedHatClusterObservabilityReceiverScore { + receiver, + sender: sender.clone(), + }; + receiver_score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(e.to_string()))?; + } + Ok(PreparationOutcome::Success { + details: "Successfully installed receivers for OpenshiftClusterMonitoring" + .to_string(), + }) + } else { + Ok(PreparationOutcome::Noop) + } + } + + async fn install_rules( + &self, + sender: &RedHatClusterObservability, + inventory: &Inventory, + rules: Option>>>, + ) -> Result { + todo!() + } + + async fn add_scrape_targets( + &self, + sender: &RedHatClusterObservability, + inventory: &Inventory, + scrape_targets: Option>>>, + ) -> Result { + todo!() + } + + async fn ensure_monitoring_installed( + &self, + sender: &RedHatClusterObservability, + inventory: &Inventory, + ) -> Result { + todo!() + } +} diff --git a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs index e382cfdc..f18d342c 100644 --- a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs +++ b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs @@ -22,7 +22,7 @@ use crate::modules::monitoring::kube_prometheus::crd::grafana_default_dashboard: use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{ ServiceMonitor, ServiceMonitorSpec, }; -use crate::topology::oberservability::monitoring::AlertReceiver; +use crate::topology::oberservability::monitoring::{AlertReceiver, Observability}; use crate::topology::{K8sclient, Topology, k8s::K8sClient}; use crate::{ data::Version, @@ -40,8 +40,6 @@ use crate::{ }; use harmony_types::id::Id; -use super::prometheus::Monitor; - #[derive(Clone, Debug, Serialize)] pub struct K8sPrometheusCRDAlertingScore { pub sender: CRDPrometheus, @@ -50,9 +48,7 @@ pub struct K8sPrometheusCRDAlertingScore { pub prometheus_rules: Vec, } -impl> Score - for K8sPrometheusCRDAlertingScore -{ +impl> Score for K8sPrometheusCRDAlertingScore { fn create_interpret(&self) -> Box> { Box::new(K8sPrometheusCRDAlertingInterpret { sender: self.sender.clone(), @@ -76,7 +72,7 @@ pub struct K8sPrometheusCRDAlertingInterpret { } #[async_trait] -impl> Interpret +impl> Interpret for K8sPrometheusCRDAlertingInterpret { async fn execute( @@ -567,7 +563,7 @@ impl K8sPrometheusCRDAlertingInterpret { receivers: &Vec>>, ) -> Result { for receiver in receivers.iter() { - receiver.build_route(sender).await.map_err(|err| { + receiver.build_route().map_err(|err| { InterpretError::new(format!("failed to install receiver: {}", err)) })?; } diff --git a/harmony/src/modules/prometheus/mod.rs b/harmony/src/modules/prometheus/mod.rs index c4f25ba2..519eb50c 100644 --- a/harmony/src/modules/prometheus/mod.rs +++ b/harmony/src/modules/prometheus/mod.rs @@ -2,4 +2,4 @@ pub mod alerts; pub mod k8s_prometheus_alerting_score; #[allow(clippy::module_inception)] pub mod prometheus; -pub mod rhob_alerting_score; +// pub mod rhob_alerting_score; diff --git a/harmony/src/modules/prometheus/rhob_alerting_score.rs b/harmony/src/modules/prometheus/rhob_alerting_score.rs index 5522e001..4cc9dbf5 100644 --- a/harmony/src/modules/prometheus/rhob_alerting_score.rs +++ b/harmony/src/modules/prometheus/rhob_alerting_score.rs @@ -29,7 +29,7 @@ use crate::modules::monitoring::kube_prometheus::crd::rhob_service_monitor::{ }; use crate::score::Score; use crate::topology::ingress::Ingress; -use crate::topology::oberservability::monitoring::AlertReceiver; +use crate::topology::oberservability::monitoring::{AlertReceiver, Observability}; use crate::topology::{K8sclient, Topology, k8s::K8sClient}; use crate::{ data::Version, @@ -38,8 +38,6 @@ use crate::{ }; use harmony_types::id::Id; -use super::prometheus::Monitor; - #[derive(Clone, Debug, Serialize)] pub struct RHOBAlertingScore { pub sender: RHOBObservability, @@ -48,7 +46,7 @@ pub struct RHOBAlertingScore { pub prometheus_rules: Vec, } -impl> Score +impl> Score for RHOBAlertingScore { fn create_interpret(&self) -> Box> { @@ -74,7 +72,7 @@ pub struct RHOBAlertingInterpret { } #[async_trait] -impl> Interpret +impl> Interpret for RHOBAlertingInterpret { async fn execute( @@ -518,7 +516,7 @@ impl RHOBAlertingInterpret { receivers: &Vec>>, ) -> Result { for receiver in receivers.iter() { - receiver.build_route(sender).await.map_err(|err| { + receiver.build_route().map_err(|err| { InterpretError::new(format!("failed to install receiver: {}", err)) })?; } -- 2.39.5 From 554c94f5a96c79dd4801fc56bc820aad28b4d2a8 Mon Sep 17 00:00:00 2001 From: wjro Date: Mon, 23 Feb 2026 14:48:05 -0500 Subject: [PATCH 04/15] wip: compiles --- examples/okd_cluster_alerts/src/main.rs | 15 +++-- .../topology/k8s_anywhere/k8s_anywhere.rs | 19 ++++-- .../topology/k8s_anywhere/kube_prometheus.rs | 56 ++++++++++++++++ .../src/domain/topology/k8s_anywhere/mod.rs | 2 + .../topology/k8s_anywhere/prometheus.rs | 56 ++++++++++++++++ .../topology/oberservability/monitoring.rs | 6 +- .../alert_channel/discord_alert_channel.rs | 55 +++++++++------ .../alert_channel/webhook_receiver.rs | 22 +++++- .../alert_rule/prometheus_alert_rule.rs | 67 +++++++++---------- .../application_monitoring_score.rs | 4 +- .../rhobs_application_monitoring_score.rs | 1 - .../monitoring/grafana/crd/crd_grafana.rs | 1 - .../monitoring/grafana/crd/rhob_grafana.rs | 2 +- harmony/src/modules/monitoring/grafana/mod.rs | 2 +- .../crd/crd_alertmanager_config.rs | 4 +- .../crd/crd_prometheus_rules.rs | 23 ++++--- .../helm_prometheus_alert_score.rs | 4 +- .../modules/monitoring/kube_prometheus/mod.rs | 4 +- .../monitoring/kube_prometheus/prometheus.rs | 1 + .../monitoring/kube_prometheus/types.rs | 1 + harmony/src/modules/monitoring/mod.rs | 6 +- harmony/src/modules/monitoring/okd/mod.rs | 7 +- .../score_openshift_cluster_alert_score.rs | 42 ++++++++++++ .../monitoring/prometheus/prometheus.rs | 5 +- .../red_hat_cluster_observability/crd/mod.rs | 2 - .../crd/rhob_alertmanager_config.rs | 2 +- .../crd/rhob_alertmanagers.rs | 1 - .../crd/rhob_default_rules.rs | 5 +- .../crd/rhob_monitoring_stack.rs | 1 - .../crd/rhob_prometheus_rules.rs | 23 ++++--- .../crd/rhob_prometheuses.rs | 1 - .../red_hat_cluster_observability/mod.rs | 2 +- .../redhat_cluster_observability.rs | 4 +- .../k8s_prometheus_alerting_score.rs | 15 +++-- harmony/src/modules/prometheus/mod.rs | 4 +- harmony_agent/deploy/src/main.rs | 16 ++--- 36 files changed, 346 insertions(+), 135 deletions(-) create mode 100644 harmony/src/domain/topology/k8s_anywhere/kube_prometheus.rs create mode 100644 harmony/src/domain/topology/k8s_anywhere/prometheus.rs create mode 100644 harmony/src/modules/monitoring/okd/score_openshift_cluster_alert_score.rs diff --git a/examples/okd_cluster_alerts/src/main.rs b/examples/okd_cluster_alerts/src/main.rs index 5f5d8dc2..139e052d 100644 --- a/examples/okd_cluster_alerts/src/main.rs +++ b/examples/okd_cluster_alerts/src/main.rs @@ -3,10 +3,10 @@ use std::collections::HashMap; use harmony::{ inventory::Inventory, modules::monitoring::{ - alert_channel::discord_alert_channel::DiscordWebhook, - okd::cluster_monitoring::OpenshiftClusterAlertScore, + alert_channel::discord_alert_channel::DiscordReceiver, + okd::score_openshift_cluster_alert_score::OpenshiftClusterAlertScore, }, - topology::K8sAnywhereTopology, + topology::{K8sAnywhereTopology, oberservability::monitoring::AlertRoute}, }; use harmony_macros::hurl; use harmony_types::k8s_name::K8sName; @@ -25,11 +25,16 @@ async fn main() { Inventory::autoload(), K8sAnywhereTopology::from_env(), vec![Box::new(OpenshiftClusterAlertScore { - receivers: vec![Box::new(DiscordWebhook { + receivers: vec![Box::new(DiscordReceiver { name: "wills-discord-webhook-example".to_string(), url: hurl!("https://something.io"), - selectors: selectors, + route: AlertRoute { + ..AlertRoute::default("wills-discord-webhook-example".to_string()) + }, })], + sender: harmony::modules::monitoring::okd::OpenshiftClusterAlertSender, + rules: vec![], + scrape_targets: None, })], None, ) diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index 6ba26187..f24ab6dd 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -30,19 +30,24 @@ use crate::{ k3d::K3DInstallationScore, k8s::ingress::{K8sIngressScore, PathType}, monitoring::{ - grafana::{crd::crd_grafana::{Grafana as GrafanaCRD,GrafanaCom, GrafanaDashboard, GrafanaDashboardDatasource, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig, GrafanaDatasourceJsonData, GrafanaDatasourceSecureJsonData, GrafanaDatasourceSpec, GrafanaSpec}, grafana::Grafana, helm::helm_grafana::grafana_helm_chart_score}, + grafana::{ + crd::crd_grafana::{ + Grafana as GrafanaCRD, GrafanaCom, GrafanaDashboard, + GrafanaDashboardDatasource, GrafanaDashboardSpec, GrafanaDatasource, + GrafanaDatasourceConfig, GrafanaDatasourceJsonData, + GrafanaDatasourceSecureJsonData, GrafanaDatasourceSpec, GrafanaSpec, + }, + grafana::Grafana, + helm::helm_grafana::grafana_helm_chart_score, + }, kube_prometheus::crd::{ - crd_alertmanager_config::CRDPrometheus, - crd_prometheuses::LabelSelector, + crd_alertmanager_config::CRDPrometheus, crd_prometheuses::LabelSelector, prometheus_operator::prometheus_operator_helm_chart_score, service_monitor::ServiceMonitor, }, }, okd::{crd::ingresses_config::Ingress as IngressResource, route::OKDTlsPassthroughScore}, - prometheus::{ - k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore, - rhob_alerting_score::RHOBAlertingScore, - }, + prometheus::k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore, }, score::Score, topology::{TlsRoute, TlsRouter, ingress::Ingress, oberservability::monitoring::Observability}, diff --git a/harmony/src/domain/topology/k8s_anywhere/kube_prometheus.rs b/harmony/src/domain/topology/k8s_anywhere/kube_prometheus.rs new file mode 100644 index 00000000..4aab9bd7 --- /dev/null +++ b/harmony/src/domain/topology/k8s_anywhere/kube_prometheus.rs @@ -0,0 +1,56 @@ +use async_trait::async_trait; + +use crate::{ + inventory::Inventory, + modules::monitoring::kube_prometheus::prometheus::KubePrometheus, + topology::{ + K8sAnywhereTopology, PreparationError, PreparationOutcome, + oberservability::monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, + }, +}; + +#[async_trait] +impl Observability for K8sAnywhereTopology { + async fn install_alert_sender( + &self, + sender: &KubePrometheus, + inventory: &Inventory, + ) -> Result { + todo!() + } + + async fn install_receivers( + &self, + sender: &KubePrometheus, + inventory: &Inventory, + receivers: Option>>>, + ) -> Result { + todo!() + } + + async fn install_rules( + &self, + sender: &KubePrometheus, + inventory: &Inventory, + rules: Option>>>, + ) -> Result { + todo!() + } + + async fn add_scrape_targets( + &self, + sender: &KubePrometheus, + inventory: &Inventory, + scrape_targets: Option>>>, + ) -> Result { + todo!() + } + + async fn ensure_monitoring_installed( + &self, + sender: &KubePrometheus, + inventory: &Inventory, + ) -> Result { + todo!() + } +} diff --git a/harmony/src/domain/topology/k8s_anywhere/mod.rs b/harmony/src/domain/topology/k8s_anywhere/mod.rs index 144f3c1c..5aa2cf80 100644 --- a/harmony/src/domain/topology/k8s_anywhere/mod.rs +++ b/harmony/src/domain/topology/k8s_anywhere/mod.rs @@ -1,5 +1,7 @@ mod k8s_anywhere; +pub mod kube_prometheus; pub mod nats; pub mod openshift_monitoring; mod postgres; +mod prometheus; pub use k8s_anywhere::*; diff --git a/harmony/src/domain/topology/k8s_anywhere/prometheus.rs b/harmony/src/domain/topology/k8s_anywhere/prometheus.rs new file mode 100644 index 00000000..a630feef --- /dev/null +++ b/harmony/src/domain/topology/k8s_anywhere/prometheus.rs @@ -0,0 +1,56 @@ +use async_trait::async_trait; + +use crate::{ + inventory::Inventory, + modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus, + topology::{ + K8sAnywhereTopology, PreparationError, PreparationOutcome, + oberservability::monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, + }, +}; + +#[async_trait] +impl Observability for K8sAnywhereTopology { + async fn install_alert_sender( + &self, + sender: &CRDPrometheus, + inventory: &Inventory, + ) -> Result { + todo!() + } + + async fn install_receivers( + &self, + sender: &CRDPrometheus, + inventory: &Inventory, + receivers: Option>>>, + ) -> Result { + todo!() + } + + async fn install_rules( + &self, + sender: &CRDPrometheus, + inventory: &Inventory, + rules: Option>>>, + ) -> Result { + todo!() + } + + async fn add_scrape_targets( + &self, + sender: &CRDPrometheus, + inventory: &Inventory, + scrape_targets: Option>>>, + ) -> Result { + todo!() + } + + async fn ensure_monitoring_installed( + &self, + sender: &CRDPrometheus, + inventory: &Inventory, + ) -> Result { + todo!() + } +} diff --git a/harmony/src/domain/topology/oberservability/monitoring.rs b/harmony/src/domain/topology/oberservability/monitoring.rs index 45a4a4a7..75933b74 100644 --- a/harmony/src/domain/topology/oberservability/monitoring.rs +++ b/harmony/src/domain/topology/oberservability/monitoring.rs @@ -47,7 +47,7 @@ pub struct AlertRoute { } impl AlertRoute { - fn default(name: String) -> Self { + pub fn default(name: String) -> Self { Self { receiver: name, matchers: vec![], @@ -75,7 +75,9 @@ pub enum MatchOp { impl Serialize for MatchOp { fn serialize(&self, serializer: S) -> Result - where S: serde::Serializer { + where + S: serde::Serializer, + { let op = match self { MatchOp::Eq => "=", MatchOp::NotEq => "!=", diff --git a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs index 5fa3570d..1d0ebbb4 100644 --- a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs +++ b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs @@ -15,6 +15,7 @@ use crate::infra::kube::kube_resource_to_dynamic; use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{ AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus, }; +use crate::modules::monitoring::kube_prometheus::prometheus::KubePrometheus; use crate::modules::monitoring::okd::OpenshiftClusterAlertSender; use crate::modules::monitoring::red_hat_cluster_observability::RedHatClusterObservability; use crate::topology::oberservability::monitoring::{ @@ -22,8 +23,6 @@ use crate::topology::oberservability::monitoring::{ }; use crate::{ interpret::{InterpretError, Outcome}, - modules::monitoring::{ - }, topology::oberservability::monitoring::AlertReceiver, }; use harmony_types::net::Url; @@ -136,11 +135,9 @@ impl AlertReceiver for DiscordReceiver { impl AlertReceiver for DiscordReceiver { fn build_route(&self) -> Result { serde_yaml::to_value(&self.route).map_err(|e| InterpretError::new(e.to_string())) - } - fn build_receiver(&self) -> Result { - + fn build_receiver(&self) -> Result { //FIXME this secret needs to be applied so that the discord Configs for RedHatCO //CRD AlertmanagerConfigs can access the URL let secret_name = format!("{}-secret", self.name.clone()); @@ -159,21 +156,20 @@ impl AlertReceiver for DiscordReceiver { ..Default::default() }; - let receiver_config = json!({ - "name": self.name, - "discordConfigs": [ - { - "apiURL": { - "key": "webhook-url", - "name": format!("{}-secret", self.name) - }, - "title": "{{ template \"discord.default.title\" . }}", - "message": "{{ template \"discord.default.message\" . }}" - } - ] - }); - serde_yaml::to_value(receiver_config) - .map_err(|e| InterpretError::new(e.to_string())) + let receiver_config = json!({ + "name": self.name, + "discordConfigs": [ + { + "apiURL": { + "key": "webhook-url", + "name": format!("{}-secret", self.name) + }, + "title": "{{ template \"discord.default.title\" . }}", + "message": "{{ template \"discord.default.message\" . }}" + } + ] + }); + serde_yaml::to_value(receiver_config).map_err(|e| InterpretError::new(e.to_string())) } fn name(&self) -> String { @@ -184,6 +180,25 @@ impl AlertReceiver for DiscordReceiver { Box::new(self.clone()) } } + +impl AlertReceiver for DiscordWebhook { + fn build_route(&self) -> Result { + todo!() + } + + fn build_receiver(&self) -> Result { + todo!() + } + + fn name(&self) -> String { + todo!() + } + + fn clone_box(&self) -> Box> { + todo!() + } +} + // // #[async_trait] // impl AlertReceiver for DiscordWebhook { diff --git a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs index 5e0602f8..518ccad0 100644 --- a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs +++ b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs @@ -10,7 +10,9 @@ use serde_yaml::{Mapping, Value}; use crate::{ interpret::{InterpretError, Outcome}, modules::monitoring::{ - okd::OpenshiftClusterAlertSender, red_hat_cluster_observability::RedHatClusterObservability, + kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus, + okd::OpenshiftClusterAlertSender, + red_hat_cluster_observability::RedHatClusterObservability, }, topology::oberservability::monitoring::{AlertManagerReceiver, AlertReceiver}, }; @@ -132,6 +134,24 @@ impl AlertReceiver for WebhookReceiver { } } +impl AlertReceiver for WebhookReceiver { + fn build_route(&self) -> Result { + todo!() + } + + fn build_receiver(&self) -> Result { + todo!() + } + + fn name(&self) -> String { + todo!() + } + + fn clone_box(&self) -> Box> { + todo!() + } +} + // #[async_trait] // impl AlertReceiver for WebhookReceiver { // fn as_alertmanager_receiver(&self) -> Result { diff --git a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs index e001c68a..e0a8dbb3 100644 --- a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs +++ b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs @@ -5,12 +5,9 @@ use serde::Serialize; use crate::{ interpret::{InterpretError, Outcome}, - modules::monitoring::{ - kube_prometheus::{ - prometheus::{KubePrometheus, KubePrometheusRule}, - types::{AlertGroup, AlertManagerAdditionalPromRules}, - }, - prometheus::prometheus::{Prometheus, PrometheusRule}, + modules::monitoring::kube_prometheus::{ + prometheus::{KubePrometheus, KubePrometheusRule}, + types::{AlertGroup, AlertManagerAdditionalPromRules}, }, topology::oberservability::monitoring::AlertRule, }; @@ -25,35 +22,35 @@ impl AlertRule for AlertManagerRuleGroup { } } -#[async_trait] -impl AlertRule for AlertManagerRuleGroup { - async fn install(&self, sender: &Prometheus) -> Result { - sender.install_rule(self).await - } - fn clone_box(&self) -> Box> { - Box::new(self.clone()) - } -} - -#[async_trait] -impl PrometheusRule for AlertManagerRuleGroup { - fn name(&self) -> String { - self.name.clone() - } - async fn configure_rule(&self) -> AlertManagerAdditionalPromRules { - let mut additional_prom_rules = BTreeMap::new(); - - additional_prom_rules.insert( - self.name.clone(), - AlertGroup { - groups: vec![self.clone()], - }, - ); - AlertManagerAdditionalPromRules { - rules: additional_prom_rules, - } - } -} +// #[async_trait] +// impl AlertRule for AlertManagerRuleGroup { +// async fn install(&self, sender: &Prometheus) -> Result { +// sender.install_rule(self).await +// } +// fn clone_box(&self) -> Box> { +// Box::new(self.clone()) +// } +// } +// +// #[async_trait] +// impl PrometheusRule for AlertManagerRuleGroup { +// fn name(&self) -> String { +// self.name.clone() +// } +// async fn configure_rule(&self) -> AlertManagerAdditionalPromRules { +// let mut additional_prom_rules = BTreeMap::new(); +// +// additional_prom_rules.insert( +// self.name.clone(), +// AlertGroup { +// groups: vec![self.clone()], +// }, +// ); +// AlertManagerAdditionalPromRules { +// rules: additional_prom_rules, +// } +// } +// } #[async_trait] impl KubePrometheusRule for AlertManagerRuleGroup { fn name(&self) -> String { diff --git a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs index b5b2aba8..b7a92e06 100644 --- a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs +++ b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs @@ -14,7 +14,9 @@ use crate::{ score::Score, topology::{ K8sclient, Topology, - oberservability::monitoring::{AlertReceiver, AlertingInterpret, Observability, ScrapeTarget}, + oberservability::monitoring::{ + AlertReceiver, AlertingInterpret, Observability, ScrapeTarget, + }, }, }; diff --git a/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs index 488eb8a9..50a25964 100644 --- a/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs +++ b/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs @@ -10,7 +10,6 @@ use crate::{ modules::{ application::Application, monitoring::kube_prometheus::crd::{ - crd_alertmanager_config::CRDPrometheus, rhob_alertmanager_config::RHOBObservability, }, }, score::Score, diff --git a/harmony/src/modules/monitoring/grafana/crd/crd_grafana.rs b/harmony/src/modules/monitoring/grafana/crd/crd_grafana.rs index e72aa0ec..0f02a844 100644 --- a/harmony/src/modules/monitoring/grafana/crd/crd_grafana.rs +++ b/harmony/src/modules/monitoring/grafana/crd/crd_grafana.rs @@ -6,7 +6,6 @@ use serde::{Deserialize, Serialize}; use crate::modules::monitoring::kube_prometheus::crd::crd_prometheuses::LabelSelector; - #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] #[kube( group = "grafana.integreatly.org", diff --git a/harmony/src/modules/monitoring/grafana/crd/rhob_grafana.rs b/harmony/src/modules/monitoring/grafana/crd/rhob_grafana.rs index 65efab9a..ce13f61d 100644 --- a/harmony/src/modules/monitoring/grafana/crd/rhob_grafana.rs +++ b/harmony/src/modules/monitoring/grafana/crd/rhob_grafana.rs @@ -4,7 +4,7 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::modules::monitoring::kube_prometheus::crd::rhob_prometheuses::LabelSelector; +use crate::modules::monitoring::red_hat_cluster_observability::crd::rhob_prometheuses::LabelSelector; #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] #[kube( diff --git a/harmony/src/modules/monitoring/grafana/mod.rs b/harmony/src/modules/monitoring/grafana/mod.rs index 552ed66c..08c5c666 100644 --- a/harmony/src/modules/monitoring/grafana/mod.rs +++ b/harmony/src/modules/monitoring/grafana/mod.rs @@ -1,3 +1,3 @@ +pub mod crd; pub mod grafana; pub mod helm; -pub mod crd; diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs index 61ad21fe..cfeef943 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs @@ -55,7 +55,9 @@ impl Serialize for Box> { } #[async_trait] -impl + Grafana> Installable for CRDPrometheus { +impl + Grafana> Installable + for CRDPrometheus +{ async fn configure(&self, inventory: &Inventory, topology: &T) -> Result<(), InterpretError> { topology.ensure_grafana_operator(inventory).await?; // topology.ensure_monitoring(self, inventory).await?; diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs index 3cb4d7ff..6d08456b 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs @@ -4,6 +4,7 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule; #[derive(CustomResource, Debug, Serialize, Deserialize, Clone, JsonSchema)] #[kube( @@ -43,14 +44,14 @@ pub struct Rule { pub annotations: Option>, } -// impl From for Rule { -// fn from(value: PrometheusAlertRule) -> Self { -// Rule { -// alert: Some(value.alert), -// expr: Some(value.expr), -// for_: value.r#for, -// labels: Some(value.labels.into_iter().collect::>()), -// annotations: Some(value.annotations.into_iter().collect::>()), -// } -// } -// } +impl From for Rule { + fn from(value: PrometheusAlertRule) -> Self { + Rule { + alert: Some(value.alert), + expr: Some(value.expr), + for_: value.r#for, + labels: Some(value.labels.into_iter().collect::>()), + annotations: Some(value.annotations.into_iter().collect::>()), + } + } +} diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs b/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs index fdc0ed2b..40415cf4 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs @@ -8,7 +8,7 @@ use crate::{ score::Score, topology::{ HelmCommand, Topology, - oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Monitor}, + oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability}, tenant::TenantManager, }, }; @@ -20,7 +20,7 @@ pub struct HelmPrometheusAlertingScore { pub service_monitors: Vec, } -impl> Score +impl> Score for HelmPrometheusAlertingScore { fn create_interpret(&self) -> Box> { diff --git a/harmony/src/modules/monitoring/kube_prometheus/mod.rs b/harmony/src/modules/monitoring/kube_prometheus/mod.rs index 1fac351a..122e9396 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/mod.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/mod.rs @@ -1,5 +1,5 @@ pub mod crd; pub mod helm; -// pub mod helm_prometheus_alert_score; -// pub mod prometheus; +pub mod helm_prometheus_alert_score; +pub mod prometheus; pub mod types; diff --git a/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs b/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs index 0e01b0f4..6533733d 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs @@ -7,6 +7,7 @@ use serde::Serialize; use crate::{ interpret::{InterpretError, Outcome}, inventory::Inventory, + modules::monitoring::alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, score, topology::{ HelmCommand, Topology, diff --git a/harmony/src/modules/monitoring/kube_prometheus/types.rs b/harmony/src/modules/monitoring/kube_prometheus/types.rs index 5e626bb3..abe5896a 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/types.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/types.rs @@ -5,6 +5,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_yaml::{Mapping, Sequence, Value}; +use crate::modules::monitoring::alert_rule::prometheus_alert_rule::AlertManagerRuleGroup; #[async_trait] pub trait AlertChannelConfig { diff --git a/harmony/src/modules/monitoring/mod.rs b/harmony/src/modules/monitoring/mod.rs index 99b412d7..a9ace9fc 100644 --- a/harmony/src/modules/monitoring/mod.rs +++ b/harmony/src/modules/monitoring/mod.rs @@ -1,10 +1,10 @@ pub mod alert_channel; -// pub mod alert_rule; +pub mod alert_rule; pub mod application_monitoring; pub mod grafana; pub mod kube_prometheus; pub mod ntfy; pub mod okd; -// pub mod prometheus; -pub mod scrape_target; +pub mod prometheus; pub mod red_hat_cluster_observability; +pub mod scrape_target; diff --git a/harmony/src/modules/monitoring/okd/mod.rs b/harmony/src/modules/monitoring/okd/mod.rs index c30067b8..e45947af 100644 --- a/harmony/src/modules/monitoring/okd/mod.rs +++ b/harmony/src/modules/monitoring/okd/mod.rs @@ -1,15 +1,18 @@ use serde::Serialize; -use crate::topology::oberservability::monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}; +use crate::topology::oberservability::monitoring::{ + AlertReceiver, AlertRule, AlertSender, ScrapeTarget, +}; pub mod score_cluster_monitoring; pub mod score_openshift_alert_rule; +pub mod score_openshift_cluster_alert_score; pub mod score_openshift_receiver; pub mod score_openshift_scrape_target; pub mod score_user_workload; pub mod score_verify_user_workload_monitoring; -#[derive(Debug)] +#[derive(Debug, Clone, Serialize)] pub struct OpenshiftClusterAlertSender; impl AlertSender for OpenshiftClusterAlertSender { diff --git a/harmony/src/modules/monitoring/okd/score_openshift_cluster_alert_score.rs b/harmony/src/modules/monitoring/okd/score_openshift_cluster_alert_score.rs new file mode 100644 index 00000000..608a6f6a --- /dev/null +++ b/harmony/src/modules/monitoring/okd/score_openshift_cluster_alert_score.rs @@ -0,0 +1,42 @@ +use async_trait::async_trait; +use harmony_types::id::Id; +use serde::Serialize; + +use crate::{ + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + modules::monitoring::okd::OpenshiftClusterAlertSender, + score::Score, + topology::{ + Topology, + oberservability::monitoring::{ + AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget, + }, + }, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct OpenshiftClusterAlertScore { + pub sender: OpenshiftClusterAlertSender, + pub receivers: Vec>>, + pub rules: Vec>>, + pub scrape_targets: Option>>>, +} + +impl> Score + for OpenshiftClusterAlertScore +{ + fn name(&self) -> String { + "OpenshiftClusterAlertScore".to_string() + } + + fn create_interpret(&self) -> Box> { + Box::new(AlertingInterpret { + sender: OpenshiftClusterAlertSender, + receivers: self.receivers.clone(), + rules: self.rules.clone(), + scrape_targets: self.scrape_targets.clone(), + }) + } +} diff --git a/harmony/src/modules/monitoring/prometheus/prometheus.rs b/harmony/src/modules/monitoring/prometheus/prometheus.rs index 23d74b1d..99a1e171 100644 --- a/harmony/src/modules/monitoring/prometheus/prometheus.rs +++ b/harmony/src/modules/monitoring/prometheus/prometheus.rs @@ -10,7 +10,10 @@ use crate::{ modules::monitoring::{ alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, grafana::helm::helm_grafana::grafana_helm_chart_score, - kube_prometheus::types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig}, + kube_prometheus::{ + prometheus::KubePrometheusRule, + types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig}, + }, }, score::Score, topology::{ diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/mod.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/mod.rs index 218f1316..ca68d7c3 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/mod.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/mod.rs @@ -1,4 +1,3 @@ - pub mod rhob_alertmanager_config; pub mod rhob_alertmanagers; pub mod rhob_cluster_observability_operator; @@ -8,4 +7,3 @@ pub mod rhob_prometheus_rules; pub mod rhob_prometheuses; pub mod rhob_role; pub mod rhob_service_monitor; - diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs index c6b0a775..e440b6ff 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs @@ -16,7 +16,7 @@ use crate::topology::{ kind = "AlertmanagerConfig", plural = "alertmanagerconfigs", namespaced, - derive = "Default", + derive = "Default" )] pub struct AlertmanagerConfigSpec { #[serde(flatten)] diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanagers.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanagers.rs index a687858f..22c3386a 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanagers.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanagers.rs @@ -4,7 +4,6 @@ use serde::{Deserialize, Serialize}; use crate::modules::monitoring::red_hat_cluster_observability::crd::rhob_prometheuses::LabelSelector; - /// Rust CRD for `Alertmanager` from Prometheus Operator #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] #[kube( diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_default_rules.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_default_rules.rs index 8cb481bd..e59938b7 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_default_rules.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_default_rules.rs @@ -1,10 +1,11 @@ use crate::modules::{ - monitoring::red_hat_cluster_observability::crd::rhob_prometheus_rules::Rule, prometheus::alerts::k8s::{ + monitoring::red_hat_cluster_observability::crd::rhob_prometheus_rules::Rule, + prometheus::alerts::k8s::{ deployment::alert_deployment_unavailable, pod::{alert_container_restarting, alert_pod_not_ready, pod_failed}, pvc::high_pvc_fill_rate_over_two_days, service::alert_service_down, - } + }, }; pub fn build_default_application_rules() -> Vec { diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_monitoring_stack.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_monitoring_stack.rs index 43d640ce..99e72460 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_monitoring_stack.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_monitoring_stack.rs @@ -4,7 +4,6 @@ use serde::{Deserialize, Serialize}; use crate::modules::monitoring::red_hat_cluster_observability::crd::rhob_prometheuses::LabelSelector; - /// MonitoringStack CRD for monitoring.rhobs/v1alpha1 #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] #[kube( diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheus_rules.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheus_rules.rs index fef14e52..e2b5b608 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheus_rules.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheus_rules.rs @@ -4,6 +4,7 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule; #[derive(CustomResource, Debug, Serialize, Deserialize, Clone, JsonSchema)] #[kube( @@ -43,14 +44,14 @@ pub struct Rule { pub annotations: Option>, } -// impl From for Rule { -// fn from(value: PrometheusAlertRule) -> Self { -// Rule { -// alert: Some(value.alert), -// expr: Some(value.expr), -// for_: value.r#for, -// labels: Some(value.labels.into_iter().collect::>()), -// annotations: Some(value.annotations.into_iter().collect::>()), -// } -// } -// } +impl From for Rule { + fn from(value: PrometheusAlertRule) -> Self { + Rule { + alert: Some(value.alert), + expr: Some(value.expr), + for_: value.r#for, + labels: Some(value.labels.into_iter().collect::>()), + annotations: Some(value.annotations.into_iter().collect::>()), + } + } +} diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheuses.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheuses.rs index d368a728..b93ecc76 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheuses.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_prometheuses.rs @@ -4,7 +4,6 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; - #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] #[kube( group = "monitoring.rhobs", diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs index ed413d0f..5286b020 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs @@ -7,7 +7,7 @@ use crate::{ topology::{ Topology, oberservability::monitoring::{ - AlertReceiver, AlertRule, AlertSender, AlertingInterpret, Observability, ScrapeTarget + AlertReceiver, AlertRule, AlertSender, AlertingInterpret, Observability, ScrapeTarget, }, }, }; diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs index 4aa9e12b..53de66de 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs @@ -6,7 +6,9 @@ use crate::{ score::Score, topology::{ Topology, - oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget}, + oberservability::monitoring::{ + AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget, + }, }, }; diff --git a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs index f18d342c..f3687821 100644 --- a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs +++ b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs @@ -8,17 +8,16 @@ use log::{debug, info}; use serde::Serialize; use std::process::Command; +use crate::modules::monitoring::grafana::crd::crd_grafana::{ + Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig, + GrafanaDatasourceJsonData, GrafanaDatasourceSpec, GrafanaSpec, +}; +use crate::modules::monitoring::grafana::crd::grafana_default_dashboard::build_default_dashboard; use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus; use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules; -use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{ - Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig, - GrafanaDatasourceJsonData, GrafanaDatasourceSpec, GrafanaSecretKeyRef, GrafanaSpec, - GrafanaValueFrom, GrafanaValueSource, -}; use crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::{ PrometheusRule, PrometheusRuleSpec, RuleGroup, }; -use crate::modules::monitoring::kube_prometheus::crd::grafana_default_dashboard::build_default_dashboard; use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{ ServiceMonitor, ServiceMonitorSpec, }; @@ -48,7 +47,9 @@ pub struct K8sPrometheusCRDAlertingScore { pub prometheus_rules: Vec, } -impl> Score for K8sPrometheusCRDAlertingScore { +impl> Score + for K8sPrometheusCRDAlertingScore +{ fn create_interpret(&self) -> Box> { Box::new(K8sPrometheusCRDAlertingInterpret { sender: self.sender.clone(), diff --git a/harmony/src/modules/prometheus/mod.rs b/harmony/src/modules/prometheus/mod.rs index 519eb50c..f0571e94 100644 --- a/harmony/src/modules/prometheus/mod.rs +++ b/harmony/src/modules/prometheus/mod.rs @@ -1,5 +1,5 @@ pub mod alerts; pub mod k8s_prometheus_alerting_score; -#[allow(clippy::module_inception)] -pub mod prometheus; +// #[allow(clippy::module_inception)] +// pub mod prometheus; // pub mod rhob_alerting_score; diff --git a/harmony_agent/deploy/src/main.rs b/harmony_agent/deploy/src/main.rs index 8baab66b..b12d82e9 100644 --- a/harmony_agent/deploy/src/main.rs +++ b/harmony_agent/deploy/src/main.rs @@ -40,14 +40,14 @@ async fn main() { Box::new(PackagingDeployment { application: application.clone(), }), - Box::new(Monitoring { - application: application.clone(), - alert_receiver: vec![Box::new(DiscordWebhook { - name: K8sName("test-discord".to_string()), - url: hurl!("https://discord.doesnt.exist.com"), - selectors: vec![], - })], - }), + // Box::new(Monitoring { + // application: application.clone(), + // alert_receiver: vec![Box::new(DiscordWebhook { + // name: K8sName("test-discord".to_string()), + // url: hurl!("https://discord.doesnt.exist.com"), + // selectors: vec![], + // })], + // }), ], application, }; -- 2.39.5 From 699822af7402e7db6db4aaae1c42926273b593bc Mon Sep 17 00:00:00 2001 From: wjro Date: Mon, 23 Feb 2026 15:03:55 -0500 Subject: [PATCH 05/15] chore: reorganized file location --- examples/okd_cluster_alerts/src/main.rs | 2 +- harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs | 4 ++-- harmony/src/domain/topology/k8s_anywhere/mod.rs | 4 +--- .../k8s_anywhere/{ => observability}/kube_prometheus.rs | 2 +- .../src/domain/topology/k8s_anywhere/observability/mod.rs | 4 ++++ .../{ => observability}/openshift_monitoring.rs | 7 ++++--- .../k8s_anywhere/{ => observability}/prometheus.rs | 2 +- .../observability/redhat_cluster_observability.rs} | 2 +- harmony/src/domain/topology/mod.rs | 2 +- .../domain/topology/{oberservability => }/monitoring.rs | 0 harmony/src/domain/topology/oberservability/mod.rs | 1 - harmony/src/modules/application/features/monitoring.rs | 4 ++-- .../src/modules/application/features/rhob_monitoring.rs | 4 ++-- .../monitoring/alert_channel/discord_alert_channel.rs | 6 ++---- .../modules/monitoring/alert_channel/webhook_receiver.rs | 2 +- .../modules/monitoring/alert_rule/prometheus_alert_rule.rs | 2 +- .../application_monitoring/application_monitoring_score.rs | 4 +--- .../kube_prometheus/crd/crd_alertmanager_config.rs | 2 +- .../monitoring/kube_prometheus/crd/crd_scrape_config.rs | 2 +- .../kube_prometheus/helm_prometheus_alert_score.rs | 2 +- .../src/modules/monitoring/kube_prometheus/prometheus.rs | 2 +- harmony/src/modules/monitoring/okd/mod.rs | 6 ++---- ...er_monitoring.rs => score_enable_cluster_monitoring.rs} | 4 ++-- .../monitoring/okd/score_openshift_cluster_alert_score.rs | 4 +--- .../src/modules/monitoring/okd/score_openshift_receiver.rs | 2 +- harmony/src/modules/monitoring/prometheus/prometheus.rs | 2 +- .../crd/rhob_alertmanager_config.rs | 2 +- .../monitoring/red_hat_cluster_observability/mod.rs | 3 +-- .../redhat_cluster_observability.rs | 4 +--- .../red_hat_cluster_observability/score_alert_receiver.rs | 2 +- harmony/src/modules/monitoring/scrape_target/server.rs | 2 +- .../modules/prometheus/k8s_prometheus_alerting_score.rs | 2 +- 32 files changed, 42 insertions(+), 51 deletions(-) rename harmony/src/domain/topology/k8s_anywhere/{ => observability}/kube_prometheus.rs (94%) create mode 100644 harmony/src/domain/topology/k8s_anywhere/observability/mod.rs rename harmony/src/domain/topology/k8s_anywhere/{ => observability}/openshift_monitoring.rs (94%) rename harmony/src/domain/topology/k8s_anywhere/{ => observability}/prometheus.rs (94%) rename harmony/src/{modules/monitoring/red_hat_cluster_observability/trait_redhat_cluster_observability.rs => domain/topology/k8s_anywhere/observability/redhat_cluster_observability.rs} (97%) rename harmony/src/domain/topology/{oberservability => }/monitoring.rs (100%) delete mode 100644 harmony/src/domain/topology/oberservability/mod.rs rename harmony/src/modules/monitoring/okd/{score_cluster_monitoring.rs => score_enable_cluster_monitoring.rs} (89%) diff --git a/examples/okd_cluster_alerts/src/main.rs b/examples/okd_cluster_alerts/src/main.rs index 139e052d..d708fccd 100644 --- a/examples/okd_cluster_alerts/src/main.rs +++ b/examples/okd_cluster_alerts/src/main.rs @@ -6,7 +6,7 @@ use harmony::{ alert_channel::discord_alert_channel::DiscordReceiver, okd::score_openshift_cluster_alert_score::OpenshiftClusterAlertScore, }, - topology::{K8sAnywhereTopology, oberservability::monitoring::AlertRoute}, + topology::{K8sAnywhereTopology, monitoring::AlertRoute}, }; use harmony_macros::hurl; use harmony_types::k8s_name::K8sName; diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index f24ab6dd..e3c1d8e9 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -50,14 +50,14 @@ use crate::{ prometheus::k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore, }, score::Score, - topology::{TlsRoute, TlsRouter, ingress::Ingress, oberservability::monitoring::Observability}, + topology::{TlsRoute, TlsRouter, ingress::Ingress, monitoring::Observability}, }; use super::super::{ DeploymentTarget, HelmCommand, K8sclient, MultiTargetTopology, PreparationError, PreparationOutcome, Topology, k8s::K8sClient, - oberservability::monitoring::AlertReceiver, + monitoring::AlertReceiver, tenant::{ TenantConfig, TenantManager, k8s::K8sTenantManager, diff --git a/harmony/src/domain/topology/k8s_anywhere/mod.rs b/harmony/src/domain/topology/k8s_anywhere/mod.rs index 5aa2cf80..a10dd38a 100644 --- a/harmony/src/domain/topology/k8s_anywhere/mod.rs +++ b/harmony/src/domain/topology/k8s_anywhere/mod.rs @@ -1,7 +1,5 @@ mod k8s_anywhere; -pub mod kube_prometheus; pub mod nats; -pub mod openshift_monitoring; +pub mod observability; mod postgres; -mod prometheus; pub use k8s_anywhere::*; diff --git a/harmony/src/domain/topology/k8s_anywhere/kube_prometheus.rs b/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs similarity index 94% rename from harmony/src/domain/topology/k8s_anywhere/kube_prometheus.rs rename to harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs index 4aab9bd7..dd30b5b5 100644 --- a/harmony/src/domain/topology/k8s_anywhere/kube_prometheus.rs +++ b/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs @@ -5,7 +5,7 @@ use crate::{ modules::monitoring::kube_prometheus::prometheus::KubePrometheus, topology::{ K8sAnywhereTopology, PreparationError, PreparationOutcome, - oberservability::monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, + monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, }, }; diff --git a/harmony/src/domain/topology/k8s_anywhere/observability/mod.rs b/harmony/src/domain/topology/k8s_anywhere/observability/mod.rs new file mode 100644 index 00000000..3be45c05 --- /dev/null +++ b/harmony/src/domain/topology/k8s_anywhere/observability/mod.rs @@ -0,0 +1,4 @@ +pub mod kube_prometheus; +pub mod openshift_monitoring; +pub mod prometheus; +pub mod redhat_cluster_observability; diff --git a/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs b/harmony/src/domain/topology/k8s_anywhere/observability/openshift_monitoring.rs similarity index 94% rename from harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs rename to harmony/src/domain/topology/k8s_anywhere/observability/openshift_monitoring.rs index 890decf2..6cde1a3a 100644 --- a/harmony/src/domain/topology/k8s_anywhere/openshift_monitoring.rs +++ b/harmony/src/domain/topology/k8s_anywhere/observability/openshift_monitoring.rs @@ -5,7 +5,8 @@ use crate::score::Score; use crate::{ inventory::Inventory, modules::monitoring::okd::{ - OpenshiftClusterAlertSender, score_cluster_monitoring::OpenshiftClusterMonitoringScore, + OpenshiftClusterAlertSender, + score_enable_cluster_monitoring::OpenshiftEnableClusterMonitoringScore, score_openshift_alert_rule::OpenshiftAlertRuleScore, score_openshift_receiver::OpenshiftReceiverScore, score_openshift_scrape_target::OpenshiftScrapeTargetScore, @@ -14,7 +15,7 @@ use crate::{ }, topology::{ K8sAnywhereTopology, PreparationError, PreparationOutcome, - oberservability::monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, + monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, }, }; @@ -26,7 +27,7 @@ impl Observability for K8sAnywhereTopology { inventory: &Inventory, ) -> Result { info!("enabling cluster monitoring"); - let cluster_monitoring_score = OpenshiftClusterMonitoringScore {}; + let cluster_monitoring_score = OpenshiftEnableClusterMonitoringScore {}; cluster_monitoring_score .create_interpret() .execute(inventory, self) diff --git a/harmony/src/domain/topology/k8s_anywhere/prometheus.rs b/harmony/src/domain/topology/k8s_anywhere/observability/prometheus.rs similarity index 94% rename from harmony/src/domain/topology/k8s_anywhere/prometheus.rs rename to harmony/src/domain/topology/k8s_anywhere/observability/prometheus.rs index a630feef..4024b892 100644 --- a/harmony/src/domain/topology/k8s_anywhere/prometheus.rs +++ b/harmony/src/domain/topology/k8s_anywhere/observability/prometheus.rs @@ -5,7 +5,7 @@ use crate::{ modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus, topology::{ K8sAnywhereTopology, PreparationError, PreparationOutcome, - oberservability::monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, + monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, }, }; diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/trait_redhat_cluster_observability.rs b/harmony/src/domain/topology/k8s_anywhere/observability/redhat_cluster_observability.rs similarity index 97% rename from harmony/src/modules/monitoring/red_hat_cluster_observability/trait_redhat_cluster_observability.rs rename to harmony/src/domain/topology/k8s_anywhere/observability/redhat_cluster_observability.rs index 087ed6dc..81362ee6 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/trait_redhat_cluster_observability.rs +++ b/harmony/src/domain/topology/k8s_anywhere/observability/redhat_cluster_observability.rs @@ -16,7 +16,7 @@ use crate::{ }, topology::{ K8sAnywhereTopology, PreparationError, PreparationOutcome, - oberservability::monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, + monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, }, }; diff --git a/harmony/src/domain/topology/mod.rs b/harmony/src/domain/topology/mod.rs index 28161fcf..c21b2846 100644 --- a/harmony/src/domain/topology/mod.rs +++ b/harmony/src/domain/topology/mod.rs @@ -2,6 +2,7 @@ pub mod decentralized; mod failover; mod ha_cluster; pub mod ingress; +pub mod monitoring; pub mod node_exporter; pub mod opnsense; pub use failover::*; @@ -11,7 +12,6 @@ mod http; pub mod installable; mod k8s_anywhere; mod localhost; -pub mod oberservability; pub mod tenant; use derive_new::new; pub use k8s_anywhere::*; diff --git a/harmony/src/domain/topology/oberservability/monitoring.rs b/harmony/src/domain/topology/monitoring.rs similarity index 100% rename from harmony/src/domain/topology/oberservability/monitoring.rs rename to harmony/src/domain/topology/monitoring.rs diff --git a/harmony/src/domain/topology/oberservability/mod.rs b/harmony/src/domain/topology/oberservability/mod.rs deleted file mode 100644 index 7f2ac952..00000000 --- a/harmony/src/domain/topology/oberservability/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod monitoring; diff --git a/harmony/src/modules/application/features/monitoring.rs b/harmony/src/modules/application/features/monitoring.rs index 8ae74072..a4aba4ab 100644 --- a/harmony/src/modules/application/features/monitoring.rs +++ b/harmony/src/modules/application/features/monitoring.rs @@ -9,8 +9,8 @@ use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{ }; use crate::topology::MultiTargetTopology; use crate::topology::ingress::Ingress; -use crate::topology::oberservability::monitoring::AlertReceiver; -use crate::topology::oberservability::monitoring::Observability; +use crate::topology::monitoring::AlertReceiver; +use crate::topology::monitoring::Observability; use crate::{ inventory::Inventory, modules::monitoring::{ diff --git a/harmony/src/modules/application/features/rhob_monitoring.rs b/harmony/src/modules/application/features/rhob_monitoring.rs index 643a5c63..02150ef0 100644 --- a/harmony/src/modules/application/features/rhob_monitoring.rs +++ b/harmony/src/modules/application/features/rhob_monitoring.rs @@ -8,8 +8,8 @@ use crate::modules::monitoring::red_hat_cluster_observability::RedHatClusterObse use crate::modules::monitoring::red_hat_cluster_observability::redhat_cluster_observability::RedHatClusterObservabilityScore; use crate::topology::MultiTargetTopology; use crate::topology::ingress::Ingress; -use crate::topology::oberservability::monitoring::AlertReceiver; -use crate::topology::oberservability::monitoring::Observability; +use crate::topology::monitoring::AlertReceiver; +use crate::topology::monitoring::Observability; use crate::{ inventory::Inventory, modules::monitoring::{ diff --git a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs index 1d0ebbb4..9b591840 100644 --- a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs +++ b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs @@ -18,12 +18,10 @@ use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{ use crate::modules::monitoring::kube_prometheus::prometheus::KubePrometheus; use crate::modules::monitoring::okd::OpenshiftClusterAlertSender; use crate::modules::monitoring::red_hat_cluster_observability::RedHatClusterObservability; -use crate::topology::oberservability::monitoring::{ - AlertManagerReceiver, AlertRoute, MatchOp, ReceiverEndpoint, -}; +use crate::topology::monitoring::{AlertManagerReceiver, AlertRoute, MatchOp, ReceiverEndpoint}; use crate::{ interpret::{InterpretError, Outcome}, - topology::oberservability::monitoring::AlertReceiver, + topology::monitoring::AlertReceiver, }; use harmony_types::net::Url; diff --git a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs index 518ccad0..cbe36e22 100644 --- a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs +++ b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs @@ -14,7 +14,7 @@ use crate::{ okd::OpenshiftClusterAlertSender, red_hat_cluster_observability::RedHatClusterObservability, }, - topology::oberservability::monitoring::{AlertManagerReceiver, AlertReceiver}, + topology::monitoring::{AlertManagerReceiver, AlertReceiver}, }; use harmony_types::net::Url; diff --git a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs index e0a8dbb3..c906531f 100644 --- a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs +++ b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs @@ -9,7 +9,7 @@ use crate::{ prometheus::{KubePrometheus, KubePrometheusRule}, types::{AlertGroup, AlertManagerAdditionalPromRules}, }, - topology::oberservability::monitoring::AlertRule, + topology::monitoring::AlertRule, }; #[async_trait] diff --git a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs index b7a92e06..81ab6c37 100644 --- a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs +++ b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs @@ -14,9 +14,7 @@ use crate::{ score::Score, topology::{ K8sclient, Topology, - oberservability::monitoring::{ - AlertReceiver, AlertingInterpret, Observability, ScrapeTarget, - }, + monitoring::{AlertReceiver, AlertingInterpret, Observability, ScrapeTarget}, }, }; diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs index cfeef943..890886d5 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs @@ -15,7 +15,7 @@ use crate::{ K8sclient, Topology, installable::Installable, k8s::K8sClient, - oberservability::monitoring::{AlertReceiver, AlertSender, Observability, ScrapeTarget}, + monitoring::{AlertReceiver, AlertSender, Observability, ScrapeTarget}, }, }; diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs index 24a28330..081c63f8 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs @@ -9,7 +9,7 @@ use crate::{ modules::monitoring::kube_prometheus::crd::{ crd_alertmanager_config::CRDPrometheus, crd_prometheuses::LabelSelector, }, - topology::oberservability::monitoring::ScrapeTarget, + topology::monitoring::ScrapeTarget, }; #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs b/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs index 40415cf4..da27dc58 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs @@ -8,7 +8,7 @@ use crate::{ score::Score, topology::{ HelmCommand, Topology, - oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability}, + monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability}, tenant::TenantManager, }, }; diff --git a/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs b/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs index 6533733d..67083a77 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs @@ -12,7 +12,7 @@ use crate::{ topology::{ HelmCommand, Topology, installable::Installable, - oberservability::monitoring::{AlertReceiver, AlertRule, AlertSender}, + monitoring::{AlertReceiver, AlertRule, AlertSender}, tenant::TenantManager, }, }; diff --git a/harmony/src/modules/monitoring/okd/mod.rs b/harmony/src/modules/monitoring/okd/mod.rs index e45947af..86d5422b 100644 --- a/harmony/src/modules/monitoring/okd/mod.rs +++ b/harmony/src/modules/monitoring/okd/mod.rs @@ -1,10 +1,8 @@ use serde::Serialize; -use crate::topology::oberservability::monitoring::{ - AlertReceiver, AlertRule, AlertSender, ScrapeTarget, -}; +use crate::topology::monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}; -pub mod score_cluster_monitoring; +pub mod score_enable_cluster_monitoring; pub mod score_openshift_alert_rule; pub mod score_openshift_cluster_alert_score; pub mod score_openshift_receiver; diff --git a/harmony/src/modules/monitoring/okd/score_cluster_monitoring.rs b/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs similarity index 89% rename from harmony/src/modules/monitoring/okd/score_cluster_monitoring.rs rename to harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs index 66c09458..3ce60968 100644 --- a/harmony/src/modules/monitoring/okd/score_cluster_monitoring.rs +++ b/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs @@ -12,9 +12,9 @@ use crate::{ }; #[derive(Clone, Debug, Serialize)] -pub struct OpenshiftClusterMonitoringScore {} +pub struct OpenshiftEnableClusterMonitoringScore {} -impl Score for OpenshiftClusterMonitoringScore { +impl Score for OpenshiftEnableClusterMonitoringScore { fn name(&self) -> String { "OpenshiftClusterMonitoringScore".to_string() } diff --git a/harmony/src/modules/monitoring/okd/score_openshift_cluster_alert_score.rs b/harmony/src/modules/monitoring/okd/score_openshift_cluster_alert_score.rs index 608a6f6a..42362854 100644 --- a/harmony/src/modules/monitoring/okd/score_openshift_cluster_alert_score.rs +++ b/harmony/src/modules/monitoring/okd/score_openshift_cluster_alert_score.rs @@ -10,9 +10,7 @@ use crate::{ score::Score, topology::{ Topology, - oberservability::monitoring::{ - AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget, - }, + monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget}, }, }; diff --git a/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs b/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs index d5743822..74525250 100644 --- a/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs +++ b/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs @@ -11,7 +11,7 @@ use crate::{ inventory::Inventory, modules::monitoring::okd::OpenshiftClusterAlertSender, score::Score, - topology::{K8sclient, Topology, oberservability::monitoring::AlertReceiver}, + topology::{K8sclient, Topology, monitoring::AlertReceiver}, }; #[derive(Debug, Clone, Serialize)] diff --git a/harmony/src/modules/monitoring/prometheus/prometheus.rs b/harmony/src/modules/monitoring/prometheus/prometheus.rs index 99a1e171..3e2fd14b 100644 --- a/harmony/src/modules/monitoring/prometheus/prometheus.rs +++ b/harmony/src/modules/monitoring/prometheus/prometheus.rs @@ -19,7 +19,7 @@ use crate::{ topology::{ HelmCommand, Topology, installable::Installable, - oberservability::monitoring::{AlertReceiver, AlertRule, AlertSender}, + monitoring::{AlertReceiver, AlertRule, AlertSender}, tenant::TenantManager, }, }; diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs index e440b6ff..f24b040c 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; use crate::topology::{ k8s::K8sClient, - oberservability::monitoring::{AlertReceiver, AlertSender}, + monitoring::{AlertReceiver, AlertSender}, }; #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema, Default)] diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs index 5286b020..789656bf 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs @@ -6,7 +6,7 @@ use crate::{ score::Score, topology::{ Topology, - oberservability::monitoring::{ + monitoring::{ AlertReceiver, AlertRule, AlertSender, AlertingInterpret, Observability, ScrapeTarget, }, }, @@ -17,7 +17,6 @@ pub mod redhat_cluster_observability; pub mod score_alert_receiver; pub mod score_coo_monitoring_stack; pub mod score_redhat_cluster_observability_operator; -pub mod trait_redhat_cluster_observability; /// RedHat Cluster Observability Operator allows fine grained control of monitoring stack /// solutions. This can be run in parallel with the default openshift-monitoring stack. diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs index 53de66de..522f0012 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs @@ -6,9 +6,7 @@ use crate::{ score::Score, topology::{ Topology, - oberservability::monitoring::{ - AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget, - }, + monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget}, }, }; diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/score_alert_receiver.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/score_alert_receiver.rs index 51da5179..0bdbeb27 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/score_alert_receiver.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/score_alert_receiver.rs @@ -18,7 +18,7 @@ use crate::{ }, }, score::Score, - topology::{K8sclient, Topology, oberservability::monitoring::AlertReceiver}, + topology::{K8sclient, Topology, monitoring::AlertReceiver}, }; #[derive(Debug, Clone, Serialize)] diff --git a/harmony/src/modules/monitoring/scrape_target/server.rs b/harmony/src/modules/monitoring/scrape_target/server.rs index 178e9140..c11c792e 100644 --- a/harmony/src/modules/monitoring/scrape_target/server.rs +++ b/harmony/src/modules/monitoring/scrape_target/server.rs @@ -10,7 +10,7 @@ use crate::{ crd_alertmanager_config::CRDPrometheus, crd_scrape_config::{Params, RelabelConfig, ScrapeConfig, ScrapeConfigSpec, StaticConfig}, }, - topology::oberservability::monitoring::ScrapeTarget, + topology::monitoring::ScrapeTarget, }; #[derive(Debug, Clone, Serialize)] diff --git a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs index f3687821..f005086c 100644 --- a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs +++ b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs @@ -21,7 +21,7 @@ use crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::{ use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{ ServiceMonitor, ServiceMonitorSpec, }; -use crate::topology::oberservability::monitoring::{AlertReceiver, Observability}; +use crate::topology::monitoring::{AlertReceiver, Observability}; use crate::topology::{K8sclient, Topology, k8s::K8sClient}; use crate::{ data::Version, -- 2.39.5 From c54936d19f21b203afb1bb11db6cae315a6b088d Mon Sep 17 00:00:00 2001 From: wjro Date: Mon, 23 Feb 2026 16:07:52 -0500 Subject: [PATCH 06/15] fix: added check to verify if cluster monitoring is enabled --- examples/okd_cluster_alerts/src/main.rs | 23 ++- .../okd/score_enable_cluster_monitoring.rs | 143 +++++++++++++++--- 2 files changed, 134 insertions(+), 32 deletions(-) diff --git a/examples/okd_cluster_alerts/src/main.rs b/examples/okd_cluster_alerts/src/main.rs index d708fccd..306fdda7 100644 --- a/examples/okd_cluster_alerts/src/main.rs +++ b/examples/okd_cluster_alerts/src/main.rs @@ -1,26 +1,24 @@ -use std::collections::HashMap; - use harmony::{ inventory::Inventory, modules::monitoring::{ alert_channel::discord_alert_channel::DiscordReceiver, okd::score_openshift_cluster_alert_score::OpenshiftClusterAlertScore, }, - topology::{K8sAnywhereTopology, monitoring::AlertRoute}, + topology::{ + K8sAnywhereTopology, + monitoring::{AlertMatcher, AlertRoute, MatchOp}, + }, }; + use harmony_macros::hurl; -use harmony_types::k8s_name::K8sName; #[tokio::main] async fn main() { - let mut sel = HashMap::new(); - sel.insert( - "openshift_io_alert_source".to_string(), - "platform".to_string(), - ); - let mut sel2 = HashMap::new(); - sel2.insert("openshift_io_alert_source".to_string(), "".to_string()); - let selectors = vec![sel, sel2]; + let matcher = AlertMatcher { + label: "openshift_io_alert_source".to_string(), + operator: MatchOp::Eq, + value: "platform".to_string(), + }; harmony_cli::run( Inventory::autoload(), K8sAnywhereTopology::from_env(), @@ -29,6 +27,7 @@ async fn main() { name: "wills-discord-webhook-example".to_string(), url: hurl!("https://something.io"), route: AlertRoute { + matchers: vec![matcher], ..AlertRoute::default("wills-discord-webhook-example".to_string()) }, })], diff --git a/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs b/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs index 3ce60968..5f05557f 100644 --- a/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs +++ b/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs @@ -1,14 +1,19 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, sync::Arc}; +use async_trait::async_trait; +use harmony_types::id::Id; use k8s_openapi::api::core::v1::ConfigMap; -use kube::api::ObjectMeta; +use kube::api::{GroupVersionKind, ObjectMeta}; +use log::debug; use serde::Serialize; use crate::{ - interpret::Interpret, + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, modules::k8s::resource::K8sResourceScore, score::Score, - topology::{K8sclient, Topology}, + topology::{K8sclient, Topology, k8s::K8sClient}, }; #[derive(Clone, Debug, Serialize)] @@ -20,27 +25,125 @@ impl Score for OpenshiftEnableClusterMonitoringScore } fn create_interpret(&self) -> Box> { + Box::new(OpenshiftEnableClusterMonitoringInterpret {}) + } +} + +#[derive(Clone, Debug, Serialize)] +pub struct OpenshiftEnableClusterMonitoringInterpret {} + +#[async_trait] +impl Interpret for OpenshiftEnableClusterMonitoringInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { let namespace = "openshift-monitoring".to_string(); - let mut data = BTreeMap::new(); - data.insert( - "config.yaml".to_string(), - r#" + let name = "cluster-monitoring-config".to_string(); + let client = topology.k8s_client().await?; + let enabled = self + .check_cluster_monitoring_enabled(client, &name, &namespace) + .await + .map_err(|e| InterpretError::new(e))?; + + debug!("enabled {:#?}", enabled); + + match enabled { + true => Ok(Outcome::success( + "Openshift Cluster Monitoring already enabled".to_string(), + )), + false => { + let mut data = BTreeMap::new(); + data.insert( + "config.yaml".to_string(), + r#" enableUserWorkload: true alertmanagerMain: enableUserAlertmanagerConfig: true "# - .to_string(), - ); + .to_string(), + ); - let cm = ConfigMap { - metadata: ObjectMeta { - name: Some("cluster-monitoring-config".to_string()), - namespace: Some(namespace.clone()), - ..Default::default() - }, - data: Some(data), - ..Default::default() - }; - K8sResourceScore::single(cm, Some(namespace)).create_interpret() + let cm = ConfigMap { + metadata: ObjectMeta { + name: Some(name), + namespace: Some(namespace.clone()), + ..Default::default() + }, + data: Some(data), + ..Default::default() + }; + K8sResourceScore::single(cm, Some(namespace)) + .create_interpret() + .execute(inventory, topology) + .await?; + + Ok(Outcome::success( + "Successfully enabled Openshift Cluster Monitoring".to_string(), + )) + } + } + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("OpenshiftEnableClusterMonitoringInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} +impl OpenshiftEnableClusterMonitoringInterpret { + async fn check_cluster_monitoring_enabled( + &self, + client: Arc, + name: &str, + namespace: &str, + ) -> Result { + let gvk = GroupVersionKind { + group: "".to_string(), + version: "v1".to_string(), + kind: "ConfigMap".to_string(), + }; + + let cm = match client + .get_resource_json_value(name, Some(namespace), &gvk) + .await + { + Ok(obj) => obj, + Err(_) => return Ok(false), // CM doesn't exist? Treat as disabled. + }; + + debug!("{:#?}", cm.data.pointer("/data/config.yaml")); + let config_yaml_str = match cm + .data + .pointer("/data/config.yaml") + .and_then(|v| v.as_str()) + { + Some(s) => s, + None => return Ok(false), // Key missing? Treat as disabled. + }; + + + debug!("{:#?}", config_yaml_str); + let parsed_config: serde_yaml::Value = serde_yaml::from_str(config_yaml_str) + .map_err(|e| format!("Failed to parse nested YAML: {}", e))?; + + let enabled = parsed_config + .get("enableUserWorkload") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + debug!("{:#?}", enabled); + Ok(enabled) } } -- 2.39.5 From 4b5e3a52a12fe5b494e72a85998b9d27c35131c6 Mon Sep 17 00:00:00 2001 From: wjro Date: Tue, 24 Feb 2026 11:14:47 -0500 Subject: [PATCH 07/15] feat: working example of enabling and adding an alert receiver for okd_cluster_alerts --- examples/okd_cluster_alerts/src/main.rs | 12 +- harmony/src/domain/topology/monitoring.rs | 1 + .../alert_channel/discord_alert_channel.rs | 2 +- .../okd/score_enable_cluster_monitoring.rs | 2 +- .../monitoring/okd/score_user_workload.rs | 149 +++++++++++++++--- 5 files changed, 138 insertions(+), 28 deletions(-) diff --git a/examples/okd_cluster_alerts/src/main.rs b/examples/okd_cluster_alerts/src/main.rs index 306fdda7..bbf7a4a8 100644 --- a/examples/okd_cluster_alerts/src/main.rs +++ b/examples/okd_cluster_alerts/src/main.rs @@ -14,21 +14,21 @@ use harmony_macros::hurl; #[tokio::main] async fn main() { - let matcher = AlertMatcher { - label: "openshift_io_alert_source".to_string(), + let platform_matcher = AlertMatcher { + label: "prometheus".to_string(), operator: MatchOp::Eq, - value: "platform".to_string(), + value: "openshift-monitoring/k8s".to_string(), }; harmony_cli::run( Inventory::autoload(), K8sAnywhereTopology::from_env(), vec![Box::new(OpenshiftClusterAlertScore { receivers: vec![Box::new(DiscordReceiver { - name: "wills-discord-webhook-example".to_string(), + name: "wills-discord-channel-example".to_string(), url: hurl!("https://something.io"), route: AlertRoute { - matchers: vec![matcher], - ..AlertRoute::default("wills-discord-webhook-example".to_string()) + matchers: vec![platform_matcher], + ..AlertRoute::default("wills-discord-channel-example".to_string()) }, })], sender: harmony::modules::monitoring::okd::OpenshiftClusterAlertSender, diff --git a/harmony/src/domain/topology/monitoring.rs b/harmony/src/domain/topology/monitoring.rs index 75933b74..8f6c6b66 100644 --- a/harmony/src/domain/topology/monitoring.rs +++ b/harmony/src/domain/topology/monitoring.rs @@ -35,6 +35,7 @@ pub struct ReceiverEndpoint {} #[derive(Debug, Clone, Serialize)] pub struct AlertRoute { pub receiver: String, + #[serde(skip_serializing_if = "Vec::is_empty")] pub matchers: Vec, #[serde(skip_serializing_if = "Vec::is_empty")] pub group_by: Vec, diff --git a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs index 9b591840..135e5a06 100644 --- a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs +++ b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs @@ -112,7 +112,7 @@ impl AlertReceiver for DiscordReceiver { let receiver_block = serde_yaml::to_value(json!({ "name": self.name, "discord_configs": [{ - "webhook_url": format!("{{{{ webhook-url \"{}\" }}}}", self.url), + "webhook_url": format!("{}", self.url), "title": "{{ template \"discord.default.title\" . }}", "message": "{{ template \"discord.default.message\" . }}" }] diff --git a/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs b/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs index 5f05557f..952bf9cb 100644 --- a/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs +++ b/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs @@ -102,6 +102,7 @@ alertmanagerMain: todo!() } } + impl OpenshiftEnableClusterMonitoringInterpret { async fn check_cluster_monitoring_enabled( &self, @@ -133,7 +134,6 @@ impl OpenshiftEnableClusterMonitoringInterpret { None => return Ok(false), // Key missing? Treat as disabled. }; - debug!("{:#?}", config_yaml_str); let parsed_config: serde_yaml::Value = serde_yaml::from_str(config_yaml_str) .map_err(|e| format!("Failed to parse nested YAML: {}", e))?; diff --git a/harmony/src/modules/monitoring/okd/score_user_workload.rs b/harmony/src/modules/monitoring/okd/score_user_workload.rs index fdea01ff..3eab008d 100644 --- a/harmony/src/modules/monitoring/okd/score_user_workload.rs +++ b/harmony/src/modules/monitoring/okd/score_user_workload.rs @@ -1,13 +1,18 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, sync::Arc}; use crate::{ - interpret::Interpret, + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, modules::k8s::resource::K8sResourceScore, score::Score, - topology::{K8sclient, Topology}, + topology::{K8sclient, Topology, k8s::K8sClient}, }; +use async_trait::async_trait; +use harmony_types::id::Id; use k8s_openapi::api::core::v1::ConfigMap; -use kube::api::ObjectMeta; +use kube::api::{GroupVersionKind, ObjectMeta}; +use log::debug; use serde::Serialize; #[derive(Clone, Debug, Serialize)] @@ -19,27 +24,131 @@ impl Score for OpenshiftUserWorkloadMonitoring { } fn create_interpret(&self) -> Box> { + Box::new(OpenshiftUserWorkloadMonitoringInterpret {}) + } +} + +#[derive(Clone, Debug, Serialize)] +pub struct OpenshiftUserWorkloadMonitoringInterpret {} + +#[async_trait] +impl Interpret for OpenshiftUserWorkloadMonitoringInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { let namespace = "openshift-user-workload-monitoring".to_string(); - let mut data = BTreeMap::new(); - data.insert( - "config.yaml".to_string(), - r#" + let cm_name = "user-workload-monitoring-config".to_string(); + let client = topology.k8s_client().await?; + let cm_enabled = self + .check_cluster_user_workload_monitoring_enabled(client, &cm_name, &namespace) + .await?; + match cm_enabled { + true => Ok(Outcome::success( + "OpenshiftUserWorkloadMonitoringEnabled".to_string(), + )), + false => { + let mut data = BTreeMap::new(); + data.insert( + "config.yaml".to_string(), + r#" alertmanager: enabled: true enableAlertmanagerConfig: true "# - .to_string(), - ); - let cm = ConfigMap { - metadata: ObjectMeta { - name: Some("user-workload-monitoring-config".to_string()), - namespace: Some(namespace.clone()), - ..Default::default() - }, - data: Some(data), - ..Default::default() - }; + .to_string(), + ); + let cm = ConfigMap { + metadata: ObjectMeta { + name: Some("user-workload-monitoring-config".to_string()), + namespace: Some(namespace.clone()), + ..Default::default() + }, + data: Some(data), + ..Default::default() + }; - K8sResourceScore::single(cm, Some(namespace)).create_interpret() + K8sResourceScore::single(cm, Some(namespace)) + .create_interpret() + .execute(inventory, topology) + .await + } + } + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("OpenshiftUserWorkloadMonitoringInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} + +impl OpenshiftUserWorkloadMonitoringInterpret { + async fn check_cluster_user_workload_monitoring_enabled( + &self, + client: Arc, + name: &str, + namespace: &str, + ) -> Result { + let gvk = GroupVersionKind { + group: "".to_string(), + version: "v1".to_string(), + kind: "ConfigMap".to_string(), + }; + + let cm = match client + .get_resource_json_value(name, Some(namespace), &gvk) + .await + { + Ok(obj) => obj, + Err(_) => return Ok(false), // CM doesn't exist? Treat as disabled. + }; + + debug!("{:#?}", cm.data.pointer("/data/config.yaml")); + let config_yaml_str = match cm + .data + .pointer("/data/config.yaml") + .and_then(|v| v.as_str()) + { + Some(s) => s, + None => return Ok(false), // Key missing? Treat as disabled. + }; + + debug!("{:#?}", config_yaml_str); + let parsed_config: serde_yaml::Value = serde_yaml::from_str(config_yaml_str) + .map_err(|e| format!("Failed to parse nested YAML: {}", e))?; + + let alert_manager_enabled = parsed_config + .get("alertmanager") + .and_then(|a| a.get("enableAlertmanagerConfig")) + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + debug!("alertmanagerenabled: {:#?}", alert_manager_enabled); + + let enabled = parsed_config + .get("alertmanager") + .and_then(|enabled| enabled.get("enabled")) + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + debug!("user workload monitoring enabled: {:#?}", enabled); + + if alert_manager_enabled && enabled == true { + Ok(true) + } else { + Ok(false) + } } } -- 2.39.5 From 0c1c8daf13ae93273272c12817826d3397038164 Mon Sep 17 00:00:00 2001 From: wjro Date: Tue, 24 Feb 2026 16:13:30 -0500 Subject: [PATCH 08/15] wip: working alert rule for okd --- examples/monitoring/src/main.rs | 40 ++-- examples/monitoring_with_tenant/src/main.rs | 18 +- examples/okd_cluster_alerts/src/main.rs | 24 ++- .../rhob_application_monitoring/src/main.rs | 13 +- examples/rust/src/main.rs | 17 +- examples/try_rust_webapp/src/main.rs | 9 +- .../observability/openshift_monitoring.rs | 16 +- harmony/src/domain/topology/monitoring.rs | 172 +++++++++--------- .../alert_channel/discord_alert_channel.rs | 114 ++++++------ .../alert_channel/webhook_receiver.rs | 2 +- .../alert_rule}/alerts/infra/dell_server.rs | 0 .../alert_rule}/alerts/infra/mod.rs | 0 .../alert_rule}/alerts/k8s/deployment.rs | 0 .../alert_rule}/alerts/k8s/memory_usage.rs | 0 .../alert_rule}/alerts/k8s/mod.rs | 0 .../alert_rule}/alerts/k8s/pod.rs | 0 .../alert_rule}/alerts/k8s/pvc.rs | 0 .../alert_rule}/alerts/k8s/service.rs | 0 .../alert_rule}/alerts/mod.rs | 0 .../src/modules/monitoring/alert_rule/mod.rs | 1 + .../alert_rule/prometheus_alert_rule.rs | 155 +++++++++------- .../kube_prometheus/crd/crd_default_rules.rs | 2 +- .../monitoring/okd/crd/alerting_rules.rs | 58 ++++++ harmony/src/modules/monitoring/okd/crd/mod.rs | 1 + harmony/src/modules/monitoring/okd/mod.rs | 1 + .../okd/score_openshift_alert_rule.rs | 34 +++- .../okd/score_openshift_scrape_target.rs | 13 +- .../crd/rhob_default_rules.rs | 6 +- harmony/src/modules/prometheus/mod.rs | 3 - harmony/src/modules/prometheus/prometheus.rs | 25 --- 30 files changed, 420 insertions(+), 304 deletions(-) rename harmony/src/modules/{prometheus => monitoring/alert_rule}/alerts/infra/dell_server.rs (100%) rename harmony/src/modules/{prometheus => monitoring/alert_rule}/alerts/infra/mod.rs (100%) rename harmony/src/modules/{prometheus => monitoring/alert_rule}/alerts/k8s/deployment.rs (100%) rename harmony/src/modules/{prometheus => monitoring/alert_rule}/alerts/k8s/memory_usage.rs (100%) rename harmony/src/modules/{prometheus => monitoring/alert_rule}/alerts/k8s/mod.rs (100%) rename harmony/src/modules/{prometheus => monitoring/alert_rule}/alerts/k8s/pod.rs (100%) rename harmony/src/modules/{prometheus => monitoring/alert_rule}/alerts/k8s/pvc.rs (100%) rename harmony/src/modules/{prometheus => monitoring/alert_rule}/alerts/k8s/service.rs (100%) rename harmony/src/modules/{prometheus => monitoring/alert_rule}/alerts/mod.rs (100%) create mode 100644 harmony/src/modules/monitoring/okd/crd/alerting_rules.rs create mode 100644 harmony/src/modules/monitoring/okd/crd/mod.rs delete mode 100644 harmony/src/modules/prometheus/prometheus.rs diff --git a/examples/monitoring/src/main.rs b/examples/monitoring/src/main.rs index fca2728f..b4fc13d8 100644 --- a/examples/monitoring/src/main.rs +++ b/examples/monitoring/src/main.rs @@ -2,36 +2,40 @@ use std::collections::HashMap; use harmony::{ inventory::Inventory, - modules::{ - monitoring::{ - alert_channel::discord_alert_channel::DiscordWebhook, - alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, - kube_prometheus::{ - helm_prometheus_alert_score::HelmPrometheusAlertingScore, - types::{ - HTTPScheme, MatchExpression, Operator, Selector, ServiceMonitor, - ServiceMonitorEndpoint, + modules::monitoring::{ + alert_channel::discord_alert_channel::DiscordReceiver, + alert_rule::{ + alerts::{ + infra::dell_server::{ + alert_global_storage_status_critical, + alert_global_storage_status_non_recoverable, + global_storage_status_degraded_non_critical, }, + k8s::pvc::high_pvc_fill_rate_over_two_days, }, + prometheus_alert_rule::AlertManagerRuleGroup, }, - prometheus::alerts::{ - infra::dell_server::{ - alert_global_storage_status_critical, alert_global_storage_status_non_recoverable, - global_storage_status_degraded_non_critical, + kube_prometheus::{ + helm_prometheus_alert_score::HelmPrometheusAlertingScore, + types::{ + HTTPScheme, MatchExpression, Operator, Selector, ServiceMonitor, + ServiceMonitorEndpoint, }, - k8s::pvc::high_pvc_fill_rate_over_two_days, }, }, - topology::K8sAnywhereTopology, + topology::{K8sAnywhereTopology, monitoring::AlertRoute}, }; use harmony_types::{k8s_name::K8sName, net::Url}; #[tokio::main] async fn main() { - let discord_receiver = DiscordWebhook { - name: "test-discord".to_string(), + let receiver_name = "test-discord".to_string(); + let discord_receiver = DiscordReceiver { + name: receiver_name.clone(), url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()), - selectors: vec![], + route: AlertRoute { + ..AlertRoute::default(receiver_name) + }, }; let high_pvc_fill_rate_over_two_days_alert = high_pvc_fill_rate_over_two_days(); diff --git a/examples/monitoring_with_tenant/src/main.rs b/examples/monitoring_with_tenant/src/main.rs index bc2b8b99..04d270d4 100644 --- a/examples/monitoring_with_tenant/src/main.rs +++ b/examples/monitoring_with_tenant/src/main.rs @@ -4,8 +4,11 @@ use harmony::{ inventory::Inventory, modules::{ monitoring::{ - alert_channel::discord_alert_channel::DiscordWebhook, - alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, + alert_channel::discord_alert_channel::DiscordReceiver, + alert_rule::{ + alerts::k8s::pvc::high_pvc_fill_rate_over_two_days, + prometheus_alert_rule::AlertManagerRuleGroup, + }, kube_prometheus::{ helm_prometheus_alert_score::HelmPrometheusAlertingScore, types::{ @@ -14,11 +17,11 @@ use harmony::{ }, }, }, - prometheus::alerts::k8s::pvc::high_pvc_fill_rate_over_two_days, tenant::TenantScore, }, topology::{ K8sAnywhereTopology, + monitoring::AlertRoute, tenant::{ResourceLimits, TenantConfig, TenantNetworkPolicy}, }, }; @@ -42,10 +45,13 @@ async fn main() { }, }; - let discord_receiver = DiscordWebhook { - name: "test-discord".to_string(), + let receiver_name = "test-discord".to_string(); + let discord_receiver = DiscordReceiver { + name: receiver_name.clone(), url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()), - selectors: vec![], + route: AlertRoute { + ..AlertRoute::default(receiver_name) + }, }; let high_pvc_fill_rate_over_two_days_alert = high_pvc_fill_rate_over_two_days(); diff --git a/examples/okd_cluster_alerts/src/main.rs b/examples/okd_cluster_alerts/src/main.rs index bbf7a4a8..f0d8123e 100644 --- a/examples/okd_cluster_alerts/src/main.rs +++ b/examples/okd_cluster_alerts/src/main.rs @@ -2,6 +2,10 @@ use harmony::{ inventory::Inventory, modules::monitoring::{ alert_channel::discord_alert_channel::DiscordReceiver, + alert_rule::{ + alerts::k8s::pvc::high_pvc_fill_rate_over_two_days, + prometheus_alert_rule::AlertManagerRuleGroup, + }, okd::score_openshift_cluster_alert_score::OpenshiftClusterAlertScore, }, topology::{ @@ -19,20 +23,30 @@ async fn main() { operator: MatchOp::Eq, value: "openshift-monitoring/k8s".to_string(), }; + let severity = AlertMatcher { + label: "severity".to_string(), + operator: MatchOp::Eq, + value: "critical".to_string(), + }; + + let test_rule = high_pvc_fill_rate_over_two_days(); + + let additional_rules = AlertManagerRuleGroup::new("test-alerts", vec![test_rule]); + harmony_cli::run( Inventory::autoload(), K8sAnywhereTopology::from_env(), vec![Box::new(OpenshiftClusterAlertScore { receivers: vec![Box::new(DiscordReceiver { - name: "wills-discord-channel-example".to_string(), - url: hurl!("https://something.io"), + name: "crit-wills-discord-channel-example".to_string(), + url: hurl!("https://test.io"), route: AlertRoute { - matchers: vec![platform_matcher], - ..AlertRoute::default("wills-discord-channel-example".to_string()) + matchers: vec![severity], + ..AlertRoute::default("crit-wills-discord-channel-example".to_string()) }, })], sender: harmony::modules::monitoring::okd::OpenshiftClusterAlertSender, - rules: vec![], + rules: vec![Box::new(additional_rules)], scrape_targets: None, })], None, diff --git a/examples/rhob_application_monitoring/src/main.rs b/examples/rhob_application_monitoring/src/main.rs index e865029d..5f0578ed 100644 --- a/examples/rhob_application_monitoring/src/main.rs +++ b/examples/rhob_application_monitoring/src/main.rs @@ -6,9 +6,9 @@ use harmony::{ application::{ ApplicationScore, RustWebFramework, RustWebapp, features::rhob_monitoring::Monitoring, }, - monitoring::alert_channel::discord_alert_channel::DiscordWebhook, + monitoring::alert_channel::discord_alert_channel::DiscordReceiver, }, - topology::K8sAnywhereTopology, + topology::{K8sAnywhereTopology, monitoring::AlertRoute}, }; use harmony_types::{k8s_name::K8sName, net::Url}; @@ -22,10 +22,13 @@ async fn main() { service_port: 3000, }); - let discord_receiver = DiscordWebhook { - name: "test-discord".to_string(), + let receiver_name = "test-discord".to_string(); + let discord_receiver = DiscordReceiver { + name: receiver_name.clone(), url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()), - selectors: vec![], + route: AlertRoute { + ..AlertRoute::default(receiver_name) + }, }; let app = ApplicationScore { diff --git a/examples/rust/src/main.rs b/examples/rust/src/main.rs index 60252141..0ab2bd8c 100644 --- a/examples/rust/src/main.rs +++ b/examples/rust/src/main.rs @@ -8,13 +8,13 @@ use harmony::{ features::{Monitoring, PackagingDeployment}, }, monitoring::alert_channel::{ - discord_alert_channel::DiscordWebhook, webhook_receiver::WebhookReceiver, + discord_alert_channel::DiscordReceiver, webhook_receiver::WebhookReceiver, }, }, - topology::K8sAnywhereTopology, + topology::{K8sAnywhereTopology, monitoring::AlertRoute}, }; use harmony_macros::hurl; -use harmony_types::k8s_name::K8sName; +use harmony_types::{k8s_name::K8sName, net::Url}; #[tokio::main] async fn main() { @@ -26,10 +26,13 @@ async fn main() { service_port: 3000, }); - let discord_receiver = DiscordWebhook { - name: "test-discord".to_string(), - url: hurl!("https://discord.doesnt.exist.com"), - selectors: vec![], + let receiver_name = "test-discord".to_string(); + let discord_receiver = DiscordReceiver { + name: receiver_name.clone(), + url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()), + route: AlertRoute { + ..AlertRoute::default(receiver_name) + }, }; let webhook_receiver = WebhookReceiver { diff --git a/examples/try_rust_webapp/src/main.rs b/examples/try_rust_webapp/src/main.rs index 2dbb3de3..aa954f63 100644 --- a/examples/try_rust_webapp/src/main.rs +++ b/examples/try_rust_webapp/src/main.rs @@ -1,11 +1,8 @@ use harmony::{ inventory::Inventory, - modules::{ - application::{ - ApplicationScore, RustWebFramework, RustWebapp, - features::{Monitoring, PackagingDeployment}, - }, - monitoring::alert_channel::discord_alert_channel::DiscordWebhook, + modules::application::{ + ApplicationScore, RustWebFramework, RustWebapp, + features::{Monitoring, PackagingDeployment}, }, topology::K8sAnywhereTopology, }; diff --git a/harmony/src/domain/topology/k8s_anywhere/observability/openshift_monitoring.rs b/harmony/src/domain/topology/k8s_anywhere/observability/openshift_monitoring.rs index 6cde1a3a..aa42a220 100644 --- a/harmony/src/domain/topology/k8s_anywhere/observability/openshift_monitoring.rs +++ b/harmony/src/domain/topology/k8s_anywhere/observability/openshift_monitoring.rs @@ -23,7 +23,7 @@ use crate::{ impl Observability for K8sAnywhereTopology { async fn install_alert_sender( &self, - sender: &OpenshiftClusterAlertSender, + _sender: &OpenshiftClusterAlertSender, inventory: &Inventory, ) -> Result { info!("enabling cluster monitoring"); @@ -49,7 +49,7 @@ impl Observability for K8sAnywhereTopology { async fn install_receivers( &self, - sender: &OpenshiftClusterAlertSender, + _sender: &OpenshiftClusterAlertSender, inventory: &Inventory, receivers: Option>>>, ) -> Result { @@ -74,14 +74,14 @@ impl Observability for K8sAnywhereTopology { async fn install_rules( &self, - sender: &OpenshiftClusterAlertSender, + _sender: &OpenshiftClusterAlertSender, inventory: &Inventory, rules: Option>>>, ) -> Result { if let Some(rules) = rules { for rule in rules { info!("Installing rule "); - let rule_score = OpenshiftAlertRuleScore {}; + let rule_score = OpenshiftAlertRuleScore { rule: rule }; rule_score .create_interpret() .execute(inventory, self) @@ -98,14 +98,16 @@ impl Observability for K8sAnywhereTopology { async fn add_scrape_targets( &self, - sender: &OpenshiftClusterAlertSender, + _sender: &OpenshiftClusterAlertSender, inventory: &Inventory, scrape_targets: Option>>>, ) -> Result { if let Some(scrape_targets) = scrape_targets { for scrape_target in scrape_targets { info!("Installing scrape target"); - let scrape_target_score = OpenshiftScrapeTargetScore {}; + let scrape_target_score = OpenshiftScrapeTargetScore { + scrape_target: scrape_target, + }; scrape_target_score .create_interpret() .execute(inventory, self) @@ -123,7 +125,7 @@ impl Observability for K8sAnywhereTopology { async fn ensure_monitoring_installed( &self, - sender: &OpenshiftClusterAlertSender, + _sender: &OpenshiftClusterAlertSender, inventory: &Inventory, ) -> Result { let verify_monitoring_score = VerifyUserWorkload {}; diff --git a/harmony/src/domain/topology/monitoring.rs b/harmony/src/domain/topology/monitoring.rs index 8f6c6b66..602aea98 100644 --- a/harmony/src/domain/topology/monitoring.rs +++ b/harmony/src/domain/topology/monitoring.rs @@ -20,97 +20,6 @@ pub trait AlertSender: Send + Sync + std::fmt::Debug { fn name(&self) -> String; } -/// Defines the entity that receives the alerts from a sender. For example Discord, Slack, etc -/// -pub trait AlertReceiver: std::fmt::Debug + Send + Sync { - fn build_route(&self) -> Result; - fn build_receiver(&self) -> Result; - fn name(&self) -> String; - fn clone_box(&self) -> Box>; -} - -pub struct ReceiverEndpoint {} - -///Generic routing that can map to various alert sender backends -#[derive(Debug, Clone, Serialize)] -pub struct AlertRoute { - pub receiver: String, - #[serde(skip_serializing_if = "Vec::is_empty")] - pub matchers: Vec, - #[serde(skip_serializing_if = "Vec::is_empty")] - pub group_by: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub repeat_interval: Option, - #[serde(rename = "continue")] - pub continue_matching: bool, - #[serde(skip_serializing_if = "Vec::is_empty")] - pub children: Vec, -} - -impl AlertRoute { - pub fn default(name: String) -> Self { - Self { - receiver: name, - matchers: vec![], - group_by: vec![], - repeat_interval: Some("30s".to_string()), - continue_matching: true, - children: vec![], - } - } -} - -#[derive(Debug, Clone, Serialize)] -pub struct AlertMatcher { - pub label: String, - pub operator: MatchOp, - pub value: String, -} - -#[derive(Debug, Clone)] -pub enum MatchOp { - Eq, - NotEq, - Regex, -} - -impl Serialize for MatchOp { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - let op = match self { - MatchOp::Eq => "=", - MatchOp::NotEq => "!=", - MatchOp::Regex => "=~", - }; - serializer.serialize_str(op) - } -} - -#[derive(Debug)] -pub struct AlertManagerReceiver { - pub receiver_config: serde_json::Value, - // FIXME we should not leak k8s here. DynamicObject is k8s specific - pub additional_ressources: Vec, - pub route_config: serde_json::Value, -} - -/// Defines a generic rule that can be applied to a sender, such as aprometheus alert rule -#[async_trait] -pub trait AlertRule: std::fmt::Debug + Send + Sync { - async fn install(&self, sender: &S) -> Result; - fn clone_box(&self) -> Box>; -} - -/// A generic scrape target that can be added to a sender to scrape metrics from, for example a -/// server outside of the cluster -#[async_trait] -pub trait ScrapeTarget: std::fmt::Debug + Send + Sync { - async fn install(&self, sender: &S) -> Result; - fn clone_box(&self) -> Box>; -} - /// Trait which defines how an alert sender is impleneted for a specific topology #[async_trait] pub trait Observability { @@ -148,6 +57,30 @@ pub trait Observability { ) -> Result; } +/// Defines the entity that receives the alerts from a sender. For example Discord, Slack, etc +/// +pub trait AlertReceiver: std::fmt::Debug + Send + Sync { + fn build_route(&self) -> Result; + fn build_receiver(&self) -> Result; + fn name(&self) -> String; + fn clone_box(&self) -> Box>; +} + +/// Defines a generic rule that can be applied to a sender, such as aprometheus alert rule +pub trait AlertRule: std::fmt::Debug + Send + Sync { + fn build_rule(&self) -> Result; + fn name(&self) -> String; + fn clone_box(&self) -> Box>; +} + +/// A generic scrape target that can be added to a sender to scrape metrics from, for example a +/// server outside of the cluster +#[async_trait] +pub trait ScrapeTarget: std::fmt::Debug + Send + Sync { + async fn install(&self, sender: &S) -> Result; + fn clone_box(&self) -> Box>; +} + /// Alerting interpret to install an alert sender on a given topology #[derive(Debug)] pub struct AlertingInterpret { @@ -229,3 +162,60 @@ impl Clone for Box> { self.clone_box() } } + +///Generic routing that can map to various alert sender backends +#[derive(Debug, Clone, Serialize)] +pub struct AlertRoute { + pub receiver: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub matchers: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub group_by: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub repeat_interval: Option, + #[serde(rename = "continue")] + pub continue_matching: bool, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub children: Vec, +} + +impl AlertRoute { + pub fn default(name: String) -> Self { + Self { + receiver: name, + matchers: vec![], + group_by: vec![], + repeat_interval: Some("30s".to_string()), + continue_matching: true, + children: vec![], + } + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct AlertMatcher { + pub label: String, + pub operator: MatchOp, + pub value: String, +} + +#[derive(Debug, Clone)] +pub enum MatchOp { + Eq, + NotEq, + Regex, +} + +impl Serialize for MatchOp { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let op = match self { + MatchOp::Eq => "=", + MatchOp::NotEq => "!=", + MatchOp::Regex => "=~", + }; + serializer.serialize_str(op) + } +} diff --git a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs index 135e5a06..de720eea 100644 --- a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs +++ b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs @@ -18,7 +18,7 @@ use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{ use crate::modules::monitoring::kube_prometheus::prometheus::KubePrometheus; use crate::modules::monitoring::okd::OpenshiftClusterAlertSender; use crate::modules::monitoring::red_hat_cluster_observability::RedHatClusterObservability; -use crate::topology::monitoring::{AlertManagerReceiver, AlertRoute, MatchOp, ReceiverEndpoint}; +use crate::topology::monitoring::{AlertRoute, MatchOp}; use crate::{ interpret::{InterpretError, Outcome}, topology::monitoring::AlertReceiver, @@ -31,61 +31,61 @@ pub struct DiscordReceiver { pub url: Url, pub route: AlertRoute, } - -#[derive(Debug, Clone, Serialize)] -pub struct DiscordWebhook { - pub name: String, - pub url: Url, - pub selectors: Vec>, -} - -impl DiscordWebhook { - fn get_receiver_config(&self) -> Result { - let secret_name = format!("{}-secret", self.name.clone()); - let webhook_key = format!("{}", self.url.clone()); - - let mut string_data = BTreeMap::new(); - string_data.insert("webhook-url".to_string(), webhook_key.clone()); - - let secret = Secret { - metadata: kube::core::ObjectMeta { - name: Some(secret_name.clone()), - ..Default::default() - }, - string_data: Some(string_data), - type_: Some("Opaque".to_string()), - ..Default::default() - }; - - let mut matchers: Vec = Vec::new(); - for selector in &self.selectors { - trace!("selector: {:#?}", selector); - for (k, v) in selector { - matchers.push(format!("{} = {}", k, v)); - } - } - - Ok(AlertManagerReceiver { - additional_ressources: vec![kube_resource_to_dynamic(&secret)?], - - receiver_config: json!({ - "name": self.name, - "discord_configs": [ - { - "webhook_url": self.url.clone(), - "title": "{{ template \"discord.default.title\" . }}", - "message": "{{ template \"discord.default.message\" . }}" - } - ] - }), - route_config: json!({ - "receiver": self.name, - "matchers": matchers, - - }), - }) - } -} +// +// #[derive(Debug, Clone, Serialize)] +// pub struct DiscordWebhook { +// pub name: String, +// pub url: Url, +// pub selectors: Vec>, +// } +// +// impl DiscordWebhook { +// fn get_receiver_config(&self) -> Result { +// let secret_name = format!("{}-secret", self.name.clone()); +// let webhook_key = format!("{}", self.url.clone()); +// +// let mut string_data = BTreeMap::new(); +// string_data.insert("webhook-url".to_string(), webhook_key.clone()); +// +// let secret = Secret { +// metadata: kube::core::ObjectMeta { +// name: Some(secret_name.clone()), +// ..Default::default() +// }, +// string_data: Some(string_data), +// type_: Some("Opaque".to_string()), +// ..Default::default() +// }; +// +// let mut matchers: Vec = Vec::new(); +// for selector in &self.selectors { +// trace!("selector: {:#?}", selector); +// for (k, v) in selector { +// matchers.push(format!("{} = {}", k, v)); +// } +// } +// +// Ok(AlertManagerReceiver { +// additional_ressources: vec![kube_resource_to_dynamic(&secret)?], +// +// receiver_config: json!({ +// "name": self.name, +// "discord_configs": [ +// { +// "webhook_url": self.url.clone(), +// "title": "{{ template \"discord.default.title\" . }}", +// "message": "{{ template \"discord.default.message\" . }}" +// } +// ] +// }), +// route_config: json!({ +// "receiver": self.name, +// "matchers": matchers, +// +// }), +// }) +// } +// } impl AlertReceiver for DiscordReceiver { fn build_route(&self) -> Result { @@ -179,7 +179,7 @@ impl AlertReceiver for DiscordReceiver { } } -impl AlertReceiver for DiscordWebhook { +impl AlertReceiver for DiscordReceiver { fn build_route(&self) -> Result { todo!() } diff --git a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs index cbe36e22..ad329575 100644 --- a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs +++ b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs @@ -14,7 +14,7 @@ use crate::{ okd::OpenshiftClusterAlertSender, red_hat_cluster_observability::RedHatClusterObservability, }, - topology::monitoring::{AlertManagerReceiver, AlertReceiver}, + topology::monitoring::AlertReceiver, }; use harmony_types::net::Url; diff --git a/harmony/src/modules/prometheus/alerts/infra/dell_server.rs b/harmony/src/modules/monitoring/alert_rule/alerts/infra/dell_server.rs similarity index 100% rename from harmony/src/modules/prometheus/alerts/infra/dell_server.rs rename to harmony/src/modules/monitoring/alert_rule/alerts/infra/dell_server.rs diff --git a/harmony/src/modules/prometheus/alerts/infra/mod.rs b/harmony/src/modules/monitoring/alert_rule/alerts/infra/mod.rs similarity index 100% rename from harmony/src/modules/prometheus/alerts/infra/mod.rs rename to harmony/src/modules/monitoring/alert_rule/alerts/infra/mod.rs diff --git a/harmony/src/modules/prometheus/alerts/k8s/deployment.rs b/harmony/src/modules/monitoring/alert_rule/alerts/k8s/deployment.rs similarity index 100% rename from harmony/src/modules/prometheus/alerts/k8s/deployment.rs rename to harmony/src/modules/monitoring/alert_rule/alerts/k8s/deployment.rs diff --git a/harmony/src/modules/prometheus/alerts/k8s/memory_usage.rs b/harmony/src/modules/monitoring/alert_rule/alerts/k8s/memory_usage.rs similarity index 100% rename from harmony/src/modules/prometheus/alerts/k8s/memory_usage.rs rename to harmony/src/modules/monitoring/alert_rule/alerts/k8s/memory_usage.rs diff --git a/harmony/src/modules/prometheus/alerts/k8s/mod.rs b/harmony/src/modules/monitoring/alert_rule/alerts/k8s/mod.rs similarity index 100% rename from harmony/src/modules/prometheus/alerts/k8s/mod.rs rename to harmony/src/modules/monitoring/alert_rule/alerts/k8s/mod.rs diff --git a/harmony/src/modules/prometheus/alerts/k8s/pod.rs b/harmony/src/modules/monitoring/alert_rule/alerts/k8s/pod.rs similarity index 100% rename from harmony/src/modules/prometheus/alerts/k8s/pod.rs rename to harmony/src/modules/monitoring/alert_rule/alerts/k8s/pod.rs diff --git a/harmony/src/modules/prometheus/alerts/k8s/pvc.rs b/harmony/src/modules/monitoring/alert_rule/alerts/k8s/pvc.rs similarity index 100% rename from harmony/src/modules/prometheus/alerts/k8s/pvc.rs rename to harmony/src/modules/monitoring/alert_rule/alerts/k8s/pvc.rs diff --git a/harmony/src/modules/prometheus/alerts/k8s/service.rs b/harmony/src/modules/monitoring/alert_rule/alerts/k8s/service.rs similarity index 100% rename from harmony/src/modules/prometheus/alerts/k8s/service.rs rename to harmony/src/modules/monitoring/alert_rule/alerts/k8s/service.rs diff --git a/harmony/src/modules/prometheus/alerts/mod.rs b/harmony/src/modules/monitoring/alert_rule/alerts/mod.rs similarity index 100% rename from harmony/src/modules/prometheus/alerts/mod.rs rename to harmony/src/modules/monitoring/alert_rule/alerts/mod.rs diff --git a/harmony/src/modules/monitoring/alert_rule/mod.rs b/harmony/src/modules/monitoring/alert_rule/mod.rs index 846c7696..cb7bba9e 100644 --- a/harmony/src/modules/monitoring/alert_rule/mod.rs +++ b/harmony/src/modules/monitoring/alert_rule/mod.rs @@ -1 +1,2 @@ +pub mod alerts; pub mod prometheus_alert_rule; diff --git a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs index c906531f..cbd95ca9 100644 --- a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs +++ b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs @@ -2,75 +2,23 @@ use std::collections::{BTreeMap, HashMap}; use async_trait::async_trait; use serde::Serialize; +use serde_json::json; use crate::{ interpret::{InterpretError, Outcome}, - modules::monitoring::kube_prometheus::{ - prometheus::{KubePrometheus, KubePrometheusRule}, - types::{AlertGroup, AlertManagerAdditionalPromRules}, + modules::monitoring::{ + kube_prometheus::{ + prometheus::{KubePrometheus, KubePrometheusRule}, + types::{AlertGroup, AlertManagerAdditionalPromRules}, + }, + okd::{ + OpenshiftClusterAlertSender, + crd::alerting_rules::{Rule, RuleGroup}, + }, }, topology::monitoring::AlertRule, }; -#[async_trait] -impl AlertRule for AlertManagerRuleGroup { - async fn install(&self, sender: &KubePrometheus) -> Result { - sender.install_rule(self).await - } - fn clone_box(&self) -> Box> { - Box::new(self.clone()) - } -} - -// #[async_trait] -// impl AlertRule for AlertManagerRuleGroup { -// async fn install(&self, sender: &Prometheus) -> Result { -// sender.install_rule(self).await -// } -// fn clone_box(&self) -> Box> { -// Box::new(self.clone()) -// } -// } -// -// #[async_trait] -// impl PrometheusRule for AlertManagerRuleGroup { -// fn name(&self) -> String { -// self.name.clone() -// } -// async fn configure_rule(&self) -> AlertManagerAdditionalPromRules { -// let mut additional_prom_rules = BTreeMap::new(); -// -// additional_prom_rules.insert( -// self.name.clone(), -// AlertGroup { -// groups: vec![self.clone()], -// }, -// ); -// AlertManagerAdditionalPromRules { -// rules: additional_prom_rules, -// } -// } -// } -#[async_trait] -impl KubePrometheusRule for AlertManagerRuleGroup { - fn name(&self) -> String { - self.name.clone() - } - async fn configure_rule(&self) -> AlertManagerAdditionalPromRules { - let mut additional_prom_rules = BTreeMap::new(); - - additional_prom_rules.insert( - self.name.clone(), - AlertGroup { - groups: vec![self.clone()], - }, - ); - AlertManagerAdditionalPromRules { - rules: additional_prom_rules, - } - } -} - impl AlertManagerRuleGroup { pub fn new(name: &str, rules: Vec) -> AlertManagerRuleGroup { AlertManagerRuleGroup { @@ -126,3 +74,86 @@ impl PrometheusAlertRule { self } } + +impl AlertRule for AlertManagerRuleGroup { + fn build_rule(&self) -> Result { + let name = self.name.clone(); + let mut rules: Vec = vec![]; + for rule in self.rules.clone() { + rules.push(rule.into()) + } + + let rule_groups = vec![RuleGroup { name, rules }]; + + Ok(serde_json::to_value(rule_groups).map_err(|e| InterpretError::new(e.to_string()))?) + } + + fn name(&self) -> String { + self.name.clone() + } + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } +} + +impl AlertRule for AlertManagerRuleGroup { + fn build_rule(&self) -> Result { + todo!() + } + + fn name(&self) -> String { + self.name.clone() + } + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } +} + +// #[async_trait] +// impl AlertRule for AlertManagerRuleGroup { +// async fn install(&self, sender: &Prometheus) -> Result { +// sender.install_rule(self).await +// } +// fn clone_box(&self) -> Box> { +// Box::new(self.clone()) +// } +// } +// +// #[async_trait] +// impl PrometheusRule for AlertManagerRuleGroup { +// fn name(&self) -> String { +// self.name.clone() +// } +// async fn configure_rule(&self) -> AlertManagerAdditionalPromRules { +// let mut additional_prom_rules = BTreeMap::new(); +// +// additional_prom_rules.insert( +// self.name.clone(), +// AlertGroup { +// groups: vec![self.clone()], +// }, +// ); +// AlertManagerAdditionalPromRules { +// rules: additional_prom_rules, +// } +// } +// } +#[async_trait] +impl KubePrometheusRule for AlertManagerRuleGroup { + fn name(&self) -> String { + self.name.clone() + } + async fn configure_rule(&self) -> AlertManagerAdditionalPromRules { + let mut additional_prom_rules = BTreeMap::new(); + + additional_prom_rules.insert( + self.name.clone(), + AlertGroup { + groups: vec![self.clone()], + }, + ); + AlertManagerAdditionalPromRules { + rules: additional_prom_rules, + } + } +} diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_default_rules.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_default_rules.rs index 014650cb..060dcc3a 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_default_rules.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_default_rules.rs @@ -1,4 +1,4 @@ -use crate::modules::prometheus::alerts::k8s::{ +use crate::modules::monitoring::alert_rule::alerts::k8s::{ deployment::alert_deployment_unavailable, pod::{alert_container_restarting, alert_pod_not_ready, pod_failed}, pvc::high_pvc_fill_rate_over_two_days, diff --git a/harmony/src/modules/monitoring/okd/crd/alerting_rules.rs b/harmony/src/modules/monitoring/okd/crd/alerting_rules.rs new file mode 100644 index 00000000..688198ea --- /dev/null +++ b/harmony/src/modules/monitoring/okd/crd/alerting_rules.rs @@ -0,0 +1,58 @@ +use std::collections::BTreeMap; + +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule; + +#[derive(CustomResource, Debug, Serialize, Deserialize, Clone, JsonSchema, Default)] +#[kube( + group = "monitoring.openshift.io", + version = "v1", + kind = "AlertingRule", + plural = "alertingrules", + namespaced, + derive = "Default" +)] +#[serde(rename_all = "camelCase")] +pub struct AlertingRuleSpec { + pub groups: serde_json::Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct RuleGroup { + pub name: String, + pub rules: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct Rule { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub alert: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expr: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub for_: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub labels: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub annotations: Option>, +} + +impl From for Rule { + fn from(value: PrometheusAlertRule) -> Self { + Rule { + alert: Some(value.alert), + expr: Some(value.expr), + for_: value.r#for, + labels: Some(value.labels.into_iter().collect::>()), + annotations: Some(value.annotations.into_iter().collect::>()), + } + } +} diff --git a/harmony/src/modules/monitoring/okd/crd/mod.rs b/harmony/src/modules/monitoring/okd/crd/mod.rs new file mode 100644 index 00000000..d2d8cabd --- /dev/null +++ b/harmony/src/modules/monitoring/okd/crd/mod.rs @@ -0,0 +1 @@ +pub mod alerting_rules; diff --git a/harmony/src/modules/monitoring/okd/mod.rs b/harmony/src/modules/monitoring/okd/mod.rs index 86d5422b..62b58ebe 100644 --- a/harmony/src/modules/monitoring/okd/mod.rs +++ b/harmony/src/modules/monitoring/okd/mod.rs @@ -2,6 +2,7 @@ use serde::Serialize; use crate::topology::monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}; +pub mod crd; pub mod score_enable_cluster_monitoring; pub mod score_openshift_alert_rule; pub mod score_openshift_cluster_alert_score; diff --git a/harmony/src/modules/monitoring/okd/score_openshift_alert_rule.rs b/harmony/src/modules/monitoring/okd/score_openshift_alert_rule.rs index 74c091e3..f9cdbdd3 100644 --- a/harmony/src/modules/monitoring/okd/score_openshift_alert_rule.rs +++ b/harmony/src/modules/monitoring/okd/score_openshift_alert_rule.rs @@ -1,16 +1,42 @@ +use kube::api::ObjectMeta; use serde::Serialize; -use crate::{interpret::Interpret, score::Score, topology::Topology}; +use crate::{ + interpret::Interpret, + modules::{ + k8s::resource::K8sResourceScore, + monitoring::okd::{ + OpenshiftClusterAlertSender, + crd::alerting_rules::{AlertingRule, AlertingRuleSpec}, + }, + }, + score::Score, + topology::{K8sclient, Topology, monitoring::AlertRule}, +}; #[derive(Debug, Clone, Serialize)] -pub struct OpenshiftAlertRuleScore {} +pub struct OpenshiftAlertRuleScore { + pub rule: Box>, +} -impl Score for OpenshiftAlertRuleScore { +impl Score for OpenshiftAlertRuleScore { fn name(&self) -> String { "OpenshiftAlertingRuleScore".to_string() } fn create_interpret(&self) -> Box> { - todo!() + let namespace = "openshfift-monitoring".to_string(); + let alerting_rule = AlertingRule { + metadata: ObjectMeta { + name: Some(self.rule.name()), + namespace: Some(namespace.clone()), + ..Default::default() + }, + spec: AlertingRuleSpec { + groups: self.rule.build_rule().unwrap(), + }, + }; + + K8sResourceScore::single(alerting_rule, Some(namespace)).create_interpret() } } diff --git a/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs b/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs index c4ff6cb6..d9750614 100644 --- a/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs +++ b/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs @@ -1,11 +1,18 @@ use serde::Serialize; -use crate::{interpret::Interpret, score::Score, topology::Topology}; +use crate::{ + interpret::Interpret, + modules::monitoring::okd::OpenshiftClusterAlertSender, + score::Score, + topology::{K8sclient, Topology, monitoring::ScrapeTarget}, +}; #[derive(Debug, Clone, Serialize)] -pub struct OpenshiftScrapeTargetScore {} +pub struct OpenshiftScrapeTargetScore { + pub scrape_target: Box>, +} -impl Score for OpenshiftScrapeTargetScore { +impl Score for OpenshiftScrapeTargetScore { fn name(&self) -> String { "OpenshiftAlertingRuleScore".to_string() } diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_default_rules.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_default_rules.rs index e59938b7..5adea7e4 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_default_rules.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_default_rules.rs @@ -1,11 +1,11 @@ -use crate::modules::{ - monitoring::red_hat_cluster_observability::crd::rhob_prometheus_rules::Rule, - prometheus::alerts::k8s::{ +use crate::modules::monitoring::{ + alert_rule::alerts::k8s::{ deployment::alert_deployment_unavailable, pod::{alert_container_restarting, alert_pod_not_ready, pod_failed}, pvc::high_pvc_fill_rate_over_two_days, service::alert_service_down, }, + red_hat_cluster_observability::crd::rhob_prometheus_rules::Rule, }; pub fn build_default_application_rules() -> Vec { diff --git a/harmony/src/modules/prometheus/mod.rs b/harmony/src/modules/prometheus/mod.rs index f0571e94..1a1fddab 100644 --- a/harmony/src/modules/prometheus/mod.rs +++ b/harmony/src/modules/prometheus/mod.rs @@ -1,5 +1,2 @@ -pub mod alerts; pub mod k8s_prometheus_alerting_score; -// #[allow(clippy::module_inception)] -// pub mod prometheus; // pub mod rhob_alerting_score; diff --git a/harmony/src/modules/prometheus/prometheus.rs b/harmony/src/modules/prometheus/prometheus.rs deleted file mode 100644 index 2e2d0b94..00000000 --- a/harmony/src/modules/prometheus/prometheus.rs +++ /dev/null @@ -1,25 +0,0 @@ -// use async_trait::async_trait; -// -// use crate::{ -// inventory::Inventory, -// topology::{ -// PreparationError, PreparationOutcome, -// oberservability::monitoring::{AlertReceiver, AlertSender}, -// }, -// }; -// -// #[async_trait] -// pub trait Monitor { -// async fn install_montoring( -// &self, -// sender: &S, -// inventory: &Inventory, -// receivers: Option>>>, -// ) -> Result; -// -// async fn ensure_monitoring( -// &self, -// sender: &S, -// inventory: &Inventory, -// ) -> Result; -// } -- 2.39.5 From e68426cc3d21f75dd23b39873480b35d5a2a1936 Mon Sep 17 00:00:00 2001 From: wjro Date: Wed, 25 Feb 2026 14:54:10 -0500 Subject: [PATCH 09/15] feat: added implentation for prometheus node exporter external scrape target for openshift cluster alert sender. added alerting rule to return high http error rate --- examples/okd_cluster_alerts/src/main.rs | 19 +- harmony/src/domain/topology/monitoring.rs | 21 ++- harmony/src/modules/k8s/resource.rs | 8 +- .../monitoring/alert_rule/alerts/infra/mod.rs | 1 + .../alert_rule/alerts/infra/opnsense.rs | 15 ++ .../alert_rule/prometheus_alert_rule.rs | 2 + harmony/src/modules/monitoring/okd/crd/mod.rs | 2 + .../monitoring/okd/crd/scrape_target.rs | 72 ++++++++ .../monitoring/okd/crd/service_monitor.rs | 97 ++++++++++ harmony/src/modules/monitoring/okd/mod.rs | 2 + .../okd/score_openshift_alert_rule.rs | 2 +- .../okd/score_openshift_scrape_target.rs | 169 +++++++++++++++++- .../modules/monitoring/scrape_target/mod.rs | 3 + .../scrape_target/prometheus_node_exporter.rs | 76 ++++++++ .../monitoring/scrape_target/server.rs | 97 +++++----- harmony_agent/deploy/src/main.rs | 11 +- 16 files changed, 524 insertions(+), 73 deletions(-) create mode 100644 harmony/src/modules/monitoring/alert_rule/alerts/infra/opnsense.rs create mode 100644 harmony/src/modules/monitoring/okd/crd/scrape_target.rs create mode 100644 harmony/src/modules/monitoring/okd/crd/service_monitor.rs create mode 100644 harmony/src/modules/monitoring/scrape_target/prometheus_node_exporter.rs diff --git a/examples/okd_cluster_alerts/src/main.rs b/examples/okd_cluster_alerts/src/main.rs index f0d8123e..8d9f4f69 100644 --- a/examples/okd_cluster_alerts/src/main.rs +++ b/examples/okd_cluster_alerts/src/main.rs @@ -3,10 +3,11 @@ use harmony::{ modules::monitoring::{ alert_channel::discord_alert_channel::DiscordReceiver, alert_rule::{ - alerts::k8s::pvc::high_pvc_fill_rate_over_two_days, + alerts::{infra::opnsense::high_http_error_rate, k8s::pvc::high_pvc_fill_rate_over_two_days}, prometheus_alert_rule::AlertManagerRuleGroup, }, okd::score_openshift_cluster_alert_score::OpenshiftClusterAlertScore, + scrape_target::prometheus_node_exporter::PrometheusNodeExporter, }, topology::{ K8sAnywhereTopology, @@ -14,7 +15,7 @@ use harmony::{ }, }; -use harmony_macros::hurl; +use harmony_macros::{hurl, ip}; #[tokio::main] async fn main() { @@ -29,9 +30,17 @@ async fn main() { value: "critical".to_string(), }; - let test_rule = high_pvc_fill_rate_over_two_days(); + let high_http_error_rate = high_http_error_rate(); - let additional_rules = AlertManagerRuleGroup::new("test-alerts", vec![test_rule]); + let additional_rules = AlertManagerRuleGroup::new("", vec![high_http_error_rate]); + + let scrape_target = PrometheusNodeExporter { + job_name: "firewall".to_string(), + metrics_path: "/metrics".to_string(), + listen_address: ip!("127.0.0.1"), + port: 9100, + ..Default::default() + }; harmony_cli::run( Inventory::autoload(), @@ -47,7 +56,7 @@ async fn main() { })], sender: harmony::modules::monitoring::okd::OpenshiftClusterAlertSender, rules: vec![Box::new(additional_rules)], - scrape_targets: None, + scrape_targets: Some(vec![Box::new(scrape_target)]), })], None, ) diff --git a/harmony/src/domain/topology/monitoring.rs b/harmony/src/domain/topology/monitoring.rs index 602aea98..805dbcc9 100644 --- a/harmony/src/domain/topology/monitoring.rs +++ b/harmony/src/domain/topology/monitoring.rs @@ -1,9 +1,13 @@ -use std::{any::Any, collections::HashMap}; +use std::{ + any::Any, + collections::{BTreeMap, HashMap}, + net::IpAddr, +}; use async_trait::async_trait; use kube::api::DynamicObject; use log::{debug, info}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::{ data::Version, @@ -75,12 +79,21 @@ pub trait AlertRule: std::fmt::Debug + Send + Sync { /// A generic scrape target that can be added to a sender to scrape metrics from, for example a /// server outside of the cluster -#[async_trait] pub trait ScrapeTarget: std::fmt::Debug + Send + Sync { - async fn install(&self, sender: &S) -> Result; + fn build_scrape_target(&self) -> Result; + fn name(&self) -> String; fn clone_box(&self) -> Box>; } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExternalScrapeTarget { + pub ip: IpAddr, + pub port: i32, + pub interval: Option, + pub path: Option, + pub labels: Option>, +} + /// Alerting interpret to install an alert sender on a given topology #[derive(Debug)] pub struct AlertingInterpret { diff --git a/harmony/src/modules/k8s/resource.rs b/harmony/src/modules/k8s/resource.rs index dde73390..9bbcd78b 100644 --- a/harmony/src/modules/k8s/resource.rs +++ b/harmony/src/modules/k8s/resource.rs @@ -9,7 +9,7 @@ use crate::{ interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::Inventory, score::Score, - topology::{K8sclient, Topology}, + topology::{K8sclient, Topology, k8s::ApplyStrategy}, }; use harmony_types::id::Id; @@ -29,7 +29,7 @@ impl K8sResourceScore { } impl< - K: Resource + K: Resource + std::fmt::Debug + Sync + DeserializeOwned @@ -42,6 +42,7 @@ impl< > Score for K8sResourceScore where ::DynamicType: Default, + ::Scope: ApplyStrategy, { fn create_interpret(&self) -> Box> { Box::new(K8sResourceInterpret { @@ -61,7 +62,7 @@ pub struct K8sResourceInterpret { #[async_trait] impl< - K: Resource + K: Resource + Clone + std::fmt::Debug + DeserializeOwned @@ -73,6 +74,7 @@ impl< > Interpret for K8sResourceInterpret where ::DynamicType: Default, + ::Scope: ApplyStrategy, { async fn execute( &self, diff --git a/harmony/src/modules/monitoring/alert_rule/alerts/infra/mod.rs b/harmony/src/modules/monitoring/alert_rule/alerts/infra/mod.rs index 47fbc2f1..2b11bded 100644 --- a/harmony/src/modules/monitoring/alert_rule/alerts/infra/mod.rs +++ b/harmony/src/modules/monitoring/alert_rule/alerts/infra/mod.rs @@ -1 +1,2 @@ pub mod dell_server; +pub mod opnsense; diff --git a/harmony/src/modules/monitoring/alert_rule/alerts/infra/opnsense.rs b/harmony/src/modules/monitoring/alert_rule/alerts/infra/opnsense.rs new file mode 100644 index 00000000..c163a76d --- /dev/null +++ b/harmony/src/modules/monitoring/alert_rule/alerts/infra/opnsense.rs @@ -0,0 +1,15 @@ +use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule; + +pub fn high_http_error_rate() -> PrometheusAlertRule { + let expression = r#"( + sum(rate(http_requests_total{status=~"5.."}[5m])) by (job, route, service) + / + sum(rate(http_requests_total[5m])) by (job, route, service) +) > 0.05 and sum(rate(http_requests_total[5m])) by (job, route, service) > 10"#; + + PrometheusAlertRule::new("HighApplicationErrorRate", expression) + .for_duration("10m") + .label("severity", "warning") + .annotation("summary", "High HTTP error rate on {{ $labels.job }}") + .annotation("description", "Job {{ $labels.job }} (route {{ $labels.route }}) has an error rate > 5% over the last 10m.") +} diff --git a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs index cbd95ca9..ea929a31 100644 --- a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs +++ b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs @@ -91,6 +91,7 @@ impl AlertRule for AlertManagerRuleGroup { fn name(&self) -> String { self.name.clone() } + fn clone_box(&self) -> Box> { Box::new(self.clone()) } @@ -104,6 +105,7 @@ impl AlertRule for AlertManagerRuleGroup { fn name(&self) -> String { self.name.clone() } + fn clone_box(&self) -> Box> { Box::new(self.clone()) } diff --git a/harmony/src/modules/monitoring/okd/crd/mod.rs b/harmony/src/modules/monitoring/okd/crd/mod.rs index d2d8cabd..a3b92bd0 100644 --- a/harmony/src/modules/monitoring/okd/crd/mod.rs +++ b/harmony/src/modules/monitoring/okd/crd/mod.rs @@ -1 +1,3 @@ pub mod alerting_rules; +pub mod scrape_target; +pub mod service_monitor; diff --git a/harmony/src/modules/monitoring/okd/crd/scrape_target.rs b/harmony/src/modules/monitoring/okd/crd/scrape_target.rs new file mode 100644 index 00000000..6a77f003 --- /dev/null +++ b/harmony/src/modules/monitoring/okd/crd/scrape_target.rs @@ -0,0 +1,72 @@ +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(CustomResource, Debug, Serialize, Deserialize, Clone, JsonSchema, Default)] +#[kube( + group = "monitoring.coreos.com", + version = "v1alpha1", + kind = "ScrapeConfig", + plural = "scrapeconfigs", + namespaced, + derive = "Default" +)] +#[serde(rename_all = "camelCase")] +pub struct ScrapeConfigSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub job_name: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metrics_path: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scrape_interval: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scrape_timeout: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scheme: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub static_configs: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub relabelings: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metric_relabelings: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct StaticConfig { + /// targets: ["host:port"] + pub targets: Vec, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub labels: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct RelabelConfig { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_labels: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub separator: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_label: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replacement: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub action: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub regex: Option, +} diff --git a/harmony/src/modules/monitoring/okd/crd/service_monitor.rs b/harmony/src/modules/monitoring/okd/crd/service_monitor.rs new file mode 100644 index 00000000..f60f1399 --- /dev/null +++ b/harmony/src/modules/monitoring/okd/crd/service_monitor.rs @@ -0,0 +1,97 @@ +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(CustomResource, Debug, Serialize, Deserialize, Clone, JsonSchema, Default)] +#[kube( + group = "monitoring.coreos.com", + version = "v1", + kind = "ServiceMonitor", + plural = "servicemonitors", + namespaced, + derive = "Default" +)] +#[serde(rename_all = "camelCase")] +pub struct ServiceMonitorSpec { + /// The label to use to retrieve the job name from. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub job_label: Option, + + /// TargetLabels transfers labels on the Kubernetes Service onto the target. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_labels: Option>, + + /// PodTargetLabels transfers labels on the Kubernetes Pod onto the target. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pod_target_labels: Option>, + + /// A list of endpoints allowed as part of this ServiceMonitor. + pub endpoints: Vec, + + /// Selector to select Endpoints objects. + pub selector: LabelSelector, + + /// Selector to select which namespaces the Kubernetes Endpoints objects are discovered from. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub namespace_selector: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct Endpoint { + /// Name of the service port this endpoint refers to. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub port: Option, + + /// HTTP path to scrape for metrics. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + + /// HTTP scheme to use for scraping. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scheme: Option, + + /// Interval at which metrics should be scraped. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub interval: Option, + + /// Timeout after which the scrape is ended. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scrape_timeout: Option, + + /// HonorLabels chooses the metric's labels on collisions with target labels. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub honor_labels: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct LabelSelector { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub match_labels: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub match_expressions: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct LabelSelectorRequirement { + pub key: String, + pub operator: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub values: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct NamespaceSelector { + /// Boolean describing whether all namespaces are selected in contrast to a list restricting them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub any: Option, + + /// List of namespace names. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub match_names: Option>, +} diff --git a/harmony/src/modules/monitoring/okd/mod.rs b/harmony/src/modules/monitoring/okd/mod.rs index 62b58ebe..b2251aa0 100644 --- a/harmony/src/modules/monitoring/okd/mod.rs +++ b/harmony/src/modules/monitoring/okd/mod.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use serde::Serialize; use crate::topology::monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}; diff --git a/harmony/src/modules/monitoring/okd/score_openshift_alert_rule.rs b/harmony/src/modules/monitoring/okd/score_openshift_alert_rule.rs index f9cdbdd3..9c902c66 100644 --- a/harmony/src/modules/monitoring/okd/score_openshift_alert_rule.rs +++ b/harmony/src/modules/monitoring/okd/score_openshift_alert_rule.rs @@ -25,7 +25,7 @@ impl Score for OpenshiftAlertRuleScore { } fn create_interpret(&self) -> Box> { - let namespace = "openshfift-monitoring".to_string(); + let namespace = "openshift-monitoring".to_string(); let alerting_rule = AlertingRule { metadata: ObjectMeta { name: Some(self.rule.name()), diff --git a/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs b/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs index d9750614..ed497320 100644 --- a/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs +++ b/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs @@ -1,10 +1,32 @@ +use std::collections::BTreeMap; + +use async_trait::async_trait; +use harmony_types::id::Id; +use k8s_openapi::{ + api::core::v1::{ + EndpointAddress, EndpointPort, EndpointSubset, Endpoints, Service, ServicePort, ServiceSpec, + }, + apimachinery::pkg::util::intstr::IntOrString, +}; +use kube::api::ObjectMeta; use serde::Serialize; use crate::{ - interpret::Interpret, - modules::monitoring::okd::OpenshiftClusterAlertSender, + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + modules::{ + k8s::resource::K8sResourceScore, + monitoring::okd::{ + OpenshiftClusterAlertSender, + crd::service_monitor::{Endpoint, LabelSelector, ServiceMonitor, ServiceMonitorSpec}, + }, + }, score::Score, - topology::{K8sclient, Topology, monitoring::ScrapeTarget}, + topology::{ + K8sclient, Topology, + monitoring::{ExternalScrapeTarget, ScrapeTarget}, + }, }; #[derive(Debug, Clone, Serialize)] @@ -18,6 +40,147 @@ impl Score for OpenshiftScrapeTargetScore { } fn create_interpret(&self) -> Box> { + Box::new(OpenshiftScrapeTargetInterpret { + scrape_target: self.scrape_target.clone(), + }) + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct OpenshiftScrapeTargetInterpret { + scrape_target: Box>, +} + +#[async_trait] +impl Interpret for OpenshiftScrapeTargetInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + let namespace = "openshift-monitoring".to_string(); + let name = self.scrape_target.name(); + let external_target = self + .scrape_target + .build_scrape_target() + .expect("failed to build scrape target ExternalScrapeTarget"); + + let (service, endpoints, service_monitor) = + self.to_k8s_resources(&name, &namespace, external_target); + + K8sResourceScore::single(service, Some(namespace.clone())) + .create_interpret() + .execute(inventory, topology) + .await?; + K8sResourceScore::single(endpoints, Some(namespace.clone())) + .create_interpret() + .execute(inventory, topology) + .await?; + K8sResourceScore::single(service_monitor, Some(namespace.clone())) + .create_interpret() + .execute(inventory, topology) + .await?; + + Ok(Outcome::success( + "Installed scrape target of Openshift".to_string(), + )) + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("OpenshiftScrapeTargetInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { todo!() } } + +impl OpenshiftScrapeTargetInterpret { + /// Maps the generic intent into the 3 required Kubernetes objects + pub fn to_k8s_resources( + &self, + name: &str, + namespace: &str, + external_target: ExternalScrapeTarget, + ) -> (Service, Endpoints, ServiceMonitor) { + let mut labels = external_target.labels.clone().unwrap_or(BTreeMap::new()); + + labels.insert("harmony/target-name".to_string(), name.to_string().clone()); + + let service = Service { + metadata: ObjectMeta { + name: Some(name.to_string().clone()), + namespace: Some(namespace.to_string()), + labels: Some(labels.clone()), + ..Default::default() + }, + spec: Some(ServiceSpec { + cluster_ip: Some("None".to_string()), // Headless + ports: Some(vec![ServicePort { + name: Some("metrics".to_string()), + port: external_target.port.clone(), + target_port: Some(IntOrString::Int(external_target.port)), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + }; + + let endpoints = Endpoints { + metadata: ObjectMeta { + name: Some(name.to_string().clone()), + namespace: Some(namespace.to_string()), + labels: Some(labels.clone()), + ..Default::default() + }, + subsets: Some(vec![EndpointSubset { + addresses: Some(vec![EndpointAddress { + ip: external_target.ip.to_string().clone(), + ..Default::default() + }]), + ports: Some(vec![EndpointPort { + name: Some("metrics".to_string()), + port: external_target.port, + ..Default::default() + }]), + ..Default::default() + }]), + }; + + let service_monitor = ServiceMonitor { + metadata: ObjectMeta { + name: Some(name.to_string().clone()), + namespace: Some(namespace.to_string()), + ..Default::default() + }, + spec: ServiceMonitorSpec { + job_label: Some("harmony/target-name".to_string()), + endpoints: vec![Endpoint { + port: Some("metrics".to_string()), + interval: external_target.interval.clone(), + path: external_target.path.clone(), + ..Default::default() + }], + selector: LabelSelector { + match_labels: Some(BTreeMap::from([( + "harmony/target-name".to_string(), + name.to_string().clone(), + )])), + ..Default::default() + }, + ..Default::default() + }, + }; + + (service, endpoints, service_monitor) + } +} diff --git a/harmony/src/modules/monitoring/scrape_target/mod.rs b/harmony/src/modules/monitoring/scrape_target/mod.rs index 74f47ad3..a270cfbd 100644 --- a/harmony/src/modules/monitoring/scrape_target/mod.rs +++ b/harmony/src/modules/monitoring/scrape_target/mod.rs @@ -1 +1,4 @@ +use std::collections::BTreeMap; + +pub mod prometheus_node_exporter; pub mod server; diff --git a/harmony/src/modules/monitoring/scrape_target/prometheus_node_exporter.rs b/harmony/src/modules/monitoring/scrape_target/prometheus_node_exporter.rs new file mode 100644 index 00000000..0011c0b0 --- /dev/null +++ b/harmony/src/modules/monitoring/scrape_target/prometheus_node_exporter.rs @@ -0,0 +1,76 @@ +use std::{ + collections::BTreeMap, + net::{IpAddr, Ipv4Addr}, +}; + +use harmony_macros::ip; +use serde::Serialize; + +use crate::{ + interpret::InterpretError, + modules::monitoring::okd::{ + OpenshiftClusterAlertSender, + crd::scrape_target::{ScrapeConfigSpec, StaticConfig}, + }, + topology::monitoring::{ExternalScrapeTarget, ScrapeTarget}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct PrometheusNodeExporter { + pub job_name: String, + pub metrics_path: String, + pub scrape_interval: Option, + pub scrape_timeout: Option, + pub listen_address: IpAddr, + pub port: i32, + pub labels: Option>, +} + +impl Default for PrometheusNodeExporter { + fn default() -> Self { + Self { + job_name: "node-exporter".into(), + metrics_path: "/metrics".into(), + scrape_interval: None, + scrape_timeout: None, + listen_address: ip!("127.0.0.1"), + port: 9100, + labels: None, + } + } +} + +impl PrometheusNodeExporter { + fn new(job_name: String, metrics_path: String, listen_address: IpAddr, port: i32) -> Self { + Self { + job_name: job_name, + metrics_path: metrics_path, + scrape_interval: None, + scrape_timeout: None, + listen_address: listen_address, + port: port, + labels: None, + } + } +} + +impl ScrapeTarget for PrometheusNodeExporter { + fn build_scrape_target(&self) -> Result { + let external_target = ExternalScrapeTarget { + ip: self.listen_address.clone(), + port: self.port.clone(), + interval: self.scrape_interval.clone(), + path: Some(self.metrics_path.clone()), + labels: self.labels.clone(), + }; + Ok(external_target) + } + + fn name(&self) -> String { + self.job_name.clone() + } + + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } +} diff --git a/harmony/src/modules/monitoring/scrape_target/server.rs b/harmony/src/modules/monitoring/scrape_target/server.rs index c11c792e..eb48cf5f 100644 --- a/harmony/src/modules/monitoring/scrape_target/server.rs +++ b/harmony/src/modules/monitoring/scrape_target/server.rs @@ -10,7 +10,7 @@ use crate::{ crd_alertmanager_config::CRDPrometheus, crd_scrape_config::{Params, RelabelConfig, ScrapeConfig, ScrapeConfigSpec, StaticConfig}, }, - topology::monitoring::ScrapeTarget, + topology::monitoring::{ExternalScrapeTarget, ScrapeTarget}, }; #[derive(Debug, Clone, Serialize)] @@ -22,59 +22,56 @@ pub struct Server { pub domain: String, } -#[async_trait] impl ScrapeTarget for Server { - async fn install(&self, sender: &CRDPrometheus) -> Result { - let scrape_config_spec = ScrapeConfigSpec { - static_configs: Some(vec![StaticConfig { - targets: vec![self.ip.to_string()], - labels: None, - }]), - scrape_interval: Some("2m".to_string()), - kubernetes_sd_configs: None, - http_sd_configs: None, - file_sd_configs: None, - dns_sd_configs: None, - params: Some(Params { - auth: Some(vec![self.auth.clone()]), - module: Some(vec![self.module.clone()]), - }), - consul_sd_configs: None, - relabel_configs: Some(vec![RelabelConfig { - action: None, - source_labels: Some(vec!["__address__".to_string()]), - separator: None, - target_label: Some("__param_target".to_string()), - regex: None, - replacement: Some(format!("snmp.{}:31080", self.domain.clone())), - modulus: None, - }]), - metric_relabel_configs: None, - metrics_path: Some("/snmp".to_string()), - scrape_timeout: Some("2m".to_string()), - job_name: Some(format!("snmp_exporter/cloud/{}", self.name.clone())), - scheme: None, - }; - - let scrape_config = ScrapeConfig { - metadata: ObjectMeta { - name: Some(self.name.clone()), - namespace: Some(sender.namespace.clone()), - ..Default::default() - }, - spec: scrape_config_spec, - }; - sender - .client - .apply(&scrape_config, Some(&sender.namespace.clone())) - .await?; - Ok(Outcome::success(format!( - "installed scrape target {}", - self.name.clone() - ))) + fn build_scrape_target(&self) -> Result { + todo!() + // let scrape_config_spec = ScrapeConfigSpec { + // static_configs: Some(vec![StaticConfig { + // targets: vec![self.ip.to_string()], + // labels: None, + // }]), + // scrape_interval: Some("2m".to_string()), + // kubernetes_sd_configs: None, + // http_sd_configs: None, + // file_sd_configs: None, + // dns_sd_configs: None, + // params: Some(Params { + // auth: Some(vec![self.auth.clone()]), + // module: Some(vec![self.module.clone()]), + // }), + // consul_sd_configs: None, + // relabel_configs: Some(vec![RelabelConfig { + // action: None, + // source_labels: Some(vec!["__address__".to_string()]), + // separator: None, + // target_label: Some("__param_target".to_string()), + // regex: None, + // replacement: Some(format!("snmp.{}:31080", self.domain.clone())), + // modulus: None, + // }]), + // metric_relabel_configs: None, + // metrics_path: Some("/snmp".to_string()), + // scrape_timeout: Some("2m".to_string()), + // job_name: Some(format!("snmp_exporter/cloud/{}", self.name.clone())), + // scheme: None, + // }; + // + // let scrape_config = ScrapeConfig { + // metadata: ObjectMeta { + // name: Some(self.name.clone()), + // namespace: Some(sender.namespace.clone()), + // ..Default::default() + // }, + // spec: scrape_config_spec, + // }; + // Ok(serde_json::to_value(scrape_config).map_err(|e| InterpretError::new(e.to_string()))?) } fn clone_box(&self) -> Box> { Box::new(self.clone()) } + + fn name(&self) -> String { + todo!() + } } diff --git a/harmony_agent/deploy/src/main.rs b/harmony_agent/deploy/src/main.rs index b12d82e9..548f3745 100644 --- a/harmony_agent/deploy/src/main.rs +++ b/harmony_agent/deploy/src/main.rs @@ -1,12 +1,9 @@ use harmony::{ inventory::Inventory, - modules::{ - application::{ - ApplicationScore, - backend_app::{BackendApp, BuildCommand}, - features::{Monitoring, PackagingDeployment}, - }, - monitoring::alert_channel::discord_alert_channel::DiscordWebhook, + modules::application::{ + ApplicationScore, + backend_app::{BackendApp, BuildCommand}, + features::{Monitoring, PackagingDeployment}, }, topology::K8sAnywhereTopology, }; -- 2.39.5 From 621aed4903c9b15a99cd62dca4db0e2eddef1566 Mon Sep 17 00:00:00 2001 From: wjro Date: Wed, 25 Feb 2026 15:48:12 -0500 Subject: [PATCH 10/15] wip: refactoring kubeprometheus --- examples/monitoring/src/main.rs | 4 +- examples/monitoring_with_tenant/src/main.rs | 4 +- .../observability/kube_prometheus.rs | 81 +++++- .../alert_channel/discord_alert_channel.rs | 245 +--------------- .../alert_channel/webhook_receiver.rs | 265 +----------------- .../alert_rule/prometheus_alert_rule.rs | 3 +- .../helm_prometheus_alert_score.rs | 22 +- .../modules/monitoring/kube_prometheus/mod.rs | 151 +++++++++- .../monitoring/kube_prometheus/prometheus.rs | 155 ---------- .../score_kube_prometheus_alert_receivers.rs | 23 ++ .../score_kube_prometheus_rule.rs | 23 ++ .../score_kube_prometheus_scrape_target.rs | 23 ++ .../monitoring/prometheus/prometheus.rs | 3 +- 13 files changed, 317 insertions(+), 685 deletions(-) delete mode 100644 harmony/src/modules/monitoring/kube_prometheus/prometheus.rs create mode 100644 harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_alert_receivers.rs create mode 100644 harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_rule.rs create mode 100644 harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs diff --git a/examples/monitoring/src/main.rs b/examples/monitoring/src/main.rs index b4fc13d8..b465ff10 100644 --- a/examples/monitoring/src/main.rs +++ b/examples/monitoring/src/main.rs @@ -16,7 +16,7 @@ use harmony::{ prometheus_alert_rule::AlertManagerRuleGroup, }, kube_prometheus::{ - helm_prometheus_alert_score::HelmPrometheusAlertingScore, + helm_prometheus_alert_score::KubePrometheusAlertingScore, types::{ HTTPScheme, MatchExpression, Operator, Selector, ServiceMonitor, ServiceMonitorEndpoint, @@ -74,7 +74,7 @@ async fn main() { endpoints: vec![service_monitor_endpoint], ..Default::default() }; - let alerting_score = HelmPrometheusAlertingScore { + let alerting_score = KubePrometheusAlertingScore { receivers: vec![Box::new(discord_receiver)], rules: vec![Box::new(additional_rules), Box::new(additional_rules2)], service_monitors: vec![service_monitor], diff --git a/examples/monitoring_with_tenant/src/main.rs b/examples/monitoring_with_tenant/src/main.rs index 04d270d4..77145fb2 100644 --- a/examples/monitoring_with_tenant/src/main.rs +++ b/examples/monitoring_with_tenant/src/main.rs @@ -10,7 +10,7 @@ use harmony::{ prometheus_alert_rule::AlertManagerRuleGroup, }, kube_prometheus::{ - helm_prometheus_alert_score::HelmPrometheusAlertingScore, + helm_prometheus_alert_score::KubePrometheusAlertingScore, types::{ HTTPScheme, MatchExpression, Operator, Selector, ServiceMonitor, ServiceMonitorEndpoint, @@ -80,7 +80,7 @@ async fn main() { ..Default::default() }; - let alerting_score = HelmPrometheusAlertingScore { + let alerting_score = KubePrometheusAlertingScore { receivers: vec![Box::new(discord_receiver)], rules: vec![Box::new(additional_rules)], service_monitors: vec![service_monitor], diff --git a/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs b/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs index dd30b5b5..496e2474 100644 --- a/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs +++ b/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs @@ -1,8 +1,14 @@ use async_trait::async_trait; use crate::{ + interpret::InterpretError, inventory::Inventory, - modules::monitoring::kube_prometheus::prometheus::KubePrometheus, + modules::monitoring::kube_prometheus::{ + KubePrometheus, helm::kube_prometheus_helm_chart::kube_prometheus_helm_chart_score, + score_kube_prometheus_alert_receivers::KubePrometheusReceiverScore, + score_kube_prometheus_rule::KubePrometheusRuleScore, score_kube_prometheus_scrape_target::KubePrometheusScrapeTargetScore, + }, + score::Score, topology::{ K8sAnywhereTopology, PreparationError, PreparationOutcome, monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, @@ -16,34 +22,93 @@ impl Observability for K8sAnywhereTopology { sender: &KubePrometheus, inventory: &Inventory, ) -> Result { - todo!() + kube_prometheus_helm_chart_score(sender.config.clone()) + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(e.to_string()))?; + + Ok(PreparationOutcome::Success { + details: "Successfully installed kubeprometheus alert sender".to_string(), + }) } async fn install_receivers( &self, - sender: &KubePrometheus, + _sender: &KubePrometheus, inventory: &Inventory, receivers: Option>>>, ) -> Result { - todo!() + let receivers = match receivers { + Some(r) if !r.is_empty() => r, + _ => return Ok(PreparationOutcome::Noop), + }; + + for receiver in receivers { + let score = KubePrometheusReceiverScore { receiver }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Failed to install receiver: {}", e)))?; + } + + Ok(PreparationOutcome::Success { + details: "All alert receivers installed successfully".to_string(), + }) } async fn install_rules( &self, - sender: &KubePrometheus, + _sender: &KubePrometheus, inventory: &Inventory, rules: Option>>>, ) -> Result { - todo!() + let rules = match rules { + Some(r) if !r.is_empty() => r, + _ => return Ok(PreparationOutcome::Noop), + }; + + for rule in rules { + let score = KubePrometheusRuleScore { rule }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Failed to install rule: {}", e)))?; + } + + Ok(PreparationOutcome::Success { + details: "All alert rules installed successfully".to_string(), + }) } async fn add_scrape_targets( &self, - sender: &KubePrometheus, + _sender: &KubePrometheus, inventory: &Inventory, scrape_targets: Option>>>, ) -> Result { - todo!() + let scrape_targets = match scrape_targets { + Some(r) if !r.is_empty() => r, + _ => return Ok(PreparationOutcome::Noop), + }; + + for scrape_target in scrape_targets { + let score = KubePrometheusScrapeTargetScore { scrape_target }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Failed to install rule: {}", e)))?; + } + + Ok(PreparationOutcome::Success { + details: "All scrap targets installed successfully".to_string(), + }) } async fn ensure_monitoring_installed( diff --git a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs index de720eea..b3765464 100644 --- a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs +++ b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs @@ -12,10 +12,10 @@ use serde_json::json; use serde_yaml::{Mapping, Value}; use crate::infra::kube::kube_resource_to_dynamic; +use crate::modules::monitoring::kube_prometheus::KubePrometheus; use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{ AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus, }; -use crate::modules::monitoring::kube_prometheus::prometheus::KubePrometheus; use crate::modules::monitoring::okd::OpenshiftClusterAlertSender; use crate::modules::monitoring::red_hat_cluster_observability::RedHatClusterObservability; use crate::topology::monitoring::{AlertRoute, MatchOp}; @@ -31,61 +31,6 @@ pub struct DiscordReceiver { pub url: Url, pub route: AlertRoute, } -// -// #[derive(Debug, Clone, Serialize)] -// pub struct DiscordWebhook { -// pub name: String, -// pub url: Url, -// pub selectors: Vec>, -// } -// -// impl DiscordWebhook { -// fn get_receiver_config(&self) -> Result { -// let secret_name = format!("{}-secret", self.name.clone()); -// let webhook_key = format!("{}", self.url.clone()); -// -// let mut string_data = BTreeMap::new(); -// string_data.insert("webhook-url".to_string(), webhook_key.clone()); -// -// let secret = Secret { -// metadata: kube::core::ObjectMeta { -// name: Some(secret_name.clone()), -// ..Default::default() -// }, -// string_data: Some(string_data), -// type_: Some("Opaque".to_string()), -// ..Default::default() -// }; -// -// let mut matchers: Vec = Vec::new(); -// for selector in &self.selectors { -// trace!("selector: {:#?}", selector); -// for (k, v) in selector { -// matchers.push(format!("{} = {}", k, v)); -// } -// } -// -// Ok(AlertManagerReceiver { -// additional_ressources: vec![kube_resource_to_dynamic(&secret)?], -// -// receiver_config: json!({ -// "name": self.name, -// "discord_configs": [ -// { -// "webhook_url": self.url.clone(), -// "title": "{{ template \"discord.default.title\" . }}", -// "message": "{{ template \"discord.default.message\" . }}" -// } -// ] -// }), -// route_config: json!({ -// "receiver": self.name, -// "matchers": matchers, -// -// }), -// }) -// } -// } impl AlertReceiver for DiscordReceiver { fn build_route(&self) -> Result { @@ -197,77 +142,6 @@ impl AlertReceiver for DiscordReceiver { } } -// -// #[async_trait] -// impl AlertReceiver for DiscordWebhook { -// fn as_alertmanager_receiver(&self) -> Result { -// todo!() -// } -// -// async fn install(&self, sender: &RHOBObservability) -> Result { -// let ns = sender.namespace.clone(); -// -// let config = self.get_receiver_config()?; -// for resource in config.additional_ressources.iter() { -// todo!("can I apply a dynamicresource"); -// // sender.client.apply(resource, Some(&ns)).await; -// } -// -// let spec = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfigSpec { -// data: json!({ -// "route": { -// "receiver": self.name, -// }, -// "receivers": [ -// config.receiver_config -// ] -// }), -// }; -// -// let alertmanager_configs = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfig { -// metadata: ObjectMeta { -// name: Some(self.name.clone().to_string()), -// labels: Some(std::collections::BTreeMap::from([( -// "alertmanagerConfig".to_string(), -// "enabled".to_string(), -// )])), -// namespace: Some(sender.namespace.clone()), -// ..Default::default() -// }, -// spec, -// }; -// debug!( -// "alertmanager_configs yaml:\n{:#?}", -// serde_yaml::to_string(&alertmanager_configs) -// ); -// debug!( -// "alert manager configs: \n{:#?}", -// alertmanager_configs.clone() -// ); -// -// sender -// .client -// .apply(&alertmanager_configs, Some(&sender.namespace)) -// .await?; -// Ok(Outcome::success(format!( -// "installed rhob-alertmanagerconfigs for {}", -// self.name -// ))) -// } -// -// fn name(&self) -> String { -// "webhook-receiver".to_string() -// } -// -// fn clone_box(&self) -> Box> { -// Box::new(self.clone()) -// } -// -// fn as_any(&self) -> &dyn Any { -// self -// } -// } -// // #[async_trait] // impl AlertReceiver for DiscordWebhook { // fn as_alertmanager_receiver(&self) -> Result { @@ -377,120 +251,3 @@ impl AlertReceiver for DiscordReceiver { // self.get_config().await // } // } -// -// #[async_trait] -// impl AlertReceiver for DiscordWebhook { -// fn as_alertmanager_receiver(&self) -> Result { -// todo!() -// } -// async fn install(&self, sender: &KubePrometheus) -> Result { -// sender.install_receiver(self).await -// } -// fn clone_box(&self) -> Box> { -// Box::new(self.clone()) -// } -// fn name(&self) -> String { -// "discord-webhook".to_string() -// } -// fn as_any(&self) -> &dyn Any { -// self -// } -// } -// -// #[async_trait] -// impl KubePrometheusReceiver for DiscordWebhook { -// fn name(&self) -> String { -// self.name.clone().to_string() -// } -// async fn configure_receiver(&self) -> AlertManagerChannelConfig { -// self.get_config().await -// } -// } -// -// #[async_trait] -// impl AlertChannelConfig for DiscordWebhook { -// async fn get_config(&self) -> AlertManagerChannelConfig { -// let channel_global_config = None; -// let channel_receiver = self.alert_channel_receiver().await; -// let channel_route = self.alert_channel_route().await; -// -// AlertManagerChannelConfig { -// channel_global_config, -// channel_receiver, -// channel_route, -// } -// } -// } -// -// impl DiscordWebhook { -// async fn alert_channel_route(&self) -> serde_yaml::Value { -// let mut route = Mapping::new(); -// route.insert( -// Value::String("receiver".to_string()), -// Value::String(self.name.clone().to_string()), -// ); -// route.insert( -// Value::String("matchers".to_string()), -// Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]), -// ); -// route.insert(Value::String("continue".to_string()), Value::Bool(true)); -// Value::Mapping(route) -// } -// -// async fn alert_channel_receiver(&self) -> serde_yaml::Value { -// let mut receiver = Mapping::new(); -// receiver.insert( -// Value::String("name".to_string()), -// Value::String(self.name.clone().to_string()), -// ); -// -// let mut discord_config = Mapping::new(); -// discord_config.insert( -// Value::String("webhook_url".to_string()), -// Value::String(self.url.to_string()), -// ); -// -// receiver.insert( -// Value::String("discord_configs".to_string()), -// Value::Sequence(vec![Value::Mapping(discord_config)]), -// ); -// -// Value::Mapping(receiver) -// } -// } -// -// #[cfg(test)] -// mod tests { -// use super::*; -// -// #[tokio::test] -// async fn discord_serialize_should_match() { -// let discord_receiver = DiscordWebhook { -// name: K8sName("test-discord".to_string()), -// url: Url::Url(url::Url::parse("https://discord.i.dont.exist.com").unwrap()), -// selectors: vec![], -// }; -// -// let discord_receiver_receiver = -// serde_yaml::to_string(&discord_receiver.alert_channel_receiver().await).unwrap(); -// println!("receiver \n{:#}", discord_receiver_receiver); -// let discord_receiver_receiver_yaml = r#"name: test-discord -// discord_configs: -// - webhook_url: https://discord.i.dont.exist.com/ -// "# -// .to_string(); -// -// let discord_receiver_route = -// serde_yaml::to_string(&discord_receiver.alert_channel_route().await).unwrap(); -// println!("route \n{:#}", discord_receiver_route); -// let discord_receiver_route_yaml = r#"receiver: test-discord -// matchers: -// - alertname!=Watchdog -// continue: true -// "# -// .to_string(); -// -// assert_eq!(discord_receiver_receiver, discord_receiver_receiver_yaml); -// assert_eq!(discord_receiver_route, discord_receiver_route_yaml); -// } -// } diff --git a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs index ad329575..1f6e56e1 100644 --- a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs +++ b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs @@ -26,7 +26,7 @@ pub struct WebhookReceiver { impl WebhookReceiver { fn build_receiver(&self) -> serde_json::Value { - let receiver = json!({ + json!({ "name": self.name, "webhookConfigs": [ { @@ -37,13 +37,12 @@ impl WebhookReceiver { } } } - ]}); - receiver + ]}) } + fn build_route(&self) -> serde_json::Value { - let route = json!({ - "name": self.name}); - route + json!({ + "name": self.name}) } } @@ -76,53 +75,6 @@ impl AlertReceiver for WebhookReceiver { fn build_route(&self) -> Result { let route = self.build_route(); serde_yaml::to_value(route).map_err(|e| InterpretError::new(e.to_string())) - // let spec = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfigSpec { - // data: json!({ - // "route": { - // "receiver": self.name, - // }, - // "receivers": [ - // { - // "name": self.name, - // "webhookConfigs": [ - // { - // "url": self.url, - // "httpConfig": { - // "tlsConfig": { - // "insecureSkipVerify": true - // } - // } - // } - // ] - // } - // ] - // }), - // }; - // - // let alertmanager_configs = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfig { - // metadata: ObjectMeta { - // name: Some(self.name.clone()), - // labels: Some(std::collections::BTreeMap::from([( - // "alertmanagerConfig".to_string(), - // "enabled".to_string(), - // )])), - // namespace: Some(sender.namespace.clone()), - // ..Default::default() - // }, - // spec, - // }; - // debug!( - // "alert manager configs: \n{:#?}", - // alertmanager_configs.clone() - // ); - // - // sender - // .client - // .apply(&alertmanager_configs, Some(&sender.namespace)); - // Ok(Outcome::success(format!( - // "installed rhob-alertmanagerconfigs for {}", - // self.name - // ))) } fn name(&self) -> String { @@ -152,210 +104,3 @@ impl AlertReceiver for WebhookReceiver { } } -// #[async_trait] -// impl AlertReceiver for WebhookReceiver { -// fn as_alertmanager_receiver(&self) -> Result { -// todo!() -// } -// async fn build_route(&self, sender: &CRDPrometheus) -> Result { -// let spec = crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::AlertmanagerConfigSpec { -// data: json!({ -// "route": { -// "receiver": self.name, -// }, -// "receivers": [ -// { -// "name": self.name, -// "webhookConfigs": [ -// { -// "url": self.url, -// } -// ] -// } -// ] -// }), -// }; -// -// let alertmanager_configs = crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::AlertmanagerConfig { -// metadata: ObjectMeta { -// name: Some(self.name.clone()), -// labels: Some(std::collections::BTreeMap::from([( -// "alertmanagerConfig".to_string(), -// "enabled".to_string(), -// )])), -// namespace: Some(sender.namespace.clone()), -// ..Default::default() -// }, -// spec, -// }; -// debug!( -// "alert manager configs: \n{:#?}", -// alertmanager_configs.clone() -// ); -// -// sender -// .client -// .apply(&alertmanager_configs, Some(&sender.namespace)) -// .await?; -// Ok(Outcome::success(format!( -// "installed crd-alertmanagerconfigs for {}", -// self.name -// ))) -// } -// -// fn name(&self) -> String { -// "webhook-receiver".to_string() -// } -// -// fn clone_box(&self) -> Box> { -// Box::new(self.clone()) -// } -// -// fn as_any(&self) -> &dyn Any { -// self -// } -// } -// -// #[async_trait] -// impl AlertReceiver for WebhookReceiver { -// fn as_alertmanager_receiver(&self) -> Result { -// todo!() -// } -// async fn build_route(&self, sender: &Prometheus) -> Result { -// sender.install_receiver(self).await -// } -// fn name(&self) -> String { -// "webhook-receiver".to_string() -// } -// fn clone_box(&self) -> Box> { -// Box::new(self.clone()) -// } -// fn as_any(&self) -> &dyn Any { -// self -// } -// } -// -// #[async_trait] -// impl PrometheusReceiver for WebhookReceiver { -// fn name(&self) -> String { -// self.name.clone() -// } -// async fn configure_receiver(&self) -> AlertManagerChannelConfig { -// self.get_config().await -// } -// } -// -// #[async_trait] -// impl AlertReceiver for WebhookReceiver { -// fn as_alertmanager_receiver(&self) -> Result { -// todo!() -// } -// async fn build_route(&self, sender: &KubePrometheus) -> Result { -// sender.install_receiver(self).await -// } -// fn name(&self) -> String { -// "webhook-receiver".to_string() -// } -// fn clone_box(&self) -> Box> { -// Box::new(self.clone()) -// } -// fn as_any(&self) -> &dyn Any { -// self -// } -// } -// -// #[async_trait] -// impl KubePrometheusReceiver for WebhookReceiver { -// fn name(&self) -> String { -// self.name.clone() -// } -// async fn configure_receiver(&self) -> AlertManagerChannelConfig { -// self.get_config().await -// } -// } -// -// #[async_trait] -// impl AlertChannelConfig for WebhookReceiver { -// async fn get_config(&self) -> AlertManagerChannelConfig { -// let channel_global_config = None; -// let channel_receiver = self.alert_channel_receiver().await; -// let channel_route = self.alert_channel_route().await; -// -// AlertManagerChannelConfig { -// channel_global_config, -// channel_receiver, -// channel_route, -// } -// } -// } -// -// impl WebhookReceiver { -// async fn alert_channel_route(&self) -> serde_yaml::Value { -// let mut route = Mapping::new(); -// route.insert( -// Value::String("receiver".to_string()), -// Value::String(self.name.clone()), -// ); -// route.insert( -// Value::String("matchers".to_string()), -// Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]), -// ); -// route.insert(Value::String("continue".to_string()), Value::Bool(true)); -// Value::Mapping(route) -// } -// -// async fn alert_channel_receiver(&self) -> serde_yaml::Value { -// let mut receiver = Mapping::new(); -// receiver.insert( -// Value::String("name".to_string()), -// Value::String(self.name.clone()), -// ); -// -// let mut webhook_config = Mapping::new(); -// webhook_config.insert( -// Value::String("url".to_string()), -// Value::String(self.url.to_string()), -// ); -// -// receiver.insert( -// Value::String("webhook_configs".to_string()), -// Value::Sequence(vec![Value::Mapping(webhook_config)]), -// ); -// -// Value::Mapping(receiver) -// } -// } -// -// #[cfg(test)] -// mod tests { -// use super::*; -// #[tokio::test] -// async fn webhook_serialize_should_match() { -// let webhook_receiver = WebhookReceiver { -// name: "test-webhook".to_string(), -// url: Url::Url(url::Url::parse("https://webhook.i.dont.exist.com").unwrap()), -// }; -// -// let webhook_receiver_receiver = -// serde_yaml::to_string(&webhook_receiver.alert_channel_receiver().await).unwrap(); -// println!("receiver \n{:#}", webhook_receiver_receiver); -// let webhook_receiver_receiver_yaml = r#"name: test-webhook -// webhook_configs: -// - url: https://webhook.i.dont.exist.com/ -// "# -// .to_string(); -// -// let webhook_receiver_route = -// serde_yaml::to_string(&webhook_receiver.alert_channel_route().await).unwrap(); -// println!("route \n{:#}", webhook_receiver_route); -// let webhook_receiver_route_yaml = r#"receiver: test-webhook -// matchers: -// - alertname!=Watchdog -// continue: true -// "# -// .to_string(); -// -// assert_eq!(webhook_receiver_receiver, webhook_receiver_receiver_yaml); -// assert_eq!(webhook_receiver_route, webhook_receiver_route_yaml); -// } -// } diff --git a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs index ea929a31..78db8426 100644 --- a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs +++ b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs @@ -8,8 +8,7 @@ use crate::{ interpret::{InterpretError, Outcome}, modules::monitoring::{ kube_prometheus::{ - prometheus::{KubePrometheus, KubePrometheusRule}, - types::{AlertGroup, AlertManagerAdditionalPromRules}, + KubePrometheus, KubePrometheusRule, types::{AlertGroup, AlertManagerAdditionalPromRules} }, okd::{ OpenshiftClusterAlertSender, diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs b/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs index da27dc58..37743eba 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs @@ -2,38 +2,42 @@ use std::sync::{Arc, Mutex}; use serde::Serialize; -use super::{helm::config::KubePrometheusConfig, prometheus::KubePrometheus}; +use super::helm::config::KubePrometheusConfig; use crate::{ - modules::monitoring::kube_prometheus::types::ServiceMonitor, + modules::monitoring::kube_prometheus::{KubePrometheus, types::ServiceMonitor}, score::Score, topology::{ HelmCommand, Topology, - monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability}, + monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget}, tenant::TenantManager, }, }; #[derive(Clone, Debug, Serialize)] -pub struct HelmPrometheusAlertingScore { +pub struct KubePrometheusAlertingScore { pub receivers: Vec>>, pub rules: Vec>>, + pub scrape_targets: Option>>>, pub service_monitors: Vec, + pub config: Arc>, } impl> Score - for HelmPrometheusAlertingScore + for KubePrometheusAlertingScore { fn create_interpret(&self) -> Box> { - let config = Arc::new(Mutex::new(KubePrometheusConfig::new())); - config + self.config .try_lock() .expect("couldn't lock config") .additional_service_monitors = self.service_monitors.clone(); + Box::new(AlertingInterpret { - sender: KubePrometheus { config }, + sender: KubePrometheus { + config: self.config.clone(), + }, receivers: self.receivers.clone(), rules: self.rules.clone(), - scrape_targets: None, + scrape_targets: self.scrape_targets.clone(), }) } fn name(&self) -> String { diff --git a/harmony/src/modules/monitoring/kube_prometheus/mod.rs b/harmony/src/modules/monitoring/kube_prometheus/mod.rs index 122e9396..a30c479c 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/mod.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/mod.rs @@ -1,5 +1,154 @@ +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use log::{debug, error}; +use serde::Serialize; + +use crate::{ + interpret::{InterpretError, Outcome}, + inventory::Inventory, + modules::monitoring::{ + alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, + kube_prometheus::{ + helm::{ + config::KubePrometheusConfig, + kube_prometheus_helm_chart::kube_prometheus_helm_chart_score, + }, + types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig}, + }, + }, + score::Score, + topology::{ + HelmCommand, Topology, + monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}, + tenant::TenantManager, + }, +}; + pub mod crd; pub mod helm; pub mod helm_prometheus_alert_score; -pub mod prometheus; pub mod types; +pub mod score_kube_prometheus_alert_receivers; +pub mod score_kube_prometheus_rule; +pub mod score_kube_prometheus_scrape_target; + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + +#[async_trait] +impl AlertSender for KubePrometheus { + fn name(&self) -> String { + "HelmKubePrometheus".to_string() + } +} + +#[derive(Debug)] +pub struct KubePrometheus { + pub config: Arc>, +} + +impl Default for KubePrometheus { + fn default() -> Self { + Self::new() + } +} + +impl KubePrometheus { + pub fn new() -> Self { + Self { + config: Arc::new(Mutex::new(KubePrometheusConfig::new())), + } + } + + pub async fn configure_with_topology(&self, topology: &T) { + let ns = topology + .get_tenant_config() + .await + .map(|cfg| cfg.name.clone()) + .unwrap_or_else(|| "monitoring".to_string()); + error!("This must be refactored, see comments in pr #74"); + debug!("NS: {}", ns); + self.config.lock().unwrap().namespace = Some(ns); + } + + pub async fn install_receiver( + &self, + prometheus_receiver: &dyn KubePrometheusReceiver, + ) -> Result { + let prom_receiver = prometheus_receiver.configure_receiver().await; + debug!( + "adding alert receiver to prometheus config: {:#?}", + &prom_receiver + ); + let mut config = self.config.lock().unwrap(); + + config.alert_receiver_configs.push(prom_receiver); + let prom_receiver_name = prometheus_receiver.name(); + debug!("installed alert receiver {}", &prom_receiver_name); + Ok(Outcome::success(format!( + "Sucessfully installed receiver {}", + prom_receiver_name + ))) + } + + pub async fn install_rule( + &self, + prometheus_rule: &AlertManagerRuleGroup, + ) -> Result { + let prometheus_rule = prometheus_rule.configure_rule().await; + let mut config = self.config.lock().unwrap(); + + config.alert_rules.push(prometheus_rule.clone()); + Ok(Outcome::success(format!( + "Successfully installed alert rule: {:#?},", + prometheus_rule + ))) + } + + pub async fn install_prometheus( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + kube_prometheus_helm_chart_score(self.config.clone()) + .interpret(inventory, topology) + .await + } +} + +#[async_trait] +pub trait KubePrometheusReceiver: Send + Sync + std::fmt::Debug { + fn name(&self) -> String; + async fn configure_receiver(&self) -> AlertManagerChannelConfig; +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + +#[async_trait] +pub trait KubePrometheusRule: Send + Sync + std::fmt::Debug { + fn name(&self) -> String; + async fn configure_rule(&self) -> AlertManagerAdditionalPromRules; +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs b/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs deleted file mode 100644 index 67083a77..00000000 --- a/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs +++ /dev/null @@ -1,155 +0,0 @@ -use std::sync::{Arc, Mutex}; - -use async_trait::async_trait; -use log::{debug, error}; -use serde::Serialize; - -use crate::{ - interpret::{InterpretError, Outcome}, - inventory::Inventory, - modules::monitoring::alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, - score, - topology::{ - HelmCommand, Topology, - installable::Installable, - monitoring::{AlertReceiver, AlertRule, AlertSender}, - tenant::TenantManager, - }, -}; - -use score::Score; - -use super::{ - helm::{ - config::KubePrometheusConfig, kube_prometheus_helm_chart::kube_prometheus_helm_chart_score, - }, - types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig}, -}; - -#[async_trait] -impl AlertSender for KubePrometheus { - fn name(&self) -> String { - "HelmKubePrometheus".to_string() - } -} - -#[async_trait] -impl Installable for KubePrometheus { - async fn configure(&self, _inventory: &Inventory, topology: &T) -> Result<(), InterpretError> { - self.configure_with_topology(topology).await; - Ok(()) - } - - async fn ensure_installed( - &self, - inventory: &Inventory, - topology: &T, - ) -> Result<(), InterpretError> { - self.install_prometheus(inventory, topology).await?; - Ok(()) - } -} - -#[derive(Debug)] -pub struct KubePrometheus { - pub config: Arc>, -} - -impl Default for KubePrometheus { - fn default() -> Self { - Self::new() - } -} - -impl KubePrometheus { - pub fn new() -> Self { - Self { - config: Arc::new(Mutex::new(KubePrometheusConfig::new())), - } - } - - pub async fn configure_with_topology(&self, topology: &T) { - let ns = topology - .get_tenant_config() - .await - .map(|cfg| cfg.name.clone()) - .unwrap_or_else(|| "monitoring".to_string()); - error!("This must be refactored, see comments in pr #74"); - debug!("NS: {}", ns); - self.config.lock().unwrap().namespace = Some(ns); - } - - pub async fn install_receiver( - &self, - prometheus_receiver: &dyn KubePrometheusReceiver, - ) -> Result { - let prom_receiver = prometheus_receiver.configure_receiver().await; - debug!( - "adding alert receiver to prometheus config: {:#?}", - &prom_receiver - ); - let mut config = self.config.lock().unwrap(); - - config.alert_receiver_configs.push(prom_receiver); - let prom_receiver_name = prometheus_receiver.name(); - debug!("installed alert receiver {}", &prom_receiver_name); - Ok(Outcome::success(format!( - "Sucessfully installed receiver {}", - prom_receiver_name - ))) - } - - pub async fn install_rule( - &self, - prometheus_rule: &AlertManagerRuleGroup, - ) -> Result { - let prometheus_rule = prometheus_rule.configure_rule().await; - let mut config = self.config.lock().unwrap(); - - config.alert_rules.push(prometheus_rule.clone()); - Ok(Outcome::success(format!( - "Successfully installed alert rule: {:#?},", - prometheus_rule - ))) - } - - pub async fn install_prometheus( - &self, - inventory: &Inventory, - topology: &T, - ) -> Result { - kube_prometheus_helm_chart_score(self.config.clone()) - .interpret(inventory, topology) - .await - } -} - -#[async_trait] -pub trait KubePrometheusReceiver: Send + Sync + std::fmt::Debug { - fn name(&self) -> String; - async fn configure_receiver(&self) -> AlertManagerChannelConfig; -} - -impl Serialize for Box> { - fn serialize(&self, _serializer: S) -> Result - where - S: serde::Serializer, - { - todo!() - } -} - -#[async_trait] -pub trait KubePrometheusRule: Send + Sync + std::fmt::Debug { - fn name(&self) -> String; - async fn configure_rule(&self) -> AlertManagerAdditionalPromRules; -} - -impl Serialize for Box> { - fn serialize(&self, _serializer: S) -> Result - where - S: serde::Serializer, - { - todo!() - } -} diff --git a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_alert_receivers.rs b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_alert_receivers.rs new file mode 100644 index 00000000..0e28bdc3 --- /dev/null +++ b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_alert_receivers.rs @@ -0,0 +1,23 @@ +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::monitoring::kube_prometheus::KubePrometheus, + score::Score, + topology::{K8sclient, Topology, monitoring::AlertReceiver}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct KubePrometheusReceiverScore { + pub receiver: Box>, +} + +impl Score for KubePrometheusReceiverScore { + fn name(&self) -> String { + "KubePrometheusReceiverScore".to_string() + } + + fn create_interpret(&self) -> Box> { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_rule.rs b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_rule.rs new file mode 100644 index 00000000..145eade1 --- /dev/null +++ b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_rule.rs @@ -0,0 +1,23 @@ +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::monitoring::kube_prometheus::KubePrometheus, + score::Score, + topology::{K8sclient, Topology, monitoring::AlertRule}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct KubePrometheusRuleScore { + pub rule: Box>, +} + +impl Score for KubePrometheusRuleScore { + fn name(&self) -> String { + "KubePrometheusRuleScore".to_string() + } + + fn create_interpret(&self) -> Box> { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs new file mode 100644 index 00000000..5b59d566 --- /dev/null +++ b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs @@ -0,0 +1,23 @@ +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::monitoring::kube_prometheus::KubePrometheus, + score::Score, + topology::{K8sclient, Topology, monitoring::{AlertRule, ScrapeTarget}}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct KubePrometheusScrapeTargetScore { + pub scrape_target: Box>, +} + +impl Score for KubePrometheusScrapeTargetScore { + fn name(&self) -> String { + "KubePrometheusScrapeTargetScore".to_string() + } + + fn create_interpret(&self) -> Box> { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/prometheus/prometheus.rs b/harmony/src/modules/monitoring/prometheus/prometheus.rs index 3e2fd14b..ff0e6749 100644 --- a/harmony/src/modules/monitoring/prometheus/prometheus.rs +++ b/harmony/src/modules/monitoring/prometheus/prometheus.rs @@ -11,8 +11,7 @@ use crate::{ alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, grafana::helm::helm_grafana::grafana_helm_chart_score, kube_prometheus::{ - prometheus::KubePrometheusRule, - types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig}, + KubePrometheusRule, types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig} }, }, score::Score, -- 2.39.5 From d80561e326a92eff0ed3acdabc5b83ad78a35050 Mon Sep 17 00:00:00 2001 From: wjro Date: Wed, 25 Feb 2026 16:16:33 -0500 Subject: [PATCH 11/15] wip(kubeprometheus): created base scores for kubeprometheus alert receivers, scrape_tarets and rules --- examples/monitoring/src/main.rs | 2 +- examples/monitoring_with_tenant/src/main.rs | 2 +- examples/okd_cluster_alerts/src/main.rs | 4 +- .../observability/kube_prometheus.rs | 3 +- .../alert_channel/discord_alert_channel.rs | 42 ++- .../alert_channel/webhook_receiver.rs | 47 ++-- .../alert_rule/prometheus_alert_rule.rs | 3 +- .../helm/kube_prometheus_helm_chart.rs | 252 +++++++++--------- ...e.rs => kube_prometheus_alerting_score.rs} | 0 .../modules/monitoring/kube_prometheus/mod.rs | 4 +- .../score_kube_prometheus_scrape_target.rs | 5 +- .../monitoring/prometheus/prometheus.rs | 3 +- 12 files changed, 192 insertions(+), 175 deletions(-) rename harmony/src/modules/monitoring/kube_prometheus/{helm_prometheus_alert_score.rs => kube_prometheus_alerting_score.rs} (100%) diff --git a/examples/monitoring/src/main.rs b/examples/monitoring/src/main.rs index b465ff10..27210fe8 100644 --- a/examples/monitoring/src/main.rs +++ b/examples/monitoring/src/main.rs @@ -16,7 +16,7 @@ use harmony::{ prometheus_alert_rule::AlertManagerRuleGroup, }, kube_prometheus::{ - helm_prometheus_alert_score::KubePrometheusAlertingScore, + kube_prometheus_alerting_score::KubePrometheusAlertingScore, types::{ HTTPScheme, MatchExpression, Operator, Selector, ServiceMonitor, ServiceMonitorEndpoint, diff --git a/examples/monitoring_with_tenant/src/main.rs b/examples/monitoring_with_tenant/src/main.rs index 77145fb2..fe65905d 100644 --- a/examples/monitoring_with_tenant/src/main.rs +++ b/examples/monitoring_with_tenant/src/main.rs @@ -10,7 +10,7 @@ use harmony::{ prometheus_alert_rule::AlertManagerRuleGroup, }, kube_prometheus::{ - helm_prometheus_alert_score::KubePrometheusAlertingScore, + kube_prometheus_alerting_score::KubePrometheusAlertingScore, types::{ HTTPScheme, MatchExpression, Operator, Selector, ServiceMonitor, ServiceMonitorEndpoint, diff --git a/examples/okd_cluster_alerts/src/main.rs b/examples/okd_cluster_alerts/src/main.rs index 8d9f4f69..4a7fcff8 100644 --- a/examples/okd_cluster_alerts/src/main.rs +++ b/examples/okd_cluster_alerts/src/main.rs @@ -3,7 +3,9 @@ use harmony::{ modules::monitoring::{ alert_channel::discord_alert_channel::DiscordReceiver, alert_rule::{ - alerts::{infra::opnsense::high_http_error_rate, k8s::pvc::high_pvc_fill_rate_over_two_days}, + alerts::{ + infra::opnsense::high_http_error_rate, k8s::pvc::high_pvc_fill_rate_over_two_days, + }, prometheus_alert_rule::AlertManagerRuleGroup, }, okd::score_openshift_cluster_alert_score::OpenshiftClusterAlertScore, diff --git a/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs b/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs index 496e2474..c1f6cc4f 100644 --- a/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs +++ b/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs @@ -6,7 +6,8 @@ use crate::{ modules::monitoring::kube_prometheus::{ KubePrometheus, helm::kube_prometheus_helm_chart::kube_prometheus_helm_chart_score, score_kube_prometheus_alert_receivers::KubePrometheusReceiverScore, - score_kube_prometheus_rule::KubePrometheusRuleScore, score_kube_prometheus_scrape_target::KubePrometheusScrapeTargetScore, + score_kube_prometheus_rule::KubePrometheusRuleScore, + score_kube_prometheus_scrape_target::KubePrometheusScrapeTargetScore, }, score::Score, topology::{ diff --git a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs index b3765464..fd3f5066 100644 --- a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs +++ b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs @@ -1,29 +1,14 @@ -use std::any::Any; -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; -use async_trait::async_trait; -use harmony_types::k8s_name::K8sName; -use k8s_openapi::api::core::v1::Secret; -use kube::Resource; -use kube::api::{DynamicObject, ObjectMeta}; -use log::{debug, trace}; -use serde::Serialize; -use serde_json::json; -use serde_yaml::{Mapping, Value}; - -use crate::infra::kube::kube_resource_to_dynamic; use crate::modules::monitoring::kube_prometheus::KubePrometheus; -use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{ - AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus, -}; use crate::modules::monitoring::okd::OpenshiftClusterAlertSender; use crate::modules::monitoring::red_hat_cluster_observability::RedHatClusterObservability; use crate::topology::monitoring::{AlertRoute, MatchOp}; -use crate::{ - interpret::{InterpretError, Outcome}, - topology::monitoring::AlertReceiver, -}; +use crate::{interpret::InterpretError, topology::monitoring::AlertReceiver}; use harmony_types::net::Url; +use k8s_openapi::api::core::v1::Secret; +use serde::Serialize; +use serde_json::json; #[derive(Debug, Clone, Serialize)] pub struct DiscordReceiver { @@ -126,19 +111,28 @@ impl AlertReceiver for DiscordReceiver { impl AlertReceiver for DiscordReceiver { fn build_route(&self) -> Result { - todo!() + serde_yaml::to_value(self.route.clone()).map_err(|e| InterpretError::new(e.to_string())) } fn build_receiver(&self) -> Result { - todo!() + let receiver_block = serde_yaml::to_value(json!({ + "name": self.name, + "discord_configs": [{ + "webhook_url": format!("{}", self.url), + "title": "{{ template \"discord.default.title\" . }}", + "message": "{{ template \"discord.default.message\" . }}" + }] + })) + .map_err(|e| InterpretError::new(e.to_string()))?; + Ok(receiver_block) } fn name(&self) -> String { - todo!() + self.name.clone() } fn clone_box(&self) -> Box> { - todo!() + Box::new(self.clone()) } } diff --git a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs index 1f6e56e1..bfbba451 100644 --- a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs +++ b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs @@ -1,16 +1,10 @@ -use std::any::Any; - -use async_trait::async_trait; -use kube::api::ObjectMeta; -use log::debug; use serde::Serialize; use serde_json::json; -use serde_yaml::{Mapping, Value}; use crate::{ - interpret::{InterpretError, Outcome}, + interpret::InterpretError, modules::monitoring::{ - kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus, + kube_prometheus::{KubePrometheus, crd::crd_alertmanager_config::CRDPrometheus}, okd::OpenshiftClusterAlertSender, red_hat_cluster_observability::RedHatClusterObservability, }, @@ -86,21 +80,42 @@ impl AlertReceiver for WebhookReceiver { } } -impl AlertReceiver for WebhookReceiver { - fn build_route(&self) -> Result { - todo!() +impl AlertReceiver for WebhookReceiver { + fn build_receiver(&self) -> Result { + let receiver = self.build_receiver(); + serde_yaml::to_value(receiver).map_err(|e| InterpretError::new(e.to_string())) } - fn build_receiver(&self) -> Result { - todo!() + fn build_route(&self) -> Result { + let route = self.build_route(); + serde_yaml::to_value(route).map_err(|e| InterpretError::new(e.to_string())) } fn name(&self) -> String { - todo!() + self.name.clone() } - fn clone_box(&self) -> Box> { - todo!() + fn clone_box(&self) -> Box> { + Box::new(self.clone()) } } +impl AlertReceiver for WebhookReceiver { + fn build_receiver(&self) -> Result { + let receiver = self.build_receiver(); + serde_yaml::to_value(receiver).map_err(|e| InterpretError::new(e.to_string())) + } + + fn build_route(&self) -> Result { + let route = self.build_route(); + serde_yaml::to_value(route).map_err(|e| InterpretError::new(e.to_string())) + } + + fn name(&self) -> String { + self.name.clone() + } + + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } +} diff --git a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs index 78db8426..50bc46c6 100644 --- a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs +++ b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs @@ -8,7 +8,8 @@ use crate::{ interpret::{InterpretError, Outcome}, modules::monitoring::{ kube_prometheus::{ - KubePrometheus, KubePrometheusRule, types::{AlertGroup, AlertManagerAdditionalPromRules} + KubePrometheus, KubePrometheusRule, + types::{AlertGroup, AlertManagerAdditionalPromRules}, }, okd::{ OpenshiftClusterAlertSender, diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs b/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs index 7d6aec89..ac52283a 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs @@ -66,7 +66,7 @@ pub fn kube_prometheus_helm_chart_score( } let _resource_section = resource_block(&resource_limit, 2); - let mut values = format!( + let values = format!( r#" global: rbac: @@ -280,131 +280,131 @@ prometheusOperator: memory: 100Mi "#, ); - - let prometheus_config = - crate::modules::monitoring::kube_prometheus::types::PrometheusConfigValues { - prometheus: PrometheusConfig { - prometheus: bool::from_str(prometheus.as_str()).expect("couldn't parse bool"), - additional_service_monitors: config.additional_service_monitors.clone(), - }, - }; - let prometheus_config_yaml = - serde_yaml::to_string(&prometheus_config).expect("Failed to serialize YAML"); - - debug!( - "serialized prometheus config: \n {:#}", - prometheus_config_yaml - ); - values.push_str(&prometheus_config_yaml); - - // add required null receiver for prometheus alert manager - let mut null_receiver = Mapping::new(); - null_receiver.insert( - Value::String("receiver".to_string()), - Value::String("null".to_string()), - ); - null_receiver.insert( - Value::String("matchers".to_string()), - Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]), - ); - null_receiver.insert(Value::String("continue".to_string()), Value::Bool(true)); - - //add alert channels - let mut alert_manager_channel_config = AlertManagerConfig { - global: Mapping::new(), - route: AlertManagerRoute { - routes: vec![Value::Mapping(null_receiver)], - }, - receivers: vec![serde_yaml::from_str("name: 'null'").unwrap()], - }; - for receiver in config.alert_receiver_configs.iter() { - if let Some(global) = receiver.channel_global_config.clone() { - alert_manager_channel_config - .global - .insert(global.0, global.1); - } - alert_manager_channel_config - .route - .routes - .push(receiver.channel_route.clone()); - alert_manager_channel_config - .receivers - .push(receiver.channel_receiver.clone()); - } - - let mut labels = BTreeMap::new(); - labels.insert("alertmanagerConfig".to_string(), "enabled".to_string()); - let alert_manager_config_selector = AlertManagerConfigSelector { - match_labels: labels, - }; - let alert_manager_values = AlertManagerValues { - alertmanager: AlertManager { - enabled: config.alert_manager, - config: alert_manager_channel_config, - alertmanager_spec: AlertManagerSpec { - resources: Resources { - limits: Limits { - memory: "100Mi".to_string(), - cpu: "100m".to_string(), - }, - requests: Requests { - memory: "100Mi".to_string(), - cpu: "100m".to_string(), - }, - }, - alert_manager_config_selector, - replicas: 2, - }, - init_config_reloader: ConfigReloader { - resources: Resources { - limits: Limits { - memory: "100Mi".to_string(), - cpu: "100m".to_string(), - }, - requests: Requests { - memory: "100Mi".to_string(), - cpu: "100m".to_string(), - }, - }, - }, - }, - }; - - let alert_manager_yaml = - serde_yaml::to_string(&alert_manager_values).expect("Failed to serialize YAML"); - debug!("serialized alert manager: \n {:#}", alert_manager_yaml); - values.push_str(&alert_manager_yaml); - - //format alert manager additional rules for helm chart - let mut merged_rules: BTreeMap = BTreeMap::new(); - - for additional_rule in config.alert_rules.clone() { - for (key, group) in additional_rule.rules { - merged_rules.insert(key, group); - } - } - - let merged_rules = AlertManagerAdditionalPromRules { - rules: merged_rules, - }; - - let mut alert_manager_additional_rules = serde_yaml::Mapping::new(); - let rules_value = serde_yaml::to_value(merged_rules).unwrap(); - - alert_manager_additional_rules.insert( - serde_yaml::Value::String("additionalPrometheusRulesMap".to_string()), - rules_value, - ); - - let alert_manager_additional_rules_yaml = - serde_yaml::to_string(&alert_manager_additional_rules).expect("Failed to serialize YAML"); - debug!( - "alert_rules_yaml:\n{:#}", - alert_manager_additional_rules_yaml - ); - - values.push_str(&alert_manager_additional_rules_yaml); - debug!("full values.yaml: \n {:#}", values); + // + // let prometheus_config = + // crate::modules::monitoring::kube_prometheus::types::PrometheusConfigValues { + // prometheus: PrometheusConfig { + // prometheus: bool::from_str(prometheus.as_str()).expect("couldn't parse bool"), + // additional_service_monitors: config.additional_service_monitors.clone(), + // }, + // }; + // let prometheus_config_yaml = + // serde_yaml::to_string(&prometheus_config).expect("Failed to serialize YAML"); + // + // debug!( + // "serialized prometheus config: \n {:#}", + // prometheus_config_yaml + // ); + // values.push_str(&prometheus_config_yaml); + // + // // add required null receiver for prometheus alert manager + // let mut null_receiver = Mapping::new(); + // null_receiver.insert( + // Value::String("receiver".to_string()), + // Value::String("null".to_string()), + // ); + // null_receiver.insert( + // Value::String("matchers".to_string()), + // Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]), + // ); + // null_receiver.insert(Value::String("continue".to_string()), Value::Bool(true)); + // + // //add alert channels + // let mut alert_manager_channel_config = AlertManagerConfig { + // global: Mapping::new(), + // route: AlertManagerRoute { + // routes: vec![Value::Mapping(null_receiver)], + // }, + // receivers: vec![serde_yaml::from_str("name: 'null'").unwrap()], + // }; + // for receiver in config.alert_receiver_configs.iter() { + // if let Some(global) = receiver.channel_global_config.clone() { + // alert_manager_channel_config + // .global + // .insert(global.0, global.1); + // } + // alert_manager_channel_config + // .route + // .routes + // .push(receiver.channel_route.clone()); + // alert_manager_channel_config + // .receivers + // .push(receiver.channel_receiver.clone()); + // } + // + // let mut labels = BTreeMap::new(); + // labels.insert("alertmanagerConfig".to_string(), "enabled".to_string()); + // let alert_manager_config_selector = AlertManagerConfigSelector { + // match_labels: labels, + // }; + // let alert_manager_values = AlertManagerValues { + // alertmanager: AlertManager { + // enabled: config.alert_manager, + // config: alert_manager_channel_config, + // alertmanager_spec: AlertManagerSpec { + // resources: Resources { + // limits: Limits { + // memory: "100Mi".to_string(), + // cpu: "100m".to_string(), + // }, + // requests: Requests { + // memory: "100Mi".to_string(), + // cpu: "100m".to_string(), + // }, + // }, + // alert_manager_config_selector, + // replicas: 2, + // }, + // init_config_reloader: ConfigReloader { + // resources: Resources { + // limits: Limits { + // memory: "100Mi".to_string(), + // cpu: "100m".to_string(), + // }, + // requests: Requests { + // memory: "100Mi".to_string(), + // cpu: "100m".to_string(), + // }, + // }, + // }, + // }, + // }; + // + // let alert_manager_yaml = + // serde_yaml::to_string(&alert_manager_values).expect("Failed to serialize YAML"); + // debug!("serialized alert manager: \n {:#}", alert_manager_yaml); + // values.push_str(&alert_manager_yaml); + // + // //format alert manager additional rules for helm chart + // let mut merged_rules: BTreeMap = BTreeMap::new(); + // + // for additional_rule in config.alert_rules.clone() { + // for (key, group) in additional_rule.rules { + // merged_rules.insert(key, group); + // } + // } + // + // let merged_rules = AlertManagerAdditionalPromRules { + // rules: merged_rules, + // }; + // + // let mut alert_manager_additional_rules = serde_yaml::Mapping::new(); + // let rules_value = serde_yaml::to_value(merged_rules).unwrap(); + // + // alert_manager_additional_rules.insert( + // serde_yaml::Value::String("additionalPrometheusRulesMap".to_string()), + // rules_value, + // ); + // + // let alert_manager_additional_rules_yaml = + // serde_yaml::to_string(&alert_manager_additional_rules).expect("Failed to serialize YAML"); + // debug!( + // "alert_rules_yaml:\n{:#}", + // alert_manager_additional_rules_yaml + // ); + // + // values.push_str(&alert_manager_additional_rules_yaml); + // debug!("full values.yaml: \n {:#}", values); HelmChartScore { namespace: Some(NonBlankString::from_str(&config.namespace.clone().unwrap()).unwrap()), diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs b/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs similarity index 100% rename from harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs rename to harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs diff --git a/harmony/src/modules/monitoring/kube_prometheus/mod.rs b/harmony/src/modules/monitoring/kube_prometheus/mod.rs index a30c479c..f410c9c1 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/mod.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/mod.rs @@ -27,11 +27,11 @@ use crate::{ pub mod crd; pub mod helm; -pub mod helm_prometheus_alert_score; -pub mod types; +pub mod kube_prometheus_alerting_score; pub mod score_kube_prometheus_alert_receivers; pub mod score_kube_prometheus_rule; pub mod score_kube_prometheus_scrape_target; +pub mod types; impl Serialize for Box> { fn serialize(&self, _serializer: S) -> Result diff --git a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs index 5b59d566..d1c16611 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs @@ -4,7 +4,10 @@ use crate::{ interpret::Interpret, modules::monitoring::kube_prometheus::KubePrometheus, score::Score, - topology::{K8sclient, Topology, monitoring::{AlertRule, ScrapeTarget}}, + topology::{ + K8sclient, Topology, + monitoring::{AlertRule, ScrapeTarget}, + }, }; #[derive(Debug, Clone, Serialize)] diff --git a/harmony/src/modules/monitoring/prometheus/prometheus.rs b/harmony/src/modules/monitoring/prometheus/prometheus.rs index ff0e6749..8704b843 100644 --- a/harmony/src/modules/monitoring/prometheus/prometheus.rs +++ b/harmony/src/modules/monitoring/prometheus/prometheus.rs @@ -11,7 +11,8 @@ use crate::{ alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, grafana::helm::helm_grafana::grafana_helm_chart_score, kube_prometheus::{ - KubePrometheusRule, types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig} + KubePrometheusRule, + types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig}, }, }, score::Score, -- 2.39.5 From 4fad077eb48b1ee57da29233bb59993ba6b4bff6 Mon Sep 17 00:00:00 2001 From: wjro Date: Thu, 26 Feb 2026 13:07:28 -0500 Subject: [PATCH 12/15] refactor(kubeprometheus): implemented Observability for KubePrometheus --- examples/monitoring/src/main.rs | 11 +- examples/monitoring_with_tenant/src/main.rs | 11 +- .../observability/kube_prometheus.rs | 36 +++-- .../alert_rule/prometheus_alert_rule.rs | 27 +++- .../crd/crd_alertmanager_config.rs | 7 +- .../crd/crd_prometheus_rules.rs | 5 +- .../kube_prometheus/crd/crd_scrape_config.rs | 9 +- .../helm/kube_prometheus_helm_chart.rs | 125 ------------------ .../modules/monitoring/kube_prometheus/mod.rs | 3 +- .../score_kube_prometheus_alert_receivers.rs | 37 +++++- .../score_kube_prometheus_ensure_ready.rs | 80 +++++++++++ .../score_kube_prometheus_rule.rs | 27 +++- .../score_kube_prometheus_scrape_target.rs | 45 ++++++- 13 files changed, 265 insertions(+), 158 deletions(-) create mode 100644 harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_ensure_ready.rs diff --git a/examples/monitoring/src/main.rs b/examples/monitoring/src/main.rs index 27210fe8..d1304db7 100644 --- a/examples/monitoring/src/main.rs +++ b/examples/monitoring/src/main.rs @@ -1,4 +1,7 @@ -use std::collections::HashMap; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; use harmony::{ inventory::Inventory, @@ -16,6 +19,7 @@ use harmony::{ prometheus_alert_rule::AlertManagerRuleGroup, }, kube_prometheus::{ + helm::config::KubePrometheusConfig, kube_prometheus_alerting_score::KubePrometheusAlertingScore, types::{ HTTPScheme, MatchExpression, Operator, Selector, ServiceMonitor, @@ -74,10 +78,15 @@ async fn main() { endpoints: vec![service_monitor_endpoint], ..Default::default() }; + + let config = Arc::new(Mutex::new(KubePrometheusConfig::new())); + let alerting_score = KubePrometheusAlertingScore { receivers: vec![Box::new(discord_receiver)], rules: vec![Box::new(additional_rules), Box::new(additional_rules2)], service_monitors: vec![service_monitor], + scrape_targets: None, + config, }; harmony_cli::run( diff --git a/examples/monitoring_with_tenant/src/main.rs b/examples/monitoring_with_tenant/src/main.rs index fe65905d..6afc302c 100644 --- a/examples/monitoring_with_tenant/src/main.rs +++ b/examples/monitoring_with_tenant/src/main.rs @@ -1,4 +1,8 @@ -use std::{collections::HashMap, str::FromStr}; +use std::{ + collections::HashMap, + str::FromStr, + sync::{Arc, Mutex}, +}; use harmony::{ inventory::Inventory, @@ -10,6 +14,7 @@ use harmony::{ prometheus_alert_rule::AlertManagerRuleGroup, }, kube_prometheus::{ + helm::config::KubePrometheusConfig, kube_prometheus_alerting_score::KubePrometheusAlertingScore, types::{ HTTPScheme, MatchExpression, Operator, Selector, ServiceMonitor, @@ -80,10 +85,14 @@ async fn main() { ..Default::default() }; + let config = Arc::new(Mutex::new(KubePrometheusConfig::new())); + let alerting_score = KubePrometheusAlertingScore { receivers: vec![Box::new(discord_receiver)], rules: vec![Box::new(additional_rules)], service_monitors: vec![service_monitor], + scrape_targets: None, + config, }; harmony_cli::run( diff --git a/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs b/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs index c1f6cc4f..05cd3f6b 100644 --- a/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs +++ b/harmony/src/domain/topology/k8s_anywhere/observability/kube_prometheus.rs @@ -1,11 +1,11 @@ use async_trait::async_trait; use crate::{ - interpret::InterpretError, inventory::Inventory, modules::monitoring::kube_prometheus::{ KubePrometheus, helm::kube_prometheus_helm_chart::kube_prometheus_helm_chart_score, score_kube_prometheus_alert_receivers::KubePrometheusReceiverScore, + score_kube_prometheus_ensure_ready::KubePrometheusEnsureReadyScore, score_kube_prometheus_rule::KubePrometheusRuleScore, score_kube_prometheus_scrape_target::KubePrometheusScrapeTargetScore, }, @@ -36,7 +36,7 @@ impl Observability for K8sAnywhereTopology { async fn install_receivers( &self, - _sender: &KubePrometheus, + sender: &KubePrometheus, inventory: &Inventory, receivers: Option>>>, ) -> Result { @@ -46,7 +46,10 @@ impl Observability for K8sAnywhereTopology { }; for receiver in receivers { - let score = KubePrometheusReceiverScore { receiver }; + let score = KubePrometheusReceiverScore { + receiver, + sender: sender.clone(), + }; score .create_interpret() @@ -62,7 +65,7 @@ impl Observability for K8sAnywhereTopology { async fn install_rules( &self, - _sender: &KubePrometheus, + sender: &KubePrometheus, inventory: &Inventory, rules: Option>>>, ) -> Result { @@ -72,7 +75,10 @@ impl Observability for K8sAnywhereTopology { }; for rule in rules { - let score = KubePrometheusRuleScore { rule }; + let score = KubePrometheusRuleScore { + sender: sender.clone(), + rule, + }; score .create_interpret() @@ -88,7 +94,7 @@ impl Observability for K8sAnywhereTopology { async fn add_scrape_targets( &self, - _sender: &KubePrometheus, + sender: &KubePrometheus, inventory: &Inventory, scrape_targets: Option>>>, ) -> Result { @@ -98,7 +104,10 @@ impl Observability for K8sAnywhereTopology { }; for scrape_target in scrape_targets { - let score = KubePrometheusScrapeTargetScore { scrape_target }; + let score = KubePrometheusScrapeTargetScore { + scrape_target, + sender: sender.clone(), + }; score .create_interpret() @@ -117,6 +126,17 @@ impl Observability for K8sAnywhereTopology { sender: &KubePrometheus, inventory: &Inventory, ) -> Result { - todo!() + let score = KubePrometheusEnsureReadyScore { + sender: sender.clone(), + }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("KubePrometheus not ready {}", e)))?; + Ok(PreparationOutcome::Success { + details: "KubePrometheus Ready".to_string(), + }) } } diff --git a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs index 50bc46c6..7585ffdc 100644 --- a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs +++ b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs @@ -11,10 +11,7 @@ use crate::{ KubePrometheus, KubePrometheusRule, types::{AlertGroup, AlertManagerAdditionalPromRules}, }, - okd::{ - OpenshiftClusterAlertSender, - crd::alerting_rules::{Rule, RuleGroup}, - }, + okd::OpenshiftClusterAlertSender, }, topology::monitoring::AlertRule, }; @@ -78,12 +75,13 @@ impl PrometheusAlertRule { impl AlertRule for AlertManagerRuleGroup { fn build_rule(&self) -> Result { let name = self.name.clone(); - let mut rules: Vec = vec![]; + let mut rules: Vec = vec![]; for rule in self.rules.clone() { rules.push(rule.into()) } - let rule_groups = vec![RuleGroup { name, rules }]; + let rule_groups = + vec![crate::modules::monitoring::okd::crd::alerting_rules::RuleGroup { name, rules }]; Ok(serde_json::to_value(rule_groups).map_err(|e| InterpretError::new(e.to_string()))?) } @@ -99,7 +97,22 @@ impl AlertRule for AlertManagerRuleGroup { impl AlertRule for AlertManagerRuleGroup { fn build_rule(&self) -> Result { - todo!() + let name = self.name.clone(); + let mut rules: Vec< + crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::Rule, + > = vec![]; + for rule in self.rules.clone() { + rules.push(rule.into()) + } + + let rule_groups = vec![ + crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::RuleGroup { + name, + rules, + }, + ]; + + Ok(serde_json::to_value(rule_groups).map_err(|e| InterpretError::new(e.to_string()))?) } fn name(&self) -> String { diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs index 890886d5..0b9002b6 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs @@ -19,13 +19,14 @@ use crate::{ }, }; -#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[derive(CustomResource, Serialize, Deserialize, Default, Debug, Clone, JsonSchema)] #[kube( group = "monitoring.coreos.com", - version = "v1alpha1", + version = "v1", kind = "AlertmanagerConfig", plural = "alertmanagerconfigs", - namespaced + namespaced, + derive = "Default" )] pub struct AlertmanagerConfigSpec { #[serde(flatten)] diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs index 6d08456b..f3202564 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs @@ -6,13 +6,14 @@ use serde::{Deserialize, Serialize}; use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule; -#[derive(CustomResource, Debug, Serialize, Deserialize, Clone, JsonSchema)] +#[derive(CustomResource, Default, Debug, Serialize, Deserialize, Clone, JsonSchema)] #[kube( group = "monitoring.coreos.com", version = "v1", kind = "PrometheusRule", plural = "prometheusrules", - namespaced + namespaced, + derive = "Default" )] #[serde(rename_all = "camelCase")] pub struct PrometheusRuleSpec { diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs index 081c63f8..929ce98d 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs @@ -1,4 +1,4 @@ -use std::net::IpAddr; +use std::{collections::BTreeMap, net::IpAddr}; use async_trait::async_trait; use kube::CustomResource; @@ -12,12 +12,13 @@ use crate::{ topology::monitoring::ScrapeTarget, }; -#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[derive(CustomResource, Default, Serialize, Deserialize, Debug, Clone, JsonSchema)] #[kube( group = "monitoring.coreos.com", version = "v1alpha1", kind = "ScrapeConfig", plural = "scrapeconfigs", + derive = "Default", namespaced )] #[serde(rename_all = "camelCase")] @@ -70,8 +71,8 @@ pub struct ScrapeConfigSpec { #[serde(rename_all = "camelCase")] pub struct StaticConfig { pub targets: Vec, - - pub labels: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub labels: Option>, } /// Relabeling configuration for target or metric relabeling. diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs b/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs index ac52283a..ea7d2530 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs @@ -280,131 +280,6 @@ prometheusOperator: memory: 100Mi "#, ); - // - // let prometheus_config = - // crate::modules::monitoring::kube_prometheus::types::PrometheusConfigValues { - // prometheus: PrometheusConfig { - // prometheus: bool::from_str(prometheus.as_str()).expect("couldn't parse bool"), - // additional_service_monitors: config.additional_service_monitors.clone(), - // }, - // }; - // let prometheus_config_yaml = - // serde_yaml::to_string(&prometheus_config).expect("Failed to serialize YAML"); - // - // debug!( - // "serialized prometheus config: \n {:#}", - // prometheus_config_yaml - // ); - // values.push_str(&prometheus_config_yaml); - // - // // add required null receiver for prometheus alert manager - // let mut null_receiver = Mapping::new(); - // null_receiver.insert( - // Value::String("receiver".to_string()), - // Value::String("null".to_string()), - // ); - // null_receiver.insert( - // Value::String("matchers".to_string()), - // Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]), - // ); - // null_receiver.insert(Value::String("continue".to_string()), Value::Bool(true)); - // - // //add alert channels - // let mut alert_manager_channel_config = AlertManagerConfig { - // global: Mapping::new(), - // route: AlertManagerRoute { - // routes: vec![Value::Mapping(null_receiver)], - // }, - // receivers: vec![serde_yaml::from_str("name: 'null'").unwrap()], - // }; - // for receiver in config.alert_receiver_configs.iter() { - // if let Some(global) = receiver.channel_global_config.clone() { - // alert_manager_channel_config - // .global - // .insert(global.0, global.1); - // } - // alert_manager_channel_config - // .route - // .routes - // .push(receiver.channel_route.clone()); - // alert_manager_channel_config - // .receivers - // .push(receiver.channel_receiver.clone()); - // } - // - // let mut labels = BTreeMap::new(); - // labels.insert("alertmanagerConfig".to_string(), "enabled".to_string()); - // let alert_manager_config_selector = AlertManagerConfigSelector { - // match_labels: labels, - // }; - // let alert_manager_values = AlertManagerValues { - // alertmanager: AlertManager { - // enabled: config.alert_manager, - // config: alert_manager_channel_config, - // alertmanager_spec: AlertManagerSpec { - // resources: Resources { - // limits: Limits { - // memory: "100Mi".to_string(), - // cpu: "100m".to_string(), - // }, - // requests: Requests { - // memory: "100Mi".to_string(), - // cpu: "100m".to_string(), - // }, - // }, - // alert_manager_config_selector, - // replicas: 2, - // }, - // init_config_reloader: ConfigReloader { - // resources: Resources { - // limits: Limits { - // memory: "100Mi".to_string(), - // cpu: "100m".to_string(), - // }, - // requests: Requests { - // memory: "100Mi".to_string(), - // cpu: "100m".to_string(), - // }, - // }, - // }, - // }, - // }; - // - // let alert_manager_yaml = - // serde_yaml::to_string(&alert_manager_values).expect("Failed to serialize YAML"); - // debug!("serialized alert manager: \n {:#}", alert_manager_yaml); - // values.push_str(&alert_manager_yaml); - // - // //format alert manager additional rules for helm chart - // let mut merged_rules: BTreeMap = BTreeMap::new(); - // - // for additional_rule in config.alert_rules.clone() { - // for (key, group) in additional_rule.rules { - // merged_rules.insert(key, group); - // } - // } - // - // let merged_rules = AlertManagerAdditionalPromRules { - // rules: merged_rules, - // }; - // - // let mut alert_manager_additional_rules = serde_yaml::Mapping::new(); - // let rules_value = serde_yaml::to_value(merged_rules).unwrap(); - // - // alert_manager_additional_rules.insert( - // serde_yaml::Value::String("additionalPrometheusRulesMap".to_string()), - // rules_value, - // ); - // - // let alert_manager_additional_rules_yaml = - // serde_yaml::to_string(&alert_manager_additional_rules).expect("Failed to serialize YAML"); - // debug!( - // "alert_rules_yaml:\n{:#}", - // alert_manager_additional_rules_yaml - // ); - // - // values.push_str(&alert_manager_additional_rules_yaml); - // debug!("full values.yaml: \n {:#}", values); HelmChartScore { namespace: Some(NonBlankString::from_str(&config.namespace.clone().unwrap()).unwrap()), diff --git a/harmony/src/modules/monitoring/kube_prometheus/mod.rs b/harmony/src/modules/monitoring/kube_prometheus/mod.rs index f410c9c1..8765f4a3 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/mod.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/mod.rs @@ -29,6 +29,7 @@ pub mod crd; pub mod helm; pub mod kube_prometheus_alerting_score; pub mod score_kube_prometheus_alert_receivers; +pub mod score_kube_prometheus_ensure_ready; pub mod score_kube_prometheus_rule; pub mod score_kube_prometheus_scrape_target; pub mod types; @@ -49,7 +50,7 @@ impl AlertSender for KubePrometheus { } } -#[derive(Debug)] +#[derive(Clone, Debug, Serialize)] pub struct KubePrometheus { pub config: Arc>, } diff --git a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_alert_receivers.rs b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_alert_receivers.rs index 0e28bdc3..09ebecc5 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_alert_receivers.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_alert_receivers.rs @@ -1,14 +1,22 @@ +use kube::api::ObjectMeta; use serde::Serialize; use crate::{ interpret::Interpret, - modules::monitoring::kube_prometheus::KubePrometheus, + modules::{ + k8s::resource::K8sResourceScore, + monitoring::kube_prometheus::{ + KubePrometheus, + crd::crd_alertmanager_config::{AlertmanagerConfig, AlertmanagerConfigSpec}, + }, + }, score::Score, topology::{K8sclient, Topology, monitoring::AlertReceiver}, }; #[derive(Debug, Clone, Serialize)] pub struct KubePrometheusReceiverScore { + pub sender: KubePrometheus, pub receiver: Box>, } @@ -18,6 +26,31 @@ impl Score for KubePrometheusReceiverScore { } fn create_interpret(&self) -> Box> { - todo!() + let name = self.receiver.name(); + let namespace = self.sender.config.lock().unwrap().namespace.clone(); + let route = self.receiver.build_route().expect(&format!( + "failed to build route for receveiver {}", + name.clone() + )); + let receiver = self.receiver.build_receiver().expect(&format!( + "failed to build receiver path for receiver {}", + name.clone() + )); + + let data = serde_json::json!({ + "route": route, + "receivers": [receiver] + }); + + let alertmanager_config = AlertmanagerConfig { + metadata: ObjectMeta { + name: Some(name), + namespace: namespace.clone(), + ..Default::default() + }, + spec: AlertmanagerConfigSpec { data: data }, + }; + + K8sResourceScore::single(alertmanager_config, namespace).create_interpret() } } diff --git a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_ensure_ready.rs b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_ensure_ready.rs new file mode 100644 index 00000000..1c3a6665 --- /dev/null +++ b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_ensure_ready.rs @@ -0,0 +1,80 @@ +use async_trait::async_trait; +use harmony_types::id::Id; +use serde::Serialize; + +use crate::{ + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + modules::monitoring::kube_prometheus::KubePrometheus, + score::Score, + topology::{K8sclient, Topology}, +}; + +#[derive(Clone, Debug, Serialize)] +pub struct KubePrometheusEnsureReadyScore { + pub sender: KubePrometheus, +} + +impl Score for KubePrometheusEnsureReadyScore { + fn name(&self) -> String { + "KubePrometheusEnsureReadyScore".to_string() + } + + fn create_interpret(&self) -> Box> { + Box::new(KubePrometheusEnsureReadyInterpret { + sender: self.sender.clone(), + }) + } +} + +#[derive(Clone, Debug, Serialize)] +pub struct KubePrometheusEnsureReadyInterpret { + pub sender: KubePrometheus, +} + +#[async_trait] +impl Interpret for KubePrometheusEnsureReadyInterpret { + async fn execute( + &self, + _inventory: &Inventory, + topology: &T, + ) -> Result { + let client = topology.k8s_client().await?; + let namespace = self + .sender + .config + .lock() + .unwrap() + .namespace + .clone() + .unwrap_or("default".to_string()); + + let prometheus_name = "kube-prometheues-kube-prometheus-operator"; + + client + .wait_until_deployment_ready(prometheus_name, Some(&namespace), None) + .await?; + + Ok(Outcome::success(format!( + "deployment: {} ready in ns: {}", + prometheus_name, namespace + ))) + } + + fn get_name(&self) -> InterpretName { + todo!() + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_rule.rs b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_rule.rs index 145eade1..86a5b42a 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_rule.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_rule.rs @@ -1,14 +1,22 @@ +use kube::api::ObjectMeta; use serde::Serialize; use crate::{ interpret::Interpret, - modules::monitoring::kube_prometheus::KubePrometheus, + modules::{ + k8s::resource::K8sResourceScore, + monitoring::kube_prometheus::{ + KubePrometheus, + crd::crd_prometheus_rules::{PrometheusRule, PrometheusRuleSpec, RuleGroup}, + }, + }, score::Score, topology::{K8sclient, Topology, monitoring::AlertRule}, }; #[derive(Debug, Clone, Serialize)] pub struct KubePrometheusRuleScore { + pub sender: KubePrometheus, pub rule: Box>, } @@ -18,6 +26,21 @@ impl Score for KubePrometheusRuleScore { } fn create_interpret(&self) -> Box> { - todo!() + let name = self.rule.name(); + let namespace = self.sender.config.lock().unwrap().namespace.clone(); + let groups: Vec = + serde_json::from_value(self.rule.build_rule().expect("failed to build alert rule")) + .expect("failed to serialize rule group"); + + let prometheus_rule = PrometheusRule { + metadata: ObjectMeta { + name: Some(name.clone()), + namespace: namespace.clone(), + ..Default::default() + }, + + spec: PrometheusRuleSpec { groups }, + }; + K8sResourceScore::single(prometheus_rule, namespace).create_interpret() } } diff --git a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs index d1c16611..7610a2a4 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs @@ -1,8 +1,18 @@ +use kube::api::ObjectMeta; use serde::Serialize; use crate::{ interpret::Interpret, - modules::monitoring::kube_prometheus::KubePrometheus, + modules::{ + k8s::resource::K8sResourceScore, + monitoring::kube_prometheus::{ + KubePrometheus, + crd::{ + crd_prometheuses::LabelSelector, + crd_scrape_config::{ScrapeConfig, ScrapeConfigSpec, StaticConfig}, + }, + }, + }, score::Score, topology::{ K8sclient, Topology, @@ -12,6 +22,7 @@ use crate::{ #[derive(Debug, Clone, Serialize)] pub struct KubePrometheusScrapeTargetScore { + pub sender: KubePrometheus, pub scrape_target: Box>, } @@ -21,6 +32,36 @@ impl Score for KubePrometheusScrapeTargetScore { } fn create_interpret(&self) -> Box> { - todo!() + let name = self.scrape_target.name(); + let namespace = self.sender.config.lock().unwrap().namespace.clone(); + + let external_target = self + .scrape_target + .build_scrape_target() + .expect("failed to build external scrape target"); + + //TODO this may need to modified to include a scrapeConfigSelector label from the + //prometheus operator + let labels = external_target.labels; + + let scrape_target = ScrapeConfig { + metadata: ObjectMeta { + name: Some(name.clone()), + namespace: namespace.clone(), + ..Default::default() + }, + spec: ScrapeConfigSpec { + static_configs: Some(vec![StaticConfig { + targets: vec![format!("{}:{}", external_target.ip, external_target.port)], + labels, + }]), + metrics_path: external_target.path, + scrape_interval: external_target.interval, + job_name: Some(name), + ..Default::default() + }, + }; + + K8sResourceScore::single(scrape_target, namespace).create_interpret() } } -- 2.39.5 From 5e861cfc6d1aa60753f31cbaf971b8f2a157581e Mon Sep 17 00:00:00 2001 From: wjro Date: Thu, 26 Feb 2026 14:38:28 -0500 Subject: [PATCH 13/15] refactor: skeleton structure for grafana observability --- examples/okd_cluster_alerts/src/main.rs | 2 +- .../topology/k8s_anywhere/k8s_anywhere.rs | 520 +--------------- .../k8s_anywhere/observability/grafana.rs | 147 +++++ .../k8s_anywhere/observability/mod.rs | 1 + .../application/features/monitoring.rs | 2 - harmony/src/modules/mod.rs | 1 - .../application_monitoring_score.rs | 2 +- .../src/modules/monitoring/grafana/grafana.rs | 51 +- .../grafana/grafana_alerting_score.rs | 32 + .../monitoring/grafana/helm/helm_grafana.rs | 28 - .../modules/monitoring/grafana/helm/mod.rs | 1 - .../grafana/{ => k8s}/crd/crd_grafana.rs | 0 .../crd/grafana_default_dashboard.rs | 0 .../monitoring/grafana/{ => k8s}/crd/mod.rs | 1 - .../grafana/{ => k8s}/crd/rhob_grafana.rs | 0 .../{crd => k8s/helm}/grafana_operator.rs | 0 .../monitoring/grafana/k8s/helm/mod.rs | 1 + .../src/modules/monitoring/grafana/k8s/mod.rs | 7 + .../grafana/k8s/score_ensure_grafana_ready.rs | 54 ++ .../k8s/score_grafana_alert_receiver.rs | 24 + .../grafana/k8s/score_grafana_datasource.rs | 83 +++ .../grafana/k8s/score_grafana_rule.rs | 24 + .../grafana/k8s/score_install_grafana.rs | 189 ++++++ harmony/src/modules/monitoring/grafana/mod.rs | 4 +- .../crd/crd_alertmanager_config.rs | 21 - .../kube_prometheus_alerting_score.rs | 3 +- harmony/src/modules/monitoring/okd/mod.rs | 2 +- ...rs => openshift_cluster_alerting_score.rs} | 0 .../monitoring/prometheus/prometheus.rs | 48 -- .../k8s_prometheus_alerting_score.rs | 573 ------------------ harmony/src/modules/prometheus/mod.rs | 2 - .../modules/prometheus/rhob_alerting_score.rs | 525 ---------------- 32 files changed, 613 insertions(+), 1735 deletions(-) create mode 100644 harmony/src/domain/topology/k8s_anywhere/observability/grafana.rs create mode 100644 harmony/src/modules/monitoring/grafana/grafana_alerting_score.rs delete mode 100644 harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs delete mode 100644 harmony/src/modules/monitoring/grafana/helm/mod.rs rename harmony/src/modules/monitoring/grafana/{ => k8s}/crd/crd_grafana.rs (100%) rename harmony/src/modules/monitoring/grafana/{ => k8s}/crd/grafana_default_dashboard.rs (100%) rename harmony/src/modules/monitoring/grafana/{ => k8s}/crd/mod.rs (75%) rename harmony/src/modules/monitoring/grafana/{ => k8s}/crd/rhob_grafana.rs (100%) rename harmony/src/modules/monitoring/grafana/{crd => k8s/helm}/grafana_operator.rs (100%) create mode 100644 harmony/src/modules/monitoring/grafana/k8s/helm/mod.rs create mode 100644 harmony/src/modules/monitoring/grafana/k8s/mod.rs create mode 100644 harmony/src/modules/monitoring/grafana/k8s/score_ensure_grafana_ready.rs create mode 100644 harmony/src/modules/monitoring/grafana/k8s/score_grafana_alert_receiver.rs create mode 100644 harmony/src/modules/monitoring/grafana/k8s/score_grafana_datasource.rs create mode 100644 harmony/src/modules/monitoring/grafana/k8s/score_grafana_rule.rs create mode 100644 harmony/src/modules/monitoring/grafana/k8s/score_install_grafana.rs rename harmony/src/modules/monitoring/okd/{score_openshift_cluster_alert_score.rs => openshift_cluster_alerting_score.rs} (100%) delete mode 100644 harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs delete mode 100644 harmony/src/modules/prometheus/mod.rs delete mode 100644 harmony/src/modules/prometheus/rhob_alerting_score.rs diff --git a/examples/okd_cluster_alerts/src/main.rs b/examples/okd_cluster_alerts/src/main.rs index 4a7fcff8..f83e1281 100644 --- a/examples/okd_cluster_alerts/src/main.rs +++ b/examples/okd_cluster_alerts/src/main.rs @@ -8,7 +8,7 @@ use harmony::{ }, prometheus_alert_rule::AlertManagerRuleGroup, }, - okd::score_openshift_cluster_alert_score::OpenshiftClusterAlertScore, + okd::openshift_cluster_alerting_score::OpenshiftClusterAlertScore, scrape_target::prometheus_node_exporter::PrometheusNodeExporter, }, topology::{ diff --git a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs index e3c1d8e9..9cab8b90 100644 --- a/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere/k8s_anywhere.rs @@ -1,13 +1,12 @@ -use std::{collections::BTreeMap, process::Command, sync::Arc, time::Duration}; +use std::{collections::BTreeMap, process::Command, sync::Arc}; use async_trait::async_trait; -use base64::{Engine, engine::general_purpose}; use harmony_types::rfc1123::Rfc1123Name; use k8s_openapi::api::{ core::v1::{Pod, Secret}, rbac::v1::{ClusterRoleBinding, RoleRef, Subject}, }; -use kube::api::{DynamicObject, GroupVersionKind, ObjectMeta}; +use kube::api::{GroupVersionKind, ObjectMeta}; use log::{debug, info, trace, warn}; use serde::Serialize; use tokio::sync::OnceCell; @@ -28,36 +27,16 @@ use crate::{ score_cert_management::CertificateManagementScore, }, k3d::K3DInstallationScore, - k8s::ingress::{K8sIngressScore, PathType}, - monitoring::{ - grafana::{ - crd::crd_grafana::{ - Grafana as GrafanaCRD, GrafanaCom, GrafanaDashboard, - GrafanaDashboardDatasource, GrafanaDashboardSpec, GrafanaDatasource, - GrafanaDatasourceConfig, GrafanaDatasourceJsonData, - GrafanaDatasourceSecureJsonData, GrafanaDatasourceSpec, GrafanaSpec, - }, - grafana::Grafana, - helm::helm_grafana::grafana_helm_chart_score, - }, - kube_prometheus::crd::{ - crd_alertmanager_config::CRDPrometheus, crd_prometheuses::LabelSelector, - prometheus_operator::prometheus_operator_helm_chart_score, - service_monitor::ServiceMonitor, - }, - }, okd::{crd::ingresses_config::Ingress as IngressResource, route::OKDTlsPassthroughScore}, - prometheus::k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore, }, score::Score, - topology::{TlsRoute, TlsRouter, ingress::Ingress, monitoring::Observability}, + topology::{TlsRoute, TlsRouter, ingress::Ingress}, }; use super::super::{ DeploymentTarget, HelmCommand, K8sclient, MultiTargetTopology, PreparationError, PreparationOutcome, Topology, k8s::K8sClient, - monitoring::AlertReceiver, tenant::{ TenantConfig, TenantManager, k8s::K8sTenantManager, @@ -171,216 +150,6 @@ impl TlsRouter for K8sAnywhereTopology { } } -#[async_trait] -impl Grafana for K8sAnywhereTopology { - async fn ensure_grafana_operator( - &self, - inventory: &Inventory, - ) -> Result { - debug!("ensure grafana operator"); - let client = self.k8s_client().await.unwrap(); - let grafana_gvk = GroupVersionKind { - group: "grafana.integreatly.org".to_string(), - version: "v1beta1".to_string(), - kind: "Grafana".to_string(), - }; - let name = "grafanas.grafana.integreatly.org"; - let ns = "grafana"; - - let grafana_crd = client - .get_resource_json_value(name, Some(ns), &grafana_gvk) - .await; - match grafana_crd { - Ok(_) => { - return Ok(PreparationOutcome::Success { - details: "Found grafana CRDs in cluster".to_string(), - }); - } - - Err(_) => { - return self - .install_grafana_operator(inventory, Some("grafana")) - .await; - } - }; - } - async fn install_grafana(&self) -> Result { - let ns = "grafana"; - - let mut label = BTreeMap::new(); - - label.insert("dashboards".to_string(), "grafana".to_string()); - - let label_selector = LabelSelector { - match_labels: label.clone(), - match_expressions: vec![], - }; - - let client = self.k8s_client().await?; - - let grafana = self.build_grafana(ns, &label); - - client.apply(&grafana, Some(ns)).await?; - //TODO change this to a ensure ready or something better than just a timeout - client - .wait_until_deployment_ready( - "grafana-grafana-deployment", - Some("grafana"), - Some(Duration::from_secs(30)), - ) - .await?; - - let sa_name = "grafana-grafana-sa"; - let token_secret_name = "grafana-sa-token-secret"; - - let sa_token_secret = self.build_sa_token_secret(token_secret_name, sa_name, ns); - - client.apply(&sa_token_secret, Some(ns)).await?; - let secret_gvk = GroupVersionKind { - group: "".to_string(), - version: "v1".to_string(), - kind: "Secret".to_string(), - }; - - let secret = client - .get_resource_json_value(token_secret_name, Some(ns), &secret_gvk) - .await?; - - let token = format!( - "Bearer {}", - self.extract_and_normalize_token(&secret).unwrap() - ); - - debug!("creating grafana clusterrole binding"); - - let clusterrolebinding = - self.build_cluster_rolebinding(sa_name, "cluster-monitoring-view", ns); - - client.apply(&clusterrolebinding, Some(ns)).await?; - - debug!("creating grafana datasource crd"); - - let thanos_url = format!( - "https://{}", - self.get_domain("thanos-querier-openshift-monitoring") - .await - .unwrap() - ); - - let thanos_openshift_datasource = self.build_grafana_datasource( - "thanos-openshift-monitoring", - ns, - &label_selector, - &thanos_url, - &token, - ); - - client.apply(&thanos_openshift_datasource, Some(ns)).await?; - - debug!("creating grafana dashboard crd"); - let dashboard = self.build_grafana_dashboard(ns, &label_selector); - - client.apply(&dashboard, Some(ns)).await?; - debug!("creating grafana ingress"); - let grafana_ingress = self.build_grafana_ingress(ns).await; - - grafana_ingress - .interpret(&Inventory::empty(), self) - .await - .map_err(|e| PreparationError::new(e.to_string()))?; - - Ok(PreparationOutcome::Success { - details: "Installed grafana composants".to_string(), - }) - } -} - -// #[async_trait] -// impl Monitor for K8sAnywhereTopology { -// async fn install_montoring( -// &self, -// sender: &CRDPrometheus, -// _inventory: &Inventory, -// _receivers: Option>>>, -// ) -> Result { -// let client = self.k8s_client().await?; -// -// for monitor in sender.service_monitor.iter() { -// client -// .apply(monitor, Some(&sender.namespace)) -// .await -// .map_err(|e| PreparationError::new(e.to_string()))?; -// } -// Ok(PreparationOutcome::Success { -// details: "successfuly installed prometheus components".to_string(), -// }) -// } -// -// async fn ensure_monitoring( -// &self, -// sender: &CRDPrometheus, -// _inventory: &Inventory, -// ) -> Result { -// let po_result = self.ensure_prometheus_operator(sender).await?; -// -// match po_result { -// PreparationOutcome::Success { details: _ } => { -// debug!("Detected prometheus crds operator present in cluster."); -// return Ok(po_result); -// } -// PreparationOutcome::Noop => { -// debug!("Skipping Prometheus CR installation due to missing operator."); -// return Ok(po_result); -// } -// } -// } -// } -// -// #[async_trait] -// impl Monitor for K8sAnywhereTopology { -// async fn install_montoring( -// &self, -// sender: &RHOBObservability, -// inventory: &Inventory, -// receivers: Option>>>, -// ) -> Result { -// let po_result = self.ensure_cluster_observability_operator(sender).await?; -// -// if po_result == PreparationOutcome::Noop { -// debug!("Skipping Prometheus CR installation due to missing operator."); -// return Ok(po_result); -// } -// -// let result = self -// .get_cluster_observability_operator_prometheus_application_score( -// sender.clone(), -// receivers, -// ) -// .await -// .interpret(inventory, self) -// .await; -// -// match result { -// Ok(outcome) => match outcome.status { -// InterpretStatus::SUCCESS => Ok(PreparationOutcome::Success { -// details: outcome.message, -// }), -// InterpretStatus::NOOP => Ok(PreparationOutcome::Noop), -// _ => Err(PreparationError::new(outcome.message)), -// }, -// Err(err) => Err(PreparationError::new(err.to_string())), -// } -// } -// -// async fn ensure_monitoring( -// &self, -// sender: &RHOBObservability, -// inventory: &Inventory, -// ) -> Result { -// todo!() -// } -// } - impl Serialize for K8sAnywhereTopology { fn serialize(&self, _serializer: S) -> Result where @@ -585,23 +354,6 @@ impl K8sAnywhereTopology { } } - fn extract_and_normalize_token(&self, secret: &DynamicObject) -> Option { - let token_b64 = secret - .data - .get("token") - .or_else(|| secret.data.get("data").and_then(|d| d.get("token"))) - .and_then(|v| v.as_str())?; - - let bytes = general_purpose::STANDARD.decode(token_b64).ok()?; - - let s = String::from_utf8(bytes).ok()?; - - let cleaned = s - .trim_matches(|c: char| c.is_whitespace() || c == '\0') - .to_string(); - Some(cleaned) - } - pub async fn get_k8s_distribution(&self) -> Result { self.k8s_client() .await? @@ -661,141 +413,6 @@ impl K8sAnywhereTopology { } } - fn build_grafana_datasource( - &self, - name: &str, - ns: &str, - label_selector: &LabelSelector, - url: &str, - token: &str, - ) -> GrafanaDatasource { - let mut json_data = BTreeMap::new(); - json_data.insert("timeInterval".to_string(), "5s".to_string()); - - GrafanaDatasource { - metadata: ObjectMeta { - name: Some(name.to_string()), - namespace: Some(ns.to_string()), - ..Default::default() - }, - spec: GrafanaDatasourceSpec { - instance_selector: label_selector.clone(), - allow_cross_namespace_import: Some(true), - values_from: None, - datasource: GrafanaDatasourceConfig { - access: "proxy".to_string(), - name: name.to_string(), - r#type: "prometheus".to_string(), - url: url.to_string(), - database: None, - json_data: Some(GrafanaDatasourceJsonData { - time_interval: Some("60s".to_string()), - http_header_name1: Some("Authorization".to_string()), - tls_skip_verify: Some(true), - oauth_pass_thru: Some(true), - }), - secure_json_data: Some(GrafanaDatasourceSecureJsonData { - http_header_value1: Some(format!("Bearer {token}")), - }), - is_default: Some(false), - editable: Some(true), - }, - }, - } - } - - fn build_grafana_dashboard( - &self, - ns: &str, - label_selector: &LabelSelector, - ) -> GrafanaDashboard { - let graf_dashboard = GrafanaDashboard { - metadata: ObjectMeta { - name: Some(format!("grafana-dashboard-{}", ns)), - namespace: Some(ns.to_string()), - ..Default::default() - }, - spec: GrafanaDashboardSpec { - resync_period: Some("30s".to_string()), - instance_selector: label_selector.clone(), - datasources: Some(vec![GrafanaDashboardDatasource { - input_name: "DS_PROMETHEUS".to_string(), - datasource_name: "thanos-openshift-monitoring".to_string(), - }]), - json: None, - grafana_com: Some(GrafanaCom { - id: 17406, - revision: None, - }), - }, - }; - graf_dashboard - } - - fn build_grafana(&self, ns: &str, labels: &BTreeMap) -> GrafanaCRD { - let grafana = GrafanaCRD { - metadata: ObjectMeta { - name: Some(format!("grafana-{}", ns)), - namespace: Some(ns.to_string()), - labels: Some(labels.clone()), - ..Default::default() - }, - spec: GrafanaSpec { - config: None, - admin_user: None, - admin_password: None, - ingress: None, - persistence: None, - resources: None, - }, - }; - grafana - } - - async fn build_grafana_ingress(&self, ns: &str) -> K8sIngressScore { - let domain = self.get_domain(&format!("grafana-{}", ns)).await.unwrap(); - let name = format!("{}-grafana", ns); - let backend_service = format!("grafana-{}-service", ns); - - K8sIngressScore { - name: fqdn::fqdn!(&name), - host: fqdn::fqdn!(&domain), - backend_service: fqdn::fqdn!(&backend_service), - port: 3000, - path: Some("/".to_string()), - path_type: Some(PathType::Prefix), - namespace: Some(fqdn::fqdn!(&ns)), - ingress_class_name: Some("openshift-default".to_string()), - } - } - - // async fn get_cluster_observability_operator_prometheus_application_score( - // &self, - // sender: RHOBObservability, - // receivers: Option>>>, - // ) -> RHOBAlertingScore { - // RHOBAlertingScore { - // sender, - // receivers: receivers.unwrap_or_default(), - // service_monitors: vec![], - // prometheus_rules: vec![], - // } - // } - - async fn get_k8s_prometheus_application_score( - &self, - sender: CRDPrometheus, - receivers: Option>>>, - service_monitors: Option>, - ) -> K8sPrometheusCRDAlertingScore { - return K8sPrometheusCRDAlertingScore { - sender, - receivers: receivers.unwrap_or_default(), - service_monitors: service_monitors.unwrap_or_default(), - prometheus_rules: vec![], - }; - } - async fn openshift_ingress_operator_available(&self) -> Result<(), PreparationError> { let client = self.k8s_client().await?; let gvk = GroupVersionKind { @@ -961,137 +578,6 @@ impl K8sAnywhereTopology { )), } } - - // async fn ensure_cluster_observability_operator( - // &self, - // sender: &RHOBObservability, - // ) -> Result { - // let status = Command::new("sh") - // .args(["-c", "kubectl get crd -A | grep -i rhobs"]) - // .status() - // .map_err(|e| PreparationError::new(format!("could not connect to cluster: {}", e)))?; - // - // if !status.success() { - // if let Some(Some(k8s_state)) = self.k8s_state.get() { - // match k8s_state.source { - // K8sSource::LocalK3d => { - // warn!( - // "Installing observability operator is not supported on LocalK3d source" - // ); - // return Ok(PreparationOutcome::Noop); - // debug!("installing cluster observability operator"); - // todo!(); - // let op_score = - // prometheus_operator_helm_chart_score(sender.namespace.clone()); - // let result = op_score.interpret(&Inventory::empty(), self).await; - // - // return match result { - // Ok(outcome) => match outcome.status { - // InterpretStatus::SUCCESS => Ok(PreparationOutcome::Success { - // details: "installed cluster observability operator".into(), - // }), - // InterpretStatus::NOOP => Ok(PreparationOutcome::Noop), - // _ => Err(PreparationError::new( - // "failed to install cluster observability operator (unknown error)".into(), - // )), - // }, - // Err(err) => Err(PreparationError::new(err.to_string())), - // }; - // } - // K8sSource::Kubeconfig => { - // debug!( - // "unable to install cluster observability operator, contact cluster admin" - // ); - // return Ok(PreparationOutcome::Noop); - // } - // } - // } else { - // warn!( - // "Unable to detect k8s_state. Skipping Cluster Observability Operator install." - // ); - // return Ok(PreparationOutcome::Noop); - // } - // } - // - // debug!("Cluster Observability Operator is already present, skipping install"); - // - // Ok(PreparationOutcome::Success { - // details: "cluster observability operator present in cluster".into(), - // }) - // } - - async fn ensure_prometheus_operator( - &self, - sender: &CRDPrometheus, - ) -> Result { - let status = Command::new("sh") - .args(["-c", "kubectl get crd -A | grep -i prometheuses"]) - .status() - .map_err(|e| PreparationError::new(format!("could not connect to cluster: {}", e)))?; - - if !status.success() { - if let Some(Some(k8s_state)) = self.k8s_state.get() { - match k8s_state.source { - K8sSource::LocalK3d => { - debug!("installing prometheus operator"); - let op_score = - prometheus_operator_helm_chart_score(sender.namespace.clone()); - let result = op_score.interpret(&Inventory::empty(), self).await; - - return match result { - Ok(outcome) => match outcome.status { - InterpretStatus::SUCCESS => Ok(PreparationOutcome::Success { - details: "installed prometheus operator".into(), - }), - InterpretStatus::NOOP => Ok(PreparationOutcome::Noop), - _ => Err(PreparationError::new( - "failed to install prometheus operator (unknown error)".into(), - )), - }, - Err(err) => Err(PreparationError::new(err.to_string())), - }; - } - K8sSource::Kubeconfig => { - debug!("unable to install prometheus operator, contact cluster admin"); - return Ok(PreparationOutcome::Noop); - } - } - } else { - warn!("Unable to detect k8s_state. Skipping Prometheus Operator install."); - return Ok(PreparationOutcome::Noop); - } - } - - debug!("Prometheus operator is already present, skipping install"); - - Ok(PreparationOutcome::Success { - details: "prometheus operator present in cluster".into(), - }) - } - - async fn install_grafana_operator( - &self, - inventory: &Inventory, - ns: Option<&str>, - ) -> Result { - let namespace = ns.unwrap_or("grafana"); - info!("installing grafana operator in ns {namespace}"); - let tenant = self.get_k8s_tenant_manager()?.get_tenant_config().await; - let mut namespace_scope = false; - if tenant.is_some() { - namespace_scope = true; - } - let _grafana_operator_score = grafana_helm_chart_score(namespace, namespace_scope) - .interpret(inventory, self) - .await - .map_err(|e| PreparationError::new(e.to_string())); - Ok(PreparationOutcome::Success { - details: format!( - "Successfully installed grafana operator in ns {}", - ns.unwrap() - ), - }) - } } #[derive(Clone, Debug)] diff --git a/harmony/src/domain/topology/k8s_anywhere/observability/grafana.rs b/harmony/src/domain/topology/k8s_anywhere/observability/grafana.rs new file mode 100644 index 00000000..7f06a2af --- /dev/null +++ b/harmony/src/domain/topology/k8s_anywhere/observability/grafana.rs @@ -0,0 +1,147 @@ +use async_trait::async_trait; + +use crate::{ + inventory::Inventory, + modules::monitoring::grafana::{ + grafana::Grafana, + k8s::{ + score_ensure_grafana_ready::GrafanaK8sEnsureReadyScore, + score_grafana_alert_receiver::GrafanaK8sReceiverScore, + score_grafana_datasource::GrafanaK8sDatasourceScore, + score_grafana_rule::GrafanaK8sRuleScore, score_install_grafana::GrafanaK8sInstallScore, + }, + }, + score::Score, + topology::{ + K8sAnywhereTopology, PreparationError, PreparationOutcome, + monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, + }, +}; + +#[async_trait] +impl Observability for K8sAnywhereTopology { + async fn install_alert_sender( + &self, + sender: &Grafana, + inventory: &Inventory, + ) -> Result { + let score = GrafanaK8sInstallScore { + sender: sender.clone(), + }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Grafana not installed {}", e)))?; + Ok(PreparationOutcome::Success { + details: "Successfully installed grafana alert sender".to_string(), + }) + } + + async fn install_receivers( + &self, + sender: &Grafana, + inventory: &Inventory, + receivers: Option>>>, + ) -> Result { + let receivers = match receivers { + Some(r) if !r.is_empty() => r, + _ => return Ok(PreparationOutcome::Noop), + }; + + for receiver in receivers { + let score = GrafanaK8sReceiverScore { + receiver, + sender: sender.clone(), + }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Failed to install receiver: {}", e)))?; + } + + Ok(PreparationOutcome::Success { + details: "All alert receivers installed successfully".to_string(), + }) + } + + async fn install_rules( + &self, + sender: &Grafana, + inventory: &Inventory, + rules: Option>>>, + ) -> Result { + let rules = match rules { + Some(r) if !r.is_empty() => r, + _ => return Ok(PreparationOutcome::Noop), + }; + + for rule in rules { + let score = GrafanaK8sRuleScore { + sender: sender.clone(), + rule, + }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Failed to install rule: {}", e)))?; + } + + Ok(PreparationOutcome::Success { + details: "All alert rules installed successfully".to_string(), + }) + } + + async fn add_scrape_targets( + &self, + sender: &Grafana, + inventory: &Inventory, + scrape_targets: Option>>>, + ) -> Result { + let scrape_targets = match scrape_targets { + Some(r) if !r.is_empty() => r, + _ => return Ok(PreparationOutcome::Noop), + }; + + for scrape_target in scrape_targets { + let score = GrafanaK8sDatasourceScore { + scrape_target, + sender: sender.clone(), + }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Failed to add DataSource: {}", e)))?; + } + + Ok(PreparationOutcome::Success { + details: "All datasources installed successfully".to_string(), + }) + } + + async fn ensure_monitoring_installed( + &self, + sender: &Grafana, + inventory: &Inventory, + ) -> Result { + let score = GrafanaK8sEnsureReadyScore { + sender: sender.clone(), + }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Grafana not ready {}", e)))?; + Ok(PreparationOutcome::Success { + details: "Grafana Ready".to_string(), + }) + } +} diff --git a/harmony/src/domain/topology/k8s_anywhere/observability/mod.rs b/harmony/src/domain/topology/k8s_anywhere/observability/mod.rs index 3be45c05..6e105557 100644 --- a/harmony/src/domain/topology/k8s_anywhere/observability/mod.rs +++ b/harmony/src/domain/topology/k8s_anywhere/observability/mod.rs @@ -1,3 +1,4 @@ +pub mod grafana; pub mod kube_prometheus; pub mod openshift_monitoring; pub mod prometheus; diff --git a/harmony/src/modules/application/features/monitoring.rs b/harmony/src/modules/application/features/monitoring.rs index a4aba4ab..17f263dc 100644 --- a/harmony/src/modules/application/features/monitoring.rs +++ b/harmony/src/modules/application/features/monitoring.rs @@ -2,7 +2,6 @@ use crate::modules::application::{ Application, ApplicationFeature, InstallationError, InstallationOutcome, }; use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore; -use crate::modules::monitoring::grafana::grafana::Grafana; use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus; use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{ ServiceMonitor, ServiceMonitorSpec, @@ -45,7 +44,6 @@ impl< + K8sclient + MultiTargetTopology + Observability - + Grafana + Ingress + std::fmt::Debug, > ApplicationFeature for Monitoring diff --git a/harmony/src/modules/mod.rs b/harmony/src/modules/mod.rs index 3fa69469..21f2295c 100644 --- a/harmony/src/modules/mod.rs +++ b/harmony/src/modules/mod.rs @@ -18,7 +18,6 @@ pub mod network; pub mod okd; pub mod opnsense; pub mod postgresql; -pub mod prometheus; pub mod storage; pub mod tenant; pub mod tftp; diff --git a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs index 81ab6c37..11be6a73 100644 --- a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs +++ b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs @@ -25,7 +25,7 @@ pub struct ApplicationMonitoringScore { pub receivers: Vec>>, } -impl + K8sclient + Grafana> Score +impl + K8sclient> Score for ApplicationMonitoringScore { fn create_interpret(&self) -> Box> { diff --git a/harmony/src/modules/monitoring/grafana/grafana.rs b/harmony/src/modules/monitoring/grafana/grafana.rs index 5ab57c27..e28f61a3 100644 --- a/harmony/src/modules/monitoring/grafana/grafana.rs +++ b/harmony/src/modules/monitoring/grafana/grafana.rs @@ -1,17 +1,48 @@ use async_trait::async_trait; -use k8s_openapi::Resource; +use serde::Serialize; use crate::{ inventory::Inventory, - topology::{PreparationError, PreparationOutcome}, + topology::{ + PreparationError, PreparationOutcome, + monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}, + }, }; -#[async_trait] -pub trait Grafana { - async fn ensure_grafana_operator( - &self, - inventory: &Inventory, - ) -> Result; - - async fn install_grafana(&self) -> Result; +#[derive(Debug, Clone, Serialize)] +pub struct Grafana { + pub namespace: String, +} + +impl AlertSender for Grafana { + fn name(&self) -> String { + "grafana".to_string() + } +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } } diff --git a/harmony/src/modules/monitoring/grafana/grafana_alerting_score.rs b/harmony/src/modules/monitoring/grafana/grafana_alerting_score.rs new file mode 100644 index 00000000..a2f00e8c --- /dev/null +++ b/harmony/src/modules/monitoring/grafana/grafana_alerting_score.rs @@ -0,0 +1,32 @@ +use serde::Serialize; + +use crate::{ + modules::monitoring::grafana::grafana::Grafana, + score::Score, + topology::{ + HelmCommand, Topology, + monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget}, + }, +}; + +#[derive(Clone, Debug, Serialize)] +pub struct GrafanaAlertingScore { + pub receivers: Vec>>, + pub rules: Vec>>, + pub scrape_targets: Option>>>, + pub sender: Grafana, +} + +impl> Score for GrafanaAlertingScore { + fn create_interpret(&self) -> Box> { + Box::new(AlertingInterpret { + sender: self.sender.clone(), + receivers: self.receivers.clone(), + rules: self.rules.clone(), + scrape_targets: self.scrape_targets.clone(), + }) + } + fn name(&self) -> String { + "HelmPrometheusAlertingScore".to_string() + } +} diff --git a/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs b/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs deleted file mode 100644 index c9ccacb0..00000000 --- a/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs +++ /dev/null @@ -1,28 +0,0 @@ -use harmony_macros::hurl; -use non_blank_string_rs::NonBlankString; -use std::{collections::HashMap, str::FromStr}; - -use crate::modules::helm::chart::{HelmChartScore, HelmRepository}; - -pub fn grafana_helm_chart_score(ns: &str, namespace_scope: bool) -> HelmChartScore { - let mut values_overrides = HashMap::new(); - values_overrides.insert( - NonBlankString::from_str("namespaceScope").unwrap(), - namespace_scope.to_string(), - ); - HelmChartScore { - namespace: Some(NonBlankString::from_str(ns).unwrap()), - release_name: NonBlankString::from_str("grafana-operator").unwrap(), - chart_name: NonBlankString::from_str("grafana/grafana-operator").unwrap(), - chart_version: None, - values_overrides: Some(values_overrides), - values_yaml: None, - create_namespace: true, - install_only: true, - repository: Some(HelmRepository::new( - "grafana".to_string(), - hurl!("https://grafana.github.io/helm-charts"), - true, - )), - } -} diff --git a/harmony/src/modules/monitoring/grafana/helm/mod.rs b/harmony/src/modules/monitoring/grafana/helm/mod.rs deleted file mode 100644 index 1d35fdf0..00000000 --- a/harmony/src/modules/monitoring/grafana/helm/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod helm_grafana; diff --git a/harmony/src/modules/monitoring/grafana/crd/crd_grafana.rs b/harmony/src/modules/monitoring/grafana/k8s/crd/crd_grafana.rs similarity index 100% rename from harmony/src/modules/monitoring/grafana/crd/crd_grafana.rs rename to harmony/src/modules/monitoring/grafana/k8s/crd/crd_grafana.rs diff --git a/harmony/src/modules/monitoring/grafana/crd/grafana_default_dashboard.rs b/harmony/src/modules/monitoring/grafana/k8s/crd/grafana_default_dashboard.rs similarity index 100% rename from harmony/src/modules/monitoring/grafana/crd/grafana_default_dashboard.rs rename to harmony/src/modules/monitoring/grafana/k8s/crd/grafana_default_dashboard.rs diff --git a/harmony/src/modules/monitoring/grafana/crd/mod.rs b/harmony/src/modules/monitoring/grafana/k8s/crd/mod.rs similarity index 75% rename from harmony/src/modules/monitoring/grafana/crd/mod.rs rename to harmony/src/modules/monitoring/grafana/k8s/crd/mod.rs index 1a91d94d..bc9ac6de 100644 --- a/harmony/src/modules/monitoring/grafana/crd/mod.rs +++ b/harmony/src/modules/monitoring/grafana/k8s/crd/mod.rs @@ -1,4 +1,3 @@ pub mod crd_grafana; pub mod grafana_default_dashboard; -pub mod grafana_operator; pub mod rhob_grafana; diff --git a/harmony/src/modules/monitoring/grafana/crd/rhob_grafana.rs b/harmony/src/modules/monitoring/grafana/k8s/crd/rhob_grafana.rs similarity index 100% rename from harmony/src/modules/monitoring/grafana/crd/rhob_grafana.rs rename to harmony/src/modules/monitoring/grafana/k8s/crd/rhob_grafana.rs diff --git a/harmony/src/modules/monitoring/grafana/crd/grafana_operator.rs b/harmony/src/modules/monitoring/grafana/k8s/helm/grafana_operator.rs similarity index 100% rename from harmony/src/modules/monitoring/grafana/crd/grafana_operator.rs rename to harmony/src/modules/monitoring/grafana/k8s/helm/grafana_operator.rs diff --git a/harmony/src/modules/monitoring/grafana/k8s/helm/mod.rs b/harmony/src/modules/monitoring/grafana/k8s/helm/mod.rs new file mode 100644 index 00000000..97edab97 --- /dev/null +++ b/harmony/src/modules/monitoring/grafana/k8s/helm/mod.rs @@ -0,0 +1 @@ +pub mod grafana_operator; diff --git a/harmony/src/modules/monitoring/grafana/k8s/mod.rs b/harmony/src/modules/monitoring/grafana/k8s/mod.rs new file mode 100644 index 00000000..107a5bd7 --- /dev/null +++ b/harmony/src/modules/monitoring/grafana/k8s/mod.rs @@ -0,0 +1,7 @@ +pub mod crd; +pub mod helm; +pub mod score_ensure_grafana_ready; +pub mod score_grafana_alert_receiver; +pub mod score_grafana_datasource; +pub mod score_grafana_rule; +pub mod score_install_grafana; diff --git a/harmony/src/modules/monitoring/grafana/k8s/score_ensure_grafana_ready.rs b/harmony/src/modules/monitoring/grafana/k8s/score_ensure_grafana_ready.rs new file mode 100644 index 00000000..81c3d396 --- /dev/null +++ b/harmony/src/modules/monitoring/grafana/k8s/score_ensure_grafana_ready.rs @@ -0,0 +1,54 @@ +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::monitoring::grafana::grafana::Grafana, + score::Score, + topology::{K8sclient, Topology}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct GrafanaK8sEnsureReadyScore { + pub sender: Grafana, +} + +impl Score for GrafanaK8sEnsureReadyScore { + fn name(&self) -> String { + "GrafanaK8sEnsureReadyScore".to_string() + } + + fn create_interpret(&self) -> Box> { + todo!() + } +} +// async fn ensure_ready( +// &self, +// inventory: &Inventory, +// ) -> Result { +// debug!("ensure grafana operator"); +// let client = self.k8s_client().await.unwrap(); +// let grafana_gvk = GroupVersionKind { +// group: "grafana.integreatly.org".to_string(), +// version: "v1beta1".to_string(), +// kind: "Grafana".to_string(), +// }; +// let name = "grafanas.grafana.integreatly.org"; +// let ns = "grafana"; +// +// let grafana_crd = client +// .get_resource_json_value(name, Some(ns), &grafana_gvk) +// .await; +// match grafana_crd { +// Ok(_) => { +// return Ok(PreparationOutcome::Success { +// details: "Found grafana CRDs in cluster".to_string(), +// }); +// } +// +// Err(_) => { +// return self +// .install_grafana_operator(inventory, Some("grafana")) +// .await; +// } +// }; +// } diff --git a/harmony/src/modules/monitoring/grafana/k8s/score_grafana_alert_receiver.rs b/harmony/src/modules/monitoring/grafana/k8s/score_grafana_alert_receiver.rs new file mode 100644 index 00000000..9e4d15fa --- /dev/null +++ b/harmony/src/modules/monitoring/grafana/k8s/score_grafana_alert_receiver.rs @@ -0,0 +1,24 @@ +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::monitoring::grafana::grafana::Grafana, + score::Score, + topology::{K8sclient, Topology, monitoring::AlertReceiver}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct GrafanaK8sReceiverScore { + pub sender: Grafana, + pub receiver: Box>, +} + +impl Score for GrafanaK8sReceiverScore { + fn name(&self) -> String { + "GrafanaK8sReceiverScore".to_string() + } + + fn create_interpret(&self) -> Box> { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/grafana/k8s/score_grafana_datasource.rs b/harmony/src/modules/monitoring/grafana/k8s/score_grafana_datasource.rs new file mode 100644 index 00000000..71b723b1 --- /dev/null +++ b/harmony/src/modules/monitoring/grafana/k8s/score_grafana_datasource.rs @@ -0,0 +1,83 @@ +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::monitoring::grafana::grafana::Grafana, + score::Score, + topology::{K8sclient, Topology, monitoring::ScrapeTarget}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct GrafanaK8sDatasourceScore { + pub sender: Grafana, + pub scrape_target: Box>, +} + +impl Score for GrafanaK8sDatasourceScore { + fn name(&self) -> String { + "GrafanaK8sDatasourceScore".to_string() + } + + fn create_interpret(&self) -> Box> { + todo!() + } +} + +// fn extract_and_normalize_token(&self, secret: &DynamicObject) -> Option { +// let token_b64 = secret +// .data +// .get("token") +// .or_else(|| secret.data.get("data").and_then(|d| d.get("token"))) +// .and_then(|v| v.as_str())?; +// +// let bytes = general_purpose::STANDARD.decode(token_b64).ok()?; +// +// let s = String::from_utf8(bytes).ok()?; +// +// let cleaned = s +// .trim_matches(|c: char| c.is_whitespace() || c == '\0') +// .to_string(); +// Some(cleaned) +// } +// fn build_grafana_datasource( +// &self, +// name: &str, +// ns: &str, +// label_selector: &LabelSelector, +// url: &str, +// token: &str, +// ) -> GrafanaDatasource { +// let mut json_data = BTreeMap::new(); +// json_data.insert("timeInterval".to_string(), "5s".to_string()); +// +// GrafanaDatasource { +// metadata: ObjectMeta { +// name: Some(name.to_string()), +// namespace: Some(ns.to_string()), +// ..Default::default() +// }, +// spec: GrafanaDatasourceSpec { +// instance_selector: label_selector.clone(), +// allow_cross_namespace_import: Some(true), +// values_from: None, +// datasource: GrafanaDatasourceConfig { +// access: "proxy".to_string(), +// name: name.to_string(), +// rype: "prometheus".to_string(), +// url: url.to_string(), +// database: None, +// json_data: Some(GrafanaDatasourceJsonData { +// time_interval: Some("60s".to_string()), +// http_header_name1: Some("Authorization".to_string()), +// tls_skip_verify: Some(true), +// oauth_pass_thru: Some(true), +// }), +// secure_json_data: Some(GrafanaDatasourceSecureJsonData { +// http_header_value1: Some(format!("Bearer {token}")), +// }), +// is_default: Some(false), +// editable: Some(true), +// }, +// }, +// } +// } diff --git a/harmony/src/modules/monitoring/grafana/k8s/score_grafana_rule.rs b/harmony/src/modules/monitoring/grafana/k8s/score_grafana_rule.rs new file mode 100644 index 00000000..464dde8d --- /dev/null +++ b/harmony/src/modules/monitoring/grafana/k8s/score_grafana_rule.rs @@ -0,0 +1,24 @@ +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::monitoring::grafana::grafana::Grafana, + score::Score, + topology::{K8sclient, Topology, monitoring::AlertRule}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct GrafanaK8sRuleScore { + pub sender: Grafana, + pub rule: Box>, +} + +impl Score for GrafanaK8sRuleScore { + fn name(&self) -> String { + "GrafanaK8sRuleScore".to_string() + } + + fn create_interpret(&self) -> Box> { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/grafana/k8s/score_install_grafana.rs b/harmony/src/modules/monitoring/grafana/k8s/score_install_grafana.rs new file mode 100644 index 00000000..60c109e5 --- /dev/null +++ b/harmony/src/modules/monitoring/grafana/k8s/score_install_grafana.rs @@ -0,0 +1,189 @@ +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::monitoring::grafana::grafana::Grafana, + score::Score, + topology::{K8sclient, Topology}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct GrafanaK8sInstallScore { + pub sender: Grafana, +} + +impl Score for GrafanaK8sInstallScore { + fn name(&self) -> String { + "GrafanaK8sEnsureReadyScore".to_string() + } + + fn create_interpret(&self) -> Box> { + todo!() + } +} +// let score = grafana_operator_helm_chart_score(sender.namespace.clone()); +// +// score +// .create_interpret() +// .execute(inventory, self) +// .await +// .map_err(|e| PreparationError::new(e.to_string()))?; +// + +// +// fn build_grafana_dashboard( +// &self, +// ns: &str, +// label_selector: &LabelSelector, +// ) -> GrafanaDashboard { +// let graf_dashboard = GrafanaDashboard { +// metadata: ObjectMeta { +// name: Some(format!("grafana-dashboard-{}", ns)), +// namespace: Some(ns.to_string()), +// ..Default::default() +// }, +// spec: GrafanaDashboardSpec { +// resync_period: Some("30s".to_string()), +// instance_selector: label_selector.clone(), +// datasources: Some(vec![GrafanaDashboardDatasource { +// input_name: "DS_PROMETHEUS".to_string(), +// datasource_name: "thanos-openshift-monitoring".to_string(), +// }]), +// json: None, +// grafana_com: Some(GrafanaCom { +// id: 17406, +// revision: None, +// }), +// }, +// }; +// graf_dashboard +// } +// +// fn build_grafana(&self, ns: &str, labels: &BTreeMap) -> GrafanaCRD { +// let grafana = GrafanaCRD { +// metadata: ObjectMeta { +// name: Some(format!("grafana-{}", ns)), +// namespace: Some(ns.to_string()), +// labels: Some(labels.clone()), +// ..Default::default() +// }, +// spec: GrafanaSpec { +// config: None, +// admin_user: None, +// admin_password: None, +// ingress: None, +// persistence: None, +// resources: None, +// }, +// }; +// grafana +// } +// +// async fn build_grafana_ingress(&self, ns: &str) -> K8sIngressScore { +// let domain = self.get_domain(&format!("grafana-{}", ns)).await.unwrap(); +// let name = format!("{}-grafana", ns); +// let backend_service = format!("grafana-{}-service", ns); +// +// K8sIngressScore { +// name: fqdn::fqdn!(&name), +// host: fqdn::fqdn!(&domain), +// backend_service: fqdn::fqdn!(&backend_service), +// port: 3000, +// path: Some("/".to_string()), +// path_type: Some(PathType::Prefix), +// namespace: Some(fqdn::fqdn!(&ns)), +// ingress_class_name: Some("openshift-default".to_string()), +// } +// } +// #[async_trait] +// impl Grafana for K8sAnywhereTopology { +// async fn install_grafana(&self) -> Result { +// let ns = "grafana"; +// +// let mut label = BTreeMap::new(); +// +// label.insert("dashboards".to_string(), "grafana".to_string()); +// +// let label_selector = LabelSelector { +// match_labels: label.clone(), +// match_expressions: vec![], +// }; +// +// let client = self.k8s_client().await?; +// +// let grafana = self.build_grafana(ns, &label); +// +// client.apply(&grafana, Some(ns)).await?; +// //TODO change this to a ensure ready or something better than just a timeout +// client +// .wait_until_deployment_ready( +// "grafana-grafana-deployment", +// Some("grafana"), +// Some(Duration::from_secs(30)), +// ) +// .await?; +// +// let sa_name = "grafana-grafana-sa"; +// let token_secret_name = "grafana-sa-token-secret"; +// +// let sa_token_secret = self.build_sa_token_secret(token_secret_name, sa_name, ns); +// +// client.apply(&sa_token_secret, Some(ns)).await?; +// let secret_gvk = GroupVersionKind { +// group: "".to_string(), +// version: "v1".to_string(), +// kind: "Secret".to_string(), +// }; +// +// let secret = client +// .get_resource_json_value(token_secret_name, Some(ns), &secret_gvk) +// .await?; +// +// let token = format!( +// "Bearer {}", +// self.extract_and_normalize_token(&secret).unwrap() +// ); +// +// debug!("creating grafana clusterrole binding"); +// +// let clusterrolebinding = +// self.build_cluster_rolebinding(sa_name, "cluster-monitoring-view", ns); +// +// client.apply(&clusterrolebinding, Some(ns)).await?; +// +// debug!("creating grafana datasource crd"); +// +// let thanos_url = format!( +// "https://{}", +// self.get_domain("thanos-querier-openshift-monitoring") +// .await +// .unwrap() +// ); +// +// let thanos_openshift_datasource = self.build_grafana_datasource( +// "thanos-openshift-monitoring", +// ns, +// &label_selector, +// &thanos_url, +// &token, +// ); +// +// client.apply(&thanos_openshift_datasource, Some(ns)).await?; +// +// debug!("creating grafana dashboard crd"); +// let dashboard = self.build_grafana_dashboard(ns, &label_selector); +// +// client.apply(&dashboard, Some(ns)).await?; +// debug!("creating grafana ingress"); +// let grafana_ingress = self.build_grafana_ingress(ns).await; +// +// grafana_ingress +// .interpret(&Inventory::empty(), self) +// .await +// .map_err(|e| PreparationError::new(e.to_string()))?; +// +// Ok(PreparationOutcome::Success { +// details: "Installed grafana composants".to_string(), +// }) +// } +// } diff --git a/harmony/src/modules/monitoring/grafana/mod.rs b/harmony/src/modules/monitoring/grafana/mod.rs index 08c5c666..3176b923 100644 --- a/harmony/src/modules/monitoring/grafana/mod.rs +++ b/harmony/src/modules/monitoring/grafana/mod.rs @@ -1,3 +1,3 @@ -pub mod crd; pub mod grafana; -pub mod helm; +pub mod grafana_alerting_score; +pub mod k8s; diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs index 0b9002b6..7f0ae6f5 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs @@ -54,24 +54,3 @@ impl Serialize for Box> { todo!() } } - -#[async_trait] -impl + Grafana> Installable - for CRDPrometheus -{ - async fn configure(&self, inventory: &Inventory, topology: &T) -> Result<(), InterpretError> { - topology.ensure_grafana_operator(inventory).await?; - // topology.ensure_monitoring(self, inventory).await?; - Ok(()) - } - - async fn ensure_installed( - &self, - inventory: &Inventory, - topology: &T, - ) -> Result<(), InterpretError> { - topology.install_grafana().await?; - // topology.install_montoring(&self, inventory, None).await?; - Ok(()) - } -} diff --git a/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs b/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs index 37743eba..f48d10fa 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs @@ -22,10 +22,11 @@ pub struct KubePrometheusAlertingScore { pub config: Arc>, } -impl> Score +impl> Score for KubePrometheusAlertingScore { fn create_interpret(&self) -> Box> { + //TODO test that additional service monitor is added self.config .try_lock() .expect("couldn't lock config") diff --git a/harmony/src/modules/monitoring/okd/mod.rs b/harmony/src/modules/monitoring/okd/mod.rs index b2251aa0..0af3033a 100644 --- a/harmony/src/modules/monitoring/okd/mod.rs +++ b/harmony/src/modules/monitoring/okd/mod.rs @@ -5,9 +5,9 @@ use serde::Serialize; use crate::topology::monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}; pub mod crd; +pub mod openshift_cluster_alerting_score; pub mod score_enable_cluster_monitoring; pub mod score_openshift_alert_rule; -pub mod score_openshift_cluster_alert_score; pub mod score_openshift_receiver; pub mod score_openshift_scrape_target; pub mod score_user_workload; diff --git a/harmony/src/modules/monitoring/okd/score_openshift_cluster_alert_score.rs b/harmony/src/modules/monitoring/okd/openshift_cluster_alerting_score.rs similarity index 100% rename from harmony/src/modules/monitoring/okd/score_openshift_cluster_alert_score.rs rename to harmony/src/modules/monitoring/okd/openshift_cluster_alerting_score.rs diff --git a/harmony/src/modules/monitoring/prometheus/prometheus.rs b/harmony/src/modules/monitoring/prometheus/prometheus.rs index 8704b843..ae7dc9b3 100644 --- a/harmony/src/modules/monitoring/prometheus/prometheus.rs +++ b/harmony/src/modules/monitoring/prometheus/prometheus.rs @@ -9,7 +9,6 @@ use crate::{ inventory::Inventory, modules::monitoring::{ alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, - grafana::helm::helm_grafana::grafana_helm_chart_score, kube_prometheus::{ KubePrometheusRule, types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig}, @@ -18,7 +17,6 @@ use crate::{ score::Score, topology::{ HelmCommand, Topology, - installable::Installable, monitoring::{AlertReceiver, AlertRule, AlertSender}, tenant::TenantManager, }, @@ -106,52 +104,6 @@ impl Prometheus { .interpret(inventory, topology) .await } - pub async fn install_grafana( - &self, - inventory: &Inventory, - topology: &T, - ) -> Result { - let namespace = { - let config = self.config.lock().unwrap(); - config.namespace.clone() - }; - - if let Some(ns) = namespace.as_deref() { - grafana_helm_chart_score(ns, false) - .interpret(inventory, topology) - .await - } else { - Err(InterpretError::new( - "could not install grafana, missing namespace".to_string(), - )) - } - } -} -#[async_trait] -impl Installable for Prometheus { - async fn configure(&self, _inventory: &Inventory, topology: &T) -> Result<(), InterpretError> { - self.configure_with_topology(topology).await; - Ok(()) - } - - async fn ensure_installed( - &self, - inventory: &Inventory, - topology: &T, - ) -> Result<(), InterpretError> { - self.install_prometheus(inventory, topology).await?; - - let install_grafana = { - let config = self.config.lock().unwrap(); - config.grafana - }; - - if install_grafana { - self.install_grafana(inventory, topology).await?; - } - - Ok(()) - } } #[async_trait] diff --git a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs deleted file mode 100644 index f005086c..00000000 --- a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs +++ /dev/null @@ -1,573 +0,0 @@ -use std::fs; -use std::{collections::BTreeMap, sync::Arc}; -use tempfile::tempdir; - -use async_trait::async_trait; -use kube::api::ObjectMeta; -use log::{debug, info}; -use serde::Serialize; -use std::process::Command; - -use crate::modules::monitoring::grafana::crd::crd_grafana::{ - Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig, - GrafanaDatasourceJsonData, GrafanaDatasourceSpec, GrafanaSpec, -}; -use crate::modules::monitoring::grafana::crd::grafana_default_dashboard::build_default_dashboard; -use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus; -use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules; -use crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::{ - PrometheusRule, PrometheusRuleSpec, RuleGroup, -}; -use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{ - ServiceMonitor, ServiceMonitorSpec, -}; -use crate::topology::monitoring::{AlertReceiver, Observability}; -use crate::topology::{K8sclient, Topology, k8s::K8sClient}; -use crate::{ - data::Version, - interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, - inventory::Inventory, - modules::monitoring::kube_prometheus::crd::{ - crd_alertmanagers::{Alertmanager, AlertmanagerSpec}, - crd_prometheuses::{ - AlertmanagerEndpoints, LabelSelector, Prometheus, PrometheusSpec, - PrometheusSpecAlerting, - }, - role::{build_prom_role, build_prom_rolebinding, build_prom_service_account}, - }, - score::Score, -}; -use harmony_types::id::Id; - -#[derive(Clone, Debug, Serialize)] -pub struct K8sPrometheusCRDAlertingScore { - pub sender: CRDPrometheus, - pub receivers: Vec>>, - pub service_monitors: Vec, - pub prometheus_rules: Vec, -} - -impl> Score - for K8sPrometheusCRDAlertingScore -{ - fn create_interpret(&self) -> Box> { - Box::new(K8sPrometheusCRDAlertingInterpret { - sender: self.sender.clone(), - receivers: self.receivers.clone(), - service_monitors: self.service_monitors.clone(), - prometheus_rules: self.prometheus_rules.clone(), - }) - } - - fn name(&self) -> String { - "prometheus alerting [CRDAlertingScore]".into() - } -} - -#[derive(Clone, Debug)] -pub struct K8sPrometheusCRDAlertingInterpret { - pub sender: CRDPrometheus, - pub receivers: Vec>>, - pub service_monitors: Vec, - pub prometheus_rules: Vec, -} - -#[async_trait] -impl> Interpret - for K8sPrometheusCRDAlertingInterpret -{ - async fn execute( - &self, - _inventory: &Inventory, - topology: &T, - ) -> Result { - let client = topology.k8s_client().await.unwrap(); - self.ensure_grafana_operator().await?; - self.install_prometheus(&client).await?; - self.install_alert_manager(&client).await?; - self.install_client_kube_metrics().await?; - self.install_grafana(&client).await?; - self.install_receivers(&self.sender, &self.receivers) - .await?; - self.install_rules(&self.prometheus_rules, &client).await?; - self.install_monitors(self.service_monitors.clone(), &client) - .await?; - Ok(Outcome::success( - "K8s monitoring components installed".to_string(), - )) - } - - fn get_name(&self) -> InterpretName { - InterpretName::K8sPrometheusCrdAlerting - } - - fn get_version(&self) -> Version { - todo!() - } - - fn get_status(&self) -> InterpretStatus { - todo!() - } - - fn get_children(&self) -> Vec { - todo!() - } -} - -impl K8sPrometheusCRDAlertingInterpret { - async fn crd_exists(&self, crd: &str) -> bool { - let status = Command::new("sh") - .args(["-c", &format!("kubectl get crd -A | grep -i {crd}")]) - .status() - .map_err(|e| InterpretError::new(format!("could not connect to cluster: {}", e))) - .unwrap(); - - status.success() - } - - async fn install_chart( - &self, - chart_path: String, - chart_name: String, - ) -> Result<(), InterpretError> { - let temp_dir = - tempdir().map_err(|e| InterpretError::new(format!("Tempdir error: {}", e)))?; - let temp_path = temp_dir.path().to_path_buf(); - debug!("Using temp directory: {}", temp_path.display()); - let chart = format!("{}/{}", chart_path, chart_name); - let pull_output = Command::new("helm") - .args(["pull", &chart, "--destination", temp_path.to_str().unwrap()]) - .output() - .map_err(|e| InterpretError::new(format!("Helm pull error: {}", e)))?; - - if !pull_output.status.success() { - return Err(InterpretError::new(format!( - "Helm pull failed: {}", - String::from_utf8_lossy(&pull_output.stderr) - ))); - } - - let tgz_path = fs::read_dir(&temp_path) - .unwrap() - .filter_map(|entry| { - let entry = entry.ok()?; - let path = entry.path(); - if path.extension()? == "tgz" { - Some(path) - } else { - None - } - }) - .next() - .ok_or_else(|| InterpretError::new("Could not find pulled Helm chart".into()))?; - - debug!("Installing chart from: {}", tgz_path.display()); - - let install_output = Command::new("helm") - .args([ - "upgrade", - "--install", - &chart_name, - tgz_path.to_str().unwrap(), - "--namespace", - &self.sender.namespace.clone(), - "--create-namespace", - "--wait", - "--atomic", - ]) - .output() - .map_err(|e| InterpretError::new(format!("Helm install error: {}", e)))?; - - if !install_output.status.success() { - return Err(InterpretError::new(format!( - "Helm install failed: {}", - String::from_utf8_lossy(&install_output.stderr) - ))); - } - - debug!( - "Installed chart {}/{} in namespace: {}", - &chart_path, - &chart_name, - self.sender.namespace.clone() - ); - Ok(()) - } - - async fn ensure_grafana_operator(&self) -> Result { - let _ = Command::new("helm") - .args([ - "repo", - "add", - "grafana-operator", - "https://grafana.github.io/helm-charts", - ]) - .output() - .unwrap(); - - let _ = Command::new("helm") - .args(["repo", "update"]) - .output() - .unwrap(); - - let output = Command::new("helm") - .args([ - "install", - "grafana-operator", - "grafana-operator/grafana-operator", - "--namespace", - &self.sender.namespace.clone(), - "--create-namespace", - "--set", - "namespaceScope=true", - ]) - .output() - .unwrap(); - - if !output.status.success() { - return Err(InterpretError::new(format!( - "helm install failed:\nstdout: {}\nstderr: {}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ))); - } - - Ok(Outcome::success(format!( - "installed grafana operator in ns {}", - self.sender.namespace.clone() - ))) - } - - async fn install_prometheus(&self, client: &Arc) -> Result { - debug!( - "installing crd-prometheuses in namespace {}", - self.sender.namespace.clone() - ); - debug!("building role/rolebinding/serviceaccount for crd-prometheus"); - let rolename = format!("{}-prom", self.sender.namespace.clone()); - let sa_name = format!("{}-prom-sa", self.sender.namespace.clone()); - let role = build_prom_role(rolename.clone(), self.sender.namespace.clone()); - let rolebinding = build_prom_rolebinding( - rolename.clone(), - self.sender.namespace.clone(), - sa_name.clone(), - ); - let sa = build_prom_service_account(sa_name.clone(), self.sender.namespace.clone()); - let prom_spec = PrometheusSpec { - alerting: Some(PrometheusSpecAlerting { - alertmanagers: Some(vec![AlertmanagerEndpoints { - name: Some("alertmanager-operated".into()), - namespace: Some(self.sender.namespace.clone()), - port: Some("web".into()), - scheme: Some("http".into()), - }]), - }), - service_account_name: sa_name.clone(), - service_monitor_namespace_selector: Some(LabelSelector { - match_labels: BTreeMap::from([( - "kubernetes.io/metadata.name".to_string(), - self.sender.namespace.clone(), - )]), - match_expressions: vec![], - }), - service_monitor_selector: Some(LabelSelector { - match_labels: BTreeMap::from([("client".to_string(), "prometheus".to_string())]), - ..Default::default() - }), - - service_discovery_role: Some("Endpoints".into()), - - pod_monitor_selector: Some(LabelSelector { - match_labels: BTreeMap::from([("client".to_string(), "prometheus".to_string())]), - ..Default::default() - }), - - rule_selector: Some(LabelSelector { - match_labels: BTreeMap::from([("role".to_string(), "prometheus-rule".to_string())]), - ..Default::default() - }), - - rule_namespace_selector: Some(LabelSelector { - match_labels: BTreeMap::from([( - "kubernetes.io/metadata.name".to_string(), - self.sender.namespace.clone(), - )]), - match_expressions: vec![], - }), - }; - let prom = Prometheus { - metadata: ObjectMeta { - name: Some(self.sender.namespace.clone()), - labels: Some(std::collections::BTreeMap::from([ - ("alertmanagerConfig".to_string(), "enabled".to_string()), - ("client".to_string(), "prometheus".to_string()), - ])), - namespace: Some(self.sender.namespace.clone()), - ..Default::default() - }, - spec: prom_spec, - }; - client - .apply(&role, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - info!( - "installed prometheus role: {:#?} in ns {:#?}", - role.metadata.name.unwrap(), - role.metadata.namespace.unwrap() - ); - client - .apply(&rolebinding, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - info!( - "installed prometheus rolebinding: {:#?} in ns {:#?}", - rolebinding.metadata.name.unwrap(), - rolebinding.metadata.namespace.unwrap() - ); - client - .apply(&sa, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - info!( - "installed prometheus service account: {:#?} in ns {:#?}", - sa.metadata.name.unwrap(), - sa.metadata.namespace.unwrap() - ); - client - .apply(&prom, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - info!( - "installed prometheus: {:#?} in ns {:#?}", - &prom.metadata.name.clone().unwrap(), - &prom.metadata.namespace.clone().unwrap() - ); - - Ok(Outcome::success(format!( - "successfully deployed crd-prometheus {:#?}", - prom - ))) - } - - async fn install_alert_manager( - &self, - client: &Arc, - ) -> Result { - let am = Alertmanager { - metadata: ObjectMeta { - name: Some(self.sender.namespace.clone()), - labels: Some(std::collections::BTreeMap::from([( - "alertmanagerConfig".to_string(), - "enabled".to_string(), - )])), - namespace: Some(self.sender.namespace.clone()), - ..Default::default() - }, - spec: AlertmanagerSpec::default(), - }; - client - .apply(&am, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - Ok(Outcome::success(format!( - "successfully deployed service monitor {:#?}", - am.metadata.name - ))) - } - async fn install_monitors( - &self, - mut monitors: Vec, - client: &Arc, - ) -> Result { - let default_service_monitor = ServiceMonitor { - metadata: ObjectMeta { - name: Some(self.sender.namespace.clone()), - labels: Some(std::collections::BTreeMap::from([ - ("alertmanagerConfig".to_string(), "enabled".to_string()), - ("client".to_string(), "prometheus".to_string()), - ( - "app.kubernetes.io/name".to_string(), - "kube-state-metrics".to_string(), - ), - ])), - namespace: Some(self.sender.namespace.clone()), - ..Default::default() - }, - spec: ServiceMonitorSpec::default(), - }; - monitors.push(default_service_monitor); - for monitor in monitors.iter() { - client - .apply(monitor, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - } - Ok(Outcome::success( - "succesfully deployed service monitors".to_string(), - )) - } - - async fn install_rules( - &self, - #[allow(clippy::ptr_arg)] rules: &Vec, - client: &Arc, - ) -> Result { - let mut prom_rule_spec = PrometheusRuleSpec { - groups: rules.clone(), - }; - - let default_rules_group = RuleGroup { - name: "default-rules".to_string(), - rules: build_default_application_rules(), - }; - - prom_rule_spec.groups.push(default_rules_group); - let prom_rules = PrometheusRule { - metadata: ObjectMeta { - name: Some(self.sender.namespace.clone()), - labels: Some(std::collections::BTreeMap::from([ - ("alertmanagerConfig".to_string(), "enabled".to_string()), - ("role".to_string(), "prometheus-rule".to_string()), - ])), - namespace: Some(self.sender.namespace.clone()), - ..Default::default() - }, - spec: prom_rule_spec, - }; - client - .apply(&prom_rules, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - Ok(Outcome::success(format!( - "successfully deployed rules {:#?}", - prom_rules.metadata.name - ))) - } - - async fn install_client_kube_metrics(&self) -> Result { - self.install_chart( - "oci://hub.nationtech.io/harmony".to_string(), - "nt-kube-metrics".to_string(), - ) - .await?; - Ok(Outcome::success(format!( - "Installed client kube metrics in ns {}", - &self.sender.namespace.clone() - ))) - } - - async fn install_grafana(&self, client: &Arc) -> Result { - let mut label = BTreeMap::new(); - label.insert("dashboards".to_string(), "grafana".to_string()); - let labels = LabelSelector { - match_labels: label.clone(), - match_expressions: vec![], - }; - let namespace = self.sender.namespace.clone(); - let json_data = GrafanaDatasourceJsonData { - time_interval: Some("5s".to_string()), - http_header_name1: None, - tls_skip_verify: Some(true), - oauth_pass_thru: Some(true), - }; - let json = build_default_dashboard(&namespace); - - let graf_data_source = GrafanaDatasource { - metadata: ObjectMeta { - name: Some(format!( - "grafana-datasource-{}", - self.sender.namespace.clone() - )), - namespace: Some(self.sender.namespace.clone()), - ..Default::default() - }, - spec: GrafanaDatasourceSpec { - instance_selector: labels.clone(), - allow_cross_namespace_import: Some(false), - datasource: GrafanaDatasourceConfig { - access: "proxy".to_string(), - database: Some("prometheus".to_string()), - json_data: Some(json_data), - //this is fragile - name: format!("prometheus-{}-0", self.sender.namespace.clone()), - r#type: "prometheus".to_string(), - url: format!( - "http://prometheus-operated.{}.svc.cluster.local:9090", - self.sender.namespace.clone() - ), - secure_json_data: None, - is_default: None, - editable: None, - }, - values_from: None, - }, - }; - - client - .apply(&graf_data_source, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - - let graf_dashboard = GrafanaDashboard { - metadata: ObjectMeta { - name: Some(format!( - "grafana-dashboard-{}", - self.sender.namespace.clone() - )), - namespace: Some(self.sender.namespace.clone()), - ..Default::default() - }, - spec: GrafanaDashboardSpec { - resync_period: Some("30s".to_string()), - instance_selector: labels.clone(), - json: Some(json), - grafana_com: None, - datasources: None, - }, - }; - - client - .apply(&graf_dashboard, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - - let grafana = Grafana { - metadata: ObjectMeta { - name: Some(format!("grafana-{}", self.sender.namespace.clone())), - namespace: Some(self.sender.namespace.clone()), - labels: Some(label.clone()), - ..Default::default() - }, - spec: GrafanaSpec { - config: None, - admin_user: None, - admin_password: None, - ingress: None, - persistence: None, - resources: None, - }, - }; - client - .apply(&grafana, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - Ok(Outcome::success(format!( - "successfully deployed grafana instance {:#?}", - grafana.metadata.name - ))) - } - - async fn install_receivers( - &self, - sender: &CRDPrometheus, - receivers: &Vec>>, - ) -> Result { - for receiver in receivers.iter() { - receiver.build_route().map_err(|err| { - InterpretError::new(format!("failed to install receiver: {}", err)) - })?; - } - Ok(Outcome::success("successfully deployed receivers".into())) - } -} diff --git a/harmony/src/modules/prometheus/mod.rs b/harmony/src/modules/prometheus/mod.rs deleted file mode 100644 index 1a1fddab..00000000 --- a/harmony/src/modules/prometheus/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod k8s_prometheus_alerting_score; -// pub mod rhob_alerting_score; diff --git a/harmony/src/modules/prometheus/rhob_alerting_score.rs b/harmony/src/modules/prometheus/rhob_alerting_score.rs deleted file mode 100644 index 4cc9dbf5..00000000 --- a/harmony/src/modules/prometheus/rhob_alerting_score.rs +++ /dev/null @@ -1,525 +0,0 @@ -use fqdn::fqdn; -use std::fs; -use std::{collections::BTreeMap, sync::Arc}; -use tempfile::tempdir; - -use async_trait::async_trait; -use kube::api::ObjectMeta; -use log::{debug, info}; -use serde::Serialize; -use std::process::Command; - -use crate::modules::k8s::ingress::{K8sIngressScore, PathType}; -use crate::modules::monitoring::kube_prometheus::crd::grafana_default_dashboard::build_default_dashboard; -use crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::RHOBObservability; -use crate::modules::monitoring::kube_prometheus::crd::rhob_grafana::{ - Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig, - GrafanaDatasourceSpec, GrafanaSpec, -}; -use crate::modules::monitoring::kube_prometheus::crd::rhob_monitoring_stack::{ - MonitoringStack, MonitoringStackSpec, -}; -use crate::modules::monitoring::kube_prometheus::crd::rhob_prometheus_rules::{ - PrometheusRule, PrometheusRuleSpec, RuleGroup, -}; -use crate::modules::monitoring::kube_prometheus::crd::rhob_prometheuses::LabelSelector; - -use crate::modules::monitoring::kube_prometheus::crd::rhob_service_monitor::{ - ServiceMonitor, ServiceMonitorSpec, -}; -use crate::score::Score; -use crate::topology::ingress::Ingress; -use crate::topology::oberservability::monitoring::{AlertReceiver, Observability}; -use crate::topology::{K8sclient, Topology, k8s::K8sClient}; -use crate::{ - data::Version, - interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, - inventory::Inventory, -}; -use harmony_types::id::Id; - -#[derive(Clone, Debug, Serialize)] -pub struct RHOBAlertingScore { - pub sender: RHOBObservability, - pub receivers: Vec>>, - pub service_monitors: Vec, - pub prometheus_rules: Vec, -} - -impl> Score - for RHOBAlertingScore -{ - fn create_interpret(&self) -> Box> { - Box::new(RHOBAlertingInterpret { - sender: self.sender.clone(), - receivers: self.receivers.clone(), - service_monitors: self.service_monitors.clone(), - prometheus_rules: self.prometheus_rules.clone(), - }) - } - - fn name(&self) -> String { - "RHOB alerting [RHOBAlertingScore]".into() - } -} - -#[derive(Clone, Debug)] -pub struct RHOBAlertingInterpret { - pub sender: RHOBObservability, - pub receivers: Vec>>, - pub service_monitors: Vec, - pub prometheus_rules: Vec, -} - -#[async_trait] -impl> Interpret - for RHOBAlertingInterpret -{ - async fn execute( - &self, - inventory: &Inventory, - topology: &T, - ) -> Result { - let client = topology.k8s_client().await.unwrap(); - self.ensure_grafana_operator().await?; - self.install_prometheus(inventory, topology, &client) - .await?; - self.install_client_kube_metrics().await?; - self.install_grafana(inventory, topology, &client).await?; - self.install_receivers(&self.sender, &self.receivers) - .await?; - self.install_rules(&self.prometheus_rules, &client).await?; - self.install_monitors(self.service_monitors.clone(), &client) - .await?; - Ok(Outcome::success( - "K8s monitoring components installed".to_string(), - )) - } - - fn get_name(&self) -> InterpretName { - InterpretName::RHOBAlerting - } - - fn get_version(&self) -> Version { - todo!() - } - - fn get_status(&self) -> InterpretStatus { - todo!() - } - - fn get_children(&self) -> Vec { - todo!() - } -} - -impl RHOBAlertingInterpret { - async fn crd_exists(&self, crd: &str) -> bool { - let status = Command::new("sh") - .args(["-c", &format!("kubectl get crd -A | grep -i {crd}")]) - .status() - .map_err(|e| InterpretError::new(format!("could not connect to cluster: {}", e))) - .unwrap(); - - status.success() - } - - async fn install_chart( - &self, - chart_path: String, - chart_name: String, - ) -> Result<(), InterpretError> { - let temp_dir = - tempdir().map_err(|e| InterpretError::new(format!("Tempdir error: {}", e)))?; - let temp_path = temp_dir.path().to_path_buf(); - debug!("Using temp directory: {}", temp_path.display()); - let chart = format!("{}/{}", chart_path, chart_name); - let pull_output = Command::new("helm") - .args(["pull", &chart, "--destination", temp_path.to_str().unwrap()]) - .output() - .map_err(|e| InterpretError::new(format!("Helm pull error: {}", e)))?; - - if !pull_output.status.success() { - return Err(InterpretError::new(format!( - "Helm pull failed: {}", - String::from_utf8_lossy(&pull_output.stderr) - ))); - } - - let tgz_path = fs::read_dir(&temp_path) - .unwrap() - .filter_map(|entry| { - let entry = entry.ok()?; - let path = entry.path(); - if path.extension()? == "tgz" { - Some(path) - } else { - None - } - }) - .next() - .ok_or_else(|| InterpretError::new("Could not find pulled Helm chart".into()))?; - - debug!("Installing chart from: {}", tgz_path.display()); - - let install_output = Command::new("helm") - .args([ - "upgrade", - "--install", - &chart_name, - tgz_path.to_str().unwrap(), - "--namespace", - &self.sender.namespace.clone(), - "--create-namespace", - "--wait", - "--atomic", - ]) - .output() - .map_err(|e| InterpretError::new(format!("Helm install error: {}", e)))?; - - if !install_output.status.success() { - return Err(InterpretError::new(format!( - "Helm install failed: {}", - String::from_utf8_lossy(&install_output.stderr) - ))); - } - - debug!( - "Installed chart {}/{} in namespace: {}", - &chart_path, - &chart_name, - self.sender.namespace.clone() - ); - Ok(()) - } - - async fn ensure_grafana_operator(&self) -> Result { - let _ = Command::new("helm") - .args([ - "repo", - "add", - "grafana-operator", - "https://grafana.github.io/helm-charts", - ]) - .output() - .unwrap(); - - let _ = Command::new("helm") - .args(["repo", "update"]) - .output() - .unwrap(); - - let output = Command::new("helm") - .args([ - "upgrade", - "--install", - "grafana-operator", - "grafana-operator/grafana-operator", - "--namespace", - &self.sender.namespace.clone(), - "--create-namespace", - "--set", - "namespaceScope=true", - ]) - .output() - .unwrap(); - - if !output.status.success() { - return Err(InterpretError::new(format!( - "helm upgrade --install failed:\nstdout: {}\nstderr: {}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ))); - } - - Ok(Outcome::success(format!( - "installed grafana operator in ns {}", - self.sender.namespace.clone() - ))) - } - - async fn install_prometheus( - &self, - inventory: &Inventory, - topology: &T, - client: &Arc, - ) -> Result { - debug!( - "installing crd-prometheuses in namespace {}", - self.sender.namespace.clone() - ); - debug!("building role/rolebinding/serviceaccount for crd-prometheus"); - - let stack = MonitoringStack { - metadata: ObjectMeta { - name: Some(format!("{}-monitoring", self.sender.namespace.clone()).into()), - namespace: Some(self.sender.namespace.clone()), - labels: Some([("monitoring-stack".into(), "true".into())].into()), - ..Default::default() - }, - spec: MonitoringStackSpec { - log_level: Some("debug".into()), - retention: Some("1d".into()), - resource_selector: Some(LabelSelector { - match_labels: Default::default(), - match_expressions: vec![], - }), - }, - }; - - client - .apply(&stack, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - - let alert_manager_domain = topology - .get_domain(&format!("alert-manager-{}", self.sender.namespace.clone())) - .await?; - let name = format!("{}-alert-manager", self.sender.namespace.clone()); - let backend_service = format!("alertmanager-operated"); - let namespace = self.sender.namespace.clone(); - let alert_manager_ingress = K8sIngressScore { - name: fqdn!(&name), - host: fqdn!(&alert_manager_domain), - backend_service: fqdn!(&backend_service), - port: 9093, - path: Some("/".to_string()), - path_type: Some(PathType::Prefix), - namespace: Some(fqdn!(&namespace)), - ingress_class_name: Some("openshift-default".to_string()), - }; - - let prometheus_domain = topology - .get_domain(&format!("prometheus-{}", self.sender.namespace.clone())) - .await?; - let name = format!("{}-prometheus", self.sender.namespace.clone()); - let backend_service = format!("prometheus-operated"); - let prometheus_ingress = K8sIngressScore { - name: fqdn!(&name), - host: fqdn!(&prometheus_domain), - backend_service: fqdn!(&backend_service), - port: 9090, - path: Some("/".to_string()), - path_type: Some(PathType::Prefix), - namespace: Some(fqdn!(&namespace)), - ingress_class_name: Some("openshift-default".to_string()), - }; - - alert_manager_ingress.interpret(inventory, topology).await?; - prometheus_ingress.interpret(inventory, topology).await?; - info!("installed rhob monitoring stack",); - Ok(Outcome::success(format!( - "successfully deployed rhob-prometheus {:#?}", - stack - ))) - } - - async fn install_monitors( - &self, - mut monitors: Vec, - client: &Arc, - ) -> Result { - let default_service_monitor = ServiceMonitor { - metadata: ObjectMeta { - name: Some(self.sender.namespace.clone()), - labels: Some(std::collections::BTreeMap::from([ - ("alertmanagerConfig".to_string(), "enabled".to_string()), - ("client".to_string(), "prometheus".to_string()), - ( - "app.kubernetes.io/name".to_string(), - "kube-state-metrics".to_string(), - ), - ])), - namespace: Some(self.sender.namespace.clone()), - ..Default::default() - }, - spec: ServiceMonitorSpec::default(), - }; - monitors.push(default_service_monitor); - for monitor in monitors.iter() { - client - .apply(monitor, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - } - Ok(Outcome::success( - "succesfully deployed service monitors".to_string(), - )) - } - - async fn install_rules( - &self, - #[allow(clippy::ptr_arg)] rules: &Vec, - client: &Arc, - ) -> Result { - let mut prom_rule_spec = PrometheusRuleSpec { - groups: rules.clone(), - }; - - let default_rules_group = RuleGroup { - name: "default-rules".to_string(), - rules: crate::modules::monitoring::kube_prometheus::crd::rhob_default_rules::build_default_application_rules(), - }; - - prom_rule_spec.groups.push(default_rules_group); - let prom_rules = PrometheusRule { - metadata: ObjectMeta { - name: Some(self.sender.namespace.clone()), - labels: Some(std::collections::BTreeMap::from([ - ("alertmanagerConfig".to_string(), "enabled".to_string()), - ("role".to_string(), "prometheus-rule".to_string()), - ])), - namespace: Some(self.sender.namespace.clone()), - ..Default::default() - }, - spec: prom_rule_spec, - }; - client - .apply(&prom_rules, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - Ok(Outcome::success(format!( - "successfully deployed rules {:#?}", - prom_rules.metadata.name - ))) - } - - async fn install_client_kube_metrics(&self) -> Result { - self.install_chart( - "oci://hub.nationtech.io/harmony".to_string(), - "nt-kube-metrics".to_string(), - ) - .await?; - Ok(Outcome::success(format!( - "Installed client kube metrics in ns {}", - &self.sender.namespace.clone() - ))) - } - - async fn install_grafana( - &self, - inventory: &Inventory, - topology: &T, - client: &Arc, - ) -> Result { - let mut label = BTreeMap::new(); - label.insert("dashboards".to_string(), "grafana".to_string()); - let labels = LabelSelector { - match_labels: label.clone(), - match_expressions: vec![], - }; - let mut json_data = BTreeMap::new(); - json_data.insert("timeInterval".to_string(), "5s".to_string()); - let namespace = self.sender.namespace.clone(); - - let json = build_default_dashboard(&namespace); - - let graf_data_source = GrafanaDatasource { - metadata: ObjectMeta { - name: Some(format!( - "grafana-datasource-{}", - self.sender.namespace.clone() - )), - namespace: Some(self.sender.namespace.clone()), - ..Default::default() - }, - spec: GrafanaDatasourceSpec { - instance_selector: labels.clone(), - allow_cross_namespace_import: Some(false), - datasource: GrafanaDatasourceConfig { - access: "proxy".to_string(), - database: Some("prometheus".to_string()), - json_data: Some(json_data), - //this is fragile - name: format!("prometheus-{}-0", self.sender.namespace.clone()), - r#type: "prometheus".to_string(), - url: format!( - "http://prometheus-operated.{}.svc.cluster.local:9090", - self.sender.namespace.clone() - ), - }, - }, - }; - - client - .apply(&graf_data_source, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - - let graf_dashboard = GrafanaDashboard { - metadata: ObjectMeta { - name: Some(format!( - "grafana-dashboard-{}", - self.sender.namespace.clone() - )), - namespace: Some(self.sender.namespace.clone()), - ..Default::default() - }, - spec: GrafanaDashboardSpec { - resync_period: Some("30s".to_string()), - instance_selector: labels.clone(), - json, - }, - }; - - client - .apply(&graf_dashboard, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - - let grafana = Grafana { - metadata: ObjectMeta { - name: Some(format!("grafana-{}", self.sender.namespace.clone())), - namespace: Some(self.sender.namespace.clone()), - labels: Some(label.clone()), - ..Default::default() - }, - spec: GrafanaSpec { - config: None, - admin_user: None, - admin_password: None, - ingress: None, - persistence: None, - resources: None, - }, - }; - client - .apply(&grafana, Some(&self.sender.namespace.clone())) - .await - .map_err(|e| InterpretError::new(e.to_string()))?; - let domain = topology - .get_domain(&format!("grafana-{}", self.sender.namespace.clone())) - .await?; - let name = format!("{}-grafana", self.sender.namespace.clone()); - let backend_service = format!("grafana-{}-service", self.sender.namespace.clone()); - let grafana_ingress = K8sIngressScore { - name: fqdn!(&name), - host: fqdn!(&domain), - backend_service: fqdn!(&backend_service), - port: 3000, - path: Some("/".to_string()), - path_type: Some(PathType::Prefix), - namespace: Some(fqdn!(&namespace)), - ingress_class_name: Some("openshift-default".to_string()), - }; - - grafana_ingress.interpret(inventory, topology).await?; - Ok(Outcome::success(format!( - "successfully deployed grafana instance {:#?}", - grafana.metadata.name - ))) - } - - async fn install_receivers( - &self, - sender: &RHOBObservability, - receivers: &Vec>>, - ) -> Result { - for receiver in receivers.iter() { - receiver.build_route().map_err(|err| { - InterpretError::new(format!("failed to install receiver: {}", err)) - })?; - } - Ok(Outcome::success("successfully deployed receivers".into())) - } -} -- 2.39.5 From b14b41d172b6ccd7108255890f036b2522a7ce05 Mon Sep 17 00:00:00 2001 From: wjro Date: Thu, 26 Feb 2026 15:10:28 -0500 Subject: [PATCH 14/15] refactor: prometheus alert sender --- .../k8s_anywhere/observability/prometheus.rs | 110 ++++++++++++-- .../application/features/monitoring.rs | 15 +- .../alert_channel/webhook_receiver.rs | 7 +- .../alert_rule/prometheus_alert_rule.rs | 63 +------- .../application_monitoring_score.rs | 15 +- .../grafana/grafana_alerting_score.rs | 2 +- .../grafana/k8s/score_grafana_rule.rs | 43 ++++++ .../crd/crd_alertmanager_config.rs | 22 --- .../kube_prometheus/crd/crd_scrape_config.rs | 10 +- .../kube_prometheus_alerting_score.rs | 4 +- .../modules/monitoring/kube_prometheus/mod.rs | 67 --------- .../modules/monitoring/prometheus/helm/mod.rs | 1 + .../{ => helm}/prometheus_config.rs | 4 +- .../prometheus/helm/prometheus_helm.rs | 5 +- .../src/modules/monitoring/prometheus/mod.rs | 57 +++++++- .../monitoring/prometheus/prometheus.rs | 137 ------------------ .../prometheus/prometheus_alerting_score.rs | 47 ++++++ .../score_prometheus_alert_receivers.rs | 58 ++++++++ .../score_prometheus_ensure_ready.rs | 80 ++++++++++ .../prometheus/score_prometheus_rule.rs | 48 ++++++ .../score_prometheus_scrape_target.rs | 66 +++++++++ .../monitoring/scrape_target/server.rs | 12 +- 22 files changed, 525 insertions(+), 348 deletions(-) rename harmony/src/modules/monitoring/prometheus/{ => helm}/prometheus_config.rs (95%) delete mode 100644 harmony/src/modules/monitoring/prometheus/prometheus.rs create mode 100644 harmony/src/modules/monitoring/prometheus/prometheus_alerting_score.rs create mode 100644 harmony/src/modules/monitoring/prometheus/score_prometheus_alert_receivers.rs create mode 100644 harmony/src/modules/monitoring/prometheus/score_prometheus_ensure_ready.rs create mode 100644 harmony/src/modules/monitoring/prometheus/score_prometheus_rule.rs create mode 100644 harmony/src/modules/monitoring/prometheus/score_prometheus_scrape_target.rs diff --git a/harmony/src/domain/topology/k8s_anywhere/observability/prometheus.rs b/harmony/src/domain/topology/k8s_anywhere/observability/prometheus.rs index 4024b892..a90520cb 100644 --- a/harmony/src/domain/topology/k8s_anywhere/observability/prometheus.rs +++ b/harmony/src/domain/topology/k8s_anywhere/observability/prometheus.rs @@ -2,7 +2,13 @@ use async_trait::async_trait; use crate::{ inventory::Inventory, - modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus, + modules::monitoring::prometheus::{ + Prometheus, score_prometheus_alert_receivers::PrometheusReceiverScore, + score_prometheus_ensure_ready::PrometheusEnsureReadyScore, + score_prometheus_rule::PrometheusRuleScore, + score_prometheus_scrape_target::PrometheusScrapeTargetScore, + }, + score::Score, topology::{ K8sAnywhereTopology, PreparationError, PreparationOutcome, monitoring::{AlertReceiver, AlertRule, Observability, ScrapeTarget}, @@ -10,47 +16,121 @@ use crate::{ }; #[async_trait] -impl Observability for K8sAnywhereTopology { +impl Observability for K8sAnywhereTopology { async fn install_alert_sender( &self, - sender: &CRDPrometheus, + sender: &Prometheus, inventory: &Inventory, ) -> Result { - todo!() + Ok(PreparationOutcome::Success { + details: "Successfully installed kubeprometheus alert sender".to_string(), + }) } async fn install_receivers( &self, - sender: &CRDPrometheus, + sender: &Prometheus, inventory: &Inventory, - receivers: Option>>>, + receivers: Option>>>, ) -> Result { - todo!() + let receivers = match receivers { + Some(r) if !r.is_empty() => r, + _ => return Ok(PreparationOutcome::Noop), + }; + + for receiver in receivers { + let score = PrometheusReceiverScore { + receiver, + sender: sender.clone(), + }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Failed to install receiver: {}", e)))?; + } + + Ok(PreparationOutcome::Success { + details: "All alert receivers installed successfully".to_string(), + }) } async fn install_rules( &self, - sender: &CRDPrometheus, + sender: &Prometheus, inventory: &Inventory, - rules: Option>>>, + rules: Option>>>, ) -> Result { - todo!() + let rules = match rules { + Some(r) if !r.is_empty() => r, + _ => return Ok(PreparationOutcome::Noop), + }; + + for rule in rules { + let score = PrometheusRuleScore { + sender: sender.clone(), + rule, + }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Failed to install rule: {}", e)))?; + } + + Ok(PreparationOutcome::Success { + details: "All alert rules installed successfully".to_string(), + }) } async fn add_scrape_targets( &self, - sender: &CRDPrometheus, + sender: &Prometheus, inventory: &Inventory, - scrape_targets: Option>>>, + scrape_targets: Option>>>, ) -> Result { - todo!() + let scrape_targets = match scrape_targets { + Some(r) if !r.is_empty() => r, + _ => return Ok(PreparationOutcome::Noop), + }; + + for scrape_target in scrape_targets { + let score = PrometheusScrapeTargetScore { + scrape_target, + sender: sender.clone(), + }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Failed to install rule: {}", e)))?; + } + + Ok(PreparationOutcome::Success { + details: "All scrap targets installed successfully".to_string(), + }) } async fn ensure_monitoring_installed( &self, - sender: &CRDPrometheus, + sender: &Prometheus, inventory: &Inventory, ) -> Result { - todo!() + let score = PrometheusEnsureReadyScore { + sender: sender.clone(), + }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Prometheus not ready {}", e)))?; + + Ok(PreparationOutcome::Success { + details: "Prometheus Ready".to_string(), + }) } } diff --git a/harmony/src/modules/application/features/monitoring.rs b/harmony/src/modules/application/features/monitoring.rs index 17f263dc..28186e4f 100644 --- a/harmony/src/modules/application/features/monitoring.rs +++ b/harmony/src/modules/application/features/monitoring.rs @@ -2,10 +2,11 @@ use crate::modules::application::{ Application, ApplicationFeature, InstallationError, InstallationOutcome, }; use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore; -use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus; use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{ ServiceMonitor, ServiceMonitorSpec, }; +use crate::modules::monitoring::prometheus::Prometheus; +use crate::modules::monitoring::prometheus::helm::prometheus_config::PrometheusConfig; use crate::topology::MultiTargetTopology; use crate::topology::ingress::Ingress; use crate::topology::monitoring::AlertReceiver; @@ -27,12 +28,12 @@ use kube::api::ObjectMeta; use log::{debug, info}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; #[derive(Debug, Clone)] pub struct Monitoring { pub application: Arc, - pub alert_receiver: Vec>>, + pub alert_receiver: Vec>>, } #[async_trait] @@ -43,7 +44,7 @@ impl< + TenantManager + K8sclient + MultiTargetTopology - + Observability + + Observability + Ingress + std::fmt::Debug, > ApplicationFeature for Monitoring @@ -70,10 +71,8 @@ impl< }; let mut alerting_score = ApplicationMonitoringScore { - sender: CRDPrometheus { - namespace: namespace.clone(), - client: topology.k8s_client().await.unwrap(), - service_monitor: vec![app_service_monitor], + sender: Prometheus { + config: Arc::new(Mutex::new(PrometheusConfig::new())), }, application: self.application.clone(), receivers: self.alert_receiver.clone(), diff --git a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs index bfbba451..24a20496 100644 --- a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs +++ b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs @@ -4,8 +4,7 @@ use serde_json::json; use crate::{ interpret::InterpretError, modules::monitoring::{ - kube_prometheus::{KubePrometheus, crd::crd_alertmanager_config::CRDPrometheus}, - okd::OpenshiftClusterAlertSender, + kube_prometheus::KubePrometheus, okd::OpenshiftClusterAlertSender, prometheus::Prometheus, red_hat_cluster_observability::RedHatClusterObservability, }, topology::monitoring::AlertReceiver, @@ -100,7 +99,7 @@ impl AlertReceiver for WebhookReceiver { } } -impl AlertReceiver for WebhookReceiver { +impl AlertReceiver for WebhookReceiver { fn build_receiver(&self) -> Result { let receiver = self.build_receiver(); serde_yaml::to_value(receiver).map_err(|e| InterpretError::new(e.to_string())) @@ -115,7 +114,7 @@ impl AlertReceiver for WebhookReceiver { self.name.clone() } - fn clone_box(&self) -> Box> { + fn clone_box(&self) -> Box> { Box::new(self.clone()) } } diff --git a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs index 7585ffdc..2cc7fceb 100644 --- a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs +++ b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs @@ -1,18 +1,10 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; -use async_trait::async_trait; use serde::Serialize; -use serde_json::json; use crate::{ - interpret::{InterpretError, Outcome}, - modules::monitoring::{ - kube_prometheus::{ - KubePrometheus, KubePrometheusRule, - types::{AlertGroup, AlertManagerAdditionalPromRules}, - }, - okd::OpenshiftClusterAlertSender, - }, + interpret::InterpretError, + modules::monitoring::{kube_prometheus::KubePrometheus, okd::OpenshiftClusterAlertSender}, topology::monitoring::AlertRule, }; @@ -123,52 +115,3 @@ impl AlertRule for AlertManagerRuleGroup { Box::new(self.clone()) } } - -// #[async_trait] -// impl AlertRule for AlertManagerRuleGroup { -// async fn install(&self, sender: &Prometheus) -> Result { -// sender.install_rule(self).await -// } -// fn clone_box(&self) -> Box> { -// Box::new(self.clone()) -// } -// } -// -// #[async_trait] -// impl PrometheusRule for AlertManagerRuleGroup { -// fn name(&self) -> String { -// self.name.clone() -// } -// async fn configure_rule(&self) -> AlertManagerAdditionalPromRules { -// let mut additional_prom_rules = BTreeMap::new(); -// -// additional_prom_rules.insert( -// self.name.clone(), -// AlertGroup { -// groups: vec![self.clone()], -// }, -// ); -// AlertManagerAdditionalPromRules { -// rules: additional_prom_rules, -// } -// } -// } -#[async_trait] -impl KubePrometheusRule for AlertManagerRuleGroup { - fn name(&self) -> String { - self.name.clone() - } - async fn configure_rule(&self) -> AlertManagerAdditionalPromRules { - let mut additional_prom_rules = BTreeMap::new(); - - additional_prom_rules.insert( - self.name.clone(), - AlertGroup { - groups: vec![self.clone()], - }, - ); - AlertManagerAdditionalPromRules { - rules: additional_prom_rules, - } - } -} diff --git a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs index 11be6a73..f5385a22 100644 --- a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs +++ b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs @@ -5,12 +5,7 @@ use serde::Serialize; use crate::{ interpret::Interpret, - modules::{ - application::Application, - monitoring::{ - grafana::grafana::Grafana, kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus, - }, - }, + modules::{application::Application, monitoring::prometheus::Prometheus}, score::Score, topology::{ K8sclient, Topology, @@ -20,14 +15,12 @@ use crate::{ #[derive(Debug, Clone, Serialize)] pub struct ApplicationMonitoringScore { - pub sender: CRDPrometheus, + pub sender: Prometheus, pub application: Arc, - pub receivers: Vec>>, + pub receivers: Vec>>, } -impl + K8sclient> Score - for ApplicationMonitoringScore -{ +impl + K8sclient> Score for ApplicationMonitoringScore { fn create_interpret(&self) -> Box> { debug!("creating alerting interpret"); Box::new(AlertingInterpret { diff --git a/harmony/src/modules/monitoring/grafana/grafana_alerting_score.rs b/harmony/src/modules/monitoring/grafana/grafana_alerting_score.rs index a2f00e8c..b032ed89 100644 --- a/harmony/src/modules/monitoring/grafana/grafana_alerting_score.rs +++ b/harmony/src/modules/monitoring/grafana/grafana_alerting_score.rs @@ -17,7 +17,7 @@ pub struct GrafanaAlertingScore { pub sender: Grafana, } -impl> Score for GrafanaAlertingScore { +impl> Score for GrafanaAlertingScore { fn create_interpret(&self) -> Box> { Box::new(AlertingInterpret { sender: self.sender.clone(), diff --git a/harmony/src/modules/monitoring/grafana/k8s/score_grafana_rule.rs b/harmony/src/modules/monitoring/grafana/k8s/score_grafana_rule.rs index 464dde8d..1a3c19d1 100644 --- a/harmony/src/modules/monitoring/grafana/k8s/score_grafana_rule.rs +++ b/harmony/src/modules/monitoring/grafana/k8s/score_grafana_rule.rs @@ -22,3 +22,46 @@ impl Score for GrafanaK8sRuleScore { todo!() } } + +// kind: Secret +// apiVersion: v1 +// metadata: +// name: credentials +// namespace: grafana +// stringData: +// PROMETHEUS_USERNAME: root +// PROMETHEUS_PASSWORD: secret +// type: Opaque +// --- +// apiVersion: grafana.integreatly.org/v1beta1 +// kind: GrafanaDatasource +// metadata: +// name: grafanadatasource-sample +// spec: +// valuesFrom: +// - targetPath: "basicAuthUser" +// valueFrom: +// secretKeyRef: +// name: "credentials" +// key: "PROMETHEUS_USERNAME" +// - targetPath: "secureJsonData.basicAuthPassword" +// valueFrom: +// secretKeyRef: +// name: "credentials" +// key: "PROMETHEUS_PASSWORD" +// instanceSelector: +// matchLabels: +// dashboards: "grafana" +// datasource: +// name: prometheus +// type: prometheus +// access: proxy +// basicAuth: true +// url: http://prometheus-service:9090 +// isDefault: true +// basicAuthUser: ${PROMETHEUS_USERNAME} +// jsonData: +// "tlsSkipVerify": true +// "timeInterval": "5s" +// secureJsonData: +// "basicAuthPassword": ${PROMETHEUS_PASSWORD} # diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs index 7f0ae6f5..e9de6e28 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs @@ -32,25 +32,3 @@ pub struct AlertmanagerConfigSpec { #[serde(flatten)] pub data: serde_json::Value, } - -#[derive(Debug, Clone, Serialize)] -pub struct CRDPrometheus { - pub namespace: String, - pub client: Arc, - pub service_monitor: Vec, -} - -impl AlertSender for CRDPrometheus { - fn name(&self) -> String { - "CRDAlertManager".to_string() - } -} - -impl Serialize for Box> { - fn serialize(&self, _serializer: S) -> Result - where - S: serde::Serializer, - { - todo!() - } -} diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs index 929ce98d..0973e118 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs @@ -1,16 +1,10 @@ -use std::{collections::BTreeMap, net::IpAddr}; +use std::collections::BTreeMap; -use async_trait::async_trait; use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::{ - modules::monitoring::kube_prometheus::crd::{ - crd_alertmanager_config::CRDPrometheus, crd_prometheuses::LabelSelector, - }, - topology::monitoring::ScrapeTarget, -}; +use crate::modules::monitoring::kube_prometheus::crd::crd_prometheuses::LabelSelector; #[derive(CustomResource, Default, Serialize, Deserialize, Debug, Clone, JsonSchema)] #[kube( diff --git a/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs b/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs index f48d10fa..31eebcdd 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs @@ -22,9 +22,7 @@ pub struct KubePrometheusAlertingScore { pub config: Arc>, } -impl> Score - for KubePrometheusAlertingScore -{ +impl> Score for KubePrometheusAlertingScore { fn create_interpret(&self) -> Box> { //TODO test that additional service monitor is added self.config diff --git a/harmony/src/modules/monitoring/kube_prometheus/mod.rs b/harmony/src/modules/monitoring/kube_prometheus/mod.rs index 8765f4a3..7f496fb2 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/mod.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/mod.rs @@ -67,67 +67,6 @@ impl KubePrometheus { config: Arc::new(Mutex::new(KubePrometheusConfig::new())), } } - - pub async fn configure_with_topology(&self, topology: &T) { - let ns = topology - .get_tenant_config() - .await - .map(|cfg| cfg.name.clone()) - .unwrap_or_else(|| "monitoring".to_string()); - error!("This must be refactored, see comments in pr #74"); - debug!("NS: {}", ns); - self.config.lock().unwrap().namespace = Some(ns); - } - - pub async fn install_receiver( - &self, - prometheus_receiver: &dyn KubePrometheusReceiver, - ) -> Result { - let prom_receiver = prometheus_receiver.configure_receiver().await; - debug!( - "adding alert receiver to prometheus config: {:#?}", - &prom_receiver - ); - let mut config = self.config.lock().unwrap(); - - config.alert_receiver_configs.push(prom_receiver); - let prom_receiver_name = prometheus_receiver.name(); - debug!("installed alert receiver {}", &prom_receiver_name); - Ok(Outcome::success(format!( - "Sucessfully installed receiver {}", - prom_receiver_name - ))) - } - - pub async fn install_rule( - &self, - prometheus_rule: &AlertManagerRuleGroup, - ) -> Result { - let prometheus_rule = prometheus_rule.configure_rule().await; - let mut config = self.config.lock().unwrap(); - - config.alert_rules.push(prometheus_rule.clone()); - Ok(Outcome::success(format!( - "Successfully installed alert rule: {:#?},", - prometheus_rule - ))) - } - - pub async fn install_prometheus( - &self, - inventory: &Inventory, - topology: &T, - ) -> Result { - kube_prometheus_helm_chart_score(self.config.clone()) - .interpret(inventory, topology) - .await - } -} - -#[async_trait] -pub trait KubePrometheusReceiver: Send + Sync + std::fmt::Debug { - fn name(&self) -> String; - async fn configure_receiver(&self) -> AlertManagerChannelConfig; } impl Serialize for Box> { @@ -139,12 +78,6 @@ impl Serialize for Box> { } } -#[async_trait] -pub trait KubePrometheusRule: Send + Sync + std::fmt::Debug { - fn name(&self) -> String; - async fn configure_rule(&self) -> AlertManagerAdditionalPromRules; -} - impl Serialize for Box> { fn serialize(&self, _serializer: S) -> Result where diff --git a/harmony/src/modules/monitoring/prometheus/helm/mod.rs b/harmony/src/modules/monitoring/prometheus/helm/mod.rs index 431fc6ca..99a01e51 100644 --- a/harmony/src/modules/monitoring/prometheus/helm/mod.rs +++ b/harmony/src/modules/monitoring/prometheus/helm/mod.rs @@ -1 +1,2 @@ +pub mod prometheus_config; pub mod prometheus_helm; diff --git a/harmony/src/modules/monitoring/prometheus/prometheus_config.rs b/harmony/src/modules/monitoring/prometheus/helm/prometheus_config.rs similarity index 95% rename from harmony/src/modules/monitoring/prometheus/prometheus_config.rs rename to harmony/src/modules/monitoring/prometheus/helm/prometheus_config.rs index 32c3fab5..031d1ac1 100644 --- a/harmony/src/modules/monitoring/prometheus/prometheus_config.rs +++ b/harmony/src/modules/monitoring/prometheus/helm/prometheus_config.rs @@ -1,8 +1,10 @@ +use serde::Serialize; + use crate::modules::monitoring::kube_prometheus::types::{ AlertManagerAdditionalPromRules, AlertManagerChannelConfig, ServiceMonitor, }; -#[derive(Debug)] +#[derive(Debug, Clone, Serialize)] pub struct PrometheusConfig { pub namespace: Option, pub default_rules: bool, diff --git a/harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs b/harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs index 611c500f..642bd985 100644 --- a/harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs +++ b/harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs @@ -3,9 +3,8 @@ use std::sync::{Arc, Mutex}; use non_blank_string_rs::NonBlankString; -use crate::modules::{ - helm::chart::HelmChartScore, monitoring::prometheus::prometheus_config::PrometheusConfig, -}; +use crate::modules::helm::chart::HelmChartScore; +use crate::modules::monitoring::prometheus::helm::prometheus_config::PrometheusConfig; pub fn prometheus_helm_chart_score(config: Arc>) -> HelmChartScore { let config = config.lock().unwrap(); diff --git a/harmony/src/modules/monitoring/prometheus/mod.rs b/harmony/src/modules/monitoring/prometheus/mod.rs index d1bda0e2..cc3bf05d 100644 --- a/harmony/src/modules/monitoring/prometheus/mod.rs +++ b/harmony/src/modules/monitoring/prometheus/mod.rs @@ -1,4 +1,55 @@ +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use serde::Serialize; + +use crate::{ + modules::monitoring::prometheus::helm::prometheus_config::PrometheusConfig, + topology::monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}, +}; + pub mod helm; -#[allow(clippy::module_inception)] -pub mod prometheus; -pub mod prometheus_config; +pub mod prometheus_alerting_score; +pub mod score_prometheus_alert_receivers; +pub mod score_prometheus_ensure_ready; +pub mod score_prometheus_rule; +pub mod score_prometheus_scrape_target; + +#[derive(Debug, Clone, Serialize)] +pub struct Prometheus { + pub config: Arc>, +} + +#[async_trait] +impl AlertSender for Prometheus { + fn name(&self) -> String { + "Prometheus".to_string() + } +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/prometheus/prometheus.rs b/harmony/src/modules/monitoring/prometheus/prometheus.rs deleted file mode 100644 index ae7dc9b3..00000000 --- a/harmony/src/modules/monitoring/prometheus/prometheus.rs +++ /dev/null @@ -1,137 +0,0 @@ -use std::sync::{Arc, Mutex}; - -use async_trait::async_trait; -use log::{debug, error}; -use serde::Serialize; - -use crate::{ - interpret::{InterpretError, Outcome}, - inventory::Inventory, - modules::monitoring::{ - alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, - kube_prometheus::{ - KubePrometheusRule, - types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig}, - }, - }, - score::Score, - topology::{ - HelmCommand, Topology, - monitoring::{AlertReceiver, AlertRule, AlertSender}, - tenant::TenantManager, - }, -}; - -use super::{ - helm::prometheus_helm::prometheus_helm_chart_score, prometheus_config::PrometheusConfig, -}; - -#[derive(Debug)] -pub struct Prometheus { - pub config: Arc>, -} - -#[async_trait] -impl AlertSender for Prometheus { - fn name(&self) -> String { - "Prometheus".to_string() - } -} - -impl Default for Prometheus { - fn default() -> Self { - Self::new() - } -} - -impl Prometheus { - pub fn new() -> Self { - Self { - config: Arc::new(Mutex::new(PrometheusConfig::new())), - } - } - pub async fn configure_with_topology(&self, topology: &T) { - let ns = topology - .get_tenant_config() - .await - .map(|cfg| cfg.name.clone()) - .unwrap_or_else(|| "monitoring".to_string()); - error!("This must be refactored, see comments in pr #74"); - debug!("NS: {}", ns); - self.config.lock().unwrap().namespace = Some(ns); - } - - pub async fn install_receiver( - &self, - prometheus_receiver: &dyn PrometheusReceiver, - ) -> Result { - let prom_receiver = prometheus_receiver.configure_receiver().await; - debug!( - "adding alert receiver to prometheus config: {:#?}", - &prom_receiver - ); - let mut config = self.config.lock().unwrap(); - - config.alert_receiver_configs.push(prom_receiver); - let prom_receiver_name = prometheus_receiver.name(); - debug!("installed alert receiver {}", &prom_receiver_name); - Ok(Outcome::success(format!( - "Sucessfully installed receiver {}", - prom_receiver_name - ))) - } - - pub async fn install_rule( - &self, - prometheus_rule: &AlertManagerRuleGroup, - ) -> Result { - let prometheus_rule = prometheus_rule.configure_rule().await; - let mut config = self.config.lock().unwrap(); - - config.alert_rules.push(prometheus_rule.clone()); - Ok(Outcome::success(format!( - "Successfully installed alert rule: {:#?},", - prometheus_rule - ))) - } - - pub async fn install_prometheus( - &self, - inventory: &Inventory, - topology: &T, - ) -> Result { - prometheus_helm_chart_score(self.config.clone()) - .interpret(inventory, topology) - .await - } -} - -#[async_trait] -pub trait PrometheusReceiver: Send + Sync + std::fmt::Debug { - fn name(&self) -> String; - async fn configure_receiver(&self) -> AlertManagerChannelConfig; -} - -impl Serialize for Box> { - fn serialize(&self, _serializer: S) -> Result - where - S: serde::Serializer, - { - todo!() - } -} - -#[async_trait] -pub trait PrometheusRule: Send + Sync + std::fmt::Debug { - fn name(&self) -> String; - async fn configure_rule(&self) -> AlertManagerAdditionalPromRules; -} - -impl Serialize for Box> { - fn serialize(&self, _serializer: S) -> Result - where - S: serde::Serializer, - { - todo!() - } -} diff --git a/harmony/src/modules/monitoring/prometheus/prometheus_alerting_score.rs b/harmony/src/modules/monitoring/prometheus/prometheus_alerting_score.rs new file mode 100644 index 00000000..fb4ec49f --- /dev/null +++ b/harmony/src/modules/monitoring/prometheus/prometheus_alerting_score.rs @@ -0,0 +1,47 @@ +use std::sync::{Arc, Mutex}; + +use serde::Serialize; + +use crate::{ + modules::monitoring::{ + kube_prometheus::types::ServiceMonitor, + prometheus::{Prometheus, helm::prometheus_config::PrometheusConfig}, + }, + score::Score, + topology::{ + HelmCommand, Topology, + monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget}, + tenant::TenantManager, + }, +}; + +#[derive(Clone, Debug, Serialize)] +pub struct PrometheusAlertingScore { + pub receivers: Vec>>, + pub rules: Vec>>, + pub scrape_targets: Option>>>, + pub service_monitors: Vec, + pub config: Arc>, +} + +impl> Score for PrometheusAlertingScore { + fn create_interpret(&self) -> Box> { + //TODO test that additional service monitor is added + self.config + .try_lock() + .expect("couldn't lock config") + .additional_service_monitors = self.service_monitors.clone(); + + Box::new(AlertingInterpret { + sender: Prometheus { + config: self.config.clone(), + }, + receivers: self.receivers.clone(), + rules: self.rules.clone(), + scrape_targets: self.scrape_targets.clone(), + }) + } + fn name(&self) -> String { + "HelmPrometheusAlertingScore".to_string() + } +} diff --git a/harmony/src/modules/monitoring/prometheus/score_prometheus_alert_receivers.rs b/harmony/src/modules/monitoring/prometheus/score_prometheus_alert_receivers.rs new file mode 100644 index 00000000..144c173c --- /dev/null +++ b/harmony/src/modules/monitoring/prometheus/score_prometheus_alert_receivers.rs @@ -0,0 +1,58 @@ +use kube::api::ObjectMeta; +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::{ + k8s::resource::K8sResourceScore, + monitoring::{ + kube_prometheus::crd::crd_alertmanager_config::{ + AlertmanagerConfig, AlertmanagerConfigSpec, + }, + prometheus::Prometheus, + }, + }, + score::Score, + topology::{K8sclient, Topology, monitoring::AlertReceiver}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct PrometheusReceiverScore { + pub sender: Prometheus, + pub receiver: Box>, +} + +impl Score for PrometheusReceiverScore { + fn name(&self) -> String { + "PrometheusReceiverScore".to_string() + } + + fn create_interpret(&self) -> Box> { + let name = self.receiver.name(); + let namespace = self.sender.config.lock().unwrap().namespace.clone(); + let route = self.receiver.build_route().expect(&format!( + "failed to build route for receveiver {}", + name.clone() + )); + let receiver = self.receiver.build_receiver().expect(&format!( + "failed to build receiver path for receiver {}", + name.clone() + )); + + let data = serde_json::json!({ + "route": route, + "receivers": [receiver] + }); + + let alertmanager_config = AlertmanagerConfig { + metadata: ObjectMeta { + name: Some(name), + namespace: namespace.clone(), + ..Default::default() + }, + spec: AlertmanagerConfigSpec { data: data }, + }; + + K8sResourceScore::single(alertmanager_config, namespace).create_interpret() + } +} diff --git a/harmony/src/modules/monitoring/prometheus/score_prometheus_ensure_ready.rs b/harmony/src/modules/monitoring/prometheus/score_prometheus_ensure_ready.rs new file mode 100644 index 00000000..7bb14113 --- /dev/null +++ b/harmony/src/modules/monitoring/prometheus/score_prometheus_ensure_ready.rs @@ -0,0 +1,80 @@ +use async_trait::async_trait; +use harmony_types::id::Id; +use serde::Serialize; + +use crate::{ + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + modules::monitoring::prometheus::Prometheus, + score::Score, + topology::{K8sclient, Topology}, +}; + +#[derive(Clone, Debug, Serialize)] +pub struct PrometheusEnsureReadyScore { + pub sender: Prometheus, +} + +impl Score for PrometheusEnsureReadyScore { + fn name(&self) -> String { + "PrometheusEnsureReadyScore".to_string() + } + + fn create_interpret(&self) -> Box> { + Box::new(PrometheusEnsureReadyInterpret { + sender: self.sender.clone(), + }) + } +} + +#[derive(Clone, Debug, Serialize)] +pub struct PrometheusEnsureReadyInterpret { + pub sender: Prometheus, +} + +#[async_trait] +impl Interpret for PrometheusEnsureReadyInterpret { + async fn execute( + &self, + _inventory: &Inventory, + topology: &T, + ) -> Result { + let client = topology.k8s_client().await?; + let namespace = self + .sender + .config + .lock() + .unwrap() + .namespace + .clone() + .unwrap_or("default".to_string()); + + let prometheus_name = "prometheues-prometheus-operator"; + + client + .wait_until_deployment_ready(prometheus_name, Some(&namespace), None) + .await?; + + Ok(Outcome::success(format!( + "deployment: {} ready in ns: {}", + prometheus_name, namespace + ))) + } + + fn get_name(&self) -> InterpretName { + todo!() + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/prometheus/score_prometheus_rule.rs b/harmony/src/modules/monitoring/prometheus/score_prometheus_rule.rs new file mode 100644 index 00000000..c9840c16 --- /dev/null +++ b/harmony/src/modules/monitoring/prometheus/score_prometheus_rule.rs @@ -0,0 +1,48 @@ +use kube::api::ObjectMeta; +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::{ + k8s::resource::K8sResourceScore, + monitoring::{ + kube_prometheus::crd::crd_prometheus_rules::{ + PrometheusRule, PrometheusRuleSpec, RuleGroup, + }, + prometheus::Prometheus, + }, + }, + score::Score, + topology::{K8sclient, Topology, monitoring::AlertRule}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct PrometheusRuleScore { + pub sender: Prometheus, + pub rule: Box>, +} + +impl Score for PrometheusRuleScore { + fn name(&self) -> String { + "PrometheusRuleScore".to_string() + } + + fn create_interpret(&self) -> Box> { + let name = self.rule.name(); + let namespace = self.sender.config.lock().unwrap().namespace.clone(); + let groups: Vec = + serde_json::from_value(self.rule.build_rule().expect("failed to build alert rule")) + .expect("failed to serialize rule group"); + + let prometheus_rule = PrometheusRule { + metadata: ObjectMeta { + name: Some(name.clone()), + namespace: namespace.clone(), + ..Default::default() + }, + + spec: PrometheusRuleSpec { groups }, + }; + K8sResourceScore::single(prometheus_rule, namespace).create_interpret() + } +} diff --git a/harmony/src/modules/monitoring/prometheus/score_prometheus_scrape_target.rs b/harmony/src/modules/monitoring/prometheus/score_prometheus_scrape_target.rs new file mode 100644 index 00000000..1a4d97d1 --- /dev/null +++ b/harmony/src/modules/monitoring/prometheus/score_prometheus_scrape_target.rs @@ -0,0 +1,66 @@ +use kube::api::ObjectMeta; +use serde::Serialize; + +use crate::{ + interpret::Interpret, + modules::{ + k8s::resource::K8sResourceScore, + monitoring::{ + kube_prometheus::crd::crd_scrape_config::{ + ScrapeConfig, ScrapeConfigSpec, StaticConfig, + }, + prometheus::Prometheus, + }, + }, + score::Score, + topology::{ + K8sclient, Topology, + monitoring::{AlertRule, ScrapeTarget}, + }, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct PrometheusScrapeTargetScore { + pub sender: Prometheus, + pub scrape_target: Box>, +} + +impl Score for PrometheusScrapeTargetScore { + fn name(&self) -> String { + "PrometheusScrapeTargetScore".to_string() + } + + fn create_interpret(&self) -> Box> { + let name = self.scrape_target.name(); + let namespace = self.sender.config.lock().unwrap().namespace.clone(); + + let external_target = self + .scrape_target + .build_scrape_target() + .expect("failed to build external scrape target"); + + //TODO this may need to modified to include a scrapeConfigSelector label from the + //prometheus operator + let labels = external_target.labels; + + let scrape_target = ScrapeConfig { + metadata: ObjectMeta { + name: Some(name.clone()), + namespace: namespace.clone(), + ..Default::default() + }, + spec: ScrapeConfigSpec { + static_configs: Some(vec![StaticConfig { + targets: vec![format!("{}:{}", external_target.ip, external_target.port)], + labels, + }]), + metrics_path: external_target.path, + scrape_interval: external_target.interval, + job_name: Some(name), + ..Default::default() + }, + }; + + K8sResourceScore::single(scrape_target, namespace).create_interpret() + } +} diff --git a/harmony/src/modules/monitoring/scrape_target/server.rs b/harmony/src/modules/monitoring/scrape_target/server.rs index eb48cf5f..ffaac18d 100644 --- a/harmony/src/modules/monitoring/scrape_target/server.rs +++ b/harmony/src/modules/monitoring/scrape_target/server.rs @@ -6,9 +6,11 @@ use serde::Serialize; use crate::{ interpret::{InterpretError, Outcome}, - modules::monitoring::kube_prometheus::crd::{ - crd_alertmanager_config::CRDPrometheus, - crd_scrape_config::{Params, RelabelConfig, ScrapeConfig, ScrapeConfigSpec, StaticConfig}, + modules::monitoring::{ + kube_prometheus::crd::crd_scrape_config::{ + Params, RelabelConfig, ScrapeConfig, ScrapeConfigSpec, StaticConfig, + }, + prometheus::Prometheus, }, topology::monitoring::{ExternalScrapeTarget, ScrapeTarget}, }; @@ -22,7 +24,7 @@ pub struct Server { pub domain: String, } -impl ScrapeTarget for Server { +impl ScrapeTarget for Server { fn build_scrape_target(&self) -> Result { todo!() // let scrape_config_spec = ScrapeConfigSpec { @@ -67,7 +69,7 @@ impl ScrapeTarget for Server { // Ok(serde_json::to_value(scrape_config).map_err(|e| InterpretError::new(e.to_string()))?) } - fn clone_box(&self) -> Box> { + fn clone_box(&self) -> Box> { Box::new(self.clone()) } -- 2.39.5 From c4dd0b0cf2789ba8028660dbb288a87a4af44608 Mon Sep 17 00:00:00 2001 From: wjro Date: Thu, 26 Feb 2026 16:06:14 -0500 Subject: [PATCH 15/15] chore: cleaned up some dead code, comments, etc --- .../k8s_anywhere/observability/prometheus.rs | 11 ++ .../redhat_cluster_observability.rs | 16 +-- .../alert_channel/discord_alert_channel.rs | 110 ------------------ .../application_monitoring_score.rs | 2 + .../monitoring/application_monitoring/mod.rs | 2 +- .../rhobs_application_monitoring_score.rs | 56 ++++----- .../crd/crd_alertmanager_config.rs | 17 --- .../helm/kube_prometheus_helm_chart.rs | 9 +- .../kube_prometheus_alerting_score.rs | 4 +- .../modules/monitoring/kube_prometheus/mod.rs | 21 +--- .../score_kube_prometheus_scrape_target.rs | 10 +- .../monitoring/kube_prometheus/types.rs | 44 ------- harmony/src/modules/monitoring/okd/mod.rs | 8 -- .../okd/openshift_cluster_alerting_score.rs | 6 +- .../okd/score_enable_cluster_monitoring.rs | 4 +- .../okd/score_openshift_receiver.rs | 7 +- .../okd/score_openshift_scrape_target.rs | 2 + .../monitoring/okd/score_user_workload.rs | 3 + .../score_verify_user_workload_monitoring.rs | 2 + .../prometheus/helm/prometheus_helm.rs | 1 + .../src/modules/monitoring/prometheus/mod.rs | 1 + .../prometheus/prometheus_alerting_score.rs | 5 +- .../score_prometheus_alert_receivers.rs | 1 + .../prometheus/score_prometheus_install.rs | 65 +++++++++++ .../score_prometheus_scrape_target.rs | 6 +- .../red_hat_cluster_observability/crd/mod.rs | 1 - .../crd/rhob_alertmanager_config.rs | 28 ----- .../rhob_cluster_observability_operator.rs | 22 ---- .../red_hat_cluster_observability/mod.rs | 9 +- .../redhat_cluster_observability.rs | 1 + .../score_alert_receiver.rs | 11 +- .../modules/monitoring/scrape_target/mod.rs | 2 - .../scrape_target/prometheus_node_exporter.rs | 10 +- .../monitoring/scrape_target/server.rs | 11 +- 34 files changed, 151 insertions(+), 357 deletions(-) create mode 100644 harmony/src/modules/monitoring/prometheus/score_prometheus_install.rs delete mode 100644 harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_cluster_observability_operator.rs diff --git a/harmony/src/domain/topology/k8s_anywhere/observability/prometheus.rs b/harmony/src/domain/topology/k8s_anywhere/observability/prometheus.rs index a90520cb..9085a607 100644 --- a/harmony/src/domain/topology/k8s_anywhere/observability/prometheus.rs +++ b/harmony/src/domain/topology/k8s_anywhere/observability/prometheus.rs @@ -5,6 +5,7 @@ use crate::{ modules::monitoring::prometheus::{ Prometheus, score_prometheus_alert_receivers::PrometheusReceiverScore, score_prometheus_ensure_ready::PrometheusEnsureReadyScore, + score_prometheus_install::PrometheusInstallScore, score_prometheus_rule::PrometheusRuleScore, score_prometheus_scrape_target::PrometheusScrapeTargetScore, }, @@ -22,6 +23,16 @@ impl Observability for K8sAnywhereTopology { sender: &Prometheus, inventory: &Inventory, ) -> Result { + let score = PrometheusInstallScore { + sender: sender.clone(), + }; + + score + .create_interpret() + .execute(inventory, self) + .await + .map_err(|e| PreparationError::new(format!("Prometheus not installed {}", e)))?; + Ok(PreparationOutcome::Success { details: "Successfully installed kubeprometheus alert sender".to_string(), }) diff --git a/harmony/src/domain/topology/k8s_anywhere/observability/redhat_cluster_observability.rs b/harmony/src/domain/topology/k8s_anywhere/observability/redhat_cluster_observability.rs index 81362ee6..0a2ada47 100644 --- a/harmony/src/domain/topology/k8s_anywhere/observability/redhat_cluster_observability.rs +++ b/harmony/src/domain/topology/k8s_anywhere/observability/redhat_cluster_observability.rs @@ -88,26 +88,26 @@ impl Observability for K8sAnywhereTopology { async fn install_rules( &self, - sender: &RedHatClusterObservability, - inventory: &Inventory, - rules: Option>>>, + _sender: &RedHatClusterObservability, + _inventory: &Inventory, + _rules: Option>>>, ) -> Result { todo!() } async fn add_scrape_targets( &self, - sender: &RedHatClusterObservability, - inventory: &Inventory, - scrape_targets: Option>>>, + _sender: &RedHatClusterObservability, + _inventory: &Inventory, + _scrape_targets: Option>>>, ) -> Result { todo!() } async fn ensure_monitoring_installed( &self, - sender: &RedHatClusterObservability, - inventory: &Inventory, + _sender: &RedHatClusterObservability, + _inventory: &Inventory, ) -> Result { todo!() } diff --git a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs index fd3f5066..ee3b2387 100644 --- a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs +++ b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs @@ -135,113 +135,3 @@ impl AlertReceiver for DiscordReceiver { Box::new(self.clone()) } } - -// #[async_trait] -// impl AlertReceiver for DiscordWebhook { -// fn as_alertmanager_receiver(&self) -> Result { -// todo!() -// } -// async fn install(&self, sender: &CRDPrometheus) -> Result { -// let ns = sender.namespace.clone(); -// let secret_name = format!("{}-secret", self.name.clone()); -// let webhook_key = format!("{}", self.url.clone()); -// -// let mut string_data = BTreeMap::new(); -// string_data.insert("webhook-url".to_string(), webhook_key.clone()); -// -// let secret = Secret { -// metadata: kube::core::ObjectMeta { -// name: Some(secret_name.clone()), -// ..Default::default() -// }, -// string_data: Some(string_data), -// type_: Some("Opaque".to_string()), -// ..Default::default() -// }; -// -// let _ = sender.client.apply(&secret, Some(&ns)).await; -// -// let spec = AlertmanagerConfigSpec { -// data: json!({ -// "route": { -// "receiver": self.name, -// }, -// "receivers": [ -// { -// "name": self.name, -// "discordConfigs": [ -// { -// "apiURL": { -// "name": secret_name, -// "key": "webhook-url", -// }, -// "title": "{{ template \"discord.default.title\" . }}", -// "message": "{{ template \"discord.default.message\" . }}" -// } -// ] -// } -// ] -// }), -// }; -// -// let alertmanager_configs = AlertmanagerConfig { -// metadata: ObjectMeta { -// name: Some(self.name.clone().to_string()), -// labels: Some(std::collections::BTreeMap::from([( -// "alertmanagerConfig".to_string(), -// "enabled".to_string(), -// )])), -// namespace: Some(ns), -// ..Default::default() -// }, -// spec, -// }; -// -// sender -// .client -// .apply(&alertmanager_configs, Some(&sender.namespace)) -// .await?; -// Ok(Outcome::success(format!( -// "installed crd-alertmanagerconfigs for {}", -// self.name -// ))) -// } -// fn name(&self) -> String { -// "discord-webhook".to_string() -// } -// fn clone_box(&self) -> Box> { -// Box::new(self.clone()) -// } -// fn as_any(&self) -> &dyn Any { -// self -// } -// } -// -// #[async_trait] -// impl AlertReceiver for DiscordWebhook { -// fn as_alertmanager_receiver(&self) -> Result { -// todo!() -// } -// async fn install(&self, sender: &Prometheus) -> Result { -// sender.install_receiver(self).await -// } -// fn name(&self) -> String { -// "discord-webhook".to_string() -// } -// fn clone_box(&self) -> Box> { -// Box::new(self.clone()) -// } -// fn as_any(&self) -> &dyn Any { -// self -// } -// } -// -// #[async_trait] -// impl PrometheusReceiver for DiscordWebhook { -// fn name(&self) -> String { -// self.name.clone().to_string() -// } -// async fn configure_receiver(&self) -> AlertManagerChannelConfig { -// self.get_config().await -// } -// } diff --git a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs index f5385a22..b72e1f43 100644 --- a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs +++ b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs @@ -23,6 +23,8 @@ pub struct ApplicationMonitoringScore { impl + K8sclient> Score for ApplicationMonitoringScore { fn create_interpret(&self) -> Box> { debug!("creating alerting interpret"); + //TODO will need to use k8sclient to apply service monitors or find a way to pass + //them to the AlertingInterpret potentially via Sender Prometheus Box::new(AlertingInterpret { sender: self.sender.clone(), receivers: self.receivers.clone(), diff --git a/harmony/src/modules/monitoring/application_monitoring/mod.rs b/harmony/src/modules/monitoring/application_monitoring/mod.rs index 02815703..5d12f78a 100644 --- a/harmony/src/modules/monitoring/application_monitoring/mod.rs +++ b/harmony/src/modules/monitoring/application_monitoring/mod.rs @@ -1,2 +1,2 @@ pub mod application_monitoring_score; -// pub mod rhobs_application_monitoring_score; +pub mod rhobs_application_monitoring_score; diff --git a/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs index 50a25964..76aab06c 100644 --- a/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs +++ b/harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs @@ -9,27 +9,27 @@ use crate::{ inventory::Inventory, modules::{ application::Application, - monitoring::kube_prometheus::crd::{ - }, + monitoring::red_hat_cluster_observability::RedHatClusterObservability, }, score::Score, topology::{ - PreparationOutcome, Topology, - oberservability::monitoring::{AlertReceiver, Observability}, + Topology, + monitoring::{AlertReceiver, AlertingInterpret, Observability}, }, }; use harmony_types::id::Id; - #[derive(Debug, Clone, Serialize)] -pub struct ApplicationRHOBMonitoringScore { - pub sender: RHOBObservability, +pub struct ApplicationRedHatClusterMonitoringScore { + pub sender: RedHatClusterObservability, pub application: Arc, - pub receivers: Vec>>, + pub receivers: Vec>>, } -impl> Score for ApplicationRHOBMonitoringScore { +impl> Score + for ApplicationRedHatClusterMonitoringScore +{ fn create_interpret(&self) -> Box> { - Box::new(ApplicationRHOBMonitoringInterpret { + Box::new(ApplicationRedHatClusterMonitoringInterpret { score: self.clone(), }) } @@ -43,36 +43,28 @@ impl> Score for ApplicationRHO } #[derive(Debug)] -pub struct ApplicationRHOBMonitoringInterpret { - score: ApplicationRHOBMonitoringScore, +pub struct ApplicationRedHatClusterMonitoringInterpret { + score: ApplicationRedHatClusterMonitoringScore, } #[async_trait] -impl> Interpret for ApplicationRHOBMonitoringInterpret { +impl> Interpret + for ApplicationRedHatClusterMonitoringInterpret +{ async fn execute( &self, inventory: &Inventory, topology: &T, ) -> Result { - todo!() - // let result = topology - // .install_montoring( - // &self.score.sender, - // inventory, - // Some(self.score.receivers.clone()), - // ) - // .await; - // match result { - // Ok(outcome) => match outcome { - // PreparationOutcome::Success { details: _ } => { - // Ok(Outcome::success("Prometheus installed".into())) - // } - // PreparationOutcome::Noop => { - // Ok(Outcome::noop("Prometheus installation skipped".into())) - // } - // }, - // Err(err) => Err(InterpretError::from(err)), - // } + //TODO will need to use k8sclient to apply crd ServiceMonitor or find a way to pass + //them to the AlertingInterpret potentially via Sender RedHatClusterObservability + let alerting_interpret = AlertingInterpret { + sender: self.score.sender.clone(), + receivers: self.score.receivers.clone(), + rules: vec![], + scrape_targets: None, + }; + alerting_interpret.execute(inventory, topology).await } fn get_name(&self) -> InterpretName { diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs index e9de6e28..f544b603 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs @@ -1,24 +1,7 @@ -use std::sync::Arc; - -use async_trait::async_trait; use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::{ - interpret::{InterpretError, Outcome}, - inventory::Inventory, - modules::monitoring::{ - grafana::grafana::Grafana, kube_prometheus::crd::service_monitor::ServiceMonitor, - }, - topology::{ - K8sclient, Topology, - installable::Installable, - k8s::K8sClient, - monitoring::{AlertReceiver, AlertSender, Observability, ScrapeTarget}, - }, -}; - #[derive(CustomResource, Serialize, Deserialize, Default, Debug, Clone, JsonSchema)] #[kube( group = "monitoring.coreos.com", diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs b/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs index ea7d2530..3f38f0b1 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs @@ -1,20 +1,13 @@ use super::config::KubePrometheusConfig; -use log::debug; use non_blank_string_rs::NonBlankString; -use serde_yaml::{Mapping, Value}; use std::{ - collections::BTreeMap, str::FromStr, sync::{Arc, Mutex}, }; use crate::modules::{ helm::chart::HelmChartScore, - monitoring::kube_prometheus::types::{ - AlertGroup, AlertManager, AlertManagerAdditionalPromRules, AlertManagerConfig, - AlertManagerConfigSelector, AlertManagerRoute, AlertManagerSpec, AlertManagerValues, - ConfigReloader, Limits, PrometheusConfig, Requests, Resources, - }, + monitoring::kube_prometheus::types::{Limits, Requests, Resources}, }; pub fn kube_prometheus_helm_chart_score( diff --git a/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs b/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs index 31eebcdd..97f12d03 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/kube_prometheus_alerting_score.rs @@ -7,12 +7,12 @@ use crate::{ modules::monitoring::kube_prometheus::{KubePrometheus, types::ServiceMonitor}, score::Score, topology::{ - HelmCommand, Topology, + Topology, monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget}, - tenant::TenantManager, }, }; +//TODO untested #[derive(Clone, Debug, Serialize)] pub struct KubePrometheusAlertingScore { pub receivers: Vec>>, diff --git a/harmony/src/modules/monitoring/kube_prometheus/mod.rs b/harmony/src/modules/monitoring/kube_prometheus/mod.rs index 7f496fb2..7fed749f 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/mod.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/mod.rs @@ -1,28 +1,11 @@ use std::sync::{Arc, Mutex}; use async_trait::async_trait; -use log::{debug, error}; use serde::Serialize; use crate::{ - interpret::{InterpretError, Outcome}, - inventory::Inventory, - modules::monitoring::{ - alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, - kube_prometheus::{ - helm::{ - config::KubePrometheusConfig, - kube_prometheus_helm_chart::kube_prometheus_helm_chart_score, - }, - types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig}, - }, - }, - score::Score, - topology::{ - HelmCommand, Topology, - monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}, - tenant::TenantManager, - }, + modules::monitoring::kube_prometheus::helm::config::KubePrometheusConfig, + topology::monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}, }; pub mod crd; diff --git a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs index 7610a2a4..9e34fb0f 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/score_kube_prometheus_scrape_target.rs @@ -7,17 +7,11 @@ use crate::{ k8s::resource::K8sResourceScore, monitoring::kube_prometheus::{ KubePrometheus, - crd::{ - crd_prometheuses::LabelSelector, - crd_scrape_config::{ScrapeConfig, ScrapeConfigSpec, StaticConfig}, - }, + crd::crd_scrape_config::{ScrapeConfig, ScrapeConfigSpec, StaticConfig}, }, }, score::Score, - topology::{ - K8sclient, Topology, - monitoring::{AlertRule, ScrapeTarget}, - }, + topology::{K8sclient, Topology, monitoring::ScrapeTarget}, }; #[derive(Debug, Clone, Serialize)] diff --git a/harmony/src/modules/monitoring/kube_prometheus/types.rs b/harmony/src/modules/monitoring/kube_prometheus/types.rs index abe5896a..e697b50c 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/types.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/types.rs @@ -12,36 +12,6 @@ pub trait AlertChannelConfig { async fn get_config(&self) -> AlertManagerChannelConfig; } -#[derive(Debug, Clone, Serialize)] -pub struct AlertManagerValues { - pub alertmanager: AlertManager, -} -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AlertManager { - pub enabled: bool, - pub config: AlertManagerConfig, - pub alertmanager_spec: AlertManagerSpec, - pub init_config_reloader: ConfigReloader, -} - -#[derive(Debug, Clone, Serialize)] -pub struct ConfigReloader { - pub resources: Resources, -} - -#[derive(Debug, Clone, Serialize)] -pub struct AlertManagerConfig { - pub global: Mapping, - pub route: AlertManagerRoute, - pub receivers: Sequence, -} - -#[derive(Debug, Clone, Serialize)] -pub struct AlertManagerRoute { - pub routes: Sequence, -} - #[derive(Debug, Clone, Serialize)] pub struct AlertManagerChannelConfig { ///expecting an option that contains two values @@ -52,20 +22,6 @@ pub struct AlertManagerChannelConfig { pub channel_receiver: Value, } -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AlertManagerSpec { - pub(crate) resources: Resources, - pub replicas: u32, - pub alert_manager_config_selector: AlertManagerConfigSelector, -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AlertManagerConfigSelector { - pub match_labels: BTreeMap, -} - #[derive(Debug, Clone, Serialize)] pub struct Resources { pub limits: Limits, diff --git a/harmony/src/modules/monitoring/okd/mod.rs b/harmony/src/modules/monitoring/okd/mod.rs index 0af3033a..62d00745 100644 --- a/harmony/src/modules/monitoring/okd/mod.rs +++ b/harmony/src/modules/monitoring/okd/mod.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use serde::Serialize; use crate::topology::monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}; @@ -22,12 +20,6 @@ impl AlertSender for OpenshiftClusterAlertSender { } } -// impl Clone for Box> { -// fn clone(&self) -> Self { -// self.clone_box() -// } -// } - impl Serialize for Box> { fn serialize(&self, _serializer: S) -> Result where diff --git a/harmony/src/modules/monitoring/okd/openshift_cluster_alerting_score.rs b/harmony/src/modules/monitoring/okd/openshift_cluster_alerting_score.rs index 42362854..ee801bb8 100644 --- a/harmony/src/modules/monitoring/okd/openshift_cluster_alerting_score.rs +++ b/harmony/src/modules/monitoring/okd/openshift_cluster_alerting_score.rs @@ -1,11 +1,7 @@ -use async_trait::async_trait; -use harmony_types::id::Id; use serde::Serialize; use crate::{ - data::Version, - interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, - inventory::Inventory, + interpret::Interpret, modules::monitoring::okd::OpenshiftClusterAlertSender, score::Score, topology::{ diff --git a/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs b/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs index 952bf9cb..5459c186 100644 --- a/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs +++ b/harmony/src/modules/monitoring/okd/score_enable_cluster_monitoring.rs @@ -121,7 +121,7 @@ impl OpenshiftEnableClusterMonitoringInterpret { .await { Ok(obj) => obj, - Err(_) => return Ok(false), // CM doesn't exist? Treat as disabled. + Err(_) => return Ok(false), }; debug!("{:#?}", cm.data.pointer("/data/config.yaml")); @@ -131,7 +131,7 @@ impl OpenshiftEnableClusterMonitoringInterpret { .and_then(|v| v.as_str()) { Some(s) => s, - None => return Ok(false), // Key missing? Treat as disabled. + None => return Ok(false), }; debug!("{:#?}", config_yaml_str); diff --git a/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs b/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs index 74525250..6f7013c3 100644 --- a/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs +++ b/harmony/src/modules/monitoring/okd/score_openshift_receiver.rs @@ -2,7 +2,6 @@ use async_trait::async_trait; use base64::{Engine as _, prelude::BASE64_STANDARD}; use harmony_types::id::Id; use kube::api::DynamicObject; -use log::{debug, info, trace}; use serde::Serialize; use crate::{ @@ -46,10 +45,10 @@ impl Interpret for OpenshiftReceiverInterpret { let client = topology.k8s_client().await?; let ns = "openshift-monitoring"; - // 1️⃣ Get the alertmanager-main secret let mut am_secret: DynamicObject = client .get_secret_json_value("alertmanager-main", Some(ns)) .await?; + let data = am_secret .data .get_mut("data") @@ -63,7 +62,9 @@ impl Interpret for OpenshiftReceiverInterpret { .get("alertmanager.yaml") .and_then(|v| v.as_str()) .unwrap_or_default(); + let config_bytes = BASE64_STANDARD.decode(config_b64).unwrap_or_default(); + let mut am_config: serde_yaml::Value = serde_yaml::from_slice(&config_bytes) .unwrap_or_else(|_| serde_yaml::Value::Mapping(serde_yaml::Mapping::new())); @@ -107,7 +108,9 @@ impl Interpret for OpenshiftReceiverInterpret { let yaml_str = serde_yaml::to_string(&am_config).map_err(|e| InterpretError::new(e.to_string()))?; + let mut yaml_b64 = String::new(); + BASE64_STANDARD.encode_string(yaml_str, &mut yaml_b64); data.insert( "alertmanager.yaml".to_string(), diff --git a/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs b/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs index ed497320..44b0106c 100644 --- a/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs +++ b/harmony/src/modules/monitoring/okd/score_openshift_scrape_target.rs @@ -72,10 +72,12 @@ impl Interpret for OpenshiftScrapeTargetInterpret { .create_interpret() .execute(inventory, topology) .await?; + K8sResourceScore::single(endpoints, Some(namespace.clone())) .create_interpret() .execute(inventory, topology) .await?; + K8sResourceScore::single(service_monitor, Some(namespace.clone())) .create_interpret() .execute(inventory, topology) diff --git a/harmony/src/modules/monitoring/okd/score_user_workload.rs b/harmony/src/modules/monitoring/okd/score_user_workload.rs index 3eab008d..54553be7 100644 --- a/harmony/src/modules/monitoring/okd/score_user_workload.rs +++ b/harmony/src/modules/monitoring/okd/score_user_workload.rs @@ -41,9 +41,11 @@ impl Interpret for OpenshiftUserWorkloadMonitoringIn let namespace = "openshift-user-workload-monitoring".to_string(); let cm_name = "user-workload-monitoring-config".to_string(); let client = topology.k8s_client().await?; + let cm_enabled = self .check_cluster_user_workload_monitoring_enabled(client, &cm_name, &namespace) .await?; + match cm_enabled { true => Ok(Outcome::success( "OpenshiftUserWorkloadMonitoringEnabled".to_string(), @@ -59,6 +61,7 @@ alertmanager: "# .to_string(), ); + let cm = ConfigMap { metadata: ObjectMeta { name: Some("user-workload-monitoring-config".to_string()), diff --git a/harmony/src/modules/monitoring/okd/score_verify_user_workload_monitoring.rs b/harmony/src/modules/monitoring/okd/score_verify_user_workload_monitoring.rs index 1d41d358..29786763 100644 --- a/harmony/src/modules/monitoring/okd/score_verify_user_workload_monitoring.rs +++ b/harmony/src/modules/monitoring/okd/score_verify_user_workload_monitoring.rs @@ -37,9 +37,11 @@ impl Interpret for VerifyUserWorkloadInterpret { let namespace = "openshift-user-workload-monitoring"; let alertmanager_name = "alertmanager-user-workload-0"; let prometheus_name = "prometheus-user-workload-0"; + client .wait_for_pod_ready(alertmanager_name, Some(namespace)) .await?; + client .wait_for_pod_ready(prometheus_name, Some(namespace)) .await?; diff --git a/harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs b/harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs index 642bd985..b0305a8d 100644 --- a/harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs +++ b/harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs @@ -29,6 +29,7 @@ server: fullnameOverride: prometheus-{ns} "# ); + HelmChartScore { namespace: Some(NonBlankString::from_str(&config.namespace.clone().unwrap()).unwrap()), release_name: NonBlankString::from_str("prometheus").unwrap(), diff --git a/harmony/src/modules/monitoring/prometheus/mod.rs b/harmony/src/modules/monitoring/prometheus/mod.rs index cc3bf05d..0cf651a6 100644 --- a/harmony/src/modules/monitoring/prometheus/mod.rs +++ b/harmony/src/modules/monitoring/prometheus/mod.rs @@ -12,6 +12,7 @@ pub mod helm; pub mod prometheus_alerting_score; pub mod score_prometheus_alert_receivers; pub mod score_prometheus_ensure_ready; +pub mod score_prometheus_install; pub mod score_prometheus_rule; pub mod score_prometheus_scrape_target; diff --git a/harmony/src/modules/monitoring/prometheus/prometheus_alerting_score.rs b/harmony/src/modules/monitoring/prometheus/prometheus_alerting_score.rs index fb4ec49f..52d9d8cc 100644 --- a/harmony/src/modules/monitoring/prometheus/prometheus_alerting_score.rs +++ b/harmony/src/modules/monitoring/prometheus/prometheus_alerting_score.rs @@ -9,12 +9,12 @@ use crate::{ }, score::Score, topology::{ - HelmCommand, Topology, + Topology, monitoring::{AlertReceiver, AlertRule, AlertingInterpret, Observability, ScrapeTarget}, - tenant::TenantManager, }, }; +//TODO untested #[derive(Clone, Debug, Serialize)] pub struct PrometheusAlertingScore { pub receivers: Vec>>, @@ -41,6 +41,7 @@ impl> Score for PrometheusAlertingSco scrape_targets: self.scrape_targets.clone(), }) } + fn name(&self) -> String { "HelmPrometheusAlertingScore".to_string() } diff --git a/harmony/src/modules/monitoring/prometheus/score_prometheus_alert_receivers.rs b/harmony/src/modules/monitoring/prometheus/score_prometheus_alert_receivers.rs index 144c173c..58acfdb9 100644 --- a/harmony/src/modules/monitoring/prometheus/score_prometheus_alert_receivers.rs +++ b/harmony/src/modules/monitoring/prometheus/score_prometheus_alert_receivers.rs @@ -34,6 +34,7 @@ impl Score for PrometheusReceiverScore { "failed to build route for receveiver {}", name.clone() )); + let receiver = self.receiver.build_receiver().expect(&format!( "failed to build receiver path for receiver {}", name.clone() diff --git a/harmony/src/modules/monitoring/prometheus/score_prometheus_install.rs b/harmony/src/modules/monitoring/prometheus/score_prometheus_install.rs new file mode 100644 index 00000000..39eeef2c --- /dev/null +++ b/harmony/src/modules/monitoring/prometheus/score_prometheus_install.rs @@ -0,0 +1,65 @@ +use async_trait::async_trait; +use harmony_types::id::Id; +use serde::Serialize; + +use crate::{ + data::Version, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::Inventory, + modules::monitoring::prometheus::{ + Prometheus, helm::prometheus_helm::prometheus_helm_chart_score, + }, + score::Score, + topology::{HelmCommand, K8sclient, Topology}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct PrometheusInstallScore { + pub sender: Prometheus, +} + +impl Score for PrometheusInstallScore { + fn name(&self) -> String { + "PrometheusInstallScore".to_string() + } + + fn create_interpret(&self) -> Box> { + Box::new(PrometheusInstallInterpret { + score: self.clone(), + }) + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct PrometheusInstallInterpret { + score: PrometheusInstallScore, +} + +#[async_trait] +impl Interpret for PrometheusInstallInterpret { + async fn execute( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + //TODO add interpret to install service monitors CRD from sender Prometheus + let score = prometheus_helm_chart_score(self.score.sender.config.clone()); + score.create_interpret().execute(inventory, topology).await + } + + fn get_name(&self) -> InterpretName { + InterpretName::Custom("PrometheusInstallInterpret") + } + + fn get_version(&self) -> Version { + todo!() + } + + fn get_status(&self) -> InterpretStatus { + todo!() + } + + fn get_children(&self) -> Vec { + todo!() + } +} diff --git a/harmony/src/modules/monitoring/prometheus/score_prometheus_scrape_target.rs b/harmony/src/modules/monitoring/prometheus/score_prometheus_scrape_target.rs index 1a4d97d1..df4db9eb 100644 --- a/harmony/src/modules/monitoring/prometheus/score_prometheus_scrape_target.rs +++ b/harmony/src/modules/monitoring/prometheus/score_prometheus_scrape_target.rs @@ -1,6 +1,7 @@ use kube::api::ObjectMeta; use serde::Serialize; +use crate::topology::monitoring::AlertRule; use crate::{ interpret::Interpret, modules::{ @@ -13,10 +14,7 @@ use crate::{ }, }, score::Score, - topology::{ - K8sclient, Topology, - monitoring::{AlertRule, ScrapeTarget}, - }, + topology::{K8sclient, Topology, monitoring::ScrapeTarget}, }; #[derive(Debug, Clone, Serialize)] diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/mod.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/mod.rs index ca68d7c3..0842d910 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/mod.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/mod.rs @@ -1,6 +1,5 @@ pub mod rhob_alertmanager_config; pub mod rhob_alertmanagers; -pub mod rhob_cluster_observability_operator; pub mod rhob_default_rules; pub mod rhob_monitoring_stack; pub mod rhob_prometheus_rules; diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs index f24b040c..0d56dda1 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_alertmanager_config.rs @@ -1,14 +1,7 @@ -use std::sync::Arc; - use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::topology::{ - k8s::K8sClient, - monitoring::{AlertReceiver, AlertSender}, -}; - #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema, Default)] #[kube( group = "monitoring.rhobs", @@ -22,24 +15,3 @@ pub struct AlertmanagerConfigSpec { #[serde(flatten)] pub data: serde_json::Value, } - -#[derive(Debug, Clone, Serialize)] -pub struct RHOBObservability { - pub namespace: String, - pub client: Arc, -} - -impl AlertSender for RHOBObservability { - fn name(&self) -> String { - "RHOBAlertManager".to_string() - } -} - -impl Serialize for Box> { - fn serialize(&self, _serializer: S) -> Result - where - S: serde::Serializer, - { - todo!() - } -} diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_cluster_observability_operator.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_cluster_observability_operator.rs deleted file mode 100644 index bc7ad9fe..00000000 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/crd/rhob_cluster_observability_operator.rs +++ /dev/null @@ -1,22 +0,0 @@ -use std::str::FromStr; - -use non_blank_string_rs::NonBlankString; - -use crate::modules::helm::chart::HelmChartScore; -//TODO package chart or something for COO okd -pub fn rhob_cluster_observability_operator() -> HelmChartScore { - HelmChartScore { - namespace: None, - release_name: NonBlankString::from_str("").unwrap(), - chart_name: NonBlankString::from_str( - "oci://hub.nationtech.io/harmony/nt-prometheus-operator", - ) - .unwrap(), - chart_version: None, - values_overrides: None, - values_yaml: None, - create_namespace: true, - install_only: true, - repository: None, - } -} diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs index 789656bf..5c28b7a7 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/mod.rs @@ -1,15 +1,8 @@ use serde::Serialize; use crate::{ - interpret::Interpret, modules::monitoring::red_hat_cluster_observability::crd::rhob_prometheuses::LabelSelector, - score::Score, - topology::{ - Topology, - monitoring::{ - AlertReceiver, AlertRule, AlertSender, AlertingInterpret, Observability, ScrapeTarget, - }, - }, + topology::monitoring::{AlertReceiver, AlertRule, AlertSender, ScrapeTarget}, }; pub mod crd; diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs index 522f0012..c8417a58 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/redhat_cluster_observability.rs @@ -10,6 +10,7 @@ use crate::{ }, }; +//TODO untested #[derive(Debug, Clone, Serialize)] pub struct RedHatClusterObservabilityScore { pub sender: RedHatClusterObservability, diff --git a/harmony/src/modules/monitoring/red_hat_cluster_observability/score_alert_receiver.rs b/harmony/src/modules/monitoring/red_hat_cluster_observability/score_alert_receiver.rs index 0bdbeb27..1eebd8ec 100644 --- a/harmony/src/modules/monitoring/red_hat_cluster_observability/score_alert_receiver.rs +++ b/harmony/src/modules/monitoring/red_hat_cluster_observability/score_alert_receiver.rs @@ -1,8 +1,5 @@ -use std::collections::BTreeMap; - use async_trait::async_trait; use harmony_types::id::Id; -use k8s_openapi::api::core::v1::Secret; use kube::api::ObjectMeta; use serde::Serialize; @@ -29,11 +26,11 @@ pub struct RedHatClusterObservabilityReceiverScore { impl Score for RedHatClusterObservabilityReceiverScore { fn name(&self) -> String { - "RedHatClusterObsReceiverScore".to_string() + "RedHatClusterObservabilityReceiverScore".to_string() } fn create_interpret(&self) -> Box> { - Box::new(RedHatClusterObsReceiverInterpret { + Box::new(RedHatClusterObservabilityReceiverInterpret { sender: self.sender.clone(), receiver: self.receiver.clone(), }) @@ -41,13 +38,13 @@ impl Score for RedHatClusterObservabilityReceiverSco } #[derive(Debug, Clone, Serialize)] -pub struct RedHatClusterObsReceiverInterpret { +pub struct RedHatClusterObservabilityReceiverInterpret { sender: RedHatClusterObservability, pub receiver: Box>, } #[async_trait] -impl Interpret for RedHatClusterObsReceiverInterpret { +impl Interpret for RedHatClusterObservabilityReceiverInterpret { async fn execute( &self, inventory: &Inventory, diff --git a/harmony/src/modules/monitoring/scrape_target/mod.rs b/harmony/src/modules/monitoring/scrape_target/mod.rs index a270cfbd..fe31ce9d 100644 --- a/harmony/src/modules/monitoring/scrape_target/mod.rs +++ b/harmony/src/modules/monitoring/scrape_target/mod.rs @@ -1,4 +1,2 @@ -use std::collections::BTreeMap; - pub mod prometheus_node_exporter; pub mod server; diff --git a/harmony/src/modules/monitoring/scrape_target/prometheus_node_exporter.rs b/harmony/src/modules/monitoring/scrape_target/prometheus_node_exporter.rs index 0011c0b0..e82a9c5b 100644 --- a/harmony/src/modules/monitoring/scrape_target/prometheus_node_exporter.rs +++ b/harmony/src/modules/monitoring/scrape_target/prometheus_node_exporter.rs @@ -1,17 +1,11 @@ -use std::{ - collections::BTreeMap, - net::{IpAddr, Ipv4Addr}, -}; +use std::{collections::BTreeMap, net::IpAddr}; use harmony_macros::ip; use serde::Serialize; use crate::{ interpret::InterpretError, - modules::monitoring::okd::{ - OpenshiftClusterAlertSender, - crd::scrape_target::{ScrapeConfigSpec, StaticConfig}, - }, + modules::monitoring::okd::OpenshiftClusterAlertSender, topology::monitoring::{ExternalScrapeTarget, ScrapeTarget}, }; diff --git a/harmony/src/modules/monitoring/scrape_target/server.rs b/harmony/src/modules/monitoring/scrape_target/server.rs index ffaac18d..b153e27e 100644 --- a/harmony/src/modules/monitoring/scrape_target/server.rs +++ b/harmony/src/modules/monitoring/scrape_target/server.rs @@ -1,17 +1,10 @@ use std::net::IpAddr; -use async_trait::async_trait; -use kube::api::ObjectMeta; use serde::Serialize; use crate::{ - interpret::{InterpretError, Outcome}, - modules::monitoring::{ - kube_prometheus::crd::crd_scrape_config::{ - Params, RelabelConfig, ScrapeConfig, ScrapeConfigSpec, StaticConfig, - }, - prometheus::Prometheus, - }, + interpret::InterpretError, + modules::monitoring::prometheus::Prometheus, topology::monitoring::{ExternalScrapeTarget, ScrapeTarget}, }; -- 2.39.5