From f702ecd8c9e938ce19ab9690ef856e18b530e8a8 Mon Sep 17 00:00:00 2001 From: Willem Date: Fri, 4 Jul 2025 16:13:41 -0400 Subject: [PATCH] fix: deploys a lighter weight prometheus and grafana which is limited to their respective namespaces --- examples/rust/src/main.rs | 16 +- .../application/features/monitoring.rs | 36 +++- .../alert_channel/discord_alert_channel.rs | 28 ++- .../alert_channel/webhook_receiver.rs | 27 ++- .../alert_rule/prometheus_alert_rule.rs | 38 +++- .../k8s_application_monitoring_score.rs | 41 ++++ .../monitoring/application_monitoring/mod.rs | 3 + .../monitoring/grafana/helm/helm_grafana.rs | 27 +++ .../modules/monitoring/grafana/helm/mod.rs | 2 + harmony/src/modules/monitoring/grafana/mod.rs | 1 + .../helm/kube_prometheus_helm_chart.rs | 5 +- .../helm_prometheus_alert_score.rs | 8 +- .../monitoring/kube_prometheus/prometheus.rs | 22 +- .../monitoring/kube_prometheus/types.rs | 11 +- harmony/src/modules/monitoring/mod.rs | 3 + .../modules/monitoring/prometheus/helm/mod.rs | 2 + .../prometheus/helm/prometheus_helm.rs | 47 +++++ .../src/modules/monitoring/prometheus/mod.rs | 3 + .../monitoring/prometheus/prometheus.rs | 190 ++++++++++++++++++ .../prometheus/prometheus_config.rs | 34 ++++ 20 files changed, 506 insertions(+), 38 deletions(-) create mode 100644 harmony/src/modules/monitoring/application_monitoring/k8s_application_monitoring_score.rs create mode 100644 harmony/src/modules/monitoring/application_monitoring/mod.rs create mode 100644 harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs create mode 100644 harmony/src/modules/monitoring/grafana/helm/mod.rs create mode 100644 harmony/src/modules/monitoring/grafana/mod.rs create mode 100644 harmony/src/modules/monitoring/prometheus/helm/mod.rs create mode 100644 harmony/src/modules/monitoring/prometheus/helm/prometheus_helm.rs create mode 100644 harmony/src/modules/monitoring/prometheus/mod.rs create mode 100644 harmony/src/modules/monitoring/prometheus/prometheus.rs create mode 100644 harmony/src/modules/monitoring/prometheus/prometheus_config.rs diff --git a/examples/rust/src/main.rs b/examples/rust/src/main.rs index 813485d..49f90d7 100644 --- a/examples/rust/src/main.rs +++ b/examples/rust/src/main.rs @@ -4,7 +4,7 @@ use harmony::{ inventory::Inventory, maestro::Maestro, modules::application::{ - ApplicationScore, RustWebFramework, RustWebapp, features::ContinuousDelivery, + features::{ContinuousDelivery, Monitoring}, ApplicationScore, RustWebFramework, RustWebapp }, topology::{K8sAnywhereTopology, Url}, }; @@ -19,11 +19,19 @@ async fn main() { framework: Some(RustWebFramework::Leptos), }); + // let app = ApplicationScore { + // features: vec![ + // Box::new(ContinuousDelivery { + // application: application.clone(), + // }), + // Box::new(Monitoring {}), + // // TODO add monitoring, backups, multisite ha, etc + // ], + // application, + // }; let app = ApplicationScore { features: vec![ - Box::new(ContinuousDelivery { - application: application.clone(), - }), + Box::new(Monitoring {}), // TODO add monitoring, backups, multisite ha, etc ], application, diff --git a/harmony/src/modules/application/features/monitoring.rs b/harmony/src/modules/application/features/monitoring.rs index 3bb7bae..0cd61bd 100644 --- a/harmony/src/modules/application/features/monitoring.rs +++ b/harmony/src/modules/application/features/monitoring.rs @@ -1,19 +1,45 @@ + use async_trait::async_trait; use log::info; use crate::{ - modules::application::{Application, ApplicationFeature}, - topology::{HelmCommand, Topology}, + inventory::Inventory, + modules::{ + application::{Application, ApplicationFeature}, + monitoring::{application_monitoring::k8s_application_monitoring_score::ApplicationPrometheusMonitoringScore, kube_prometheus::{ + helm_prometheus_alert_score::HelmPrometheusAlertingScore, + types::{NamespaceSelector, ServiceMonitor}, + }}, + }, + score::Score, + topology::{tenant::TenantManager, HelmCommand, Topology}, }; #[derive(Debug, Default, Clone)] pub struct Monitoring {} + #[async_trait] -impl ApplicationFeature for Monitoring { - async fn ensure_installed(&self, _topology: &T) -> Result<(), String> { +impl ApplicationFeature for Monitoring { + async fn ensure_installed(&self, topology: &T) -> Result<(), String> { info!("Ensuring monitoring is available for application"); - todo!("create and execute k8s prometheus score, depends on Will's work") + let mut service_monitor = ServiceMonitor::default(); + service_monitor.namespace_selector = Some(NamespaceSelector { + any: true, + match_names: vec![], + }); + let alerting_score = ApplicationPrometheusMonitoringScore { + receivers: vec![], + rules: vec![], + service_monitors: vec![service_monitor], + }; + + alerting_score + .create_interpret() + .execute(&Inventory::empty(), topology) + .await + .unwrap(); + Ok(()) } fn name(&self) -> String { "Monitoring".to_string() diff --git a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs index fb85a98..cb3d0db 100644 --- a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs +++ b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs @@ -4,11 +4,11 @@ use serde_yaml::{Mapping, Value}; use crate::{ interpret::{InterpretError, Outcome}, - modules::monitoring::kube_prometheus::{ - prometheus::{Prometheus, PrometheusReceiver}, + modules::monitoring::{kube_prometheus::{ + prometheus::{KubePrometheus, KubePrometheusReceiver}, types::{AlertChannelConfig, AlertManagerChannelConfig}, - }, - topology::{Url, oberservability::monitoring::AlertReceiver}, + }, prometheus::prometheus::{Prometheus, PrometheusReceiver}}, + topology::{oberservability::monitoring::AlertReceiver, Url}, }; #[derive(Debug, Clone, Serialize)] @@ -37,6 +37,26 @@ impl PrometheusReceiver for DiscordWebhook { } } +#[async_trait] +impl AlertReceiver for DiscordWebhook { + async fn install(&self, sender: &KubePrometheus) -> Result { + sender.install_receiver(self).await + } + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } +} + +#[async_trait] +impl KubePrometheusReceiver for DiscordWebhook { + fn name(&self) -> String { + self.name.clone() + } + async fn configure_receiver(&self) -> AlertManagerChannelConfig { + self.get_config().await + } +} + #[async_trait] impl AlertChannelConfig for DiscordWebhook { async fn get_config(&self) -> AlertManagerChannelConfig { diff --git a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs index 8f608d0..bb2415f 100644 --- a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs +++ b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs @@ -4,11 +4,11 @@ use serde_yaml::{Mapping, Value}; use crate::{ interpret::{InterpretError, Outcome}, - modules::monitoring::kube_prometheus::{ - prometheus::{Prometheus, PrometheusReceiver}, + modules::monitoring::{kube_prometheus::{ + prometheus::{KubePrometheus, KubePrometheusReceiver}, types::{AlertChannelConfig, AlertManagerChannelConfig}, - }, - topology::{Url, oberservability::monitoring::AlertReceiver}, + }, prometheus::prometheus::{Prometheus, PrometheusReceiver}}, + topology::{oberservability::monitoring::AlertReceiver, Url}, }; #[derive(Debug, Clone, Serialize)] @@ -36,6 +36,25 @@ impl PrometheusReceiver for WebhookReceiver { self.get_config().await } } +#[async_trait] +impl AlertReceiver for WebhookReceiver { + async fn install(&self, sender: &KubePrometheus) -> Result { + sender.install_receiver(self).await + } + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } +} + +#[async_trait] +impl KubePrometheusReceiver for WebhookReceiver { + fn name(&self) -> String { + self.name.clone() + } + async fn configure_receiver(&self) -> AlertManagerChannelConfig { + self.get_config().await + } +} #[async_trait] impl AlertChannelConfig for WebhookReceiver { diff --git a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs index ccb63cc..c0627b8 100644 --- a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs +++ b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs @@ -5,13 +5,26 @@ use serde::Serialize; use crate::{ interpret::{InterpretError, Outcome}, - modules::monitoring::kube_prometheus::{ - prometheus::{Prometheus, PrometheusRule}, - types::{AlertGroup, AlertManagerAdditionalPromRules}, + modules::monitoring::{ + kube_prometheus::{ + prometheus::{KubePrometheus, KubePrometheusRule}, + types::{AlertGroup, AlertManagerAdditionalPromRules}, + }, + prometheus::prometheus::{Prometheus, PrometheusRule}, }, topology::oberservability::monitoring::AlertRule, }; +#[async_trait] +impl AlertRule for AlertManagerRuleGroup { + async fn install(&self, sender: &KubePrometheus) -> Result { + sender.install_rule(&self).await + } + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } +} + #[async_trait] impl AlertRule for AlertManagerRuleGroup { async fn install(&self, sender: &Prometheus) -> Result { @@ -41,6 +54,25 @@ impl PrometheusRule for AlertManagerRuleGroup { } } } +#[async_trait] +impl KubePrometheusRule for AlertManagerRuleGroup { + fn name(&self) -> String { + self.name.clone() + } + async fn configure_rule(&self) -> AlertManagerAdditionalPromRules { + let mut additional_prom_rules = BTreeMap::new(); + + additional_prom_rules.insert( + self.name.clone(), + AlertGroup { + groups: vec![self.clone()], + }, + ); + AlertManagerAdditionalPromRules { + rules: additional_prom_rules, + } + } +} impl AlertManagerRuleGroup { pub fn new(name: &str, rules: Vec) -> AlertManagerRuleGroup { diff --git a/harmony/src/modules/monitoring/application_monitoring/k8s_application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/k8s_application_monitoring_score.rs new file mode 100644 index 0000000..29e2893 --- /dev/null +++ b/harmony/src/modules/monitoring/application_monitoring/k8s_application_monitoring_score.rs @@ -0,0 +1,41 @@ +use std::sync::{Arc, Mutex}; + +use serde::Serialize; + +use crate::{ + modules::monitoring::{ + kube_prometheus::types::ServiceMonitor, + prometheus::{prometheus::Prometheus, prometheus_config::PrometheusConfig}, + }, + score::Score, + topology::{ + HelmCommand, Topology, + oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret}, + tenant::TenantManager, + }, +}; + +#[derive(Clone, Debug, Serialize)] +pub struct ApplicationPrometheusMonitoringScore { + pub receivers: Vec>>, + pub rules: Vec>>, + pub service_monitors: Vec, +} + +impl Score for ApplicationPrometheusMonitoringScore { + fn create_interpret(&self) -> Box> { + let config = Arc::new(Mutex::new(PrometheusConfig::new())); + config + .try_lock() + .expect("couldn't lock config") + .additional_service_monitors = self.service_monitors.clone(); + Box::new(AlertingInterpret { + sender: Prometheus::new(), + receivers: self.receivers.clone(), + rules: self.rules.clone(), + }) + } + fn name(&self) -> String { + "ApplicationPrometheusMonitoringScore".to_string() + } +} diff --git a/harmony/src/modules/monitoring/application_monitoring/mod.rs b/harmony/src/modules/monitoring/application_monitoring/mod.rs new file mode 100644 index 0000000..f6154da --- /dev/null +++ b/harmony/src/modules/monitoring/application_monitoring/mod.rs @@ -0,0 +1,3 @@ + +pub mod k8s_application_monitoring_score; + diff --git a/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs b/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs new file mode 100644 index 0000000..8df2b54 --- /dev/null +++ b/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs @@ -0,0 +1,27 @@ +use non_blank_string_rs::NonBlankString; +use std::str::FromStr; + +use crate::modules::helm::chart::HelmChartScore; + +pub fn grafana_helm_chart_score(ns: &str) -> HelmChartScore { + let values = format!( + r#" +rbac: + namespaced: true +sidecar: + dashboards: + enabled: true + "#); + + HelmChartScore { + namespace: Some(NonBlankString::from_str(ns).unwrap()), + release_name: NonBlankString::from_str("grafana").unwrap(), + chart_name: NonBlankString::from_str("oci://ghcr.io/grafana/helm-charts/grafana").unwrap(), + chart_version: None, + values_overrides: None, + values_yaml: Some(values.to_string()), + create_namespace: true, + install_only: true, + repository: None, + } +} diff --git a/harmony/src/modules/monitoring/grafana/helm/mod.rs b/harmony/src/modules/monitoring/grafana/helm/mod.rs new file mode 100644 index 0000000..6a93df6 --- /dev/null +++ b/harmony/src/modules/monitoring/grafana/helm/mod.rs @@ -0,0 +1,2 @@ +pub mod helm_grafana; + diff --git a/harmony/src/modules/monitoring/grafana/mod.rs b/harmony/src/modules/monitoring/grafana/mod.rs new file mode 100644 index 0000000..c821bcb --- /dev/null +++ b/harmony/src/modules/monitoring/grafana/mod.rs @@ -0,0 +1 @@ +pub mod helm; diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs b/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs index 3fd773f..14d9f5f 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs @@ -68,6 +68,9 @@ pub fn kube_prometheus_helm_chart_score( let mut values = format!( r#" +global: + rbac: + create: false prometheus: enabled: {prometheus} prometheusSpec: @@ -242,7 +245,7 @@ prometheus-node-exporter: cpu: 200m memory: 250Mi prometheusOperator: - enabled: {prometheus_operator} + enabled: false resources: requests: cpu: 100m diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs b/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs index 6203c43..0ab5f1b 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs @@ -2,7 +2,7 @@ use std::sync::{Arc, Mutex}; use serde::Serialize; -use super::{helm::config::KubePrometheusConfig, prometheus::Prometheus}; +use super::{helm::config::KubePrometheusConfig, prometheus::KubePrometheus}; use crate::{ modules::monitoring::kube_prometheus::types::ServiceMonitor, score::Score, @@ -15,8 +15,8 @@ use crate::{ #[derive(Clone, Debug, Serialize)] pub struct HelmPrometheusAlertingScore { - pub receivers: Vec>>, - pub rules: Vec>>, + pub receivers: Vec>>, + pub rules: Vec>>, pub service_monitors: Vec, } @@ -28,7 +28,7 @@ impl Score for HelmPrometheusAlert .expect("couldn't lock config") .additional_service_monitors = self.service_monitors.clone(); Box::new(AlertingInterpret { - sender: Prometheus::new(), + sender: KubePrometheus::new(), receivers: self.receivers.clone(), rules: self.rules.clone(), }) diff --git a/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs b/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs index fdf2057..f9d8944 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs @@ -27,14 +27,14 @@ use super::{ }; #[async_trait] -impl AlertSender for Prometheus { +impl AlertSender for KubePrometheus { fn name(&self) -> String { "HelmKubePrometheus".to_string() } } #[async_trait] -impl Installable for Prometheus { +impl Installable for KubePrometheus { async fn configure(&self, _inventory: &Inventory, topology: &T) -> Result<(), InterpretError> { self.configure_with_topology(topology).await; Ok(()) @@ -51,11 +51,11 @@ impl Installable for Prometheus { } #[derive(Debug)] -pub struct Prometheus { +pub struct KubePrometheus { pub config: Arc>, } -impl Prometheus { +impl KubePrometheus { pub fn new() -> Self { Self { config: Arc::new(Mutex::new(KubePrometheusConfig::new())), @@ -75,7 +75,7 @@ impl Prometheus { pub async fn install_receiver( &self, - prometheus_receiver: &dyn PrometheusReceiver, + prometheus_receiver: &dyn KubePrometheusReceiver, ) -> Result { let prom_receiver = prometheus_receiver.configure_receiver().await; debug!( @@ -120,12 +120,12 @@ impl Prometheus { } #[async_trait] -pub trait PrometheusReceiver: Send + Sync + std::fmt::Debug { +pub trait KubePrometheusReceiver: Send + Sync + std::fmt::Debug { fn name(&self) -> String; async fn configure_receiver(&self) -> AlertManagerChannelConfig; } -impl Serialize for Box> { +impl Serialize for Box> { fn serialize(&self, _serializer: S) -> Result where S: serde::Serializer, @@ -134,19 +134,19 @@ impl Serialize for Box> { } } -impl Clone for Box> { +impl Clone for Box> { fn clone(&self) -> Self { self.clone_box() } } #[async_trait] -pub trait PrometheusRule: Send + Sync + std::fmt::Debug { +pub trait KubePrometheusRule: Send + Sync + std::fmt::Debug { fn name(&self) -> String; async fn configure_rule(&self) -> AlertManagerAdditionalPromRules; } -impl Serialize for Box> { +impl Serialize for Box> { fn serialize(&self, _serializer: S) -> Result where S: serde::Serializer, @@ -155,7 +155,7 @@ impl Serialize for Box> { } } -impl Clone for Box> { +impl Clone for Box> { fn clone(&self) -> Self { self.clone_box() } diff --git a/harmony/src/modules/monitoring/kube_prometheus/types.rs b/harmony/src/modules/monitoring/kube_prometheus/types.rs index 2aa8574..33bfcc3 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/types.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/types.rs @@ -212,7 +212,7 @@ pub struct ServiceMonitor { pub name: String, // # Additional labels to set used for the ServiceMonitorSelector. Together with standard labels from the chart - pub additional_labels: Option, + pub additional_labels: Option>, // # Service label for use in assembling a job name of the form