From f94c899bf70571407616d6396a6f3a6efd3802a0 Mon Sep 17 00:00:00 2001 From: Willem Date: Fri, 23 May 2025 16:17:17 -0400 Subject: [PATCH] wip:using types to serialze to json and traits/impl to build alert channel routes for prometheus alert manager --- harmony/src/modules/monitoring/config.rs | 2 +- .../monitoring/discord_alert_manager.rs | 3 +- .../src/modules/monitoring/kube_prometheus.rs | 181 +++++++----------- harmony/src/modules/monitoring/mod.rs | 149 +++++++++++++- .../modules/monitoring/monitoring_alerting.rs | 31 +-- .../modules/monitoring/prometheus_msteams.rs | 2 +- 6 files changed, 225 insertions(+), 143 deletions(-) diff --git a/harmony/src/modules/monitoring/config.rs b/harmony/src/modules/monitoring/config.rs index c06377c..56435de 100644 --- a/harmony/src/modules/monitoring/config.rs +++ b/harmony/src/modules/monitoring/config.rs @@ -1,6 +1,6 @@ use serde::Serialize; -use super::monitoring_alerting::AlertChannel; +use super::AlertChannel; #[derive(Debug, Clone, Serialize)] diff --git a/harmony/src/modules/monitoring/discord_alert_manager.rs b/harmony/src/modules/monitoring/discord_alert_manager.rs index 5eaffa0..d17b077 100644 --- a/harmony/src/modules/monitoring/discord_alert_manager.rs +++ b/harmony/src/modules/monitoring/discord_alert_manager.rs @@ -4,7 +4,8 @@ use non_blank_string_rs::NonBlankString; use crate::modules::helm::chart::HelmChartScore; -use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel}; +use super::AlertChannel; +use super::config::KubePrometheusConfig; fn get_discord_alert_manager_score(config: &KubePrometheusConfig) -> Option { let (url, name) = config.alert_channel.iter().find_map(|channel| { diff --git a/harmony/src/modules/monitoring/kube_prometheus.rs b/harmony/src/modules/monitoring/kube_prometheus.rs index c82a782..06ce054 100644 --- a/harmony/src/modules/monitoring/kube_prometheus.rs +++ b/harmony/src/modules/monitoring/kube_prometheus.rs @@ -1,17 +1,22 @@ -use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel}; +use super::{ + AlertChannelGlobalConfig, AlertChannelReceiver, AlertChannelRoute, config::KubePrometheusConfig, +}; +use crate::modules::{ + helm::chart::HelmChartScore, + monitoring::{AlertChannel, AlertChannelConfig, AlertEndpoint}, +}; use log::info; use non_blank_string_rs::NonBlankString; +use serde::Serialize; +use serde_yaml::{self, Value, to_value}; use std::{collections::HashMap, str::FromStr}; use url::Url; -use crate::modules::helm::chart::HelmChartScore; - pub fn kube_prometheus_helm_chart_score(config: &KubePrometheusConfig) -> HelmChartScore { //TODO this should be make into a rule with default formatting that can be easily passed as a vec //to the overrides or something leaving the user to deal with formatting here seems bad let default_rules = config.default_rules.to_string(); let windows_monitoring = config.windows_monitoring.to_string(); - let alert_manager = config.alert_manager.to_string(); let grafana = config.grafana.to_string(); let kubernetes_service_monitors = config.kubernetes_service_monitors.to_string(); let kubernetes_api_server = config.kubernetes_api_server.to_string(); @@ -145,71 +150,53 @@ prometheus: "#, ); - let alertmanager_config = alert_manager_yaml_builder(&config); - values.push_str(&alertmanager_config); + let alertmanager_config = build_alert_manager_config(&config); - fn alert_manager_yaml_builder(config: &KubePrometheusConfig) -> String { - let mut receivers = String::new(); - let mut routes = String::new(); - let mut global_configs = String::new(); - let alert_manager = config.alert_manager; - for alert_channel in &config.alert_channel { - match alert_channel { - AlertChannel::Discord { name, .. } => { - let (receiver, route) = discord_alert_receiver_builder(name); - info!("discord receiver: {} \nroute: {}", receiver, route); - receivers.push_str(&receiver); - routes.push_str(&route); - } - AlertChannel::Slack { - slack_channel, - webhook_url, - } => { - let (receiver, route) = slack_alert_receiver_builder(slack_channel); - info!("slack receiver: {} \nroute: {}", receiver, route); - receivers.push_str(&receiver); - routes.push_str(&route); - let global_config = format!( - r#" - global: - slack_api_url: {webhook_url}"# - ); + fn build_alert_manager_config(config: &KubePrometheusConfig) -> AlertManagerConfig { + let mut receivers = Vec::new(); + let mut routes = Vec::new(); + let mut global_configs = None; - global_configs.push_str(&global_config); - } - AlertChannel::MSTeams { connector, .. } => { - let (receiver, route) = msteams_alert_receiver_builder(connector); - info!("msteams receiver: {} \nroute: {}", receiver, route); - receivers.push_str(&receiver); - routes.push_str(&route); - } - AlertChannel::Smpt { .. } => todo!(), + let alert_channel_configs: Vec = config + .alert_channel + .iter() + .map(|s| s.build_alert_receiver()) + .collect(); + + for alert_channel_config in alert_channel_configs { + receivers.push(alert_channel_config.receiver); + routes.push(alert_channel_config.route); + if let Some(global) = alert_channel_config.global_config { + global_configs = Some(global); } } - info!("after alert receiver: {}", receivers); - info!("after alert routes: {}", routes); - let alertmanager_config = format!( - r#" -alertmanager: - enabled: {alert_manager} - config: {global_configs} - route: - group_by: ['job'] - group_wait: 30s - group_interval: 5m - repeat_interval: 12h - routes: -{routes} - receivers: - - name: 'null' -{receivers}"# - ); + info!("after alert receiver: {:#?}", receivers); + info!("after alert routes: {:#?}", routes); - info!("alert manager config: {}", alertmanager_config); + let alertmanager_config = AlertManagerConfig { + global: global_configs, + route: AlertManagerRoute { + group_by: vec!["job".to_string()], + group_wait: "30s".to_string(), + group_interval: "5m".to_string(), + repeat_interval: "12h".to_string(), + routes, + }, + receivers, + }; + + info!("alert manager config: {:?}", alertmanager_config); alertmanager_config } + let alert_manager_values = AlertManagerValues{ alertmanager: AlertManager{ enabled: true, config: alertmanager_config } }; + let yaml_config = + serde_yaml::to_string(&alert_manager_values).expect("Failed to serialize YAML"); + + values.push_str(&yaml_config); + + info!("{}", values); HelmChartScore { namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()), release_name: NonBlankString::from_str("kube-prometheus").unwrap(), @@ -226,61 +213,29 @@ alertmanager: } } -fn discord_alert_receiver_builder(release_name: &String) -> (String, String) { - let discord_receiver_name = format!("Discord-{}", release_name); - let receiver = format!( - r#" - - name: '{discord_receiver_name}' - webhook_configs: - - url: 'http://{release_name}-alertmanager-discord:9094' - send_resolved: true"#, - ); - let route = format!( - r#" - - receiver: '{discord_receiver_name}' - matchers: - - alertname!=Watchdog - continue: true"#, - ); - (receiver, route) +#[derive(Debug, Serialize)] +struct AlertManagerConfig { + global: Option, + route: AlertManagerRoute, + receivers: Vec, } -fn slack_alert_receiver_builder(slack_channel: &String) -> (String, String) { - let slack_receiver_name = format!("Slack-{}", slack_channel); - let receiver = format!( - r#" - - name: '{slack_receiver_name}' - slack_configs: - - channel: '{slack_channel}' - send_resolved: true - title: '{{{{ .CommonAnnotations.title }}}}' - text: '{{{{ .CommonAnnotations.description }}}}'"#, - ); - let route = format!( - r#" - - receiver: '{slack_receiver_name}' - matchers: - - alertname!=Watchdog - continue: true"#, - ); - (receiver, route) +#[derive(Debug, Serialize)] +struct AlertManagerRoute { + group_by: Vec, + group_wait: String, + group_interval: String, + repeat_interval: String, + routes: Vec, } -fn msteams_alert_receiver_builder(connector: &String) -> (String, String) { - let msteams_receiver_name = format!("MSTeams-{}", connector); - let receiver = format!( - r#" - - name: '{msteams_receiver_name}' - webhook_configs: - - send_resolved: true - url: 'http://prometheus-msteams-prometheus-msteams.monitoring.svc.cluster.local:2000/alertmanager'"#, - ); - let route = format!( - r#" - - receiver: '{msteams_receiver_name}' - matchers: - - alertname!=Watchdog - continue: true"#, - ); - (receiver, route) +#[derive(Debug, Serialize)] +struct AlertManagerValues { + alertmanager: AlertManager, +} + +#[derive(Debug, Serialize)] +struct AlertManager { + enabled: bool, + config: AlertManagerConfig, } diff --git a/harmony/src/modules/monitoring/mod.rs b/harmony/src/modules/monitoring/mod.rs index 4c20ca7..95f95d2 100644 --- a/harmony/src/modules/monitoring/mod.rs +++ b/harmony/src/modules/monitoring/mod.rs @@ -1,5 +1,148 @@ -mod prometheus_msteams; +use email_address::EmailAddress; +use serde::{Deserialize, Serialize}; +use url::Url; + +mod config; +mod discord_alert_manager; mod kube_prometheus; pub mod monitoring_alerting; -mod discord_alert_manager; -mod config; +mod prometheus_msteams; + +#[derive(Debug)] +struct AlertChannelConfig { + receiver: AlertChannelReceiver, + route: AlertChannelRoute, + global_config: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +struct AlertChannelReceiver { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub slack_configs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub webhook_configs: Option>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct SlackConfig { + pub channel: String, + send_resolved: bool, + title: String, + text: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct WebhookConfig { + pub url: Url, + send_resolved: bool, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct AlertChannelRoute { + pub receiver: String, + pub matchers: Vec, + #[serde(default)] + pub continue_: bool, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct AlertChannelGlobalConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub slack_api_url: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub enum AlertChannel { + Discord { + name: String, + webhook_url: Url, + }, + Slack { + slack_channel: String, + webhook_url: Url, + }, + MSTeams { + connector: String, + webhook_url: Url, + }, + //TODO test and implement in helm chart + //currently does not work + Smpt { + email_address: EmailAddress, + service_name: String, + }, +} + +trait AlertEndpoint { + //fn register_webhook(&self, webhook_url: Url); + fn build_alert_receiver(&self) -> AlertChannelConfig; +} + +impl AlertEndpoint for AlertChannel { + fn build_alert_receiver(&self) -> AlertChannelConfig { + match self { + AlertChannel::Discord { name, .. } => AlertChannelConfig { + receiver: AlertChannelReceiver { + name: format!("Discord-{name}"), + slack_configs: None, + webhook_configs: Some(vec![WebhookConfig { + url: url::Url::parse("http://{name}-alertmanager-discord:9094") + .expect("invalid url"), + send_resolved: true, + }]), + }, + route: AlertChannelRoute { + receiver: format!("Discord-{name}"), + matchers: vec!["alertname!=Watchdog".to_string()], + continue_: true, + }, + global_config: None, + }, + AlertChannel::Slack { + slack_channel, + webhook_url, + } => AlertChannelConfig { + receiver: AlertChannelReceiver { + name: format!("Slack-{slack_channel}"), + slack_configs: Some(vec![SlackConfig { + channel: slack_channel.clone(), + send_resolved: true, + title: "{{ .CommonAnnotations.title }}".to_string(), + text: "{{ .CommonAnnotations.description }}".to_string(), + }]), + webhook_configs: None, + }, + route: AlertChannelRoute { + receiver: format!("Slack-{slack_channel}"), + matchers: vec!["alertname!=Watchdog".to_string()], + continue_: true, + }, + global_config: Some(AlertChannelGlobalConfig { + slack_api_url: Some(webhook_url.clone()), + }), + }, + AlertChannel::MSTeams { + connector, .. + } => AlertChannelConfig{ + receiver: AlertChannelReceiver{ + name: format!("MSTeams-{connector}"), + slack_configs: None, + webhook_configs: Some(vec![WebhookConfig{ + url: url::Url::parse("http://prometheus-msteams-prometheus-msteams.monitoring.svc.cluster.local:2000/alertmanager").expect("invalid url"), + send_resolved: true,}]) + }, + route: AlertChannelRoute{ + receiver: format!("MSTeams-{connector}"), + matchers: vec!["alertname!=Watchdog".to_string()], + continue_: true, + }, + global_config: None, }, + + AlertChannel::Smpt { + email_address, + service_name, + } => todo!(), + } + } +} diff --git a/harmony/src/modules/monitoring/monitoring_alerting.rs b/harmony/src/modules/monitoring/monitoring_alerting.rs index 7f1e358..89bd1fa 100644 --- a/harmony/src/modules/monitoring/monitoring_alerting.rs +++ b/harmony/src/modules/monitoring/monitoring_alerting.rs @@ -15,30 +15,10 @@ use crate::{ use super::{ config::KubePrometheusConfig, discord_alert_manager::discord_alert_manager_score, - kube_prometheus::kube_prometheus_helm_chart_score, prometheus_msteams::prometheus_msteams_score, + kube_prometheus::kube_prometheus_helm_chart_score, + prometheus_msteams::prometheus_msteams_score, AlertChannel, }; -#[derive(Debug, Clone, Serialize)] -pub enum AlertChannel { - Discord { - name: String, - webhook_url: Url, - }, - Slack { - slack_channel: String, - webhook_url: Url, - }, - MSTeams { - connector: String, - webhook_url: Url, - }, - //TODO test and implement in helm chart - //currently does not work - Smpt { - email_address: EmailAddress, - service_name: String, - }, -} #[derive(Debug, Clone, Serialize)] pub struct MonitoringAlertingStackScore { @@ -114,7 +94,10 @@ impl MonitoringAlertingStackInterpret { "No extra configs for slack alerting".to_string(), )), AlertChannel::MSTeams { .. } => { - prometheus_msteams_score(config).create_interpret().execute(inventory, topology).await + prometheus_msteams_score(config) + .create_interpret() + .execute(inventory, topology) + .await } AlertChannel::Smpt { .. } => { todo!() @@ -123,7 +106,7 @@ impl MonitoringAlertingStackInterpret { outcomes.push(outcome); } for result in outcomes { - result?; + result?; } Ok(Outcome::success("All alert channels deployed".to_string())) diff --git a/harmony/src/modules/monitoring/prometheus_msteams.rs b/harmony/src/modules/monitoring/prometheus_msteams.rs index 6f023ad..941262e 100644 --- a/harmony/src/modules/monitoring/prometheus_msteams.rs +++ b/harmony/src/modules/monitoring/prometheus_msteams.rs @@ -4,7 +4,7 @@ use non_blank_string_rs::NonBlankString; use crate::modules::helm::chart::HelmChartScore; -use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel}; +use super::{config::KubePrometheusConfig, AlertChannel}; fn build_prometheus_msteams_score(config: &KubePrometheusConfig) -> Option { let (url, name) = config.alert_channel.iter().find_map(|channel| {