NationTech/harmony
6
0
Fork 1
Code Issues 35 Pull Requests 6 Actions Packages Projects Releases 1 Wiki Activity

wip:using types to serialze to json and traits/impl to build alert channel routes for prometheus alert manager

Browse Source
This commit is contained in:
Willem 2025-05-23 16:17:17 -04:00
parent 77eb1228be
commit f94c899bf7
6 changed files with 225 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
harmony/src/modules/monitoring/config.rs
View File

@ -1,6 +1,6 @@
use serde::Serialize;
use super::monitoring_alerting::AlertChannel;
use super::AlertChannel;
#[derive(Debug, Clone, Serialize)]

3
harmony/src/modules/monitoring/discord_alert_manager.rs
View File

@ -4,7 +4,8 @@ use non_blank_string_rs::NonBlankString;
use crate::modules::helm::chart::HelmChartScore;
use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel};
use super::AlertChannel;
use super::config::KubePrometheusConfig;
fn get_discord_alert_manager_score(config: &KubePrometheusConfig) -> Option<HelmChartScore> {
let (url, name) = config.alert_channel.iter().find_map(|channel| {

181
harmony/src/modules/monitoring/kube_prometheus.rs
View File

@ -1,17 +1,22 @@
use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel};
use super::{
AlertChannelGlobalConfig, AlertChannelReceiver, AlertChannelRoute, config::KubePrometheusConfig,
};
use crate::modules::{
helm::chart::HelmChartScore,
monitoring::{AlertChannel, AlertChannelConfig, AlertEndpoint},
};
use log::info;
use non_blank_string_rs::NonBlankString;
use serde::Serialize;
use serde_yaml::{self, Value, to_value};
use std::{collections::HashMap, str::FromStr};
use url::Url;
use crate::modules::helm::chart::HelmChartScore;
pub fn kube_prometheus_helm_chart_score(config: &KubePrometheusConfig) -> HelmChartScore {
//TODO this should be make into a rule with default formatting that can be easily passed as a vec
//to the overrides or something leaving the user to deal with formatting here seems bad
let default_rules = config.default_rules.to_string();
let windows_monitoring = config.windows_monitoring.to_string();
let alert_manager = config.alert_manager.to_string();
let grafana = config.grafana.to_string();
let kubernetes_service_monitors = config.kubernetes_service_monitors.to_string();
let kubernetes_api_server = config.kubernetes_api_server.to_string();
@ -145,71 +150,53 @@ prometheus:
"#,
);
let alertmanager_config = alert_manager_yaml_builder(&config);
values.push_str(&alertmanager_config);
let alertmanager_config = build_alert_manager_config(&config);
fn alert_manager_yaml_builder(config: &KubePrometheusConfig) -> String {
let mut receivers = String::new();
let mut routes = String::new();
let mut global_configs = String::new();
let alert_manager = config.alert_manager;
for alert_channel in &config.alert_channel {
match alert_channel {
AlertChannel::Discord { name, .. } => {
let (receiver, route) = discord_alert_receiver_builder(name);
info!("discord receiver: {} \nroute: {}", receiver, route);
receivers.push_str(&receiver);
routes.push_str(&route);
}
AlertChannel::Slack {
slack_channel,
webhook_url,
} => {
let (receiver, route) = slack_alert_receiver_builder(slack_channel);
info!("slack receiver: {} \nroute: {}", receiver, route);
receivers.push_str(&receiver);
routes.push_str(&route);
let global_config = format!(
r#"
global:
slack_api_url: {webhook_url}"#
);
fn build_alert_manager_config(config: &KubePrometheusConfig) -> AlertManagerConfig {
let mut receivers = Vec::new();
let mut routes = Vec::new();
let mut global_configs = None;
global_configs.push_str(&global_config);
}
AlertChannel::MSTeams { connector, .. } => {
let (receiver, route) = msteams_alert_receiver_builder(connector);
info!("msteams receiver: {} \nroute: {}", receiver, route);
receivers.push_str(&receiver);
routes.push_str(&route);
}
AlertChannel::Smpt { .. } => todo!(),
let alert_channel_configs: Vec<AlertChannelConfig> = config
.alert_channel
.iter()
.map(|s| s.build_alert_receiver())
.collect();
for alert_channel_config in alert_channel_configs {
receivers.push(alert_channel_config.receiver);
routes.push(alert_channel_config.route);
if let Some(global) = alert_channel_config.global_config {
global_configs = Some(global);
}
}
info!("after alert receiver: {}", receivers);
info!("after alert routes: {}", routes);
let alertmanager_config = format!(
r#"
alertmanager:
enabled: {alert_manager}
config: {global_configs}
route:
group_by: ['job']
group_wait: 30s
group_interval: 5m
repeat_interval: 12h
routes:
{routes}
receivers:
- name: 'null'
{receivers}"#
);
info!("after alert receiver: {:#?}", receivers);
info!("after alert routes: {:#?}", routes);
info!("alert manager config: {}", alertmanager_config);
let alertmanager_config = AlertManagerConfig {
global: global_configs,
route: AlertManagerRoute {
group_by: vec!["job".to_string()],
group_wait: "30s".to_string(),
group_interval: "5m".to_string(),
repeat_interval: "12h".to_string(),
routes,
},
receivers,
};
info!("alert manager config: {:?}", alertmanager_config);
alertmanager_config
}
let alert_manager_values = AlertManagerValues{ alertmanager: AlertManager{ enabled: true, config: alertmanager_config } };
let yaml_config =
serde_yaml::to_string(&alert_manager_values).expect("Failed to serialize YAML");
values.push_str(&yaml_config);
info!("{}", values);
HelmChartScore {
namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()),
release_name: NonBlankString::from_str("kube-prometheus").unwrap(),
@ -226,61 +213,29 @@ alertmanager:
}
}
fn discord_alert_receiver_builder(release_name: &String) -> (String, String) {
let discord_receiver_name = format!("Discord-{}", release_name);
let receiver = format!(
r#"
- name: '{discord_receiver_name}'
webhook_configs:
- url: 'http://{release_name}-alertmanager-discord:9094'
send_resolved: true"#,
);
let route = format!(
r#"
- receiver: '{discord_receiver_name}'
matchers:
- alertname!=Watchdog
continue: true"#,
);
(receiver, route)
#[derive(Debug, Serialize)]
struct AlertManagerConfig {
global: Option<AlertChannelGlobalConfig>,
route: AlertManagerRoute,
receivers: Vec<AlertChannelReceiver>,
}
fn slack_alert_receiver_builder(slack_channel: &String) -> (String, String) {
let slack_receiver_name = format!("Slack-{}", slack_channel);
let receiver = format!(
r#"
- name: '{slack_receiver_name}'
slack_configs:
- channel: '{slack_channel}'
send_resolved: true
title: '{{{{ .CommonAnnotations.title }}}}'
text: '{{{{ .CommonAnnotations.description }}}}'"#,
);
let route = format!(
r#"
- receiver: '{slack_receiver_name}'
matchers:
- alertname!=Watchdog
continue: true"#,
);
(receiver, route)
#[derive(Debug, Serialize)]
struct AlertManagerRoute {
group_by: Vec<String>,
group_wait: String,
group_interval: String,
repeat_interval: String,
routes: Vec<AlertChannelRoute>,
}
fn msteams_alert_receiver_builder(connector: &String) -> (String, String) {
let msteams_receiver_name = format!("MSTeams-{}", connector);
let receiver = format!(
r#"
- name: '{msteams_receiver_name}'
webhook_configs:
- send_resolved: true
url: 'http://prometheus-msteams-prometheus-msteams.monitoring.svc.cluster.local:2000/alertmanager'"#,
);
let route = format!(
r#"
- receiver: '{msteams_receiver_name}'
matchers:
- alertname!=Watchdog
continue: true"#,
);
(receiver, route)
#[derive(Debug, Serialize)]
struct AlertManagerValues {
alertmanager: AlertManager,
}
#[derive(Debug, Serialize)]
struct AlertManager {
enabled: bool,
config: AlertManagerConfig,
}

149
harmony/src/modules/monitoring/mod.rs
View File

@ -1,5 +1,148 @@
mod prometheus_msteams;
use email_address::EmailAddress;
use serde::{Deserialize, Serialize};
use url::Url;
mod config;
mod discord_alert_manager;
mod kube_prometheus;
pub mod monitoring_alerting;
mod discord_alert_manager;
mod config;
mod prometheus_msteams;
#[derive(Debug)]
struct AlertChannelConfig {
receiver: AlertChannelReceiver,
route: AlertChannelRoute,
global_config: Option<AlertChannelGlobalConfig>,
}
#[derive(Debug, Serialize, Deserialize)]
struct AlertChannelReceiver {
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub slack_configs: Option<Vec<SlackConfig>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub webhook_configs: Option<Vec<WebhookConfig>>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct SlackConfig {
pub channel: String,
send_resolved: bool,
title: String,
text: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct WebhookConfig {
pub url: Url,
send_resolved: bool,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct AlertChannelRoute {
pub receiver: String,
pub matchers: Vec<String>,
#[serde(default)]
pub continue_: bool,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct AlertChannelGlobalConfig {
#[serde(skip_serializing_if = "Option::is_none")]
pub slack_api_url: Option<Url>,
}
#[derive(Debug, Clone, Serialize)]
pub enum AlertChannel {
Discord {
name: String,
webhook_url: Url,
},
Slack {
slack_channel: String,
webhook_url: Url,
},
MSTeams {
connector: String,
webhook_url: Url,
},
//TODO test and implement in helm chart
//currently does not work
Smpt {
email_address: EmailAddress,
service_name: String,
},
}
trait AlertEndpoint {
//fn register_webhook(&self, webhook_url: Url);
fn build_alert_receiver(&self) -> AlertChannelConfig;
}
impl AlertEndpoint for AlertChannel {
fn build_alert_receiver(&self) -> AlertChannelConfig {
match self {
AlertChannel::Discord { name, .. } => AlertChannelConfig {
receiver: AlertChannelReceiver {
name: format!("Discord-{name}"),
slack_configs: None,
webhook_configs: Some(vec![WebhookConfig {
url: url::Url::parse("http://{name}-alertmanager-discord:9094")
.expect("invalid url"),
send_resolved: true,
}]),
},
route: AlertChannelRoute {
receiver: format!("Discord-{name}"),
matchers: vec!["alertname!=Watchdog".to_string()],
continue_: true,
},
global_config: None,
},
AlertChannel::Slack {
slack_channel,
webhook_url,
} => AlertChannelConfig {
receiver: AlertChannelReceiver {
name: format!("Slack-{slack_channel}"),
slack_configs: Some(vec![SlackConfig {
channel: slack_channel.clone(),
send_resolved: true,
title: "{{ .CommonAnnotations.title }}".to_string(),
text: "{{ .CommonAnnotations.description }}".to_string(),
}]),
webhook_configs: None,
},
route: AlertChannelRoute {
receiver: format!("Slack-{slack_channel}"),
matchers: vec!["alertname!=Watchdog".to_string()],
continue_: true,
},
global_config: Some(AlertChannelGlobalConfig {
slack_api_url: Some(webhook_url.clone()),
}),
},
AlertChannel::MSTeams {
connector, ..
} => AlertChannelConfig{
receiver: AlertChannelReceiver{
name: format!("MSTeams-{connector}"),
slack_configs: None,
webhook_configs: Some(vec![WebhookConfig{
url: url::Url::parse("http://prometheus-msteams-prometheus-msteams.monitoring.svc.cluster.local:2000/alertmanager").expect("invalid url"),
send_resolved: true,}])
},
route: AlertChannelRoute{
receiver: format!("MSTeams-{connector}"),
matchers: vec!["alertname!=Watchdog".to_string()],
continue_: true,
},
global_config: None, },
AlertChannel::Smpt {
email_address,
service_name,
} => todo!(),
}
}
}

31
harmony/src/modules/monitoring/monitoring_alerting.rs
View File

@ -15,30 +15,10 @@ use crate::{
use super::{
config::KubePrometheusConfig, discord_alert_manager::discord_alert_manager_score,
kube_prometheus::kube_prometheus_helm_chart_score, prometheus_msteams::prometheus_msteams_score,
kube_prometheus::kube_prometheus_helm_chart_score,
prometheus_msteams::prometheus_msteams_score, AlertChannel,
};
#[derive(Debug, Clone, Serialize)]
pub enum AlertChannel {
Discord {
name: String,
webhook_url: Url,
},
Slack {
slack_channel: String,
webhook_url: Url,
},
MSTeams {
connector: String,
webhook_url: Url,
},
//TODO test and implement in helm chart
//currently does not work
Smpt {
email_address: EmailAddress,
service_name: String,
},
}
#[derive(Debug, Clone, Serialize)]
pub struct MonitoringAlertingStackScore {
@ -114,7 +94,10 @@ impl MonitoringAlertingStackInterpret {
"No extra configs for slack alerting".to_string(),
)),
AlertChannel::MSTeams { .. } => {
prometheus_msteams_score(config).create_interpret().execute(inventory, topology).await
prometheus_msteams_score(config)
.create_interpret()
.execute(inventory, topology)
.await
}
AlertChannel::Smpt { .. } => {
todo!()
@ -123,7 +106,7 @@ impl MonitoringAlertingStackInterpret {
outcomes.push(outcome);
}
for result in outcomes {
result?;
result?;
}
Ok(Outcome::success("All alert channels deployed".to_string()))

2
harmony/src/modules/monitoring/prometheus_msteams.rs
View File

@ -4,7 +4,7 @@ use non_blank_string_rs::NonBlankString;
use crate::modules::helm::chart::HelmChartScore;
use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel};
use super::{config::KubePrometheusConfig, AlertChannel};
fn build_prometheus_msteams_score(config: &KubePrometheusConfig) -> Option<HelmChartScore> {
let (url, name) = config.alert_channel.iter().find_map(|channel| {