wip:using types to serialze to json and traits/impl to build alert channel routes for prometheus alert manager
This commit is contained in:
parent
77eb1228be
commit
f94c899bf7
@ -1,6 +1,6 @@
|
|||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
|
||||||
use super::monitoring_alerting::AlertChannel;
|
use super::AlertChannel;
|
||||||
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
|||||||
@ -4,7 +4,8 @@ use non_blank_string_rs::NonBlankString;
|
|||||||
|
|
||||||
use crate::modules::helm::chart::HelmChartScore;
|
use crate::modules::helm::chart::HelmChartScore;
|
||||||
|
|
||||||
use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel};
|
use super::AlertChannel;
|
||||||
|
use super::config::KubePrometheusConfig;
|
||||||
|
|
||||||
fn get_discord_alert_manager_score(config: &KubePrometheusConfig) -> Option<HelmChartScore> {
|
fn get_discord_alert_manager_score(config: &KubePrometheusConfig) -> Option<HelmChartScore> {
|
||||||
let (url, name) = config.alert_channel.iter().find_map(|channel| {
|
let (url, name) = config.alert_channel.iter().find_map(|channel| {
|
||||||
|
|||||||
@ -1,17 +1,22 @@
|
|||||||
use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel};
|
use super::{
|
||||||
|
AlertChannelGlobalConfig, AlertChannelReceiver, AlertChannelRoute, config::KubePrometheusConfig,
|
||||||
|
};
|
||||||
|
use crate::modules::{
|
||||||
|
helm::chart::HelmChartScore,
|
||||||
|
monitoring::{AlertChannel, AlertChannelConfig, AlertEndpoint},
|
||||||
|
};
|
||||||
use log::info;
|
use log::info;
|
||||||
use non_blank_string_rs::NonBlankString;
|
use non_blank_string_rs::NonBlankString;
|
||||||
|
use serde::Serialize;
|
||||||
|
use serde_yaml::{self, Value, to_value};
|
||||||
use std::{collections::HashMap, str::FromStr};
|
use std::{collections::HashMap, str::FromStr};
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use crate::modules::helm::chart::HelmChartScore;
|
|
||||||
|
|
||||||
pub fn kube_prometheus_helm_chart_score(config: &KubePrometheusConfig) -> HelmChartScore {
|
pub fn kube_prometheus_helm_chart_score(config: &KubePrometheusConfig) -> HelmChartScore {
|
||||||
//TODO this should be make into a rule with default formatting that can be easily passed as a vec
|
//TODO this should be make into a rule with default formatting that can be easily passed as a vec
|
||||||
//to the overrides or something leaving the user to deal with formatting here seems bad
|
//to the overrides or something leaving the user to deal with formatting here seems bad
|
||||||
let default_rules = config.default_rules.to_string();
|
let default_rules = config.default_rules.to_string();
|
||||||
let windows_monitoring = config.windows_monitoring.to_string();
|
let windows_monitoring = config.windows_monitoring.to_string();
|
||||||
let alert_manager = config.alert_manager.to_string();
|
|
||||||
let grafana = config.grafana.to_string();
|
let grafana = config.grafana.to_string();
|
||||||
let kubernetes_service_monitors = config.kubernetes_service_monitors.to_string();
|
let kubernetes_service_monitors = config.kubernetes_service_monitors.to_string();
|
||||||
let kubernetes_api_server = config.kubernetes_api_server.to_string();
|
let kubernetes_api_server = config.kubernetes_api_server.to_string();
|
||||||
@ -145,71 +150,53 @@ prometheus:
|
|||||||
"#,
|
"#,
|
||||||
);
|
);
|
||||||
|
|
||||||
let alertmanager_config = alert_manager_yaml_builder(&config);
|
let alertmanager_config = build_alert_manager_config(&config);
|
||||||
values.push_str(&alertmanager_config);
|
|
||||||
|
|
||||||
fn alert_manager_yaml_builder(config: &KubePrometheusConfig) -> String {
|
fn build_alert_manager_config(config: &KubePrometheusConfig) -> AlertManagerConfig {
|
||||||
let mut receivers = String::new();
|
let mut receivers = Vec::new();
|
||||||
let mut routes = String::new();
|
let mut routes = Vec::new();
|
||||||
let mut global_configs = String::new();
|
let mut global_configs = None;
|
||||||
let alert_manager = config.alert_manager;
|
|
||||||
for alert_channel in &config.alert_channel {
|
|
||||||
match alert_channel {
|
|
||||||
AlertChannel::Discord { name, .. } => {
|
|
||||||
let (receiver, route) = discord_alert_receiver_builder(name);
|
|
||||||
info!("discord receiver: {} \nroute: {}", receiver, route);
|
|
||||||
receivers.push_str(&receiver);
|
|
||||||
routes.push_str(&route);
|
|
||||||
}
|
|
||||||
AlertChannel::Slack {
|
|
||||||
slack_channel,
|
|
||||||
webhook_url,
|
|
||||||
} => {
|
|
||||||
let (receiver, route) = slack_alert_receiver_builder(slack_channel);
|
|
||||||
info!("slack receiver: {} \nroute: {}", receiver, route);
|
|
||||||
receivers.push_str(&receiver);
|
|
||||||
routes.push_str(&route);
|
|
||||||
let global_config = format!(
|
|
||||||
r#"
|
|
||||||
global:
|
|
||||||
slack_api_url: {webhook_url}"#
|
|
||||||
);
|
|
||||||
|
|
||||||
global_configs.push_str(&global_config);
|
let alert_channel_configs: Vec<AlertChannelConfig> = config
|
||||||
}
|
.alert_channel
|
||||||
AlertChannel::MSTeams { connector, .. } => {
|
.iter()
|
||||||
let (receiver, route) = msteams_alert_receiver_builder(connector);
|
.map(|s| s.build_alert_receiver())
|
||||||
info!("msteams receiver: {} \nroute: {}", receiver, route);
|
.collect();
|
||||||
receivers.push_str(&receiver);
|
|
||||||
routes.push_str(&route);
|
for alert_channel_config in alert_channel_configs {
|
||||||
}
|
receivers.push(alert_channel_config.receiver);
|
||||||
AlertChannel::Smpt { .. } => todo!(),
|
routes.push(alert_channel_config.route);
|
||||||
|
if let Some(global) = alert_channel_config.global_config {
|
||||||
|
global_configs = Some(global);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
info!("after alert receiver: {}", receivers);
|
|
||||||
info!("after alert routes: {}", routes);
|
|
||||||
|
|
||||||
let alertmanager_config = format!(
|
info!("after alert receiver: {:#?}", receivers);
|
||||||
r#"
|
info!("after alert routes: {:#?}", routes);
|
||||||
alertmanager:
|
|
||||||
enabled: {alert_manager}
|
|
||||||
config: {global_configs}
|
|
||||||
route:
|
|
||||||
group_by: ['job']
|
|
||||||
group_wait: 30s
|
|
||||||
group_interval: 5m
|
|
||||||
repeat_interval: 12h
|
|
||||||
routes:
|
|
||||||
{routes}
|
|
||||||
receivers:
|
|
||||||
- name: 'null'
|
|
||||||
{receivers}"#
|
|
||||||
);
|
|
||||||
|
|
||||||
info!("alert manager config: {}", alertmanager_config);
|
let alertmanager_config = AlertManagerConfig {
|
||||||
|
global: global_configs,
|
||||||
|
route: AlertManagerRoute {
|
||||||
|
group_by: vec!["job".to_string()],
|
||||||
|
group_wait: "30s".to_string(),
|
||||||
|
group_interval: "5m".to_string(),
|
||||||
|
repeat_interval: "12h".to_string(),
|
||||||
|
routes,
|
||||||
|
},
|
||||||
|
receivers,
|
||||||
|
};
|
||||||
|
|
||||||
|
info!("alert manager config: {:?}", alertmanager_config);
|
||||||
alertmanager_config
|
alertmanager_config
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let alert_manager_values = AlertManagerValues{ alertmanager: AlertManager{ enabled: true, config: alertmanager_config } };
|
||||||
|
let yaml_config =
|
||||||
|
serde_yaml::to_string(&alert_manager_values).expect("Failed to serialize YAML");
|
||||||
|
|
||||||
|
values.push_str(&yaml_config);
|
||||||
|
|
||||||
|
info!("{}", values);
|
||||||
HelmChartScore {
|
HelmChartScore {
|
||||||
namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()),
|
namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()),
|
||||||
release_name: NonBlankString::from_str("kube-prometheus").unwrap(),
|
release_name: NonBlankString::from_str("kube-prometheus").unwrap(),
|
||||||
@ -226,61 +213,29 @@ alertmanager:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn discord_alert_receiver_builder(release_name: &String) -> (String, String) {
|
#[derive(Debug, Serialize)]
|
||||||
let discord_receiver_name = format!("Discord-{}", release_name);
|
struct AlertManagerConfig {
|
||||||
let receiver = format!(
|
global: Option<AlertChannelGlobalConfig>,
|
||||||
r#"
|
route: AlertManagerRoute,
|
||||||
- name: '{discord_receiver_name}'
|
receivers: Vec<AlertChannelReceiver>,
|
||||||
webhook_configs:
|
|
||||||
- url: 'http://{release_name}-alertmanager-discord:9094'
|
|
||||||
send_resolved: true"#,
|
|
||||||
);
|
|
||||||
let route = format!(
|
|
||||||
r#"
|
|
||||||
- receiver: '{discord_receiver_name}'
|
|
||||||
matchers:
|
|
||||||
- alertname!=Watchdog
|
|
||||||
continue: true"#,
|
|
||||||
);
|
|
||||||
(receiver, route)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn slack_alert_receiver_builder(slack_channel: &String) -> (String, String) {
|
#[derive(Debug, Serialize)]
|
||||||
let slack_receiver_name = format!("Slack-{}", slack_channel);
|
struct AlertManagerRoute {
|
||||||
let receiver = format!(
|
group_by: Vec<String>,
|
||||||
r#"
|
group_wait: String,
|
||||||
- name: '{slack_receiver_name}'
|
group_interval: String,
|
||||||
slack_configs:
|
repeat_interval: String,
|
||||||
- channel: '{slack_channel}'
|
routes: Vec<AlertChannelRoute>,
|
||||||
send_resolved: true
|
|
||||||
title: '{{{{ .CommonAnnotations.title }}}}'
|
|
||||||
text: '{{{{ .CommonAnnotations.description }}}}'"#,
|
|
||||||
);
|
|
||||||
let route = format!(
|
|
||||||
r#"
|
|
||||||
- receiver: '{slack_receiver_name}'
|
|
||||||
matchers:
|
|
||||||
- alertname!=Watchdog
|
|
||||||
continue: true"#,
|
|
||||||
);
|
|
||||||
(receiver, route)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn msteams_alert_receiver_builder(connector: &String) -> (String, String) {
|
#[derive(Debug, Serialize)]
|
||||||
let msteams_receiver_name = format!("MSTeams-{}", connector);
|
struct AlertManagerValues {
|
||||||
let receiver = format!(
|
alertmanager: AlertManager,
|
||||||
r#"
|
}
|
||||||
- name: '{msteams_receiver_name}'
|
|
||||||
webhook_configs:
|
#[derive(Debug, Serialize)]
|
||||||
- send_resolved: true
|
struct AlertManager {
|
||||||
url: 'http://prometheus-msteams-prometheus-msteams.monitoring.svc.cluster.local:2000/alertmanager'"#,
|
enabled: bool,
|
||||||
);
|
config: AlertManagerConfig,
|
||||||
let route = format!(
|
|
||||||
r#"
|
|
||||||
- receiver: '{msteams_receiver_name}'
|
|
||||||
matchers:
|
|
||||||
- alertname!=Watchdog
|
|
||||||
continue: true"#,
|
|
||||||
);
|
|
||||||
(receiver, route)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,5 +1,148 @@
|
|||||||
mod prometheus_msteams;
|
use email_address::EmailAddress;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
|
mod config;
|
||||||
|
mod discord_alert_manager;
|
||||||
mod kube_prometheus;
|
mod kube_prometheus;
|
||||||
pub mod monitoring_alerting;
|
pub mod monitoring_alerting;
|
||||||
mod discord_alert_manager;
|
mod prometheus_msteams;
|
||||||
mod config;
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct AlertChannelConfig {
|
||||||
|
receiver: AlertChannelReceiver,
|
||||||
|
route: AlertChannelRoute,
|
||||||
|
global_config: Option<AlertChannelGlobalConfig>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
struct AlertChannelReceiver {
|
||||||
|
pub name: String,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub slack_configs: Option<Vec<SlackConfig>>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub webhook_configs: Option<Vec<WebhookConfig>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct SlackConfig {
|
||||||
|
pub channel: String,
|
||||||
|
send_resolved: bool,
|
||||||
|
title: String,
|
||||||
|
text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct WebhookConfig {
|
||||||
|
pub url: Url,
|
||||||
|
send_resolved: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct AlertChannelRoute {
|
||||||
|
pub receiver: String,
|
||||||
|
pub matchers: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub continue_: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct AlertChannelGlobalConfig {
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub slack_api_url: Option<Url>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub enum AlertChannel {
|
||||||
|
Discord {
|
||||||
|
name: String,
|
||||||
|
webhook_url: Url,
|
||||||
|
},
|
||||||
|
Slack {
|
||||||
|
slack_channel: String,
|
||||||
|
webhook_url: Url,
|
||||||
|
},
|
||||||
|
MSTeams {
|
||||||
|
connector: String,
|
||||||
|
webhook_url: Url,
|
||||||
|
},
|
||||||
|
//TODO test and implement in helm chart
|
||||||
|
//currently does not work
|
||||||
|
Smpt {
|
||||||
|
email_address: EmailAddress,
|
||||||
|
service_name: String,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
trait AlertEndpoint {
|
||||||
|
//fn register_webhook(&self, webhook_url: Url);
|
||||||
|
fn build_alert_receiver(&self) -> AlertChannelConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AlertEndpoint for AlertChannel {
|
||||||
|
fn build_alert_receiver(&self) -> AlertChannelConfig {
|
||||||
|
match self {
|
||||||
|
AlertChannel::Discord { name, .. } => AlertChannelConfig {
|
||||||
|
receiver: AlertChannelReceiver {
|
||||||
|
name: format!("Discord-{name}"),
|
||||||
|
slack_configs: None,
|
||||||
|
webhook_configs: Some(vec![WebhookConfig {
|
||||||
|
url: url::Url::parse("http://{name}-alertmanager-discord:9094")
|
||||||
|
.expect("invalid url"),
|
||||||
|
send_resolved: true,
|
||||||
|
}]),
|
||||||
|
},
|
||||||
|
route: AlertChannelRoute {
|
||||||
|
receiver: format!("Discord-{name}"),
|
||||||
|
matchers: vec!["alertname!=Watchdog".to_string()],
|
||||||
|
continue_: true,
|
||||||
|
},
|
||||||
|
global_config: None,
|
||||||
|
},
|
||||||
|
AlertChannel::Slack {
|
||||||
|
slack_channel,
|
||||||
|
webhook_url,
|
||||||
|
} => AlertChannelConfig {
|
||||||
|
receiver: AlertChannelReceiver {
|
||||||
|
name: format!("Slack-{slack_channel}"),
|
||||||
|
slack_configs: Some(vec![SlackConfig {
|
||||||
|
channel: slack_channel.clone(),
|
||||||
|
send_resolved: true,
|
||||||
|
title: "{{ .CommonAnnotations.title }}".to_string(),
|
||||||
|
text: "{{ .CommonAnnotations.description }}".to_string(),
|
||||||
|
}]),
|
||||||
|
webhook_configs: None,
|
||||||
|
},
|
||||||
|
route: AlertChannelRoute {
|
||||||
|
receiver: format!("Slack-{slack_channel}"),
|
||||||
|
matchers: vec!["alertname!=Watchdog".to_string()],
|
||||||
|
continue_: true,
|
||||||
|
},
|
||||||
|
global_config: Some(AlertChannelGlobalConfig {
|
||||||
|
slack_api_url: Some(webhook_url.clone()),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
AlertChannel::MSTeams {
|
||||||
|
connector, ..
|
||||||
|
} => AlertChannelConfig{
|
||||||
|
receiver: AlertChannelReceiver{
|
||||||
|
name: format!("MSTeams-{connector}"),
|
||||||
|
slack_configs: None,
|
||||||
|
webhook_configs: Some(vec![WebhookConfig{
|
||||||
|
url: url::Url::parse("http://prometheus-msteams-prometheus-msteams.monitoring.svc.cluster.local:2000/alertmanager").expect("invalid url"),
|
||||||
|
send_resolved: true,}])
|
||||||
|
},
|
||||||
|
route: AlertChannelRoute{
|
||||||
|
receiver: format!("MSTeams-{connector}"),
|
||||||
|
matchers: vec!["alertname!=Watchdog".to_string()],
|
||||||
|
continue_: true,
|
||||||
|
},
|
||||||
|
global_config: None, },
|
||||||
|
|
||||||
|
AlertChannel::Smpt {
|
||||||
|
email_address,
|
||||||
|
service_name,
|
||||||
|
} => todo!(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -15,30 +15,10 @@ use crate::{
|
|||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
config::KubePrometheusConfig, discord_alert_manager::discord_alert_manager_score,
|
config::KubePrometheusConfig, discord_alert_manager::discord_alert_manager_score,
|
||||||
kube_prometheus::kube_prometheus_helm_chart_score, prometheus_msteams::prometheus_msteams_score,
|
kube_prometheus::kube_prometheus_helm_chart_score,
|
||||||
|
prometheus_msteams::prometheus_msteams_score, AlertChannel,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
|
||||||
pub enum AlertChannel {
|
|
||||||
Discord {
|
|
||||||
name: String,
|
|
||||||
webhook_url: Url,
|
|
||||||
},
|
|
||||||
Slack {
|
|
||||||
slack_channel: String,
|
|
||||||
webhook_url: Url,
|
|
||||||
},
|
|
||||||
MSTeams {
|
|
||||||
connector: String,
|
|
||||||
webhook_url: Url,
|
|
||||||
},
|
|
||||||
//TODO test and implement in helm chart
|
|
||||||
//currently does not work
|
|
||||||
Smpt {
|
|
||||||
email_address: EmailAddress,
|
|
||||||
service_name: String,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub struct MonitoringAlertingStackScore {
|
pub struct MonitoringAlertingStackScore {
|
||||||
@ -114,7 +94,10 @@ impl MonitoringAlertingStackInterpret {
|
|||||||
"No extra configs for slack alerting".to_string(),
|
"No extra configs for slack alerting".to_string(),
|
||||||
)),
|
)),
|
||||||
AlertChannel::MSTeams { .. } => {
|
AlertChannel::MSTeams { .. } => {
|
||||||
prometheus_msteams_score(config).create_interpret().execute(inventory, topology).await
|
prometheus_msteams_score(config)
|
||||||
|
.create_interpret()
|
||||||
|
.execute(inventory, topology)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
AlertChannel::Smpt { .. } => {
|
AlertChannel::Smpt { .. } => {
|
||||||
todo!()
|
todo!()
|
||||||
@ -123,7 +106,7 @@ impl MonitoringAlertingStackInterpret {
|
|||||||
outcomes.push(outcome);
|
outcomes.push(outcome);
|
||||||
}
|
}
|
||||||
for result in outcomes {
|
for result in outcomes {
|
||||||
result?;
|
result?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Outcome::success("All alert channels deployed".to_string()))
|
Ok(Outcome::success("All alert channels deployed".to_string()))
|
||||||
|
|||||||
@ -4,7 +4,7 @@ use non_blank_string_rs::NonBlankString;
|
|||||||
|
|
||||||
use crate::modules::helm::chart::HelmChartScore;
|
use crate::modules::helm::chart::HelmChartScore;
|
||||||
|
|
||||||
use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel};
|
use super::{config::KubePrometheusConfig, AlertChannel};
|
||||||
|
|
||||||
fn build_prometheus_msteams_score(config: &KubePrometheusConfig) -> Option<HelmChartScore> {
|
fn build_prometheus_msteams_score(config: &KubePrometheusConfig) -> Option<HelmChartScore> {
|
||||||
let (url, name) = config.alert_channel.iter().find_map(|channel| {
|
let (url, name) = config.alert_channel.iter().find_map(|channel| {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user