refactor: used AlertEndPoint trait and impl, built custom types for prometheus alert manager
This commit is contained in:
parent
f94c899bf7
commit
b5c6e1c99d
@ -1,16 +1,15 @@
|
||||
use super::{
|
||||
AlertChannelGlobalConfig, AlertChannelReceiver, AlertChannelRoute, config::KubePrometheusConfig,
|
||||
use crate::modules::monitoring::kube_prometheus::types::{
|
||||
AlertChannelReceiver, AlertChannelRoute, AlertManager, AlertManagerConfig,
|
||||
AlertManagerRoute, AlertManagerValues,
|
||||
};
|
||||
use crate::modules::{
|
||||
helm::chart::HelmChartScore,
|
||||
monitoring::{AlertChannel, AlertChannelConfig, AlertEndpoint},
|
||||
monitoring::{config::KubePrometheusConfig, kube_prometheus::traits::AlertEndpoint},
|
||||
};
|
||||
use log::info;
|
||||
use non_blank_string_rs::NonBlankString;
|
||||
use serde::Serialize;
|
||||
use serde_yaml::{self, Value, to_value};
|
||||
use std::{collections::HashMap, str::FromStr};
|
||||
use url::Url;
|
||||
use serde_yaml::{self};
|
||||
use std::str::FromStr;
|
||||
|
||||
pub fn kube_prometheus_helm_chart_score(config: &KubePrometheusConfig) -> HelmChartScore {
|
||||
//TODO this should be make into a rule with default formatting that can be easily passed as a vec
|
||||
@ -147,35 +146,45 @@ prometheusOperator:
|
||||
enabled: {prometheus_operator}
|
||||
prometheus:
|
||||
enabled: {prometheus}
|
||||
prometheusSpec:
|
||||
maximumStartupDurationSeconds: 240
|
||||
"#,
|
||||
);
|
||||
|
||||
let alertmanager_config = build_alert_manager_config(&config);
|
||||
let alert_manager_config = build_alert_manager_config(&config);
|
||||
|
||||
fn build_alert_manager_config(config: &KubePrometheusConfig) -> AlertManagerConfig {
|
||||
let mut receivers = Vec::new();
|
||||
let mut routes = Vec::new();
|
||||
let mut global_configs = None;
|
||||
fn build_alert_manager_config(config: &KubePrometheusConfig) -> AlertManagerValues {
|
||||
let mut global_config = None;
|
||||
|
||||
let alert_channel_configs: Vec<AlertChannelConfig> = config
|
||||
let (mut receivers, mut routes): (Vec<_>, Vec<_>) = config
|
||||
.alert_channel
|
||||
.iter()
|
||||
.map(|s| s.build_alert_receiver())
|
||||
.collect();
|
||||
.map(|chan| {
|
||||
if let Some(global) = chan.global_config {
|
||||
global_config = Some(global);
|
||||
}
|
||||
(chan.receiver, chan.route)
|
||||
})
|
||||
.unzip();
|
||||
|
||||
for alert_channel_config in alert_channel_configs {
|
||||
receivers.push(alert_channel_config.receiver);
|
||||
routes.push(alert_channel_config.route);
|
||||
if let Some(global) = alert_channel_config.global_config {
|
||||
global_configs = Some(global);
|
||||
}
|
||||
}
|
||||
receivers.push(AlertChannelReceiver {
|
||||
name: "null".to_string(),
|
||||
slack_configs: None,
|
||||
webhook_configs: None,
|
||||
});
|
||||
|
||||
routes.push(AlertChannelRoute {
|
||||
receiver: "null".to_string(),
|
||||
matchers: vec!["alertname=Watchdog".to_string()],
|
||||
r#continue: false,
|
||||
});
|
||||
|
||||
info!("after alert receiver: {:#?}", receivers);
|
||||
info!("after alert routes: {:#?}", routes);
|
||||
|
||||
let alertmanager_config = AlertManagerConfig {
|
||||
global: global_configs,
|
||||
let config = AlertManagerConfig {
|
||||
global: global_config,
|
||||
route: AlertManagerRoute {
|
||||
group_by: vec!["job".to_string()],
|
||||
group_wait: "30s".to_string(),
|
||||
@ -186,13 +195,18 @@ prometheus:
|
||||
receivers,
|
||||
};
|
||||
|
||||
info!("alert manager config: {:?}", alertmanager_config);
|
||||
alertmanager_config
|
||||
info!("alert manager config: {:?}", config);
|
||||
|
||||
AlertManagerValues {
|
||||
alertmanager: AlertManager {
|
||||
enabled: true,
|
||||
config,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
let alert_manager_values = AlertManagerValues{ alertmanager: AlertManager{ enabled: true, config: alertmanager_config } };
|
||||
let yaml_config =
|
||||
serde_yaml::to_string(&alert_manager_values).expect("Failed to serialize YAML");
|
||||
serde_yaml::to_string(&alert_manager_config).expect("Failed to serialize YAML");
|
||||
|
||||
values.push_str(&yaml_config);
|
||||
|
||||
@ -212,30 +226,3 @@ prometheus:
|
||||
repository: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AlertManagerConfig {
|
||||
global: Option<AlertChannelGlobalConfig>,
|
||||
route: AlertManagerRoute,
|
||||
receivers: Vec<AlertChannelReceiver>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AlertManagerRoute {
|
||||
group_by: Vec<String>,
|
||||
group_wait: String,
|
||||
group_interval: String,
|
||||
repeat_interval: String,
|
||||
routes: Vec<AlertChannelRoute>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AlertManagerValues {
|
||||
alertmanager: AlertManager,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AlertManager {
|
||||
enabled: bool,
|
||||
config: AlertManagerConfig,
|
||||
}
|
||||
4
harmony/src/modules/monitoring/kube_prometheus/mod.rs
Normal file
4
harmony/src/modules/monitoring/kube_prometheus/mod.rs
Normal file
@ -0,0 +1,4 @@
|
||||
pub mod traits;
|
||||
pub mod kube_prometheus;
|
||||
pub mod types;
|
||||
|
||||
92
harmony/src/modules/monitoring/kube_prometheus/traits.rs
Normal file
92
harmony/src/modules/monitoring/kube_prometheus/traits.rs
Normal file
@ -0,0 +1,92 @@
|
||||
use crate::modules::monitoring::AlertChannel;
|
||||
|
||||
use super::types::{AlertChannelConfig, AlertChannelGlobalConfig, AlertChannelReceiver, AlertChannelRoute, SlackConfig, WebhookConfig};
|
||||
|
||||
|
||||
pub trait AlertEndpoint {
|
||||
//fn register_webhook(&self, webhook_url: Url);
|
||||
fn build_alert_receiver(&self) -> AlertChannelConfig;
|
||||
}
|
||||
|
||||
impl AlertEndpoint for AlertChannel {
|
||||
fn build_alert_receiver(&self) -> AlertChannelConfig {
|
||||
match self {
|
||||
AlertChannel::Discord { name, .. } => AlertChannelConfig {
|
||||
receiver: AlertChannelReceiver {
|
||||
name: format!("Discord-{name}"),
|
||||
slack_configs: None,
|
||||
webhook_configs: Some(vec![WebhookConfig {
|
||||
url: url::Url::parse("http://{name}-alertmanager-discord:9094")
|
||||
.expect("invalid url"),
|
||||
send_resolved: true,
|
||||
}]),
|
||||
},
|
||||
route: AlertChannelRoute {
|
||||
receiver: format!("Discord-{name}"),
|
||||
matchers: vec!["alertname!=Watchdog".to_string()],
|
||||
r#continue: true,
|
||||
},
|
||||
global_config: None,
|
||||
},
|
||||
AlertChannel::Slack {
|
||||
slack_channel,
|
||||
webhook_url,
|
||||
} => AlertChannelConfig {
|
||||
receiver: AlertChannelReceiver {
|
||||
name: format!("Slack-{slack_channel}"),
|
||||
slack_configs: Some(vec![SlackConfig {
|
||||
channel: slack_channel.clone(),
|
||||
send_resolved: true,
|
||||
title: "{{ .CommonAnnotations.title }}".to_string(),
|
||||
text: ">-
|
||||
*Alert:* {{ .CommonLabels.alertname }}
|
||||
*Severity:* {{ .CommonLabels.severity }}
|
||||
*Namespace:* {{ .CommonLabels.namespace }}
|
||||
*Pod:* {{ .CommonLabels.pod }}
|
||||
*ExternalURL:* {{ .ExternalURL }}
|
||||
|
||||
{{ range .Alerts }}
|
||||
*Instance:* {{ .Labels.instance }}
|
||||
*Summary:* {{ .Annotations.summary }}
|
||||
*Description:* {{ .Annotations.description }}
|
||||
*Starts At:* {{ .StartsAt }}
|
||||
*Status:* {{ .Status }}
|
||||
{{ end }}".to_string()
|
||||
}]),
|
||||
webhook_configs: None,
|
||||
},
|
||||
route: AlertChannelRoute {
|
||||
receiver: format!("Slack-{slack_channel}"),
|
||||
matchers: vec!["alertname!=Watchdog".to_string()],
|
||||
r#continue: true,
|
||||
},
|
||||
global_config: Some(AlertChannelGlobalConfig {
|
||||
slack_api_url: Some(webhook_url.clone()),
|
||||
}),
|
||||
},
|
||||
AlertChannel::MSTeams {
|
||||
connector, ..
|
||||
} => AlertChannelConfig{
|
||||
receiver: AlertChannelReceiver{
|
||||
name: format!("MSTeams-{connector}"),
|
||||
slack_configs: None,
|
||||
webhook_configs: Some(vec![WebhookConfig{
|
||||
url: url::Url::parse("http://prometheus-msteams-prometheus-msteams.monitoring.svc.cluster.local:2000/alertmanager").expect("invalid url"),
|
||||
send_resolved: true,}])
|
||||
},
|
||||
route: AlertChannelRoute{
|
||||
receiver: format!("MSTeams-{connector}"),
|
||||
matchers: vec!["alertname!=Watchdog".to_string()],
|
||||
r#continue: true,
|
||||
},
|
||||
global_config: None, },
|
||||
|
||||
AlertChannel::Smpt {
|
||||
email_address,
|
||||
service_name,
|
||||
} => todo!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
73
harmony/src/modules/monitoring/kube_prometheus/types.rs
Normal file
73
harmony/src/modules/monitoring/kube_prometheus/types.rs
Normal file
@ -0,0 +1,73 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use url::Url;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct AlertManagerValues {
|
||||
pub alertmanager: AlertManager,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct AlertManager {
|
||||
pub enabled: bool,
|
||||
pub config: AlertManagerConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct AlertChannelConfig {
|
||||
pub receiver: AlertChannelReceiver,
|
||||
pub route: AlertChannelRoute,
|
||||
pub global_config: Option<AlertChannelGlobalConfig>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct AlertChannelReceiver {
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub slack_configs: Option<Vec<SlackConfig>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub webhook_configs: Option<Vec<WebhookConfig>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct AlertManagerRoute {
|
||||
pub group_by: Vec<String>,
|
||||
pub group_wait: String,
|
||||
pub group_interval: String,
|
||||
pub repeat_interval: String,
|
||||
pub routes: Vec<AlertChannelRoute>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct AlertChannelGlobalConfig {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub slack_api_url: Option<Url>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct SlackConfig {
|
||||
pub channel: String,
|
||||
pub send_resolved: bool,
|
||||
pub title: String,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct WebhookConfig {
|
||||
pub url: Url,
|
||||
pub send_resolved: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct AlertChannelRoute {
|
||||
pub receiver: String,
|
||||
pub matchers: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub r#continue: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct AlertManagerConfig {
|
||||
pub global: Option<AlertChannelGlobalConfig>,
|
||||
pub route: AlertManagerRoute,
|
||||
pub receivers: Vec<AlertChannelReceiver>,
|
||||
}
|
||||
@ -1,57 +1,13 @@
|
||||
use email_address::EmailAddress;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::Serialize;
|
||||
use url::Url;
|
||||
|
||||
mod config;
|
||||
mod discord_alert_manager;
|
||||
mod kube_prometheus;
|
||||
pub mod kube_prometheus;
|
||||
pub mod monitoring_alerting;
|
||||
mod prometheus_msteams;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct AlertChannelConfig {
|
||||
receiver: AlertChannelReceiver,
|
||||
route: AlertChannelRoute,
|
||||
global_config: Option<AlertChannelGlobalConfig>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct AlertChannelReceiver {
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub slack_configs: Option<Vec<SlackConfig>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub webhook_configs: Option<Vec<WebhookConfig>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct SlackConfig {
|
||||
pub channel: String,
|
||||
send_resolved: bool,
|
||||
title: String,
|
||||
text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct WebhookConfig {
|
||||
pub url: Url,
|
||||
send_resolved: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct AlertChannelRoute {
|
||||
pub receiver: String,
|
||||
pub matchers: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub continue_: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct AlertChannelGlobalConfig {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub slack_api_url: Option<Url>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub enum AlertChannel {
|
||||
Discord {
|
||||
@ -74,75 +30,3 @@ pub enum AlertChannel {
|
||||
},
|
||||
}
|
||||
|
||||
trait AlertEndpoint {
|
||||
//fn register_webhook(&self, webhook_url: Url);
|
||||
fn build_alert_receiver(&self) -> AlertChannelConfig;
|
||||
}
|
||||
|
||||
impl AlertEndpoint for AlertChannel {
|
||||
fn build_alert_receiver(&self) -> AlertChannelConfig {
|
||||
match self {
|
||||
AlertChannel::Discord { name, .. } => AlertChannelConfig {
|
||||
receiver: AlertChannelReceiver {
|
||||
name: format!("Discord-{name}"),
|
||||
slack_configs: None,
|
||||
webhook_configs: Some(vec![WebhookConfig {
|
||||
url: url::Url::parse("http://{name}-alertmanager-discord:9094")
|
||||
.expect("invalid url"),
|
||||
send_resolved: true,
|
||||
}]),
|
||||
},
|
||||
route: AlertChannelRoute {
|
||||
receiver: format!("Discord-{name}"),
|
||||
matchers: vec!["alertname!=Watchdog".to_string()],
|
||||
continue_: true,
|
||||
},
|
||||
global_config: None,
|
||||
},
|
||||
AlertChannel::Slack {
|
||||
slack_channel,
|
||||
webhook_url,
|
||||
} => AlertChannelConfig {
|
||||
receiver: AlertChannelReceiver {
|
||||
name: format!("Slack-{slack_channel}"),
|
||||
slack_configs: Some(vec![SlackConfig {
|
||||
channel: slack_channel.clone(),
|
||||
send_resolved: true,
|
||||
title: "{{ .CommonAnnotations.title }}".to_string(),
|
||||
text: "{{ .CommonAnnotations.description }}".to_string(),
|
||||
}]),
|
||||
webhook_configs: None,
|
||||
},
|
||||
route: AlertChannelRoute {
|
||||
receiver: format!("Slack-{slack_channel}"),
|
||||
matchers: vec!["alertname!=Watchdog".to_string()],
|
||||
continue_: true,
|
||||
},
|
||||
global_config: Some(AlertChannelGlobalConfig {
|
||||
slack_api_url: Some(webhook_url.clone()),
|
||||
}),
|
||||
},
|
||||
AlertChannel::MSTeams {
|
||||
connector, ..
|
||||
} => AlertChannelConfig{
|
||||
receiver: AlertChannelReceiver{
|
||||
name: format!("MSTeams-{connector}"),
|
||||
slack_configs: None,
|
||||
webhook_configs: Some(vec![WebhookConfig{
|
||||
url: url::Url::parse("http://prometheus-msteams-prometheus-msteams.monitoring.svc.cluster.local:2000/alertmanager").expect("invalid url"),
|
||||
send_resolved: true,}])
|
||||
},
|
||||
route: AlertChannelRoute{
|
||||
receiver: format!("MSTeams-{connector}"),
|
||||
matchers: vec!["alertname!=Watchdog".to_string()],
|
||||
continue_: true,
|
||||
},
|
||||
global_config: None, },
|
||||
|
||||
AlertChannel::Smpt {
|
||||
email_address,
|
||||
service_name,
|
||||
} => todo!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -14,9 +14,7 @@ use crate::{
|
||||
};
|
||||
|
||||
use super::{
|
||||
config::KubePrometheusConfig, discord_alert_manager::discord_alert_manager_score,
|
||||
kube_prometheus::kube_prometheus_helm_chart_score,
|
||||
prometheus_msteams::prometheus_msteams_score, AlertChannel,
|
||||
config::KubePrometheusConfig, discord_alert_manager::discord_alert_manager_score, kube_prometheus::kube_prometheus::kube_prometheus_helm_chart_score, prometheus_msteams::prometheus_msteams_score, AlertChannel
|
||||
};
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user