refactor: used AlertEndPoint trait and impl, built custom types for prometheus alert manager
This commit is contained in:
parent
f94c899bf7
commit
b5c6e1c99d
@ -1,16 +1,15 @@
|
|||||||
use super::{
|
use crate::modules::monitoring::kube_prometheus::types::{
|
||||||
AlertChannelGlobalConfig, AlertChannelReceiver, AlertChannelRoute, config::KubePrometheusConfig,
|
AlertChannelReceiver, AlertChannelRoute, AlertManager, AlertManagerConfig,
|
||||||
|
AlertManagerRoute, AlertManagerValues,
|
||||||
};
|
};
|
||||||
use crate::modules::{
|
use crate::modules::{
|
||||||
helm::chart::HelmChartScore,
|
helm::chart::HelmChartScore,
|
||||||
monitoring::{AlertChannel, AlertChannelConfig, AlertEndpoint},
|
monitoring::{config::KubePrometheusConfig, kube_prometheus::traits::AlertEndpoint},
|
||||||
};
|
};
|
||||||
use log::info;
|
use log::info;
|
||||||
use non_blank_string_rs::NonBlankString;
|
use non_blank_string_rs::NonBlankString;
|
||||||
use serde::Serialize;
|
use serde_yaml::{self};
|
||||||
use serde_yaml::{self, Value, to_value};
|
use std::str::FromStr;
|
||||||
use std::{collections::HashMap, str::FromStr};
|
|
||||||
use url::Url;
|
|
||||||
|
|
||||||
pub fn kube_prometheus_helm_chart_score(config: &KubePrometheusConfig) -> HelmChartScore {
|
pub fn kube_prometheus_helm_chart_score(config: &KubePrometheusConfig) -> HelmChartScore {
|
||||||
//TODO this should be make into a rule with default formatting that can be easily passed as a vec
|
//TODO this should be make into a rule with default formatting that can be easily passed as a vec
|
||||||
@ -147,35 +146,45 @@ prometheusOperator:
|
|||||||
enabled: {prometheus_operator}
|
enabled: {prometheus_operator}
|
||||||
prometheus:
|
prometheus:
|
||||||
enabled: {prometheus}
|
enabled: {prometheus}
|
||||||
|
prometheusSpec:
|
||||||
|
maximumStartupDurationSeconds: 240
|
||||||
"#,
|
"#,
|
||||||
);
|
);
|
||||||
|
|
||||||
let alertmanager_config = build_alert_manager_config(&config);
|
let alert_manager_config = build_alert_manager_config(&config);
|
||||||
|
|
||||||
fn build_alert_manager_config(config: &KubePrometheusConfig) -> AlertManagerConfig {
|
fn build_alert_manager_config(config: &KubePrometheusConfig) -> AlertManagerValues {
|
||||||
let mut receivers = Vec::new();
|
let mut global_config = None;
|
||||||
let mut routes = Vec::new();
|
|
||||||
let mut global_configs = None;
|
|
||||||
|
|
||||||
let alert_channel_configs: Vec<AlertChannelConfig> = config
|
let (mut receivers, mut routes): (Vec<_>, Vec<_>) = config
|
||||||
.alert_channel
|
.alert_channel
|
||||||
.iter()
|
.iter()
|
||||||
.map(|s| s.build_alert_receiver())
|
.map(|s| s.build_alert_receiver())
|
||||||
.collect();
|
.map(|chan| {
|
||||||
|
if let Some(global) = chan.global_config {
|
||||||
|
global_config = Some(global);
|
||||||
|
}
|
||||||
|
(chan.receiver, chan.route)
|
||||||
|
})
|
||||||
|
.unzip();
|
||||||
|
|
||||||
for alert_channel_config in alert_channel_configs {
|
receivers.push(AlertChannelReceiver {
|
||||||
receivers.push(alert_channel_config.receiver);
|
name: "null".to_string(),
|
||||||
routes.push(alert_channel_config.route);
|
slack_configs: None,
|
||||||
if let Some(global) = alert_channel_config.global_config {
|
webhook_configs: None,
|
||||||
global_configs = Some(global);
|
});
|
||||||
}
|
|
||||||
}
|
routes.push(AlertChannelRoute {
|
||||||
|
receiver: "null".to_string(),
|
||||||
|
matchers: vec!["alertname=Watchdog".to_string()],
|
||||||
|
r#continue: false,
|
||||||
|
});
|
||||||
|
|
||||||
info!("after alert receiver: {:#?}", receivers);
|
info!("after alert receiver: {:#?}", receivers);
|
||||||
info!("after alert routes: {:#?}", routes);
|
info!("after alert routes: {:#?}", routes);
|
||||||
|
|
||||||
let alertmanager_config = AlertManagerConfig {
|
let config = AlertManagerConfig {
|
||||||
global: global_configs,
|
global: global_config,
|
||||||
route: AlertManagerRoute {
|
route: AlertManagerRoute {
|
||||||
group_by: vec!["job".to_string()],
|
group_by: vec!["job".to_string()],
|
||||||
group_wait: "30s".to_string(),
|
group_wait: "30s".to_string(),
|
||||||
@ -186,13 +195,18 @@ prometheus:
|
|||||||
receivers,
|
receivers,
|
||||||
};
|
};
|
||||||
|
|
||||||
info!("alert manager config: {:?}", alertmanager_config);
|
info!("alert manager config: {:?}", config);
|
||||||
alertmanager_config
|
|
||||||
|
AlertManagerValues {
|
||||||
|
alertmanager: AlertManager {
|
||||||
|
enabled: true,
|
||||||
|
config,
|
||||||
|
},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let alert_manager_values = AlertManagerValues{ alertmanager: AlertManager{ enabled: true, config: alertmanager_config } };
|
|
||||||
let yaml_config =
|
let yaml_config =
|
||||||
serde_yaml::to_string(&alert_manager_values).expect("Failed to serialize YAML");
|
serde_yaml::to_string(&alert_manager_config).expect("Failed to serialize YAML");
|
||||||
|
|
||||||
values.push_str(&yaml_config);
|
values.push_str(&yaml_config);
|
||||||
|
|
||||||
@ -212,30 +226,3 @@ prometheus:
|
|||||||
repository: None,
|
repository: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
|
||||||
struct AlertManagerConfig {
|
|
||||||
global: Option<AlertChannelGlobalConfig>,
|
|
||||||
route: AlertManagerRoute,
|
|
||||||
receivers: Vec<AlertChannelReceiver>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
|
||||||
struct AlertManagerRoute {
|
|
||||||
group_by: Vec<String>,
|
|
||||||
group_wait: String,
|
|
||||||
group_interval: String,
|
|
||||||
repeat_interval: String,
|
|
||||||
routes: Vec<AlertChannelRoute>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
|
||||||
struct AlertManagerValues {
|
|
||||||
alertmanager: AlertManager,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
|
||||||
struct AlertManager {
|
|
||||||
enabled: bool,
|
|
||||||
config: AlertManagerConfig,
|
|
||||||
}
|
|
||||||
4
harmony/src/modules/monitoring/kube_prometheus/mod.rs
Normal file
4
harmony/src/modules/monitoring/kube_prometheus/mod.rs
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
pub mod traits;
|
||||||
|
pub mod kube_prometheus;
|
||||||
|
pub mod types;
|
||||||
|
|
||||||
92
harmony/src/modules/monitoring/kube_prometheus/traits.rs
Normal file
92
harmony/src/modules/monitoring/kube_prometheus/traits.rs
Normal file
@ -0,0 +1,92 @@
|
|||||||
|
use crate::modules::monitoring::AlertChannel;
|
||||||
|
|
||||||
|
use super::types::{AlertChannelConfig, AlertChannelGlobalConfig, AlertChannelReceiver, AlertChannelRoute, SlackConfig, WebhookConfig};
|
||||||
|
|
||||||
|
|
||||||
|
pub trait AlertEndpoint {
|
||||||
|
//fn register_webhook(&self, webhook_url: Url);
|
||||||
|
fn build_alert_receiver(&self) -> AlertChannelConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AlertEndpoint for AlertChannel {
|
||||||
|
fn build_alert_receiver(&self) -> AlertChannelConfig {
|
||||||
|
match self {
|
||||||
|
AlertChannel::Discord { name, .. } => AlertChannelConfig {
|
||||||
|
receiver: AlertChannelReceiver {
|
||||||
|
name: format!("Discord-{name}"),
|
||||||
|
slack_configs: None,
|
||||||
|
webhook_configs: Some(vec![WebhookConfig {
|
||||||
|
url: url::Url::parse("http://{name}-alertmanager-discord:9094")
|
||||||
|
.expect("invalid url"),
|
||||||
|
send_resolved: true,
|
||||||
|
}]),
|
||||||
|
},
|
||||||
|
route: AlertChannelRoute {
|
||||||
|
receiver: format!("Discord-{name}"),
|
||||||
|
matchers: vec!["alertname!=Watchdog".to_string()],
|
||||||
|
r#continue: true,
|
||||||
|
},
|
||||||
|
global_config: None,
|
||||||
|
},
|
||||||
|
AlertChannel::Slack {
|
||||||
|
slack_channel,
|
||||||
|
webhook_url,
|
||||||
|
} => AlertChannelConfig {
|
||||||
|
receiver: AlertChannelReceiver {
|
||||||
|
name: format!("Slack-{slack_channel}"),
|
||||||
|
slack_configs: Some(vec![SlackConfig {
|
||||||
|
channel: slack_channel.clone(),
|
||||||
|
send_resolved: true,
|
||||||
|
title: "{{ .CommonAnnotations.title }}".to_string(),
|
||||||
|
text: ">-
|
||||||
|
*Alert:* {{ .CommonLabels.alertname }}
|
||||||
|
*Severity:* {{ .CommonLabels.severity }}
|
||||||
|
*Namespace:* {{ .CommonLabels.namespace }}
|
||||||
|
*Pod:* {{ .CommonLabels.pod }}
|
||||||
|
*ExternalURL:* {{ .ExternalURL }}
|
||||||
|
|
||||||
|
{{ range .Alerts }}
|
||||||
|
*Instance:* {{ .Labels.instance }}
|
||||||
|
*Summary:* {{ .Annotations.summary }}
|
||||||
|
*Description:* {{ .Annotations.description }}
|
||||||
|
*Starts At:* {{ .StartsAt }}
|
||||||
|
*Status:* {{ .Status }}
|
||||||
|
{{ end }}".to_string()
|
||||||
|
}]),
|
||||||
|
webhook_configs: None,
|
||||||
|
},
|
||||||
|
route: AlertChannelRoute {
|
||||||
|
receiver: format!("Slack-{slack_channel}"),
|
||||||
|
matchers: vec!["alertname!=Watchdog".to_string()],
|
||||||
|
r#continue: true,
|
||||||
|
},
|
||||||
|
global_config: Some(AlertChannelGlobalConfig {
|
||||||
|
slack_api_url: Some(webhook_url.clone()),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
AlertChannel::MSTeams {
|
||||||
|
connector, ..
|
||||||
|
} => AlertChannelConfig{
|
||||||
|
receiver: AlertChannelReceiver{
|
||||||
|
name: format!("MSTeams-{connector}"),
|
||||||
|
slack_configs: None,
|
||||||
|
webhook_configs: Some(vec![WebhookConfig{
|
||||||
|
url: url::Url::parse("http://prometheus-msteams-prometheus-msteams.monitoring.svc.cluster.local:2000/alertmanager").expect("invalid url"),
|
||||||
|
send_resolved: true,}])
|
||||||
|
},
|
||||||
|
route: AlertChannelRoute{
|
||||||
|
receiver: format!("MSTeams-{connector}"),
|
||||||
|
matchers: vec!["alertname!=Watchdog".to_string()],
|
||||||
|
r#continue: true,
|
||||||
|
},
|
||||||
|
global_config: None, },
|
||||||
|
|
||||||
|
AlertChannel::Smpt {
|
||||||
|
email_address,
|
||||||
|
service_name,
|
||||||
|
} => todo!(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
73
harmony/src/modules/monitoring/kube_prometheus/types.rs
Normal file
73
harmony/src/modules/monitoring/kube_prometheus/types.rs
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct AlertManagerValues {
|
||||||
|
pub alertmanager: AlertManager,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct AlertManager {
|
||||||
|
pub enabled: bool,
|
||||||
|
pub config: AlertManagerConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct AlertChannelConfig {
|
||||||
|
pub receiver: AlertChannelReceiver,
|
||||||
|
pub route: AlertChannelRoute,
|
||||||
|
pub global_config: Option<AlertChannelGlobalConfig>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct AlertChannelReceiver {
|
||||||
|
pub name: String,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub slack_configs: Option<Vec<SlackConfig>>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub webhook_configs: Option<Vec<WebhookConfig>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct AlertManagerRoute {
|
||||||
|
pub group_by: Vec<String>,
|
||||||
|
pub group_wait: String,
|
||||||
|
pub group_interval: String,
|
||||||
|
pub repeat_interval: String,
|
||||||
|
pub routes: Vec<AlertChannelRoute>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct AlertChannelGlobalConfig {
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub slack_api_url: Option<Url>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct SlackConfig {
|
||||||
|
pub channel: String,
|
||||||
|
pub send_resolved: bool,
|
||||||
|
pub title: String,
|
||||||
|
pub text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct WebhookConfig {
|
||||||
|
pub url: Url,
|
||||||
|
pub send_resolved: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct AlertChannelRoute {
|
||||||
|
pub receiver: String,
|
||||||
|
pub matchers: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub r#continue: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct AlertManagerConfig {
|
||||||
|
pub global: Option<AlertChannelGlobalConfig>,
|
||||||
|
pub route: AlertManagerRoute,
|
||||||
|
pub receivers: Vec<AlertChannelReceiver>,
|
||||||
|
}
|
||||||
@ -1,57 +1,13 @@
|
|||||||
use email_address::EmailAddress;
|
use email_address::EmailAddress;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::Serialize;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
mod config;
|
mod config;
|
||||||
mod discord_alert_manager;
|
mod discord_alert_manager;
|
||||||
mod kube_prometheus;
|
pub mod kube_prometheus;
|
||||||
pub mod monitoring_alerting;
|
pub mod monitoring_alerting;
|
||||||
mod prometheus_msteams;
|
mod prometheus_msteams;
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
struct AlertChannelConfig {
|
|
||||||
receiver: AlertChannelReceiver,
|
|
||||||
route: AlertChannelRoute,
|
|
||||||
global_config: Option<AlertChannelGlobalConfig>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
|
||||||
struct AlertChannelReceiver {
|
|
||||||
pub name: String,
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
pub slack_configs: Option<Vec<SlackConfig>>,
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
pub webhook_configs: Option<Vec<WebhookConfig>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
|
||||||
pub struct SlackConfig {
|
|
||||||
pub channel: String,
|
|
||||||
send_resolved: bool,
|
|
||||||
title: String,
|
|
||||||
text: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
|
||||||
pub struct WebhookConfig {
|
|
||||||
pub url: Url,
|
|
||||||
send_resolved: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
|
||||||
pub struct AlertChannelRoute {
|
|
||||||
pub receiver: String,
|
|
||||||
pub matchers: Vec<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
pub continue_: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
|
||||||
pub struct AlertChannelGlobalConfig {
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
pub slack_api_url: Option<Url>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub enum AlertChannel {
|
pub enum AlertChannel {
|
||||||
Discord {
|
Discord {
|
||||||
@ -74,75 +30,3 @@ pub enum AlertChannel {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
trait AlertEndpoint {
|
|
||||||
//fn register_webhook(&self, webhook_url: Url);
|
|
||||||
fn build_alert_receiver(&self) -> AlertChannelConfig;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl AlertEndpoint for AlertChannel {
|
|
||||||
fn build_alert_receiver(&self) -> AlertChannelConfig {
|
|
||||||
match self {
|
|
||||||
AlertChannel::Discord { name, .. } => AlertChannelConfig {
|
|
||||||
receiver: AlertChannelReceiver {
|
|
||||||
name: format!("Discord-{name}"),
|
|
||||||
slack_configs: None,
|
|
||||||
webhook_configs: Some(vec![WebhookConfig {
|
|
||||||
url: url::Url::parse("http://{name}-alertmanager-discord:9094")
|
|
||||||
.expect("invalid url"),
|
|
||||||
send_resolved: true,
|
|
||||||
}]),
|
|
||||||
},
|
|
||||||
route: AlertChannelRoute {
|
|
||||||
receiver: format!("Discord-{name}"),
|
|
||||||
matchers: vec!["alertname!=Watchdog".to_string()],
|
|
||||||
continue_: true,
|
|
||||||
},
|
|
||||||
global_config: None,
|
|
||||||
},
|
|
||||||
AlertChannel::Slack {
|
|
||||||
slack_channel,
|
|
||||||
webhook_url,
|
|
||||||
} => AlertChannelConfig {
|
|
||||||
receiver: AlertChannelReceiver {
|
|
||||||
name: format!("Slack-{slack_channel}"),
|
|
||||||
slack_configs: Some(vec![SlackConfig {
|
|
||||||
channel: slack_channel.clone(),
|
|
||||||
send_resolved: true,
|
|
||||||
title: "{{ .CommonAnnotations.title }}".to_string(),
|
|
||||||
text: "{{ .CommonAnnotations.description }}".to_string(),
|
|
||||||
}]),
|
|
||||||
webhook_configs: None,
|
|
||||||
},
|
|
||||||
route: AlertChannelRoute {
|
|
||||||
receiver: format!("Slack-{slack_channel}"),
|
|
||||||
matchers: vec!["alertname!=Watchdog".to_string()],
|
|
||||||
continue_: true,
|
|
||||||
},
|
|
||||||
global_config: Some(AlertChannelGlobalConfig {
|
|
||||||
slack_api_url: Some(webhook_url.clone()),
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
AlertChannel::MSTeams {
|
|
||||||
connector, ..
|
|
||||||
} => AlertChannelConfig{
|
|
||||||
receiver: AlertChannelReceiver{
|
|
||||||
name: format!("MSTeams-{connector}"),
|
|
||||||
slack_configs: None,
|
|
||||||
webhook_configs: Some(vec![WebhookConfig{
|
|
||||||
url: url::Url::parse("http://prometheus-msteams-prometheus-msteams.monitoring.svc.cluster.local:2000/alertmanager").expect("invalid url"),
|
|
||||||
send_resolved: true,}])
|
|
||||||
},
|
|
||||||
route: AlertChannelRoute{
|
|
||||||
receiver: format!("MSTeams-{connector}"),
|
|
||||||
matchers: vec!["alertname!=Watchdog".to_string()],
|
|
||||||
continue_: true,
|
|
||||||
},
|
|
||||||
global_config: None, },
|
|
||||||
|
|
||||||
AlertChannel::Smpt {
|
|
||||||
email_address,
|
|
||||||
service_name,
|
|
||||||
} => todo!(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@ -14,9 +14,7 @@ use crate::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
config::KubePrometheusConfig, discord_alert_manager::discord_alert_manager_score,
|
config::KubePrometheusConfig, discord_alert_manager::discord_alert_manager_score, kube_prometheus::kube_prometheus::kube_prometheus_helm_chart_score, prometheus_msteams::prometheus_msteams_score, AlertChannel
|
||||||
kube_prometheus::kube_prometheus_helm_chart_score,
|
|
||||||
prometheus_msteams::prometheus_msteams_score, AlertChannel,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user