forked from NationTech/harmony
feat: send alerts to multiple alert channels
This commit is contained in:
parent
44b2b092a8
commit
e1a8ee1c15
@ -22,7 +22,7 @@ pub struct KubePrometheusConfig {
|
|||||||
pub kube_proxy: bool,
|
pub kube_proxy: bool,
|
||||||
pub kube_state_metrics: bool,
|
pub kube_state_metrics: bool,
|
||||||
pub prometheus_operator: bool,
|
pub prometheus_operator: bool,
|
||||||
pub alert_channel: Option<AlertChannel>,
|
pub alert_channel: Vec<AlertChannel>,
|
||||||
}
|
}
|
||||||
impl KubePrometheusConfig {
|
impl KubePrometheusConfig {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
@ -31,7 +31,7 @@ impl KubePrometheusConfig {
|
|||||||
default_rules: true,
|
default_rules: true,
|
||||||
windows_monitoring: false,
|
windows_monitoring: false,
|
||||||
alert_manager: true,
|
alert_manager: true,
|
||||||
alert_channel: None,
|
alert_channel: Vec::new(),
|
||||||
grafana: true,
|
grafana: true,
|
||||||
node_exporter: false,
|
node_exporter: false,
|
||||||
prometheus: true,
|
prometheus: true,
|
||||||
|
@ -6,27 +6,27 @@ use crate::modules::helm::chart::HelmChartScore;
|
|||||||
|
|
||||||
use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel};
|
use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel};
|
||||||
|
|
||||||
pub fn discord_alert_manager_score(config: &KubePrometheusConfig) -> HelmChartScore {
|
fn get_discord_alert_manager_score(config: &KubePrometheusConfig) -> Option<HelmChartScore> {
|
||||||
let (url, release_name) = match &config.alert_channel {
|
let (url, name) = config.alert_channel.iter().find_map(|channel| {
|
||||||
Some(AlertChannel::Discord { webhook_url, name }) => {
|
if let AlertChannel::Discord { webhook_url, name } = channel {
|
||||||
(webhook_url.to_string(), name.to_string())
|
Some((webhook_url, name))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
}
|
}
|
||||||
_ => panic!("Expected Discord alert channel configuration"),
|
})?;
|
||||||
};
|
|
||||||
|
|
||||||
let values = format!(
|
let values = format!(
|
||||||
r#"
|
r#"
|
||||||
|
|
||||||
environment:
|
environment:
|
||||||
- name: "DISCORD_WEBHOOK"
|
- name: "DISCORD_WEBHOOK"
|
||||||
value: "{url}"
|
value: "{url}"
|
||||||
"#,
|
"#,
|
||||||
);
|
);
|
||||||
|
|
||||||
HelmChartScore {
|
Some(HelmChartScore {
|
||||||
namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()),
|
namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()),
|
||||||
release_name: NonBlankString::from_str(&release_name).unwrap(),
|
release_name: NonBlankString::from_str(&name).unwrap(),
|
||||||
chart_name: NonBlankString::from_str("oci://hub.nationtech.io/nt/alertmanager-discord")
|
chart_name: NonBlankString::from_str("oci://hub.nationtech.io/library/alertmanager-discord")
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
chart_version: None,
|
chart_version: None,
|
||||||
values_overrides: None,
|
values_overrides: None,
|
||||||
@ -34,5 +34,13 @@ environment:
|
|||||||
create_namespace: true,
|
create_namespace: true,
|
||||||
install_only: true,
|
install_only: true,
|
||||||
repository: None,
|
repository: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn discord_alert_manager_score(config: &KubePrometheusConfig) -> HelmChartScore {
|
||||||
|
if let Some(chart) = get_discord_alert_manager_score(config) {
|
||||||
|
chart
|
||||||
|
} else {
|
||||||
|
panic!("Expected discord alert manager helm chart");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,8 +1,8 @@
|
|||||||
use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel};
|
use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel};
|
||||||
use log::info;
|
use log::info;
|
||||||
use non_blank_string_rs::NonBlankString;
|
use non_blank_string_rs::NonBlankString;
|
||||||
use url::Url;
|
|
||||||
use std::{collections::HashMap, str::FromStr};
|
use std::{collections::HashMap, str::FromStr};
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
use crate::modules::helm::chart::HelmChartScore;
|
use crate::modules::helm::chart::HelmChartScore;
|
||||||
|
|
||||||
@ -116,21 +116,6 @@ defaultRules:
|
|||||||
windows: true
|
windows: true
|
||||||
windowsMonitoring:
|
windowsMonitoring:
|
||||||
enabled: {windows_monitoring}
|
enabled: {windows_monitoring}
|
||||||
alertmanager:
|
|
||||||
enabled: {alert_manager}
|
|
||||||
config:
|
|
||||||
route:
|
|
||||||
group_by: ['job']
|
|
||||||
group_wait: 30s
|
|
||||||
group_interval: 5m
|
|
||||||
repeat_interval: 12h
|
|
||||||
routes:
|
|
||||||
- receiver: 'null'
|
|
||||||
matchers:
|
|
||||||
- alertname="Watchdog"
|
|
||||||
continue: false
|
|
||||||
receivers:
|
|
||||||
- name: 'null'
|
|
||||||
grafana:
|
grafana:
|
||||||
enabled: {grafana}
|
enabled: {grafana}
|
||||||
kubernetesServiceMonitors:
|
kubernetesServiceMonitors:
|
||||||
@ -160,15 +145,65 @@ prometheus:
|
|||||||
"#,
|
"#,
|
||||||
);
|
);
|
||||||
|
|
||||||
if let Some(alert_channel) = &config.alert_channel {
|
let alertmanager_config = alert_manager_yaml_builder(&config);
|
||||||
match alert_channel {
|
values.push_str(&alertmanager_config);
|
||||||
AlertChannel::Discord { name, .. } => {
|
|
||||||
values.push_str(&discord_alert_builder(name).to_string());}
|
fn alert_manager_yaml_builder(config: &KubePrometheusConfig) -> String {
|
||||||
AlertChannel::Slack { slack_channel, webhook_url } => {
|
let mut receivers = String::new();
|
||||||
values.push_str(&slack_alert_builder(slack_channel, webhook_url).to_string());}
|
let mut routes = String::new();
|
||||||
AlertChannel::Smpt { .. } => todo!(),
|
let mut global_configs = String::new();
|
||||||
|
let alert_manager = config.alert_manager;
|
||||||
|
for alert_channel in &config.alert_channel {
|
||||||
|
match alert_channel {
|
||||||
|
AlertChannel::Discord { name, .. } => {
|
||||||
|
let (receiver, route) = discord_alert_builder(name);
|
||||||
|
info!("discord receiver: {} \nroute: {}", receiver, route);
|
||||||
|
receivers.push_str(&receiver);
|
||||||
|
routes.push_str(&route);
|
||||||
|
}
|
||||||
|
AlertChannel::Slack {
|
||||||
|
slack_channel,
|
||||||
|
webhook_url,
|
||||||
|
} => {
|
||||||
|
let (receiver, route) = slack_alert_builder(slack_channel);
|
||||||
|
info!("slack receiver: {} \nroute: {}", receiver, route);
|
||||||
|
receivers.push_str(&receiver);
|
||||||
|
|
||||||
|
routes.push_str(&route);
|
||||||
|
let global_config = format!(
|
||||||
|
r#"
|
||||||
|
global:
|
||||||
|
slack_api_url: {webhook_url}"#
|
||||||
|
);
|
||||||
|
|
||||||
|
global_configs.push_str(&global_config);
|
||||||
|
}
|
||||||
|
AlertChannel::Smpt { .. } => todo!(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
info!("after alert receiver: {}", receivers);
|
||||||
|
info!("after alert routes: {}", routes);
|
||||||
|
|
||||||
|
let alertmanager_config = format!(
|
||||||
|
r#"
|
||||||
|
alertmanager:
|
||||||
|
enabled: {alert_manager}
|
||||||
|
config: {global_configs}
|
||||||
|
route:
|
||||||
|
group_by: ['job']
|
||||||
|
group_wait: 30s
|
||||||
|
group_interval: 5m
|
||||||
|
repeat_interval: 12h
|
||||||
|
routes:
|
||||||
|
{routes}
|
||||||
|
receivers:
|
||||||
|
- name: 'null'
|
||||||
|
{receivers}"#
|
||||||
|
);
|
||||||
|
|
||||||
|
info!("alert manager config: {}", alertmanager_config);
|
||||||
|
alertmanager_config
|
||||||
|
}
|
||||||
|
|
||||||
HelmChartScore {
|
HelmChartScore {
|
||||||
namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()),
|
namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()),
|
||||||
@ -186,43 +221,42 @@ prometheus:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn discord_alert_builder(release_name: &String) -> String {
|
fn discord_alert_builder(release_name: &String) -> (String, String) {
|
||||||
let discord_alert_manager_release_name = release_name;
|
let discord_receiver_name = format!("Discord-{}", release_name);
|
||||||
let discord_alert_values = format!(
|
let receiver = format!(
|
||||||
r#"
|
r#"
|
||||||
alertmanager:
|
- name: '{discord_receiver_name}'
|
||||||
config:
|
|
||||||
route:
|
|
||||||
receiver: 'Discord'
|
|
||||||
receivers:
|
|
||||||
- name: 'null'
|
|
||||||
- name: 'Discord'
|
|
||||||
webhook_configs:
|
webhook_configs:
|
||||||
- url: 'http://{discord_alert_manager_release_name}-alertmanager-discord:9094'
|
- url: 'http://{release_name}-alertmanager-discord:9094'
|
||||||
send_resolved: true
|
send_resolved: true"#,
|
||||||
"#,
|
|
||||||
);
|
);
|
||||||
discord_alert_values
|
let route = format!(
|
||||||
|
r#"
|
||||||
|
- receiver: '{discord_receiver_name}'
|
||||||
|
matchers:
|
||||||
|
- alertname!=Watchdog
|
||||||
|
continue: true"#,
|
||||||
|
);
|
||||||
|
(receiver, route)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn slack_alert_builder(slack_channel: &String, webhook_url: &Url) -> String {
|
fn slack_alert_builder(slack_channel: &String) -> (String, String) {
|
||||||
let slack_alert_values = format!(
|
let slack_receiver_name = format!("Slack-{}", slack_channel);
|
||||||
|
let receiver = format!(
|
||||||
r#"
|
r#"
|
||||||
alertmanager:
|
- name: '{slack_receiver_name}'
|
||||||
config:
|
|
||||||
global:
|
|
||||||
slack_api_url: {webhook_url}
|
|
||||||
route:
|
|
||||||
receiver: 'Slack'
|
|
||||||
receivers:
|
|
||||||
- name: 'null'
|
|
||||||
- name: 'Slack'
|
|
||||||
slack_configs:
|
slack_configs:
|
||||||
- channel: '{slack_channel}'
|
- channel: '{slack_channel}'
|
||||||
send_resolved: true
|
send_resolved: true
|
||||||
title: '{{{{ .CommonAnnotations.title }}}}'
|
title: '{{{{ .CommonAnnotations.title }}}}'
|
||||||
text: '{{{{ .CommonAnnotations.description }}}}'
|
text: '{{{{ .CommonAnnotations.description }}}}'"#,
|
||||||
"#,
|
|
||||||
);
|
);
|
||||||
slack_alert_values
|
let route = format!(
|
||||||
|
r#"
|
||||||
|
- receiver: '{slack_receiver_name}'
|
||||||
|
matchers:
|
||||||
|
- alertname!=Watchdog
|
||||||
|
continue: true"#,
|
||||||
|
);
|
||||||
|
(receiver, route)
|
||||||
}
|
}
|
||||||
|
@ -38,14 +38,14 @@ pub enum AlertChannel {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub struct MonitoringAlertingStackScore {
|
pub struct MonitoringAlertingStackScore {
|
||||||
pub alert_channel: Option<AlertChannel>,
|
pub alert_channel: Vec<AlertChannel>,
|
||||||
pub namespace: Option<String>,
|
pub namespace: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MonitoringAlertingStackScore {
|
impl MonitoringAlertingStackScore {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
alert_channel: None,
|
alert_channel: Vec::new(),
|
||||||
namespace: None,
|
namespace: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -96,8 +96,10 @@ impl MonitoringAlertingStackInterpret {
|
|||||||
topology: &T,
|
topology: &T,
|
||||||
config: &KubePrometheusConfig,
|
config: &KubePrometheusConfig,
|
||||||
) -> Result<Outcome, InterpretError> {
|
) -> Result<Outcome, InterpretError> {
|
||||||
match &self.score.alert_channel {
|
let mut outcomes = vec![];
|
||||||
Some(channel) => match channel {
|
|
||||||
|
for channel in &self.score.alert_channel {
|
||||||
|
let outcome = match channel {
|
||||||
AlertChannel::Discord { .. } => {
|
AlertChannel::Discord { .. } => {
|
||||||
discord_alert_manager_score(config)
|
discord_alert_manager_score(config)
|
||||||
.create_interpret()
|
.create_interpret()
|
||||||
@ -110,9 +112,14 @@ impl MonitoringAlertingStackInterpret {
|
|||||||
AlertChannel::Smpt { .. } => {
|
AlertChannel::Smpt { .. } => {
|
||||||
todo!()
|
todo!()
|
||||||
}
|
}
|
||||||
},
|
};
|
||||||
None => Ok(Outcome::success("No alert channel configured".to_string())),
|
outcomes.push(outcome);
|
||||||
}
|
}
|
||||||
|
for result in outcomes {
|
||||||
|
result?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Outcome::success("All alert channels deployed".to_string()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user