feat: deploys kube-prometheus with discord alert receiver
This commit is contained in:
parent
b1755c183d
commit
2cb95151ae
@ -12,7 +12,7 @@ use harmony::{
|
|||||||
async fn main() {
|
async fn main() {
|
||||||
let discord_receiver = DiscordWebhook {
|
let discord_receiver = DiscordWebhook {
|
||||||
name: "test-discord".to_string(),
|
name: "test-discord".to_string(),
|
||||||
url: Url::Url(url::Url::parse("https://discord.i.dont.exist.com").unwrap()),
|
url: Url::Url(url::Url::parse("discord.doesnt.exist.com").unwrap()),
|
||||||
};
|
};
|
||||||
let alerting_score = HelmPrometheusAlertingScore {
|
let alerting_score = HelmPrometheusAlertingScore {
|
||||||
receivers: vec![Box::new(discord_receiver)],
|
receivers: vec![Box::new(discord_receiver)],
|
||||||
|
|||||||
@ -1,10 +1,12 @@
|
|||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|
||||||
use crate::interpret::InterpretError;
|
use crate::{interpret::InterpretError, inventory::Inventory};
|
||||||
|
|
||||||
use super::oberservability::monitoring::AlertSender;
|
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
pub trait Installable {
|
pub trait Installable<T>: Send + Sync {
|
||||||
async fn ensure_installed(&self) -> Result<(), InterpretError>;
|
async fn ensure_installed(
|
||||||
|
&self,
|
||||||
|
inventory: &Inventory,
|
||||||
|
topology: &T,
|
||||||
|
) -> Result<(), InterpretError>;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,18 +4,12 @@ use crate::{
|
|||||||
data::{Id, Version},
|
data::{Id, Version},
|
||||||
interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
|
interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
|
||||||
inventory::Inventory,
|
inventory::Inventory,
|
||||||
modules::monitoring::kube_prometheus::prometheus::Installer,
|
|
||||||
topology::{HelmCommand, Topology, installable::Installable},
|
topology::{HelmCommand, Topology, installable::Installable},
|
||||||
};
|
};
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
pub trait AlertSender: Send + Sync + std::fmt::Debug + Installable {
|
pub trait AlertSender: Send + Sync + std::fmt::Debug {
|
||||||
fn name(&self) -> String;
|
fn name(&self) -> String;
|
||||||
async fn install<T: Topology + HelmCommand>(
|
|
||||||
&self,
|
|
||||||
inventory: &Inventory,
|
|
||||||
topology: &T,
|
|
||||||
) -> Result<Outcome, InterpretError>;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@ -25,7 +19,7 @@ pub struct AlertingInterpret<S: AlertSender> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl<S: AlertSender + Installable, T: Topology> Interpret<T> for AlertingInterpret<S> {
|
impl<S: AlertSender + Installable<T>, T: Topology + HelmCommand> Interpret<T> for AlertingInterpret<S> {
|
||||||
async fn execute(
|
async fn execute(
|
||||||
&self,
|
&self,
|
||||||
inventory: &Inventory,
|
inventory: &Inventory,
|
||||||
@ -34,7 +28,7 @@ impl<S: AlertSender + Installable, T: Topology> Interpret<T> for AlertingInterpr
|
|||||||
for receiver in self.receivers.iter() {
|
for receiver in self.receivers.iter() {
|
||||||
receiver.install(&self.sender).await?;
|
receiver.install(&self.sender).await?;
|
||||||
}
|
}
|
||||||
self.sender.ensure_installed().await?;
|
self.sender.ensure_installed(inventory, topology).await?;
|
||||||
Ok(Outcome::success(format!(
|
Ok(Outcome::success(format!(
|
||||||
"successfully installed alert sender {}",
|
"successfully installed alert sender {}",
|
||||||
self.sender.name()
|
self.sender.name()
|
||||||
|
|||||||
@ -5,7 +5,7 @@ use crate::{
|
|||||||
interpret::{InterpretError, Outcome},
|
interpret::{InterpretError, Outcome},
|
||||||
modules::monitoring::kube_prometheus::{
|
modules::monitoring::kube_prometheus::{
|
||||||
prometheus::{Prometheus, PrometheusReceiver},
|
prometheus::{Prometheus, PrometheusReceiver},
|
||||||
types::AlertChannelConfig,
|
types::{AlertChannelConfig, AlertManagerChannelConfig},
|
||||||
},
|
},
|
||||||
topology::{Url, oberservability::monitoring::AlertReceiver},
|
topology::{Url, oberservability::monitoring::AlertReceiver},
|
||||||
};
|
};
|
||||||
@ -28,20 +28,30 @@ impl AlertReceiver<Prometheus> for DiscordWebhook {
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl PrometheusReceiver for DiscordWebhook {
|
impl PrometheusReceiver for DiscordWebhook {
|
||||||
//TODO not return a tuple
|
|
||||||
fn name(&self) -> String {
|
fn name(&self) -> String {
|
||||||
self.name.clone()
|
self.name.clone()
|
||||||
}
|
}
|
||||||
async fn receiver_config(&self) -> Value {
|
async fn configure_receiver(&self) -> AlertManagerChannelConfig {
|
||||||
self.alert_channel_receiver().await
|
self.get_config().await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl AlertChannelConfig for DiscordWebhook {
|
impl AlertChannelConfig for DiscordWebhook {
|
||||||
async fn alert_channel_global_config(&self) -> Option<serde_yaml::Value> {
|
async fn get_config(&self) -> AlertManagerChannelConfig {
|
||||||
None
|
let channel_global_config = None;
|
||||||
|
let channel_receiver = self.alert_channel_receiver().await;
|
||||||
|
let channel_route = self.alert_channel_route().await;
|
||||||
|
|
||||||
|
AlertManagerChannelConfig {
|
||||||
|
channel_global_config,
|
||||||
|
channel_receiver,
|
||||||
|
channel_route,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DiscordWebhook {
|
||||||
|
|
||||||
async fn alert_channel_route(&self) -> serde_yaml::Value {
|
async fn alert_channel_route(&self) -> serde_yaml::Value {
|
||||||
let mut route = Mapping::new();
|
let mut route = Mapping::new();
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use serde_yaml::Value;
|
use serde_yaml::Value;
|
||||||
|
|
||||||
use crate::modules::monitoring::kube_prometheus::prometheus::PrometheusReceiver;
|
use crate::modules::monitoring::kube_prometheus::{prometheus::PrometheusReceiver, types::AlertManagerChannelConfig};
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub struct KubePrometheusConfig {
|
pub struct KubePrometheusConfig {
|
||||||
@ -22,7 +22,7 @@ pub struct KubePrometheusConfig {
|
|||||||
pub kube_proxy: bool,
|
pub kube_proxy: bool,
|
||||||
pub kube_state_metrics: bool,
|
pub kube_state_metrics: bool,
|
||||||
pub prometheus_operator: bool,
|
pub prometheus_operator: bool,
|
||||||
pub alert_receiver_configs: Vec<Value>,
|
pub alert_receiver_configs: Vec<AlertManagerChannelConfig>,
|
||||||
}
|
}
|
||||||
impl KubePrometheusConfig {
|
impl KubePrometheusConfig {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
|
|||||||
@ -1,11 +1,19 @@
|
|||||||
use super::config::KubePrometheusConfig;
|
use super::config::KubePrometheusConfig;
|
||||||
|
use log::debug;
|
||||||
use non_blank_string_rs::NonBlankString;
|
use non_blank_string_rs::NonBlankString;
|
||||||
|
use serde_yaml::{Mapping, Value};
|
||||||
use std::{
|
use std::{
|
||||||
str::FromStr,
|
str::FromStr,
|
||||||
sync::{Arc, Mutex},
|
sync::{Arc, Mutex},
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::modules::helm::chart::HelmChartScore;
|
use crate::modules::{
|
||||||
|
helm::chart::HelmChartScore,
|
||||||
|
monitoring::kube_prometheus::types::{
|
||||||
|
AlertManager, AlertManagerChannelConfig, AlertManagerConfig, AlertManagerRoute,
|
||||||
|
AlertManagerValues,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
pub fn kube_prometheus_helm_chart_score(
|
pub fn kube_prometheus_helm_chart_score(
|
||||||
config: Arc<Mutex<KubePrometheusConfig>>,
|
config: Arc<Mutex<KubePrometheusConfig>>,
|
||||||
@ -15,11 +23,11 @@ pub fn kube_prometheus_helm_chart_score(
|
|||||||
//to the overrides or something leaving the user to deal with formatting here seems bad
|
//to the overrides or something leaving the user to deal with formatting here seems bad
|
||||||
let default_rules = config.default_rules.to_string();
|
let default_rules = config.default_rules.to_string();
|
||||||
let windows_monitoring = config.windows_monitoring.to_string();
|
let windows_monitoring = config.windows_monitoring.to_string();
|
||||||
let alert_manager = config.alert_manager.to_string();
|
|
||||||
let grafana = config.grafana.to_string();
|
let grafana = config.grafana.to_string();
|
||||||
let kubernetes_service_monitors = config.kubernetes_service_monitors.to_string();
|
let kubernetes_service_monitors = config.kubernetes_service_monitors.to_string();
|
||||||
let kubernetes_api_server = config.kubernetes_api_server.to_string();
|
let kubernetes_api_server = config.kubernetes_api_server.to_string();
|
||||||
let kubelet = config.kubelet.to_string();
|
let kubelet = config.kubelet.to_string();
|
||||||
|
let alert_manager = config.alert_manager.to_string();
|
||||||
let kube_controller_manager = config.kube_controller_manager.to_string();
|
let kube_controller_manager = config.kube_controller_manager.to_string();
|
||||||
let core_dns = config.core_dns.to_string();
|
let core_dns = config.core_dns.to_string();
|
||||||
let kube_etcd = config.kube_etcd.to_string();
|
let kube_etcd = config.kube_etcd.to_string();
|
||||||
@ -29,7 +37,7 @@ pub fn kube_prometheus_helm_chart_score(
|
|||||||
let node_exporter = config.node_exporter.to_string();
|
let node_exporter = config.node_exporter.to_string();
|
||||||
let prometheus_operator = config.prometheus_operator.to_string();
|
let prometheus_operator = config.prometheus_operator.to_string();
|
||||||
let prometheus = config.prometheus.to_string();
|
let prometheus = config.prometheus.to_string();
|
||||||
let values = format!(
|
let mut values = format!(
|
||||||
r#"
|
r#"
|
||||||
additionalPrometheusRulesMap:
|
additionalPrometheusRulesMap:
|
||||||
pods-status-alerts:
|
pods-status-alerts:
|
||||||
@ -148,6 +156,54 @@ prometheus:
|
|||||||
enabled: {prometheus}
|
enabled: {prometheus}
|
||||||
"#,
|
"#,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
||||||
|
let mut null_receiver = Mapping::new();
|
||||||
|
null_receiver.insert(
|
||||||
|
Value::String("receiver".to_string()),
|
||||||
|
Value::String("null".to_string()),
|
||||||
|
);
|
||||||
|
null_receiver.insert(
|
||||||
|
Value::String("matchers".to_string()),
|
||||||
|
Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]),
|
||||||
|
);
|
||||||
|
null_receiver.insert(Value::String("continue".to_string()), Value::Bool(true));
|
||||||
|
|
||||||
|
|
||||||
|
let mut alert_manager_channel_config = AlertManagerConfig {
|
||||||
|
global: Mapping::new(),
|
||||||
|
route: AlertManagerRoute {
|
||||||
|
routes: vec![Value::Mapping(null_receiver)],
|
||||||
|
},
|
||||||
|
receivers: vec![serde_yaml::from_str("name: 'null'").unwrap()],
|
||||||
|
};
|
||||||
|
for receiver in config.alert_receiver_configs.iter() {
|
||||||
|
if let Some(global) = receiver.channel_global_config.clone() {
|
||||||
|
alert_manager_channel_config
|
||||||
|
.global
|
||||||
|
.insert(global.0, global.1);
|
||||||
|
}
|
||||||
|
alert_manager_channel_config
|
||||||
|
.route
|
||||||
|
.routes
|
||||||
|
.push(receiver.channel_route.clone());
|
||||||
|
alert_manager_channel_config
|
||||||
|
.receivers
|
||||||
|
.push(receiver.channel_receiver.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
let alert_manager_values = AlertManagerValues {
|
||||||
|
alertmanager: AlertManager {
|
||||||
|
enabled: config.alert_manager,
|
||||||
|
config: alert_manager_channel_config,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
let alert_manager_yaml =
|
||||||
|
serde_yaml::to_string(&alert_manager_values).expect("Failed to serialize YAML");
|
||||||
|
debug!("serialed alert manager: \n {:#}", alert_manager_yaml);
|
||||||
|
values.push_str(&alert_manager_yaml);
|
||||||
|
debug!("full values.yaml: \n {:#}", values);
|
||||||
HelmChartScore {
|
HelmChartScore {
|
||||||
namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()),
|
namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()),
|
||||||
release_name: NonBlankString::from_str("kube-prometheus").unwrap(),
|
release_name: NonBlankString::from_str("kube-prometheus").unwrap(),
|
||||||
|
|||||||
@ -16,54 +16,46 @@ use crate::{
|
|||||||
|
|
||||||
use score::Score;
|
use score::Score;
|
||||||
|
|
||||||
use super::helm::{
|
use super::{helm::{
|
||||||
config::KubePrometheusConfig, kube_prometheus_helm_chart::kube_prometheus_helm_chart_score,
|
config::KubePrometheusConfig, kube_prometheus_helm_chart::kube_prometheus_helm_chart_score,
|
||||||
};
|
}, types::AlertManagerChannelConfig};
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl AlertSender for Prometheus {
|
impl AlertSender for Prometheus {
|
||||||
fn name(&self) -> String {
|
fn name(&self) -> String {
|
||||||
"HelmKubePrometheus".to_string()
|
"HelmKubePrometheus".to_string()
|
||||||
}
|
}
|
||||||
async fn install<T: Topology + HelmCommand + Send + Sync>(
|
|
||||||
&self,
|
|
||||||
inventory: &Inventory,
|
|
||||||
topology: &T,
|
|
||||||
) -> Result<Outcome, InterpretError> {
|
|
||||||
let _ = self.install_prometheus(inventory, topology).await;
|
|
||||||
todo!()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//im not totally sure what to do in the impl installable
|
|
||||||
//should we have a oncecell that checks insured is true?
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl Installable for Prometheus {
|
|
||||||
async fn ensure_installed(&self) -> Result<(), InterpretError> {
|
|
||||||
todo!()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//before we talked about having a trait installable and a trait installer for the topology
|
|
||||||
// i feel like that might still be necessary to meet the requirement of inventory and topology on
|
|
||||||
// the score.create_interpret().execute(inventory, topology) method
|
|
||||||
#[async_trait]
|
|
||||||
pub trait Installer {
|
|
||||||
async fn install<I: Installable + Send + Sync>(&self, sender: I) -> Result<(), InterpretError>;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl Installer for K8sAnywhereTopology {
|
impl<T: Topology + HelmCommand> Installable<T> for Prometheus {
|
||||||
async fn install<I: Installable + Send + Sync>(
|
async fn ensure_installed(&self, inventory: &Inventory, topology: &T) -> Result<(), InterpretError> {
|
||||||
&self,
|
//install_prometheus
|
||||||
installable: I,
|
self.install_prometheus(inventory, topology).await?;
|
||||||
) -> Result<(), InterpretError> {
|
|
||||||
installable.ensure_installed().await?;
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// //before we talked about having a trait installable and a trait installer for the topology
|
||||||
|
// // i feel like that might still be necessary to meet the requirement of inventory and topology on
|
||||||
|
// // the score.create_interpret().execute(inventory, topology) method
|
||||||
|
// #[async_trait]
|
||||||
|
// pub trait Installer {
|
||||||
|
// async fn install(&self, inventory: &Inventory, sender: Box<dyn Installable>) -> Result<(), InterpretError>;
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// #[async_trait]
|
||||||
|
// impl Installer for K8sAnywhereTopology {
|
||||||
|
// async fn install(
|
||||||
|
// &self,
|
||||||
|
// inventory: &Inventory,
|
||||||
|
// installable: Box<dyn Installable<T>>,
|
||||||
|
// ) -> Result<(), InterpretError> {
|
||||||
|
// installable.ensure_installed(inventory, self).await?;
|
||||||
|
// Ok(())
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct Prometheus {
|
pub struct Prometheus {
|
||||||
pub config: Arc<Mutex<KubePrometheusConfig>>,
|
pub config: Arc<Mutex<KubePrometheusConfig>>,
|
||||||
@ -74,7 +66,7 @@ impl Prometheus {
|
|||||||
&self,
|
&self,
|
||||||
prometheus_receiver: &dyn PrometheusReceiver,
|
prometheus_receiver: &dyn PrometheusReceiver,
|
||||||
) -> Result<Outcome, InterpretError> {
|
) -> Result<Outcome, InterpretError> {
|
||||||
let prom_receiver = prometheus_receiver.receiver_config().await;
|
let prom_receiver = prometheus_receiver.configure_receiver().await;
|
||||||
debug!(
|
debug!(
|
||||||
"adding alert receiver to prometheus config: {:#?}",
|
"adding alert receiver to prometheus config: {:#?}",
|
||||||
&prom_receiver
|
&prom_receiver
|
||||||
@ -105,7 +97,7 @@ impl Prometheus {
|
|||||||
#[async_trait]
|
#[async_trait]
|
||||||
pub trait PrometheusReceiver: Send + Sync + std::fmt::Debug {
|
pub trait PrometheusReceiver: Send + Sync + std::fmt::Debug {
|
||||||
fn name(&self) -> String;
|
fn name(&self) -> String;
|
||||||
async fn receiver_config(&self) -> Value;
|
async fn configure_receiver(&self) -> AlertManagerChannelConfig;
|
||||||
//this probably needs to be a type
|
//this probably needs to be a type
|
||||||
//that
|
//that
|
||||||
//represents
|
//represents
|
||||||
|
|||||||
@ -1,9 +1,40 @@
|
|||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde_yaml::Value;
|
use serde::Serialize;
|
||||||
|
use serde_yaml::{Mapping, Sequence, Value};
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
pub trait AlertChannelConfig {
|
pub trait AlertChannelConfig {
|
||||||
async fn alert_channel_global_config(&self) -> Option<Value>;
|
async fn get_config(&self) -> AlertManagerChannelConfig;
|
||||||
async fn alert_channel_route(&self) -> Value;
|
}
|
||||||
async fn alert_channel_receiver(&self) -> Value;
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct AlertManagerValues {
|
||||||
|
pub alertmanager: AlertManager,
|
||||||
|
}
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct AlertManager {
|
||||||
|
pub enabled: bool,
|
||||||
|
pub config: AlertManagerConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct AlertManagerConfig {
|
||||||
|
pub global: Mapping,
|
||||||
|
pub route: AlertManagerRoute,
|
||||||
|
pub receivers: Sequence,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct AlertManagerRoute {
|
||||||
|
pub routes: Sequence,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct AlertManagerChannelConfig {
|
||||||
|
///expecting an option that contains two values
|
||||||
|
///if necessary for the alertchannel
|
||||||
|
///[ jira_api_url: <string> ]
|
||||||
|
pub channel_global_config: Option<(Value, Value)>,
|
||||||
|
pub channel_route: Value,
|
||||||
|
pub channel_receiver: Value,
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user