feat: Alerting module architecture to make it easy to use and extensible by external crates
Co-authored-by: Jean-Gabriel Gill-Couture <jg@nationtech.io> Reviewed-on: https://git.nationtech.io/NationTech/harmony/pulls/61 Reviewed-by: johnride <jg@nationtech.io> Co-authored-by: Willem <wrolleman@nationtech.io> Co-committed-by: Willem <wrolleman@nationtech.io>
This commit is contained in:
parent
155e9bac28
commit
e06548ac44
9
Cargo.lock
generated
9
Cargo.lock
generated
@ -1154,6 +1154,15 @@ dependencies = [
|
|||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "example-monitoring"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"harmony",
|
||||||
|
"harmony_cli",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "example-nanodc"
|
name = "example-nanodc"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
|
73
adr/010-monitoring-alerting/architecture.rs
Normal file
73
adr/010-monitoring-alerting/architecture.rs
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
pub trait MonitoringSystem {}
|
||||||
|
|
||||||
|
// 1. Modified AlertReceiver trait:
|
||||||
|
// - Removed the problematic `clone` method.
|
||||||
|
// - Added `box_clone` which returns a Box<dyn AlertReceiver>.
|
||||||
|
pub trait AlertReceiver {
|
||||||
|
type M: MonitoringSystem;
|
||||||
|
fn install(&self, sender: &Self::M) -> Result<(), String>;
|
||||||
|
// This method allows concrete types to clone themselves into a Box<dyn AlertReceiver>
|
||||||
|
fn box_clone(&self) -> Box<dyn AlertReceiver<M = Self::M>>;
|
||||||
|
}
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct Prometheus{}
|
||||||
|
impl MonitoringSystem for Prometheus {}
|
||||||
|
|
||||||
|
#[derive(Clone)] // Keep derive(Clone) for DiscordWebhook itself
|
||||||
|
struct DiscordWebhook{}
|
||||||
|
|
||||||
|
impl AlertReceiver for DiscordWebhook {
|
||||||
|
type M = Prometheus;
|
||||||
|
fn install(&self, sender: &Self::M) -> Result<(), String> {
|
||||||
|
// Placeholder for actual installation logic
|
||||||
|
println!("DiscordWebhook installed for Prometheus monitoring.");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
// 2. Implement `box_clone` for DiscordWebhook:
|
||||||
|
// This uses the derived `Clone` for DiscordWebhook to create a new boxed instance.
|
||||||
|
fn box_clone(&self) -> Box<dyn AlertReceiver<M = Self::M>> {
|
||||||
|
Box::new(self.clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Implement `std::clone::Clone` for `Box<dyn AlertReceiver<M= M>>`:
|
||||||
|
// This allows `Box<dyn AlertReceiver>` to be cloned.
|
||||||
|
// The `+ 'static` lifetime bound is often necessary for trait objects stored in collections,
|
||||||
|
// ensuring they live long enough.
|
||||||
|
impl<M: MonitoringSystem + 'static> Clone for Box<dyn AlertReceiver<M= M>> {
|
||||||
|
fn clone(&self) -> Self {
|
||||||
|
self.box_clone() // Call the custom `box_clone` method
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MonitoringConfig can now derive Clone because its `receivers` field
|
||||||
|
// (Vec<Box<dyn AlertReceiver<M = M>>>) is now cloneable.
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct MonitoringConfig <M: MonitoringSystem + 'static>{
|
||||||
|
receivers: Vec<Box<dyn AlertReceiver<M = M>>>
|
||||||
|
}
|
||||||
|
|
||||||
|
// Example usage to demonstrate compilation and functionality
|
||||||
|
fn main() {
|
||||||
|
let prometheus_instance = Prometheus{};
|
||||||
|
let discord_webhook_instance = DiscordWebhook{};
|
||||||
|
|
||||||
|
let mut config = MonitoringConfig {
|
||||||
|
receivers: Vec::new()
|
||||||
|
};
|
||||||
|
|
||||||
|
// Create a boxed alert receiver
|
||||||
|
let boxed_receiver: Box<dyn AlertReceiver<M = Prometheus>> = Box::new(discord_webhook_instance);
|
||||||
|
config.receivers.push(boxed_receiver);
|
||||||
|
|
||||||
|
// Clone the config, which will now correctly clone the boxed receiver
|
||||||
|
let cloned_config = config.clone();
|
||||||
|
|
||||||
|
println!("Original config has {} receivers.", config.receivers.len());
|
||||||
|
println!("Cloned config has {} receivers.", cloned_config.receivers.len());
|
||||||
|
|
||||||
|
// Example of using the installed receiver
|
||||||
|
if let Some(receiver) = config.receivers.get(0) {
|
||||||
|
let _ = receiver.install(&prometheus_instance);
|
||||||
|
}
|
||||||
|
}
|
@ -4,7 +4,7 @@ use harmony::{
|
|||||||
maestro::Maestro,
|
maestro::Maestro,
|
||||||
modules::{
|
modules::{
|
||||||
lamp::{LAMPConfig, LAMPScore},
|
lamp::{LAMPConfig, LAMPScore},
|
||||||
monitoring::monitoring_alerting::MonitoringAlertingStackScore,
|
monitoring::alert_channel::discord_alert_channel::DiscordWebhook,
|
||||||
},
|
},
|
||||||
topology::{K8sAnywhereTopology, Url},
|
topology::{K8sAnywhereTopology, Url},
|
||||||
};
|
};
|
||||||
@ -32,6 +32,16 @@ async fn main() {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
//let monitoring = MonitoringAlertingScore {
|
||||||
|
// alert_receivers: vec![Box::new(DiscordWebhook {
|
||||||
|
// url: Url::Url(url::Url::parse("https://discord.idonotexist.com").unwrap()),
|
||||||
|
// // TODO write url macro
|
||||||
|
// // url: url!("https://discord.idonotexist.com"),
|
||||||
|
// })],
|
||||||
|
// alert_rules: vec![],
|
||||||
|
// scrape_targets: vec![],
|
||||||
|
//};
|
||||||
|
|
||||||
// You can choose the type of Topology you want, we suggest starting with the
|
// You can choose the type of Topology you want, we suggest starting with the
|
||||||
// K8sAnywhereTopology as it is the most automatic one that enables you to easily deploy
|
// K8sAnywhereTopology as it is the most automatic one that enables you to easily deploy
|
||||||
// locally, to development environment from a CI, to staging, and to production with settings
|
// locally, to development environment from a CI, to staging, and to production with settings
|
||||||
@ -43,10 +53,7 @@ async fn main() {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let mut monitoring_stack_score = MonitoringAlertingStackScore::new();
|
// maestro.register_all(vec![Box::new(lamp_stack)]);
|
||||||
monitoring_stack_score.namespace = Some(lamp_stack.config.namespace.clone());
|
|
||||||
|
|
||||||
maestro.register_all(vec![Box::new(lamp_stack), Box::new(monitoring_stack_score)]);
|
|
||||||
// Here we bootstrap the CLI, this gives some nice features if you need them
|
// Here we bootstrap the CLI, this gives some nice features if you need them
|
||||||
harmony_cli::init(maestro, None).await.unwrap();
|
harmony_cli::init(maestro, None).await.unwrap();
|
||||||
}
|
}
|
||||||
|
11
examples/monitoring/Cargo.toml
Normal file
11
examples/monitoring/Cargo.toml
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
[package]
|
||||||
|
name = "example-monitoring"
|
||||||
|
edition = "2024"
|
||||||
|
version.workspace = true
|
||||||
|
readme.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
harmony = { version = "0.1.0", path = "../../harmony" }
|
||||||
|
harmony_cli = { version = "0.1.0", path = "../../harmony_cli" }
|
||||||
|
tokio.workspace = true
|
25
examples/monitoring/src/main.rs
Normal file
25
examples/monitoring/src/main.rs
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
use harmony::{
|
||||||
|
inventory::Inventory, maestro::Maestro,
|
||||||
|
modules::monitoring::kube_prometheus::helm_prometheus_alert_score::HelmPrometheusAlertingScore,
|
||||||
|
topology::K8sAnywhereTopology,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[tokio::main]
|
||||||
|
async fn main() {
|
||||||
|
let alerting_score = HelmPrometheusAlertingScore { receivers: vec![] };
|
||||||
|
let mut maestro = Maestro::<K8sAnywhereTopology>::initialize(
|
||||||
|
Inventory::autoload(),
|
||||||
|
K8sAnywhereTopology::from_env(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
//let monitoring = MonitoringAlertingScore {
|
||||||
|
// alert_receivers: vec![],
|
||||||
|
// alert_rules: vec![],
|
||||||
|
// scrape_targets: vec![],
|
||||||
|
//};
|
||||||
|
//maestro.register_all(vec![Box::new(monitoring)]);
|
||||||
|
maestro.register_all(vec![Box::new(alerting_score)]);
|
||||||
|
harmony_cli::init(maestro, None).await.unwrap();
|
||||||
|
}
|
8
harmony/src/domain/topology/installable.rs
Normal file
8
harmony/src/domain/topology/installable.rs
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
use async_trait::async_trait;
|
||||||
|
|
||||||
|
use crate::interpret::InterpretError;
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
pub trait Installable {
|
||||||
|
async fn ensure_installed(&self) -> Result<(), InterpretError>;
|
||||||
|
}
|
@ -1,6 +1,7 @@
|
|||||||
mod ha_cluster;
|
mod ha_cluster;
|
||||||
mod host_binding;
|
mod host_binding;
|
||||||
mod http;
|
mod http;
|
||||||
|
pub mod installable;
|
||||||
mod k8s_anywhere;
|
mod k8s_anywhere;
|
||||||
mod localhost;
|
mod localhost;
|
||||||
pub mod oberservability;
|
pub mod oberservability;
|
||||||
|
@ -1,30 +1,61 @@
|
|||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|
||||||
use std::fmt::Debug;
|
use crate::{
|
||||||
|
data::{Id, Version},
|
||||||
|
interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
|
||||||
|
inventory::Inventory,
|
||||||
|
topology::{Topology, installable::Installable},
|
||||||
|
};
|
||||||
|
|
||||||
use crate::interpret::InterpretError;
|
pub trait AlertSender: Send + Sync + std::fmt::Debug + Installable {}
|
||||||
|
|
||||||
use crate::{interpret::Outcome, topology::Topology};
|
#[derive(Debug)]
|
||||||
|
pub struct AlertingInterpret<S: AlertSender> {
|
||||||
|
pub sender: S,
|
||||||
|
pub receivers: Vec<Box<dyn AlertReceiver<S>>>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Represents an entity responsible for collecting and organizing observability data
|
|
||||||
/// from various telemetry sources
|
|
||||||
/// A `Monitor` abstracts the logic required to scrape, aggregate, and structure
|
|
||||||
/// monitoring data, enabling consistent processing regardless of the underlying data source.
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
pub trait Monitor<T: Topology>: Debug + Send + Sync {
|
impl<S: AlertSender, T: Topology> Interpret<T> for AlertingInterpret<S> {
|
||||||
async fn deploy_monitor(
|
async fn execute(
|
||||||
&self,
|
&self,
|
||||||
|
inventory: &Inventory,
|
||||||
topology: &T,
|
topology: &T,
|
||||||
alert_receivers: Vec<AlertReceiver>,
|
) -> Result<Outcome, InterpretError> {
|
||||||
) -> Result<Outcome, InterpretError>;
|
for receiver in self.receivers.iter() {
|
||||||
|
receiver.install(&self.sender).await?;
|
||||||
|
}
|
||||||
|
todo!()
|
||||||
|
}
|
||||||
|
|
||||||
async fn delete_monitor(
|
fn get_name(&self) -> InterpretName {
|
||||||
&self,
|
todo!()
|
||||||
topolgy: &T,
|
}
|
||||||
alert_receivers: Vec<AlertReceiver>,
|
|
||||||
) -> Result<Outcome, InterpretError>;
|
fn get_version(&self) -> Version {
|
||||||
|
todo!()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_status(&self) -> InterpretStatus {
|
||||||
|
todo!()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_children(&self) -> Vec<Id> {
|
||||||
|
todo!()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct AlertReceiver {
|
#[async_trait]
|
||||||
pub receiver_id: String,
|
pub trait AlertReceiver<S: AlertSender>: std::fmt::Debug + Send + Sync {
|
||||||
|
async fn install(&self, sender: &S) -> Result<(), InterpretError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
pub trait AlertRule<S: AlertSender> {
|
||||||
|
async fn install(&self, sender: &S) -> Result<(), InterpretError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
pub trait ScrapeTarger<S: AlertSender> {
|
||||||
|
async fn install(&self, sender: &S) -> Result<(), InterpretError>;
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,20 @@
|
|||||||
|
use async_trait::async_trait;
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
interpret::InterpretError,
|
||||||
|
modules::monitoring::kube_prometheus::prometheus::{Prometheus, PrometheusReceiver},
|
||||||
|
topology::{Url, oberservability::monitoring::AlertReceiver},
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct DiscordWebhook {
|
||||||
|
pub name: String,
|
||||||
|
pub url: Url,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl AlertReceiver<Prometheus> for DiscordWebhook {
|
||||||
|
async fn install(&self, sender: &Prometheus) -> Result<(), InterpretError> {
|
||||||
|
sender.install_receiver(PrometheusReceiver {}).await
|
||||||
|
}
|
||||||
|
}
|
1
harmony/src/modules/monitoring/alert_channel/mod.rs
Normal file
1
harmony/src/modules/monitoring/alert_channel/mod.rs
Normal file
@ -0,0 +1 @@
|
|||||||
|
pub mod discord_alert_channel;
|
@ -1,35 +0,0 @@
|
|||||||
use std::str::FromStr;
|
|
||||||
|
|
||||||
use non_blank_string_rs::NonBlankString;
|
|
||||||
use url::Url;
|
|
||||||
|
|
||||||
use crate::modules::helm::chart::HelmChartScore;
|
|
||||||
|
|
||||||
pub fn discord_alert_manager_score(
|
|
||||||
webhook_url: Url,
|
|
||||||
namespace: String,
|
|
||||||
name: String,
|
|
||||||
) -> HelmChartScore {
|
|
||||||
let values = format!(
|
|
||||||
r#"
|
|
||||||
environment:
|
|
||||||
- name: "DISCORD_WEBHOOK"
|
|
||||||
value: "{webhook_url}"
|
|
||||||
"#,
|
|
||||||
);
|
|
||||||
|
|
||||||
HelmChartScore {
|
|
||||||
namespace: Some(NonBlankString::from_str(&namespace).unwrap()),
|
|
||||||
release_name: NonBlankString::from_str(&name).unwrap(),
|
|
||||||
chart_name: NonBlankString::from_str(
|
|
||||||
"oci://hub.nationtech.io/library/alertmanager-discord",
|
|
||||||
)
|
|
||||||
.unwrap(),
|
|
||||||
chart_version: None,
|
|
||||||
values_overrides: None,
|
|
||||||
values_yaml: Some(values.to_string()),
|
|
||||||
create_namespace: true,
|
|
||||||
install_only: true,
|
|
||||||
repository: None,
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,55 +0,0 @@
|
|||||||
use async_trait::async_trait;
|
|
||||||
use serde_json::Value;
|
|
||||||
use url::Url;
|
|
||||||
|
|
||||||
use crate::{
|
|
||||||
interpret::{InterpretError, Outcome},
|
|
||||||
topology::K8sAnywhereTopology,
|
|
||||||
};
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct DiscordWebhookConfig {
|
|
||||||
pub webhook_url: Url,
|
|
||||||
pub name: String,
|
|
||||||
pub send_resolved_notifications: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait DiscordWebhookReceiver {
|
|
||||||
fn deploy_discord_webhook_receiver(
|
|
||||||
&self,
|
|
||||||
_notification_adapter_id: &str,
|
|
||||||
) -> Result<Outcome, InterpretError>;
|
|
||||||
|
|
||||||
fn delete_discord_webhook_receiver(
|
|
||||||
&self,
|
|
||||||
_notification_adapter_id: &str,
|
|
||||||
) -> Result<Outcome, InterpretError>;
|
|
||||||
}
|
|
||||||
|
|
||||||
// trait used to generate alert manager config values impl<T: Topology + AlertManagerConfig> Monitor for KubePrometheus
|
|
||||||
pub trait AlertManagerConfig<T> {
|
|
||||||
fn get_alert_manager_config(&self) -> Result<Value, InterpretError>;
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl<T: DiscordWebhookReceiver> AlertManagerConfig<T> for DiscordWebhookConfig {
|
|
||||||
fn get_alert_manager_config(&self) -> Result<Value, InterpretError> {
|
|
||||||
todo!()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl DiscordWebhookReceiver for K8sAnywhereTopology {
|
|
||||||
fn deploy_discord_webhook_receiver(
|
|
||||||
&self,
|
|
||||||
_notification_adapter_id: &str,
|
|
||||||
) -> Result<Outcome, InterpretError> {
|
|
||||||
todo!()
|
|
||||||
}
|
|
||||||
fn delete_discord_webhook_receiver(
|
|
||||||
&self,
|
|
||||||
_notification_adapter_id: &str,
|
|
||||||
) -> Result<Outcome, InterpretError> {
|
|
||||||
todo!()
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,7 +1,5 @@
|
|||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
|
||||||
use super::monitoring_alerting::AlertChannel;
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub struct KubePrometheusConfig {
|
pub struct KubePrometheusConfig {
|
||||||
pub namespace: String,
|
pub namespace: String,
|
||||||
@ -21,7 +19,6 @@ pub struct KubePrometheusConfig {
|
|||||||
pub kube_proxy: bool,
|
pub kube_proxy: bool,
|
||||||
pub kube_state_metrics: bool,
|
pub kube_state_metrics: bool,
|
||||||
pub prometheus_operator: bool,
|
pub prometheus_operator: bool,
|
||||||
pub alert_channel: Vec<AlertChannel>,
|
|
||||||
}
|
}
|
||||||
impl KubePrometheusConfig {
|
impl KubePrometheusConfig {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
@ -30,7 +27,6 @@ impl KubePrometheusConfig {
|
|||||||
default_rules: true,
|
default_rules: true,
|
||||||
windows_monitoring: false,
|
windows_monitoring: false,
|
||||||
alert_manager: true,
|
alert_manager: true,
|
||||||
alert_channel: Vec::new(),
|
|
||||||
grafana: true,
|
grafana: true,
|
||||||
node_exporter: false,
|
node_exporter: false,
|
||||||
prometheus: true,
|
prometheus: true,
|
@ -1,11 +1,12 @@
|
|||||||
use super::{config::KubePrometheusConfig, monitoring_alerting::AlertChannel};
|
use super::config::KubePrometheusConfig;
|
||||||
use log::info;
|
|
||||||
use non_blank_string_rs::NonBlankString;
|
use non_blank_string_rs::NonBlankString;
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
|
|
||||||
use crate::modules::helm::chart::HelmChartScore;
|
use crate::modules::helm::chart::HelmChartScore;
|
||||||
|
|
||||||
pub fn kube_prometheus_helm_chart_score(config: &KubePrometheusConfig) -> HelmChartScore {
|
pub fn kube_prometheus_helm_chart_score() -> HelmChartScore {
|
||||||
|
let config = KubePrometheusConfig::new();
|
||||||
|
|
||||||
//TODO this should be make into a rule with default formatting that can be easily passed as a vec
|
//TODO this should be make into a rule with default formatting that can be easily passed as a vec
|
||||||
//to the overrides or something leaving the user to deal with formatting here seems bad
|
//to the overrides or something leaving the user to deal with formatting here seems bad
|
||||||
let default_rules = config.default_rules.to_string();
|
let default_rules = config.default_rules.to_string();
|
||||||
@ -24,7 +25,7 @@ pub fn kube_prometheus_helm_chart_score(config: &KubePrometheusConfig) -> HelmCh
|
|||||||
let node_exporter = config.node_exporter.to_string();
|
let node_exporter = config.node_exporter.to_string();
|
||||||
let prometheus_operator = config.prometheus_operator.to_string();
|
let prometheus_operator = config.prometheus_operator.to_string();
|
||||||
let prometheus = config.prometheus.to_string();
|
let prometheus = config.prometheus.to_string();
|
||||||
let mut values = format!(
|
let values = format!(
|
||||||
r#"
|
r#"
|
||||||
additionalPrometheusRulesMap:
|
additionalPrometheusRulesMap:
|
||||||
pods-status-alerts:
|
pods-status-alerts:
|
||||||
@ -143,67 +144,6 @@ prometheus:
|
|||||||
enabled: {prometheus}
|
enabled: {prometheus}
|
||||||
"#,
|
"#,
|
||||||
);
|
);
|
||||||
|
|
||||||
let alertmanager_config = alert_manager_yaml_builder(&config);
|
|
||||||
values.push_str(&alertmanager_config);
|
|
||||||
|
|
||||||
fn alert_manager_yaml_builder(config: &KubePrometheusConfig) -> String {
|
|
||||||
let mut receivers = String::new();
|
|
||||||
let mut routes = String::new();
|
|
||||||
let mut global_configs = String::new();
|
|
||||||
let alert_manager = config.alert_manager;
|
|
||||||
for alert_channel in &config.alert_channel {
|
|
||||||
match alert_channel {
|
|
||||||
AlertChannel::Discord { name, .. } => {
|
|
||||||
let (receiver, route) = discord_alert_builder(name);
|
|
||||||
info!("discord receiver: {} \nroute: {}", receiver, route);
|
|
||||||
receivers.push_str(&receiver);
|
|
||||||
routes.push_str(&route);
|
|
||||||
}
|
|
||||||
AlertChannel::Slack {
|
|
||||||
slack_channel,
|
|
||||||
webhook_url,
|
|
||||||
} => {
|
|
||||||
let (receiver, route) = slack_alert_builder(slack_channel);
|
|
||||||
info!("slack receiver: {} \nroute: {}", receiver, route);
|
|
||||||
receivers.push_str(&receiver);
|
|
||||||
|
|
||||||
routes.push_str(&route);
|
|
||||||
let global_config = format!(
|
|
||||||
r#"
|
|
||||||
global:
|
|
||||||
slack_api_url: {webhook_url}"#
|
|
||||||
);
|
|
||||||
|
|
||||||
global_configs.push_str(&global_config);
|
|
||||||
}
|
|
||||||
AlertChannel::Smpt { .. } => todo!(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
info!("after alert receiver: {}", receivers);
|
|
||||||
info!("after alert routes: {}", routes);
|
|
||||||
|
|
||||||
let alertmanager_config = format!(
|
|
||||||
r#"
|
|
||||||
alertmanager:
|
|
||||||
enabled: {alert_manager}
|
|
||||||
config: {global_configs}
|
|
||||||
route:
|
|
||||||
group_by: ['job']
|
|
||||||
group_wait: 30s
|
|
||||||
group_interval: 5m
|
|
||||||
repeat_interval: 12h
|
|
||||||
routes:
|
|
||||||
{routes}
|
|
||||||
receivers:
|
|
||||||
- name: 'null'
|
|
||||||
{receivers}"#
|
|
||||||
);
|
|
||||||
|
|
||||||
info!("alert manager config: {}", alertmanager_config);
|
|
||||||
alertmanager_config
|
|
||||||
}
|
|
||||||
|
|
||||||
HelmChartScore {
|
HelmChartScore {
|
||||||
namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()),
|
namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()),
|
||||||
release_name: NonBlankString::from_str("kube-prometheus").unwrap(),
|
release_name: NonBlankString::from_str("kube-prometheus").unwrap(),
|
||||||
@ -219,43 +159,3 @@ alertmanager:
|
|||||||
repository: None,
|
repository: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn discord_alert_builder(release_name: &String) -> (String, String) {
|
|
||||||
let discord_receiver_name = format!("Discord-{}", release_name);
|
|
||||||
let receiver = format!(
|
|
||||||
r#"
|
|
||||||
- name: '{discord_receiver_name}'
|
|
||||||
webhook_configs:
|
|
||||||
- url: 'http://{release_name}-alertmanager-discord:9094'
|
|
||||||
send_resolved: true"#,
|
|
||||||
);
|
|
||||||
let route = format!(
|
|
||||||
r#"
|
|
||||||
- receiver: '{discord_receiver_name}'
|
|
||||||
matchers:
|
|
||||||
- alertname!=Watchdog
|
|
||||||
continue: true"#,
|
|
||||||
);
|
|
||||||
(receiver, route)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn slack_alert_builder(slack_channel: &String) -> (String, String) {
|
|
||||||
let slack_receiver_name = format!("Slack-{}", slack_channel);
|
|
||||||
let receiver = format!(
|
|
||||||
r#"
|
|
||||||
- name: '{slack_receiver_name}'
|
|
||||||
slack_configs:
|
|
||||||
- channel: '{slack_channel}'
|
|
||||||
send_resolved: true
|
|
||||||
title: '{{{{ .CommonAnnotations.title }}}}'
|
|
||||||
text: '{{{{ .CommonAnnotations.description }}}}'"#,
|
|
||||||
);
|
|
||||||
let route = format!(
|
|
||||||
r#"
|
|
||||||
- receiver: '{slack_receiver_name}'
|
|
||||||
matchers:
|
|
||||||
- alertname!=Watchdog
|
|
||||||
continue: true"#,
|
|
||||||
);
|
|
||||||
(receiver, route)
|
|
||||||
}
|
|
@ -0,0 +1,2 @@
|
|||||||
|
pub mod config;
|
||||||
|
pub mod kube_prometheus_helm_chart;
|
@ -0,0 +1,47 @@
|
|||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
modules::monitoring::alert_channel::discord_alert_channel::DiscordWebhook,
|
||||||
|
score::Score,
|
||||||
|
topology::{
|
||||||
|
HelmCommand, Topology,
|
||||||
|
oberservability::monitoring::{AlertReceiver, AlertingInterpret},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::prometheus::Prometheus;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
pub struct HelmPrometheusAlertingScore {
|
||||||
|
pub receivers: Vec<Box<dyn AlertReceiver<Prometheus>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: Topology + HelmCommand> Score<T> for HelmPrometheusAlertingScore {
|
||||||
|
fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
|
||||||
|
Box::new(AlertingInterpret {
|
||||||
|
sender: Prometheus {},
|
||||||
|
receivers: vec![Box::new(DiscordWebhook {
|
||||||
|
url: todo!(),
|
||||||
|
name: todo!(),
|
||||||
|
})],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn name(&self) -> String {
|
||||||
|
"HelmPrometheusAlertingScore".to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Serialize for Box<dyn AlertReceiver<Prometheus>> {
|
||||||
|
fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
|
||||||
|
where
|
||||||
|
S: serde::Serializer,
|
||||||
|
{
|
||||||
|
todo!()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl Clone for Box<dyn AlertReceiver<Prometheus>> {
|
||||||
|
fn clone(&self) -> Self {
|
||||||
|
todo!()
|
||||||
|
}
|
||||||
|
}
|
4
harmony/src/modules/monitoring/kube_prometheus/mod.rs
Normal file
4
harmony/src/modules/monitoring/kube_prometheus/mod.rs
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
pub mod helm;
|
||||||
|
pub mod helm_prometheus_alert_score;
|
||||||
|
pub mod prometheus;
|
||||||
|
pub mod types;
|
34
harmony/src/modules/monitoring/kube_prometheus/prometheus.rs
Normal file
34
harmony/src/modules/monitoring/kube_prometheus/prometheus.rs
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
use async_trait::async_trait;
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
interpret::InterpretError,
|
||||||
|
topology::{installable::Installable, oberservability::monitoring::AlertSender},
|
||||||
|
};
|
||||||
|
|
||||||
|
impl AlertSender for Prometheus {}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Installable for Prometheus {
|
||||||
|
async fn ensure_installed(&self) -> Result<(), InterpretError> {
|
||||||
|
todo!()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct Prometheus;
|
||||||
|
|
||||||
|
impl Prometheus {
|
||||||
|
pub async fn install_receiver(
|
||||||
|
&self,
|
||||||
|
prometheus_receiver: PrometheusReceiver,
|
||||||
|
) -> Result<(), InterpretError> {
|
||||||
|
todo!()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct PrometheusReceiver {}
|
||||||
|
|
||||||
|
impl PrometheusReceiver {
|
||||||
|
fn get_prometheus_receiver_config(&self) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct AlertChannelGlobalConfig {}
|
12
harmony/src/modules/monitoring/kube_prometheus/types.rs
Normal file
12
harmony/src/modules/monitoring/kube_prometheus/types.rs
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct AlertReceiverRoute {
|
||||||
|
pub receiver: String,
|
||||||
|
pub matchers: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub r#continue: bool,
|
||||||
|
}
|
||||||
|
pub struct AlertChannelReceiver {
|
||||||
|
pub name: String,
|
||||||
|
}
|
@ -1,5 +1,2 @@
|
|||||||
mod config;
|
pub mod alert_channel;
|
||||||
mod discord_alert_manager;
|
pub mod kube_prometheus;
|
||||||
pub mod discord_webhook_sender;
|
|
||||||
mod kube_prometheus;
|
|
||||||
pub mod monitoring_alerting;
|
|
||||||
|
@ -1,158 +0,0 @@
|
|||||||
use async_trait::async_trait;
|
|
||||||
use email_address::EmailAddress;
|
|
||||||
|
|
||||||
use log::info;
|
|
||||||
use serde::Serialize;
|
|
||||||
use url::Url;
|
|
||||||
|
|
||||||
use crate::{
|
|
||||||
data::{Id, Version},
|
|
||||||
interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
|
|
||||||
inventory::Inventory,
|
|
||||||
score::Score,
|
|
||||||
topology::{HelmCommand, Topology},
|
|
||||||
};
|
|
||||||
|
|
||||||
use super::{config::KubePrometheusConfig, kube_prometheus::kube_prometheus_helm_chart_score};
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
|
||||||
pub enum AlertChannel {
|
|
||||||
Discord {
|
|
||||||
name: String,
|
|
||||||
webhook_url: Url,
|
|
||||||
},
|
|
||||||
Slack {
|
|
||||||
slack_channel: String,
|
|
||||||
webhook_url: Url,
|
|
||||||
},
|
|
||||||
//TODO test and implement in helm chart
|
|
||||||
//currently does not work
|
|
||||||
Smpt {
|
|
||||||
email_address: EmailAddress,
|
|
||||||
service_name: String,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
|
||||||
pub struct MonitoringAlertingStackScore {
|
|
||||||
pub alert_channel: Vec<AlertChannel>,
|
|
||||||
pub namespace: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MonitoringAlertingStackScore {
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self {
|
|
||||||
alert_channel: Vec::new(),
|
|
||||||
namespace: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T: Topology + HelmCommand> Score<T> for MonitoringAlertingStackScore {
|
|
||||||
fn create_interpret(&self) -> Box<dyn Interpret<T>> {
|
|
||||||
Box::new(MonitoringAlertingStackInterpret {
|
|
||||||
score: self.clone(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
fn name(&self) -> String {
|
|
||||||
format!("MonitoringAlertingStackScore")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
|
||||||
struct MonitoringAlertingStackInterpret {
|
|
||||||
score: MonitoringAlertingStackScore,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MonitoringAlertingStackInterpret {
|
|
||||||
async fn build_kube_prometheus_helm_chart_config(&self) -> KubePrometheusConfig {
|
|
||||||
let mut config = KubePrometheusConfig::new();
|
|
||||||
if let Some(ns) = &self.score.namespace {
|
|
||||||
config.namespace = ns.clone();
|
|
||||||
}
|
|
||||||
config.alert_channel = self.score.alert_channel.clone();
|
|
||||||
config
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn deploy_kube_prometheus_helm_chart_score<T: Topology + HelmCommand>(
|
|
||||||
&self,
|
|
||||||
inventory: &Inventory,
|
|
||||||
topology: &T,
|
|
||||||
config: &KubePrometheusConfig,
|
|
||||||
) -> Result<Outcome, InterpretError> {
|
|
||||||
let helm_chart = kube_prometheus_helm_chart_score(config);
|
|
||||||
helm_chart
|
|
||||||
.create_interpret()
|
|
||||||
.execute(inventory, topology)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn deploy_alert_channel_service<T: Topology + HelmCommand>(
|
|
||||||
&self,
|
|
||||||
inventory: &Inventory,
|
|
||||||
topology: &T,
|
|
||||||
config: &KubePrometheusConfig,
|
|
||||||
) -> Result<Outcome, InterpretError> {
|
|
||||||
//let mut outcomes = vec![];
|
|
||||||
|
|
||||||
//for channel in &self.score.alert_channel {
|
|
||||||
// let outcome = match channel {
|
|
||||||
// AlertChannel::Discord { .. } => {
|
|
||||||
// discord_alert_manager_score(config)
|
|
||||||
// .create_interpret()
|
|
||||||
// .execute(inventory, topology)
|
|
||||||
// .await
|
|
||||||
// }
|
|
||||||
// AlertChannel::Slack { .. } => Ok(Outcome::success(
|
|
||||||
// "No extra configs for slack alerting".to_string(),
|
|
||||||
// )),
|
|
||||||
// AlertChannel::Smpt { .. } => {
|
|
||||||
// todo!()
|
|
||||||
// }
|
|
||||||
// };
|
|
||||||
// outcomes.push(outcome);
|
|
||||||
//}
|
|
||||||
//for result in outcomes {
|
|
||||||
// result?;
|
|
||||||
//}
|
|
||||||
|
|
||||||
Ok(Outcome::success("All alert channels deployed".to_string()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl<T: Topology + HelmCommand> Interpret<T> for MonitoringAlertingStackInterpret {
|
|
||||||
async fn execute(
|
|
||||||
&self,
|
|
||||||
inventory: &Inventory,
|
|
||||||
topology: &T,
|
|
||||||
) -> Result<Outcome, InterpretError> {
|
|
||||||
let config = self.build_kube_prometheus_helm_chart_config().await;
|
|
||||||
info!("Built kube prometheus config");
|
|
||||||
info!("Installing kube prometheus chart");
|
|
||||||
self.deploy_kube_prometheus_helm_chart_score(inventory, topology, &config)
|
|
||||||
.await?;
|
|
||||||
info!("Installing alert channel service");
|
|
||||||
self.deploy_alert_channel_service(inventory, topology, &config)
|
|
||||||
.await?;
|
|
||||||
Ok(Outcome::success(format!(
|
|
||||||
"succesfully deployed monitoring and alerting stack"
|
|
||||||
)))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_name(&self) -> InterpretName {
|
|
||||||
todo!()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_version(&self) -> Version {
|
|
||||||
todo!()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_status(&self) -> InterpretStatus {
|
|
||||||
todo!()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_children(&self) -> Vec<Id> {
|
|
||||||
todo!()
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in New Issue
Block a user