forked from NationTech/harmony
working on implementing grafana crds via grafana operator need to link prometheus rules and alert managers in prometheus, testing it shows that prometheus isnt detecting them automatically
87 lines
3.0 KiB
Rust
87 lines
3.0 KiB
Rust
use std::collections::HashMap;
|
|
|
|
use harmony::{
|
|
inventory::Inventory,
|
|
maestro::Maestro,
|
|
modules::{
|
|
monitoring::{
|
|
alert_channel::discord_alert_channel::DiscordWebhook,
|
|
alert_rule::prometheus_alert_rule::AlertManagerRuleGroup,
|
|
kube_prometheus::{
|
|
helm_prometheus_alert_score::HelmPrometheusAlertingScore,
|
|
types::{
|
|
HTTPScheme, MatchExpression, Operator, Selector, ServiceMonitor,
|
|
ServiceMonitorEndpoint,
|
|
},
|
|
},
|
|
},
|
|
prometheus::alerts::{
|
|
infra::dell_server::{
|
|
alert_global_storage_status_critical, alert_global_storage_status_non_recoverable,
|
|
global_storage_status_degraded_non_critical,
|
|
},
|
|
k8s::pvc::high_pvc_fill_rate_over_two_days,
|
|
},
|
|
},
|
|
topology::{K8sAnywhereTopology, Url},
|
|
};
|
|
|
|
#[tokio::main]
|
|
async fn main() {
|
|
let discord_receiver = DiscordWebhook {
|
|
name: "test-discord".to_string(),
|
|
url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()),
|
|
};
|
|
|
|
let high_pvc_fill_rate_over_two_days_alert = high_pvc_fill_rate_over_two_days();
|
|
let dell_system_storage_degraded = global_storage_status_degraded_non_critical();
|
|
let alert_global_storage_status_critical = alert_global_storage_status_critical();
|
|
let alert_global_storage_status_non_recoverable = alert_global_storage_status_non_recoverable();
|
|
|
|
let additional_rules =
|
|
AlertManagerRuleGroup::new("pvc-alerts", vec![high_pvc_fill_rate_over_two_days_alert]);
|
|
let additional_rules2 = AlertManagerRuleGroup::new(
|
|
"dell-server-alerts",
|
|
vec![
|
|
dell_system_storage_degraded,
|
|
alert_global_storage_status_critical,
|
|
alert_global_storage_status_non_recoverable,
|
|
],
|
|
);
|
|
|
|
let service_monitor_endpoint = ServiceMonitorEndpoint {
|
|
port: Some("80".to_string()),
|
|
path: Some("/metrics".to_string()),
|
|
scheme: Some(HTTPScheme::HTTP),
|
|
..Default::default()
|
|
};
|
|
|
|
let service_monitor = ServiceMonitor {
|
|
name: "test-service-monitor".to_string(),
|
|
selector: Selector {
|
|
match_labels: HashMap::new(),
|
|
match_expressions: vec![MatchExpression {
|
|
key: "test".to_string(),
|
|
operator: Operator::In,
|
|
values: vec!["test-service".to_string()],
|
|
}],
|
|
},
|
|
endpoints: vec![service_monitor_endpoint],
|
|
..Default::default()
|
|
};
|
|
let alerting_score = HelmPrometheusAlertingScore {
|
|
receivers: vec![Box::new(discord_receiver)],
|
|
rules: vec![Box::new(additional_rules), Box::new(additional_rules2)],
|
|
service_monitors: vec![service_monitor],
|
|
};
|
|
let mut maestro = Maestro::<K8sAnywhereTopology>::initialize(
|
|
Inventory::autoload(),
|
|
K8sAnywhereTopology::from_env(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
maestro.register_all(vec![Box::new(alerting_score)]);
|
|
harmony_cli::init(maestro, None).await.unwrap();
|
|
}
|