monitoring-alerting #30

Merged
wjro merged 12 commits from monitoring-alerting into master 2025-05-06 17:50:57 +00:00
Showing only changes of commit ef9c1cce77 - Show all commits

View File

@ -4,37 +4,41 @@ use non_blank_string_rs::NonBlankString;
use crate::modules::helm::chart::HelmChartScore;
pub fn kube_prometheus_score(ns: &str) -> HelmChartScore {
//TODO this should be make into a rule with default formatting that can be easily passed as a vec
//to the overrides or something leaving the user to deal with formatting here seems bad
//TODO this should be make into a rule with default formatting that can be easily passed as a vec
//to the overrides or something leaving the user to deal with formatting here seems bad
let values = r#"
additionalPrometheusRules:
- name: kubelet-alerts
groups:
- name: pvc-alerts
rules:
- alert: 'PVC Fill Over 95 Percent In 2 Days'
annotations:
description: The PVC {{ $labels.persistentvolumeclaim }} in namespace {{
$labels.namespace }} is predicted to fill over 95% in less than 2 days.
title: PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace
}} will fill over 95% in less than 2 days
expr: "(\n kubelet_volume_stats_used_bytes \n / \n kubelet_volume_stats_capacity_bytes\n)
> 0.95\nAND\npredict_linear(kubelet_volume_stats_used_bytes[2d], 2 * 24
* 60 * 60)\n/\nkubelet_volume_stats_capacity_bytes\n> 0.95\n"
for: 5m
labels:
severity: warning
additionalPrometheusRulesMap:
pvc-alerts:
groups:
- name: pvc-alerts
rules:
- alert: 'PVC Fill Over 95 Percent In 2 Days'
expr: |
(
kubelet_volume_stats_used_bytes
/
kubelet_volume_stats_capacity_bytes
) > 0.95
AND
predict_linear(kubelet_volume_stats_used_bytes[2d], 2 * 24 * 60 * 60)
/
kubelet_volume_stats_capacity_bytes
> 0.95
for: 1m
labels:
severity: warning
annotations:
description: The PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} is predicted to fill over 95% in less than 2 days.
title: PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} will fill over 95% in less than 2 days
"#;
HelmChartScore {
namespace: Some(NonBlankString::from_str(ns).unwrap()),
release_name: NonBlankString::from_str("kube-prometheus").unwrap(),
chart_name: NonBlankString::from_str(
"oci://registry-1.docker.io/bitnamicharts/kube-prometheus"
//use kube prometheus chart which includes grafana, prometheus, alert
//manager, etc
)
"oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack", //use kube prometheus chart which includes grafana, prometheus, alert
//manager, etc
)
.unwrap(),
chart_version: None,
values_overrides: None,