monitoring-alerting #30

Merged
wjro merged 12 commits from monitoring-alerting into master 2025-05-06 17:50:57 +00:00
Showing only changes of commit ef9c1cce77 - Show all commits

View File

@ -4,37 +4,41 @@ use non_blank_string_rs::NonBlankString;
use crate::modules::helm::chart::HelmChartScore; use crate::modules::helm::chart::HelmChartScore;
pub fn kube_prometheus_score(ns: &str) -> HelmChartScore { pub fn kube_prometheus_score(ns: &str) -> HelmChartScore {
//TODO this should be make into a rule with default formatting that can be easily passed as a vec //TODO this should be make into a rule with default formatting that can be easily passed as a vec
//to the overrides or something leaving the user to deal with formatting here seems bad //to the overrides or something leaving the user to deal with formatting here seems bad
let values = r#" let values = r#"
additionalPrometheusRules: additionalPrometheusRulesMap:
- name: kubelet-alerts pvc-alerts:
groups: groups:
- name: pvc-alerts - name: pvc-alerts
rules: rules:
- alert: 'PVC Fill Over 95 Percent In 2 Days' - alert: 'PVC Fill Over 95 Percent In 2 Days'
annotations: expr: |
description: The PVC {{ $labels.persistentvolumeclaim }} in namespace {{ (
$labels.namespace }} is predicted to fill over 95% in less than 2 days. kubelet_volume_stats_used_bytes
title: PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace /
}} will fill over 95% in less than 2 days kubelet_volume_stats_capacity_bytes
expr: "(\n kubelet_volume_stats_used_bytes \n / \n kubelet_volume_stats_capacity_bytes\n) ) > 0.95
> 0.95\nAND\npredict_linear(kubelet_volume_stats_used_bytes[2d], 2 * 24 AND
* 60 * 60)\n/\nkubelet_volume_stats_capacity_bytes\n> 0.95\n" predict_linear(kubelet_volume_stats_used_bytes[2d], 2 * 24 * 60 * 60)
for: 5m /
labels: kubelet_volume_stats_capacity_bytes
severity: warning > 0.95
for: 1m
labels:
severity: warning
annotations:
description: The PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} is predicted to fill over 95% in less than 2 days.
title: PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} will fill over 95% in less than 2 days
"#; "#;
HelmChartScore { HelmChartScore {
namespace: Some(NonBlankString::from_str(ns).unwrap()), namespace: Some(NonBlankString::from_str(ns).unwrap()),
release_name: NonBlankString::from_str("kube-prometheus").unwrap(), release_name: NonBlankString::from_str("kube-prometheus").unwrap(),
chart_name: NonBlankString::from_str( chart_name: NonBlankString::from_str(
"oci://registry-1.docker.io/bitnamicharts/kube-prometheus" "oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack", //use kube prometheus chart which includes grafana, prometheus, alert
//use kube prometheus chart which includes grafana, prometheus, alert //manager, etc
//manager, etc )
)
.unwrap(), .unwrap(),
chart_version: None, chart_version: None,
values_overrides: None, values_overrides: None,