Monitor an application within a tenant #86

Merged
letian merged 22 commits from feat/crd-alertmanager-configs into master 2025-08-04 21:42:05 +00:00
9 changed files with 39 additions and 167 deletions
Showing only changes of commit c6b255d0bd - Show all commits

View File

@ -43,9 +43,9 @@ async fn main() {
let app = ApplicationScore { let app = ApplicationScore {
features: vec![ features: vec![
// Box::new(ContinuousDelivery { Box::new(ContinuousDelivery {
// application: application.clone(), application: application.clone(),
// }), }),
Box::new(Monitoring { Box::new(Monitoring {
application: application.clone(), application: application.clone(),
alert_receiver: vec![Box::new(discord_receiver), Box::new(webhook_receiver)], alert_receiver: vec![Box::new(discord_receiver), Box::new(webhook_receiver)],
Outdated
Review

As JG mentioned, we want to keep this as simple as possible, as opinionated as possible.

If we keep it named Monitoring we can just replace the inner implementation later.

As JG mentioned, we want to keep this as simple as possible, as opinionated as possible. If we keep it named `Monitoring` we can just replace the inner implementation later.

View File

@ -14,7 +14,7 @@ use crate::{
modules::{ modules::{
k3d::K3DInstallationScore, k3d::K3DInstallationScore,
monitoring::kube_prometheus::crd::{ monitoring::kube_prometheus::crd::{
crd_alertmanager_config::{CRDAlertManagerReceiver, CRDPrometheus}, crd_alertmanager_config::CRDPrometheus,
prometheus_operator::prometheus_operator_helm_chart_score, prometheus_operator::prometheus_operator_helm_chart_score,
}, },
prometheus::{ prometheus::{
@ -127,10 +127,7 @@ impl K8sAnywhereTopology {
) -> K8sPrometheusCRDAlertingScore { ) -> K8sPrometheusCRDAlertingScore {
K8sPrometheusCRDAlertingScore { K8sPrometheusCRDAlertingScore {
sender, sender,
receivers: self receivers: receivers.unwrap_or_else(Vec::new),
.configure_receivers(receivers)
.await
.unwrap_or_else(Vec::new),
service_monitors: vec![], service_monitors: vec![],
prometheus_rules: vec![], prometheus_rules: vec![],
} }
@ -308,17 +305,6 @@ impl K8sAnywhereTopology {
"prometheus operator present in cluster".to_string(), "prometheus operator present in cluster".to_string(),
)) ))
} }
async fn configure_receivers(
&self,
receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
) -> Option<Vec<Box<dyn CRDAlertManagerReceiver>>> {
let Some(receivers) = receivers else {
return None;
};
todo!()
}
} }
#[derive(Clone, Debug)] #[derive(Clone, Debug)]

View File

@ -2,9 +2,7 @@ use std::sync::Arc;
use crate::modules::application::{Application, ApplicationFeature}; use crate::modules::application::{Application, ApplicationFeature};
use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore; use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore;
use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{ use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
};
use crate::{ use crate::{
inventory::Inventory, inventory::Inventory,
@ -20,9 +18,7 @@ use crate::{
}; };
use async_trait::async_trait; use async_trait::async_trait;
use base64::{Engine as _, engine::general_purpose}; use base64::{Engine as _, engine::general_purpose};
use kube::api::ObjectMeta;
use log::{debug, info}; use log::{debug, info};
use serde_json::json;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct Monitoring { pub struct Monitoring {
@ -51,12 +47,6 @@ impl<
let mut alerting_score = ApplicationMonitoringScore { let mut alerting_score = ApplicationMonitoringScore {
sender: CRDPrometheus { sender: CRDPrometheus {
alertmanager_configs: AlertmanagerConfig {
metadata: ObjectMeta {
..Default::default()
},
spec: AlertmanagerConfigSpec { data: json! {""} },
},
namespace: namespace.clone(), namespace: namespace.clone(),
client: topology.k8s_client().await.unwrap(), client: topology.k8s_client().await.unwrap(),
}, },

View File

@ -1,20 +1,16 @@
use std::any::Any; use std::any::Any;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::sync::Arc;
use async_trait::async_trait; use async_trait::async_trait;
use k8s_openapi::api::core::v1::Secret; use k8s_openapi::api::core::v1::Secret;
use kube::api::ObjectMeta; use kube::api::ObjectMeta;
use kube::{Api, Client, ResourceExt};
use log::{debug, info};
use serde::Serialize; use serde::Serialize;
use serde_json::json; use serde_json::json;
use serde_yaml::{Mapping, Value}; use serde_yaml::{Mapping, Value};
use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{ use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{
AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManagerReceiver, CRDPrometheus, AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
}; };
use crate::topology::k8s::K8sClient;
use crate::{ use crate::{
interpret::{InterpretError, Outcome}, interpret::{InterpretError, Outcome},
modules::monitoring::{ modules::monitoring::{
@ -34,12 +30,9 @@ pub struct DiscordWebhook {
} }
#[async_trait] #[async_trait]
impl CRDAlertManagerReceiver for DiscordWebhook { impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
fn name(&self) -> String { async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
self.name.clone() let ns = sender.namespace.clone();
}
async fn configure_receiver(&self, client: &Arc<K8sClient>, ns: String) -> AlertmanagerConfig {
let secret_name = format!("{}-secret", self.name.clone()); let secret_name = format!("{}-secret", self.name.clone());
let webhook_key = format!("{}", self.url.clone()); let webhook_key = format!("{}", self.url.clone());
@ -56,7 +49,7 @@ impl CRDAlertManagerReceiver for DiscordWebhook {
..Default::default() ..Default::default()
}; };
let _ = client.apply(&secret, Some(&ns)).await; let _ = sender.client.apply(&secret, Some(&ns)).await;
let spec = AlertmanagerConfigSpec { let spec = AlertmanagerConfigSpec {
data: json!({ data: json!({
@ -81,7 +74,7 @@ impl CRDAlertManagerReceiver for DiscordWebhook {
}), }),
}; };
AlertmanagerConfig { let alertmanager_configs = AlertmanagerConfig {
metadata: ObjectMeta { metadata: ObjectMeta {
name: Some(self.name.clone()), name: Some(self.name.clone()),
labels: Some(std::collections::BTreeMap::from([( labels: Some(std::collections::BTreeMap::from([(
@ -92,20 +85,11 @@ impl CRDAlertManagerReceiver for DiscordWebhook {
..Default::default() ..Default::default()
}, },
spec, spec,
} };
}
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
Box::new(self.clone())
}
}
#[async_trait]
impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
sender sender
.client .client
.apply(&sender.alertmanager_configs, Some(&sender.namespace)) .apply(&alertmanager_configs, Some(&sender.namespace))
.await?; .await?;
Ok(Outcome::success(format!( Ok(Outcome::success(format!(
"installed crd-alertmanagerconfigs for {}", "installed crd-alertmanagerconfigs for {}",

View File

@ -1,7 +1,6 @@
use std::{any::Any, sync::Arc}; use std::any::Any;
use async_trait::async_trait; use async_trait::async_trait;
use k8s_openapi::api::core::v1::Secret;
use kube::api::ObjectMeta; use kube::api::ObjectMeta;
use log::debug; use log::debug;
use serde::Serialize; use serde::Serialize;
@ -13,14 +12,14 @@ use crate::{
modules::monitoring::{ modules::monitoring::{
kube_prometheus::{ kube_prometheus::{
crd::crd_alertmanager_config::{ crd::crd_alertmanager_config::{
AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManagerReceiver, CRDPrometheus, AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
}, },
prometheus::{KubePrometheus, KubePrometheusReceiver}, prometheus::{KubePrometheus, KubePrometheusReceiver},
types::{AlertChannelConfig, AlertManagerChannelConfig}, types::{AlertChannelConfig, AlertManagerChannelConfig},
}, },
prometheus::prometheus::{Prometheus, PrometheusReceiver}, prometheus::prometheus::{Prometheus, PrometheusReceiver},
}, },
topology::{Url, k8s::K8sClient, oberservability::monitoring::AlertReceiver}, topology::{Url, oberservability::monitoring::AlertReceiver},
}; };
#[derive(Debug, Clone, Serialize)] #[derive(Debug, Clone, Serialize)]
@ -30,12 +29,8 @@ pub struct WebhookReceiver {
} }
#[async_trait] #[async_trait]
impl CRDAlertManagerReceiver for WebhookReceiver { impl AlertReceiver<CRDPrometheus> for WebhookReceiver {
fn name(&self) -> String { async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
self.name.clone()
}
async fn configure_receiver(&self, client: &Arc<K8sClient>, ns: String) -> AlertmanagerConfig {
let spec = AlertmanagerConfigSpec { let spec = AlertmanagerConfigSpec {
data: json!({ data: json!({
"route": { "route": {
@ -54,45 +49,41 @@ impl CRDAlertManagerReceiver for WebhookReceiver {
}), }),
}; };
let am = AlertmanagerConfig { let alertmanager_configs = AlertmanagerConfig {
metadata: ObjectMeta { metadata: ObjectMeta {
name: Some(self.name.clone()), name: Some(self.name.clone()),
labels: Some(std::collections::BTreeMap::from([( labels: Some(std::collections::BTreeMap::from([(
"alertmanagerConfig".to_string(), "alertmanagerConfig".to_string(),
"enabled".to_string(), "enabled".to_string(),
)])), )])),
namespace: Some(ns), namespace: Some(sender.namespace.clone()),
..Default::default() ..Default::default()
}, },
spec, spec,
}; };
debug!(" am: \n{:#?}", am.clone()); debug!(
"alert manager configs: \n{:#?}",
alertmanager_configs.clone()
);
am
}
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
Box::new(self.clone())
}
}
#[async_trait]
impl AlertReceiver<CRDPrometheus> for WebhookReceiver {
async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
sender sender
.client .client
.apply(&sender.alertmanager_configs, Some(&sender.namespace)) .apply(&alertmanager_configs, Some(&sender.namespace))
.await?; .await?;
Ok(Outcome::success(format!( Ok(Outcome::success(format!(
"installed crd-alertmanagerconfigs for {}", "installed crd-alertmanagerconfigs for {}",
self.name self.name
))) )))
} }
fn name(&self) -> String { fn name(&self) -> String {
"webhook-receiver".to_string() "webhook-receiver".to_string()
} }
fn clone_box(&self) -> Box<dyn AlertReceiver<CRDPrometheus>> { fn clone_box(&self) -> Box<dyn AlertReceiver<CRDPrometheus>> {
Box::new(self.clone()) Box::new(self.clone())
} }
fn as_any(&self) -> &dyn Any { fn as_any(&self) -> &dyn Any {
self self
} }

View File

@ -1,44 +0,0 @@
use std::sync::{Arc, Mutex};
use serde::Serialize;
use crate::{
modules::monitoring::{
kube_prometheus::types::ServiceMonitor,
prometheus::{prometheus::Prometheus, prometheus_config::PrometheusConfig},
},
score::Score,
topology::{
HelmCommand, Topology,
oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret},
tenant::TenantManager,
},
};
#[derive(Clone, Debug, Serialize)]
pub struct ApplicationPrometheusMonitoringScore {
pub receivers: Vec<Box<dyn AlertReceiver<Prometheus>>>,
pub rules: Vec<Box<dyn AlertRule<Prometheus>>>,
pub service_monitors: Vec<ServiceMonitor>,
}
impl<T: Topology + HelmCommand + TenantManager> Score<T> for ApplicationPrometheusMonitoringScore {
fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
let mut prom_config = PrometheusConfig::new();
prom_config.alert_manager = true;
let config = Arc::new(Mutex::new(prom_config));
config
.try_lock()
.expect("couldn't lock config")
.additional_service_monitors = self.service_monitors.clone();
Box::new(AlertingInterpret {
sender: Prometheus::new(),
receivers: self.receivers.clone(),
rules: self.rules.clone(),
})
}
fn name(&self) -> String {
"ApplicationPrometheusMonitoringScore".to_string()
}
}

View File

@ -1,2 +1 @@
pub mod application_monitoring_score; pub mod application_monitoring_score;
pub mod k8s_application_monitoring_score;

View File

@ -1,6 +1,5 @@
use std::sync::Arc; use std::sync::Arc;
use async_trait::async_trait;
use kube::CustomResource; use kube::CustomResource;
use schemars::JsonSchema; use schemars::JsonSchema;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -25,7 +24,6 @@ pub struct AlertmanagerConfigSpec {
#[derive(Debug, Clone, Serialize)] #[derive(Debug, Clone, Serialize)]
pub struct CRDPrometheus { pub struct CRDPrometheus {
pub alertmanager_configs: AlertmanagerConfig,
pub namespace: String, pub namespace: String,
pub client: Arc<K8sClient>, pub client: Arc<K8sClient>,
} }
@ -42,31 +40,6 @@ impl Clone for Box<dyn AlertReceiver<CRDPrometheus>> {
} }
} }
#[async_trait]
pub trait CRDAlertManagerReceiver:
AlertReceiver<CRDPrometheus> + Send + Sync + std::fmt::Debug
{
fn name(&self) -> String;
async fn configure_receiver(&self, client: &Arc<K8sClient>, ns: String) -> AlertmanagerConfig;
// This new method is for cloning the trait object
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver>;
}
impl Clone for Box<dyn CRDAlertManagerReceiver> {
fn clone(&self) -> Self {
CRDAlertManagerReceiver::clone_box(self.as_ref())
}
}
impl Serialize for Box<dyn CRDAlertManagerReceiver> {
fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
todo!()
}
}
impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> { impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> {
fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error> fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
where where

View File

@ -8,9 +8,7 @@ use log::{debug, info};
use serde::Serialize; use serde::Serialize;
use tokio::process::Command; use tokio::process::Command;
use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{ use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
AlertmanagerConfig, CRDAlertManagerReceiver, CRDPrometheus,
};
use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules; use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules;
use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{ use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{
Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig, Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig,
@ -23,6 +21,7 @@ use crate::modules::monitoring::kube_prometheus::crd::grafana_default_dashboard:
use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{ use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{
ServiceMonitor, ServiceMonitorSpec, ServiceMonitor, ServiceMonitorSpec,
}; };
use crate::topology::oberservability::monitoring::AlertReceiver;
use crate::topology::{K8sclient, Topology, k8s::K8sClient}; use crate::topology::{K8sclient, Topology, k8s::K8sClient};
use crate::{ use crate::{
data::{Id, Version}, data::{Id, Version},
@ -44,7 +43,7 @@ use super::prometheus::PrometheusApplicationMonitoring;
#[derive(Clone, Debug, Serialize)] #[derive(Clone, Debug, Serialize)]
pub struct K8sPrometheusCRDAlertingScore { pub struct K8sPrometheusCRDAlertingScore {
pub sender: CRDPrometheus, pub sender: CRDPrometheus,
pub receivers: Vec<Box<dyn CRDAlertManagerReceiver>>, pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
pub service_monitors: Vec<ServiceMonitor>, pub service_monitors: Vec<ServiceMonitor>,
pub prometheus_rules: Vec<RuleGroup>, pub prometheus_rules: Vec<RuleGroup>,
} }
@ -69,7 +68,7 @@ impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> S
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct K8sPrometheusCRDAlertingInterpret { pub struct K8sPrometheusCRDAlertingInterpret {
pub sender: CRDPrometheus, pub sender: CRDPrometheus,
pub receivers: Vec<Box<dyn CRDAlertManagerReceiver>>, pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
pub service_monitors: Vec<ServiceMonitor>, pub service_monitors: Vec<ServiceMonitor>,
pub prometheus_rules: Vec<RuleGroup>, pub prometheus_rules: Vec<RuleGroup>,
} }
@ -89,7 +88,8 @@ impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> I
self.install_alert_manager(&client).await?; self.install_alert_manager(&client).await?;
self.install_client_kube_metrics().await?; self.install_client_kube_metrics().await?;
self.install_grafana(&client).await?; self.install_grafana(&client).await?;
self.install_receivers(&self.receivers, &client).await?; self.install_receivers(&self.sender, &self.receivers)
.await?;
self.install_rules(&self.prometheus_rules, &client).await?; self.install_rules(&self.prometheus_rules, &client).await?;
self.install_monitors(self.service_monitors.clone(), &client) self.install_monitors(self.service_monitors.clone(), &client)
.await?; .await?;
@ -246,6 +246,7 @@ impl K8sPrometheusCRDAlertingInterpret {
self.sender.namespace.clone().clone() self.sender.namespace.clone().clone()

is there a reason why these double clone().clone()? there's a lot of them in this file

is there a reason why these double `clone().clone()`? there's a lot of them in this file
))) )))
} }
async fn install_prometheus(&self, client: &Arc<K8sClient>) -> Result<Outcome, InterpretError> { async fn install_prometheus(&self, client: &Arc<K8sClient>) -> Result<Outcome, InterpretError> {
debug!( debug!(
"installing crd-prometheuses in namespace {}", "installing crd-prometheuses in namespace {}",
@ -559,19 +560,11 @@ impl K8sPrometheusCRDAlertingInterpret {
async fn install_receivers( async fn install_receivers(
&self, &self,
receivers: &Vec<Box<dyn CRDAlertManagerReceiver>>, sender: &CRDPrometheus,
client: &Arc<K8sClient>, receivers: &Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
) -> Result<Outcome, InterpretError> { ) -> Result<Outcome, InterpretError> {
for receiver in receivers.iter() { for receiver in receivers.iter() {
let alertmanager_config: AlertmanagerConfig = receiver receiver.install(sender).await.map_err(|err| {
.configure_receiver(client, self.sender.namespace.clone())
.await;
let sender = CRDPrometheus {
alertmanager_configs: alertmanager_config,
namespace: self.sender.namespace.clone().clone(),
client: self.sender.client.clone(),
};
receiver.install(&sender).await.map_err(|err| {
InterpretError::new(format!("failed to install receiver: {}", err)) InterpretError::new(format!("failed to install receiver: {}", err))
})?; })?;
} }