Monitor an application within a tenant #86

Merged
letian merged 22 commits from feat/crd-alertmanager-configs into master 2025-08-04 21:42:05 +00:00
7 changed files with 156 additions and 29 deletions
Showing only changes of commit 7b91088828 - Show all commits

View File

@ -3,9 +3,15 @@ use std::{path::PathBuf, sync::Arc};
use harmony::{
inventory::Inventory,
maestro::Maestro,
modules::{application::{
features::{ContinuousDelivery, PrometheusApplicationMonitoring}, ApplicationScore, RustWebFramework, RustWebapp
}, monitoring::alert_channel::discord_alert_channel::DiscordWebhook},
modules::{
application::{
ApplicationScore, RustWebFramework, RustWebapp,
features::{ContinuousDelivery, PrometheusApplicationMonitoring},
},
monitoring::alert_channel::{
discord_alert_channel::DiscordWebhook, webhook_receiver::WebhookReceiver,
},
},
topology::{K8sAnywhereTopology, Url},
};
@ -24,16 +30,21 @@ async fn main() {
url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()),
};
let webhook_receiver = WebhookReceiver {
name: "sample-webhook-receiver".to_string(),
url: Url::Url(url::Url::parse("https://webhook-doesnt-exist.com").unwrap()),
};
let app = ApplicationScore {
features: vec![
// Box::new(ContinuousDelivery {
// application: application.clone(),
// }),
Box::new(ContinuousDelivery {
application: application.clone(),
}),
Box::new(PrometheusApplicationMonitoring {
application: application.clone(),
alert_receiver: vec![Box::new(discord_receiver),]
alert_receiver: vec![Box::new(discord_receiver), Box::new(webhook_receiver)],
}),
// TODO add monitoring, backups, multisite ha, etc
// TODO add backups, multisite ha, etc
],
application,
};

View File

@ -7,23 +7,28 @@ use crate::{
inventory::Inventory,
modules::{
application::{Application, ApplicationFeature},
monitoring::
kube_prometheus::{
alert_manager_config::{CRDAlertManager, CRDAlertManagerReceiver}, helm_prometheus_application_alerting::HelmPrometheusApplicationAlertingScore
}
,
monitoring::kube_prometheus::{
alert_manager_config::{CRDAlertManager, CRDAlertManagerReceiver},
helm_prometheus_application_alerting::HelmPrometheusApplicationAlertingScore,
},
},
score::Score,
topology::{oberservability::monitoring::AlertReceiver, tenant::TenantManager, HelmCommand, K8sclient, Topology},
topology::{
HelmCommand, K8sclient, Topology, oberservability::monitoring::AlertReceiver,
tenant::TenantManager,
},
};
#[derive(Debug, Clone)]
pub struct PrometheusApplicationMonitoring {
pub application: Arc<dyn Application>,
pub alert_receiver: Vec<Box<dyn CRDAlertManagerReceiver>>,}
pub alert_receiver: Vec<Box<dyn CRDAlertManagerReceiver>>,
}
#[async_trait]
impl<T: Topology + HelmCommand + 'static + TenantManager + K8sclient + std::fmt::Debug> ApplicationFeature<T> for PrometheusApplicationMonitoring {
impl<T: Topology + HelmCommand + 'static + TenantManager + K8sclient + std::fmt::Debug>
ApplicationFeature<T> for PrometheusApplicationMonitoring
{
async fn ensure_installed(&self, topology: &T) -> Result<(), String> {
info!("Ensuring monitoring is available for application");

View File

@ -91,8 +91,6 @@ impl CRDAlertManagerReceiver for DiscordWebhook {
},
spec,
};
debug!(" am: \n{:#?}", am.clone());
am
}
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
@ -103,7 +101,14 @@ impl CRDAlertManagerReceiver for DiscordWebhook {
#[async_trait]
impl AlertReceiver<CRDAlertManager> for DiscordWebhook {
async fn install(&self, sender: &CRDAlertManager) -> Result<Outcome, InterpretError> {
todo!()
sender
.client
.apply(&sender.alertmanager_configs, Some(&sender.namespace))
.await?;
Ok(Outcome::success(format!(
"installed crd-alertmanagerconfigs for {}",
self.name
)))
Review

this Prometheus sender doesn't seem to be used, meaning the AlertReceiver<Prometheus> implementation doesn't seem to be used either (and same goes for the webhook alert channel)

this `Prometheus` sender doesn't seem to be used, meaning the `AlertReceiver<Prometheus>` implementation doesn't seem to be used either (and same goes for the webhook alert channel)
}
fn name(&self) -> String {
"discord-webhook".to_string()

View File

@ -1,17 +1,27 @@
use std::{collections::BTreeMap, sync::Arc};
use async_trait::async_trait;
use k8s_openapi::api::core::v1::Secret;
use kube::api::ObjectMeta;
use log::debug;
use serde::Serialize;
use serde_json::json;
use serde_yaml::{Mapping, Value};
use crate::{
interpret::{InterpretError, Outcome},
modules::monitoring::{
kube_prometheus::{
alert_manager_config::{
AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManager,
CRDAlertManagerReceiver,
},
prometheus::{KubePrometheus, KubePrometheusReceiver},
types::{AlertChannelConfig, AlertManagerChannelConfig},
},
prometheus::prometheus::{Prometheus, PrometheusReceiver},
},
topology::{Url, oberservability::monitoring::AlertReceiver},
topology::{Url, k8s::K8sClient, oberservability::monitoring::AlertReceiver},
};
#[derive(Debug, Clone, Serialize)]
@ -20,6 +30,90 @@ pub struct WebhookReceiver {
pub url: Url,
}
#[async_trait]
impl CRDAlertManagerReceiver for WebhookReceiver {
fn name(&self) -> String {
self.name.clone()
}
async fn configure_receiver(&self, client: &Arc<K8sClient>, ns: String) -> AlertmanagerConfig {
// let secret_name = format!("{}-secret", self.name.clone());
// let webhook_key = format!("{}", self.url.clone());
//
// let mut string_data = BTreeMap::new();
// string_data.insert("webhook-url".to_string(), webhook_key.clone());
//
// let secret = Secret {
// metadata: kube::core::ObjectMeta {
// name: Some(secret_name.clone()),
// ..Default::default()
// },
// string_data: Some(string_data),
// type_: Some("Opaque".to_string()),
// ..Default::default()
// };
//
// let _ = client.apply(&secret, Some(&ns)).await;
let spec = AlertmanagerConfigSpec {
data: json!({
"route": {
"receiver": self.name,
},
"receivers": [
{
"name": self.name,
"webhookConfigs": [
{
"url": self.url,
}
]
}
]
}),
};
let am = AlertmanagerConfig {
metadata: ObjectMeta {
name: Some(self.name.clone()),
labels: Some(std::collections::BTreeMap::from([(
"alertmanagerConfig".to_string(),
"enabled".to_string(),
)])),
namespace: Some(ns),
..Default::default()
},
spec,
};
debug!(" am: \n{:#?}", am.clone());
am
}
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
Box::new(self.clone())
}
}
#[async_trait]
impl AlertReceiver<CRDAlertManager> for WebhookReceiver {
async fn install(&self, sender: &CRDAlertManager) -> Result<Outcome, InterpretError> {
sender
.client
.apply(&sender.alertmanager_configs, Some(&sender.namespace))
.await?;
Ok(Outcome::success(format!(
"installed crd-alertmanagerconfigs for {}",
self.name
)))
}
fn name(&self) -> String {
"webhook-receiver".to_string()
}
fn clone_box(&self) -> Box<dyn AlertReceiver<CRDAlertManager>> {
Box::new(self.clone())
}
}
#[async_trait]
impl AlertReceiver<Prometheus> for WebhookReceiver {
async fn install(&self, sender: &Prometheus) -> Result<Outcome, InterpretError> {

View File

@ -91,6 +91,8 @@ use crate::{
},
};
use super::types::AlertManagerConfig;
#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
#[kube(
group = "monitoring.coreos.com",
@ -108,8 +110,9 @@ pub struct AlertmanagerConfigSpec {
#[derive(Debug, Clone)]
pub struct CRDAlertManager {
namespace: String,
client: K8sClient,
pub alertmanager_configs: AlertmanagerConfig,
pub namespace: String,
pub client: Arc<K8sClient>,
}
impl AlertSender for CRDAlertManager {

View File

@ -12,7 +12,9 @@ use crate::{
};
use super::{
alert_manager_config::{AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManager, CRDAlertManagerReceiver},
alert_manager_config::{
AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManager, CRDAlertManagerReceiver,
},
prometheus::KubePrometheus,
};
@ -50,10 +52,17 @@ impl<T: Topology + K8sclient> Interpret<T> for HelmPrometheusApplicationAlerting
) -> Result<Outcome, InterpretError> {
let client = topology.k8s_client().await.unwrap();
for receiver in self.receivers.iter() {
let alertmanager_config: AlertmanagerConfig = receiver.configure_receiver(&client, self.namespace.clone()).await;
client
.apply(&alertmanager_config, Some(&self.namespace))
.await?;
let alertmanager_config: AlertmanagerConfig = receiver
.configure_receiver(&client, self.namespace.clone())
.await;
let sender = CRDAlertManager {
alertmanager_configs: alertmanager_config,
namespace: self.namespace.clone(),
client: client.clone(),
};
receiver.install(&sender).await.map_err(|err| {
InterpretError::new(format!("failed to install receiver: {}", err))
})?;
}
Ok(Outcome::success(format!("deployed alert channels")))
}

View File

@ -1,6 +1,6 @@
pub mod alert_manager_config;
pub mod helm;
pub mod helm_prometheus_alert_score;
pub mod helm_prometheus_application_alerting;
pub mod prometheus;
pub mod types;
pub mod alert_manager_config;
pub mod helm_prometheus_application_alerting;