NationTech/harmony
6
0
Fork 1
Code Issues 35 Pull Requests 6 Actions Packages Projects Releases 1 Wiki Activity

Monitor an application within a tenant #86

Merged
letian merged 22 commits from feat/crd-alertmanager-configs into master 2025-08-04 21:42:05 +00:00
Conversation 0 Commits 22 Files Changed 43 +39 -167
9 changed files with 39 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit c6b255d0bd - Show all commits

6
examples/rust/src/main.rs
View File

@ -43,9 +43,9 @@ async fn main() {
let app = ApplicationScore {
features: vec![
// Box::new(ContinuousDelivery {
// application: application.clone(),
// }),
Box::new(ContinuousDelivery {
application: application.clone(),
}),
Box::new(Monitoring {
application: application.clone(),
alert_receiver: vec![Box::new(discord_receiver), Box::new(webhook_receiver)],
taha commented 2025-07-14 19:55:34 +00:00
Outdated
Review
Copy Link

As JG mentioned, we want to keep this as simple as possible, as opinionated as possible.

If we keep it named Monitoring we can just replace the inner implementation later.

As JG mentioned, we want to keep this as simple as possible, as opinionated as possible. If we keep it named `Monitoring` we can just replace the inner implementation later.

18
harmony/src/domain/topology/k8s_anywhere.rs
View File

@ -14,7 +14,7 @@ use crate::{
modules::{
k3d::K3DInstallationScore,
monitoring::kube_prometheus::crd::{
crd_alertmanager_config::{CRDAlertManagerReceiver, CRDPrometheus},
crd_alertmanager_config::CRDPrometheus,
prometheus_operator::prometheus_operator_helm_chart_score,
},
prometheus::{
@ -127,10 +127,7 @@ impl K8sAnywhereTopology {
) -> K8sPrometheusCRDAlertingScore {
K8sPrometheusCRDAlertingScore {
sender,
receivers: self
.configure_receivers(receivers)
.await
.unwrap_or_else(Vec::new),
receivers: receivers.unwrap_or_else(Vec::new),
service_monitors: vec![],
prometheus_rules: vec![],
}
@ -308,17 +305,6 @@ impl K8sAnywhereTopology {
"prometheus operator present in cluster".to_string(),
))
}
async fn configure_receivers(
&self,
receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
) -> Option<Vec<Box<dyn CRDAlertManagerReceiver>>> {
let Some(receivers) = receivers else {
return None;
};
todo!()
}
}
#[derive(Clone, Debug)]

12
harmony/src/modules/application/features/monitoring.rs
View File

@ -2,9 +2,7 @@ use std::sync::Arc;
use crate::modules::application::{Application, ApplicationFeature};
use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore;
use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{
AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
};
use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
use crate::{
inventory::Inventory,
@ -20,9 +18,7 @@ use crate::{
};
use async_trait::async_trait;
use base64::{Engine as _, engine::general_purpose};
use kube::api::ObjectMeta;
use log::{debug, info};
use serde_json::json;
#[derive(Debug, Clone)]
pub struct Monitoring {
@ -51,12 +47,6 @@ impl<
let mut alerting_score = ApplicationMonitoringScore {
sender: CRDPrometheus {
alertmanager_configs: AlertmanagerConfig {
metadata: ObjectMeta {
..Default::default()
},
spec: AlertmanagerConfigSpec { data: json! {""} },
},
namespace: namespace.clone(),
client: topology.k8s_client().await.unwrap(),
},

32
harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs
View File

@ -1,20 +1,16 @@
use std::any::Any;
use std::collections::BTreeMap;
use std::sync::Arc;
use async_trait::async_trait;
use k8s_openapi::api::core::v1::Secret;
use kube::api::ObjectMeta;
use kube::{Api, Client, ResourceExt};
use log::{debug, info};
use serde::Serialize;
use serde_json::json;
use serde_yaml::{Mapping, Value};
use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{
AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManagerReceiver, CRDPrometheus,
AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
};
use crate::topology::k8s::K8sClient;
use crate::{
interpret::{InterpretError, Outcome},
modules::monitoring::{
@ -34,12 +30,9 @@ pub struct DiscordWebhook {
}
#[async_trait]
impl CRDAlertManagerReceiver for DiscordWebhook {
fn name(&self) -> String {
self.name.clone()
}
async fn configure_receiver(&self, client: &Arc<K8sClient>, ns: String) -> AlertmanagerConfig {
impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
let ns = sender.namespace.clone();
let secret_name = format!("{}-secret", self.name.clone());
let webhook_key = format!("{}", self.url.clone());
@ -56,7 +49,7 @@ impl CRDAlertManagerReceiver for DiscordWebhook {
..Default::default()
};
let _ = client.apply(&secret, Some(&ns)).await;
let _ = sender.client.apply(&secret, Some(&ns)).await;
let spec = AlertmanagerConfigSpec {
data: json!({
@ -81,7 +74,7 @@ impl CRDAlertManagerReceiver for DiscordWebhook {
}),
};
AlertmanagerConfig {
let alertmanager_configs = AlertmanagerConfig {
metadata: ObjectMeta {
name: Some(self.name.clone()),
labels: Some(std::collections::BTreeMap::from([(
@ -92,20 +85,11 @@ impl CRDAlertManagerReceiver for DiscordWebhook {
..Default::default()
},
spec,
}
}
};
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
Box::new(self.clone())
}
}
#[async_trait]
impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
sender
.client
.apply(&sender.alertmanager_configs, Some(&sender.namespace))
.apply(&alertmanager_configs, Some(&sender.namespace))
.await?;
Ok(Outcome::success(format!(
"installed crd-alertmanagerconfigs for {}",

39
harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs
View File

@ -1,7 +1,6 @@
use std::{any::Any, sync::Arc};
use std::any::Any;
use async_trait::async_trait;
use k8s_openapi::api::core::v1::Secret;
use kube::api::ObjectMeta;
use log::debug;
use serde::Serialize;
@ -13,14 +12,14 @@ use crate::{
modules::monitoring::{
kube_prometheus::{
crd::crd_alertmanager_config::{
AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManagerReceiver, CRDPrometheus,
AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
},
prometheus::{KubePrometheus, KubePrometheusReceiver},
types::{AlertChannelConfig, AlertManagerChannelConfig},
},
prometheus::prometheus::{Prometheus, PrometheusReceiver},
},
topology::{Url, k8s::K8sClient, oberservability::monitoring::AlertReceiver},
topology::{Url, oberservability::monitoring::AlertReceiver},
};
#[derive(Debug, Clone, Serialize)]
@ -30,12 +29,8 @@ pub struct WebhookReceiver {
}
#[async_trait]
impl CRDAlertManagerReceiver for WebhookReceiver {
fn name(&self) -> String {
self.name.clone()
}
async fn configure_receiver(&self, client: &Arc<K8sClient>, ns: String) -> AlertmanagerConfig {
impl AlertReceiver<CRDPrometheus> for WebhookReceiver {
async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
let spec = AlertmanagerConfigSpec {
data: json!({
"route": {
@ -54,45 +49,41 @@ impl CRDAlertManagerReceiver for WebhookReceiver {
}),
};
let am = AlertmanagerConfig {
let alertmanager_configs = AlertmanagerConfig {
metadata: ObjectMeta {
name: Some(self.name.clone()),
labels: Some(std::collections::BTreeMap::from([(
"alertmanagerConfig".to_string(),
"enabled".to_string(),
)])),
namespace: Some(ns),
namespace: Some(sender.namespace.clone()),
..Default::default()
},
spec,
};
debug!(" am: \n{:#?}", am.clone());
debug!(
"alert manager configs: \n{:#?}",
alertmanager_configs.clone()
);
am
}
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
Box::new(self.clone())
}
}
#[async_trait]
impl AlertReceiver<CRDPrometheus> for WebhookReceiver {
async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
sender
.client
.apply(&sender.alertmanager_configs, Some(&sender.namespace))
.apply(&alertmanager_configs, Some(&sender.namespace))
.await?;
Ok(Outcome::success(format!(
"installed crd-alertmanagerconfigs for {}",
self.name
)))
}
fn name(&self) -> String {
"webhook-receiver".to_string()
}
fn clone_box(&self) -> Box<dyn AlertReceiver<CRDPrometheus>> {
Box::new(self.clone())
}
fn as_any(&self) -> &dyn Any {
self
}

44
harmony/src/modules/monitoring/application_monitoring/k8s_application_monitoring_score.rs
View File

@ -1,44 +0,0 @@
use std::sync::{Arc, Mutex};
use serde::Serialize;
use crate::{
modules::monitoring::{
kube_prometheus::types::ServiceMonitor,
prometheus::{prometheus::Prometheus, prometheus_config::PrometheusConfig},
},
score::Score,
topology::{
HelmCommand, Topology,
oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret},
tenant::TenantManager,
},
};
#[derive(Clone, Debug, Serialize)]
pub struct ApplicationPrometheusMonitoringScore {
pub receivers: Vec<Box<dyn AlertReceiver<Prometheus>>>,
pub rules: Vec<Box<dyn AlertRule<Prometheus>>>,
pub service_monitors: Vec<ServiceMonitor>,
}
impl<T: Topology + HelmCommand + TenantManager> Score<T> for ApplicationPrometheusMonitoringScore {
fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
let mut prom_config = PrometheusConfig::new();
prom_config.alert_manager = true;
let config = Arc::new(Mutex::new(prom_config));
config
.try_lock()
.expect("couldn't lock config")
.additional_service_monitors = self.service_monitors.clone();
Box::new(AlertingInterpret {
sender: Prometheus::new(),
receivers: self.receivers.clone(),
rules: self.rules.clone(),
})
}
fn name(&self) -> String {
"ApplicationPrometheusMonitoringScore".to_string()
}
}

1
harmony/src/modules/monitoring/application_monitoring/mod.rs
View File

@ -1,2 +1 @@
pub mod application_monitoring_score;
pub mod k8s_application_monitoring_score;

27
harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs
View File

@ -1,6 +1,5 @@
use std::sync::Arc;
use async_trait::async_trait;
use kube::CustomResource;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -25,7 +24,6 @@ pub struct AlertmanagerConfigSpec {
#[derive(Debug, Clone, Serialize)]
pub struct CRDPrometheus {
pub alertmanager_configs: AlertmanagerConfig,
pub namespace: String,
pub client: Arc<K8sClient>,
}
@ -42,31 +40,6 @@ impl Clone for Box<dyn AlertReceiver<CRDPrometheus>> {
}
}
#[async_trait]
pub trait CRDAlertManagerReceiver:
AlertReceiver<CRDPrometheus> + Send + Sync + std::fmt::Debug
{
fn name(&self) -> String;
async fn configure_receiver(&self, client: &Arc<K8sClient>, ns: String) -> AlertmanagerConfig;
// This new method is for cloning the trait object
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver>;
}
impl Clone for Box<dyn CRDAlertManagerReceiver> {
fn clone(&self) -> Self {
CRDAlertManagerReceiver::clone_box(self.as_ref())
}
}
impl Serialize for Box<dyn CRDAlertManagerReceiver> {
fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
todo!()
}
}
impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> {
fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
where

27
harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs
View File

@ -8,9 +8,7 @@ use log::{debug, info};
use serde::Serialize;
use tokio::process::Command;
use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{
AlertmanagerConfig, CRDAlertManagerReceiver, CRDPrometheus,
};
use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules;
use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{
Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig,
@ -23,6 +21,7 @@ use crate::modules::monitoring::kube_prometheus::crd::grafana_default_dashboard:
use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{
ServiceMonitor, ServiceMonitorSpec,
};
use crate::topology::oberservability::monitoring::AlertReceiver;
use crate::topology::{K8sclient, Topology, k8s::K8sClient};
use crate::{
data::{Id, Version},
@ -44,7 +43,7 @@ use super::prometheus::PrometheusApplicationMonitoring;
#[derive(Clone, Debug, Serialize)]
pub struct K8sPrometheusCRDAlertingScore {
pub sender: CRDPrometheus,
pub receivers: Vec<Box<dyn CRDAlertManagerReceiver>>,
pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
pub service_monitors: Vec<ServiceMonitor>,
pub prometheus_rules: Vec<RuleGroup>,
}
@ -69,7 +68,7 @@ impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> S
#[derive(Clone, Debug)]
pub struct K8sPrometheusCRDAlertingInterpret {
pub sender: CRDPrometheus,
pub receivers: Vec<Box<dyn CRDAlertManagerReceiver>>,
pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
pub service_monitors: Vec<ServiceMonitor>,
pub prometheus_rules: Vec<RuleGroup>,
}
@ -89,7 +88,8 @@ impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> I
self.install_alert_manager(&client).await?;
self.install_client_kube_metrics().await?;
self.install_grafana(&client).await?;
self.install_receivers(&self.receivers, &client).await?;
self.install_receivers(&self.sender, &self.receivers)
.await?;
self.install_rules(&self.prometheus_rules, &client).await?;
self.install_monitors(self.service_monitors.clone(), &client)
.await?;
@ -246,6 +246,7 @@ impl K8sPrometheusCRDAlertingInterpret {
self.sender.namespace.clone().clone()
letian commented 2025-08-02 14:08:23 +00:00
Outdated
Review
Copy Link

is there a reason why these double clone().clone()? there's a lot of them in this file

is there a reason why these double `clone().clone()`? there's a lot of them in this file
)))
}
async fn install_prometheus(&self, client: &Arc<K8sClient>) -> Result<Outcome, InterpretError> {
debug!(
"installing crd-prometheuses in namespace {}",
@ -559,19 +560,11 @@ impl K8sPrometheusCRDAlertingInterpret {
async fn install_receivers(
&self,
receivers: &Vec<Box<dyn CRDAlertManagerReceiver>>,
client: &Arc<K8sClient>,
sender: &CRDPrometheus,
receivers: &Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
) -> Result<Outcome, InterpretError> {
for receiver in receivers.iter() {
let alertmanager_config: AlertmanagerConfig = receiver
.configure_receiver(client, self.sender.namespace.clone())
.await;
let sender = CRDPrometheus {
alertmanager_configs: alertmanager_config,
namespace: self.sender.namespace.clone().clone(),
client: self.sender.client.clone(),
};
receiver.install(&sender).await.map_err(|err| {
receiver.install(sender).await.map_err(|err| {
InterpretError::new(format!("failed to install receiver: {}", err))
})?;
}