examples/rust/src/main.rs

@@ -43,9 +43,9 @@ async fn main() {
 
     let app = ApplicationScore {
         features: vec![
-            Box::new(ContinuousDelivery {
+            // Box::new(ContinuousDelivery {
-                application: application.clone(),
+            //     application: application.clone(),
-            }),
+            // }),
             Box::new(Monitoring {
                 application: application.clone(),
                 alert_receiver: vec![Box::new(discord_receiver), Box::new(webhook_receiver)],

harmony/src/domain/topology/k8s_anywhere.rs

@@ -12,14 +12,16 @@ use crate::{
     inventory::Inventory,
     maestro::Maestro,
     modules::{
+        application::Application,
         k3d::K3DInstallationScore,
         monitoring::kube_prometheus::crd::{
-            crd_alertmanager_config::CRDPrometheus,
+            crd_alertmanager_config::{CRDAlertManagerReceiver, CRDPrometheus},
             prometheus_operator::prometheus_operator_helm_chart_score,
         },
         prometheus::{
+            k3d_prometheus_alerting_score::K3dPrometheusCRDAlertingScore,
             k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore,
-            prometheus::PrometheusApplicationMonitoring,
+            prometheus::{PrometheusApplicationMonitoring, PrometheusMonitoring},
         },
     },
     score::Score,
@@ -29,7 +31,7 @@ use crate::{
 use super::{
     DeploymentTarget, HelmCommand, K8sclient, MultiTargetTopology, Topology,
     k8s::K8sClient,
-    oberservability::monitoring::AlertReceiver,
+    oberservability::monitoring::{AlertReceiver, AlertSender},
     tenant::{TenantConfig, TenantManager, k8s::K8sTenantManager},
 };
 
@@ -70,8 +72,59 @@ impl K8sclient for K8sAnywhereTopology {
     }
 }
 
+#[async_trait]
+impl PrometheusMonitoring<CRDPrometheus> for K8sAnywhereTopology {
+    async fn ensure_prometheus_operator(
+        &self,
+        sender: &CRDPrometheus,
+    ) -> Result<Outcome, InterpretError> {
+        let output = Command::new("sh")
+            .args(["-c", "kubectl get crd -A | grep -i prometheuses"])
+            .output()
+            .map_err(|e| InterpretError::new(format!("could not connect to cluster: {}", e)))?;
+        if output.status.success() && output.stdout.is_empty() {
+            if let Some(Some(k8s_state)) = self.k8s_state.get() {
+                match k8s_state.source {
+                    K8sSource::LocalK3d => {
+                        debug!("installing prometheus operator");
+                        let op_score =
+                            prometheus_operator_helm_chart_score(sender.namespace.clone());
+                        op_score
+                            .create_interpret()
+                            .execute(&Inventory::empty(), self)
+                            .await?;
+                        return Ok(Outcome::success(
+                            "installed prometheus operator".to_string(),
+                        ));
+                    }
+                    K8sSource::Kubeconfig => {
+                        debug!("unable to install prometheus operator, contact cluster admin");
+                        return Ok(Outcome::noop());
+                    }
+                }
+            } else {
+                warn!("Unable to detect k8s_state. Skipping Prometheus Operator install.");
+                return Ok(Outcome::noop());
+            }
+        }
+        debug!("Prometheus operator is already present, skipping install");
+        Ok(Outcome::success(
+            "prometheus operator present in cluster".to_string(),
+        ))
+    }
+}
 #[async_trait]
 impl PrometheusApplicationMonitoring<CRDPrometheus> for K8sAnywhereTopology {
+    async fn configure_receivers(
+        &self,
+        receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
+    ) -> Option<Vec<Box<dyn CRDAlertManagerReceiver>>> {
+        let Some(receivers) = receivers else {
+            return None;
+        };
+
+        todo!()
+    }
     async fn install_prometheus(
         &self,
         sender: &CRDPrometheus,
@@ -127,7 +180,10 @@ impl K8sAnywhereTopology {
     ) -> K8sPrometheusCRDAlertingScore {
         K8sPrometheusCRDAlertingScore {
             sender,
-            receivers: receivers.unwrap_or_else(Vec::new),
+            receivers: self
+                .configure_receivers(receivers)
+                .await
+                .unwrap_or_else(|| vec![]),
             service_monitors: vec![],
             prometheus_rules: vec![],
         }
@@ -266,45 +322,6 @@ impl K8sAnywhereTopology {
             )),
         }
     }
-
-    async fn ensure_prometheus_operator(
-        &self,
-        sender: &CRDPrometheus,
-    ) -> Result<Outcome, InterpretError> {
-        let output = Command::new("sh")
-            .args(["-c", "kubectl get crd -A | grep -i prometheuses"])
-            .output()
-            .map_err(|e| InterpretError::new(format!("could not connect to cluster: {}", e)))?;
-        if output.status.success() && output.stdout.is_empty() {
-            if let Some(Some(k8s_state)) = self.k8s_state.get() {
-                match k8s_state.source {
-                    K8sSource::LocalK3d => {
-                        debug!("installing prometheus operator");
-                        let op_score =
-                            prometheus_operator_helm_chart_score(sender.namespace.clone());
-                        op_score
-                            .create_interpret()
-                            .execute(&Inventory::empty(), self)
-                            .await?;
-                        return Ok(Outcome::success(
-                            "installed prometheus operator".to_string(),
-                        ));
-                    }
-                    K8sSource::Kubeconfig => {
-                        debug!("unable to install prometheus operator, contact cluster admin");
-                        return Ok(Outcome::noop());
-                    }
-                }
-            } else {
-                warn!("Unable to detect k8s_state. Skipping Prometheus Operator install.");
-                return Ok(Outcome::noop());
-            }
-        }
-        debug!("Prometheus operator is already present, skipping install");
-        Ok(Outcome::success(
-            "prometheus operator present in cluster".to_string(),
-        ))
-    }
 }
 
 #[derive(Clone, Debug)]

harmony/src/modules/application/features/monitoring.rs

@@ -2,7 +2,9 @@ use std::sync::Arc;
 
 use crate::modules::application::{Application, ApplicationFeature};
 use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore;
-use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
+use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{
+    AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
+};
 
 use crate::{
     inventory::Inventory,
@@ -18,7 +20,9 @@ use crate::{
 };
 use async_trait::async_trait;
 use base64::{Engine as _, engine::general_purpose};
+use kube::api::ObjectMeta;
 use log::{debug, info};
+use serde_json::json;
 
 #[derive(Debug, Clone)]
 pub struct Monitoring {
@@ -47,6 +51,12 @@ impl<
 
         let mut alerting_score = ApplicationMonitoringScore {
             sender: CRDPrometheus {
+                alertmanager_configs: AlertmanagerConfig {
+                    metadata: ObjectMeta {
+                        ..Default::default()
+                    },
+                    spec: AlertmanagerConfigSpec { data: json! {""} },
+                },
                 namespace: namespace.clone(),
                 client: topology.k8s_client().await.unwrap(),
             },

harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs

@@ -1,16 +1,20 @@
 use std::any::Any;
 use std::collections::BTreeMap;
+use std::sync::Arc;
 
 use async_trait::async_trait;
 use k8s_openapi::api::core::v1::Secret;
 use kube::api::ObjectMeta;
+use kube::{Api, Client, ResourceExt};
+use log::{debug, info};
 use serde::Serialize;
 use serde_json::json;
 use serde_yaml::{Mapping, Value};
 
 use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{
-    AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
+    AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManagerReceiver, CRDPrometheus,
 };
+use crate::topology::k8s::K8sClient;
 use crate::{
     interpret::{InterpretError, Outcome},
     modules::monitoring::{
@@ -30,9 +34,12 @@ pub struct DiscordWebhook {
 }
 
 #[async_trait]
-impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
+impl CRDAlertManagerReceiver for DiscordWebhook {
-    async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
+    fn name(&self) -> String {
-        let ns = sender.namespace.clone();
+        self.name.clone()
+    }
+
+    async fn configure_receiver(&self, client: &Arc<K8sClient>, ns: String) -> AlertmanagerConfig {
         let secret_name = format!("{}-secret", self.name.clone());
         let webhook_key = format!("{}", self.url.clone());
 
@@ -49,7 +56,7 @@ impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
             ..Default::default()
         };
 
-        let _ = sender.client.apply(&secret, Some(&ns)).await;
+        let _ = client.apply(&secret, Some(&ns)).await;
 
         let spec = AlertmanagerConfigSpec {
             data: json!({
@@ -74,7 +81,7 @@ impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
             }),
         };
 
-        let alertmanager_configs = AlertmanagerConfig {
+        let am = AlertmanagerConfig {
             metadata: ObjectMeta {
                 name: Some(self.name.clone()),
                 labels: Some(std::collections::BTreeMap::from([(
@@ -86,10 +93,19 @@ impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
             },
             spec,
         };
+        am
+    }
+    fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
+        Box::new(self.clone())
+    }
+}
 
+#[async_trait]
+impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
+    async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
         sender
             .client
-            .apply(&alertmanager_configs, Some(&sender.namespace))
+            .apply(&sender.alertmanager_configs, Some(&sender.namespace))
             .await?;
         Ok(Outcome::success(format!(
             "installed crd-alertmanagerconfigs for {}",

harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs

@@ -1,6 +1,7 @@
-use std::any::Any;
+use std::{any::Any, sync::Arc};
 
 use async_trait::async_trait;
+use k8s_openapi::api::core::v1::Secret;
 use kube::api::ObjectMeta;
 use log::debug;
 use serde::Serialize;
@@ -12,14 +13,14 @@ use crate::{
     modules::monitoring::{
         kube_prometheus::{
             crd::crd_alertmanager_config::{
-                AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
+                AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManagerReceiver, CRDPrometheus,
             },
             prometheus::{KubePrometheus, KubePrometheusReceiver},
             types::{AlertChannelConfig, AlertManagerChannelConfig},
         },
         prometheus::prometheus::{Prometheus, PrometheusReceiver},
     },
-    topology::{Url, oberservability::monitoring::AlertReceiver},
+    topology::{Url, k8s::K8sClient, oberservability::monitoring::AlertReceiver},
 };
 
 #[derive(Debug, Clone, Serialize)]
@@ -29,8 +30,12 @@ pub struct WebhookReceiver {
 }
 
 #[async_trait]
-impl AlertReceiver<CRDPrometheus> for WebhookReceiver {
+impl CRDAlertManagerReceiver for WebhookReceiver {
-    async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
+    fn name(&self) -> String {
+        self.name.clone()
+    }
+
+    async fn configure_receiver(&self, client: &Arc<K8sClient>, ns: String) -> AlertmanagerConfig {
         let spec = AlertmanagerConfigSpec {
             data: json!({
                 "route": {
@@ -49,41 +54,45 @@ impl AlertReceiver<CRDPrometheus> for WebhookReceiver {
             }),
         };
 
-        let alertmanager_configs = AlertmanagerConfig {
+        let am = AlertmanagerConfig {
             metadata: ObjectMeta {
                 name: Some(self.name.clone()),
                 labels: Some(std::collections::BTreeMap::from([(
                     "alertmanagerConfig".to_string(),
                     "enabled".to_string(),
                 )])),
-                namespace: Some(sender.namespace.clone()),
+                namespace: Some(ns),
                 ..Default::default()
             },
             spec,
         };
-        debug!(
+        debug!(" am: \n{:#?}", am.clone());
-            "alert manager configs: \n{:#?}",
-            alertmanager_configs.clone()
-        );
 
+        am
+    }
+    fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
+        Box::new(self.clone())
+    }
+}
+
+#[async_trait]
+impl AlertReceiver<CRDPrometheus> for WebhookReceiver {
+    async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
         sender
             .client
-            .apply(&alertmanager_configs, Some(&sender.namespace))
+            .apply(&sender.alertmanager_configs, Some(&sender.namespace))
             .await?;
         Ok(Outcome::success(format!(
             "installed crd-alertmanagerconfigs for {}",
             self.name
         )))
     }
-
     fn name(&self) -> String {
         "webhook-receiver".to_string()
     }
-
     fn clone_box(&self) -> Box<dyn AlertReceiver<CRDPrometheus>> {
         Box::new(self.clone())
     }
-
     fn as_any(&self) -> &dyn Any {
         self
     }

harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs

@@ -9,11 +9,17 @@ use crate::{
     inventory::Inventory,
     modules::{
         application::Application,
-        monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus,
+        monitoring::kube_prometheus::crd::crd_alertmanager_config::{
-        prometheus::prometheus::PrometheusApplicationMonitoring,
+            CRDAlertManagerReceiver, CRDPrometheus,
+        },
+        prometheus::prometheus::{PrometheusApplicationMonitoring, PrometheusMonitoring},
     },
     score::Score,
-    topology::{Topology, oberservability::monitoring::AlertReceiver},
+    topology::{
+        Topology,
+        oberservability::monitoring::{AlertReceiver, AlertSender},
+        tenant::TenantManager,
+    },
 };
 
 #[derive(Debug, Clone, Serialize)]

harmony/src/modules/monitoring/application_monitoring/k8s_application_monitoring_score.rs

@@ -0,0 +1,44 @@
+use std::sync::{Arc, Mutex};
+
+use serde::Serialize;
+
+use crate::{
+    modules::monitoring::{
+        kube_prometheus::types::ServiceMonitor,
+        prometheus::{prometheus::Prometheus, prometheus_config::PrometheusConfig},
+    },
+    score::Score,
+    topology::{
+        HelmCommand, Topology,
+        oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret},
+        tenant::TenantManager,
+    },
+};
+
+#[derive(Clone, Debug, Serialize)]
+pub struct ApplicationPrometheusMonitoringScore {
+    pub receivers: Vec<Box<dyn AlertReceiver<Prometheus>>>,
+    pub rules: Vec<Box<dyn AlertRule<Prometheus>>>,
+    pub service_monitors: Vec<ServiceMonitor>,
+}
+
+impl<T: Topology + HelmCommand + TenantManager> Score<T> for ApplicationPrometheusMonitoringScore {
+    fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
+        let mut prom_config = PrometheusConfig::new();
+        prom_config.alert_manager = true;
+
+        let config = Arc::new(Mutex::new(prom_config));
+        config
+            .try_lock()
+            .expect("couldn't lock config")
+            .additional_service_monitors = self.service_monitors.clone();
+        Box::new(AlertingInterpret {
+            sender: Prometheus::new(),
+            receivers: self.receivers.clone(),
+            rules: self.rules.clone(),
+        })
+    }
+    fn name(&self) -> String {
+        "ApplicationPrometheusMonitoringScore".to_string()
+    }
+}

harmony/src/modules/monitoring/application_monitoring/mod.rs

@@ -1 +1,2 @@
 pub mod application_monitoring_score;
+pub mod k8s_application_monitoring_score;

harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs

@@ -1,10 +1,13 @@
 use std::sync::Arc;
 
+use async_trait::async_trait;
 use kube::CustomResource;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
 use crate::topology::{
+    Topology,
+    installable::Installable,
     k8s::K8sClient,
     oberservability::monitoring::{AlertReceiver, AlertSender},
 };
@@ -24,6 +27,7 @@ pub struct AlertmanagerConfigSpec {
 
 #[derive(Debug, Clone, Serialize)]
 pub struct CRDPrometheus {
+    pub alertmanager_configs: AlertmanagerConfig,
     pub namespace: String,
     pub client: Arc<K8sClient>,
 }
@@ -40,6 +44,31 @@ impl Clone for Box<dyn AlertReceiver<CRDPrometheus>> {
     }
 }
 
+#[async_trait]
+pub trait CRDAlertManagerReceiver:
+    AlertReceiver<CRDPrometheus> + Send + Sync + std::fmt::Debug
+{
+    fn name(&self) -> String;
+    async fn configure_receiver(&self, client: &Arc<K8sClient>, ns: String) -> AlertmanagerConfig;
+    // This new method is for cloning the trait object
+    fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver>;
+}
+
+impl Clone for Box<dyn CRDAlertManagerReceiver> {
+    fn clone(&self) -> Self {
+        CRDAlertManagerReceiver::clone_box(self.as_ref())
+    }
+}
+
+impl Serialize for Box<dyn CRDAlertManagerReceiver> {
+    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        todo!()
+    }
+}
+
 impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> {
     fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
     where

harmony/src/modules/prometheus/k3d_prometheus_alerting_score.rs

@@ -0,0 +1,582 @@
+use std::fs;
+use std::{collections::BTreeMap, sync::Arc};
+use tempfile::tempdir;
+
+use async_trait::async_trait;
+use kube::api::ObjectMeta;
+use log::{debug, info};
+use serde::Serialize;
+use tokio::process::Command;
+
+use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{
+    AlertmanagerConfig, CRDAlertManagerReceiver, CRDPrometheus,
+};
+use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules;
+use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{
+    Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig,
+    GrafanaDatasourceSpec, GrafanaSpec,
+};
+use crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::{
+    PrometheusRule, PrometheusRuleSpec, RuleGroup,
+};
+use crate::modules::monitoring::kube_prometheus::crd::grafana_default_dashboard::build_default_dashboard;
+use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{
+    ServiceMonitor, ServiceMonitorSpec,
+};
+use crate::topology::{K8sclient, Topology, k8s::K8sClient};
+use crate::{
+    data::{Id, Version},
+    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
+    inventory::Inventory,
+    modules::monitoring::kube_prometheus::crd::{
+        crd_alertmanagers::{Alertmanager, AlertmanagerSpec},
+        crd_prometheuses::{
+            AlertmanagerEndpoints, LabelSelector, Prometheus, PrometheusSpec,
+            PrometheusSpecAlerting,
+        },
+        role::{build_prom_role, build_prom_rolebinding, build_prom_service_account},
+    },
+    score::Score,
+};
+
+use super::prometheus::PrometheusMonitoring;
+
+#[derive(Clone, Debug, Serialize)]
+pub struct K3dPrometheusCRDAlertingScore {
+    pub namespace: String,
+    pub receivers: Vec<Box<dyn CRDAlertManagerReceiver>>,
+    pub service_monitors: Vec<ServiceMonitor>,
+    pub prometheus_rules: Vec<RuleGroup>,
+}
+
+impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus>> Score<T>
+    for K3dPrometheusCRDAlertingScore
+{
+    fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
+        Box::new(K3dPrometheusCRDAlertingInterpret {
+            namespace: self.namespace.clone(),
+            receivers: self.receivers.clone(),
+            service_monitors: self.service_monitors.clone(),
+            prometheus_rules: self.prometheus_rules.clone(),
+        })
+    }
+
+    fn name(&self) -> String {
+        "CRDApplicationAlertingScore".into()
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct K3dPrometheusCRDAlertingInterpret {
+    pub namespace: String,
+    pub receivers: Vec<Box<dyn CRDAlertManagerReceiver>>,
+    pub service_monitors: Vec<ServiceMonitor>,
+    pub prometheus_rules: Vec<RuleGroup>,
+}
+
+#[async_trait]
+impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus>> Interpret<T>
+    for K3dPrometheusCRDAlertingInterpret
+{
+    async fn execute(
+        &self,
+        _inventory: &Inventory,
+        topology: &T,
+    ) -> Result<Outcome, InterpretError> {
+        let client = topology.k8s_client().await.unwrap();
+        self.ensure_prometheus_operator().await?;
+        self.ensure_grafana_operator().await?;
+        self.install_prometheus(&client).await?;
+        self.install_alert_manager(&client).await?;
+        self.install_client_kube_metrics().await?;
+        self.install_grafana(&client).await?;
+        self.install_receivers(&self.receivers, &client).await?;
+        self.install_rules(&self.prometheus_rules, &client).await?;
+        self.install_monitors(self.service_monitors.clone(), &client)
+            .await?;
+        Ok(Outcome::success(format!(
+            "deployed application monitoring composants"
+        )))
+    }
+
+    fn get_name(&self) -> InterpretName {
+        todo!()
+    }
+
+    fn get_version(&self) -> Version {
+        todo!()
+    }
+
+    fn get_status(&self) -> InterpretStatus {
+        todo!()
+    }
+
+    fn get_children(&self) -> Vec<Id> {
+        todo!()
+    }
+}
+
+impl K3dPrometheusCRDAlertingInterpret {
+    async fn crd_exists(&self, crd: &str) -> bool {
+        let output = Command::new("kubectl")
+            .args(["get", "crd", crd])
+            .output()
+            .await;
+
+        matches!(output, Ok(o) if o.status.success())
+    }
+
+    async fn install_chart(
+        &self,
+        chart_path: String,
+        chart_name: String,
+    ) -> Result<(), InterpretError> {
+        let temp_dir =
+            tempdir().map_err(|e| InterpretError::new(format!("Tempdir error: {}", e)))?;
+        let temp_path = temp_dir.path().to_path_buf();
+        debug!("Using temp directory: {}", temp_path.display());
+        let chart = format!("{}/{}", chart_path, chart_name);
+        let pull_output = Command::new("helm")
+            .args(&["pull", &chart, "--destination", temp_path.to_str().unwrap()])
+            .output()
+            .await
+            .map_err(|e| InterpretError::new(format!("Helm pull error: {}", e)))?;
+
+        if !pull_output.status.success() {
+            return Err(InterpretError::new(format!(
+                "Helm pull failed: {}",
+                String::from_utf8_lossy(&pull_output.stderr)
+            )));
+        }
+
+        let tgz_path = fs::read_dir(&temp_path)
+            .unwrap()
+            .filter_map(|entry| {
+                let entry = entry.ok()?;
+                let path = entry.path();
+                if path.extension()? == "tgz" {
+                    Some(path)
+                } else {
+                    None
+                }
+            })
+            .next()
+            .ok_or_else(|| InterpretError::new("Could not find pulled Helm chart".into()))?;
+
+        debug!("Installing chart from: {}", tgz_path.display());
+
+        let install_output = Command::new("helm")
+            .args(&[
+                "install",
+                &chart_name,
+                tgz_path.to_str().unwrap(),
+                "--namespace",
+                &self.namespace,
+                "--create-namespace",
+                "--wait",
+                "--atomic",
+            ])
+            .output()
+            .await
+            .map_err(|e| InterpretError::new(format!("Helm install error: {}", e)))?;
+
+        if !install_output.status.success() {
+            return Err(InterpretError::new(format!(
+                "Helm install failed: {}",
+                String::from_utf8_lossy(&install_output.stderr)
+            )));
+        }
+
+        debug!(
+            "Installed chart {}/{} in namespace: {}",
+            &chart_path, &chart_name, self.namespace
+        );
+        Ok(())
+    }
+
+    async fn ensure_prometheus_operator(&self) -> Result<Outcome, InterpretError> {
+        self.install_chart(
+            "oci://hub.nationtech.io/harmony".to_string(),
+            "nt-prometheus-operator".to_string(),
+        )
+        .await?;
+        Ok(Outcome::success(format!(
+            "installed prometheus operator to ns {}",
+            self.namespace.clone()
+        )))
+    }
+
+    async fn ensure_grafana_operator(&self) -> Result<Outcome, InterpretError> {
+        if self.crd_exists("grafanas.grafana.integreatly.org").await {
+            debug!("grafana CRDs already exist — skipping install.");
+            return Ok(Outcome::success("Grafana CRDs already exist".to_string()));
+        }
+
+        let _ = Command::new("helm")
+            .args(&[
+                "repo",
+                "add",
+                "grafana-operator",
+                "https://grafana.github.io/helm-charts",
+            ])
+            .output()
+            .await
+            .unwrap();
+
+        let _ = Command::new("helm")
+            .args(&["repo", "update"])
+            .output()
+            .await
+            .unwrap();
+
+        let output = Command::new("helm")
+            .args(&[
+                "install",
+                "grafana-operator",
+                "grafana-operator/grafana-operator",
+                "--namespace",
+                &self.namespace,
+                "--create-namespace",
+                "--set",
+                "namespaceScope=true",
+            ])
+            .output()
+            .await
+            .unwrap();
+
+        if !output.status.success() {
+            return Err(InterpretError::new(format!(
+                "helm install failed:\nstdout: {}\nstderr: {}",
+                String::from_utf8_lossy(&output.stdout),
+                String::from_utf8_lossy(&output.stderr)
+            )));
+        }
+
+        Ok(Outcome::success(format!(
+            "installed grafana operator in ns {}",
+            self.namespace.clone()
+        )))
+    }
+    async fn install_prometheus(&self, client: &Arc<K8sClient>) -> Result<Outcome, InterpretError> {
+        debug!(
+            "installing crd-prometheuses in namespace {}",
+            self.namespace.clone()
+        );
+        debug!("building role/rolebinding/serviceaccount for crd-prometheus");
+        let rolename = format!("{}-prom", self.namespace.clone());
+        let sa_name = format!("{}-prom-sa", self.namespace.clone());
+        let role = build_prom_role(rolename.clone(), self.namespace.clone());
+        let rolebinding =
+            build_prom_rolebinding(rolename.clone(), self.namespace.clone(), sa_name.clone());
+        let sa = build_prom_service_account(sa_name.clone(), self.namespace.clone());
+        let prom_spec = PrometheusSpec {
+            alerting: Some(PrometheusSpecAlerting {
+                alertmanagers: Some(vec![AlertmanagerEndpoints {
+                    name: Some(format!("alertmanager-operated")),
+                    namespace: Some(format!("{}", self.namespace.clone())),
+                    port: Some("web".into()),
+                    scheme: Some("http".into()),
+                }]),
+            }),
+            service_account_name: sa_name.clone(),
+            service_monitor_namespace_selector: Some(LabelSelector {
+                match_labels: BTreeMap::from([(
+                    "kubernetes.io/metadata.name".to_string(),
+                    format!("{}", self.namespace.clone()),
+                )]),
+                match_expressions: vec![],
+            }),
+            service_monitor_selector: Some(LabelSelector {
+                match_labels: BTreeMap::from([("client".to_string(), "prometheus".to_string())]),
+                ..Default::default()
+            }),
+
+            service_discovery_role: Some("Endpoints".into()),
+
+            pod_monitor_selector: Some(LabelSelector {
+                match_labels: BTreeMap::from([("client".to_string(), "prometheus".to_string())]),
+                ..Default::default()
+            }),
+
+            rule_selector: Some(LabelSelector {
+                match_labels: BTreeMap::from([("role".to_string(), "prometheus-rule".to_string())]),
+                ..Default::default()
+            }),
+
+            rule_namespace_selector: Some(LabelSelector {
+                match_labels: BTreeMap::from([(
+                    "kubernetes.io/metadata.name".to_string(),
+                    format!("{}", self.namespace.clone()),
+                )]),
+                match_expressions: vec![],
+            }),
+        };
+        let prom = Prometheus {
+            metadata: ObjectMeta {
+                name: Some(self.namespace.clone()),
+                labels: Some(std::collections::BTreeMap::from([
+                    ("alertmanagerConfig".to_string(), "enabled".to_string()),
+                    ("client".to_string(), "prometheus".to_string()),
+                ])),
+                namespace: Some(self.namespace.clone()),
+                ..Default::default()
+            },
+            spec: prom_spec,
+        };
+        client
+            .apply(&role, Some(&self.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        info!(
+            "installed prometheus role: {:#?} in ns {:#?}",
+            role.metadata.name.unwrap(),
+            role.metadata.namespace.unwrap()
+        );
+        client
+            .apply(&rolebinding, Some(&self.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        info!(
+            "installed prometheus rolebinding: {:#?} in ns {:#?}",
+            rolebinding.metadata.name.unwrap(),
+            rolebinding.metadata.namespace.unwrap()
+        );
+        client
+            .apply(&sa, Some(&self.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        info!(
+            "installed prometheus service account: {:#?} in ns {:#?}",
+            sa.metadata.name.unwrap(),
+            sa.metadata.namespace.unwrap()
+        );
+        client
+            .apply(&prom, Some(&self.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        info!(
+            "installed prometheus: {:#?} in ns {:#?}",
+            &prom.metadata.name.clone().unwrap(),
+            &prom.metadata.namespace.clone().unwrap()
+        );
+
+        Ok(Outcome::success(format!(
+            "successfully deployed crd-prometheus {:#?}",
+            prom
+        )))
+    }
+
+    async fn install_alert_manager(
+        &self,
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let am = Alertmanager {
+            metadata: ObjectMeta {
+                name: Some(self.namespace.clone()),
+                labels: Some(std::collections::BTreeMap::from([(
+                    "alertmanagerConfig".to_string(),
+                    "enabled".to_string(),
+                )])),
+                namespace: Some(self.namespace.clone()),
+                ..Default::default()
+            },
+            spec: AlertmanagerSpec::default(),
+        };
+        client
+            .apply(&am, Some(&self.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        Ok(Outcome::success(format!(
+            "successfully deployed service monitor {:#?}",
+            am.metadata.name
+        )))
+    }
+    async fn install_monitors(
+        &self,
+        mut monitors: Vec<ServiceMonitor>,
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let default_service_monitor = ServiceMonitor {
+            metadata: ObjectMeta {
+                name: Some(self.namespace.clone()),
+                labels: Some(std::collections::BTreeMap::from([
+                    ("alertmanagerConfig".to_string(), "enabled".to_string()),
+                    ("client".to_string(), "prometheus".to_string()),
+                    (
+                        "app.kubernetes.io/name".to_string(),
+                        "kube-state-metrics".to_string(),
+                    ),
+                ])),
+                namespace: Some(self.namespace.clone()),
+                ..Default::default()
+            },
+            spec: ServiceMonitorSpec::default(),
+        };
+        monitors.push(default_service_monitor);
+        for monitor in monitors.iter() {
+            client
+                .apply(monitor, Some(&self.namespace.clone()))
+                .await
+                .map_err(|e| InterpretError::new(e.to_string()))?;
+        }
+        Ok(Outcome::success(
+            "succesfully deployed service monitors".to_string(),
+        ))
+    }
+
+    async fn install_rules(
+        &self,
+        rules: &Vec<RuleGroup>,
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let mut prom_rule_spec = PrometheusRuleSpec {
+            groups: rules.clone(),
+        };
+
+        let default_rules_group = RuleGroup {
+            name: format!("default-rules"),
+            rules: build_default_application_rules(),
+        };
+
+        prom_rule_spec.groups.push(default_rules_group);
+        let prom_rules = PrometheusRule {
+            metadata: ObjectMeta {
+                name: Some(self.namespace.clone()),
+                labels: Some(std::collections::BTreeMap::from([
+                    ("alertmanagerConfig".to_string(), "enabled".to_string()),
+                    ("role".to_string(), "prometheus-rule".to_string()),
+                ])),
+                namespace: Some(self.namespace.clone()),
+                ..Default::default()
+            },
+            spec: prom_rule_spec,
+        };
+        client
+            .apply(&prom_rules, Some(&self.namespace))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        Ok(Outcome::success(format!(
+            "successfully deployed rules {:#?}",
+            prom_rules.metadata.name
+        )))
+    }
+
+    async fn install_client_kube_metrics(&self) -> Result<Outcome, InterpretError> {
+        self.install_chart(
+            "oci://hub.nationtech.io/harmony".to_string(),
+            "nt-kube-metrics".to_string(),
+        )
+        .await?;
+        Ok(Outcome::success(format!(
+            "Installed client kube metrics in ns {}",
+            &self.namespace
+        )))
+    }
+
+    async fn install_grafana(&self, client: &Arc<K8sClient>) -> Result<Outcome, InterpretError> {
+        let mut label = BTreeMap::new();
+        label.insert("dashboards".to_string(), "grafana".to_string());
+        let labels = LabelSelector {
+            match_labels: label.clone(),
+            match_expressions: vec![],
+        };
+        let mut json_data = BTreeMap::new();
+        json_data.insert("timeInterval".to_string(), "5s".to_string());
+        let namespace = self.namespace.clone();
+
+        let json = build_default_dashboard(&namespace);
+
+        let graf_data_source = GrafanaDatasource {
+            metadata: ObjectMeta {
+                name: Some(format!("grafana-datasource-{}", self.namespace.clone())),
+                namespace: Some(self.namespace.clone()),
+                ..Default::default()
+            },
+            spec: GrafanaDatasourceSpec {
+                instance_selector: labels.clone(),
+                allow_cross_namespace_import: Some(false),
+                datasource: GrafanaDatasourceConfig {
+                    access: "proxy".to_string(),
+                    database: Some("prometheus".to_string()),
+                    json_data: Some(json_data),
+                    //TODO this is fragile
+                    name: format!("prometheus-{}-0", self.namespace.clone()),
+                    r#type: "prometheus".to_string(),
+                    url: format!(
+                        "http://prometheus-operated.{}.svc.cluster.local:9090",
+                        self.namespace.clone()
+                    ),
+                },
+            },
+        };
+
+        client
+            .apply(&graf_data_source, Some(&self.namespace))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+
+        let graf_dashboard = GrafanaDashboard {
+            metadata: ObjectMeta {
+                name: Some(format!("grafana-dashboard-{}", self.namespace.clone())),
+                namespace: Some(self.namespace.clone()),
+                ..Default::default()
+            },
+            spec: GrafanaDashboardSpec {
+                resync_period: Some("30s".to_string()),
+                instance_selector: labels.clone(),
+                json,
+            },
+        };
+
+        client
+            .apply(&graf_dashboard, Some(&self.namespace))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+
+        let grafana = Grafana {
+            metadata: ObjectMeta {
+                name: Some(format!("grafana-{}", self.namespace.clone())),
+                namespace: Some(self.namespace.clone()),
+                labels: Some(label.clone()),
+                ..Default::default()
+            },
+            spec: GrafanaSpec {
+                config: None,
+                admin_user: None,
+                admin_password: None,
+                ingress: None,
+                persistence: None,
+                resources: None,
+            },
+        };
+        client
+            .apply(&grafana, Some(&self.namespace))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        Ok(Outcome::success(format!(
+            "successfully deployed grafana instance {:#?}",
+            grafana.metadata.name
+        )))
+    }
+
+    async fn install_receivers(
+        &self,
+        receivers: &Vec<Box<dyn CRDAlertManagerReceiver>>,
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        for receiver in receivers.iter() {
+            let alertmanager_config: AlertmanagerConfig = receiver
+                .configure_receiver(&client, self.namespace.clone())
+                .await;
+            let sender = CRDPrometheus {
+                alertmanager_configs: alertmanager_config,
+                namespace: self.namespace.clone(),
+                client: client.clone(),
+            };
+            receiver.install(&sender).await.map_err(|err| {
+                InterpretError::new(format!("failed to install receiver: {}", err))
+            })?;
+        }
+        Ok(Outcome::success(format!("successfully deployed receivers")))
+    }
+}

harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs

@@ -8,7 +8,9 @@ use log::{debug, info};
 use serde::Serialize;
 use tokio::process::Command;
 
-use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
+use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{
+    AlertmanagerConfig, CRDAlertManagerReceiver, CRDPrometheus,
+};
 use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules;
 use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{
     Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig,
@@ -21,7 +23,6 @@ use crate::modules::monitoring::kube_prometheus::crd::grafana_default_dashboard:
 use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{
     ServiceMonitor, ServiceMonitorSpec,
 };
-use crate::topology::oberservability::monitoring::AlertReceiver;
 use crate::topology::{K8sclient, Topology, k8s::K8sClient};
 use crate::{
     data::{Id, Version},
@@ -38,12 +39,12 @@ use crate::{
     score::Score,
 };
 
-use super::prometheus::PrometheusApplicationMonitoring;
+use super::prometheus::{PrometheusApplicationMonitoring, PrometheusMonitoring};
 
 #[derive(Clone, Debug, Serialize)]
 pub struct K8sPrometheusCRDAlertingScore {
     pub sender: CRDPrometheus,
-    pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
+    pub receivers: Vec<Box<dyn CRDAlertManagerReceiver>>,
     pub service_monitors: Vec<ServiceMonitor>,
     pub prometheus_rules: Vec<RuleGroup>,
 }
@@ -68,7 +69,7 @@ impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> S
 #[derive(Clone, Debug)]
 pub struct K8sPrometheusCRDAlertingInterpret {
     pub sender: CRDPrometheus,
-    pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
+    pub receivers: Vec<Box<dyn CRDAlertManagerReceiver>>,
     pub service_monitors: Vec<ServiceMonitor>,
     pub prometheus_rules: Vec<RuleGroup>,
 }
@@ -88,8 +89,7 @@ impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> I
         self.install_alert_manager(&client).await?;
         self.install_client_kube_metrics().await?;
         self.install_grafana(&client).await?;
-        self.install_receivers(&self.sender, &self.receivers)
+        self.install_receivers(&self.receivers, &client).await?;
-            .await?;
         self.install_rules(&self.prometheus_rules, &client).await?;
         self.install_monitors(self.service_monitors.clone(), &client)
             .await?;
@@ -246,7 +246,6 @@ impl K8sPrometheusCRDAlertingInterpret {
             self.sender.namespace.clone().clone()
         )))
     }
-
     async fn install_prometheus(&self, client: &Arc<K8sClient>) -> Result<Outcome, InterpretError> {
         debug!(
             "installing crd-prometheuses in namespace {}",
@@ -560,14 +559,22 @@ impl K8sPrometheusCRDAlertingInterpret {
 
     async fn install_receivers(
         &self,
-        sender: &CRDPrometheus,
+        receivers: &Vec<Box<dyn CRDAlertManagerReceiver>>,
-        receivers: &Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
+        client: &Arc<K8sClient>,
     ) -> Result<Outcome, InterpretError> {
         for receiver in receivers.iter() {
-            receiver.install(sender).await.map_err(|err| {
+            let alertmanager_config: AlertmanagerConfig = receiver
+                .configure_receiver(&client, self.sender.namespace.clone().clone())
+                .await;
+            let sender = CRDPrometheus {
+                alertmanager_configs: alertmanager_config,
+                namespace: self.sender.namespace.clone().clone(),
+                client: self.sender.client.clone(),
+            };
+            receiver.install(&sender).await.map_err(|err| {
                 InterpretError::new(format!("failed to install receiver: {}", err))
             })?;
         }
-        Ok(Outcome::success("successfully deployed receivers".into()))
+        Ok(Outcome::success(format!("successfully deployed receivers")))
     }
 }

harmony/src/modules/prometheus/mod.rs

@@ -1,3 +1,4 @@
 pub mod alerts;
+pub mod k3d_prometheus_alerting_score;
 pub mod k8s_prometheus_alerting_score;
 pub mod prometheus;

harmony/src/modules/prometheus/prometheus.rs

@@ -3,11 +3,21 @@ use async_trait::async_trait;
 use crate::{
     interpret::{InterpretError, Outcome},
     inventory::Inventory,
+    modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDAlertManagerReceiver,
     topology::oberservability::monitoring::{AlertReceiver, AlertSender},
 };
 
 #[async_trait]
-pub trait PrometheusApplicationMonitoring<S: AlertSender> {
+pub trait PrometheusMonitoring<S: AlertSender> {
+    async fn ensure_prometheus_operator(&self, sender: &S) -> Result<Outcome, InterpretError>;
+}
+
+#[async_trait]
+pub trait PrometheusApplicationMonitoring<S: AlertSender>: PrometheusMonitoring<S> {
+    async fn configure_receivers(
+        &self,
+        receivers: Option<Vec<Box<dyn AlertReceiver<S>>>>,
+    ) -> Option<Vec<Box<dyn CRDAlertManagerReceiver>>>;
     async fn install_prometheus(
         &self,
         sender: &S,