feat: example OpenshiftClusterAlertScore

Reviewed-on: #170

examples/okd_cluster_alerts/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "example-okd-cluster-alerts"
+edition = "2024"
+version.workspace = true
+readme.workspace = true
+license.workspace = true
+publish = false
+
+[dependencies]
+harmony = { path = "../../harmony" }
+harmony_cli = { path = "../../harmony_cli" }
+harmony_types = { path = "../../harmony_types" }
+harmony_secret = { path = "../../harmony_secret" }
+harmony_secret_derive = { path = "../../harmony_secret_derive" }
+cidr = { workspace = true }
+tokio = { workspace = true }
+harmony_macros = { path = "../../harmony_macros" }
+log = { workspace = true }
+env_logger = { workspace = true }
+url = { workspace = true }
+serde.workspace = true
+brocade = { path = "../../brocade" }

examples/okd_cluster_alerts/src/main.rs

@@ -0,0 +1,26 @@
+use harmony::{
+    inventory::Inventory,
+    modules::monitoring::{
+        alert_channel::discord_alert_channel::DiscordWebhook,
+        okd::cluster_monitoring::OpenshiftClusterAlertScore,
+    },
+    topology::K8sAnywhereTopology,
+};
+use harmony_macros::hurl;
+
+#[tokio::main]
+async fn main() {
+    harmony_cli::run(
+        Inventory::autoload(),
+        K8sAnywhereTopology::from_env(),
+        vec![Box::new(OpenshiftClusterAlertScore {
+            receivers: vec![Box::new(DiscordWebhook {
+                name: "Webhook example".to_string(),
+                url: hurl!("http://something.o"),
+            })],
+        })],
+        None,
+    )
+    .await
+    .unwrap();
+}

examples/try_rust_webapp/src/main.rs

@@ -3,7 +3,7 @@ use harmony::{
     modules::{
         application::{
             ApplicationScore, RustWebFramework, RustWebapp,
-            features::{PackagingDeployment, rhob_monitoring::Monitoring},
+            features::{Monitoring, PackagingDeployment},
         },
         monitoring::alert_channel::discord_alert_channel::DiscordWebhook,
     },

harmony/src/domain/topology/k8s.rs

@@ -3,7 +3,10 @@ use std::time::Duration;
 use derive_new::new;
 use k8s_openapi::{
     ClusterResourceScope, NamespaceResourceScope,
-    api::{apps::v1::Deployment, core::v1::Pod},
+    api::{
+        apps::v1::Deployment,
+        core::v1::{Pod, ServiceAccount},
+    },
     apimachinery::pkg::version::Info,
 };
 use kube::{
@@ -21,7 +24,7 @@ use kube::{
 };
 use log::{debug, error, info, trace};
 use serde::{Serialize, de::DeserializeOwned};
-use serde_json::{Value, json};
+use serde_json::json;
 use similar::TextDiff;
 use tokio::{io::AsyncReadExt, time::sleep};
 
@@ -57,6 +60,11 @@ impl K8sClient {
         })
     }
 
+    pub async fn service_account_api(&self, namespace: &str) -> Api<ServiceAccount> {
+        let api: Api<ServiceAccount> = Api::namespaced(self.client.clone(), namespace);
+        api
+    }
+
     pub async fn get_apiserver_version(&self) -> Result<Info, Error> {
         let client: Client = self.client.clone();
         let version_info: Info = client.apiserver_version().await?;
@@ -83,6 +91,23 @@ impl K8sClient {
         Ok(resource.get(name).await?)
     }
 
+    pub async fn get_secret_json_value(
+        &self,
+        name: &str,
+        namespace: Option<&str>,
+    ) -> Result<DynamicObject, Error> {
+        self.get_resource_json_value(
+            name,
+            namespace,
+            &GroupVersionKind {
+                group: "".to_string(),
+                version: "v1".to_string(),
+                kind: "Secret".to_string(),
+            },
+        )
+        .await
+    }
+
     pub async fn get_deployment(
         &self,
         name: &str,

harmony/src/domain/topology/k8s_anywhere.rs

@@ -1,7 +1,12 @@
-use std::{process::Command, sync::Arc};
+use std::{collections::BTreeMap, process::Command, sync::Arc};
 
 use async_trait::async_trait;
-use kube::api::GroupVersionKind;
+use base64::{Engine, engine::general_purpose};
+use k8s_openapi::api::{
+    core::v1::Secret,
+    rbac::v1::{ClusterRoleBinding, RoleRef, Subject},
+};
+use kube::api::{DynamicObject, GroupVersionKind, ObjectMeta};
 use log::{debug, info, warn};
 use serde::Serialize;
 use tokio::sync::OnceCell;
@@ -12,14 +17,26 @@ use crate::{
     inventory::Inventory,
     modules::{
         k3d::K3DInstallationScore,
-        monitoring::kube_prometheus::crd::{
-            crd_alertmanager_config::CRDPrometheus,
-            prometheus_operator::prometheus_operator_helm_chart_score,
-            rhob_alertmanager_config::RHOBObservability,
+        k8s::ingress::{K8sIngressScore, PathType},
+        monitoring::{
+            grafana::{grafana::Grafana, helm::helm_grafana::grafana_helm_chart_score},
+            kube_prometheus::crd::{
+                crd_alertmanager_config::CRDPrometheus,
+                crd_grafana::{
+                    Grafana as GrafanaCRD, GrafanaCom, GrafanaDashboard,
+                    GrafanaDashboardDatasource, GrafanaDashboardSpec, GrafanaDatasource,
+                    GrafanaDatasourceConfig, GrafanaDatasourceJsonData,
+                    GrafanaDatasourceSecureJsonData, GrafanaDatasourceSpec, GrafanaSpec,
+                },
+                crd_prometheuses::LabelSelector,
+                prometheus_operator::prometheus_operator_helm_chart_score,
+                rhob_alertmanager_config::RHOBObservability,
+                service_monitor::ServiceMonitor,
+            },
         },
         prometheus::{
             k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore,
-            prometheus::PrometheusApplicationMonitoring, rhob_alerting_score::RHOBAlertingScore,
+            prometheus::PrometheusMonitoring, rhob_alerting_score::RHOBAlertingScore,
         },
     },
     score::Score,
@@ -86,41 +103,172 @@ impl K8sclient for K8sAnywhereTopology {
 }
 
 #[async_trait]
-impl PrometheusApplicationMonitoring<CRDPrometheus> for K8sAnywhereTopology {
+impl Grafana for K8sAnywhereTopology {
+    async fn ensure_grafana_operator(
+        &self,
+        inventory: &Inventory,
+    ) -> Result<PreparationOutcome, PreparationError> {
+        debug!("ensure grafana operator");
+        let client = self.k8s_client().await.unwrap();
+        let grafana_gvk = GroupVersionKind {
+            group: "grafana.integreatly.org".to_string(),
+            version: "v1beta1".to_string(),
+            kind: "Grafana".to_string(),
+        };
+        let name = "grafanas.grafana.integreatly.org";
+        let ns = "grafana";
+
+        let grafana_crd = client
+            .get_resource_json_value(name, Some(ns), &grafana_gvk)
+            .await;
+        match grafana_crd {
+            Ok(_) => {
+                return Ok(PreparationOutcome::Success {
+                    details: "Found grafana CRDs in cluster".to_string(),
+                });
+            }
+
+            Err(_) => {
+                return self
+                    .install_grafana_operator(inventory, Some("grafana"))
+                    .await;
+            }
+        };
+    }
+    async fn install_grafana(&self) -> Result<PreparationOutcome, PreparationError> {
+        let ns = "grafana";
+
+        let mut label = BTreeMap::new();
+
+        label.insert("dashboards".to_string(), "grafana".to_string());
+
+        let label_selector = LabelSelector {
+            match_labels: label.clone(),
+            match_expressions: vec![],
+        };
+
+        let client = self.k8s_client().await?;
+
+        let grafana = self.build_grafana(ns, &label);
+
+        client.apply(&grafana, Some(ns)).await?;
+        //TODO change this to a ensure ready or something better than just a timeout
+        client
+            .wait_until_deployment_ready(
+                "grafana-grafana-deployment".to_string(),
+                Some("grafana"),
+                Some(30),
+            )
+            .await?;
+
+        let sa_name = "grafana-grafana-sa";
+        let token_secret_name = "grafana-sa-token-secret";
+
+        let sa_token_secret = self.build_sa_token_secret(token_secret_name, sa_name, ns);
+
+        client.apply(&sa_token_secret, Some(ns)).await?;
+        let secret_gvk = GroupVersionKind {
+            group: "".to_string(),
+            version: "v1".to_string(),
+            kind: "Secret".to_string(),
+        };
+
+        let secret = client
+            .get_resource_json_value(token_secret_name, Some(ns), &secret_gvk)
+            .await?;
+
+        let token = format!(
+            "Bearer {}",
+            self.extract_and_normalize_token(&secret).unwrap()
+        );
+
+        debug!("creating grafana clusterrole binding");
+
+        let clusterrolebinding =
+            self.build_cluster_rolebinding(sa_name, "cluster-monitoring-view", ns);
+
+        client.apply(&clusterrolebinding, Some(ns)).await?;
+
+        debug!("creating grafana datasource crd");
+
+        let thanos_url = format!(
+            "https://{}",
+            self.get_domain("thanos-querier-openshift-monitoring")
+                .await
+                .unwrap()
+        );
+
+        let thanos_openshift_datasource = self.build_grafana_datasource(
+            "thanos-openshift-monitoring",
+            ns,
+            &label_selector,
+            &thanos_url,
+            &token,
+        );
+
+        client.apply(&thanos_openshift_datasource, Some(ns)).await?;
+
+        debug!("creating grafana dashboard crd");
+        let dashboard = self.build_grafana_dashboard(ns, &label_selector);
+
+        client.apply(&dashboard, Some(ns)).await?;
+        debug!("creating grafana ingress");
+        let grafana_ingress = self.build_grafana_ingress(ns).await;
+
+        grafana_ingress
+            .interpret(&Inventory::empty(), self)
+            .await
+            .map_err(|e| PreparationError::new(e.to_string()))?;
+
+        Ok(PreparationOutcome::Success {
+            details: "Installed grafana composants".to_string(),
+        })
+    }
+}
+
+#[async_trait]
+impl PrometheusMonitoring<CRDPrometheus> for K8sAnywhereTopology {
     async fn install_prometheus(
         &self,
         sender: &CRDPrometheus,
-        inventory: &Inventory,
-        receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
+        _inventory: &Inventory,
+        _receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
+    ) -> Result<PreparationOutcome, PreparationError> {
+        let client = self.k8s_client().await?;
+
+        for monitor in sender.service_monitor.iter() {
+            client
+                .apply(monitor, Some(&sender.namespace))
+                .await
+                .map_err(|e| PreparationError::new(e.to_string()))?;
+        }
+        Ok(PreparationOutcome::Success {
+            details: "successfuly installed prometheus components".to_string(),
+        })
+    }
+
+    async fn ensure_prometheus_operator(
+        &self,
+        sender: &CRDPrometheus,
+        _inventory: &Inventory,
     ) -> Result<PreparationOutcome, PreparationError> {
         let po_result = self.ensure_prometheus_operator(sender).await?;
 
-        if po_result == PreparationOutcome::Noop {
-            debug!("Skipping Prometheus CR installation due to missing operator.");
-            return Ok(po_result);
-        }
-
-        let result = self
-            .get_k8s_prometheus_application_score(sender.clone(), receivers)
-            .await
-            .interpret(inventory, self)
-            .await;
-
-        match result {
-            Ok(outcome) => match outcome.status {
-                InterpretStatus::SUCCESS => Ok(PreparationOutcome::Success {
-                    details: outcome.message,
-                }),
-                InterpretStatus::NOOP => Ok(PreparationOutcome::Noop),
-                _ => Err(PreparationError::new(outcome.message)),
-            },
-            Err(err) => Err(PreparationError::new(err.to_string())),
+        match po_result {
+            PreparationOutcome::Success { details: _ } => {
+                debug!("Detected prometheus crds operator present in cluster.");
+                return Ok(po_result);
+            }
+            PreparationOutcome::Noop => {
+                debug!("Skipping Prometheus CR installation due to missing operator.");
+                return Ok(po_result);
+            }
         }
     }
 }
 
 #[async_trait]
-impl PrometheusApplicationMonitoring<RHOBObservability> for K8sAnywhereTopology {
+impl PrometheusMonitoring<RHOBObservability> for K8sAnywhereTopology {
     async fn install_prometheus(
         &self,
         sender: &RHOBObservability,
@@ -154,6 +302,14 @@ impl PrometheusApplicationMonitoring<RHOBObservability> for K8sAnywhereTopology
             Err(err) => Err(PreparationError::new(err.to_string())),
         }
     }
+
+    async fn ensure_prometheus_operator(
+        &self,
+        sender: &RHOBObservability,
+        inventory: &Inventory,
+    ) -> Result<PreparationOutcome, PreparationError> {
+        todo!()
+    }
 }
 
 impl Serialize for K8sAnywhereTopology {
@@ -215,6 +371,180 @@ impl K8sAnywhereTopology {
             .await
     }
 
+    fn extract_and_normalize_token(&self, secret: &DynamicObject) -> Option<String> {
+        let token_b64 = secret
+            .data
+            .get("token")
+            .or_else(|| secret.data.get("data").and_then(|d| d.get("token")))
+            .and_then(|v| v.as_str())?;
+
+        let bytes = general_purpose::STANDARD.decode(token_b64).ok()?;
+
+        let s = String::from_utf8(bytes).ok()?;
+
+        let cleaned = s
+            .trim_matches(|c: char| c.is_whitespace() || c == '\0')
+            .to_string();
+        Some(cleaned)
+    }
+
+    pub fn build_cluster_rolebinding(
+        &self,
+        service_account_name: &str,
+        clusterrole_name: &str,
+        ns: &str,
+    ) -> ClusterRoleBinding {
+        ClusterRoleBinding {
+            metadata: ObjectMeta {
+                name: Some(format!("{}-view-binding", service_account_name)),
+                ..Default::default()
+            },
+            role_ref: RoleRef {
+                api_group: "rbac.authorization.k8s.io".into(),
+                kind: "ClusterRole".into(),
+                name: clusterrole_name.into(),
+            },
+            subjects: Some(vec![Subject {
+                kind: "ServiceAccount".into(),
+                name: service_account_name.into(),
+                namespace: Some(ns.into()),
+                ..Default::default()
+            }]),
+        }
+    }
+
+    pub fn build_sa_token_secret(
+        &self,
+        secret_name: &str,
+        service_account_name: &str,
+        ns: &str,
+    ) -> Secret {
+        let mut annotations = BTreeMap::new();
+        annotations.insert(
+            "kubernetes.io/service-account.name".to_string(),
+            service_account_name.to_string(),
+        );
+
+        Secret {
+            metadata: ObjectMeta {
+                name: Some(secret_name.into()),
+                namespace: Some(ns.into()),
+                annotations: Some(annotations),
+                ..Default::default()
+            },
+            type_: Some("kubernetes.io/service-account-token".to_string()),
+            ..Default::default()
+        }
+    }
+
+    fn build_grafana_datasource(
+        &self,
+        name: &str,
+        ns: &str,
+        label_selector: &LabelSelector,
+        url: &str,
+        token: &str,
+    ) -> GrafanaDatasource {
+        let mut json_data = BTreeMap::new();
+        json_data.insert("timeInterval".to_string(), "5s".to_string());
+
+        GrafanaDatasource {
+            metadata: ObjectMeta {
+                name: Some(name.to_string()),
+                namespace: Some(ns.to_string()),
+                ..Default::default()
+            },
+            spec: GrafanaDatasourceSpec {
+                instance_selector: label_selector.clone(),
+                allow_cross_namespace_import: Some(true),
+                values_from: None,
+                datasource: GrafanaDatasourceConfig {
+                    access: "proxy".to_string(),
+                    name: name.to_string(),
+                    r#type: "prometheus".to_string(),
+                    url: url.to_string(),
+                    database: None,
+                    json_data: Some(GrafanaDatasourceJsonData {
+                        time_interval: Some("60s".to_string()),
+                        http_header_name1: Some("Authorization".to_string()),
+                        tls_skip_verify: Some(true),
+                        oauth_pass_thru: Some(true),
+                    }),
+                    secure_json_data: Some(GrafanaDatasourceSecureJsonData {
+                        http_header_value1: Some(format!("Bearer {token}")),
+                    }),
+                    is_default: Some(false),
+                    editable: Some(true),
+                },
+            },
+        }
+    }
+
+    fn build_grafana_dashboard(
+        &self,
+        ns: &str,
+        label_selector: &LabelSelector,
+    ) -> GrafanaDashboard {
+        let graf_dashboard = GrafanaDashboard {
+            metadata: ObjectMeta {
+                name: Some(format!("grafana-dashboard-{}", ns)),
+                namespace: Some(ns.to_string()),
+                ..Default::default()
+            },
+            spec: GrafanaDashboardSpec {
+                resync_period: Some("30s".to_string()),
+                instance_selector: label_selector.clone(),
+                datasources: Some(vec![GrafanaDashboardDatasource {
+                    input_name: "DS_PROMETHEUS".to_string(),
+                    datasource_name: "thanos-openshift-monitoring".to_string(),
+                }]),
+                json: None,
+                grafana_com: Some(GrafanaCom {
+                    id: 17406,
+                    revision: None,
+                }),
+            },
+        };
+        graf_dashboard
+    }
+
+    fn build_grafana(&self, ns: &str, labels: &BTreeMap<String, String>) -> GrafanaCRD {
+        let grafana = GrafanaCRD {
+            metadata: ObjectMeta {
+                name: Some(format!("grafana-{}", ns)),
+                namespace: Some(ns.to_string()),
+                labels: Some(labels.clone()),
+                ..Default::default()
+            },
+            spec: GrafanaSpec {
+                config: None,
+                admin_user: None,
+                admin_password: None,
+                ingress: None,
+                persistence: None,
+                resources: None,
+            },
+        };
+        grafana
+    }
+
+    async fn build_grafana_ingress(&self, ns: &str) -> K8sIngressScore {
+        let domain = self.get_domain(&format!("grafana-{}", ns)).await.unwrap();
+        let name = format!("{}-grafana", ns);
+        let backend_service = format!("grafana-{}-service", ns);
+
+        K8sIngressScore {
+            name: fqdn::fqdn!(&name),
+            host: fqdn::fqdn!(&domain),
+            backend_service: fqdn::fqdn!(&backend_service),
+            port: 3000,
+            path: Some("/".to_string()),
+            path_type: Some(PathType::Prefix),
+            namespace: Some(fqdn::fqdn!(&ns)),
+            ingress_class_name: Some("openshift-default".to_string()),
+        }
+    }
+
     async fn get_cluster_observability_operator_prometheus_application_score(
         &self,
         sender: RHOBObservability,
@@ -232,13 +562,14 @@ impl K8sAnywhereTopology {
         &self,
         sender: CRDPrometheus,
         receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
+        service_monitors: Option<Vec<ServiceMonitor>>,
     ) -> K8sPrometheusCRDAlertingScore {
-        K8sPrometheusCRDAlertingScore {
+        return K8sPrometheusCRDAlertingScore {
             sender,
             receivers: receivers.unwrap_or_default(),
-            service_monitors: vec![],
+            service_monitors: service_monitors.unwrap_or_default(),
             prometheus_rules: vec![],
-        }
+        };
     }
 
     async fn openshift_ingress_operator_available(&self) -> Result<(), PreparationError> {
@@ -506,6 +837,30 @@ impl K8sAnywhereTopology {
             details: "prometheus operator present in cluster".into(),
         })
     }
+
+    async fn install_grafana_operator(
+        &self,
+        inventory: &Inventory,
+        ns: Option<&str>,
+    ) -> Result<PreparationOutcome, PreparationError> {
+        let namespace = ns.unwrap_or("grafana");
+        info!("installing grafana operator in ns {namespace}");
+        let tenant = self.get_k8s_tenant_manager()?.get_tenant_config().await;
+        let mut namespace_scope = false;
+        if tenant.is_some() {
+            namespace_scope = true;
+        }
+        let _grafana_operator_score = grafana_helm_chart_score(namespace, namespace_scope)
+            .interpret(inventory, self)
+            .await
+            .map_err(|e| PreparationError::new(e.to_string()));
+        Ok(PreparationOutcome::Success {
+            details: format!(
+                "Successfully installed grafana operator in ns {}",
+                ns.unwrap()
+            ),
+        })
+    }
 }
 
 #[derive(Clone, Debug)]

harmony/src/domain/topology/oberservability/monitoring.rs

@@ -1,6 +1,7 @@
 use std::any::Any;
 
 use async_trait::async_trait;
+use kube::api::DynamicObject;
 use log::debug;
 
 use crate::{
@@ -31,6 +32,7 @@ impl<S: AlertSender + Installable<T>, T: Topology> Interpret<T> for AlertingInte
         inventory: &Inventory,
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
+        debug!("hit sender configure for AlertingInterpret");
         self.sender.configure(inventory, topology).await?;
         for receiver in self.receivers.iter() {
             receiver.install(&self.sender).await?;
@@ -75,6 +77,14 @@ pub trait AlertReceiver<S: AlertSender>: std::fmt::Debug + Send + Sync {
     fn name(&self) -> String;
     fn clone_box(&self) -> Box<dyn AlertReceiver<S>>;
     fn as_any(&self) -> &dyn Any;
+    fn as_alertmanager_receiver(&self) -> AlertManagerReceiver;
+}
+
+#[derive(Debug)]
+pub struct AlertManagerReceiver {
+    pub receiver_config: serde_json::Value,
+    // FIXME we should not leak k8s here. DynamicObject is k8s specific
+    pub additional_ressources: Vec<DynamicObject>,
 }
 
 #[async_trait]
@@ -86,4 +96,5 @@ pub trait AlertRule<S: AlertSender>: std::fmt::Debug + Send + Sync {
 #[async_trait]
 pub trait ScrapeTarget<S: AlertSender>: std::fmt::Debug + Send + Sync {
     async fn install(&self, sender: &S) -> Result<Outcome, InterpretError>;
+    fn clone_box(&self) -> Box<dyn ScrapeTarget<S>>;
 }

harmony/src/infra/kube.rs

@@ -0,0 +1,182 @@
+use k8s_openapi::Resource as K8sResource;
+use kube::api::{ApiResource, DynamicObject, GroupVersionKind};
+use kube::core::TypeMeta;
+use serde::Serialize;
+use serde::de::DeserializeOwned;
+use serde_json::Value;
+
+/// Convert a typed Kubernetes resource `K` into a `DynamicObject`.
+///
+/// Requirements:
+/// - `K` must be a k8s_openapi resource (provides static GVK via `Resource`).
+/// - `K` must have standard Kubernetes shape (metadata + payload fields).
+///
+/// Notes:
+/// - We set `types` (apiVersion/kind) and copy `metadata`.
+/// - We place the remaining top-level fields into `obj.data` as JSON.
+/// - Scope is not encoded on the object itself; you still need the corresponding
+///   `DynamicResource` (derived from K::group/version/kind) when constructing an Api.
+///
+/// Example usage:
+///   let dyn_obj = kube_resource_to_dynamic(secret)?;
+///   let api: Api<DynamicObject> = Api::namespaced_with(client, "ns", &dr);
+///   api.patch(&dyn_obj.name_any(), &PatchParams::apply("mgr"), &Patch::Apply(dyn_obj)).await?;
+pub fn kube_resource_to_dynamic<K>(res: &K) -> Result<DynamicObject, String>
+where
+    K: K8sResource + Serialize + DeserializeOwned,
+{
+    // Serialize the typed resource to JSON so we can split metadata and payload
+    let mut v = serde_json::to_value(res).map_err(|e| format!("Failed to serialize : {e}"))?;
+    let obj = v
+        .as_object_mut()
+        .ok_or_else(|| "expected object JSON".to_string())?;
+
+    // Extract and parse metadata into kube::core::ObjectMeta
+    let metadata_value = obj
+        .remove("metadata")
+        .ok_or_else(|| "missing metadata".to_string())?;
+    let metadata: kube::core::ObjectMeta = serde_json::from_value(metadata_value)
+        .map_err(|e| format!("Failed to deserialize : {e}"))?;
+
+    // Name is required for DynamicObject::new; prefer metadata.name
+    let name = metadata
+        .name
+        .clone()
+        .ok_or_else(|| "metadata.name is required".to_string())?;
+
+    // Remaining fields (spec/status/data/etc.) become the dynamic payload
+    let payload = Value::Object(obj.clone());
+
+    // Construct the DynamicObject
+    let mut dyn_obj = DynamicObject::new(
+        &name,
+        &ApiResource::from_gvk(&GroupVersionKind::gvk(K::GROUP, K::VERSION, K::KIND)),
+    );
+    dyn_obj.types = Some(TypeMeta {
+        api_version: api_version_for::<K>(),
+        kind: K::KIND.into(),
+    });
+
+    // Preserve namespace/labels/annotations/etc.
+    dyn_obj.metadata = metadata;
+
+    // Attach payload
+    dyn_obj.data = payload;
+
+    Ok(dyn_obj)
+}
+
+/// Helper: compute apiVersion string ("group/version" or "v1" for core).
+fn api_version_for<K>() -> String
+where
+    K: K8sResource,
+{
+    let group = K::GROUP;
+    let version = K::VERSION;
+    if group.is_empty() {
+        version.to_string() // core/v1 => "v1"
+    } else {
+        format!("{}/{}", group, version)
+    }
+}
+#[cfg(test)]
+mod test {
+    use super::*;
+    use k8s_openapi::api::{
+        apps::v1::{Deployment, DeploymentSpec},
+        core::v1::{PodTemplateSpec, Secret},
+    };
+    use kube::api::ObjectMeta;
+    use pretty_assertions::assert_eq;
+
+    #[test]
+    fn secret_to_dynamic_roundtrip() {
+        // Create a sample Secret resource
+        let mut secret = Secret {
+            metadata: ObjectMeta {
+                name: Some("my-secret".to_string()),
+                ..Default::default()
+            },
+            type_: Some("kubernetes.io/service-account-token".to_string()),
+            ..Default::default()
+        };
+
+        // Convert to DynamicResource
+        let dynamic: DynamicObject =
+            kube_resource_to_dynamic(&secret).expect("Failed to convert Secret to DynamicResource");
+
+        // Serialize both the original and dynamic resources to Value
+        let original_value = serde_json::to_value(&secret).expect("Failed to serialize Secret");
+        let dynamic_value =
+            serde_json::to_value(&dynamic).expect("Failed to serialize DynamicResource");
+
+        // Assert that they are identical
+        assert_eq!(original_value, dynamic_value);
+
+        secret.metadata.namespace = Some("false".to_string());
+        let modified_value = serde_json::to_value(&secret).expect("Failed to serialize Secret");
+        assert_ne!(modified_value, dynamic_value);
+    }
+
+    #[test]
+    fn deployment_to_dynamic_roundtrip() {
+        // Create a sample Deployment with nested structures
+        let mut deployment = Deployment {
+            metadata: ObjectMeta {
+                name: Some("my-deployment".to_string()),
+                labels: Some({
+                    let mut map = std::collections::BTreeMap::new();
+                    map.insert("app".to_string(), "nginx".to_string());
+                    map
+                }),
+                ..Default::default()
+            },
+            spec: Some(DeploymentSpec {
+                replicas: Some(3),
+                selector: Default::default(),
+                template: PodTemplateSpec {
+                    metadata: Some(ObjectMeta {
+                        labels: Some({
+                            let mut map = std::collections::BTreeMap::new();
+                            map.insert("app".to_string(), "nginx".to_string());
+                            map
+                        }),
+                        ..Default::default()
+                    }),
+                    spec: Some(Default::default()), // PodSpec with empty containers for simplicity
+                },
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let dynamic = kube_resource_to_dynamic(&deployment).expect("Failed to convert Deployment");
+
+        let original_value = serde_json::to_value(&deployment).unwrap();
+        let dynamic_value = serde_json::to_value(&dynamic).unwrap();
+
+        assert_eq!(original_value, dynamic_value);
+
+        assert_eq!(
+            dynamic.data.get("spec").unwrap().get("replicas").unwrap(),
+            3
+        );
+        assert_eq!(
+            dynamic
+                .data
+                .get("spec")
+                .unwrap()
+                .get("template")
+                .unwrap()
+                .get("metadata")
+                .unwrap()
+                .get("labels")
+                .unwrap()
+                .get("app")
+                .unwrap()
+                .as_str()
+                .unwrap(),
+            "nginx".to_string()
+        );
+    }
+}

harmony/src/infra/mod.rs

@@ -3,5 +3,6 @@ pub mod executors;
 pub mod hp_ilo;
 pub mod intel_amt;
 pub mod inventory;
+pub mod kube;
 pub mod opnsense;
 mod sqlx;

harmony/src/modules/application/features/monitoring.rs

@@ -2,7 +2,11 @@ use crate::modules::application::{
     Application, ApplicationFeature, InstallationError, InstallationOutcome,
 };
 use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore;
+use crate::modules::monitoring::grafana::grafana::Grafana;
 use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
+use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{
+    ServiceMonitor, ServiceMonitorSpec,
+};
 use crate::topology::MultiTargetTopology;
 use crate::topology::ingress::Ingress;
 use crate::{
@@ -14,7 +18,7 @@ use crate::{
     topology::{HelmCommand, K8sclient, Topology, tenant::TenantManager},
 };
 use crate::{
-    modules::prometheus::prometheus::PrometheusApplicationMonitoring,
+    modules::prometheus::prometheus::PrometheusMonitoring,
     topology::oberservability::monitoring::AlertReceiver,
 };
 use async_trait::async_trait;
@@ -22,6 +26,7 @@ use base64::{Engine as _, engine::general_purpose};
 use harmony_secret::SecretManager;
 use harmony_secret_derive::Secret;
 use harmony_types::net::Url;
+use kube::api::ObjectMeta;
 use log::{debug, info};
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
@@ -40,7 +45,8 @@ impl<
         + TenantManager
         + K8sclient
         + MultiTargetTopology
-        + PrometheusApplicationMonitoring<CRDPrometheus>
+        + PrometheusMonitoring<CRDPrometheus>
+        + Grafana
         + Ingress
         + std::fmt::Debug,
 > ApplicationFeature<T> for Monitoring
@@ -57,10 +63,20 @@ impl<
             .unwrap_or_else(|| self.application.name());
         let domain = topology.get_domain("ntfy").await.unwrap();
 
+        let app_service_monitor = ServiceMonitor {
+            metadata: ObjectMeta {
+                name: Some(self.application.name()),
+                namespace: Some(namespace.clone()),
+                ..Default::default()
+            },
+            spec: ServiceMonitorSpec::default(),
+        };
+
         let mut alerting_score = ApplicationMonitoringScore {
             sender: CRDPrometheus {
                 namespace: namespace.clone(),
                 client: topology.k8s_client().await.unwrap(),
+                service_monitor: vec![app_service_monitor],
             },
             application: self.application.clone(),
             receivers: self.alert_receiver.clone(),

harmony/src/modules/application/features/rhob_monitoring.rs

@@ -18,7 +18,7 @@ use crate::{
     topology::{HelmCommand, K8sclient, Topology, tenant::TenantManager},
 };
 use crate::{
-    modules::prometheus::prometheus::PrometheusApplicationMonitoring,
+    modules::prometheus::prometheus::PrometheusMonitoring,
     topology::oberservability::monitoring::AlertReceiver,
 };
 use async_trait::async_trait;
@@ -42,7 +42,7 @@ impl<
         + MultiTargetTopology
         + Ingress
         + std::fmt::Debug
-        + PrometheusApplicationMonitoring<RHOBObservability>,
+        + PrometheusMonitoring<RHOBObservability>,
 > ApplicationFeature<T> for Monitoring
 {
     async fn ensure_installed(

harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs

@@ -3,7 +3,8 @@ use std::collections::BTreeMap;
 
 use async_trait::async_trait;
 use k8s_openapi::api::core::v1::Secret;
-use kube::api::ObjectMeta;
+use kube::Resource;
+use kube::api::{DynamicObject, ObjectMeta};
 use log::debug;
 use serde::Serialize;
 use serde_json::json;
@@ -13,6 +14,8 @@ use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{
     AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
 };
 use crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::RHOBObservability;
+use crate::modules::monitoring::okd::OpenshiftClusterAlertSender;
+use crate::topology::oberservability::monitoring::AlertManagerReceiver;
 use crate::{
     interpret::{InterpretError, Outcome},
     modules::monitoring::{
@@ -32,10 +35,8 @@ pub struct DiscordWebhook {
     pub url: Url,
 }
 
-#[async_trait]
-impl AlertReceiver<RHOBObservability> for DiscordWebhook {
-    async fn install(&self, sender: &RHOBObservability) -> Result<Outcome, InterpretError> {
-        let ns = sender.namespace.clone();
+impl DiscordWebhook {
+    fn get_receiver_config(&self) -> AlertManagerReceiver {
         let secret_name = format!("{}-secret", self.name.clone());
         let webhook_key = format!("{}", self.url.clone());
 
@@ -52,26 +53,74 @@ impl AlertReceiver<RHOBObservability> for DiscordWebhook {
             ..Default::default()
         };
 
-        let _ = sender.client.apply(&secret, Some(&ns)).await;
+        AlertManagerReceiver {
+            additional_ressources: vec![],
+
+            receiver_config: json!({
+                "name": self.name,
+                "discordConfigs": [
+                    {
+                    "apiURL": {
+                        "name": secret_name,
+                        "key":  "webhook-url",
+                    },
+                    "title": "{{ template \"discord.default.title\" . }}",
+                    "message": "{{ template \"discord.default.message\" . }}"
+                    }
+                ]
+            }),
+        }
+    }
+}
+
+#[async_trait]
+impl AlertReceiver<OpenshiftClusterAlertSender> for DiscordWebhook {
+    async fn install(
+        &self,
+        sender: &OpenshiftClusterAlertSender,
+    ) -> Result<Outcome, InterpretError> {
+        todo!()
+    }
+
+    fn name(&self) -> String {
+        todo!()
+    }
+
+    fn clone_box(&self) -> Box<dyn AlertReceiver<OpenshiftClusterAlertSender>> {
+        Box::new(self.clone())
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        todo!()
+    }
+
+    fn as_alertmanager_receiver(&self) -> AlertManagerReceiver {
+        self.get_receiver_config()
+    }
+}
+
+#[async_trait]
+impl AlertReceiver<RHOBObservability> for DiscordWebhook {
+    fn as_alertmanager_receiver(&self) -> AlertManagerReceiver {
+        todo!()
+    }
+
+    async fn install(&self, sender: &RHOBObservability) -> Result<Outcome, InterpretError> {
+        let ns = sender.namespace.clone();
+
+        let config = self.get_receiver_config();
+        for resource in config.additional_ressources.iter() {
+            todo!("can I apply a dynamicresource");
+            // sender.client.apply(resource, Some(&ns)).await;
+        }
+
         let spec = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfigSpec {
             data: json!({
                 "route": {
                     "receiver": self.name,
                 },
                 "receivers": [
-                    {
-                        "name": self.name,
-                        "discordConfigs": [
-                            {
-                            "apiURL": {
-                                "name": secret_name,
-                                "key":  "webhook-url",
-                            },
-                            "title": "{{ template \"discord.default.title\" . }}",
-                            "message": "{{ template \"discord.default.message\" . }}"
-                            }
-                        ]
-                    }
+                    config.receiver_config
                 ]
             }),
         };
@@ -122,6 +171,9 @@ impl AlertReceiver<RHOBObservability> for DiscordWebhook {
 
 #[async_trait]
 impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
+    fn as_alertmanager_receiver(&self) -> AlertManagerReceiver {
+        todo!()
+    }
     async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
         let ns = sender.namespace.clone();
         let secret_name = format!("{}-secret", self.name.clone());
@@ -200,6 +252,9 @@ impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
 
 #[async_trait]
 impl AlertReceiver<Prometheus> for DiscordWebhook {
+    fn as_alertmanager_receiver(&self) -> AlertManagerReceiver {
+        todo!()
+    }
     async fn install(&self, sender: &Prometheus) -> Result<Outcome, InterpretError> {
         sender.install_receiver(self).await
     }
@@ -226,6 +281,9 @@ impl PrometheusReceiver for DiscordWebhook {
 
 #[async_trait]
 impl AlertReceiver<KubePrometheus> for DiscordWebhook {
+    fn as_alertmanager_receiver(&self) -> AlertManagerReceiver {
+        todo!()
+    }
     async fn install(&self, sender: &KubePrometheus) -> Result<Outcome, InterpretError> {
         sender.install_receiver(self).await
     }

harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs

@@ -19,7 +19,7 @@ use crate::{
         },
         prometheus::prometheus::{Prometheus, PrometheusReceiver},
     },
-    topology::oberservability::monitoring::AlertReceiver,
+    topology::oberservability::monitoring::{AlertManagerReceiver, AlertReceiver},
 };
 use harmony_types::net::Url;
 
@@ -31,6 +31,9 @@ pub struct WebhookReceiver {
 
 #[async_trait]
 impl AlertReceiver<RHOBObservability> for WebhookReceiver {
+    fn as_alertmanager_receiver(&self) -> AlertManagerReceiver {
+        todo!()
+    }
     async fn install(&self, sender: &RHOBObservability) -> Result<Outcome, InterpretError> {
         let spec = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfigSpec {
             data: json!({
@@ -97,6 +100,9 @@ impl AlertReceiver<RHOBObservability> for WebhookReceiver {
 
 #[async_trait]
 impl AlertReceiver<CRDPrometheus> for WebhookReceiver {
+    fn as_alertmanager_receiver(&self) -> AlertManagerReceiver {
+        todo!()
+    }
     async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
         let spec = crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::AlertmanagerConfigSpec {
             data: json!({
@@ -158,6 +164,9 @@ impl AlertReceiver<CRDPrometheus> for WebhookReceiver {
 
 #[async_trait]
 impl AlertReceiver<Prometheus> for WebhookReceiver {
+    fn as_alertmanager_receiver(&self) -> AlertManagerReceiver {
+        todo!()
+    }
     async fn install(&self, sender: &Prometheus) -> Result<Outcome, InterpretError> {
         sender.install_receiver(self).await
     }
@@ -184,6 +193,9 @@ impl PrometheusReceiver for WebhookReceiver {
 
 #[async_trait]
 impl AlertReceiver<KubePrometheus> for WebhookReceiver {
+    fn as_alertmanager_receiver(&self) -> AlertManagerReceiver {
+        todo!()
+    }
     async fn install(&self, sender: &KubePrometheus) -> Result<Outcome, InterpretError> {
         sender.install_receiver(self).await
     }

harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs

@@ -1,21 +1,23 @@
 use std::sync::Arc;
 
-use async_trait::async_trait;
+use log::debug;
 use serde::Serialize;
 
 use crate::{
-    data::Version,
-    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
-    inventory::Inventory,
+    interpret::Interpret,
     modules::{
         application::Application,
-        monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus,
-        prometheus::prometheus::PrometheusApplicationMonitoring,
+        monitoring::{
+            grafana::grafana::Grafana, kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus,
+        },
+        prometheus::prometheus::PrometheusMonitoring,
     },
     score::Score,
-    topology::{PreparationOutcome, Topology, oberservability::monitoring::AlertReceiver},
+    topology::{
+        K8sclient, Topology,
+        oberservability::monitoring::{AlertReceiver, AlertingInterpret, ScrapeTarget},
+    },
 };
-use harmony_types::id::Id;
 
 #[derive(Debug, Clone, Serialize)]
 pub struct ApplicationMonitoringScore {
@@ -24,12 +26,16 @@ pub struct ApplicationMonitoringScore {
     pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
 }
 
-impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
+impl<T: Topology + PrometheusMonitoring<CRDPrometheus> + K8sclient + Grafana> Score<T>
     for ApplicationMonitoringScore
 {
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
-        Box::new(ApplicationMonitoringInterpret {
-            score: self.clone(),
+        debug!("creating alerting interpret");
+        Box::new(AlertingInterpret {
+            sender: self.sender.clone(),
+            receivers: self.receivers.clone(),
+            rules: vec![],
+            scrape_targets: None,
         })
     }
 
@@ -40,55 +46,3 @@ impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
         )
     }
 }
-
-#[derive(Debug)]
-pub struct ApplicationMonitoringInterpret {
-    score: ApplicationMonitoringScore,
-}
-
-#[async_trait]
-impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Interpret<T>
-    for ApplicationMonitoringInterpret
-{
-    async fn execute(
-        &self,
-        inventory: &Inventory,
-        topology: &T,
-    ) -> Result<Outcome, InterpretError> {
-        let result = topology
-            .install_prometheus(
-                &self.score.sender,
-                inventory,
-                Some(self.score.receivers.clone()),
-            )
-            .await;
-
-        match result {
-            Ok(outcome) => match outcome {
-                PreparationOutcome::Success { details: _ } => {
-                    Ok(Outcome::success("Prometheus installed".into()))
-                }
-                PreparationOutcome::Noop => {
-                    Ok(Outcome::noop("Prometheus installation skipped".into()))
-                }
-            },
-            Err(err) => Err(InterpretError::from(err)),
-        }
-    }
-
-    fn get_name(&self) -> InterpretName {
-        InterpretName::ApplicationMonitoring
-    }
-
-    fn get_version(&self) -> Version {
-        todo!()
-    }
-
-    fn get_status(&self) -> InterpretStatus {
-        todo!()
-    }
-
-    fn get_children(&self) -> Vec<Id> {
-        todo!()
-    }
-}

harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs

@@ -12,7 +12,7 @@ use crate::{
         monitoring::kube_prometheus::crd::{
             crd_alertmanager_config::CRDPrometheus, rhob_alertmanager_config::RHOBObservability,
         },
-        prometheus::prometheus::PrometheusApplicationMonitoring,
+        prometheus::prometheus::PrometheusMonitoring,
     },
     score::Score,
     topology::{PreparationOutcome, Topology, oberservability::monitoring::AlertReceiver},
@@ -26,7 +26,7 @@ pub struct ApplicationRHOBMonitoringScore {
     pub receivers: Vec<Box<dyn AlertReceiver<RHOBObservability>>>,
 }
 
-impl<T: Topology + PrometheusApplicationMonitoring<RHOBObservability>> Score<T>
+impl<T: Topology + PrometheusMonitoring<RHOBObservability>> Score<T>
     for ApplicationRHOBMonitoringScore
 {
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
@@ -49,7 +49,7 @@ pub struct ApplicationRHOBMonitoringInterpret {
 }
 
 #[async_trait]
-impl<T: Topology + PrometheusApplicationMonitoring<RHOBObservability>> Interpret<T>
+impl<T: Topology + PrometheusMonitoring<RHOBObservability>> Interpret<T>
     for ApplicationRHOBMonitoringInterpret
 {
     async fn execute(

harmony/src/modules/monitoring/grafana/grafana.rs

@@ -0,0 +1,17 @@
+use async_trait::async_trait;
+use k8s_openapi::Resource;
+
+use crate::{
+    inventory::Inventory,
+    topology::{PreparationError, PreparationOutcome},
+};
+
+#[async_trait]
+pub trait Grafana {
+    async fn ensure_grafana_operator(
+        &self,
+        inventory: &Inventory,
+    ) -> Result<PreparationOutcome, PreparationError>;
+
+    async fn install_grafana(&self) -> Result<PreparationOutcome, PreparationError>;
+}

harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs

@@ -1,27 +1,28 @@
+use harmony_macros::hurl;
 use non_blank_string_rs::NonBlankString;
-use std::str::FromStr;
+use std::{collections::HashMap, str::FromStr};
 
-use crate::modules::helm::chart::HelmChartScore;
-
-pub fn grafana_helm_chart_score(ns: &str) -> HelmChartScore {
-    let values = r#"
-rbac:
-  namespaced: true
-sidecar:
-  dashboards:
-    enabled: true
-        "#
-    .to_string();
+use crate::modules::helm::chart::{HelmChartScore, HelmRepository};
 
+pub fn grafana_helm_chart_score(ns: &str, namespace_scope: bool) -> HelmChartScore {
+    let mut values_overrides = HashMap::new();
+    values_overrides.insert(
+        NonBlankString::from_str("namespaceScope").unwrap(),
+        namespace_scope.to_string(),
+    );
     HelmChartScore {
         namespace: Some(NonBlankString::from_str(ns).unwrap()),
-        release_name: NonBlankString::from_str("grafana").unwrap(),
-        chart_name: NonBlankString::from_str("oci://ghcr.io/grafana/helm-charts/grafana").unwrap(),
+        release_name: NonBlankString::from_str("grafana-operator").unwrap(),
+        chart_name: NonBlankString::from_str("grafana/grafana-operator").unwrap(),
         chart_version: None,
-        values_overrides: None,
-        values_yaml: Some(values.to_string()),
+        values_overrides: Some(values_overrides),
+        values_yaml: None,
         create_namespace: true,
         install_only: true,
-        repository: None,
+        repository: Some(HelmRepository::new(
+            "grafana".to_string(),
+            hurl!("https://grafana.github.io/helm-charts"),
+            true,
+        )),
     }
 }

harmony/src/modules/monitoring/grafana/mod.rs

@@ -1 +1,2 @@
+pub mod grafana;
 pub mod helm;

harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs

@@ -1,12 +1,25 @@
 use std::sync::Arc;
 
+use async_trait::async_trait;
 use kube::CustomResource;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
-use crate::topology::{
-    k8s::K8sClient,
-    oberservability::monitoring::{AlertReceiver, AlertSender},
+use crate::{
+    interpret::{InterpretError, Outcome},
+    inventory::Inventory,
+    modules::{
+        monitoring::{
+            grafana::grafana::Grafana, kube_prometheus::crd::service_monitor::ServiceMonitor,
+        },
+        prometheus::prometheus::PrometheusMonitoring,
+    },
+    topology::{
+        K8sclient, Topology,
+        installable::Installable,
+        k8s::K8sClient,
+        oberservability::monitoring::{AlertReceiver, AlertSender, ScrapeTarget},
+    },
 };
 
 #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
@@ -26,6 +39,7 @@ pub struct AlertmanagerConfigSpec {
 pub struct CRDPrometheus {
     pub namespace: String,
     pub client: Arc<K8sClient>,
+    pub service_monitor: Vec<ServiceMonitor>,
 }
 
 impl AlertSender for CRDPrometheus {
@@ -40,6 +54,12 @@ impl Clone for Box<dyn AlertReceiver<CRDPrometheus>> {
     }
 }
 
+impl Clone for Box<dyn ScrapeTarget<CRDPrometheus>> {
+    fn clone(&self) -> Self {
+        self.clone_box()
+    }
+}
+
 impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> {
     fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
     where
@@ -48,3 +68,24 @@ impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> {
         todo!()
     }
 }
+
+#[async_trait]
+impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus> + Grafana> Installable<T>
+    for CRDPrometheus
+{
+    async fn configure(&self, inventory: &Inventory, topology: &T) -> Result<(), InterpretError> {
+        topology.ensure_grafana_operator(inventory).await?;
+        topology.ensure_prometheus_operator(self, inventory).await?;
+        Ok(())
+    }
+
+    async fn ensure_installed(
+        &self,
+        inventory: &Inventory,
+        topology: &T,
+    ) -> Result<(), InterpretError> {
+        topology.install_grafana().await?;
+        topology.install_prometheus(&self, inventory, None).await?;
+        Ok(())
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/crd_grafana.rs

@@ -103,9 +103,34 @@ pub struct GrafanaDashboardSpec {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub resync_period: Option<String>,
 
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub datasources: Option<Vec<GrafanaDashboardDatasource>>,
+
     pub instance_selector: LabelSelector,
 
-    pub json: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub json: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub grafana_com: Option<GrafanaCom>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDashboardDatasource {
+    pub input_name: String,
+    pub datasource_name: String,
+}
+
+// ------------------------------------------------------------------------------------------------
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaCom {
+    pub id: u32,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub revision: Option<u32>,
 }
 
 // ------------------------------------------------------------------------------------------------
@@ -126,20 +151,79 @@ pub struct GrafanaDatasourceSpec {
     pub allow_cross_namespace_import: Option<bool>,
 
     pub datasource: GrafanaDatasourceConfig,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub values_from: Option<Vec<GrafanaValueFrom>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaValueFrom {
+    pub target_path: String,
+    pub value_from: GrafanaValueSource,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaValueSource {
+    pub secret_key_ref: GrafanaSecretKeyRef,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaSecretKeyRef {
+    pub name: String,
+    pub key: String,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct GrafanaDatasourceConfig {
     pub access: String,
-    pub database: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub json_data: Option<BTreeMap<String, String>>,
+    pub database: Option<String>,
     pub name: String,
     pub r#type: String,
     pub url: String,
+    /// Represents jsonData in the GrafanaDatasource spec
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub json_data: Option<GrafanaDatasourceJsonData>,
+
+    /// Represents secureJsonData (secrets)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub secure_json_data: Option<GrafanaDatasourceSecureJsonData>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub is_default: Option<bool>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub editable: Option<bool>,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDatasourceJsonData {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub time_interval: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub http_header_name1: Option<String>,
+
+    /// Disable TLS skip verification (false = verify)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub tls_skip_verify: Option<bool>,
+
+    /// Auth type - set to "forward" for OpenShift OAuth identity
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub oauth_pass_thru: Option<bool>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDatasourceSecureJsonData {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub http_header_value1: Option<String>,
+}
 // ------------------------------------------------------------------------------------------------
 
 #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, Default)]

harmony/src/modules/monitoring/okd/cluster_monitoring.rs

@@ -0,0 +1,139 @@
+use base64::prelude::*;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use harmony_types::id::Id;
+use kube::api::DynamicObject;
+use log::{debug, info, trace};
+use serde::Serialize;
+
+use crate::{
+    data::Version,
+    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
+    inventory::Inventory,
+    modules::{
+        application::Application,
+        monitoring::{
+            grafana::grafana::Grafana,
+            kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus,
+            okd::OpenshiftClusterAlertSender,
+        },
+        prometheus::prometheus::PrometheusMonitoring,
+    },
+    score::Score,
+    topology::{
+        K8sclient, Topology,
+        k8s::K8sClient,
+        oberservability::monitoring::{AlertReceiver, AlertingInterpret, ScrapeTarget},
+    },
+};
+
+impl Clone for Box<dyn AlertReceiver<OpenshiftClusterAlertSender>> {
+    fn clone(&self) -> Self {
+        self.clone_box()
+    }
+}
+
+impl Serialize for Box<dyn AlertReceiver<OpenshiftClusterAlertSender>> {
+    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        todo!()
+    }
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct OpenshiftClusterAlertScore {
+    pub receivers: Vec<Box<dyn AlertReceiver<OpenshiftClusterAlertSender>>>,
+}
+
+impl<T: Topology + K8sclient> Score<T> for OpenshiftClusterAlertScore {
+    fn name(&self) -> String {
+        "ClusterAlertScore".to_string()
+    }
+
+    #[doc(hidden)]
+    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
+        Box::new(OpenshiftClusterAlertInterpret {
+            receivers: self.receivers.clone(),
+        })
+    }
+}
+
+#[derive(Debug)]
+pub struct OpenshiftClusterAlertInterpret {
+    receivers: Vec<Box<dyn AlertReceiver<OpenshiftClusterAlertSender>>>,
+}
+
+#[async_trait]
+impl<T: Topology + K8sclient> Interpret<T> for OpenshiftClusterAlertInterpret {
+    async fn execute(
+        &self,
+        _inventory: &Inventory,
+        topology: &T,
+    ) -> Result<Outcome, InterpretError> {
+        let client = topology.k8s_client().await?;
+
+        let secret: DynamicObject = client
+            .get_secret_json_value("alertmanager-main", Some("openshift-monitoring"))
+            .await?;
+        trace!("Got secret {secret:?}");
+
+        let data: serde_json::Value = secret.data;
+
+        // TODO : get config in base64 by drilling into the value
+        let config_b64 = match data.get("alertmanager.yaml") {
+            Some(value) => value.as_str().unwrap_or(""),
+            None => "",
+        };
+
+        // TODO : base64 decode it
+        let config_bytes = BASE64_STANDARD.decode(config_b64).unwrap_or_default();
+
+        // TODO : use serde_yaml to deserialize the string
+        let am_config: serde_yaml::Value =
+            serde_yaml::from_str(&String::from_utf8(config_bytes).unwrap_or_default())
+                .unwrap_or_default();
+
+        // Merge current alert receivers from this config with self.receivers
+        if let Some(existing_receivers) = am_config.get("receivers") {
+            for receiver in existing_receivers.as_sequence().unwrap_or(&vec![]) {
+                match serde_json::to_string(receiver) {
+                    Ok(yaml_str) => {
+                        // TODO: validate that each receiver implements to_alertmanager_yaml()
+                        // and compare with our receivers
+                        info!("Found existing receiver config: {}", yaml_str);
+                    }
+                    Err(e) => debug!("Failed to serialize receiver: {}", e),
+                }
+            }
+        }
+
+        for custom_receiver in &self.receivers {
+            trace!("Processing custom receiver");
+            debug!(
+                "Custom receiver YAML output: {:?}",
+                custom_receiver.as_alertmanager_receiver()
+            );
+        }
+
+        Ok(Outcome::success(todo!("whats up")))
+    }
+
+    fn get_name(&self) -> InterpretName {
+        InterpretName::Custom("OpenshiftClusterAlertInterpret")
+    }
+
+    fn get_version(&self) -> Version {
+        todo!()
+    }
+
+    fn get_status(&self) -> InterpretStatus {
+        todo!()
+    }
+
+    fn get_children(&self) -> Vec<Id> {
+        todo!()
+    }
+}

harmony/src/modules/monitoring/okd/config.rs

@@ -0,0 +1,90 @@
+use std::{collections::BTreeMap, sync::Arc};
+
+use crate::{
+    interpret::{InterpretError, Outcome},
+    topology::k8s::K8sClient,
+};
+use k8s_openapi::api::core::v1::ConfigMap;
+use kube::api::ObjectMeta;
+
+pub(crate) struct Config;
+
+impl Config {
+    pub async fn create_cluster_monitoring_config_cm(
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let mut data = BTreeMap::new();
+        data.insert(
+            "config.yaml".to_string(),
+            r#"
+enableUserWorkload: true
+alertmanagerMain:
+  enableUserAlertmanagerConfig: true
+"#
+            .to_string(),
+        );
+
+        let cm = ConfigMap {
+            metadata: ObjectMeta {
+                name: Some("cluster-monitoring-config".to_string()),
+                namespace: Some("openshift-monitoring".to_string()),
+                ..Default::default()
+            },
+            data: Some(data),
+            ..Default::default()
+        };
+        client.apply(&cm, Some("openshift-monitoring")).await?;
+
+        Ok(Outcome::success(
+            "updated cluster-monitoring-config-map".to_string(),
+        ))
+    }
+
+    pub async fn create_user_workload_monitoring_config_cm(
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let mut data = BTreeMap::new();
+        data.insert(
+            "config.yaml".to_string(),
+            r#"
+alertmanager: 
+  enabled: true
+  enableAlertmanagerConfig: true
+"#
+            .to_string(),
+        );
+        let cm = ConfigMap {
+            metadata: ObjectMeta {
+                name: Some("user-workload-monitoring-config".to_string()),
+                namespace: Some("openshift-user-workload-monitoring".to_string()),
+                ..Default::default()
+            },
+            data: Some(data),
+            ..Default::default()
+        };
+        client
+            .apply(&cm, Some("openshift-user-workload-monitoring"))
+            .await?;
+
+        Ok(Outcome::success(
+            "updated openshift-user-monitoring-config-map".to_string(),
+        ))
+    }
+
+    pub async fn verify_user_workload(client: &Arc<K8sClient>) -> Result<Outcome, InterpretError> {
+        let namespace = "openshift-user-workload-monitoring";
+        let alertmanager_name = "alertmanager-user-workload-0";
+        let prometheus_name = "prometheus-user-workload-0";
+        client
+            .wait_for_pod_ready(alertmanager_name, Some(namespace))
+            .await?;
+        client
+            .wait_for_pod_ready(prometheus_name, Some(namespace))
+            .await?;
+
+        Ok(Outcome::success(format!(
+            "pods: {}, {} ready in ns: {}",
+            alertmanager_name, prometheus_name, namespace
+        )))
+    }
+}

harmony/src/modules/monitoring/okd/enable_user_workload.rs

@@ -1,16 +1,13 @@
-use std::{collections::BTreeMap, sync::Arc};
-
 use crate::{
     data::Version,
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::Inventory,
+    modules::monitoring::okd::config::Config,
     score::Score,
-    topology::{K8sclient, Topology, k8s::K8sClient},
+    topology::{K8sclient, Topology},
 };
 use async_trait::async_trait;
 use harmony_types::id::Id;
-use k8s_openapi::api::core::v1::ConfigMap;
-use kube::api::ObjectMeta;
 use serde::Serialize;
 
 #[derive(Clone, Debug, Serialize)]
@@ -37,10 +34,9 @@ impl<T: Topology + K8sclient> Interpret<T> for OpenshiftUserWorkloadMonitoringIn
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
         let client = topology.k8s_client().await.unwrap();
-        self.update_cluster_monitoring_config_cm(&client).await?;
-        self.update_user_workload_monitoring_config_cm(&client)
-            .await?;
-        self.verify_user_workload(&client).await?;
+        Config::create_cluster_monitoring_config_cm(&client).await?;
+        Config::create_user_workload_monitoring_config_cm(&client).await?;
+        Config::verify_user_workload(&client).await?;
         Ok(Outcome::success(
             "successfully enabled user-workload-monitoring".to_string(),
         ))
@@ -62,88 +58,3 @@ impl<T: Topology + K8sclient> Interpret<T> for OpenshiftUserWorkloadMonitoringIn
         todo!()
     }
 }
-
-impl OpenshiftUserWorkloadMonitoringInterpret {
-    pub async fn update_cluster_monitoring_config_cm(
-        &self,
-        client: &Arc<K8sClient>,
-    ) -> Result<Outcome, InterpretError> {
-        let mut data = BTreeMap::new();
-        data.insert(
-            "config.yaml".to_string(),
-            r#"
-enableUserWorkload: true
-alertmanagerMain:
-  enableUserAlertmanagerConfig: true
-"#
-            .to_string(),
-        );
-
-        let cm = ConfigMap {
-            metadata: ObjectMeta {
-                name: Some("cluster-monitoring-config".to_string()),
-                namespace: Some("openshift-monitoring".to_string()),
-                ..Default::default()
-            },
-            data: Some(data),
-            ..Default::default()
-        };
-        client.apply(&cm, Some("openshift-monitoring")).await?;
-
-        Ok(Outcome::success(
-            "updated cluster-monitoring-config-map".to_string(),
-        ))
-    }
-
-    pub async fn update_user_workload_monitoring_config_cm(
-        &self,
-        client: &Arc<K8sClient>,
-    ) -> Result<Outcome, InterpretError> {
-        let mut data = BTreeMap::new();
-        data.insert(
-            "config.yaml".to_string(),
-            r#"
-alertmanager: 
-  enabled: true
-  enableAlertmanagerConfig: true
-"#
-            .to_string(),
-        );
-        let cm = ConfigMap {
-            metadata: ObjectMeta {
-                name: Some("user-workload-monitoring-config".to_string()),
-                namespace: Some("openshift-user-workload-monitoring".to_string()),
-                ..Default::default()
-            },
-            data: Some(data),
-            ..Default::default()
-        };
-        client
-            .apply(&cm, Some("openshift-user-workload-monitoring"))
-            .await?;
-
-        Ok(Outcome::success(
-            "updated openshift-user-monitoring-config-map".to_string(),
-        ))
-    }
-
-    pub async fn verify_user_workload(
-        &self,
-        client: &Arc<K8sClient>,
-    ) -> Result<Outcome, InterpretError> {
-        let namespace = "openshift-user-workload-monitoring";
-        let alertmanager_name = "alertmanager-user-workload-0";
-        let prometheus_name = "prometheus-user-workload-0";
-        client
-            .wait_for_pod_ready(alertmanager_name, Some(namespace))
-            .await?;
-        client
-            .wait_for_pod_ready(prometheus_name, Some(namespace))
-            .await?;
-
-        Ok(Outcome::success(format!(
-            "pods: {}, {} ready in ns: {}",
-            alertmanager_name, prometheus_name, namespace
-        )))
-    }
-}

harmony/src/modules/monitoring/okd/mod.rs

@@ -1 +1,14 @@
+use crate::topology::oberservability::monitoring::AlertSender;
+
+pub mod cluster_monitoring;
+pub(crate) mod config;
 pub mod enable_user_workload;
+
+#[derive(Debug)]
+pub struct OpenshiftClusterAlertSender;
+
+impl AlertSender for OpenshiftClusterAlertSender {
+    fn name(&self) -> String {
+        "OpenshiftClusterAlertSender".to_string()
+    }
+}

harmony/src/modules/monitoring/prometheus/prometheus.rs

@@ -114,7 +114,7 @@ impl Prometheus {
         };
 
         if let Some(ns) = namespace.as_deref() {
-            grafana_helm_chart_score(ns)
+            grafana_helm_chart_score(ns, false)
                 .interpret(inventory, topology)
                 .await
         } else {

harmony/src/modules/monitoring/scrape_target/server.rs

@@ -73,4 +73,8 @@ impl ScrapeTarget<CRDPrometheus> for Server {
             self.name.clone()
         )))
     }
+
+    fn clone_box(&self) -> Box<dyn ScrapeTarget<CRDPrometheus>> {
+        Box::new(self.clone())
+    }
 }

harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs

@@ -12,7 +12,8 @@ use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::C
 use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules;
 use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{
     Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig,
-    GrafanaDatasourceSpec, GrafanaSpec,
+    GrafanaDatasourceJsonData, GrafanaDatasourceSpec, GrafanaSecretKeyRef, GrafanaSpec,
+    GrafanaValueFrom, GrafanaValueSource,
 };
 use crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::{
     PrometheusRule, PrometheusRuleSpec, RuleGroup,
@@ -39,7 +40,7 @@ use crate::{
 };
 use harmony_types::id::Id;
 
-use super::prometheus::PrometheusApplicationMonitoring;
+use super::prometheus::PrometheusMonitoring;
 
 #[derive(Clone, Debug, Serialize)]
 pub struct K8sPrometheusCRDAlertingScore {
@@ -49,7 +50,7 @@ pub struct K8sPrometheusCRDAlertingScore {
     pub prometheus_rules: Vec<RuleGroup>,
 }
 
-impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
+impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus>> Score<T>
     for K8sPrometheusCRDAlertingScore
 {
     fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
@@ -75,7 +76,7 @@ pub struct K8sPrometheusCRDAlertingInterpret {
 }
 
 #[async_trait]
-impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> Interpret<T>
+impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus>> Interpret<T>
     for K8sPrometheusCRDAlertingInterpret
 {
     async fn execute(
@@ -466,10 +467,13 @@ impl K8sPrometheusCRDAlertingInterpret {
             match_labels: label.clone(),
             match_expressions: vec![],
         };
-        let mut json_data = BTreeMap::new();
-        json_data.insert("timeInterval".to_string(), "5s".to_string());
         let namespace = self.sender.namespace.clone();
-
+        let json_data = GrafanaDatasourceJsonData {
+            time_interval: Some("5s".to_string()),
+            http_header_name1: None,
+            tls_skip_verify: Some(true),
+            oauth_pass_thru: Some(true),
+        };
         let json = build_default_dashboard(&namespace);
 
         let graf_data_source = GrafanaDatasource {
@@ -495,7 +499,11 @@ impl K8sPrometheusCRDAlertingInterpret {
                         "http://prometheus-operated.{}.svc.cluster.local:9090",
                         self.sender.namespace.clone()
                     ),
+                    secure_json_data: None,
+                    is_default: None,
+                    editable: None,
                 },
+                values_from: None,
             },
         };
 
@@ -516,7 +524,9 @@ impl K8sPrometheusCRDAlertingInterpret {
             spec: GrafanaDashboardSpec {
                 resync_period: Some("30s".to_string()),
                 instance_selector: labels.clone(),
-                json,
+                json: Some(json),
+                grafana_com: None,
+                datasources: None,
             },
         };
 

harmony/src/modules/prometheus/prometheus.rs

@@ -9,11 +9,17 @@ use crate::{
 };
 
 #[async_trait]
-pub trait PrometheusApplicationMonitoring<S: AlertSender> {
+pub trait PrometheusMonitoring<S: AlertSender> {
     async fn install_prometheus(
         &self,
         sender: &S,
         inventory: &Inventory,
         receivers: Option<Vec<Box<dyn AlertReceiver<S>>>>,
     ) -> Result<PreparationOutcome, PreparationError>;
+
+    async fn ensure_prometheus_operator(
+        &self,
+        sender: &S,
+        inventory: &Inventory,
+    ) -> Result<PreparationOutcome, PreparationError>;
 }

harmony/src/modules/prometheus/rhob_alerting_score.rs

@@ -38,7 +38,7 @@ use crate::{
 };
 use harmony_types::id::Id;
 
-use super::prometheus::PrometheusApplicationMonitoring;
+use super::prometheus::PrometheusMonitoring;
 
 #[derive(Clone, Debug, Serialize)]
 pub struct RHOBAlertingScore {
@@ -48,8 +48,8 @@ pub struct RHOBAlertingScore {
     pub prometheus_rules: Vec<RuleGroup>,
 }
 
-impl<T: Topology + K8sclient + Ingress + PrometheusApplicationMonitoring<RHOBObservability>>
-    Score<T> for RHOBAlertingScore
+impl<T: Topology + K8sclient + Ingress + PrometheusMonitoring<RHOBObservability>> Score<T>
+    for RHOBAlertingScore
 {
     fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
         Box::new(RHOBAlertingInterpret {
@@ -74,8 +74,8 @@ pub struct RHOBAlertingInterpret {
 }
 
 #[async_trait]
-impl<T: Topology + K8sclient + Ingress + PrometheusApplicationMonitoring<RHOBObservability>>
-    Interpret<T> for RHOBAlertingInterpret
+impl<T: Topology + K8sclient + Ingress + PrometheusMonitoring<RHOBObservability>> Interpret<T>
+    for RHOBAlertingInterpret
 {
     async fn execute(
         &self,