wip:added impl for prometheuses, alertmanagers, prometheusrules, servicemonitors, and some default rules that are deployed for application monitor

working on implementing grafana crds via grafana operator
need to link prometheus rules and alert managers in prometheus, testing it shows that prometheus isnt detecting them automatically

examples/monitoring/src/main.rs

@@ -51,8 +51,8 @@ async fn main() {
 
     let service_monitor_endpoint = ServiceMonitorEndpoint {
         port: Some("80".to_string()),
-        path: "/metrics".to_string(),
-        scheme: HTTPScheme::HTTP,
+        path: Some("/metrics".to_string()),
+        scheme: Some(HTTPScheme::HTTP),
         ..Default::default()
     };
 

examples/monitoring_with_tenant/src/main.rs

@@ -54,8 +54,8 @@ async fn main() {
 
     let service_monitor_endpoint = ServiceMonitorEndpoint {
         port: Some("80".to_string()),
-        path: "/metrics".to_string(),
-        scheme: HTTPScheme::HTTP,
+        path: Some("/metrics".to_string()),
+        scheme: Some(HTTPScheme::HTTP),
         ..Default::default()
     };
 

examples/rust/src/main.rs

@@ -6,7 +6,7 @@ use harmony::{
     modules::{
         application::{
             ApplicationScore, RustWebFramework, RustWebapp,
-            features::{ContinuousDelivery, PrometheusApplicationMonitoring},
+            features::{ContinuousDelivery, Monitoring},
         },
         monitoring::alert_channel::{
             discord_alert_channel::DiscordWebhook, webhook_receiver::WebhookReceiver,
@@ -46,9 +46,11 @@ async fn main() {
             Box::new(ContinuousDelivery {
                 application: application.clone(),
             }), // TODO add monitoring, backups, multisite ha, etc
-            Box::new(PrometheusApplicationMonitoring {
+            Box::new(Monitoring {
                 application: application.clone(),
                 alert_receiver: vec![Box::new(discord_receiver), Box::new(webhook_receiver)],
+                service_monitors: vec![],
+                alert_rules: vec![],
             }),
             // TODO add backups, multisite ha, etc
         ],

harmony/src/modules/application/features/monitoring.rs

@@ -1,9 +1,13 @@
 use std::sync::Arc;
 
-use async_trait::async_trait;
-use base64::{Engine as _, engine::general_purpose};
-use log::{debug, info};
-
+use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::{
+    build_rule_container_restarting, build_rule_pod_failed,
+};
+use crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::RuleGroup;
+use crate::modules::monitoring::kube_prometheus::service_monitor::{
+    ServiceMonitor, ServiceMonitorSpec,
+};
+use crate::modules::monitoring::kube_prometheus::types::{Selector, ServiceMonitorEndpoint};
 use crate::{
     inventory::Inventory,
     modules::{
@@ -13,7 +17,7 @@ use crate::{
             kube_prometheus::{
                 alert_manager_config::{CRDAlertManager, CRDAlertManagerReceiver},
                 helm_prometheus_application_alerting::HelmPrometheusApplicationAlertingScore,
-                types::{NamespaceSelector, ServiceMonitor},
+                types::{NamespaceSelector, ServiceMonitor as KubePrometheusServiceMonitor},
             },
             ntfy::ntfy::NtfyScore,
         },
@@ -24,23 +28,31 @@ use crate::{
         tenant::TenantManager,
     },
 };
+use async_trait::async_trait;
+use base64::{Engine as _, engine::general_purpose};
+use kube::api::ObjectMeta;
+use log::{debug, info};
 
 #[derive(Debug, Clone)]
-pub struct PrometheusApplicationMonitoring {
+pub struct Monitoring {
     pub application: Arc<dyn OCICompliant>,
     pub alert_receiver: Vec<Box<dyn CRDAlertManagerReceiver>>,
+    pub service_monitors: Vec<ServiceMonitor>,
+    pub alert_rules: Vec<RuleGroup>,
 }
 
 #[async_trait]
 impl<T: Topology + HelmCommand + 'static + TenantManager + K8sclient + std::fmt::Debug>
-    ApplicationFeature<T> for PrometheusApplicationMonitoring
+    ApplicationFeature<T> for Monitoring
 {
     async fn ensure_installed(&self, topology: &T) -> Result<(), String> {
         info!("Ensuring monitoring is available for application");
-
+        let namespace = self.application.name().clone();
         let mut alerting_score = HelmPrometheusApplicationAlertingScore {
-            namespace: self.application.name().clone(),
+            namespace: namespace.clone(),
             receivers: self.alert_receiver.clone(),
+            service_monitors: self.service_monitors.clone(),
+            prometheus_rules: self.alert_rules.clone(),
         };
         let ntfy = NtfyScore {
             // namespace: topology
@@ -91,14 +103,27 @@ impl<T: Topology + HelmCommand + 'static + TenantManager + K8sclient + std::fmt:
 
         //TODO add service monitors to PrometheusApplicationMonitoring which can be
         //deployed for the namespace using prometheus crd-servicemonitors
-        let mut service_monitor = ServiceMonitor::default();
-        service_monitor.namespace_selector = Some(NamespaceSelector {
-            any: true,
-            match_names: vec![],
-        });
+        let service_monitor = ServiceMonitor {
+            metadata: ObjectMeta {
+                name: Some(self.application.name().clone()),
+                labels: Some(std::collections::BTreeMap::from([(
+                    "alertmanagerConfig".to_string(),
+                    "enabled".to_string(),
+                )])),
+                namespace: Some(namespace),
+                ..Default::default()
+            },
+            spec: ServiceMonitorSpec::default(),
+        };
 
-        service_monitor.name = "rust-webapp".to_string();
+        alerting_score.service_monitors.push(service_monitor);
 
+        let rules_group = RuleGroup {
+            name: format!("{}-rules", self.application.name().clone()),
+            rules: vec![build_rule_container_restarting(), build_rule_pod_failed()],
+        };
+
+        alerting_score.prometheus_rules.push(rules_group);
         alerting_score
             .create_interpret()
             .execute(&Inventory::empty(), topology)

harmony/src/modules/monitoring/kube_prometheus/alert_manager_config.rs

@@ -17,8 +17,6 @@ use crate::{
     },
 };
 
-use super::types::AlertManagerConfig;
-
 #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
 #[kube(
     group = "monitoring.coreos.com",

harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanagers.rs

@@ -0,0 +1,53 @@
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+
+use super::crd_prometheuses::LabelSelector;
+
+/// Rust CRD for `Alertmanager` from Prometheus Operator
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "monitoring.coreos.com",
+    version = "v1",
+    kind = "Alertmanager",
+    plural = "alertmanagers",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct AlertmanagerSpec {
+    /// Number of replicas for HA
+    pub replicas: i32,
+
+    /// Selectors for AlertmanagerConfig CRDs
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub alertmanager_config_selector: Option<LabelSelector>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub alertmanager_config_namespace_selector: Option<LabelSelector>,
+
+    /// Optional pod template metadata (annotations, labels)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub pod_metadata: Option<LabelSelector>,
+
+    /// Optional topology spread settings
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub version: Option<String>,
+}
+
+impl Default for AlertmanagerSpec {
+    fn default() -> Self {
+        AlertmanagerSpec {
+            replicas: 1,
+
+            // Match all AlertmanagerConfigs in the same namespace
+            alertmanager_config_namespace_selector: None,
+
+            // Empty selector matches all AlertmanagerConfigs in that namespace
+            alertmanager_config_selector: Some(LabelSelector::default()),
+
+            pod_metadata: None,
+            version: None,
+        }
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/crd_default_rules.rs

@@ -0,0 +1,38 @@
+use std::collections::BTreeMap;
+
+use super::crd_prometheus_rules::Rule;
+
+pub fn build_rule_container_restarting() -> Rule {
+    Rule {
+        alert: Some("ContainerRestarting".into()),
+        expr: Some("increase(kube_pod_container_status_restarts_total[5m]) > 3".into()),
+        for_: Some("5m".into()),
+        labels: Some(BTreeMap::from([("severity".into(), "warning".into())])),
+        annotations: Some(BTreeMap::from([
+            (
+                "summary".into(),
+                "Container is restarting frequently".into(),
+            ),
+            (
+                "description".into(),
+                "Container in this namespace is restarting more than 3 times in 5 minutes.".into(),
+            ),
+        ])),
+    }
+}
+
+pub fn build_rule_pod_failed() -> Rule {
+    Rule {
+        alert: Some("PodFailed".into()),
+        expr: Some("kube_pod_status_phase{phase=\"Failed\"} > 0".into()),
+        for_: Some("0m".into()),
+        labels: Some(BTreeMap::from([("severity".into(), "critical".into())])),
+        annotations: Some(BTreeMap::from([
+            ("summary".into(), "A pod has failed".into()),
+            (
+                "description".into(),
+                "One or more pods are in Failed phase.".into(),
+            ),
+        ])),
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/crd_grafana.rs

@@ -0,0 +1,170 @@
+use std::collections::BTreeMap;
+
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "grafana.integreatly.org",
+    version = "v1beta1",
+    kind = "Grafana",
+    plural = "grafanas",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaSpec {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub config: Option<GrafanaConfig>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub admin_user: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub admin_password: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub ingress: Option<GrafanaIngress>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub persistence: Option<GrafanaPersistence>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub resources: Option<ResourceRequirements>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaConfig {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub log: Option<GrafanaLogConfig>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub security: Option<GrafanaSecurityConfig>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaLogConfig {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub mode: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub level: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaSecurityConfig {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub admin_user: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub admin_password: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaIngress {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub enabled: Option<bool>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub hosts: Option<Vec<String>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaPersistence {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub enabled: Option<bool>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub storage_class_name: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub size: Option<String>,
+}
+
+// ------------------------------------------------------------------------------------------------
+
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "grafana.integreatly.org",
+    version = "v1beta1",
+    kind = "GrafanaDashboard",
+    plural = "grafanadashboards",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDashboardSpec {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub resync_period: Option<String>,
+
+    pub instance_selector: LabelSelector,
+
+    pub json: String,
+}
+
+// ------------------------------------------------------------------------------------------------
+
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "grafana.integreatly.org",
+    version = "v1beta1",
+    kind = "GrafanaDatasource",
+    plural = "grafanadatasources",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDatasourceSpec {
+    pub instance_selector: LabelSelector,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub allow_cross_namespace_import: Option<bool>,
+
+    pub datasource: GrafanaDatasourceConfig,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDatasourceConfig {
+    pub access: String,
+    pub database: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub json_data: Option<BTreeMap<String, serde_json::Value>>,
+    pub name: String,
+    pub r#type: String,
+    pub url: String,
+}
+
+// ------------------------------------------------------------------------------------------------
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct LabelSelector {
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub match_labels: BTreeMap<String, String>,
+
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub match_expressions: Vec<LabelSelectorRequirement>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct LabelSelectorRequirement {
+    pub key: String,
+    pub operator: String,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub values: Vec<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct ResourceRequirements {
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub limits: BTreeMap<String, String>,
+
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub requests: BTreeMap<String, String>,
+}

harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheus_rules.rs

@@ -0,0 +1,54 @@
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use super::crd_default_rules::{build_rule_container_restarting, build_rule_pod_failed};
+
+#[derive(CustomResource, Debug, Serialize, Deserialize, Clone, JsonSchema)]
+#[kube(
+    group = "monitoring.coreos.com",
+    version = "v1",
+    kind = "PrometheusRule",
+    plural = "prometheusrules",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct PrometheusRuleSpec {
+    pub groups: Vec<RuleGroup>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct RuleGroup {
+    pub name: String,
+    pub rules: Vec<Rule>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Rule {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub alert: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub expr: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub for_: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub labels: Option<std::collections::BTreeMap<String, String>>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub annotations: Option<std::collections::BTreeMap<String, String>>,
+}
+
+impl PrometheusRuleSpec {
+    pub fn with_default_rules() -> Self {
+        PrometheusRuleSpec {
+            groups: vec![RuleGroup {
+                name: "default.rules".into(),
+                rules: vec![build_rule_container_restarting(), build_rule_pod_failed()],
+            }],
+        }
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/crd_prometheuses.rs

@@ -0,0 +1,78 @@
+use std::collections::BTreeMap;
+
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use crate::modules::monitoring::kube_prometheus::types::Operator;
+
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "monitoring.coreos.com",
+    version = "v1",
+    kind = "Prometheus",
+    plural = "prometheuses",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct PrometheusSpec {
+    pub service_account_name: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub service_monitor_namespace_selector: Option<LabelSelector>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub service_monitor_selector: Option<LabelSelector>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub service_discovery_role: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub pod_monitor_selector: Option<LabelSelector>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub rule_selector: Option<LabelSelector>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub rule_namespace_selector: Option<LabelSelector>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct LabelSelector {
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub match_labels: BTreeMap<String, String>,
+
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub match_expressions: Vec<LabelSelectorRequirement>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct LabelSelectorRequirement {
+    pub key: String,
+    pub operator: Operator,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub values: Vec<String>,
+}
+
+impl Default for PrometheusSpec {
+    fn default() -> Self {
+        PrometheusSpec {
+            service_account_name: "prometheus".into(),
+
+            // null means "only my namespace"
+            service_monitor_namespace_selector: None,
+
+            // empty selector means match all ServiceMonitors in that namespace
+            service_monitor_selector: Some(LabelSelector::default()),
+
+            service_discovery_role: Some("Endpoints".into()),
+
+            pod_monitor_selector: None,
+
+            rule_selector: None,
+
+            rule_namespace_selector: Some(LabelSelector::default()),
+        }
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/grafana_operator.rs

@@ -0,0 +1,22 @@
+use std::str::FromStr;
+
+use non_blank_string_rs::NonBlankString;
+
+use crate::modules::helm::chart::HelmChartScore;
+
+pub fn grafana_operator_helm_chart_score(ns: String) -> HelmChartScore {
+    HelmChartScore {
+        namespace: Some(NonBlankString::from_str(&ns).unwrap()),
+        release_name: NonBlankString::from_str("kube-prometheus").unwrap(),
+        chart_name: NonBlankString::from_str(
+            "grafana-operator oci://ghcr.io/grafana/helm-charts/grafana-operator",
+        )
+        .unwrap(),
+        chart_version: None,
+        values_overrides: None,
+        values_yaml: None,
+        create_namespace: true,
+        install_only: true,
+        repository: None,
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/mod.rs

@@ -0,0 +1,7 @@
+pub mod crd_alertmanagers;
+pub mod crd_default_rules;
+pub mod crd_grafana;
+pub mod crd_prometheus_rules;
+pub mod crd_prometheuses;
+pub mod grafana_operator;
+pub mod role;

harmony/src/modules/monitoring/kube_prometheus/crd/role.rs

@@ -0,0 +1,62 @@
+use k8s_openapi::api::{
+    core::v1::ServiceAccount,
+    rbac::v1::{PolicyRule, Role, RoleBinding, RoleRef, Subject},
+};
+use kube::api::ObjectMeta;
+
+pub fn build_prom_role(role_name: String, namespace: String) -> Role {
+    Role {
+        metadata: ObjectMeta {
+            name: Some(role_name),
+            namespace: Some(namespace),
+            ..Default::default()
+        },
+        rules: Some(vec![PolicyRule {
+            api_groups: Some(vec!["".into()]), // core API group
+            resources: Some(vec!["services".into(), "endpoints".into(), "pods".into()]),
+            verbs: vec!["get".into(), "list".into(), "watch".into()],
+            ..Default::default()
+        }]),
+    }
+}
+
+pub fn build_prom_rolebinding(
+    role_name: String,
+    namespace: String,
+    service_account_name: String,
+) -> RoleBinding {
+    RoleBinding {
+        metadata: ObjectMeta {
+            name: Some(format!("{}-rolebinding", role_name)),
+            namespace: Some(namespace.clone()),
+            ..Default::default()
+        },
+        role_ref: RoleRef {
+            api_group: "rbac.authorization.k8s.io".into(),
+            kind: "Role".into(),
+            name: role_name,
+        },
+        subjects: Some(vec![Subject {
+            kind: "ServiceAccount".into(),
+            name: service_account_name,
+            namespace: Some(namespace.clone()),
+            ..Default::default()
+        }]),
+    }
+}
+
+pub fn build_prom_service_account(
+    service_account_name: String,
+    namespace: String,
+) -> ServiceAccount {
+    ServiceAccount {
+        automount_service_account_token: None,
+        image_pull_secrets: None,
+        metadata: ObjectMeta {
+            name: Some(service_account_name),
+            namespace: Some(namespace),
+            ..Default::default()
+        },
+        secrets: None,
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_application_alerting.rs

@@ -1,20 +1,28 @@
+use std::sync::Arc;
+
 use async_trait::async_trait;
 use kube::{Api, api::ObjectMeta};
-use log::debug;
+use log::{debug, info};
 use serde::Serialize;
 
 use crate::{
     data::{Id, Version},
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::Inventory,
+    modules::monitoring::kube_prometheus::crd::{
+        crd_alertmanagers::{Alertmanager, AlertmanagerSpec},
+        crd_prometheuses::{Prometheus, PrometheusSpec},
+        role::{build_prom_role, build_prom_rolebinding, build_prom_service_account},
+    },
     score::Score,
-    topology::{K8sclient, Topology, oberservability::monitoring::AlertReceiver},
+    topology::{K8sclient, Topology, k8s::K8sClient, oberservability::monitoring::AlertReceiver},
 };
 
 use super::{
     alert_manager_config::{
         AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManager, CRDAlertManagerReceiver,
     },
+    crd::crd_prometheus_rules::{PrometheusRule, PrometheusRuleSpec, RuleGroup},
     prometheus::KubePrometheus,
 };
 
@@ -22,6 +30,8 @@ use super::{
 pub struct HelmPrometheusApplicationAlertingScore {
     pub namespace: String,
     pub receivers: Vec<Box<dyn CRDAlertManagerReceiver>>,
+    pub service_monitors: Vec<super::service_monitor::ServiceMonitor>,
+    pub prometheus_rules: Vec<RuleGroup>,
 }
 
 impl<T: Topology + K8sclient> Score<T> for HelmPrometheusApplicationAlertingScore {
@@ -29,6 +39,8 @@ impl<T: Topology + K8sclient> Score<T> for HelmPrometheusApplicationAlertingScor
         Box::new(HelmPrometheusApplicationAlertingInterpret {
             namespace: self.namespace.clone(),
             receivers: self.receivers.clone(),
+            service_monitors: self.service_monitors.clone(),
+            prometheus_rules: self.prometheus_rules.clone(),
         })
     }
 
@@ -41,6 +53,8 @@ impl<T: Topology + K8sclient> Score<T> for HelmPrometheusApplicationAlertingScor
 pub struct HelmPrometheusApplicationAlertingInterpret {
     pub namespace: String,
     pub receivers: Vec<Box<dyn CRDAlertManagerReceiver>>,
+    pub service_monitors: Vec<super::service_monitor::ServiceMonitor>,
+    pub prometheus_rules: Vec<RuleGroup>,
 }
 
 #[async_trait]
@@ -51,6 +65,8 @@ impl<T: Topology + K8sclient> Interpret<T> for HelmPrometheusApplicationAlerting
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
         let client = topology.k8s_client().await.unwrap();
+        self.install_prometheus(&client).await?;
+        self.install_alert_manager(&client).await?;
         for receiver in self.receivers.iter() {
             let alertmanager_config: AlertmanagerConfig = receiver
                 .configure_receiver(&client, self.namespace.clone())
@@ -64,6 +80,15 @@ impl<T: Topology + K8sclient> Interpret<T> for HelmPrometheusApplicationAlerting
                 InterpretError::new(format!("failed to install receiver: {}", err))
             })?;
         }
+        self.install_rules(self.prometheus_rules.clone(), client.clone())
+            .await
+            .map_err(|err| InterpretError::new(format!("failed to install rules: {}", err)))?;
+
+        debug!("\n\n\n monitors: {:#?}", self.service_monitors.clone());
+        for monitor in self.service_monitors.iter() {
+            self.install_monitor(monitor.clone(), client.clone())
+                .await?;
+        }
         Ok(Outcome::success(format!("deployed alert channels")))
     }
 
@@ -83,3 +108,146 @@ impl<T: Topology + K8sclient> Interpret<T> for HelmPrometheusApplicationAlerting
         todo!()
     }
 }
+
+impl HelmPrometheusApplicationAlertingInterpret {
+    async fn install_prometheus(&self, client: &Arc<K8sClient>) -> Result<Outcome, InterpretError> {
+        debug!(
+            "installing crd-prometheuses in namespace {}",
+            self.namespace.clone()
+        );
+        debug!("building role/rolebinding/serviceaccount for crd-prometheus");
+        let rolename = format!("{}-prom", self.namespace.clone());
+        let sa_name = format!("{}-prom-sa", self.namespace.clone());
+        let role = build_prom_role(rolename.clone(), self.namespace.clone());
+        let rolebinding =
+            build_prom_rolebinding(rolename.clone(), self.namespace.clone(), sa_name.clone());
+        let sa = build_prom_service_account(sa_name.clone(), self.namespace.clone());
+        let mut prom_spec = PrometheusSpec::default();
+        prom_spec.service_account_name = sa_name.clone();
+        let prom = Prometheus {
+            metadata: ObjectMeta {
+                name: Some(self.namespace.clone()),
+                labels: Some(std::collections::BTreeMap::from([(
+                    "alertmanagerConfig".to_string(),
+                    "enabled".to_string(),
+                )])),
+                namespace: Some(self.namespace.clone()),
+                ..Default::default()
+            },
+            spec: prom_spec,
+        };
+        client
+            .apply(&role, Some(&self.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        info!(
+            "installed prometheus role: {:#?} in ns {:#?}",
+            role.metadata.name.unwrap(),
+            role.metadata.namespace.unwrap()
+        );
+        client
+            .apply(&rolebinding, Some(&self.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        info!(
+            "installed prometheus rolebinding: {:#?} in ns {:#?}",
+            rolebinding.metadata.name.unwrap(),
+            rolebinding.metadata.namespace.unwrap()
+        );
+        client
+            .apply(&sa, Some(&self.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        info!(
+            "installed prometheus service account: {:#?} in ns {:#?}",
+            sa.metadata.name.unwrap(),
+            sa.metadata.namespace.unwrap()
+        );
+        client
+            .apply(&prom, Some(&self.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        info!(
+            "installed prometheus: {:#?} in ns {:#?}",
+            &prom.metadata.name.clone().unwrap(),
+            &prom.metadata.namespace.clone().unwrap()
+        );
+
+        Ok(Outcome::success(format!(
+            "successfully deployed crd-prometheus {:#?}",
+            prom
+        )))
+    }
+
+    async fn install_alert_manager(
+        &self,
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let am = Alertmanager {
+            metadata: ObjectMeta {
+                name: Some(self.namespace.clone()),
+                labels: Some(std::collections::BTreeMap::from([(
+                    "alertmanagerConfig".to_string(),
+                    "enabled".to_string(),
+                )])),
+                namespace: Some(self.namespace.clone()),
+                ..Default::default()
+            },
+            spec: AlertmanagerSpec::default(),
+        };
+        client
+            .apply(&am, Some(&self.namespace.clone()))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        Ok(Outcome::success(format!(
+            "successfully deployed service monitor {:#?}",
+            am.metadata.name
+        )))
+    }
+
+    async fn install_monitor(
+        &self,
+        monitor: super::service_monitor::ServiceMonitor,
+        client: Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        debug!("service monitor: \n{:#?}", monitor.clone());
+        let namespace = self.namespace.clone();
+        client
+            .apply(&monitor, Some(&namespace))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        Ok(Outcome::success(format!(
+            "successfully deployed service monitor {:#?}",
+            monitor.metadata.name
+        )))
+    }
+
+    async fn install_rules(
+        &self,
+        rules: Vec<RuleGroup>,
+        client: Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let prom_rule_spec = PrometheusRuleSpec { groups: rules };
+
+        let prom_rules = PrometheusRule {
+            metadata: ObjectMeta {
+                name: Some(self.namespace.clone()),
+                labels: Some(std::collections::BTreeMap::from([(
+                    "alertmanagerConfig".to_string(),
+                    "enabled".to_string(),
+                )])),
+                namespace: Some(self.namespace.clone()),
+                ..Default::default()
+            },
+            spec: prom_rule_spec,
+        };
+        client
+            .apply(&prom_rules, Some(&self.namespace))
+            .await
+            .map_err(|e| InterpretError::new(e.to_string()))?;
+        Ok(Outcome::success(format!(
+            "successfully deployed service monitor {:#?}",
+            prom_rules.metadata.name
+        )))
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/mod.rs

@@ -1,6 +1,8 @@
 pub mod alert_manager_config;
+pub mod crd;
 pub mod helm;
 pub mod helm_prometheus_alert_score;
 pub mod helm_prometheus_application_alerting;
 pub mod prometheus;
+pub mod service_monitor;
 pub mod types;

harmony/src/modules/monitoring/kube_prometheus/service_monitor.rs

@@ -0,0 +1,89 @@
+use std::collections::{BTreeMap, HashMap};
+
+use kube::{CustomResource, Resource, api::ObjectMeta};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use crate::interpret::InterpretError;
+
+use super::types::{
+    HTTPScheme, MatchExpression, NamespaceSelector, Operator, Selector,
+    ServiceMonitor as KubeServiceMonitor, ServiceMonitorEndpoint,
+};
+
+/// This is the top-level struct for the ServiceMonitor Custom Resource.
+/// The `#[derive(CustomResource)]` macro handles all the boilerplate for you,
+/// including the `impl Resource`.
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "monitoring.coreos.com",
+    version = "v1",
+    kind = "ServiceMonitor",
+    plural = "servicemonitors",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct ServiceMonitorSpec {
+    /// A label selector to select services to monitor.
+    pub selector: Selector,
+
+    /// A list of endpoints on the selected services to be monitored.
+    pub endpoints: Vec<ServiceMonitorEndpoint>,
+
+    /// Selector to select which namespaces the Kubernetes Endpoints objects
+    /// are discovered from.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub namespace_selector: Option<NamespaceSelector>,
+
+    /// The label to use to retrieve the job name from.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub job_label: Option<String>,
+
+    /// Pod-based target labels to transfer from the Kubernetes Pod onto the target.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub pod_target_labels: Vec<String>,
+
+    /// TargetLabels transfers labels on the Kubernetes Service object to the target.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub target_labels: Vec<String>,
+}
+
+impl Default for ServiceMonitorSpec {
+    fn default() -> Self {
+        let mut labels = HashMap::new();
+        Self {
+            selector: Selector {
+                match_labels: { labels },
+                match_expressions: vec![MatchExpression {
+                    key: "app.kubernetes.io/name".into(),
+                    operator: Operator::Exists,
+                    values: vec![],
+                }],
+            },
+            endpoints: vec![ServiceMonitorEndpoint {
+                port: Some("http".to_string()),
+                path: Some("/metrics".into()),
+                interval: Some("30s".into()),
+                scheme: Some(HTTPScheme::HTTP),
+                ..Default::default()
+            }],
+            namespace_selector: None, // only the same namespace
+            job_label: Some("app".into()),
+            pod_target_labels: vec![],
+            target_labels: vec![],
+        }
+    }
+}
+
+impl From<KubeServiceMonitor> for ServiceMonitorSpec {
+    fn from(value: KubeServiceMonitor) -> Self {
+        Self {
+            selector: value.selector,
+            endpoints: value.endpoints,
+            namespace_selector: value.namespace_selector,
+            job_label: value.job_label,
+            pod_target_labels: value.pod_target_labels,
+            target_labels: value.target_labels,
+        }
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/types.rs

@@ -1,7 +1,8 @@
 use std::collections::{BTreeMap, HashMap};
 
 use async_trait::async_trait;
-use serde::Serialize;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
 use serde_yaml::{Mapping, Sequence, Value};
 
 use crate::modules::monitoring::alert_rule::prometheus_alert_rule::AlertManagerRuleGroup;
@@ -94,7 +95,7 @@ pub struct AlertGroup {
     pub groups: Vec<AlertManagerRuleGroup>,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 pub enum HTTPScheme {
     #[serde(rename = "http")]
     HTTP,
@@ -102,7 +103,7 @@ pub enum HTTPScheme {
     HTTPS,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 pub enum Operator {
     In,
     NotIn,
@@ -147,70 +148,79 @@ pub struct ServiceMonitorTLSConfig {
     pub server_name: Option<String>,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct ServiceMonitorEndpoint {
-    // ## Name of the endpoint's service port
-    // ## Mutually exclusive with targetPort
+    /// Name of the service port this endpoint refers to.
     pub port: Option<String>,
 
-    // ## Name or number of the endpoint's target port
-    // ## Mutually exclusive with port
-    pub target_port: Option<String>,
-
-    // ## File containing bearer token to be used when scraping targets
-    // ##
-    pub bearer_token_file: Option<String>,
-
-    // ## Interval at which metrics should be scraped
-    // ##
+    /// Interval at which metrics should be scraped.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub interval: Option<String>,
 
-    // ## HTTP path to scrape for metrics
-    // ##
-    pub path: String,
+    /// The HTTP path to scrape for metrics.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub path: Option<String>,
 
-    // ## HTTP scheme to use for scraping
-    // ##
-    pub scheme: HTTPScheme,
+    /// HTTP scheme to use for scraping.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub scheme: Option<HTTPScheme>,
 
-    // ## TLS configuration to use when scraping the endpoint
-    // ##
-    pub tls_config: Option<ServiceMonitorTLSConfig>,
+    /// Relabelings to apply to samples before scraping.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub relabelings: Vec<RelabelConfig>,
 
-    // ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion.
-    // ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api-reference/api.md#relabelconfig
-    // ##
-    // # - action: keep
-    // #   regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+'
-    // #   sourceLabels: [__name__]
-    pub metric_relabelings: Vec<Mapping>,
-
-    // ## RelabelConfigs to apply to samples before scraping
-    // ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api-reference/api.md#relabelconfig
-    // ##
-    // # - sourceLabels: [__meta_kubernetes_pod_node_name]
-    // #   separator: ;
-    // #   regex: ^(.*)$
-    // #   targetLabel: nodename
-    // #   replacement: $1
-    // #   action: replace
-    pub relabelings: Vec<Mapping>,
+    /// MetricRelabelings to apply to samples after scraping, but before ingestion.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub metric_relabelings: Vec<RelabelConfig>,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct RelabelConfig {
+    /// The action to perform based on the regex matching.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub action: Option<String>,
+
+    /// A list of labels from which to extract values.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub source_labels: Vec<String>,
+
+    /// Separator to be used when concatenating source_labels.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub separator: Option<String>,
+
+    /// The label to which the resulting value is written.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub target_label: Option<String>,
+
+    /// A regular expression to match against the concatenated source label values.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub regex: Option<String>,
+
+    /// The replacement value to use.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub replacement: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct MatchExpression {
     pub key: String,
-    pub operator: Operator,
+    pub operator: Operator, // "In", "NotIn", "Exists", "DoesNotExist"
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub values: Vec<String>,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct Selector {
-    //   # label selector for services
+    /// A map of key-value pairs to match.
+    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
     pub match_labels: HashMap<String, String>,
+
+    /// A list of label selector requirements.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub match_expressions: Vec<MatchExpression>,
 }
 
@@ -258,10 +268,15 @@ pub struct ServiceMonitor {
     pub fallback_scrape_protocol: Option<String>,
 }
 
-#[derive(Debug, Serialize, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct NamespaceSelector {
+    /// Select all namespaces.
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
     pub any: bool,
+
+    /// List of namespace names to select from.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub match_names: Vec<String>,
 }
 
@@ -283,19 +298,3 @@ impl Default for ServiceMonitor {
         }
     }
 }
-
-impl Default for ServiceMonitorEndpoint {
-    fn default() -> Self {
-        Self {
-            port: Some("80".to_string()),
-            target_port: Default::default(),
-            bearer_token_file: Default::default(),
-            interval: Default::default(),
-            path: "/metrics".to_string(),
-            scheme: HTTPScheme::HTTP,
-            tls_config: Default::default(),
-            metric_relabelings: Default::default(),
-            relabelings: Default::default(),
-        }
-    }
-}