Merge branch 'master' into feat/detect_k8s_flavour

Reviewed-on: #160
Co-authored-by: Willem <wrolleman@nationtech.io>
Co-committed-by: Willem <wrolleman@nationtech.io>

harmony/src/domain/topology/k8s.rs

@@ -1,13 +1,17 @@
+use std::time::Duration;
+
 use derive_new::new;
 use k8s_openapi::{
     ClusterResourceScope, NamespaceResourceScope,
     api::{apps::v1::Deployment, core::v1::Pod},
+    apimachinery::pkg::version::Info,
 };
 use kube::{
-    Client, Config, Error, Resource,
+    Client, Config, Discovery, Error, Resource,
     api::{Api, AttachParams, DeleteParams, ListParams, Patch, PatchParams, ResourceExt},
     config::{KubeConfigOptions, Kubeconfig},
     core::ErrorResponse,
+    error::DiscoveryError,
     runtime::reflector::Lookup,
 };
 use kube::{api::DynamicObject, runtime::conditions};
@@ -19,7 +23,7 @@ use log::{debug, error, trace};
 use serde::{Serialize, de::DeserializeOwned};
 use serde_json::{Value, json};
 use similar::TextDiff;
-use tokio::io::AsyncReadExt;
+use tokio::{io::AsyncReadExt, time::sleep};
 
 #[derive(new, Clone)]
 pub struct K8sClient {
@@ -53,6 +57,17 @@ impl K8sClient {
         })
     }
 
+    pub async fn get_apiserver_version(&self) -> Result<Info, Error> {
+        let client: Client = self.client.clone();
+        let version_info: Info = client.apiserver_version().await?;
+        Ok(version_info)
+    }
+
+    pub async fn discovery(&self) -> Result<Discovery, Error> {
+        let discovery: Discovery = Discovery::new(self.client.clone()).run().await?;
+        Ok(discovery)
+    }
+
     pub async fn get_resource_json_value(
         &self,
         name: &str,
@@ -153,6 +168,41 @@ impl K8sClient {
         }
     }
 
+    pub async fn wait_for_pod_ready(
+        &self,
+        pod_name: &str,
+        namespace: Option<&str>,
+    ) -> Result<(), Error> {
+        let mut elapsed = 0;
+        let interval = 5; // seconds between checks
+        let timeout_secs = 120;
+        loop {
+            let pod = self.get_pod(pod_name, namespace).await?;
+
+            if let Some(p) = pod {
+                if let Some(status) = p.status {
+                    if let Some(phase) = status.phase {
+                        if phase.to_lowercase() == "running" {
+                            return Ok(());
+                        }
+                    }
+                }
+            }
+
+            if elapsed >= timeout_secs {
+                return Err(Error::Discovery(DiscoveryError::MissingResource(format!(
+                    "'{}' in ns '{}' did not become ready within {}s",
+                    pod_name,
+                    namespace.unwrap(),
+                    timeout_secs
+                ))));
+            }
+
+            sleep(Duration::from_secs(interval)).await;
+            elapsed += interval;
+        }
+    }
+
     /// Will execute a commond in the first pod found that matches the specified label
     /// '{label}={name}'
     pub async fn exec_app_capture_output(
@@ -419,9 +469,12 @@ impl K8sClient {
             .as_str()
             .expect("couldn't get kind as str");
 
-        let split: Vec<&str> = api_version.splitn(2, "/").collect();
+        let mut it = api_version.splitn(2, '/');
-        let g = split[0];
+        let first = it.next().unwrap();
-        let v = split[1];
+        let (g, v) = match it.next() {
+            Some(second) => (first, second),
+            None => ("", first),
+        };
 
         let gvk = GroupVersionKind::gvk(g, v, kind);
         let api_resource = ApiResource::from_gvk(&gvk);

harmony/src/domain/topology/k8s_anywhere.rs

@@ -47,6 +47,13 @@ struct K8sState {
     message: String,
 }
 
+#[derive(Debug, Clone)]
+pub enum KubernetesDistribution {
+    OpenshiftFamily,
+    K3sFamily,
+    Default,
+}
+
 #[derive(Debug, Clone)]
 enum K8sSource {
     LocalK3d,
@@ -57,6 +64,7 @@ enum K8sSource {
 pub struct K8sAnywhereTopology {
     k8s_state: Arc<OnceCell<Option<K8sState>>>,
     tenant_manager: Arc<OnceCell<K8sTenantManager>>,
+    flavour: Arc<OnceCell<KubernetesDistribution>>,
     config: Arc<K8sAnywhereConfig>,
 }
 
@@ -162,6 +170,7 @@ impl K8sAnywhereTopology {
         Self {
             k8s_state: Arc::new(OnceCell::new()),
             tenant_manager: Arc::new(OnceCell::new()),
+            flavour: Arc::new(OnceCell::new()),
             config: Arc::new(K8sAnywhereConfig::from_env()),
         }
     }
@@ -170,10 +179,42 @@ impl K8sAnywhereTopology {
         Self {
             k8s_state: Arc::new(OnceCell::new()),
             tenant_manager: Arc::new(OnceCell::new()),
+            flavour: Arc::new(OnceCell::new()),
             config: Arc::new(config),
         }
     }
 
+    pub async fn get_k8s_distribution(&self) -> Result<&KubernetesDistribution, PreparationError> {
+        self.flavour
+            .get_or_try_init(async || {
+                let client = self.k8s_client().await.unwrap();
+
+                let discovery = client.discovery().await.map_err(|e| {
+                    PreparationError::new(format!("Could not discover API groups: {}", e))
+                })?;
+
+                let version = client.get_apiserver_version().await.map_err(|e| {
+                    PreparationError::new(format!("Could not get server version: {}", e))
+                })?;
+
+                // OpenShift / OKD
+                if discovery
+                    .groups()
+                    .any(|g| g.name() == "project.openshift.io")
+                {
+                    return Ok(KubernetesDistribution::OpenshiftFamily);
+                }
+
+                // K3d / K3s
+                if version.git_version.contains("k3s") {
+                    return Ok(KubernetesDistribution::K3sFamily);
+                }
+
+                return Ok(KubernetesDistribution::Default);
+            })
+            .await
+    }
+
     async fn get_cluster_observability_operator_prometheus_application_score(
         &self,
         sender: RHOBObservability,

harmony/src/modules/application/features/helm_argocd_score.rs

@@ -160,6 +160,9 @@ global:
   ## Used for ingresses, certificates, SSO, notifications, etc.
   domain: {domain}
 
+  securityContext: 
+    runAsUser: null
+
   # -- Runtime class name for all components
   runtimeClassName: ""
 
@@ -471,6 +474,13 @@ redis:
   # -- Redis name
   name: redis
 
+  serviceAccount:
+    create: true
+
+  securityContext:
+    runAsUser: null
+
+
   ## Redis image
   image:
     # -- Redis repository

harmony/src/modules/monitoring/mod.rs

@@ -4,4 +4,5 @@ pub mod application_monitoring;
 pub mod grafana;
 pub mod kube_prometheus;
 pub mod ntfy;
+pub mod okd;
 pub mod prometheus;

harmony/src/modules/monitoring/okd/enable_user_workload.rs

@@ -0,0 +1,149 @@
+use std::{collections::BTreeMap, sync::Arc};
+
+use crate::{
+    data::Version,
+    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
+    inventory::Inventory,
+    score::Score,
+    topology::{K8sclient, Topology, k8s::K8sClient},
+};
+use async_trait::async_trait;
+use harmony_types::id::Id;
+use k8s_openapi::api::core::v1::ConfigMap;
+use kube::api::ObjectMeta;
+use serde::Serialize;
+
+#[derive(Clone, Debug, Serialize)]
+pub struct OpenshiftUserWorkloadMonitoring {}
+
+impl<T: Topology + K8sclient> Score<T> for OpenshiftUserWorkloadMonitoring {
+    fn name(&self) -> String {
+        "OpenshiftUserWorkloadMonitoringScore".to_string()
+    }
+
+    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
+        Box::new(OpenshiftUserWorkloadMonitoringInterpret {})
+    }
+}
+
+#[derive(Clone, Debug, Serialize)]
+pub struct OpenshiftUserWorkloadMonitoringInterpret {}
+
+#[async_trait]
+impl<T: Topology + K8sclient> Interpret<T> for OpenshiftUserWorkloadMonitoringInterpret {
+    async fn execute(
+        &self,
+        _inventory: &Inventory,
+        topology: &T,
+    ) -> Result<Outcome, InterpretError> {
+        let client = topology.k8s_client().await.unwrap();
+        self.update_cluster_monitoring_config_cm(&client).await?;
+        self.update_user_workload_monitoring_config_cm(&client)
+            .await?;
+        self.verify_user_workload(&client).await?;
+        Ok(Outcome::success(
+            "successfully enabled user-workload-monitoring".to_string(),
+        ))
+    }
+
+    fn get_name(&self) -> InterpretName {
+        InterpretName::Custom("OpenshiftUserWorkloadMonitoring")
+    }
+
+    fn get_version(&self) -> Version {
+        todo!()
+    }
+
+    fn get_status(&self) -> InterpretStatus {
+        todo!()
+    }
+
+    fn get_children(&self) -> Vec<Id> {
+        todo!()
+    }
+}
+
+impl OpenshiftUserWorkloadMonitoringInterpret {
+    pub async fn update_cluster_monitoring_config_cm(
+        &self,
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let mut data = BTreeMap::new();
+        data.insert(
+            "config.yaml".to_string(),
+            r#"
+enableUserWorkload: true
+alertmanagerMain:
+  enableUserAlertmanagerConfig: true
+"#
+            .to_string(),
+        );
+
+        let cm = ConfigMap {
+            metadata: ObjectMeta {
+                name: Some("cluster-monitoring-config".to_string()),
+                namespace: Some("openshift-monitoring".to_string()),
+                ..Default::default()
+            },
+            data: Some(data),
+            ..Default::default()
+        };
+        client.apply(&cm, Some("openshift-monitoring")).await?;
+
+        Ok(Outcome::success(
+            "updated cluster-monitoring-config-map".to_string(),
+        ))
+    }
+
+    pub async fn update_user_workload_monitoring_config_cm(
+        &self,
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let mut data = BTreeMap::new();
+        data.insert(
+            "config.yaml".to_string(),
+            r#"
+alertmanager: 
+  enabled: true
+  enableAlertmanagerConfig: true
+"#
+            .to_string(),
+        );
+        let cm = ConfigMap {
+            metadata: ObjectMeta {
+                name: Some("user-workload-monitoring-config".to_string()),
+                namespace: Some("openshift-user-workload-monitoring".to_string()),
+                ..Default::default()
+            },
+            data: Some(data),
+            ..Default::default()
+        };
+        client
+            .apply(&cm, Some("openshift-user-workload-monitoring"))
+            .await?;
+
+        Ok(Outcome::success(
+            "updated openshift-user-monitoring-config-map".to_string(),
+        ))
+    }
+
+    pub async fn verify_user_workload(
+        &self,
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let namespace = "openshift-user-workload-monitoring";
+        let alertmanager_name = "alertmanager-user-workload-0";
+        let prometheus_name = "prometheus-user-workload-0";
+        client
+            .wait_for_pod_ready(alertmanager_name, Some(namespace))
+            .await?;
+        client
+            .wait_for_pod_ready(prometheus_name, Some(namespace))
+            .await?;
+
+        Ok(Outcome::success(format!(
+            "pods: {}, {} ready in ns: {}",
+            alertmanager_name, prometheus_name, namespace
+        )))
+    }
+}

harmony/src/modules/monitoring/okd/mod.rs

@@ -0,0 +1 @@
+pub mod enable_user_workload;

harmony/src/modules/prometheus/rhob_alerting_score.rs

@@ -12,9 +12,6 @@ use std::process::Command;
 use crate::modules::k8s::ingress::{K8sIngressScore, PathType};
 use crate::modules::monitoring::kube_prometheus::crd::grafana_default_dashboard::build_default_dashboard;
 use crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::RHOBObservability;
-use crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanagers::{
-    Alertmanager, AlertmanagerSpec,
-};
 use crate::modules::monitoring::kube_prometheus::crd::rhob_grafana::{
     Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig,
     GrafanaDatasourceSpec, GrafanaSpec,
@@ -25,13 +22,8 @@ use crate::modules::monitoring::kube_prometheus::crd::rhob_monitoring_stack::{
 use crate::modules::monitoring::kube_prometheus::crd::rhob_prometheus_rules::{
     PrometheusRule, PrometheusRuleSpec, RuleGroup,
 };
-use crate::modules::monitoring::kube_prometheus::crd::rhob_prometheuses::{
+use crate::modules::monitoring::kube_prometheus::crd::rhob_prometheuses::LabelSelector;
-    AlertmanagerEndpoints, LabelSelector, PrometheusSpec, PrometheusSpecAlerting,
-};
 
-use crate::modules::monitoring::kube_prometheus::crd::rhob_role::{
-    build_prom_role, build_prom_rolebinding, build_prom_service_account,
-};
 use crate::modules::monitoring::kube_prometheus::crd::rhob_service_monitor::{
     ServiceMonitor, ServiceMonitorSpec,
 };