diff --git a/examples/try_rust_webapp/src/main.rs b/examples/try_rust_webapp/src/main.rs index 56a058d..7bfdf57 100644 --- a/examples/try_rust_webapp/src/main.rs +++ b/examples/try_rust_webapp/src/main.rs @@ -3,7 +3,7 @@ use harmony::{ modules::{ application::{ ApplicationScore, RustWebFramework, RustWebapp, - features::{PackagingDeployment, rhob_monitoring::Monitoring}, + features::{Monitoring, PackagingDeployment}, }, monitoring::alert_channel::discord_alert_channel::DiscordWebhook, }, diff --git a/harmony/src/domain/topology/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere.rs index 6dfb1a8..895f7da 100644 --- a/harmony/src/domain/topology/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere.rs @@ -1,7 +1,7 @@ -use std::{process::Command, sync::Arc}; +use std::{collections::BTreeMap, process::Command, sync::Arc}; use async_trait::async_trait; -use kube::api::GroupVersionKind; +use kube::api::{GroupVersionKind, ObjectMeta}; use log::{debug, info, warn}; use serde::Serialize; use tokio::sync::OnceCell; @@ -12,12 +12,20 @@ use crate::{ inventory::Inventory, modules::{ k3d::K3DInstallationScore, + k8s::ingress::{K8sIngressScore, PathType}, monitoring::{ grafana::{grafana::Grafana, helm::helm_grafana::grafana_helm_chart_score}, kube_prometheus::crd::{ crd_alertmanager_config::CRDPrometheus, + crd_grafana::{ + Grafana as GrafanaCRD, GrafanaDashboard, GrafanaDashboardSpec, + GrafanaDatasource, GrafanaDatasourceConfig, GrafanaDatasourceSpec, GrafanaSpec, + }, + crd_prometheuses::LabelSelector, + grafana_default_dashboard::build_default_dashboard, prometheus_operator::prometheus_operator_helm_chart_score, - rhob_alertmanager_config::RHOBObservability, service_monitor::ServiceMonitor, + rhob_alertmanager_config::RHOBObservability, + service_monitor::ServiceMonitor, }, }, prometheus::{ @@ -90,10 +98,11 @@ impl K8sclient for K8sAnywhereTopology { #[async_trait] impl Grafana for K8sAnywhereTopology { - async fn ensure_grafana_operator_ready( + async fn ensure_grafana_operator( &self, inventory: &Inventory, ) -> Result { + debug!("ensure grafana operator"); let client = self.k8s_client().await.unwrap(); let grafana_gvk = GroupVersionKind { group: "grafana.integreatly.org".to_string(), @@ -112,6 +121,7 @@ impl Grafana for K8sAnywhereTopology { details: "Found grafana CRDs in cluster".to_string(), }); } + Err(_) => { return self .install_grafana_operator(inventory, Some("grafana")) @@ -120,7 +130,41 @@ impl Grafana for K8sAnywhereTopology { }; } async fn install_grafana(&self) -> Result { - todo!() + debug!("install grafana"); + let ns = "grafana"; + + let mut label = BTreeMap::new(); + + label.insert("dashboards".to_string(), "grafana".to_string()); + let label_selector = LabelSelector { + match_labels: label.clone(), + match_expressions: vec![], + }; + + let client = self.k8s_client().await?; + + let datasource = self.build_grafana_datasource(ns, &label_selector); + + client.apply(&datasource, Some(ns)).await?; + + let dashboard = self.build_grafana_dashboard(ns, &label_selector); + + client.apply(&dashboard, Some(ns)).await?; + + let grafana = self.build_grafana(ns, &label); + + client.apply(&grafana, Some(ns)).await?; + + let grafana_ingress = self.build_grafana_ingress(ns).await; + + grafana_ingress + .interpret(&Inventory::empty(), self) + .await + .map_err(|e| PreparationError::new(e.to_string()))?; + + Ok(PreparationOutcome::Success { + details: "Installed grafana composants".to_string(), + }) } } @@ -129,49 +173,38 @@ impl PrometheusMonitoring for K8sAnywhereTopology { async fn install_prometheus( &self, sender: &CRDPrometheus, - inventory: &Inventory, - receivers: Option>>>, + _inventory: &Inventory, + _receivers: Option>>>, ) -> Result { - let po_result = self.ensure_prometheus_operator(sender).await?; + let client = self.k8s_client().await?; - if po_result == PreparationOutcome::Noop { - debug!("Skipping Prometheus CR installation due to missing operator."); - return Ok(po_result); - } - - let result = self - .get_k8s_prometheus_application_score( - sender.clone(), - receivers, - Some(sender.service_monitor.clone()), - ) - .await - .interpret(inventory, self) - .await; - - match result { - Ok(outcome) => match outcome.status { - InterpretStatus::SUCCESS => Ok(PreparationOutcome::Success { - details: outcome.message, - }), - InterpretStatus::NOOP => Ok(PreparationOutcome::Noop), - _ => Err(PreparationError::new(outcome.message)), - }, - Err(err) => Err(PreparationError::new(err.to_string())), + for monitor in sender.service_monitor.iter() { + client + .apply(monitor, Some(&sender.namespace)) + .await + .map_err(|e| PreparationError::new(e.to_string()))?; } + Ok(PreparationOutcome::Success { + details: "successfuly installed prometheus components".to_string(), + }) } + async fn ensure_prometheus_operator( &self, sender: &CRDPrometheus, - inventory: &Inventory, + _inventory: &Inventory, ) -> Result { let po_result = self.ensure_prometheus_operator(sender).await?; - if po_result == PreparationOutcome::Noop { - debug!("Skipping Prometheus CR installation due to missing operator."); - return Ok(po_result); - } else { - todo!() + match po_result { + PreparationOutcome::Success { details: _ } => { + debug!("Detected prometheus crds operator present in cluster."); + return Ok(po_result); + } + PreparationOutcome::Noop => { + debug!("Skipping Prometheus CR installation due to missing operator."); + return Ok(po_result); + } } } } @@ -211,6 +244,7 @@ impl PrometheusMonitoring for K8sAnywhereTopology { Err(err) => Err(PreparationError::new(err.to_string())), } } + async fn ensure_prometheus_operator( &self, sender: &RHOBObservability, @@ -300,6 +334,95 @@ impl K8sAnywhereTopology { .clone() } + fn build_grafana_datasource( + &self, + ns: &str, + label_selector: &LabelSelector, + ) -> GrafanaDatasource { + let mut json_data = BTreeMap::new(); + json_data.insert("timeInterval".to_string(), "5s".to_string()); + + let graf_data_source = GrafanaDatasource { + metadata: ObjectMeta { + name: Some(format!("grafana-datasource-{}", ns)), + namespace: Some(ns.to_string()), + ..Default::default() + }, + spec: GrafanaDatasourceSpec { + instance_selector: label_selector.clone(), + allow_cross_namespace_import: Some(false), + datasource: GrafanaDatasourceConfig { + access: "proxy".to_string(), + database: Some("prometheus".to_string()), + json_data: Some(json_data), + //this is fragile + name: format!("prometheus-{}-0", ns), + r#type: "prometheus".to_string(), + url: format!("http://prometheus-operated.{}.svc.cluster.local:9090", ns), + }, + }, + }; + graf_data_source + } + + fn build_grafana_dashboard( + &self, + ns: &str, + label_selector: &LabelSelector, + ) -> GrafanaDashboard { + let json = build_default_dashboard(ns); + let graf_dashboard = GrafanaDashboard { + metadata: ObjectMeta { + name: Some(format!("grafana-dashboard-{}", ns)), + namespace: Some(ns.to_string()), + ..Default::default() + }, + spec: GrafanaDashboardSpec { + resync_period: Some("30s".to_string()), + instance_selector: label_selector.clone(), + json, + }, + }; + graf_dashboard + } + + fn build_grafana(&self, ns: &str, labels: &BTreeMap) -> GrafanaCRD { + let grafana = GrafanaCRD { + metadata: ObjectMeta { + name: Some(format!("grafana-{}", ns)), + namespace: Some(ns.to_string()), + labels: Some(labels.clone()), + ..Default::default() + }, + spec: GrafanaSpec { + config: None, + admin_user: None, + admin_password: None, + ingress: None, + persistence: None, + resources: None, + }, + }; + grafana + } + + async fn build_grafana_ingress(&self, ns: &str) -> K8sIngressScore { + let domain = self.get_domain(&format!("grafana-{}", ns)).await.unwrap(); + let name = format!("{}-grafana", ns); + let backend_service = format!("grafana-{}-service", ns); + + K8sIngressScore { + name: fqdn::fqdn!(&name), + host: fqdn::fqdn!(&domain), + backend_service: fqdn::fqdn!(&backend_service), + port: 3000, + path: Some("/".to_string()), + path_type: Some(PathType::Prefix), + namespace: Some(fqdn::fqdn!(&ns)), + ingress_class_name: Some("openshift-default".to_string()), + } + } + async fn get_cluster_observability_operator_prometheus_application_score( &self, sender: RHOBObservability, @@ -607,7 +730,14 @@ impl K8sAnywhereTopology { inventory: &Inventory, ns: Option<&str>, ) -> Result { - let _grafana_operator_score = grafana_helm_chart_score(ns.unwrap(), true) + let namespace = ns.unwrap_or("grafana"); + info!("installing grafana operator in ns {namespace}"); + let tenant = self.get_k8s_tenant_manager()?.get_tenant_config().await; + let mut namespace_scope = false; + if tenant.is_some() { + namespace_scope = true; + } + let _grafana_operator_score = grafana_helm_chart_score(namespace, namespace_scope) .interpret(inventory, self) .await; Ok(PreparationOutcome::Success { diff --git a/harmony/src/domain/topology/oberservability/monitoring.rs b/harmony/src/domain/topology/oberservability/monitoring.rs index 1489e83..0c57ea4 100644 --- a/harmony/src/domain/topology/oberservability/monitoring.rs +++ b/harmony/src/domain/topology/oberservability/monitoring.rs @@ -30,6 +30,7 @@ impl, T: Topology> Interpret for AlertingInte inventory: &Inventory, topology: &T, ) -> Result { + debug!("hit sender configure for AlertingInterpret"); self.sender.configure(inventory, topology).await?; for receiver in self.receivers.iter() { receiver.install(&self.sender).await?; @@ -38,6 +39,7 @@ impl, T: Topology> Interpret for AlertingInte debug!("installing rule: {:#?}", rule); rule.install(&self.sender).await?; } + debug!("hit sender ensure installed for AlertingInterpret"); self.sender.ensure_installed(inventory, topology).await?; Ok(Outcome::success(format!( "successfully installed alert sender {}", diff --git a/harmony/src/modules/application/features/monitoring.rs b/harmony/src/modules/application/features/monitoring.rs index 0fd155d..fd6ae2a 100644 --- a/harmony/src/modules/application/features/monitoring.rs +++ b/harmony/src/modules/application/features/monitoring.rs @@ -2,7 +2,11 @@ use crate::modules::application::{ Application, ApplicationFeature, InstallationError, InstallationOutcome, }; use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore; +use crate::modules::monitoring::grafana::grafana::Grafana; use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus; +use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{ + ServiceMonitor, ServiceMonitorSpec, +}; use crate::topology::MultiTargetTopology; use crate::topology::ingress::Ingress; use crate::{ @@ -22,6 +26,7 @@ use base64::{Engine as _, engine::general_purpose}; use harmony_secret::SecretManager; use harmony_secret_derive::Secret; use harmony_types::net::Url; +use kube::api::ObjectMeta; use log::{debug, info}; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -41,6 +46,7 @@ impl< + K8sclient + MultiTargetTopology + PrometheusMonitoring + + Grafana + Ingress + std::fmt::Debug, > ApplicationFeature for Monitoring @@ -57,11 +63,20 @@ impl< .unwrap_or_else(|| self.application.name()); let domain = topology.get_domain("ntfy").await.unwrap(); + let app_service_monitor = ServiceMonitor { + metadata: ObjectMeta { + name: Some(self.application.name()), + namespace: Some(namespace.clone()), + ..Default::default() + }, + spec: ServiceMonitorSpec::default(), + }; + let mut alerting_score = ApplicationMonitoringScore { sender: CRDPrometheus { namespace: namespace.clone(), client: topology.k8s_client().await.unwrap(), - service_monitor: vec![], + service_monitor: vec![app_service_monitor], }, application: self.application.clone(), receivers: self.alert_receiver.clone(), diff --git a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs index 2780edd..0f6e0ec 100644 --- a/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs +++ b/harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs @@ -1,21 +1,23 @@ use std::sync::Arc; -use async_trait::async_trait; +use log::debug; use serde::Serialize; use crate::{ - data::Version, - interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, - inventory::Inventory, + interpret::Interpret, modules::{ application::Application, - monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus, + monitoring::{ + grafana::grafana::Grafana, kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus, + }, prometheus::prometheus::PrometheusMonitoring, }, score::Score, - topology::{PreparationOutcome, Topology, oberservability::monitoring::AlertReceiver}, + topology::{ + K8sclient, Topology, + oberservability::monitoring::{AlertReceiver, AlertingInterpret}, + }, }; -use harmony_types::id::Id; #[derive(Debug, Clone, Serialize)] pub struct ApplicationMonitoringScore { @@ -24,10 +26,15 @@ pub struct ApplicationMonitoringScore { pub receivers: Vec>>, } -impl> Score for ApplicationMonitoringScore { +impl + K8sclient + Grafana> Score + for ApplicationMonitoringScore +{ fn create_interpret(&self) -> Box> { - Box::new(ApplicationMonitoringInterpret { - score: self.clone(), + debug!("creating alerting interpret"); + Box::new(AlertingInterpret { + sender: self.sender.clone(), + receivers: self.receivers.clone(), + rules: vec![], }) } @@ -38,55 +45,3 @@ impl> Score for Application ) } } - -#[derive(Debug)] -pub struct ApplicationMonitoringInterpret { - score: ApplicationMonitoringScore, -} - -#[async_trait] -impl> Interpret - for ApplicationMonitoringInterpret -{ - async fn execute( - &self, - inventory: &Inventory, - topology: &T, - ) -> Result { - let result = topology - .install_prometheus( - &self.score.sender, - inventory, - Some(self.score.receivers.clone()), - ) - .await; - - match result { - Ok(outcome) => match outcome { - PreparationOutcome::Success { details: _ } => { - Ok(Outcome::success("Prometheus installed".into())) - } - PreparationOutcome::Noop => { - Ok(Outcome::noop("Prometheus installation skipped".into())) - } - }, - Err(err) => Err(InterpretError::from(err)), - } - } - - fn get_name(&self) -> InterpretName { - InterpretName::ApplicationMonitoring - } - - fn get_version(&self) -> Version { - todo!() - } - - fn get_status(&self) -> InterpretStatus { - todo!() - } - - fn get_children(&self) -> Vec { - todo!() - } -} diff --git a/harmony/src/modules/monitoring/grafana/grafana.rs b/harmony/src/modules/monitoring/grafana/grafana.rs index 411d7a6..5ab57c2 100644 --- a/harmony/src/modules/monitoring/grafana/grafana.rs +++ b/harmony/src/modules/monitoring/grafana/grafana.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use k8s_openapi::Resource; use crate::{ inventory::Inventory, @@ -7,9 +8,10 @@ use crate::{ #[async_trait] pub trait Grafana { - async fn ensure_grafana_operator_ready( + async fn ensure_grafana_operator( &self, inventory: &Inventory, ) -> Result; + async fn install_grafana(&self) -> Result; } diff --git a/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs b/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs index 094beca..2965ada 100644 --- a/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs +++ b/harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs @@ -1,24 +1,27 @@ use non_blank_string_rs::NonBlankString; use std::{collections::HashMap, str::FromStr}; -use crate::modules::helm::chart::HelmChartScore; +use crate::modules::helm::chart::{HelmChartScore, HelmRepository}; -pub fn grafana_helm_chart_score(ns: &str, scope: bool) -> HelmChartScore { +pub fn grafana_helm_chart_score(ns: &str, namespace_scope: bool) -> HelmChartScore { let mut values_overrides = HashMap::new(); values_overrides.insert( NonBlankString::from_str("namespaceScope").unwrap(), - scope.to_string(), + namespace_scope.to_string(), ); HelmChartScore { namespace: Some(NonBlankString::from_str(ns).unwrap()), - release_name: NonBlankString::from_str("grafana").unwrap(), - chart_name: NonBlankString::from_str("oci://ghcr.io/grafana/helm-charts/grafana-operator") - .unwrap(), + release_name: NonBlankString::from_str("grafana-operator").unwrap(), + chart_name: NonBlankString::from_str("grafana/grafana-operator").unwrap(), chart_version: None, values_overrides: Some(values_overrides), values_yaml: None, create_namespace: true, install_only: true, - repository: None, + repository: Some(HelmRepository::new( + "grafana".to_string(), + url::Url::parse("https://grafana.github.io/helm-charts").unwrap(), + true, + )), } } diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs index 0ac8fc7..ceeca41 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs @@ -68,7 +68,7 @@ impl + Grafana> In for CRDPrometheus { async fn configure(&self, inventory: &Inventory, topology: &T) -> Result<(), InterpretError> { - topology.ensure_grafana_operator_ready(inventory).await?; + topology.ensure_grafana_operator(inventory).await?; topology.ensure_prometheus_operator(self, inventory).await?; Ok(()) }