diff --git a/harmony/src/domain/topology/k8s_anywhere.rs b/harmony/src/domain/topology/k8s_anywhere.rs index cb37ece..cb4ab2d 100644 --- a/harmony/src/domain/topology/k8s_anywhere.rs +++ b/harmony/src/domain/topology/k8s_anywhere.rs @@ -6,7 +6,9 @@ use std::{ use async_trait::async_trait; use k8s_openapi::api::{ - authentication::v1::{TokenRequest, TokenRequestSpec, TokenRequestStatus}, + authentication::v1::{ + BoundObjectReference, TokenRequest, TokenRequestSpec, TokenRequestStatus, + }, core::v1::{Secret, ServiceAccount}, rbac::v1::{ClusterRoleBinding, RoleRef, Subject}, }; @@ -30,9 +32,11 @@ use crate::{ kube_prometheus::crd::{ crd_alertmanager_config::CRDPrometheus, crd_grafana::{ - Grafana as GrafanaCRD, GrafanaDashboard, GrafanaDashboardSpec, - GrafanaDatasource, GrafanaDatasourceConfig, GrafanaDatasourceJsonData, - GrafanaDatasourceSecureJsonData, GrafanaDatasourceSpec, GrafanaSpec, + Grafana as GrafanaCRD, GrafanaCom, GrafanaDashboard, + GrafanaDashboardDatasource, GrafanaDashboardSpec, GrafanaDatasource, + GrafanaDatasourceConfig, GrafanaDatasourceJsonData, + GrafanaDatasourceSecureJsonData, GrafanaDatasourceSpec, GrafanaSecretKeyRef, + GrafanaSpec, GrafanaValueFrom, GrafanaValueSource, }, crd_prometheuses::LabelSelector, grafana_default_dashboard::build_default_dashboard, @@ -166,22 +170,24 @@ impl Grafana for K8sAnywhereTopology { .wait_until_deployment_ready( "grafana-grafana-deployment".to_string(), Some("grafana"), - Some(15), + Some(30), ) .await?; let sa_name = "grafana-grafana-sa"; - debug!("creating token for sevice account {sa_name}"); - let token = self.create_service_account_token(sa_name, ns).await?; + let token_secret_name = "grafana-sa-token-secret"; - debug!("creating secret"); - let secret_name = "grafana-sa-secret"; - let secret = self.build_token_secret(secret_name, &token.token, ns).await; + // let sa_token_secret = self.build_sa_token_secret(token_secret_name, sa_name, ns); + // + // client.apply(&sa_token_secret, Some(ns)).await?; + let secret = self.build_token_secret(token_secret_name, ns).await; client.apply(&secret, Some(ns)).await?; + let token_request_status = self.create_service_account_token(sa_name, ns).await?; debug!("creating grafana clusterrole binding"); + let clusterrolebinding = self.build_cluster_rolebinding(sa_name, "cluster-monitoring-view", ns); @@ -189,7 +195,7 @@ impl Grafana for K8sAnywhereTopology { debug!("creating grafana datasource crd"); - let token_str = format!("Bearer {}", token.token); + // let token_str = format!("Bearer {}", token.token); let thanos_url = format!( "https://{}", @@ -203,36 +209,11 @@ impl Grafana for K8sAnywhereTopology { ns, &label_selector, &thanos_url, - token_str.clone(), + &token_request_status.token, // Pass the secret name here ); client.apply(&thanos_openshift_datasource, Some(ns)).await?; - //TODO user workload datasource returns 503 -> need to figure out how to correctly add the - //userworkload thanos-ruler or prometheus-federate to the grafana datasource - //it may alrady be included in the overall monitoring stack - - let user_thanos_url = format!( - "https://{}", - self.get_domain( - "thanos-ruler-openshift-user-workload-monitoring.apps.ncd0.harmony.mcd" - ) - .await - .unwrap() - ); - - let thanos_openshift_userworkload_datasource = self.build_grafana_datasource( - "thanos-openshift-userworkload-monitoring", - ns, - &label_selector, - &user_thanos_url, - token_str.clone(), - ); - - client - .apply(&thanos_openshift_userworkload_datasource, Some(ns)) - .await?; - debug!("creating grafana dashboard crd"); let dashboard = self.build_grafana_dashboard(ns, &label_selector); @@ -446,6 +427,30 @@ impl K8sAnywhereTopology { } } + pub fn build_sa_token_secret( + &self, + secret_name: &str, + service_account_name: &str, + ns: &str, + ) -> Secret { + let mut annotations = BTreeMap::new(); + annotations.insert( + "kubernetes.io/service-account.name".to_string(), + service_account_name.to_string(), + ); + + Secret { + metadata: ObjectMeta { + name: Some(secret_name.into()), + namespace: Some(ns.into()), + annotations: Some(annotations), + ..Default::default() + }, + type_: Some("kubernetes.io/service-account-token".to_string()), + ..Default::default() + } + } + pub fn get_token_request(&self, ns: &str) -> TokenRequest { debug!("building token request"); TokenRequest { @@ -456,7 +461,11 @@ impl K8sAnywhereTopology { spec: TokenRequestSpec { audiences: vec!["https://kubernetes.default.svc".to_string()], expiration_seconds: Some(3600), - ..Default::default() + bound_object_ref: Some(BoundObjectReference { + kind: Some("Secret".to_string()), + name: Some("grafana-sa-token-secret".to_string()), + ..Default::default() + }), }, ..Default::default() } @@ -486,17 +495,14 @@ impl K8sAnywhereTopology { Ok(status) } - pub async fn build_token_secret(&self, secret_name: &str, token: &str, ns: &str) -> Secret { + pub async fn build_token_secret(&self, secret_name: &str, ns: &str) -> Secret { Secret { metadata: ObjectMeta { name: Some(secret_name.into()), namespace: Some(ns.into()), ..Default::default() }, - string_data: Some(std::collections::BTreeMap::from([( - secret_name.into(), - format!("Bearer {}", token), - )])), + string_data: None, ..Default::default() } } @@ -507,7 +513,7 @@ impl K8sAnywhereTopology { ns: &str, label_selector: &LabelSelector, url: &str, - token: String, + token: &str, // Pass in the secret name ) -> GrafanaDatasource { let mut json_data = BTreeMap::new(); json_data.insert("timeInterval".to_string(), "5s".to_string()); @@ -521,6 +527,7 @@ impl K8sAnywhereTopology { spec: GrafanaDatasourceSpec { instance_selector: label_selector.clone(), allow_cross_namespace_import: Some(true), + values_from: None, datasource: GrafanaDatasourceConfig { access: "proxy".to_string(), name: name.to_string(), @@ -534,7 +541,7 @@ impl K8sAnywhereTopology { oauth_pass_thru: Some(true), }), secure_json_data: Some(GrafanaDatasourceSecureJsonData { - http_header_value1: Some(token), + http_header_value1: Some(format!("Bearer {token}")), }), is_default: Some(false), editable: Some(true), @@ -548,7 +555,6 @@ impl K8sAnywhereTopology { ns: &str, label_selector: &LabelSelector, ) -> GrafanaDashboard { - let json = build_default_dashboard(ns); let graf_dashboard = GrafanaDashboard { metadata: ObjectMeta { name: Some(format!("grafana-dashboard-{}", ns)), @@ -558,7 +564,15 @@ impl K8sAnywhereTopology { spec: GrafanaDashboardSpec { resync_period: Some("30s".to_string()), instance_selector: label_selector.clone(), - json, + datasources: Some(vec![GrafanaDashboardDatasource { + input_name: "DS_PROMETHEUS".to_string(), + datasource_name: "thanos-openshift-monitoring".to_string(), + }]), + json: None, + grafana_com: Some(GrafanaCom { + id: 17406, + revision: None, + }), }, }; graf_dashboard diff --git a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_grafana.rs b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_grafana.rs index e58f4ca..c99adc1 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/crd/crd_grafana.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/crd/crd_grafana.rs @@ -103,9 +103,34 @@ pub struct GrafanaDashboardSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub resync_period: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub datasources: Option>, + pub instance_selector: LabelSelector, - pub json: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub json: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub grafana_com: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct GrafanaDashboardDatasource { + pub input_name: String, + pub datasource_name: String, +} + +// ------------------------------------------------------------------------------------------------ + +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct GrafanaCom { + pub id: u32, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub revision: Option, } // ------------------------------------------------------------------------------------------------ @@ -126,6 +151,30 @@ pub struct GrafanaDatasourceSpec { pub allow_cross_namespace_import: Option, pub datasource: GrafanaDatasourceConfig, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub values_from: Option>, +} + + +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct GrafanaValueFrom { + pub target_path: String, + pub value_from: GrafanaValueSource, +} + +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct GrafanaValueSource { + pub secret_key_ref: GrafanaSecretKeyRef, +} + +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct GrafanaSecretKeyRef { + pub name: String, + pub key: String, } #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] diff --git a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs index f9e8531..7873235 100644 --- a/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs +++ b/harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs @@ -11,8 +11,7 @@ use std::process::Command; use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus; use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules; use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{ - Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig, - GrafanaDatasourceJsonData, GrafanaDatasourceSpec, GrafanaSpec, + Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig, GrafanaDatasourceJsonData, GrafanaDatasourceSpec, GrafanaSecretKeyRef, GrafanaSpec, GrafanaValueFrom, GrafanaValueSource }; use crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::{ PrometheusRule, PrometheusRuleSpec, RuleGroup, @@ -504,6 +503,7 @@ impl K8sPrometheusCRDAlertingInterpret { is_default: None, editable: None, }, + values_from: None, }, }; @@ -524,7 +524,9 @@ impl K8sPrometheusCRDAlertingInterpret { spec: GrafanaDashboardSpec { resync_period: Some("30s".to_string()), instance_selector: labels.clone(), - json, + json: Some(json), + grafana_com: None, + datasources: None, }, };