feat: added alert manager routes to openshift cluster monitoring
All checks were successful
Run Check Script / check (pull_request) Successful in 1m37s
All checks were successful
Run Check Script / check (pull_request) Successful in 1m37s
This commit is contained in:
parent
90b89224d8
commit
78e595e696
@ -24,13 +24,14 @@ use harmony::{
|
||||
},
|
||||
topology::K8sAnywhereTopology,
|
||||
};
|
||||
use harmony_types::net::Url;
|
||||
use harmony_types::{k8s_name::K8sName, net::Url};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let discord_receiver = DiscordWebhook {
|
||||
name: "test-discord".to_string(),
|
||||
name: K8sName("test-discord".to_string()),
|
||||
url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()),
|
||||
selectors: vec![],
|
||||
};
|
||||
|
||||
let high_pvc_fill_rate_over_two_days_alert = high_pvc_fill_rate_over_two_days();
|
||||
|
||||
@ -22,8 +22,8 @@ use harmony::{
|
||||
tenant::{ResourceLimits, TenantConfig, TenantNetworkPolicy},
|
||||
},
|
||||
};
|
||||
use harmony_types::id::Id;
|
||||
use harmony_types::net::Url;
|
||||
use harmony_types::{id::Id, k8s_name::K8sName};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
@ -43,8 +43,9 @@ async fn main() {
|
||||
};
|
||||
|
||||
let discord_receiver = DiscordWebhook {
|
||||
name: "test-discord".to_string(),
|
||||
name: K8sName("test-discord".to_string()),
|
||||
url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()),
|
||||
selectors: vec![],
|
||||
};
|
||||
|
||||
let high_pvc_fill_rate_over_two_days_alert = high_pvc_fill_rate_over_two_days();
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use harmony::{
|
||||
inventory::Inventory,
|
||||
modules::monitoring::{
|
||||
@ -7,16 +9,26 @@ use harmony::{
|
||||
topology::K8sAnywhereTopology,
|
||||
};
|
||||
use harmony_macros::hurl;
|
||||
use harmony_types::k8s_name::K8sName;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let mut sel = HashMap::new();
|
||||
sel.insert(
|
||||
"openshift_io_alert_source".to_string(),
|
||||
"platform".to_string(),
|
||||
);
|
||||
let mut sel2 = HashMap::new();
|
||||
sel2.insert("openshift_io_alert_source".to_string(), "".to_string());
|
||||
let selectors = vec![sel, sel2];
|
||||
harmony_cli::run(
|
||||
Inventory::autoload(),
|
||||
K8sAnywhereTopology::from_env(),
|
||||
vec![Box::new(OpenshiftClusterAlertScore {
|
||||
receivers: vec![Box::new(DiscordWebhook {
|
||||
name: "discord-webhook-example".to_string(),
|
||||
url: hurl!("http://something.o"),
|
||||
name: K8sName("wills-discord-webhook-example".to_string()),
|
||||
url: hurl!("https://something.io"),
|
||||
selectors: selectors,
|
||||
})],
|
||||
})],
|
||||
None,
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
use std::{path::PathBuf, sync::Arc};
|
||||
use std::{collections::HashMap, path::PathBuf, sync::Arc};
|
||||
|
||||
use harmony::{
|
||||
inventory::Inventory,
|
||||
@ -10,7 +10,7 @@ use harmony::{
|
||||
},
|
||||
topology::K8sAnywhereTopology,
|
||||
};
|
||||
use harmony_types::net::Url;
|
||||
use harmony_types::{k8s_name::K8sName, net::Url};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
@ -22,8 +22,9 @@ async fn main() {
|
||||
});
|
||||
|
||||
let discord_receiver = DiscordWebhook {
|
||||
name: "test-discord".to_string(),
|
||||
name: K8sName("test-discord".to_string()),
|
||||
url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()),
|
||||
selectors: vec![],
|
||||
};
|
||||
|
||||
let app = ApplicationScore {
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
use std::{path::PathBuf, sync::Arc};
|
||||
use std::{collections::HashMap, path::PathBuf, sync::Arc};
|
||||
|
||||
use harmony::{
|
||||
inventory::Inventory,
|
||||
@ -14,6 +14,7 @@ use harmony::{
|
||||
topology::K8sAnywhereTopology,
|
||||
};
|
||||
use harmony_macros::hurl;
|
||||
use harmony_types::k8s_name::K8sName;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
@ -25,8 +26,9 @@ async fn main() {
|
||||
});
|
||||
|
||||
let discord_receiver = DiscordWebhook {
|
||||
name: "test-discord".to_string(),
|
||||
name: K8sName("test-discord".to_string()),
|
||||
url: hurl!("https://discord.doesnt.exist.com"),
|
||||
selectors: vec![],
|
||||
};
|
||||
|
||||
let webhook_receiver = WebhookReceiver {
|
||||
|
||||
Binary file not shown.
@ -0,0 +1,7 @@
|
||||
|
||||
apiVersion: v2
|
||||
name: harmony-example-rust-webapp-chart
|
||||
description: A Helm chart for the harmony-example-rust-webapp web application.
|
||||
type: application
|
||||
version: 0.1.0
|
||||
appVersion: "latest"
|
||||
@ -0,0 +1,16 @@
|
||||
|
||||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "chart.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
*/}}
|
||||
{{- define "chart.fullname" -}}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
@ -0,0 +1,23 @@
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "chart.fullname" . }}
|
||||
spec:
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "chart.name" . }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "chart.name" . }}
|
||||
spec:
|
||||
containers:
|
||||
- name: {{ .Chart.Name }}
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3000
|
||||
protocol: TCP
|
||||
@ -0,0 +1,35 @@
|
||||
|
||||
{{- if .Values.ingress.enabled -}}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "chart.fullname" . }}
|
||||
annotations:
|
||||
{{- toYaml .Values.ingress.annotations | nindent 4 }}
|
||||
spec:
|
||||
{{- if .Values.ingress.tls }}
|
||||
tls:
|
||||
{{- range .Values.ingress.tls }}
|
||||
- hosts:
|
||||
{{- range .hosts }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
secretName: {{ .secretName }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.ingress.hosts }}
|
||||
- host: {{ .host | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- range .paths }}
|
||||
- path: {{ .path }}
|
||||
pathType: {{ .pathType }}
|
||||
backend:
|
||||
service:
|
||||
name: {{ include "chart.fullname" $ }}
|
||||
port:
|
||||
number: 3000
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@ -0,0 +1,14 @@
|
||||
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "chart.fullname" . }}
|
||||
spec:
|
||||
type: {{ .Values.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.service.port }}
|
||||
targetPort: 3000
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
app: {{ include "chart.name" . }}
|
||||
@ -0,0 +1,34 @@
|
||||
|
||||
# Default values for harmony-example-rust-webapp-chart.
|
||||
# This is a YAML-formatted file.
|
||||
# Declare variables to be passed into your templates.
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: hub.nationtech.io/harmony/harmony-example-rust-webapp
|
||||
pullPolicy: IfNotPresent
|
||||
# Overridden by the chart's appVersion
|
||||
tag: "latest"
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 3000
|
||||
|
||||
ingress:
|
||||
enabled: true
|
||||
# Annotations for cert-manager to handle SSL.
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
||||
# Add other annotations like nginx ingress class if needed
|
||||
# kubernetes.io/ingress.class: nginx
|
||||
hosts:
|
||||
- host: chart-example.local
|
||||
paths:
|
||||
- path: /
|
||||
pathType: ImplementationSpecific
|
||||
tls:
|
||||
- secretName: harmony-example-rust-webapp-tls
|
||||
hosts:
|
||||
- chart-example.local
|
||||
|
||||
@ -10,6 +10,7 @@ use harmony::{
|
||||
topology::K8sAnywhereTopology,
|
||||
};
|
||||
use harmony_macros::hurl;
|
||||
use harmony_types::k8s_name::K8sName;
|
||||
use std::{path::PathBuf, sync::Arc};
|
||||
|
||||
#[tokio::main]
|
||||
@ -31,8 +32,9 @@ async fn main() {
|
||||
Box::new(Monitoring {
|
||||
application: application.clone(),
|
||||
alert_receiver: vec![Box::new(DiscordWebhook {
|
||||
name: "test-discord".to_string(),
|
||||
name: K8sName("test-discord".to_string()),
|
||||
url: hurl!("https://discord.doesnt.exist.com"),
|
||||
selectors: vec![],
|
||||
})],
|
||||
}),
|
||||
],
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
use std::any::Any;
|
||||
use std::{any::Any, collections::HashMap};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use kube::api::DynamicObject;
|
||||
@ -85,6 +85,7 @@ pub struct AlertManagerReceiver {
|
||||
pub receiver_config: serde_json::Value,
|
||||
// FIXME we should not leak k8s here. DynamicObject is k8s specific
|
||||
pub additional_ressources: Vec<DynamicObject>,
|
||||
pub route_config: serde_json::Value,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@ -14,7 +14,7 @@ use k8s_openapi::{
|
||||
},
|
||||
apimachinery::pkg::util::intstr::IntOrString,
|
||||
};
|
||||
use kube::{api::DynamicObject, Resource};
|
||||
use kube::{Resource, api::DynamicObject};
|
||||
use log::debug;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::json;
|
||||
|
||||
@ -1,11 +1,12 @@
|
||||
use std::any::Any;
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use harmony_types::k8s_name::K8sName;
|
||||
use k8s_openapi::api::core::v1::Secret;
|
||||
use kube::Resource;
|
||||
use kube::api::{DynamicObject, ObjectMeta};
|
||||
use log::debug;
|
||||
use log::{debug, trace};
|
||||
use serde::Serialize;
|
||||
use serde_json::json;
|
||||
use serde_yaml::{Mapping, Value};
|
||||
@ -32,21 +33,9 @@ use harmony_types::net::Url;
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct DiscordWebhook {
|
||||
// FIXME use a stricter type as this is used as a k8s resource name. It could also be converted
|
||||
// to remove whitespace and other invalid characters, but this is a potential bug that is not
|
||||
// very easy to figure out for beginners.
|
||||
//
|
||||
// It gives out error messages like this :
|
||||
//
|
||||
// [2025-10-30 15:10:49 ERROR harmony::domain::topology::k8s] Failed to get dynamic resource 'Webhook example-secret': Failed to build request: failed to build request: invalid uri character
|
||||
// [2025-10-30 15:10:49 ERROR harmony_cli::cli_logger] ⚠️ InterpretError : Failed to build request: failed to build request: invalid uri character
|
||||
// [2025-10-30 15:10:49 DEBUG harmony::domain::maestro] Got result Err(InterpretError { msg: "InterpretError : Failed to build request: failed to build request: invalid uri character" })
|
||||
// [2025-10-30 15:10:49 INFO harmony_cli::cli_logger] 🎼 Harmony completed
|
||||
//
|
||||
// thread 'main' panicked at examples/okd_cluster_alerts/src/main.rs:25:6:
|
||||
// called `Result::unwrap()` on an `Err` value: InterpretError { msg: "InterpretError : Failed to build request: failed to build request: invalid uri character" }
|
||||
pub name: String,
|
||||
pub name: K8sName,
|
||||
pub url: Url,
|
||||
pub selectors: Vec<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
impl DiscordWebhook {
|
||||
@ -67,22 +56,32 @@ impl DiscordWebhook {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut matchers: Vec<String> = Vec::new();
|
||||
for selector in &self.selectors {
|
||||
trace!("selector: {:#?}", selector);
|
||||
for (k, v) in selector {
|
||||
matchers.push(format!("{} = {}", k, v));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(AlertManagerReceiver {
|
||||
additional_ressources: vec![kube_resource_to_dynamic(&secret)?],
|
||||
|
||||
receiver_config: json!({
|
||||
"name": self.name,
|
||||
"discordConfigs": [
|
||||
"discord_configs": [
|
||||
{
|
||||
"apiURL": {
|
||||
"name": secret_name,
|
||||
"key": "webhook-url",
|
||||
},
|
||||
"webhook_url": self.url.clone(),
|
||||
"title": "{{ template \"discord.default.title\" . }}",
|
||||
"message": "{{ template \"discord.default.message\" . }}"
|
||||
}
|
||||
]
|
||||
}),
|
||||
route_config: json!({
|
||||
"receiver": self.name,
|
||||
"matchers": matchers,
|
||||
|
||||
}),
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -97,7 +96,7 @@ impl AlertReceiver<OpenshiftClusterAlertSender> for DiscordWebhook {
|
||||
}
|
||||
|
||||
fn name(&self) -> String {
|
||||
self.name.clone()
|
||||
self.name.clone().to_string()
|
||||
}
|
||||
|
||||
fn clone_box(&self) -> Box<dyn AlertReceiver<OpenshiftClusterAlertSender>> {
|
||||
@ -141,7 +140,7 @@ impl AlertReceiver<RHOBObservability> for DiscordWebhook {
|
||||
|
||||
let alertmanager_configs = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfig {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(self.name.clone()),
|
||||
name: Some(self.name.clone().to_string()),
|
||||
labels: Some(std::collections::BTreeMap::from([(
|
||||
"alertmanagerConfig".to_string(),
|
||||
"enabled".to_string(),
|
||||
@ -233,7 +232,7 @@ impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
|
||||
|
||||
let alertmanager_configs = AlertmanagerConfig {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(self.name.clone()),
|
||||
name: Some(self.name.clone().to_string()),
|
||||
labels: Some(std::collections::BTreeMap::from([(
|
||||
"alertmanagerConfig".to_string(),
|
||||
"enabled".to_string(),
|
||||
@ -286,7 +285,7 @@ impl AlertReceiver<Prometheus> for DiscordWebhook {
|
||||
#[async_trait]
|
||||
impl PrometheusReceiver for DiscordWebhook {
|
||||
fn name(&self) -> String {
|
||||
self.name.clone()
|
||||
self.name.clone().to_string()
|
||||
}
|
||||
async fn configure_receiver(&self) -> AlertManagerChannelConfig {
|
||||
self.get_config().await
|
||||
@ -315,7 +314,7 @@ impl AlertReceiver<KubePrometheus> for DiscordWebhook {
|
||||
#[async_trait]
|
||||
impl KubePrometheusReceiver for DiscordWebhook {
|
||||
fn name(&self) -> String {
|
||||
self.name.clone()
|
||||
self.name.clone().to_string()
|
||||
}
|
||||
async fn configure_receiver(&self) -> AlertManagerChannelConfig {
|
||||
self.get_config().await
|
||||
@ -342,7 +341,7 @@ impl DiscordWebhook {
|
||||
let mut route = Mapping::new();
|
||||
route.insert(
|
||||
Value::String("receiver".to_string()),
|
||||
Value::String(self.name.clone()),
|
||||
Value::String(self.name.clone().to_string()),
|
||||
);
|
||||
route.insert(
|
||||
Value::String("matchers".to_string()),
|
||||
@ -356,7 +355,7 @@ impl DiscordWebhook {
|
||||
let mut receiver = Mapping::new();
|
||||
receiver.insert(
|
||||
Value::String("name".to_string()),
|
||||
Value::String(self.name.clone()),
|
||||
Value::String(self.name.clone().to_string()),
|
||||
);
|
||||
|
||||
let mut discord_config = Mapping::new();
|
||||
@ -381,8 +380,9 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn discord_serialize_should_match() {
|
||||
let discord_receiver = DiscordWebhook {
|
||||
name: "test-discord".to_string(),
|
||||
name: K8sName("test-discord".to_string()),
|
||||
url: Url::Url(url::Url::parse("https://discord.i.dont.exist.com").unwrap()),
|
||||
selectors: vec![],
|
||||
};
|
||||
|
||||
let discord_receiver_receiver =
|
||||
|
||||
@ -118,14 +118,15 @@ impl<T: Topology + K8sclient> Interpret<T> for OpenshiftClusterAlertInterpret {
|
||||
let name = custom_receiver.name();
|
||||
let alertmanager_receiver = custom_receiver.as_alertmanager_receiver()?;
|
||||
|
||||
let json_value = alertmanager_receiver.receiver_config;
|
||||
let receiver_json_value = alertmanager_receiver.receiver_config;
|
||||
|
||||
let yaml_string = serde_json::to_string(&json_value).map_err(|e| {
|
||||
InterpretError::new(format!("Failed to serialize receiver config: {}", e))
|
||||
})?;
|
||||
let receiver_yaml_string =
|
||||
serde_json::to_string(&receiver_json_value).map_err(|e| {
|
||||
InterpretError::new(format!("Failed to serialize receiver config: {}", e))
|
||||
})?;
|
||||
|
||||
let yaml_value: serde_yaml::Value =
|
||||
serde_yaml::from_str(&yaml_string).map_err(|e| {
|
||||
let receiver_yaml_value: serde_yaml::Value =
|
||||
serde_yaml::from_str(&receiver_yaml_string).map_err(|e| {
|
||||
InterpretError::new(format!("Failed to parse receiver config as YAML: {}", e))
|
||||
})?;
|
||||
|
||||
@ -135,15 +136,70 @@ impl<T: Topology + K8sclient> Interpret<T> for OpenshiftClusterAlertInterpret {
|
||||
.map_or(false, |n| n == name)
|
||||
}) {
|
||||
info!("Replacing existing AlertManager receiver: {}", name);
|
||||
existing_receivers_sequence[idx] = yaml_value;
|
||||
existing_receivers_sequence[idx] = receiver_yaml_value;
|
||||
} else {
|
||||
debug!("Adding new AlertManager receiver: {}", name);
|
||||
existing_receivers_sequence.push(yaml_value);
|
||||
existing_receivers_sequence.push(receiver_yaml_value);
|
||||
}
|
||||
|
||||
additional_resources.push(alertmanager_receiver.additional_ressources);
|
||||
}
|
||||
|
||||
let existing_route_mapping = if let Some(route) = am_config.get_mut("route") {
|
||||
match route.as_mapping_mut() {
|
||||
Some(map) => map,
|
||||
None => {
|
||||
return Err(InterpretError::new(format!(
|
||||
"Expected alertmanager config route to be a mapping, got {:?}",
|
||||
route
|
||||
)));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
&mut serde_yaml::Mapping::default()
|
||||
};
|
||||
|
||||
let existing_route_sequence = if let Some(routes) = existing_route_mapping.get_mut("routes")
|
||||
{
|
||||
match routes.as_sequence_mut() {
|
||||
Some(seq) => seq,
|
||||
None => {
|
||||
return Err(InterpretError::new(format!(
|
||||
"Expected alertmanager config routes to be a sequence, got {:?}",
|
||||
routes
|
||||
)));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
&mut serde_yaml::Sequence::default()
|
||||
};
|
||||
|
||||
for custom_receiver in &self.receivers {
|
||||
let name = custom_receiver.name();
|
||||
let alertmanager_receiver = custom_receiver.as_alertmanager_receiver()?;
|
||||
|
||||
let route_json_value = alertmanager_receiver.route_config;
|
||||
let route_yaml_string = serde_json::to_string(&route_json_value).map_err(|e| {
|
||||
InterpretError::new(format!("Failed to serialize route config: {}", e))
|
||||
})?;
|
||||
|
||||
let route_yaml_value: serde_yaml::Value = serde_yaml::from_str(&route_yaml_string)
|
||||
.map_err(|e| {
|
||||
InterpretError::new(format!("Failed to parse route config as YAML: {}", e))
|
||||
})?;
|
||||
|
||||
if let Some(idy) = existing_route_sequence.iter().position(|r| {
|
||||
r.get("receiver")
|
||||
.and_then(|n| n.as_str())
|
||||
.map_or(false, |n| n == name)
|
||||
}) {
|
||||
info!("Replacing existing AlertManager receiver: {}", name);
|
||||
existing_route_sequence[idy] = route_yaml_value;
|
||||
} else {
|
||||
debug!("Adding new AlertManager receiver: {}", name);
|
||||
existing_route_sequence.push(route_yaml_value);
|
||||
}
|
||||
}
|
||||
debug!("Current alertmanager config {am_config:#?}");
|
||||
// TODO
|
||||
// - save new version of alertmanager config
|
||||
|
||||
Loading…
Reference in New Issue
Block a user