Monitor an application within a tenant #86
@ -3,9 +3,15 @@ use std::{path::PathBuf, sync::Arc};
|
||||
use harmony::{
|
||||
inventory::Inventory,
|
||||
maestro::Maestro,
|
||||
modules::{application::{
|
||||
features::{ContinuousDelivery, PrometheusApplicationMonitoring}, ApplicationScore, RustWebFramework, RustWebapp
|
||||
}, monitoring::alert_channel::discord_alert_channel::DiscordWebhook},
|
||||
modules::{
|
||||
application::{
|
||||
ApplicationScore, RustWebFramework, RustWebapp,
|
||||
features::{ContinuousDelivery, PrometheusApplicationMonitoring},
|
||||
},
|
||||
monitoring::alert_channel::{
|
||||
discord_alert_channel::DiscordWebhook, webhook_receiver::WebhookReceiver,
|
||||
},
|
||||
},
|
||||
topology::{K8sAnywhereTopology, Url},
|
||||
};
|
||||
|
||||
@ -24,16 +30,21 @@ async fn main() {
|
||||
url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()),
|
||||
};
|
||||
|
||||
let webhook_receiver = WebhookReceiver {
|
||||
name: "sample-webhook-receiver".to_string(),
|
||||
url: Url::Url(url::Url::parse("https://webhook-doesnt-exist.com").unwrap()),
|
||||
};
|
||||
|
||||
let app = ApplicationScore {
|
||||
features: vec![
|
||||
// Box::new(ContinuousDelivery {
|
||||
// application: application.clone(),
|
||||
// }),
|
||||
Box::new(ContinuousDelivery {
|
||||
application: application.clone(),
|
||||
}),
|
||||
Box::new(PrometheusApplicationMonitoring {
|
||||
application: application.clone(),
|
||||
alert_receiver: vec![Box::new(discord_receiver),]
|
||||
alert_receiver: vec![Box::new(discord_receiver), Box::new(webhook_receiver)],
|
||||
}),
|
||||
// TODO add monitoring, backups, multisite ha, etc
|
||||
// TODO add backups, multisite ha, etc
|
||||
],
|
||||
application,
|
||||
};
|
||||
|
@ -7,23 +7,28 @@ use crate::{
|
||||
inventory::Inventory,
|
||||
modules::{
|
||||
application::{Application, ApplicationFeature},
|
||||
monitoring::
|
||||
kube_prometheus::{
|
||||
alert_manager_config::{CRDAlertManager, CRDAlertManagerReceiver}, helm_prometheus_application_alerting::HelmPrometheusApplicationAlertingScore
|
||||
}
|
||||
,
|
||||
monitoring::kube_prometheus::{
|
||||
alert_manager_config::{CRDAlertManager, CRDAlertManagerReceiver},
|
||||
helm_prometheus_application_alerting::HelmPrometheusApplicationAlertingScore,
|
||||
},
|
||||
},
|
||||
score::Score,
|
||||
topology::{oberservability::monitoring::AlertReceiver, tenant::TenantManager, HelmCommand, K8sclient, Topology},
|
||||
topology::{
|
||||
HelmCommand, K8sclient, Topology, oberservability::monitoring::AlertReceiver,
|
||||
tenant::TenantManager,
|
||||
},
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PrometheusApplicationMonitoring {
|
||||
pub application: Arc<dyn Application>,
|
||||
pub alert_receiver: Vec<Box<dyn CRDAlertManagerReceiver>>,}
|
||||
pub alert_receiver: Vec<Box<dyn CRDAlertManagerReceiver>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<T: Topology + HelmCommand + 'static + TenantManager + K8sclient + std::fmt::Debug> ApplicationFeature<T> for PrometheusApplicationMonitoring {
|
||||
impl<T: Topology + HelmCommand + 'static + TenantManager + K8sclient + std::fmt::Debug>
|
||||
ApplicationFeature<T> for PrometheusApplicationMonitoring
|
||||
{
|
||||
async fn ensure_installed(&self, topology: &T) -> Result<(), String> {
|
||||
info!("Ensuring monitoring is available for application");
|
||||
|
||||
|
@ -91,8 +91,6 @@ impl CRDAlertManagerReceiver for DiscordWebhook {
|
||||
},
|
||||
spec,
|
||||
};
|
||||
debug!(" am: \n{:#?}", am.clone());
|
||||
|
||||
am
|
||||
}
|
||||
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
|
||||
@ -103,7 +101,14 @@ impl CRDAlertManagerReceiver for DiscordWebhook {
|
||||
#[async_trait]
|
||||
impl AlertReceiver<CRDAlertManager> for DiscordWebhook {
|
||||
async fn install(&self, sender: &CRDAlertManager) -> Result<Outcome, InterpretError> {
|
||||
todo!()
|
||||
sender
|
||||
.client
|
||||
.apply(&sender.alertmanager_configs, Some(&sender.namespace))
|
||||
.await?;
|
||||
Ok(Outcome::success(format!(
|
||||
"installed crd-alertmanagerconfigs for {}",
|
||||
self.name
|
||||
)))
|
||||
|
||||
}
|
||||
fn name(&self) -> String {
|
||||
"discord-webhook".to_string()
|
||||
|
@ -1,17 +1,27 @@
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use k8s_openapi::api::core::v1::Secret;
|
||||
use kube::api::ObjectMeta;
|
||||
use log::debug;
|
||||
use serde::Serialize;
|
||||
use serde_json::json;
|
||||
use serde_yaml::{Mapping, Value};
|
||||
|
||||
use crate::{
|
||||
interpret::{InterpretError, Outcome},
|
||||
modules::monitoring::{
|
||||
kube_prometheus::{
|
||||
alert_manager_config::{
|
||||
AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManager,
|
||||
CRDAlertManagerReceiver,
|
||||
},
|
||||
prometheus::{KubePrometheus, KubePrometheusReceiver},
|
||||
types::{AlertChannelConfig, AlertManagerChannelConfig},
|
||||
},
|
||||
prometheus::prometheus::{Prometheus, PrometheusReceiver},
|
||||
},
|
||||
topology::{Url, oberservability::monitoring::AlertReceiver},
|
||||
topology::{Url, k8s::K8sClient, oberservability::monitoring::AlertReceiver},
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
@ -20,6 +30,90 @@ pub struct WebhookReceiver {
|
||||
pub url: Url,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl CRDAlertManagerReceiver for WebhookReceiver {
|
||||
fn name(&self) -> String {
|
||||
self.name.clone()
|
||||
}
|
||||
|
||||
async fn configure_receiver(&self, client: &Arc<K8sClient>, ns: String) -> AlertmanagerConfig {
|
||||
// let secret_name = format!("{}-secret", self.name.clone());
|
||||
// let webhook_key = format!("{}", self.url.clone());
|
||||
//
|
||||
// let mut string_data = BTreeMap::new();
|
||||
// string_data.insert("webhook-url".to_string(), webhook_key.clone());
|
||||
//
|
||||
// let secret = Secret {
|
||||
// metadata: kube::core::ObjectMeta {
|
||||
// name: Some(secret_name.clone()),
|
||||
// ..Default::default()
|
||||
// },
|
||||
// string_data: Some(string_data),
|
||||
// type_: Some("Opaque".to_string()),
|
||||
// ..Default::default()
|
||||
// };
|
||||
//
|
||||
// let _ = client.apply(&secret, Some(&ns)).await;
|
||||
|
||||
let spec = AlertmanagerConfigSpec {
|
||||
data: json!({
|
||||
"route": {
|
||||
"receiver": self.name,
|
||||
},
|
||||
"receivers": [
|
||||
{
|
||||
"name": self.name,
|
||||
"webhookConfigs": [
|
||||
{
|
||||
"url": self.url,
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}),
|
||||
};
|
||||
|
||||
let am = AlertmanagerConfig {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(self.name.clone()),
|
||||
labels: Some(std::collections::BTreeMap::from([(
|
||||
"alertmanagerConfig".to_string(),
|
||||
"enabled".to_string(),
|
||||
)])),
|
||||
namespace: Some(ns),
|
||||
..Default::default()
|
||||
},
|
||||
spec,
|
||||
};
|
||||
debug!(" am: \n{:#?}", am.clone());
|
||||
|
||||
am
|
||||
}
|
||||
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AlertReceiver<CRDAlertManager> for WebhookReceiver {
|
||||
async fn install(&self, sender: &CRDAlertManager) -> Result<Outcome, InterpretError> {
|
||||
sender
|
||||
.client
|
||||
.apply(&sender.alertmanager_configs, Some(&sender.namespace))
|
||||
.await?;
|
||||
Ok(Outcome::success(format!(
|
||||
"installed crd-alertmanagerconfigs for {}",
|
||||
self.name
|
||||
)))
|
||||
}
|
||||
fn name(&self) -> String {
|
||||
"webhook-receiver".to_string()
|
||||
}
|
||||
fn clone_box(&self) -> Box<dyn AlertReceiver<CRDAlertManager>> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AlertReceiver<Prometheus> for WebhookReceiver {
|
||||
async fn install(&self, sender: &Prometheus) -> Result<Outcome, InterpretError> {
|
||||
|
@ -91,6 +91,8 @@ use crate::{
|
||||
},
|
||||
};
|
||||
|
||||
use super::types::AlertManagerConfig;
|
||||
|
||||
#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
|
||||
#[kube(
|
||||
group = "monitoring.coreos.com",
|
||||
@ -108,8 +110,9 @@ pub struct AlertmanagerConfigSpec {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CRDAlertManager {
|
||||
namespace: String,
|
||||
client: K8sClient,
|
||||
pub alertmanager_configs: AlertmanagerConfig,
|
||||
pub namespace: String,
|
||||
pub client: Arc<K8sClient>,
|
||||
}
|
||||
|
||||
impl AlertSender for CRDAlertManager {
|
||||
|
@ -12,7 +12,9 @@ use crate::{
|
||||
};
|
||||
|
||||
use super::{
|
||||
alert_manager_config::{AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManager, CRDAlertManagerReceiver},
|
||||
alert_manager_config::{
|
||||
AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManager, CRDAlertManagerReceiver,
|
||||
},
|
||||
prometheus::KubePrometheus,
|
||||
};
|
||||
|
||||
@ -50,10 +52,17 @@ impl<T: Topology + K8sclient> Interpret<T> for HelmPrometheusApplicationAlerting
|
||||
) -> Result<Outcome, InterpretError> {
|
||||
let client = topology.k8s_client().await.unwrap();
|
||||
for receiver in self.receivers.iter() {
|
||||
let alertmanager_config: AlertmanagerConfig = receiver.configure_receiver(&client, self.namespace.clone()).await;
|
||||
client
|
||||
.apply(&alertmanager_config, Some(&self.namespace))
|
||||
.await?;
|
||||
let alertmanager_config: AlertmanagerConfig = receiver
|
||||
.configure_receiver(&client, self.namespace.clone())
|
||||
.await;
|
||||
let sender = CRDAlertManager {
|
||||
alertmanager_configs: alertmanager_config,
|
||||
namespace: self.namespace.clone(),
|
||||
client: client.clone(),
|
||||
};
|
||||
receiver.install(&sender).await.map_err(|err| {
|
||||
InterpretError::new(format!("failed to install receiver: {}", err))
|
||||
})?;
|
||||
}
|
||||
Ok(Outcome::success(format!("deployed alert channels")))
|
||||
}
|
||||
|
@ -1,6 +1,6 @@
|
||||
pub mod alert_manager_config;
|
||||
pub mod helm;
|
||||
pub mod helm_prometheus_alert_score;
|
||||
pub mod helm_prometheus_application_alerting;
|
||||
pub mod prometheus;
|
||||
pub mod types;
|
||||
pub mod alert_manager_config;
|
||||
pub mod helm_prometheus_application_alerting;
|
||||
|
Loading…
Reference in New Issue
Block a user
this
Prometheus
sender doesn't seem to be used, meaning theAlertReceiver<Prometheus>
implementation doesn't seem to be used either (and same goes for the webhook alert channel)