Monitor an application within a tenant #86
@ -3,9 +3,15 @@ use std::{path::PathBuf, sync::Arc};
|
|||||||
use harmony::{
|
use harmony::{
|
||||||
inventory::Inventory,
|
inventory::Inventory,
|
||||||
maestro::Maestro,
|
maestro::Maestro,
|
||||||
modules::{application::{
|
modules::{
|
||||||
features::{ContinuousDelivery, PrometheusApplicationMonitoring}, ApplicationScore, RustWebFramework, RustWebapp
|
application::{
|
||||||
}, monitoring::alert_channel::discord_alert_channel::DiscordWebhook},
|
ApplicationScore, RustWebFramework, RustWebapp,
|
||||||
|
features::{ContinuousDelivery, PrometheusApplicationMonitoring},
|
||||||
|
},
|
||||||
|
monitoring::alert_channel::{
|
||||||
|
discord_alert_channel::DiscordWebhook, webhook_receiver::WebhookReceiver,
|
||||||
|
},
|
||||||
|
},
|
||||||
topology::{K8sAnywhereTopology, Url},
|
topology::{K8sAnywhereTopology, Url},
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -24,16 +30,21 @@ async fn main() {
|
|||||||
url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()),
|
url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let webhook_receiver = WebhookReceiver {
|
||||||
|
name: "sample-webhook-receiver".to_string(),
|
||||||
|
url: Url::Url(url::Url::parse("https://webhook-doesnt-exist.com").unwrap()),
|
||||||
|
};
|
||||||
|
|
||||||
let app = ApplicationScore {
|
let app = ApplicationScore {
|
||||||
features: vec![
|
features: vec![
|
||||||
// Box::new(ContinuousDelivery {
|
Box::new(ContinuousDelivery {
|
||||||
// application: application.clone(),
|
application: application.clone(),
|
||||||
// }),
|
}),
|
||||||
Box::new(PrometheusApplicationMonitoring {
|
Box::new(PrometheusApplicationMonitoring {
|
||||||
application: application.clone(),
|
application: application.clone(),
|
||||||
alert_receiver: vec![Box::new(discord_receiver),]
|
alert_receiver: vec![Box::new(discord_receiver), Box::new(webhook_receiver)],
|
||||||
}),
|
}),
|
||||||
// TODO add monitoring, backups, multisite ha, etc
|
// TODO add backups, multisite ha, etc
|
||||||
],
|
],
|
||||||
application,
|
application,
|
||||||
};
|
};
|
||||||
|
|||||||
@ -7,23 +7,28 @@ use crate::{
|
|||||||
inventory::Inventory,
|
inventory::Inventory,
|
||||||
modules::{
|
modules::{
|
||||||
application::{Application, ApplicationFeature},
|
application::{Application, ApplicationFeature},
|
||||||
monitoring::
|
monitoring::kube_prometheus::{
|
||||||
kube_prometheus::{
|
alert_manager_config::{CRDAlertManager, CRDAlertManagerReceiver},
|
||||||
alert_manager_config::{CRDAlertManager, CRDAlertManagerReceiver}, helm_prometheus_application_alerting::HelmPrometheusApplicationAlertingScore
|
helm_prometheus_application_alerting::HelmPrometheusApplicationAlertingScore,
|
||||||
}
|
},
|
||||||
,
|
|
||||||
},
|
},
|
||||||
score::Score,
|
score::Score,
|
||||||
topology::{oberservability::monitoring::AlertReceiver, tenant::TenantManager, HelmCommand, K8sclient, Topology},
|
topology::{
|
||||||
|
HelmCommand, K8sclient, Topology, oberservability::monitoring::AlertReceiver,
|
||||||
|
tenant::TenantManager,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct PrometheusApplicationMonitoring {
|
pub struct PrometheusApplicationMonitoring {
|
||||||
pub application: Arc<dyn Application>,
|
pub application: Arc<dyn Application>,
|
||||||
pub alert_receiver: Vec<Box<dyn CRDAlertManagerReceiver>>,}
|
pub alert_receiver: Vec<Box<dyn CRDAlertManagerReceiver>>,
|
||||||
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl<T: Topology + HelmCommand + 'static + TenantManager + K8sclient + std::fmt::Debug> ApplicationFeature<T> for PrometheusApplicationMonitoring {
|
impl<T: Topology + HelmCommand + 'static + TenantManager + K8sclient + std::fmt::Debug>
|
||||||
|
ApplicationFeature<T> for PrometheusApplicationMonitoring
|
||||||
|
{
|
||||||
async fn ensure_installed(&self, topology: &T) -> Result<(), String> {
|
async fn ensure_installed(&self, topology: &T) -> Result<(), String> {
|
||||||
info!("Ensuring monitoring is available for application");
|
info!("Ensuring monitoring is available for application");
|
||||||
|
|
||||||
|
|||||||
@ -91,8 +91,6 @@ impl CRDAlertManagerReceiver for DiscordWebhook {
|
|||||||
},
|
},
|
||||||
spec,
|
spec,
|
||||||
};
|
};
|
||||||
debug!(" am: \n{:#?}", am.clone());
|
|
||||||
|
|
||||||
am
|
am
|
||||||
}
|
}
|
||||||
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
|
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
|
||||||
@ -103,7 +101,14 @@ impl CRDAlertManagerReceiver for DiscordWebhook {
|
|||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl AlertReceiver<CRDAlertManager> for DiscordWebhook {
|
impl AlertReceiver<CRDAlertManager> for DiscordWebhook {
|
||||||
async fn install(&self, sender: &CRDAlertManager) -> Result<Outcome, InterpretError> {
|
async fn install(&self, sender: &CRDAlertManager) -> Result<Outcome, InterpretError> {
|
||||||
todo!()
|
sender
|
||||||
|
.client
|
||||||
|
.apply(&sender.alertmanager_configs, Some(&sender.namespace))
|
||||||
|
.await?;
|
||||||
|
Ok(Outcome::success(format!(
|
||||||
|
"installed crd-alertmanagerconfigs for {}",
|
||||||
|
self.name
|
||||||
|
)))
|
||||||
|
|
|||||||
}
|
}
|
||||||
fn name(&self) -> String {
|
fn name(&self) -> String {
|
||||||
"discord-webhook".to_string()
|
"discord-webhook".to_string()
|
||||||
|
|||||||
@ -1,17 +1,27 @@
|
|||||||
|
use std::{collections::BTreeMap, sync::Arc};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
use k8s_openapi::api::core::v1::Secret;
|
||||||
|
use kube::api::ObjectMeta;
|
||||||
|
use log::debug;
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
use serde_json::json;
|
||||||
use serde_yaml::{Mapping, Value};
|
use serde_yaml::{Mapping, Value};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
interpret::{InterpretError, Outcome},
|
interpret::{InterpretError, Outcome},
|
||||||
modules::monitoring::{
|
modules::monitoring::{
|
||||||
kube_prometheus::{
|
kube_prometheus::{
|
||||||
|
alert_manager_config::{
|
||||||
|
AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManager,
|
||||||
|
CRDAlertManagerReceiver,
|
||||||
|
},
|
||||||
prometheus::{KubePrometheus, KubePrometheusReceiver},
|
prometheus::{KubePrometheus, KubePrometheusReceiver},
|
||||||
types::{AlertChannelConfig, AlertManagerChannelConfig},
|
types::{AlertChannelConfig, AlertManagerChannelConfig},
|
||||||
},
|
},
|
||||||
prometheus::prometheus::{Prometheus, PrometheusReceiver},
|
prometheus::prometheus::{Prometheus, PrometheusReceiver},
|
||||||
},
|
},
|
||||||
topology::{Url, oberservability::monitoring::AlertReceiver},
|
topology::{Url, k8s::K8sClient, oberservability::monitoring::AlertReceiver},
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
@ -20,6 +30,90 @@ pub struct WebhookReceiver {
|
|||||||
pub url: Url,
|
pub url: Url,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl CRDAlertManagerReceiver for WebhookReceiver {
|
||||||
|
fn name(&self) -> String {
|
||||||
|
self.name.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn configure_receiver(&self, client: &Arc<K8sClient>, ns: String) -> AlertmanagerConfig {
|
||||||
|
// let secret_name = format!("{}-secret", self.name.clone());
|
||||||
|
// let webhook_key = format!("{}", self.url.clone());
|
||||||
|
//
|
||||||
|
// let mut string_data = BTreeMap::new();
|
||||||
|
// string_data.insert("webhook-url".to_string(), webhook_key.clone());
|
||||||
|
//
|
||||||
|
// let secret = Secret {
|
||||||
|
// metadata: kube::core::ObjectMeta {
|
||||||
|
// name: Some(secret_name.clone()),
|
||||||
|
// ..Default::default()
|
||||||
|
// },
|
||||||
|
// string_data: Some(string_data),
|
||||||
|
// type_: Some("Opaque".to_string()),
|
||||||
|
// ..Default::default()
|
||||||
|
// };
|
||||||
|
//
|
||||||
|
// let _ = client.apply(&secret, Some(&ns)).await;
|
||||||
|
|
||||||
|
let spec = AlertmanagerConfigSpec {
|
||||||
|
data: json!({
|
||||||
|
"route": {
|
||||||
|
"receiver": self.name,
|
||||||
|
},
|
||||||
|
"receivers": [
|
||||||
|
{
|
||||||
|
"name": self.name,
|
||||||
|
"webhookConfigs": [
|
||||||
|
{
|
||||||
|
"url": self.url,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
let am = AlertmanagerConfig {
|
||||||
|
metadata: ObjectMeta {
|
||||||
|
name: Some(self.name.clone()),
|
||||||
|
labels: Some(std::collections::BTreeMap::from([(
|
||||||
|
"alertmanagerConfig".to_string(),
|
||||||
|
"enabled".to_string(),
|
||||||
|
)])),
|
||||||
|
namespace: Some(ns),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
spec,
|
||||||
|
};
|
||||||
|
debug!(" am: \n{:#?}", am.clone());
|
||||||
|
|
||||||
|
am
|
||||||
|
}
|
||||||
|
fn clone_box(&self) -> Box<dyn CRDAlertManagerReceiver> {
|
||||||
|
Box::new(self.clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl AlertReceiver<CRDAlertManager> for WebhookReceiver {
|
||||||
|
async fn install(&self, sender: &CRDAlertManager) -> Result<Outcome, InterpretError> {
|
||||||
|
sender
|
||||||
|
.client
|
||||||
|
.apply(&sender.alertmanager_configs, Some(&sender.namespace))
|
||||||
|
.await?;
|
||||||
|
Ok(Outcome::success(format!(
|
||||||
|
"installed crd-alertmanagerconfigs for {}",
|
||||||
|
self.name
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
fn name(&self) -> String {
|
||||||
|
"webhook-receiver".to_string()
|
||||||
|
}
|
||||||
|
fn clone_box(&self) -> Box<dyn AlertReceiver<CRDAlertManager>> {
|
||||||
|
Box::new(self.clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl AlertReceiver<Prometheus> for WebhookReceiver {
|
impl AlertReceiver<Prometheus> for WebhookReceiver {
|
||||||
async fn install(&self, sender: &Prometheus) -> Result<Outcome, InterpretError> {
|
async fn install(&self, sender: &Prometheus) -> Result<Outcome, InterpretError> {
|
||||||
|
|||||||
@ -91,6 +91,8 @@ use crate::{
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use super::types::AlertManagerConfig;
|
||||||
|
|
||||||
#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
|
#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
|
||||||
#[kube(
|
#[kube(
|
||||||
group = "monitoring.coreos.com",
|
group = "monitoring.coreos.com",
|
||||||
@ -108,8 +110,9 @@ pub struct AlertmanagerConfigSpec {
|
|||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct CRDAlertManager {
|
pub struct CRDAlertManager {
|
||||||
namespace: String,
|
pub alertmanager_configs: AlertmanagerConfig,
|
||||||
client: K8sClient,
|
pub namespace: String,
|
||||||
|
pub client: Arc<K8sClient>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AlertSender for CRDAlertManager {
|
impl AlertSender for CRDAlertManager {
|
||||||
|
|||||||
@ -12,7 +12,9 @@ use crate::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
alert_manager_config::{AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManager, CRDAlertManagerReceiver},
|
alert_manager_config::{
|
||||||
|
AlertmanagerConfig, AlertmanagerConfigSpec, CRDAlertManager, CRDAlertManagerReceiver,
|
||||||
|
},
|
||||||
prometheus::KubePrometheus,
|
prometheus::KubePrometheus,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -50,10 +52,17 @@ impl<T: Topology + K8sclient> Interpret<T> for HelmPrometheusApplicationAlerting
|
|||||||
) -> Result<Outcome, InterpretError> {
|
) -> Result<Outcome, InterpretError> {
|
||||||
let client = topology.k8s_client().await.unwrap();
|
let client = topology.k8s_client().await.unwrap();
|
||||||
for receiver in self.receivers.iter() {
|
for receiver in self.receivers.iter() {
|
||||||
let alertmanager_config: AlertmanagerConfig = receiver.configure_receiver(&client, self.namespace.clone()).await;
|
let alertmanager_config: AlertmanagerConfig = receiver
|
||||||
client
|
.configure_receiver(&client, self.namespace.clone())
|
||||||
.apply(&alertmanager_config, Some(&self.namespace))
|
.await;
|
||||||
.await?;
|
let sender = CRDAlertManager {
|
||||||
|
alertmanager_configs: alertmanager_config,
|
||||||
|
namespace: self.namespace.clone(),
|
||||||
|
client: client.clone(),
|
||||||
|
};
|
||||||
|
receiver.install(&sender).await.map_err(|err| {
|
||||||
|
InterpretError::new(format!("failed to install receiver: {}", err))
|
||||||
|
})?;
|
||||||
}
|
}
|
||||||
Ok(Outcome::success(format!("deployed alert channels")))
|
Ok(Outcome::success(format!("deployed alert channels")))
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
|
pub mod alert_manager_config;
|
||||||
pub mod helm;
|
pub mod helm;
|
||||||
pub mod helm_prometheus_alert_score;
|
pub mod helm_prometheus_alert_score;
|
||||||
|
pub mod helm_prometheus_application_alerting;
|
||||||
pub mod prometheus;
|
pub mod prometheus;
|
||||||
pub mod types;
|
pub mod types;
|
||||||
pub mod alert_manager_config;
|
|
||||||
pub mod helm_prometheus_application_alerting;
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user
this
Prometheussender doesn't seem to be used, meaning theAlertReceiver<Prometheus>implementation doesn't seem to be used either (and same goes for the webhook alert channel)