diff --git a/.gitea/workflows/harmony_composer.yaml b/.gitea/workflows/harmony_composer.yaml index 6c7bc0f..fbb809b 100644 --- a/.gitea/workflows/harmony_composer.yaml +++ b/.gitea/workflows/harmony_composer.yaml @@ -19,13 +19,6 @@ jobs: - name: Build for Windows x86_64 GNU run: cargo build --release --bin harmony_composer --target x86_64-pc-windows-gnu - - uses: actions/upload-artifact@v3 - with: - name: binaries - path: | - target/x86_64-unknown-linux-gnu/release/harmony_composer - target/x86_64-pc-windows-gnu/release/harmony_composer.exe - - name: Setup log into hub.nationtech.io uses: docker/login-action@v3 with: @@ -35,6 +28,62 @@ jobs: # TODO: build ARM images and MacOS binaries (or other targets) too + - name: Update snapshot-latest tag + run: | + git config user.name "Gitea CI" + git config user.email "ci@nationtech.io" + git tag -f snapshot-latest + git push origin snapshot-latest --force + + - name: Install jq + run: apt install -y jq # The current image includes apt lists so we don't have to apt update and rm /var/lib/apt... every time. But if the image is optimized it won't work anymore + + - name: Create or update release + run: | + # First, check if release exists and delete it if it does + RELEASE_ID=$(curl -s -X GET \ + -H "Authorization: token ${{ secrets.GITEATOKEN }}" \ + "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases/tags/snapshot-latest" \ + | jq -r '.id // empty') + + if [ -n "$RELEASE_ID" ]; then + # Delete existing release + curl -X DELETE \ + -H "Authorization: token ${{ secrets.GITEATOKEN }}" \ + "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases/$RELEASE_ID" + fi + + # Create new release + RESPONSE=$(curl -X POST \ + -H "Authorization: token ${{ secrets.GITEATOKEN }}" \ + -H "Content-Type: application/json" \ + -d '{ + "tag_name": "snapshot-latest", + "name": "Latest Snapshot", + "body": "Automated snapshot build from master branch", + "draft": false, + "prerelease": true + }' \ + "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases") + + echo "RELEASE_ID=$(echo $RESPONSE | jq -r '.id')" >> $GITHUB_ENV + + - name: Upload Linux binary + run: | + curl -X POST \ + -H "Authorization: token ${{ secrets.GITEATOKEN }}" \ + -H "Content-Type: application/octet-stream" \ + --data-binary "@target/x86_64-unknown-linux-gnu/release/harmony_composer" \ + "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases/${{ env.RELEASE_ID }}/assets?name=harmony_composer" + + - name: Upload Windows binary + run: | + curl -X POST \ + -H "Authorization: token ${{ secrets.GITEATOKEN }}" \ + -H "Content-Type: application/octet-stream" \ + --data-binary "@target/x86_64-pc-windows-gnu/release/harmony_composer.exe" \ + "https://git.nationtech.io/api/v1/repos/nationtech/harmony/releases/${{ env.RELEASE_ID }}/assets?name=harmony_composer.exe" + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/Cargo.lock b/Cargo.lock index 0ee6318..2769781 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1161,6 +1161,7 @@ dependencies = [ "harmony", "harmony_cli", "tokio", + "url", ] [[package]] @@ -1576,6 +1577,7 @@ dependencies = [ "serde-value", "serde_json", "serde_yaml", + "similar", "temp-dir", "temp-file", "tokio", @@ -4090,6 +4092,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + [[package]] name = "simple_asn1" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index 970300d..c081c86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,34 +20,23 @@ readme = "README.md" license = "GNU AGPL v3" [workspace.dependencies] -log = "0.4.22" -env_logger = "0.11.5" -derive-new = "0.7.0" -async-trait = "0.1.82" -tokio = { version = "1.40.0", features = [ - "io-std", - "fs", - "macros", - "rt-multi-thread", -] } +log = "0.4" +env_logger = "0.11" +derive-new = "0.7" +async-trait = "0.1" +tokio = { version = "1.40", features = ["io-std", "fs", "macros", "rt-multi-thread"] } cidr = { features = ["serde"], version = "0.2" } -russh = "0.45.0" -russh-keys = "0.45.0" -rand = "0.8.5" -url = "2.5.4" -kube = "0.98.0" -k8s-openapi = { version = "0.24.0", features = ["v1_30"] } -serde_yaml = "0.9.34" -serde-value = "0.7.0" -http = "1.2.0" -inquire = "0.7.5" -convert_case = "0.8.0" +russh = "0.45" +russh-keys = "0.45" +rand = "0.8" +url = "2.5" +kube = "0.98" +k8s-openapi = { version = "0.24", features = ["v1_30"] } +serde_yaml = "0.9" +serde-value = "0.7" +http = "1.2" +inquire = "0.7" +convert_case = "0.8" chrono = "0.4" - -[workspace.dependencies.uuid] -version = "1.11.0" -features = [ - "v4", # Lets you generate random UUIDs - "fast-rng", # Use a faster (but still sufficiently random) RNG - "macro-diagnostics", # Enable better diagnostics for compile-time UUIDs -] +similar = "2" +uuid = { version = "1.11", features = [ "v4", "fast-rng", "macro-diagnostics" ] } diff --git a/examples/lamp/src/main.rs b/examples/lamp/src/main.rs index 0b1f93c..51816e6 100644 --- a/examples/lamp/src/main.rs +++ b/examples/lamp/src/main.rs @@ -2,10 +2,7 @@ use harmony::{ data::Version, inventory::Inventory, maestro::Maestro, - modules::{ - lamp::{LAMPConfig, LAMPScore}, - monitoring::alert_channel::discord_alert_channel::DiscordWebhook, - }, + modules::lamp::{LAMPConfig, LAMPScore}, topology::{K8sAnywhereTopology, Url}, }; @@ -53,7 +50,7 @@ async fn main() { .await .unwrap(); - // maestro.register_all(vec![Box::new(lamp_stack)]); + maestro.register_all(vec![Box::new(lamp_stack)]); // Here we bootstrap the CLI, this gives some nice features if you need them harmony_cli::init(maestro, None).await.unwrap(); } diff --git a/examples/monitoring/Cargo.toml b/examples/monitoring/Cargo.toml index af42491..d188b78 100644 --- a/examples/monitoring/Cargo.toml +++ b/examples/monitoring/Cargo.toml @@ -9,3 +9,4 @@ license.workspace = true harmony = { version = "0.1.0", path = "../../harmony" } harmony_cli = { version = "0.1.0", path = "../../harmony_cli" } tokio.workspace = true +url.workspace = true diff --git a/examples/monitoring/src/main.rs b/examples/monitoring/src/main.rs index d52c649..c2522f3 100644 --- a/examples/monitoring/src/main.rs +++ b/examples/monitoring/src/main.rs @@ -1,12 +1,50 @@ use harmony::{ - inventory::Inventory, maestro::Maestro, - modules::monitoring::kube_prometheus::helm_prometheus_alert_score::HelmPrometheusAlertingScore, - topology::K8sAnywhereTopology, + inventory::Inventory, + maestro::Maestro, + modules::{ + monitoring::{ + alert_channel::discord_alert_channel::DiscordWebhook, + alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, + kube_prometheus::helm_prometheus_alert_score::HelmPrometheusAlertingScore, + }, + prometheus::alerts::{ + infra::dell_server::{ + alert_global_storage_status_critical, alert_global_storage_status_non_recoverable, + global_storage_status_degraded_non_critical, + }, + k8s::pvc::high_pvc_fill_rate_over_two_days, + }, + }, + topology::{K8sAnywhereTopology, Url}, }; #[tokio::main] async fn main() { - let alerting_score = HelmPrometheusAlertingScore { receivers: vec![] }; + let discord_receiver = DiscordWebhook { + name: "test-discord".to_string(), + url: Url::Url(url::Url::parse("https://discord.doesnt.exist.com").unwrap()), + }; + + let high_pvc_fill_rate_over_two_days_alert = high_pvc_fill_rate_over_two_days(); + let dell_system_storage_degraded = global_storage_status_degraded_non_critical(); + let alert_global_storage_status_critical = alert_global_storage_status_critical(); + let alert_global_storage_status_non_recoverable = alert_global_storage_status_non_recoverable(); + + let additional_rules = + AlertManagerRuleGroup::new("pvc-alerts", vec![high_pvc_fill_rate_over_two_days_alert]); + let additional_rules2 = AlertManagerRuleGroup::new( + "dell-server-alerts", + vec![ + dell_system_storage_degraded, + alert_global_storage_status_critical, + alert_global_storage_status_non_recoverable, + ], + ); + + let alerting_score = HelmPrometheusAlertingScore { + receivers: vec![Box::new(discord_receiver)], + rules: vec![Box::new(additional_rules), Box::new(additional_rules2)], + }; let mut maestro = Maestro::::initialize( Inventory::autoload(), K8sAnywhereTopology::from_env(), @@ -14,12 +52,6 @@ async fn main() { .await .unwrap(); - //let monitoring = MonitoringAlertingScore { - // alert_receivers: vec![], - // alert_rules: vec![], - // scrape_targets: vec![], - //}; - //maestro.register_all(vec![Box::new(monitoring)]); maestro.register_all(vec![Box::new(alerting_score)]); harmony_cli::init(maestro, None).await.unwrap(); } diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index 54cf36d..f84bd63 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -53,3 +53,4 @@ fqdn = { version = "0.4.6", features = [ ] } temp-dir = "0.1.14" dyn-clone = "1.0.19" +similar.workspace = true diff --git a/harmony/src/domain/config.rs b/harmony/src/domain/config.rs index 53d7446..7812616 100644 --- a/harmony/src/domain/config.rs +++ b/harmony/src/domain/config.rs @@ -10,4 +10,6 @@ lazy_static! { std::env::var("HARMONY_REGISTRY_URL").unwrap_or_else(|_| "hub.nationtech.io".to_string()); pub static ref REGISTRY_PROJECT: String = std::env::var("HARMONY_REGISTRY_PROJECT").unwrap_or_else(|_| "harmony".to_string()); + pub static ref DRY_RUN: bool = + std::env::var("HARMONY_DRY_RUN").map_or(true, |value| value.parse().unwrap_or(true)); } diff --git a/harmony/src/domain/topology/installable.rs b/harmony/src/domain/topology/installable.rs index 9b9054f..8d8178c 100644 --- a/harmony/src/domain/topology/installable.rs +++ b/harmony/src/domain/topology/installable.rs @@ -1,8 +1,12 @@ use async_trait::async_trait; -use crate::interpret::InterpretError; +use crate::{interpret::InterpretError, inventory::Inventory}; #[async_trait] -pub trait Installable { - async fn ensure_installed(&self) -> Result<(), InterpretError>; +pub trait Installable: Send + Sync { + async fn ensure_installed( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result<(), InterpretError>; } diff --git a/harmony/src/domain/topology/k8s.rs b/harmony/src/domain/topology/k8s.rs index bf91726..fc91b0c 100644 --- a/harmony/src/domain/topology/k8s.rs +++ b/harmony/src/domain/topology/k8s.rs @@ -4,9 +4,11 @@ use kube::{ Api, Client, Config, Error, Resource, api::{Patch, PatchParams}, config::{KubeConfigOptions, Kubeconfig}, + core::ErrorResponse, }; use log::{debug, error, trace}; use serde::de::DeserializeOwned; +use similar::TextDiff; #[derive(new, Clone)] pub struct K8sClient { @@ -59,8 +61,79 @@ impl K8sClient { .name .as_ref() .expect("K8s Resource should have a name"); - api.patch(name, &patch_params, &Patch::Apply(resource)) - .await + + if *crate::config::DRY_RUN { + match api.get(name).await { + Ok(current) => { + trace!("Received current value {current:#?}"); + // The resource exists, so we calculate and display a diff. + println!("\nPerforming dry-run for resource: '{}'", name); + let mut current_yaml = serde_yaml::to_value(¤t) + .expect(&format!("Could not serialize current value : {current:#?}")); + if current_yaml.is_mapping() && current_yaml.get("status").is_some() { + let map = current_yaml.as_mapping_mut().unwrap(); + let removed = map.remove_entry("status"); + trace!("Removed status {:?}", removed); + } else { + trace!( + "Did not find status entry for current object {}/{}", + current.meta().namespace.as_ref().unwrap_or(&"".to_string()), + current.meta().name.as_ref().unwrap_or(&"".to_string()) + ); + } + let current_yaml = serde_yaml::to_string(¤t_yaml) + .unwrap_or_else(|_| "Failed to serialize current resource".to_string()); + let new_yaml = serde_yaml::to_string(resource) + .unwrap_or_else(|_| "Failed to serialize new resource".to_string()); + + if current_yaml == new_yaml { + println!("No changes detected."); + // Return the current resource state as there are no changes. + return Ok(current); + } + + println!("Changes detected:"); + let diff = TextDiff::from_lines(¤t_yaml, &new_yaml); + + // Iterate over the changes and print them in a git-like diff format. + for change in diff.iter_all_changes() { + let sign = match change.tag() { + similar::ChangeTag::Delete => "-", + similar::ChangeTag::Insert => "+", + similar::ChangeTag::Equal => " ", + }; + print!("{}{}", sign, change); + } + // In a dry run, we return the new resource state that would have been applied. + Ok(resource.clone()) + } + Err(Error::Api(ErrorResponse { code: 404, .. })) => { + // The resource does not exist, so the "diff" is the entire new resource. + println!("\nPerforming dry-run for new resource: '{}'", name); + println!( + "Resource does not exist. It would be created with the following content:" + ); + let new_yaml = serde_yaml::to_string(resource) + .unwrap_or_else(|_| "Failed to serialize new resource".to_string()); + + // Print each line of the new resource with a '+' prefix. + for line in new_yaml.lines() { + println!("+{}", line); + } + // In a dry run, we return the new resource state that would have been created. + Ok(resource.clone()) + } + Err(e) => { + // Another API error occurred. + error!("Failed to get resource '{}': {}", name, e); + Err(e) + } + } + } else { + return api + .patch(name, &patch_params, &Patch::Apply(resource)) + .await; + } } pub async fn apply_many(&self, resource: &Vec, ns: Option<&str>) -> Result, Error> diff --git a/harmony/src/domain/topology/oberservability/monitoring.rs b/harmony/src/domain/topology/oberservability/monitoring.rs index a3a6164..ed7e936 100644 --- a/harmony/src/domain/topology/oberservability/monitoring.rs +++ b/harmony/src/domain/topology/oberservability/monitoring.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use log::debug; use crate::{ data::{Id, Version}, @@ -7,16 +8,20 @@ use crate::{ topology::{Topology, installable::Installable}, }; -pub trait AlertSender: Send + Sync + std::fmt::Debug + Installable {} +#[async_trait] +pub trait AlertSender: Send + Sync + std::fmt::Debug { + fn name(&self) -> String; +} #[derive(Debug)] pub struct AlertingInterpret { pub sender: S, pub receivers: Vec>>, + pub rules: Vec>>, } #[async_trait] -impl Interpret for AlertingInterpret { +impl, T: Topology> Interpret for AlertingInterpret { async fn execute( &self, inventory: &Inventory, @@ -25,7 +30,15 @@ impl Interpret for AlertingInterpret { for receiver in self.receivers.iter() { receiver.install(&self.sender).await?; } - todo!() + for rule in self.rules.iter() { + debug!("installing rule: {:#?}", rule); + rule.install(&self.sender).await?; + } + self.sender.ensure_installed(inventory, topology).await?; + Ok(Outcome::success(format!( + "successfully installed alert sender {}", + self.sender.name() + ))) } fn get_name(&self) -> InterpretName { @@ -47,12 +60,14 @@ impl Interpret for AlertingInterpret { #[async_trait] pub trait AlertReceiver: std::fmt::Debug + Send + Sync { - async fn install(&self, sender: &S) -> Result<(), InterpretError>; + async fn install(&self, sender: &S) -> Result; + fn clone_box(&self) -> Box>; } #[async_trait] -pub trait AlertRule { - async fn install(&self, sender: &S) -> Result<(), InterpretError>; +pub trait AlertRule: std::fmt::Debug + Send + Sync { + async fn install(&self, sender: &S) -> Result; + fn clone_box(&self) -> Box>; } #[async_trait] diff --git a/harmony/src/domain/topology/tenant/k8s.rs b/harmony/src/domain/topology/tenant/k8s.rs index 260cf3f..705b0c4 100644 --- a/harmony/src/domain/topology/tenant/k8s.rs +++ b/harmony/src/domain/topology/tenant/k8s.rs @@ -138,6 +138,7 @@ impl K8sTenantManager { "kind": "NetworkPolicy", "metadata": { "name": format!("{}-network-policy", config.name), + "namespace": self.get_namespace_name(config), }, "spec": { "podSelector": {}, @@ -219,8 +220,29 @@ impl K8sTenantManager { }) }) .collect(); + let ports: Option> = + c.1.as_ref().map(|spec| match &spec.data { + super::PortSpecData::SinglePort(port) => vec![NetworkPolicyPort { + port: Some(IntOrString::Int(port.clone().into())), + ..Default::default() + }], + super::PortSpecData::PortRange(start, end) => vec![NetworkPolicyPort { + port: Some(IntOrString::Int(start.clone().into())), + end_port: Some(end.clone().into()), + protocol: None, // Not currently supported by Harmony + }], + + super::PortSpecData::ListOfPorts(items) => items + .iter() + .map(|i| NetworkPolicyPort { + port: Some(IntOrString::Int(i.clone().into())), + ..Default::default() + }) + .collect(), + }); let rule = serde_json::from_value::(json!({ - "from": cidr_list + "from": cidr_list, + "ports": ports, })) .map_err(|e| { ExecutorError::ConfigurationError(format!( diff --git a/harmony/src/modules/mod.rs b/harmony/src/modules/mod.rs index 8b5720e..e9b6c52 100644 --- a/harmony/src/modules/mod.rs +++ b/harmony/src/modules/mod.rs @@ -13,5 +13,6 @@ pub mod load_balancer; pub mod monitoring; pub mod okd; pub mod opnsense; +pub mod prometheus; pub mod tenant; pub mod tftp; diff --git a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs index 42f4450..fb85a98 100644 --- a/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs +++ b/harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs @@ -1,12 +1,17 @@ use async_trait::async_trait; +use serde::Serialize; +use serde_yaml::{Mapping, Value}; use crate::{ - interpret::InterpretError, - modules::monitoring::kube_prometheus::prometheus::{Prometheus, PrometheusReceiver}, + interpret::{InterpretError, Outcome}, + modules::monitoring::kube_prometheus::{ + prometheus::{Prometheus, PrometheusReceiver}, + types::{AlertChannelConfig, AlertManagerChannelConfig}, + }, topology::{Url, oberservability::monitoring::AlertReceiver}, }; -#[derive(Debug)] +#[derive(Debug, Clone, Serialize)] pub struct DiscordWebhook { pub name: String, pub url: Url, @@ -14,7 +19,107 @@ pub struct DiscordWebhook { #[async_trait] impl AlertReceiver for DiscordWebhook { - async fn install(&self, sender: &Prometheus) -> Result<(), InterpretError> { - sender.install_receiver(PrometheusReceiver {}).await + async fn install(&self, sender: &Prometheus) -> Result { + sender.install_receiver(self).await + } + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } +} + +#[async_trait] +impl PrometheusReceiver for DiscordWebhook { + fn name(&self) -> String { + self.name.clone() + } + async fn configure_receiver(&self) -> AlertManagerChannelConfig { + self.get_config().await + } +} + +#[async_trait] +impl AlertChannelConfig for DiscordWebhook { + async fn get_config(&self) -> AlertManagerChannelConfig { + let channel_global_config = None; + let channel_receiver = self.alert_channel_receiver().await; + let channel_route = self.alert_channel_route().await; + + AlertManagerChannelConfig { + channel_global_config, + channel_receiver, + channel_route, + } + } +} + +impl DiscordWebhook { + async fn alert_channel_route(&self) -> serde_yaml::Value { + let mut route = Mapping::new(); + route.insert( + Value::String("receiver".to_string()), + Value::String(self.name.clone()), + ); + route.insert( + Value::String("matchers".to_string()), + Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]), + ); + route.insert(Value::String("continue".to_string()), Value::Bool(true)); + Value::Mapping(route) + } + + async fn alert_channel_receiver(&self) -> serde_yaml::Value { + let mut receiver = Mapping::new(); + receiver.insert( + Value::String("name".to_string()), + Value::String(self.name.clone()), + ); + + let mut discord_config = Mapping::new(); + discord_config.insert( + Value::String("webhook_url".to_string()), + Value::String(self.url.to_string()), + ); + + receiver.insert( + Value::String("discord_configs".to_string()), + Value::Sequence(vec![Value::Mapping(discord_config)]), + ); + + Value::Mapping(receiver) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn discord_serialize_should_match() { + let discord_receiver = DiscordWebhook { + name: "test-discord".to_string(), + url: Url::Url(url::Url::parse("https://discord.i.dont.exist.com").unwrap()), + }; + + let discord_receiver_receiver = + serde_yaml::to_string(&discord_receiver.alert_channel_receiver().await).unwrap(); + println!("receiver \n{:#}", discord_receiver_receiver); + let discord_receiver_receiver_yaml = r#"name: test-discord +discord_configs: +- webhook_url: https://discord.i.dont.exist.com/ +"# + .to_string(); + + let discord_receiver_route = + serde_yaml::to_string(&discord_receiver.alert_channel_route().await).unwrap(); + println!("route \n{:#}", discord_receiver_route); + let discord_receiver_route_yaml = r#"receiver: test-discord +matchers: +- alertname!=Watchdog +continue: true +"# + .to_string(); + + assert_eq!(discord_receiver_receiver, discord_receiver_receiver_yaml); + assert_eq!(discord_receiver_route, discord_receiver_route_yaml); } } diff --git a/harmony/src/modules/monitoring/alert_channel/mod.rs b/harmony/src/modules/monitoring/alert_channel/mod.rs index fabc6dd..1bca2bc 100644 --- a/harmony/src/modules/monitoring/alert_channel/mod.rs +++ b/harmony/src/modules/monitoring/alert_channel/mod.rs @@ -1 +1,2 @@ pub mod discord_alert_channel; +pub mod webhook_receiver; diff --git a/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs new file mode 100644 index 0000000..8f608d0 --- /dev/null +++ b/harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs @@ -0,0 +1,124 @@ +use async_trait::async_trait; +use serde::Serialize; +use serde_yaml::{Mapping, Value}; + +use crate::{ + interpret::{InterpretError, Outcome}, + modules::monitoring::kube_prometheus::{ + prometheus::{Prometheus, PrometheusReceiver}, + types::{AlertChannelConfig, AlertManagerChannelConfig}, + }, + topology::{Url, oberservability::monitoring::AlertReceiver}, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct WebhookReceiver { + pub name: String, + pub url: Url, +} + +#[async_trait] +impl AlertReceiver for WebhookReceiver { + async fn install(&self, sender: &Prometheus) -> Result { + sender.install_receiver(self).await + } + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } +} + +#[async_trait] +impl PrometheusReceiver for WebhookReceiver { + fn name(&self) -> String { + self.name.clone() + } + async fn configure_receiver(&self) -> AlertManagerChannelConfig { + self.get_config().await + } +} + +#[async_trait] +impl AlertChannelConfig for WebhookReceiver { + async fn get_config(&self) -> AlertManagerChannelConfig { + let channel_global_config = None; + let channel_receiver = self.alert_channel_receiver().await; + let channel_route = self.alert_channel_route().await; + + AlertManagerChannelConfig { + channel_global_config, + channel_receiver, + channel_route, + } + } +} + +impl WebhookReceiver { + async fn alert_channel_route(&self) -> serde_yaml::Value { + let mut route = Mapping::new(); + route.insert( + Value::String("receiver".to_string()), + Value::String(self.name.clone()), + ); + route.insert( + Value::String("matchers".to_string()), + Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]), + ); + route.insert(Value::String("continue".to_string()), Value::Bool(true)); + Value::Mapping(route) + } + + async fn alert_channel_receiver(&self) -> serde_yaml::Value { + let mut receiver = Mapping::new(); + receiver.insert( + Value::String("name".to_string()), + Value::String(self.name.clone()), + ); + + let mut webhook_config = Mapping::new(); + webhook_config.insert( + Value::String("url".to_string()), + Value::String(self.url.to_string()), + ); + + receiver.insert( + Value::String("webhook_configs".to_string()), + Value::Sequence(vec![Value::Mapping(webhook_config)]), + ); + + Value::Mapping(receiver) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn webhook_serialize_should_match() { + let webhook_receiver = WebhookReceiver { + name: "test-webhook".to_string(), + url: Url::Url(url::Url::parse("https://webhook.i.dont.exist.com").unwrap()), + }; + + let webhook_receiver_receiver = + serde_yaml::to_string(&webhook_receiver.alert_channel_receiver().await).unwrap(); + println!("receiver \n{:#}", webhook_receiver_receiver); + let webhook_receiver_receiver_yaml = r#"name: test-webhook +webhook_configs: +- url: https://webhook.i.dont.exist.com/ +"# + .to_string(); + + let webhook_receiver_route = + serde_yaml::to_string(&webhook_receiver.alert_channel_route().await).unwrap(); + println!("route \n{:#}", webhook_receiver_route); + let webhook_receiver_route_yaml = r#"receiver: test-webhook +matchers: +- alertname!=Watchdog +continue: true +"# + .to_string(); + + assert_eq!(webhook_receiver_receiver, webhook_receiver_receiver_yaml); + assert_eq!(webhook_receiver_route, webhook_receiver_route_yaml); + } +} diff --git a/harmony/src/modules/monitoring/alert_rule/mod.rs b/harmony/src/modules/monitoring/alert_rule/mod.rs new file mode 100644 index 0000000..846c769 --- /dev/null +++ b/harmony/src/modules/monitoring/alert_rule/mod.rs @@ -0,0 +1 @@ +pub mod prometheus_alert_rule; diff --git a/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs new file mode 100644 index 0000000..ccb63cc --- /dev/null +++ b/harmony/src/modules/monitoring/alert_rule/prometheus_alert_rule.rs @@ -0,0 +1,99 @@ +use std::collections::{BTreeMap, HashMap}; + +use async_trait::async_trait; +use serde::Serialize; + +use crate::{ + interpret::{InterpretError, Outcome}, + modules::monitoring::kube_prometheus::{ + prometheus::{Prometheus, PrometheusRule}, + types::{AlertGroup, AlertManagerAdditionalPromRules}, + }, + topology::oberservability::monitoring::AlertRule, +}; + +#[async_trait] +impl AlertRule for AlertManagerRuleGroup { + async fn install(&self, sender: &Prometheus) -> Result { + sender.install_rule(&self).await + } + fn clone_box(&self) -> Box> { + Box::new(self.clone()) + } +} + +#[async_trait] +impl PrometheusRule for AlertManagerRuleGroup { + fn name(&self) -> String { + self.name.clone() + } + async fn configure_rule(&self) -> AlertManagerAdditionalPromRules { + let mut additional_prom_rules = BTreeMap::new(); + + additional_prom_rules.insert( + self.name.clone(), + AlertGroup { + groups: vec![self.clone()], + }, + ); + AlertManagerAdditionalPromRules { + rules: additional_prom_rules, + } + } +} + +impl AlertManagerRuleGroup { + pub fn new(name: &str, rules: Vec) -> AlertManagerRuleGroup { + AlertManagerRuleGroup { + name: name.to_string().to_lowercase(), + rules, + } + } +} + +#[derive(Debug, Clone, Serialize)] +///logical group of alert rules +///evaluates to: +///name: +/// groups: +/// - name: name +/// rules: PrometheusAlertRule +pub struct AlertManagerRuleGroup { + pub name: String, + pub rules: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct PrometheusAlertRule { + pub alert: String, + pub expr: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub r#for: Option, + pub labels: HashMap, + pub annotations: HashMap, +} + +impl PrometheusAlertRule { + pub fn new(alert_name: &str, expr: &str) -> Self { + Self { + alert: alert_name.into(), + expr: expr.into(), + r#for: Some("1m".into()), + labels: HashMap::new(), + annotations: HashMap::new(), + } + } + pub fn for_duration(mut self, duration: &str) -> Self { + self.r#for = Some(duration.into()); + self + } + pub fn label(mut self, key: &str, value: &str) -> Self { + self.labels.insert(key.into(), value.into()); + self + } + + pub fn annotation(mut self, key: &str, value: &str) -> Self { + self.annotations.insert(key.into(), value.into()); + self + } +} diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm/config.rs b/harmony/src/modules/monitoring/kube_prometheus/helm/config.rs index 0e62c0f..ecbf8d8 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm/config.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm/config.rs @@ -1,5 +1,10 @@ use serde::Serialize; +use crate::modules::monitoring::{ + alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, + kube_prometheus::types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig}, +}; + #[derive(Debug, Clone, Serialize)] pub struct KubePrometheusConfig { pub namespace: String, @@ -19,6 +24,8 @@ pub struct KubePrometheusConfig { pub kube_proxy: bool, pub kube_state_metrics: bool, pub prometheus_operator: bool, + pub alert_receiver_configs: Vec, + pub alert_rules: Vec, } impl KubePrometheusConfig { pub fn new() -> Self { @@ -40,6 +47,8 @@ impl KubePrometheusConfig { prometheus_operator: true, core_dns: false, kube_scheduler: false, + alert_receiver_configs: vec![], + alert_rules: vec![], } } } diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs b/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs index 2377627..843a677 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm/kube_prometheus_helm_chart.rs @@ -1,17 +1,28 @@ use super::config::KubePrometheusConfig; +use log::debug; use non_blank_string_rs::NonBlankString; -use std::str::FromStr; +use serde_yaml::{Mapping, Value}; +use std::{ + collections::BTreeMap, + str::FromStr, + sync::{Arc, Mutex}, +}; -use crate::modules::helm::chart::HelmChartScore; +use crate::modules::{ + helm::chart::HelmChartScore, + monitoring::kube_prometheus::types::{ + AlertGroup, AlertManager, AlertManagerAdditionalPromRules, AlertManagerConfig, + AlertManagerRoute, AlertManagerValues, + }, +}; -pub fn kube_prometheus_helm_chart_score() -> HelmChartScore { - let config = KubePrometheusConfig::new(); +pub fn kube_prometheus_helm_chart_score( + config: Arc>, +) -> HelmChartScore { + let config = config.lock().unwrap(); - //TODO this should be make into a rule with default formatting that can be easily passed as a vec - //to the overrides or something leaving the user to deal with formatting here seems bad let default_rules = config.default_rules.to_string(); let windows_monitoring = config.windows_monitoring.to_string(); - let alert_manager = config.alert_manager.to_string(); let grafana = config.grafana.to_string(); let kubernetes_service_monitors = config.kubernetes_service_monitors.to_string(); let kubernetes_api_server = config.kubernetes_api_server.to_string(); @@ -25,58 +36,8 @@ pub fn kube_prometheus_helm_chart_score() -> HelmChartScore { let node_exporter = config.node_exporter.to_string(); let prometheus_operator = config.prometheus_operator.to_string(); let prometheus = config.prometheus.to_string(); - let values = format!( + let mut values = format!( r#" -additionalPrometheusRulesMap: - pods-status-alerts: - groups: - - name: pods - rules: - - alert: "[CRIT] POD not healthy" - expr: min_over_time(sum by (namespace, pod) (kube_pod_status_phase{{phase=~"Pending|Unknown|Failed"}})[15m:1m]) > 0 - for: 0m - labels: - severity: critical - annotations: - title: "[CRIT] POD not healthy : {{{{ $labels.pod }}}}" - description: | - A POD is in a non-ready state! - - **Pod**: {{{{ $labels.pod }}}} - - **Namespace**: {{{{ $labels.namespace }}}} - - alert: "[CRIT] POD crash looping" - expr: increase(kube_pod_container_status_restarts_total[5m]) > 3 - for: 0m - labels: - severity: critical - annotations: - title: "[CRIT] POD crash looping : {{{{ $labels.pod }}}}" - description: | - A POD is drowning in a crash loop! - - **Pod**: {{{{ $labels.pod }}}} - - **Namespace**: {{{{ $labels.namespace }}}} - - **Instance**: {{{{ $labels.instance }}}} - pvc-alerts: - groups: - - name: pvc-alerts - rules: - - alert: 'PVC Fill Over 95 Percent In 2 Days' - expr: | - ( - kubelet_volume_stats_used_bytes - / - kubelet_volume_stats_capacity_bytes - ) > 0.95 - AND - predict_linear(kubelet_volume_stats_used_bytes[2d], 2 * 24 * 60 * 60) - / - kubelet_volume_stats_capacity_bytes - > 0.95 - for: 1m - labels: - severity: warning - annotations: - description: The PVC {{{{ $labels.persistentvolumeclaim }}}} in namespace {{{{ $labels.namespace }}}} is predicted to fill over 95% in less than 2 days. - title: PVC {{{{ $labels.persistentvolumeclaim }}}} in namespace {{{{ $labels.namespace }}}} will fill over 95% in less than 2 days defaultRules: create: {default_rules} rules: @@ -144,6 +105,85 @@ prometheus: enabled: {prometheus} "#, ); + + // add required null receiver for prometheus alert manager + let mut null_receiver = Mapping::new(); + null_receiver.insert( + Value::String("receiver".to_string()), + Value::String("null".to_string()), + ); + null_receiver.insert( + Value::String("matchers".to_string()), + Value::Sequence(vec![Value::String("alertname!=Watchdog".to_string())]), + ); + null_receiver.insert(Value::String("continue".to_string()), Value::Bool(true)); + + //add alert channels + let mut alert_manager_channel_config = AlertManagerConfig { + global: Mapping::new(), + route: AlertManagerRoute { + routes: vec![Value::Mapping(null_receiver)], + }, + receivers: vec![serde_yaml::from_str("name: 'null'").unwrap()], + }; + for receiver in config.alert_receiver_configs.iter() { + if let Some(global) = receiver.channel_global_config.clone() { + alert_manager_channel_config + .global + .insert(global.0, global.1); + } + alert_manager_channel_config + .route + .routes + .push(receiver.channel_route.clone()); + alert_manager_channel_config + .receivers + .push(receiver.channel_receiver.clone()); + } + + let alert_manager_values = AlertManagerValues { + alertmanager: AlertManager { + enabled: config.alert_manager, + config: alert_manager_channel_config, + }, + }; + + let alert_manager_yaml = + serde_yaml::to_string(&alert_manager_values).expect("Failed to serialize YAML"); + debug!("serialized alert manager: \n {:#}", alert_manager_yaml); + values.push_str(&alert_manager_yaml); + + //format alert manager additional rules for helm chart + let mut merged_rules: BTreeMap = BTreeMap::new(); + + for additional_rule in config.alert_rules.clone() { + for (key, group) in additional_rule.rules { + merged_rules.insert(key, group); + } + } + + let merged_rules = AlertManagerAdditionalPromRules { + rules: merged_rules, + }; + + let mut alert_manager_additional_rules = serde_yaml::Mapping::new(); + let rules_value = serde_yaml::to_value(merged_rules).unwrap(); + + alert_manager_additional_rules.insert( + serde_yaml::Value::String("additionalPrometheusRulesMap".to_string()), + rules_value, + ); + + let alert_manager_additional_rules_yaml = + serde_yaml::to_string(&alert_manager_additional_rules).expect("Failed to serialize YAML"); + debug!( + "alert_rules_yaml:\n{:#}", + alert_manager_additional_rules_yaml + ); + + values.push_str(&alert_manager_additional_rules_yaml); + debug!("full values.yaml: \n {:#}", values); + HelmChartScore { namespace: Some(NonBlankString::from_str(&config.namespace).unwrap()), release_name: NonBlankString::from_str("kube-prometheus").unwrap(), diff --git a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs b/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs index c090f13..8844309 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs @@ -1,47 +1,33 @@ +use std::sync::{Arc, Mutex}; + use serde::Serialize; +use super::{helm::config::KubePrometheusConfig, prometheus::Prometheus}; use crate::{ - modules::monitoring::alert_channel::discord_alert_channel::DiscordWebhook, score::Score, topology::{ HelmCommand, Topology, - oberservability::monitoring::{AlertReceiver, AlertingInterpret}, + oberservability::monitoring::{AlertReceiver, AlertRule, AlertingInterpret}, }, }; -use super::prometheus::Prometheus; - #[derive(Clone, Debug, Serialize)] pub struct HelmPrometheusAlertingScore { pub receivers: Vec>>, + pub rules: Vec>>, } impl Score for HelmPrometheusAlertingScore { fn create_interpret(&self) -> Box> { Box::new(AlertingInterpret { - sender: Prometheus {}, - receivers: vec![Box::new(DiscordWebhook { - url: todo!(), - name: todo!(), - })], + sender: Prometheus { + config: Arc::new(Mutex::new(KubePrometheusConfig::new())), + }, + receivers: self.receivers.clone(), + rules: self.rules.clone(), }) } - fn name(&self) -> String { "HelmPrometheusAlertingScore".to_string() } } - -impl Serialize for Box> { - fn serialize(&self, _serializer: S) -> Result - where - S: serde::Serializer, - { - todo!() - } -} -impl Clone for Box> { - fn clone(&self) -> Self { - todo!() - } -} diff --git a/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs b/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs index c5be07e..148f91c 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/prometheus.rs @@ -1,34 +1,139 @@ +use std::sync::{Arc, Mutex}; + use async_trait::async_trait; +use log::debug; +use serde::Serialize; use crate::{ - interpret::InterpretError, - topology::{installable::Installable, oberservability::monitoring::AlertSender}, + interpret::{InterpretError, Outcome}, + inventory::Inventory, + modules::monitoring::alert_rule::prometheus_alert_rule::AlertManagerRuleGroup, + score, + topology::{ + HelmCommand, Topology, + installable::Installable, + oberservability::monitoring::{AlertReceiver, AlertRule, AlertSender}, + }, }; -impl AlertSender for Prometheus {} +use score::Score; + +use super::{ + helm::{ + config::KubePrometheusConfig, kube_prometheus_helm_chart::kube_prometheus_helm_chart_score, + }, + types::{AlertManagerAdditionalPromRules, AlertManagerChannelConfig}, +}; #[async_trait] -impl Installable for Prometheus { - async fn ensure_installed(&self) -> Result<(), InterpretError> { - todo!() +impl AlertSender for Prometheus { + fn name(&self) -> String { + "HelmKubePrometheus".to_string() } } + +#[async_trait] +impl Installable for Prometheus { + async fn ensure_installed( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result<(), InterpretError> { + self.install_prometheus(inventory, topology).await?; + Ok(()) + } +} + #[derive(Debug)] -pub struct Prometheus; +pub struct Prometheus { + pub config: Arc>, +} impl Prometheus { pub async fn install_receiver( &self, - prometheus_receiver: PrometheusReceiver, - ) -> Result<(), InterpretError> { + prometheus_receiver: &dyn PrometheusReceiver, + ) -> Result { + let prom_receiver = prometheus_receiver.configure_receiver().await; + debug!( + "adding alert receiver to prometheus config: {:#?}", + &prom_receiver + ); + let mut config = self.config.lock().unwrap(); + + config.alert_receiver_configs.push(prom_receiver); + let prom_receiver_name = prometheus_receiver.name(); + debug!("installed alert receiver {}", &prom_receiver_name); + Ok(Outcome::success(format!( + "Sucessfully installed receiver {}", + prom_receiver_name + ))) + } + + pub async fn install_rule( + &self, + prometheus_rule: &AlertManagerRuleGroup, + ) -> Result { + let prometheus_rule = prometheus_rule.configure_rule().await; + let mut config = self.config.lock().unwrap(); + + config.alert_rules.push(prometheus_rule.clone()); + Ok(Outcome::success(format!( + "Successfully installed alert rule: {:#?},", + prometheus_rule + ))) + } + + pub async fn install_prometheus( + &self, + inventory: &Inventory, + topology: &T, + ) -> Result { + kube_prometheus_helm_chart_score(self.config.clone()) + .create_interpret() + .execute(inventory, topology) + .await + } +} + +#[async_trait] +pub trait PrometheusReceiver: Send + Sync + std::fmt::Debug { + fn name(&self) -> String; + async fn configure_receiver(&self) -> AlertManagerChannelConfig; +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { todo!() } } -pub struct PrometheusReceiver {} - -impl PrometheusReceiver { - fn get_prometheus_receiver_config(&self) {} +impl Clone for Box> { + fn clone(&self) -> Self { + self.clone_box() + } } -pub struct AlertChannelGlobalConfig {} +#[async_trait] +pub trait PrometheusRule: Send + Sync + std::fmt::Debug { + fn name(&self) -> String; + async fn configure_rule(&self) -> AlertManagerAdditionalPromRules; +} + +impl Serialize for Box> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + +impl Clone for Box> { + fn clone(&self) -> Self { + self.clone_box() + } +} diff --git a/harmony/src/modules/monitoring/kube_prometheus/types.rs b/harmony/src/modules/monitoring/kube_prometheus/types.rs index 224b125..878d527 100644 --- a/harmony/src/modules/monitoring/kube_prometheus/types.rs +++ b/harmony/src/modules/monitoring/kube_prometheus/types.rs @@ -1,12 +1,55 @@ -use serde::Serialize; +use std::collections::BTreeMap; -#[derive(Serialize)] -pub struct AlertReceiverRoute { - pub receiver: String, - pub matchers: Vec, - #[serde(default)] - pub r#continue: bool, +use async_trait::async_trait; +use serde::Serialize; +use serde_yaml::{Mapping, Sequence, Value}; + +use crate::modules::monitoring::alert_rule::prometheus_alert_rule::AlertManagerRuleGroup; + +#[async_trait] +pub trait AlertChannelConfig { + async fn get_config(&self) -> AlertManagerChannelConfig; } -pub struct AlertChannelReceiver { - pub name: String, + +#[derive(Debug, Clone, Serialize)] +pub struct AlertManagerValues { + pub alertmanager: AlertManager, +} +#[derive(Debug, Clone, Serialize)] +pub struct AlertManager { + pub enabled: bool, + pub config: AlertManagerConfig, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AlertManagerConfig { + pub global: Mapping, + pub route: AlertManagerRoute, + pub receivers: Sequence, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AlertManagerRoute { + pub routes: Sequence, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AlertManagerChannelConfig { + ///expecting an option that contains two values + ///if necessary for the alertchannel + ///[ jira_api_url: ] + pub channel_global_config: Option<(Value, Value)>, + pub channel_route: Value, + pub channel_receiver: Value, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AlertManagerAdditionalPromRules { + #[serde(flatten)] + pub rules: BTreeMap, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AlertGroup { + pub groups: Vec, } diff --git a/harmony/src/modules/monitoring/mod.rs b/harmony/src/modules/monitoring/mod.rs index 7cdb3a9..0e4f8a4 100644 --- a/harmony/src/modules/monitoring/mod.rs +++ b/harmony/src/modules/monitoring/mod.rs @@ -1,2 +1,3 @@ pub mod alert_channel; +pub mod alert_rule; pub mod kube_prometheus; diff --git a/harmony/src/modules/prometheus/alerts/infra/dell_server.rs b/harmony/src/modules/prometheus/alerts/infra/dell_server.rs new file mode 100644 index 0000000..1092c4c --- /dev/null +++ b/harmony/src/modules/prometheus/alerts/infra/dell_server.rs @@ -0,0 +1,40 @@ +use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule; + +pub fn global_storage_status_degraded_non_critical() -> PrometheusAlertRule { + PrometheusAlertRule::new("GlobalStorageStatusNonCritical", "globalStorageStatus == 4") + .for_duration("5m") + .label("severity", "warning") + .annotation( + "description", + "- **system**: {{ $labels.instance }}\n- **Status**: nonCritical\n- **Value**: {{ $value }}\n- **Job**: {{ $labels.job }}", + ) + .annotation("title", " System storage status is in degraded state") +} + +pub fn alert_global_storage_status_critical() -> PrometheusAlertRule { + PrometheusAlertRule::new( + "GlobalStorageStatus critical", + "globalStorageStatus == 5", + ) + .for_duration("5m") + .label("severity", "warning") + .annotation("title", "System storage status is critical at {{ $labels.instance }}") + .annotation( + "description", + "- **System**: {{ $labels.instance }}\n- **Status**: Critical\n- **Value**: {{ $value }}\n- **Job**: {{ $labels.job }}", + ) +} + +pub fn alert_global_storage_status_non_recoverable() -> PrometheusAlertRule { + PrometheusAlertRule::new( + "GlobalStorageStatus nonRecoverable", + "globalStorageStatus == 6", + ) + .for_duration("5m") + .label("severity", "warning") + .annotation("title", "System storage status is nonRecoverable at {{ $labels.instance }}") + .annotation( + "description", + "- **System**: {{ $labels.instance }}\n- **Status**: nonRecoverable\n- **Value**: {{ $value }}\n- **Job**: {{ $labels.job }}", + ) +} diff --git a/harmony/src/modules/prometheus/alerts/infra/mod.rs b/harmony/src/modules/prometheus/alerts/infra/mod.rs new file mode 100644 index 0000000..47fbc2f --- /dev/null +++ b/harmony/src/modules/prometheus/alerts/infra/mod.rs @@ -0,0 +1 @@ +pub mod dell_server; diff --git a/harmony/src/modules/prometheus/alerts/k8s/mod.rs b/harmony/src/modules/prometheus/alerts/k8s/mod.rs new file mode 100644 index 0000000..f01a9c8 --- /dev/null +++ b/harmony/src/modules/prometheus/alerts/k8s/mod.rs @@ -0,0 +1 @@ +pub mod pvc; diff --git a/harmony/src/modules/prometheus/alerts/k8s/pvc.rs b/harmony/src/modules/prometheus/alerts/k8s/pvc.rs new file mode 100644 index 0000000..f99ee39 --- /dev/null +++ b/harmony/src/modules/prometheus/alerts/k8s/pvc.rs @@ -0,0 +1,11 @@ +use crate::modules::monitoring::alert_rule::prometheus_alert_rule::PrometheusAlertRule; + +pub fn high_pvc_fill_rate_over_two_days() -> PrometheusAlertRule { + PrometheusAlertRule::new( + "PVC Fill Over 95 Percent In 2 Days", + "(kubelet_volume_stats_used_bytes/kubelet_volume_stats_capacity_bytes) > 0.95 AND predict_linear(kubelet_volume_stats_used_bytes[2d], 2 * 24 * 60 * 60)/kubelet_volume_stats_capacity_bytes > 0.95",) + .for_duration("1m") + .label("severity", "warning") + .annotation("summary", "The PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} is predicted to fill over 95% in less than 2 days.") + .annotation("description", "PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} will fill over 95% in less than 2 days",) +} diff --git a/harmony/src/modules/prometheus/alerts/mod.rs b/harmony/src/modules/prometheus/alerts/mod.rs new file mode 100644 index 0000000..416e56f --- /dev/null +++ b/harmony/src/modules/prometheus/alerts/mod.rs @@ -0,0 +1,2 @@ +pub mod infra; +pub mod k8s; diff --git a/harmony/src/modules/prometheus/mod.rs b/harmony/src/modules/prometheus/mod.rs new file mode 100644 index 0000000..3a0c1d7 --- /dev/null +++ b/harmony/src/modules/prometheus/mod.rs @@ -0,0 +1 @@ +pub mod alerts; diff --git a/harmony_composer/README.md b/harmony_composer/README.md new file mode 100644 index 0000000..45ac542 --- /dev/null +++ b/harmony_composer/README.md @@ -0,0 +1,74 @@ +# harmony_composer + +`harmony_composer` is a command-line utility for compiling and bootstrapping deployments for the Harmony orchestration framework. + +It's designed to simplify the build process by either compiling a Harmony project found in a local harmony directory or by bootstrapping a new deployment through auto-detection of the current project type. + +## ⚡ Quick Install & Run (Linux x86-64) + +You can download and run the latest snapshot build with a single command. This will place the binary in ~/.local/bin, which should be in your PATH on most modern Linux distributions. + +```bash +mkdir -p ~/.local/bin && \ +curl -L https://git.nationtech.io/NationTech/harmony/releases/download/snapshot-latest/harmony_composer \ + -o ~/.local/bin/harmony_composer && \ +chmod +x ~/.local/bin/harmony_composer && \ +alias hc=~/.local/bin/harmony_composer && \ +echo "\n\nharmony_composer installed successfully\!\n\nUse \`hc\` to run it.\n\nNote : this hc alias only works for the current shell session. Add 'alias hc=~/.local/bin/harmony_composer' to your '~/.bashrc' or '~/.zshrc' file to make it permanently available to your user." +``` + +Then you can start using it with either : + +- `harmony_composer` if `~/.local/bin` is in you `$PATH` +- `hc` alias set up in your current shell session. + - If you want to make the `hc` command always available, add `alias hc=~/.local/bin/harmony_composer` to your shell profile. Usually `~/.bashrc` for bash, `~/.zshrc` for zsh. + +> ⚠️ Warning: Unstable Builds +> The snapshot-latest tag points to the latest build from the master branch. It is unstable, unsupported, and intended only for early testing of new features. Please do not use it in production environments. + +## ⚙️ How It Works + +harmony_composer requires either cargo or docker to be available on your system to compile the Harmony project. + +- If cargo is found: It will be used to compile the project locally. +- If cargo is not found: It will automatically download and run the harmony_composer Docker image. This image is a self-contained build environment with the required Cargo binary and build targets for both Linux and Windows. +- If both cargo and docker are unavailable, `harmony_composer` will fail. Please install one of them. + +## 📖 Basic Usage + +Here are some common commands: + +```bash + +# Compile the repo's Harmony module +harmony_composer compile + +# Run check script on the project +harmony_composer check + +# Run the repo's entire harmony deployment sequence +harmony_composer deploy + +# Run the full check, compile, and deploy pipeline +harmony_composer all +``` + +For a full list of commands and their options, run: + +```bash + +harmony_composer --help +``` + +## 🏗️ Supported Architectures + +The build system currently supports compiling for: + + x86_64-unknown-linux-gnu + x86_64-pc-windows-gnu + +More target architectures are planned. If your platform is not yet supported, please open a feature request in the main repository. + +## 🔗 Main Project + +This tool is a small part of the main Harmony project. For complete documentation, contribution guidelines, and license information, please refer to the main repository.