Merge remote-tracking branch 'origin/master' into feat/cluster_monitoring

Reviewed-on: #180
Reviewed-by: Ian Letourneau <ian@noma.to>

Cargo.lock

@@ -1780,6 +1780,7 @@ dependencies = [
 name = "example-nanodc"
 version = "0.1.0"
 dependencies = [
+ "brocade",
  "cidr",
  "env_logger",
  "harmony",
@@ -1788,6 +1789,7 @@ dependencies = [
  "harmony_tui",
  "harmony_types",
  "log",
+ "serde",
  "tokio",
  "url",
 ]
@@ -1802,10 +1804,30 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "example-okd-cluster-alerts"
+version = "0.1.0"
+dependencies = [
+ "brocade",
+ "cidr",
+ "env_logger",
+ "harmony",
+ "harmony_cli",
+ "harmony_macros",
+ "harmony_secret",
+ "harmony_secret_derive",
+ "harmony_types",
+ "log",
+ "serde",
+ "tokio",
+ "url",
+]
+
 [[package]]
 name = "example-okd-install"
 version = "0.1.0"
 dependencies = [
+ "brocade",
  "cidr",
  "env_logger",
  "harmony",
@@ -1836,13 +1858,16 @@ dependencies = [
 name = "example-opnsense"
 version = "0.1.0"
 dependencies = [
+ "brocade",
  "cidr",
  "env_logger",
  "harmony",
  "harmony_macros",
+ "harmony_secret",
  "harmony_tui",
  "harmony_types",
  "log",
+ "serde",
  "tokio",
  "url",
 ]
@@ -1851,6 +1876,7 @@ dependencies = [
 name = "example-pxe"
 version = "0.1.0"
 dependencies = [
+ "brocade",
  "cidr",
  "env_logger",
  "harmony",
@@ -1865,6 +1891,15 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "example-remove-rook-osd"
+version = "0.1.0"
+dependencies = [
+ "harmony",
+ "harmony_cli",
+ "tokio",
+]
+
 [[package]]
 name = "example-rust"
 version = "0.1.0"
@@ -1918,8 +1953,6 @@ dependencies = [
  "env_logger",
  "harmony",
  "harmony_macros",
- "harmony_secret",
- "harmony_secret_derive",
  "harmony_tui",
  "harmony_types",
  "log",
@@ -4613,15 +4646,6 @@ version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001"
 
-[[package]]
-name = "remove_rook_osd"
-version = "0.1.0"
-dependencies = [
- "harmony",
- "harmony_cli",
- "tokio",
-]
-
 [[package]]
 name = "reqwest"
 version = "0.11.27"

brocade/src/fast_iron.rs

@@ -10,6 +10,7 @@ use log::{debug, info};
 use regex::Regex;
 use std::{collections::HashSet, str::FromStr};
 
+#[derive(Debug)]
 pub struct FastIronClient {
     shell: BrocadeShell,
     version: BrocadeInfo,

brocade/src/lib.rs

@@ -31,6 +31,7 @@ pub struct BrocadeOptions {
 pub struct TimeoutConfig {
     pub shell_ready: Duration,
     pub command_execution: Duration,
+    pub command_output: Duration,
     pub cleanup: Duration,
     pub message_wait: Duration,
 }
@@ -40,6 +41,7 @@ impl Default for TimeoutConfig {
         Self {
             shell_ready: Duration::from_secs(10),
             command_execution: Duration::from_secs(60), // Commands like `deploy` (for a LAG) can take a while
+            command_output: Duration::from_secs(5), // Delay to start logging "waiting for command output"
             cleanup: Duration::from_secs(10),
             message_wait: Duration::from_millis(500),
         }
@@ -162,7 +164,7 @@ pub async fn init(
 }
 
 #[async_trait]
-pub trait BrocadeClient {
+pub trait BrocadeClient: std::fmt::Debug {
     /// Retrieves the operating system and version details from the connected Brocade switch.
     ///
     /// This is typically the first call made after establishing a connection to determine

brocade/src/network_operating_system.rs

@@ -3,6 +3,7 @@ use std::str::FromStr;
 use async_trait::async_trait;
 use harmony_types::switch::{PortDeclaration, PortLocation};
 use log::{debug, info};
+use regex::Regex;
 
 use crate::{
     BrocadeClient, BrocadeInfo, Error, ExecutionMode, InterSwitchLink, InterfaceInfo,
@@ -10,6 +11,7 @@ use crate::{
     parse_brocade_mac_address, shell::BrocadeShell,
 };
 
+#[derive(Debug)]
 pub struct NetworkOperatingSystemClient {
     shell: BrocadeShell,
     version: BrocadeInfo,
@@ -102,13 +104,37 @@ impl NetworkOperatingSystemClient {
         };
 
         Some(Ok(InterfaceInfo {
-            name: format!("{} {}", interface_type, port_location),
+            name: format!("{interface_type} {port_location}"),
             port_location,
             interface_type,
             operating_mode,
             status,
         }))
     }
+
+    fn map_configure_interfaces_error(&self, err: Error) -> Error {
+        debug!("[Brocade] {err}");
+
+        if let Error::CommandError(message) = &err {
+            if message.contains("switchport")
+                && message.contains("Cannot configure aggregator member")
+            {
+                let re = Regex::new(r"\(conf-if-([a-zA-Z]+)-([\d/]+)\)#").unwrap();
+
+                if let Some(caps) = re.captures(message) {
+                    let interface_type = &caps[1];
+                    let port_location = &caps[2];
+                    let interface = format!("{interface_type} {port_location}");
+
+                    return Error::CommandError(format!(
+                        "Cannot configure interface '{interface}', it is a member of a port-channel (LAG)"
+                    ));
+                }
+            }
+        }
+
+        err
+    }
 }
 
 #[async_trait]
@@ -196,11 +222,10 @@ impl BrocadeClient for NetworkOperatingSystemClient {
             commands.push("exit".into());
         }
 
-        commands.push("write memory".into());
-
         self.shell
             .run_commands(commands, ExecutionMode::Regular)
-            .await?;
+            .await
+            .map_err(|err| self.map_configure_interfaces_error(err))?;
 
         info!("[Brocade] Interfaces configured.");
 
@@ -212,7 +237,7 @@ impl BrocadeClient for NetworkOperatingSystemClient {
 
         let output = self
             .shell
-            .run_command("show port-channel", ExecutionMode::Regular)
+            .run_command("show port-channel summary", ExecutionMode::Regular)
             .await?;
 
         let used_ids: Vec<u8> = output
@@ -247,7 +272,12 @@ impl BrocadeClient for NetworkOperatingSystemClient {
         ports: &[PortLocation],
     ) -> Result<(), Error> {
         info!(
-            "[Brocade] Configuring port-channel '{channel_name} {channel_id}' with ports: {ports:?}"
+            "[Brocade] Configuring port-channel '{channel_id} {channel_name}' with ports: {}",
+            ports
+                .iter()
+                .map(|p| format!("{p}"))
+                .collect::<Vec<String>>()
+                .join(", ")
         );
 
         let interfaces = self.get_interfaces().await?;
@@ -275,8 +305,6 @@ impl BrocadeClient for NetworkOperatingSystemClient {
             commands.push("exit".into());
         }
 
-        commands.push("write memory".into());
-
         self.shell
             .run_commands(commands, ExecutionMode::Regular)
             .await?;
@@ -293,7 +321,6 @@ impl BrocadeClient for NetworkOperatingSystemClient {
             "configure terminal".into(),
             format!("no interface port-channel {}", channel_name),
             "exit".into(),
-            "write memory".into(),
         ];
 
         self.shell

brocade/src/shell.rs

@@ -13,6 +13,7 @@ use log::info;
 use russh::ChannelMsg;
 use tokio::time::timeout;
 
+#[derive(Debug)]
 pub struct BrocadeShell {
     ip: IpAddr,
     port: u16,
@@ -210,7 +211,7 @@ impl BrocadeSession {
         let mut output = Vec::new();
         let start = Instant::now();
         let read_timeout = Duration::from_millis(500);
-        let log_interval = Duration::from_secs(3);
+        let log_interval = Duration::from_secs(5);
         let mut last_log = Instant::now();
 
         loop {
@@ -220,7 +221,9 @@ impl BrocadeSession {
                 ));
             }
 
-            if start.elapsed() > Duration::from_secs(5) && last_log.elapsed() > log_interval {
+            if start.elapsed() > self.options.timeouts.command_output
+                && last_log.elapsed() > log_interval
+            {
                 info!("[Brocade] Waiting for command output...");
                 last_log = Instant::now();
             }
@@ -275,7 +278,7 @@ impl BrocadeSession {
         let output_lower = output.to_lowercase();
         if ERROR_PATTERNS.iter().any(|&p| output_lower.contains(p)) {
             return Err(Error::CommandError(format!(
-                "Command '{command}' failed: {}",
+                "Command error: {}",
                 output.trim()
             )));
         }

docs/pxe_test/pxe_vm_lab_setup.sh

@@ -122,7 +122,6 @@ EOF
   fi
 
   echo "Creating OPNsense VM using serial image..."
-
   virt-install \
     --connect "${CONNECT_URI}" \
     --name "${VM_OPN}" \
@@ -132,12 +131,11 @@ EOF
     --os-variant "${OS_VARIANT_OPN}" \
     --graphics none \
     --noautoconsole \
-    --import \
-    --disk path="${OPN_IMG_PATH}",device=disk,bus=sata,boot.order=1 \
-    --disk path="${disk_opn}",device=disk,bus=virtio,boot.order=2 \
+    --disk path="${disk_opn}",device=disk,bus=virtio,boot.order=1 \
+    --disk path="${OPN_IMG_PATH}",device=disk,bus=usb,readonly=on,boot.order=2 \
     --network network=default,model=virtio \
     --network network="${NET_HARMONYLAN}",model=virtio \
-    --boot hd,menu=on
+    --boot uefi,menu=on
 
   echo "OPNsense VM created. Connect with: sudo virsh console ${VM_OPN}"
   echo "The VM will boot from the serial installation image."

examples/nanodc/Cargo.toml

@@ -17,3 +17,5 @@ harmony_secret = { path = "../../harmony_secret" }
 log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
+serde = { workspace = true }
+brocade = { path = "../../brocade" }

examples/nanodc/src/main.rs

@@ -3,12 +3,13 @@ use std::{
     sync::Arc,
 };
 
+use brocade::BrocadeOptions;
 use cidr::Ipv4Cidr;
 use harmony::{
     config::secret::SshKeyPair,
     data::{FileContent, FilePath},
     hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
-    infra::opnsense::OPNSenseManagementInterface,
+    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
     inventory::Inventory,
     modules::{
         http::StaticFilesHttpScore,
@@ -22,8 +23,9 @@ use harmony::{
     topology::{LogicalHost, UnmanagedRouter},
 };
 use harmony_macros::{ip, mac_address};
-use harmony_secret::SecretManager;
+use harmony_secret::{Secret, SecretManager};
 use harmony_types::net::Url;
+use serde::{Deserialize, Serialize};
 
 #[tokio::main]
 async fn main() {
@@ -32,6 +34,26 @@ async fn main() {
         name: String::from("fw0"),
     };
 
+    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+        .await
+        .expect("Failed to get credentials");
+
+    let switches: Vec<IpAddr> = vec![ip!("192.168.33.101")];
+    let brocade_options = Some(BrocadeOptions {
+        dry_run: *harmony::config::DRY_RUN,
+        ..Default::default()
+    });
+    let switch_client = BrocadeSwitchClient::init(
+        &switches,
+        &switch_auth.username,
+        &switch_auth.password,
+        brocade_options,
+    )
+    .await
+    .expect("Failed to connect to switch");
+
+    let switch_client = Arc::new(switch_client);
+
     let opnsense = Arc::new(
         harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
     );
@@ -39,6 +61,7 @@ async fn main() {
     let gateway_ipv4 = Ipv4Addr::new(192, 168, 33, 1);
     let gateway_ip = IpAddr::V4(gateway_ipv4);
     let topology = harmony::topology::HAClusterTopology {
+        kubeconfig: None,
         domain_name: "ncd0.harmony.mcd".to_string(), // TODO this must be set manually correctly
         // when setting up the opnsense firewall
         router: Arc::new(UnmanagedRouter::new(
@@ -83,7 +106,7 @@ async fn main() {
                 name: "wk2".to_string(),
             },
         ],
-        switch: vec![],
+        switch_client: switch_client.clone(),
     };
 
     let inventory = Inventory {
@@ -166,3 +189,9 @@ async fn main() {
     .await
     .unwrap();
 }
+
+#[derive(Secret, Serialize, Deserialize, Debug)]
+pub struct BrocadeSwitchAuth {
+    pub username: String,
+    pub password: String,
+}

examples/okd_cluster_alerts/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "example-okd-cluster-alerts"
+edition = "2024"
+version.workspace = true
+readme.workspace = true
+license.workspace = true
+publish = false
+
+[dependencies]
+harmony = { path = "../../harmony" }
+harmony_cli = { path = "../../harmony_cli" }
+harmony_types = { path = "../../harmony_types" }
+harmony_secret = { path = "../../harmony_secret" }
+harmony_secret_derive = { path = "../../harmony_secret_derive" }
+cidr = { workspace = true }
+tokio = { workspace = true }
+harmony_macros = { path = "../../harmony_macros" }
+log = { workspace = true }
+env_logger = { workspace = true }
+url = { workspace = true }
+serde.workspace = true
+brocade = { path = "../../brocade" }

examples/okd_cluster_alerts/src/main.rs

@@ -0,0 +1,26 @@
+use harmony::{
+    inventory::Inventory,
+    modules::monitoring::{
+        alert_channel::discord_alert_channel::DiscordWebhook,
+        okd::cluster_monitoring::OpenshiftClusterAlertScore,
+    },
+    topology::K8sAnywhereTopology,
+};
+use harmony_macros::hurl;
+
+#[tokio::main]
+async fn main() {
+    harmony_cli::run(
+        Inventory::autoload(),
+        K8sAnywhereTopology::from_env(),
+        vec![Box::new(OpenshiftClusterAlertScore {
+            receivers: vec![Box::new(DiscordWebhook {
+                name: "discord-webhook-example".to_string(),
+                url: hurl!("http://something.o"),
+            })],
+        })],
+        None,
+    )
+    .await
+    .unwrap();
+}

examples/okd_installation/Cargo.toml

@@ -19,3 +19,4 @@ log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
 serde.workspace = true
+brocade = { path = "../../brocade" }

examples/okd_installation/src/topology.rs

@@ -1,7 +1,8 @@
+use brocade::BrocadeOptions;
 use cidr::Ipv4Cidr;
 use harmony::{
     hardware::{Location, SwitchGroup},
-    infra::opnsense::OPNSenseManagementInterface,
+    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
     inventory::Inventory,
     topology::{HAClusterTopology, LogicalHost, UnmanagedRouter},
 };
@@ -22,6 +23,26 @@ pub async fn get_topology() -> HAClusterTopology {
         name: String::from("opnsense-1"),
     };
 
+    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+        .await
+        .expect("Failed to get credentials");
+
+    let switches: Vec<IpAddr> = vec![ip!("192.168.1.101")]; // TODO: Adjust me
+    let brocade_options = Some(BrocadeOptions {
+        dry_run: *harmony::config::DRY_RUN,
+        ..Default::default()
+    });
+    let switch_client = BrocadeSwitchClient::init(
+        &switches,
+        &switch_auth.username,
+        &switch_auth.password,
+        brocade_options,
+    )
+    .await
+    .expect("Failed to connect to switch");
+
+    let switch_client = Arc::new(switch_client);
+
     let config = SecretManager::get_or_prompt::<OPNSenseFirewallConfig>().await;
     let config = config.unwrap();
 
@@ -38,6 +59,7 @@ pub async fn get_topology() -> HAClusterTopology {
     let gateway_ipv4 = ipv4!("192.168.1.1");
     let gateway_ip = IpAddr::V4(gateway_ipv4);
     harmony::topology::HAClusterTopology {
+        kubeconfig: None,
         domain_name: "demo.harmony.mcd".to_string(),
         router: Arc::new(UnmanagedRouter::new(
             gateway_ip,
@@ -58,7 +80,7 @@ pub async fn get_topology() -> HAClusterTopology {
             name: "bootstrap".to_string(),
         },
         workers: vec![],
-        switch: vec![],
+        switch_client: switch_client.clone(),
     }
 }
 
@@ -75,3 +97,9 @@ pub fn get_inventory() -> Inventory {
         control_plane_host: vec![],
     }
 }
+
+#[derive(Secret, Serialize, Deserialize, Debug)]
+pub struct BrocadeSwitchAuth {
+    pub username: String,
+    pub password: String,
+}

examples/okd_pxe/Cargo.toml

@@ -19,3 +19,4 @@ log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
 serde.workspace = true
+brocade = { path = "../../brocade" }

examples/okd_pxe/src/topology.rs

@@ -1,13 +1,15 @@
+use brocade::BrocadeOptions;
 use cidr::Ipv4Cidr;
 use harmony::{
     config::secret::OPNSenseFirewallCredentials,
     hardware::{Location, SwitchGroup},
-    infra::opnsense::OPNSenseManagementInterface,
+    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
     inventory::Inventory,
     topology::{HAClusterTopology, LogicalHost, UnmanagedRouter},
 };
 use harmony_macros::{ip, ipv4};
-use harmony_secret::SecretManager;
+use harmony_secret::{Secret, SecretManager};
+use serde::{Deserialize, Serialize};
 use std::{net::IpAddr, sync::Arc};
 
 pub async fn get_topology() -> HAClusterTopology {
@@ -16,6 +18,26 @@ pub async fn get_topology() -> HAClusterTopology {
         name: String::from("opnsense-1"),
     };
 
+    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+        .await
+        .expect("Failed to get credentials");
+
+    let switches: Vec<IpAddr> = vec![ip!("192.168.1.101")]; // TODO: Adjust me
+    let brocade_options = Some(BrocadeOptions {
+        dry_run: *harmony::config::DRY_RUN,
+        ..Default::default()
+    });
+    let switch_client = BrocadeSwitchClient::init(
+        &switches,
+        &switch_auth.username,
+        &switch_auth.password,
+        brocade_options,
+    )
+    .await
+    .expect("Failed to connect to switch");
+
+    let switch_client = Arc::new(switch_client);
+
     let config = SecretManager::get_or_prompt::<OPNSenseFirewallCredentials>().await;
     let config = config.unwrap();
 
@@ -32,6 +54,7 @@ pub async fn get_topology() -> HAClusterTopology {
     let gateway_ipv4 = ipv4!("192.168.1.1");
     let gateway_ip = IpAddr::V4(gateway_ipv4);
     harmony::topology::HAClusterTopology {
+        kubeconfig: None,
         domain_name: "demo.harmony.mcd".to_string(),
         router: Arc::new(UnmanagedRouter::new(
             gateway_ip,
@@ -52,7 +75,7 @@ pub async fn get_topology() -> HAClusterTopology {
             name: "cp0".to_string(),
         },
         workers: vec![],
-        switch: vec![],
+        switch_client: switch_client.clone(),
     }
 }
 
@@ -69,3 +92,9 @@ pub fn get_inventory() -> Inventory {
         control_plane_host: vec![],
     }
 }
+
+#[derive(Secret, Serialize, Deserialize, Debug)]
+pub struct BrocadeSwitchAuth {
+    pub username: String,
+    pub password: String,
+}

examples/opnsense/Cargo.toml

@@ -16,3 +16,6 @@ harmony_macros = { path = "../../harmony_macros" }
 log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
+harmony_secret = { path = "../../harmony_secret" }
+brocade = { path = "../../brocade" }
+serde = { workspace = true }

examples/opnsense/src/main.rs

@@ -3,10 +3,11 @@ use std::{
     sync::Arc,
 };
 
+use brocade::BrocadeOptions;
 use cidr::Ipv4Cidr;
 use harmony::{
     hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
-    infra::opnsense::OPNSenseManagementInterface,
+    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
     inventory::Inventory,
     modules::{
         dummy::{ErrorScore, PanicScore, SuccessScore},
@@ -18,7 +19,9 @@ use harmony::{
     topology::{LogicalHost, UnmanagedRouter},
 };
 use harmony_macros::{ip, mac_address};
+use harmony_secret::{Secret, SecretManager};
 use harmony_types::net::Url;
+use serde::{Deserialize, Serialize};
 
 #[tokio::main]
 async fn main() {
@@ -27,6 +30,26 @@ async fn main() {
         name: String::from("opnsense-1"),
     };
 
+    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+        .await
+        .expect("Failed to get credentials");
+
+    let switches: Vec<IpAddr> = vec![ip!("192.168.5.101")]; // TODO: Adjust me
+    let brocade_options = Some(BrocadeOptions {
+        dry_run: *harmony::config::DRY_RUN,
+        ..Default::default()
+    });
+    let switch_client = BrocadeSwitchClient::init(
+        &switches,
+        &switch_auth.username,
+        &switch_auth.password,
+        brocade_options,
+    )
+    .await
+    .expect("Failed to connect to switch");
+
+    let switch_client = Arc::new(switch_client);
+
     let opnsense = Arc::new(
         harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
     );
@@ -34,6 +57,7 @@ async fn main() {
     let gateway_ipv4 = Ipv4Addr::new(10, 100, 8, 1);
     let gateway_ip = IpAddr::V4(gateway_ipv4);
     let topology = harmony::topology::HAClusterTopology {
+        kubeconfig: None,
         domain_name: "demo.harmony.mcd".to_string(),
         router: Arc::new(UnmanagedRouter::new(
             gateway_ip,
@@ -54,7 +78,7 @@ async fn main() {
             name: "cp0".to_string(),
         },
         workers: vec![],
-        switch: vec![],
+        switch_client: switch_client.clone(),
     };
 
     let inventory = Inventory {
@@ -109,3 +133,9 @@ async fn main() {
     .await
     .unwrap();
 }
+
+#[derive(Secret, Serialize, Deserialize, Debug)]
+pub struct BrocadeSwitchAuth {
+    pub username: String,
+    pub password: String,
+}

examples/try_rust_webapp/src/main.rs

@@ -3,7 +3,7 @@ use harmony::{
     modules::{
         application::{
             ApplicationScore, RustWebFramework, RustWebapp,
-            features::{PackagingDeployment, rhob_monitoring::Monitoring},
+            features::{Monitoring, PackagingDeployment},
         },
         monitoring::alert_channel::discord_alert_channel::DiscordWebhook,
     },

harmony/src/domain/topology/ha_cluster.rs

@@ -1,26 +1,19 @@
 use async_trait::async_trait;
-use brocade::BrocadeOptions;
 use harmony_macros::ip;
-use harmony_secret::SecretManager;
 use harmony_types::{
     net::{MacAddress, Url},
     switch::PortLocation,
 };
-use k8s_openapi::api::core::v1::Namespace;
 use kube::api::ObjectMeta;
 use log::debug;
 use log::info;
 
-use crate::data::FileContent;
-use crate::executors::ExecutorError;
-use crate::hardware::PhysicalHost;
-use crate::infra::brocade::BrocadeSwitchAuth;
-use crate::infra::brocade::BrocadeSwitchClient;
-use crate::modules::okd::crd::{
-    InstallPlanApproval, OperatorGroup, OperatorGroupSpec, Subscription, SubscriptionSpec,
-    nmstate::{self, NMState, NodeNetworkConfigurationPolicy, NodeNetworkConfigurationPolicySpec},
-};
+use crate::modules::okd::crd::nmstate::{self, NodeNetworkConfigurationPolicy};
 use crate::topology::PxeOptions;
+use crate::{data::FileContent, modules::okd::crd::nmstate::NMState};
+use crate::{
+    executors::ExecutorError, modules::okd::crd::nmstate::NodeNetworkConfigurationPolicySpec,
+};
 
 use super::{
     DHCPStaticEntry, DhcpServer, DnsRecord, DnsRecordType, DnsServer, Firewall, HostNetworkConfig,
@@ -30,7 +23,6 @@ use super::{
 };
 
 use std::collections::BTreeMap;
-use std::net::IpAddr;
 use std::sync::Arc;
 
 #[derive(Debug, Clone)]
@@ -43,10 +35,11 @@ pub struct HAClusterTopology {
     pub tftp_server: Arc<dyn TftpServer>,
     pub http_server: Arc<dyn HttpServer>,
     pub dns_server: Arc<dyn DnsServer>,
+    pub switch_client: Arc<dyn SwitchClient>,
     pub bootstrap_host: LogicalHost,
     pub control_plane: Vec<LogicalHost>,
     pub workers: Vec<LogicalHost>,
-    pub switch: Vec<LogicalHost>,
+    pub kubeconfig: Option<String>,
 }
 
 #[async_trait]
@@ -65,9 +58,17 @@ impl Topology for HAClusterTopology {
 #[async_trait]
 impl K8sclient for HAClusterTopology {
     async fn k8s_client(&self) -> Result<Arc<K8sClient>, String> {
-        Ok(Arc::new(
-            K8sClient::try_default().await.map_err(|e| e.to_string())?,
-        ))
+        match &self.kubeconfig {
+            None => Ok(Arc::new(
+                K8sClient::try_default().await.map_err(|e| e.to_string())?,
+            )),
+            Some(kubeconfig) => {
+                let Some(client) = K8sClient::from_kubeconfig(&kubeconfig).await else {
+                    return Err("Failed to create k8s client".to_string());
+                };
+                Ok(Arc::new(client))
+            }
+        }
     }
 }
 
@@ -93,60 +94,48 @@ impl HAClusterTopology {
     }
 
     async fn ensure_nmstate_operator_installed(&self) -> Result<(), String> {
-        // FIXME: Find a way to check nmstate is already available (get pod -n openshift-nmstate)
-        debug!("Installing NMState operator...");
         let k8s_client = self.k8s_client().await?;
 
-        let nmstate_namespace = Namespace {
-            metadata: ObjectMeta {
-                name: Some("openshift-nmstate".to_string()),
-                finalizers: Some(vec!["kubernetes".to_string()]),
-                ..Default::default()
-            },
-            ..Default::default()
-        };
-        debug!("Creating NMState namespace: {nmstate_namespace:#?}");
-        k8s_client
-            .apply(&nmstate_namespace, None)
+        debug!("Installing NMState controller...");
+        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/nmstate.io_nmstates.yaml
+").unwrap(), Some("nmstate"))
             .await
             .map_err(|e| e.to_string())?;
 
-        let nmstate_operator_group = OperatorGroup {
-            metadata: ObjectMeta {
-                name: Some("openshift-nmstate".to_string()),
-                namespace: Some("openshift-nmstate".to_string()),
-                ..Default::default()
-            },
-            spec: OperatorGroupSpec {
-                target_namespaces: vec!["openshift-nmstate".to_string()],
-            },
-        };
-        debug!("Creating NMState operator group: {nmstate_operator_group:#?}");
-        k8s_client
-            .apply(&nmstate_operator_group, None)
+        debug!("Creating NMState namespace...");
+        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/namespace.yaml
+").unwrap(), Some("nmstate"))
             .await
             .map_err(|e| e.to_string())?;
 
-        let nmstate_subscription = Subscription {
-            metadata: ObjectMeta {
-                name: Some("kubernetes-nmstate-operator".to_string()),
-                namespace: Some("openshift-nmstate".to_string()),
-                ..Default::default()
-            },
-            spec: SubscriptionSpec {
-                channel: Some("stable".to_string()),
-                install_plan_approval: Some(InstallPlanApproval::Automatic),
-                name: "kubernetes-nmstate-operator".to_string(),
-                source: "redhat-operators".to_string(),
-                source_namespace: "openshift-marketplace".to_string(),
-            },
-        };
-        debug!("Subscribing to NMState Operator: {nmstate_subscription:#?}");
-        k8s_client
-            .apply(&nmstate_subscription, None)
+        debug!("Creating NMState service account...");
+        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/service_account.yaml
+").unwrap(), Some("nmstate"))
             .await
             .map_err(|e| e.to_string())?;
 
+        debug!("Creating NMState role...");
+        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/role.yaml
+").unwrap(), Some("nmstate"))
+            .await
+            .map_err(|e| e.to_string())?;
+
+        debug!("Creating NMState role binding...");
+        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/role_binding.yaml
+").unwrap(), Some("nmstate"))
+            .await
+            .map_err(|e| e.to_string())?;
+
+        debug!("Creating NMState operator...");
+        k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/operator.yaml
+").unwrap(), Some("nmstate"))
+            .await
+            .map_err(|e| e.to_string())?;
+
+        k8s_client
+            .wait_until_deployment_ready("nmstate-operator", Some("nmstate"), None)
+            .await?;
+
         let nmstate = NMState {
             metadata: ObjectMeta {
                 name: Some("nmstate".to_string()),
@@ -167,11 +156,7 @@ impl HAClusterTopology {
         42 // FIXME: Find a better way to declare the bond id
     }
 
-    async fn configure_bond(
-        &self,
-        host: &PhysicalHost,
-        config: &HostNetworkConfig,
-    ) -> Result<(), SwitchError> {
+    async fn configure_bond(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
         self.ensure_nmstate_operator_installed()
             .await
             .map_err(|e| {
@@ -180,29 +165,33 @@ impl HAClusterTopology {
                 ))
             })?;
 
-        let bond_config = self.create_bond_configuration(host, config);
-        debug!("Configuring bond for host {host:?}: {bond_config:#?}");
+        let bond_config = self.create_bond_configuration(config);
+        debug!(
+            "Applying NMState bond config for host {}: {bond_config:#?}",
+            config.host_id
+        );
         self.k8s_client()
             .await
             .unwrap()
             .apply(&bond_config, None)
             .await
-            .unwrap();
+            .map_err(|e| SwitchError::new(format!("Failed to configure bond: {e}")))?;
 
-        todo!()
+        Ok(())
     }
 
     fn create_bond_configuration(
         &self,
-        host: &PhysicalHost,
         config: &HostNetworkConfig,
     ) -> NodeNetworkConfigurationPolicy {
-        let host_name = host.id.clone();
-
+        let host_name = &config.host_id;
         let bond_id = self.get_next_bond_id();
         let bond_name = format!("bond{bond_id}");
+
+        info!("Configuring bond '{bond_name}' for host '{host_name}'...");
+
         let mut bond_mtu: Option<u32> = None;
-        let mut bond_mac_address: Option<String> = None;
+        let mut copy_mac_from: Option<String> = None;
         let mut bond_ports = Vec::new();
         let mut interfaces: Vec<nmstate::InterfaceSpec> = Vec::new();
 
@@ -228,14 +217,14 @@ impl HAClusterTopology {
                 ..Default::default()
             });
 
-            bond_ports.push(interface_name);
+            bond_ports.push(interface_name.clone());
 
             // Use the first port's details for the bond mtu and mac address
             if bond_mtu.is_none() {
                 bond_mtu = Some(switch_port.interface.mtu);
             }
-            if bond_mac_address.is_none() {
-                bond_mac_address = Some(switch_port.interface.mac_address.to_string());
+            if copy_mac_from.is_none() {
+                copy_mac_from = Some(interface_name);
             }
         }
 
@@ -244,8 +233,7 @@ impl HAClusterTopology {
             description: Some(format!("Network bond for host {host_name}")),
             r#type: "bond".to_string(),
             state: "up".to_string(),
-            mtu: bond_mtu,
-            mac_address: bond_mac_address,
+            copy_mac_from,
             ipv4: Some(nmstate::IpStackSpec {
                 dhcp: Some(true),
                 enabled: Some(true),
@@ -280,37 +268,12 @@ impl HAClusterTopology {
         }
     }
 
-    async fn get_switch_client(&self) -> Result<Box<dyn SwitchClient>, SwitchError> {
-        let auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
-            .await
-            .map_err(|e| SwitchError::new(format!("Failed to get credentials: {e}")))?;
-
-        // FIXME: We assume Brocade switches
-        let switches: Vec<IpAddr> = self.switch.iter().map(|s| s.ip).collect();
-        let brocade_options = Some(BrocadeOptions {
-            dry_run: *crate::config::DRY_RUN,
-            ..Default::default()
-        });
-        let client =
-            BrocadeSwitchClient::init(&switches, &auth.username, &auth.password, brocade_options)
-                .await
-                .map_err(|e| SwitchError::new(format!("Failed to connect to switch: {e}")))?;
-
-        Ok(Box::new(client))
-    }
-
-    async fn configure_port_channel(
-        &self,
-        host: &PhysicalHost,
-        config: &HostNetworkConfig,
-    ) -> Result<(), SwitchError> {
+    async fn configure_port_channel(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
         debug!("Configuring port channel: {config:#?}");
-        let client = self.get_switch_client().await?;
-
         let switch_ports = config.switch_ports.iter().map(|s| s.port.clone()).collect();
 
-        client
-            .configure_port_channel(&format!("Harmony_{}", host.id), switch_ports)
+        self.switch_client
+            .configure_port_channel(&format!("Harmony_{}", config.host_id), switch_ports)
             .await
             .map_err(|e| SwitchError::new(format!("Failed to configure switch: {e}")))?;
 
@@ -325,6 +288,7 @@ impl HAClusterTopology {
         };
 
         Self {
+            kubeconfig: None,
             domain_name: "DummyTopology".to_string(),
             router: dummy_infra.clone(),
             load_balancer: dummy_infra.clone(),
@@ -333,10 +297,10 @@ impl HAClusterTopology {
             tftp_server: dummy_infra.clone(),
             http_server: dummy_infra.clone(),
             dns_server: dummy_infra.clone(),
+            switch_client: dummy_infra.clone(),
             bootstrap_host: dummy_host,
             control_plane: vec![],
             workers: vec![],
-            switch: vec![],
         }
     }
 }
@@ -494,8 +458,7 @@ impl HttpServer for HAClusterTopology {
 #[async_trait]
 impl Switch for HAClusterTopology {
     async fn setup_switch(&self) -> Result<(), SwitchError> {
-        let client = self.get_switch_client().await?;
-        client.setup().await?;
+        self.switch_client.setup().await?;
         Ok(())
     }
 
@@ -503,18 +466,13 @@ impl Switch for HAClusterTopology {
         &self,
         mac_address: &MacAddress,
     ) -> Result<Option<PortLocation>, SwitchError> {
-        let client = self.get_switch_client().await?;
-        let port = client.find_port(mac_address).await?;
+        let port = self.switch_client.find_port(mac_address).await?;
         Ok(port)
     }
 
-    async fn configure_host_network(
-        &self,
-        host: &PhysicalHost,
-        config: HostNetworkConfig,
-    ) -> Result<(), SwitchError> {
-        self.configure_bond(host, &config).await?;
-        self.configure_port_channel(host, &config).await
+    async fn configure_host_network(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> {
+        self.configure_bond(config).await?;
+        self.configure_port_channel(config).await
     }
 }
 
@@ -704,3 +662,25 @@ impl DnsServer for DummyInfra {
         unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
     }
 }
+
+#[async_trait]
+impl SwitchClient for DummyInfra {
+    async fn setup(&self) -> Result<(), SwitchError> {
+        unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
+    }
+
+    async fn find_port(
+        &self,
+        _mac_address: &MacAddress,
+    ) -> Result<Option<PortLocation>, SwitchError> {
+        unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
+    }
+
+    async fn configure_port_channel(
+        &self,
+        _channel_name: &str,
+        _switch_ports: Vec<PortLocation>,
+    ) -> Result<u8, SwitchError> {
+        unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
+    }
+}

harmony/src/domain/topology/k8s.rs

@@ -3,14 +3,18 @@ use std::time::Duration;
 use derive_new::new;
 use k8s_openapi::{
     ClusterResourceScope, NamespaceResourceScope,
-    api::{apps::v1::Deployment, core::v1::Pod},
+    api::{
+        apps::v1::Deployment,
+        core::v1::{Pod, ServiceAccount},
+    },
     apimachinery::pkg::version::Info,
 };
 use kube::{
     Client, Config, Discovery, Error, Resource,
     api::{Api, AttachParams, DeleteParams, ListParams, Patch, PatchParams, ResourceExt},
     config::{KubeConfigOptions, Kubeconfig},
-    core::ErrorResponse,
+    core::{DynamicResourceScope, ErrorResponse},
+    discovery::{ApiCapabilities, Scope},
     error::DiscoveryError,
     runtime::reflector::Lookup,
 };
@@ -19,11 +23,12 @@ use kube::{
     api::{ApiResource, GroupVersionKind},
     runtime::wait::await_condition,
 };
-use log::{debug, error, info, trace};
+use log::{debug, error, info, trace, warn};
 use serde::{Serialize, de::DeserializeOwned};
-use serde_json::{Value, json};
+use serde_json::json;
 use similar::TextDiff;
 use tokio::{io::AsyncReadExt, time::sleep};
+use url::Url;
 
 #[derive(new, Clone)]
 pub struct K8sClient {
@@ -57,6 +62,11 @@ impl K8sClient {
         })
     }
 
+    pub async fn service_account_api(&self, namespace: &str) -> Api<ServiceAccount> {
+        let api: Api<ServiceAccount> = Api::namespaced(self.client.clone(), namespace);
+        api
+    }
+
     pub async fn get_apiserver_version(&self) -> Result<Info, Error> {
         let client: Client = self.client.clone();
         let version_info: Info = client.apiserver_version().await?;
@@ -80,7 +90,25 @@ impl K8sClient {
         } else {
             Api::default_namespaced_with(self.client.clone(), &gvk)
         };
-        Ok(resource.get(name).await?)
+
+        resource.get(name).await
+    }
+
+    pub async fn get_secret_json_value(
+        &self,
+        name: &str,
+        namespace: Option<&str>,
+    ) -> Result<DynamicObject, Error> {
+        self.get_resource_json_value(
+            name,
+            namespace,
+            &GroupVersionKind {
+                group: "".to_string(),
+                version: "v1".to_string(),
+                kind: "Secret".to_string(),
+            },
+        )
+        .await
     }
 
     pub async fn get_deployment(
@@ -95,8 +123,9 @@ impl K8sClient {
             debug!("getting default namespace deployment");
             Api::default_namespaced(self.client.clone())
         };
+
         debug!("getting deployment {} in ns {}", name, namespace.unwrap());
-        Ok(deps.get_opt(name).await?)
+        deps.get_opt(name).await
     }
 
     pub async fn get_pod(&self, name: &str, namespace: Option<&str>) -> Result<Option<Pod>, Error> {
@@ -105,7 +134,8 @@ impl K8sClient {
         } else {
             Api::default_namespaced(self.client.clone())
         };
-        Ok(pods.get_opt(name).await?)
+
+        pods.get_opt(name).await
     }
 
     pub async fn scale_deployment(
@@ -148,9 +178,9 @@ impl K8sClient {
 
     pub async fn wait_until_deployment_ready(
         &self,
-        name: String,
+        name: &str,
         namespace: Option<&str>,
-        timeout: Option<u64>,
+        timeout: Option<Duration>,
     ) -> Result<(), String> {
         let api: Api<Deployment>;
 
@@ -160,9 +190,9 @@ impl K8sClient {
             api = Api::default_namespaced(self.client.clone());
         }
 
-        let establish = await_condition(api, name.as_str(), conditions::is_deployment_completed());
-        let t = timeout.unwrap_or(300);
-        let res = tokio::time::timeout(std::time::Duration::from_secs(t), establish).await;
+        let establish = await_condition(api, name, conditions::is_deployment_completed());
+        let timeout = timeout.unwrap_or(Duration::from_secs(120));
+        let res = tokio::time::timeout(timeout, establish).await;
 
         if res.is_ok() {
             Ok(())
@@ -252,7 +282,7 @@ impl K8sClient {
 
                 if let Some(s) = status.status {
                     let mut stdout_buf = String::new();
-                    if let Some(mut stdout) = process.stdout().take() {
+                    if let Some(mut stdout) = process.stdout() {
                         stdout
                             .read_to_string(&mut stdout_buf)
                             .await
@@ -324,6 +354,169 @@ impl K8sClient {
         }
     }
 
+    fn get_api_for_dynamic_object(
+        &self,
+        object: &DynamicObject,
+        ns: Option<&str>,
+    ) -> Result<Api<DynamicObject>, Error> {
+        let api_resource = object
+            .types
+            .as_ref()
+            .and_then(|t| {
+                let parts: Vec<&str> = t.api_version.split('/').collect();
+                match parts.as_slice() {
+                    [version] => Some(ApiResource::from_gvk(&GroupVersionKind::gvk(
+                        "", version, &t.kind,
+                    ))),
+                    [group, version] => Some(ApiResource::from_gvk(&GroupVersionKind::gvk(
+                        group, version, &t.kind,
+                    ))),
+                    _ => None,
+                }
+            })
+            .ok_or_else(|| {
+                Error::BuildRequest(kube::core::request::Error::Validation(
+                    "Invalid apiVersion in DynamicObject {object:#?}".to_string(),
+                ))
+            })?;
+
+        match ns {
+            Some(ns) => Ok(Api::namespaced_with(self.client.clone(), ns, &api_resource)),
+            None => Ok(Api::default_namespaced_with(
+                self.client.clone(),
+                &api_resource,
+            )),
+        }
+    }
+
+    pub async fn apply_dynamic_many(
+        &self,
+        resource: &[DynamicObject],
+        namespace: Option<&str>,
+        force_conflicts: bool,
+    ) -> Result<Vec<DynamicObject>, Error> {
+        let mut result = Vec::new();
+        for r in resource.iter() {
+            result.push(self.apply_dynamic(r, namespace, force_conflicts).await?);
+        }
+
+        Ok(result)
+    }
+
+    /// Apply DynamicObject resource to the cluster
+    pub async fn apply_dynamic(
+        &self,
+        resource: &DynamicObject,
+        namespace: Option<&str>,
+        force_conflicts: bool,
+    ) -> Result<DynamicObject, Error> {
+        // Build API for this dynamic object
+        let api = self.get_api_for_dynamic_object(resource, namespace)?;
+        let name = resource
+            .metadata
+            .name
+            .as_ref()
+            .ok_or_else(|| {
+                Error::BuildRequest(kube::core::request::Error::Validation(
+                    "DynamicObject must have metadata.name".to_string(),
+                ))
+            })?
+            .as_str();
+
+        debug!(
+            "Applying dynamic resource kind={:?} apiVersion={:?} name='{}' ns={:?}",
+            resource.types.as_ref().map(|t| &t.kind),
+            resource.types.as_ref().map(|t| &t.api_version),
+            name,
+            namespace
+        );
+        trace!(
+            "Dynamic resource payload:\n{:#}",
+            serde_json::to_value(resource).unwrap_or(serde_json::Value::Null)
+        );
+
+        // Using same field manager as in apply()
+        let mut patch_params = PatchParams::apply("harmony");
+        patch_params.force = force_conflicts;
+
+        if *crate::config::DRY_RUN {
+            // Dry-run path: fetch current, show diff, and return appropriate object
+            match api.get(name).await {
+                Ok(current) => {
+                    trace!("Received current dynamic value {current:#?}");
+
+                    println!("\nPerforming dry-run for resource: '{}'", name);
+
+                    // Serialize current and new, and strip status from current if present
+                    let mut current_yaml =
+                        serde_yaml::to_value(&current).unwrap_or_else(|_| serde_yaml::Value::Null);
+                    if let Some(map) = current_yaml.as_mapping_mut() {
+                        if map.contains_key(&serde_yaml::Value::String("status".to_string())) {
+                            let removed =
+                                map.remove(&serde_yaml::Value::String("status".to_string()));
+                            trace!("Removed status from current dynamic object: {:?}", removed);
+                        } else {
+                            trace!(
+                                "Did not find status entry for current dynamic object {}/{}",
+                                current.metadata.namespace.as_deref().unwrap_or(""),
+                                current.metadata.name.as_deref().unwrap_or("")
+                            );
+                        }
+                    }
+
+                    let current_yaml = serde_yaml::to_string(&current_yaml)
+                        .unwrap_or_else(|_| "Failed to serialize current resource".to_string());
+                    let new_yaml = serde_yaml::to_string(resource)
+                        .unwrap_or_else(|_| "Failed to serialize new resource".to_string());
+
+                    if current_yaml == new_yaml {
+                        println!("No changes detected.");
+                        return Ok(current);
+                    }
+
+                    println!("Changes detected:");
+                    let diff = TextDiff::from_lines(&current_yaml, &new_yaml);
+                    for change in diff.iter_all_changes() {
+                        let sign = match change.tag() {
+                            similar::ChangeTag::Delete => "-",
+                            similar::ChangeTag::Insert => "+",
+                            similar::ChangeTag::Equal => " ",
+                        };
+                        print!("{}{}", sign, change);
+                    }
+
+                    // Return the incoming resource as the would-be applied state
+                    Ok(resource.clone())
+                }
+                Err(Error::Api(ErrorResponse { code: 404, .. })) => {
+                    println!("\nPerforming dry-run for new resource: '{}'", name);
+                    println!(
+                        "Resource does not exist. It would be created with the following content:"
+                    );
+                    let new_yaml = serde_yaml::to_string(resource)
+                        .unwrap_or_else(|_| "Failed to serialize new resource".to_string());
+                    for line in new_yaml.lines() {
+                        println!("+{}", line);
+                    }
+                    Ok(resource.clone())
+                }
+                Err(e) => {
+                    error!("Failed to get dynamic resource '{}': {}", name, e);
+                    Err(e)
+                }
+            }
+        } else {
+            // Real apply via server-side apply
+            debug!("Patching (server-side apply) dynamic resource '{}'", name);
+            api.patch(name, &patch_params, &Patch::Apply(resource))
+                .await
+                .map_err(|e| {
+                    error!("Failed to apply dynamic resource '{}': {}", name, e);
+                    e
+                })
+        }
+    }
+
     /// Apply a resource in namespace
     ///
     /// See `kubectl apply` for more information on the expected behavior of this function
@@ -358,14 +551,14 @@ impl K8sClient {
                 Ok(current) => {
                     trace!("Received current value {current:#?}");
                     // The resource exists, so we calculate and display a diff.
-                    println!("\nPerforming dry-run for resource: '{}'", name);
+                    println!("\nPerforming dry-run for resource: '{name}'");
                     let mut current_yaml = serde_yaml::to_value(&current).unwrap_or_else(|_| {
                         panic!("Could not serialize current value : {current:#?}")
                     });
                     if current_yaml.is_mapping() && current_yaml.get("status").is_some() {
                         let map = current_yaml.as_mapping_mut().unwrap();
                         let removed = map.remove_entry("status");
-                        trace!("Removed status {:?}", removed);
+                        trace!("Removed status {removed:?}");
                     } else {
                         trace!(
                             "Did not find status entry for current object {}/{}",
@@ -394,14 +587,14 @@ impl K8sClient {
                             similar::ChangeTag::Insert => "+",
                             similar::ChangeTag::Equal => " ",
                         };
-                        print!("{}{}", sign, change);
+                        print!("{sign}{change}");
                     }
                     // In a dry run, we return the new resource state that would have been applied.
                     Ok(resource.clone())
                 }
                 Err(Error::Api(ErrorResponse { code: 404, .. })) => {
                     // The resource does not exist, so the "diff" is the entire new resource.
-                    println!("\nPerforming dry-run for new resource: '{}'", name);
+                    println!("\nPerforming dry-run for new resource: '{name}'");
                     println!(
                         "Resource does not exist. It would be created with the following content:"
                     );
@@ -410,14 +603,14 @@ impl K8sClient {
 
                     // Print each line of the new resource with a '+' prefix.
                     for line in new_yaml.lines() {
-                        println!("+{}", line);
+                        println!("+{line}");
                     }
                     // In a dry run, we return the new resource state that would have been created.
                     Ok(resource.clone())
                 }
                 Err(e) => {
                     // Another API error occurred.
-                    error!("Failed to get resource '{}': {}", name, e);
+                    error!("Failed to get resource '{name}': {e}");
                     Err(e)
                 }
             }
@@ -432,7 +625,7 @@ impl K8sClient {
     where
         K: Resource + Clone + std::fmt::Debug + DeserializeOwned + serde::Serialize,
         <K as Resource>::Scope: ApplyStrategy<K>,
-        <K as kube::Resource>::DynamicType: Default,
+        <K as Resource>::DynamicType: Default,
     {
         let mut result = Vec::new();
         for r in resource.iter() {
@@ -497,10 +690,7 @@ impl K8sClient {
 
         // 6. Apply the object to the cluster using Server-Side Apply.
         //    This will create the resource if it doesn't exist, or update it if it does.
-        println!(
-            "Applying Argo Application '{}' in namespace '{}'...",
-            name, namespace
-        );
+        println!("Applying '{name}' in namespace '{namespace}'...",);
         let patch_params = PatchParams::apply("harmony"); // Use a unique field manager name
         let result = api.patch(name, &patch_params, &Patch::Apply(&obj)).await?;
 
@@ -509,6 +699,51 @@ impl K8sClient {
         Ok(())
     }
 
+    /// Apply a resource from a URL
+    ///
+    /// It is the equivalent of `kubectl apply -f <url>`
+    pub async fn apply_url(&self, url: Url, ns: Option<&str>) -> Result<(), Error> {
+        let patch_params = PatchParams::apply("harmony");
+        let discovery = kube::Discovery::new(self.client.clone()).run().await?;
+
+        let yaml = reqwest::get(url)
+            .await
+            .expect("Could not get URL")
+            .text()
+            .await
+            .expect("Could not get content from URL");
+
+        for doc in multidoc_deserialize(&yaml).expect("failed to parse YAML from file") {
+            let obj: DynamicObject =
+                serde_yaml::from_value(doc).expect("cannot apply without valid YAML");
+            let namespace = obj.metadata.namespace.as_deref().or(ns);
+            let type_meta = obj
+                .types
+                .as_ref()
+                .expect("cannot apply object without valid TypeMeta");
+            let gvk = GroupVersionKind::try_from(type_meta)
+                .expect("cannot apply object without valid GroupVersionKind");
+            let name = obj.name_any();
+
+            if let Some((ar, caps)) = discovery.resolve_gvk(&gvk) {
+                let api = get_dynamic_api(ar, caps, self.client.clone(), namespace, false);
+                trace!(
+                    "Applying {}: \n{}",
+                    gvk.kind,
+                    serde_yaml::to_string(&obj).expect("Failed to serialize YAML")
+                );
+                let data: serde_json::Value =
+                    serde_json::to_value(&obj).expect("Failed to serialize JSON");
+                let _r = api.patch(&name, &patch_params, &Patch::Apply(data)).await?;
+                debug!("applied {} {}", gvk.kind, name);
+            } else {
+                warn!("Cannot apply document for unknown {gvk:?}");
+            }
+        }
+
+        Ok(())
+    }
+
     pub(crate) async fn from_kubeconfig(path: &str) -> Option<K8sClient> {
         let k = match Kubeconfig::read_from(path) {
             Ok(k) => k,
@@ -528,6 +763,31 @@ impl K8sClient {
     }
 }
 
+fn get_dynamic_api(
+    resource: ApiResource,
+    capabilities: ApiCapabilities,
+    client: Client,
+    ns: Option<&str>,
+    all: bool,
+) -> Api<DynamicObject> {
+    if capabilities.scope == Scope::Cluster || all {
+        Api::all_with(client, &resource)
+    } else if let Some(namespace) = ns {
+        Api::namespaced_with(client, namespace, &resource)
+    } else {
+        Api::default_namespaced_with(client, &resource)
+    }
+}
+
+fn multidoc_deserialize(data: &str) -> Result<Vec<serde_yaml::Value>, serde_yaml::Error> {
+    use serde::Deserialize;
+    let mut docs = vec![];
+    for de in serde_yaml::Deserializer::from_str(data) {
+        docs.push(serde_yaml::Value::deserialize(de)?);
+    }
+    Ok(docs)
+}
+
 pub trait ApplyStrategy<K: Resource> {
     fn get_api(client: &Client, ns: Option<&str>) -> Api<K>;
 }

harmony/src/domain/topology/k8s_anywhere.rs

@@ -1,7 +1,12 @@
-use std::{process::Command, sync::Arc};
+use std::{collections::BTreeMap, process::Command, sync::Arc, time::Duration};
 
 use async_trait::async_trait;
-use kube::api::GroupVersionKind;
+use base64::{Engine, engine::general_purpose};
+use k8s_openapi::api::{
+    core::v1::Secret,
+    rbac::v1::{ClusterRoleBinding, RoleRef, Subject},
+};
+use kube::api::{DynamicObject, GroupVersionKind, ObjectMeta};
 use log::{debug, info, warn};
 use serde::Serialize;
 use tokio::sync::OnceCell;
@@ -12,14 +17,26 @@ use crate::{
     inventory::Inventory,
     modules::{
         k3d::K3DInstallationScore,
-        monitoring::kube_prometheus::crd::{
-            crd_alertmanager_config::CRDPrometheus,
-            prometheus_operator::prometheus_operator_helm_chart_score,
-            rhob_alertmanager_config::RHOBObservability,
+        k8s::ingress::{K8sIngressScore, PathType},
+        monitoring::{
+            grafana::{grafana::Grafana, helm::helm_grafana::grafana_helm_chart_score},
+            kube_prometheus::crd::{
+                crd_alertmanager_config::CRDPrometheus,
+                crd_grafana::{
+                    Grafana as GrafanaCRD, GrafanaCom, GrafanaDashboard,
+                    GrafanaDashboardDatasource, GrafanaDashboardSpec, GrafanaDatasource,
+                    GrafanaDatasourceConfig, GrafanaDatasourceJsonData,
+                    GrafanaDatasourceSecureJsonData, GrafanaDatasourceSpec, GrafanaSpec,
+                },
+                crd_prometheuses::LabelSelector,
+                prometheus_operator::prometheus_operator_helm_chart_score,
+                rhob_alertmanager_config::RHOBObservability,
+                service_monitor::ServiceMonitor,
+            },
         },
         prometheus::{
             k8s_prometheus_alerting_score::K8sPrometheusCRDAlertingScore,
-            prometheus::PrometheusApplicationMonitoring, rhob_alerting_score::RHOBAlertingScore,
+            prometheus::PrometheusMonitoring, rhob_alerting_score::RHOBAlertingScore,
         },
     },
     score::Score,
@@ -86,41 +103,172 @@ impl K8sclient for K8sAnywhereTopology {
 }
 
 #[async_trait]
-impl PrometheusApplicationMonitoring<CRDPrometheus> for K8sAnywhereTopology {
+impl Grafana for K8sAnywhereTopology {
+    async fn ensure_grafana_operator(
+        &self,
+        inventory: &Inventory,
+    ) -> Result<PreparationOutcome, PreparationError> {
+        debug!("ensure grafana operator");
+        let client = self.k8s_client().await.unwrap();
+        let grafana_gvk = GroupVersionKind {
+            group: "grafana.integreatly.org".to_string(),
+            version: "v1beta1".to_string(),
+            kind: "Grafana".to_string(),
+        };
+        let name = "grafanas.grafana.integreatly.org";
+        let ns = "grafana";
+
+        let grafana_crd = client
+            .get_resource_json_value(name, Some(ns), &grafana_gvk)
+            .await;
+        match grafana_crd {
+            Ok(_) => {
+                return Ok(PreparationOutcome::Success {
+                    details: "Found grafana CRDs in cluster".to_string(),
+                });
+            }
+
+            Err(_) => {
+                return self
+                    .install_grafana_operator(inventory, Some("grafana"))
+                    .await;
+            }
+        };
+    }
+    async fn install_grafana(&self) -> Result<PreparationOutcome, PreparationError> {
+        let ns = "grafana";
+
+        let mut label = BTreeMap::new();
+
+        label.insert("dashboards".to_string(), "grafana".to_string());
+
+        let label_selector = LabelSelector {
+            match_labels: label.clone(),
+            match_expressions: vec![],
+        };
+
+        let client = self.k8s_client().await?;
+
+        let grafana = self.build_grafana(ns, &label);
+
+        client.apply(&grafana, Some(ns)).await?;
+        //TODO change this to a ensure ready or something better than just a timeout
+        client
+            .wait_until_deployment_ready(
+                "grafana-grafana-deployment",
+                Some("grafana"),
+                Some(Duration::from_secs(30)),
+            )
+            .await?;
+
+        let sa_name = "grafana-grafana-sa";
+        let token_secret_name = "grafana-sa-token-secret";
+
+        let sa_token_secret = self.build_sa_token_secret(token_secret_name, sa_name, ns);
+
+        client.apply(&sa_token_secret, Some(ns)).await?;
+        let secret_gvk = GroupVersionKind {
+            group: "".to_string(),
+            version: "v1".to_string(),
+            kind: "Secret".to_string(),
+        };
+
+        let secret = client
+            .get_resource_json_value(token_secret_name, Some(ns), &secret_gvk)
+            .await?;
+
+        let token = format!(
+            "Bearer {}",
+            self.extract_and_normalize_token(&secret).unwrap()
+        );
+
+        debug!("creating grafana clusterrole binding");
+
+        let clusterrolebinding =
+            self.build_cluster_rolebinding(sa_name, "cluster-monitoring-view", ns);
+
+        client.apply(&clusterrolebinding, Some(ns)).await?;
+
+        debug!("creating grafana datasource crd");
+
+        let thanos_url = format!(
+            "https://{}",
+            self.get_domain("thanos-querier-openshift-monitoring")
+                .await
+                .unwrap()
+        );
+
+        let thanos_openshift_datasource = self.build_grafana_datasource(
+            "thanos-openshift-monitoring",
+            ns,
+            &label_selector,
+            &thanos_url,
+            &token,
+        );
+
+        client.apply(&thanos_openshift_datasource, Some(ns)).await?;
+
+        debug!("creating grafana dashboard crd");
+        let dashboard = self.build_grafana_dashboard(ns, &label_selector);
+
+        client.apply(&dashboard, Some(ns)).await?;
+        debug!("creating grafana ingress");
+        let grafana_ingress = self.build_grafana_ingress(ns).await;
+
+        grafana_ingress
+            .interpret(&Inventory::empty(), self)
+            .await
+            .map_err(|e| PreparationError::new(e.to_string()))?;
+
+        Ok(PreparationOutcome::Success {
+            details: "Installed grafana composants".to_string(),
+        })
+    }
+}
+
+#[async_trait]
+impl PrometheusMonitoring<CRDPrometheus> for K8sAnywhereTopology {
     async fn install_prometheus(
         &self,
         sender: &CRDPrometheus,
-        inventory: &Inventory,
-        receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
+        _inventory: &Inventory,
+        _receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
+    ) -> Result<PreparationOutcome, PreparationError> {
+        let client = self.k8s_client().await?;
+
+        for monitor in sender.service_monitor.iter() {
+            client
+                .apply(monitor, Some(&sender.namespace))
+                .await
+                .map_err(|e| PreparationError::new(e.to_string()))?;
+        }
+        Ok(PreparationOutcome::Success {
+            details: "successfuly installed prometheus components".to_string(),
+        })
+    }
+
+    async fn ensure_prometheus_operator(
+        &self,
+        sender: &CRDPrometheus,
+        _inventory: &Inventory,
     ) -> Result<PreparationOutcome, PreparationError> {
         let po_result = self.ensure_prometheus_operator(sender).await?;
 
-        if po_result == PreparationOutcome::Noop {
-            debug!("Skipping Prometheus CR installation due to missing operator.");
-            return Ok(po_result);
-        }
-
-        let result = self
-            .get_k8s_prometheus_application_score(sender.clone(), receivers)
-            .await
-            .interpret(inventory, self)
-            .await;
-
-        match result {
-            Ok(outcome) => match outcome.status {
-                InterpretStatus::SUCCESS => Ok(PreparationOutcome::Success {
-                    details: outcome.message,
-                }),
-                InterpretStatus::NOOP => Ok(PreparationOutcome::Noop),
-                _ => Err(PreparationError::new(outcome.message)),
-            },
-            Err(err) => Err(PreparationError::new(err.to_string())),
+        match po_result {
+            PreparationOutcome::Success { details: _ } => {
+                debug!("Detected prometheus crds operator present in cluster.");
+                return Ok(po_result);
+            }
+            PreparationOutcome::Noop => {
+                debug!("Skipping Prometheus CR installation due to missing operator.");
+                return Ok(po_result);
+            }
         }
     }
 }
 
 #[async_trait]
-impl PrometheusApplicationMonitoring<RHOBObservability> for K8sAnywhereTopology {
+impl PrometheusMonitoring<RHOBObservability> for K8sAnywhereTopology {
     async fn install_prometheus(
         &self,
         sender: &RHOBObservability,
@@ -154,6 +302,14 @@ impl PrometheusApplicationMonitoring<RHOBObservability> for K8sAnywhereTopology
             Err(err) => Err(PreparationError::new(err.to_string())),
         }
     }
+
+    async fn ensure_prometheus_operator(
+        &self,
+        sender: &RHOBObservability,
+        inventory: &Inventory,
+    ) -> Result<PreparationOutcome, PreparationError> {
+        todo!()
+    }
 }
 
 impl Serialize for K8sAnywhereTopology {
@@ -215,6 +371,180 @@ impl K8sAnywhereTopology {
             .await
     }
 
+    fn extract_and_normalize_token(&self, secret: &DynamicObject) -> Option<String> {
+        let token_b64 = secret
+            .data
+            .get("token")
+            .or_else(|| secret.data.get("data").and_then(|d| d.get("token")))
+            .and_then(|v| v.as_str())?;
+
+        let bytes = general_purpose::STANDARD.decode(token_b64).ok()?;
+
+        let s = String::from_utf8(bytes).ok()?;
+
+        let cleaned = s
+            .trim_matches(|c: char| c.is_whitespace() || c == '\0')
+            .to_string();
+        Some(cleaned)
+    }
+
+    pub fn build_cluster_rolebinding(
+        &self,
+        service_account_name: &str,
+        clusterrole_name: &str,
+        ns: &str,
+    ) -> ClusterRoleBinding {
+        ClusterRoleBinding {
+            metadata: ObjectMeta {
+                name: Some(format!("{}-view-binding", service_account_name)),
+                ..Default::default()
+            },
+            role_ref: RoleRef {
+                api_group: "rbac.authorization.k8s.io".into(),
+                kind: "ClusterRole".into(),
+                name: clusterrole_name.into(),
+            },
+            subjects: Some(vec![Subject {
+                kind: "ServiceAccount".into(),
+                name: service_account_name.into(),
+                namespace: Some(ns.into()),
+                ..Default::default()
+            }]),
+        }
+    }
+
+    pub fn build_sa_token_secret(
+        &self,
+        secret_name: &str,
+        service_account_name: &str,
+        ns: &str,
+    ) -> Secret {
+        let mut annotations = BTreeMap::new();
+        annotations.insert(
+            "kubernetes.io/service-account.name".to_string(),
+            service_account_name.to_string(),
+        );
+
+        Secret {
+            metadata: ObjectMeta {
+                name: Some(secret_name.into()),
+                namespace: Some(ns.into()),
+                annotations: Some(annotations),
+                ..Default::default()
+            },
+            type_: Some("kubernetes.io/service-account-token".to_string()),
+            ..Default::default()
+        }
+    }
+
+    fn build_grafana_datasource(
+        &self,
+        name: &str,
+        ns: &str,
+        label_selector: &LabelSelector,
+        url: &str,
+        token: &str,
+    ) -> GrafanaDatasource {
+        let mut json_data = BTreeMap::new();
+        json_data.insert("timeInterval".to_string(), "5s".to_string());
+
+        GrafanaDatasource {
+            metadata: ObjectMeta {
+                name: Some(name.to_string()),
+                namespace: Some(ns.to_string()),
+                ..Default::default()
+            },
+            spec: GrafanaDatasourceSpec {
+                instance_selector: label_selector.clone(),
+                allow_cross_namespace_import: Some(true),
+                values_from: None,
+                datasource: GrafanaDatasourceConfig {
+                    access: "proxy".to_string(),
+                    name: name.to_string(),
+                    r#type: "prometheus".to_string(),
+                    url: url.to_string(),
+                    database: None,
+                    json_data: Some(GrafanaDatasourceJsonData {
+                        time_interval: Some("60s".to_string()),
+                        http_header_name1: Some("Authorization".to_string()),
+                        tls_skip_verify: Some(true),
+                        oauth_pass_thru: Some(true),
+                    }),
+                    secure_json_data: Some(GrafanaDatasourceSecureJsonData {
+                        http_header_value1: Some(format!("Bearer {token}")),
+                    }),
+                    is_default: Some(false),
+                    editable: Some(true),
+                },
+            },
+        }
+    }
+
+    fn build_grafana_dashboard(
+        &self,
+        ns: &str,
+        label_selector: &LabelSelector,
+    ) -> GrafanaDashboard {
+        let graf_dashboard = GrafanaDashboard {
+            metadata: ObjectMeta {
+                name: Some(format!("grafana-dashboard-{}", ns)),
+                namespace: Some(ns.to_string()),
+                ..Default::default()
+            },
+            spec: GrafanaDashboardSpec {
+                resync_period: Some("30s".to_string()),
+                instance_selector: label_selector.clone(),
+                datasources: Some(vec![GrafanaDashboardDatasource {
+                    input_name: "DS_PROMETHEUS".to_string(),
+                    datasource_name: "thanos-openshift-monitoring".to_string(),
+                }]),
+                json: None,
+                grafana_com: Some(GrafanaCom {
+                    id: 17406,
+                    revision: None,
+                }),
+            },
+        };
+        graf_dashboard
+    }
+
+    fn build_grafana(&self, ns: &str, labels: &BTreeMap<String, String>) -> GrafanaCRD {
+        let grafana = GrafanaCRD {
+            metadata: ObjectMeta {
+                name: Some(format!("grafana-{}", ns)),
+                namespace: Some(ns.to_string()),
+                labels: Some(labels.clone()),
+                ..Default::default()
+            },
+            spec: GrafanaSpec {
+                config: None,
+                admin_user: None,
+                admin_password: None,
+                ingress: None,
+                persistence: None,
+                resources: None,
+            },
+        };
+        grafana
+    }
+
+    async fn build_grafana_ingress(&self, ns: &str) -> K8sIngressScore {
+        let domain = self.get_domain(&format!("grafana-{}", ns)).await.unwrap();
+        let name = format!("{}-grafana", ns);
+        let backend_service = format!("grafana-{}-service", ns);
+
+        K8sIngressScore {
+            name: fqdn::fqdn!(&name),
+            host: fqdn::fqdn!(&domain),
+            backend_service: fqdn::fqdn!(&backend_service),
+            port: 3000,
+            path: Some("/".to_string()),
+            path_type: Some(PathType::Prefix),
+            namespace: Some(fqdn::fqdn!(&ns)),
+            ingress_class_name: Some("openshift-default".to_string()),
+        }
+    }
+
     async fn get_cluster_observability_operator_prometheus_application_score(
         &self,
         sender: RHOBObservability,
@@ -232,13 +562,14 @@ impl K8sAnywhereTopology {
         &self,
         sender: CRDPrometheus,
         receivers: Option<Vec<Box<dyn AlertReceiver<CRDPrometheus>>>>,
+        service_monitors: Option<Vec<ServiceMonitor>>,
     ) -> K8sPrometheusCRDAlertingScore {
-        K8sPrometheusCRDAlertingScore {
+        return K8sPrometheusCRDAlertingScore {
             sender,
             receivers: receivers.unwrap_or_default(),
-            service_monitors: vec![],
+            service_monitors: service_monitors.unwrap_or_default(),
             prometheus_rules: vec![],
-        }
+        };
     }
 
     async fn openshift_ingress_operator_available(&self) -> Result<(), PreparationError> {
@@ -506,6 +837,30 @@ impl K8sAnywhereTopology {
             details: "prometheus operator present in cluster".into(),
         })
     }
+
+    async fn install_grafana_operator(
+        &self,
+        inventory: &Inventory,
+        ns: Option<&str>,
+    ) -> Result<PreparationOutcome, PreparationError> {
+        let namespace = ns.unwrap_or("grafana");
+        info!("installing grafana operator in ns {namespace}");
+        let tenant = self.get_k8s_tenant_manager()?.get_tenant_config().await;
+        let mut namespace_scope = false;
+        if tenant.is_some() {
+            namespace_scope = true;
+        }
+        let _grafana_operator_score = grafana_helm_chart_score(namespace, namespace_scope)
+            .interpret(inventory, self)
+            .await
+            .map_err(|e| PreparationError::new(e.to_string()));
+        Ok(PreparationOutcome::Success {
+            details: format!(
+                "Successfully installed grafana operator in ns {}",
+                ns.unwrap()
+            ),
+        })
+    }
 }
 
 #[derive(Clone, Debug)]

harmony/src/domain/topology/network.rs

@@ -1,8 +1,15 @@
-use std::{error::Error, net::Ipv4Addr, str::FromStr, sync::Arc};
+use std::{
+    error::Error,
+    fmt::{self, Debug},
+    net::Ipv4Addr,
+    str::FromStr,
+    sync::Arc,
+};
 
 use async_trait::async_trait;
 use derive_new::new;
 use harmony_types::{
+    id::Id,
     net::{IpAddress, MacAddress},
     switch::PortLocation,
 };
@@ -19,8 +26,8 @@ pub struct DHCPStaticEntry {
     pub ip: Ipv4Addr,
 }
 
-impl std::fmt::Display for DHCPStaticEntry {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl fmt::Display for DHCPStaticEntry {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let mac = self
             .mac
             .iter()
@@ -42,8 +49,8 @@ pub trait Firewall: Send + Sync {
     fn get_host(&self) -> LogicalHost;
 }
 
-impl std::fmt::Debug for dyn Firewall {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl Debug for dyn Firewall {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_fmt(format_args!("Firewall {}", self.get_ip()))
     }
 }
@@ -65,7 +72,7 @@ pub struct PxeOptions {
 }
 
 #[async_trait]
-pub trait DhcpServer: Send + Sync + std::fmt::Debug {
+pub trait DhcpServer: Send + Sync + Debug {
     async fn add_static_mapping(&self, entry: &DHCPStaticEntry) -> Result<(), ExecutorError>;
     async fn remove_static_mapping(&self, mac: &MacAddress) -> Result<(), ExecutorError>;
     async fn list_static_mappings(&self) -> Vec<(MacAddress, IpAddress)>;
@@ -104,8 +111,8 @@ pub trait DnsServer: Send + Sync {
     }
 }
 
-impl std::fmt::Debug for dyn DnsServer {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl Debug for dyn DnsServer {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_fmt(format_args!("DnsServer {}", self.get_ip()))
     }
 }
@@ -141,8 +148,8 @@ pub enum DnsRecordType {
     TXT,
 }
 
-impl std::fmt::Display for DnsRecordType {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl fmt::Display for DnsRecordType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             DnsRecordType::A => write!(f, "A"),
             DnsRecordType::AAAA => write!(f, "AAAA"),
@@ -185,15 +192,12 @@ pub trait Switch: Send + Sync {
         mac_address: &MacAddress,
     ) -> Result<Option<PortLocation>, SwitchError>;
 
-    async fn configure_host_network(
-        &self,
-        host: &PhysicalHost,
-        config: HostNetworkConfig,
-    ) -> Result<(), SwitchError>;
+    async fn configure_host_network(&self, config: &HostNetworkConfig) -> Result<(), SwitchError>;
 }
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct HostNetworkConfig {
+    pub host_id: Id,
     pub switch_ports: Vec<SwitchPort>,
 }
 
@@ -216,8 +220,8 @@ pub struct SwitchError {
     msg: String,
 }
 
-impl std::fmt::Display for SwitchError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl fmt::Display for SwitchError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str(&self.msg)
     }
 }
@@ -225,7 +229,7 @@ impl std::fmt::Display for SwitchError {
 impl Error for SwitchError {}
 
 #[async_trait]
-pub trait SwitchClient: Send + Sync {
+pub trait SwitchClient: Debug + Send + Sync {
     /// Executes essential, idempotent, one-time initial configuration steps.
     ///
     /// This is an opiniated procedure that setups a switch to provide high availability

harmony/src/domain/topology/oberservability/monitoring.rs

@@ -1,6 +1,7 @@
 use std::any::Any;
 
 use async_trait::async_trait;
+use kube::api::DynamicObject;
 use log::debug;
 
 use crate::{
@@ -21,6 +22,7 @@ pub struct AlertingInterpret<S: AlertSender> {
     pub sender: S,
     pub receivers: Vec<Box<dyn AlertReceiver<S>>>,
     pub rules: Vec<Box<dyn AlertRule<S>>>,
+    pub scrape_targets: Option<Vec<Box<dyn ScrapeTarget<S>>>>,
 }
 
 #[async_trait]
@@ -30,6 +32,7 @@ impl<S: AlertSender + Installable<T>, T: Topology> Interpret<T> for AlertingInte
         inventory: &Inventory,
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
+        debug!("hit sender configure for AlertingInterpret");
         self.sender.configure(inventory, topology).await?;
         for receiver in self.receivers.iter() {
             receiver.install(&self.sender).await?;
@@ -38,6 +41,12 @@ impl<S: AlertSender + Installable<T>, T: Topology> Interpret<T> for AlertingInte
             debug!("installing rule: {:#?}", rule);
             rule.install(&self.sender).await?;
         }
+        if let Some(targets) = &self.scrape_targets {
+            for target in targets.iter() {
+                debug!("installing scrape_target: {:#?}", target);
+                target.install(&self.sender).await?;
+            }
+        }
         self.sender.ensure_installed(inventory, topology).await?;
         Ok(Outcome::success(format!(
             "successfully installed alert sender {}",
@@ -68,6 +77,14 @@ pub trait AlertReceiver<S: AlertSender>: std::fmt::Debug + Send + Sync {
     fn name(&self) -> String;
     fn clone_box(&self) -> Box<dyn AlertReceiver<S>>;
     fn as_any(&self) -> &dyn Any;
+    fn as_alertmanager_receiver(&self) -> Result<AlertManagerReceiver, String>;
+}
+
+#[derive(Debug)]
+pub struct AlertManagerReceiver {
+    pub receiver_config: serde_json::Value,
+    // FIXME we should not leak k8s here. DynamicObject is k8s specific
+    pub additional_ressources: Vec<DynamicObject>,
 }
 
 #[async_trait]
@@ -77,6 +94,7 @@ pub trait AlertRule<S: AlertSender>: std::fmt::Debug + Send + Sync {
 }
 
 #[async_trait]
-pub trait ScrapeTarget<S: AlertSender> {
-    async fn install(&self, sender: &S) -> Result<(), InterpretError>;
+pub trait ScrapeTarget<S: AlertSender>: std::fmt::Debug + Send + Sync {
+    async fn install(&self, sender: &S) -> Result<Outcome, InterpretError>;
+    fn clone_box(&self) -> Box<dyn ScrapeTarget<S>>;
 }

harmony/src/domain/topology/tenant/k8s.rs

@@ -14,7 +14,7 @@ use k8s_openapi::{
     },
     apimachinery::pkg::util::intstr::IntOrString,
 };
-use kube::Resource;
+use kube::{api::DynamicObject, Resource};
 use log::debug;
 use serde::de::DeserializeOwned;
 use serde_json::json;

harmony/src/infra/brocade.rs

@@ -1,15 +1,14 @@
 use async_trait::async_trait;
 use brocade::{BrocadeClient, BrocadeOptions, InterSwitchLink, InterfaceStatus, PortOperatingMode};
-use harmony_secret::Secret;
 use harmony_types::{
     net::{IpAddress, MacAddress},
     switch::{PortDeclaration, PortLocation},
 };
 use option_ext::OptionExt;
-use serde::{Deserialize, Serialize};
 
 use crate::topology::{SwitchClient, SwitchError};
 
+#[derive(Debug)]
 pub struct BrocadeSwitchClient {
     brocade: Box<dyn BrocadeClient + Send + Sync>,
 }
@@ -114,12 +113,6 @@ impl SwitchClient for BrocadeSwitchClient {
     }
 }
 
-#[derive(Secret, Serialize, Deserialize, Debug)]
-pub struct BrocadeSwitchAuth {
-    pub username: String,
-    pub password: String,
-}
-
 #[cfg(test)]
 mod tests {
     use std::sync::{Arc, Mutex};
@@ -235,7 +228,7 @@ mod tests {
         assert_that!(*configured_interfaces).is_empty();
     }
 
-    #[derive(Clone)]
+    #[derive(Debug, Clone)]
     struct FakeBrocadeClient {
         stack_topology: Vec<InterSwitchLink>,
         interfaces: Vec<InterfaceInfo>,

harmony/src/infra/inventory/mod.rs

@@ -11,7 +11,7 @@ pub struct InventoryRepositoryFactory;
 impl InventoryRepositoryFactory {
     pub async fn build() -> Result<Box<dyn InventoryRepository>, RepoError> {
         Ok(Box::new(
-            SqliteInventoryRepository::new(&(*DATABASE_URL)).await?,
+            SqliteInventoryRepository::new(&DATABASE_URL).await?,
         ))
     }
 }

harmony/src/infra/kube.rs

@@ -0,0 +1,182 @@
+use k8s_openapi::Resource as K8sResource;
+use kube::api::{ApiResource, DynamicObject, GroupVersionKind};
+use kube::core::TypeMeta;
+use serde::Serialize;
+use serde::de::DeserializeOwned;
+use serde_json::Value;
+
+/// Convert a typed Kubernetes resource `K` into a `DynamicObject`.
+///
+/// Requirements:
+/// - `K` must be a k8s_openapi resource (provides static GVK via `Resource`).
+/// - `K` must have standard Kubernetes shape (metadata + payload fields).
+///
+/// Notes:
+/// - We set `types` (apiVersion/kind) and copy `metadata`.
+/// - We place the remaining top-level fields into `obj.data` as JSON.
+/// - Scope is not encoded on the object itself; you still need the corresponding
+///   `DynamicResource` (derived from K::group/version/kind) when constructing an Api.
+///
+/// Example usage:
+///   let dyn_obj = kube_resource_to_dynamic(secret)?;
+///   let api: Api<DynamicObject> = Api::namespaced_with(client, "ns", &dr);
+///   api.patch(&dyn_obj.name_any(), &PatchParams::apply("mgr"), &Patch::Apply(dyn_obj)).await?;
+pub fn kube_resource_to_dynamic<K>(res: &K) -> Result<DynamicObject, String>
+where
+    K: K8sResource + Serialize + DeserializeOwned,
+{
+    // Serialize the typed resource to JSON so we can split metadata and payload
+    let mut v = serde_json::to_value(res).map_err(|e| format!("Failed to serialize : {e}"))?;
+    let obj = v
+        .as_object_mut()
+        .ok_or_else(|| "expected object JSON".to_string())?;
+
+    // Extract and parse metadata into kube::core::ObjectMeta
+    let metadata_value = obj
+        .remove("metadata")
+        .ok_or_else(|| "missing metadata".to_string())?;
+    let metadata: kube::core::ObjectMeta = serde_json::from_value(metadata_value)
+        .map_err(|e| format!("Failed to deserialize : {e}"))?;
+
+    // Name is required for DynamicObject::new; prefer metadata.name
+    let name = metadata
+        .name
+        .clone()
+        .ok_or_else(|| "metadata.name is required".to_string())?;
+
+    // Remaining fields (spec/status/data/etc.) become the dynamic payload
+    let payload = Value::Object(obj.clone());
+
+    // Construct the DynamicObject
+    let mut dyn_obj = DynamicObject::new(
+        &name,
+        &ApiResource::from_gvk(&GroupVersionKind::gvk(K::GROUP, K::VERSION, K::KIND)),
+    );
+    dyn_obj.types = Some(TypeMeta {
+        api_version: api_version_for::<K>(),
+        kind: K::KIND.into(),
+    });
+
+    // Preserve namespace/labels/annotations/etc.
+    dyn_obj.metadata = metadata;
+
+    // Attach payload
+    dyn_obj.data = payload;
+
+    Ok(dyn_obj)
+}
+
+/// Helper: compute apiVersion string ("group/version" or "v1" for core).
+fn api_version_for<K>() -> String
+where
+    K: K8sResource,
+{
+    let group = K::GROUP;
+    let version = K::VERSION;
+    if group.is_empty() {
+        version.to_string() // core/v1 => "v1"
+    } else {
+        format!("{}/{}", group, version)
+    }
+}
+#[cfg(test)]
+mod test {
+    use super::*;
+    use k8s_openapi::api::{
+        apps::v1::{Deployment, DeploymentSpec},
+        core::v1::{PodTemplateSpec, Secret},
+    };
+    use kube::api::ObjectMeta;
+    use pretty_assertions::assert_eq;
+
+    #[test]
+    fn secret_to_dynamic_roundtrip() {
+        // Create a sample Secret resource
+        let mut secret = Secret {
+            metadata: ObjectMeta {
+                name: Some("my-secret".to_string()),
+                ..Default::default()
+            },
+            type_: Some("kubernetes.io/service-account-token".to_string()),
+            ..Default::default()
+        };
+
+        // Convert to DynamicResource
+        let dynamic: DynamicObject =
+            kube_resource_to_dynamic(&secret).expect("Failed to convert Secret to DynamicResource");
+
+        // Serialize both the original and dynamic resources to Value
+        let original_value = serde_json::to_value(&secret).expect("Failed to serialize Secret");
+        let dynamic_value =
+            serde_json::to_value(&dynamic).expect("Failed to serialize DynamicResource");
+
+        // Assert that they are identical
+        assert_eq!(original_value, dynamic_value);
+
+        secret.metadata.namespace = Some("false".to_string());
+        let modified_value = serde_json::to_value(&secret).expect("Failed to serialize Secret");
+        assert_ne!(modified_value, dynamic_value);
+    }
+
+    #[test]
+    fn deployment_to_dynamic_roundtrip() {
+        // Create a sample Deployment with nested structures
+        let mut deployment = Deployment {
+            metadata: ObjectMeta {
+                name: Some("my-deployment".to_string()),
+                labels: Some({
+                    let mut map = std::collections::BTreeMap::new();
+                    map.insert("app".to_string(), "nginx".to_string());
+                    map
+                }),
+                ..Default::default()
+            },
+            spec: Some(DeploymentSpec {
+                replicas: Some(3),
+                selector: Default::default(),
+                template: PodTemplateSpec {
+                    metadata: Some(ObjectMeta {
+                        labels: Some({
+                            let mut map = std::collections::BTreeMap::new();
+                            map.insert("app".to_string(), "nginx".to_string());
+                            map
+                        }),
+                        ..Default::default()
+                    }),
+                    spec: Some(Default::default()), // PodSpec with empty containers for simplicity
+                },
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let dynamic = kube_resource_to_dynamic(&deployment).expect("Failed to convert Deployment");
+
+        let original_value = serde_json::to_value(&deployment).unwrap();
+        let dynamic_value = serde_json::to_value(&dynamic).unwrap();
+
+        assert_eq!(original_value, dynamic_value);
+
+        assert_eq!(
+            dynamic.data.get("spec").unwrap().get("replicas").unwrap(),
+            3
+        );
+        assert_eq!(
+            dynamic
+                .data
+                .get("spec")
+                .unwrap()
+                .get("template")
+                .unwrap()
+                .get("metadata")
+                .unwrap()
+                .get("labels")
+                .unwrap()
+                .get("app")
+                .unwrap()
+                .as_str()
+                .unwrap(),
+            "nginx".to_string()
+        );
+    }
+}

harmony/src/infra/mod.rs

@@ -3,5 +3,6 @@ pub mod executors;
 pub mod hp_ilo;
 pub mod intel_amt;
 pub mod inventory;
+pub mod kube;
 pub mod opnsense;
 mod sqlx;

harmony/src/modules/application/features/monitoring.rs

@@ -2,7 +2,11 @@ use crate::modules::application::{
     Application, ApplicationFeature, InstallationError, InstallationOutcome,
 };
 use crate::modules::monitoring::application_monitoring::application_monitoring_score::ApplicationMonitoringScore;
+use crate::modules::monitoring::grafana::grafana::Grafana;
 use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus;
+use crate::modules::monitoring::kube_prometheus::crd::service_monitor::{
+    ServiceMonitor, ServiceMonitorSpec,
+};
 use crate::topology::MultiTargetTopology;
 use crate::topology::ingress::Ingress;
 use crate::{
@@ -14,7 +18,7 @@ use crate::{
     topology::{HelmCommand, K8sclient, Topology, tenant::TenantManager},
 };
 use crate::{
-    modules::prometheus::prometheus::PrometheusApplicationMonitoring,
+    modules::prometheus::prometheus::PrometheusMonitoring,
     topology::oberservability::monitoring::AlertReceiver,
 };
 use async_trait::async_trait;
@@ -22,6 +26,7 @@ use base64::{Engine as _, engine::general_purpose};
 use harmony_secret::SecretManager;
 use harmony_secret_derive::Secret;
 use harmony_types::net::Url;
+use kube::api::ObjectMeta;
 use log::{debug, info};
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
@@ -40,7 +45,8 @@ impl<
         + TenantManager
         + K8sclient
         + MultiTargetTopology
-        + PrometheusApplicationMonitoring<CRDPrometheus>
+        + PrometheusMonitoring<CRDPrometheus>
+        + Grafana
         + Ingress
         + std::fmt::Debug,
 > ApplicationFeature<T> for Monitoring
@@ -57,10 +63,20 @@ impl<
             .unwrap_or_else(|| self.application.name());
         let domain = topology.get_domain("ntfy").await.unwrap();
 
+        let app_service_monitor = ServiceMonitor {
+            metadata: ObjectMeta {
+                name: Some(self.application.name()),
+                namespace: Some(namespace.clone()),
+                ..Default::default()
+            },
+            spec: ServiceMonitorSpec::default(),
+        };
+
         let mut alerting_score = ApplicationMonitoringScore {
             sender: CRDPrometheus {
                 namespace: namespace.clone(),
                 client: topology.k8s_client().await.unwrap(),
+                service_monitor: vec![app_service_monitor],
             },
             application: self.application.clone(),
             receivers: self.alert_receiver.clone(),

harmony/src/modules/application/features/rhob_monitoring.rs

@@ -18,7 +18,7 @@ use crate::{
     topology::{HelmCommand, K8sclient, Topology, tenant::TenantManager},
 };
 use crate::{
-    modules::prometheus::prometheus::PrometheusApplicationMonitoring,
+    modules::prometheus::prometheus::PrometheusMonitoring,
     topology::oberservability::monitoring::AlertReceiver,
 };
 use async_trait::async_trait;
@@ -42,7 +42,7 @@ impl<
         + MultiTargetTopology
         + Ingress
         + std::fmt::Debug
-        + PrometheusApplicationMonitoring<RHOBObservability>,
+        + PrometheusMonitoring<RHOBObservability>,
 > ApplicationFeature<T> for Monitoring
 {
     async fn ensure_installed(

harmony/src/modules/k8s/resource.rs

@@ -38,13 +38,15 @@ impl<
         + 'static
         + Send
         + Clone,
-    T: Topology,
+    T: Topology + K8sclient,
 > Score<T> for K8sResourceScore<K>
 where
     <K as kube::Resource>::DynamicType: Default,
 {
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
-        todo!()
+        Box::new(K8sResourceInterpret {
+            score: self.clone(),
+        })
     }
 
     fn name(&self) -> String {

harmony/src/modules/monitoring/alert_channel/discord_alert_channel.rs

@@ -3,16 +3,20 @@ use std::collections::BTreeMap;
 
 use async_trait::async_trait;
 use k8s_openapi::api::core::v1::Secret;
-use kube::api::ObjectMeta;
+use kube::Resource;
+use kube::api::{DynamicObject, ObjectMeta};
 use log::debug;
 use serde::Serialize;
 use serde_json::json;
 use serde_yaml::{Mapping, Value};
 
+use crate::infra::kube::kube_resource_to_dynamic;
 use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::{
     AlertmanagerConfig, AlertmanagerConfigSpec, CRDPrometheus,
 };
 use crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::RHOBObservability;
+use crate::modules::monitoring::okd::OpenshiftClusterAlertSender;
+use crate::topology::oberservability::monitoring::AlertManagerReceiver;
 use crate::{
     interpret::{InterpretError, Outcome},
     modules::monitoring::{
@@ -28,14 +32,25 @@ use harmony_types::net::Url;
 
 #[derive(Debug, Clone, Serialize)]
 pub struct DiscordWebhook {
+    // FIXME use a stricter type as this is used as a k8s resource name. It could also be converted
+    // to remove whitespace and other invalid characters, but this is a potential bug that is not
+    // very easy to figure out for beginners.
+    //
+    // It gives out error messages like this :
+    //
+    // [2025-10-30 15:10:49 ERROR harmony::domain::topology::k8s] Failed to get dynamic resource 'Webhook example-secret': Failed to build request: failed to build request: invalid uri character
+    // [2025-10-30 15:10:49 ERROR harmony_cli::cli_logger] ⚠️ InterpretError : Failed to build request: failed to build request: invalid uri character
+    // [2025-10-30 15:10:49 DEBUG harmony::domain::maestro] Got result Err(InterpretError { msg: "InterpretError : Failed to build request: failed to build request: invalid uri character" })
+    // [2025-10-30 15:10:49 INFO  harmony_cli::cli_logger] 🎼 Harmony completed
+    //
+    // thread 'main' panicked at examples/okd_cluster_alerts/src/main.rs:25:6:
+    // called `Result::unwrap()` on an `Err` value: InterpretError { msg: "InterpretError : Failed to build request: failed to build request: invalid uri character" }
     pub name: String,
     pub url: Url,
 }
 
-#[async_trait]
-impl AlertReceiver<RHOBObservability> for DiscordWebhook {
-    async fn install(&self, sender: &RHOBObservability) -> Result<Outcome, InterpretError> {
-        let ns = sender.namespace.clone();
+impl DiscordWebhook {
+    fn get_receiver_config(&self) -> Result<AlertManagerReceiver, String> {
         let secret_name = format!("{}-secret", self.name.clone());
         let webhook_key = format!("{}", self.url.clone());
 
@@ -52,26 +67,74 @@ impl AlertReceiver<RHOBObservability> for DiscordWebhook {
             ..Default::default()
         };
 
-        let _ = sender.client.apply(&secret, Some(&ns)).await;
+        Ok(AlertManagerReceiver {
+            additional_ressources: vec![kube_resource_to_dynamic(&secret)?],
+
+            receiver_config: json!({
+                "name": self.name,
+                "discordConfigs": [
+                    {
+                    "apiURL": {
+                        "name": secret_name,
+                        "key":  "webhook-url",
+                    },
+                    "title": "{{ template \"discord.default.title\" . }}",
+                    "message": "{{ template \"discord.default.message\" . }}"
+                    }
+                ]
+            }),
+        })
+    }
+}
+
+#[async_trait]
+impl AlertReceiver<OpenshiftClusterAlertSender> for DiscordWebhook {
+    async fn install(
+        &self,
+        sender: &OpenshiftClusterAlertSender,
+    ) -> Result<Outcome, InterpretError> {
+        todo!()
+    }
+
+    fn name(&self) -> String {
+        self.name.clone()
+    }
+
+    fn clone_box(&self) -> Box<dyn AlertReceiver<OpenshiftClusterAlertSender>> {
+        Box::new(self.clone())
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        todo!()
+    }
+
+    fn as_alertmanager_receiver(&self) -> Result<AlertManagerReceiver, String> {
+        self.get_receiver_config()
+    }
+}
+
+#[async_trait]
+impl AlertReceiver<RHOBObservability> for DiscordWebhook {
+    fn as_alertmanager_receiver(&self) -> Result<AlertManagerReceiver, String> {
+        todo!()
+    }
+
+    async fn install(&self, sender: &RHOBObservability) -> Result<Outcome, InterpretError> {
+        let ns = sender.namespace.clone();
+
+        let config = self.get_receiver_config()?;
+        for resource in config.additional_ressources.iter() {
+            todo!("can I apply a dynamicresource");
+            // sender.client.apply(resource, Some(&ns)).await;
+        }
+
         let spec = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfigSpec {
             data: json!({
                 "route": {
                     "receiver": self.name,
                 },
                 "receivers": [
-                    {
-                        "name": self.name,
-                        "discordConfigs": [
-                            {
-                            "apiURL": {
-                                "name": secret_name,
-                                "key":  "webhook-url",
-                            },
-                            "title": "{{ template \"discord.default.title\" . }}",
-                            "message": "{{ template \"discord.default.message\" . }}"
-                            }
-                        ]
-                    }
+                    config.receiver_config
                 ]
             }),
         };
@@ -122,6 +185,9 @@ impl AlertReceiver<RHOBObservability> for DiscordWebhook {
 
 #[async_trait]
 impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
+    fn as_alertmanager_receiver(&self) -> Result<AlertManagerReceiver, String> {
+        todo!()
+    }
     async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
         let ns = sender.namespace.clone();
         let secret_name = format!("{}-secret", self.name.clone());
@@ -200,6 +266,9 @@ impl AlertReceiver<CRDPrometheus> for DiscordWebhook {
 
 #[async_trait]
 impl AlertReceiver<Prometheus> for DiscordWebhook {
+    fn as_alertmanager_receiver(&self) -> Result<AlertManagerReceiver, String> {
+        todo!()
+    }
     async fn install(&self, sender: &Prometheus) -> Result<Outcome, InterpretError> {
         sender.install_receiver(self).await
     }
@@ -226,6 +295,9 @@ impl PrometheusReceiver for DiscordWebhook {
 
 #[async_trait]
 impl AlertReceiver<KubePrometheus> for DiscordWebhook {
+    fn as_alertmanager_receiver(&self) -> Result<AlertManagerReceiver, String> {
+        todo!()
+    }
     async fn install(&self, sender: &KubePrometheus) -> Result<Outcome, InterpretError> {
         sender.install_receiver(self).await
     }

harmony/src/modules/monitoring/alert_channel/webhook_receiver.rs

@@ -19,7 +19,7 @@ use crate::{
         },
         prometheus::prometheus::{Prometheus, PrometheusReceiver},
     },
-    topology::oberservability::monitoring::AlertReceiver,
+    topology::oberservability::monitoring::{AlertManagerReceiver, AlertReceiver},
 };
 use harmony_types::net::Url;
 
@@ -31,6 +31,9 @@ pub struct WebhookReceiver {
 
 #[async_trait]
 impl AlertReceiver<RHOBObservability> for WebhookReceiver {
+    fn as_alertmanager_receiver(&self) -> Result<AlertManagerReceiver, String> {
+        todo!()
+    }
     async fn install(&self, sender: &RHOBObservability) -> Result<Outcome, InterpretError> {
         let spec = crate::modules::monitoring::kube_prometheus::crd::rhob_alertmanager_config::AlertmanagerConfigSpec {
             data: json!({
@@ -97,6 +100,9 @@ impl AlertReceiver<RHOBObservability> for WebhookReceiver {
 
 #[async_trait]
 impl AlertReceiver<CRDPrometheus> for WebhookReceiver {
+    fn as_alertmanager_receiver(&self) -> Result<AlertManagerReceiver, String> {
+        todo!()
+    }
     async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
         let spec = crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::AlertmanagerConfigSpec {
             data: json!({
@@ -158,6 +164,9 @@ impl AlertReceiver<CRDPrometheus> for WebhookReceiver {
 
 #[async_trait]
 impl AlertReceiver<Prometheus> for WebhookReceiver {
+    fn as_alertmanager_receiver(&self) -> Result<AlertManagerReceiver, String> {
+        todo!()
+    }
     async fn install(&self, sender: &Prometheus) -> Result<Outcome, InterpretError> {
         sender.install_receiver(self).await
     }
@@ -184,6 +193,9 @@ impl PrometheusReceiver for WebhookReceiver {
 
 #[async_trait]
 impl AlertReceiver<KubePrometheus> for WebhookReceiver {
+    fn as_alertmanager_receiver(&self) -> Result<AlertManagerReceiver, String> {
+        todo!()
+    }
     async fn install(&self, sender: &KubePrometheus) -> Result<Outcome, InterpretError> {
         sender.install_receiver(self).await
     }

harmony/src/modules/monitoring/application_monitoring/application_monitoring_score.rs

@@ -1,21 +1,23 @@
 use std::sync::Arc;
 
-use async_trait::async_trait;
+use log::debug;
 use serde::Serialize;
 
 use crate::{
-    data::Version,
-    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
-    inventory::Inventory,
+    interpret::Interpret,
     modules::{
         application::Application,
-        monitoring::kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus,
-        prometheus::prometheus::PrometheusApplicationMonitoring,
+        monitoring::{
+            grafana::grafana::Grafana, kube_prometheus::crd::crd_alertmanager_config::CRDPrometheus,
+        },
+        prometheus::prometheus::PrometheusMonitoring,
     },
     score::Score,
-    topology::{PreparationOutcome, Topology, oberservability::monitoring::AlertReceiver},
+    topology::{
+        K8sclient, Topology,
+        oberservability::monitoring::{AlertReceiver, AlertingInterpret, ScrapeTarget},
+    },
 };
-use harmony_types::id::Id;
 
 #[derive(Debug, Clone, Serialize)]
 pub struct ApplicationMonitoringScore {
@@ -24,12 +26,16 @@ pub struct ApplicationMonitoringScore {
     pub receivers: Vec<Box<dyn AlertReceiver<CRDPrometheus>>>,
 }
 
-impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
+impl<T: Topology + PrometheusMonitoring<CRDPrometheus> + K8sclient + Grafana> Score<T>
     for ApplicationMonitoringScore
 {
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
-        Box::new(ApplicationMonitoringInterpret {
-            score: self.clone(),
+        debug!("creating alerting interpret");
+        Box::new(AlertingInterpret {
+            sender: self.sender.clone(),
+            receivers: self.receivers.clone(),
+            rules: vec![],
+            scrape_targets: None,
         })
     }
 
@@ -40,55 +46,3 @@ impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
         )
     }
 }
-
-#[derive(Debug)]
-pub struct ApplicationMonitoringInterpret {
-    score: ApplicationMonitoringScore,
-}
-
-#[async_trait]
-impl<T: Topology + PrometheusApplicationMonitoring<CRDPrometheus>> Interpret<T>
-    for ApplicationMonitoringInterpret
-{
-    async fn execute(
-        &self,
-        inventory: &Inventory,
-        topology: &T,
-    ) -> Result<Outcome, InterpretError> {
-        let result = topology
-            .install_prometheus(
-                &self.score.sender,
-                inventory,
-                Some(self.score.receivers.clone()),
-            )
-            .await;
-
-        match result {
-            Ok(outcome) => match outcome {
-                PreparationOutcome::Success { details: _ } => {
-                    Ok(Outcome::success("Prometheus installed".into()))
-                }
-                PreparationOutcome::Noop => {
-                    Ok(Outcome::noop("Prometheus installation skipped".into()))
-                }
-            },
-            Err(err) => Err(InterpretError::from(err)),
-        }
-    }
-
-    fn get_name(&self) -> InterpretName {
-        InterpretName::ApplicationMonitoring
-    }
-
-    fn get_version(&self) -> Version {
-        todo!()
-    }
-
-    fn get_status(&self) -> InterpretStatus {
-        todo!()
-    }
-
-    fn get_children(&self) -> Vec<Id> {
-        todo!()
-    }
-}

harmony/src/modules/monitoring/application_monitoring/rhobs_application_monitoring_score.rs

@@ -12,7 +12,7 @@ use crate::{
         monitoring::kube_prometheus::crd::{
             crd_alertmanager_config::CRDPrometheus, rhob_alertmanager_config::RHOBObservability,
         },
-        prometheus::prometheus::PrometheusApplicationMonitoring,
+        prometheus::prometheus::PrometheusMonitoring,
     },
     score::Score,
     topology::{PreparationOutcome, Topology, oberservability::monitoring::AlertReceiver},
@@ -26,7 +26,7 @@ pub struct ApplicationRHOBMonitoringScore {
     pub receivers: Vec<Box<dyn AlertReceiver<RHOBObservability>>>,
 }
 
-impl<T: Topology + PrometheusApplicationMonitoring<RHOBObservability>> Score<T>
+impl<T: Topology + PrometheusMonitoring<RHOBObservability>> Score<T>
     for ApplicationRHOBMonitoringScore
 {
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
@@ -49,7 +49,7 @@ pub struct ApplicationRHOBMonitoringInterpret {
 }
 
 #[async_trait]
-impl<T: Topology + PrometheusApplicationMonitoring<RHOBObservability>> Interpret<T>
+impl<T: Topology + PrometheusMonitoring<RHOBObservability>> Interpret<T>
     for ApplicationRHOBMonitoringInterpret
 {
     async fn execute(

harmony/src/modules/monitoring/grafana/grafana.rs

@@ -0,0 +1,17 @@
+use async_trait::async_trait;
+use k8s_openapi::Resource;
+
+use crate::{
+    inventory::Inventory,
+    topology::{PreparationError, PreparationOutcome},
+};
+
+#[async_trait]
+pub trait Grafana {
+    async fn ensure_grafana_operator(
+        &self,
+        inventory: &Inventory,
+    ) -> Result<PreparationOutcome, PreparationError>;
+
+    async fn install_grafana(&self) -> Result<PreparationOutcome, PreparationError>;
+}

harmony/src/modules/monitoring/grafana/helm/helm_grafana.rs

@@ -1,27 +1,28 @@
+use harmony_macros::hurl;
 use non_blank_string_rs::NonBlankString;
-use std::str::FromStr;
+use std::{collections::HashMap, str::FromStr};
 
-use crate::modules::helm::chart::HelmChartScore;
-
-pub fn grafana_helm_chart_score(ns: &str) -> HelmChartScore {
-    let values = r#"
-rbac:
-  namespaced: true
-sidecar:
-  dashboards:
-    enabled: true
-        "#
-    .to_string();
+use crate::modules::helm::chart::{HelmChartScore, HelmRepository};
 
+pub fn grafana_helm_chart_score(ns: &str, namespace_scope: bool) -> HelmChartScore {
+    let mut values_overrides = HashMap::new();
+    values_overrides.insert(
+        NonBlankString::from_str("namespaceScope").unwrap(),
+        namespace_scope.to_string(),
+    );
     HelmChartScore {
         namespace: Some(NonBlankString::from_str(ns).unwrap()),
-        release_name: NonBlankString::from_str("grafana").unwrap(),
-        chart_name: NonBlankString::from_str("oci://ghcr.io/grafana/helm-charts/grafana").unwrap(),
+        release_name: NonBlankString::from_str("grafana-operator").unwrap(),
+        chart_name: NonBlankString::from_str("grafana/grafana-operator").unwrap(),
         chart_version: None,
-        values_overrides: None,
-        values_yaml: Some(values.to_string()),
+        values_overrides: Some(values_overrides),
+        values_yaml: None,
         create_namespace: true,
         install_only: true,
-        repository: None,
+        repository: Some(HelmRepository::new(
+            "grafana".to_string(),
+            hurl!("https://grafana.github.io/helm-charts"),
+            true,
+        )),
     }
 }

harmony/src/modules/monitoring/grafana/mod.rs

@@ -1 +1,2 @@
+pub mod grafana;
 pub mod helm;

harmony/src/modules/monitoring/kube_prometheus/crd/crd_alertmanager_config.rs

@@ -1,12 +1,25 @@
 use std::sync::Arc;
 
+use async_trait::async_trait;
 use kube::CustomResource;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
-use crate::topology::{
-    k8s::K8sClient,
-    oberservability::monitoring::{AlertReceiver, AlertSender},
+use crate::{
+    interpret::{InterpretError, Outcome},
+    inventory::Inventory,
+    modules::{
+        monitoring::{
+            grafana::grafana::Grafana, kube_prometheus::crd::service_monitor::ServiceMonitor,
+        },
+        prometheus::prometheus::PrometheusMonitoring,
+    },
+    topology::{
+        K8sclient, Topology,
+        installable::Installable,
+        k8s::K8sClient,
+        oberservability::monitoring::{AlertReceiver, AlertSender, ScrapeTarget},
+    },
 };
 
 #[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
@@ -26,6 +39,7 @@ pub struct AlertmanagerConfigSpec {
 pub struct CRDPrometheus {
     pub namespace: String,
     pub client: Arc<K8sClient>,
+    pub service_monitor: Vec<ServiceMonitor>,
 }
 
 impl AlertSender for CRDPrometheus {
@@ -40,6 +54,12 @@ impl Clone for Box<dyn AlertReceiver<CRDPrometheus>> {
     }
 }
 
+impl Clone for Box<dyn ScrapeTarget<CRDPrometheus>> {
+    fn clone(&self) -> Self {
+        self.clone_box()
+    }
+}
+
 impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> {
     fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
     where
@@ -48,3 +68,24 @@ impl Serialize for Box<dyn AlertReceiver<CRDPrometheus>> {
         todo!()
     }
 }
+
+#[async_trait]
+impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus> + Grafana> Installable<T>
+    for CRDPrometheus
+{
+    async fn configure(&self, inventory: &Inventory, topology: &T) -> Result<(), InterpretError> {
+        topology.ensure_grafana_operator(inventory).await?;
+        topology.ensure_prometheus_operator(self, inventory).await?;
+        Ok(())
+    }
+
+    async fn ensure_installed(
+        &self,
+        inventory: &Inventory,
+        topology: &T,
+    ) -> Result<(), InterpretError> {
+        topology.install_grafana().await?;
+        topology.install_prometheus(&self, inventory, None).await?;
+        Ok(())
+    }
+}

harmony/src/modules/monitoring/kube_prometheus/crd/crd_grafana.rs

@@ -103,9 +103,34 @@ pub struct GrafanaDashboardSpec {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub resync_period: Option<String>,
 
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub datasources: Option<Vec<GrafanaDashboardDatasource>>,
+
     pub instance_selector: LabelSelector,
 
-    pub json: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub json: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub grafana_com: Option<GrafanaCom>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDashboardDatasource {
+    pub input_name: String,
+    pub datasource_name: String,
+}
+
+// ------------------------------------------------------------------------------------------------
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaCom {
+    pub id: u32,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub revision: Option<u32>,
 }
 
 // ------------------------------------------------------------------------------------------------
@@ -126,20 +151,79 @@ pub struct GrafanaDatasourceSpec {
     pub allow_cross_namespace_import: Option<bool>,
 
     pub datasource: GrafanaDatasourceConfig,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub values_from: Option<Vec<GrafanaValueFrom>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaValueFrom {
+    pub target_path: String,
+    pub value_from: GrafanaValueSource,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaValueSource {
+    pub secret_key_ref: GrafanaSecretKeyRef,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaSecretKeyRef {
+    pub name: String,
+    pub key: String,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct GrafanaDatasourceConfig {
     pub access: String,
-    pub database: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub json_data: Option<BTreeMap<String, String>>,
+    pub database: Option<String>,
     pub name: String,
     pub r#type: String,
     pub url: String,
+    /// Represents jsonData in the GrafanaDatasource spec
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub json_data: Option<GrafanaDatasourceJsonData>,
+
+    /// Represents secureJsonData (secrets)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub secure_json_data: Option<GrafanaDatasourceSecureJsonData>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub is_default: Option<bool>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub editable: Option<bool>,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDatasourceJsonData {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub time_interval: Option<String>,
+
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub http_header_name1: Option<String>,
+
+    /// Disable TLS skip verification (false = verify)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub tls_skip_verify: Option<bool>,
+
+    /// Auth type - set to "forward" for OpenShift OAuth identity
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub oauth_pass_thru: Option<bool>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrafanaDatasourceSecureJsonData {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub http_header_value1: Option<String>,
+}
 // ------------------------------------------------------------------------------------------------
 
 #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, Default)]

harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs

@@ -0,0 +1,187 @@
+use std::net::IpAddr;
+
+use async_trait::async_trait;
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    modules::monitoring::kube_prometheus::crd::{
+        crd_alertmanager_config::CRDPrometheus, crd_prometheuses::LabelSelector,
+    },
+    topology::oberservability::monitoring::ScrapeTarget,
+};
+
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "monitoring.coreos.com",
+    version = "v1alpha1",
+    kind = "ScrapeConfig",
+    plural = "scrapeconfigs",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct ScrapeConfigSpec {
+    /// List of static configurations.
+    pub static_configs: Option<Vec<StaticConfig>>,
+
+    /// Kubernetes service discovery.
+    pub kubernetes_sd_configs: Option<Vec<KubernetesSDConfig>>,
+
+    /// HTTP-based service discovery.
+    pub http_sd_configs: Option<Vec<HttpSDConfig>>,
+
+    /// File-based service discovery.
+    pub file_sd_configs: Option<Vec<FileSDConfig>>,
+
+    /// DNS-based service discovery.
+    pub dns_sd_configs: Option<Vec<DnsSDConfig>>,
+
+    /// Consul service discovery.
+    pub consul_sd_configs: Option<Vec<ConsulSDConfig>>,
+
+    /// Relabeling configuration applied to discovered targets.
+    pub relabel_configs: Option<Vec<RelabelConfig>>,
+
+    /// Metric relabeling configuration applied to scraped samples.
+    pub metric_relabel_configs: Option<Vec<RelabelConfig>>,
+
+    /// Path to scrape metrics from (defaults to `/metrics`).
+    pub metrics_path: Option<String>,
+
+    /// Interval at which Prometheus scrapes targets (e.g., "30s").
+    pub scrape_interval: Option<String>,
+
+    /// Timeout for scraping (e.g., "10s").
+    pub scrape_timeout: Option<String>,
+
+    /// Optional job name override.
+    pub job_name: Option<String>,
+
+    /// Optional scheme (http or https).
+    pub scheme: Option<String>,
+
+    /// Authorization paramaters for snmp walk
+    pub params: Option<Params>,
+}
+
+/// Static configuration section of a ScrapeConfig.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct StaticConfig {
+    pub targets: Vec<String>,
+
+    pub labels: Option<LabelSelector>,
+}
+
+/// Relabeling configuration for target or metric relabeling.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct RelabelConfig {
+    pub source_labels: Option<Vec<String>>,
+    pub separator: Option<String>,
+    pub target_label: Option<String>,
+    pub regex: Option<String>,
+    pub modulus: Option<u64>,
+    pub replacement: Option<String>,
+    pub action: Option<String>,
+}
+
+/// Kubernetes service discovery configuration.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct KubernetesSDConfig {
+    ///"pod", "service", "endpoints"pub role: String,
+    pub namespaces: Option<NamespaceSelector>,
+    pub selectors: Option<Vec<LabelSelector>>,
+    pub api_server: Option<String>,
+    pub bearer_token_file: Option<String>,
+    pub tls_config: Option<TLSConfig>,
+}
+
+/// Namespace selector for Kubernetes service discovery.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct NamespaceSelector {
+    pub any: Option<bool>,
+    pub match_names: Option<Vec<String>>,
+}
+
+/// HTTP-based service discovery configuration.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct HttpSDConfig {
+    pub url: String,
+    pub refresh_interval: Option<String>,
+    pub basic_auth: Option<BasicAuth>,
+    pub authorization: Option<Authorization>,
+    pub tls_config: Option<TLSConfig>,
+}
+
+/// File-based service discovery configuration.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct FileSDConfig {
+    pub files: Vec<String>,
+    pub refresh_interval: Option<String>,
+}
+
+/// DNS-based service discovery configuration.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct DnsSDConfig {
+    pub names: Vec<String>,
+    pub refresh_interval: Option<String>,
+    pub type_: Option<String>, // SRV, A, AAAA
+    pub port: Option<u16>,
+}
+
+/// Consul service discovery configuration.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ConsulSDConfig {
+    pub server: String,
+    pub services: Option<Vec<String>>,
+    pub scheme: Option<String>,
+    pub datacenter: Option<String>,
+    pub tag_separator: Option<String>,
+    pub refresh_interval: Option<String>,
+    pub tls_config: Option<TLSConfig>,
+}
+
+/// Basic authentication credentials.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct BasicAuth {
+    pub username: String,
+    pub password: Option<String>,
+    pub password_file: Option<String>,
+}
+
+/// Bearer token or other auth mechanisms.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Authorization {
+    pub credentials: Option<String>,
+    pub credentials_file: Option<String>,
+    pub type_: Option<String>,
+}
+
+/// TLS configuration for secure scraping.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct TLSConfig {
+    pub ca_file: Option<String>,
+    pub cert_file: Option<String>,
+    pub key_file: Option<String>,
+    pub server_name: Option<String>,
+    pub insecure_skip_verify: Option<bool>,
+}
+
+/// Authorization parameters for SNMP walk.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Params {
+    pub auth: Option<Vec<String>>,
+    pub module: Option<Vec<String>>,
+}

harmony/src/modules/monitoring/kube_prometheus/crd/mod.rs

@@ -4,6 +4,7 @@ pub mod crd_default_rules;
 pub mod crd_grafana;
 pub mod crd_prometheus_rules;
 pub mod crd_prometheuses;
+pub mod crd_scrape_config;
 pub mod grafana_default_dashboard;
 pub mod grafana_operator;
 pub mod prometheus_operator;

harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs

@@ -31,6 +31,7 @@ impl<T: Topology + HelmCommand + TenantManager> Score<T> for HelmPrometheusAlert
             sender: KubePrometheus { config },
             receivers: self.receivers.clone(),
             rules: self.rules.clone(),
+            scrape_targets: None,
         })
     }
     fn name(&self) -> String {

harmony/src/modules/monitoring/mod.rs

@@ -6,3 +6,4 @@ pub mod kube_prometheus;
 pub mod ntfy;
 pub mod okd;
 pub mod prometheus;
+pub mod scrape_target;

harmony/src/modules/monitoring/ntfy/ntfy.rs

@@ -100,11 +100,7 @@ impl<T: Topology + HelmCommand + K8sclient + MultiTargetTopology> Interpret<T> f
 
         info!("deploying ntfy...");
         client
-            .wait_until_deployment_ready(
-                "ntfy".to_string(),
-                Some(self.score.namespace.as_str()),
-                None,
-            )
+            .wait_until_deployment_ready("ntfy", Some(self.score.namespace.as_str()), None)
             .await?;
         info!("ntfy deployed");
 

harmony/src/modules/monitoring/okd/cluster_monitoring.rs

@@ -0,0 +1,214 @@
+use base64::prelude::*;
+
+use async_trait::async_trait;
+use harmony_types::id::Id;
+use kube::api::DynamicObject;
+use log::{debug, info, trace};
+use serde::Serialize;
+
+use crate::{
+    data::Version,
+    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
+    inventory::Inventory,
+    modules::monitoring::okd::OpenshiftClusterAlertSender,
+    score::Score,
+    topology::{K8sclient, Topology, oberservability::monitoring::AlertReceiver},
+};
+
+impl Clone for Box<dyn AlertReceiver<OpenshiftClusterAlertSender>> {
+    fn clone(&self) -> Self {
+        self.clone_box()
+    }
+}
+
+impl Serialize for Box<dyn AlertReceiver<OpenshiftClusterAlertSender>> {
+    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        todo!()
+    }
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct OpenshiftClusterAlertScore {
+    pub receivers: Vec<Box<dyn AlertReceiver<OpenshiftClusterAlertSender>>>,
+}
+
+impl<T: Topology + K8sclient> Score<T> for OpenshiftClusterAlertScore {
+    fn name(&self) -> String {
+        "ClusterAlertScore".to_string()
+    }
+
+    #[doc(hidden)]
+    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
+        Box::new(OpenshiftClusterAlertInterpret {
+            receivers: self.receivers.clone(),
+        })
+    }
+}
+
+#[derive(Debug)]
+pub struct OpenshiftClusterAlertInterpret {
+    receivers: Vec<Box<dyn AlertReceiver<OpenshiftClusterAlertSender>>>,
+}
+
+#[async_trait]
+impl<T: Topology + K8sclient> Interpret<T> for OpenshiftClusterAlertInterpret {
+    async fn execute(
+        &self,
+        _inventory: &Inventory,
+        topology: &T,
+    ) -> Result<Outcome, InterpretError> {
+        let client = topology.k8s_client().await?;
+        let openshift_monitoring_namespace = "openshift-monitoring";
+
+        let mut alertmanager_main_secret: DynamicObject = client
+            .get_secret_json_value("alertmanager-main", Some(openshift_monitoring_namespace))
+            .await?;
+        trace!("Got secret {alertmanager_main_secret:#?}");
+
+        let data: &mut serde_json::Value = &mut alertmanager_main_secret.data;
+        trace!("Alertmanager-main secret data {data:#?}");
+        let data_obj = data
+            .get_mut("data")
+            .ok_or(InterpretError::new(
+                "Missing 'data' field in alertmanager-main secret.".to_string(),
+            ))?
+            .as_object_mut()
+            .ok_or(InterpretError::new(
+                "'data' field in alertmanager-main secret is expected to be an object ."
+                    .to_string(),
+            ))?;
+
+        let config_b64 = data_obj
+            .get("alertmanager.yaml")
+            .ok_or(InterpretError::new(
+                "Missing 'alertmanager.yaml' in alertmanager-main secret data".to_string(),
+            ))?
+            .as_str()
+            .unwrap_or("");
+        trace!("Config base64 {config_b64}");
+
+        let config_bytes = BASE64_STANDARD.decode(config_b64).unwrap_or_default();
+
+        let mut am_config: serde_yaml::Value =
+            serde_yaml::from_str(&String::from_utf8(config_bytes).unwrap_or_default())
+                .unwrap_or_default();
+
+        debug!("Current alertmanager config {am_config:#?}");
+
+        let existing_receivers_sequence = if let Some(receivers) = am_config.get_mut("receivers") {
+            match receivers.as_sequence_mut() {
+                Some(seq) => seq,
+                None => {
+                    return Err(InterpretError::new(format!(
+                        "Expected alertmanager config receivers to be a sequence, got {:?}",
+                        receivers
+                    )));
+                }
+            }
+        } else {
+            &mut serde_yaml::Sequence::default()
+        };
+
+        let mut additional_resources = vec![];
+
+        for custom_receiver in &self.receivers {
+            let name = custom_receiver.name();
+            let alertmanager_receiver = custom_receiver.as_alertmanager_receiver()?;
+
+            let json_value = alertmanager_receiver.receiver_config;
+
+            let yaml_string = serde_json::to_string(&json_value).map_err(|e| {
+                InterpretError::new(format!("Failed to serialize receiver config: {}", e))
+            })?;
+
+            let yaml_value: serde_yaml::Value =
+                serde_yaml::from_str(&yaml_string).map_err(|e| {
+                    InterpretError::new(format!("Failed to parse receiver config as YAML: {}", e))
+                })?;
+
+            if let Some(idx) = existing_receivers_sequence.iter().position(|r| {
+                r.get("name")
+                    .and_then(|n| n.as_str())
+                    .map_or(false, |n| n == name)
+            }) {
+                info!("Replacing existing AlertManager receiver: {}", name);
+                existing_receivers_sequence[idx] = yaml_value;
+            } else {
+                debug!("Adding new AlertManager receiver: {}", name);
+                existing_receivers_sequence.push(yaml_value);
+            }
+
+            additional_resources.push(alertmanager_receiver.additional_ressources);
+        }
+
+        debug!("Current alertmanager config {am_config:#?}");
+        // TODO
+        // - save new version of alertmanager config
+        // - write additional ressources to the cluster
+        let am_config = serde_yaml::to_string(&am_config).map_err(|e| {
+            InterpretError::new(format!(
+                "Failed to serialize new alertmanager config to string : {e}"
+            ))
+        })?;
+
+        let mut am_config_b64 = String::new();
+        BASE64_STANDARD.encode_string(am_config, &mut am_config_b64);
+
+        // TODO put update configmap value and save new value
+        data_obj.insert(
+            "alertmanager.yaml".to_string(),
+            serde_json::Value::String(am_config_b64),
+        );
+
+        // https://kubernetes.io/docs/reference/using-api/server-side-apply/#field-management
+        alertmanager_main_secret.metadata.managed_fields = None;
+
+        trace!("Applying new alertmanager_main_secret {alertmanager_main_secret:#?}");
+        client
+            .apply_dynamic(
+                &alertmanager_main_secret,
+                Some(openshift_monitoring_namespace),
+                true,
+            )
+            .await?;
+
+        let additional_resources = additional_resources.concat();
+        trace!("Applying additional ressources for alert receivers {additional_resources:#?}");
+        client
+            .apply_dynamic_many(
+                &additional_resources,
+                Some(openshift_monitoring_namespace),
+                true,
+            )
+            .await?;
+
+        Ok(Outcome::success(format!(
+            "Successfully configured {} cluster alert receivers: {}",
+            self.receivers.len(),
+            self.receivers
+                .iter()
+                .map(|r| r.name())
+                .collect::<Vec<_>>()
+                .join(", ")
+        )))
+    }
+
+    fn get_name(&self) -> InterpretName {
+        InterpretName::Custom("OpenshiftClusterAlertInterpret")
+    }
+
+    fn get_version(&self) -> Version {
+        todo!()
+    }
+
+    fn get_status(&self) -> InterpretStatus {
+        todo!()
+    }
+
+    fn get_children(&self) -> Vec<Id> {
+        todo!()
+    }
+}

harmony/src/modules/monitoring/okd/config.rs

@@ -0,0 +1,90 @@
+use std::{collections::BTreeMap, sync::Arc};
+
+use crate::{
+    interpret::{InterpretError, Outcome},
+    topology::k8s::K8sClient,
+};
+use k8s_openapi::api::core::v1::ConfigMap;
+use kube::api::ObjectMeta;
+
+pub(crate) struct Config;
+
+impl Config {
+    pub async fn create_cluster_monitoring_config_cm(
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let mut data = BTreeMap::new();
+        data.insert(
+            "config.yaml".to_string(),
+            r#"
+enableUserWorkload: true
+alertmanagerMain:
+  enableUserAlertmanagerConfig: true
+"#
+            .to_string(),
+        );
+
+        let cm = ConfigMap {
+            metadata: ObjectMeta {
+                name: Some("cluster-monitoring-config".to_string()),
+                namespace: Some("openshift-monitoring".to_string()),
+                ..Default::default()
+            },
+            data: Some(data),
+            ..Default::default()
+        };
+        client.apply(&cm, Some("openshift-monitoring")).await?;
+
+        Ok(Outcome::success(
+            "updated cluster-monitoring-config-map".to_string(),
+        ))
+    }
+
+    pub async fn create_user_workload_monitoring_config_cm(
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let mut data = BTreeMap::new();
+        data.insert(
+            "config.yaml".to_string(),
+            r#"
+alertmanager: 
+  enabled: true
+  enableAlertmanagerConfig: true
+"#
+            .to_string(),
+        );
+        let cm = ConfigMap {
+            metadata: ObjectMeta {
+                name: Some("user-workload-monitoring-config".to_string()),
+                namespace: Some("openshift-user-workload-monitoring".to_string()),
+                ..Default::default()
+            },
+            data: Some(data),
+            ..Default::default()
+        };
+        client
+            .apply(&cm, Some("openshift-user-workload-monitoring"))
+            .await?;
+
+        Ok(Outcome::success(
+            "updated openshift-user-monitoring-config-map".to_string(),
+        ))
+    }
+
+    pub async fn verify_user_workload(client: &Arc<K8sClient>) -> Result<Outcome, InterpretError> {
+        let namespace = "openshift-user-workload-monitoring";
+        let alertmanager_name = "alertmanager-user-workload-0";
+        let prometheus_name = "prometheus-user-workload-0";
+        client
+            .wait_for_pod_ready(alertmanager_name, Some(namespace))
+            .await?;
+        client
+            .wait_for_pod_ready(prometheus_name, Some(namespace))
+            .await?;
+
+        Ok(Outcome::success(format!(
+            "pods: {}, {} ready in ns: {}",
+            alertmanager_name, prometheus_name, namespace
+        )))
+    }
+}

harmony/src/modules/monitoring/okd/enable_user_workload.rs

@@ -1,16 +1,13 @@
-use std::{collections::BTreeMap, sync::Arc};
-
 use crate::{
     data::Version,
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::Inventory,
+    modules::monitoring::okd::config::Config,
     score::Score,
-    topology::{K8sclient, Topology, k8s::K8sClient},
+    topology::{K8sclient, Topology},
 };
 use async_trait::async_trait;
 use harmony_types::id::Id;
-use k8s_openapi::api::core::v1::ConfigMap;
-use kube::api::ObjectMeta;
 use serde::Serialize;
 
 #[derive(Clone, Debug, Serialize)]
@@ -37,10 +34,9 @@ impl<T: Topology + K8sclient> Interpret<T> for OpenshiftUserWorkloadMonitoringIn
         topology: &T,
     ) -> Result<Outcome, InterpretError> {
         let client = topology.k8s_client().await.unwrap();
-        self.update_cluster_monitoring_config_cm(&client).await?;
-        self.update_user_workload_monitoring_config_cm(&client)
-            .await?;
-        self.verify_user_workload(&client).await?;
+        Config::create_cluster_monitoring_config_cm(&client).await?;
+        Config::create_user_workload_monitoring_config_cm(&client).await?;
+        Config::verify_user_workload(&client).await?;
         Ok(Outcome::success(
             "successfully enabled user-workload-monitoring".to_string(),
         ))
@@ -62,88 +58,3 @@ impl<T: Topology + K8sclient> Interpret<T> for OpenshiftUserWorkloadMonitoringIn
         todo!()
     }
 }
-
-impl OpenshiftUserWorkloadMonitoringInterpret {
-    pub async fn update_cluster_monitoring_config_cm(
-        &self,
-        client: &Arc<K8sClient>,
-    ) -> Result<Outcome, InterpretError> {
-        let mut data = BTreeMap::new();
-        data.insert(
-            "config.yaml".to_string(),
-            r#"
-enableUserWorkload: true
-alertmanagerMain:
-  enableUserAlertmanagerConfig: true
-"#
-            .to_string(),
-        );
-
-        let cm = ConfigMap {
-            metadata: ObjectMeta {
-                name: Some("cluster-monitoring-config".to_string()),
-                namespace: Some("openshift-monitoring".to_string()),
-                ..Default::default()
-            },
-            data: Some(data),
-            ..Default::default()
-        };
-        client.apply(&cm, Some("openshift-monitoring")).await?;
-
-        Ok(Outcome::success(
-            "updated cluster-monitoring-config-map".to_string(),
-        ))
-    }
-
-    pub async fn update_user_workload_monitoring_config_cm(
-        &self,
-        client: &Arc<K8sClient>,
-    ) -> Result<Outcome, InterpretError> {
-        let mut data = BTreeMap::new();
-        data.insert(
-            "config.yaml".to_string(),
-            r#"
-alertmanager: 
-  enabled: true
-  enableAlertmanagerConfig: true
-"#
-            .to_string(),
-        );
-        let cm = ConfigMap {
-            metadata: ObjectMeta {
-                name: Some("user-workload-monitoring-config".to_string()),
-                namespace: Some("openshift-user-workload-monitoring".to_string()),
-                ..Default::default()
-            },
-            data: Some(data),
-            ..Default::default()
-        };
-        client
-            .apply(&cm, Some("openshift-user-workload-monitoring"))
-            .await?;
-
-        Ok(Outcome::success(
-            "updated openshift-user-monitoring-config-map".to_string(),
-        ))
-    }
-
-    pub async fn verify_user_workload(
-        &self,
-        client: &Arc<K8sClient>,
-    ) -> Result<Outcome, InterpretError> {
-        let namespace = "openshift-user-workload-monitoring";
-        let alertmanager_name = "alertmanager-user-workload-0";
-        let prometheus_name = "prometheus-user-workload-0";
-        client
-            .wait_for_pod_ready(alertmanager_name, Some(namespace))
-            .await?;
-        client
-            .wait_for_pod_ready(prometheus_name, Some(namespace))
-            .await?;
-
-        Ok(Outcome::success(format!(
-            "pods: {}, {} ready in ns: {}",
-            alertmanager_name, prometheus_name, namespace
-        )))
-    }
-}

harmony/src/modules/monitoring/okd/mod.rs

@@ -1 +1,14 @@
+use crate::topology::oberservability::monitoring::AlertSender;
+
+pub mod cluster_monitoring;
+pub(crate) mod config;
 pub mod enable_user_workload;
+
+#[derive(Debug)]
+pub struct OpenshiftClusterAlertSender;
+
+impl AlertSender for OpenshiftClusterAlertSender {
+    fn name(&self) -> String {
+        "OpenshiftClusterAlertSender".to_string()
+    }
+}

harmony/src/modules/monitoring/prometheus/prometheus.rs

@@ -114,7 +114,7 @@ impl Prometheus {
         };
 
         if let Some(ns) = namespace.as_deref() {
-            grafana_helm_chart_score(ns)
+            grafana_helm_chart_score(ns, false)
                 .interpret(inventory, topology)
                 .await
         } else {

harmony/src/modules/monitoring/scrape_target/mod.rs

@@ -0,0 +1 @@
+pub mod server;

harmony/src/modules/monitoring/scrape_target/server.rs

@@ -0,0 +1,80 @@
+use std::net::IpAddr;
+
+use async_trait::async_trait;
+use kube::api::ObjectMeta;
+use serde::Serialize;
+
+use crate::{
+    interpret::{InterpretError, Outcome},
+    modules::monitoring::kube_prometheus::crd::{
+        crd_alertmanager_config::CRDPrometheus,
+        crd_scrape_config::{Params, RelabelConfig, ScrapeConfig, ScrapeConfigSpec, StaticConfig},
+    },
+    topology::oberservability::monitoring::ScrapeTarget,
+};
+
+#[derive(Debug, Clone, Serialize)]
+pub struct Server {
+    pub name: String,
+    pub ip: IpAddr,
+    pub auth: String,
+    pub module: String,
+    pub domain: String,
+}
+
+#[async_trait]
+impl ScrapeTarget<CRDPrometheus> for Server {
+    async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
+        let scrape_config_spec = ScrapeConfigSpec {
+            static_configs: Some(vec![StaticConfig {
+                targets: vec![self.ip.to_string()],
+                labels: None,
+            }]),
+            scrape_interval: Some("2m".to_string()),
+            kubernetes_sd_configs: None,
+            http_sd_configs: None,
+            file_sd_configs: None,
+            dns_sd_configs: None,
+            params: Some(Params {
+                auth: Some(vec![self.auth.clone()]),
+                module: Some(vec![self.module.clone()]),
+            }),
+            consul_sd_configs: None,
+            relabel_configs: Some(vec![RelabelConfig {
+                action: None,
+                source_labels: Some(vec!["__address__".to_string()]),
+                separator: None,
+                target_label: Some("__param_target".to_string()),
+                regex: None,
+                replacement: Some(format!("snmp.{}:31080", self.domain.clone())),
+                modulus: None,
+            }]),
+            metric_relabel_configs: None,
+            metrics_path: Some("/snmp".to_string()),
+            scrape_timeout: Some("2m".to_string()),
+            job_name: Some(format!("snmp_exporter/cloud/{}", self.name.clone())),
+            scheme: None,
+        };
+
+        let scrape_config = ScrapeConfig {
+            metadata: ObjectMeta {
+                name: Some(self.name.clone()),
+                namespace: Some(sender.namespace.clone()),
+                ..Default::default()
+            },
+            spec: scrape_config_spec,
+        };
+        sender
+            .client
+            .apply(&scrape_config, Some(&sender.namespace.clone()))
+            .await?;
+        Ok(Outcome::success(format!(
+            "installed scrape target {}",
+            self.name.clone()
+        )))
+    }
+
+    fn clone_box(&self) -> Box<dyn ScrapeTarget<CRDPrometheus>> {
+        Box::new(self.clone())
+    }
+}

harmony/src/modules/okd/bootstrap_03_control_plane.rs

@@ -5,10 +5,8 @@ use crate::{
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::{HostRole, Inventory},
     modules::{
-        dhcp::DhcpHostBindingScore,
-        http::IPxeMacBootFileScore,
-        inventory::DiscoverHostForRoleScore,
-        okd::{host_network::HostNetworkConfigurationScore, templates::BootstrapIpxeTpl},
+        dhcp::DhcpHostBindingScore, http::IPxeMacBootFileScore,
+        inventory::DiscoverHostForRoleScore, okd::templates::BootstrapIpxeTpl,
     },
     score::Score,
     topology::{HAClusterTopology, HostBinding},
@@ -205,28 +203,6 @@ impl OKDSetup03ControlPlaneInterpret {
 
         Ok(())
     }
-
-    /// Placeholder for automating network bonding configuration.
-    async fn persist_network_bond(
-        &self,
-        inventory: &Inventory,
-        topology: &HAClusterTopology,
-        hosts: &Vec<PhysicalHost>,
-    ) -> Result<(), InterpretError> {
-        info!("[ControlPlane] Ensuring persistent bonding");
-        let score = HostNetworkConfigurationScore {
-            hosts: hosts.clone(),
-        };
-        score.interpret(inventory, topology).await?;
-
-        inquire::Confirm::new(
-            "Network configuration for control plane nodes is not automated yet. Configure it manually if needed.",
-        )
-        .prompt()
-        .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?;
-
-        Ok(())
-    }
 }
 
 #[async_trait]
@@ -265,10 +241,6 @@ impl Interpret<HAClusterTopology> for OKDSetup03ControlPlaneInterpret {
         // 4. Reboot the nodes to start the OS installation.
         self.reboot_targets(&nodes).await?;
 
-        // 5. Placeholder for post-boot network configuration (e.g., bonding).
-        self.persist_network_bond(inventory, topology, &nodes)
-            .await?;
-
         // TODO: Implement a step to wait for the control plane nodes to join the cluster
         // and for the cluster operators to become available. This would be similar to
         // the `wait-for bootstrap-complete` command.

harmony/src/modules/okd/bootstrap_persist_network_bond.rs

@@ -0,0 +1,130 @@
+use crate::{
+    data::Version,
+    hardware::PhysicalHost,
+    infra::inventory::InventoryRepositoryFactory,
+    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
+    inventory::{HostRole, Inventory},
+    modules::okd::host_network::HostNetworkConfigurationScore,
+    score::Score,
+    topology::HAClusterTopology,
+};
+use async_trait::async_trait;
+use derive_new::new;
+use harmony_types::id::Id;
+use log::info;
+use serde::Serialize;
+
+// -------------------------------------------------------------------------------------------------
+// Persist Network Bond
+// - Persist bonding via NMState
+// - Persist port channels on the Switch
+// -------------------------------------------------------------------------------------------------
+
+#[derive(Debug, Clone, Serialize, new)]
+pub struct OKDSetupPersistNetworkBondScore {}
+
+impl Score<HAClusterTopology> for OKDSetupPersistNetworkBondScore {
+    fn create_interpret(&self) -> Box<dyn Interpret<HAClusterTopology>> {
+        Box::new(OKDSetupPersistNetworkBondInterpet::new())
+    }
+
+    fn name(&self) -> String {
+        "OKDSetupPersistNetworkBondScore".to_string()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct OKDSetupPersistNetworkBondInterpet {
+    version: Version,
+    status: InterpretStatus,
+}
+
+impl OKDSetupPersistNetworkBondInterpet {
+    pub fn new() -> Self {
+        let version = Version::from("1.0.0").unwrap();
+        Self {
+            version,
+            status: InterpretStatus::QUEUED,
+        }
+    }
+
+    /// Ensures that three physical hosts are discovered and available for the ControlPlane role.
+    /// It will trigger discovery if not enough hosts are found.
+    async fn get_nodes(
+        &self,
+        _inventory: &Inventory,
+        _topology: &HAClusterTopology,
+    ) -> Result<Vec<PhysicalHost>, InterpretError> {
+        const REQUIRED_HOSTS: usize = 3;
+        let repo = InventoryRepositoryFactory::build().await?;
+        let control_plane_hosts = repo.get_host_for_role(&HostRole::ControlPlane).await?;
+
+        if control_plane_hosts.len() < REQUIRED_HOSTS {
+            Err(InterpretError::new(format!(
+                "OKD Requires at least {} control plane hosts, but only found {}. Cannot proceed.",
+                REQUIRED_HOSTS,
+                control_plane_hosts.len()
+            )))
+        } else {
+            // Take exactly the number of required hosts to ensure consistency.
+            Ok(control_plane_hosts
+                .into_iter()
+                .take(REQUIRED_HOSTS)
+                .collect())
+        }
+    }
+
+    async fn persist_network_bond(
+        &self,
+        inventory: &Inventory,
+        topology: &HAClusterTopology,
+        hosts: &Vec<PhysicalHost>,
+    ) -> Result<(), InterpretError> {
+        info!("Ensuring persistent bonding");
+
+        let score = HostNetworkConfigurationScore {
+            hosts: hosts.clone(),
+        };
+        score.interpret(inventory, topology).await?;
+
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl Interpret<HAClusterTopology> for OKDSetupPersistNetworkBondInterpet {
+    fn get_name(&self) -> InterpretName {
+        InterpretName::Custom("OKDSetupPersistNetworkBondInterpet")
+    }
+
+    fn get_version(&self) -> Version {
+        self.version.clone()
+    }
+
+    fn get_status(&self) -> InterpretStatus {
+        self.status.clone()
+    }
+
+    fn get_children(&self) -> Vec<Id> {
+        vec![]
+    }
+
+    async fn execute(
+        &self,
+        inventory: &Inventory,
+        topology: &HAClusterTopology,
+    ) -> Result<Outcome, InterpretError> {
+        let nodes = self.get_nodes(inventory, topology).await?;
+
+        let res = self.persist_network_bond(inventory, topology, &nodes).await;
+
+        match res {
+            Ok(_) => Ok(Outcome::success(
+                "Network bond successfully persisted".into(),
+            )),
+            Err(_) => Err(InterpretError::new(
+                "Failed to persist network bond".to_string(),
+            )),
+        }
+    }
+}

harmony/src/modules/okd/crd/mod.rs

@@ -1,41 +1 @@
-use kube::CustomResource;
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
-
 pub mod nmstate;
-
-#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[kube(
-    group = "operators.coreos.com",
-    version = "v1",
-    kind = "OperatorGroup",
-    namespaced
-)]
-#[serde(rename_all = "camelCase")]
-pub struct OperatorGroupSpec {
-    pub target_namespaces: Vec<String>,
-}
-
-#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[kube(
-    group = "operators.coreos.com",
-    version = "v1alpha1",
-    kind = "Subscription",
-    namespaced
-)]
-#[serde(rename_all = "camelCase")]
-pub struct SubscriptionSpec {
-    pub name: String,
-    pub source: String,
-    pub source_namespace: String,
-    pub channel: Option<String>,
-    pub install_plan_approval: Option<InstallPlanApproval>,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-pub enum InstallPlanApproval {
-    #[serde(rename = "Automatic")]
-    Automatic,
-    #[serde(rename = "Manual")]
-    Manual,
-}

harmony/src/modules/okd/crd/nmstate.rs

@@ -6,9 +6,16 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[kube(group = "nmstate.io", version = "v1", kind = "NMState", namespaced)]
+#[kube(
+    group = "nmstate.io",
+    version = "v1",
+    kind = "NMState",
+    plural = "nmstates",
+    namespaced = false
+)]
 #[serde(rename_all = "camelCase")]
 pub struct NMStateSpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub probe_configuration: Option<ProbeConfig>,
 }
 
@@ -44,6 +51,7 @@ pub struct ProbeDns {
 )]
 #[serde(rename_all = "camelCase")]
 pub struct NodeNetworkConfigurationPolicySpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub node_selector: Option<BTreeMap<String, String>>,
     pub desired_state: DesiredStateSpec,
 }
@@ -58,37 +66,64 @@ pub struct DesiredStateSpec {
 #[serde(rename_all = "kebab-case")]
 pub struct InterfaceSpec {
     pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub description: Option<String>,
     pub r#type: String,
     pub state: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mac_address: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub copy_mac_from: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mtu: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub controller: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ipv4: Option<IpStackSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ipv6: Option<IpStackSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ethernet: Option<EthernetSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub link_aggregation: Option<BondSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub vlan: Option<VlanSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub vxlan: Option<VxlanSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mac_vtap: Option<MacVtapSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mac_vlan: Option<MacVlanSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub infiniband: Option<InfinibandSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub linux_bridge: Option<LinuxBridgeSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ovs_bridge: Option<OvsBridgeSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ethtool: Option<EthtoolSpec>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct IpStackSpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub dhcp: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub autoconf: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub address: Option<Vec<IpAddressSpec>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub auto_dns: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub auto_gateway: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub auto_routes: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub dhcp_client_id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub dhcp_duid: Option<String>,
 }
 
@@ -102,8 +137,11 @@ pub struct IpAddressSpec {
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct EthernetSpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub speed: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub duplex: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub auto_negotiation: Option<bool>,
 }
 
@@ -112,6 +150,7 @@ pub struct EthernetSpec {
 pub struct BondSpec {
     pub mode: String,
     pub ports: Vec<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub options: Option<BTreeMap<String, Value>>,
 }
 
@@ -120,6 +159,7 @@ pub struct BondSpec {
 pub struct VlanSpec {
     pub base_iface: String,
     pub id: u16,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub protocol: Option<String>,
 }
 
@@ -129,8 +169,11 @@ pub struct VxlanSpec {
     pub base_iface: String,
     pub id: u32,
     pub remote: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub local: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub learning: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub destination_port: Option<u16>,
 }
 
@@ -139,6 +182,7 @@ pub struct VxlanSpec {
 pub struct MacVtapSpec {
     pub base_iface: String,
     pub mode: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub promiscuous: Option<bool>,
 }
 
@@ -147,6 +191,7 @@ pub struct MacVtapSpec {
 pub struct MacVlanSpec {
     pub base_iface: String,
     pub mode: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub promiscuous: Option<bool>,
 }
 
@@ -161,25 +206,35 @@ pub struct InfinibandSpec {
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct LinuxBridgeSpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub options: Option<LinuxBridgeOptions>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ports: Option<Vec<LinuxBridgePort>>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct LinuxBridgeOptions {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mac_ageing_time: Option<u32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub multicast_snooping: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub stp: Option<StpOptions>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct StpOptions {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub forward_delay: Option<u16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub hello_time: Option<u16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub max_age: Option<u16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub priority: Option<u16>,
 }
 
@@ -187,15 +242,20 @@ pub struct StpOptions {
 #[serde(rename_all = "kebab-case")]
 pub struct LinuxBridgePort {
     pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub vlan: Option<LinuxBridgePortVlan>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct LinuxBridgePortVlan {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mode: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub trunk_tags: Option<Vec<VlanTag>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub tag: Option<u16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enable_native: Option<bool>,
 }
 
@@ -203,6 +263,7 @@ pub struct LinuxBridgePortVlan {
 #[serde(rename_all = "kebab-case")]
 pub struct VlanTag {
     pub id: u16,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub id_range: Option<VlanIdRange>,
 }
 
@@ -216,15 +277,20 @@ pub struct VlanIdRange {
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct OvsBridgeSpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub options: Option<OvsBridgeOptions>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ports: Option<Vec<OvsPortSpec>>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct OvsBridgeOptions {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub stp: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub rstp: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mcast_snooping_enable: Option<bool>,
 }
 
@@ -232,8 +298,11 @@ pub struct OvsBridgeOptions {
 #[serde(rename_all = "kebab-case")]
 pub struct OvsPortSpec {
     pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub link_aggregation: Option<BondSpec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub vlan: Option<LinuxBridgePortVlan>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub r#type: Option<String>,
 }
 
@@ -246,6 +315,8 @@ pub struct EthtoolSpec {
 #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)]
 #[serde(rename_all = "kebab-case")]
 pub struct EthtoolFecSpec {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub auto: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mode: Option<String>,
 }

harmony/src/modules/okd/host_network.rs

@@ -39,30 +39,70 @@ impl HostNetworkConfigurationInterpret {
         &self,
         topology: &T,
         host: &PhysicalHost,
-    ) -> Result<(), InterpretError> {
-        let switch_ports = self.collect_switch_ports_for_host(topology, host).await?;
-        if !switch_ports.is_empty() {
-            topology
-                .configure_host_network(host, HostNetworkConfig { switch_ports })
-                .await
-                .map_err(|e| InterpretError::new(format!("Failed to configure host: {e}")))?;
+        current_host: &usize,
+        total_hosts: &usize,
+    ) -> Result<HostNetworkConfig, InterpretError> {
+        if host.network.is_empty() {
+            info!("[Host {current_host}/{total_hosts}] No interfaces to configure, skipping");
+            return Ok(HostNetworkConfig {
+                host_id: host.id.clone(),
+                switch_ports: vec![],
+            });
         }
 
-        Ok(())
+        let switch_ports = self
+            .collect_switch_ports_for_host(topology, host, current_host, total_hosts)
+            .await?;
+
+        let config = HostNetworkConfig {
+            host_id: host.id.clone(),
+            switch_ports,
+        };
+
+        if !config.switch_ports.is_empty() {
+            info!(
+                "[Host {current_host}/{total_hosts}] Found {} ports for {} interfaces",
+                config.switch_ports.len(),
+                host.network.len()
+            );
+
+            info!("[Host {current_host}/{total_hosts}] Configuring host network...");
+            topology
+                .configure_host_network(&config)
+                .await
+                .map_err(|e| InterpretError::new(format!("Failed to configure host: {e}")))?;
+        } else {
+            info!(
+                "[Host {current_host}/{total_hosts}] No ports found for {} interfaces, skipping",
+                host.network.len()
+            );
+        }
+
+        Ok(config)
     }
 
     async fn collect_switch_ports_for_host<T: Topology + Switch>(
         &self,
         topology: &T,
         host: &PhysicalHost,
+        current_host: &usize,
+        total_hosts: &usize,
     ) -> Result<Vec<SwitchPort>, InterpretError> {
         let mut switch_ports = vec![];
 
+        if host.network.is_empty() {
+            return Ok(switch_ports);
+        }
+
+        info!("[Host {current_host}/{total_hosts}] Collecting ports on switch...");
         for network_interface in &host.network {
             let mac_address = network_interface.mac_address;
 
             match topology.get_port_for_mac_address(&mac_address).await {
                 Ok(Some(port)) => {
+                    info!(
+                        "[Host {current_host}/{total_hosts}] Found port '{port}' for '{mac_address}'"
+                    );
                     switch_ports.push(SwitchPort {
                         interface: NetworkInterface {
                             name: network_interface.name.clone(),
@@ -73,7 +113,7 @@ impl HostNetworkConfigurationInterpret {
                         port,
                     });
                 }
-                Ok(None) => debug!("No port found for host '{}', skipping", host.id),
+                Ok(None) => debug!("No port found for '{mac_address}', skipping"),
                 Err(e) => {
                     return Err(InterpretError::new(format!(
                         "Failed to get port for host '{}': {}",
@@ -85,6 +125,47 @@ impl HostNetworkConfigurationInterpret {
 
         Ok(switch_ports)
     }
+
+    fn format_host_configuration(&self, configs: Vec<HostNetworkConfig>) -> Vec<String> {
+        let mut report = vec![
+            "Network Configuration Report".to_string(),
+            "------------------------------------------------------------------".to_string(),
+        ];
+
+        for config in configs {
+            let host = self
+                .score
+                .hosts
+                .iter()
+                .find(|h| h.id == config.host_id)
+                .unwrap();
+
+            println!("[Host] {host}");
+
+            if config.switch_ports.is_empty() {
+                report.push(format!(
+                    "⏭️ Host {}: SKIPPED (No matching switch ports found)",
+                    config.host_id
+                ));
+            } else {
+                let mappings: Vec<String> = config
+                    .switch_ports
+                    .iter()
+                    .map(|p| format!("[{} -> {}]", p.interface.name, p.port))
+                    .collect();
+
+                report.push(format!(
+                    "✅ Host {}: Bonded {} port(s) {}",
+                    config.host_id,
+                    config.switch_ports.len(),
+                    mappings.join(", ")
+                ));
+            }
+        }
+        report
+            .push("------------------------------------------------------------------".to_string());
+        report
+    }
 }
 
 #[async_trait]
@@ -114,27 +195,38 @@ impl<T: Topology + Switch> Interpret<T> for HostNetworkConfigurationInterpret {
             return Ok(Outcome::noop("No hosts to configure".into()));
         }
 
-        info!(
-            "Started network configuration for {} host(s)...",
-            self.score.hosts.len()
-        );
+        let host_count = self.score.hosts.len();
+        info!("Started network configuration for {host_count} host(s)...",);
 
+        info!("Setting up switch with sane defaults...");
         topology
             .setup_switch()
             .await
             .map_err(|e| InterpretError::new(format!("Switch setup failed: {e}")))?;
+        info!("Switch ready");
+
+        let mut current_host = 1;
+        let mut host_configurations = vec![];
 
-        let mut configured_host_count = 0;
         for host in &self.score.hosts {
-            self.configure_network_for_host(topology, host).await?;
-            configured_host_count += 1;
-        }
+            let host_configuration = self
+                .configure_network_for_host(topology, host, &current_host, &host_count)
+                .await?;
 
-        if configured_host_count > 0 {
-            Ok(Outcome::success(format!(
-                "Configured {configured_host_count}/{} host(s)",
-                self.score.hosts.len()
-            )))
+            host_configurations.push(host_configuration);
+            current_host += 1;
+        }
+        if current_host > 1 {
+            let details = self.format_host_configuration(host_configurations);
+
+            Ok(Outcome::success_with_details(
+                format!(
+                    "Configured {}/{} host(s)",
+                    current_host - 1,
+                    self.score.hosts.len()
+                ),
+                details,
+            ))
         } else {
             Ok(Outcome::noop("No hosts configured".into()))
         }
@@ -209,6 +301,7 @@ mod tests {
         assert_that!(*configured_host_networks).contains_exactly(vec![(
             HOST_ID.clone(),
             HostNetworkConfig {
+                host_id: HOST_ID.clone(),
                 switch_ports: vec![SwitchPort {
                     interface: EXISTING_INTERFACE.clone(),
                     port: PORT.clone(),
@@ -234,6 +327,7 @@ mod tests {
         assert_that!(*configured_host_networks).contains_exactly(vec![(
             HOST_ID.clone(),
             HostNetworkConfig {
+                host_id: HOST_ID.clone(),
                 switch_ports: vec![
                     SwitchPort {
                         interface: EXISTING_INTERFACE.clone(),
@@ -263,6 +357,7 @@ mod tests {
             (
                 HOST_ID.clone(),
                 HostNetworkConfig {
+                    host_id: HOST_ID.clone(),
                     switch_ports: vec![SwitchPort {
                         interface: EXISTING_INTERFACE.clone(),
                         port: PORT.clone(),
@@ -272,6 +367,7 @@ mod tests {
             (
                 ANOTHER_HOST_ID.clone(),
                 HostNetworkConfig {
+                    host_id: ANOTHER_HOST_ID.clone(),
                     switch_ports: vec![SwitchPort {
                         interface: ANOTHER_EXISTING_INTERFACE.clone(),
                         port: ANOTHER_PORT.clone(),
@@ -382,11 +478,10 @@ mod tests {
 
         async fn configure_host_network(
             &self,
-            host: &PhysicalHost,
-            config: HostNetworkConfig,
+            config: &HostNetworkConfig,
         ) -> Result<(), SwitchError> {
             let mut configured_host_networks = self.configured_host_networks.lock().unwrap();
-            configured_host_networks.push((host.id.clone(), config.clone()));
+            configured_host_networks.push((config.host_id.clone(), config.clone()));
 
             Ok(())
         }

harmony/src/modules/okd/installation.rs

@@ -50,7 +50,7 @@
 use crate::{
     modules::okd::{
         OKDSetup01InventoryScore, OKDSetup02BootstrapScore, OKDSetup03ControlPlaneScore,
-        OKDSetup04WorkersScore, OKDSetup05SanityCheckScore,
+        OKDSetup04WorkersScore, OKDSetup05SanityCheckScore, OKDSetupPersistNetworkBondScore,
         bootstrap_06_installation_report::OKDSetup06InstallationReportScore,
     },
     score::Score,
@@ -65,6 +65,7 @@ impl OKDInstallationPipeline {
             Box::new(OKDSetup01InventoryScore::new()),
             Box::new(OKDSetup02BootstrapScore::new()),
             Box::new(OKDSetup03ControlPlaneScore::new()),
+            Box::new(OKDSetupPersistNetworkBondScore::new()),
             Box::new(OKDSetup04WorkersScore::new()),
             Box::new(OKDSetup05SanityCheckScore::new()),
             Box::new(OKDSetup06InstallationReportScore::new()),

harmony/src/modules/okd/mod.rs

@@ -6,6 +6,7 @@ mod bootstrap_05_sanity_check;
 mod bootstrap_06_installation_report;
 pub mod bootstrap_dhcp;
 pub mod bootstrap_load_balancer;
+mod bootstrap_persist_network_bond;
 pub mod dhcp;
 pub mod dns;
 pub mod installation;
@@ -19,5 +20,6 @@ pub use bootstrap_03_control_plane::*;
 pub use bootstrap_04_workers::*;
 pub use bootstrap_05_sanity_check::*;
 pub use bootstrap_06_installation_report::*;
+pub use bootstrap_persist_network_bond::*;
 pub mod crd;
 pub mod host_network;

harmony/src/modules/prometheus/k8s_prometheus_alerting_score.rs

@@ -12,7 +12,8 @@ use crate::modules::monitoring::kube_prometheus::crd::crd_alertmanager_config::C
 use crate::modules::monitoring::kube_prometheus::crd::crd_default_rules::build_default_application_rules;
 use crate::modules::monitoring::kube_prometheus::crd::crd_grafana::{
     Grafana, GrafanaDashboard, GrafanaDashboardSpec, GrafanaDatasource, GrafanaDatasourceConfig,
-    GrafanaDatasourceSpec, GrafanaSpec,
+    GrafanaDatasourceJsonData, GrafanaDatasourceSpec, GrafanaSecretKeyRef, GrafanaSpec,
+    GrafanaValueFrom, GrafanaValueSource,
 };
 use crate::modules::monitoring::kube_prometheus::crd::crd_prometheus_rules::{
     PrometheusRule, PrometheusRuleSpec, RuleGroup,
@@ -39,7 +40,7 @@ use crate::{
 };
 use harmony_types::id::Id;
 
-use super::prometheus::PrometheusApplicationMonitoring;
+use super::prometheus::PrometheusMonitoring;
 
 #[derive(Clone, Debug, Serialize)]
 pub struct K8sPrometheusCRDAlertingScore {
@@ -49,7 +50,7 @@ pub struct K8sPrometheusCRDAlertingScore {
     pub prometheus_rules: Vec<RuleGroup>,
 }
 
-impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> Score<T>
+impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus>> Score<T>
     for K8sPrometheusCRDAlertingScore
 {
     fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
@@ -75,7 +76,7 @@ pub struct K8sPrometheusCRDAlertingInterpret {
 }
 
 #[async_trait]
-impl<T: Topology + K8sclient + PrometheusApplicationMonitoring<CRDPrometheus>> Interpret<T>
+impl<T: Topology + K8sclient + PrometheusMonitoring<CRDPrometheus>> Interpret<T>
     for K8sPrometheusCRDAlertingInterpret
 {
     async fn execute(
@@ -466,10 +467,13 @@ impl K8sPrometheusCRDAlertingInterpret {
             match_labels: label.clone(),
             match_expressions: vec![],
         };
-        let mut json_data = BTreeMap::new();
-        json_data.insert("timeInterval".to_string(), "5s".to_string());
         let namespace = self.sender.namespace.clone();
-
+        let json_data = GrafanaDatasourceJsonData {
+            time_interval: Some("5s".to_string()),
+            http_header_name1: None,
+            tls_skip_verify: Some(true),
+            oauth_pass_thru: Some(true),
+        };
         let json = build_default_dashboard(&namespace);
 
         let graf_data_source = GrafanaDatasource {
@@ -495,7 +499,11 @@ impl K8sPrometheusCRDAlertingInterpret {
                         "http://prometheus-operated.{}.svc.cluster.local:9090",
                         self.sender.namespace.clone()
                     ),
+                    secure_json_data: None,
+                    is_default: None,
+                    editable: None,
                 },
+                values_from: None,
             },
         };
 
@@ -516,7 +524,9 @@ impl K8sPrometheusCRDAlertingInterpret {
             spec: GrafanaDashboardSpec {
                 resync_period: Some("30s".to_string()),
                 instance_selector: labels.clone(),
-                json,
+                json: Some(json),
+                grafana_com: None,
+                datasources: None,
             },
         };
 

harmony/src/modules/prometheus/prometheus.rs

@@ -9,11 +9,17 @@ use crate::{
 };
 
 #[async_trait]
-pub trait PrometheusApplicationMonitoring<S: AlertSender> {
+pub trait PrometheusMonitoring<S: AlertSender> {
     async fn install_prometheus(
         &self,
         sender: &S,
         inventory: &Inventory,
         receivers: Option<Vec<Box<dyn AlertReceiver<S>>>>,
     ) -> Result<PreparationOutcome, PreparationError>;
+
+    async fn ensure_prometheus_operator(
+        &self,
+        sender: &S,
+        inventory: &Inventory,
+    ) -> Result<PreparationOutcome, PreparationError>;
 }

harmony/src/modules/prometheus/rhob_alerting_score.rs

@@ -38,7 +38,7 @@ use crate::{
 };
 use harmony_types::id::Id;
 
-use super::prometheus::PrometheusApplicationMonitoring;
+use super::prometheus::PrometheusMonitoring;
 
 #[derive(Clone, Debug, Serialize)]
 pub struct RHOBAlertingScore {
@@ -48,8 +48,8 @@ pub struct RHOBAlertingScore {
     pub prometheus_rules: Vec<RuleGroup>,
 }
 
-impl<T: Topology + K8sclient + Ingress + PrometheusApplicationMonitoring<RHOBObservability>>
-    Score<T> for RHOBAlertingScore
+impl<T: Topology + K8sclient + Ingress + PrometheusMonitoring<RHOBObservability>> Score<T>
+    for RHOBAlertingScore
 {
     fn create_interpret(&self) -> Box<dyn crate::interpret::Interpret<T>> {
         Box::new(RHOBAlertingInterpret {
@@ -74,8 +74,8 @@ pub struct RHOBAlertingInterpret {
 }
 
 #[async_trait]
-impl<T: Topology + K8sclient + Ingress + PrometheusApplicationMonitoring<RHOBObservability>>
-    Interpret<T> for RHOBAlertingInterpret
+impl<T: Topology + K8sclient + Ingress + PrometheusMonitoring<RHOBObservability>> Interpret<T>
+    for RHOBAlertingInterpret
 {
     async fn execute(
         &self,

harmony_cli/src/cli_reporter.rs

@@ -40,7 +40,7 @@ pub fn init() {
                 HarmonyEvent::HarmonyFinished => {
                     if !details.is_empty() {
                         println!(
-                            "\n{} All done! Here's what's next for you:",
+                            "\n{} All done! Here's a few info for you:",
                             theme::EMOJI_SUMMARY
                         );
                         for detail in details.iter() {

opnsense-config-xml/src/data/interfaces.rs

@@ -9,7 +9,7 @@ pub struct Interface {
     pub physical_interface_name: String,
     pub descr: Option<MaybeString>,
     pub mtu: Option<MaybeString>,
-    pub enable: MaybeString,
+    pub enable: Option<MaybeString>,
     pub lock: Option<MaybeString>,
     #[yaserde(rename = "spoofmac")]
     pub spoof_mac: Option<MaybeString>,
@@ -134,19 +134,15 @@ mod test {
   <interfaces>
     <paul>
       <if></if>
-      <enable/>
     </paul>
     <anotherpaul>
       <if></if>
-      <enable/>
     </anotherpaul>
     <thirdone>
       <if></if>
-      <enable/>
     </thirdone>
     <andgofor4>
       <if></if>
-      <enable/>
     </andgofor4>
   </interfaces>
   <bar>foo</bar>