fix: added some extra details

Reviewed-on: #174

Cargo.lock

@@ -1780,6 +1780,7 @@ dependencies = [
 name = "example-nanodc"
 version = "0.1.0"
 dependencies = [
+ "brocade",
  "cidr",
  "env_logger",
  "harmony",
@@ -1788,6 +1789,7 @@ dependencies = [
  "harmony_tui",
  "harmony_types",
  "log",
+ "serde",
  "tokio",
  "url",
 ]
@@ -1806,6 +1808,7 @@ dependencies = [
 name = "example-okd-install"
 version = "0.1.0"
 dependencies = [
+ "brocade",
  "cidr",
  "env_logger",
  "harmony",
@@ -1836,13 +1839,16 @@ dependencies = [
 name = "example-opnsense"
 version = "0.1.0"
 dependencies = [
+ "brocade",
  "cidr",
  "env_logger",
  "harmony",
  "harmony_macros",
+ "harmony_secret",
  "harmony_tui",
  "harmony_types",
  "log",
+ "serde",
  "tokio",
  "url",
 ]
@@ -1851,6 +1857,7 @@ dependencies = [
 name = "example-pxe"
 version = "0.1.0"
 dependencies = [
+ "brocade",
  "cidr",
  "env_logger",
  "harmony",
@@ -1865,6 +1872,15 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "example-remove-rook-osd"
+version = "0.1.0"
+dependencies = [
+ "harmony",
+ "harmony_cli",
+ "tokio",
+]
+
 [[package]]
 name = "example-rust"
 version = "0.1.0"
@@ -1918,8 +1934,6 @@ dependencies = [
  "env_logger",
  "harmony",
  "harmony_macros",
- "harmony_secret",
- "harmony_secret_derive",
  "harmony_tui",
  "harmony_types",
  "log",

brocade/src/fast_iron.rs

@@ -10,6 +10,7 @@ use log::{debug, info};
 use regex::Regex;
 use std::{collections::HashSet, str::FromStr};
 
+#[derive(Debug)]
 pub struct FastIronClient {
     shell: BrocadeShell,
     version: BrocadeInfo,

brocade/src/lib.rs

@@ -162,7 +162,7 @@ pub async fn init(
 }
 
 #[async_trait]
-pub trait BrocadeClient {
+pub trait BrocadeClient: std::fmt::Debug {
     /// Retrieves the operating system and version details from the connected Brocade switch.
     ///
     /// This is typically the first call made after establishing a connection to determine

brocade/src/network_operating_system.rs

@@ -10,6 +10,7 @@ use crate::{
     parse_brocade_mac_address, shell::BrocadeShell,
 };
 
+#[derive(Debug)]
 pub struct NetworkOperatingSystemClient {
     shell: BrocadeShell,
     version: BrocadeInfo,

brocade/src/shell.rs

@@ -13,6 +13,7 @@ use log::info;
 use russh::ChannelMsg;
 use tokio::time::timeout;
 
+#[derive(Debug)]
 pub struct BrocadeShell {
     ip: IpAddr,
     port: u16,

docs/doc-clone-and-restore-coreos.md

@@ -0,0 +1,133 @@
+## Working procedure to clone and restore CoreOS disk from OKD Cluster
+
+### **Step 1 - take a backup**
+```
+sudo dd if=/dev/old of=/dev/backup status=progress 
+```
+
+### **Step 2 - clone beginning of old disk to new**
+```
+sudo dd if=/dev/old of=/dev/backup status=progress count=1000 bs=1M
+```
+
+### **Step 3 - verify and modify disk partitions**
+list disk partitions
+```
+sgdisk -p /dev/new
+```
+if new disk is smaller than old disk and there is space on the xfs partition of the old disk, modify partitions of new disk
+```
+gdisk /dev/new
+```
+inside of gdisk commands
+```
+-v -> verify table
+-p -> print table
+-d -> select partition to delete partition
+-n -> recreate partition with same partition number as deleted partition
+```
+For end sector, either specify the new end or just press Enter for maximum available 
+When asked about partition type, enter the same type code (it will show the old one) 
+```
+p - >to verify 
+w -> to write
+```
+make xfs file system for new partition <new4>
+```
+sudo mkfs.xfs -f /dev/new4
+```
+
+### **Step 4 - copy old PARTUUID **
+
+**careful here**
+get old patuuid:
+```
+sgdisk -i <partition_number> /dev/old_disk  # Note the "Partition unique GUID"
+```
+get labels
+```
+sgdisk -p /dev/old_disk  # Shows partition names in the table
+
+blkid /dev/old_disk*  # Shows PARTUUIDs and labels for all partitions
+```
+set it on new disk
+```
+sgdisk -u <partition_number>:<old_partuuid> /dev/sdc
+```
+partition name:
+```
+sgdisk -c <partition_number>:"<old_name>" /dev/sdc
+```
+verify all:
+```
+lsblk -o NAME,SIZE,PARTUUID,PARTLABEL /dev/old_disk
+```
+
+### **Step 5 - Mount disks and copy files from old to new disk**
+
+mount files before copy:
+
+```
+mkdir -p /mnt/new
+mkdir -p /mnt/old
+mount /dev/old4 /mnt/old
+mount /dev/new4 /mnt/new
+```
+copy:
+
+with -n flag can run as dry-run
+```
+rsync -aAXHvn --numeric-ids /source/ /destination/
+```
+
+```
+rsync -aAXHv --numeric-ids /source/ /destination/
+```
+
+### **Step 6 - Set correct UUID for new partition 4**
+to set uuid with xfs_admin you must unmount first
+
+unmount old devices
+```
+umount /mnt/new
+umount /mnt/old
+```
+
+to set correct uuid for partition 4
+```
+blkid /dev/old4
+```
+```
+xfs_admin -U <old_uuid> /dev/new_partition
+```
+to set labels
+get it 
+```
+sgdisk -i 4 /dev/sda | grep "Partition name"
+```
+set it
+```
+sgdisk -c 4:"<label_name>" /dev/sdc
+
+or
+
+(check existing with xfs_admin -l /dev/old_partition)
+Use xfs_admin -L <label> /dev/new_partition 
+```
+
+### **Step 7 - Verify**
+
+verify everything:
+```
+sgdisk -p /dev/sda  # Old disk
+sgdisk -p /dev/sdc  # New disk
+```
+```
+lsblk -o NAME,SIZE,PARTUUID,PARTLABEL /dev/sda
+lsblk -o NAME,SIZE,PARTUUID,PARTLABEL /dev/sdc
+```
+```
+blkid /dev/sda* | grep UUID=
+blkid /dev/sdc* | grep UUID=
+```
+

examples/nanodc/Cargo.toml

@@ -17,3 +17,5 @@ harmony_secret = { path = "../../harmony_secret" }
 log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
+serde = { workspace = true }
+brocade = { path = "../../brocade" }

examples/nanodc/src/main.rs

@@ -3,12 +3,13 @@ use std::{
     sync::Arc,
 };
 
+use brocade::BrocadeOptions;
 use cidr::Ipv4Cidr;
 use harmony::{
     config::secret::SshKeyPair,
     data::{FileContent, FilePath},
     hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
-    infra::opnsense::OPNSenseManagementInterface,
+    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
     inventory::Inventory,
     modules::{
         http::StaticFilesHttpScore,
@@ -22,8 +23,9 @@ use harmony::{
     topology::{LogicalHost, UnmanagedRouter},
 };
 use harmony_macros::{ip, mac_address};
-use harmony_secret::SecretManager;
+use harmony_secret::{Secret, SecretManager};
 use harmony_types::net::Url;
+use serde::{Deserialize, Serialize};
 
 #[tokio::main]
 async fn main() {
@@ -32,6 +34,26 @@ async fn main() {
         name: String::from("fw0"),
     };
 
+    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+        .await
+        .expect("Failed to get credentials");
+
+    let switches: Vec<IpAddr> = vec![ip!("192.168.33.101")];
+    let brocade_options = Some(BrocadeOptions {
+        dry_run: *harmony::config::DRY_RUN,
+        ..Default::default()
+    });
+    let switch_client = BrocadeSwitchClient::init(
+        &switches,
+        &switch_auth.username,
+        &switch_auth.password,
+        brocade_options,
+    )
+    .await
+    .expect("Failed to connect to switch");
+
+    let switch_client = Arc::new(switch_client);
+
     let opnsense = Arc::new(
         harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
     );
@@ -83,7 +105,7 @@ async fn main() {
                 name: "wk2".to_string(),
             },
         ],
-        switch: vec![],
+        switch_client: switch_client.clone(),
     };
 
     let inventory = Inventory {
@@ -166,3 +188,9 @@ async fn main() {
     .await
     .unwrap();
 }
+
+#[derive(Secret, Serialize, Deserialize, Debug)]
+pub struct BrocadeSwitchAuth {
+    pub username: String,
+    pub password: String,
+}

examples/okd_installation/Cargo.toml

@@ -19,3 +19,4 @@ log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
 serde.workspace = true
+brocade = { path = "../../brocade" }

examples/okd_installation/src/topology.rs

@@ -1,7 +1,8 @@
+use brocade::BrocadeOptions;
 use cidr::Ipv4Cidr;
 use harmony::{
     hardware::{Location, SwitchGroup},
-    infra::opnsense::OPNSenseManagementInterface,
+    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
     inventory::Inventory,
     topology::{HAClusterTopology, LogicalHost, UnmanagedRouter},
 };
@@ -22,6 +23,26 @@ pub async fn get_topology() -> HAClusterTopology {
         name: String::from("opnsense-1"),
     };
 
+    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+        .await
+        .expect("Failed to get credentials");
+
+    let switches: Vec<IpAddr> = vec![ip!("192.168.1.101")]; // TODO: Adjust me
+    let brocade_options = Some(BrocadeOptions {
+        dry_run: *harmony::config::DRY_RUN,
+        ..Default::default()
+    });
+    let switch_client = BrocadeSwitchClient::init(
+        &switches,
+        &switch_auth.username,
+        &switch_auth.password,
+        brocade_options,
+    )
+    .await
+    .expect("Failed to connect to switch");
+
+    let switch_client = Arc::new(switch_client);
+
     let config = SecretManager::get_or_prompt::<OPNSenseFirewallConfig>().await;
     let config = config.unwrap();
 
@@ -58,7 +79,7 @@ pub async fn get_topology() -> HAClusterTopology {
             name: "bootstrap".to_string(),
         },
         workers: vec![],
-        switch: vec![],
+        switch_client: switch_client.clone(),
     }
 }
 
@@ -75,3 +96,9 @@ pub fn get_inventory() -> Inventory {
         control_plane_host: vec![],
     }
 }
+
+#[derive(Secret, Serialize, Deserialize, Debug)]
+pub struct BrocadeSwitchAuth {
+    pub username: String,
+    pub password: String,
+}

examples/okd_pxe/Cargo.toml

@@ -19,3 +19,4 @@ log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
 serde.workspace = true
+brocade = { path = "../../brocade" }

examples/okd_pxe/src/topology.rs

@@ -1,13 +1,15 @@
+use brocade::BrocadeOptions;
 use cidr::Ipv4Cidr;
 use harmony::{
     config::secret::OPNSenseFirewallCredentials,
     hardware::{Location, SwitchGroup},
-    infra::opnsense::OPNSenseManagementInterface,
+    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
     inventory::Inventory,
     topology::{HAClusterTopology, LogicalHost, UnmanagedRouter},
 };
 use harmony_macros::{ip, ipv4};
-use harmony_secret::SecretManager;
+use harmony_secret::{Secret, SecretManager};
+use serde::{Deserialize, Serialize};
 use std::{net::IpAddr, sync::Arc};
 
 pub async fn get_topology() -> HAClusterTopology {
@@ -16,6 +18,26 @@ pub async fn get_topology() -> HAClusterTopology {
         name: String::from("opnsense-1"),
     };
 
+    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+        .await
+        .expect("Failed to get credentials");
+
+    let switches: Vec<IpAddr> = vec![ip!("192.168.1.101")]; // TODO: Adjust me
+    let brocade_options = Some(BrocadeOptions {
+        dry_run: *harmony::config::DRY_RUN,
+        ..Default::default()
+    });
+    let switch_client = BrocadeSwitchClient::init(
+        &switches,
+        &switch_auth.username,
+        &switch_auth.password,
+        brocade_options,
+    )
+    .await
+    .expect("Failed to connect to switch");
+
+    let switch_client = Arc::new(switch_client);
+
     let config = SecretManager::get_or_prompt::<OPNSenseFirewallCredentials>().await;
     let config = config.unwrap();
 
@@ -52,7 +74,7 @@ pub async fn get_topology() -> HAClusterTopology {
             name: "cp0".to_string(),
         },
         workers: vec![],
-        switch: vec![],
+        switch_client: switch_client.clone(),
     }
 }
 
@@ -69,3 +91,9 @@ pub fn get_inventory() -> Inventory {
         control_plane_host: vec![],
     }
 }
+
+#[derive(Secret, Serialize, Deserialize, Debug)]
+pub struct BrocadeSwitchAuth {
+    pub username: String,
+    pub password: String,
+}

examples/opnsense/Cargo.toml

@@ -16,3 +16,6 @@ harmony_macros = { path = "../../harmony_macros" }
 log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
+harmony_secret = { path = "../../harmony_secret" }
+brocade = { path = "../../brocade" }
+serde = { workspace = true }

examples/opnsense/src/main.rs

@@ -3,10 +3,11 @@ use std::{
     sync::Arc,
 };
 
+use brocade::BrocadeOptions;
 use cidr::Ipv4Cidr;
 use harmony::{
     hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
-    infra::opnsense::OPNSenseManagementInterface,
+    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
     inventory::Inventory,
     modules::{
         dummy::{ErrorScore, PanicScore, SuccessScore},
@@ -18,7 +19,9 @@ use harmony::{
     topology::{LogicalHost, UnmanagedRouter},
 };
 use harmony_macros::{ip, mac_address};
+use harmony_secret::{Secret, SecretManager};
 use harmony_types::net::Url;
+use serde::{Deserialize, Serialize};
 
 #[tokio::main]
 async fn main() {
@@ -27,6 +30,26 @@ async fn main() {
         name: String::from("opnsense-1"),
     };
 
+    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+        .await
+        .expect("Failed to get credentials");
+
+    let switches: Vec<IpAddr> = vec![ip!("192.168.5.101")]; // TODO: Adjust me
+    let brocade_options = Some(BrocadeOptions {
+        dry_run: *harmony::config::DRY_RUN,
+        ..Default::default()
+    });
+    let switch_client = BrocadeSwitchClient::init(
+        &switches,
+        &switch_auth.username,
+        &switch_auth.password,
+        brocade_options,
+    )
+    .await
+    .expect("Failed to connect to switch");
+
+    let switch_client = Arc::new(switch_client);
+
     let opnsense = Arc::new(
         harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
     );
@@ -54,7 +77,7 @@ async fn main() {
             name: "cp0".to_string(),
         },
         workers: vec![],
-        switch: vec![],
+        switch_client: switch_client.clone(),
     };
 
     let inventory = Inventory {
@@ -109,3 +132,9 @@ async fn main() {
     .await
     .unwrap();
 }
+
+#[derive(Secret, Serialize, Deserialize, Debug)]
+pub struct BrocadeSwitchAuth {
+    pub username: String,
+    pub password: String,
+}

examples/remove_rook_osd/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "example-remove-rook-osd"
+edition = "2024"
+version.workspace = true
+readme.workspace = true
+license.workspace = true
+
+[dependencies]
+harmony = { version = "0.1.0", path = "../../harmony" }
+harmony_cli = { version = "0.1.0", path = "../../harmony_cli" }
+tokio.workspace = true

examples/remove_rook_osd/src/main.rs

@@ -0,0 +1,18 @@
+use harmony::{
+    inventory::Inventory, modules::storage::ceph::ceph_remove_osd_score::CephRemoveOsd,
+    topology::K8sAnywhereTopology,
+};
+
+#[tokio::main]
+async fn main() {
+    let ceph_score = CephRemoveOsd {
+        osd_deployment_name: "rook-ceph-osd-2".to_string(),
+        rook_ceph_namespace: "rook-ceph".to_string(),
+    };
+
+    let topology = K8sAnywhereTopology::from_env();
+    let inventory = Inventory::autoload();
+    harmony_cli::run(inventory, topology, vec![Box::new(ceph_score)], None)
+        .await
+        .unwrap();
+}

harmony/src/domain/interpret/mod.rs

@@ -30,6 +30,7 @@ pub enum InterpretName {
     Lamp,
     ApplicationMonitoring,
     K8sPrometheusCrdAlerting,
+    CephRemoveOsd,
     DiscoverInventoryAgent,
     CephClusterHealth,
     Custom(&'static str),
@@ -61,6 +62,7 @@ impl std::fmt::Display for InterpretName {
             InterpretName::Lamp => f.write_str("LAMP"),
             InterpretName::ApplicationMonitoring => f.write_str("ApplicationMonitoring"),
             InterpretName::K8sPrometheusCrdAlerting => f.write_str("K8sPrometheusCrdAlerting"),
+            InterpretName::CephRemoveOsd => f.write_str("CephRemoveOsd"),
             InterpretName::DiscoverInventoryAgent => f.write_str("DiscoverInventoryAgent"),
             InterpretName::CephClusterHealth => f.write_str("CephClusterHealth"),
             InterpretName::Custom(name) => f.write_str(name),

harmony/src/domain/topology/ha_cluster.rs

@@ -1,7 +1,5 @@
 use async_trait::async_trait;
-use brocade::BrocadeOptions;
 use harmony_macros::ip;
-use harmony_secret::SecretManager;
 use harmony_types::{
     net::{MacAddress, Url},
     switch::PortLocation,
@@ -14,8 +12,6 @@ use log::info;
 use crate::data::FileContent;
 use crate::executors::ExecutorError;
 use crate::hardware::PhysicalHost;
-use crate::infra::brocade::BrocadeSwitchAuth;
-use crate::infra::brocade::BrocadeSwitchClient;
 use crate::modules::okd::crd::{
     InstallPlanApproval, OperatorGroup, OperatorGroupSpec, Subscription, SubscriptionSpec,
     nmstate::{self, NMState, NodeNetworkConfigurationPolicy, NodeNetworkConfigurationPolicySpec},
@@ -30,7 +26,6 @@ use super::{
 };
 
 use std::collections::BTreeMap;
-use std::net::IpAddr;
 use std::sync::Arc;
 
 #[derive(Debug, Clone)]
@@ -43,10 +38,10 @@ pub struct HAClusterTopology {
     pub tftp_server: Arc<dyn TftpServer>,
     pub http_server: Arc<dyn HttpServer>,
     pub dns_server: Arc<dyn DnsServer>,
+    pub switch_client: Arc<dyn SwitchClient>,
     pub bootstrap_host: LogicalHost,
     pub control_plane: Vec<LogicalHost>,
     pub workers: Vec<LogicalHost>,
-    pub switch: Vec<LogicalHost>,
 }
 
 #[async_trait]
@@ -280,36 +275,15 @@ impl HAClusterTopology {
         }
     }
 
-    async fn get_switch_client(&self) -> Result<Box<dyn SwitchClient>, SwitchError> {
-        let auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
-            .await
-            .map_err(|e| SwitchError::new(format!("Failed to get credentials: {e}")))?;
-
-        // FIXME: We assume Brocade switches
-        let switches: Vec<IpAddr> = self.switch.iter().map(|s| s.ip).collect();
-        let brocade_options = Some(BrocadeOptions {
-            dry_run: *crate::config::DRY_RUN,
-            ..Default::default()
-        });
-        let client =
-            BrocadeSwitchClient::init(&switches, &auth.username, &auth.password, brocade_options)
-                .await
-                .map_err(|e| SwitchError::new(format!("Failed to connect to switch: {e}")))?;
-
-        Ok(Box::new(client))
-    }
-
     async fn configure_port_channel(
         &self,
         host: &PhysicalHost,
         config: &HostNetworkConfig,
     ) -> Result<(), SwitchError> {
         debug!("Configuring port channel: {config:#?}");
-        let client = self.get_switch_client().await?;
-
         let switch_ports = config.switch_ports.iter().map(|s| s.port.clone()).collect();
 
-        client
+        self.switch_client
             .configure_port_channel(&format!("Harmony_{}", host.id), switch_ports)
             .await
             .map_err(|e| SwitchError::new(format!("Failed to configure switch: {e}")))?;
@@ -333,10 +307,10 @@ impl HAClusterTopology {
             tftp_server: dummy_infra.clone(),
             http_server: dummy_infra.clone(),
             dns_server: dummy_infra.clone(),
+            switch_client: dummy_infra.clone(),
             bootstrap_host: dummy_host,
             control_plane: vec![],
             workers: vec![],
-            switch: vec![],
         }
     }
 }
@@ -494,8 +468,7 @@ impl HttpServer for HAClusterTopology {
 #[async_trait]
 impl Switch for HAClusterTopology {
     async fn setup_switch(&self) -> Result<(), SwitchError> {
-        let client = self.get_switch_client().await?;
-        client.setup().await?;
+        self.switch_client.setup().await?;
         Ok(())
     }
 
@@ -503,8 +476,7 @@ impl Switch for HAClusterTopology {
         &self,
         mac_address: &MacAddress,
     ) -> Result<Option<PortLocation>, SwitchError> {
-        let client = self.get_switch_client().await?;
-        let port = client.find_port(mac_address).await?;
+        let port = self.switch_client.find_port(mac_address).await?;
         Ok(port)
     }
 
@@ -704,3 +676,25 @@ impl DnsServer for DummyInfra {
         unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
     }
 }
+
+#[async_trait]
+impl SwitchClient for DummyInfra {
+    async fn setup(&self) -> Result<(), SwitchError> {
+        unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
+    }
+
+    async fn find_port(
+        &self,
+        _mac_address: &MacAddress,
+    ) -> Result<Option<PortLocation>, SwitchError> {
+        unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
+    }
+
+    async fn configure_port_channel(
+        &self,
+        _channel_name: &str,
+        _switch_ports: Vec<PortLocation>,
+    ) -> Result<u8, SwitchError> {
+        unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
+    }
+}

harmony/src/domain/topology/k8s.rs

@@ -3,13 +3,11 @@ use std::time::Duration;
 use derive_new::new;
 use k8s_openapi::{
     ClusterResourceScope, NamespaceResourceScope,
-    api::{
-        apps::v1::Deployment,
-        core::v1::{Pod, PodStatus},
-    },
+    api::{apps::v1::Deployment, core::v1::Pod},
+    apimachinery::pkg::version::Info,
 };
 use kube::{
-    Client, Config, Error, Resource,
+    Client, Config, Discovery, Error, Resource,
     api::{Api, AttachParams, DeleteParams, ListParams, Patch, PatchParams, ResourceExt},
     config::{KubeConfigOptions, Kubeconfig},
     core::ErrorResponse,
@@ -21,7 +19,7 @@ use kube::{
     api::{ApiResource, GroupVersionKind},
     runtime::wait::await_condition,
 };
-use log::{debug, error, trace};
+use log::{debug, error, info, trace};
 use serde::{Serialize, de::DeserializeOwned};
 use serde_json::{Value, json};
 use similar::TextDiff;
@@ -59,6 +57,17 @@ impl K8sClient {
         })
     }
 
+    pub async fn get_apiserver_version(&self) -> Result<Info, Error> {
+        let client: Client = self.client.clone();
+        let version_info: Info = client.apiserver_version().await?;
+        Ok(version_info)
+    }
+
+    pub async fn discovery(&self) -> Result<Discovery, Error> {
+        let discovery: Discovery = Discovery::new(self.client.clone()).run().await?;
+        Ok(discovery)
+    }
+
     pub async fn get_resource_json_value(
         &self,
         name: &str,
@@ -80,10 +89,13 @@ impl K8sClient {
         namespace: Option<&str>,
     ) -> Result<Option<Deployment>, Error> {
         let deps: Api<Deployment> = if let Some(ns) = namespace {
+            debug!("getting namespaced deployment");
             Api::namespaced(self.client.clone(), ns)
         } else {
+            debug!("getting default namespace deployment");
             Api::default_namespaced(self.client.clone())
         };
+        debug!("getting deployment {} in ns {}", name, namespace.unwrap());
         Ok(deps.get_opt(name).await?)
     }
 
@@ -114,7 +126,7 @@ impl K8sClient {
             }
         });
         let pp = PatchParams::default();
-        let scale = Patch::Apply(&patch);
+        let scale = Patch::Merge(&patch);
         deployments.patch_scale(name, &pp, &scale).await?;
         Ok(())
     }

harmony/src/domain/topology/k8s_anywhere.rs

@@ -47,6 +47,13 @@ struct K8sState {
     message: String,
 }
 
+#[derive(Debug, Clone)]
+pub enum KubernetesDistribution {
+    OpenshiftFamily,
+    K3sFamily,
+    Default,
+}
+
 #[derive(Debug, Clone)]
 enum K8sSource {
     LocalK3d,
@@ -57,6 +64,7 @@ enum K8sSource {
 pub struct K8sAnywhereTopology {
     k8s_state: Arc<OnceCell<Option<K8sState>>>,
     tenant_manager: Arc<OnceCell<K8sTenantManager>>,
+    k8s_distribution: Arc<OnceCell<KubernetesDistribution>>,
     config: Arc<K8sAnywhereConfig>,
 }
 
@@ -162,6 +170,7 @@ impl K8sAnywhereTopology {
         Self {
             k8s_state: Arc::new(OnceCell::new()),
             tenant_manager: Arc::new(OnceCell::new()),
+            k8s_distribution: Arc::new(OnceCell::new()),
             config: Arc::new(K8sAnywhereConfig::from_env()),
         }
     }
@@ -170,10 +179,42 @@ impl K8sAnywhereTopology {
         Self {
             k8s_state: Arc::new(OnceCell::new()),
             tenant_manager: Arc::new(OnceCell::new()),
+            k8s_distribution: Arc::new(OnceCell::new()),
             config: Arc::new(config),
         }
     }
 
+    pub async fn get_k8s_distribution(&self) -> Result<&KubernetesDistribution, PreparationError> {
+        self.k8s_distribution
+            .get_or_try_init(async || {
+                let client = self.k8s_client().await.unwrap();
+
+                let discovery = client.discovery().await.map_err(|e| {
+                    PreparationError::new(format!("Could not discover API groups: {}", e))
+                })?;
+
+                let version = client.get_apiserver_version().await.map_err(|e| {
+                    PreparationError::new(format!("Could not get server version: {}", e))
+                })?;
+
+                // OpenShift / OKD
+                if discovery
+                    .groups()
+                    .any(|g| g.name() == "project.openshift.io")
+                {
+                    return Ok(KubernetesDistribution::OpenshiftFamily);
+                }
+
+                // K3d / K3s
+                if version.git_version.contains("k3s") {
+                    return Ok(KubernetesDistribution::K3sFamily);
+                }
+
+                return Ok(KubernetesDistribution::Default);
+            })
+            .await
+    }
+
     async fn get_cluster_observability_operator_prometheus_application_score(
         &self,
         sender: RHOBObservability,

harmony/src/domain/topology/network.rs

@@ -1,4 +1,10 @@
-use std::{error::Error, net::Ipv4Addr, str::FromStr, sync::Arc};
+use std::{
+    error::Error,
+    fmt::{self, Debug},
+    net::Ipv4Addr,
+    str::FromStr,
+    sync::Arc,
+};
 
 use async_trait::async_trait;
 use derive_new::new;
@@ -19,8 +25,8 @@ pub struct DHCPStaticEntry {
     pub ip: Ipv4Addr,
 }
 
-impl std::fmt::Display for DHCPStaticEntry {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl fmt::Display for DHCPStaticEntry {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let mac = self
             .mac
             .iter()
@@ -42,8 +48,8 @@ pub trait Firewall: Send + Sync {
     fn get_host(&self) -> LogicalHost;
 }
 
-impl std::fmt::Debug for dyn Firewall {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl Debug for dyn Firewall {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_fmt(format_args!("Firewall {}", self.get_ip()))
     }
 }
@@ -65,7 +71,7 @@ pub struct PxeOptions {
 }
 
 #[async_trait]
-pub trait DhcpServer: Send + Sync + std::fmt::Debug {
+pub trait DhcpServer: Send + Sync + Debug {
     async fn add_static_mapping(&self, entry: &DHCPStaticEntry) -> Result<(), ExecutorError>;
     async fn remove_static_mapping(&self, mac: &MacAddress) -> Result<(), ExecutorError>;
     async fn list_static_mappings(&self) -> Vec<(MacAddress, IpAddress)>;
@@ -104,8 +110,8 @@ pub trait DnsServer: Send + Sync {
     }
 }
 
-impl std::fmt::Debug for dyn DnsServer {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl Debug for dyn DnsServer {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_fmt(format_args!("DnsServer {}", self.get_ip()))
     }
 }
@@ -141,8 +147,8 @@ pub enum DnsRecordType {
     TXT,
 }
 
-impl std::fmt::Display for DnsRecordType {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl fmt::Display for DnsRecordType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             DnsRecordType::A => write!(f, "A"),
             DnsRecordType::AAAA => write!(f, "AAAA"),
@@ -216,8 +222,8 @@ pub struct SwitchError {
     msg: String,
 }
 
-impl std::fmt::Display for SwitchError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl fmt::Display for SwitchError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str(&self.msg)
     }
 }
@@ -225,7 +231,7 @@ impl std::fmt::Display for SwitchError {
 impl Error for SwitchError {}
 
 #[async_trait]
-pub trait SwitchClient: Send + Sync {
+pub trait SwitchClient: Debug + Send + Sync {
     /// Executes essential, idempotent, one-time initial configuration steps.
     ///
     /// This is an opiniated procedure that setups a switch to provide high availability

harmony/src/domain/topology/oberservability/monitoring.rs

@@ -21,6 +21,7 @@ pub struct AlertingInterpret<S: AlertSender> {
     pub sender: S,
     pub receivers: Vec<Box<dyn AlertReceiver<S>>>,
     pub rules: Vec<Box<dyn AlertRule<S>>>,
+    pub scrape_targets: Option<Vec<Box<dyn ScrapeTarget<S>>>>,
 }
 
 #[async_trait]
@@ -38,6 +39,12 @@ impl<S: AlertSender + Installable<T>, T: Topology> Interpret<T> for AlertingInte
             debug!("installing rule: {:#?}", rule);
             rule.install(&self.sender).await?;
         }
+        if let Some(targets) = &self.scrape_targets {
+            for target in targets.iter() {
+                debug!("installing scrape_target: {:#?}", target);
+                target.install(&self.sender).await?;
+            }
+        }
         self.sender.ensure_installed(inventory, topology).await?;
         Ok(Outcome::success(format!(
             "successfully installed alert sender {}",
@@ -77,6 +84,6 @@ pub trait AlertRule<S: AlertSender>: std::fmt::Debug + Send + Sync {
 }
 
 #[async_trait]
-pub trait ScrapeTarget<S: AlertSender> {
-    async fn install(&self, sender: &S) -> Result<(), InterpretError>;
+pub trait ScrapeTarget<S: AlertSender>: std::fmt::Debug + Send + Sync {
+    async fn install(&self, sender: &S) -> Result<Outcome, InterpretError>;
 }

harmony/src/infra/brocade.rs

@@ -1,15 +1,14 @@
 use async_trait::async_trait;
 use brocade::{BrocadeClient, BrocadeOptions, InterSwitchLink, InterfaceStatus, PortOperatingMode};
-use harmony_secret::Secret;
 use harmony_types::{
     net::{IpAddress, MacAddress},
     switch::{PortDeclaration, PortLocation},
 };
 use option_ext::OptionExt;
-use serde::{Deserialize, Serialize};
 
 use crate::topology::{SwitchClient, SwitchError};
 
+#[derive(Debug)]
 pub struct BrocadeSwitchClient {
     brocade: Box<dyn BrocadeClient + Send + Sync>,
 }
@@ -114,12 +113,6 @@ impl SwitchClient for BrocadeSwitchClient {
     }
 }
 
-#[derive(Secret, Serialize, Deserialize, Debug)]
-pub struct BrocadeSwitchAuth {
-    pub username: String,
-    pub password: String,
-}
-
 #[cfg(test)]
 mod tests {
     use std::sync::{Arc, Mutex};
@@ -235,7 +228,7 @@ mod tests {
         assert_that!(*configured_interfaces).is_empty();
     }
 
-    #[derive(Clone)]
+    #[derive(Debug, Clone)]
     struct FakeBrocadeClient {
         stack_topology: Vec<InterSwitchLink>,
         interfaces: Vec<InterfaceInfo>,

harmony/src/modules/cert_manager/cluster_issuer.rs

@@ -0,0 +1,209 @@
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use harmony_types::id::Id;
+use kube::{CustomResource, api::ObjectMeta};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    data::Version,
+    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
+    inventory::Inventory,
+    score::Score,
+    topology::{K8sclient, Topology, k8s::K8sClient},
+};
+
+#[derive(Clone, Debug, Serialize)]
+pub struct ClusterIssuerScore {
+    email: String,
+    server: String,
+    issuer_name: String,
+    namespace: String,
+}
+
+impl<T: Topology + K8sclient> Score<T> for ClusterIssuerScore {
+    fn name(&self) -> String {
+        "ClusterIssuerScore".to_string()
+    }
+
+    #[doc(hidden)]
+    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
+        Box::new(ClusterIssuerInterpret {
+            score: self.clone(),
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ClusterIssuerInterpret {
+    score: ClusterIssuerScore,
+}
+
+#[async_trait]
+impl<T: Topology + K8sclient> Interpret<T> for ClusterIssuerInterpret {
+    async fn execute(
+        &self,
+        _inventory: &Inventory,
+        topology: &T,
+    ) -> Result<Outcome, InterpretError> {
+        self.apply_cluster_issuer(topology.k8s_client().await.unwrap())
+            .await
+    }
+
+    fn get_name(&self) -> InterpretName {
+        InterpretName::Custom("ClusterIssuer")
+    }
+
+    fn get_version(&self) -> Version {
+        todo!()
+    }
+
+    fn get_status(&self) -> InterpretStatus {
+        todo!()
+    }
+
+    fn get_children(&self) -> Vec<Id> {
+        todo!()
+    }
+}
+
+impl ClusterIssuerInterpret {
+    async fn validate_cert_manager(
+        &self,
+        client: &Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let cert_manager = "cert-manager".to_string();
+        let operator_namespace = "openshift-operators".to_string();
+        match client
+            .get_deployment(&cert_manager, Some(&operator_namespace))
+            .await
+        {
+            Ok(Some(deployment)) => {
+                if let Some(status) = deployment.status {
+                    let ready_count = status.ready_replicas.unwrap_or(0);
+                    if ready_count >= 1 {
+                        return Ok(Outcome::success(format!(
+                            "'{}' is ready with {} replica(s).",
+                            &cert_manager, ready_count
+                        )));
+                    } else {
+                        return Err(InterpretError::new(
+                            "cert-manager operator not ready in cluster".to_string(),
+                        ));
+                    }
+                } else {
+                    Err(InterpretError::new(format!(
+                        "failed to get deployment status {} in ns {}",
+                        &cert_manager, &operator_namespace
+                    )))
+                }
+            }
+            Ok(None) => Err(InterpretError::new(format!(
+                "Deployment '{}' not found in namespace '{}'.",
+                &cert_manager, &operator_namespace
+            ))),
+            Err(e) => Err(InterpretError::new(format!(
+                "Failed to query for deployment '{}': {}",
+                &cert_manager, e
+            ))),
+        }
+    }
+
+    fn build_cluster_issuer(&self) -> Result<ClusterIssuer, InterpretError> {
+        let issuer_name = &self.score.issuer_name;
+        let email = &self.score.email;
+        let server = &self.score.server;
+        let namespace = &self.score.namespace;
+        let cluster_issuer = ClusterIssuer {
+            metadata: ObjectMeta {
+                name: Some(issuer_name.to_string()),
+                namespace: Some(namespace.to_string()),
+                ..Default::default()
+            },
+            spec: ClusterIssuerSpec {
+                acme: AcmeSpec {
+                    email: email.to_string(),
+                    private_key_secret_ref: PrivateKeySecretRef {
+                        name: issuer_name.to_string(),
+                    },
+                    server: server.to_string(),
+                    solvers: vec![SolverSpec {
+                        http01: Some(Http01Solver {
+                            ingress: Http01Ingress {
+                                class: "nginx".to_string(),
+                            },
+                        }),
+                    }],
+                },
+            },
+        };
+
+        Ok(cluster_issuer)
+    }
+
+    pub async fn apply_cluster_issuer(
+        &self,
+        client: Arc<K8sClient>,
+    ) -> Result<Outcome, InterpretError> {
+        let namespace = self.score.namespace.clone();
+        self.validate_cert_manager(&client).await?;
+        let cluster_issuer = self.build_cluster_issuer().unwrap();
+        client
+            .apply_yaml(
+                &serde_yaml::to_value(cluster_issuer).unwrap(),
+                Some(&namespace),
+            )
+            .await?;
+        Ok(Outcome::success(format!(
+            "successfully deployed cluster operator: {} in namespace: {}",
+            self.score.issuer_name, self.score.namespace
+        )))
+    }
+}
+
+#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[kube(
+    group = "cert-manager.io",
+    version = "v1",
+    kind = "ClusterIssuer",
+    plural = "clusterissuers"
+)]
+#[serde(rename_all = "camelCase")]
+pub struct ClusterIssuerSpec {
+    pub acme: AcmeSpec,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct AcmeSpec {
+    pub email: String,
+    pub private_key_secret_ref: PrivateKeySecretRef,
+    pub server: String,
+    pub solvers: Vec<SolverSpec>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct PrivateKeySecretRef {
+    pub name: String,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct SolverSpec {
+    pub http01: Option<Http01Solver>,
+    // Other solver types (e.g., dns01) would go here as Options
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Http01Solver {
+    pub ingress: Http01Ingress,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Http01Ingress {
+    pub class: String,
+}

harmony/src/modules/cert_manager/mod.rs

@@ -1,2 +1,3 @@
+pub mod cluster_issuer;
 mod helm;
 pub use helm::*;

harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs

@@ -0,0 +1,187 @@
+use std::net::IpAddr;
+
+use async_trait::async_trait;
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    modules::monitoring::kube_prometheus::crd::{
+        crd_alertmanager_config::CRDPrometheus, crd_prometheuses::LabelSelector,
+    },
+    topology::oberservability::monitoring::ScrapeTarget,
+};
+
+#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
+#[kube(
+    group = "monitoring.coreos.com",
+    version = "v1alpha1",
+    kind = "ScrapeConfig",
+    plural = "scrapeconfigs",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct ScrapeConfigSpec {
+    /// List of static configurations.
+    pub static_configs: Option<Vec<StaticConfig>>,
+
+    /// Kubernetes service discovery.
+    pub kubernetes_sd_configs: Option<Vec<KubernetesSDConfig>>,
+
+    /// HTTP-based service discovery.
+    pub http_sd_configs: Option<Vec<HttpSDConfig>>,
+
+    /// File-based service discovery.
+    pub file_sd_configs: Option<Vec<FileSDConfig>>,
+
+    /// DNS-based service discovery.
+    pub dns_sd_configs: Option<Vec<DnsSDConfig>>,
+
+    /// Consul service discovery.
+    pub consul_sd_configs: Option<Vec<ConsulSDConfig>>,
+
+    /// Relabeling configuration applied to discovered targets.
+    pub relabel_configs: Option<Vec<RelabelConfig>>,
+
+    /// Metric relabeling configuration applied to scraped samples.
+    pub metric_relabel_configs: Option<Vec<RelabelConfig>>,
+
+    /// Path to scrape metrics from (defaults to `/metrics`).
+    pub metrics_path: Option<String>,
+
+    /// Interval at which Prometheus scrapes targets (e.g., "30s").
+    pub scrape_interval: Option<String>,
+
+    /// Timeout for scraping (e.g., "10s").
+    pub scrape_timeout: Option<String>,
+
+    /// Optional job name override.
+    pub job_name: Option<String>,
+
+    /// Optional scheme (http or https).
+    pub scheme: Option<String>,
+
+    /// Authorization paramaters for snmp walk
+    pub params: Option<Params>,
+}
+
+/// Static configuration section of a ScrapeConfig.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct StaticConfig {
+    pub targets: Vec<String>,
+
+    pub labels: Option<LabelSelector>,
+}
+
+/// Relabeling configuration for target or metric relabeling.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct RelabelConfig {
+    pub source_labels: Option<Vec<String>>,
+    pub separator: Option<String>,
+    pub target_label: Option<String>,
+    pub regex: Option<String>,
+    pub modulus: Option<u64>,
+    pub replacement: Option<String>,
+    pub action: Option<String>,
+}
+
+/// Kubernetes service discovery configuration.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct KubernetesSDConfig {
+    ///"pod", "service", "endpoints"pub role: String,
+    pub namespaces: Option<NamespaceSelector>,
+    pub selectors: Option<Vec<LabelSelector>>,
+    pub api_server: Option<String>,
+    pub bearer_token_file: Option<String>,
+    pub tls_config: Option<TLSConfig>,
+}
+
+/// Namespace selector for Kubernetes service discovery.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct NamespaceSelector {
+    pub any: Option<bool>,
+    pub match_names: Option<Vec<String>>,
+}
+
+/// HTTP-based service discovery configuration.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct HttpSDConfig {
+    pub url: String,
+    pub refresh_interval: Option<String>,
+    pub basic_auth: Option<BasicAuth>,
+    pub authorization: Option<Authorization>,
+    pub tls_config: Option<TLSConfig>,
+}
+
+/// File-based service discovery configuration.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct FileSDConfig {
+    pub files: Vec<String>,
+    pub refresh_interval: Option<String>,
+}
+
+/// DNS-based service discovery configuration.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct DnsSDConfig {
+    pub names: Vec<String>,
+    pub refresh_interval: Option<String>,
+    pub type_: Option<String>, // SRV, A, AAAA
+    pub port: Option<u16>,
+}
+
+/// Consul service discovery configuration.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ConsulSDConfig {
+    pub server: String,
+    pub services: Option<Vec<String>>,
+    pub scheme: Option<String>,
+    pub datacenter: Option<String>,
+    pub tag_separator: Option<String>,
+    pub refresh_interval: Option<String>,
+    pub tls_config: Option<TLSConfig>,
+}
+
+/// Basic authentication credentials.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct BasicAuth {
+    pub username: String,
+    pub password: Option<String>,
+    pub password_file: Option<String>,
+}
+
+/// Bearer token or other auth mechanisms.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Authorization {
+    pub credentials: Option<String>,
+    pub credentials_file: Option<String>,
+    pub type_: Option<String>,
+}
+
+/// TLS configuration for secure scraping.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct TLSConfig {
+    pub ca_file: Option<String>,
+    pub cert_file: Option<String>,
+    pub key_file: Option<String>,
+    pub server_name: Option<String>,
+    pub insecure_skip_verify: Option<bool>,
+}
+
+/// Authorization parameters for SNMP walk.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Params {
+    pub auth: Option<Vec<String>>,
+    pub module: Option<Vec<String>>,
+}

harmony/src/modules/monitoring/kube_prometheus/crd/mod.rs

@@ -4,6 +4,7 @@ pub mod crd_default_rules;
 pub mod crd_grafana;
 pub mod crd_prometheus_rules;
 pub mod crd_prometheuses;
+pub mod crd_scrape_config;
 pub mod grafana_default_dashboard;
 pub mod grafana_operator;
 pub mod prometheus_operator;

harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs

@@ -31,6 +31,7 @@ impl<T: Topology + HelmCommand + TenantManager> Score<T> for HelmPrometheusAlert
             sender: KubePrometheus { config },
             receivers: self.receivers.clone(),
             rules: self.rules.clone(),
+            scrape_targets: None,
         })
     }
     fn name(&self) -> String {

harmony/src/modules/monitoring/mod.rs

@@ -6,3 +6,4 @@ pub mod kube_prometheus;
 pub mod ntfy;
 pub mod okd;
 pub mod prometheus;
+pub mod scrape_target;

harmony/src/modules/monitoring/scrape_target/mod.rs

@@ -0,0 +1 @@
+pub mod server;

harmony/src/modules/monitoring/scrape_target/server.rs

@@ -0,0 +1,76 @@
+use std::net::IpAddr;
+
+use async_trait::async_trait;
+use kube::api::ObjectMeta;
+use serde::Serialize;
+
+use crate::{
+    interpret::{InterpretError, Outcome},
+    modules::monitoring::kube_prometheus::crd::{
+        crd_alertmanager_config::CRDPrometheus,
+        crd_scrape_config::{Params, RelabelConfig, ScrapeConfig, ScrapeConfigSpec, StaticConfig},
+    },
+    topology::oberservability::monitoring::ScrapeTarget,
+};
+
+#[derive(Debug, Clone, Serialize)]
+pub struct Server {
+    pub name: String,
+    pub ip: IpAddr,
+    pub auth: String,
+    pub module: String,
+    pub domain: String,
+}
+
+#[async_trait]
+impl ScrapeTarget<CRDPrometheus> for Server {
+    async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
+        let scrape_config_spec = ScrapeConfigSpec {
+            static_configs: Some(vec![StaticConfig {
+                targets: vec![self.ip.to_string()],
+                labels: None,
+            }]),
+            scrape_interval: Some("2m".to_string()),
+            kubernetes_sd_configs: None,
+            http_sd_configs: None,
+            file_sd_configs: None,
+            dns_sd_configs: None,
+            params: Some(Params {
+                auth: Some(vec![self.auth.clone()]),
+                module: Some(vec![self.module.clone()]),
+            }),
+            consul_sd_configs: None,
+            relabel_configs: Some(vec![RelabelConfig {
+                action: None,
+                source_labels: Some(vec!["__address__".to_string()]),
+                separator: None,
+                target_label: Some("__param_target".to_string()),
+                regex: None,
+                replacement: Some(format!("snmp.{}:31080", self.domain.clone())),
+                modulus: None,
+            }]),
+            metric_relabel_configs: None,
+            metrics_path: Some("/snmp".to_string()),
+            scrape_timeout: Some("2m".to_string()),
+            job_name: Some(format!("snmp_exporter/cloud/{}", self.name.clone())),
+            scheme: None,
+        };
+
+        let scrape_config = ScrapeConfig {
+            metadata: ObjectMeta {
+                name: Some(self.name.clone()),
+                namespace: Some(sender.namespace.clone()),
+                ..Default::default()
+            },
+            spec: scrape_config_spec,
+        };
+        sender
+            .client
+            .apply(&scrape_config, Some(&sender.namespace.clone()))
+            .await?;
+        Ok(Outcome::success(format!(
+            "installed scrape target {}",
+            self.name.clone()
+        )))
+    }
+}

harmony/src/modules/storage/ceph/ceph_remove_osd_score.rs

@@ -4,7 +4,7 @@ use std::{
 };
 
 use async_trait::async_trait;
-use log::{info, warn};
+use log::{debug, warn};
 use serde::{Deserialize, Serialize};
 use tokio::time::sleep;
 
@@ -19,8 +19,8 @@ use harmony_types::id::Id;
 
 #[derive(Debug, Clone, Serialize)]
 pub struct CephRemoveOsd {
-    osd_deployment_name: String,
-    rook_ceph_namespace: String,
+    pub osd_deployment_name: String,
+    pub rook_ceph_namespace: String,
 }
 
 impl<T: Topology + K8sclient> Score<T> for CephRemoveOsd {
@@ -54,18 +54,17 @@ impl<T: Topology + K8sclient> Interpret<T> for CephRemoveOsdInterpret {
         self.verify_deployment_scaled(client.clone()).await?;
         self.delete_deployment(client.clone()).await?;
         self.verify_deployment_deleted(client.clone()).await?;
-        let osd_id_full = self.get_ceph_osd_id().unwrap();
-        self.purge_ceph_osd(client.clone(), &osd_id_full).await?;
-        self.verify_ceph_osd_removal(client.clone(), &osd_id_full)
-            .await?;
+        self.purge_ceph_osd(client.clone()).await?;
+        self.verify_ceph_osd_removal(client.clone()).await?;
 
+        let osd_id_full = self.get_ceph_osd_id().unwrap();
         Ok(Outcome::success(format!(
             "Successfully removed OSD {} from rook-ceph cluster by deleting deployment {}",
             osd_id_full, self.score.osd_deployment_name
         )))
     }
     fn get_name(&self) -> InterpretName {
-        todo!()
+        InterpretName::CephRemoveOsd
     }
 
     fn get_version(&self) -> Version {
@@ -82,7 +81,7 @@ impl<T: Topology + K8sclient> Interpret<T> for CephRemoveOsdInterpret {
 }
 
 impl CephRemoveOsdInterpret {
-    pub fn get_ceph_osd_id(&self) -> Result<String, InterpretError> {
+    pub fn get_ceph_osd_id_numeric(&self) -> Result<String, InterpretError> {
         let osd_id_numeric = self
             .score
             .osd_deployment_name
@@ -94,9 +93,14 @@ impl CephRemoveOsdInterpret {
                     self.score.osd_deployment_name
                 ))
             })?;
+        Ok(osd_id_numeric.to_string())
+    }
+
+    pub fn get_ceph_osd_id(&self) -> Result<String, InterpretError> {
+        let osd_id_numeric = self.get_ceph_osd_id_numeric().unwrap();
         let osd_id_full = format!("osd.{}", osd_id_numeric);
 
-        info!(
+        debug!(
             "Targeting Ceph OSD: {} (parsed from deployment {})",
             osd_id_full, self.score.osd_deployment_name
         );
@@ -108,6 +112,7 @@ impl CephRemoveOsdInterpret {
         &self,
         client: Arc<K8sClient>,
     ) -> Result<Outcome, InterpretError> {
+        debug!("verifying toolbox exists");
         let toolbox_dep = "rook-ceph-tools".to_string();
 
         match client
@@ -149,7 +154,7 @@ impl CephRemoveOsdInterpret {
         &self,
         client: Arc<K8sClient>,
     ) -> Result<Outcome, InterpretError> {
-        info!(
+        debug!(
             "Scaling down OSD deployment: {}",
             self.score.osd_deployment_name
         );
@@ -172,7 +177,7 @@ impl CephRemoveOsdInterpret {
     ) -> Result<Outcome, InterpretError> {
         let (timeout, interval, start) = self.build_timer();
 
-        info!("Waiting for OSD deployment to scale down to 0 replicas");
+        debug!("Waiting for OSD deployment to scale down to 0 replicas");
         loop {
             let dep = client
                 .get_deployment(
@@ -180,11 +185,9 @@ impl CephRemoveOsdInterpret {
                     Some(&self.score.rook_ceph_namespace),
                 )
                 .await?;
-
             if let Some(deployment) = dep {
                 if let Some(status) = deployment.status {
-                    if status.replicas.unwrap_or(1) == 0 && status.ready_replicas.unwrap_or(1) == 0
-                    {
+                    if status.replicas == None && status.ready_replicas == None {
                         return Ok(Outcome::success(
                             "Deployment successfully scaled down.".to_string(),
                         ));
@@ -212,7 +215,7 @@ impl CephRemoveOsdInterpret {
         &self,
         client: Arc<K8sClient>,
     ) -> Result<Outcome, InterpretError> {
-        info!(
+        debug!(
             "Deleting OSD deployment: {}",
             self.score.osd_deployment_name
         );
@@ -234,7 +237,7 @@ impl CephRemoveOsdInterpret {
     ) -> Result<Outcome, InterpretError> {
         let (timeout, interval, start) = self.build_timer();
 
-        info!("Waiting for OSD deployment to scale down to 0 replicas");
+        debug!("Verifying OSD deployment deleted");
         loop {
             let dep = client
                 .get_deployment(
@@ -244,7 +247,7 @@ impl CephRemoveOsdInterpret {
                 .await?;
 
             if dep.is_none() {
-                info!(
+                debug!(
                     "Deployment {} successfully deleted.",
                     self.score.osd_deployment_name
                 );
@@ -276,12 +279,10 @@ impl CephRemoveOsdInterpret {
         Ok(tree)
     }
 
-    pub async fn purge_ceph_osd(
-        &self,
-        client: Arc<K8sClient>,
-        osd_id_full: &str,
-    ) -> Result<Outcome, InterpretError> {
-        info!(
+    pub async fn purge_ceph_osd(&self, client: Arc<K8sClient>) -> Result<Outcome, InterpretError> {
+        let osd_id_numeric = self.get_ceph_osd_id_numeric().unwrap();
+        let osd_id_full = self.get_ceph_osd_id().unwrap();
+        debug!(
             "Purging OSD {} from Ceph cluster and removing its auth key",
             osd_id_full
         );
@@ -291,8 +292,9 @@ impl CephRemoveOsdInterpret {
                 "app".to_string(),
                 Some(&self.score.rook_ceph_namespace),
                 vec![
-                    format!("ceph osd purge {osd_id_full} --yes-i-really-mean-it").as_str(),
-                    format!("ceph auth del osd.{osd_id_full}").as_str(),
+                    "sh",
+                    "-c",
+                    format!("ceph osd purge {osd_id_numeric} --yes-i-really-mean-it && ceph auth del {osd_id_full}").as_str(),
                 ],
             )
             .await?;
@@ -305,10 +307,10 @@ impl CephRemoveOsdInterpret {
     pub async fn verify_ceph_osd_removal(
         &self,
         client: Arc<K8sClient>,
-        osd_id_full: &str,
     ) -> Result<Outcome, InterpretError> {
         let (timeout, interval, start) = self.build_timer();
-        info!(
+        let osd_id_full = self.get_ceph_osd_id().unwrap();
+        debug!(
             "Verifying OSD {} has been removed from the Ceph tree...",
             osd_id_full
         );
@@ -318,7 +320,7 @@ impl CephRemoveOsdInterpret {
                     "rook-ceph-tools".to_string(),
                     "app".to_string(),
                     Some(&self.score.rook_ceph_namespace),
-                    vec!["ceph osd tree -f json"],
+                    vec!["sh", "-c", "ceph osd tree -f json"],
                 )
                 .await?;
             let tree =

harmony/src/modules/storage/ceph/mod.rs

@@ -1,2 +1,2 @@
-pub mod ceph_osd_replacement_score;
+pub mod ceph_remove_osd_score;
 pub mod ceph_validate_health_score;