describe grpc pod config

Cargo.lock

@@ -1780,7 +1780,6 @@ dependencies = [
 name = "example-nanodc"
 version = "0.1.0"
 dependencies = [
- "brocade",
  "cidr",
  "env_logger",
  "harmony",
@@ -1789,7 +1788,6 @@ dependencies = [
  "harmony_tui",
  "harmony_types",
  "log",
- "serde",
  "tokio",
  "url",
 ]
@@ -1808,7 +1806,6 @@ dependencies = [
 name = "example-okd-install"
 version = "0.1.0"
 dependencies = [
- "brocade",
  "cidr",
  "env_logger",
  "harmony",
@@ -1839,16 +1836,13 @@ dependencies = [
 name = "example-opnsense"
 version = "0.1.0"
 dependencies = [
- "brocade",
  "cidr",
  "env_logger",
  "harmony",
  "harmony_macros",
- "harmony_secret",
  "harmony_tui",
  "harmony_types",
  "log",
- "serde",
  "tokio",
  "url",
 ]
@@ -1857,7 +1851,6 @@ dependencies = [
 name = "example-pxe"
 version = "0.1.0"
 dependencies = [
- "brocade",
  "cidr",
  "env_logger",
  "harmony",
@@ -1872,15 +1865,6 @@ dependencies = [
  "url",
 ]
 
-[[package]]
-name = "example-remove-rook-osd"
-version = "0.1.0"
-dependencies = [
- "harmony",
- "harmony_cli",
- "tokio",
-]
-
 [[package]]
 name = "example-rust"
 version = "0.1.0"
@@ -3934,7 +3918,6 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
 name = "opnsense-config"
 version = "0.1.0"
 dependencies = [
- "assertor",
  "async-trait",
  "chrono",
  "env_logger",

Cargo.toml

@@ -15,8 +15,7 @@ members = [
   "harmony_inventory_agent",
   "harmony_secret_derive",
   "harmony_secret",
-  "adr/agent_discovery/mdns",
+  "adr/agent_discovery/mdns", "brocade",
-  "brocade",
 ]
 
 [workspace.package]

brocade/src/fast_iron.rs

@@ -10,7 +10,6 @@ use log::{debug, info};
 use regex::Regex;
 use std::{collections::HashSet, str::FromStr};
 
-#[derive(Debug)]
 pub struct FastIronClient {
     shell: BrocadeShell,
     version: BrocadeInfo,

brocade/src/lib.rs

@@ -162,7 +162,7 @@ pub async fn init(
 }
 
 #[async_trait]
-pub trait BrocadeClient: std::fmt::Debug {
+pub trait BrocadeClient {
     /// Retrieves the operating system and version details from the connected Brocade switch.
     ///
     /// This is typically the first call made after establishing a connection to determine

brocade/src/network_operating_system.rs

@@ -10,7 +10,6 @@ use crate::{
     parse_brocade_mac_address, shell::BrocadeShell,
 };
 
-#[derive(Debug)]
 pub struct NetworkOperatingSystemClient {
     shell: BrocadeShell,
     version: BrocadeInfo,

brocade/src/shell.rs

@@ -13,7 +13,6 @@ use log::info;
 use russh::ChannelMsg;
 use tokio::time::timeout;
 
-#[derive(Debug)]
 pub struct BrocadeShell {
     ip: IpAddr,
     port: u16,

examples/nanodc/Cargo.toml

@@ -17,5 +17,3 @@ harmony_secret = { path = "../../harmony_secret" }
 log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
-serde = { workspace = true }
-brocade = { path = "../../brocade" }

examples/nanodc/src/main.rs

@@ -3,13 +3,12 @@ use std::{
     sync::Arc,
 };
 
-use brocade::BrocadeOptions;
 use cidr::Ipv4Cidr;
 use harmony::{
     config::secret::SshKeyPair,
     data::{FileContent, FilePath},
     hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
-    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
+    infra::opnsense::OPNSenseManagementInterface,
     inventory::Inventory,
     modules::{
         http::StaticFilesHttpScore,
@@ -23,9 +22,8 @@ use harmony::{
     topology::{LogicalHost, UnmanagedRouter},
 };
 use harmony_macros::{ip, mac_address};
-use harmony_secret::{Secret, SecretManager};
+use harmony_secret::SecretManager;
 use harmony_types::net::Url;
-use serde::{Deserialize, Serialize};
 
 #[tokio::main]
 async fn main() {
@@ -34,26 +32,6 @@ async fn main() {
         name: String::from("fw0"),
     };
 
-    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
-        .await
-        .expect("Failed to get credentials");
-
-    let switches: Vec<IpAddr> = vec![ip!("192.168.33.101")];
-    let brocade_options = Some(BrocadeOptions {
-        dry_run: *harmony::config::DRY_RUN,
-        ..Default::default()
-    });
-    let switch_client = BrocadeSwitchClient::init(
-        &switches,
-        &switch_auth.username,
-        &switch_auth.password,
-        brocade_options,
-    )
-    .await
-    .expect("Failed to connect to switch");
-
-    let switch_client = Arc::new(switch_client);
-
     let opnsense = Arc::new(
         harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
     );
@@ -61,6 +39,7 @@ async fn main() {
     let gateway_ipv4 = Ipv4Addr::new(192, 168, 33, 1);
     let gateway_ip = IpAddr::V4(gateway_ipv4);
     let topology = harmony::topology::HAClusterTopology {
+        kubeconfig: None,
         domain_name: "ncd0.harmony.mcd".to_string(), // TODO this must be set manually correctly
         // when setting up the opnsense firewall
         router: Arc::new(UnmanagedRouter::new(
@@ -105,7 +84,7 @@ async fn main() {
                 name: "wk2".to_string(),
             },
         ],
-        switch_client: switch_client.clone(),
+        switch: vec![],
     };
 
     let inventory = Inventory {
@@ -188,9 +167,3 @@ async fn main() {
     .await
     .unwrap();
 }
-
-#[derive(Secret, Serialize, Deserialize, Debug)]
-pub struct BrocadeSwitchAuth {
-    pub username: String,
-    pub password: String,
-}

examples/okd_installation/Cargo.toml

@@ -19,4 +19,3 @@ log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
 serde.workspace = true
-brocade = { path = "../../brocade" }

examples/okd_installation/src/topology.rs

@@ -1,8 +1,7 @@
-use brocade::BrocadeOptions;
 use cidr::Ipv4Cidr;
 use harmony::{
     hardware::{Location, SwitchGroup},
-    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
+    infra::opnsense::OPNSenseManagementInterface,
     inventory::Inventory,
     topology::{HAClusterTopology, LogicalHost, UnmanagedRouter},
 };
@@ -23,26 +22,6 @@ pub async fn get_topology() -> HAClusterTopology {
         name: String::from("opnsense-1"),
     };
 
-    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
-        .await
-        .expect("Failed to get credentials");
-
-    let switches: Vec<IpAddr> = vec![ip!("192.168.1.101")]; // TODO: Adjust me
-    let brocade_options = Some(BrocadeOptions {
-        dry_run: *harmony::config::DRY_RUN,
-        ..Default::default()
-    });
-    let switch_client = BrocadeSwitchClient::init(
-        &switches,
-        &switch_auth.username,
-        &switch_auth.password,
-        brocade_options,
-    )
-    .await
-    .expect("Failed to connect to switch");
-
-    let switch_client = Arc::new(switch_client);
-
     let config = SecretManager::get_or_prompt::<OPNSenseFirewallConfig>().await;
     let config = config.unwrap();
 
@@ -59,6 +38,7 @@ pub async fn get_topology() -> HAClusterTopology {
     let gateway_ipv4 = ipv4!("192.168.1.1");
     let gateway_ip = IpAddr::V4(gateway_ipv4);
     harmony::topology::HAClusterTopology {
+        kubeconfig: None,
         domain_name: "demo.harmony.mcd".to_string(),
         router: Arc::new(UnmanagedRouter::new(
             gateway_ip,
@@ -79,7 +59,7 @@ pub async fn get_topology() -> HAClusterTopology {
             name: "bootstrap".to_string(),
         },
         workers: vec![],
-        switch_client: switch_client.clone(),
+        switch: vec![],
     }
 }
 
@@ -96,9 +76,3 @@ pub fn get_inventory() -> Inventory {
         control_plane_host: vec![],
     }
 }
-
-#[derive(Secret, Serialize, Deserialize, Debug)]
-pub struct BrocadeSwitchAuth {
-    pub username: String,
-    pub password: String,
-}

examples/okd_pxe/Cargo.toml

@@ -19,4 +19,3 @@ log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
 serde.workspace = true
-brocade = { path = "../../brocade" }

examples/okd_pxe/src/topology.rs

@@ -1,15 +1,13 @@
-use brocade::BrocadeOptions;
 use cidr::Ipv4Cidr;
 use harmony::{
     config::secret::OPNSenseFirewallCredentials,
     hardware::{Location, SwitchGroup},
-    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
+    infra::opnsense::OPNSenseManagementInterface,
     inventory::Inventory,
     topology::{HAClusterTopology, LogicalHost, UnmanagedRouter},
 };
 use harmony_macros::{ip, ipv4};
-use harmony_secret::{Secret, SecretManager};
+use harmony_secret::SecretManager;
-use serde::{Deserialize, Serialize};
 use std::{net::IpAddr, sync::Arc};
 
 pub async fn get_topology() -> HAClusterTopology {
@@ -18,26 +16,6 @@ pub async fn get_topology() -> HAClusterTopology {
         name: String::from("opnsense-1"),
     };
 
-    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
-        .await
-        .expect("Failed to get credentials");
-
-    let switches: Vec<IpAddr> = vec![ip!("192.168.1.101")]; // TODO: Adjust me
-    let brocade_options = Some(BrocadeOptions {
-        dry_run: *harmony::config::DRY_RUN,
-        ..Default::default()
-    });
-    let switch_client = BrocadeSwitchClient::init(
-        &switches,
-        &switch_auth.username,
-        &switch_auth.password,
-        brocade_options,
-    )
-    .await
-    .expect("Failed to connect to switch");
-
-    let switch_client = Arc::new(switch_client);
-
     let config = SecretManager::get_or_prompt::<OPNSenseFirewallCredentials>().await;
     let config = config.unwrap();
 
@@ -54,6 +32,7 @@ pub async fn get_topology() -> HAClusterTopology {
     let gateway_ipv4 = ipv4!("192.168.1.1");
     let gateway_ip = IpAddr::V4(gateway_ipv4);
     harmony::topology::HAClusterTopology {
+        kubeconfig: None,
         domain_name: "demo.harmony.mcd".to_string(),
         router: Arc::new(UnmanagedRouter::new(
             gateway_ip,
@@ -74,7 +53,7 @@ pub async fn get_topology() -> HAClusterTopology {
             name: "cp0".to_string(),
         },
         workers: vec![],
-        switch_client: switch_client.clone(),
+        switch: vec![],
     }
 }
 
@@ -91,9 +70,3 @@ pub fn get_inventory() -> Inventory {
         control_plane_host: vec![],
     }
 }
-
-#[derive(Secret, Serialize, Deserialize, Debug)]
-pub struct BrocadeSwitchAuth {
-    pub username: String,
-    pub password: String,
-}

examples/opnsense/Cargo.toml

@@ -16,6 +16,3 @@ harmony_macros = { path = "../../harmony_macros" }
 log = { workspace = true }
 env_logger = { workspace = true }
 url = { workspace = true }
-harmony_secret = { path = "../../harmony_secret" }
-brocade = { path = "../../brocade" }
-serde = { workspace = true }

examples/opnsense/src/main.rs

@@ -3,11 +3,10 @@ use std::{
     sync::Arc,
 };
 
-use brocade::BrocadeOptions;
 use cidr::Ipv4Cidr;
 use harmony::{
     hardware::{HostCategory, Location, PhysicalHost, SwitchGroup},
-    infra::{brocade::BrocadeSwitchClient, opnsense::OPNSenseManagementInterface},
+    infra::opnsense::OPNSenseManagementInterface,
     inventory::Inventory,
     modules::{
         dummy::{ErrorScore, PanicScore, SuccessScore},
@@ -19,9 +18,7 @@ use harmony::{
     topology::{LogicalHost, UnmanagedRouter},
 };
 use harmony_macros::{ip, mac_address};
-use harmony_secret::{Secret, SecretManager};
 use harmony_types::net::Url;
-use serde::{Deserialize, Serialize};
 
 #[tokio::main]
 async fn main() {
@@ -30,26 +27,6 @@ async fn main() {
         name: String::from("opnsense-1"),
     };
 
-    let switch_auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
-        .await
-        .expect("Failed to get credentials");
-
-    let switches: Vec<IpAddr> = vec![ip!("192.168.5.101")]; // TODO: Adjust me
-    let brocade_options = Some(BrocadeOptions {
-        dry_run: *harmony::config::DRY_RUN,
-        ..Default::default()
-    });
-    let switch_client = BrocadeSwitchClient::init(
-        &switches,
-        &switch_auth.username,
-        &switch_auth.password,
-        brocade_options,
-    )
-    .await
-    .expect("Failed to connect to switch");
-
-    let switch_client = Arc::new(switch_client);
-
     let opnsense = Arc::new(
         harmony::infra::opnsense::OPNSenseFirewall::new(firewall, None, "root", "opnsense").await,
     );
@@ -57,6 +34,7 @@ async fn main() {
     let gateway_ipv4 = Ipv4Addr::new(10, 100, 8, 1);
     let gateway_ip = IpAddr::V4(gateway_ipv4);
     let topology = harmony::topology::HAClusterTopology {
+        kubeconfig: None,
         domain_name: "demo.harmony.mcd".to_string(),
         router: Arc::new(UnmanagedRouter::new(
             gateway_ip,
@@ -77,7 +55,7 @@ async fn main() {
             name: "cp0".to_string(),
         },
         workers: vec![],
-        switch_client: switch_client.clone(),
+        switch: vec![],
     };
 
     let inventory = Inventory {
@@ -132,9 +110,3 @@ async fn main() {
     .await
     .unwrap();
 }
-
-#[derive(Secret, Serialize, Deserialize, Debug)]
-pub struct BrocadeSwitchAuth {
-    pub username: String,
-    pub password: String,
-}

examples/remove_rook_osd/Cargo.toml

@@ -1,11 +0,0 @@
-[package]
-name = "example-remove-rook-osd"
-edition = "2024"
-version.workspace = true
-readme.workspace = true
-license.workspace = true
-
-[dependencies]
-harmony = { version = "0.1.0", path = "../../harmony" }
-harmony_cli = { version = "0.1.0", path = "../../harmony_cli" }
-tokio.workspace = true

examples/remove_rook_osd/src/main.rs

@@ -1,18 +0,0 @@
-use harmony::{
-    inventory::Inventory, modules::storage::ceph::ceph_remove_osd_score::CephRemoveOsd,
-    topology::K8sAnywhereTopology,
-};
-
-#[tokio::main]
-async fn main() {
-    let ceph_score = CephRemoveOsd {
-        osd_deployment_name: "rook-ceph-osd-2".to_string(),
-        rook_ceph_namespace: "rook-ceph".to_string(),
-    };
-
-    let topology = K8sAnywhereTopology::from_env();
-    let inventory = Inventory::autoload();
-    harmony_cli::run(inventory, topology, vec![Box::new(ceph_score)], None)
-        .await
-        .unwrap();
-}

harmony/src/domain/interpret/mod.rs

@@ -30,7 +30,6 @@ pub enum InterpretName {
     Lamp,
     ApplicationMonitoring,
     K8sPrometheusCrdAlerting,
-    CephRemoveOsd,
     DiscoverInventoryAgent,
     CephClusterHealth,
     Custom(&'static str),
@@ -62,7 +61,6 @@ impl std::fmt::Display for InterpretName {
             InterpretName::Lamp => f.write_str("LAMP"),
             InterpretName::ApplicationMonitoring => f.write_str("ApplicationMonitoring"),
             InterpretName::K8sPrometheusCrdAlerting => f.write_str("K8sPrometheusCrdAlerting"),
-            InterpretName::CephRemoveOsd => f.write_str("CephRemoveOsd"),
             InterpretName::DiscoverInventoryAgent => f.write_str("DiscoverInventoryAgent"),
             InterpretName::CephClusterHealth => f.write_str("CephClusterHealth"),
             InterpretName::Custom(name) => f.write_str(name),

harmony/src/domain/topology/ha_cluster.rs

@@ -1,5 +1,7 @@
 use async_trait::async_trait;
+use brocade::BrocadeOptions;
 use harmony_macros::ip;
+use harmony_secret::SecretManager;
 use harmony_types::{
     net::{MacAddress, Url},
     switch::PortLocation,
@@ -12,6 +14,8 @@ use log::info;
 use crate::data::FileContent;
 use crate::executors::ExecutorError;
 use crate::hardware::PhysicalHost;
+use crate::infra::brocade::BrocadeSwitchAuth;
+use crate::infra::brocade::BrocadeSwitchClient;
 use crate::modules::okd::crd::{
     InstallPlanApproval, OperatorGroup, OperatorGroupSpec, Subscription, SubscriptionSpec,
     nmstate::{self, NMState, NodeNetworkConfigurationPolicy, NodeNetworkConfigurationPolicySpec},
@@ -25,8 +29,9 @@ use super::{
     Topology, k8s::K8sClient,
 };
 
-use std::collections::BTreeMap;
+use std::net::IpAddr;
 use std::sync::Arc;
+use std::{collections::BTreeMap, time::Duration};
 
 #[derive(Debug, Clone)]
 pub struct HAClusterTopology {
@@ -38,10 +43,11 @@ pub struct HAClusterTopology {
     pub tftp_server: Arc<dyn TftpServer>,
     pub http_server: Arc<dyn HttpServer>,
     pub dns_server: Arc<dyn DnsServer>,
-    pub switch_client: Arc<dyn SwitchClient>,
     pub bootstrap_host: LogicalHost,
     pub control_plane: Vec<LogicalHost>,
     pub workers: Vec<LogicalHost>,
+    pub switch: Vec<LogicalHost>,
+    pub kubeconfig: Option<String>,
 }
 
 #[async_trait]
@@ -60,9 +66,17 @@ impl Topology for HAClusterTopology {
 #[async_trait]
 impl K8sclient for HAClusterTopology {
     async fn k8s_client(&self) -> Result<Arc<K8sClient>, String> {
-        Ok(Arc::new(
+        match &self.kubeconfig {
-            K8sClient::try_default().await.map_err(|e| e.to_string())?,
+            None => Ok(Arc::new(
-        ))
+                K8sClient::try_default().await.map_err(|e| e.to_string())?,
+            )),
+            Some(kubeconfig) => {
+                let Some(client) = K8sClient::from_kubeconfig(&kubeconfig).await else {
+                    return Err("Failed to create k8s client".to_string());
+                };
+                Ok(Arc::new(client))
+            }
+        }
     }
 }
 
@@ -88,13 +102,13 @@ impl HAClusterTopology {
     }
 
     async fn ensure_nmstate_operator_installed(&self) -> Result<(), String> {
-        // FIXME: Find a way to check nmstate is already available (get pod -n openshift-nmstate)
+        // FIXME: Find a way to check nmstate is already available (get pod -n nmstate)
         debug!("Installing NMState operator...");
         let k8s_client = self.k8s_client().await?;
 
         let nmstate_namespace = Namespace {
             metadata: ObjectMeta {
-                name: Some("openshift-nmstate".to_string()),
+                name: Some("nmstate".to_string()),
                 finalizers: Some(vec!["kubernetes".to_string()]),
                 ..Default::default()
             },
@@ -108,50 +122,60 @@ impl HAClusterTopology {
 
         let nmstate_operator_group = OperatorGroup {
             metadata: ObjectMeta {
-                name: Some("openshift-nmstate".to_string()),
+                name: Some("nmstate".to_string()),
-                namespace: Some("openshift-nmstate".to_string()),
+                namespace: Some("nmstate".to_string()),
                 ..Default::default()
             },
             spec: OperatorGroupSpec {
-                target_namespaces: vec!["openshift-nmstate".to_string()],
+                target_namespaces: vec!["nmstate".to_string()],
             },
         };
+
         debug!("Creating NMState operator group: {nmstate_operator_group:#?}");
         k8s_client
-            .apply(&nmstate_operator_group, None)
+            .apply(&nmstate_operator_group, Some("nmstate"))
             .await
             .map_err(|e| e.to_string())?;
 
         let nmstate_subscription = Subscription {
             metadata: ObjectMeta {
                 name: Some("kubernetes-nmstate-operator".to_string()),
-                namespace: Some("openshift-nmstate".to_string()),
+                namespace: Some("nmstate".to_string()),
                 ..Default::default()
             },
             spec: SubscriptionSpec {
-                channel: Some("stable".to_string()),
+                channel: Some("alpha".to_string()),
-                install_plan_approval: Some(InstallPlanApproval::Automatic),
                 name: "kubernetes-nmstate-operator".to_string(),
-                source: "redhat-operators".to_string(),
+                source: "operatorhubio-catalog".to_string(),
                 source_namespace: "openshift-marketplace".to_string(),
+                install_plan_approval: Some(InstallPlanApproval::Automatic),
             },
         };
         debug!("Subscribing to NMState Operator: {nmstate_subscription:#?}");
         k8s_client
-            .apply(&nmstate_subscription, None)
+            .apply(&nmstate_subscription, Some("nmstate"))
             .await
             .map_err(|e| e.to_string())?;
 
+        k8s_client
+            .wait_for_operator(
+                "kubernetes-nmstate-operator",
+                Some("nmstate"),
+                Some(Duration::from_secs(30)),
+            )
+            .await?;
+
         let nmstate = NMState {
             metadata: ObjectMeta {
                 name: Some("nmstate".to_string()),
+                namespace: Some("nmstate".to_string()),
                 ..Default::default()
             },
             ..Default::default()
         };
         debug!("Creating NMState: {nmstate:#?}");
         k8s_client
-            .apply(&nmstate, None)
+            .apply(&nmstate, Some("nmstate"))
             .await
             .map_err(|e| e.to_string())?;
 
@@ -182,9 +206,9 @@ impl HAClusterTopology {
             .unwrap()
             .apply(&bond_config, None)
             .await
-            .unwrap();
+            .map_err(|e| SwitchError::new(format!("Failed to configure bond: {e}")))?;
 
-        todo!()
+        Ok(())
     }
 
     fn create_bond_configuration(
@@ -275,15 +299,36 @@ impl HAClusterTopology {
         }
     }
 
+    async fn get_switch_client(&self) -> Result<Box<dyn SwitchClient>, SwitchError> {
+        let auth = SecretManager::get_or_prompt::<BrocadeSwitchAuth>()
+            .await
+            .map_err(|e| SwitchError::new(format!("Failed to get credentials: {e}")))?;
+
+        // FIXME: We assume Brocade switches
+        let switches: Vec<IpAddr> = self.switch.iter().map(|s| s.ip).collect();
+        let brocade_options = Some(BrocadeOptions {
+            dry_run: *crate::config::DRY_RUN,
+            ..Default::default()
+        });
+        let client =
+            BrocadeSwitchClient::init(&switches, &auth.username, &auth.password, brocade_options)
+                .await
+                .map_err(|e| SwitchError::new(format!("Failed to connect to switch: {e}")))?;
+
+        Ok(Box::new(client))
+    }
+
     async fn configure_port_channel(
         &self,
         host: &PhysicalHost,
         config: &HostNetworkConfig,
     ) -> Result<(), SwitchError> {
         debug!("Configuring port channel: {config:#?}");
+        let client = self.get_switch_client().await?;
+
         let switch_ports = config.switch_ports.iter().map(|s| s.port.clone()).collect();
 
-        self.switch_client
+        client
             .configure_port_channel(&format!("Harmony_{}", host.id), switch_ports)
             .await
             .map_err(|e| SwitchError::new(format!("Failed to configure switch: {e}")))?;
@@ -299,6 +344,7 @@ impl HAClusterTopology {
         };
 
         Self {
+            kubeconfig: None,
             domain_name: "DummyTopology".to_string(),
             router: dummy_infra.clone(),
             load_balancer: dummy_infra.clone(),
@@ -307,10 +353,10 @@ impl HAClusterTopology {
             tftp_server: dummy_infra.clone(),
             http_server: dummy_infra.clone(),
             dns_server: dummy_infra.clone(),
-            switch_client: dummy_infra.clone(),
             bootstrap_host: dummy_host,
             control_plane: vec![],
             workers: vec![],
+            switch: vec![],
         }
     }
 }
@@ -468,7 +514,8 @@ impl HttpServer for HAClusterTopology {
 #[async_trait]
 impl Switch for HAClusterTopology {
     async fn setup_switch(&self) -> Result<(), SwitchError> {
-        self.switch_client.setup().await?;
+        let client = self.get_switch_client().await?;
+        client.setup().await?;
         Ok(())
     }
 
@@ -476,7 +523,8 @@ impl Switch for HAClusterTopology {
         &self,
         mac_address: &MacAddress,
     ) -> Result<Option<PortLocation>, SwitchError> {
-        let port = self.switch_client.find_port(mac_address).await?;
+        let client = self.get_switch_client().await?;
+        let port = client.find_port(mac_address).await?;
         Ok(port)
     }
 
@@ -676,25 +724,3 @@ impl DnsServer for DummyInfra {
         unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
     }
 }
-
-#[async_trait]
-impl SwitchClient for DummyInfra {
-    async fn setup(&self) -> Result<(), SwitchError> {
-        unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
-    }
-
-    async fn find_port(
-        &self,
-        _mac_address: &MacAddress,
-    ) -> Result<Option<PortLocation>, SwitchError> {
-        unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
-    }
-
-    async fn configure_port_channel(
-        &self,
-        _channel_name: &str,
-        _switch_ports: Vec<PortLocation>,
-    ) -> Result<u8, SwitchError> {
-        unimplemented!("{}", UNIMPLEMENTED_DUMMY_INFRA)
-    }
-}

harmony/src/domain/topology/k8s.rs

@@ -3,28 +3,34 @@ use std::time::Duration;
 use derive_new::new;
 use k8s_openapi::{
     ClusterResourceScope, NamespaceResourceScope,
-    api::{apps::v1::Deployment, core::v1::Pod},
+    api::{
-    apimachinery::pkg::version::Info,
+        apps::v1::Deployment,
+        core::v1::{Pod, PodStatus},
+    },
 };
 use kube::{
-    Client, Config, Discovery, Error, Resource,
+    Client, Config, Error, Resource,
-    api::{Api, AttachParams, DeleteParams, ListParams, Patch, PatchParams, ResourceExt},
+    api::{
+        Api, AttachParams, DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, ResourceExt,
+    },
     config::{KubeConfigOptions, Kubeconfig},
     core::ErrorResponse,
     error::DiscoveryError,
-    runtime::reflector::Lookup,
+    runtime::{reflector::Lookup, wait::Condition},
 };
 use kube::{api::DynamicObject, runtime::conditions};
 use kube::{
     api::{ApiResource, GroupVersionKind},
     runtime::wait::await_condition,
 };
-use log::{debug, error, info, trace};
+use log::{debug, error, trace};
-use serde::{Serialize, de::DeserializeOwned};
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
 use serde_json::{Value, json};
 use similar::TextDiff;
 use tokio::{io::AsyncReadExt, time::sleep};
 
+use crate::modules::okd::crd::ClusterServiceVersion;
+
 #[derive(new, Clone)]
 pub struct K8sClient {
     client: Client,
@@ -57,17 +63,6 @@ impl K8sClient {
         })
     }
 
-    pub async fn get_apiserver_version(&self) -> Result<Info, Error> {
-        let client: Client = self.client.clone();
-        let version_info: Info = client.apiserver_version().await?;
-        Ok(version_info)
-    }
-
-    pub async fn discovery(&self) -> Result<Discovery, Error> {
-        let discovery: Discovery = Discovery::new(self.client.clone()).run().await?;
-        Ok(discovery)
-    }
-
     pub async fn get_resource_json_value(
         &self,
         name: &str,
@@ -89,13 +84,10 @@ impl K8sClient {
         namespace: Option<&str>,
     ) -> Result<Option<Deployment>, Error> {
         let deps: Api<Deployment> = if let Some(ns) = namespace {
-            debug!("getting namespaced deployment");
             Api::namespaced(self.client.clone(), ns)
         } else {
-            debug!("getting default namespace deployment");
             Api::default_namespaced(self.client.clone())
         };
-        debug!("getting deployment {} in ns {}", name, namespace.unwrap());
         Ok(deps.get_opt(name).await?)
     }
 
@@ -126,7 +118,7 @@ impl K8sClient {
             }
         });
         let pp = PatchParams::default();
-        let scale = Patch::Merge(&patch);
+        let scale = Patch::Apply(&patch);
         deployments.patch_scale(name, &pp, &scale).await?;
         Ok(())
     }
@@ -206,6 +198,33 @@ impl K8sClient {
         }
     }
 
+    pub async fn wait_for_operator(
+        &self,
+        operator_name: &str,
+        namespace: Option<&str>,
+        timeout: Option<Duration>,
+    ) -> Result<(), String> {
+        let api: Api<ClusterServiceVersion>;
+
+        if let Some(ns) = namespace {
+            api = Api::namespaced(self.client.clone(), ns);
+        } else {
+            api = Api::default_namespaced(self.client.clone());
+        }
+
+        let establish = await_condition(api, operator_name, is_operator_ready());
+        let t = timeout.unwrap_or(Duration::from_secs(5));
+        let res = tokio::time::timeout(t, establish).await;
+
+        if res.is_ok() {
+            Ok(())
+        } else {
+            Err(format!(
+                "timed out while waiting for operator {operator_name}"
+            ))
+        }
+    }
+
     /// Will execute a commond in the first pod found that matches the specified label
     /// '{label}={name}'
     pub async fn exec_app_capture_output(
@@ -559,3 +578,14 @@ where
         }
     }
 }
+
+fn is_operator_ready() -> impl Condition<ClusterServiceVersion> {
+    |obj: Option<&ClusterServiceVersion>| {
+        if let Some(csv) = obj {
+            if let Some(status) = &csv.spec.status {
+                return status.phase == "Succeeded";
+            }
+        }
+        false
+    }
+}

harmony/src/domain/topology/k8s_anywhere.rs

@@ -47,13 +47,6 @@ struct K8sState {
     message: String,
 }
 
-#[derive(Debug, Clone)]
-pub enum KubernetesDistribution {
-    OpenshiftFamily,
-    K3sFamily,
-    Default,
-}
-
 #[derive(Debug, Clone)]
 enum K8sSource {
     LocalK3d,
@@ -64,7 +57,6 @@ enum K8sSource {
 pub struct K8sAnywhereTopology {
     k8s_state: Arc<OnceCell<Option<K8sState>>>,
     tenant_manager: Arc<OnceCell<K8sTenantManager>>,
-    k8s_distribution: Arc<OnceCell<KubernetesDistribution>>,
     config: Arc<K8sAnywhereConfig>,
 }
 
@@ -170,7 +162,6 @@ impl K8sAnywhereTopology {
         Self {
             k8s_state: Arc::new(OnceCell::new()),
             tenant_manager: Arc::new(OnceCell::new()),
-            k8s_distribution: Arc::new(OnceCell::new()),
             config: Arc::new(K8sAnywhereConfig::from_env()),
         }
     }
@@ -179,42 +170,10 @@ impl K8sAnywhereTopology {
         Self {
             k8s_state: Arc::new(OnceCell::new()),
             tenant_manager: Arc::new(OnceCell::new()),
-            k8s_distribution: Arc::new(OnceCell::new()),
             config: Arc::new(config),
         }
     }
 
-    pub async fn get_k8s_distribution(&self) -> Result<&KubernetesDistribution, PreparationError> {
-        self.k8s_distribution
-            .get_or_try_init(async || {
-                let client = self.k8s_client().await.unwrap();
-
-                let discovery = client.discovery().await.map_err(|e| {
-                    PreparationError::new(format!("Could not discover API groups: {}", e))
-                })?;
-
-                let version = client.get_apiserver_version().await.map_err(|e| {
-                    PreparationError::new(format!("Could not get server version: {}", e))
-                })?;
-
-                // OpenShift / OKD
-                if discovery
-                    .groups()
-                    .any(|g| g.name() == "project.openshift.io")
-                {
-                    return Ok(KubernetesDistribution::OpenshiftFamily);
-                }
-
-                // K3d / K3s
-                if version.git_version.contains("k3s") {
-                    return Ok(KubernetesDistribution::K3sFamily);
-                }
-
-                return Ok(KubernetesDistribution::Default);
-            })
-            .await
-    }
-
     async fn get_cluster_observability_operator_prometheus_application_score(
         &self,
         sender: RHOBObservability,

harmony/src/domain/topology/load_balancer.rs

@@ -28,7 +28,13 @@ pub trait LoadBalancer: Send + Sync {
         &self,
         service: &LoadBalancerService,
     ) -> Result<(), ExecutorError> {
-        self.add_service(service).await?;
+        debug!(
+            "Listing LoadBalancer services {:?}",
+            self.list_services().await
+        );
+        if !self.list_services().await.contains(service) {
+            self.add_service(service).await?;
+        }
         Ok(())
     }
 }

harmony/src/domain/topology/network.rs

@@ -1,10 +1,4 @@
-use std::{
+use std::{error::Error, net::Ipv4Addr, str::FromStr, sync::Arc};
-    error::Error,
-    fmt::{self, Debug},
-    net::Ipv4Addr,
-    str::FromStr,
-    sync::Arc,
-};
 
 use async_trait::async_trait;
 use derive_new::new;
@@ -25,8 +19,8 @@ pub struct DHCPStaticEntry {
     pub ip: Ipv4Addr,
 }
 
-impl fmt::Display for DHCPStaticEntry {
+impl std::fmt::Display for DHCPStaticEntry {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let mac = self
             .mac
             .iter()
@@ -48,8 +42,8 @@ pub trait Firewall: Send + Sync {
     fn get_host(&self) -> LogicalHost;
 }
 
-impl Debug for dyn Firewall {
+impl std::fmt::Debug for dyn Firewall {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.write_fmt(format_args!("Firewall {}", self.get_ip()))
     }
 }
@@ -71,7 +65,7 @@ pub struct PxeOptions {
 }
 
 #[async_trait]
-pub trait DhcpServer: Send + Sync + Debug {
+pub trait DhcpServer: Send + Sync + std::fmt::Debug {
     async fn add_static_mapping(&self, entry: &DHCPStaticEntry) -> Result<(), ExecutorError>;
     async fn remove_static_mapping(&self, mac: &MacAddress) -> Result<(), ExecutorError>;
     async fn list_static_mappings(&self) -> Vec<(MacAddress, IpAddress)>;
@@ -110,8 +104,8 @@ pub trait DnsServer: Send + Sync {
     }
 }
 
-impl Debug for dyn DnsServer {
+impl std::fmt::Debug for dyn DnsServer {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.write_fmt(format_args!("DnsServer {}", self.get_ip()))
     }
 }
@@ -147,8 +141,8 @@ pub enum DnsRecordType {
     TXT,
 }
 
-impl fmt::Display for DnsRecordType {
+impl std::fmt::Display for DnsRecordType {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             DnsRecordType::A => write!(f, "A"),
             DnsRecordType::AAAA => write!(f, "AAAA"),
@@ -222,8 +216,8 @@ pub struct SwitchError {
     msg: String,
 }
 
-impl fmt::Display for SwitchError {
+impl std::fmt::Display for SwitchError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.write_str(&self.msg)
     }
 }
@@ -231,7 +225,7 @@ impl fmt::Display for SwitchError {
 impl Error for SwitchError {}
 
 #[async_trait]
-pub trait SwitchClient: Debug + Send + Sync {
+pub trait SwitchClient: Send + Sync {
     /// Executes essential, idempotent, one-time initial configuration steps.
     ///
     /// This is an opiniated procedure that setups a switch to provide high availability

harmony/src/domain/topology/oberservability/monitoring.rs

@@ -21,7 +21,6 @@ pub struct AlertingInterpret<S: AlertSender> {
     pub sender: S,
     pub receivers: Vec<Box<dyn AlertReceiver<S>>>,
     pub rules: Vec<Box<dyn AlertRule<S>>>,
-    pub scrape_targets: Option<Vec<Box<dyn ScrapeTarget<S>>>>,
 }
 
 #[async_trait]
@@ -39,12 +38,6 @@ impl<S: AlertSender + Installable<T>, T: Topology> Interpret<T> for AlertingInte
             debug!("installing rule: {:#?}", rule);
             rule.install(&self.sender).await?;
         }
-        if let Some(targets) = &self.scrape_targets {
-            for target in targets.iter() {
-                debug!("installing scrape_target: {:#?}", target);
-                target.install(&self.sender).await?;
-            }
-        }
         self.sender.ensure_installed(inventory, topology).await?;
         Ok(Outcome::success(format!(
             "successfully installed alert sender {}",
@@ -84,6 +77,6 @@ pub trait AlertRule<S: AlertSender>: std::fmt::Debug + Send + Sync {
 }
 
 #[async_trait]
-pub trait ScrapeTarget<S: AlertSender>: std::fmt::Debug + Send + Sync {
+pub trait ScrapeTarget<S: AlertSender> {
-    async fn install(&self, sender: &S) -> Result<Outcome, InterpretError>;
+    async fn install(&self, sender: &S) -> Result<(), InterpretError>;
 }

harmony/src/infra/brocade.rs

@@ -1,14 +1,15 @@
 use async_trait::async_trait;
 use brocade::{BrocadeClient, BrocadeOptions, InterSwitchLink, InterfaceStatus, PortOperatingMode};
+use harmony_secret::Secret;
 use harmony_types::{
     net::{IpAddress, MacAddress},
     switch::{PortDeclaration, PortLocation},
 };
 use option_ext::OptionExt;
+use serde::{Deserialize, Serialize};
 
 use crate::topology::{SwitchClient, SwitchError};
 
-#[derive(Debug)]
 pub struct BrocadeSwitchClient {
     brocade: Box<dyn BrocadeClient + Send + Sync>,
 }
@@ -113,6 +114,12 @@ impl SwitchClient for BrocadeSwitchClient {
     }
 }
 
+#[derive(Secret, Serialize, Deserialize, Debug)]
+pub struct BrocadeSwitchAuth {
+    pub username: String,
+    pub password: String,
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::{Arc, Mutex};
@@ -228,7 +235,7 @@ mod tests {
         assert_that!(*configured_interfaces).is_empty();
     }
 
-    #[derive(Debug, Clone)]
+    #[derive(Clone)]
     struct FakeBrocadeClient {
         stack_topology: Vec<InterSwitchLink>,
         interfaces: Vec<InterfaceInfo>,

harmony/src/infra/kubers/mod.rs

@@ -0,0 +1 @@
+pub mod types;

harmony/src/infra/kubers/types.rs

@@ -0,0 +1,40 @@
+use std::collections::BTreeMap;
+
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+#[derive(CustomResource, Default, Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[kube(
+    group = "operators.coreos.com",
+    version = "v1alpha1",
+    kind = "CatalogSource",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct CatalogSourceSpec {
+    pub source_type: String,
+    pub image: String,
+    pub display_name: String,
+    pub publisher: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub grpc_pod_config: Option<GrpcPodConfig>,
+}
+
+impl Default for CatalogSource {
+    fn default() -> Self {
+        Self {
+            metadata: Default::default(),
+            spec: Default::default(),
+        }
+    }
+}
+
+#[derive(Default, Serialize, Deserialize, Clone, Debug, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct GrpcPodConfig {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub memory_target: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub node_selector: Option<BTreeMap<String, String>>,
+}

harmony/src/infra/mod.rs

@@ -3,5 +3,6 @@ pub mod executors;
 pub mod hp_ilo;
 pub mod intel_amt;
 pub mod inventory;
+pub mod kubers;
 pub mod opnsense;
 mod sqlx;

harmony/src/infra/opnsense/load_balancer.rs

@@ -26,13 +26,19 @@ impl LoadBalancer for OPNSenseFirewall {
     }
 
     async fn add_service(&self, service: &LoadBalancerService) -> Result<(), ExecutorError> {
+        warn!(
+            "TODO : the current implementation does not check / cleanup / merge with existing haproxy services properly. Make sure to manually verify that the configuration is correct after executing any operation here"
+        );
         let mut config = self.opnsense_config.write().await;
-        let mut load_balancer = config.load_balancer();
-
         let (frontend, backend, servers, healthcheck) =
             harmony_load_balancer_service_to_haproxy_xml(service);
-
+        let mut load_balancer = config.load_balancer();
-        load_balancer.configure_service(frontend, backend, servers, healthcheck);
+        load_balancer.add_backend(backend);
+        load_balancer.add_frontend(frontend);
+        load_balancer.add_servers(servers);
+        if let Some(healthcheck) = healthcheck {
+            load_balancer.add_healthcheck(healthcheck);
+        }
 
         Ok(())
     }
@@ -100,7 +106,7 @@ pub(crate) fn haproxy_xml_config_to_harmony_loadbalancer(
                 .backends
                 .backends
                 .iter()
-                .find(|b| Some(b.uuid.clone()) == frontend.default_backend);
+                .find(|b| b.uuid == frontend.default_backend);
 
             let mut health_check = None;
             match matching_backend {
@@ -110,7 +116,8 @@ pub(crate) fn haproxy_xml_config_to_harmony_loadbalancer(
                 }
                 None => {
                     warn!(
-                        "HAProxy config could not find a matching backend for frontend {frontend:?}"
+                        "HAProxy config could not find a matching backend for frontend {:?}",
+                        frontend
                     );
                 }
             }
@@ -145,11 +152,11 @@ pub(crate) fn get_servers_for_backend(
         .servers
         .iter()
         .filter_map(|server| {
-            let address = server.address.clone()?;
-            let port = server.port?;
-
             if backend_servers.contains(&server.uuid.as_str()) {
-                return Some(BackendServer { address, port });
+                return Some(BackendServer {
+                    address: server.address.clone(),
+                    port: server.port,
+                });
             }
             None
         })
@@ -340,7 +347,7 @@ pub(crate) fn harmony_load_balancer_service_to_haproxy_xml(
         name: format!("frontend_{}", service.listening_port),
         bind: service.listening_port.to_string(),
         mode: "tcp".to_string(), // TODO do not depend on health check here
-        default_backend: Some(backend.uuid.clone()),
+        default_backend: backend.uuid.clone(),
         ..Default::default()
     };
     info!("HAPRoxy frontend and backend mode currently hardcoded to tcp");
@@ -354,8 +361,8 @@ fn server_to_haproxy_server(server: &BackendServer) -> HAProxyServer {
         uuid: Uuid::new_v4().to_string(),
         name: format!("{}_{}", &server.address, &server.port),
         enabled: 1,
-        address: Some(server.address.clone()),
+        address: server.address.clone(),
-        port: Some(server.port),
+        port: server.port,
         mode: "active".to_string(),
         server_type: "static".to_string(),
         ..Default::default()
@@ -378,8 +385,8 @@ mod tests {
         let mut haproxy = HAProxy::default();
         let server = HAProxyServer {
             uuid: "server1".to_string(),
-            address: Some("192.168.1.1".to_string()),
+            address: "192.168.1.1".to_string(),
-            port: Some(80),
+            port: 80,
             ..Default::default()
         };
         haproxy.servers.servers.push(server);
@@ -404,8 +411,8 @@ mod tests {
         let mut haproxy = HAProxy::default();
         let server = HAProxyServer {
             uuid: "server1".to_string(),
-            address: Some("192.168.1.1".to_string()),
+            address: "192.168.1.1".to_string(),
-            port: Some(80),
+            port: 80,
             ..Default::default()
         };
         haproxy.servers.servers.push(server);
@@ -424,8 +431,8 @@ mod tests {
         let mut haproxy = HAProxy::default();
         let server = HAProxyServer {
             uuid: "server1".to_string(),
-            address: Some("192.168.1.1".to_string()),
+            address: "192.168.1.1".to_string(),
-            port: Some(80),
+            port: 80,
             ..Default::default()
         };
         haproxy.servers.servers.push(server);
@@ -446,16 +453,16 @@ mod tests {
         let mut haproxy = HAProxy::default();
         let server = HAProxyServer {
             uuid: "server1".to_string(),
-            address: Some("some-hostname.test.mcd".to_string()),
+            address: "some-hostname.test.mcd".to_string(),
-            port: Some(80),
+            port: 80,
             ..Default::default()
         };
         haproxy.servers.servers.push(server);
 
         let server = HAProxyServer {
             uuid: "server2".to_string(),
-            address: Some("192.168.1.2".to_string()),
+            address: "192.168.1.2".to_string(),
-            port: Some(8080),
+            port: 8080,
             ..Default::default()
         };
         haproxy.servers.servers.push(server);

harmony/src/modules/cert_manager/cluster_issuer.rs

@@ -1,209 +0,0 @@
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use harmony_types::id::Id;
-use kube::{CustomResource, api::ObjectMeta};
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
-
-use crate::{
-    data::Version,
-    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
-    inventory::Inventory,
-    score::Score,
-    topology::{K8sclient, Topology, k8s::K8sClient},
-};
-
-#[derive(Clone, Debug, Serialize)]
-pub struct ClusterIssuerScore {
-    email: String,
-    server: String,
-    issuer_name: String,
-    namespace: String,
-}
-
-impl<T: Topology + K8sclient> Score<T> for ClusterIssuerScore {
-    fn name(&self) -> String {
-        "ClusterIssuerScore".to_string()
-    }
-
-    #[doc(hidden)]
-    fn create_interpret(&self) -> Box<dyn Interpret<T>> {
-        Box::new(ClusterIssuerInterpret {
-            score: self.clone(),
-        })
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct ClusterIssuerInterpret {
-    score: ClusterIssuerScore,
-}
-
-#[async_trait]
-impl<T: Topology + K8sclient> Interpret<T> for ClusterIssuerInterpret {
-    async fn execute(
-        &self,
-        _inventory: &Inventory,
-        topology: &T,
-    ) -> Result<Outcome, InterpretError> {
-        self.apply_cluster_issuer(topology.k8s_client().await.unwrap())
-            .await
-    }
-
-    fn get_name(&self) -> InterpretName {
-        InterpretName::Custom("ClusterIssuer")
-    }
-
-    fn get_version(&self) -> Version {
-        todo!()
-    }
-
-    fn get_status(&self) -> InterpretStatus {
-        todo!()
-    }
-
-    fn get_children(&self) -> Vec<Id> {
-        todo!()
-    }
-}
-
-impl ClusterIssuerInterpret {
-    async fn validate_cert_manager(
-        &self,
-        client: &Arc<K8sClient>,
-    ) -> Result<Outcome, InterpretError> {
-        let cert_manager = "cert-manager".to_string();
-        let operator_namespace = "openshift-operators".to_string();
-        match client
-            .get_deployment(&cert_manager, Some(&operator_namespace))
-            .await
-        {
-            Ok(Some(deployment)) => {
-                if let Some(status) = deployment.status {
-                    let ready_count = status.ready_replicas.unwrap_or(0);
-                    if ready_count >= 1 {
-                        return Ok(Outcome::success(format!(
-                            "'{}' is ready with {} replica(s).",
-                            &cert_manager, ready_count
-                        )));
-                    } else {
-                        return Err(InterpretError::new(
-                            "cert-manager operator not ready in cluster".to_string(),
-                        ));
-                    }
-                } else {
-                    Err(InterpretError::new(format!(
-                        "failed to get deployment status {} in ns {}",
-                        &cert_manager, &operator_namespace
-                    )))
-                }
-            }
-            Ok(None) => Err(InterpretError::new(format!(
-                "Deployment '{}' not found in namespace '{}'.",
-                &cert_manager, &operator_namespace
-            ))),
-            Err(e) => Err(InterpretError::new(format!(
-                "Failed to query for deployment '{}': {}",
-                &cert_manager, e
-            ))),
-        }
-    }
-
-    fn build_cluster_issuer(&self) -> Result<ClusterIssuer, InterpretError> {
-        let issuer_name = &self.score.issuer_name;
-        let email = &self.score.email;
-        let server = &self.score.server;
-        let namespace = &self.score.namespace;
-        let cluster_issuer = ClusterIssuer {
-            metadata: ObjectMeta {
-                name: Some(issuer_name.to_string()),
-                namespace: Some(namespace.to_string()),
-                ..Default::default()
-            },
-            spec: ClusterIssuerSpec {
-                acme: AcmeSpec {
-                    email: email.to_string(),
-                    private_key_secret_ref: PrivateKeySecretRef {
-                        name: issuer_name.to_string(),
-                    },
-                    server: server.to_string(),
-                    solvers: vec![SolverSpec {
-                        http01: Some(Http01Solver {
-                            ingress: Http01Ingress {
-                                class: "nginx".to_string(),
-                            },
-                        }),
-                    }],
-                },
-            },
-        };
-
-        Ok(cluster_issuer)
-    }
-
-    pub async fn apply_cluster_issuer(
-        &self,
-        client: Arc<K8sClient>,
-    ) -> Result<Outcome, InterpretError> {
-        let namespace = self.score.namespace.clone();
-        self.validate_cert_manager(&client).await?;
-        let cluster_issuer = self.build_cluster_issuer().unwrap();
-        client
-            .apply_yaml(
-                &serde_yaml::to_value(cluster_issuer).unwrap(),
-                Some(&namespace),
-            )
-            .await?;
-        Ok(Outcome::success(format!(
-            "successfully deployed cluster operator: {} in namespace: {}",
-            self.score.issuer_name, self.score.namespace
-        )))
-    }
-}
-
-#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[kube(
-    group = "cert-manager.io",
-    version = "v1",
-    kind = "ClusterIssuer",
-    plural = "clusterissuers"
-)]
-#[serde(rename_all = "camelCase")]
-pub struct ClusterIssuerSpec {
-    pub acme: AcmeSpec,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct AcmeSpec {
-    pub email: String,
-    pub private_key_secret_ref: PrivateKeySecretRef,
-    pub server: String,
-    pub solvers: Vec<SolverSpec>,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct PrivateKeySecretRef {
-    pub name: String,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct SolverSpec {
-    pub http01: Option<Http01Solver>,
-    // Other solver types (e.g., dns01) would go here as Options
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct Http01Solver {
-    pub ingress: Http01Ingress,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct Http01Ingress {
-    pub class: String,
-}

harmony/src/modules/cert_manager/mod.rs

@@ -1,3 +1,2 @@
-pub mod cluster_issuer;
 mod helm;
 pub use helm::*;

harmony/src/modules/k8s/resource.rs

@@ -38,13 +38,15 @@ impl<
         + 'static
         + Send
         + Clone,
-    T: Topology,
+    T: Topology + K8sclient,
 > Score<T> for K8sResourceScore<K>
 where
     <K as kube::Resource>::DynamicType: Default,
 {
     fn create_interpret(&self) -> Box<dyn Interpret<T>> {
-        todo!()
+        Box::new(K8sResourceInterpret {
+            score: self.clone(),
+        })
     }
 
     fn name(&self) -> String {

harmony/src/modules/monitoring/kube_prometheus/crd/crd_scrape_config.rs

@@ -1,187 +0,0 @@
-use std::net::IpAddr;
-
-use async_trait::async_trait;
-use kube::CustomResource;
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
-
-use crate::{
-    modules::monitoring::kube_prometheus::crd::{
-        crd_alertmanager_config::CRDPrometheus, crd_prometheuses::LabelSelector,
-    },
-    topology::oberservability::monitoring::ScrapeTarget,
-};
-
-#[derive(CustomResource, Serialize, Deserialize, Debug, Clone, JsonSchema)]
-#[kube(
-    group = "monitoring.coreos.com",
-    version = "v1alpha1",
-    kind = "ScrapeConfig",
-    plural = "scrapeconfigs",
-    namespaced
-)]
-#[serde(rename_all = "camelCase")]
-pub struct ScrapeConfigSpec {
-    /// List of static configurations.
-    pub static_configs: Option<Vec<StaticConfig>>,
-
-    /// Kubernetes service discovery.
-    pub kubernetes_sd_configs: Option<Vec<KubernetesSDConfig>>,
-
-    /// HTTP-based service discovery.
-    pub http_sd_configs: Option<Vec<HttpSDConfig>>,
-
-    /// File-based service discovery.
-    pub file_sd_configs: Option<Vec<FileSDConfig>>,
-
-    /// DNS-based service discovery.
-    pub dns_sd_configs: Option<Vec<DnsSDConfig>>,
-
-    /// Consul service discovery.
-    pub consul_sd_configs: Option<Vec<ConsulSDConfig>>,
-
-    /// Relabeling configuration applied to discovered targets.
-    pub relabel_configs: Option<Vec<RelabelConfig>>,
-
-    /// Metric relabeling configuration applied to scraped samples.
-    pub metric_relabel_configs: Option<Vec<RelabelConfig>>,
-
-    /// Path to scrape metrics from (defaults to `/metrics`).
-    pub metrics_path: Option<String>,
-
-    /// Interval at which Prometheus scrapes targets (e.g., "30s").
-    pub scrape_interval: Option<String>,
-
-    /// Timeout for scraping (e.g., "10s").
-    pub scrape_timeout: Option<String>,
-
-    /// Optional job name override.
-    pub job_name: Option<String>,
-
-    /// Optional scheme (http or https).
-    pub scheme: Option<String>,
-
-    /// Authorization paramaters for snmp walk
-    pub params: Option<Params>,
-}
-
-/// Static configuration section of a ScrapeConfig.
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct StaticConfig {
-    pub targets: Vec<String>,
-
-    pub labels: Option<LabelSelector>,
-}
-
-/// Relabeling configuration for target or metric relabeling.
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct RelabelConfig {
-    pub source_labels: Option<Vec<String>>,
-    pub separator: Option<String>,
-    pub target_label: Option<String>,
-    pub regex: Option<String>,
-    pub modulus: Option<u64>,
-    pub replacement: Option<String>,
-    pub action: Option<String>,
-}
-
-/// Kubernetes service discovery configuration.
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct KubernetesSDConfig {
-    ///"pod", "service", "endpoints"pub role: String,
-    pub namespaces: Option<NamespaceSelector>,
-    pub selectors: Option<Vec<LabelSelector>>,
-    pub api_server: Option<String>,
-    pub bearer_token_file: Option<String>,
-    pub tls_config: Option<TLSConfig>,
-}
-
-/// Namespace selector for Kubernetes service discovery.
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct NamespaceSelector {
-    pub any: Option<bool>,
-    pub match_names: Option<Vec<String>>,
-}
-
-/// HTTP-based service discovery configuration.
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct HttpSDConfig {
-    pub url: String,
-    pub refresh_interval: Option<String>,
-    pub basic_auth: Option<BasicAuth>,
-    pub authorization: Option<Authorization>,
-    pub tls_config: Option<TLSConfig>,
-}
-
-/// File-based service discovery configuration.
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct FileSDConfig {
-    pub files: Vec<String>,
-    pub refresh_interval: Option<String>,
-}
-
-/// DNS-based service discovery configuration.
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct DnsSDConfig {
-    pub names: Vec<String>,
-    pub refresh_interval: Option<String>,
-    pub type_: Option<String>, // SRV, A, AAAA
-    pub port: Option<u16>,
-}
-
-/// Consul service discovery configuration.
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct ConsulSDConfig {
-    pub server: String,
-    pub services: Option<Vec<String>>,
-    pub scheme: Option<String>,
-    pub datacenter: Option<String>,
-    pub tag_separator: Option<String>,
-    pub refresh_interval: Option<String>,
-    pub tls_config: Option<TLSConfig>,
-}
-
-/// Basic authentication credentials.
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct BasicAuth {
-    pub username: String,
-    pub password: Option<String>,
-    pub password_file: Option<String>,
-}
-
-/// Bearer token or other auth mechanisms.
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct Authorization {
-    pub credentials: Option<String>,
-    pub credentials_file: Option<String>,
-    pub type_: Option<String>,
-}
-
-/// TLS configuration for secure scraping.
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct TLSConfig {
-    pub ca_file: Option<String>,
-    pub cert_file: Option<String>,
-    pub key_file: Option<String>,
-    pub server_name: Option<String>,
-    pub insecure_skip_verify: Option<bool>,
-}
-
-/// Authorization parameters for SNMP walk.
-#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct Params {
-    pub auth: Option<Vec<String>>,
-    pub module: Option<Vec<String>>,
-}

harmony/src/modules/monitoring/kube_prometheus/crd/mod.rs

@@ -4,7 +4,6 @@ pub mod crd_default_rules;
 pub mod crd_grafana;
 pub mod crd_prometheus_rules;
 pub mod crd_prometheuses;
-pub mod crd_scrape_config;
 pub mod grafana_default_dashboard;
 pub mod grafana_operator;
 pub mod prometheus_operator;

harmony/src/modules/monitoring/kube_prometheus/helm_prometheus_alert_score.rs

@@ -31,7 +31,6 @@ impl<T: Topology + HelmCommand + TenantManager> Score<T> for HelmPrometheusAlert
             sender: KubePrometheus { config },
             receivers: self.receivers.clone(),
             rules: self.rules.clone(),
-            scrape_targets: None,
         })
     }
     fn name(&self) -> String {

harmony/src/modules/monitoring/mod.rs

@@ -6,4 +6,3 @@ pub mod kube_prometheus;
 pub mod ntfy;
 pub mod okd;
 pub mod prometheus;
-pub mod scrape_target;

harmony/src/modules/monitoring/scrape_target/mod.rs

@@ -1 +0,0 @@
-pub mod server;

harmony/src/modules/monitoring/scrape_target/server.rs

@@ -1,76 +0,0 @@
-use std::net::IpAddr;
-
-use async_trait::async_trait;
-use kube::api::ObjectMeta;
-use serde::Serialize;
-
-use crate::{
-    interpret::{InterpretError, Outcome},
-    modules::monitoring::kube_prometheus::crd::{
-        crd_alertmanager_config::CRDPrometheus,
-        crd_scrape_config::{Params, RelabelConfig, ScrapeConfig, ScrapeConfigSpec, StaticConfig},
-    },
-    topology::oberservability::monitoring::ScrapeTarget,
-};
-
-#[derive(Debug, Clone, Serialize)]
-pub struct Server {
-    pub name: String,
-    pub ip: IpAddr,
-    pub auth: String,
-    pub module: String,
-    pub domain: String,
-}
-
-#[async_trait]
-impl ScrapeTarget<CRDPrometheus> for Server {
-    async fn install(&self, sender: &CRDPrometheus) -> Result<Outcome, InterpretError> {
-        let scrape_config_spec = ScrapeConfigSpec {
-            static_configs: Some(vec![StaticConfig {
-                targets: vec![self.ip.to_string()],
-                labels: None,
-            }]),
-            scrape_interval: Some("2m".to_string()),
-            kubernetes_sd_configs: None,
-            http_sd_configs: None,
-            file_sd_configs: None,
-            dns_sd_configs: None,
-            params: Some(Params {
-                auth: Some(vec![self.auth.clone()]),
-                module: Some(vec![self.module.clone()]),
-            }),
-            consul_sd_configs: None,
-            relabel_configs: Some(vec![RelabelConfig {
-                action: None,
-                source_labels: Some(vec!["__address__".to_string()]),
-                separator: None,
-                target_label: Some("__param_target".to_string()),
-                regex: None,
-                replacement: Some(format!("snmp.{}:31080", self.domain.clone())),
-                modulus: None,
-            }]),
-            metric_relabel_configs: None,
-            metrics_path: Some("/snmp".to_string()),
-            scrape_timeout: Some("2m".to_string()),
-            job_name: Some(format!("snmp_exporter/cloud/{}", self.name.clone())),
-            scheme: None,
-        };
-
-        let scrape_config = ScrapeConfig {
-            metadata: ObjectMeta {
-                name: Some(self.name.clone()),
-                namespace: Some(sender.namespace.clone()),
-                ..Default::default()
-            },
-            spec: scrape_config_spec,
-        };
-        sender
-            .client
-            .apply(&scrape_config, Some(&sender.namespace.clone()))
-            .await?;
-        Ok(Outcome::success(format!(
-            "installed scrape target {}",
-            self.name.clone()
-        )))
-    }
-}

harmony/src/modules/okd/bootstrap_03_control_plane.rs

@@ -5,10 +5,8 @@ use crate::{
     interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
     inventory::{HostRole, Inventory},
     modules::{
-        dhcp::DhcpHostBindingScore,
+        dhcp::DhcpHostBindingScore, http::IPxeMacBootFileScore,
-        http::IPxeMacBootFileScore,
+        inventory::DiscoverHostForRoleScore, okd::templates::BootstrapIpxeTpl,
-        inventory::DiscoverHostForRoleScore,
-        okd::{host_network::HostNetworkConfigurationScore, templates::BootstrapIpxeTpl},
     },
     score::Score,
     topology::{HAClusterTopology, HostBinding},
@@ -205,28 +203,6 @@ impl OKDSetup03ControlPlaneInterpret {
 
         Ok(())
     }
-
-    /// Placeholder for automating network bonding configuration.
-    async fn persist_network_bond(
-        &self,
-        inventory: &Inventory,
-        topology: &HAClusterTopology,
-        hosts: &Vec<PhysicalHost>,
-    ) -> Result<(), InterpretError> {
-        info!("[ControlPlane] Ensuring persistent bonding");
-        let score = HostNetworkConfigurationScore {
-            hosts: hosts.clone(),
-        };
-        score.interpret(inventory, topology).await?;
-
-        inquire::Confirm::new(
-            "Network configuration for control plane nodes is not automated yet. Configure it manually if needed.",
-        )
-        .prompt()
-        .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?;
-
-        Ok(())
-    }
 }
 
 #[async_trait]
@@ -265,10 +241,6 @@ impl Interpret<HAClusterTopology> for OKDSetup03ControlPlaneInterpret {
         // 4. Reboot the nodes to start the OS installation.
         self.reboot_targets(&nodes).await?;
 
-        // 5. Placeholder for post-boot network configuration (e.g., bonding).
-        self.persist_network_bond(inventory, topology, &nodes)
-            .await?;
-
         // TODO: Implement a step to wait for the control plane nodes to join the cluster
         // and for the cluster operators to become available. This would be similar to
         // the `wait-for bootstrap-complete` command.

harmony/src/modules/okd/bootstrap_load_balancer.rs

@@ -77,8 +77,6 @@ impl OKDBootstrapLoadBalancerScore {
             address: topology.bootstrap_host.ip.to_string(),
             port,
         });
-
-        backend.dedup();
         backend
     }
 }

harmony/src/modules/okd/bootstrap_persist_network_bond.rs

@@ -0,0 +1,131 @@
+use crate::{
+    data::Version,
+    hardware::PhysicalHost,
+    infra::inventory::InventoryRepositoryFactory,
+    interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome},
+    inventory::{HostRole, Inventory},
+    modules::okd::host_network::HostNetworkConfigurationScore,
+    score::Score,
+    topology::HAClusterTopology,
+};
+use async_trait::async_trait;
+use derive_new::new;
+use harmony_types::id::Id;
+use log::info;
+use serde::Serialize;
+
+// -------------------------------------------------------------------------------------------------
+// Step XX: Persist Network Bond
+// - Persist bonding via NMState
+// - Persist port channels on the Switch
+// -------------------------------------------------------------------------------------------------
+
+#[derive(Debug, Clone, Serialize, new)]
+pub struct OKDSetupPersistNetworkBondScore {}
+
+impl Score<HAClusterTopology> for OKDSetupPersistNetworkBondScore {
+    fn create_interpret(&self) -> Box<dyn Interpret<HAClusterTopology>> {
+        Box::new(OKDSetupPersistNetworkBondInterpet::new())
+    }
+
+    fn name(&self) -> String {
+        "OKDSetupPersistNetworkBondScore".to_string()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct OKDSetupPersistNetworkBondInterpet {
+    version: Version,
+    status: InterpretStatus,
+}
+
+impl OKDSetupPersistNetworkBondInterpet {
+    pub fn new() -> Self {
+        let version = Version::from("1.0.0").unwrap();
+        Self {
+            version,
+            status: InterpretStatus::QUEUED,
+        }
+    }
+
+    /// Ensures that three physical hosts are discovered and available for the ControlPlane role.
+    /// It will trigger discovery if not enough hosts are found.
+    async fn get_nodes(
+        &self,
+        _inventory: &Inventory,
+        _topology: &HAClusterTopology,
+    ) -> Result<Vec<PhysicalHost>, InterpretError> {
+        const REQUIRED_HOSTS: usize = 3;
+        let repo = InventoryRepositoryFactory::build().await?;
+        let control_plane_hosts = repo.get_host_for_role(&HostRole::ControlPlane).await?;
+
+        if control_plane_hosts.len() < REQUIRED_HOSTS {
+            Err(InterpretError::new(format!(
+                "OKD Requires at least {} control plane hosts, but only found {}. Cannot proceed.",
+                REQUIRED_HOSTS,
+                control_plane_hosts.len()
+            )))
+        } else {
+            // Take exactly the number of required hosts to ensure consistency.
+            Ok(control_plane_hosts
+                .into_iter()
+                .take(REQUIRED_HOSTS)
+                .collect())
+        }
+    }
+
+    async fn persist_network_bond(
+        &self,
+        inventory: &Inventory,
+        topology: &HAClusterTopology,
+        hosts: &Vec<PhysicalHost>,
+    ) -> Result<(), InterpretError> {
+        info!("[ControlPlane] Ensuring persistent bonding");
+        let score = HostNetworkConfigurationScore {
+            hosts: hosts.clone(),
+        };
+        score.interpret(inventory, topology).await?;
+
+        inquire::Confirm::new(
+            "Network configuration for control plane nodes is not automated yet. Configure it manually if needed.",
+        )
+        .prompt()
+        .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?;
+
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl Interpret<HAClusterTopology> for OKDSetupPersistNetworkBondInterpet {
+    fn get_name(&self) -> InterpretName {
+        InterpretName::Custom("OKDSetup03ControlPlane")
+    }
+
+    fn get_version(&self) -> Version {
+        self.version.clone()
+    }
+
+    fn get_status(&self) -> InterpretStatus {
+        self.status.clone()
+    }
+
+    fn get_children(&self) -> Vec<Id> {
+        vec![]
+    }
+
+    async fn execute(
+        &self,
+        inventory: &Inventory,
+        topology: &HAClusterTopology,
+    ) -> Result<Outcome, InterpretError> {
+        let nodes = self.get_nodes(inventory, topology).await?;
+
+        self.persist_network_bond(inventory, topology, &nodes)
+            .await?;
+
+        Ok(Outcome::success(
+            "Network bond successfully persisted".into(),
+        ))
+    }
+}

harmony/src/modules/okd/crd/mod.rs

@@ -28,7 +28,9 @@ pub struct SubscriptionSpec {
     pub name: String,
     pub source: String,
     pub source_namespace: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub channel: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub install_plan_approval: Option<InstallPlanApproval>,
 }
 
@@ -39,3 +41,22 @@ pub enum InstallPlanApproval {
     #[serde(rename = "Manual")]
     Manual,
 }
+
+#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[kube(
+    group = "operators.coreos.com",
+    version = "v1alpha1",
+    kind = "ClusterServiceVersion",
+    namespaced
+)]
+#[serde(rename_all = "camelCase")]
+pub struct ClusterServiceVersionSpec {
+    pub status: Option<ClusterServiceVersionStatus>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ClusterServiceVersionStatus {
+    pub phase: String,
+    pub reason: String,
+}

harmony/src/modules/okd/installation.rs

@@ -50,7 +50,7 @@
 use crate::{
     modules::okd::{
         OKDSetup01InventoryScore, OKDSetup02BootstrapScore, OKDSetup03ControlPlaneScore,
-        OKDSetup04WorkersScore, OKDSetup05SanityCheckScore,
+        OKDSetup04WorkersScore, OKDSetup05SanityCheckScore, OKDSetupPersistNetworkBondScore,
         bootstrap_06_installation_report::OKDSetup06InstallationReportScore,
     },
     score::Score,
@@ -65,6 +65,7 @@ impl OKDInstallationPipeline {
             Box::new(OKDSetup01InventoryScore::new()),
             Box::new(OKDSetup02BootstrapScore::new()),
             Box::new(OKDSetup03ControlPlaneScore::new()),
+            Box::new(OKDSetupPersistNetworkBondScore::new()),
             Box::new(OKDSetup04WorkersScore::new()),
             Box::new(OKDSetup05SanityCheckScore::new()),
             Box::new(OKDSetup06InstallationReportScore::new()),

harmony/src/modules/okd/mod.rs

@@ -6,6 +6,7 @@ mod bootstrap_05_sanity_check;
 mod bootstrap_06_installation_report;
 pub mod bootstrap_dhcp;
 pub mod bootstrap_load_balancer;
+mod bootstrap_persist_network_bond;
 pub mod dhcp;
 pub mod dns;
 pub mod installation;
@@ -19,5 +20,6 @@ pub use bootstrap_03_control_plane::*;
 pub use bootstrap_04_workers::*;
 pub use bootstrap_05_sanity_check::*;
 pub use bootstrap_06_installation_report::*;
+pub use bootstrap_persist_network_bond::*;
 pub mod crd;
 pub mod host_network;

harmony/src/modules/storage/ceph/ceph_osd_replacement_score.rs

@@ -4,7 +4,7 @@ use std::{
 };
 
 use async_trait::async_trait;
-use log::{debug, warn};
+use log::{info, warn};
 use serde::{Deserialize, Serialize};
 use tokio::time::sleep;
 
@@ -19,8 +19,8 @@ use harmony_types::id::Id;
 
 #[derive(Debug, Clone, Serialize)]
 pub struct CephRemoveOsd {
-    pub osd_deployment_name: String,
+    osd_deployment_name: String,
-    pub rook_ceph_namespace: String,
+    rook_ceph_namespace: String,
 }
 
 impl<T: Topology + K8sclient> Score<T> for CephRemoveOsd {
@@ -54,17 +54,18 @@ impl<T: Topology + K8sclient> Interpret<T> for CephRemoveOsdInterpret {
         self.verify_deployment_scaled(client.clone()).await?;
         self.delete_deployment(client.clone()).await?;
         self.verify_deployment_deleted(client.clone()).await?;
-        self.purge_ceph_osd(client.clone()).await?;
-        self.verify_ceph_osd_removal(client.clone()).await?;
-
         let osd_id_full = self.get_ceph_osd_id().unwrap();
+        self.purge_ceph_osd(client.clone(), &osd_id_full).await?;
+        self.verify_ceph_osd_removal(client.clone(), &osd_id_full)
+            .await?;
+
         Ok(Outcome::success(format!(
             "Successfully removed OSD {} from rook-ceph cluster by deleting deployment {}",
             osd_id_full, self.score.osd_deployment_name
         )))
     }
     fn get_name(&self) -> InterpretName {
-        InterpretName::CephRemoveOsd
+        todo!()
     }
 
     fn get_version(&self) -> Version {
@@ -81,7 +82,7 @@ impl<T: Topology + K8sclient> Interpret<T> for CephRemoveOsdInterpret {
 }
 
 impl CephRemoveOsdInterpret {
-    pub fn get_ceph_osd_id_numeric(&self) -> Result<String, InterpretError> {
+    pub fn get_ceph_osd_id(&self) -> Result<String, InterpretError> {
         let osd_id_numeric = self
             .score
             .osd_deployment_name
@@ -93,14 +94,9 @@ impl CephRemoveOsdInterpret {
                     self.score.osd_deployment_name
                 ))
             })?;
-        Ok(osd_id_numeric.to_string())
-    }
-
-    pub fn get_ceph_osd_id(&self) -> Result<String, InterpretError> {
-        let osd_id_numeric = self.get_ceph_osd_id_numeric().unwrap();
         let osd_id_full = format!("osd.{}", osd_id_numeric);
 
-        debug!(
+        info!(
             "Targeting Ceph OSD: {} (parsed from deployment {})",
             osd_id_full, self.score.osd_deployment_name
         );
@@ -112,7 +108,6 @@ impl CephRemoveOsdInterpret {
         &self,
         client: Arc<K8sClient>,
     ) -> Result<Outcome, InterpretError> {
-        debug!("verifying toolbox exists");
         let toolbox_dep = "rook-ceph-tools".to_string();
 
         match client
@@ -154,7 +149,7 @@ impl CephRemoveOsdInterpret {
         &self,
         client: Arc<K8sClient>,
     ) -> Result<Outcome, InterpretError> {
-        debug!(
+        info!(
             "Scaling down OSD deployment: {}",
             self.score.osd_deployment_name
         );
@@ -177,7 +172,7 @@ impl CephRemoveOsdInterpret {
     ) -> Result<Outcome, InterpretError> {
         let (timeout, interval, start) = self.build_timer();
 
-        debug!("Waiting for OSD deployment to scale down to 0 replicas");
+        info!("Waiting for OSD deployment to scale down to 0 replicas");
         loop {
             let dep = client
                 .get_deployment(
@@ -185,9 +180,11 @@ impl CephRemoveOsdInterpret {
                     Some(&self.score.rook_ceph_namespace),
                 )
                 .await?;
+
             if let Some(deployment) = dep {
                 if let Some(status) = deployment.status {
-                    if status.replicas == None && status.ready_replicas == None {
+                    if status.replicas.unwrap_or(1) == 0 && status.ready_replicas.unwrap_or(1) == 0
+                    {
                         return Ok(Outcome::success(
                             "Deployment successfully scaled down.".to_string(),
                         ));
@@ -215,7 +212,7 @@ impl CephRemoveOsdInterpret {
         &self,
         client: Arc<K8sClient>,
     ) -> Result<Outcome, InterpretError> {
-        debug!(
+        info!(
             "Deleting OSD deployment: {}",
             self.score.osd_deployment_name
         );
@@ -237,7 +234,7 @@ impl CephRemoveOsdInterpret {
     ) -> Result<Outcome, InterpretError> {
         let (timeout, interval, start) = self.build_timer();
 
-        debug!("Verifying OSD deployment deleted");
+        info!("Waiting for OSD deployment to scale down to 0 replicas");
         loop {
             let dep = client
                 .get_deployment(
@@ -247,7 +244,7 @@ impl CephRemoveOsdInterpret {
                 .await?;
 
             if dep.is_none() {
-                debug!(
+                info!(
                     "Deployment {} successfully deleted.",
                     self.score.osd_deployment_name
                 );
@@ -279,10 +276,12 @@ impl CephRemoveOsdInterpret {
         Ok(tree)
     }
 
-    pub async fn purge_ceph_osd(&self, client: Arc<K8sClient>) -> Result<Outcome, InterpretError> {
+    pub async fn purge_ceph_osd(
-        let osd_id_numeric = self.get_ceph_osd_id_numeric().unwrap();
+        &self,
-        let osd_id_full = self.get_ceph_osd_id().unwrap();
+        client: Arc<K8sClient>,
-        debug!(
+        osd_id_full: &str,
+    ) -> Result<Outcome, InterpretError> {
+        info!(
             "Purging OSD {} from Ceph cluster and removing its auth key",
             osd_id_full
         );
@@ -292,9 +291,8 @@ impl CephRemoveOsdInterpret {
                 "app".to_string(),
                 Some(&self.score.rook_ceph_namespace),
                 vec![
-                    "sh",
+                    format!("ceph osd purge {osd_id_full} --yes-i-really-mean-it").as_str(),
-                    "-c",
+                    format!("ceph auth del osd.{osd_id_full}").as_str(),
-                    format!("ceph osd purge {osd_id_numeric} --yes-i-really-mean-it && ceph auth del {osd_id_full}").as_str(),
                 ],
             )
             .await?;
@@ -307,10 +305,10 @@ impl CephRemoveOsdInterpret {
     pub async fn verify_ceph_osd_removal(
         &self,
         client: Arc<K8sClient>,
+        osd_id_full: &str,
     ) -> Result<Outcome, InterpretError> {
         let (timeout, interval, start) = self.build_timer();
-        let osd_id_full = self.get_ceph_osd_id().unwrap();
+        info!(
-        debug!(
             "Verifying OSD {} has been removed from the Ceph tree...",
             osd_id_full
         );
@@ -320,7 +318,7 @@ impl CephRemoveOsdInterpret {
                     "rook-ceph-tools".to_string(),
                     "app".to_string(),
                     Some(&self.score.rook_ceph_namespace),
-                    vec!["sh", "-c", "ceph osd tree -f json"],
+                    vec!["ceph osd tree -f json"],
                 )
                 .await?;
             let tree =

harmony/src/modules/storage/ceph/mod.rs

@@ -1,2 +1,2 @@
-pub mod ceph_remove_osd_score;
+pub mod ceph_osd_replacement_score;
 pub mod ceph_validate_health_score;

opnsense-config-xml/src/data/haproxy.rs

@@ -77,7 +77,7 @@ impl YaSerializeTrait for HAProxyId {
     }
 }
 
-#[derive(PartialEq, Debug, Clone)]
+#[derive(PartialEq, Debug)]
 pub struct HAProxyId(String);
 
 impl Default for HAProxyId {
@@ -297,7 +297,7 @@ pub struct HAProxyFrontends {
     pub frontend: Vec<Frontend>,
 }
 
-#[derive(Clone, Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
+#[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
 pub struct Frontend {
     #[yaserde(attribute = true)]
     pub uuid: String,
@@ -310,7 +310,7 @@ pub struct Frontend {
     pub bind_options: MaybeString,
     pub mode: String,
     #[yaserde(rename = "defaultBackend")]
-    pub default_backend: Option<String>,
+    pub default_backend: String,
     pub ssl_enabled: i32,
     pub ssl_certificates: MaybeString,
     pub ssl_default_certificate: MaybeString,
@@ -416,7 +416,7 @@ pub struct HAProxyBackends {
     pub backends: Vec<HAProxyBackend>,
 }
 
-#[derive(Clone, Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
+#[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
 pub struct HAProxyBackend {
     #[yaserde(attribute = true, rename = "uuid")]
     pub uuid: String,
@@ -535,7 +535,7 @@ pub struct HAProxyServers {
     pub servers: Vec<HAProxyServer>,
 }
 
-#[derive(Clone, Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
+#[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
 pub struct HAProxyServer {
     #[yaserde(attribute = true, rename = "uuid")]
     pub uuid: String,
@@ -543,8 +543,8 @@ pub struct HAProxyServer {
     pub enabled: u8,
     pub name: String,
     pub description: MaybeString,
-    pub address: Option<String>,
+    pub address: String,
-    pub port: Option<u16>,
+    pub port: u16,
     pub checkport: MaybeString,
     pub mode: String,
     pub multiplexer_protocol: MaybeString,
@@ -589,7 +589,7 @@ pub struct HAProxyHealthChecks {
     pub healthchecks: Vec<HAProxyHealthCheck>,
 }
 
-#[derive(Clone, Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
+#[derive(Default, PartialEq, Debug, YaSerialize, YaDeserialize)]
 pub struct HAProxyHealthCheck {
     #[yaserde(attribute = true)]
     pub uuid: String,

opnsense-config-xml/src/data/interfaces.rs

@@ -9,7 +9,7 @@ pub struct Interface {
     pub physical_interface_name: String,
     pub descr: Option<MaybeString>,
     pub mtu: Option<MaybeString>,
-    pub enable: MaybeString,
+    pub enable: Option<MaybeString>,
     pub lock: Option<MaybeString>,
     #[yaserde(rename = "spoofmac")]
     pub spoof_mac: Option<MaybeString>,

opnsense-config/Cargo.toml

@@ -25,7 +25,6 @@ sha2 = "0.10.9"
 
 [dev-dependencies]
 pretty_assertions.workspace = true
-assertor.workspace = true
 
 [lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(e2e_test)'] }

opnsense-config/src/config/manager/ssh.rs

@@ -30,7 +30,8 @@ impl SshConfigManager {
 
         self.opnsense_shell
             .exec(&format!(
-                "cp /conf/config.xml /conf/backup/{backup_filename}"
+                "cp /conf/config.xml /conf/backup/{}",
+                backup_filename
             ))
             .await
     }

opnsense-config/src/config/shell/mod.rs

@@ -1,8 +1,10 @@
 mod ssh;
-use crate::Error;
-use async_trait::async_trait;
 pub use ssh::*;
 
+use async_trait::async_trait;
+
+use crate::Error;
+
 #[async_trait]
 pub trait OPNsenseShell: std::fmt::Debug + Send + Sync {
     async fn exec(&self, command: &str) -> Result<String, Error>;

opnsense-config/src/modules/load_balancer.rs

@@ -1,8 +1,11 @@
-use crate::{config::OPNsenseShell, Error};
+use std::sync::Arc;
+
+use log::warn;
 use opnsense_config_xml::{
     Frontend, HAProxy, HAProxyBackend, HAProxyHealthCheck, HAProxyServer, OPNsense,
 };
-use std::{collections::HashSet, sync::Arc};
+
+use crate::{config::OPNsenseShell, Error};
 
 pub struct LoadBalancerConfig<'a> {
     opnsense: &'a mut OPNsense,
@@ -28,7 +31,7 @@ impl<'a> LoadBalancerConfig<'a> {
         match &mut self.opnsense.opnsense.haproxy.as_mut() {
             Some(haproxy) => f(haproxy),
             None => unimplemented!(
-                "Cannot configure load balancer when haproxy config does not exist yet"
+                "Adding a backend is not supported when haproxy config does not exist yet"
             ),
         }
     }
@@ -37,67 +40,21 @@ impl<'a> LoadBalancerConfig<'a> {
         self.with_haproxy(|haproxy| haproxy.general.enabled = enabled as i32);
     }
 
-    /// Configures a service by removing any existing service on the same port
+    pub fn add_backend(&mut self, backend: HAProxyBackend) {
-    /// and then adding the new definition. This ensures idempotency.
+        warn!("TODO make sure this new backend does not refer non-existing entities like servers or health checks");
-    pub fn configure_service(
+        self.with_haproxy(|haproxy| haproxy.backends.backends.push(backend));
-        &mut self,
-        frontend: Frontend,
-        backend: HAProxyBackend,
-        servers: Vec<HAProxyServer>,
-        healthcheck: Option<HAProxyHealthCheck>,
-    ) {
-        self.remove_service_by_bind_address(&frontend.bind);
-        self.remove_servers(&servers);
-
-        self.add_new_service(frontend, backend, servers, healthcheck);
     }
 
-    // Remove the corresponding real servers based on their name if they already exist.
+    pub fn add_frontend(&mut self, frontend: Frontend) {
-    fn remove_servers(&mut self, servers: &[HAProxyServer]) {
+        self.with_haproxy(|haproxy| haproxy.frontends.frontend.push(frontend));
-        let server_names: HashSet<_> = servers.iter().map(|s| s.name.clone()).collect();
-        self.with_haproxy(|haproxy| {
-            haproxy
-                .servers
-                .servers
-                .retain(|s| !server_names.contains(&s.name));
-        });
     }
 
-    /// Removes a service and its dependent components based on the frontend's bind address.
+    pub fn add_healthcheck(&mut self, healthcheck: HAProxyHealthCheck) {
-    /// This performs a cascading delete of the frontend, backend, servers, and health check.
+        self.with_haproxy(|haproxy| haproxy.healthchecks.healthchecks.push(healthcheck));
-    fn remove_service_by_bind_address(&mut self, bind_address: &str) {
-        self.with_haproxy(|haproxy| {
-            let Some(old_frontend) = remove_frontend_by_bind_address(haproxy, bind_address) else {
-                return;
-            };
-
-            let Some(old_backend) = remove_backend(haproxy, old_frontend) else {
-                return;
-            };
-
-            remove_healthcheck(haproxy, &old_backend);
-            remove_linked_servers(haproxy, &old_backend);
-        });
     }
 
-    /// Adds the components of a new service to the HAProxy configuration.
+    pub fn add_servers(&mut self, mut servers: Vec<HAProxyServer>) {
-    /// This function de-duplicates servers by name to prevent configuration errors.
+        self.with_haproxy(|haproxy| haproxy.servers.servers.append(&mut servers));
-    fn add_new_service(
-        &mut self,
-        frontend: Frontend,
-        backend: HAProxyBackend,
-        servers: Vec<HAProxyServer>,
-        healthcheck: Option<HAProxyHealthCheck>,
-    ) {
-        self.with_haproxy(|haproxy| {
-            if let Some(check) = healthcheck {
-                haproxy.healthchecks.healthchecks.push(check);
-            }
-
-            haproxy.servers.servers.extend(servers);
-            haproxy.backends.backends.push(backend);
-            haproxy.frontends.frontend.push(frontend);
-        });
     }
 
     pub async fn reload_restart(&self) -> Result<(), Error> {
@@ -125,262 +82,3 @@ impl<'a> LoadBalancerConfig<'a> {
         Ok(())
     }
 }
-
-fn remove_frontend_by_bind_address(haproxy: &mut HAProxy, bind_address: &str) -> Option<Frontend> {
-    let pos = haproxy
-        .frontends
-        .frontend
-        .iter()
-        .position(|f| f.bind == bind_address);
-
-    match pos {
-        Some(pos) => Some(haproxy.frontends.frontend.remove(pos)),
-        None => None,
-    }
-}
-
-fn remove_backend(haproxy: &mut HAProxy, old_frontend: Frontend) -> Option<HAProxyBackend> {
-    let default_backend = old_frontend.default_backend?;
-    let pos = haproxy
-        .backends
-        .backends
-        .iter()
-        .position(|b| b.uuid == default_backend);
-
-    match pos {
-        Some(pos) => Some(haproxy.backends.backends.remove(pos)),
-        None => None, // orphaned frontend, shouldn't happen
-    }
-}
-
-fn remove_healthcheck(haproxy: &mut HAProxy, backend: &HAProxyBackend) {
-    if let Some(uuid) = &backend.health_check.content {
-        haproxy
-            .healthchecks
-            .healthchecks
-            .retain(|h| h.uuid != *uuid);
-    }
-}
-
-/// Remove the backend's servers. This assumes servers are not shared between services.
-fn remove_linked_servers(haproxy: &mut HAProxy, backend: &HAProxyBackend) {
-    if let Some(server_uuids_str) = &backend.linked_servers.content {
-        let server_uuids_to_remove: HashSet<_> = server_uuids_str.split(',').collect();
-        haproxy
-            .servers
-            .servers
-            .retain(|s| !server_uuids_to_remove.contains(s.uuid.as_str()));
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::config::DummyOPNSenseShell;
-    use assertor::*;
-    use opnsense_config_xml::{
-        Frontend, HAProxy, HAProxyBackend, HAProxyBackends, HAProxyFrontends, HAProxyHealthCheck,
-        HAProxyHealthChecks, HAProxyId, HAProxyServer, HAProxyServers, MaybeString, OPNsense,
-    };
-    use std::sync::Arc;
-
-    use super::LoadBalancerConfig;
-
-    static SERVICE_BIND_ADDRESS: &str = "192.168.1.1:80";
-    static OTHER_SERVICE_BIND_ADDRESS: &str = "192.168.1.1:443";
-
-    static SERVER_ADDRESS: &str = "1.1.1.1:80";
-    static OTHER_SERVER_ADDRESS: &str = "1.1.1.1:443";
-
-    #[test]
-    fn configure_service_should_add_all_service_components_to_haproxy() {
-        let mut opnsense = given_opnsense();
-        let mut load_balancer = given_load_balancer(&mut opnsense);
-        let (healthcheck, servers, backend, frontend) =
-            given_service(SERVICE_BIND_ADDRESS, SERVER_ADDRESS);
-
-        load_balancer.configure_service(
-            frontend.clone(),
-            backend.clone(),
-            servers.clone(),
-            Some(healthcheck.clone()),
-        );
-
-        assert_haproxy_configured_with(
-            opnsense,
-            vec![frontend],
-            vec![backend],
-            servers,
-            vec![healthcheck],
-        );
-    }
-
-    #[test]
-    fn configure_service_should_replace_service_on_same_bind_address() {
-        let (healthcheck, servers, backend, frontend) =
-            given_service(SERVICE_BIND_ADDRESS, SERVER_ADDRESS);
-        let mut opnsense = given_opnsense_with(given_haproxy(
-            vec![frontend.clone()],
-            vec![backend.clone()],
-            servers.clone(),
-            vec![healthcheck.clone()],
-        ));
-        let mut load_balancer = given_load_balancer(&mut opnsense);
-
-        let (updated_healthcheck, updated_servers, updated_backend, updated_frontend) =
-            given_service(SERVICE_BIND_ADDRESS, OTHER_SERVER_ADDRESS);
-
-        load_balancer.configure_service(
-            updated_frontend.clone(),
-            updated_backend.clone(),
-            updated_servers.clone(),
-            Some(updated_healthcheck.clone()),
-        );
-
-        assert_haproxy_configured_with(
-            opnsense,
-            vec![updated_frontend],
-            vec![updated_backend],
-            updated_servers,
-            vec![updated_healthcheck],
-        );
-    }
-
-    #[test]
-    fn configure_service_should_keep_existing_service_on_different_bind_addresses() {
-        let (healthcheck, servers, backend, frontend) =
-            given_service(SERVICE_BIND_ADDRESS, SERVER_ADDRESS);
-        let (other_healthcheck, other_servers, other_backend, other_frontend) =
-            given_service(OTHER_SERVICE_BIND_ADDRESS, OTHER_SERVER_ADDRESS);
-        let mut opnsense = given_opnsense_with(given_haproxy(
-            vec![frontend.clone()],
-            vec![backend.clone()],
-            servers.clone(),
-            vec![healthcheck.clone()],
-        ));
-        let mut load_balancer = given_load_balancer(&mut opnsense);
-
-        load_balancer.configure_service(
-            other_frontend.clone(),
-            other_backend.clone(),
-            other_servers.clone(),
-            Some(other_healthcheck.clone()),
-        );
-
-        assert_haproxy_configured_with(
-            opnsense,
-            vec![frontend, other_frontend],
-            vec![backend, other_backend],
-            [servers, other_servers].concat(),
-            vec![healthcheck, other_healthcheck],
-        );
-    }
-
-    fn assert_haproxy_configured_with(
-        opnsense: OPNsense,
-        frontends: Vec<Frontend>,
-        backends: Vec<HAProxyBackend>,
-        servers: Vec<HAProxyServer>,
-        healthchecks: Vec<HAProxyHealthCheck>,
-    ) {
-        let haproxy = opnsense.opnsense.haproxy.as_ref().unwrap();
-        assert_that!(haproxy.frontends.frontend).contains_exactly(frontends);
-        assert_that!(haproxy.backends.backends).contains_exactly(backends);
-        assert_that!(haproxy.servers.servers).is_equal_to(servers);
-        assert_that!(haproxy.healthchecks.healthchecks).contains_exactly(healthchecks);
-    }
-
-    fn given_opnsense() -> OPNsense {
-        OPNsense::default()
-    }
-
-    fn given_opnsense_with(haproxy: HAProxy) -> OPNsense {
-        let mut opnsense = OPNsense::default();
-        opnsense.opnsense.haproxy = Some(haproxy);
-
-        opnsense
-    }
-
-    fn given_load_balancer<'a>(opnsense: &'a mut OPNsense) -> LoadBalancerConfig<'a> {
-        let opnsense_shell = Arc::new(DummyOPNSenseShell {});
-        if opnsense.opnsense.haproxy.is_none() {
-            opnsense.opnsense.haproxy = Some(HAProxy::default());
-        }
-        LoadBalancerConfig::new(opnsense, opnsense_shell)
-    }
-
-    fn given_service(
-        bind_address: &str,
-        server_address: &str,
-    ) -> (
-        HAProxyHealthCheck,
-        Vec<HAProxyServer>,
-        HAProxyBackend,
-        Frontend,
-    ) {
-        let healthcheck = given_healthcheck();
-        let servers = vec![given_server(server_address)];
-        let backend = given_backend();
-        let frontend = given_frontend(bind_address);
-        (healthcheck, servers, backend, frontend)
-    }
-
-    fn given_haproxy(
-        frontends: Vec<Frontend>,
-        backends: Vec<HAProxyBackend>,
-        servers: Vec<HAProxyServer>,
-        healthchecks: Vec<HAProxyHealthCheck>,
-    ) -> HAProxy {
-        HAProxy {
-            frontends: HAProxyFrontends {
-                frontend: frontends,
-            },
-            backends: HAProxyBackends { backends },
-            servers: HAProxyServers { servers },
-            healthchecks: HAProxyHealthChecks { healthchecks },
-            ..Default::default()
-        }
-    }
-
-    fn given_frontend(bind_address: &str) -> Frontend {
-        Frontend {
-            uuid: "uuid".into(),
-            id: HAProxyId::default(),
-            enabled: 1,
-            name: format!("frontend_{bind_address}"),
-            bind: bind_address.into(),
-            default_backend: Some("backend-uuid".into()),
-            ..Default::default()
-        }
-    }
-
-    fn given_backend() -> HAProxyBackend {
-        HAProxyBackend {
-            uuid: "backend-uuid".into(),
-            id: HAProxyId::default(),
-            enabled: 1,
-            name: "backend_192.168.1.1:80".into(),
-            linked_servers: MaybeString::from("server-uuid"),
-            health_check_enabled: 1,
-            health_check: MaybeString::from("healthcheck-uuid"),
-            ..Default::default()
-        }
-    }
-
-    fn given_server(address: &str) -> HAProxyServer {
-        HAProxyServer {
-            uuid: "server-uuid".into(),
-            id: HAProxyId::default(),
-            name: address.into(),
-            address: Some(address.into()),
-            ..Default::default()
-        }
-    }
-
-    fn given_healthcheck() -> HAProxyHealthCheck {
-        HAProxyHealthCheck {
-            uuid: "healthcheck-uuid".into(),
-            name: "healthcheck".into(),
-            ..Default::default()
-        }
-    }
-}