From dbd1f1b010ef1589ea03c28d6e2f66fa81c9fa48 Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Thu, 16 Oct 2025 17:01:05 -0400 Subject: [PATCH 01/11] install operatorhub to configure nmstate --- harmony/src/domain/topology/ha_cluster.rs | 32 +++-- harmony/src/infra/kubers/mod.rs | 1 + harmony/src/infra/kubers/types.rs | 27 ++++ harmony/src/infra/mod.rs | 1 + harmony/src/modules/k8s/resource.rs | 6 +- .../modules/okd/bootstrap_03_control_plane.rs | 32 +---- .../okd/bootstrap_persist_network_bond.rs | 131 ++++++++++++++++++ harmony/src/modules/okd/crd/mod.rs | 1 - harmony/src/modules/okd/installation.rs | 3 +- harmony/src/modules/okd/mod.rs | 2 + opnsense-config-xml/src/data/interfaces.rs | 2 +- 11 files changed, 192 insertions(+), 46 deletions(-) create mode 100644 harmony/src/infra/kubers/mod.rs create mode 100644 harmony/src/infra/kubers/types.rs create mode 100644 harmony/src/modules/okd/bootstrap_persist_network_bond.rs diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index 7be2725..5e9a567 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -47,6 +47,7 @@ pub struct HAClusterTopology { pub control_plane: Vec, pub workers: Vec, pub switch: Vec, + pub kubeconfig: Option, } #[async_trait] @@ -65,9 +66,17 @@ impl Topology for HAClusterTopology { #[async_trait] impl K8sclient for HAClusterTopology { async fn k8s_client(&self) -> Result, String> { - Ok(Arc::new( - K8sClient::try_default().await.map_err(|e| e.to_string())?, - )) + match &self.kubeconfig { + None => Ok(Arc::new( + K8sClient::try_default().await.map_err(|e| e.to_string())?, + )), + Some(kubeconfig) => { + let Some(client) = K8sClient::from_kubeconfig(&kubeconfig).await else { + return Err("Failed to create k8s client".to_string()); + }; + Ok(Arc::new(client)) + } + } } } @@ -123,7 +132,7 @@ impl HAClusterTopology { }; debug!("Creating NMState operator group: {nmstate_operator_group:#?}"); k8s_client - .apply(&nmstate_operator_group, None) + .apply(&nmstate_operator_group, Some("openshift-nmstate")) .await .map_err(|e| e.to_string())?; @@ -134,29 +143,29 @@ impl HAClusterTopology { ..Default::default() }, spec: SubscriptionSpec { - channel: Some("stable".to_string()), - install_plan_approval: Some(InstallPlanApproval::Automatic), + channel: Some("alpha".to_string()), name: "kubernetes-nmstate-operator".to_string(), - source: "redhat-operators".to_string(), + source: "operatorhubio-catalog".to_string(), source_namespace: "openshift-marketplace".to_string(), }, }; debug!("Subscribing to NMState Operator: {nmstate_subscription:#?}"); k8s_client - .apply(&nmstate_subscription, None) + .apply(&nmstate_subscription, Some("openshift-nmstate")) .await .map_err(|e| e.to_string())?; let nmstate = NMState { metadata: ObjectMeta { name: Some("nmstate".to_string()), + namespace: Some("openshift-nmstate".to_string()), ..Default::default() }, ..Default::default() }; debug!("Creating NMState: {nmstate:#?}"); k8s_client - .apply(&nmstate, None) + .apply(&nmstate, Some("openshift-nmstate")) .await .map_err(|e| e.to_string())?; @@ -187,9 +196,9 @@ impl HAClusterTopology { .unwrap() .apply(&bond_config, None) .await - .unwrap(); + .map_err(|e| SwitchError::new(format!("Failed to configure bond: {e}")))?; - todo!() + Ok(()) } fn create_bond_configuration( @@ -325,6 +334,7 @@ impl HAClusterTopology { }; Self { + kubeconfig: None, domain_name: "DummyTopology".to_string(), router: dummy_infra.clone(), load_balancer: dummy_infra.clone(), diff --git a/harmony/src/infra/kubers/mod.rs b/harmony/src/infra/kubers/mod.rs new file mode 100644 index 0000000..cd40856 --- /dev/null +++ b/harmony/src/infra/kubers/mod.rs @@ -0,0 +1 @@ +pub mod types; diff --git a/harmony/src/infra/kubers/types.rs b/harmony/src/infra/kubers/types.rs new file mode 100644 index 0000000..4e7db49 --- /dev/null +++ b/harmony/src/infra/kubers/types.rs @@ -0,0 +1,27 @@ +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(CustomResource, Default, Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[kube( + group = "operators.coreos.com", + version = "v1alpha1", + kind = "CatalogSource", + namespaced +)] +#[serde(rename_all = "camelCase")] +pub struct CatalogSourceSpec { + pub source_type: String, + pub image: String, + pub display_name: String, + pub publisher: String, +} + +impl Default for CatalogSource { + fn default() -> Self { + Self { + metadata: Default::default(), + spec: Default::default(), + } + } +} diff --git a/harmony/src/infra/mod.rs b/harmony/src/infra/mod.rs index 203cf90..5b38eab 100644 --- a/harmony/src/infra/mod.rs +++ b/harmony/src/infra/mod.rs @@ -3,5 +3,6 @@ pub mod executors; pub mod hp_ilo; pub mod intel_amt; pub mod inventory; +pub mod kubers; pub mod opnsense; mod sqlx; diff --git a/harmony/src/modules/k8s/resource.rs b/harmony/src/modules/k8s/resource.rs index d679326..57f9731 100644 --- a/harmony/src/modules/k8s/resource.rs +++ b/harmony/src/modules/k8s/resource.rs @@ -38,13 +38,15 @@ impl< + 'static + Send + Clone, - T: Topology, + T: Topology + K8sclient, > Score for K8sResourceScore where ::DynamicType: Default, { fn create_interpret(&self) -> Box> { - todo!() + Box::new(K8sResourceInterpret { + score: self.clone(), + }) } fn name(&self) -> String { diff --git a/harmony/src/modules/okd/bootstrap_03_control_plane.rs b/harmony/src/modules/okd/bootstrap_03_control_plane.rs index af8e71f..5abe848 100644 --- a/harmony/src/modules/okd/bootstrap_03_control_plane.rs +++ b/harmony/src/modules/okd/bootstrap_03_control_plane.rs @@ -5,10 +5,8 @@ use crate::{ interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, inventory::{HostRole, Inventory}, modules::{ - dhcp::DhcpHostBindingScore, - http::IPxeMacBootFileScore, - inventory::DiscoverHostForRoleScore, - okd::{host_network::HostNetworkConfigurationScore, templates::BootstrapIpxeTpl}, + dhcp::DhcpHostBindingScore, http::IPxeMacBootFileScore, + inventory::DiscoverHostForRoleScore, okd::templates::BootstrapIpxeTpl, }, score::Score, topology::{HAClusterTopology, HostBinding}, @@ -205,28 +203,6 @@ impl OKDSetup03ControlPlaneInterpret { Ok(()) } - - /// Placeholder for automating network bonding configuration. - async fn persist_network_bond( - &self, - inventory: &Inventory, - topology: &HAClusterTopology, - hosts: &Vec, - ) -> Result<(), InterpretError> { - info!("[ControlPlane] Ensuring persistent bonding"); - let score = HostNetworkConfigurationScore { - hosts: hosts.clone(), - }; - score.interpret(inventory, topology).await?; - - inquire::Confirm::new( - "Network configuration for control plane nodes is not automated yet. Configure it manually if needed.", - ) - .prompt() - .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?; - - Ok(()) - } } #[async_trait] @@ -265,10 +241,6 @@ impl Interpret for OKDSetup03ControlPlaneInterpret { // 4. Reboot the nodes to start the OS installation. self.reboot_targets(&nodes).await?; - // 5. Placeholder for post-boot network configuration (e.g., bonding). - self.persist_network_bond(inventory, topology, &nodes) - .await?; - // TODO: Implement a step to wait for the control plane nodes to join the cluster // and for the cluster operators to become available. This would be similar to // the `wait-for bootstrap-complete` command. diff --git a/harmony/src/modules/okd/bootstrap_persist_network_bond.rs b/harmony/src/modules/okd/bootstrap_persist_network_bond.rs new file mode 100644 index 0000000..6c486ea --- /dev/null +++ b/harmony/src/modules/okd/bootstrap_persist_network_bond.rs @@ -0,0 +1,131 @@ +use crate::{ + data::Version, + hardware::PhysicalHost, + infra::inventory::InventoryRepositoryFactory, + interpret::{Interpret, InterpretError, InterpretName, InterpretStatus, Outcome}, + inventory::{HostRole, Inventory}, + modules::okd::host_network::HostNetworkConfigurationScore, + score::Score, + topology::HAClusterTopology, +}; +use async_trait::async_trait; +use derive_new::new; +use harmony_types::id::Id; +use log::info; +use serde::Serialize; + +// ------------------------------------------------------------------------------------------------- +// Step XX: Persist Network Bond +// - Persist bonding via NMState +// - Persist port channels on the Switch +// ------------------------------------------------------------------------------------------------- + +#[derive(Debug, Clone, Serialize, new)] +pub struct OKDSetupPersistNetworkBondScore {} + +impl Score for OKDSetupPersistNetworkBondScore { + fn create_interpret(&self) -> Box> { + Box::new(OKDSetupPersistNetworkBondInterpet::new()) + } + + fn name(&self) -> String { + "OKDSetupPersistNetworkBondScore".to_string() + } +} + +#[derive(Debug, Clone)] +pub struct OKDSetupPersistNetworkBondInterpet { + version: Version, + status: InterpretStatus, +} + +impl OKDSetupPersistNetworkBondInterpet { + pub fn new() -> Self { + let version = Version::from("1.0.0").unwrap(); + Self { + version, + status: InterpretStatus::QUEUED, + } + } + + /// Ensures that three physical hosts are discovered and available for the ControlPlane role. + /// It will trigger discovery if not enough hosts are found. + async fn get_nodes( + &self, + _inventory: &Inventory, + _topology: &HAClusterTopology, + ) -> Result, InterpretError> { + const REQUIRED_HOSTS: usize = 3; + let repo = InventoryRepositoryFactory::build().await?; + let control_plane_hosts = repo.get_host_for_role(&HostRole::ControlPlane).await?; + + if control_plane_hosts.len() < REQUIRED_HOSTS { + Err(InterpretError::new(format!( + "OKD Requires at least {} control plane hosts, but only found {}. Cannot proceed.", + REQUIRED_HOSTS, + control_plane_hosts.len() + ))) + } else { + // Take exactly the number of required hosts to ensure consistency. + Ok(control_plane_hosts + .into_iter() + .take(REQUIRED_HOSTS) + .collect()) + } + } + + async fn persist_network_bond( + &self, + inventory: &Inventory, + topology: &HAClusterTopology, + hosts: &Vec, + ) -> Result<(), InterpretError> { + info!("[ControlPlane] Ensuring persistent bonding"); + let score = HostNetworkConfigurationScore { + hosts: hosts.clone(), + }; + score.interpret(inventory, topology).await?; + + inquire::Confirm::new( + "Network configuration for control plane nodes is not automated yet. Configure it manually if needed.", + ) + .prompt() + .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?; + + Ok(()) + } +} + +#[async_trait] +impl Interpret for OKDSetupPersistNetworkBondInterpet { + fn get_name(&self) -> InterpretName { + InterpretName::Custom("OKDSetup03ControlPlane") + } + + fn get_version(&self) -> Version { + self.version.clone() + } + + fn get_status(&self) -> InterpretStatus { + self.status.clone() + } + + fn get_children(&self) -> Vec { + vec![] + } + + async fn execute( + &self, + inventory: &Inventory, + topology: &HAClusterTopology, + ) -> Result { + let nodes = self.get_nodes(inventory, topology).await?; + + self.persist_network_bond(inventory, topology, &nodes) + .await?; + + Ok(Outcome::success( + "Network bond successfully persisted".into(), + )) + } +} diff --git a/harmony/src/modules/okd/crd/mod.rs b/harmony/src/modules/okd/crd/mod.rs index c1a68ce..c6b3416 100644 --- a/harmony/src/modules/okd/crd/mod.rs +++ b/harmony/src/modules/okd/crd/mod.rs @@ -29,7 +29,6 @@ pub struct SubscriptionSpec { pub source: String, pub source_namespace: String, pub channel: Option, - pub install_plan_approval: Option, } #[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] diff --git a/harmony/src/modules/okd/installation.rs b/harmony/src/modules/okd/installation.rs index 72603c8..3deb59a 100644 --- a/harmony/src/modules/okd/installation.rs +++ b/harmony/src/modules/okd/installation.rs @@ -50,7 +50,7 @@ use crate::{ modules::okd::{ OKDSetup01InventoryScore, OKDSetup02BootstrapScore, OKDSetup03ControlPlaneScore, - OKDSetup04WorkersScore, OKDSetup05SanityCheckScore, + OKDSetup04WorkersScore, OKDSetup05SanityCheckScore, OKDSetupPersistNetworkBondScore, bootstrap_06_installation_report::OKDSetup06InstallationReportScore, }, score::Score, @@ -65,6 +65,7 @@ impl OKDInstallationPipeline { Box::new(OKDSetup01InventoryScore::new()), Box::new(OKDSetup02BootstrapScore::new()), Box::new(OKDSetup03ControlPlaneScore::new()), + Box::new(OKDSetupPersistNetworkBondScore::new()), Box::new(OKDSetup04WorkersScore::new()), Box::new(OKDSetup05SanityCheckScore::new()), Box::new(OKDSetup06InstallationReportScore::new()), diff --git a/harmony/src/modules/okd/mod.rs b/harmony/src/modules/okd/mod.rs index a12f132..8bb85ef 100644 --- a/harmony/src/modules/okd/mod.rs +++ b/harmony/src/modules/okd/mod.rs @@ -6,6 +6,7 @@ mod bootstrap_05_sanity_check; mod bootstrap_06_installation_report; pub mod bootstrap_dhcp; pub mod bootstrap_load_balancer; +mod bootstrap_persist_network_bond; pub mod dhcp; pub mod dns; pub mod installation; @@ -19,5 +20,6 @@ pub use bootstrap_03_control_plane::*; pub use bootstrap_04_workers::*; pub use bootstrap_05_sanity_check::*; pub use bootstrap_06_installation_report::*; +pub use bootstrap_persist_network_bond::*; pub mod crd; pub mod host_network; diff --git a/opnsense-config-xml/src/data/interfaces.rs b/opnsense-config-xml/src/data/interfaces.rs index b06f392..fb49a4d 100644 --- a/opnsense-config-xml/src/data/interfaces.rs +++ b/opnsense-config-xml/src/data/interfaces.rs @@ -9,7 +9,7 @@ pub struct Interface { pub physical_interface_name: String, pub descr: Option, pub mtu: Option, - pub enable: MaybeString, + pub enable: Option, pub lock: Option, #[yaserde(rename = "spoofmac")] pub spoof_mac: Option, -- 2.39.5 From 83fcf9e8acdc33dec0c3865a14f7f47b18576c15 Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Fri, 17 Oct 2025 11:39:26 -0400 Subject: [PATCH 02/11] wip --- harmony/src/domain/topology/ha_cluster.rs | 31 +++++++++------ harmony/src/domain/topology/k8s.rs | 48 +++++++++++++++++++++-- harmony/src/modules/okd/crd/mod.rs | 22 +++++++++++ 3 files changed, 87 insertions(+), 14 deletions(-) diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index 5e9a567..4107953 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -29,9 +29,9 @@ use super::{ Topology, k8s::K8sClient, }; -use std::collections::BTreeMap; use std::net::IpAddr; use std::sync::Arc; +use std::{collections::BTreeMap, time::Duration}; #[derive(Debug, Clone)] pub struct HAClusterTopology { @@ -102,13 +102,13 @@ impl HAClusterTopology { } async fn ensure_nmstate_operator_installed(&self) -> Result<(), String> { - // FIXME: Find a way to check nmstate is already available (get pod -n openshift-nmstate) + // FIXME: Find a way to check nmstate is already available (get pod -n nmstate) debug!("Installing NMState operator..."); let k8s_client = self.k8s_client().await?; let nmstate_namespace = Namespace { metadata: ObjectMeta { - name: Some("openshift-nmstate".to_string()), + name: Some("nmstate".to_string()), finalizers: Some(vec!["kubernetes".to_string()]), ..Default::default() }, @@ -122,24 +122,24 @@ impl HAClusterTopology { let nmstate_operator_group = OperatorGroup { metadata: ObjectMeta { - name: Some("openshift-nmstate".to_string()), - namespace: Some("openshift-nmstate".to_string()), + name: Some("nmstate".to_string()), + namespace: Some("nmstate".to_string()), ..Default::default() }, spec: OperatorGroupSpec { - target_namespaces: vec!["openshift-nmstate".to_string()], + target_namespaces: vec!["nmstate".to_string()], }, }; debug!("Creating NMState operator group: {nmstate_operator_group:#?}"); k8s_client - .apply(&nmstate_operator_group, Some("openshift-nmstate")) + .apply(&nmstate_operator_group, Some("nmstate")) .await .map_err(|e| e.to_string())?; let nmstate_subscription = Subscription { metadata: ObjectMeta { name: Some("kubernetes-nmstate-operator".to_string()), - namespace: Some("openshift-nmstate".to_string()), + namespace: Some("nmstate".to_string()), ..Default::default() }, spec: SubscriptionSpec { @@ -147,25 +147,34 @@ impl HAClusterTopology { name: "kubernetes-nmstate-operator".to_string(), source: "operatorhubio-catalog".to_string(), source_namespace: "openshift-marketplace".to_string(), + install_plan_approval: Some(InstallPlanApproval::Automatic), }, }; debug!("Subscribing to NMState Operator: {nmstate_subscription:#?}"); k8s_client - .apply(&nmstate_subscription, Some("openshift-nmstate")) + .apply(&nmstate_subscription, Some("nmstate")) .await .map_err(|e| e.to_string())?; + k8s_client + .wait_for_operator( + "kubernetes-nmstate-operator", + Some("nmstate"), + Some(Duration::from_secs(30)), + ) + .await?; + let nmstate = NMState { metadata: ObjectMeta { name: Some("nmstate".to_string()), - namespace: Some("openshift-nmstate".to_string()), + namespace: Some("nmstate".to_string()), ..Default::default() }, ..Default::default() }; debug!("Creating NMState: {nmstate:#?}"); k8s_client - .apply(&nmstate, Some("openshift-nmstate")) + .apply(&nmstate, Some("nmstate")) .await .map_err(|e| e.to_string())?; diff --git a/harmony/src/domain/topology/k8s.rs b/harmony/src/domain/topology/k8s.rs index 5a1e6ec..22c1799 100644 --- a/harmony/src/domain/topology/k8s.rs +++ b/harmony/src/domain/topology/k8s.rs @@ -10,11 +10,13 @@ use k8s_openapi::{ }; use kube::{ Client, Config, Error, Resource, - api::{Api, AttachParams, DeleteParams, ListParams, Patch, PatchParams, ResourceExt}, + api::{ + Api, AttachParams, DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, ResourceExt, + }, config::{KubeConfigOptions, Kubeconfig}, core::ErrorResponse, error::DiscoveryError, - runtime::reflector::Lookup, + runtime::{reflector::Lookup, wait::Condition}, }; use kube::{api::DynamicObject, runtime::conditions}; use kube::{ @@ -22,11 +24,13 @@ use kube::{ runtime::wait::await_condition, }; use log::{debug, error, trace}; -use serde::{Serialize, de::DeserializeOwned}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; use serde_json::{Value, json}; use similar::TextDiff; use tokio::{io::AsyncReadExt, time::sleep}; +use crate::modules::okd::crd::ClusterServiceVersion; + #[derive(new, Clone)] pub struct K8sClient { client: Client, @@ -194,6 +198,33 @@ impl K8sClient { } } + pub async fn wait_for_operator( + &self, + operator_name: &str, + namespace: Option<&str>, + timeout: Option, + ) -> Result<(), String> { + let api: Api; + + if let Some(ns) = namespace { + api = Api::namespaced(self.client.clone(), ns); + } else { + api = Api::default_namespaced(self.client.clone()); + } + + let establish = await_condition(api, operator_name, is_operator_ready()); + let t = timeout.unwrap_or(Duration::from_secs(5)); + let res = tokio::time::timeout(t, establish).await; + + if res.is_ok() { + Ok(()) + } else { + Err(format!( + "timed out while waiting for operator {operator_name}" + )) + } + } + /// Will execute a commond in the first pod found that matches the specified label /// '{label}={name}' pub async fn exec_app_capture_output( @@ -547,3 +578,14 @@ where } } } + +fn is_operator_ready() -> impl Condition { + |obj: Option<&ClusterServiceVersion>| { + if let Some(csv) = obj { + if let Some(status) = &csv.spec.status { + return status.phase == "Succeeded"; + } + } + false + } +} diff --git a/harmony/src/modules/okd/crd/mod.rs b/harmony/src/modules/okd/crd/mod.rs index c6b3416..b26ad8d 100644 --- a/harmony/src/modules/okd/crd/mod.rs +++ b/harmony/src/modules/okd/crd/mod.rs @@ -28,7 +28,10 @@ pub struct SubscriptionSpec { pub name: String, pub source: String, pub source_namespace: String, + #[serde(skip_serializing_if = "Option::is_none")] pub channel: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub install_plan_approval: Option, } #[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] @@ -38,3 +41,22 @@ pub enum InstallPlanApproval { #[serde(rename = "Manual")] Manual, } + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[kube( + group = "operators.coreos.com", + version = "v1alpha1", + kind = "ClusterServiceVersion", + namespaced +)] +#[serde(rename_all = "camelCase")] +pub struct ClusterServiceVersionSpec { + pub status: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ClusterServiceVersionStatus { + pub phase: String, + pub reason: String, +} -- 2.39.5 From 035de57925a268380214e2a6db3ddcde9acd690a Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Tue, 21 Oct 2025 13:42:18 -0400 Subject: [PATCH 03/11] add missing field kubeconfig --- examples/nanodc/src/main.rs | 1 + examples/okd_installation/src/topology.rs | 1 + examples/okd_pxe/src/topology.rs | 1 + examples/opnsense/src/main.rs | 1 + 4 files changed, 4 insertions(+) diff --git a/examples/nanodc/src/main.rs b/examples/nanodc/src/main.rs index 57574d2..d52766c 100644 --- a/examples/nanodc/src/main.rs +++ b/examples/nanodc/src/main.rs @@ -39,6 +39,7 @@ async fn main() { let gateway_ipv4 = Ipv4Addr::new(192, 168, 33, 1); let gateway_ip = IpAddr::V4(gateway_ipv4); let topology = harmony::topology::HAClusterTopology { + kubeconfig: None, domain_name: "ncd0.harmony.mcd".to_string(), // TODO this must be set manually correctly // when setting up the opnsense firewall router: Arc::new(UnmanagedRouter::new( diff --git a/examples/okd_installation/src/topology.rs b/examples/okd_installation/src/topology.rs index 31062f5..a568e97 100644 --- a/examples/okd_installation/src/topology.rs +++ b/examples/okd_installation/src/topology.rs @@ -38,6 +38,7 @@ pub async fn get_topology() -> HAClusterTopology { let gateway_ipv4 = ipv4!("192.168.1.1"); let gateway_ip = IpAddr::V4(gateway_ipv4); harmony::topology::HAClusterTopology { + kubeconfig: None, domain_name: "demo.harmony.mcd".to_string(), router: Arc::new(UnmanagedRouter::new( gateway_ip, diff --git a/examples/okd_pxe/src/topology.rs b/examples/okd_pxe/src/topology.rs index 707969a..6be6af9 100644 --- a/examples/okd_pxe/src/topology.rs +++ b/examples/okd_pxe/src/topology.rs @@ -32,6 +32,7 @@ pub async fn get_topology() -> HAClusterTopology { let gateway_ipv4 = ipv4!("192.168.1.1"); let gateway_ip = IpAddr::V4(gateway_ipv4); harmony::topology::HAClusterTopology { + kubeconfig: None, domain_name: "demo.harmony.mcd".to_string(), router: Arc::new(UnmanagedRouter::new( gateway_ip, diff --git a/examples/opnsense/src/main.rs b/examples/opnsense/src/main.rs index fcfaf09..d269a9d 100644 --- a/examples/opnsense/src/main.rs +++ b/examples/opnsense/src/main.rs @@ -34,6 +34,7 @@ async fn main() { let gateway_ipv4 = Ipv4Addr::new(10, 100, 8, 1); let gateway_ip = IpAddr::V4(gateway_ipv4); let topology = harmony::topology::HAClusterTopology { + kubeconfig: None, domain_name: "demo.harmony.mcd".to_string(), router: Arc::new(UnmanagedRouter::new( gateway_ip, -- 2.39.5 From 48b5993a25ff9f835239f8f504e163c0c545fe5d Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Thu, 23 Oct 2025 11:00:54 -0400 Subject: [PATCH 04/11] describe grpc pod config --- harmony/src/domain/topology/ha_cluster.rs | 1 + harmony/src/infra/kubers/types.rs | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index 4107953..a8c3a91 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -130,6 +130,7 @@ impl HAClusterTopology { target_namespaces: vec!["nmstate".to_string()], }, }; + debug!("Creating NMState operator group: {nmstate_operator_group:#?}"); k8s_client .apply(&nmstate_operator_group, Some("nmstate")) diff --git a/harmony/src/infra/kubers/types.rs b/harmony/src/infra/kubers/types.rs index 4e7db49..48384ea 100644 --- a/harmony/src/infra/kubers/types.rs +++ b/harmony/src/infra/kubers/types.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -15,6 +17,8 @@ pub struct CatalogSourceSpec { pub image: String, pub display_name: String, pub publisher: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub grpc_pod_config: Option, } impl Default for CatalogSource { @@ -25,3 +29,12 @@ impl Default for CatalogSource { } } } + +#[derive(Default, Serialize, Deserialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct GrpcPodConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_target: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub node_selector: Option>, +} -- 2.39.5 From 0184e18c66dc0603f9baf4b316c38f9ff0818a3e Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Thu, 23 Oct 2025 14:26:19 -0400 Subject: [PATCH 05/11] remove CatalogSource CRD to replace it by resources loaded from URLs --- harmony/src/domain/topology/ha_cluster.rs | 80 +++++++----------- harmony/src/domain/topology/k8s.rs | 92 ++++++++++++++++++--- harmony/src/infra/kubers/mod.rs | 1 - harmony/src/infra/kubers/types.rs | 40 --------- harmony/src/infra/mod.rs | 1 - harmony/src/modules/monitoring/ntfy/ntfy.rs | 6 +- harmony/src/modules/okd/crd/nmstate.rs | 9 +- 7 files changed, 119 insertions(+), 110 deletions(-) delete mode 100644 harmony/src/infra/kubers/mod.rs delete mode 100644 harmony/src/infra/kubers/types.rs diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index a8c3a91..bfc2e57 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -102,80 +102,58 @@ impl HAClusterTopology { } async fn ensure_nmstate_operator_installed(&self) -> Result<(), String> { - // FIXME: Find a way to check nmstate is already available (get pod -n nmstate) - debug!("Installing NMState operator..."); let k8s_client = self.k8s_client().await?; - let nmstate_namespace = Namespace { - metadata: ObjectMeta { - name: Some("nmstate".to_string()), - finalizers: Some(vec!["kubernetes".to_string()]), - ..Default::default() - }, - ..Default::default() - }; - debug!("Creating NMState namespace: {nmstate_namespace:#?}"); - k8s_client - .apply(&nmstate_namespace, None) + debug!("Installing NMState controller..."); + k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/nmstate.io_nmstates.yaml +").unwrap(), Some("nmstate")) .await .map_err(|e| e.to_string())?; - let nmstate_operator_group = OperatorGroup { - metadata: ObjectMeta { - name: Some("nmstate".to_string()), - namespace: Some("nmstate".to_string()), - ..Default::default() - }, - spec: OperatorGroupSpec { - target_namespaces: vec!["nmstate".to_string()], - }, - }; - - debug!("Creating NMState operator group: {nmstate_operator_group:#?}"); - k8s_client - .apply(&nmstate_operator_group, Some("nmstate")) + debug!("Creating NMState namespace..."); + k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/namespace.yaml +").unwrap(), Some("nmstate")) .await .map_err(|e| e.to_string())?; - let nmstate_subscription = Subscription { - metadata: ObjectMeta { - name: Some("kubernetes-nmstate-operator".to_string()), - namespace: Some("nmstate".to_string()), - ..Default::default() - }, - spec: SubscriptionSpec { - channel: Some("alpha".to_string()), - name: "kubernetes-nmstate-operator".to_string(), - source: "operatorhubio-catalog".to_string(), - source_namespace: "openshift-marketplace".to_string(), - install_plan_approval: Some(InstallPlanApproval::Automatic), - }, - }; - debug!("Subscribing to NMState Operator: {nmstate_subscription:#?}"); - k8s_client - .apply(&nmstate_subscription, Some("nmstate")) + debug!("Creating NMState service account..."); + k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/service_account.yaml +").unwrap(), Some("nmstate")) + .await + .map_err(|e| e.to_string())?; + + debug!("Creating NMState role..."); + k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/role.yaml +").unwrap(), Some("nmstate")) + .await + .map_err(|e| e.to_string())?; + + debug!("Creating NMState role binding..."); + k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/role_binding.yaml +").unwrap(), Some("nmstate")) + .await + .map_err(|e| e.to_string())?; + + debug!("Creating NMState operator..."); + k8s_client.apply_url(url::Url::parse("https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.84.0/operator.yaml +").unwrap(), Some("nmstate")) .await .map_err(|e| e.to_string())?; k8s_client - .wait_for_operator( - "kubernetes-nmstate-operator", - Some("nmstate"), - Some(Duration::from_secs(30)), - ) + .wait_until_deployment_ready("nmstate-operator", Some("nmstate"), None) .await?; let nmstate = NMState { metadata: ObjectMeta { name: Some("nmstate".to_string()), - namespace: Some("nmstate".to_string()), ..Default::default() }, ..Default::default() }; debug!("Creating NMState: {nmstate:#?}"); k8s_client - .apply(&nmstate, Some("nmstate")) + .apply(&nmstate, None) .await .map_err(|e| e.to_string())?; diff --git a/harmony/src/domain/topology/k8s.rs b/harmony/src/domain/topology/k8s.rs index 22c1799..cfd0180 100644 --- a/harmony/src/domain/topology/k8s.rs +++ b/harmony/src/domain/topology/k8s.rs @@ -15,6 +15,7 @@ use kube::{ }, config::{KubeConfigOptions, Kubeconfig}, core::ErrorResponse, + discovery::{ApiCapabilities, Scope}, error::DiscoveryError, runtime::{reflector::Lookup, wait::Condition}, }; @@ -23,11 +24,13 @@ use kube::{ api::{ApiResource, GroupVersionKind}, runtime::wait::await_condition, }; -use log::{debug, error, trace}; +use log::{debug, error, info, trace, warn}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use serde_json::{Value, json}; +use serde_value::DeserializerError; use similar::TextDiff; use tokio::{io::AsyncReadExt, time::sleep}; +use url::Url; use crate::modules::okd::crd::ClusterServiceVersion; @@ -140,9 +143,9 @@ impl K8sClient { pub async fn wait_until_deployment_ready( &self, - name: String, + name: &str, namespace: Option<&str>, - timeout: Option, + timeout: Option, ) -> Result<(), String> { let api: Api; @@ -152,9 +155,9 @@ impl K8sClient { api = Api::default_namespaced(self.client.clone()); } - let establish = await_condition(api, name.as_str(), conditions::is_deployment_completed()); - let t = timeout.unwrap_or(300); - let res = tokio::time::timeout(std::time::Duration::from_secs(t), establish).await; + let establish = await_condition(api, name, conditions::is_deployment_completed()); + let timeout = timeout.unwrap_or(Duration::from_secs(120)); + let res = tokio::time::timeout(timeout, establish).await; if res.is_ok() { Ok(()) @@ -451,7 +454,7 @@ impl K8sClient { where K: Resource + Clone + std::fmt::Debug + DeserializeOwned + serde::Serialize, ::Scope: ApplyStrategy, - ::DynamicType: Default, + ::DynamicType: Default, { let mut result = Vec::new(); for r in resource.iter() { @@ -516,10 +519,7 @@ impl K8sClient { // 6. Apply the object to the cluster using Server-Side Apply. // This will create the resource if it doesn't exist, or update it if it does. - println!( - "Applying Argo Application '{}' in namespace '{}'...", - name, namespace - ); + println!("Applying Argo Application '{name}' in namespace '{namespace}'...",); let patch_params = PatchParams::apply("harmony"); // Use a unique field manager name let result = api.patch(name, &patch_params, &Patch::Apply(&obj)).await?; @@ -528,6 +528,51 @@ impl K8sClient { Ok(()) } + /// Apply a resource from a URL + /// + /// It is the equivalent of `kubectl apply -f ` + pub async fn apply_url(&self, url: Url, ns: Option<&str>) -> Result<(), Error> { + let patch_params = PatchParams::apply("harmony"); + let discovery = kube::Discovery::new(self.client.clone()).run().await?; + + let yaml = reqwest::get(url) + .await + .expect("Could not get URL") + .text() + .await + .expect("Could not get content from URL"); + + for doc in multidoc_deserialize(&yaml).expect("failed to parse YAML from file") { + let obj: DynamicObject = + serde_yaml::from_value(doc).expect("cannot apply without valid YAML"); + let namespace = obj.metadata.namespace.as_deref().or(ns); + let type_meta = obj + .types + .as_ref() + .expect("cannot apply object without valid TypeMeta"); + let gvk = GroupVersionKind::try_from(type_meta) + .expect("cannot apply object without valid GroupVersionKind"); + let name = obj.name_any(); + + if let Some((ar, caps)) = discovery.resolve_gvk(&gvk) { + let api = get_dynamic_api(ar, caps, self.client.clone(), namespace, false); + trace!( + "Applying {}: \n{}", + gvk.kind, + serde_yaml::to_string(&obj).expect("Failed to serialize YAML") + ); + let data: serde_json::Value = + serde_json::to_value(&obj).expect("Failed to serialize JSON"); + let _r = api.patch(&name, &patch_params, &Patch::Apply(data)).await?; + debug!("applied {} {}", gvk.kind, name); + } else { + warn!("Cannot apply document for unknown {gvk:?}"); + } + } + + Ok(()) + } + pub(crate) async fn from_kubeconfig(path: &str) -> Option { let k = match Kubeconfig::read_from(path) { Ok(k) => k, @@ -547,6 +592,31 @@ impl K8sClient { } } +fn get_dynamic_api( + resource: ApiResource, + capabilities: ApiCapabilities, + client: Client, + ns: Option<&str>, + all: bool, +) -> Api { + if capabilities.scope == Scope::Cluster || all { + Api::all_with(client, &resource) + } else if let Some(namespace) = ns { + Api::namespaced_with(client, namespace, &resource) + } else { + Api::default_namespaced_with(client, &resource) + } +} + +fn multidoc_deserialize(data: &str) -> Result, serde_yaml::Error> { + use serde::Deserialize; + let mut docs = vec![]; + for de in serde_yaml::Deserializer::from_str(data) { + docs.push(serde_yaml::Value::deserialize(de)?); + } + Ok(docs) +} + pub trait ApplyStrategy { fn get_api(client: &Client, ns: Option<&str>) -> Api; } diff --git a/harmony/src/infra/kubers/mod.rs b/harmony/src/infra/kubers/mod.rs deleted file mode 100644 index cd40856..0000000 --- a/harmony/src/infra/kubers/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod types; diff --git a/harmony/src/infra/kubers/types.rs b/harmony/src/infra/kubers/types.rs deleted file mode 100644 index 48384ea..0000000 --- a/harmony/src/infra/kubers/types.rs +++ /dev/null @@ -1,40 +0,0 @@ -use std::collections::BTreeMap; - -use kube::CustomResource; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -#[derive(CustomResource, Default, Deserialize, Serialize, Clone, Debug, JsonSchema)] -#[kube( - group = "operators.coreos.com", - version = "v1alpha1", - kind = "CatalogSource", - namespaced -)] -#[serde(rename_all = "camelCase")] -pub struct CatalogSourceSpec { - pub source_type: String, - pub image: String, - pub display_name: String, - pub publisher: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub grpc_pod_config: Option, -} - -impl Default for CatalogSource { - fn default() -> Self { - Self { - metadata: Default::default(), - spec: Default::default(), - } - } -} - -#[derive(Default, Serialize, Deserialize, Clone, Debug, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct GrpcPodConfig { - #[serde(skip_serializing_if = "Option::is_none")] - pub memory_target: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub node_selector: Option>, -} diff --git a/harmony/src/infra/mod.rs b/harmony/src/infra/mod.rs index 5b38eab..203cf90 100644 --- a/harmony/src/infra/mod.rs +++ b/harmony/src/infra/mod.rs @@ -3,6 +3,5 @@ pub mod executors; pub mod hp_ilo; pub mod intel_amt; pub mod inventory; -pub mod kubers; pub mod opnsense; mod sqlx; diff --git a/harmony/src/modules/monitoring/ntfy/ntfy.rs b/harmony/src/modules/monitoring/ntfy/ntfy.rs index 4ed342b..f82aaf7 100644 --- a/harmony/src/modules/monitoring/ntfy/ntfy.rs +++ b/harmony/src/modules/monitoring/ntfy/ntfy.rs @@ -100,11 +100,7 @@ impl Interpret f info!("deploying ntfy..."); client - .wait_until_deployment_ready( - "ntfy".to_string(), - Some(self.score.namespace.as_str()), - None, - ) + .wait_until_deployment_ready("ntfy", Some(self.score.namespace.as_str()), None) .await?; info!("ntfy deployed"); diff --git a/harmony/src/modules/okd/crd/nmstate.rs b/harmony/src/modules/okd/crd/nmstate.rs index 5f71e4e..4fb65c6 100644 --- a/harmony/src/modules/okd/crd/nmstate.rs +++ b/harmony/src/modules/okd/crd/nmstate.rs @@ -6,9 +6,16 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] -#[kube(group = "nmstate.io", version = "v1", kind = "NMState", namespaced)] +#[kube( + group = "nmstate.io", + version = "v1", + kind = "NMState", + plural = "nmstates", + namespaced = false +)] #[serde(rename_all = "camelCase")] pub struct NMStateSpec { + #[serde(skip_serializing_if = "Option::is_none")] pub probe_configuration: Option, } -- 2.39.5 From 2fe1c5d1475f69ea4873dff2a96904c4898ad468 Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Thu, 23 Oct 2025 14:27:34 -0400 Subject: [PATCH 06/11] remove unused wait_for_operator --- harmony/src/domain/topology/ha_cluster.rs | 8 +-- harmony/src/domain/topology/k8s.rs | 83 ++++++----------------- harmony/src/modules/okd/crd/mod.rs | 61 ----------------- 3 files changed, 23 insertions(+), 129 deletions(-) diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index bfc2e57..54b36d8 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -6,7 +6,6 @@ use harmony_types::{ net::{MacAddress, Url}, switch::PortLocation, }; -use k8s_openapi::api::core::v1::Namespace; use kube::api::ObjectMeta; use log::debug; use log::info; @@ -16,9 +15,8 @@ use crate::executors::ExecutorError; use crate::hardware::PhysicalHost; use crate::infra::brocade::BrocadeSwitchAuth; use crate::infra::brocade::BrocadeSwitchClient; -use crate::modules::okd::crd::{ - InstallPlanApproval, OperatorGroup, OperatorGroupSpec, Subscription, SubscriptionSpec, - nmstate::{self, NMState, NodeNetworkConfigurationPolicy, NodeNetworkConfigurationPolicySpec}, +use crate::modules::okd::crd::nmstate::{ + self, NMState, NodeNetworkConfigurationPolicy, NodeNetworkConfigurationPolicySpec, }; use crate::topology::PxeOptions; @@ -29,9 +27,9 @@ use super::{ Topology, k8s::K8sClient, }; +use std::collections::BTreeMap; use std::net::IpAddr; use std::sync::Arc; -use std::{collections::BTreeMap, time::Duration}; #[derive(Debug, Clone)] pub struct HAClusterTopology { diff --git a/harmony/src/domain/topology/k8s.rs b/harmony/src/domain/topology/k8s.rs index cfd0180..10a7df3 100644 --- a/harmony/src/domain/topology/k8s.rs +++ b/harmony/src/domain/topology/k8s.rs @@ -3,37 +3,29 @@ use std::time::Duration; use derive_new::new; use k8s_openapi::{ ClusterResourceScope, NamespaceResourceScope, - api::{ - apps::v1::Deployment, - core::v1::{Pod, PodStatus}, - }, + api::{apps::v1::Deployment, core::v1::Pod}, }; use kube::{ Client, Config, Error, Resource, - api::{ - Api, AttachParams, DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, ResourceExt, - }, + api::{Api, AttachParams, DeleteParams, ListParams, Patch, PatchParams, ResourceExt}, config::{KubeConfigOptions, Kubeconfig}, core::ErrorResponse, discovery::{ApiCapabilities, Scope}, error::DiscoveryError, - runtime::{reflector::Lookup, wait::Condition}, + runtime::reflector::Lookup, }; use kube::{api::DynamicObject, runtime::conditions}; use kube::{ api::{ApiResource, GroupVersionKind}, runtime::wait::await_condition, }; -use log::{debug, error, info, trace, warn}; -use serde::{Deserialize, Serialize, de::DeserializeOwned}; -use serde_json::{Value, json}; -use serde_value::DeserializerError; +use log::{debug, error, trace, warn}; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::json; use similar::TextDiff; use tokio::{io::AsyncReadExt, time::sleep}; use url::Url; -use crate::modules::okd::crd::ClusterServiceVersion; - #[derive(new, Clone)] pub struct K8sClient { client: Client, @@ -78,7 +70,8 @@ impl K8sClient { } else { Api::default_namespaced_with(self.client.clone(), &gvk) }; - Ok(resource.get(name).await?) + + resource.get(name).await } pub async fn get_deployment( @@ -91,7 +84,8 @@ impl K8sClient { } else { Api::default_namespaced(self.client.clone()) }; - Ok(deps.get_opt(name).await?) + + deps.get_opt(name).await } pub async fn get_pod(&self, name: &str, namespace: Option<&str>) -> Result, Error> { @@ -100,7 +94,8 @@ impl K8sClient { } else { Api::default_namespaced(self.client.clone()) }; - Ok(pods.get_opt(name).await?) + + pods.get_opt(name).await } pub async fn scale_deployment( @@ -201,33 +196,6 @@ impl K8sClient { } } - pub async fn wait_for_operator( - &self, - operator_name: &str, - namespace: Option<&str>, - timeout: Option, - ) -> Result<(), String> { - let api: Api; - - if let Some(ns) = namespace { - api = Api::namespaced(self.client.clone(), ns); - } else { - api = Api::default_namespaced(self.client.clone()); - } - - let establish = await_condition(api, operator_name, is_operator_ready()); - let t = timeout.unwrap_or(Duration::from_secs(5)); - let res = tokio::time::timeout(t, establish).await; - - if res.is_ok() { - Ok(()) - } else { - Err(format!( - "timed out while waiting for operator {operator_name}" - )) - } - } - /// Will execute a commond in the first pod found that matches the specified label /// '{label}={name}' pub async fn exec_app_capture_output( @@ -274,7 +242,7 @@ impl K8sClient { if let Some(s) = status.status { let mut stdout_buf = String::new(); - if let Some(mut stdout) = process.stdout().take() { + if let Some(mut stdout) = process.stdout() { stdout .read_to_string(&mut stdout_buf) .await @@ -380,14 +348,14 @@ impl K8sClient { Ok(current) => { trace!("Received current value {current:#?}"); // The resource exists, so we calculate and display a diff. - println!("\nPerforming dry-run for resource: '{}'", name); + println!("\nPerforming dry-run for resource: '{name}'"); let mut current_yaml = serde_yaml::to_value(¤t).unwrap_or_else(|_| { panic!("Could not serialize current value : {current:#?}") }); if current_yaml.is_mapping() && current_yaml.get("status").is_some() { let map = current_yaml.as_mapping_mut().unwrap(); let removed = map.remove_entry("status"); - trace!("Removed status {:?}", removed); + trace!("Removed status {removed:?}"); } else { trace!( "Did not find status entry for current object {}/{}", @@ -416,14 +384,14 @@ impl K8sClient { similar::ChangeTag::Insert => "+", similar::ChangeTag::Equal => " ", }; - print!("{}{}", sign, change); + print!("{sign}{change}"); } // In a dry run, we return the new resource state that would have been applied. Ok(resource.clone()) } Err(Error::Api(ErrorResponse { code: 404, .. })) => { // The resource does not exist, so the "diff" is the entire new resource. - println!("\nPerforming dry-run for new resource: '{}'", name); + println!("\nPerforming dry-run for new resource: '{name}'"); println!( "Resource does not exist. It would be created with the following content:" ); @@ -432,14 +400,14 @@ impl K8sClient { // Print each line of the new resource with a '+' prefix. for line in new_yaml.lines() { - println!("+{}", line); + println!("+{line}"); } // In a dry run, we return the new resource state that would have been created. Ok(resource.clone()) } Err(e) => { // Another API error occurred. - error!("Failed to get resource '{}': {}", name, e); + error!("Failed to get resource '{name}': {e}"); Err(e) } } @@ -519,7 +487,7 @@ impl K8sClient { // 6. Apply the object to the cluster using Server-Side Apply. // This will create the resource if it doesn't exist, or update it if it does. - println!("Applying Argo Application '{name}' in namespace '{namespace}'...",); + println!("Applying '{name}' in namespace '{namespace}'...",); let patch_params = PatchParams::apply("harmony"); // Use a unique field manager name let result = api.patch(name, &patch_params, &Patch::Apply(&obj)).await?; @@ -648,14 +616,3 @@ where } } } - -fn is_operator_ready() -> impl Condition { - |obj: Option<&ClusterServiceVersion>| { - if let Some(csv) = obj { - if let Some(status) = &csv.spec.status { - return status.phase == "Succeeded"; - } - } - false - } -} diff --git a/harmony/src/modules/okd/crd/mod.rs b/harmony/src/modules/okd/crd/mod.rs index b26ad8d..568db3f 100644 --- a/harmony/src/modules/okd/crd/mod.rs +++ b/harmony/src/modules/okd/crd/mod.rs @@ -1,62 +1 @@ -use kube::CustomResource; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - pub mod nmstate; - -#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] -#[kube( - group = "operators.coreos.com", - version = "v1", - kind = "OperatorGroup", - namespaced -)] -#[serde(rename_all = "camelCase")] -pub struct OperatorGroupSpec { - pub target_namespaces: Vec, -} - -#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] -#[kube( - group = "operators.coreos.com", - version = "v1alpha1", - kind = "Subscription", - namespaced -)] -#[serde(rename_all = "camelCase")] -pub struct SubscriptionSpec { - pub name: String, - pub source: String, - pub source_namespace: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub channel: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub install_plan_approval: Option, -} - -#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] -pub enum InstallPlanApproval { - #[serde(rename = "Automatic")] - Automatic, - #[serde(rename = "Manual")] - Manual, -} - -#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] -#[kube( - group = "operators.coreos.com", - version = "v1alpha1", - kind = "ClusterServiceVersion", - namespaced -)] -#[serde(rename_all = "camelCase")] -pub struct ClusterServiceVersionSpec { - pub status: Option, -} - -#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct ClusterServiceVersionStatus { - pub phase: String, - pub reason: String, -} -- 2.39.5 From 89cb23f4f75b4f4be00da5ca4d5e91772d2d8b00 Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Thu, 23 Oct 2025 16:37:43 -0400 Subject: [PATCH 07/11] add more logging and add reporting message --- brocade/src/network_operating_system.rs | 7 +- harmony/src/domain/topology/ha_cluster.rs | 35 ++---- harmony/src/domain/topology/network.rs | 8 +- .../okd/bootstrap_persist_network_bond.rs | 11 +- harmony/src/modules/okd/host_network.rs | 113 ++++++++++++++---- 5 files changed, 116 insertions(+), 58 deletions(-) diff --git a/brocade/src/network_operating_system.rs b/brocade/src/network_operating_system.rs index b14bc08..b1100b4 100644 --- a/brocade/src/network_operating_system.rs +++ b/brocade/src/network_operating_system.rs @@ -247,7 +247,12 @@ impl BrocadeClient for NetworkOperatingSystemClient { ports: &[PortLocation], ) -> Result<(), Error> { info!( - "[Brocade] Configuring port-channel '{channel_name} {channel_id}' with ports: {ports:?}" + "[Brocade] Configuring port-channel '{channel_id} {channel_name}' with ports: {}", + ports + .iter() + .map(|p| format!("{p}")) + .collect::>() + .join(", ") ); let interfaces = self.get_interfaces().await?; diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index 54b36d8..1892cef 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -12,7 +12,6 @@ use log::info; use crate::data::FileContent; use crate::executors::ExecutorError; -use crate::hardware::PhysicalHost; use crate::infra::brocade::BrocadeSwitchAuth; use crate::infra::brocade::BrocadeSwitchClient; use crate::modules::okd::crd::nmstate::{ @@ -162,11 +161,7 @@ impl HAClusterTopology { 42 // FIXME: Find a better way to declare the bond id } - async fn configure_bond( - &self, - host: &PhysicalHost, - config: &HostNetworkConfig, - ) -> Result<(), SwitchError> { + async fn configure_bond(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> { self.ensure_nmstate_operator_installed() .await .map_err(|e| { @@ -175,8 +170,11 @@ impl HAClusterTopology { )) })?; - let bond_config = self.create_bond_configuration(host, config); - debug!("Configuring bond for host {host:?}: {bond_config:#?}"); + let bond_config = self.create_bond_configuration(config); + debug!( + "Configuring bond for host {}: {bond_config:#?}", + config.host_id + ); self.k8s_client() .await .unwrap() @@ -189,10 +187,9 @@ impl HAClusterTopology { fn create_bond_configuration( &self, - host: &PhysicalHost, config: &HostNetworkConfig, ) -> NodeNetworkConfigurationPolicy { - let host_name = host.id.clone(); + let host_name = &config.host_id; let bond_id = self.get_next_bond_id(); let bond_name = format!("bond{bond_id}"); @@ -294,18 +291,14 @@ impl HAClusterTopology { Ok(Box::new(client)) } - async fn configure_port_channel( - &self, - host: &PhysicalHost, - config: &HostNetworkConfig, - ) -> Result<(), SwitchError> { + async fn configure_port_channel(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> { debug!("Configuring port channel: {config:#?}"); let client = self.get_switch_client().await?; let switch_ports = config.switch_ports.iter().map(|s| s.port.clone()).collect(); client - .configure_port_channel(&format!("Harmony_{}", host.id), switch_ports) + .configure_port_channel(&format!("Harmony_{}", config.host_id), switch_ports) .await .map_err(|e| SwitchError::new(format!("Failed to configure switch: {e}")))?; @@ -504,13 +497,9 @@ impl Switch for HAClusterTopology { Ok(port) } - async fn configure_host_network( - &self, - host: &PhysicalHost, - config: HostNetworkConfig, - ) -> Result<(), SwitchError> { - self.configure_bond(host, &config).await?; - self.configure_port_channel(host, &config).await + async fn configure_host_network(&self, config: &HostNetworkConfig) -> Result<(), SwitchError> { + self.configure_bond(config).await?; + self.configure_port_channel(config).await } } diff --git a/harmony/src/domain/topology/network.rs b/harmony/src/domain/topology/network.rs index 99db03a..8a939a3 100644 --- a/harmony/src/domain/topology/network.rs +++ b/harmony/src/domain/topology/network.rs @@ -3,6 +3,7 @@ use std::{error::Error, net::Ipv4Addr, str::FromStr, sync::Arc}; use async_trait::async_trait; use derive_new::new; use harmony_types::{ + id::Id, net::{IpAddress, MacAddress}, switch::PortLocation, }; @@ -185,15 +186,12 @@ pub trait Switch: Send + Sync { mac_address: &MacAddress, ) -> Result, SwitchError>; - async fn configure_host_network( - &self, - host: &PhysicalHost, - config: HostNetworkConfig, - ) -> Result<(), SwitchError>; + async fn configure_host_network(&self, config: &HostNetworkConfig) -> Result<(), SwitchError>; } #[derive(Clone, Debug, PartialEq)] pub struct HostNetworkConfig { + pub host_id: Id, pub switch_ports: Vec, } diff --git a/harmony/src/modules/okd/bootstrap_persist_network_bond.rs b/harmony/src/modules/okd/bootstrap_persist_network_bond.rs index 6c486ea..fd901f1 100644 --- a/harmony/src/modules/okd/bootstrap_persist_network_bond.rs +++ b/harmony/src/modules/okd/bootstrap_persist_network_bond.rs @@ -15,7 +15,7 @@ use log::info; use serde::Serialize; // ------------------------------------------------------------------------------------------------- -// Step XX: Persist Network Bond +// Persist Network Bond // - Persist bonding via NMState // - Persist port channels on the Switch // ------------------------------------------------------------------------------------------------- @@ -80,18 +80,13 @@ impl OKDSetupPersistNetworkBondInterpet { topology: &HAClusterTopology, hosts: &Vec, ) -> Result<(), InterpretError> { - info!("[ControlPlane] Ensuring persistent bonding"); + info!("Ensuring persistent bonding"); + let score = HostNetworkConfigurationScore { hosts: hosts.clone(), }; score.interpret(inventory, topology).await?; - inquire::Confirm::new( - "Network configuration for control plane nodes is not automated yet. Configure it manually if needed.", - ) - .prompt() - .map_err(|e| InterpretError::new(format!("User prompt failed: {e}")))?; - Ok(()) } } diff --git a/harmony/src/modules/okd/host_network.rs b/harmony/src/modules/okd/host_network.rs index 3bc8c3c..81ce4b5 100644 --- a/harmony/src/modules/okd/host_network.rs +++ b/harmony/src/modules/okd/host_network.rs @@ -39,16 +39,34 @@ impl HostNetworkConfigurationInterpret { &self, topology: &T, host: &PhysicalHost, - ) -> Result<(), InterpretError> { + current_host: &usize, + total_hosts: &usize, + ) -> Result { + info!("[Host {current_host}/{total_hosts}] Collecting ports on switch..."); let switch_ports = self.collect_switch_ports_for_host(topology, host).await?; - if !switch_ports.is_empty() { + + let config = HostNetworkConfig { + host_id: host.id.clone(), + switch_ports, + }; + + if !config.switch_ports.is_empty() { + info!( + "[Host {current_host}/{total_hosts}] Found {} ports for {} interfaces", + config.switch_ports.len(), + host.network.len() + ); + + info!("[Host {current_host}/{total_hosts}] Configuring host network..."); topology - .configure_host_network(host, HostNetworkConfig { switch_ports }) + .configure_host_network(&config) .await .map_err(|e| InterpretError::new(format!("Failed to configure host: {e}")))?; + } else { + info!("[Host {current_host}/{total_hosts}] No ports found"); } - Ok(()) + Ok(config) } async fn collect_switch_ports_for_host( @@ -85,6 +103,47 @@ impl HostNetworkConfigurationInterpret { Ok(switch_ports) } + + fn format_host_configuration(&self, configs: Vec) -> Vec { + let mut report = vec![ + "Network Configuration Report".to_string(), + "------------------------------------------------------------------".to_string(), + ]; + + for config in configs { + let host = self + .score + .hosts + .iter() + .find(|h| h.id == config.host_id) + .unwrap(); + + println!("[Host] {host}"); + + if config.switch_ports.is_empty() { + report.push(format!( + "⏭️ Host {}: SKIPPED (No matching switch ports found)", + config.host_id + )); + } else { + let mappings: Vec = config + .switch_ports + .iter() + .map(|p| format!("[{} -> {}]", p.interface.name, p.port)) + .collect(); + + report.push(format!( + "✅ Host {}: Bonded {} port(s) {}", + config.host_id, + config.switch_ports.len(), + mappings.join(", ") + )); + } + } + report + .push("------------------------------------------------------------------".to_string()); + report + } } #[async_trait] @@ -114,27 +173,36 @@ impl Interpret for HostNetworkConfigurationInterpret { return Ok(Outcome::noop("No hosts to configure".into())); } - info!( - "Started network configuration for {} host(s)...", - self.score.hosts.len() - ); + let host_count = self.score.hosts.len(); + info!("Started network configuration for {host_count} host(s)...",); topology .setup_switch() .await .map_err(|e| InterpretError::new(format!("Switch setup failed: {e}")))?; - let mut configured_host_count = 0; - for host in &self.score.hosts { - self.configure_network_for_host(topology, host).await?; - configured_host_count += 1; - } + let mut current_host = 1; + let mut host_configurations = vec![]; - if configured_host_count > 0 { - Ok(Outcome::success(format!( - "Configured {configured_host_count}/{} host(s)", - self.score.hosts.len() - ))) + for host in &self.score.hosts { + let host_configuration = self + .configure_network_for_host(topology, host, ¤t_host, &host_count) + .await?; + + host_configurations.push(host_configuration); + current_host += 1; + } + if current_host > 1 { + let details = self.format_host_configuration(host_configurations); + + Ok(Outcome::success_with_details( + format!( + "Configured {}/{} host(s)", + current_host - 1, + self.score.hosts.len() + ), + details, + )) } else { Ok(Outcome::noop("No hosts configured".into())) } @@ -209,6 +277,7 @@ mod tests { assert_that!(*configured_host_networks).contains_exactly(vec![( HOST_ID.clone(), HostNetworkConfig { + host_id: HOST_ID.clone(), switch_ports: vec![SwitchPort { interface: EXISTING_INTERFACE.clone(), port: PORT.clone(), @@ -234,6 +303,7 @@ mod tests { assert_that!(*configured_host_networks).contains_exactly(vec![( HOST_ID.clone(), HostNetworkConfig { + host_id: HOST_ID.clone(), switch_ports: vec![ SwitchPort { interface: EXISTING_INTERFACE.clone(), @@ -263,6 +333,7 @@ mod tests { ( HOST_ID.clone(), HostNetworkConfig { + host_id: HOST_ID.clone(), switch_ports: vec![SwitchPort { interface: EXISTING_INTERFACE.clone(), port: PORT.clone(), @@ -272,6 +343,7 @@ mod tests { ( ANOTHER_HOST_ID.clone(), HostNetworkConfig { + host_id: ANOTHER_HOST_ID.clone(), switch_ports: vec![SwitchPort { interface: ANOTHER_EXISTING_INTERFACE.clone(), port: ANOTHER_PORT.clone(), @@ -382,11 +454,10 @@ mod tests { async fn configure_host_network( &self, - host: &PhysicalHost, - config: HostNetworkConfig, + config: &HostNetworkConfig, ) -> Result<(), SwitchError> { let mut configured_host_networks = self.configured_host_networks.lock().unwrap(); - configured_host_networks.push((host.id.clone(), config.clone())); + configured_host_networks.push((config.host_id.clone(), config.clone())); Ok(()) } -- 2.39.5 From ec77afa605ff8512263288042fa86c9d5c2efe10 Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Tue, 28 Oct 2025 15:42:47 -0400 Subject: [PATCH 08/11] fix command to find next available port-channel id --- brocade/src/network_operating_system.rs | 9 ++------- .../src/modules/okd/bootstrap_persist_network_bond.rs | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/brocade/src/network_operating_system.rs b/brocade/src/network_operating_system.rs index b1100b4..f7f8570 100644 --- a/brocade/src/network_operating_system.rs +++ b/brocade/src/network_operating_system.rs @@ -102,7 +102,7 @@ impl NetworkOperatingSystemClient { }; Some(Ok(InterfaceInfo { - name: format!("{} {}", interface_type, port_location), + name: format!("{interface_type} {port_location}"), port_location, interface_type, operating_mode, @@ -196,8 +196,6 @@ impl BrocadeClient for NetworkOperatingSystemClient { commands.push("exit".into()); } - commands.push("write memory".into()); - self.shell .run_commands(commands, ExecutionMode::Regular) .await?; @@ -212,7 +210,7 @@ impl BrocadeClient for NetworkOperatingSystemClient { let output = self .shell - .run_command("show port-channel", ExecutionMode::Regular) + .run_command("show port-channel summary", ExecutionMode::Regular) .await?; let used_ids: Vec = output @@ -280,8 +278,6 @@ impl BrocadeClient for NetworkOperatingSystemClient { commands.push("exit".into()); } - commands.push("write memory".into()); - self.shell .run_commands(commands, ExecutionMode::Regular) .await?; @@ -298,7 +294,6 @@ impl BrocadeClient for NetworkOperatingSystemClient { "configure terminal".into(), format!("no interface port-channel {}", channel_name), "exit".into(), - "write memory".into(), ]; self.shell diff --git a/harmony/src/modules/okd/bootstrap_persist_network_bond.rs b/harmony/src/modules/okd/bootstrap_persist_network_bond.rs index fd901f1..788f2c8 100644 --- a/harmony/src/modules/okd/bootstrap_persist_network_bond.rs +++ b/harmony/src/modules/okd/bootstrap_persist_network_bond.rs @@ -94,7 +94,7 @@ impl OKDSetupPersistNetworkBondInterpet { #[async_trait] impl Interpret for OKDSetupPersistNetworkBondInterpet { fn get_name(&self) -> InterpretName { - InterpretName::Custom("OKDSetup03ControlPlane") + InterpretName::Custom("OKDSetupPersistNetworkBondInterpet") } fn get_version(&self) -> Version { -- 2.39.5 From 981529751a249672c5fb65e0766ef1928fdd8267 Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Tue, 28 Oct 2025 18:32:25 -0400 Subject: [PATCH 09/11] improve more logs, better error messages when switch setup failed --- brocade/src/lib.rs | 2 ++ brocade/src/network_operating_system.rs | 28 +++++++++++++++- brocade/src/shell.rs | 8 +++-- harmony/src/domain/topology/ha_cluster.rs | 6 ++-- .../okd/bootstrap_persist_network_bond.rs | 14 +++++--- harmony/src/modules/okd/host_network.rs | 32 ++++++++++++++++--- harmony_cli/src/cli_reporter.rs | 2 +- 7 files changed, 76 insertions(+), 16 deletions(-) diff --git a/brocade/src/lib.rs b/brocade/src/lib.rs index 57b464a..c0b5b70 100644 --- a/brocade/src/lib.rs +++ b/brocade/src/lib.rs @@ -31,6 +31,7 @@ pub struct BrocadeOptions { pub struct TimeoutConfig { pub shell_ready: Duration, pub command_execution: Duration, + pub command_output: Duration, pub cleanup: Duration, pub message_wait: Duration, } @@ -40,6 +41,7 @@ impl Default for TimeoutConfig { Self { shell_ready: Duration::from_secs(10), command_execution: Duration::from_secs(60), // Commands like `deploy` (for a LAG) can take a while + command_output: Duration::from_secs(5), // Delay to start logging "waiting for command output" cleanup: Duration::from_secs(10), message_wait: Duration::from_millis(500), } diff --git a/brocade/src/network_operating_system.rs b/brocade/src/network_operating_system.rs index a6f824a..f4db713 100644 --- a/brocade/src/network_operating_system.rs +++ b/brocade/src/network_operating_system.rs @@ -3,6 +3,7 @@ use std::str::FromStr; use async_trait::async_trait; use harmony_types::switch::{PortDeclaration, PortLocation}; use log::{debug, info}; +use regex::Regex; use crate::{ BrocadeClient, BrocadeInfo, Error, ExecutionMode, InterSwitchLink, InterfaceInfo, @@ -110,6 +111,30 @@ impl NetworkOperatingSystemClient { status, })) } + + fn map_configure_interfaces_error(&self, err: Error) -> Error { + debug!("[Brocade] {err}"); + + if let Error::CommandError(message) = &err { + if message.contains("switchport") + && message.contains("Cannot configure aggregator member") + { + let re = Regex::new(r"\(conf-if-([a-zA-Z]+)-([\d/]+)\)#").unwrap(); + + if let Some(caps) = re.captures(message) { + let interface_type = &caps[1]; + let port_location = &caps[2]; + let interface = format!("{interface_type} {port_location}"); + + return Error::CommandError(format!( + "Cannot configure interface '{interface}', it is a member of a port-channel (LAG)" + )); + } + } + } + + err + } } #[async_trait] @@ -199,7 +224,8 @@ impl BrocadeClient for NetworkOperatingSystemClient { self.shell .run_commands(commands, ExecutionMode::Regular) - .await?; + .await + .map_err(|err| self.map_configure_interfaces_error(err))?; info!("[Brocade] Interfaces configured."); diff --git a/brocade/src/shell.rs b/brocade/src/shell.rs index 28eceb8..9cd94a9 100644 --- a/brocade/src/shell.rs +++ b/brocade/src/shell.rs @@ -211,7 +211,7 @@ impl BrocadeSession { let mut output = Vec::new(); let start = Instant::now(); let read_timeout = Duration::from_millis(500); - let log_interval = Duration::from_secs(3); + let log_interval = Duration::from_secs(5); let mut last_log = Instant::now(); loop { @@ -221,7 +221,9 @@ impl BrocadeSession { )); } - if start.elapsed() > Duration::from_secs(5) && last_log.elapsed() > log_interval { + if start.elapsed() > self.options.timeouts.command_output + && last_log.elapsed() > log_interval + { info!("[Brocade] Waiting for command output..."); last_log = Instant::now(); } @@ -276,7 +278,7 @@ impl BrocadeSession { let output_lower = output.to_lowercase(); if ERROR_PATTERNS.iter().any(|&p| output_lower.contains(p)) { return Err(Error::CommandError(format!( - "Command '{command}' failed: {}", + "Command error: {}", output.trim() ))); } diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index ee5f274..e16b6d2 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -167,7 +167,7 @@ impl HAClusterTopology { let bond_config = self.create_bond_configuration(config); debug!( - "Configuring bond for host {}: {bond_config:#?}", + "Applying NMState bond config for host {}: {bond_config:#?}", config.host_id ); self.k8s_client() @@ -185,9 +185,11 @@ impl HAClusterTopology { config: &HostNetworkConfig, ) -> NodeNetworkConfigurationPolicy { let host_name = &config.host_id; - let bond_id = self.get_next_bond_id(); let bond_name = format!("bond{bond_id}"); + + info!("Configuring bond '{bond_name}' for host '{host_name}'..."); + let mut bond_mtu: Option = None; let mut bond_mac_address: Option = None; let mut bond_ports = Vec::new(); diff --git a/harmony/src/modules/okd/bootstrap_persist_network_bond.rs b/harmony/src/modules/okd/bootstrap_persist_network_bond.rs index 788f2c8..4aa47b6 100644 --- a/harmony/src/modules/okd/bootstrap_persist_network_bond.rs +++ b/harmony/src/modules/okd/bootstrap_persist_network_bond.rs @@ -116,11 +116,15 @@ impl Interpret for OKDSetupPersistNetworkBondInterpet { ) -> Result { let nodes = self.get_nodes(inventory, topology).await?; - self.persist_network_bond(inventory, topology, &nodes) - .await?; + let res = self.persist_network_bond(inventory, topology, &nodes).await; - Ok(Outcome::success( - "Network bond successfully persisted".into(), - )) + match res { + Ok(_) => Ok(Outcome::success( + "Network bond successfully persisted".into(), + )), + Err(_) => Err(InterpretError::new( + "Failed to persist network bond".to_string(), + )), + } } } diff --git a/harmony/src/modules/okd/host_network.rs b/harmony/src/modules/okd/host_network.rs index 81ce4b5..39fb027 100644 --- a/harmony/src/modules/okd/host_network.rs +++ b/harmony/src/modules/okd/host_network.rs @@ -42,8 +42,17 @@ impl HostNetworkConfigurationInterpret { current_host: &usize, total_hosts: &usize, ) -> Result { - info!("[Host {current_host}/{total_hosts}] Collecting ports on switch..."); - let switch_ports = self.collect_switch_ports_for_host(topology, host).await?; + if host.network.is_empty() { + info!("[Host {current_host}/{total_hosts}] No interfaces to configure, skipping"); + return Ok(HostNetworkConfig { + host_id: host.id.clone(), + switch_ports: vec![], + }); + } + + let switch_ports = self + .collect_switch_ports_for_host(topology, host, current_host, total_hosts) + .await?; let config = HostNetworkConfig { host_id: host.id.clone(), @@ -63,7 +72,10 @@ impl HostNetworkConfigurationInterpret { .await .map_err(|e| InterpretError::new(format!("Failed to configure host: {e}")))?; } else { - info!("[Host {current_host}/{total_hosts}] No ports found"); + info!( + "[Host {current_host}/{total_hosts}] No ports found for {} interfaces, skipping", + host.network.len() + ); } Ok(config) @@ -73,14 +85,24 @@ impl HostNetworkConfigurationInterpret { &self, topology: &T, host: &PhysicalHost, + current_host: &usize, + total_hosts: &usize, ) -> Result, InterpretError> { let mut switch_ports = vec![]; + if host.network.is_empty() { + return Ok(switch_ports); + } + + info!("[Host {current_host}/{total_hosts}] Collecting ports on switch..."); for network_interface in &host.network { let mac_address = network_interface.mac_address; match topology.get_port_for_mac_address(&mac_address).await { Ok(Some(port)) => { + info!( + "[Host {current_host}/{total_hosts}] Found port '{port}' for '{mac_address}'" + ); switch_ports.push(SwitchPort { interface: NetworkInterface { name: network_interface.name.clone(), @@ -91,7 +113,7 @@ impl HostNetworkConfigurationInterpret { port, }); } - Ok(None) => debug!("No port found for host '{}', skipping", host.id), + Ok(None) => debug!("No port found for '{mac_address}', skipping"), Err(e) => { return Err(InterpretError::new(format!( "Failed to get port for host '{}': {}", @@ -176,10 +198,12 @@ impl Interpret for HostNetworkConfigurationInterpret { let host_count = self.score.hosts.len(); info!("Started network configuration for {host_count} host(s)...",); + info!("Setting up switch with sane defaults..."); topology .setup_switch() .await .map_err(|e| InterpretError::new(format!("Switch setup failed: {e}")))?; + info!("Switch ready"); let mut current_host = 1; let mut host_configurations = vec![]; diff --git a/harmony_cli/src/cli_reporter.rs b/harmony_cli/src/cli_reporter.rs index f6095cc..6c9823b 100644 --- a/harmony_cli/src/cli_reporter.rs +++ b/harmony_cli/src/cli_reporter.rs @@ -40,7 +40,7 @@ pub fn init() { HarmonyEvent::HarmonyFinished => { if !details.is_empty() { println!( - "\n{} All done! Here's what's next for you:", + "\n{} All done! Here's a few info for you:", theme::EMOJI_SUMMARY ); for detail in details.iter() { -- 2.39.5 From d6c6192c6b77028542a1c1e4256a2e1a15c7900e Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Wed, 29 Oct 2025 11:27:20 -0400 Subject: [PATCH 10/11] adjust nm config policy serialization --- harmony/src/domain/topology/ha_cluster.rs | 11 ++-- harmony/src/infra/inventory/mod.rs | 2 +- harmony/src/modules/okd/crd/nmstate.rs | 64 +++++++++++++++++++++++ 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/harmony/src/domain/topology/ha_cluster.rs b/harmony/src/domain/topology/ha_cluster.rs index e16b6d2..d65d9aa 100644 --- a/harmony/src/domain/topology/ha_cluster.rs +++ b/harmony/src/domain/topology/ha_cluster.rs @@ -191,7 +191,7 @@ impl HAClusterTopology { info!("Configuring bond '{bond_name}' for host '{host_name}'..."); let mut bond_mtu: Option = None; - let mut bond_mac_address: Option = None; + let mut copy_mac_from: Option = None; let mut bond_ports = Vec::new(); let mut interfaces: Vec = Vec::new(); @@ -217,14 +217,14 @@ impl HAClusterTopology { ..Default::default() }); - bond_ports.push(interface_name); + bond_ports.push(interface_name.clone()); // Use the first port's details for the bond mtu and mac address if bond_mtu.is_none() { bond_mtu = Some(switch_port.interface.mtu); } - if bond_mac_address.is_none() { - bond_mac_address = Some(switch_port.interface.mac_address.to_string()); + if copy_mac_from.is_none() { + copy_mac_from = Some(interface_name); } } @@ -233,8 +233,7 @@ impl HAClusterTopology { description: Some(format!("Network bond for host {host_name}")), r#type: "bond".to_string(), state: "up".to_string(), - mtu: bond_mtu, - mac_address: bond_mac_address, + copy_mac_from, ipv4: Some(nmstate::IpStackSpec { dhcp: Some(true), enabled: Some(true), diff --git a/harmony/src/infra/inventory/mod.rs b/harmony/src/infra/inventory/mod.rs index 90d78ea..e0d80fe 100644 --- a/harmony/src/infra/inventory/mod.rs +++ b/harmony/src/infra/inventory/mod.rs @@ -11,7 +11,7 @@ pub struct InventoryRepositoryFactory; impl InventoryRepositoryFactory { pub async fn build() -> Result, RepoError> { Ok(Box::new( - SqliteInventoryRepository::new(&(*DATABASE_URL)).await?, + SqliteInventoryRepository::new(&DATABASE_URL).await?, )) } } diff --git a/harmony/src/modules/okd/crd/nmstate.rs b/harmony/src/modules/okd/crd/nmstate.rs index 4fb65c6..9f986e5 100644 --- a/harmony/src/modules/okd/crd/nmstate.rs +++ b/harmony/src/modules/okd/crd/nmstate.rs @@ -51,6 +51,7 @@ pub struct ProbeDns { )] #[serde(rename_all = "camelCase")] pub struct NodeNetworkConfigurationPolicySpec { + #[serde(skip_serializing_if = "Option::is_none")] pub node_selector: Option>, pub desired_state: DesiredStateSpec, } @@ -65,37 +66,64 @@ pub struct DesiredStateSpec { #[serde(rename_all = "kebab-case")] pub struct InterfaceSpec { pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] pub description: Option, pub r#type: String, pub state: String, + #[serde(skip_serializing_if = "Option::is_none")] pub mac_address: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub copy_mac_from: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub mtu: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub controller: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ipv4: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ipv6: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ethernet: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub link_aggregation: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub vlan: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub vxlan: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub mac_vtap: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub mac_vlan: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub infiniband: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub linux_bridge: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ovs_bridge: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ethtool: Option, } #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct IpStackSpec { + #[serde(skip_serializing_if = "Option::is_none")] pub enabled: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub dhcp: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub autoconf: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub address: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub auto_dns: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub auto_gateway: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub auto_routes: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub dhcp_client_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub dhcp_duid: Option, } @@ -109,8 +137,11 @@ pub struct IpAddressSpec { #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct EthernetSpec { + #[serde(skip_serializing_if = "Option::is_none")] pub speed: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub duplex: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub auto_negotiation: Option, } @@ -119,6 +150,7 @@ pub struct EthernetSpec { pub struct BondSpec { pub mode: String, pub ports: Vec, + #[serde(skip_serializing_if = "Option::is_none")] pub options: Option>, } @@ -127,6 +159,7 @@ pub struct BondSpec { pub struct VlanSpec { pub base_iface: String, pub id: u16, + #[serde(skip_serializing_if = "Option::is_none")] pub protocol: Option, } @@ -136,8 +169,11 @@ pub struct VxlanSpec { pub base_iface: String, pub id: u32, pub remote: String, + #[serde(skip_serializing_if = "Option::is_none")] pub local: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub learning: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub destination_port: Option, } @@ -146,6 +182,7 @@ pub struct VxlanSpec { pub struct MacVtapSpec { pub base_iface: String, pub mode: String, + #[serde(skip_serializing_if = "Option::is_none")] pub promiscuous: Option, } @@ -154,6 +191,7 @@ pub struct MacVtapSpec { pub struct MacVlanSpec { pub base_iface: String, pub mode: String, + #[serde(skip_serializing_if = "Option::is_none")] pub promiscuous: Option, } @@ -168,25 +206,35 @@ pub struct InfinibandSpec { #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct LinuxBridgeSpec { + #[serde(skip_serializing_if = "Option::is_none")] pub options: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ports: Option>, } #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct LinuxBridgeOptions { + #[serde(skip_serializing_if = "Option::is_none")] pub mac_ageing_time: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub multicast_snooping: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub stp: Option, } #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct StpOptions { + #[serde(skip_serializing_if = "Option::is_none")] pub enabled: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub forward_delay: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub hello_time: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub max_age: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub priority: Option, } @@ -194,15 +242,20 @@ pub struct StpOptions { #[serde(rename_all = "kebab-case")] pub struct LinuxBridgePort { pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] pub vlan: Option, } #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct LinuxBridgePortVlan { + #[serde(skip_serializing_if = "Option::is_none")] pub mode: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub trunk_tags: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub tag: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub enable_native: Option, } @@ -210,6 +263,7 @@ pub struct LinuxBridgePortVlan { #[serde(rename_all = "kebab-case")] pub struct VlanTag { pub id: u16, + #[serde(skip_serializing_if = "Option::is_none")] pub id_range: Option, } @@ -223,15 +277,20 @@ pub struct VlanIdRange { #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct OvsBridgeSpec { + #[serde(skip_serializing_if = "Option::is_none")] pub options: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ports: Option>, } #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct OvsBridgeOptions { + #[serde(skip_serializing_if = "Option::is_none")] pub stp: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub rstp: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub mcast_snooping_enable: Option, } @@ -239,8 +298,11 @@ pub struct OvsBridgeOptions { #[serde(rename_all = "kebab-case")] pub struct OvsPortSpec { pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] pub link_aggregation: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub vlan: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub r#type: Option, } @@ -253,6 +315,8 @@ pub struct EthtoolSpec { #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct EthtoolFecSpec { + #[serde(skip_serializing_if = "Option::is_none")] pub auto: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub mode: Option, } -- 2.39.5 From b8ac3580f8457ff6191ef9735107683c224a331c Mon Sep 17 00:00:00 2001 From: Ian Letourneau Date: Wed, 29 Oct 2025 12:21:23 -0400 Subject: [PATCH 11/11] revert opnsense config change --- opnsense-config-xml/src/data/interfaces.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opnsense-config-xml/src/data/interfaces.rs b/opnsense-config-xml/src/data/interfaces.rs index fb49a4d..b06f392 100644 --- a/opnsense-config-xml/src/data/interfaces.rs +++ b/opnsense-config-xml/src/data/interfaces.rs @@ -9,7 +9,7 @@ pub struct Interface { pub physical_interface_name: String, pub descr: Option, pub mtu: Option, - pub enable: Option, + pub enable: MaybeString, pub lock: Option, #[yaserde(rename = "spoofmac")] pub spoof_mac: Option, -- 2.39.5